feat(rt): add data monitor
This commit is contained in:
1 parent
fb6e9a2163
commit
ca21145bbe
9 files changed
+618
-11
No files matched your search
@@ -15,6 +15,7 @@ from prettytable import PrettyTable
|
||||
from csaxs_bec.devices.omny.rt.rt_ophyd import (
|
||||
format_scan_timing,
|
||||
RtCommunicationError,
|
||||
RtDataMonitor,
|
||||
RtError,
|
||||
RtReadbackSignal,
|
||||
RtSetpointSignal,
|
||||
@@ -485,6 +486,7 @@ class RtFlomniController(Controller):
|
||||
raise RtError("Cannot start scan because no target positions are planned.")
|
||||
# hier exception
|
||||
# start a point-by-point scan (for cont scan in flomni it would be "sa")
|
||||
logger.info("RtFlomniController: Starting scan.")
|
||||
self.socket_put_and_receive("sd")
|
||||
|
||||
@retry_once
|
||||
@@ -774,7 +776,7 @@ class RtFlomniMotor(Device, PositionerBase):
|
||||
|
||||
|
||||
class RtFlomniFlyer(Device):
|
||||
USER_ACCESS = ["controller"]
|
||||
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
|
||||
data = Cpt(
|
||||
AsyncMultiSignal,
|
||||
name="data",
|
||||
@@ -811,9 +813,12 @@ class RtFlomniFlyer(Device):
|
||||
port=2222,
|
||||
socket_cls=SocketIO,
|
||||
device_manager=None,
|
||||
data_timeout: float | None = 5.0,
|
||||
scan_timeout: float | None = 60.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
|
||||
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
|
||||
self.shutdown_event = threading.Event()
|
||||
self.controller = RtFlomniController(
|
||||
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
|
||||
@@ -829,6 +834,53 @@ class RtFlomniFlyer(Device):
|
||||
self.scan_done_event = threading.Event()
|
||||
self.scan_done_event.set()
|
||||
|
||||
@property
|
||||
def data_timeout(self) -> float | None:
|
||||
"""Seconds without progress before logging a warning (default: 5).
|
||||
|
||||
Set to None to suppress the warning while leaving scan_timeout independent.
|
||||
"""
|
||||
return self._data_monitor.data_timeout
|
||||
|
||||
@data_timeout.setter
|
||||
def data_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.data_timeout = value
|
||||
|
||||
@property
|
||||
def scan_timeout(self) -> float | None:
|
||||
"""Seconds without progress before failing the scan (default: 60).
|
||||
|
||||
Progress means an advancing controller position until the first sample,
|
||||
then receipt of new samples. Set to None to disable timeout failures.
|
||||
Readout can then wait until progress resumes, the controller finishes,
|
||||
or the scan is stopped.
|
||||
"""
|
||||
return self._data_monitor.scan_timeout
|
||||
|
||||
@scan_timeout.setter
|
||||
def scan_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.scan_timeout = value
|
||||
|
||||
def fetch_diagnostics(self) -> dict:
|
||||
"""Return scan status, feedback state, and recent controller commands."""
|
||||
diagnostics = {}
|
||||
try:
|
||||
mode, planned, current = self.controller.get_scan_status()
|
||||
diagnostics["scan_status"] = {
|
||||
"mode": mode,
|
||||
"planned_positions": planned,
|
||||
"current_position": current,
|
||||
}
|
||||
except Exception as exc:
|
||||
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
diagnostics["feedback_running"] = self.controller.feedback_is_running()
|
||||
except Exception as exc:
|
||||
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
|
||||
with self.controller._lock:
|
||||
diagnostics["command_history"] = list(self.controller.command_history)
|
||||
return diagnostics
|
||||
|
||||
def read_positions_from_sampler(self, status: DeviceStatus):
|
||||
"""
|
||||
Read the positions from the sampler and update the data signal.
|
||||
@@ -864,6 +916,18 @@ class RtFlomniFlyer(Device):
|
||||
read_counter = read_counter + 1
|
||||
signals = self._get_signals_from_table(return_table)
|
||||
self.data.set(signals)
|
||||
self._data_monitor.sample_received()
|
||||
|
||||
if mode > 0 and not self.shutdown_event.is_set():
|
||||
exception = self._data_monitor.check(
|
||||
self.name,
|
||||
mode=mode,
|
||||
planned_positions=number_of_positions_planned,
|
||||
position=current_position_in_scan,
|
||||
)
|
||||
if exception is not None:
|
||||
status.set_exception(exception)
|
||||
return
|
||||
|
||||
self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics)
|
||||
|
||||
@@ -958,6 +1022,7 @@ class RtFlomniFlyer(Device):
|
||||
self.scan_done_event.clear()
|
||||
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
|
||||
...
|
||||
self._data_monitor.reset()
|
||||
self.controller.start_scan()
|
||||
self._t_kickoff = time.monotonic()
|
||||
self._t_scan_end = None
|
||||
@@ -981,7 +1046,10 @@ class RtFlomniFlyer(Device):
|
||||
def stop(self, *, success=False):
|
||||
self.shutdown_event.set()
|
||||
self.scan_done_event.set()
|
||||
if self.readout_thread is not None:
|
||||
if (
|
||||
self.readout_thread is not None
|
||||
and self.readout_thread is not threading.current_thread()
|
||||
):
|
||||
self.readout_thread.join()
|
||||
return super().stop(success=success)
|
||||
|
||||
@@ -991,4 +1059,4 @@ if __name__ == "__main__":
|
||||
socket_cls=SocketIO, socket_host="mpc-flomni.psi.ch", socket_port=2222, device_manager=None
|
||||
)
|
||||
rtcontroller.on()
|
||||
rtcontroller.laser_tracker_on()
|
||||
rtcontroller.laser_tracker_on()
|
||||
@@ -13,7 +13,12 @@ from ophyd_devices.utils.controller import Controller, retry_once, threadlocked
|
||||
from ophyd_devices.utils.socket import SocketIO, SocketSignal, raise_if_disconnected
|
||||
from prettytable import PrettyTable
|
||||
|
||||
from csaxs_bec.devices.omny.rt.rt_ophyd import RtCommunicationError, RtError, format_scan_timing
|
||||
from csaxs_bec.devices.omny.rt.rt_ophyd import (
|
||||
RtCommunicationError,
|
||||
RtDataMonitor,
|
||||
RtError,
|
||||
format_scan_timing,
|
||||
)
|
||||
|
||||
logger = bec_logger.logger
|
||||
|
||||
@@ -217,6 +222,7 @@ class RtLamniController(Controller):
|
||||
raise RtError("Cannot start scan because no target positions are planned.")
|
||||
# hier exception
|
||||
# start a point-by-point scan (for cont scan in flomni it would be "sa")
|
||||
logger.info("RtLamniController: Starting scan.")
|
||||
self.socket_put_and_receive("sd")
|
||||
|
||||
def feedback_is_running(self) -> bool:
|
||||
@@ -694,7 +700,7 @@ class RtLamniMotor(Device, PositionerBase):
|
||||
|
||||
|
||||
class RtLamniFlyer(Device):
|
||||
USER_ACCESS = ["controller"]
|
||||
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
|
||||
data = Cpt(
|
||||
AsyncMultiSignal,
|
||||
name="data",
|
||||
@@ -740,9 +746,12 @@ class RtLamniFlyer(Device):
|
||||
port=3333,
|
||||
socket_cls=SocketIO,
|
||||
device_manager=None,
|
||||
data_timeout: float | None = 5.0,
|
||||
scan_timeout: float | None = 60.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
|
||||
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
|
||||
self.shutdown_event = threading.Event()
|
||||
self.controller = RtLamniController(
|
||||
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
|
||||
@@ -759,6 +768,53 @@ class RtLamniFlyer(Device):
|
||||
self.scan_done_event = threading.Event()
|
||||
self.scan_done_event.set()
|
||||
|
||||
@property
|
||||
def data_timeout(self) -> float | None:
|
||||
"""Seconds without progress before logging a warning (default: 5).
|
||||
|
||||
Set to None to suppress the warning while leaving scan_timeout independent.
|
||||
"""
|
||||
return self._data_monitor.data_timeout
|
||||
|
||||
@data_timeout.setter
|
||||
def data_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.data_timeout = value
|
||||
|
||||
@property
|
||||
def scan_timeout(self) -> float | None:
|
||||
"""Seconds without progress before failing the scan (default: 60).
|
||||
|
||||
Progress means an advancing controller position until the first sample,
|
||||
then receipt of new samples. Set to None to disable timeout failures.
|
||||
Readout can then wait until progress resumes, the controller finishes,
|
||||
or the scan is stopped.
|
||||
"""
|
||||
return self._data_monitor.scan_timeout
|
||||
|
||||
@scan_timeout.setter
|
||||
def scan_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.scan_timeout = value
|
||||
|
||||
def fetch_diagnostics(self) -> dict:
|
||||
"""Return scan status, feedback state, and recent controller commands."""
|
||||
diagnostics = {}
|
||||
try:
|
||||
mode, planned, current = self.controller.get_scan_status()
|
||||
diagnostics["scan_status"] = {
|
||||
"mode": mode,
|
||||
"planned_positions": planned,
|
||||
"current_position": current,
|
||||
}
|
||||
except Exception as exc:
|
||||
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
diagnostics["feedback_running"] = self.controller.feedback_is_running()
|
||||
except Exception as exc:
|
||||
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
|
||||
with self.controller._lock:
|
||||
diagnostics["command_history"] = list(self.controller.command_history)
|
||||
return diagnostics
|
||||
|
||||
def stage(self):
|
||||
self.shutdown_event.clear()
|
||||
self.scan_done_event.set()
|
||||
@@ -783,6 +839,7 @@ class RtLamniFlyer(Device):
|
||||
self.scan_done_event.clear()
|
||||
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
|
||||
...
|
||||
self._data_monitor.reset()
|
||||
self.controller.start_scan()
|
||||
self._t_kickoff = time.monotonic()
|
||||
self._t_scan_end = None
|
||||
@@ -806,7 +863,10 @@ class RtLamniFlyer(Device):
|
||||
def stop(self, *, success=False):
|
||||
self.shutdown_event.set()
|
||||
self.scan_done_event.set()
|
||||
if self.readout_thread is not None:
|
||||
if (
|
||||
self.readout_thread is not None
|
||||
and self.readout_thread is not threading.current_thread()
|
||||
):
|
||||
self.readout_thread.join()
|
||||
return super().stop(success=success)
|
||||
|
||||
@@ -846,6 +906,18 @@ class RtLamniFlyer(Device):
|
||||
read_counter = read_counter + 1
|
||||
signals = self._get_signals_from_table(return_table)
|
||||
self.data.set(signals)
|
||||
self._data_monitor.sample_received()
|
||||
|
||||
if mode > 0 and not self.shutdown_event.is_set():
|
||||
exception = self._data_monitor.check(
|
||||
self.name,
|
||||
mode=mode,
|
||||
planned_positions=number_of_positions_planned,
|
||||
position=current_position_in_scan,
|
||||
)
|
||||
if exception is not None:
|
||||
status.set_exception(exception)
|
||||
return
|
||||
|
||||
self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics)
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ if builtins.__dict__.get("bec") is not None:
|
||||
|
||||
from csaxs_bec.devices.omny.rt.rt_ophyd import (
|
||||
RtCommunicationError,
|
||||
RtDataMonitor,
|
||||
RtError,
|
||||
RtReadbackSignal,
|
||||
RtSetpointSignal,
|
||||
@@ -874,6 +875,7 @@ class RtOMNYController(Controller):
|
||||
raise RtError("Cannot start scan because no target positions are planned.")
|
||||
# hier exception
|
||||
# start a point-by-point scan (for cont scan in flomni it would be "sa")
|
||||
logger.info("RtOMNYController: Starting scan.")
|
||||
self.socket_put_and_receive("sd")
|
||||
|
||||
@retry_once
|
||||
@@ -1161,7 +1163,7 @@ class RtOMNYMotor(Device, PositionerBase):
|
||||
|
||||
|
||||
class RtOMNYFlyer(Device):
|
||||
USER_ACCESS = ["controller"]
|
||||
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
|
||||
|
||||
data = Cpt(
|
||||
AsyncMultiSignal,
|
||||
@@ -1205,9 +1207,12 @@ class RtOMNYFlyer(Device):
|
||||
port=2222,
|
||||
socket_cls=SocketIO,
|
||||
device_manager=None,
|
||||
data_timeout: float | None = 5.0,
|
||||
scan_timeout: float | None = 60.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
|
||||
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
|
||||
self.shutdown_event = threading.Event()
|
||||
self.controller = RtOMNYController(
|
||||
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
|
||||
@@ -1218,6 +1223,53 @@ class RtOMNYFlyer(Device):
|
||||
self.scan_done_event = threading.Event()
|
||||
self.scan_done_event.set()
|
||||
|
||||
@property
|
||||
def data_timeout(self) -> float | None:
|
||||
"""Seconds without progress before logging a warning (default: 5).
|
||||
|
||||
Set to None to suppress the warning while leaving scan_timeout independent.
|
||||
"""
|
||||
return self._data_monitor.data_timeout
|
||||
|
||||
@data_timeout.setter
|
||||
def data_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.data_timeout = value
|
||||
|
||||
@property
|
||||
def scan_timeout(self) -> float | None:
|
||||
"""Seconds without progress before failing the scan (default: 60).
|
||||
|
||||
Progress means an advancing controller position until the first sample,
|
||||
then receipt of new samples. Set to None to disable timeout failures.
|
||||
Readout can then wait until progress resumes, the controller finishes,
|
||||
or the scan is stopped.
|
||||
"""
|
||||
return self._data_monitor.scan_timeout
|
||||
|
||||
@scan_timeout.setter
|
||||
def scan_timeout(self, value: float | None) -> None:
|
||||
self._data_monitor.scan_timeout = value
|
||||
|
||||
def fetch_diagnostics(self) -> dict:
|
||||
"""Return scan status, feedback state, and recent controller commands."""
|
||||
diagnostics = {}
|
||||
try:
|
||||
mode, planned, current = self.controller.get_scan_status()
|
||||
diagnostics["scan_status"] = {
|
||||
"mode": mode,
|
||||
"planned_positions": planned,
|
||||
"current_position": current,
|
||||
}
|
||||
except Exception as exc:
|
||||
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
diagnostics["feedback_running"] = self.controller.feedback_is_running()
|
||||
except Exception as exc:
|
||||
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
|
||||
with self.controller._lock:
|
||||
diagnostics["command_history"] = list(self.controller.command_history)
|
||||
return diagnostics
|
||||
|
||||
def stage(self):
|
||||
self.shutdown_event.clear()
|
||||
self.scan_done_event.set()
|
||||
@@ -1234,6 +1286,7 @@ class RtOMNYFlyer(Device):
|
||||
self.scan_done_event.clear()
|
||||
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
|
||||
...
|
||||
self._data_monitor.reset()
|
||||
self.controller.start_scan()
|
||||
self.shutdown_event.wait(0.1)
|
||||
status = DeviceStatus(self)
|
||||
@@ -1255,7 +1308,10 @@ class RtOMNYFlyer(Device):
|
||||
def stop(self, *, success=False):
|
||||
self.shutdown_event.set()
|
||||
self.scan_done_event.set()
|
||||
if self.readout_thread is not None:
|
||||
if (
|
||||
self.readout_thread is not None
|
||||
and self.readout_thread is not threading.current_thread()
|
||||
):
|
||||
self.readout_thread.join()
|
||||
return super().stop(success=success)
|
||||
|
||||
@@ -1292,6 +1348,18 @@ class RtOMNYFlyer(Device):
|
||||
read_counter = read_counter + 1
|
||||
signals = self._get_signals_from_table(return_table)
|
||||
self.data.set(signals)
|
||||
self._data_monitor.sample_received()
|
||||
|
||||
if mode > 0 and not self.shutdown_event.is_set():
|
||||
exception = self._data_monitor.check(
|
||||
self.name,
|
||||
mode=mode,
|
||||
planned_positions=number_of_positions_planned,
|
||||
position=current_position_in_scan,
|
||||
)
|
||||
if exception is not None:
|
||||
status.set_exception(exception)
|
||||
return
|
||||
|
||||
if self.shutdown_event.wait(0.05):
|
||||
logger.info("Shutdown event set, stopping readout.")
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""
|
||||
This module contains base signals for RT devices. Controller and motors are implemented in the
|
||||
bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`.
|
||||
This module contains base signals and data monitoring for RT devices. Controllers and motors are
|
||||
implemented in bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import math
|
||||
import time
|
||||
|
||||
from bec_lib import bec_logger
|
||||
@@ -46,6 +46,128 @@ class BECConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class RtDataMonitor:
|
||||
"""Track gaps in scan progress for an owning RT flyer.
|
||||
|
||||
Both deadlines start at kickoff. While waiting for the first sampler batch,
|
||||
each new highest controller position resets the deadlines. After the first
|
||||
sample, each received sample resets them. The readout loop checks both
|
||||
deadlines after draining available samples, between socket calls.
|
||||
|
||||
Set either timeout to `None` to disable it independently. With
|
||||
`data_timeout=None`, the monitor skips the stall warning. With
|
||||
`scan_timeout=None`, a stall can leave readout waiting until progress
|
||||
resumes, the controller finishes, or the scan is stopped. Other
|
||||
controller errors and cancellation still apply.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, data_timeout: float | None = 5.0, scan_timeout: float | None = 60.0
|
||||
) -> None:
|
||||
"""Configure the warning and failure thresholds for stalled progress.
|
||||
|
||||
Args:
|
||||
data_timeout (float | None): Seconds without progress before logging one
|
||||
warning per stall. Defaults to 5 seconds. Pass None to disable warnings.
|
||||
scan_timeout (float | None): Seconds without progress before failing the
|
||||
flyer completion status. Defaults to 60 seconds. Pass None to disable
|
||||
timeout failures.
|
||||
|
||||
Raises:
|
||||
ValueError: If an enabled timeout is non-finite or less than or equal to zero.
|
||||
"""
|
||||
self.data_timeout = data_timeout
|
||||
self.scan_timeout = scan_timeout
|
||||
self._last_progress_time: float | None = None
|
||||
self._initial_position: int | None = None
|
||||
self._data_timeout_logged = False
|
||||
|
||||
@staticmethod
|
||||
def _validate_timeout(value: float | None, name: str) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
value = float(value)
|
||||
if not math.isfinite(value) or value <= 0:
|
||||
raise ValueError(f"{name} must be a finite, positive number of seconds.")
|
||||
return value
|
||||
|
||||
@property
|
||||
def data_timeout(self) -> float | None:
|
||||
"""Seconds without progress before logging a warning, or None to skip the warning."""
|
||||
return self._data_timeout
|
||||
|
||||
@data_timeout.setter
|
||||
def data_timeout(self, value: float | None) -> None:
|
||||
self._data_timeout = self._validate_timeout(value, "data_timeout")
|
||||
|
||||
@property
|
||||
def scan_timeout(self) -> float | None:
|
||||
"""Seconds without progress before failing the scan, or None to keep waiting for progress."""
|
||||
return self._scan_timeout
|
||||
|
||||
@scan_timeout.setter
|
||||
def scan_timeout(self, value: float | None) -> None:
|
||||
self._scan_timeout = self._validate_timeout(value, "scan_timeout")
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Start monitoring a new scan at kickoff."""
|
||||
self._initial_position = 0
|
||||
self._record_progress()
|
||||
|
||||
def sample_received(self) -> None:
|
||||
"""Switch to monitoring sample receipt and record this sample."""
|
||||
self._initial_position = None
|
||||
self._record_progress()
|
||||
|
||||
def _record_progress(self) -> None:
|
||||
self._last_progress_time = time.monotonic()
|
||||
self._data_timeout_logged = False
|
||||
|
||||
def check(
|
||||
self, device_name: str, *, mode: int, planned_positions: int, position: int
|
||||
) -> TimeoutError | None:
|
||||
"""Check the deadlines using the latest controller status.
|
||||
|
||||
Args:
|
||||
device_name (str): Flyer name used in warnings and errors.
|
||||
mode (int): Latest controller scan mode.
|
||||
planned_positions (int): Number of points planned by the controller.
|
||||
position (int): Current scan position reported by the controller.
|
||||
|
||||
Returns:
|
||||
TimeoutError | None: Exception to set on the flyer completion status
|
||||
if the scan timeout expires, or None to continue. A warning can
|
||||
be logged while readout continues.
|
||||
"""
|
||||
if self._last_progress_time is None:
|
||||
return
|
||||
if self._initial_position is not None and position > self._initial_position:
|
||||
self._initial_position = position
|
||||
self._record_progress()
|
||||
elapsed = time.monotonic() - self._last_progress_time
|
||||
progress = (
|
||||
"RT controller progress"
|
||||
if self._initial_position is not None
|
||||
else "RT samples received"
|
||||
)
|
||||
scan_timeout = self.scan_timeout
|
||||
if scan_timeout is not None and elapsed >= scan_timeout:
|
||||
return TimeoutError(
|
||||
f"No {progress} on {device_name} for {elapsed:.1f} s "
|
||||
f"(scan_timeout={scan_timeout:g} s)."
|
||||
)
|
||||
data_timeout = self.data_timeout
|
||||
if data_timeout is None or elapsed < data_timeout or self._data_timeout_logged:
|
||||
return
|
||||
self._data_timeout_logged = True
|
||||
logger.warning(
|
||||
f"No {progress} on {device_name} for {elapsed:.1f} s "
|
||||
f"(data_timeout={data_timeout:g} s). "
|
||||
f"Last RT status: mode={mode}, planned_positions={planned_positions}, "
|
||||
f"current_position={position}"
|
||||
)
|
||||
|
||||
|
||||
class RtSignalBase(SocketSignal):
|
||||
def __init__(self, signal_name, **kwargs):
|
||||
self.signal_name = signal_name
|
||||
|
||||
@@ -248,6 +248,7 @@ class FlomniFermatScan(ScanBase):
|
||||
status = self.actions.complete(device=self.dev.rt_positions, wait=False)
|
||||
while not status.done:
|
||||
self.at_each_point()
|
||||
status.wait()
|
||||
|
||||
@scan_hook
|
||||
def at_each_point(self):
|
||||
|
||||
@@ -229,6 +229,7 @@ class LamniFermatScan(ScanBase):
|
||||
while not status.done:
|
||||
self.at_each_point()
|
||||
time.sleep(1)
|
||||
status.wait()
|
||||
|
||||
@scan_hook
|
||||
def at_each_point(self):
|
||||
|
||||
@@ -221,6 +221,7 @@ class OmnyFermatScan(ScanBase):
|
||||
while not status.done:
|
||||
self.at_each_point()
|
||||
time.sleep(1)
|
||||
status.wait()
|
||||
|
||||
@scan_hook
|
||||
def at_each_point(self):
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
"""RT progress warnings and timeouts use the status already polled by the flyer."""
|
||||
|
||||
import importlib
|
||||
import threading
|
||||
from collections import deque
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from csaxs_bec.devices.omny.rt import rt_ophyd
|
||||
|
||||
|
||||
@pytest.fixture(params=[("flomni", "RtFlomni"), ("lamni", "RtLamni"), ("omny", "RtOMNY")])
|
||||
def flyer(request, monkeypatch):
|
||||
suffix, class_prefix = request.param
|
||||
module = importlib.import_module(f"csaxs_bec.devices.omny.rt.rt_{suffix}_ophyd")
|
||||
controller = mock.Mock()
|
||||
controller._lock = threading.RLock()
|
||||
controller._min_scan_buffer_reached = True
|
||||
controller.command_history = deque(["[PUT]: sd", "[GET]: scan started"])
|
||||
controller.get_scan_status.return_value = (2, 10, 6)
|
||||
controller.feedback_is_running.return_value = True
|
||||
controller.socket_put_and_receive.side_effect = lambda command: ",".join(
|
||||
[command[1:]] + ["0"] * 24
|
||||
)
|
||||
monkeypatch.setattr(module, f"{class_prefix}Controller", lambda **kwargs: controller)
|
||||
device = getattr(module, f"{class_prefix}Flyer")(name="rt_positions", data_timeout=3)
|
||||
monkeypatch.setattr(device.data, "set", mock.Mock())
|
||||
monkeypatch.setattr(device.progress, "put", mock.Mock())
|
||||
yield device
|
||||
device.destroy()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock(monkeypatch):
|
||||
clock = SimpleNamespace(now=100)
|
||||
monkeypatch.setattr(rt_ophyd, "time", SimpleNamespace(monotonic=lambda: clock.now))
|
||||
return clock
|
||||
|
||||
|
||||
def run_readout(flyer, clock, states, *, finish=False, failure=False):
|
||||
"""Poll scripted (elapsed seconds, mode, position) without sleeping or hardware."""
|
||||
origin = clock.now
|
||||
states = iter(states)
|
||||
|
||||
def next_status():
|
||||
try:
|
||||
elapsed, mode, position = next(states)
|
||||
except StopIteration:
|
||||
flyer.shutdown_event.set()
|
||||
return (2, 10, 0)
|
||||
clock.now = origin + elapsed
|
||||
return mode, 10, position
|
||||
|
||||
flyer.controller.get_scan_status.side_effect = next_status
|
||||
flyer.shutdown_event = mock.Mock(wraps=threading.Event())
|
||||
flyer.shutdown_event.wait.side_effect = lambda timeout: flyer.shutdown_event.is_set()
|
||||
status = flyer.kickoff()
|
||||
assert status.done and status.success
|
||||
status = flyer.complete()
|
||||
flyer.readout_thread.join(timeout=2)
|
||||
assert not flyer.readout_thread.is_alive()
|
||||
if failure:
|
||||
with pytest.raises(TimeoutError, match="scan_timeout"):
|
||||
status.wait(timeout=1)
|
||||
assert status.done and not status.success
|
||||
else:
|
||||
status.wait(timeout=1)
|
||||
assert status.success
|
||||
if finish:
|
||||
assert not flyer.shutdown_event.is_set()
|
||||
return status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [2, 3])
|
||||
def test_warns_once_at_configured_timeout_without_samples(flyer, clock, mode):
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(flyer, clock, [(0, mode, 0), (2.999, mode, 0)])
|
||||
warning.assert_not_called()
|
||||
|
||||
run_readout(flyer, clock, [(0, mode, 0), (3, mode, 0), (4, mode, 0), (20, mode, 0)])
|
||||
warning.assert_called_once()
|
||||
assert "data_timeout=3 s" in warning.call_args.args[0]
|
||||
assert f"mode={mode}, planned_positions=10, current_position=0" in warning.call_args.args[0]
|
||||
flyer.data.set.assert_not_called()
|
||||
|
||||
|
||||
def test_samples_rearm_warning_and_new_scan_resets_timeout(flyer, clock):
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(
|
||||
flyer, clock, [(0, 2, 0), (3, 2, 0), (4, 2, 6), (6.999, 2, 6), (7, 2, 6), (20, 2, 6)]
|
||||
)
|
||||
assert flyer.data.set.call_count == 5
|
||||
assert warning.call_count == 2
|
||||
warning.reset_mock()
|
||||
run_readout(flyer, clock, [(0, 2, 0), (2.999, 2, 0)])
|
||||
warning.assert_not_called()
|
||||
run_readout(flyer, clock, [(0, 2, 0), (3, 2, 0)])
|
||||
warning.assert_called_once()
|
||||
|
||||
|
||||
def test_available_samples_are_read_before_timeout_check(flyer, clock):
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(flyer, clock, [(0, 2, 0), (4, 2, 6), (6.999, 2, 6)])
|
||||
warning.assert_not_called()
|
||||
assert flyer.data.set.call_count == 5
|
||||
|
||||
|
||||
def test_finished_scan_drains_samples_without_stall_warning(flyer, clock):
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(flyer, clock, [(0, 2, 0), (70, 0, 3)], finish=True)
|
||||
warning.assert_not_called()
|
||||
assert flyer.data.set.call_count == 10
|
||||
|
||||
|
||||
def test_diagnostics_include_status_feedback_and_history(flyer):
|
||||
assert flyer.fetch_diagnostics() == {
|
||||
"scan_status": {"mode": 2, "planned_positions": 10, "current_position": 6},
|
||||
"feedback_running": True,
|
||||
"command_history": ["[PUT]: sd", "[GET]: scan started"],
|
||||
}
|
||||
|
||||
|
||||
def test_diagnostics_preserve_other_fields_when_a_query_fails(flyer):
|
||||
flyer.controller.get_scan_status.side_effect = TimeoutError("status unavailable")
|
||||
diagnostics = flyer.fetch_diagnostics()
|
||||
assert diagnostics["scan_status_error"] == "TimeoutError: status unavailable"
|
||||
assert diagnostics["feedback_running"] is True
|
||||
assert diagnostics["command_history"]
|
||||
flyer.controller.feedback_is_running.side_effect = RuntimeError("feedback unavailable")
|
||||
assert flyer.fetch_diagnostics()["feedback_error"] == "RuntimeError: feedback unavailable"
|
||||
|
||||
|
||||
def test_stopped_device_does_not_warn(flyer, clock):
|
||||
flyer._data_monitor.reset()
|
||||
clock.now += 10
|
||||
flyer.shutdown_event.set()
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
status = mock.Mock()
|
||||
flyer.read_positions_from_sampler(status)
|
||||
status.set_finished.assert_called_once_with()
|
||||
warning.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", [2, 3])
|
||||
def test_scan_timeout_fails_status_without_stopping_axes(flyer, clock, mode):
|
||||
flyer.scan_timeout = 8
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
status = run_readout(
|
||||
flyer, clock, [(0, mode, 0), (3, mode, 0), (7.999, mode, 0), (8, mode, 0)], failure=True
|
||||
)
|
||||
warning.assert_called_once()
|
||||
assert "scan_timeout=8 s" in str(status.exception())
|
||||
assert flyer.controller.get_scan_status.call_count == 4
|
||||
flyer.controller.feedback_is_running.assert_not_called()
|
||||
flyer.controller.socket_put_and_receive.assert_not_called()
|
||||
assert flyer.shutdown_event.is_set()
|
||||
assert flyer.scan_done_event.is_set()
|
||||
flyer.controller.stop_all_axes.assert_not_called()
|
||||
|
||||
|
||||
def test_received_points_reset_scan_timeout(flyer, clock):
|
||||
flyer.scan_timeout = 8
|
||||
status = run_readout(
|
||||
flyer, clock, [(0, 2, 0), (7, 2, 6), (14.999, 2, 6), (15, 2, 6)], failure=True
|
||||
)
|
||||
assert "for 8.0 s" in str(status.exception())
|
||||
assert flyer.data.set.call_count == 5
|
||||
|
||||
|
||||
def test_default_scan_timeout_is_sixty_seconds(flyer, clock):
|
||||
assert flyer.scan_timeout == 60
|
||||
status = run_readout(flyer, clock, [(0, 2, 0), (59.999, 2, 0), (60, 2, 0)], failure=True)
|
||||
assert "scan_timeout=60 s" in str(status.exception())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["data_timeout", "scan_timeout"])
|
||||
@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf")])
|
||||
def test_invalid_timeouts_are_rejected(name, timeout):
|
||||
with pytest.raises(ValueError, match="finite, positive"):
|
||||
rt_ophyd.RtDataMonitor(**{name: timeout})
|
||||
|
||||
|
||||
def test_slow_first_batch_keeps_scan_alive_while_controller_advances(flyer, clock):
|
||||
flyer.data_timeout = 15
|
||||
states = [(0, 2, 0), (11, 2, 1), (22, 2, 2), (33, 2, 3), (44, 2, 4)]
|
||||
states += [(55, 2, 5), (60, 2, 5), (66, 2, 6), (77, 0, 10)]
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(flyer, clock, states, finish=True)
|
||||
warning.assert_not_called()
|
||||
assert flyer.data.set.call_count == 10
|
||||
|
||||
|
||||
def test_initial_position_must_reach_a_new_high_to_reset_timeout(flyer, clock):
|
||||
flyer.scan_timeout = 8
|
||||
states = [(0, 2, 0), (2, 2, 1), (8, 2, 0), (9, 2, 1), (10, 2, 1)]
|
||||
status = run_readout(flyer, clock, states, failure=True)
|
||||
assert "No RT controller progress" in str(status.exception())
|
||||
|
||||
|
||||
def test_controller_progress_cannot_mask_stalled_sample_readout(clock):
|
||||
monitor = rt_ophyd.RtDataMonitor(data_timeout=3, scan_timeout=8)
|
||||
monitor.reset()
|
||||
monitor.sample_received()
|
||||
|
||||
clock.now += 3
|
||||
assert monitor.check("rt_positions", mode=2, planned_positions=10, position=7) is None
|
||||
|
||||
clock.now += 5
|
||||
error = monitor.check("rt_positions", mode=2, planned_positions=10, position=8)
|
||||
assert isinstance(error, TimeoutError)
|
||||
assert "No RT samples received" in str(error)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_timeout", "scan_timeout", "warns", "fails"),
|
||||
[(None, 8, False, True), (3, None, True, False), (None, None, False, False)],
|
||||
)
|
||||
def test_timeouts_can_be_disabled_independently(flyer, clock, data_timeout, scan_timeout, warns, fails):
|
||||
flyer.data_timeout = data_timeout
|
||||
flyer.scan_timeout = scan_timeout
|
||||
states = [(0, 2, 0), (3, 2, 0), (8, 2, 0), (70, 2, 0), (71, 0, 10)]
|
||||
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
|
||||
run_readout(flyer, clock, states, finish=not fails, failure=fails)
|
||||
assert warning.call_count == int(warns)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""RT completion errors must propagate out of the scan's polling loop."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from bec_lib import messages
|
||||
from bec_server.scan_server.errors import DeviceInstructionError
|
||||
from bec_server.scan_server.scans.scan_status import ScanStatus
|
||||
|
||||
from csaxs_bec.scans.flomni_fermat_scan import FlomniFermatScan
|
||||
from csaxs_bec.scans.lamni_fermat_scan import LamniFermatScan
|
||||
from csaxs_bec.scans.omny_fermat_scan import OmnyFermatScan
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scan_cls", [FlomniFermatScan, LamniFermatScan, OmnyFermatScan])
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
def test_rt_scan_checks_completed_readout_result(scan_cls, failed):
|
||||
status = ScanStatus(instruction_handler=mock.Mock())
|
||||
if failed:
|
||||
status.set_failed(
|
||||
messages.ErrorInfo(
|
||||
error_message="No RT scan points received within scan_timeout",
|
||||
compact_error_message="RT scan timed out",
|
||||
exception_type="TimeoutError",
|
||||
device="rt_positions",
|
||||
)
|
||||
)
|
||||
else:
|
||||
status.set_done()
|
||||
scan = SimpleNamespace(
|
||||
actions=mock.Mock(),
|
||||
exp_time=0.1,
|
||||
frames_per_trigger=1,
|
||||
dev=SimpleNamespace(rt_positions="rt_positions"),
|
||||
at_each_point=mock.Mock(),
|
||||
_scan_modifier=None,
|
||||
_scan_modifier_hooks={},
|
||||
)
|
||||
scan.actions._scan_hook_context.side_effect = lambda _: nullcontext()
|
||||
scan.actions.complete.return_value = status
|
||||
if failed:
|
||||
with pytest.raises(DeviceInstructionError, match="RT scan timed out"):
|
||||
scan_cls.scan_core(scan)
|
||||
else:
|
||||
scan_cls.scan_core(scan)
|
||||
scan.actions.complete.assert_called_once_with(device="rt_positions", wait=False)
|
||||
Reference in new issue
Block a user