From ca21145bbe8f92dc2db11413faa7fc70b1c395b0 Mon Sep 17 00:00:00 2001 From: wakonig_k Date: Sat, 19 Sep 2026 09:40:52 +0200 Subject: [PATCH] feat(rt): add data monitor --- csaxs_bec/devices/omny/rt/rt_flomni_ophyd.py | 74 +++++- csaxs_bec/devices/omny/rt/rt_lamni_ophyd.py | 78 ++++++- csaxs_bec/devices/omny/rt/rt_omny_ophyd.py | 72 +++++- csaxs_bec/devices/omny/rt/rt_ophyd.py | 128 ++++++++++- csaxs_bec/scans/flomni_fermat_scan.py | 1 + csaxs_bec/scans/lamni_fermat_scan.py | 1 + csaxs_bec/scans/omny_fermat_scan.py | 1 + tests/tests_devices/test_rt_data_monitor.py | 226 +++++++++++++++++++ tests/tests_scans/test_rt_scan_timeout.py | 48 ++++ 9 files changed, 618 insertions(+), 11 deletions(-) create mode 100644 tests/tests_devices/test_rt_data_monitor.py create mode 100644 tests/tests_scans/test_rt_scan_timeout.py diff --git a/csaxs_bec/devices/omny/rt/rt_flomni_ophyd.py b/csaxs_bec/devices/omny/rt/rt_flomni_ophyd.py index f1e13e14..5b43b93c 100644 --- a/csaxs_bec/devices/omny/rt/rt_flomni_ophyd.py +++ b/csaxs_bec/devices/omny/rt/rt_flomni_ophyd.py @@ -15,6 +15,7 @@ from prettytable import PrettyTable from csaxs_bec.devices.omny.rt.rt_ophyd import ( format_scan_timing, RtCommunicationError, + RtDataMonitor, RtError, RtReadbackSignal, RtSetpointSignal, @@ -485,6 +486,7 @@ class RtFlomniController(Controller): raise RtError("Cannot start scan because no target positions are planned.") # hier exception # start a point-by-point scan (for cont scan in flomni it would be "sa") + logger.info("RtFlomniController: Starting scan.") self.socket_put_and_receive("sd") @retry_once @@ -774,7 +776,7 @@ class RtFlomniMotor(Device, PositionerBase): class RtFlomniFlyer(Device): - USER_ACCESS = ["controller"] + USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"] data = Cpt( AsyncMultiSignal, name="data", @@ -811,9 +813,12 @@ class RtFlomniFlyer(Device): port=2222, socket_cls=SocketIO, device_manager=None, + data_timeout: float | None = 5.0, + scan_timeout: float | None = 60.0, **kwargs, ): super().__init__(prefix=prefix, name=name, parent=parent, **kwargs) + self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout) self.shutdown_event = threading.Event() self.controller = RtFlomniController( socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager @@ -829,6 +834,53 @@ class RtFlomniFlyer(Device): self.scan_done_event = threading.Event() self.scan_done_event.set() + @property + def data_timeout(self) -> float | None: + """Seconds without progress before logging a warning (default: 5). + + Set to None to suppress the warning while leaving scan_timeout independent. + """ + return self._data_monitor.data_timeout + + @data_timeout.setter + def data_timeout(self, value: float | None) -> None: + self._data_monitor.data_timeout = value + + @property + def scan_timeout(self) -> float | None: + """Seconds without progress before failing the scan (default: 60). + + Progress means an advancing controller position until the first sample, + then receipt of new samples. Set to None to disable timeout failures. + Readout can then wait until progress resumes, the controller finishes, + or the scan is stopped. + """ + return self._data_monitor.scan_timeout + + @scan_timeout.setter + def scan_timeout(self, value: float | None) -> None: + self._data_monitor.scan_timeout = value + + def fetch_diagnostics(self) -> dict: + """Return scan status, feedback state, and recent controller commands.""" + diagnostics = {} + try: + mode, planned, current = self.controller.get_scan_status() + diagnostics["scan_status"] = { + "mode": mode, + "planned_positions": planned, + "current_position": current, + } + except Exception as exc: + diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}" + try: + diagnostics["feedback_running"] = self.controller.feedback_is_running() + except Exception as exc: + diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}" + with self.controller._lock: + diagnostics["command_history"] = list(self.controller.command_history) + return diagnostics + def read_positions_from_sampler(self, status: DeviceStatus): """ Read the positions from the sampler and update the data signal. @@ -864,6 +916,18 @@ class RtFlomniFlyer(Device): read_counter = read_counter + 1 signals = self._get_signals_from_table(return_table) self.data.set(signals) + self._data_monitor.sample_received() + + if mode > 0 and not self.shutdown_event.is_set(): + exception = self._data_monitor.check( + self.name, + mode=mode, + planned_positions=number_of_positions_planned, + position=current_position_in_scan, + ) + if exception is not None: + status.set_exception(exception) + return self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics) @@ -958,6 +1022,7 @@ class RtFlomniFlyer(Device): self.scan_done_event.clear() while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001): ... + self._data_monitor.reset() self.controller.start_scan() self._t_kickoff = time.monotonic() self._t_scan_end = None @@ -981,7 +1046,10 @@ class RtFlomniFlyer(Device): def stop(self, *, success=False): self.shutdown_event.set() self.scan_done_event.set() - if self.readout_thread is not None: + if ( + self.readout_thread is not None + and self.readout_thread is not threading.current_thread() + ): self.readout_thread.join() return super().stop(success=success) @@ -991,4 +1059,4 @@ if __name__ == "__main__": socket_cls=SocketIO, socket_host="mpc-flomni.psi.ch", socket_port=2222, device_manager=None ) rtcontroller.on() - rtcontroller.laser_tracker_on() \ No newline at end of file + rtcontroller.laser_tracker_on() diff --git a/csaxs_bec/devices/omny/rt/rt_lamni_ophyd.py b/csaxs_bec/devices/omny/rt/rt_lamni_ophyd.py index 798553a2..d4a26aa2 100644 --- a/csaxs_bec/devices/omny/rt/rt_lamni_ophyd.py +++ b/csaxs_bec/devices/omny/rt/rt_lamni_ophyd.py @@ -13,7 +13,12 @@ from ophyd_devices.utils.controller import Controller, retry_once, threadlocked from ophyd_devices.utils.socket import SocketIO, SocketSignal, raise_if_disconnected from prettytable import PrettyTable -from csaxs_bec.devices.omny.rt.rt_ophyd import RtCommunicationError, RtError, format_scan_timing +from csaxs_bec.devices.omny.rt.rt_ophyd import ( + RtCommunicationError, + RtDataMonitor, + RtError, + format_scan_timing, +) logger = bec_logger.logger @@ -217,6 +222,7 @@ class RtLamniController(Controller): raise RtError("Cannot start scan because no target positions are planned.") # hier exception # start a point-by-point scan (for cont scan in flomni it would be "sa") + logger.info("RtLamniController: Starting scan.") self.socket_put_and_receive("sd") def feedback_is_running(self) -> bool: @@ -694,7 +700,7 @@ class RtLamniMotor(Device, PositionerBase): class RtLamniFlyer(Device): - USER_ACCESS = ["controller"] + USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"] data = Cpt( AsyncMultiSignal, name="data", @@ -740,9 +746,12 @@ class RtLamniFlyer(Device): port=3333, socket_cls=SocketIO, device_manager=None, + data_timeout: float | None = 5.0, + scan_timeout: float | None = 60.0, **kwargs, ): super().__init__(prefix=prefix, name=name, parent=parent, **kwargs) + self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout) self.shutdown_event = threading.Event() self.controller = RtLamniController( socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager @@ -759,6 +768,53 @@ class RtLamniFlyer(Device): self.scan_done_event = threading.Event() self.scan_done_event.set() + @property + def data_timeout(self) -> float | None: + """Seconds without progress before logging a warning (default: 5). + + Set to None to suppress the warning while leaving scan_timeout independent. + """ + return self._data_monitor.data_timeout + + @data_timeout.setter + def data_timeout(self, value: float | None) -> None: + self._data_monitor.data_timeout = value + + @property + def scan_timeout(self) -> float | None: + """Seconds without progress before failing the scan (default: 60). + + Progress means an advancing controller position until the first sample, + then receipt of new samples. Set to None to disable timeout failures. + Readout can then wait until progress resumes, the controller finishes, + or the scan is stopped. + """ + return self._data_monitor.scan_timeout + + @scan_timeout.setter + def scan_timeout(self, value: float | None) -> None: + self._data_monitor.scan_timeout = value + + def fetch_diagnostics(self) -> dict: + """Return scan status, feedback state, and recent controller commands.""" + diagnostics = {} + try: + mode, planned, current = self.controller.get_scan_status() + diagnostics["scan_status"] = { + "mode": mode, + "planned_positions": planned, + "current_position": current, + } + except Exception as exc: + diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}" + try: + diagnostics["feedback_running"] = self.controller.feedback_is_running() + except Exception as exc: + diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}" + with self.controller._lock: + diagnostics["command_history"] = list(self.controller.command_history) + return diagnostics + def stage(self): self.shutdown_event.clear() self.scan_done_event.set() @@ -783,6 +839,7 @@ class RtLamniFlyer(Device): self.scan_done_event.clear() while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001): ... + self._data_monitor.reset() self.controller.start_scan() self._t_kickoff = time.monotonic() self._t_scan_end = None @@ -806,7 +863,10 @@ class RtLamniFlyer(Device): def stop(self, *, success=False): self.shutdown_event.set() self.scan_done_event.set() - if self.readout_thread is not None: + if ( + self.readout_thread is not None + and self.readout_thread is not threading.current_thread() + ): self.readout_thread.join() return super().stop(success=success) @@ -846,6 +906,18 @@ class RtLamniFlyer(Device): read_counter = read_counter + 1 signals = self._get_signals_from_table(return_table) self.data.set(signals) + self._data_monitor.sample_received() + + if mode > 0 and not self.shutdown_event.is_set(): + exception = self._data_monitor.check( + self.name, + mode=mode, + planned_positions=number_of_positions_planned, + position=current_position_in_scan, + ) + if exception is not None: + status.set_exception(exception) + return self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics) diff --git a/csaxs_bec/devices/omny/rt/rt_omny_ophyd.py b/csaxs_bec/devices/omny/rt/rt_omny_ophyd.py index 24804e2b..32ae657a 100644 --- a/csaxs_bec/devices/omny/rt/rt_omny_ophyd.py +++ b/csaxs_bec/devices/omny/rt/rt_omny_ophyd.py @@ -23,6 +23,7 @@ if builtins.__dict__.get("bec") is not None: from csaxs_bec.devices.omny.rt.rt_ophyd import ( RtCommunicationError, + RtDataMonitor, RtError, RtReadbackSignal, RtSetpointSignal, @@ -874,6 +875,7 @@ class RtOMNYController(Controller): raise RtError("Cannot start scan because no target positions are planned.") # hier exception # start a point-by-point scan (for cont scan in flomni it would be "sa") + logger.info("RtOMNYController: Starting scan.") self.socket_put_and_receive("sd") @retry_once @@ -1161,7 +1163,7 @@ class RtOMNYMotor(Device, PositionerBase): class RtOMNYFlyer(Device): - USER_ACCESS = ["controller"] + USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"] data = Cpt( AsyncMultiSignal, @@ -1205,9 +1207,12 @@ class RtOMNYFlyer(Device): port=2222, socket_cls=SocketIO, device_manager=None, + data_timeout: float | None = 5.0, + scan_timeout: float | None = 60.0, **kwargs, ): super().__init__(prefix=prefix, name=name, parent=parent, **kwargs) + self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout) self.shutdown_event = threading.Event() self.controller = RtOMNYController( socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager @@ -1218,6 +1223,53 @@ class RtOMNYFlyer(Device): self.scan_done_event = threading.Event() self.scan_done_event.set() + @property + def data_timeout(self) -> float | None: + """Seconds without progress before logging a warning (default: 5). + + Set to None to suppress the warning while leaving scan_timeout independent. + """ + return self._data_monitor.data_timeout + + @data_timeout.setter + def data_timeout(self, value: float | None) -> None: + self._data_monitor.data_timeout = value + + @property + def scan_timeout(self) -> float | None: + """Seconds without progress before failing the scan (default: 60). + + Progress means an advancing controller position until the first sample, + then receipt of new samples. Set to None to disable timeout failures. + Readout can then wait until progress resumes, the controller finishes, + or the scan is stopped. + """ + return self._data_monitor.scan_timeout + + @scan_timeout.setter + def scan_timeout(self, value: float | None) -> None: + self._data_monitor.scan_timeout = value + + def fetch_diagnostics(self) -> dict: + """Return scan status, feedback state, and recent controller commands.""" + diagnostics = {} + try: + mode, planned, current = self.controller.get_scan_status() + diagnostics["scan_status"] = { + "mode": mode, + "planned_positions": planned, + "current_position": current, + } + except Exception as exc: + diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}" + try: + diagnostics["feedback_running"] = self.controller.feedback_is_running() + except Exception as exc: + diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}" + with self.controller._lock: + diagnostics["command_history"] = list(self.controller.command_history) + return diagnostics + def stage(self): self.shutdown_event.clear() self.scan_done_event.set() @@ -1234,6 +1286,7 @@ class RtOMNYFlyer(Device): self.scan_done_event.clear() while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001): ... + self._data_monitor.reset() self.controller.start_scan() self.shutdown_event.wait(0.1) status = DeviceStatus(self) @@ -1255,7 +1308,10 @@ class RtOMNYFlyer(Device): def stop(self, *, success=False): self.shutdown_event.set() self.scan_done_event.set() - if self.readout_thread is not None: + if ( + self.readout_thread is not None + and self.readout_thread is not threading.current_thread() + ): self.readout_thread.join() return super().stop(success=success) @@ -1292,6 +1348,18 @@ class RtOMNYFlyer(Device): read_counter = read_counter + 1 signals = self._get_signals_from_table(return_table) self.data.set(signals) + self._data_monitor.sample_received() + + if mode > 0 and not self.shutdown_event.is_set(): + exception = self._data_monitor.check( + self.name, + mode=mode, + planned_positions=number_of_positions_planned, + position=current_position_in_scan, + ) + if exception is not None: + status.set_exception(exception) + return if self.shutdown_event.wait(0.05): logger.info("Shutdown event set, stopping readout.") diff --git a/csaxs_bec/devices/omny/rt/rt_ophyd.py b/csaxs_bec/devices/omny/rt/rt_ophyd.py index 839e7a50..85cb85f5 100644 --- a/csaxs_bec/devices/omny/rt/rt_ophyd.py +++ b/csaxs_bec/devices/omny/rt/rt_ophyd.py @@ -1,9 +1,9 @@ """ -This module contains base signals for RT devices. Controller and motors are implemented in the -bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`. +This module contains base signals and data monitoring for RT devices. Controllers and motors are +implemented in bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`. """ -import functools +import math import time from bec_lib import bec_logger @@ -46,6 +46,128 @@ class BECConfigError(Exception): pass +class RtDataMonitor: + """Track gaps in scan progress for an owning RT flyer. + + Both deadlines start at kickoff. While waiting for the first sampler batch, + each new highest controller position resets the deadlines. After the first + sample, each received sample resets them. The readout loop checks both + deadlines after draining available samples, between socket calls. + + Set either timeout to `None` to disable it independently. With + `data_timeout=None`, the monitor skips the stall warning. With + `scan_timeout=None`, a stall can leave readout waiting until progress + resumes, the controller finishes, or the scan is stopped. Other + controller errors and cancellation still apply. + """ + + def __init__( + self, data_timeout: float | None = 5.0, scan_timeout: float | None = 60.0 + ) -> None: + """Configure the warning and failure thresholds for stalled progress. + + Args: + data_timeout (float | None): Seconds without progress before logging one + warning per stall. Defaults to 5 seconds. Pass None to disable warnings. + scan_timeout (float | None): Seconds without progress before failing the + flyer completion status. Defaults to 60 seconds. Pass None to disable + timeout failures. + + Raises: + ValueError: If an enabled timeout is non-finite or less than or equal to zero. + """ + self.data_timeout = data_timeout + self.scan_timeout = scan_timeout + self._last_progress_time: float | None = None + self._initial_position: int | None = None + self._data_timeout_logged = False + + @staticmethod + def _validate_timeout(value: float | None, name: str) -> float | None: + if value is None: + return None + value = float(value) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be a finite, positive number of seconds.") + return value + + @property + def data_timeout(self) -> float | None: + """Seconds without progress before logging a warning, or None to skip the warning.""" + return self._data_timeout + + @data_timeout.setter + def data_timeout(self, value: float | None) -> None: + self._data_timeout = self._validate_timeout(value, "data_timeout") + + @property + def scan_timeout(self) -> float | None: + """Seconds without progress before failing the scan, or None to keep waiting for progress.""" + return self._scan_timeout + + @scan_timeout.setter + def scan_timeout(self, value: float | None) -> None: + self._scan_timeout = self._validate_timeout(value, "scan_timeout") + + def reset(self) -> None: + """Start monitoring a new scan at kickoff.""" + self._initial_position = 0 + self._record_progress() + + def sample_received(self) -> None: + """Switch to monitoring sample receipt and record this sample.""" + self._initial_position = None + self._record_progress() + + def _record_progress(self) -> None: + self._last_progress_time = time.monotonic() + self._data_timeout_logged = False + + def check( + self, device_name: str, *, mode: int, planned_positions: int, position: int + ) -> TimeoutError | None: + """Check the deadlines using the latest controller status. + + Args: + device_name (str): Flyer name used in warnings and errors. + mode (int): Latest controller scan mode. + planned_positions (int): Number of points planned by the controller. + position (int): Current scan position reported by the controller. + + Returns: + TimeoutError | None: Exception to set on the flyer completion status + if the scan timeout expires, or None to continue. A warning can + be logged while readout continues. + """ + if self._last_progress_time is None: + return + if self._initial_position is not None and position > self._initial_position: + self._initial_position = position + self._record_progress() + elapsed = time.monotonic() - self._last_progress_time + progress = ( + "RT controller progress" + if self._initial_position is not None + else "RT samples received" + ) + scan_timeout = self.scan_timeout + if scan_timeout is not None and elapsed >= scan_timeout: + return TimeoutError( + f"No {progress} on {device_name} for {elapsed:.1f} s " + f"(scan_timeout={scan_timeout:g} s)." + ) + data_timeout = self.data_timeout + if data_timeout is None or elapsed < data_timeout or self._data_timeout_logged: + return + self._data_timeout_logged = True + logger.warning( + f"No {progress} on {device_name} for {elapsed:.1f} s " + f"(data_timeout={data_timeout:g} s). " + f"Last RT status: mode={mode}, planned_positions={planned_positions}, " + f"current_position={position}" + ) + + class RtSignalBase(SocketSignal): def __init__(self, signal_name, **kwargs): self.signal_name = signal_name diff --git a/csaxs_bec/scans/flomni_fermat_scan.py b/csaxs_bec/scans/flomni_fermat_scan.py index 8d056a2a..b4a1f236 100644 --- a/csaxs_bec/scans/flomni_fermat_scan.py +++ b/csaxs_bec/scans/flomni_fermat_scan.py @@ -248,6 +248,7 @@ class FlomniFermatScan(ScanBase): status = self.actions.complete(device=self.dev.rt_positions, wait=False) while not status.done: self.at_each_point() + status.wait() @scan_hook def at_each_point(self): diff --git a/csaxs_bec/scans/lamni_fermat_scan.py b/csaxs_bec/scans/lamni_fermat_scan.py index 817347eb..f994246d 100644 --- a/csaxs_bec/scans/lamni_fermat_scan.py +++ b/csaxs_bec/scans/lamni_fermat_scan.py @@ -229,6 +229,7 @@ class LamniFermatScan(ScanBase): while not status.done: self.at_each_point() time.sleep(1) + status.wait() @scan_hook def at_each_point(self): diff --git a/csaxs_bec/scans/omny_fermat_scan.py b/csaxs_bec/scans/omny_fermat_scan.py index cada6751..a6fa5462 100644 --- a/csaxs_bec/scans/omny_fermat_scan.py +++ b/csaxs_bec/scans/omny_fermat_scan.py @@ -221,6 +221,7 @@ class OmnyFermatScan(ScanBase): while not status.done: self.at_each_point() time.sleep(1) + status.wait() @scan_hook def at_each_point(self): diff --git a/tests/tests_devices/test_rt_data_monitor.py b/tests/tests_devices/test_rt_data_monitor.py new file mode 100644 index 00000000..766263ff --- /dev/null +++ b/tests/tests_devices/test_rt_data_monitor.py @@ -0,0 +1,226 @@ +"""RT progress warnings and timeouts use the status already polled by the flyer.""" + +import importlib +import threading +from collections import deque +from types import SimpleNamespace +from unittest import mock + +import pytest + +from csaxs_bec.devices.omny.rt import rt_ophyd + + +@pytest.fixture(params=[("flomni", "RtFlomni"), ("lamni", "RtLamni"), ("omny", "RtOMNY")]) +def flyer(request, monkeypatch): + suffix, class_prefix = request.param + module = importlib.import_module(f"csaxs_bec.devices.omny.rt.rt_{suffix}_ophyd") + controller = mock.Mock() + controller._lock = threading.RLock() + controller._min_scan_buffer_reached = True + controller.command_history = deque(["[PUT]: sd", "[GET]: scan started"]) + controller.get_scan_status.return_value = (2, 10, 6) + controller.feedback_is_running.return_value = True + controller.socket_put_and_receive.side_effect = lambda command: ",".join( + [command[1:]] + ["0"] * 24 + ) + monkeypatch.setattr(module, f"{class_prefix}Controller", lambda **kwargs: controller) + device = getattr(module, f"{class_prefix}Flyer")(name="rt_positions", data_timeout=3) + monkeypatch.setattr(device.data, "set", mock.Mock()) + monkeypatch.setattr(device.progress, "put", mock.Mock()) + yield device + device.destroy() + + +@pytest.fixture +def clock(monkeypatch): + clock = SimpleNamespace(now=100) + monkeypatch.setattr(rt_ophyd, "time", SimpleNamespace(monotonic=lambda: clock.now)) + return clock + + +def run_readout(flyer, clock, states, *, finish=False, failure=False): + """Poll scripted (elapsed seconds, mode, position) without sleeping or hardware.""" + origin = clock.now + states = iter(states) + + def next_status(): + try: + elapsed, mode, position = next(states) + except StopIteration: + flyer.shutdown_event.set() + return (2, 10, 0) + clock.now = origin + elapsed + return mode, 10, position + + flyer.controller.get_scan_status.side_effect = next_status + flyer.shutdown_event = mock.Mock(wraps=threading.Event()) + flyer.shutdown_event.wait.side_effect = lambda timeout: flyer.shutdown_event.is_set() + status = flyer.kickoff() + assert status.done and status.success + status = flyer.complete() + flyer.readout_thread.join(timeout=2) + assert not flyer.readout_thread.is_alive() + if failure: + with pytest.raises(TimeoutError, match="scan_timeout"): + status.wait(timeout=1) + assert status.done and not status.success + else: + status.wait(timeout=1) + assert status.success + if finish: + assert not flyer.shutdown_event.is_set() + return status + + +@pytest.mark.parametrize("mode", [2, 3]) +def test_warns_once_at_configured_timeout_without_samples(flyer, clock, mode): + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout(flyer, clock, [(0, mode, 0), (2.999, mode, 0)]) + warning.assert_not_called() + + run_readout(flyer, clock, [(0, mode, 0), (3, mode, 0), (4, mode, 0), (20, mode, 0)]) + warning.assert_called_once() + assert "data_timeout=3 s" in warning.call_args.args[0] + assert f"mode={mode}, planned_positions=10, current_position=0" in warning.call_args.args[0] + flyer.data.set.assert_not_called() + + +def test_samples_rearm_warning_and_new_scan_resets_timeout(flyer, clock): + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout( + flyer, clock, [(0, 2, 0), (3, 2, 0), (4, 2, 6), (6.999, 2, 6), (7, 2, 6), (20, 2, 6)] + ) + assert flyer.data.set.call_count == 5 + assert warning.call_count == 2 + warning.reset_mock() + run_readout(flyer, clock, [(0, 2, 0), (2.999, 2, 0)]) + warning.assert_not_called() + run_readout(flyer, clock, [(0, 2, 0), (3, 2, 0)]) + warning.assert_called_once() + + +def test_available_samples_are_read_before_timeout_check(flyer, clock): + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout(flyer, clock, [(0, 2, 0), (4, 2, 6), (6.999, 2, 6)]) + warning.assert_not_called() + assert flyer.data.set.call_count == 5 + + +def test_finished_scan_drains_samples_without_stall_warning(flyer, clock): + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout(flyer, clock, [(0, 2, 0), (70, 0, 3)], finish=True) + warning.assert_not_called() + assert flyer.data.set.call_count == 10 + + +def test_diagnostics_include_status_feedback_and_history(flyer): + assert flyer.fetch_diagnostics() == { + "scan_status": {"mode": 2, "planned_positions": 10, "current_position": 6}, + "feedback_running": True, + "command_history": ["[PUT]: sd", "[GET]: scan started"], + } + + +def test_diagnostics_preserve_other_fields_when_a_query_fails(flyer): + flyer.controller.get_scan_status.side_effect = TimeoutError("status unavailable") + diagnostics = flyer.fetch_diagnostics() + assert diagnostics["scan_status_error"] == "TimeoutError: status unavailable" + assert diagnostics["feedback_running"] is True + assert diagnostics["command_history"] + flyer.controller.feedback_is_running.side_effect = RuntimeError("feedback unavailable") + assert flyer.fetch_diagnostics()["feedback_error"] == "RuntimeError: feedback unavailable" + + +def test_stopped_device_does_not_warn(flyer, clock): + flyer._data_monitor.reset() + clock.now += 10 + flyer.shutdown_event.set() + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + status = mock.Mock() + flyer.read_positions_from_sampler(status) + status.set_finished.assert_called_once_with() + warning.assert_not_called() + + +@pytest.mark.parametrize("mode", [2, 3]) +def test_scan_timeout_fails_status_without_stopping_axes(flyer, clock, mode): + flyer.scan_timeout = 8 + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + status = run_readout( + flyer, clock, [(0, mode, 0), (3, mode, 0), (7.999, mode, 0), (8, mode, 0)], failure=True + ) + warning.assert_called_once() + assert "scan_timeout=8 s" in str(status.exception()) + assert flyer.controller.get_scan_status.call_count == 4 + flyer.controller.feedback_is_running.assert_not_called() + flyer.controller.socket_put_and_receive.assert_not_called() + assert flyer.shutdown_event.is_set() + assert flyer.scan_done_event.is_set() + flyer.controller.stop_all_axes.assert_not_called() + + +def test_received_points_reset_scan_timeout(flyer, clock): + flyer.scan_timeout = 8 + status = run_readout( + flyer, clock, [(0, 2, 0), (7, 2, 6), (14.999, 2, 6), (15, 2, 6)], failure=True + ) + assert "for 8.0 s" in str(status.exception()) + assert flyer.data.set.call_count == 5 + + +def test_default_scan_timeout_is_sixty_seconds(flyer, clock): + assert flyer.scan_timeout == 60 + status = run_readout(flyer, clock, [(0, 2, 0), (59.999, 2, 0), (60, 2, 0)], failure=True) + assert "scan_timeout=60 s" in str(status.exception()) + + +@pytest.mark.parametrize("name", ["data_timeout", "scan_timeout"]) +@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf")]) +def test_invalid_timeouts_are_rejected(name, timeout): + with pytest.raises(ValueError, match="finite, positive"): + rt_ophyd.RtDataMonitor(**{name: timeout}) + + +def test_slow_first_batch_keeps_scan_alive_while_controller_advances(flyer, clock): + flyer.data_timeout = 15 + states = [(0, 2, 0), (11, 2, 1), (22, 2, 2), (33, 2, 3), (44, 2, 4)] + states += [(55, 2, 5), (60, 2, 5), (66, 2, 6), (77, 0, 10)] + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout(flyer, clock, states, finish=True) + warning.assert_not_called() + assert flyer.data.set.call_count == 10 + + +def test_initial_position_must_reach_a_new_high_to_reset_timeout(flyer, clock): + flyer.scan_timeout = 8 + states = [(0, 2, 0), (2, 2, 1), (8, 2, 0), (9, 2, 1), (10, 2, 1)] + status = run_readout(flyer, clock, states, failure=True) + assert "No RT controller progress" in str(status.exception()) + + +def test_controller_progress_cannot_mask_stalled_sample_readout(clock): + monitor = rt_ophyd.RtDataMonitor(data_timeout=3, scan_timeout=8) + monitor.reset() + monitor.sample_received() + + clock.now += 3 + assert monitor.check("rt_positions", mode=2, planned_positions=10, position=7) is None + + clock.now += 5 + error = monitor.check("rt_positions", mode=2, planned_positions=10, position=8) + assert isinstance(error, TimeoutError) + assert "No RT samples received" in str(error) + + +@pytest.mark.parametrize( + ("data_timeout", "scan_timeout", "warns", "fails"), + [(None, 8, False, True), (3, None, True, False), (None, None, False, False)], +) +def test_timeouts_can_be_disabled_independently(flyer, clock, data_timeout, scan_timeout, warns, fails): + flyer.data_timeout = data_timeout + flyer.scan_timeout = scan_timeout + states = [(0, 2, 0), (3, 2, 0), (8, 2, 0), (70, 2, 0), (71, 0, 10)] + with mock.patch.object(rt_ophyd.logger, "warning") as warning: + run_readout(flyer, clock, states, finish=not fails, failure=fails) + assert warning.call_count == int(warns) diff --git a/tests/tests_scans/test_rt_scan_timeout.py b/tests/tests_scans/test_rt_scan_timeout.py new file mode 100644 index 00000000..231e3a58 --- /dev/null +++ b/tests/tests_scans/test_rt_scan_timeout.py @@ -0,0 +1,48 @@ +"""RT completion errors must propagate out of the scan's polling loop.""" + +from contextlib import nullcontext +from types import SimpleNamespace +from unittest import mock + +import pytest +from bec_lib import messages +from bec_server.scan_server.errors import DeviceInstructionError +from bec_server.scan_server.scans.scan_status import ScanStatus + +from csaxs_bec.scans.flomni_fermat_scan import FlomniFermatScan +from csaxs_bec.scans.lamni_fermat_scan import LamniFermatScan +from csaxs_bec.scans.omny_fermat_scan import OmnyFermatScan + + +@pytest.mark.parametrize("scan_cls", [FlomniFermatScan, LamniFermatScan, OmnyFermatScan]) +@pytest.mark.parametrize("failed", [False, True]) +def test_rt_scan_checks_completed_readout_result(scan_cls, failed): + status = ScanStatus(instruction_handler=mock.Mock()) + if failed: + status.set_failed( + messages.ErrorInfo( + error_message="No RT scan points received within scan_timeout", + compact_error_message="RT scan timed out", + exception_type="TimeoutError", + device="rt_positions", + ) + ) + else: + status.set_done() + scan = SimpleNamespace( + actions=mock.Mock(), + exp_time=0.1, + frames_per_trigger=1, + dev=SimpleNamespace(rt_positions="rt_positions"), + at_each_point=mock.Mock(), + _scan_modifier=None, + _scan_modifier_hooks={}, + ) + scan.actions._scan_hook_context.side_effect = lambda _: nullcontext() + scan.actions.complete.return_value = status + if failed: + with pytest.raises(DeviceInstructionError, match="RT scan timed out"): + scan_cls.scan_core(scan) + else: + scan_cls.scan_core(scan) + scan.actions.complete.assert_called_once_with(device="rt_positions", wait=False)