feat(rt): add data monitor
CI for csaxs_bec / test (push) Successful in 3m48s
CI for csaxs_bec / test (pull_request) Successful in 3m59s

This commit is contained in:
wakonig_k committed 2026-11-16 16:42:52 +01:00
1 parent fb6e9a2163
commit ca21145bbe
9 files changed
+618 -11

No files matched your search

+71 -3
View File
@@ -15,6 +15,7 @@ from prettytable import PrettyTable
from csaxs_bec.devices.omny.rt.rt_ophyd import (
format_scan_timing,
RtCommunicationError,
RtDataMonitor,
RtError,
RtReadbackSignal,
RtSetpointSignal,
@@ -485,6 +486,7 @@ class RtFlomniController(Controller):
raise RtError("Cannot start scan because no target positions are planned.")
# hier exception
# start a point-by-point scan (for cont scan in flomni it would be "sa")
logger.info("RtFlomniController: Starting scan.")
self.socket_put_and_receive("sd")
@retry_once
@@ -774,7 +776,7 @@ class RtFlomniMotor(Device, PositionerBase):
class RtFlomniFlyer(Device):
USER_ACCESS = ["controller"]
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
data = Cpt(
AsyncMultiSignal,
name="data",
@@ -811,9 +813,12 @@ class RtFlomniFlyer(Device):
port=2222,
socket_cls=SocketIO,
device_manager=None,
data_timeout: float | None = 5.0,
scan_timeout: float | None = 60.0,
**kwargs,
):
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
self.shutdown_event = threading.Event()
self.controller = RtFlomniController(
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
@@ -829,6 +834,53 @@ class RtFlomniFlyer(Device):
self.scan_done_event = threading.Event()
self.scan_done_event.set()
@property
def data_timeout(self) -> float | None:
"""Seconds without progress before logging a warning (default: 5).
Set to None to suppress the warning while leaving scan_timeout independent.
"""
return self._data_monitor.data_timeout
@data_timeout.setter
def data_timeout(self, value: float | None) -> None:
self._data_monitor.data_timeout = value
@property
def scan_timeout(self) -> float | None:
"""Seconds without progress before failing the scan (default: 60).
Progress means an advancing controller position until the first sample,
then receipt of new samples. Set to None to disable timeout failures.
Readout can then wait until progress resumes, the controller finishes,
or the scan is stopped.
"""
return self._data_monitor.scan_timeout
@scan_timeout.setter
def scan_timeout(self, value: float | None) -> None:
self._data_monitor.scan_timeout = value
def fetch_diagnostics(self) -> dict:
"""Return scan status, feedback state, and recent controller commands."""
diagnostics = {}
try:
mode, planned, current = self.controller.get_scan_status()
diagnostics["scan_status"] = {
"mode": mode,
"planned_positions": planned,
"current_position": current,
}
except Exception as exc:
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
try:
diagnostics["feedback_running"] = self.controller.feedback_is_running()
except Exception as exc:
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
with self.controller._lock:
diagnostics["command_history"] = list(self.controller.command_history)
return diagnostics
def read_positions_from_sampler(self, status: DeviceStatus):
"""
Read the positions from the sampler and update the data signal.
@@ -864,6 +916,18 @@ class RtFlomniFlyer(Device):
read_counter = read_counter + 1
signals = self._get_signals_from_table(return_table)
self.data.set(signals)
self._data_monitor.sample_received()
if mode > 0 and not self.shutdown_event.is_set():
exception = self._data_monitor.check(
self.name,
mode=mode,
planned_positions=number_of_positions_planned,
position=current_position_in_scan,
)
if exception is not None:
status.set_exception(exception)
return
self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics)
@@ -958,6 +1022,7 @@ class RtFlomniFlyer(Device):
self.scan_done_event.clear()
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
...
self._data_monitor.reset()
self.controller.start_scan()
self._t_kickoff = time.monotonic()
self._t_scan_end = None
@@ -981,7 +1046,10 @@ class RtFlomniFlyer(Device):
def stop(self, *, success=False):
self.shutdown_event.set()
self.scan_done_event.set()
if self.readout_thread is not None:
if (
self.readout_thread is not None
and self.readout_thread is not threading.current_thread()
):
self.readout_thread.join()
return super().stop(success=success)
@@ -991,4 +1059,4 @@ if __name__ == "__main__":
socket_cls=SocketIO, socket_host="mpc-flomni.psi.ch", socket_port=2222, device_manager=None
)
rtcontroller.on()
rtcontroller.laser_tracker_on()
rtcontroller.laser_tracker_on()
+75 -3
View File
@@ -13,7 +13,12 @@ from ophyd_devices.utils.controller import Controller, retry_once, threadlocked
from ophyd_devices.utils.socket import SocketIO, SocketSignal, raise_if_disconnected
from prettytable import PrettyTable
from csaxs_bec.devices.omny.rt.rt_ophyd import RtCommunicationError, RtError, format_scan_timing
from csaxs_bec.devices.omny.rt.rt_ophyd import (
RtCommunicationError,
RtDataMonitor,
RtError,
format_scan_timing,
)
logger = bec_logger.logger
@@ -217,6 +222,7 @@ class RtLamniController(Controller):
raise RtError("Cannot start scan because no target positions are planned.")
# hier exception
# start a point-by-point scan (for cont scan in flomni it would be "sa")
logger.info("RtLamniController: Starting scan.")
self.socket_put_and_receive("sd")
def feedback_is_running(self) -> bool:
@@ -694,7 +700,7 @@ class RtLamniMotor(Device, PositionerBase):
class RtLamniFlyer(Device):
USER_ACCESS = ["controller"]
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
data = Cpt(
AsyncMultiSignal,
name="data",
@@ -740,9 +746,12 @@ class RtLamniFlyer(Device):
port=3333,
socket_cls=SocketIO,
device_manager=None,
data_timeout: float | None = 5.0,
scan_timeout: float | None = 60.0,
**kwargs,
):
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
self.shutdown_event = threading.Event()
self.controller = RtLamniController(
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
@@ -759,6 +768,53 @@ class RtLamniFlyer(Device):
self.scan_done_event = threading.Event()
self.scan_done_event.set()
@property
def data_timeout(self) -> float | None:
"""Seconds without progress before logging a warning (default: 5).
Set to None to suppress the warning while leaving scan_timeout independent.
"""
return self._data_monitor.data_timeout
@data_timeout.setter
def data_timeout(self, value: float | None) -> None:
self._data_monitor.data_timeout = value
@property
def scan_timeout(self) -> float | None:
"""Seconds without progress before failing the scan (default: 60).
Progress means an advancing controller position until the first sample,
then receipt of new samples. Set to None to disable timeout failures.
Readout can then wait until progress resumes, the controller finishes,
or the scan is stopped.
"""
return self._data_monitor.scan_timeout
@scan_timeout.setter
def scan_timeout(self, value: float | None) -> None:
self._data_monitor.scan_timeout = value
def fetch_diagnostics(self) -> dict:
"""Return scan status, feedback state, and recent controller commands."""
diagnostics = {}
try:
mode, planned, current = self.controller.get_scan_status()
diagnostics["scan_status"] = {
"mode": mode,
"planned_positions": planned,
"current_position": current,
}
except Exception as exc:
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
try:
diagnostics["feedback_running"] = self.controller.feedback_is_running()
except Exception as exc:
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
with self.controller._lock:
diagnostics["command_history"] = list(self.controller.command_history)
return diagnostics
def stage(self):
self.shutdown_event.clear()
self.scan_done_event.set()
@@ -783,6 +839,7 @@ class RtLamniFlyer(Device):
self.scan_done_event.clear()
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
...
self._data_monitor.reset()
self.controller.start_scan()
self._t_kickoff = time.monotonic()
self._t_scan_end = None
@@ -806,7 +863,10 @@ class RtLamniFlyer(Device):
def stop(self, *, success=False):
self.shutdown_event.set()
self.scan_done_event.set()
if self.readout_thread is not None:
if (
self.readout_thread is not None
and self.readout_thread is not threading.current_thread()
):
self.readout_thread.join()
return super().stop(success=success)
@@ -846,6 +906,18 @@ class RtLamniFlyer(Device):
read_counter = read_counter + 1
signals = self._get_signals_from_table(return_table)
self.data.set(signals)
self._data_monitor.sample_received()
if mode > 0 and not self.shutdown_event.is_set():
exception = self._data_monitor.check(
self.name,
mode=mode,
planned_positions=number_of_positions_planned,
position=current_position_in_scan,
)
if exception is not None:
status.set_exception(exception)
return
self._t_scan_end = time.monotonic() # scan finished in the RT loop (statistics)
+70 -2
View File
@@ -23,6 +23,7 @@ if builtins.__dict__.get("bec") is not None:
from csaxs_bec.devices.omny.rt.rt_ophyd import (
RtCommunicationError,
RtDataMonitor,
RtError,
RtReadbackSignal,
RtSetpointSignal,
@@ -874,6 +875,7 @@ class RtOMNYController(Controller):
raise RtError("Cannot start scan because no target positions are planned.")
# hier exception
# start a point-by-point scan (for cont scan in flomni it would be "sa")
logger.info("RtOMNYController: Starting scan.")
self.socket_put_and_receive("sd")
@retry_once
@@ -1161,7 +1163,7 @@ class RtOMNYMotor(Device, PositionerBase):
class RtOMNYFlyer(Device):
USER_ACCESS = ["controller"]
USER_ACCESS = ["controller", "data_timeout", "scan_timeout", "fetch_diagnostics"]
data = Cpt(
AsyncMultiSignal,
@@ -1205,9 +1207,12 @@ class RtOMNYFlyer(Device):
port=2222,
socket_cls=SocketIO,
device_manager=None,
data_timeout: float | None = 5.0,
scan_timeout: float | None = 60.0,
**kwargs,
):
super().__init__(prefix=prefix, name=name, parent=parent, **kwargs)
self._data_monitor = RtDataMonitor(data_timeout=data_timeout, scan_timeout=scan_timeout)
self.shutdown_event = threading.Event()
self.controller = RtOMNYController(
socket_cls=socket_cls, socket_host=host, socket_port=port, device_manager=device_manager
@@ -1218,6 +1223,53 @@ class RtOMNYFlyer(Device):
self.scan_done_event = threading.Event()
self.scan_done_event.set()
@property
def data_timeout(self) -> float | None:
"""Seconds without progress before logging a warning (default: 5).
Set to None to suppress the warning while leaving scan_timeout independent.
"""
return self._data_monitor.data_timeout
@data_timeout.setter
def data_timeout(self, value: float | None) -> None:
self._data_monitor.data_timeout = value
@property
def scan_timeout(self) -> float | None:
"""Seconds without progress before failing the scan (default: 60).
Progress means an advancing controller position until the first sample,
then receipt of new samples. Set to None to disable timeout failures.
Readout can then wait until progress resumes, the controller finishes,
or the scan is stopped.
"""
return self._data_monitor.scan_timeout
@scan_timeout.setter
def scan_timeout(self, value: float | None) -> None:
self._data_monitor.scan_timeout = value
def fetch_diagnostics(self) -> dict:
"""Return scan status, feedback state, and recent controller commands."""
diagnostics = {}
try:
mode, planned, current = self.controller.get_scan_status()
diagnostics["scan_status"] = {
"mode": mode,
"planned_positions": planned,
"current_position": current,
}
except Exception as exc:
diagnostics["scan_status_error"] = f"{type(exc).__name__}: {exc}"
try:
diagnostics["feedback_running"] = self.controller.feedback_is_running()
except Exception as exc:
diagnostics["feedback_error"] = f"{type(exc).__name__}: {exc}"
with self.controller._lock:
diagnostics["command_history"] = list(self.controller.command_history)
return diagnostics
def stage(self):
self.shutdown_event.clear()
self.scan_done_event.set()
@@ -1234,6 +1286,7 @@ class RtOMNYFlyer(Device):
self.scan_done_event.clear()
while not self.controller._min_scan_buffer_reached and not self.shutdown_event.wait(0.001):
...
self._data_monitor.reset()
self.controller.start_scan()
self.shutdown_event.wait(0.1)
status = DeviceStatus(self)
@@ -1255,7 +1308,10 @@ class RtOMNYFlyer(Device):
def stop(self, *, success=False):
self.shutdown_event.set()
self.scan_done_event.set()
if self.readout_thread is not None:
if (
self.readout_thread is not None
and self.readout_thread is not threading.current_thread()
):
self.readout_thread.join()
return super().stop(success=success)
@@ -1292,6 +1348,18 @@ class RtOMNYFlyer(Device):
read_counter = read_counter + 1
signals = self._get_signals_from_table(return_table)
self.data.set(signals)
self._data_monitor.sample_received()
if mode > 0 and not self.shutdown_event.is_set():
exception = self._data_monitor.check(
self.name,
mode=mode,
planned_positions=number_of_positions_planned,
position=current_position_in_scan,
)
if exception is not None:
status.set_exception(exception)
return
if self.shutdown_event.wait(0.05):
logger.info("Shutdown event set, stopping readout.")
+125 -3
View File
@@ -1,9 +1,9 @@
"""
This module contains base signals for RT devices. Controller and motors are implemented in the
bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`.
This module contains base signals and data monitoring for RT devices. Controllers and motors are
implemented in bespoke modules such as `rt_flomni_ophyd.py` or `rt_lamni_ophyd.py`.
"""
import functools
import math
import time
from bec_lib import bec_logger
@@ -46,6 +46,128 @@ class BECConfigError(Exception):
pass
class RtDataMonitor:
"""Track gaps in scan progress for an owning RT flyer.
Both deadlines start at kickoff. While waiting for the first sampler batch,
each new highest controller position resets the deadlines. After the first
sample, each received sample resets them. The readout loop checks both
deadlines after draining available samples, between socket calls.
Set either timeout to `None` to disable it independently. With
`data_timeout=None`, the monitor skips the stall warning. With
`scan_timeout=None`, a stall can leave readout waiting until progress
resumes, the controller finishes, or the scan is stopped. Other
controller errors and cancellation still apply.
"""
def __init__(
self, data_timeout: float | None = 5.0, scan_timeout: float | None = 60.0
) -> None:
"""Configure the warning and failure thresholds for stalled progress.
Args:
data_timeout (float | None): Seconds without progress before logging one
warning per stall. Defaults to 5 seconds. Pass None to disable warnings.
scan_timeout (float | None): Seconds without progress before failing the
flyer completion status. Defaults to 60 seconds. Pass None to disable
timeout failures.
Raises:
ValueError: If an enabled timeout is non-finite or less than or equal to zero.
"""
self.data_timeout = data_timeout
self.scan_timeout = scan_timeout
self._last_progress_time: float | None = None
self._initial_position: int | None = None
self._data_timeout_logged = False
@staticmethod
def _validate_timeout(value: float | None, name: str) -> float | None:
if value is None:
return None
value = float(value)
if not math.isfinite(value) or value <= 0:
raise ValueError(f"{name} must be a finite, positive number of seconds.")
return value
@property
def data_timeout(self) -> float | None:
"""Seconds without progress before logging a warning, or None to skip the warning."""
return self._data_timeout
@data_timeout.setter
def data_timeout(self, value: float | None) -> None:
self._data_timeout = self._validate_timeout(value, "data_timeout")
@property
def scan_timeout(self) -> float | None:
"""Seconds without progress before failing the scan, or None to keep waiting for progress."""
return self._scan_timeout
@scan_timeout.setter
def scan_timeout(self, value: float | None) -> None:
self._scan_timeout = self._validate_timeout(value, "scan_timeout")
def reset(self) -> None:
"""Start monitoring a new scan at kickoff."""
self._initial_position = 0
self._record_progress()
def sample_received(self) -> None:
"""Switch to monitoring sample receipt and record this sample."""
self._initial_position = None
self._record_progress()
def _record_progress(self) -> None:
self._last_progress_time = time.monotonic()
self._data_timeout_logged = False
def check(
self, device_name: str, *, mode: int, planned_positions: int, position: int
) -> TimeoutError | None:
"""Check the deadlines using the latest controller status.
Args:
device_name (str): Flyer name used in warnings and errors.
mode (int): Latest controller scan mode.
planned_positions (int): Number of points planned by the controller.
position (int): Current scan position reported by the controller.
Returns:
TimeoutError | None: Exception to set on the flyer completion status
if the scan timeout expires, or None to continue. A warning can
be logged while readout continues.
"""
if self._last_progress_time is None:
return
if self._initial_position is not None and position > self._initial_position:
self._initial_position = position
self._record_progress()
elapsed = time.monotonic() - self._last_progress_time
progress = (
"RT controller progress"
if self._initial_position is not None
else "RT samples received"
)
scan_timeout = self.scan_timeout
if scan_timeout is not None and elapsed >= scan_timeout:
return TimeoutError(
f"No {progress} on {device_name} for {elapsed:.1f} s "
f"(scan_timeout={scan_timeout:g} s)."
)
data_timeout = self.data_timeout
if data_timeout is None or elapsed < data_timeout or self._data_timeout_logged:
return
self._data_timeout_logged = True
logger.warning(
f"No {progress} on {device_name} for {elapsed:.1f} s "
f"(data_timeout={data_timeout:g} s). "
f"Last RT status: mode={mode}, planned_positions={planned_positions}, "
f"current_position={position}"
)
class RtSignalBase(SocketSignal):
def __init__(self, signal_name, **kwargs):
self.signal_name = signal_name
+1
View File
@@ -248,6 +248,7 @@ class FlomniFermatScan(ScanBase):
status = self.actions.complete(device=self.dev.rt_positions, wait=False)
while not status.done:
self.at_each_point()
status.wait()
@scan_hook
def at_each_point(self):
+1
View File
@@ -229,6 +229,7 @@ class LamniFermatScan(ScanBase):
while not status.done:
self.at_each_point()
time.sleep(1)
status.wait()
@scan_hook
def at_each_point(self):
+1
View File
@@ -221,6 +221,7 @@ class OmnyFermatScan(ScanBase):
while not status.done:
self.at_each_point()
time.sleep(1)
status.wait()
@scan_hook
def at_each_point(self):
+226
View File
@@ -0,0 +1,226 @@
"""RT progress warnings and timeouts use the status already polled by the flyer."""
import importlib
import threading
from collections import deque
from types import SimpleNamespace
from unittest import mock
import pytest
from csaxs_bec.devices.omny.rt import rt_ophyd
@pytest.fixture(params=[("flomni", "RtFlomni"), ("lamni", "RtLamni"), ("omny", "RtOMNY")])
def flyer(request, monkeypatch):
suffix, class_prefix = request.param
module = importlib.import_module(f"csaxs_bec.devices.omny.rt.rt_{suffix}_ophyd")
controller = mock.Mock()
controller._lock = threading.RLock()
controller._min_scan_buffer_reached = True
controller.command_history = deque(["[PUT]: sd", "[GET]: scan started"])
controller.get_scan_status.return_value = (2, 10, 6)
controller.feedback_is_running.return_value = True
controller.socket_put_and_receive.side_effect = lambda command: ",".join(
[command[1:]] + ["0"] * 24
)
monkeypatch.setattr(module, f"{class_prefix}Controller", lambda **kwargs: controller)
device = getattr(module, f"{class_prefix}Flyer")(name="rt_positions", data_timeout=3)
monkeypatch.setattr(device.data, "set", mock.Mock())
monkeypatch.setattr(device.progress, "put", mock.Mock())
yield device
device.destroy()
@pytest.fixture
def clock(monkeypatch):
clock = SimpleNamespace(now=100)
monkeypatch.setattr(rt_ophyd, "time", SimpleNamespace(monotonic=lambda: clock.now))
return clock
def run_readout(flyer, clock, states, *, finish=False, failure=False):
"""Poll scripted (elapsed seconds, mode, position) without sleeping or hardware."""
origin = clock.now
states = iter(states)
def next_status():
try:
elapsed, mode, position = next(states)
except StopIteration:
flyer.shutdown_event.set()
return (2, 10, 0)
clock.now = origin + elapsed
return mode, 10, position
flyer.controller.get_scan_status.side_effect = next_status
flyer.shutdown_event = mock.Mock(wraps=threading.Event())
flyer.shutdown_event.wait.side_effect = lambda timeout: flyer.shutdown_event.is_set()
status = flyer.kickoff()
assert status.done and status.success
status = flyer.complete()
flyer.readout_thread.join(timeout=2)
assert not flyer.readout_thread.is_alive()
if failure:
with pytest.raises(TimeoutError, match="scan_timeout"):
status.wait(timeout=1)
assert status.done and not status.success
else:
status.wait(timeout=1)
assert status.success
if finish:
assert not flyer.shutdown_event.is_set()
return status
@pytest.mark.parametrize("mode", [2, 3])
def test_warns_once_at_configured_timeout_without_samples(flyer, clock, mode):
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(flyer, clock, [(0, mode, 0), (2.999, mode, 0)])
warning.assert_not_called()
run_readout(flyer, clock, [(0, mode, 0), (3, mode, 0), (4, mode, 0), (20, mode, 0)])
warning.assert_called_once()
assert "data_timeout=3 s" in warning.call_args.args[0]
assert f"mode={mode}, planned_positions=10, current_position=0" in warning.call_args.args[0]
flyer.data.set.assert_not_called()
def test_samples_rearm_warning_and_new_scan_resets_timeout(flyer, clock):
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(
flyer, clock, [(0, 2, 0), (3, 2, 0), (4, 2, 6), (6.999, 2, 6), (7, 2, 6), (20, 2, 6)]
)
assert flyer.data.set.call_count == 5
assert warning.call_count == 2
warning.reset_mock()
run_readout(flyer, clock, [(0, 2, 0), (2.999, 2, 0)])
warning.assert_not_called()
run_readout(flyer, clock, [(0, 2, 0), (3, 2, 0)])
warning.assert_called_once()
def test_available_samples_are_read_before_timeout_check(flyer, clock):
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(flyer, clock, [(0, 2, 0), (4, 2, 6), (6.999, 2, 6)])
warning.assert_not_called()
assert flyer.data.set.call_count == 5
def test_finished_scan_drains_samples_without_stall_warning(flyer, clock):
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(flyer, clock, [(0, 2, 0), (70, 0, 3)], finish=True)
warning.assert_not_called()
assert flyer.data.set.call_count == 10
def test_diagnostics_include_status_feedback_and_history(flyer):
assert flyer.fetch_diagnostics() == {
"scan_status": {"mode": 2, "planned_positions": 10, "current_position": 6},
"feedback_running": True,
"command_history": ["[PUT]: sd", "[GET]: scan started"],
}
def test_diagnostics_preserve_other_fields_when_a_query_fails(flyer):
flyer.controller.get_scan_status.side_effect = TimeoutError("status unavailable")
diagnostics = flyer.fetch_diagnostics()
assert diagnostics["scan_status_error"] == "TimeoutError: status unavailable"
assert diagnostics["feedback_running"] is True
assert diagnostics["command_history"]
flyer.controller.feedback_is_running.side_effect = RuntimeError("feedback unavailable")
assert flyer.fetch_diagnostics()["feedback_error"] == "RuntimeError: feedback unavailable"
def test_stopped_device_does_not_warn(flyer, clock):
flyer._data_monitor.reset()
clock.now += 10
flyer.shutdown_event.set()
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
status = mock.Mock()
flyer.read_positions_from_sampler(status)
status.set_finished.assert_called_once_with()
warning.assert_not_called()
@pytest.mark.parametrize("mode", [2, 3])
def test_scan_timeout_fails_status_without_stopping_axes(flyer, clock, mode):
flyer.scan_timeout = 8
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
status = run_readout(
flyer, clock, [(0, mode, 0), (3, mode, 0), (7.999, mode, 0), (8, mode, 0)], failure=True
)
warning.assert_called_once()
assert "scan_timeout=8 s" in str(status.exception())
assert flyer.controller.get_scan_status.call_count == 4
flyer.controller.feedback_is_running.assert_not_called()
flyer.controller.socket_put_and_receive.assert_not_called()
assert flyer.shutdown_event.is_set()
assert flyer.scan_done_event.is_set()
flyer.controller.stop_all_axes.assert_not_called()
def test_received_points_reset_scan_timeout(flyer, clock):
flyer.scan_timeout = 8
status = run_readout(
flyer, clock, [(0, 2, 0), (7, 2, 6), (14.999, 2, 6), (15, 2, 6)], failure=True
)
assert "for 8.0 s" in str(status.exception())
assert flyer.data.set.call_count == 5
def test_default_scan_timeout_is_sixty_seconds(flyer, clock):
assert flyer.scan_timeout == 60
status = run_readout(flyer, clock, [(0, 2, 0), (59.999, 2, 0), (60, 2, 0)], failure=True)
assert "scan_timeout=60 s" in str(status.exception())
@pytest.mark.parametrize("name", ["data_timeout", "scan_timeout"])
@pytest.mark.parametrize("timeout", [0, -1, float("nan"), float("inf")])
def test_invalid_timeouts_are_rejected(name, timeout):
with pytest.raises(ValueError, match="finite, positive"):
rt_ophyd.RtDataMonitor(**{name: timeout})
def test_slow_first_batch_keeps_scan_alive_while_controller_advances(flyer, clock):
flyer.data_timeout = 15
states = [(0, 2, 0), (11, 2, 1), (22, 2, 2), (33, 2, 3), (44, 2, 4)]
states += [(55, 2, 5), (60, 2, 5), (66, 2, 6), (77, 0, 10)]
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(flyer, clock, states, finish=True)
warning.assert_not_called()
assert flyer.data.set.call_count == 10
def test_initial_position_must_reach_a_new_high_to_reset_timeout(flyer, clock):
flyer.scan_timeout = 8
states = [(0, 2, 0), (2, 2, 1), (8, 2, 0), (9, 2, 1), (10, 2, 1)]
status = run_readout(flyer, clock, states, failure=True)
assert "No RT controller progress" in str(status.exception())
def test_controller_progress_cannot_mask_stalled_sample_readout(clock):
monitor = rt_ophyd.RtDataMonitor(data_timeout=3, scan_timeout=8)
monitor.reset()
monitor.sample_received()
clock.now += 3
assert monitor.check("rt_positions", mode=2, planned_positions=10, position=7) is None
clock.now += 5
error = monitor.check("rt_positions", mode=2, planned_positions=10, position=8)
assert isinstance(error, TimeoutError)
assert "No RT samples received" in str(error)
@pytest.mark.parametrize(
("data_timeout", "scan_timeout", "warns", "fails"),
[(None, 8, False, True), (3, None, True, False), (None, None, False, False)],
)
def test_timeouts_can_be_disabled_independently(flyer, clock, data_timeout, scan_timeout, warns, fails):
flyer.data_timeout = data_timeout
flyer.scan_timeout = scan_timeout
states = [(0, 2, 0), (3, 2, 0), (8, 2, 0), (70, 2, 0), (71, 0, 10)]
with mock.patch.object(rt_ophyd.logger, "warning") as warning:
run_readout(flyer, clock, states, finish=not fails, failure=fails)
assert warning.call_count == int(warns)
+48
View File
@@ -0,0 +1,48 @@
"""RT completion errors must propagate out of the scan's polling loop."""
from contextlib import nullcontext
from types import SimpleNamespace
from unittest import mock
import pytest
from bec_lib import messages
from bec_server.scan_server.errors import DeviceInstructionError
from bec_server.scan_server.scans.scan_status import ScanStatus
from csaxs_bec.scans.flomni_fermat_scan import FlomniFermatScan
from csaxs_bec.scans.lamni_fermat_scan import LamniFermatScan
from csaxs_bec.scans.omny_fermat_scan import OmnyFermatScan
@pytest.mark.parametrize("scan_cls", [FlomniFermatScan, LamniFermatScan, OmnyFermatScan])
@pytest.mark.parametrize("failed", [False, True])
def test_rt_scan_checks_completed_readout_result(scan_cls, failed):
status = ScanStatus(instruction_handler=mock.Mock())
if failed:
status.set_failed(
messages.ErrorInfo(
error_message="No RT scan points received within scan_timeout",
compact_error_message="RT scan timed out",
exception_type="TimeoutError",
device="rt_positions",
)
)
else:
status.set_done()
scan = SimpleNamespace(
actions=mock.Mock(),
exp_time=0.1,
frames_per_trigger=1,
dev=SimpleNamespace(rt_positions="rt_positions"),
at_each_point=mock.Mock(),
_scan_modifier=None,
_scan_modifier_hooks={},
)
scan.actions._scan_hook_context.side_effect = lambda _: nullcontext()
scan.actions.complete.return_value = status
if failed:
with pytest.raises(DeviceInstructionError, match="RT scan timed out"):
scan_cls.scan_core(scan)
else:
scan_cls.scan_core(scan)
scan.actions.complete.assert_called_once_with(device="rt_positions", wait=False)