From 3a3c52093b68407af4857b01bde0304944cfeafd Mon Sep 17 00:00:00 2001 From: menzel Date: Wed, 11 Nov 2026 10:05:20 +0100 Subject: [PATCH] fix(jfj-client): treat wait_till_done HTTP 500 as idle when the broker is Idle The previous fix only covered on_complete. At the beamline the scan with packet loss then completed (S00998), but the next one failed in on_stage (S00999): on_stage calls wait_for_idle() with raise_on_timeout=True, and /wait_till_done keeps answering HTTP 500 "Missing packets in data collection; reduce frame rate" until the next collection starts, although the broker is Idle. wait_for_idle() now checks the broker state when /wait_till_done raises a ServiceException and returns True if it is Idle. This covers every caller (on_stage, on_complete, initialise). The broker message stays available through jfj_status, so on_complete still applies the missing-packet policy and the image-count check. on_complete no longer needs its own state check; it keeps the 0.2 s pause between failed attempts. Co-Authored-By: Claude Opus 5.5 --- csaxs_bec/devices/jungfraujoch/eiger.py | 14 +++--- .../jungfraujoch/jungfrau_joch_client.py | 22 +++++++++- tests/tests_devices/test_eiger.py | 43 +++++++++++++++++-- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/csaxs_bec/devices/jungfraujoch/eiger.py b/csaxs_bec/devices/jungfraujoch/eiger.py index 023c5de9..dc79cdbd 100644 --- a/csaxs_bec/devices/jungfraujoch/eiger.py +++ b/csaxs_bec/devices/jungfraujoch/eiger.py @@ -572,17 +572,13 @@ class Eiger(PSIDeviceBase): f"Device {self.name} is in ERROR state during acquisition. Broker status: " f"{yaml.dump(self.jfj_client.jfj_status.to_dict(), indent=4)}" ) - # NOTE JFJoch's /wait_till_done answers HTTP 500 carrying the broker message - # when the last collection ended with an error (e.g. missing packets), even - # though the broker is already Idle, and keeps doing so on every call. So - # wait_for_idle returning False does not mean "not finished yet": check the - # broker state too, and let the message checks below decide. Without this, - # such a scan spun here (logging an error per call) until the timeout. + # NOTE A broker error that ends a collection (e.g. missing packets) makes + # /wait_till_done fail while the broker is Idle; wait_for_idle treats that + # as idle, and the message checks below decide. Other failures can return + # immediately, so pause before retrying rather than spinning. finished = self.jfj_client.wait_for_idle(timeout=1, raise_on_timeout=False) if not finished: - finished = self.jfj_client.detector_state == DetectorState.IDLE - if not finished: - time.sleep(0.2) # wait_for_idle can fail immediately; don't spin + time.sleep(0.2) if finished: # Call API endpoints to get broker status broker_status = self.jfj_client.jfj_status diff --git a/csaxs_bec/devices/jungfraujoch/jungfrau_joch_client.py b/csaxs_bec/devices/jungfraujoch/jungfrau_joch_client.py index 2e99453b..81211a23 100644 --- a/csaxs_bec/devices/jungfraujoch/jungfrau_joch_client.py +++ b/csaxs_bec/devices/jungfraujoch/jungfrau_joch_client.py @@ -402,6 +402,14 @@ class JungfrauJochClient: ) thread.start() + def _broker_is_idle(self) -> bool: + """True if the broker reports Idle; False if it does not or cannot be asked.""" + try: + state = self.api.status_get(_request_timeout=2).state + except Exception: # pylint: disable=broad-except + return False + return getattr(state, "value", state) == DetectorState.IDLE.value + def wait_for_idle(self, timeout: int = 10, raise_on_timeout: bool = True) -> bool: """ Method to wait until the detector is in IDLE state. This is a blocking call with a @@ -427,8 +435,20 @@ class JungfrauJochClient: ) return False except Exception as exc: - content = traceback.format_exc() message = self._api_error_message(exc) + # NOTE After a collection that ended with a broker error (e.g. "Missing packets in + # data collection; reduce frame rate"), /wait_till_done answers HTTP 500 with that + # message on every call until the next collection, although the broker is Idle. + # The detector is then idle, which is all this method promises; the broker message + # stays readable via jfj_status for whoever has to judge the finished collection. + # Without this, on_complete timed out and the next scan failed in on_stage. + if isinstance(exc, ServiceException) and self._broker_is_idle(): + logger.info( + f"Device {self._parent_name}: wait_till_done reported '{message}', but the " + f"broker is Idle, so the detector is treated as idle." + ) + return True + content = traceback.format_exc() logger.info( f"Error on device {self._parent_name} in wait_for_idle. " f"{message or ''} Full traceback: {content}" diff --git a/tests/tests_devices/test_eiger.py b/tests/tests_devices/test_eiger.py index 629442af..a390f913 100644 --- a/tests/tests_devices/test_eiger.py +++ b/tests/tests_devices/test_eiger.py @@ -785,15 +785,52 @@ def test_eiger_tolerance_does_not_hide_a_short_acquisition(eiger_1_5m): _run_on_complete(eiger, _missing_packets_status(), images_collected=9) +def _missing_packets_http_500(): + """The ServiceException JFJoch's /wait_till_done raises after a lossy collection.""" + return ServiceException( + status=500, + reason="Internal Server Error", + body='{"msg":"Missing packets in data collection; reduce frame rate","reason":"Other"}', + ) + + +def test_jfj_client_wait_for_idle_is_idle_when_broker_idle_despite_http_500(): + """on_stage's raising wait_for_idle must pass when the previous collection lost packets + (2026-11-11: S00998 completed with the tolerance, S00999 then failed in on_stage).""" + client = _jfj_client() + idle = BrokerStatus(state="Idle", message_severity="error", message="Missing packets") + with ( + mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()), + mock.patch.object(client.api, "status_get", return_value=idle), + ): + assert client.wait_for_idle(timeout=1) is True + assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is True + + +def test_jfj_client_wait_for_idle_still_raises_on_http_500_when_not_idle(): + """The Idle shortcut must not hide a 500 while the broker is busy or in error.""" + client = _jfj_client() + busy = BrokerStatus(state="Error", message_severity="error", message="Detector error") + with ( + mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()), + mock.patch.object(client.api, "status_get", return_value=busy), + ): + with pytest.raises(JungfrauJochClientError, match="Missing packets"): + client.wait_for_idle(timeout=1) + assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is False + + def _run_on_complete_wait_till_done_failing(eiger, broker_status, images_collected=10): """Like _run_on_complete, but as JFJoch behaves after a collection that ended in an - error: /wait_till_done answers HTTP 500 on every call, so wait_for_idle always - returns False, while the broker itself already reports Idle (seen 2026-11-11, S00985). + error: /wait_till_done answers HTTP 500 on every call while the broker itself already + reports Idle (seen 2026-11-11, S00985). """ eiger._wait_for_on_complete = 2 with ( mock.patch.object(eiger.jfj_client.api, "status_get", return_value=broker_status), - mock.patch.object(eiger.jfj_client, "wait_for_idle", return_value=False), + mock.patch.object( + eiger.jfj_client.api, "wait_till_done_post", side_effect=_missing_packets_http_500() + ), mock.patch.object( eiger.jfj_client.api, "statistics_data_collection_get",