fix(jfj-client): treat wait_till_done HTTP 500 as idle when the broker is Idle
CI for csaxs_bec / test (push) Successful in 3m47s

The previous fix only covered on_complete. At the beamline the scan with
packet loss then completed (S00998), but the next one failed in on_stage
(S00999): on_stage calls wait_for_idle() with raise_on_timeout=True, and
/wait_till_done keeps answering HTTP 500 "Missing packets in data
collection; reduce frame rate" until the next collection starts, although
the broker is Idle.

wait_for_idle() now checks the broker state when /wait_till_done raises a
ServiceException and returns True if it is Idle. This covers every caller
(on_stage, on_complete, initialise). The broker message stays available
through jfj_status, so on_complete still applies the missing-packet policy
and the image-count check. on_complete no longer needs its own state
check; it keeps the 0.2 s pause between failed attempts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
menzelandClaude Opus 5.5 authored and x12sa committed 2026-11-11 10:06:46 +01:00
1 parent 09d201b470
commit 3a3c52093b
3 files changed
+66 -13

No files matched your search

+5 -9
View File
@@ -572,17 +572,13 @@ class Eiger(PSIDeviceBase):
f"Device {self.name} is in ERROR state during acquisition. Broker status: "
f"{yaml.dump(self.jfj_client.jfj_status.to_dict(), indent=4)}"
)
# NOTE JFJoch's /wait_till_done answers HTTP 500 carrying the broker message
# when the last collection ended with an error (e.g. missing packets), even
# though the broker is already Idle, and keeps doing so on every call. So
# wait_for_idle returning False does not mean "not finished yet": check the
# broker state too, and let the message checks below decide. Without this,
# such a scan spun here (logging an error per call) until the timeout.
# NOTE A broker error that ends a collection (e.g. missing packets) makes
# /wait_till_done fail while the broker is Idle; wait_for_idle treats that
# as idle, and the message checks below decide. Other failures can return
# immediately, so pause before retrying rather than spinning.
finished = self.jfj_client.wait_for_idle(timeout=1, raise_on_timeout=False)
if not finished:
finished = self.jfj_client.detector_state == DetectorState.IDLE
if not finished:
time.sleep(0.2) # wait_for_idle can fail immediately; don't spin
time.sleep(0.2)
if finished:
# Call API endpoints to get broker status
broker_status = self.jfj_client.jfj_status
@@ -402,6 +402,14 @@ class JungfrauJochClient:
)
thread.start()
def _broker_is_idle(self) -> bool:
"""True if the broker reports Idle; False if it does not or cannot be asked."""
try:
state = self.api.status_get(_request_timeout=2).state
except Exception: # pylint: disable=broad-except
return False
return getattr(state, "value", state) == DetectorState.IDLE.value
def wait_for_idle(self, timeout: int = 10, raise_on_timeout: bool = True) -> bool:
"""
Method to wait until the detector is in IDLE state. This is a blocking call with a
@@ -427,8 +435,20 @@ class JungfrauJochClient:
)
return False
except Exception as exc:
content = traceback.format_exc()
message = self._api_error_message(exc)
# NOTE After a collection that ended with a broker error (e.g. "Missing packets in
# data collection; reduce frame rate"), /wait_till_done answers HTTP 500 with that
# message on every call until the next collection, although the broker is Idle.
# The detector is then idle, which is all this method promises; the broker message
# stays readable via jfj_status for whoever has to judge the finished collection.
# Without this, on_complete timed out and the next scan failed in on_stage.
if isinstance(exc, ServiceException) and self._broker_is_idle():
logger.info(
f"Device {self._parent_name}: wait_till_done reported '{message}', but the "
f"broker is Idle, so the detector is treated as idle."
)
return True
content = traceback.format_exc()
logger.info(
f"Error on device {self._parent_name} in wait_for_idle. "
f"{message or ''} Full traceback: {content}"
+40 -3
View File
@@ -785,15 +785,52 @@ def test_eiger_tolerance_does_not_hide_a_short_acquisition(eiger_1_5m):
_run_on_complete(eiger, _missing_packets_status(), images_collected=9)
def _missing_packets_http_500():
"""The ServiceException JFJoch's /wait_till_done raises after a lossy collection."""
return ServiceException(
status=500,
reason="Internal Server Error",
body='{"msg":"Missing packets in data collection; reduce frame rate","reason":"Other"}',
)
def test_jfj_client_wait_for_idle_is_idle_when_broker_idle_despite_http_500():
"""on_stage's raising wait_for_idle must pass when the previous collection lost packets
(2026-11-11: S00998 completed with the tolerance, S00999 then failed in on_stage)."""
client = _jfj_client()
idle = BrokerStatus(state="Idle", message_severity="error", message="Missing packets")
with (
mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()),
mock.patch.object(client.api, "status_get", return_value=idle),
):
assert client.wait_for_idle(timeout=1) is True
assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is True
def test_jfj_client_wait_for_idle_still_raises_on_http_500_when_not_idle():
"""The Idle shortcut must not hide a 500 while the broker is busy or in error."""
client = _jfj_client()
busy = BrokerStatus(state="Error", message_severity="error", message="Detector error")
with (
mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()),
mock.patch.object(client.api, "status_get", return_value=busy),
):
with pytest.raises(JungfrauJochClientError, match="Missing packets"):
client.wait_for_idle(timeout=1)
assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is False
def _run_on_complete_wait_till_done_failing(eiger, broker_status, images_collected=10):
"""Like _run_on_complete, but as JFJoch behaves after a collection that ended in an
error: /wait_till_done answers HTTP 500 on every call, so wait_for_idle always
returns False, while the broker itself already reports Idle (seen 2026-11-11, S00985).
error: /wait_till_done answers HTTP 500 on every call while the broker itself already
reports Idle (seen 2026-11-11, S00985).
"""
eiger._wait_for_on_complete = 2
with (
mock.patch.object(eiger.jfj_client.api, "status_get", return_value=broker_status),
mock.patch.object(eiger.jfj_client, "wait_for_idle", return_value=False),
mock.patch.object(
eiger.jfj_client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()
),
mock.patch.object(
eiger.jfj_client.api,
"statistics_data_collection_get",