fix(jfj-client): treat wait_till_done HTTP 500 as idle when the broker is Idle
CI for csaxs_bec / test (push) Successful in 3m47s
CI for csaxs_bec / test (push) Successful in 3m47s
The previous fix only covered on_complete. At the beamline the scan with packet loss then completed (S00998), but the next one failed in on_stage (S00999): on_stage calls wait_for_idle() with raise_on_timeout=True, and /wait_till_done keeps answering HTTP 500 "Missing packets in data collection; reduce frame rate" until the next collection starts, although the broker is Idle. wait_for_idle() now checks the broker state when /wait_till_done raises a ServiceException and returns True if it is Idle. This covers every caller (on_stage, on_complete, initialise). The broker message stays available through jfj_status, so on_complete still applies the missing-packet policy and the image-count check. on_complete no longer needs its own state check; it keeps the 0.2 s pause between failed attempts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
09d201b470
commit
3a3c52093b
3 files changed
+66
-13
No files matched your search
@@ -572,17 +572,13 @@ class Eiger(PSIDeviceBase):
|
||||
f"Device {self.name} is in ERROR state during acquisition. Broker status: "
|
||||
f"{yaml.dump(self.jfj_client.jfj_status.to_dict(), indent=4)}"
|
||||
)
|
||||
# NOTE JFJoch's /wait_till_done answers HTTP 500 carrying the broker message
|
||||
# when the last collection ended with an error (e.g. missing packets), even
|
||||
# though the broker is already Idle, and keeps doing so on every call. So
|
||||
# wait_for_idle returning False does not mean "not finished yet": check the
|
||||
# broker state too, and let the message checks below decide. Without this,
|
||||
# such a scan spun here (logging an error per call) until the timeout.
|
||||
# NOTE A broker error that ends a collection (e.g. missing packets) makes
|
||||
# /wait_till_done fail while the broker is Idle; wait_for_idle treats that
|
||||
# as idle, and the message checks below decide. Other failures can return
|
||||
# immediately, so pause before retrying rather than spinning.
|
||||
finished = self.jfj_client.wait_for_idle(timeout=1, raise_on_timeout=False)
|
||||
if not finished:
|
||||
finished = self.jfj_client.detector_state == DetectorState.IDLE
|
||||
if not finished:
|
||||
time.sleep(0.2) # wait_for_idle can fail immediately; don't spin
|
||||
time.sleep(0.2)
|
||||
if finished:
|
||||
# Call API endpoints to get broker status
|
||||
broker_status = self.jfj_client.jfj_status
|
||||
|
||||
@@ -402,6 +402,14 @@ class JungfrauJochClient:
|
||||
)
|
||||
thread.start()
|
||||
|
||||
def _broker_is_idle(self) -> bool:
|
||||
"""True if the broker reports Idle; False if it does not or cannot be asked."""
|
||||
try:
|
||||
state = self.api.status_get(_request_timeout=2).state
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return False
|
||||
return getattr(state, "value", state) == DetectorState.IDLE.value
|
||||
|
||||
def wait_for_idle(self, timeout: int = 10, raise_on_timeout: bool = True) -> bool:
|
||||
"""
|
||||
Method to wait until the detector is in IDLE state. This is a blocking call with a
|
||||
@@ -427,8 +435,20 @@ class JungfrauJochClient:
|
||||
)
|
||||
return False
|
||||
except Exception as exc:
|
||||
content = traceback.format_exc()
|
||||
message = self._api_error_message(exc)
|
||||
# NOTE After a collection that ended with a broker error (e.g. "Missing packets in
|
||||
# data collection; reduce frame rate"), /wait_till_done answers HTTP 500 with that
|
||||
# message on every call until the next collection, although the broker is Idle.
|
||||
# The detector is then idle, which is all this method promises; the broker message
|
||||
# stays readable via jfj_status for whoever has to judge the finished collection.
|
||||
# Without this, on_complete timed out and the next scan failed in on_stage.
|
||||
if isinstance(exc, ServiceException) and self._broker_is_idle():
|
||||
logger.info(
|
||||
f"Device {self._parent_name}: wait_till_done reported '{message}', but the "
|
||||
f"broker is Idle, so the detector is treated as idle."
|
||||
)
|
||||
return True
|
||||
content = traceback.format_exc()
|
||||
logger.info(
|
||||
f"Error on device {self._parent_name} in wait_for_idle. "
|
||||
f"{message or ''} Full traceback: {content}"
|
||||
|
||||
@@ -785,15 +785,52 @@ def test_eiger_tolerance_does_not_hide_a_short_acquisition(eiger_1_5m):
|
||||
_run_on_complete(eiger, _missing_packets_status(), images_collected=9)
|
||||
|
||||
|
||||
def _missing_packets_http_500():
|
||||
"""The ServiceException JFJoch's /wait_till_done raises after a lossy collection."""
|
||||
return ServiceException(
|
||||
status=500,
|
||||
reason="Internal Server Error",
|
||||
body='{"msg":"Missing packets in data collection; reduce frame rate","reason":"Other"}',
|
||||
)
|
||||
|
||||
|
||||
def test_jfj_client_wait_for_idle_is_idle_when_broker_idle_despite_http_500():
|
||||
"""on_stage's raising wait_for_idle must pass when the previous collection lost packets
|
||||
(2026-11-11: S00998 completed with the tolerance, S00999 then failed in on_stage)."""
|
||||
client = _jfj_client()
|
||||
idle = BrokerStatus(state="Idle", message_severity="error", message="Missing packets")
|
||||
with (
|
||||
mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()),
|
||||
mock.patch.object(client.api, "status_get", return_value=idle),
|
||||
):
|
||||
assert client.wait_for_idle(timeout=1) is True
|
||||
assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is True
|
||||
|
||||
|
||||
def test_jfj_client_wait_for_idle_still_raises_on_http_500_when_not_idle():
|
||||
"""The Idle shortcut must not hide a 500 while the broker is busy or in error."""
|
||||
client = _jfj_client()
|
||||
busy = BrokerStatus(state="Error", message_severity="error", message="Detector error")
|
||||
with (
|
||||
mock.patch.object(client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()),
|
||||
mock.patch.object(client.api, "status_get", return_value=busy),
|
||||
):
|
||||
with pytest.raises(JungfrauJochClientError, match="Missing packets"):
|
||||
client.wait_for_idle(timeout=1)
|
||||
assert client.wait_for_idle(timeout=1, raise_on_timeout=False) is False
|
||||
|
||||
|
||||
def _run_on_complete_wait_till_done_failing(eiger, broker_status, images_collected=10):
|
||||
"""Like _run_on_complete, but as JFJoch behaves after a collection that ended in an
|
||||
error: /wait_till_done answers HTTP 500 on every call, so wait_for_idle always
|
||||
returns False, while the broker itself already reports Idle (seen 2026-11-11, S00985).
|
||||
error: /wait_till_done answers HTTP 500 on every call while the broker itself already
|
||||
reports Idle (seen 2026-11-11, S00985).
|
||||
"""
|
||||
eiger._wait_for_on_complete = 2
|
||||
with (
|
||||
mock.patch.object(eiger.jfj_client.api, "status_get", return_value=broker_status),
|
||||
mock.patch.object(eiger.jfj_client, "wait_for_idle", return_value=False),
|
||||
mock.patch.object(
|
||||
eiger.jfj_client.api, "wait_till_done_post", side_effect=_missing_packets_http_500()
|
||||
),
|
||||
mock.patch.object(
|
||||
eiger.jfj_client.api,
|
||||
"statistics_data_collection_get",
|
||||
|
||||
Reference in new issue
Block a user