DAQ: handle critical detector errors, GUI now intercepts and halts automation. Should also throw an error when a detector has a fault during manual collection.
Build and Publish / test (push) Successful in 8m9s
Build and Publish / build (push) Successful in 16s
Build and Publish / Build and Deploy Docs (push) Successful in 34s

This commit is contained in:
2026-05-04 15:36:20 +02:00
parent c60db6798a
commit e99681ddd8
4 changed files with 354 additions and 88 deletions
+62 -12
View File
@@ -123,6 +123,39 @@ class AareDAQ:
),
)
@staticmethod
def _is_jfjoch_detector_state_error(message: str | None) -> bool:
text = str(message or "").lower()
return (
"daq state error" in text
or "must be idle to start measurement" in text
or "must be idle" in text
)
def _raise_if_critical_jfjoch_detector_error(
self,
error: Exception,
*,
command: str,
) -> None:
if not isinstance(error, JFJochCommunicationError):
return
if (
getattr(error, "status_code", None) != 500
and not self._is_jfjoch_detector_state_error(str(error))
):
return
message = (
f"Critical detector error while running JFJoch command '{command}'. "
f"Automation has been stopped. "
f"There is an error with the detector. Please call your local contact. "
f"Original error: {error}"
)
logger.critical(message)
raise CriticalTellException(message) from error
def _run_noncritical(
self,
action: Callable[[], object],
@@ -1820,7 +1853,11 @@ class AareDAQ:
self.__aare.create_gridscan_run(self.sample, request, status)
if not self.__cfg.simulated_detector:
self.__jfjoch.wait_till_running(timeout=60.0)
try:
self.__jfjoch.wait_till_running(timeout=60.0)
except Exception as e:
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
raise
else:
logger.info("Simulated detector mode enabled; faking jfjoch intilalisation.")
@@ -1861,7 +1898,11 @@ class AareDAQ:
else:
#TODO how to handle no beam grids - do we always want to go to the center?
#TODO How to handle grids with no/minimal diffraction - use bkg instead?
scan_result = self.__jfjoch.wait_till_done(60)
try:
scan_result = self.__jfjoch.wait_till_done(60)
except Exception as e:
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done")
raise
com = None
if scan_result is None:
logger.error(
@@ -1982,7 +2023,7 @@ class AareDAQ:
except Exception as e:
logger.exception(
"Failed during raster",
f"Failed during raster: {e}",
extra=merge_log_context(
sample_log_context(self.sample),
raster_request_log_context(request),
@@ -2029,7 +2070,6 @@ class AareDAQ:
omega_start = self.omega
status = self.status
#self.__aare.create_rotation_run(self.sample, request, status)
if request.exp_time_s < 0.004:
logger.warning("Exposure time too short for PXII rotation scan")
request.exp_time_s = 0.004
@@ -2037,10 +2077,15 @@ class AareDAQ:
if self.sample is not None and self.sample.db_id is not None:
self.__aare.create_rotation_run(self.sample, request, status)
try:
self.__jfjoch.wait_till_running(timeout=60.0)
try:
self.__jfjoch.wait_till_running(timeout=60.0)
except Exception as e:
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
raise
if request.screening:
self.__devs.aerotech.screening_scan(
rotation_deg=request.steps*request.incr_omega_deg,
rotation_deg=request.steps * request.incr_omega_deg,
wedge_deg=request.wedge_omega_deg,
time_sec=total_time,
steps=request.steps,
@@ -2048,13 +2093,13 @@ class AareDAQ:
)
else:
self.__devs.aerotech.rotation_scan(
rotation_deg=request.steps*request.incr_omega_deg,
rotation_deg=request.steps * request.incr_omega_deg,
time_sec=total_time,
start_pos_deg=request.start_omega_deg,
run_async=True,
)
#Is this for helical scans...? do we do smargon scans?
# Is this for helical scans...? do we do smargon scans?
if request.start is not None and request.end is not None:
smargon_time_step = request.exp_time_s / float(request.steps)
pos_step = (request.end.sh_mm - request.start.sh_mm) * (1.0 / float(request.steps))
@@ -2065,14 +2110,18 @@ class AareDAQ:
)
time.sleep(smargon_time_step)
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60,0)))
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60, 0)))
self.__devs.aerotech_omega = omega_start
if self.__cfg.simulated_detector:
logger.warning("Detector in simulation mode, returning fake zero rotation result.")
return self._build_fake_rotation_result(request)
else:
scan_result = self.__jfjoch.wait_till_done(60)
try:
scan_result = self.__jfjoch.wait_till_done(60)
except Exception as e:
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done")
raise
return CompletedRotationScan(
request=copy.deepcopy(request),
result=scan_result,
@@ -2772,9 +2821,7 @@ class AareDAQ:
)
try:
logger.info(f"setting busy at {time.perf_counter() - start}")
self.__cfg.try_set_busy(timeout=360)
logger.info(f"set busy at {time.perf_counter() - start}")
self._set_progress_context(
progress,
@@ -2875,6 +2922,9 @@ class AareDAQ:
self._mark_progress_success(progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete")
logger.info(f"Rotation scan done at {time.perf_counter() - start}")
except JFJochCommunicationError as e:
self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown")
raise
except Exception as e:
logger.error(f"Error in measure: {e}")
if progress.current_step is not None:
+58 -13
View File
@@ -31,11 +31,56 @@ class JFJochWrapper:
self.__client = jfjoch_client.ApiClient(jfjoch_client.Configuration(host=self.__url))
self.__api = jfjoch_client.DefaultApi(self.__client)
@staticmethod
def _extract_status_code(error: Exception) -> int | None:
for attr_name in ("status", "status_code"):
value = getattr(error, attr_name, None)
if isinstance(value, int):
return value
body = getattr(error, "body", None)
if isinstance(body, str) and "500" in body:
return 500
return None
def _raise_jfjoch_error(
self,
message: str,
*,
error: Exception,
operation: str,
endpoint: str,
) -> None:
raise JFJochCommunicationError(
message,
operation=operation,
endpoint=endpoint,
base_url=self.__url,
status_code=self._extract_status_code(error),
) from error
def initialize(self):
self.__api.initialize_post()
try:
self.__api.initialize_post()
except Exception as e:
self._raise_jfjoch_error(
"JFJoch initialize failed",
error=e,
operation="POST",
endpoint="initialize_post",
)
def cancel(self):
self.__api.cancel_post()
try:
self.__api.cancel_post()
except Exception as e:
self._raise_jfjoch_error(
"JFJoch cancel failed",
error=e,
operation="POST",
endpoint="cancel_post",
)
def is_idle(self) -> bool:
status = self.__api.status_get()
@@ -141,12 +186,12 @@ class JFJochWrapper:
try:
self.__api.start_post(dataset_settings=dataset_settings)
except Exception as e:
raise JFJochCommunicationError(
self._raise_jfjoch_error(
f"JFJoch data collection failed to initialize for {scan_type.value} scan with exception: {e}",
error=e,
operation="POST",
endpoint="start_post",
base_url=self.__url,
) from e
)
def measure_rotation(self,
r: RotationScanRequest,
@@ -171,12 +216,12 @@ class JFJochWrapper:
self.__api.wait_until_running_post_with_http_info(timeout=math.ceil(timeout))
return True
except Exception as e:
raise JFJochCommunicationError(
self._raise_jfjoch_error(
f"JFJoch wait until running returned an exception: {e}",
operation="GET",
error=e,
operation="POST",
endpoint="wait_until_running_post",
base_url=self.__url,
) from e
)
def wait_till_done(self, timeout : int | float) -> jfjoch_client.models.ScanResult | None:
if self.__simulated:
@@ -185,12 +230,12 @@ class JFJochWrapper:
self.__api.wait_till_done_post_with_http_info(timeout=math.ceil(timeout))
return self.__api.result_scan_get()
except Exception as e:
raise JFJochCommunicationError(
self._raise_jfjoch_error(
f"JFJoch wait till done retrieval returned an exception: {e}",
operation="GET",
error=e,
operation="POST",
endpoint="wait_till_done_post / result_scan_get",
base_url=self.__url,
) from e
)
def detector(self) -> jfjoch_client.models.DetectorListElement:
l = self.__api.config_select_detector_get()
+116 -12
View File
@@ -84,6 +84,7 @@ class MainWindow(QMainWindow):
self.__mounting = False
self.__samcam_feed_banner_active = False
self.__samcam_feed_banner_message = "Sample camera feed unavailable"
self._automation_critical_banner_active = False
self._dev_help_dialog = None
self._beamline_recovery_dialog = None
self._controls_help_dialog = None
@@ -324,6 +325,12 @@ class MainWindow(QMainWindow):
self.job_list_panel.samples_in_queue_changed.connect(
self.automation_progress_panel.set_samples_in_queue
)
self.job_list_panel.automation_running_changed.connect(
self.automation_progress_panel.set_running
)
self.job_list_panel.automation_running_changed.connect(
self._on_automation_running_changed
)
self.automation_progress_panel.set_samples_in_queue(
len(self.job_list_panel.table_model.samples)
)
@@ -615,9 +622,12 @@ class MainWindow(QMainWindow):
self.daq.sample_missing.connect(self.show_sample_missing_dialog)
self.daq.standard_scan_completed.connect(self.rotation.scan_completed)
self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done)
self.daq.automation_critical_failure.connect(self._on_automation_critical_failure)
self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
self.daq.manual_collection_critical_failure.connect(self._on_manual_collection_critical_failure)
self.daq.recovery_action_completed.connect(lambda _msg: self._clear_automation_critical_banner())
self.manual_sample_panel.sample_manual.connect(self.daq.sample_manual)
self.face_panel.face_detection.connect(self.daq.face_detection)
@@ -997,6 +1007,71 @@ class MainWindow(QMainWindow):
self._dev_help_dialog.raise_()
self._dev_help_dialog.activateWindow()
def _clear_automation_critical_banner(self) -> None:
if not self._automation_critical_banner_active:
return
self.alert_banner.clear_message()
self._automation_critical_banner_active = False
@Slot(bool)
def _on_automation_running_changed(self, running: bool) -> None:
if running:
self._clear_automation_critical_banner()
def _is_detector_critical_failure(self, message: str) -> bool:
text = (message or "").lower()
return "detector" in text or "jfjoch" in text
@staticmethod
def _is_detector_state_error(message: str) -> bool:
text = (message or "").lower()
return (
"daq state error" in text
or "must be idle to start measurement" in text
or "must be idle" in text
)
def _detector_error_dialog_title(self, message: str) -> str:
if self._is_detector_state_error(message):
return "Detector state error"
return "Detector error"
def _detector_error_banner_text(self, *, automation: bool, message: str) -> str:
if self._is_detector_state_error(message):
prefix = "Automation halted" if automation else "Manual collection stopped"
return f"{prefix}: detector state error."
return (
"Automation halted: detector error."
if automation
else "Manual collection stopped: detector error."
)
@Slot(str)
def _on_manual_collection_critical_failure(self, message: str) -> None:
logger.critical(f"Manual collection critical detector failure: {message}")
try:
self.alert_banner_secondary.show_message(
self._detector_error_banner_text(automation=False, message=message),
True,
auto_clear_ms=0,
)
except Exception:
pass
try:
QMessageBox.critical(
self,
self._detector_error_dialog_title(message),
(
"The manual collection has stopped because there is an error with the detector.\n\n"
"Please call your local contact.\n\n"
f"Details:\n{message}"
),
)
except Exception as e:
logger.error(f"Failed to show manual collection detector popup: {e}")
@Slot(str)
def _on_automation_critical_failure(self, message: str) -> None:
"""
@@ -1008,6 +1083,13 @@ class MainWindow(QMainWindow):
"""
logger.critical(f"Automation critical failure: {message}")
is_detector_failure = self._is_detector_critical_failure(message)
banner_message = (
self._detector_error_banner_text(automation=True, message=message)
if is_detector_failure
else f"Automation halted: {message}"
)
# 1. Stop the sample queue so we don't keep dispatching new samples
try:
if self.job_list_panel is not None and self.job_list_panel.is_running():
@@ -1056,26 +1138,48 @@ class MainWindow(QMainWindow):
# 3. Banner so the operator sees it immediately
try:
self.alert_banner.show_message(
f"Automation halted: {message}", True, auto_clear_ms=0
banner_message, True, auto_clear_ms=0
)
self._automation_critical_banner_active = True
except Exception:
pass
# 4. Surface recovery UI
try:
if bool(getattr(self.__decoded_token, "staff", False)):
if is_detector_failure:
QMessageBox.critical(
self,
self._detector_error_dialog_title(message),
(
"Automation has been stopped because there is an error with the detector.\n\n"
"Please call your local contact.\n\n"
f"Details:\n{message}"
),
)
self.show_beamline_recovery()
else:
QMessageBox.critical(
self,
"Automation halted",
(
"A critical error occurred during automation and the "
"beamline could not recover automatically:\n\n"
f"{message}\n\n"
"Please contact your local contact to recover the beamline."
),
)
if is_detector_failure:
QMessageBox.critical(
self,
self._detector_error_dialog_title(message),
(
"Automation has been stopped because there is an error with the detector.\n\n"
"Please call your local contact.\n\n"
f"Details:\n{message}"
),
)
else:
QMessageBox.critical(
self,
"Automation halted",
(
"A critical error occurred during automation and the "
"beamline could not recover automatically:\n\n"
f"{message}\n\n"
"Please contact your local contact to recover the beamline."
),
)
except Exception as e:
logger.error(f"Failed to surface recovery UI after critical failure: {e}")
+118 -51
View File
@@ -69,6 +69,7 @@ class DAQWorker(QObject):
sample_resync_completed = Signal(str)
automation_critical_failure = Signal(str)
manual_collection_critical_failure = Signal(str)
error_codes_loaded = Signal(dict)
last_error_payload_changed = Signal(dict)
last_error_payloads_changed = Signal(list)
@@ -141,6 +142,8 @@ class DAQWorker(QObject):
self._last_detector_msg: str | None = None
self._last_detector_is_error: bool | None = None
self._last_manual_collection_critical_msg: str | None = None
self._baton_stream_reply: QNetworkReply | None = None
self._last_baton_status: BatonStatus | None = None
@@ -829,22 +832,23 @@ class DAQWorker(QObject):
try:
# Check for HTTP errors first
if reply.error() != QNetworkReply.NetworkError.NoError:
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
err_msg = reply.errorString()
status, err_msg, body_json = self._extract_reply_error_details(reply)
try:
raw_body = reply.readAll().data().decode("utf-8")
if raw_body:
body_json = json.loads(raw_body)
if isinstance(body_json, dict):
err_msg = body_json.get("message", err_msg)
code = body_json.get("code", "")
if self._is_detector_state_failure_message(err_msg):
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
else:
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
if code == "JFJOCH_UNAVAILABLE" or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
# Check if this is a JFJoch error
if code == "JFJOCH_UNAVAILABLE" or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
except Exception:
pass
if self._is_critical_detector_failure(status, body_json, err_msg):
critical_msg = (
"Manual collection stopped because there is an error with the detector. "
"Please call your local contact.\n\n"
f"Details: {err_msg}"
)
self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True)
self._emit_manual_collection_critical_failure(critical_msg)
short_msg = err_msg.split("input':", 1)[0].strip() if "input':" in err_msg else err_msg
logger.error(f"Rotation scan failed: {short_msg}")
@@ -860,6 +864,7 @@ class DAQWorker(QObject):
self.http_error.emit(str(e))
finally:
reply.deleteLater()
@Slot(RotationScanRequest)
def standard_scan(self, r: RotationScanRequest):
"""
@@ -881,24 +886,23 @@ class DAQWorker(QObject):
try:
# Check for HTTP errors first
if reply.error() != QNetworkReply.NetworkError.NoError:
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
raw_body = ""
body_json = None
err_msg = reply.errorString()
status, err_msg, body_json = self._extract_reply_error_details(reply)
try:
raw_body = reply.readAll().data().decode("utf-8")
if raw_body:
body_json = json.loads(raw_body)
if isinstance(body_json, dict):
err_msg = body_json.get("message", err_msg)
code = body_json.get("code", "")
if self._is_detector_state_failure_message(err_msg):
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
else:
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
if code == "JFJOCH_UNAVAILABLE" or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
# Check if this is a JFJoch error
if code == "JFJOCH_UNAVAILABLE" or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
except Exception:
pass
if self._is_critical_detector_failure(status, body_json, err_msg):
critical_msg = (
"Manual collection stopped because there is an error with the detector. "
"Please call your local contact.\n\n"
f"Details: {err_msg}"
)
self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True)
self._emit_manual_collection_critical_failure(critical_msg)
logger.error(f"Raster scan failed: {err_msg}")
self.http_error.emit(err_msg)
@@ -1017,10 +1021,85 @@ class DAQWorker(QObject):
reply = self.__net_manager.get(request)
reply.finished.connect(lambda: self.handle_reference_tools_response(reply))
def _emit_manual_collection_critical_failure(self, msg: str) -> None:
if not msg:
return
if msg == self._last_manual_collection_critical_msg:
return
self._last_manual_collection_critical_msg = msg
self.manual_collection_critical_failure.emit(msg)
@staticmethod
def _extract_reply_error_details(reply: QNetworkReply) -> tuple[int | None, str, dict | None]:
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
err_msg = reply.errorString()
body_json = None
try:
raw_body = reply.readAll().data().decode("utf-8")
if raw_body:
try:
parsed = json.loads(raw_body)
if isinstance(parsed, dict):
body_json = parsed
err_msg = (
parsed.get("message")
or parsed.get("detail")
or parsed.get("error")
or raw_body
)
else:
err_msg = raw_body
except Exception:
err_msg = raw_body
except Exception:
pass
return status, str(err_msg), body_json
@staticmethod
def _is_detector_state_failure_message(message: str | None) -> bool:
text = str(message or "").lower()
return (
"daq state error" in text
or "must be idle to start measurement" in text
or "must be idle" in text
)
@staticmethod
def _is_critical_detector_failure(
status: int | None,
body_json: dict | None = None,
err_msg: str | None = None,
) -> bool:
try:
status_int = int(status) if status is not None else None
except Exception:
status_int = None
code = None
if isinstance(body_json, dict):
code = body_json.get("code")
text = str(err_msg or "").lower()
if status_int == 500:
return True
if code == "JFJOCH_UNAVAILABLE" and status_int is not None and 500 <= status_int < 600:
return True
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
return True
return False
@staticmethod
def _is_critical_automation_failure(
status: int | None,
body_json: dict | None = None,
err_msg: str | None = None,
) -> bool:
"""Detect server-side critical automation errors that require operator recovery."""
try:
@@ -1035,11 +1114,13 @@ class DAQWorker(QObject):
if code == DAQErrorCode.AUTOMATION_CRITICAL.value:
return True
# Conservative fallback for /scan/auto: an unstructured 5xx means the
# server raised unexpectedly during automation.
if status_int is not None and 500 <= status_int < 600 and code is None:
return True
text = str(err_msg or "").lower()
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
return True
return False
def handle_auto_scan_response(self, reply, sample_id: int):
@@ -1048,24 +1129,10 @@ class DAQWorker(QObject):
logger.info(f"Sample time {resp} s")
self.automated_scan_done.emit(sample_id, True, "")
else:
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
err_str = reply.errorString()
body_json = None
try:
response_body = reply.readAll().data().decode("utf-8")
if response_body:
parsed_body = json.loads(response_body)
if isinstance(parsed_body, dict):
body_json = parsed_body
err_str = (
body_json.get("message")
or body_json.get("detail")
or response_body
)
else:
err_str = response_body
except Exception:
pass
status, err_str, body_json = self._extract_reply_error_details(reply)
if self._is_detector_state_failure_message(err_str):
self._emit_detector_message(f"JFJoch: {err_str}", is_error=True)
if status == 401:
logger.error(f"Error in auto scan: {err_str}")
@@ -1080,7 +1147,7 @@ class DAQWorker(QObject):
elif status == 417:
self.sample_missing.emit(err_str)
self.automated_scan_done.emit(sample_id, False, "Critical")
elif self._is_critical_automation_failure(status, body_json):
elif self._is_critical_automation_failure(status, body_json, err_str):
logger.critical(f"Critical automation failure: {err_str}")
self.http_error.emit(err_str)
self.automated_scan_done.emit(sample_id, False, "Critical")