DAQ: handle critical detector errors, GUI now intercepts and halts automation. Should also throw an error when a detector has a fault during manual collection.
This commit is contained in:
+62
-12
@@ -123,6 +123,39 @@ class AareDAQ:
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_jfjoch_detector_state_error(message: str | None) -> bool:
|
||||
text = str(message or "").lower()
|
||||
return (
|
||||
"daq state error" in text
|
||||
or "must be idle to start measurement" in text
|
||||
or "must be idle" in text
|
||||
)
|
||||
|
||||
def _raise_if_critical_jfjoch_detector_error(
|
||||
self,
|
||||
error: Exception,
|
||||
*,
|
||||
command: str,
|
||||
) -> None:
|
||||
if not isinstance(error, JFJochCommunicationError):
|
||||
return
|
||||
|
||||
if (
|
||||
getattr(error, "status_code", None) != 500
|
||||
and not self._is_jfjoch_detector_state_error(str(error))
|
||||
):
|
||||
return
|
||||
|
||||
message = (
|
||||
f"Critical detector error while running JFJoch command '{command}'. "
|
||||
f"Automation has been stopped. "
|
||||
f"There is an error with the detector. Please call your local contact. "
|
||||
f"Original error: {error}"
|
||||
)
|
||||
logger.critical(message)
|
||||
raise CriticalTellException(message) from error
|
||||
|
||||
def _run_noncritical(
|
||||
self,
|
||||
action: Callable[[], object],
|
||||
@@ -1820,7 +1853,11 @@ class AareDAQ:
|
||||
self.__aare.create_gridscan_run(self.sample, request, status)
|
||||
|
||||
if not self.__cfg.simulated_detector:
|
||||
self.__jfjoch.wait_till_running(timeout=60.0)
|
||||
try:
|
||||
self.__jfjoch.wait_till_running(timeout=60.0)
|
||||
except Exception as e:
|
||||
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
|
||||
raise
|
||||
else:
|
||||
logger.info("Simulated detector mode enabled; faking jfjoch intilalisation.")
|
||||
|
||||
@@ -1861,7 +1898,11 @@ class AareDAQ:
|
||||
else:
|
||||
#TODO how to handle no beam grids - do we always want to go to the center?
|
||||
#TODO How to handle grids with no/minimal diffraction - use bkg instead?
|
||||
scan_result = self.__jfjoch.wait_till_done(60)
|
||||
try:
|
||||
scan_result = self.__jfjoch.wait_till_done(60)
|
||||
except Exception as e:
|
||||
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done")
|
||||
raise
|
||||
com = None
|
||||
if scan_result is None:
|
||||
logger.error(
|
||||
@@ -1982,7 +2023,7 @@ class AareDAQ:
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Failed during raster",
|
||||
f"Failed during raster: {e}",
|
||||
extra=merge_log_context(
|
||||
sample_log_context(self.sample),
|
||||
raster_request_log_context(request),
|
||||
@@ -2029,7 +2070,6 @@ class AareDAQ:
|
||||
omega_start = self.omega
|
||||
status = self.status
|
||||
|
||||
#self.__aare.create_rotation_run(self.sample, request, status)
|
||||
if request.exp_time_s < 0.004:
|
||||
logger.warning("Exposure time too short for PXII rotation scan")
|
||||
request.exp_time_s = 0.004
|
||||
@@ -2037,10 +2077,15 @@ class AareDAQ:
|
||||
if self.sample is not None and self.sample.db_id is not None:
|
||||
self.__aare.create_rotation_run(self.sample, request, status)
|
||||
try:
|
||||
self.__jfjoch.wait_till_running(timeout=60.0)
|
||||
try:
|
||||
self.__jfjoch.wait_till_running(timeout=60.0)
|
||||
except Exception as e:
|
||||
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
|
||||
raise
|
||||
|
||||
if request.screening:
|
||||
self.__devs.aerotech.screening_scan(
|
||||
rotation_deg=request.steps*request.incr_omega_deg,
|
||||
rotation_deg=request.steps * request.incr_omega_deg,
|
||||
wedge_deg=request.wedge_omega_deg,
|
||||
time_sec=total_time,
|
||||
steps=request.steps,
|
||||
@@ -2048,13 +2093,13 @@ class AareDAQ:
|
||||
)
|
||||
else:
|
||||
self.__devs.aerotech.rotation_scan(
|
||||
rotation_deg=request.steps*request.incr_omega_deg,
|
||||
rotation_deg=request.steps * request.incr_omega_deg,
|
||||
time_sec=total_time,
|
||||
start_pos_deg=request.start_omega_deg,
|
||||
run_async=True,
|
||||
)
|
||||
|
||||
#Is this for helical scans...? do we do smargon scans?
|
||||
# Is this for helical scans...? do we do smargon scans?
|
||||
if request.start is not None and request.end is not None:
|
||||
smargon_time_step = request.exp_time_s / float(request.steps)
|
||||
pos_step = (request.end.sh_mm - request.start.sh_mm) * (1.0 / float(request.steps))
|
||||
@@ -2065,14 +2110,18 @@ class AareDAQ:
|
||||
)
|
||||
time.sleep(smargon_time_step)
|
||||
|
||||
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60,0)))
|
||||
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60, 0)))
|
||||
self.__devs.aerotech_omega = omega_start
|
||||
|
||||
if self.__cfg.simulated_detector:
|
||||
logger.warning("Detector in simulation mode, returning fake zero rotation result.")
|
||||
return self._build_fake_rotation_result(request)
|
||||
else:
|
||||
scan_result = self.__jfjoch.wait_till_done(60)
|
||||
try:
|
||||
scan_result = self.__jfjoch.wait_till_done(60)
|
||||
except Exception as e:
|
||||
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done")
|
||||
raise
|
||||
return CompletedRotationScan(
|
||||
request=copy.deepcopy(request),
|
||||
result=scan_result,
|
||||
@@ -2772,9 +2821,7 @@ class AareDAQ:
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"setting busy at {time.perf_counter() - start}")
|
||||
self.__cfg.try_set_busy(timeout=360)
|
||||
logger.info(f"set busy at {time.perf_counter() - start}")
|
||||
|
||||
self._set_progress_context(
|
||||
progress,
|
||||
@@ -2875,6 +2922,9 @@ class AareDAQ:
|
||||
self._mark_progress_success(progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete")
|
||||
logger.info(f"Rotation scan done at {time.perf_counter() - start}")
|
||||
|
||||
except JFJochCommunicationError as e:
|
||||
self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error in measure: {e}")
|
||||
if progress.current_step is not None:
|
||||
|
||||
+58
-13
@@ -31,11 +31,56 @@ class JFJochWrapper:
|
||||
self.__client = jfjoch_client.ApiClient(jfjoch_client.Configuration(host=self.__url))
|
||||
self.__api = jfjoch_client.DefaultApi(self.__client)
|
||||
|
||||
@staticmethod
|
||||
def _extract_status_code(error: Exception) -> int | None:
|
||||
for attr_name in ("status", "status_code"):
|
||||
value = getattr(error, attr_name, None)
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
|
||||
body = getattr(error, "body", None)
|
||||
if isinstance(body, str) and "500" in body:
|
||||
return 500
|
||||
|
||||
return None
|
||||
|
||||
def _raise_jfjoch_error(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
error: Exception,
|
||||
operation: str,
|
||||
endpoint: str,
|
||||
) -> None:
|
||||
raise JFJochCommunicationError(
|
||||
message,
|
||||
operation=operation,
|
||||
endpoint=endpoint,
|
||||
base_url=self.__url,
|
||||
status_code=self._extract_status_code(error),
|
||||
) from error
|
||||
|
||||
def initialize(self):
|
||||
self.__api.initialize_post()
|
||||
try:
|
||||
self.__api.initialize_post()
|
||||
except Exception as e:
|
||||
self._raise_jfjoch_error(
|
||||
"JFJoch initialize failed",
|
||||
error=e,
|
||||
operation="POST",
|
||||
endpoint="initialize_post",
|
||||
)
|
||||
|
||||
def cancel(self):
|
||||
self.__api.cancel_post()
|
||||
try:
|
||||
self.__api.cancel_post()
|
||||
except Exception as e:
|
||||
self._raise_jfjoch_error(
|
||||
"JFJoch cancel failed",
|
||||
error=e,
|
||||
operation="POST",
|
||||
endpoint="cancel_post",
|
||||
)
|
||||
|
||||
def is_idle(self) -> bool:
|
||||
status = self.__api.status_get()
|
||||
@@ -141,12 +186,12 @@ class JFJochWrapper:
|
||||
try:
|
||||
self.__api.start_post(dataset_settings=dataset_settings)
|
||||
except Exception as e:
|
||||
raise JFJochCommunicationError(
|
||||
self._raise_jfjoch_error(
|
||||
f"JFJoch data collection failed to initialize for {scan_type.value} scan with exception: {e}",
|
||||
error=e,
|
||||
operation="POST",
|
||||
endpoint="start_post",
|
||||
base_url=self.__url,
|
||||
) from e
|
||||
)
|
||||
|
||||
def measure_rotation(self,
|
||||
r: RotationScanRequest,
|
||||
@@ -171,12 +216,12 @@ class JFJochWrapper:
|
||||
self.__api.wait_until_running_post_with_http_info(timeout=math.ceil(timeout))
|
||||
return True
|
||||
except Exception as e:
|
||||
raise JFJochCommunicationError(
|
||||
self._raise_jfjoch_error(
|
||||
f"JFJoch wait until running returned an exception: {e}",
|
||||
operation="GET",
|
||||
error=e,
|
||||
operation="POST",
|
||||
endpoint="wait_until_running_post",
|
||||
base_url=self.__url,
|
||||
) from e
|
||||
)
|
||||
|
||||
def wait_till_done(self, timeout : int | float) -> jfjoch_client.models.ScanResult | None:
|
||||
if self.__simulated:
|
||||
@@ -185,12 +230,12 @@ class JFJochWrapper:
|
||||
self.__api.wait_till_done_post_with_http_info(timeout=math.ceil(timeout))
|
||||
return self.__api.result_scan_get()
|
||||
except Exception as e:
|
||||
raise JFJochCommunicationError(
|
||||
self._raise_jfjoch_error(
|
||||
f"JFJoch wait till done retrieval returned an exception: {e}",
|
||||
operation="GET",
|
||||
error=e,
|
||||
operation="POST",
|
||||
endpoint="wait_till_done_post / result_scan_get",
|
||||
base_url=self.__url,
|
||||
) from e
|
||||
)
|
||||
|
||||
def detector(self) -> jfjoch_client.models.DetectorListElement:
|
||||
l = self.__api.config_select_detector_get()
|
||||
|
||||
+116
-12
@@ -84,6 +84,7 @@ class MainWindow(QMainWindow):
|
||||
self.__mounting = False
|
||||
self.__samcam_feed_banner_active = False
|
||||
self.__samcam_feed_banner_message = "Sample camera feed unavailable"
|
||||
self._automation_critical_banner_active = False
|
||||
self._dev_help_dialog = None
|
||||
self._beamline_recovery_dialog = None
|
||||
self._controls_help_dialog = None
|
||||
@@ -324,6 +325,12 @@ class MainWindow(QMainWindow):
|
||||
self.job_list_panel.samples_in_queue_changed.connect(
|
||||
self.automation_progress_panel.set_samples_in_queue
|
||||
)
|
||||
self.job_list_panel.automation_running_changed.connect(
|
||||
self.automation_progress_panel.set_running
|
||||
)
|
||||
self.job_list_panel.automation_running_changed.connect(
|
||||
self._on_automation_running_changed
|
||||
)
|
||||
self.automation_progress_panel.set_samples_in_queue(
|
||||
len(self.job_list_panel.table_model.samples)
|
||||
)
|
||||
@@ -615,9 +622,12 @@ class MainWindow(QMainWindow):
|
||||
self.daq.sample_missing.connect(self.show_sample_missing_dialog)
|
||||
|
||||
self.daq.standard_scan_completed.connect(self.rotation.scan_completed)
|
||||
self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
|
||||
|
||||
self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done)
|
||||
self.daq.automation_critical_failure.connect(self._on_automation_critical_failure)
|
||||
self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
|
||||
self.daq.manual_collection_critical_failure.connect(self._on_manual_collection_critical_failure)
|
||||
self.daq.recovery_action_completed.connect(lambda _msg: self._clear_automation_critical_banner())
|
||||
|
||||
self.manual_sample_panel.sample_manual.connect(self.daq.sample_manual)
|
||||
self.face_panel.face_detection.connect(self.daq.face_detection)
|
||||
@@ -997,6 +1007,71 @@ class MainWindow(QMainWindow):
|
||||
self._dev_help_dialog.raise_()
|
||||
self._dev_help_dialog.activateWindow()
|
||||
|
||||
def _clear_automation_critical_banner(self) -> None:
|
||||
if not self._automation_critical_banner_active:
|
||||
return
|
||||
self.alert_banner.clear_message()
|
||||
self._automation_critical_banner_active = False
|
||||
|
||||
@Slot(bool)
|
||||
def _on_automation_running_changed(self, running: bool) -> None:
|
||||
if running:
|
||||
self._clear_automation_critical_banner()
|
||||
|
||||
def _is_detector_critical_failure(self, message: str) -> bool:
|
||||
text = (message or "").lower()
|
||||
return "detector" in text or "jfjoch" in text
|
||||
|
||||
@staticmethod
|
||||
def _is_detector_state_error(message: str) -> bool:
|
||||
text = (message or "").lower()
|
||||
return (
|
||||
"daq state error" in text
|
||||
or "must be idle to start measurement" in text
|
||||
or "must be idle" in text
|
||||
)
|
||||
|
||||
def _detector_error_dialog_title(self, message: str) -> str:
|
||||
if self._is_detector_state_error(message):
|
||||
return "Detector state error"
|
||||
return "Detector error"
|
||||
|
||||
def _detector_error_banner_text(self, *, automation: bool, message: str) -> str:
|
||||
if self._is_detector_state_error(message):
|
||||
prefix = "Automation halted" if automation else "Manual collection stopped"
|
||||
return f"{prefix}: detector state error."
|
||||
return (
|
||||
"Automation halted: detector error."
|
||||
if automation
|
||||
else "Manual collection stopped: detector error."
|
||||
)
|
||||
|
||||
@Slot(str)
|
||||
def _on_manual_collection_critical_failure(self, message: str) -> None:
|
||||
logger.critical(f"Manual collection critical detector failure: {message}")
|
||||
|
||||
try:
|
||||
self.alert_banner_secondary.show_message(
|
||||
self._detector_error_banner_text(automation=False, message=message),
|
||||
True,
|
||||
auto_clear_ms=0,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
QMessageBox.critical(
|
||||
self,
|
||||
self._detector_error_dialog_title(message),
|
||||
(
|
||||
"The manual collection has stopped because there is an error with the detector.\n\n"
|
||||
"Please call your local contact.\n\n"
|
||||
f"Details:\n{message}"
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to show manual collection detector popup: {e}")
|
||||
|
||||
@Slot(str)
|
||||
def _on_automation_critical_failure(self, message: str) -> None:
|
||||
"""
|
||||
@@ -1008,6 +1083,13 @@ class MainWindow(QMainWindow):
|
||||
"""
|
||||
logger.critical(f"Automation critical failure: {message}")
|
||||
|
||||
is_detector_failure = self._is_detector_critical_failure(message)
|
||||
banner_message = (
|
||||
self._detector_error_banner_text(automation=True, message=message)
|
||||
if is_detector_failure
|
||||
else f"Automation halted: {message}"
|
||||
)
|
||||
|
||||
# 1. Stop the sample queue so we don't keep dispatching new samples
|
||||
try:
|
||||
if self.job_list_panel is not None and self.job_list_panel.is_running():
|
||||
@@ -1056,26 +1138,48 @@ class MainWindow(QMainWindow):
|
||||
# 3. Banner so the operator sees it immediately
|
||||
try:
|
||||
self.alert_banner.show_message(
|
||||
f"Automation halted: {message}", True, auto_clear_ms=0
|
||||
banner_message, True, auto_clear_ms=0
|
||||
)
|
||||
self._automation_critical_banner_active = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. Surface recovery UI
|
||||
try:
|
||||
if bool(getattr(self.__decoded_token, "staff", False)):
|
||||
if is_detector_failure:
|
||||
QMessageBox.critical(
|
||||
self,
|
||||
self._detector_error_dialog_title(message),
|
||||
(
|
||||
"Automation has been stopped because there is an error with the detector.\n\n"
|
||||
"Please call your local contact.\n\n"
|
||||
f"Details:\n{message}"
|
||||
),
|
||||
)
|
||||
self.show_beamline_recovery()
|
||||
else:
|
||||
QMessageBox.critical(
|
||||
self,
|
||||
"Automation halted",
|
||||
(
|
||||
"A critical error occurred during automation and the "
|
||||
"beamline could not recover automatically:\n\n"
|
||||
f"{message}\n\n"
|
||||
"Please contact your local contact to recover the beamline."
|
||||
),
|
||||
)
|
||||
if is_detector_failure:
|
||||
QMessageBox.critical(
|
||||
self,
|
||||
self._detector_error_dialog_title(message),
|
||||
(
|
||||
"Automation has been stopped because there is an error with the detector.\n\n"
|
||||
"Please call your local contact.\n\n"
|
||||
f"Details:\n{message}"
|
||||
),
|
||||
)
|
||||
else:
|
||||
QMessageBox.critical(
|
||||
self,
|
||||
"Automation halted",
|
||||
(
|
||||
"A critical error occurred during automation and the "
|
||||
"beamline could not recover automatically:\n\n"
|
||||
f"{message}\n\n"
|
||||
"Please contact your local contact to recover the beamline."
|
||||
),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to surface recovery UI after critical failure: {e}")
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ class DAQWorker(QObject):
|
||||
sample_resync_completed = Signal(str)
|
||||
|
||||
automation_critical_failure = Signal(str)
|
||||
manual_collection_critical_failure = Signal(str)
|
||||
error_codes_loaded = Signal(dict)
|
||||
last_error_payload_changed = Signal(dict)
|
||||
last_error_payloads_changed = Signal(list)
|
||||
@@ -141,6 +142,8 @@ class DAQWorker(QObject):
|
||||
self._last_detector_msg: str | None = None
|
||||
self._last_detector_is_error: bool | None = None
|
||||
|
||||
self._last_manual_collection_critical_msg: str | None = None
|
||||
|
||||
self._baton_stream_reply: QNetworkReply | None = None
|
||||
self._last_baton_status: BatonStatus | None = None
|
||||
|
||||
@@ -829,22 +832,23 @@ class DAQWorker(QObject):
|
||||
try:
|
||||
# Check for HTTP errors first
|
||||
if reply.error() != QNetworkReply.NetworkError.NoError:
|
||||
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
|
||||
err_msg = reply.errorString()
|
||||
status, err_msg, body_json = self._extract_reply_error_details(reply)
|
||||
|
||||
try:
|
||||
raw_body = reply.readAll().data().decode("utf-8")
|
||||
if raw_body:
|
||||
body_json = json.loads(raw_body)
|
||||
if isinstance(body_json, dict):
|
||||
err_msg = body_json.get("message", err_msg)
|
||||
code = body_json.get("code", "")
|
||||
if self._is_detector_state_failure_message(err_msg):
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
else:
|
||||
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
|
||||
# Check if this is a JFJoch error
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
except Exception:
|
||||
pass
|
||||
if self._is_critical_detector_failure(status, body_json, err_msg):
|
||||
critical_msg = (
|
||||
"Manual collection stopped because there is an error with the detector. "
|
||||
"Please call your local contact.\n\n"
|
||||
f"Details: {err_msg}"
|
||||
)
|
||||
self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True)
|
||||
self._emit_manual_collection_critical_failure(critical_msg)
|
||||
|
||||
short_msg = err_msg.split("input':", 1)[0].strip() if "input':" in err_msg else err_msg
|
||||
logger.error(f"Rotation scan failed: {short_msg}")
|
||||
@@ -860,6 +864,7 @@ class DAQWorker(QObject):
|
||||
self.http_error.emit(str(e))
|
||||
finally:
|
||||
reply.deleteLater()
|
||||
|
||||
@Slot(RotationScanRequest)
|
||||
def standard_scan(self, r: RotationScanRequest):
|
||||
"""
|
||||
@@ -881,24 +886,23 @@ class DAQWorker(QObject):
|
||||
try:
|
||||
# Check for HTTP errors first
|
||||
if reply.error() != QNetworkReply.NetworkError.NoError:
|
||||
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
|
||||
raw_body = ""
|
||||
body_json = None
|
||||
err_msg = reply.errorString()
|
||||
status, err_msg, body_json = self._extract_reply_error_details(reply)
|
||||
|
||||
try:
|
||||
raw_body = reply.readAll().data().decode("utf-8")
|
||||
if raw_body:
|
||||
body_json = json.loads(raw_body)
|
||||
if isinstance(body_json, dict):
|
||||
err_msg = body_json.get("message", err_msg)
|
||||
code = body_json.get("code", "")
|
||||
if self._is_detector_state_failure_message(err_msg):
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
else:
|
||||
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
|
||||
# Check if this is a JFJoch error
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
except Exception:
|
||||
pass
|
||||
if self._is_critical_detector_failure(status, body_json, err_msg):
|
||||
critical_msg = (
|
||||
"Manual collection stopped because there is an error with the detector. "
|
||||
"Please call your local contact.\n\n"
|
||||
f"Details: {err_msg}"
|
||||
)
|
||||
self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True)
|
||||
self._emit_manual_collection_critical_failure(critical_msg)
|
||||
|
||||
logger.error(f"Raster scan failed: {err_msg}")
|
||||
self.http_error.emit(err_msg)
|
||||
@@ -1017,10 +1021,85 @@ class DAQWorker(QObject):
|
||||
reply = self.__net_manager.get(request)
|
||||
reply.finished.connect(lambda: self.handle_reference_tools_response(reply))
|
||||
|
||||
def _emit_manual_collection_critical_failure(self, msg: str) -> None:
|
||||
if not msg:
|
||||
return
|
||||
if msg == self._last_manual_collection_critical_msg:
|
||||
return
|
||||
self._last_manual_collection_critical_msg = msg
|
||||
self.manual_collection_critical_failure.emit(msg)
|
||||
|
||||
@staticmethod
|
||||
def _extract_reply_error_details(reply: QNetworkReply) -> tuple[int | None, str, dict | None]:
|
||||
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
|
||||
err_msg = reply.errorString()
|
||||
body_json = None
|
||||
|
||||
try:
|
||||
raw_body = reply.readAll().data().decode("utf-8")
|
||||
if raw_body:
|
||||
try:
|
||||
parsed = json.loads(raw_body)
|
||||
if isinstance(parsed, dict):
|
||||
body_json = parsed
|
||||
err_msg = (
|
||||
parsed.get("message")
|
||||
or parsed.get("detail")
|
||||
or parsed.get("error")
|
||||
or raw_body
|
||||
)
|
||||
else:
|
||||
err_msg = raw_body
|
||||
except Exception:
|
||||
err_msg = raw_body
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return status, str(err_msg), body_json
|
||||
|
||||
@staticmethod
|
||||
def _is_detector_state_failure_message(message: str | None) -> bool:
|
||||
text = str(message or "").lower()
|
||||
return (
|
||||
"daq state error" in text
|
||||
or "must be idle to start measurement" in text
|
||||
or "must be idle" in text
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _is_critical_detector_failure(
|
||||
status: int | None,
|
||||
body_json: dict | None = None,
|
||||
err_msg: str | None = None,
|
||||
) -> bool:
|
||||
try:
|
||||
status_int = int(status) if status is not None else None
|
||||
except Exception:
|
||||
status_int = None
|
||||
|
||||
code = None
|
||||
if isinstance(body_json, dict):
|
||||
code = body_json.get("code")
|
||||
|
||||
text = str(err_msg or "").lower()
|
||||
|
||||
if status_int == 500:
|
||||
return True
|
||||
|
||||
if code == "JFJOCH_UNAVAILABLE" and status_int is not None and 500 <= status_int < 600:
|
||||
return True
|
||||
|
||||
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _is_critical_automation_failure(
|
||||
status: int | None,
|
||||
body_json: dict | None = None,
|
||||
err_msg: str | None = None,
|
||||
) -> bool:
|
||||
"""Detect server-side critical automation errors that require operator recovery."""
|
||||
try:
|
||||
@@ -1035,11 +1114,13 @@ class DAQWorker(QObject):
|
||||
if code == DAQErrorCode.AUTOMATION_CRITICAL.value:
|
||||
return True
|
||||
|
||||
# Conservative fallback for /scan/auto: an unstructured 5xx means the
|
||||
# server raised unexpectedly during automation.
|
||||
if status_int is not None and 500 <= status_int < 600 and code is None:
|
||||
return True
|
||||
|
||||
text = str(err_msg or "").lower()
|
||||
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def handle_auto_scan_response(self, reply, sample_id: int):
|
||||
@@ -1048,24 +1129,10 @@ class DAQWorker(QObject):
|
||||
logger.info(f"Sample time {resp} s")
|
||||
self.automated_scan_done.emit(sample_id, True, "")
|
||||
else:
|
||||
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
|
||||
err_str = reply.errorString()
|
||||
body_json = None
|
||||
try:
|
||||
response_body = reply.readAll().data().decode("utf-8")
|
||||
if response_body:
|
||||
parsed_body = json.loads(response_body)
|
||||
if isinstance(parsed_body, dict):
|
||||
body_json = parsed_body
|
||||
err_str = (
|
||||
body_json.get("message")
|
||||
or body_json.get("detail")
|
||||
or response_body
|
||||
)
|
||||
else:
|
||||
err_str = response_body
|
||||
except Exception:
|
||||
pass
|
||||
status, err_str, body_json = self._extract_reply_error_details(reply)
|
||||
|
||||
if self._is_detector_state_failure_message(err_str):
|
||||
self._emit_detector_message(f"JFJoch: {err_str}", is_error=True)
|
||||
|
||||
if status == 401:
|
||||
logger.error(f"Error in auto scan: {err_str}")
|
||||
@@ -1080,7 +1147,7 @@ class DAQWorker(QObject):
|
||||
elif status == 417:
|
||||
self.sample_missing.emit(err_str)
|
||||
self.automated_scan_done.emit(sample_id, False, "Critical")
|
||||
elif self._is_critical_automation_failure(status, body_json):
|
||||
elif self._is_critical_automation_failure(status, body_json, err_str):
|
||||
logger.critical(f"Critical automation failure: {err_str}")
|
||||
self.http_error.emit(err_str)
|
||||
self.automated_scan_done.emit(sample_id, False, "Critical")
|
||||
|
||||
Reference in New Issue
Block a user