diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index 61faa431..8d4d0885 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -123,6 +123,39 @@ class AareDAQ: ), ) + @staticmethod + def _is_jfjoch_detector_state_error(message: str | None) -> bool: + text = str(message or "").lower() + return ( + "daq state error" in text + or "must be idle to start measurement" in text + or "must be idle" in text + ) + + def _raise_if_critical_jfjoch_detector_error( + self, + error: Exception, + *, + command: str, + ) -> None: + if not isinstance(error, JFJochCommunicationError): + return + + if ( + getattr(error, "status_code", None) != 500 + and not self._is_jfjoch_detector_state_error(str(error)) + ): + return + + message = ( + f"Critical detector error while running JFJoch command '{command}'. " + f"Automation has been stopped. " + f"There is an error with the detector. Please call your local contact. " + f"Original error: {error}" + ) + logger.critical(message) + raise CriticalTellException(message) from error + def _run_noncritical( self, action: Callable[[], object], @@ -1820,7 +1853,11 @@ class AareDAQ: self.__aare.create_gridscan_run(self.sample, request, status) if not self.__cfg.simulated_detector: - self.__jfjoch.wait_till_running(timeout=60.0) + try: + self.__jfjoch.wait_till_running(timeout=60.0) + except Exception as e: + self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running") + raise else: logger.info("Simulated detector mode enabled; faking jfjoch intilalisation.") @@ -1861,7 +1898,11 @@ class AareDAQ: else: #TODO how to handle no beam grids - do we always want to go to the center? #TODO How to handle grids with no/minimal diffraction - use bkg instead? - scan_result = self.__jfjoch.wait_till_done(60) + try: + scan_result = self.__jfjoch.wait_till_done(60) + except Exception as e: + self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done") + raise com = None if scan_result is None: logger.error( @@ -1982,7 +2023,7 @@ class AareDAQ: except Exception as e: logger.exception( - "Failed during raster", + f"Failed during raster: {e}", extra=merge_log_context( sample_log_context(self.sample), raster_request_log_context(request), @@ -2029,7 +2070,6 @@ class AareDAQ: omega_start = self.omega status = self.status - #self.__aare.create_rotation_run(self.sample, request, status) if request.exp_time_s < 0.004: logger.warning("Exposure time too short for PXII rotation scan") request.exp_time_s = 0.004 @@ -2037,10 +2077,15 @@ class AareDAQ: if self.sample is not None and self.sample.db_id is not None: self.__aare.create_rotation_run(self.sample, request, status) try: - self.__jfjoch.wait_till_running(timeout=60.0) + try: + self.__jfjoch.wait_till_running(timeout=60.0) + except Exception as e: + self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running") + raise + if request.screening: self.__devs.aerotech.screening_scan( - rotation_deg=request.steps*request.incr_omega_deg, + rotation_deg=request.steps * request.incr_omega_deg, wedge_deg=request.wedge_omega_deg, time_sec=total_time, steps=request.steps, @@ -2048,13 +2093,13 @@ class AareDAQ: ) else: self.__devs.aerotech.rotation_scan( - rotation_deg=request.steps*request.incr_omega_deg, + rotation_deg=request.steps * request.incr_omega_deg, time_sec=total_time, start_pos_deg=request.start_omega_deg, run_async=True, ) - #Is this for helical scans...? do we do smargon scans? + # Is this for helical scans...? do we do smargon scans? if request.start is not None and request.end is not None: smargon_time_step = request.exp_time_s / float(request.steps) pos_step = (request.end.sh_mm - request.start.sh_mm) * (1.0 / float(request.steps)) @@ -2065,14 +2110,18 @@ class AareDAQ: ) time.sleep(smargon_time_step) - self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60,0))) + self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + 60, 0))) self.__devs.aerotech_omega = omega_start if self.__cfg.simulated_detector: logger.warning("Detector in simulation mode, returning fake zero rotation result.") return self._build_fake_rotation_result(request) else: - scan_result = self.__jfjoch.wait_till_done(60) + try: + scan_result = self.__jfjoch.wait_till_done(60) + except Exception as e: + self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done") + raise return CompletedRotationScan( request=copy.deepcopy(request), result=scan_result, @@ -2772,9 +2821,7 @@ class AareDAQ: ) try: - logger.info(f"setting busy at {time.perf_counter() - start}") self.__cfg.try_set_busy(timeout=360) - logger.info(f"set busy at {time.perf_counter() - start}") self._set_progress_context( progress, @@ -2875,6 +2922,9 @@ class AareDAQ: self._mark_progress_success(progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete") logger.info(f"Rotation scan done at {time.perf_counter() - start}") + except JFJochCommunicationError as e: + self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown") + raise except Exception as e: logger.error(f"Error in measure: {e}") if progress.current_step is not None: diff --git a/src/aare/devices/jfjoch.py b/src/aare/devices/jfjoch.py index a450beb6..8db2bfc3 100644 --- a/src/aare/devices/jfjoch.py +++ b/src/aare/devices/jfjoch.py @@ -31,11 +31,56 @@ class JFJochWrapper: self.__client = jfjoch_client.ApiClient(jfjoch_client.Configuration(host=self.__url)) self.__api = jfjoch_client.DefaultApi(self.__client) + @staticmethod + def _extract_status_code(error: Exception) -> int | None: + for attr_name in ("status", "status_code"): + value = getattr(error, attr_name, None) + if isinstance(value, int): + return value + + body = getattr(error, "body", None) + if isinstance(body, str) and "500" in body: + return 500 + + return None + + def _raise_jfjoch_error( + self, + message: str, + *, + error: Exception, + operation: str, + endpoint: str, + ) -> None: + raise JFJochCommunicationError( + message, + operation=operation, + endpoint=endpoint, + base_url=self.__url, + status_code=self._extract_status_code(error), + ) from error + def initialize(self): - self.__api.initialize_post() + try: + self.__api.initialize_post() + except Exception as e: + self._raise_jfjoch_error( + "JFJoch initialize failed", + error=e, + operation="POST", + endpoint="initialize_post", + ) def cancel(self): - self.__api.cancel_post() + try: + self.__api.cancel_post() + except Exception as e: + self._raise_jfjoch_error( + "JFJoch cancel failed", + error=e, + operation="POST", + endpoint="cancel_post", + ) def is_idle(self) -> bool: status = self.__api.status_get() @@ -141,12 +186,12 @@ class JFJochWrapper: try: self.__api.start_post(dataset_settings=dataset_settings) except Exception as e: - raise JFJochCommunicationError( + self._raise_jfjoch_error( f"JFJoch data collection failed to initialize for {scan_type.value} scan with exception: {e}", + error=e, operation="POST", endpoint="start_post", - base_url=self.__url, - ) from e + ) def measure_rotation(self, r: RotationScanRequest, @@ -171,12 +216,12 @@ class JFJochWrapper: self.__api.wait_until_running_post_with_http_info(timeout=math.ceil(timeout)) return True except Exception as e: - raise JFJochCommunicationError( + self._raise_jfjoch_error( f"JFJoch wait until running returned an exception: {e}", - operation="GET", + error=e, + operation="POST", endpoint="wait_until_running_post", - base_url=self.__url, - ) from e + ) def wait_till_done(self, timeout : int | float) -> jfjoch_client.models.ScanResult | None: if self.__simulated: @@ -185,12 +230,12 @@ class JFJochWrapper: self.__api.wait_till_done_post_with_http_info(timeout=math.ceil(timeout)) return self.__api.result_scan_get() except Exception as e: - raise JFJochCommunicationError( + self._raise_jfjoch_error( f"JFJoch wait till done retrieval returned an exception: {e}", - operation="GET", + error=e, + operation="POST", endpoint="wait_till_done_post / result_scan_get", - base_url=self.__url, - ) from e + ) def detector(self) -> jfjoch_client.models.DetectorListElement: l = self.__api.config_select_detector_get() diff --git a/src/aare/gui/main_window.py b/src/aare/gui/main_window.py index 9ec5780a..aabe8f87 100644 --- a/src/aare/gui/main_window.py +++ b/src/aare/gui/main_window.py @@ -84,6 +84,7 @@ class MainWindow(QMainWindow): self.__mounting = False self.__samcam_feed_banner_active = False self.__samcam_feed_banner_message = "Sample camera feed unavailable" + self._automation_critical_banner_active = False self._dev_help_dialog = None self._beamline_recovery_dialog = None self._controls_help_dialog = None @@ -324,6 +325,12 @@ class MainWindow(QMainWindow): self.job_list_panel.samples_in_queue_changed.connect( self.automation_progress_panel.set_samples_in_queue ) + self.job_list_panel.automation_running_changed.connect( + self.automation_progress_panel.set_running + ) + self.job_list_panel.automation_running_changed.connect( + self._on_automation_running_changed + ) self.automation_progress_panel.set_samples_in_queue( len(self.job_list_panel.table_model.samples) ) @@ -615,9 +622,12 @@ class MainWindow(QMainWindow): self.daq.sample_missing.connect(self.show_sample_missing_dialog) self.daq.standard_scan_completed.connect(self.rotation.scan_completed) + self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed) + self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done) self.daq.automation_critical_failure.connect(self._on_automation_critical_failure) - self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed) + self.daq.manual_collection_critical_failure.connect(self._on_manual_collection_critical_failure) + self.daq.recovery_action_completed.connect(lambda _msg: self._clear_automation_critical_banner()) self.manual_sample_panel.sample_manual.connect(self.daq.sample_manual) self.face_panel.face_detection.connect(self.daq.face_detection) @@ -997,6 +1007,71 @@ class MainWindow(QMainWindow): self._dev_help_dialog.raise_() self._dev_help_dialog.activateWindow() + def _clear_automation_critical_banner(self) -> None: + if not self._automation_critical_banner_active: + return + self.alert_banner.clear_message() + self._automation_critical_banner_active = False + + @Slot(bool) + def _on_automation_running_changed(self, running: bool) -> None: + if running: + self._clear_automation_critical_banner() + + def _is_detector_critical_failure(self, message: str) -> bool: + text = (message or "").lower() + return "detector" in text or "jfjoch" in text + + @staticmethod + def _is_detector_state_error(message: str) -> bool: + text = (message or "").lower() + return ( + "daq state error" in text + or "must be idle to start measurement" in text + or "must be idle" in text + ) + + def _detector_error_dialog_title(self, message: str) -> str: + if self._is_detector_state_error(message): + return "Detector state error" + return "Detector error" + + def _detector_error_banner_text(self, *, automation: bool, message: str) -> str: + if self._is_detector_state_error(message): + prefix = "Automation halted" if automation else "Manual collection stopped" + return f"{prefix}: detector state error." + return ( + "Automation halted: detector error." + if automation + else "Manual collection stopped: detector error." + ) + + @Slot(str) + def _on_manual_collection_critical_failure(self, message: str) -> None: + logger.critical(f"Manual collection critical detector failure: {message}") + + try: + self.alert_banner_secondary.show_message( + self._detector_error_banner_text(automation=False, message=message), + True, + auto_clear_ms=0, + ) + except Exception: + pass + + try: + QMessageBox.critical( + self, + self._detector_error_dialog_title(message), + ( + "The manual collection has stopped because there is an error with the detector.\n\n" + "Please call your local contact.\n\n" + f"Details:\n{message}" + ), + ) + except Exception as e: + logger.error(f"Failed to show manual collection detector popup: {e}") + @Slot(str) def _on_automation_critical_failure(self, message: str) -> None: """ @@ -1008,6 +1083,13 @@ class MainWindow(QMainWindow): """ logger.critical(f"Automation critical failure: {message}") + is_detector_failure = self._is_detector_critical_failure(message) + banner_message = ( + self._detector_error_banner_text(automation=True, message=message) + if is_detector_failure + else f"Automation halted: {message}" + ) + # 1. Stop the sample queue so we don't keep dispatching new samples try: if self.job_list_panel is not None and self.job_list_panel.is_running(): @@ -1056,26 +1138,48 @@ class MainWindow(QMainWindow): # 3. Banner so the operator sees it immediately try: self.alert_banner.show_message( - f"Automation halted: {message}", True, auto_clear_ms=0 + banner_message, True, auto_clear_ms=0 ) + self._automation_critical_banner_active = True except Exception: pass # 4. Surface recovery UI try: if bool(getattr(self.__decoded_token, "staff", False)): + if is_detector_failure: + QMessageBox.critical( + self, + self._detector_error_dialog_title(message), + ( + "Automation has been stopped because there is an error with the detector.\n\n" + "Please call your local contact.\n\n" + f"Details:\n{message}" + ), + ) self.show_beamline_recovery() else: - QMessageBox.critical( - self, - "Automation halted", - ( - "A critical error occurred during automation and the " - "beamline could not recover automatically:\n\n" - f"{message}\n\n" - "Please contact your local contact to recover the beamline." - ), - ) + if is_detector_failure: + QMessageBox.critical( + self, + self._detector_error_dialog_title(message), + ( + "Automation has been stopped because there is an error with the detector.\n\n" + "Please call your local contact.\n\n" + f"Details:\n{message}" + ), + ) + else: + QMessageBox.critical( + self, + "Automation halted", + ( + "A critical error occurred during automation and the " + "beamline could not recover automatically:\n\n" + f"{message}\n\n" + "Please contact your local contact to recover the beamline." + ), + ) except Exception as e: logger.error(f"Failed to surface recovery UI after critical failure: {e}") diff --git a/src/aare/gui/threads/daq_worker.py b/src/aare/gui/threads/daq_worker.py index edcfc6ad..b6190523 100644 --- a/src/aare/gui/threads/daq_worker.py +++ b/src/aare/gui/threads/daq_worker.py @@ -69,6 +69,7 @@ class DAQWorker(QObject): sample_resync_completed = Signal(str) automation_critical_failure = Signal(str) + manual_collection_critical_failure = Signal(str) error_codes_loaded = Signal(dict) last_error_payload_changed = Signal(dict) last_error_payloads_changed = Signal(list) @@ -141,6 +142,8 @@ class DAQWorker(QObject): self._last_detector_msg: str | None = None self._last_detector_is_error: bool | None = None + self._last_manual_collection_critical_msg: str | None = None + self._baton_stream_reply: QNetworkReply | None = None self._last_baton_status: BatonStatus | None = None @@ -829,22 +832,23 @@ class DAQWorker(QObject): try: # Check for HTTP errors first if reply.error() != QNetworkReply.NetworkError.NoError: - status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) - err_msg = reply.errorString() + status, err_msg, body_json = self._extract_reply_error_details(reply) - try: - raw_body = reply.readAll().data().decode("utf-8") - if raw_body: - body_json = json.loads(raw_body) - if isinstance(body_json, dict): - err_msg = body_json.get("message", err_msg) - code = body_json.get("code", "") + if self._is_detector_state_failure_message(err_msg): + self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) + else: + code = body_json.get("code", "") if isinstance(body_json, dict) else "" + if code == "JFJOCH_UNAVAILABLE" or status == 503: + self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) - # Check if this is a JFJoch error - if code == "JFJOCH_UNAVAILABLE" or status == 503: - self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) - except Exception: - pass + if self._is_critical_detector_failure(status, body_json, err_msg): + critical_msg = ( + "Manual collection stopped because there is an error with the detector. " + "Please call your local contact.\n\n" + f"Details: {err_msg}" + ) + self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True) + self._emit_manual_collection_critical_failure(critical_msg) short_msg = err_msg.split("input':", 1)[0].strip() if "input':" in err_msg else err_msg logger.error(f"Rotation scan failed: {short_msg}") @@ -860,6 +864,7 @@ class DAQWorker(QObject): self.http_error.emit(str(e)) finally: reply.deleteLater() + @Slot(RotationScanRequest) def standard_scan(self, r: RotationScanRequest): """ @@ -881,24 +886,23 @@ class DAQWorker(QObject): try: # Check for HTTP errors first if reply.error() != QNetworkReply.NetworkError.NoError: - status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) - raw_body = "" - body_json = None - err_msg = reply.errorString() + status, err_msg, body_json = self._extract_reply_error_details(reply) - try: - raw_body = reply.readAll().data().decode("utf-8") - if raw_body: - body_json = json.loads(raw_body) - if isinstance(body_json, dict): - err_msg = body_json.get("message", err_msg) - code = body_json.get("code", "") + if self._is_detector_state_failure_message(err_msg): + self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) + else: + code = body_json.get("code", "") if isinstance(body_json, dict) else "" + if code == "JFJOCH_UNAVAILABLE" or status == 503: + self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) - # Check if this is a JFJoch error - if code == "JFJOCH_UNAVAILABLE" or status == 503: - self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True) - except Exception: - pass + if self._is_critical_detector_failure(status, body_json, err_msg): + critical_msg = ( + "Manual collection stopped because there is an error with the detector. " + "Please call your local contact.\n\n" + f"Details: {err_msg}" + ) + self._emit_detector_message(f"Detector error during manual collection: {err_msg}", is_error=True) + self._emit_manual_collection_critical_failure(critical_msg) logger.error(f"Raster scan failed: {err_msg}") self.http_error.emit(err_msg) @@ -1017,10 +1021,85 @@ class DAQWorker(QObject): reply = self.__net_manager.get(request) reply.finished.connect(lambda: self.handle_reference_tools_response(reply)) + def _emit_manual_collection_critical_failure(self, msg: str) -> None: + if not msg: + return + if msg == self._last_manual_collection_critical_msg: + return + self._last_manual_collection_critical_msg = msg + self.manual_collection_critical_failure.emit(msg) + + @staticmethod + def _extract_reply_error_details(reply: QNetworkReply) -> tuple[int | None, str, dict | None]: + status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) + err_msg = reply.errorString() + body_json = None + + try: + raw_body = reply.readAll().data().decode("utf-8") + if raw_body: + try: + parsed = json.loads(raw_body) + if isinstance(parsed, dict): + body_json = parsed + err_msg = ( + parsed.get("message") + or parsed.get("detail") + or parsed.get("error") + or raw_body + ) + else: + err_msg = raw_body + except Exception: + err_msg = raw_body + except Exception: + pass + + return status, str(err_msg), body_json + + @staticmethod + def _is_detector_state_failure_message(message: str | None) -> bool: + text = str(message or "").lower() + return ( + "daq state error" in text + or "must be idle to start measurement" in text + or "must be idle" in text + ) + + @staticmethod + def _is_critical_detector_failure( + status: int | None, + body_json: dict | None = None, + err_msg: str | None = None, + ) -> bool: + try: + status_int = int(status) if status is not None else None + except Exception: + status_int = None + + code = None + if isinstance(body_json, dict): + code = body_json.get("code") + + text = str(err_msg or "").lower() + + if status_int == 500: + return True + + if code == "JFJOCH_UNAVAILABLE" and status_int is not None and 500 <= status_int < 600: + return True + + if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text: + return True + + return False + + @staticmethod def _is_critical_automation_failure( status: int | None, body_json: dict | None = None, + err_msg: str | None = None, ) -> bool: """Detect server-side critical automation errors that require operator recovery.""" try: @@ -1035,11 +1114,13 @@ class DAQWorker(QObject): if code == DAQErrorCode.AUTOMATION_CRITICAL.value: return True - # Conservative fallback for /scan/auto: an unstructured 5xx means the - # server raised unexpectedly during automation. if status_int is not None and 500 <= status_int < 600 and code is None: return True + text = str(err_msg or "").lower() + if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text: + return True + return False def handle_auto_scan_response(self, reply, sample_id: int): @@ -1048,24 +1129,10 @@ class DAQWorker(QObject): logger.info(f"Sample time {resp} s") self.automated_scan_done.emit(sample_id, True, "") else: - status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) - err_str = reply.errorString() - body_json = None - try: - response_body = reply.readAll().data().decode("utf-8") - if response_body: - parsed_body = json.loads(response_body) - if isinstance(parsed_body, dict): - body_json = parsed_body - err_str = ( - body_json.get("message") - or body_json.get("detail") - or response_body - ) - else: - err_str = response_body - except Exception: - pass + status, err_str, body_json = self._extract_reply_error_details(reply) + + if self._is_detector_state_failure_message(err_str): + self._emit_detector_message(f"JFJoch: {err_str}", is_error=True) if status == 401: logger.error(f"Error in auto scan: {err_str}") @@ -1080,7 +1147,7 @@ class DAQWorker(QObject): elif status == 417: self.sample_missing.emit(err_str) self.automated_scan_done.emit(sample_id, False, "Critical") - elif self._is_critical_automation_failure(status, body_json): + elif self._is_critical_automation_failure(status, body_json, err_str): logger.critical(f"Critical automation failure: {err_str}") self.http_error.emit(err_str) self.automated_scan_done.emit(sample_id, False, "Critical")