diff --git a/src/aare/common/error_codes.py b/src/aare/common/error_codes.py index d67bf298..cd9e678f 100644 --- a/src/aare/common/error_codes.py +++ b/src/aare/common/error_codes.py @@ -35,6 +35,7 @@ class DAQErrorCode(StrEnum): LOOP_CENTERING_FAILED = "LOOP_CENTERING_FAILED" AXC_FAILED = "AXC_FAILED" TRANSFORMATION_INVALID = "TRANSFORMATION_INVALID" + AUTOMATION_CRITICAL = "AUTOMATION_CRITICAL" SAMPLE_NOT_FOUND = "SAMPLE_NOT_FOUND" DATA_COLLECTION_FAILED = "DATA_COLLECTION_FAILED" RASTER_SCAN_FAILED = "RASTER_SCAN_FAILED" @@ -52,6 +53,7 @@ class DAQErrorCode(StrEnum): AAREDB_UNAVAILABLE = "AAREDB_UNAVAILABLE" MAGNET_POSITION_SENSOR_ERROR = "MAGNET_POSITION_SENSOR_ERROR" + _ERROR_CODE_HELP: dict[str, str] = { # Auth/JWT AuthErrorCode.AUTHENTICATION_ERROR: ( @@ -110,6 +112,10 @@ _ERROR_CODE_HELP: dict[str, str] = { DAQErrorCode.TRANSFORMATION_INVALID: ( "The requested beamline state transition is not allowed from the current state." ), + DAQErrorCode.AUTOMATION_CRITICAL: ( + "A critical failure occurred during fully automated measurement. The GUI should stop " + "dispatching further queue items and staff/operator recovery may be required." + ), DAQErrorCode.SAMPLE_NOT_FOUND: ( "The requested sample could not be found in the available sample lists." ), diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index ca6bbd69..5e082412 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -461,11 +461,21 @@ class AareDAQ: self.__mount(sample) if previous_sample is not None and previous_sample.db_id is not None: self.__aare.send_sample_event(previous_sample, SampleEventType.UNMOUNTED) + self.__set_state(BeamlineStateEnum.SampleAlignment) if sample is not None and sample.db_id is not None: self.__aare.send_sample_event(sample, SampleEventType.MOUNTED) self.save_screenshot_db(sample.db_id, f"{sample.db_id}_mounted") - self.__set_state(BeamlineStateEnum.SampleAlignment) + self.__set_state(BeamlineStateEnum.SampleAlignment) return True + except TransformationInvalidException as e: + logger.error(f"Mount failed due to invalid transformation: {e}") + self._handle_operation_error( + operation=DAQOperation.MOUNT, + sample=sample, + error=e, + event_type=SampleEventType.MOUNTFAILED, + ) + raise except Exception as e: logger.error(f"Mount failed: {e}") try: @@ -1273,19 +1283,26 @@ class AareDAQ: try: logger.debug(f"Mount target {target}") + if target is None: logger.debug("Unmounting sample") type = "Unmount" else: logger.debug(f"Mounting sample: {target}") type = "Mount" + if not self._execute_mount_and_prepare(target): - raise MountingFailed(f"Failed to {type} sample {target or self.__cfg.current_sample}") - logger.info(f"Sample mounted: {target}") + current_sample = target or self.__cfg.current_sample + if target is None: + raise UnmountingFailed(f"Failed to {type} sample {current_sample}") + raise MountingFailed(f"Failed to {type} sample {current_sample}") + + logger.info(f"Sample operation completed: {target}") self.__cfg.state_busy = False + except Exception as e: self.__cfg.state_busy = False - logger.debug(f"Failed to mount sample: {e}") + logger.debug(f"Failed to change mounted sample: {e}") raise @property @@ -2602,6 +2619,18 @@ class AareDAQ: if error: msg += f"with an error" self.__set_state(BeamlineStateEnum.RobotSampleExchange) + try: + if self.__cfg.state_busy: + self.__set_state(BeamlineStateEnum.RobotSampleExchange) + else: + logger.warning( + "Skipping recovery transition to RobotSampleExchange: " + "beamline is no longer busy (busy was released earlier)." + ) + except Exception: + logger.exception( + "Failed to transition to RobotSampleExchange during error recovery" + ) else: msg += f"successfully" logger.error(f"{msg}, time taken {time.perf_counter() - start} seconds.") @@ -2777,6 +2806,9 @@ class AareDAQ: if target == BeamlineStateEnum.Maintenance: self.__cfg.state = BeamlineStateEnum.Maintenance + elif target == curr_state: + logger.debug(f"State already set to {target}") + return elif target != curr_state: self.__cfg.state = BeamlineStateEnum.Moving try: @@ -2890,10 +2922,10 @@ class AareDAQ: except TransformationInvalidException as e: logger.error(f"Cannot go from {curr_state} to {target}: {e}") self.__cfg.state = curr_state - self.__cfg.state_busy = False raise except Exception as e: self.__cfg.state = BeamlineStateEnum.Maintenance + #TODO check if this is the right thing to do self.__cfg.state_busy = False raise diff --git a/src/aare/daq/server.py b/src/aare/daq/server.py index e2172dda..3a22bb8f 100644 --- a/src/aare/daq/server.py +++ b/src/aare/daq/server.py @@ -13,7 +13,7 @@ import uvicorn from aare.common.coordinate import AerotechCoordinate from aare.common.auth_models import BatonStatus, BatonRequestStatus from aare.common.coordinate import SmargonCoordinate, Coordinate -from aare.common.error_codes import export_error_codes_grouped +from aare.common.error_codes import export_error_codes_grouped, DAQErrorCode from aare.common.logger_config import setup_logger, get_uvicorn_logging_config from aare.common.models import SampleShortInfo, DAQStatusModel, BeamlineStateEnum, BeamlineSettingsModel, \ SampleShortInfoList, SessionStatus, SampleCameraSettings, AutofocusSettings, TokenData, \ @@ -1279,7 +1279,17 @@ async def auto(s: SampleShortInfo, token: str = Depends(oauth2_scheme)): Formatted string of the total runtime. """ auth.check_jwt_rw(cfg, auth.parse_token(token)) - runtime = daq.measure(s) + try: + runtime = daq.measure(s) + except Exception as e: + logger.exception("Critical automation failure in /scan/auto") + raise HTTPException( + status_code=api_status.HTTP_500_INTERNAL_SERVER_ERROR, + detail={ + "code": DAQErrorCode.AUTOMATION_CRITICAL.value, + "message": str(e) or "Critical automation failure", + }, + ) from e return f"{runtime:0.3f}" diff --git a/src/aare/gui/main_window.py b/src/aare/gui/main_window.py index 89b25d8d..61ae7669 100644 --- a/src/aare/gui/main_window.py +++ b/src/aare/gui/main_window.py @@ -620,6 +620,7 @@ class MainWindow(QMainWindow): self.daq.standard_scan_completed.connect(self.rotation.scan_completed) self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done) + self.daq.automation_critical_failure.connect(self._on_automation_critical_failure) self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed) self.manual_sample_panel.sample_manual.connect(self.daq.sample_manual) @@ -1057,6 +1058,88 @@ class MainWindow(QMainWindow): self._dev_help_dialog.raise_() self._dev_help_dialog.activateWindow() + @Slot(str) + def _on_automation_critical_failure(self, message: str) -> None: + """ + Hard-stop handler for fatal automation failures coming from the server + (transformation errors, busy-flag corruption, unhandled 500s on /scan/auto). + + Stops the queue, marks the progress widget as failed and surfaces the + beamline recovery dialog (staff) or a critical error message (users). + """ + logger.critical(f"Automation critical failure: {message}") + + # 1. Stop the sample queue so we don't keep dispatching new samples + try: + if self.job_list_panel is not None and self.job_list_panel.is_running(): + self.job_list_panel.pause_automation() + if self.daq is not None: + self.daq.send_status_request() + except Exception as e: + logger.error(f"Failed to pause automation queue after critical failure: {e}") + + # 2. Mark the automation progress widget as finished-with-error so + # _is_automation_active() returns False and idle/close timers behave. + try: + from aare.common.automation_models import ( + AutomationProgress, + StepState, + StepStatus, + WorkflowStateKind, + ) + progress = getattr(self.automation_progress_panel, "_progress", None) + if progress is None: + progress = AutomationProgress( + current_step=None, + steps=[ + StepState(step=WorkflowStateKind.MOUNT, status=StepStatus.PENDING), + StepState(step=WorkflowStateKind.LOOP_CENTRE, status=StepStatus.PENDING), + StepState(step=WorkflowStateKind.RASTER, status=StepStatus.PENDING), + StepState(step=WorkflowStateKind.DATA_COLLECTION, status=StepStatus.PENDING), + StepState(step=WorkflowStateKind.FINAL, status=StepStatus.PENDING), + ], + finished=False, + success=None, + ) + for step in progress.steps: + if step.status == StepStatus.RUNNING: + step.status = StepStatus.FAILED + step.message = message + if step.step == WorkflowStateKind.FINAL: + step.status = StepStatus.FAILED + step.message = message + progress.finished = True + progress.success = False + self.automation_progress_panel.set_progress(progress) + except Exception as e: + logger.error(f"Failed to update automation progress after critical failure: {e}") + + # 3. Banner so the operator sees it immediately + try: + self.alert_banner.show_message( + f"Automation halted: {message}", True, auto_clear_ms=0 + ) + except Exception: + pass + + # 4. Surface recovery UI + try: + if bool(getattr(self.__decoded_token, "staff", False)): + self.show_beamline_recovery() + else: + QMessageBox.critical( + self, + "Automation halted", + ( + "A critical error occurred during automation and the " + "beamline could not recover automatically:\n\n" + f"{message}\n\n" + "Please contact your local contact to recover the beamline." + ), + ) + except Exception as e: + logger.error(f"Failed to surface recovery UI after critical failure: {e}") + def show_beamline_recovery(self) -> None: if not bool(getattr(self.__decoded_token, "staff", False)): return @@ -1075,6 +1158,7 @@ class MainWindow(QMainWindow): logger.warning(f"Automation: Sample missing: {msg}") return QMessageBox.warning(self, "No Sample", f"{msg}") + #TODO tidy up mount and sampel view fucntions @Slot() def mount_view(self): diff --git a/src/aare/gui/panels/sample_queue_panel.py b/src/aare/gui/panels/sample_queue_panel.py index fae580e3..0471aae0 100644 --- a/src/aare/gui/panels/sample_queue_panel.py +++ b/src/aare/gui/panels/sample_queue_panel.py @@ -243,8 +243,7 @@ class SampleQueuePanel(QFrame): self._current_db_id = next_item.db_id self.auto_scan.emit(next_item) else: - self.pause_automation() - self.unmount.emit() + self._finish_empty_queue() else: if reply == "Warning": @@ -267,9 +266,16 @@ class SampleQueuePanel(QFrame): "\n If not please continue with new " "samples.\nOtherwise contact your" " local contact for support") + self.show_error_dialog(title=reply, msg="Critical Error", info="Repeated mount failuers.\n" + "Please check to see if pins" + " are loaded in these postions." + "\n If not please continue with new " + "samples.\nOtherwise contact your" + " local contact for support") elif reply == "Authentication Error": self.pause_automation() - self.show_error_dialog(title=reply,msg="Authentication Error", info="Please take the session to continue") + self.show_error_dialog(title=reply, msg="Authentication Error", + info="Please take the session to continue") else: self.table_model.remove_sample(db_id) self._current_db_id = None diff --git a/src/aare/gui/threads/daq_worker.py b/src/aare/gui/threads/daq_worker.py index c6e72b29..3fbe0b94 100644 --- a/src/aare/gui/threads/daq_worker.py +++ b/src/aare/gui/threads/daq_worker.py @@ -11,7 +11,7 @@ from jfjoch_client import ScanResult, ScanResultImagesInner from aare.common.coordinate import SmargonCoordinate, Coordinate, AerotechCoordinate from aare.common.auth_models import BatonStatus from aare.common.coordinate import SmargonCoordinate, Coordinate -from aare.common.error_codes import export_error_codes +from aare.common.error_codes import export_error_codes, DAQErrorCode from aare.common.exception_handler import JFJochCommunicationError from aare.common.models import ( DAQStatusModel, @@ -68,6 +68,7 @@ class DAQWorker(QObject): fluorimeter_spectrum_update = Signal(FluorescenceSpectrumOutputModel) sample_resync_completed = Signal(str) + automation_critical_failure = Signal(str) error_codes_loaded = Signal(dict) last_error_payload_changed = Signal(dict) last_error_payloads_changed = Signal(list) @@ -1024,6 +1025,31 @@ class DAQWorker(QObject): reply = self.__net_manager.get(request) reply.finished.connect(lambda: self.handle_reference_tools_response(reply)) + @staticmethod + def _is_critical_automation_failure( + status: int | None, + body_json: dict | None = None, + ) -> bool: + """Detect server-side critical automation errors that require operator recovery.""" + try: + status_int = int(status) if status is not None else None + except Exception: + status_int = None + + code = None + if isinstance(body_json, dict): + code = body_json.get("code") + + if code == DAQErrorCode.AUTOMATION_CRITICAL.value: + return True + + # Conservative fallback for /scan/auto: an unstructured 5xx means the + # server raised unexpectedly during automation. + if status_int is not None and 500 <= status_int < 600 and code is None: + return True + + return False + def handle_auto_scan_response(self, reply, sample_id: int): if reply.error() == QNetworkReply.NetworkError.NoError: resp = reply.readAll().data().decode("utf-8") @@ -1032,12 +1058,18 @@ class DAQWorker(QObject): else: status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) err_str = reply.errorString() + body_json = None try: response_body = reply.readAll().data().decode("utf-8") if response_body: - body_json = json.loads(response_body) - if "detail" in body_json: - err_str = body_json["detail"] + parsed_body = json.loads(response_body) + if isinstance(parsed_body, dict): + body_json = parsed_body + err_str = ( + body_json.get("message") + or body_json.get("detail") + or response_body + ) else: err_str = response_body except Exception: @@ -1056,6 +1088,11 @@ class DAQWorker(QObject): elif status == 417: self.sample_missing.emit(err_str) self.automated_scan_done.emit(sample_id, False, "Critical") + elif self._is_critical_automation_failure(status, body_json): + logger.critical(f"Critical automation failure: {err_str}") + self.http_error.emit(err_str) + self.automated_scan_done.emit(sample_id, False, "Critical") + self.automation_critical_failure.emit(err_str) else: logger.error(f"Error in auto scan: {err_str}") self.http_error.emit(err_str) @@ -1208,12 +1245,20 @@ class DAQWorker(QObject): def _parse_automation_progress(progress_payload: dict) -> AutomationProgress: steps: list[StepState] = [] + step_aliases = { + "final": WorkflowStateKind.FINAL, + "Paused/Finished": WorkflowStateKind.FINAL, + } + for raw_step in progress_payload.get("steps", []): raw_kind = raw_step.get("step") raw_status = raw_step.get("status", StepStatus.PENDING.value) raw_message = raw_step.get("message", "") - step_kind = WorkflowStateKind(raw_kind) + step_kind = step_aliases.get(raw_kind) + if step_kind is None: + step_kind = WorkflowStateKind(raw_kind) + step_status = StepStatus(raw_status) steps.append( StepState(