diff --git a/daq/src/aaredaq/daq.py b/daq/src/aaredaq/daq.py index 278ff370..7b22ff25 100644 --- a/daq/src/aaredaq/daq.py +++ b/daq/src/aaredaq/daq.py @@ -322,23 +322,26 @@ class AareDAQ: self.__cfg.state_busy = False raise - def __mount_failure_handler(self): + def __mount_failure_handler(self, mount_error): failed = self.__cfg.increment_failed_mount_count() - if failed == 2: - self.__devs.tell.dry(wait=True) - elif failed > 2: + if failed >=4: self.__devs.tell.dry(wait=True, wait_cold=False) - raise MountingFailed(f"Repeated drying failure, moved to park. " + raise CriticalTellException(f"Repeated drying failure, moved to park" + f"\nThere either really is no sample or" + f"a critical failure.\nPlease check the dewar and if a sample, " + f"please contact the MX team") + elif failed >=3: + self.__devs.tell.dry(wait=True, wait_cold=False) + raise WarningTellException(f"Continued to not find a sample, drying robot in park for 10 minutes. " f"\n Please get a cup of coffee.\nAfter the robot has dried, " f"check the sample positions that have been missed, if no samples continue!" f"\nOtherwise there may be a critical error so please let your local " f"contact for support") - elif failed > 3: - self.__devs.tell.dry(wait=True, wait_cold=False) - raise MountingFailed(f"Repeated drying failure, moved to park" - f"\nThere either really is no sample or" - f"a critical failure.\nPlease check the dewar and if a sample, " - f"please contact the MX team") + elif failed == 2: + self.__devs.tell.dry(wait=True) + raise MountingFailed(f"Drying gripper: repeated error: {mount_error}") + else: + raise MountingFailed(f"{mount_error}") def __mount(self, target: SampleShortInfo | None): self.__set_state(BeamlineStateEnum.RobotSampleExchange) @@ -409,10 +412,11 @@ class AareDAQ: logger.debug(f"Post tell mount, pre db input") if value is not None: if value == "No Pin in Gripper": - logger.error("No sample was detected in gripper") - self.__mount_failure_handler() - self.__aare.sample_failed(target, failed_comment = "No sample was detected in gripper") - raise MountingFailed(f"No sample was detected in gripper, drying gripper {target.sample_name} {target.location} {target.pin}") + mount_error = "No sample was detected in gripper" + logger.error(mount_error) + self.__aare.sample_failed(target, failed_comment = mount_error) + self.__mount_failure_handler(string=mount_error) + raise MountingFailed(f"{mount_error}: drying gripper {target.sample_name} {target.location} {target.pin}") elif value == "dry": logger.info("Robot is drying") else: @@ -421,9 +425,10 @@ class AareDAQ: mounted = self.__devs.tell.get_mounted_sample() logger.info(f"response from tell {mounted}") if not mounted: - logger.error(f"Failed to mount target: {target.db_id} {target.location} {target.pin}") + mount_error = "Failed to mount target" + logger.error(f"{mount_error}: {target.db_id} {target.location} {target.pin}") self.__aare.sample_failed(target, failed_comment = "No sample was mounted") - self.__mount_failure_handler() + self.__mount_failure_handler(string=mount_error) raise MountingFailed(f"No Sample detected on smart magent: {target.sample_name} {target.location} {target.pin}") if target is not None: diff --git a/daq/src/aaredaq/server.py b/daq/src/aaredaq/server.py index f7ec39d8..c071c304 100644 --- a/daq/src/aaredaq/server.py +++ b/daq/src/aaredaq/server.py @@ -25,7 +25,8 @@ from urllib3.exceptions import InsecureRequestWarning from aaredaq import auth from aaredaqlib.beamline import mx_beamline from aaredaq.config import BeamlineConfig -from aaredaq.daq import AareDAQ, LoopCenteringFailed, TransformationInvalidException, MountingFailed +from aaredaq.daq import (AareDAQ, LoopCenteringFailed, TransformationInvalidException, + MountingFailed, WarningTellException, CriticalTellException) app = FastAPI() @@ -259,6 +260,16 @@ async def mount(dbid: int, token: str = Depends(oauth2_scheme), reference: bool status_code=api_status.HTTP_404_NOT_FOUND, detail=f"{e}", ) + except WarningTellException(Exception) as e: + raise HTTPException( + status_code=api_status.HTTP_410_GONE, + detail=f"{e}", + ) + except CriticalTellException as e: + raise HTTPException( + status_code=api_status.HTTP_417_EXPECTATION_FAILED, + detail=f"{e}" + ) except Exception as e: raise HTTPException( status_code=api_status.HTTP_500_INTERNAL_SERVER_ERROR, @@ -411,6 +422,16 @@ async def auto(s: SampleShortInfo, token: str = Depends(oauth2_scheme)): status_code=api_status.HTTP_404_NOT_FOUND, detail=f"{e}", ) + except WarningTellException(Exception) as e: + raise HTTPException( + status_code=api_status.HTTP_410_GONE, + detail=f"{e}", + ) + except CriticalTellException as e: + raise HTTPException( + status_code=api_status.HTTP_417_EXPECTATION_FAILED, + detail=f"{e}" + ) except Exception as e: logger.error(f"Exception in auto: {e}") raise HTTPException( diff --git a/gui/src/aaregui/main_window.py b/gui/src/aaregui/main_window.py index 8bdaed3c..f44a97a9 100644 --- a/gui/src/aaregui/main_window.py +++ b/gui/src/aaregui/main_window.py @@ -350,6 +350,8 @@ class MainWindow(QMainWindow): self.daq.update.connect(self.fluor_panel.update_daq_status) self.daq.update.connect(self.update_daq_status) + self.daq.sample_missing.connect(self.show_sample_missing_dialog) + self.daq.standard_scan_completed.connect(self.rotation.scan_completed) self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done) self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed) @@ -435,6 +437,13 @@ class MainWindow(QMainWindow): "About", "Aare Macromolecular Crystallography GUI\nVersion: 1.0\nCopyright: Paul Scherrer Institute 2024-2025", ) + + @Slot(str) + def show_sample_missing_dialog(self, msg: str): + if self.job_list_panel.is_running(): + logger.warning(f"Automation: Sample missing: {msg}") + return + QMessageBox.warning(self, "No Sample", f"No sample was found.\n\n(Server message: {msg})") #TODO tidy up mount and sampel view fucntions @Slot() def mount_view(self): diff --git a/gui/src/aaregui/panels/sample_queue_panel.py b/gui/src/aaregui/panels/sample_queue_panel.py index a3147966..49986a1f 100644 --- a/gui/src/aaregui/panels/sample_queue_panel.py +++ b/gui/src/aaregui/panels/sample_queue_panel.py @@ -1,4 +1,4 @@ -from PySide6.QtCore import Signal, Slot, Qt +from PySide6.QtCore import Signal, Slot, Qt, QTimer from PySide6.QtWidgets import QFrame, QVBoxLayout, QHBoxLayout, QPushButton, QHeaderView, QSizePolicy, \ QTableView, QMessageBox from PySide6.QtGui import QKeySequence, QShortcut @@ -26,6 +26,12 @@ class SampleQueuePanel(QFrame): self._current_db_id: int | None = None self.ring_current = None self._experiment_shutter_state = None + self.__consecutive_missing_samples = 0 + + self.__recovery_timer = QTimer(self) + self.__recovery_timer.setSingleShot(True) + self.__recovery_timer.setInterval(600 * 1000) # 10 minutes + self.__recovery_timer.timeout.connect(self.resume_automation) self.setFrameShape(QFrame.Shape.StyledPanel) self.setFrameShadow(QFrame.Shadow.Raised) @@ -104,7 +110,42 @@ class SampleQueuePanel(QFrame): return False return True + def resume_automation(self): + if hasattr(self, '__warning_msg_box') and self.__warning_msg_box: + self.__warning_msg_box.done(0) + self.__warning_msg_box = None + + if self.__pause and len(self.table_model.samples) > 0: + logger.info("Resuming automation after timeout") + self.run() + + def manual_resume_from_warning(self): + logger.info("User chose to resume automation manually from dialog") + self.resume_automation() + + def show_warning_recovery_dialog(self): + self.__warning_msg_box = QMessageBox(self) + self.__warning_msg_box.setIcon(QMessageBox.Icon.Warning) + self.__warning_msg_box.setWindowTitle("TELL Warning") + self.__warning_msg_box.setText("TELL reported a warning (e.g. drying).") + self.__warning_msg_box.setInformativeText( + "Automation paused for 10 minutes.\nClick 'Continue Now' to resume immediately, or wait for auto-resume.") + + continue_btn = self.__warning_msg_box.addButton("Continue Now", QMessageBox.ButtonRole.AcceptRole) + self.__warning_msg_box.setWindowModality(Qt.WindowModality.NonModal) + continue_btn.clicked.connect(self.manual_resume_from_warning) + self.__warning_msg_box.show() + + def is_running(self): + return not self.__pause + def run(self): + self.__consecutive_missing_samples = 0 + self.__recovery_timer.stop() + if hasattr(self, '__warning_msg_box') and self.__warning_msg_box: + self.__warning_msg_box.done(0) + self.__warning_msg_box = None + if not self.ring_current_check(): logger.debug("low ring current, skipping") return @@ -135,8 +176,15 @@ class SampleQueuePanel(QFrame): self.ring_current = s.bl.ring_current_mA self._experiment_shutter_state = s.bl.exp_shutter_open + def pause_automation(self, set_id_to_None: bool = True): + self.table_model.set_running(False) + self.__pause = True + self.play_button.setText("▶ Run") + if set_id_to_None: + self._current_db_id = None + @Slot(int, bool) - def automated_scan_done(self, db_id: int, success: bool): + def automated_scan_done(self, db_id: int, success: bool, reply:str): if self._current_db_id is not None and db_id != self._current_db_id: return @@ -145,11 +193,9 @@ class SampleQueuePanel(QFrame): if self.ring_current is None or (self.ring_current < LOW_CURRENT_THRESHOLD): # Pause UI state - self.table_model.set_running(False) - self.__pause = True - self.play_button.setText("▶ Run") + self.pause_automation(set_id_to_None=False) - # This dialog will auto-accept when current_ok() returns True. + # This dialog should auto-accept when current_ok() returns True. # No user button press is required to continue. if ring_current_auto_check(self, self.ring_current, current_ok): logger.debug("Ring current recovered or user chose to continue," @@ -165,6 +211,7 @@ class SampleQueuePanel(QFrame): if success: self.table_model.remove_sample(db_id) self._current_db_id = None + if not self.__pause and len(self.table_model.samples) > 0: next_item = self.table_model.samples[0] self._current_db_id = next_item.db_id @@ -174,8 +221,23 @@ class SampleQueuePanel(QFrame): self.__pause = True self.play_button.setText("▶ Run") self.unmount.emit() + else: - self.table_model.set_running(False) - self.__pause = True - self.play_button.setText("▶ Run") - self._current_db_id = None + if reply == "Warning": + self.pause_automation() + logger.warning("TELL Warning: Pausing for 10 minutes.") + # recovery timer of ten minutes to allow Tell to dry. However user should be able to interupt + self.__recovery_timer.start() + self.show_warning_recovery_dialog() + elif reply == "Critical": + self.pause_automation() + logger.critical("TELL Critical Error: Stopping automation.") + else: + # Missing or other errors -> continue to next sample + if not self.__pause and len(self.table_model.samples) > 0: + next_item = self.table_model.samples[0] + self._current_db_id = next_item.db_id + self.auto_scan.emit(next_item) + else: + self.pause_automation() + self.unmount.emit() \ No newline at end of file diff --git a/gui/src/aaregui/threads/daq_worker.py b/gui/src/aaregui/threads/daq_worker.py index ad8982ea..5222bd9f 100644 --- a/gui/src/aaregui/threads/daq_worker.py +++ b/gui/src/aaregui/threads/daq_worker.py @@ -27,7 +27,7 @@ class DAQWorker(QObject): http_error = Signal(str) auth_error = Signal() sample_missing = Signal(str) - automated_scan_done = Signal(int, bool) # sample ID, success + automated_scan_done = Signal(int, bool, str) # sample ID, success run_number_incremented = Signal() raster_scan_completed = Signal(CompletedRasterGrid) standard_scan_completed = Signal(CompletedRotationScan) @@ -131,7 +131,7 @@ class DAQWorker(QObject): logger.error(f"{err_str}: baton taken by another user") self._last_auth_error_log_ts = now self.auth_error.emit() - elif status == 404: + elif status == (404, 410, 417): self.sample_missing.emit(err_str) else: logger.error(f"{err_str}") @@ -391,17 +391,28 @@ class DAQWorker(QObject): if reply.error() == QNetworkReply.NetworkError.NoError: resp = reply.readAll().data().decode("utf-8") logger.info(f"Sample time {resp} s") - self.automated_scan_done.emit(sample_id, True) + self.automated_scan_done.emit(sample_id, True, "") else: - if reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) == 401: - logger.error(f"Error in auto scan: {reply.errorString()}") + status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) + err_str = self._get_reply_error(reply) + + if status == 401: + logger.error(f"Error in auto scan: {err_str}") + self.automated_scan_done.emit(sample_id, False, "Authentication Error") self.auth_error.emit() - elif reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) == 404: - self.sample_missing.emit(reply.errorString()) + elif status == 404: + self.sample_missing.emit(err_str) + self.automated_scan_done.emit(sample_id, False, "Missing") + elif status == 410: + self.sample_missing.emit(err_str) + self.automated_scan_done.emit(sample_id, False, "Warning") + elif status == 417: + self.sample_missing.emit(err_str) + self.automated_scan_done.emit(sample_id, False, "Critical") else: - logger.error(f"Error in auto scan: {reply.errorString()}") - self.http_error.emit(reply.errorString()) - self.automated_scan_done.emit(sample_id, False) + logger.error(f"Error in auto scan: {err_str}") + self.http_error.emit(err_str) + self.automated_scan_done.emit(sample_id, False, err_str) reply.deleteLater() @Slot(SampleShortInfo)