GUI/DAQ: changed how tell mounting errors are handled. need to be tested

2025-12-10 16:05:36 +01:00
parent 521a488ec6
commit 116aa51fdd
5 changed files with 146 additions and 38 deletions
@@ -322,23 +322,26 @@ class AareDAQ:
            self.__cfg.state_busy = False
            raise

-    def __mount_failure_handler(self):
+    def __mount_failure_handler(self, mount_error):
        failed = self.__cfg.increment_failed_mount_count()
-        if failed == 2:
-            self.__devs.tell.dry(wait=True)
-        elif failed > 2:
+        if failed >=4:
            self.__devs.tell.dry(wait=True, wait_cold=False)
-            raise MountingFailed(f"Repeated drying failure, moved to park. "
+            raise CriticalTellException(f"Repeated drying failure, moved to park"
+                                 f"\nThere either really is no sample or"
+                                 f"a critical failure.\nPlease check the dewar and if a sample, "
+                                 f"please contact the MX team")
+        elif failed >=3:
+            self.__devs.tell.dry(wait=True, wait_cold=False)
+            raise WarningTellException(f"Continued to not find a sample, drying robot in park for 10 minutes. "
                                 f"\n Please get a cup of coffee.\nAfter the robot has dried, "
                                 f"check the sample positions that have been missed, if no samples continue!"
                                 f"\nOtherwise there may be a critical error so please let your local "
                                 f"contact for support")
-        elif failed > 3:
-            self.__devs.tell.dry(wait=True, wait_cold=False)
-            raise MountingFailed(f"Repeated drying failure, moved to park"
-                                 f"\nThere either really is no sample or"
-                                 f"a critical failure.\nPlease check the dewar and if a sample, "
-                                 f"please contact the MX team")
+        elif failed == 2:
+            self.__devs.tell.dry(wait=True)
+            raise MountingFailed(f"Drying gripper: repeated error: {mount_error}")
+        else:
+            raise MountingFailed(f"{mount_error}")

    def __mount(self, target: SampleShortInfo | None):
        self.__set_state(BeamlineStateEnum.RobotSampleExchange)
@@ -409,10 +412,11 @@ class AareDAQ:
            logger.debug(f"Post tell mount, pre db input")
            if value is not None:
                if value == "No Pin in Gripper":
-                    logger.error("No sample was detected in gripper")
-                    self.__mount_failure_handler()
-                    self.__aare.sample_failed(target, failed_comment = "No sample was detected in gripper")
-                    raise MountingFailed(f"No sample was detected in gripper, drying gripper {target.sample_name} {target.location} {target.pin}")
+                    mount_error = "No sample was detected in gripper"
+                    logger.error(mount_error)
+                    self.__aare.sample_failed(target, failed_comment = mount_error)
+                    self.__mount_failure_handler(string=mount_error)
+                    raise MountingFailed(f"{mount_error}: drying gripper {target.sample_name} {target.location} {target.pin}")
                elif value == "dry":
                    logger.info("Robot is drying")
                else:
@@ -421,9 +425,10 @@ class AareDAQ:
            mounted = self.__devs.tell.get_mounted_sample()
            logger.info(f"response from tell {mounted}")
            if not mounted:
-                logger.error(f"Failed to mount target: {target.db_id} {target.location} {target.pin}")
+                mount_error = "Failed to mount target"
+                logger.error(f"{mount_error}: {target.db_id} {target.location} {target.pin}")
                self.__aare.sample_failed(target, failed_comment = "No sample was mounted")
-                self.__mount_failure_handler()
+                self.__mount_failure_handler(string=mount_error)
                raise MountingFailed(f"No Sample detected on smart magent: {target.sample_name} {target.location} {target.pin}")

            if target is not None:
@@ -25,7 +25,8 @@ from urllib3.exceptions import InsecureRequestWarning
 from aaredaq import auth
 from aaredaqlib.beamline import mx_beamline
 from aaredaq.config import BeamlineConfig
-from aaredaq.daq import AareDAQ, LoopCenteringFailed, TransformationInvalidException, MountingFailed
+from aaredaq.daq import (AareDAQ, LoopCenteringFailed, TransformationInvalidException,
+                         MountingFailed, WarningTellException, CriticalTellException)

 app = FastAPI()

@@ -259,6 +260,16 @@ async def mount(dbid: int, token: str = Depends(oauth2_scheme), reference: bool
                status_code=api_status.HTTP_404_NOT_FOUND,
                detail=f"{e}",
            )
+        except WarningTellException(Exception) as e:
+            raise HTTPException(
+                status_code=api_status.HTTP_410_GONE,
+                detail=f"{e}",
+            )
+        except CriticalTellException as e:
+            raise HTTPException(
+                status_code=api_status.HTTP_417_EXPECTATION_FAILED,
+                detail=f"{e}"
+            )
        except Exception as e:
            raise HTTPException(
                status_code=api_status.HTTP_500_INTERNAL_SERVER_ERROR,
@@ -411,6 +422,16 @@ async def auto(s: SampleShortInfo, token: str = Depends(oauth2_scheme)):
            status_code=api_status.HTTP_404_NOT_FOUND,
            detail=f"{e}",
        )
+    except WarningTellException(Exception) as e:
+        raise HTTPException(
+            status_code=api_status.HTTP_410_GONE,
+            detail=f"{e}",
+        )
+    except CriticalTellException as e:
+        raise HTTPException(
+            status_code=api_status.HTTP_417_EXPECTATION_FAILED,
+            detail=f"{e}"
+        )
    except Exception as e:
        logger.error(f"Exception in auto: {e}")
        raise HTTPException(
@@ -350,6 +350,8 @@ class MainWindow(QMainWindow):
        self.daq.update.connect(self.fluor_panel.update_daq_status)
        self.daq.update.connect(self.update_daq_status)

+        self.daq.sample_missing.connect(self.show_sample_missing_dialog)
+
        self.daq.standard_scan_completed.connect(self.rotation.scan_completed)
        self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done)
        self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
@@ -435,6 +437,13 @@ class MainWindow(QMainWindow):
            "About",
            "Aare Macromolecular Crystallography GUI\nVersion: 1.0\nCopyright: Paul Scherrer Institute 2024-2025",
        )
+
+    @Slot(str)
+    def show_sample_missing_dialog(self, msg: str):
+        if self.job_list_panel.is_running():
+            logger.warning(f"Automation: Sample missing: {msg}")
+            return
+        QMessageBox.warning(self, "No Sample", f"No sample was found.\n\n(Server message: {msg})")
 #TODO tidy up mount and sampel view fucntions
    @Slot()
    def mount_view(self):
@@ -1,4 +1,4 @@
-from PySide6.QtCore import Signal, Slot, Qt
+from PySide6.QtCore import Signal, Slot, Qt, QTimer
 from PySide6.QtWidgets import QFrame, QVBoxLayout, QHBoxLayout, QPushButton, QHeaderView, QSizePolicy, \
    QTableView, QMessageBox
 from PySide6.QtGui import QKeySequence, QShortcut
@@ -26,6 +26,12 @@ class SampleQueuePanel(QFrame):
        self._current_db_id: int | None = None
        self.ring_current = None
        self._experiment_shutter_state = None
+        self.__consecutive_missing_samples = 0
+
+        self.__recovery_timer = QTimer(self)
+        self.__recovery_timer.setSingleShot(True)
+        self.__recovery_timer.setInterval(600 * 1000)  # 10 minutes
+        self.__recovery_timer.timeout.connect(self.resume_automation)

        self.setFrameShape(QFrame.Shape.StyledPanel)
        self.setFrameShadow(QFrame.Shadow.Raised)
@@ -104,7 +110,42 @@ class SampleQueuePanel(QFrame):
            return False
        return True

+    def resume_automation(self):
+        if hasattr(self, '__warning_msg_box') and self.__warning_msg_box:
+            self.__warning_msg_box.done(0)
+            self.__warning_msg_box = None
+
+        if self.__pause and len(self.table_model.samples) > 0:
+            logger.info("Resuming automation after timeout")
+            self.run()
+
+    def manual_resume_from_warning(self):
+        logger.info("User chose to resume automation manually from dialog")
+        self.resume_automation()
+
+    def show_warning_recovery_dialog(self):
+        self.__warning_msg_box = QMessageBox(self)
+        self.__warning_msg_box.setIcon(QMessageBox.Icon.Warning)
+        self.__warning_msg_box.setWindowTitle("TELL Warning")
+        self.__warning_msg_box.setText("TELL reported a warning (e.g. drying).")
+        self.__warning_msg_box.setInformativeText(
+            "Automation paused for 10 minutes.\nClick 'Continue Now' to resume immediately, or wait for auto-resume.")
+
+        continue_btn = self.__warning_msg_box.addButton("Continue Now", QMessageBox.ButtonRole.AcceptRole)
+        self.__warning_msg_box.setWindowModality(Qt.WindowModality.NonModal)
+        continue_btn.clicked.connect(self.manual_resume_from_warning)
+        self.__warning_msg_box.show()
+
+    def is_running(self):
+        return not self.__pause
+
    def run(self):
+        self.__consecutive_missing_samples = 0
+        self.__recovery_timer.stop()
+        if hasattr(self, '__warning_msg_box') and self.__warning_msg_box:
+            self.__warning_msg_box.done(0)
+            self.__warning_msg_box = None
+
        if not self.ring_current_check():
            logger.debug("low ring current, skipping")
            return
@@ -135,8 +176,15 @@ class SampleQueuePanel(QFrame):
        self.ring_current = s.bl.ring_current_mA
        self._experiment_shutter_state = s.bl.exp_shutter_open

+    def pause_automation(self, set_id_to_None: bool = True):
+        self.table_model.set_running(False)
+        self.__pause = True
+        self.play_button.setText("▶ Run")
+        if set_id_to_None:
+            self._current_db_id = None
+
    @Slot(int, bool)
-    def automated_scan_done(self, db_id: int, success: bool):
+    def automated_scan_done(self, db_id: int, success: bool, reply:str):
        if self._current_db_id is not None and db_id != self._current_db_id:
            return

@@ -145,11 +193,9 @@ class SampleQueuePanel(QFrame):

        if self.ring_current is None or (self.ring_current < LOW_CURRENT_THRESHOLD):
            # Pause UI state
-            self.table_model.set_running(False)
-            self.__pause = True
-            self.play_button.setText("▶ Run")
+            self.pause_automation(set_id_to_None=False)

-            # This dialog will auto-accept when current_ok() returns True.
+            # This dialog should auto-accept when current_ok() returns True.
            # No user button press is required to continue.
            if ring_current_auto_check(self, self.ring_current, current_ok):
                logger.debug("Ring current recovered or user chose to continue,"
@@ -165,6 +211,7 @@ class SampleQueuePanel(QFrame):
        if success:
            self.table_model.remove_sample(db_id)
            self._current_db_id = None
+
            if not self.__pause and len(self.table_model.samples) > 0:
                next_item = self.table_model.samples[0]
                self._current_db_id = next_item.db_id
@@ -174,8 +221,23 @@ class SampleQueuePanel(QFrame):
                self.__pause = True
                self.play_button.setText("▶ Run")
                self.unmount.emit()
+
        else:
-            self.table_model.set_running(False)
-            self.__pause = True
-            self.play_button.setText("▶ Run")
-            self._current_db_id = None
+            if reply == "Warning":
+                self.pause_automation()
+                logger.warning("TELL Warning: Pausing for 10 minutes.")
+                # recovery timer of ten minutes to allow Tell to dry. However user should be able to interupt
+                self.__recovery_timer.start()
+                self.show_warning_recovery_dialog()
+            elif reply == "Critical":
+                self.pause_automation()
+                logger.critical("TELL Critical Error: Stopping automation.")
+            else:
+                # Missing or other errors -> continue to next sample
+                if not self.__pause and len(self.table_model.samples) > 0:
+                    next_item = self.table_model.samples[0]
+                    self._current_db_id = next_item.db_id
+                    self.auto_scan.emit(next_item)
+                else:
+                    self.pause_automation()
+                    self.unmount.emit()
@@ -27,7 +27,7 @@ class DAQWorker(QObject):
    http_error = Signal(str)
    auth_error = Signal()
    sample_missing = Signal(str)
-    automated_scan_done = Signal(int, bool)  # sample ID, success
+    automated_scan_done = Signal(int, bool, str)  # sample ID, success
    run_number_incremented = Signal()
    raster_scan_completed = Signal(CompletedRasterGrid)
    standard_scan_completed = Signal(CompletedRotationScan)
@@ -131,7 +131,7 @@ class DAQWorker(QObject):
                    logger.error(f"{err_str}: baton taken by another user")
                    self._last_auth_error_log_ts = now
                self.auth_error.emit()
-            elif status == 404:
+            elif status == (404, 410, 417):
                self.sample_missing.emit(err_str)
            else:
                logger.error(f"{err_str}")
@@ -391,17 +391,28 @@ class DAQWorker(QObject):
        if reply.error() == QNetworkReply.NetworkError.NoError:
            resp = reply.readAll().data().decode("utf-8")
            logger.info(f"Sample time {resp} s")
-            self.automated_scan_done.emit(sample_id, True)
+            self.automated_scan_done.emit(sample_id, True, "")
        else:
-            if reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) == 401:
-                logger.error(f"Error in auto scan: {reply.errorString()}")
+            status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
+            err_str = self._get_reply_error(reply)
+
+            if status == 401:
+                logger.error(f"Error in auto scan: {err_str}")
+                self.automated_scan_done.emit(sample_id, False, "Authentication Error")
                self.auth_error.emit()
-            elif reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) == 404:
-                self.sample_missing.emit(reply.errorString())
+            elif status == 404:
+                self.sample_missing.emit(err_str)
+                self.automated_scan_done.emit(sample_id, False, "Missing")
+            elif status == 410:
+                self.sample_missing.emit(err_str)
+                self.automated_scan_done.emit(sample_id, False, "Warning")
+            elif status == 417:
+                self.sample_missing.emit(err_str)
+                self.automated_scan_done.emit(sample_id, False, "Critical")
            else:
-                logger.error(f"Error in auto scan: {reply.errorString()}")
-                self.http_error.emit(reply.errorString())
-                self.automated_scan_done.emit(sample_id, False)
+                logger.error(f"Error in auto scan: {err_str}")
+                self.http_error.emit(err_str)
+                self.automated_scan_done.emit(sample_id, False, err_str)
        reply.deleteLater()

    @Slot(SampleShortInfo)