daq: testing helpers to make some functions noncritical and therefore not break automation...

src/aare/daq/daq.py

@@ -92,6 +92,66 @@ class AareDAQ:
         self._last_sample_sync_ts = 0.0
         self._sample_sync_min_interval_s = 2.0
 
+    def _is_hardware_failure(self, error: Exception) -> bool:
+        return isinstance(
+            error,
+            (
+                CriticalTellException,
+                TellCommunicationError,
+                SmargonCommunicationError,
+                AerotechCommunicationError,
+                JFJochCommunicationError,
+                MountingFailed,
+                UnmountingFailed,
+                MagnetPositionSensorErorr,
+                TransformationInvalidException,
+                DataCollectionException,
+                RasterScanException,
+            ),
+        )
+
+    def _run_noncritical(
+            self,
+            action: Callable[[], object],
+            *,
+            description: str,
+            sample: SampleShortInfo | None = None,
+    ) -> object | None:
+        try:
+            return action()
+        except Exception as e:
+            logger.warning(
+                f"Non-critical failure during {description}: {e}",
+                extra={
+                    "sample_name": getattr(sample, "sample_name", None),
+                    "sample_id": getattr(sample, "db_id", None),
+                },
+                exc_info=True,
+            )
+            return None
+
+    def _run_critical(
+            self,
+            action: Callable[[], object],
+            *,
+            description: str,
+            sample: SampleShortInfo | None = None,
+    ) -> object:
+        try:
+            return action()
+        except Exception as e:
+            if isinstance(e, WarningTellException):
+                logger.warning(
+                    f"Non-critical warning during {description}: {e}",
+                    extra={
+                        "sample_name": getattr(sample, "sample_name", None),
+                        "sample_id": getattr(sample, "db_id", None),
+                    },
+                    exc_info=True,
+                )
+                return None
+            raise
+
     def set_face_detection_progress_callback(self, cb: Callable[[dict], None] | None) -> None:
         self._face_detection_progress_cb = cb
 
@@ -2529,8 +2589,15 @@ class AareDAQ:
             filename: Name to give to the uploaded image.
         """
         #time.sleep(0.2)  # Wait 200 ms to ensure camera image is stable
-        bgr_image = self._get_inference_image()
-        self.__aare.upload_image(sample_id, filename, bgr_image)
+        def _upload() -> None:
+            bgr_image = self._get_inference_image()
+            self.__aare.upload_image(sample_id, filename, bgr_image)
+
+        self._run_noncritical(
+            _upload,
+            description=f"screenshot upload '{filename}'",
+            sample=self.sample,
+        )
 
     def send_screenshot_db(self, filename: str | None = None, message: str | None = None) -> None:
         sample = self.sample
@@ -2554,8 +2621,11 @@ class AareDAQ:
 
         upload_name = filename or f"{sample_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
         final_message = (message or "").strip() or self._default_screenshot_message(sample_id)
-        self.__aare.upload_image(sample_id, upload_name, bgr_image, message=final_message)
-
+        self._run_noncritical(
+            lambda: self.__aare.upload_image(sample_id, upload_name, bgr_image, message=final_message),
+            description=f"send screenshot '{upload_name}'",
+            sample=sample,
+        )
 
     @property
     def sample_spreadsheet(self) -> SampleShortInfoList:
@@ -2801,8 +2871,8 @@ class AareDAQ:
                 sample=sample,
                 error=e,
                 event_type=SampleEventType.FAILED)
-        finally:
-            logger.debug(f"finally at {time.perf_counter() - start}")
+
+            raise Exception(f"Critical Error in automation: {e}") from e
 
         self._mark_progress_finished(progress, True, "Automation complete")
         return self._end_operation(start, DAQOperation.AUTOMATION, error=False)