diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index fbe3a0c5..ea3a94e3 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -92,6 +92,66 @@ class AareDAQ: self._last_sample_sync_ts = 0.0 self._sample_sync_min_interval_s = 2.0 + def _is_hardware_failure(self, error: Exception) -> bool: + return isinstance( + error, + ( + CriticalTellException, + TellCommunicationError, + SmargonCommunicationError, + AerotechCommunicationError, + JFJochCommunicationError, + MountingFailed, + UnmountingFailed, + MagnetPositionSensorErorr, + TransformationInvalidException, + DataCollectionException, + RasterScanException, + ), + ) + + def _run_noncritical( + self, + action: Callable[[], object], + *, + description: str, + sample: SampleShortInfo | None = None, + ) -> object | None: + try: + return action() + except Exception as e: + logger.warning( + f"Non-critical failure during {description}: {e}", + extra={ + "sample_name": getattr(sample, "sample_name", None), + "sample_id": getattr(sample, "db_id", None), + }, + exc_info=True, + ) + return None + + def _run_critical( + self, + action: Callable[[], object], + *, + description: str, + sample: SampleShortInfo | None = None, + ) -> object: + try: + return action() + except Exception as e: + if isinstance(e, WarningTellException): + logger.warning( + f"Non-critical warning during {description}: {e}", + extra={ + "sample_name": getattr(sample, "sample_name", None), + "sample_id": getattr(sample, "db_id", None), + }, + exc_info=True, + ) + return None + raise + def set_face_detection_progress_callback(self, cb: Callable[[dict], None] | None) -> None: self._face_detection_progress_cb = cb @@ -2529,8 +2589,15 @@ class AareDAQ: filename: Name to give to the uploaded image. """ #time.sleep(0.2) # Wait 200 ms to ensure camera image is stable - bgr_image = self._get_inference_image() - self.__aare.upload_image(sample_id, filename, bgr_image) + def _upload() -> None: + bgr_image = self._get_inference_image() + self.__aare.upload_image(sample_id, filename, bgr_image) + + self._run_noncritical( + _upload, + description=f"screenshot upload '{filename}'", + sample=self.sample, + ) def send_screenshot_db(self, filename: str | None = None, message: str | None = None) -> None: sample = self.sample @@ -2554,8 +2621,11 @@ class AareDAQ: upload_name = filename or f"{sample_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" final_message = (message or "").strip() or self._default_screenshot_message(sample_id) - self.__aare.upload_image(sample_id, upload_name, bgr_image, message=final_message) - + self._run_noncritical( + lambda: self.__aare.upload_image(sample_id, upload_name, bgr_image, message=final_message), + description=f"send screenshot '{upload_name}'", + sample=sample, + ) @property def sample_spreadsheet(self) -> SampleShortInfoList: @@ -2801,8 +2871,8 @@ class AareDAQ: sample=sample, error=e, event_type=SampleEventType.FAILED) - finally: - logger.debug(f"finally at {time.perf_counter() - start}") + + raise Exception(f"Critical Error in automation: {e}") from e self._mark_progress_finished(progress, True, "Automation complete") return self._end_operation(start, DAQOperation.AUTOMATION, error=False)