DAQ: for large grids we were hitting a busy tiemout exception - have uppded the tiemout to 10 minutes, have added busy state timeout exceptions and now skip samples if raster too large - or when starting the DAQ can toggle this off so that we scale the grid instead to the max number of images - set to jungfrau buffer limit of 4500 iamges, needs testing.

2026-05-08 13:13:36 +02:00
parent ad5d0ab667
commit c4c8214957
1 changed files with 235 additions and 40 deletions
@@ -69,8 +69,14 @@ from aare.common.exception_handler import (
    AXCFailed,
    SmargonCommunicationError,
    TellCommunicationError,
-    JFJochCommunicationError, AerotechCommunicationError, MagnetPositionSensorErorr, UnmountingFailed,
-    DataCollectionException, RasterScanException, TellMountFailedException
+    JFJochCommunicationError,
+    AerotechCommunicationError,
+    MagnetPositionSensorErorr,
+    UnmountingFailed,
+    DataCollectionException,
+    RasterScanException,
+    TellMountFailedException,
+    BeamlineBusyTimeoutException, BeamlineBusyException, AutoRasterSampleSkipped
 )
 from aare.devices.tell_client import TellEventValueEnum

@@ -86,6 +92,11 @@ class AareDAQ:
    """

    MIN_SPOTS_LOW_RES_THRESHOLD = 10.0
+    AUTOMATION_BUSY_TIMEOUT_S = 600
+    MANUAL_RASTER_BUSY_TIMEOUT_S = 600
+    AUTO_RASTER_MAX_IMAGES = 4500
+    AUTO_RASTER_MIN_CELL_SIZE_MM = 0.005
+    AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD = True

    def __init__(self, cfg: BeamlineConfig, bl: MXBeamline):
        self.last_time = 0.0
@@ -610,7 +621,11 @@ class AareDAQ:
                    error=e,
                    event_type=SampleEventType.MOUNTFAILED
                )
-            return False
+
+            sample_name = self._sample_mount_display_name(sample)
+            if sample is None:
+                raise UnmountingFailed(f"Failed to unmount {sample_name}: {e}") from e
+            raise MountingFailed(f"Failed to mount {sample_name}: {e}") from e

    def _execute_loop_centering(self, sample: SampleShortInfo) -> bool:
        """
@@ -835,7 +850,32 @@ class AareDAQ:
                event_type=SampleEventType.RASTERINGFAILED,
                additional_comment=f"JFJoch communication error: {e}"
            )
-            return None
+            raise
+
+        except BeamlineBusyTimeoutException as e:
+            logger.exception(
+                "Raster sequence failed because the beamline busy state timed out",
+                extra=merge_log_context(
+                    sample_log_context(self.sample),
+                    raster_request_log_context(grid_request),
+                    {"auto_center": auto_center},
+                ),
+            )
+            self._handle_operation_error(
+                operation=DAQOperation.RASTER,
+                sample=self.sample,
+                error=e,
+                event_type=SampleEventType.RASTERINGFAILED,
+                additional_comment=(
+                    "Beamline busy timeout during raster sequence. "
+                    "The operation exceeded the configured busy-state TTL."
+                ),
+            )
+            raise
+
+        except AutoRasterSampleSkipped:
+            raise
+
        except Exception as e:
            logger.exception(
                "Raster sequence failed",
@@ -983,17 +1023,16 @@ class AareDAQ:

        return None

-    def _placeholder_sample_from_mounted_address(self, mounted_address) -> SampleShortInfo:
-        return SampleShortInfo(
-            db_id=-1,
-            puck_name="",
-            dewar_name="",
-            sample_name=f"Mounted sample {mounted_address.puck.segment}{mounted_address.puck.pos}-{mounted_address.pin}",
-            run_number=0,
-            user="",
-            pin=mounted_address.pin,
-            location=mounted_address.puck,
-        )
+    @staticmethod
+    def _sample_mount_display_name(sample: SampleShortInfo | None) -> str:
+        if sample is None:
+            return "current sample"
+
+        sample_name = str(getattr(sample, "sample_name", "") or "").strip()
+        if sample_name:
+            return sample_name
+
+        return "sample"

    def sync_current_sample_from_tell(
            self,
@@ -1306,26 +1345,30 @@ class AareDAQ:
            end_automation_after_fail_count: int = 5
    ):
        if mount_error:
-            count = self.__cfg.record_mount_failure()
-            if count > end_automation_after_fail_count:
-                logger.error(f"Mounting continued to fail after stopping automation, contact your local contact")
-                logger.debug(f"Clearing mount fail count")
-                self.__cfg.record_mount_success()
-                raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.")
-            elif count == end_automation_after_fail_count:
-                logger.error(f"Mounting failed {count} times, stopping automation")
-                raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.")
-            elif count == dry_after_fail_count:
-                logger.warning(f"Mount failed {count} times in a row; drying")
-                try:
-                    self.park_and_dry()
-                except Exception as e:
-                    logger.exception(f"Failed to dry after mount failure: {e}")
-            else:
-                return
-        else:
-            logger.debug("Mounting succeeded, resetting mount failure counter")
-            self.__cfg.record_mount_success()
+            logger.error("Mount failed")
+            logger.debug("Mount error: ", extra={"mount_error": mount_error})
+            logger.debug("Currently fail handler disabled due to bugs..?")
+        # if mount_error:
+        #     count = self.__cfg.record_mount_failure()
+        #     if count > end_automation_after_fail_count:
+        #         logger.error(f"Mounting continued to fail after stopping automation, contact your local contact")
+        #         logger.debug(f"Clearing mount fail count")
+        #         self.__cfg.record_mount_success()
+        #         raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.")
+        #     elif count == end_automation_after_fail_count:
+        #         logger.error(f"Mounting failed {count} times, stopping automation")
+        #         raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.")
+        #     elif count == dry_after_fail_count:
+        #         logger.warning(f"Mount failed {count} times in a row; drying")
+        #         try:
+        #             self.park_and_dry()
+        #         except Exception as e:
+        #             logger.exception(f"Failed to dry after mount failure: {e}")
+        #     else:
+        #         return
+        # else:
+        #     logger.debug("Mounting succeeded, resetting mount failure counter")
+        #     self.__cfg.record_mount_success()

    def __mount_handler(self, target : SampleShortInfo):

@@ -1414,9 +1457,10 @@ class AareDAQ:

            if not self._execute_mount_and_prepare(target):
                current_sample = target or self.__cfg.current_sample
+                sample_name = self._sample_mount_display_name(current_sample)
                if target is None:
-                    raise UnmountingFailed(f"Failed to {type} sample {current_sample}")
-                raise MountingFailed(f"Failed to {type} sample {current_sample}")
+                    raise UnmountingFailed(f"Failed to {type.lower()} {sample_name}")
+                raise MountingFailed(f"Failed to {type.lower()} {sample_name}")

            logger.info(f"Sample operation completed: {target}")
            self.__cfg.state_busy = False
@@ -2259,6 +2303,57 @@ class AareDAQ:
    def get_beam_mark(self):
        return self.__cfg.get_beam_mark(self.__devs.zoom)

+    @classmethod
+    def _scale_auto_raster_grid(
+            cls,
+            *,
+            n_x: int,
+            n_y: int,
+            grid_size: Coordinate,
+            skip: bool = False,
+    ) -> tuple[int, int, Coordinate]:
+        """
+        Limit auto-raster image count while preserving the physical raster footprint.
+
+        If the ML box would generate too many images, reduce the number of cells and
+        increase the cell size proportionally. The total raster width/height stays
+        the same, but the scan is sampled more coarsely.
+        """
+        n_x = max(1, int(n_x))
+        n_y = max(1, int(n_y))
+
+        image_count = n_x * n_y
+        if image_count <= cls.AUTO_RASTER_MAX_IMAGES:
+            return n_x, n_y, grid_size
+
+        if skip:
+            raise AutoRasterSampleSkipped(
+                    f"Auto-raster grid has {image_count} images, which exceeds "
+                    f"the automation limit of {cls.AUTO_RASTER_MAX_IMAGES}; skipping sample"
+                )
+
+        physical_size_x_mm = n_x * grid_size.x
+        physical_size_y_mm = n_y * grid_size.y
+
+        scale = (image_count / cls.AUTO_RASTER_MAX_IMAGES) ** 0.5
+        scaled_n_x = max(1, int(floor(n_x / scale)))
+        scaled_n_y = max(1, int(floor(n_y / scale)))
+
+        while scaled_n_x * scaled_n_y > cls.AUTO_RASTER_MAX_IMAGES:
+            if scaled_n_x >= scaled_n_y and scaled_n_x > 1:
+                scaled_n_x -= 1
+            elif scaled_n_y > 1:
+                scaled_n_y -= 1
+            else:
+                break
+
+        scaled_grid_size = Coordinate(
+            x=max(cls.AUTO_RASTER_MIN_CELL_SIZE_MM, physical_size_x_mm / scaled_n_x),
+            y=max(cls.AUTO_RASTER_MIN_CELL_SIZE_MM, physical_size_y_mm / scaled_n_y),
+        )
+
+        return scaled_n_x, scaled_n_y, scaled_grid_size
+
    def __ml_bounding_box(self, sample_id: int | None = None, filename: str | None = None) -> RasterGridRequest | None:
        time.sleep(0.2)  # Just to be sure image is stable
        prediction_result: MLBoxPredictionResult = self.__mlbox.predict(
@@ -2319,6 +2414,45 @@ class AareDAQ:
        n_x = max(1, abs(ceil((x2 - x1) * geom.pixel_in_mm / grid_size.x)))
        n_y = max(1, abs(ceil((y2 - y1) * geom.pixel_in_mm / grid_size.y)))

+        original_n_x = n_x
+        original_n_y = n_y
+        original_grid_size = grid_size
+        #TODO should we have seperate handling for manual raster?
+        n_x, n_y, grid_size = self._scale_auto_raster_grid(
+            n_x=n_x,
+            n_y=n_y,
+            grid_size=grid_size,
+            skip = self.AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD
+        )
+
+        if (n_x, n_y, grid_size.x, grid_size.y) != (
+                original_n_x,
+                original_n_y,
+                original_grid_size.x,
+                original_grid_size.y,
+        ):
+            logger.info(
+                "Scaled ML raster grid to stay within auto-raster image limit",
+                extra=merge_log_context(
+                    sample_log_context(self.sample),
+                    {
+                        "sample_id": sample_id,
+                        "ml_image_name": filename,
+                        "max_images": self.AUTO_RASTER_MAX_IMAGES,
+                        "original_n_x": original_n_x,
+                        "original_n_y": original_n_y,
+                        "original_image_count": original_n_x * original_n_y,
+                        "original_grid_size_x_mm": original_grid_size.x,
+                        "original_grid_size_y_mm": original_grid_size.y,
+                        "scaled_n_x": n_x,
+                        "scaled_n_y": n_y,
+                        "scaled_image_count": n_x * n_y,
+                        "scaled_grid_size_x_mm": grid_size.x,
+                        "scaled_grid_size_y_mm": grid_size.y,
+                    },
+                ),
+            )
+
        logger.info(
            "Converted ML bounding box to raster request",
            extra=merge_log_context(
@@ -2779,7 +2913,8 @@ class AareDAQ:
                else:
                    logger.warning(
                        "Skipping recovery transition to RobotSampleExchange: "
-                        "beamline is no longer busy (busy was released earlier)."
+                        "beamline is no longer busy. The busy key may have expired"
+                        "befor recovery could run"
                    )
            except Exception:
                logger.exception(
@@ -2822,7 +2957,7 @@ class AareDAQ:
        )

        try:
-            self.__cfg.try_set_busy(timeout=360)
+            self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) #upped tiemout for large grids

            self._set_progress_context(
                progress,
@@ -2880,7 +3015,33 @@ class AareDAQ:
                transmission=raster_params.transmission,
            )
            self.__cfg.simulated_detector = True
-            raster_result = self._execute_raster_sequence(raster_grid, auto_center=True)
+
+            try:
+                raster_result = self._execute_raster_sequence(raster_grid, auto_center=True)
+            except AutoRasterSampleSkipped as e:
+                logger.warning(
+                    "Skipping sample during automation because auto-raster grid is too large",
+                    extra=merge_log_context(
+                        sample_log_context(sample),
+                        raster_request_log_context(raster_grid),
+                        {"reason": str(e)},
+                    ),
+                )
+                self._set_progress_step(
+                    progress,
+                    WorkflowStateKind.RASTER,
+                    StepStatus.SKIPPED,
+                    str(e),
+                )
+                self._set_progress_step(
+                    progress,
+                    WorkflowStateKind.DATA_COLLECTION,
+                    StepStatus.SKIPPED,
+                    "Data collection skipped because auto-raster was too large",
+                )
+                self._mark_progress_finished(progress, True, "Sample skipped: auto-raster too large")
+                return self._end_operation(start, DAQOperation.AUTOMATION, error=False)
+
            if raster_result is None:
                logger.error(
                    "Raster result was None during automation",
@@ -2926,6 +3087,40 @@ class AareDAQ:
        except JFJochCommunicationError as e:
            self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown")
            raise
+
+        except (BeamlineBusyTimeoutException,BeamlineBusyException) as e:
+
+            time_of_measure = abs(time.perf_counter()-start)
+
+            if  time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S:
+                logger.error(f"Error in measure due to Beamline Busy State timeout:"
+                             f"Time of: {time_of_measure} is greater than timeout duration {self.AUTOMATION_BUSY_TIMEOUT_S}"
+                             f"Error thrown: {e}")
+
+            else:
+                logger.error(f"Error in measure due to Beamline Busy State: {e}")
+
+            if progress.current_step is not None:
+                current_kind = next(
+                    (
+                        item.step
+                        for item in progress.steps
+                        if self._step_display_name(item.step) == progress.current_step
+                    ),
+                    WorkflowStateKind.FINAL,
+                )
+                self._mark_progress_failed(progress, current_kind, str(e))
+            else:
+                self._mark_progress_finished(progress, False, str(e))
+
+            self._handle_operation_error(
+                operation=DAQOperation.AUTOMATION,
+                sample=sample,
+                error=e,
+                event_type=SampleEventType.FAILED)
+
+            raise
+
        except Exception as e:
            logger.error(f"Error in measure: {e}")
            if progress.current_step is not None: