From c4c82149576cabacb4f3c7151c3b9b3cddd295eb Mon Sep 17 00:00:00 2001 From: appleb_m Date: Fri, 8 May 2026 13:13:36 +0200 Subject: [PATCH] DAQ: for large grids we were hitting a busy tiemout exception - have uppded the tiemout to 10 minutes, have added busy state timeout exceptions and now skip samples if raster too large - or when starting the DAQ can toggle this off so that we scale the grid instead to the max number of images - set to jungfrau buffer limit of 4500 iamges, needs testing. --- src/aare/daq/daq.py | 275 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 235 insertions(+), 40 deletions(-) diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index 3a32d70d..437dc8df 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -69,8 +69,14 @@ from aare.common.exception_handler import ( AXCFailed, SmargonCommunicationError, TellCommunicationError, - JFJochCommunicationError, AerotechCommunicationError, MagnetPositionSensorErorr, UnmountingFailed, - DataCollectionException, RasterScanException, TellMountFailedException + JFJochCommunicationError, + AerotechCommunicationError, + MagnetPositionSensorErorr, + UnmountingFailed, + DataCollectionException, + RasterScanException, + TellMountFailedException, + BeamlineBusyTimeoutException, BeamlineBusyException, AutoRasterSampleSkipped ) from aare.devices.tell_client import TellEventValueEnum @@ -86,6 +92,11 @@ class AareDAQ: """ MIN_SPOTS_LOW_RES_THRESHOLD = 10.0 + AUTOMATION_BUSY_TIMEOUT_S = 600 + MANUAL_RASTER_BUSY_TIMEOUT_S = 600 + AUTO_RASTER_MAX_IMAGES = 4500 + AUTO_RASTER_MIN_CELL_SIZE_MM = 0.005 + AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD = True def __init__(self, cfg: BeamlineConfig, bl: MXBeamline): self.last_time = 0.0 @@ -610,7 +621,11 @@ class AareDAQ: error=e, event_type=SampleEventType.MOUNTFAILED ) - return False + + sample_name = self._sample_mount_display_name(sample) + if sample is None: + raise UnmountingFailed(f"Failed to unmount {sample_name}: {e}") from e + raise MountingFailed(f"Failed to mount {sample_name}: {e}") from e def _execute_loop_centering(self, sample: SampleShortInfo) -> bool: """ @@ -835,7 +850,32 @@ class AareDAQ: event_type=SampleEventType.RASTERINGFAILED, additional_comment=f"JFJoch communication error: {e}" ) - return None + raise + + except BeamlineBusyTimeoutException as e: + logger.exception( + "Raster sequence failed because the beamline busy state timed out", + extra=merge_log_context( + sample_log_context(self.sample), + raster_request_log_context(grid_request), + {"auto_center": auto_center}, + ), + ) + self._handle_operation_error( + operation=DAQOperation.RASTER, + sample=self.sample, + error=e, + event_type=SampleEventType.RASTERINGFAILED, + additional_comment=( + "Beamline busy timeout during raster sequence. " + "The operation exceeded the configured busy-state TTL." + ), + ) + raise + + except AutoRasterSampleSkipped: + raise + except Exception as e: logger.exception( "Raster sequence failed", @@ -983,17 +1023,16 @@ class AareDAQ: return None - def _placeholder_sample_from_mounted_address(self, mounted_address) -> SampleShortInfo: - return SampleShortInfo( - db_id=-1, - puck_name="", - dewar_name="", - sample_name=f"Mounted sample {mounted_address.puck.segment}{mounted_address.puck.pos}-{mounted_address.pin}", - run_number=0, - user="", - pin=mounted_address.pin, - location=mounted_address.puck, - ) + @staticmethod + def _sample_mount_display_name(sample: SampleShortInfo | None) -> str: + if sample is None: + return "current sample" + + sample_name = str(getattr(sample, "sample_name", "") or "").strip() + if sample_name: + return sample_name + + return "sample" def sync_current_sample_from_tell( self, @@ -1306,26 +1345,30 @@ class AareDAQ: end_automation_after_fail_count: int = 5 ): if mount_error: - count = self.__cfg.record_mount_failure() - if count > end_automation_after_fail_count: - logger.error(f"Mounting continued to fail after stopping automation, contact your local contact") - logger.debug(f"Clearing mount fail count") - self.__cfg.record_mount_success() - raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.") - elif count == end_automation_after_fail_count: - logger.error(f"Mounting failed {count} times, stopping automation") - raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.") - elif count == dry_after_fail_count: - logger.warning(f"Mount failed {count} times in a row; drying") - try: - self.park_and_dry() - except Exception as e: - logger.exception(f"Failed to dry after mount failure: {e}") - else: - return - else: - logger.debug("Mounting succeeded, resetting mount failure counter") - self.__cfg.record_mount_success() + logger.error("Mount failed") + logger.debug("Mount error: ", extra={"mount_error": mount_error}) + logger.debug("Currently fail handler disabled due to bugs..?") + # if mount_error: + # count = self.__cfg.record_mount_failure() + # if count > end_automation_after_fail_count: + # logger.error(f"Mounting continued to fail after stopping automation, contact your local contact") + # logger.debug(f"Clearing mount fail count") + # self.__cfg.record_mount_success() + # raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.") + # elif count == end_automation_after_fail_count: + # logger.error(f"Mounting failed {count} times, stopping automation") + # raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.") + # elif count == dry_after_fail_count: + # logger.warning(f"Mount failed {count} times in a row; drying") + # try: + # self.park_and_dry() + # except Exception as e: + # logger.exception(f"Failed to dry after mount failure: {e}") + # else: + # return + # else: + # logger.debug("Mounting succeeded, resetting mount failure counter") + # self.__cfg.record_mount_success() def __mount_handler(self, target : SampleShortInfo): @@ -1414,9 +1457,10 @@ class AareDAQ: if not self._execute_mount_and_prepare(target): current_sample = target or self.__cfg.current_sample + sample_name = self._sample_mount_display_name(current_sample) if target is None: - raise UnmountingFailed(f"Failed to {type} sample {current_sample}") - raise MountingFailed(f"Failed to {type} sample {current_sample}") + raise UnmountingFailed(f"Failed to {type.lower()} {sample_name}") + raise MountingFailed(f"Failed to {type.lower()} {sample_name}") logger.info(f"Sample operation completed: {target}") self.__cfg.state_busy = False @@ -2259,6 +2303,57 @@ class AareDAQ: def get_beam_mark(self): return self.__cfg.get_beam_mark(self.__devs.zoom) + @classmethod + def _scale_auto_raster_grid( + cls, + *, + n_x: int, + n_y: int, + grid_size: Coordinate, + skip: bool = False, + ) -> tuple[int, int, Coordinate]: + """ + Limit auto-raster image count while preserving the physical raster footprint. + + If the ML box would generate too many images, reduce the number of cells and + increase the cell size proportionally. The total raster width/height stays + the same, but the scan is sampled more coarsely. + """ + n_x = max(1, int(n_x)) + n_y = max(1, int(n_y)) + + image_count = n_x * n_y + if image_count <= cls.AUTO_RASTER_MAX_IMAGES: + return n_x, n_y, grid_size + + if skip: + raise AutoRasterSampleSkipped( + f"Auto-raster grid has {image_count} images, which exceeds " + f"the automation limit of {cls.AUTO_RASTER_MAX_IMAGES}; skipping sample" + ) + + physical_size_x_mm = n_x * grid_size.x + physical_size_y_mm = n_y * grid_size.y + + scale = (image_count / cls.AUTO_RASTER_MAX_IMAGES) ** 0.5 + scaled_n_x = max(1, int(floor(n_x / scale))) + scaled_n_y = max(1, int(floor(n_y / scale))) + + while scaled_n_x * scaled_n_y > cls.AUTO_RASTER_MAX_IMAGES: + if scaled_n_x >= scaled_n_y and scaled_n_x > 1: + scaled_n_x -= 1 + elif scaled_n_y > 1: + scaled_n_y -= 1 + else: + break + + scaled_grid_size = Coordinate( + x=max(cls.AUTO_RASTER_MIN_CELL_SIZE_MM, physical_size_x_mm / scaled_n_x), + y=max(cls.AUTO_RASTER_MIN_CELL_SIZE_MM, physical_size_y_mm / scaled_n_y), + ) + + return scaled_n_x, scaled_n_y, scaled_grid_size + def __ml_bounding_box(self, sample_id: int | None = None, filename: str | None = None) -> RasterGridRequest | None: time.sleep(0.2) # Just to be sure image is stable prediction_result: MLBoxPredictionResult = self.__mlbox.predict( @@ -2319,6 +2414,45 @@ class AareDAQ: n_x = max(1, abs(ceil((x2 - x1) * geom.pixel_in_mm / grid_size.x))) n_y = max(1, abs(ceil((y2 - y1) * geom.pixel_in_mm / grid_size.y))) + original_n_x = n_x + original_n_y = n_y + original_grid_size = grid_size + #TODO should we have seperate handling for manual raster? + n_x, n_y, grid_size = self._scale_auto_raster_grid( + n_x=n_x, + n_y=n_y, + grid_size=grid_size, + skip = self.AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD + ) + + if (n_x, n_y, grid_size.x, grid_size.y) != ( + original_n_x, + original_n_y, + original_grid_size.x, + original_grid_size.y, + ): + logger.info( + "Scaled ML raster grid to stay within auto-raster image limit", + extra=merge_log_context( + sample_log_context(self.sample), + { + "sample_id": sample_id, + "ml_image_name": filename, + "max_images": self.AUTO_RASTER_MAX_IMAGES, + "original_n_x": original_n_x, + "original_n_y": original_n_y, + "original_image_count": original_n_x * original_n_y, + "original_grid_size_x_mm": original_grid_size.x, + "original_grid_size_y_mm": original_grid_size.y, + "scaled_n_x": n_x, + "scaled_n_y": n_y, + "scaled_image_count": n_x * n_y, + "scaled_grid_size_x_mm": grid_size.x, + "scaled_grid_size_y_mm": grid_size.y, + }, + ), + ) + logger.info( "Converted ML bounding box to raster request", extra=merge_log_context( @@ -2779,7 +2913,8 @@ class AareDAQ: else: logger.warning( "Skipping recovery transition to RobotSampleExchange: " - "beamline is no longer busy (busy was released earlier)." + "beamline is no longer busy. The busy key may have expired" + "befor recovery could run" ) except Exception: logger.exception( @@ -2822,7 +2957,7 @@ class AareDAQ: ) try: - self.__cfg.try_set_busy(timeout=360) + self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) #upped tiemout for large grids self._set_progress_context( progress, @@ -2880,7 +3015,33 @@ class AareDAQ: transmission=raster_params.transmission, ) self.__cfg.simulated_detector = True - raster_result = self._execute_raster_sequence(raster_grid, auto_center=True) + + try: + raster_result = self._execute_raster_sequence(raster_grid, auto_center=True) + except AutoRasterSampleSkipped as e: + logger.warning( + "Skipping sample during automation because auto-raster grid is too large", + extra=merge_log_context( + sample_log_context(sample), + raster_request_log_context(raster_grid), + {"reason": str(e)}, + ), + ) + self._set_progress_step( + progress, + WorkflowStateKind.RASTER, + StepStatus.SKIPPED, + str(e), + ) + self._set_progress_step( + progress, + WorkflowStateKind.DATA_COLLECTION, + StepStatus.SKIPPED, + "Data collection skipped because auto-raster was too large", + ) + self._mark_progress_finished(progress, True, "Sample skipped: auto-raster too large") + return self._end_operation(start, DAQOperation.AUTOMATION, error=False) + if raster_result is None: logger.error( "Raster result was None during automation", @@ -2926,6 +3087,40 @@ class AareDAQ: except JFJochCommunicationError as e: self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown") raise + + except (BeamlineBusyTimeoutException,BeamlineBusyException) as e: + + time_of_measure = abs(time.perf_counter()-start) + + if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S: + logger.error(f"Error in measure due to Beamline Busy State timeout:" + f"Time of: {time_of_measure} is greater than timeout duration {self.AUTOMATION_BUSY_TIMEOUT_S}" + f"Error thrown: {e}") + + else: + logger.error(f"Error in measure due to Beamline Busy State: {e}") + + if progress.current_step is not None: + current_kind = next( + ( + item.step + for item in progress.steps + if self._step_display_name(item.step) == progress.current_step + ), + WorkflowStateKind.FINAL, + ) + self._mark_progress_failed(progress, current_kind, str(e)) + else: + self._mark_progress_finished(progress, False, str(e)) + + self._handle_operation_error( + operation=DAQOperation.AUTOMATION, + sample=sample, + error=e, + event_type=SampleEventType.FAILED) + + raise + except Exception as e: logger.error(f"Error in measure: {e}") if progress.current_step is not None: