diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index 9ce793d2..08d6c8e2 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -70,6 +70,7 @@ from aare.common.exception_handler import ( AXCFailed, SmargonCommunicationError, TellCommunicationError, + BECCommunicationError, JFJochCommunicationError, AerotechCommunicationError, MagnetPositionSensorErorr, @@ -77,7 +78,9 @@ from aare.common.exception_handler import ( DataCollectionException, RasterScanException, TellMountFailedException, - BeamlineBusyTimeoutException, BeamlineBusyException, AutoRasterSampleSkipped + BeamlineBusyTimeoutException, + BeamlineBusyException, + AutoRasterSampleSkipped ) from aare.devices.tell_client import TellEventValueEnum @@ -117,6 +120,7 @@ class AareDAQ: self._automation_total_sample_time_s = 0.0 self._automation_last_sample_name = "" self._automation_samples_in_queue = 0 + self._last_mount_error_message = "" self.__reset_mount_failure_counter("DAQ startup") def _is_hardware_failure(self, error: Exception) -> bool: @@ -125,6 +129,7 @@ class AareDAQ: ( CriticalTellException, TellCommunicationError, + BECCommunicationError, SmargonCommunicationError, AerotechCommunicationError, JFJochCommunicationError, @@ -170,6 +175,23 @@ class AareDAQ: logger.critical(message) raise CriticalTellException(message) from error + def _raise_if_critical_bec_error( + self, + error: Exception, + *, + command: str, + ) -> None: + if not isinstance(error, BECCommunicationError): + return + + message = ( + f"Critical BEC error while running '{command}'. " + f"Automation has been stopped because the beamline state may be inconsistent. " + f"Original error: {error}" + ) + logger.critical(message) + raise CriticalTellException(message) from error + def _run_noncritical( self, action: Callable[[], object], @@ -649,6 +671,7 @@ class AareDAQ: """ previous_sample = None mount_started_at = datetime.now(timezone.utc) + self._last_mount_error_message = "" try: previous_sample = self.sample @@ -676,6 +699,7 @@ class AareDAQ: self.__set_state(BeamlineStateEnum.SampleAlignment) return True except TransformationInvalidException as e: + self._last_mount_error_message = str(e) or "Mount failed" logger.error(f"Mount failed due to invalid transformation: {e}") self._handle_operation_error( operation=DAQOperation.MOUNT, @@ -685,6 +709,7 @@ class AareDAQ: ) raise except Exception as e: + self._last_mount_error_message = str(e) or "Mount failed" logger.error(f"Mount failed: {e}") previous_sample_unmounted = ( @@ -981,7 +1006,8 @@ class AareDAQ: operation=DAQOperation.RASTER, sample=self.sample, error=e, - event_type=SampleEventType.RASTERINGFAILED + event_type=SampleEventType.RASTERINGFAILED, + additional_comment=f"Raster sequence failed: {e}" ) return None @@ -1889,7 +1915,19 @@ class AareDAQ: # self.__devs.transmission.wait() return - def _build_fake_scan_result(self, *, file_prefix: str | None, image_count: int) -> ScanResult: + def _build_fake_scan_result(self, *, file_prefix: str | None, image_count: int, + rotation: bool = False) -> ScanResult: + total_images = max(1, image_count) + + start_angle = 0.0 + if rotation: + try: + start_angle = float(self.omega) + except Exception: + start_angle = 0.0 + + angle_step = 360.0 / total_images if rotation else 0.0 + images = [ ScanResultImagesInner( number=i, @@ -1900,8 +1938,9 @@ class AareDAQ: spots_indexed=0, index=0, b=0.0, + angle=start_angle + i * angle_step if rotation else None, ) - for i in range(max(1, image_count)) + for i in range(total_images) ] return ScanResult(file_prefix=file_prefix, images=images) @@ -1909,6 +1948,7 @@ class AareDAQ: result = self._build_fake_scan_result( file_prefix=request.file_prefix, image_count=request.steps, + rotation=True ) return CompletedRotationScan( request=copy.deepcopy(request), @@ -2002,6 +2042,11 @@ class AareDAQ: def __raster(self, request: RasterGridRequest, wait_for_screenshot: float | None = 0.3) -> CompletedRasterGridElem: + total_time = request.exp_time_s * request.n_x * request.n_y + request.n_y * 0.3 + + if total_time > 1200: + raise RasterScanException(f"Raster scan is too long {total_time}s > 20min") + status = self.status smargon_top_left =request.smargon_top_left @@ -2022,8 +2067,6 @@ class AareDAQ: ), ) - total_time = request.exp_time_s * request.n_x * request.n_y + request.n_y * 0.3 - try: if self.sample is not None and self.sample.db_id is not None: self.__aare.create_gridscan_run(self.sample, request, status) @@ -2036,7 +2079,7 @@ class AareDAQ: raise else: logger.info("Simulated detector mode enabled; faking jfjoch intilalisation.") - + logger.debug(f"Starting grid scan with {request.n_x}x{request.n_y} points, exp time {request.exp_time_s}s") self.__devs.aerotech.grid_scan( grid_elem_size_y_um=request.grid_size_mm.y * 1000, grid_elem_size_x_um=request.grid_size_mm.x * 1000, @@ -2045,14 +2088,15 @@ class AareDAQ: time_sec=request.exp_time_s, run_async=True, ) - self.__devs.aerotech.wait_till_done(timeout=int(round(total_time * 2, 0))) + + self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + total_time * 0.1 + 60, 0))) # go back to aerotech x,y,z home not U home (0 degrees). if isinstance(self.__cfg.abr_meas_pos, Coordinate): coord = self.__cfg.abr_meas_pos else: coord = self.__cfg.abr_meas_pos.at_mm self.__devs.aerotech_pos = AerotechCoordinate(at_mm=coord, omega_deg=self.__devs.aerotech_omega) - self.__devs.aerotech.wait_till_done(timeout=int(360)) + self.__devs.aerotech.wait_till_done(timeout=int(60)) if self.__cfg.simulated_detector: logger.info("Simulated detector mode enabled; using fake raster result.") @@ -2165,7 +2209,7 @@ class AareDAQ: y_mm=y, request=request, ) - + self._upload_raster_diffraction_preview( sample_id=self.sample.db_id, filename=diffraction_image_filename, @@ -2228,9 +2272,7 @@ class AareDAQ: ) self.__cfg.try_set_busy(timeout=ceil(360)) try: - self.__cfg.simulated_detector = True result = self._execute_raster_sequence(request, auto_center=auto_center) - self.__cfg.simulated_detector = False if result is None: raise RasterScanException("Raster scan failed") @@ -2252,12 +2294,14 @@ class AareDAQ: total_time = request.exp_time_s * request.steps if self.sample is not None and self.sample.db_id is not None: self.__aare.create_rotation_run(self.sample, request, status) - try: + + if not self.__cfg.simulated_detector: try: self.__jfjoch.wait_till_running(timeout=60.0) except Exception as e: self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running") raise + try: if request.screening: self.__devs.aerotech.screening_scan( @@ -2995,7 +3039,7 @@ class AareDAQ: params.exp_time_s = 0.08 else: params.dtz = 150 - params.exp_time_s = 0.04 + params.exp_time_s = 0.01 return params @@ -3086,9 +3130,9 @@ class AareDAQ: sample.pin, sample.sample_name ) - + self.__cfg.simulated_detector = True try: - self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) #upped tiemout for large grids + self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) # upped tiemout for large grids self._set_progress_context( progress, @@ -3097,7 +3141,8 @@ class AareDAQ: self._mark_progress_running(progress, WorkflowStateKind.MOUNT, "Mounting sample") if not self._execute_mount_and_prepare(sample): - self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, "Mount failed") + mount_error_message = self._last_mount_error_message or "Mount failed" + self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, mount_error_message) return self._end_operation(start, DAQOperation.MOUNT, error=True) self._mark_progress_success(progress, WorkflowStateKind.MOUNT, "Mount complete") self.__set_state(BeamlineStateEnum.SampleAlignment) @@ -3205,7 +3250,7 @@ class AareDAQ: steps=params.steps, transmission=params.transmission, ) - self.__cfg.simulated_detector = False + rotation_result = self._execute_rotation_sequence(rotation_request) if rotation_result is None: logger.error("Rotation result was None") @@ -3215,15 +3260,18 @@ class AareDAQ: self._mark_progress_success(progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete") logger.info(f"Rotation scan done at {time.perf_counter() - start}") + except BECCommunicationError as e: + self._raise_if_critical_bec_error(e, command=getattr(e, "operation", None) or "bec") + except JFJochCommunicationError as e: self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown") raise - except (BeamlineBusyTimeoutException,BeamlineBusyException) as e: + except (BeamlineBusyTimeoutException, BeamlineBusyException) as e: - time_of_measure = abs(time.perf_counter()-start) + time_of_measure = abs(time.perf_counter() - start) - if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S: + if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S: logger.error(f"Error in measure due to Beamline Busy State timeout:" f"Time of: {time_of_measure} is greater than timeout duration {self.AUTOMATION_BUSY_TIMEOUT_S}" f"Error thrown: {e}") @@ -3279,6 +3327,7 @@ class AareDAQ: self._mark_progress_finished(progress, True, "Automation complete") return self._end_operation(start, DAQOperation.AUTOMATION, error=False) + @log_timing(logger, "Changing Beamline State") def __set_state(self, target: BeamlineStateEnum): """__set_state assumes that beamline is already in busy state it will apply a proper transformation and change state afterward @@ -3293,14 +3342,22 @@ class AareDAQ: curr_state = self.__cfg.state + if hasattr(curr_state, "value") and hasattr(target, "value"): + logger.info( + f"State transition requested: {curr_state} -> {target}", + extra={"from_state": curr_state, "to_state": target}, + ) + if not self.__cfg.state_busy: raise Exception("Beamline should be busy") if target == BeamlineStateEnum.Maintenance: self.__cfg.state = BeamlineStateEnum.Maintenance elif target == curr_state: - logger.debug(f"State already set to {target}") - return + logger.debug( + f"State already set to {target}", + extra={"from_state": curr_state, "to_state": target}, + ) elif target != curr_state: self.__cfg.state = BeamlineStateEnum.Moving try: @@ -3320,6 +3377,10 @@ class AareDAQ: elif target == BeamlineStateEnum.BeamLocation: workflows.se2sa(self.__devs, self.__cfg) workflows.sa2bl(self.__devs, self.__cfg) + elif target == BeamlineStateEnum.DewarTransfer: + workflows.common2dh(self.__devs, self.__cfg) + elif target == BeamlineStateEnum.RobotSampleExchange: + workflows.common_2rse(self.__devs, self.__cfg) else: raise TransformationInvalidException( f"Cannot go from {curr_state} to {target}, not implemented" @@ -3387,6 +3448,10 @@ class AareDAQ: case BeamlineStateEnum.RobotSampleExchange: if target == BeamlineStateEnum.SampleAlignment: workflows.rse2sa(self.__devs, self.__cfg) + elif target == BeamlineStateEnum.SampleExchange: + workflows.rse2se(self.__devs, self.__cfg) + elif target == BeamlineStateEnum.DewarTransfer: + workflows.common2dh(self.__devs, self.__cfg) else: raise TransformationInvalidException( f"Cannot go from {curr_state} to {target}, not implemented" @@ -3411,11 +3476,23 @@ class AareDAQ: f"Cannot go from {curr_state} to {target}, not implemented" ) self.__cfg.state = target + logger.info( + f"State transition completed: {curr_state} -> {target}", + extra={"from_state": curr_state, "to_state": target}, + ) except TransformationInvalidException as e: - logger.error(f"Cannot go from {curr_state} to {target}: {e}") + logger.error( + f"State transition invali: {curr_state} -> {target}: {e}", + extra={"from_state": curr_state, "to_state": target}, + ) self.__cfg.state = curr_state raise except Exception as e: + logger.exception( + f"Exception during state transition : {curr_state} -> {target}." + f"Changing state to maintenance due to error: {e}", + extra={"from_state": curr_state, "to_state": target}, + ) self.__cfg.state = BeamlineStateEnum.Maintenance #TODO check if this is the right thing to do self.__cfg.state_busy = False @@ -3431,18 +3508,34 @@ class AareDAQ: @property def diffraction_geometry(self) -> DiffractionGeometry: - det_cfg = self.__jfjoch.detector() - return DiffractionGeometry( - energy_keV=self.__devs.energy_kev, - dtz_mm=self.__devs.dtz, - detector_size_pxl=(det_cfg.width, det_cfg.height), - pixel_size_mm=det_cfg.pixel_size_mm, - beam_center_pxl=self.__cfg.beam_center, - detector_description=det_cfg.description, - detector_serial_number=det_cfg.serial_number, - poni_rot1_rad=-0.001396263, - poni_rot2_rad=-0.003839724, - ) + try: + det_cfg = self.__jfjoch.detector() + return DiffractionGeometry( + energy_keV=self.__devs.energy_kev, + dtz_mm=self.__devs.dtz, + detector_size_pxl=(det_cfg.width, det_cfg.height), + pixel_size_mm=det_cfg.pixel_size_mm, + beam_center_pxl=self.__cfg.beam_center, + detector_description=det_cfg.description, + detector_serial_number=det_cfg.serial_number, + poni_rot1_rad=-0.001396263, + poni_rot2_rad=-0.003839724, + ) + except JFJochCommunicationError as e: + logger.warning( + f"Falling back to default diffraction geometry because detector metadata is unavailable: {e}" + ) + return DiffractionGeometry( + energy_keV=self.__devs.energy_kev, + dtz_mm=self.__devs.dtz, + detector_size_pxl=(1, 1), + pixel_size_mm=0.15, + beam_center_pxl=self.__cfg.beam_center, + detector_description="unavailable", + detector_serial_number="unavailable", + poni_rot1_rad=-0.001396263, + poni_rot2_rad=-0.003839724, + ) @property def beamline_status(self) -> BeamlineStatus: