mx/AareDAQ
0
1
Fork 0
Code Issues 52 Pull Requests 1 Actions Packages Projects Releases 75 Wiki Activity

DAQ: added StateTransitionFailed and Maintenance Exceptions. ALso BECCommunicationException

Browse Source
This commit is contained in:
appleb_m
2026-05-20 11:46:34 +02:00
parent c60d2b5303
commit 3fa4a57532
1 changed files with 129 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aare/daq/daq.py
+129 -36
View File
@@ -70,6 +70,7 @@ from aare.common.exception_handler import (
AXCFailed,
SmargonCommunicationError,
TellCommunicationError,
BECCommunicationError,
JFJochCommunicationError,
AerotechCommunicationError,
MagnetPositionSensorErorr,
@@ -77,7 +78,9 @@ from aare.common.exception_handler import (
DataCollectionException,
RasterScanException,
TellMountFailedException,
BeamlineBusyTimeoutException, BeamlineBusyException, AutoRasterSampleSkipped
BeamlineBusyTimeoutException,
BeamlineBusyException,
AutoRasterSampleSkipped
)
from aare.devices.tell_client import TellEventValueEnum
@@ -117,6 +120,7 @@ class AareDAQ:
self._automation_total_sample_time_s = 0.0
self._automation_last_sample_name = ""
self._automation_samples_in_queue = 0
self._last_mount_error_message = ""
self.__reset_mount_failure_counter("DAQ startup")
def _is_hardware_failure(self, error: Exception) -> bool:
@@ -125,6 +129,7 @@ class AareDAQ:
(
CriticalTellException,
TellCommunicationError,
BECCommunicationError,
SmargonCommunicationError,
AerotechCommunicationError,
JFJochCommunicationError,
@@ -170,6 +175,23 @@ class AareDAQ:
logger.critical(message)
raise CriticalTellException(message) from error
def _raise_if_critical_bec_error(
self,
error: Exception,
*,
command: str,
) -> None:
if not isinstance(error, BECCommunicationError):
return
message = (
f"Critical BEC error while running '{command}'. "
f"Automation has been stopped because the beamline state may be inconsistent. "
f"Original error: {error}"
)
logger.critical(message)
raise CriticalTellException(message) from error
def _run_noncritical(
self,
action: Callable[[], object],
@@ -649,6 +671,7 @@ class AareDAQ:
"""
previous_sample = None
mount_started_at = datetime.now(timezone.utc)
self._last_mount_error_message = ""
try:
previous_sample = self.sample
@@ -676,6 +699,7 @@ class AareDAQ:
self.__set_state(BeamlineStateEnum.SampleAlignment)
return True
except TransformationInvalidException as e:
self._last_mount_error_message = str(e) or "Mount failed"
logger.error(f"Mount failed due to invalid transformation: {e}")
self._handle_operation_error(
operation=DAQOperation.MOUNT,
@@ -685,6 +709,7 @@ class AareDAQ:
)
raise
except Exception as e:
self._last_mount_error_message = str(e) or "Mount failed"
logger.error(f"Mount failed: {e}")
previous_sample_unmounted = (
@@ -981,7 +1006,8 @@ class AareDAQ:
operation=DAQOperation.RASTER,
sample=self.sample,
error=e,
event_type=SampleEventType.RASTERINGFAILED
event_type=SampleEventType.RASTERINGFAILED,
additional_comment=f"Raster sequence failed: {e}"
)
return None
@@ -1889,7 +1915,19 @@ class AareDAQ:
# self.__devs.transmission.wait()
return
def _build_fake_scan_result(self, *, file_prefix: str | None, image_count: int) -> ScanResult:
def _build_fake_scan_result(self, *, file_prefix: str | None, image_count: int,
rotation: bool = False) -> ScanResult:
total_images = max(1, image_count)
start_angle = 0.0
if rotation:
try:
start_angle = float(self.omega)
except Exception:
start_angle = 0.0
angle_step = 360.0 / total_images if rotation else 0.0
images = [
ScanResultImagesInner(
number=i,
@@ -1900,8 +1938,9 @@ class AareDAQ:
spots_indexed=0,
index=0,
b=0.0,
angle=start_angle + i * angle_step if rotation else None,
)
for i in range(max(1, image_count))
for i in range(total_images)
]
return ScanResult(file_prefix=file_prefix, images=images)
@@ -1909,6 +1948,7 @@ class AareDAQ:
result = self._build_fake_scan_result(
file_prefix=request.file_prefix,
image_count=request.steps,
rotation=True
)
return CompletedRotationScan(
request=copy.deepcopy(request),
@@ -2002,6 +2042,11 @@ class AareDAQ:
def __raster(self, request: RasterGridRequest, wait_for_screenshot: float | None = 0.3) -> CompletedRasterGridElem:
total_time = request.exp_time_s * request.n_x * request.n_y + request.n_y * 0.3
if total_time > 1200:
raise RasterScanException(f"Raster scan is too long {total_time}s > 20min")
status = self.status
smargon_top_left =request.smargon_top_left
@@ -2022,8 +2067,6 @@ class AareDAQ:
),
)
total_time = request.exp_time_s * request.n_x * request.n_y + request.n_y * 0.3
try:
if self.sample is not None and self.sample.db_id is not None:
self.__aare.create_gridscan_run(self.sample, request, status)
@@ -2036,7 +2079,7 @@ class AareDAQ:
raise
else:
logger.info("Simulated detector mode enabled; faking jfjoch intilalisation.")
logger.debug(f"Starting grid scan with {request.n_x}x{request.n_y} points, exp time {request.exp_time_s}s")
self.__devs.aerotech.grid_scan(
grid_elem_size_y_um=request.grid_size_mm.y * 1000,
grid_elem_size_x_um=request.grid_size_mm.x * 1000,
@@ -2045,14 +2088,15 @@ class AareDAQ:
time_sec=request.exp_time_s,
run_async=True,
)
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time * 2, 0)))
self.__devs.aerotech.wait_till_done(timeout=int(round(total_time + total_time * 0.1 + 60, 0)))
# go back to aerotech x,y,z home not U home (0 degrees).
if isinstance(self.__cfg.abr_meas_pos, Coordinate):
coord = self.__cfg.abr_meas_pos
else:
coord = self.__cfg.abr_meas_pos.at_mm
self.__devs.aerotech_pos = AerotechCoordinate(at_mm=coord, omega_deg=self.__devs.aerotech_omega)
self.__devs.aerotech.wait_till_done(timeout=int(360))
self.__devs.aerotech.wait_till_done(timeout=int(60))
if self.__cfg.simulated_detector:
logger.info("Simulated detector mode enabled; using fake raster result.")
@@ -2165,7 +2209,7 @@ class AareDAQ:
y_mm=y,
request=request,
)
self._upload_raster_diffraction_preview(
sample_id=self.sample.db_id,
filename=diffraction_image_filename,
@@ -2228,9 +2272,7 @@ class AareDAQ:
)
self.__cfg.try_set_busy(timeout=ceil(360))
try:
self.__cfg.simulated_detector = True
result = self._execute_raster_sequence(request, auto_center=auto_center)
self.__cfg.simulated_detector = False
if result is None:
raise RasterScanException("Raster scan failed")
@@ -2252,12 +2294,14 @@ class AareDAQ:
total_time = request.exp_time_s * request.steps
if self.sample is not None and self.sample.db_id is not None:
self.__aare.create_rotation_run(self.sample, request, status)
try:
if not self.__cfg.simulated_detector:
try:
self.__jfjoch.wait_till_running(timeout=60.0)
except Exception as e:
self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
raise
try:
if request.screening:
self.__devs.aerotech.screening_scan(
@@ -2995,7 +3039,7 @@ class AareDAQ:
params.exp_time_s = 0.08
else:
params.dtz = 150
params.exp_time_s = 0.04
params.exp_time_s = 0.01
return params
@@ -3086,9 +3130,9 @@ class AareDAQ:
sample.pin,
sample.sample_name
)
self.__cfg.simulated_detector = True
try:
self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) #upped tiemout for large grids
self.__cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S) # upped tiemout for large grids
self._set_progress_context(
progress,
@@ -3097,7 +3141,8 @@ class AareDAQ:
self._mark_progress_running(progress, WorkflowStateKind.MOUNT, "Mounting sample")
if not self._execute_mount_and_prepare(sample):
self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, "Mount failed")
mount_error_message = self._last_mount_error_message or "Mount failed"
self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, mount_error_message)
return self._end_operation(start, DAQOperation.MOUNT, error=True)
self._mark_progress_success(progress, WorkflowStateKind.MOUNT, "Mount complete")
self.__set_state(BeamlineStateEnum.SampleAlignment)
@@ -3205,7 +3250,7 @@ class AareDAQ:
steps=params.steps,
transmission=params.transmission,
)
self.__cfg.simulated_detector = False
rotation_result = self._execute_rotation_sequence(rotation_request)
if rotation_result is None:
logger.error("Rotation result was None")
@@ -3215,15 +3260,18 @@ class AareDAQ:
self._mark_progress_success(progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete")
logger.info(f"Rotation scan done at {time.perf_counter() - start}")
except BECCommunicationError as e:
self._raise_if_critical_bec_error(e, command=getattr(e, "operation", None) or "bec")
except JFJochCommunicationError as e:
self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown")
raise
except (BeamlineBusyTimeoutException,BeamlineBusyException) as e:
except (BeamlineBusyTimeoutException, BeamlineBusyException) as e:
time_of_measure = abs(time.perf_counter()-start)
time_of_measure = abs(time.perf_counter() - start)
if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S:
if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S:
logger.error(f"Error in measure due to Beamline Busy State timeout:"
f"Time of: {time_of_measure} is greater than timeout duration {self.AUTOMATION_BUSY_TIMEOUT_S}"
f"Error thrown: {e}")
@@ -3279,6 +3327,7 @@ class AareDAQ:
self._mark_progress_finished(progress, True, "Automation complete")
return self._end_operation(start, DAQOperation.AUTOMATION, error=False)
@log_timing(logger, "Changing Beamline State")
def __set_state(self, target: BeamlineStateEnum):
"""__set_state assumes that beamline is already in busy state
it will apply a proper transformation and change state afterward
@@ -3293,14 +3342,22 @@ class AareDAQ:
curr_state = self.__cfg.state
if hasattr(curr_state, "value") and hasattr(target, "value"):
logger.info(
f"State transition requested: {curr_state} -> {target}",
extra={"from_state": curr_state, "to_state": target},
)
if not self.__cfg.state_busy:
raise Exception("Beamline should be busy")
if target == BeamlineStateEnum.Maintenance:
self.__cfg.state = BeamlineStateEnum.Maintenance
elif target == curr_state:
logger.debug(f"State already set to {target}")
return
logger.debug(
f"State already set to {target}",
extra={"from_state": curr_state, "to_state": target},
)
elif target != curr_state:
self.__cfg.state = BeamlineStateEnum.Moving
try:
@@ -3320,6 +3377,10 @@ class AareDAQ:
elif target == BeamlineStateEnum.BeamLocation:
workflows.se2sa(self.__devs, self.__cfg)
workflows.sa2bl(self.__devs, self.__cfg)
elif target == BeamlineStateEnum.DewarTransfer:
workflows.common2dh(self.__devs, self.__cfg)
elif target == BeamlineStateEnum.RobotSampleExchange:
workflows.common_2rse(self.__devs, self.__cfg)
else:
raise TransformationInvalidException(
f"Cannot go from {curr_state} to {target}, not implemented"
@@ -3387,6 +3448,10 @@ class AareDAQ:
case BeamlineStateEnum.RobotSampleExchange:
if target == BeamlineStateEnum.SampleAlignment:
workflows.rse2sa(self.__devs, self.__cfg)
elif target == BeamlineStateEnum.SampleExchange:
workflows.rse2se(self.__devs, self.__cfg)
elif target == BeamlineStateEnum.DewarTransfer:
workflows.common2dh(self.__devs, self.__cfg)
else:
raise TransformationInvalidException(
f"Cannot go from {curr_state} to {target}, not implemented"
@@ -3411,11 +3476,23 @@ class AareDAQ:
f"Cannot go from {curr_state} to {target}, not implemented"
)
self.__cfg.state = target
logger.info(
f"State transition completed: {curr_state} -> {target}",
extra={"from_state": curr_state, "to_state": target},
)
except TransformationInvalidException as e:
logger.error(f"Cannot go from {curr_state} to {target}: {e}")
logger.error(
f"State transition invali: {curr_state} -> {target}: {e}",
extra={"from_state": curr_state, "to_state": target},
)
self.__cfg.state = curr_state
raise
except Exception as e:
logger.exception(
f"Exception during state transition : {curr_state} -> {target}."
f"Changing state to maintenance due to error: {e}",
extra={"from_state": curr_state, "to_state": target},
)
self.__cfg.state = BeamlineStateEnum.Maintenance
#TODO check if this is the right thing to do
self.__cfg.state_busy = False
@@ -3431,18 +3508,34 @@ class AareDAQ:
@property
def diffraction_geometry(self) -> DiffractionGeometry:
det_cfg = self.__jfjoch.detector()
return DiffractionGeometry(
energy_keV=self.__devs.energy_kev,
dtz_mm=self.__devs.dtz,
detector_size_pxl=(det_cfg.width, det_cfg.height),
pixel_size_mm=det_cfg.pixel_size_mm,
beam_center_pxl=self.__cfg.beam_center,
detector_description=det_cfg.description,
detector_serial_number=det_cfg.serial_number,
poni_rot1_rad=-0.001396263,
poni_rot2_rad=-0.003839724,
)
try:
det_cfg = self.__jfjoch.detector()
return DiffractionGeometry(
energy_keV=self.__devs.energy_kev,
dtz_mm=self.__devs.dtz,
detector_size_pxl=(det_cfg.width, det_cfg.height),
pixel_size_mm=det_cfg.pixel_size_mm,
beam_center_pxl=self.__cfg.beam_center,
detector_description=det_cfg.description,
detector_serial_number=det_cfg.serial_number,
poni_rot1_rad=-0.001396263,
poni_rot2_rad=-0.003839724,
)
except JFJochCommunicationError as e:
logger.warning(
f"Falling back to default diffraction geometry because detector metadata is unavailable: {e}"
)
return DiffractionGeometry(
energy_keV=self.__devs.energy_kev,
dtz_mm=self.__devs.dtz,
detector_size_pxl=(1, 1),
pixel_size_mm=0.15,
beam_center_pxl=self.__cfg.beam_center,
detector_description="unavailable",
detector_serial_number="unavailable",
poni_rot1_rad=-0.001396263,
poni_rot2_rad=-0.003839724,
)
@property
def beamline_status(self) -> BeamlineStatus: