mx/AareDAQ
0
1
Fork 0
Code Issues 52 Pull Requests 1 Actions Packages Projects Releases 65 Wiki Activity

DAQ: changed handling and formatting of tell client mount values. and added b ack mount fail handler - to be tested!

Browse Source
This commit is contained in:
appleb_m
2026-05-11 16:09:41 +02:00
parent 5f81f0046f
commit d9558b980a
2 changed files with 185 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aare/daq/daq.py
+170 -54
View File
@@ -1,7 +1,8 @@
import copy
import json
import secrets
import time
from datetime import datetime
from datetime import datetime, timezone
from math import ceil, floor
from pathlib import Path
from typing import List, Tuple, Optional, Callable
@@ -12,7 +13,7 @@ from aareDB import SampleEventType
from jfjoch_client.exceptions import NotFoundException
from jfjoch_client import ScanResult, ScanResultImagesInner
from aare.common.tell_models import TellStateModel
from aare.common.tell_models import TellStateModel, TellPhaseEnum
from aare.daq import workflows
from aare.daq.aaredb import AareWrapper
@@ -97,7 +98,7 @@ class AareDAQ:
AUTO_RASTER_MAX_IMAGES = 4500
AUTO_RASTER_MIN_CELL_SIZE_MM = 0.005
AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD = True
def __init__(self, cfg: BeamlineConfig, bl: MXBeamline):
self.last_time = 0.0
self.__cfg = cfg
@@ -116,6 +117,7 @@ class AareDAQ:
self._automation_total_sample_time_s = 0.0
self._automation_last_sample_name = ""
self._automation_samples_in_queue = 0
self.__reset_mount_failure_counter("DAQ startup")
def _is_hardware_failure(self, error: Exception) -> bool:
return isinstance(
@@ -210,6 +212,77 @@ class AareDAQ:
return None
raise
@staticmethod
def _parse_iso_timestamp(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value)
except ValueError:
return None
def _get_tell_events_from_redis(self) -> list[dict]:
try:
redis_client = getattr(self._AareDAQ__cfg, "_BeamlineConfig__client", None)
beamline_key = getattr(self._AareDAQ__cfg, "_BeamlineConfig__bl", None)
if redis_client is None or beamline_key is None:
return []
redis_key = f"{beamline_key}:tell_events"
raw_value = redis_client.get(redis_key)
if raw_value in (None, "", b""):
return []
if isinstance(raw_value, bytes):
raw_value = raw_value.decode("utf-8")
parsed = json.loads(str(raw_value))
return parsed if isinstance(parsed, list) else []
except Exception as e:
logger.debug(f"Failed to read tell_events from Redis: {e}")
return []
@staticmethod
def _tell_phase_confirms_previous_sample_unmounted(phase: TellPhaseEnum | None) -> bool:
return phase in {
TellPhaseEnum.OLD_SAMPLE_RETURNED,
TellPhaseEnum.PICKING_NEW_SAMPLE,
TellPhaseEnum.PLACING_NEW_SAMPLE,
TellPhaseEnum.FINALIZING,
TellPhaseEnum.COMPLETE,
}
def _was_previous_sample_unmounted_since(self, started_at: datetime) -> bool:
tell_state = self._safe_tell_state()
if tell_state is not None:
state_ts = self._parse_iso_timestamp(tell_state.last_update_ts)
if (
state_ts is not None
and state_ts >= started_at
and tell_state.operation == "mount"
and self._tell_phase_confirms_previous_sample_unmounted(tell_state.phase)
):
return True
if (
tell_state.last_event_class == "Motion Sync"
and tell_state.last_event_value == "Sample put on Puck"
):
if state_ts is not None and state_ts >= started_at:
return True
for event in reversed(self._get_tell_events_from_redis()):
if (
event.get("class") == "Motion Sync"
and event.get("event") == "Sample put on Puck"
):
event_ts = self._parse_iso_timestamp(event.get("timestamp"))
if event_ts is not None and event_ts >= started_at:
return True
return False
def set_face_detection_progress_callback(self, cb: Callable[[dict], None] | None) -> None:
self._face_detection_progress_cb = cb
@@ -574,6 +647,9 @@ class AareDAQ:
Returns:
True if successful, False otherwise
"""
previous_sample = None
mount_started_at = datetime.now(timezone.utc)
try:
previous_sample = self.sample
if previous_sample is None or previous_sample.db_id is None:
@@ -610,6 +686,25 @@ class AareDAQ:
raise
except Exception as e:
logger.error(f"Mount failed: {e}")
previous_sample_unmounted = (
previous_sample is not None
and previous_sample.db_id is not None
and self._was_previous_sample_unmounted_since(mount_started_at)
)
if previous_sample_unmounted:
logger.info(
"Mount failed after TELL confirmed previous sample was unmounted; "
"marking previous sample as unmounted and clearing cached current_sample"
)
self.__cfg.current_sample = None
self.__aare.send_sample_event(
previous_sample,
SampleEventType.UNMOUNTED,
comment="Auto-unmount succeeded before mount failed",
)
try:
self.__set_state(BeamlineStateEnum.SampleAlignment)
except Exception:
@@ -622,10 +717,7 @@ class AareDAQ:
event_type=SampleEventType.MOUNTFAILED
)
sample_name = self._sample_mount_display_name(sample)
if sample is None:
raise UnmountingFailed(f"Failed to unmount {sample_name}: {e}") from e
raise MountingFailed(f"Failed to mount {sample_name}: {e}") from e
return False
def _execute_loop_centering(self, sample: SampleShortInfo) -> bool:
"""
@@ -1012,17 +1104,6 @@ class AareDAQ:
location=mounted_address.puck,
)
def _find_sample_by_mounted_address(self, mounted_address) -> SampleShortInfo | None:
for sample in self.__cfg.spreadsheet.s:
if self._sample_matches_mounted_address(sample, mounted_address):
return sample
for sample in self.__cfg.reference_tools.s:
if self._sample_matches_mounted_address(sample, mounted_address):
return sample
return None
@staticmethod
def _sample_mount_display_name(sample: SampleShortInfo | None) -> str:
if sample is None:
@@ -1301,20 +1382,41 @@ class AareDAQ:
self.__cfg.state_busy = False
raise
def park_and_dry(self):
def _execute_dry(self, park=True):
self.__devs.tell.check_enable_motion()
self.__devs.tell.wait_not_busy()
self.__devs.tell.set_in_mount_position(True)
if self.sample:
self.__devs.tell.unmount(wait=True, timeout=60.0)
self.__cfg.current_sample = None
if park:
"""set wait_cold to -1 to go to park after drying"""
self.__devs.tell.dry(wait_cold=-1, wait=True)
else:
"""use default tell wait cold after drying"""
self.__devs.tell.dry(wait=True)
def park_and_dry(self, park = True):
self.__cfg.try_set_busy(timeout=360)
try:
self.__devs.tell.check_enable_motion()
self.__devs.tell.wait_not_busy()
self.__devs.tell.set_in_mount_position(True)
self.__devs.tell.unmount(wait=True, timeout=60.0)
self.__devs.tell.dry(wait_cold=-1, wait=True)
self.__cfg.current_sample = None
self._execute_dry(park=park)
self.__cfg.state_busy = False
except Exception as e:
self.__cfg.state_busy = False
logger.error(f"Failed to park and dry: {e}")
raise
def __reset_mount_failure_counter(self, reason: str) -> None:
previous_count = self.__cfg.get_mount_fail_count()
if previous_count > 0:
logger.warning(
f"Resetting mount failure counter from {previous_count} due to: {reason}"
)
else:
logger.debug(f"Mount failure counter already clear: {reason}")
self.__cfg.record_mount_success()
def blower_control(self):
try:
@@ -1337,38 +1439,42 @@ class AareDAQ:
logger.error("Goniometer didn't reach position based on magnet position sensor readout")
raise Exception("Goniometer is not in position based on magnet position sensor readout")
def __mount_fail_handler(
self,
mount_error: bool = False,
dry_after_fail_count: int = 3,
end_automation_after_fail_count: int = 5
):
if mount_error:
logger.error("Mount failed")
logger.debug("Mount error: ", extra={"mount_error": mount_error})
logger.debug("Currently fail handler disabled due to bugs..?")
# if mount_error:
# count = self.__cfg.record_mount_failure()
# if count > end_automation_after_fail_count:
# logger.error(f"Mounting continued to fail after stopping automation, contact your local contact")
# logger.debug(f"Clearing mount fail count")
# self.__cfg.record_mount_success()
# raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.")
# elif count == end_automation_after_fail_count:
# logger.error(f"Mounting failed {count} times, stopping automation")
# raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.")
# elif count == dry_after_fail_count:
# logger.warning(f"Mount failed {count} times in a row; drying")
# try:
# self.park_and_dry()
# except Exception as e:
# logger.exception(f"Failed to dry after mount failure: {e}")
# else:
# return
# else:
# logger.debug("Mounting succeeded, resetting mount failure counter")
# self.__cfg.record_mount_success()
# logger.error("Mount failed")
# logger.debug("Mount error: ", extra={"mount_error": mount_error})
# logger.debug("Currently fail handler disabled due to bugs..?")
if mount_error:
count = self.__cfg.record_mount_failure()
if count > end_automation_after_fail_count:
logger.error(f"Mounting continued to fail after stopping automation, contact your local contact")
logger.debug(f"Clearing mount fail count")
self.__cfg.record_mount_success()
raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.")
elif count == end_automation_after_fail_count:
logger.error(f"Mounting failed {count} times, stopping automation")
try:
self._execute_dry(park=True)
except Exception as e:
logger.exception(f"Failed to dry after mount failure: {e}")
raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.")
elif count == dry_after_fail_count:
#Dry after several fails
logger.warning(f"Mount failed {count} times in a row; drying")
try:
self._execute_dry(park=False)
except Exception as e:
logger.exception(f"Failed to dry after mount failure: {e}")
else:
return
else:
logger.debug("Mounting succeeded, resetting mount failure counter")
self.__cfg.record_mount_success()
def __mount_handler(self, target : SampleShortInfo):
@@ -1376,17 +1482,29 @@ class AareDAQ:
wait=True, timeout=360.0)
logger.debug(f"Mount response: {value.value}")
if value == TellEventValueEnum.NO_PIN_IN_GRIPPER:
"""No Pin in gripper is classed as a non critical mount fail
Unless it happens multiple times in a row, in which case the mount fail handler should dry the gripper
and if it continues stop automation as either there is several missing samples, they are stuck in the dewar
or a problem with the gripper
"""
raise MountingFailed("No Pin in Gripper")
elif value == TellEventValueEnum.PIN_IS_LOST_GRIPPER:
"""Pin is lost gripper is classed as a non critical mount fail
Unless it happens multiple times in a row, in which case the mount fail handler should dry the gripper
asit may be a problem with the gripper or the type of pins. If it continues after this automation should stop.
"""
logger.error("Pin is lost")
raise MountingFailed("Pin is lost")
elif value == TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT or value == TellEventValueEnum.DRY or value == TellEventValueEnum.COLD:
"""Robot clear after mount, tell dry or tell cold are not fails, they are events telling the daq to allow
automation to continue as the robot is busy but will not crash if the end station state is changed
this allows us to save a lot of time in autoamtion"""
logger.info(f"Robot is {value.value} - freeing beamline for user")
else:
"""Currently there are no other fail messages from Tell"""
logger.info(f"Mount response: {value.value} is not currently handled")
return
def __mount(self, target: SampleShortInfo | None):
self.__devs.smargon_move_home()
self.__devs.aerotech_pos = ABR_POS_MOUNT
@@ -1418,14 +1536,12 @@ class AareDAQ:
self.__cfg.current_sample = target
except CriticalTellException as e:
logger.error(f"Critical Tell error: {e}")
self.__mount_fail_handler(mount_error=True)
raise
except Exception as e:
logger.error(f"Mount failed: {e}")
self.__mount_fail_handler(mount_error=True)
raise
def recovery_unmount_sample(self) -> None:
self.__cfg.try_set_busy(timeout=360)
try:
@@ -2474,7 +2590,7 @@ class AareDAQ:
)
return RasterGridRequest(
exp_time_s=0.02,
exp_time_s=0.01, #TODO different settings for different beamlines
transmission=1.0,
smargon_top_left=SmargonCoordinate(chi_deg=geom.smargon.chi_deg,
phi_deg=geom.smargon.phi_deg,
src/aare/devices/tell_client.py
+15 -1
View File
@@ -219,7 +219,10 @@ class TellClient:
timeout=wait_timeout, msg=f"Mount {segment}{puck}-{sample}: "
)
return value
elif event == TellEventTypeEnum.GIPPER_DETECTION.value and value == TellEventValueEnum.NO_PIN_IN_GRIPPER.value:
elif (
event == TellEventTypeEnum.GIPPER_DETECTION.value
and value == TellEventValueEnum.NO_PIN_IN_GRIPPER.value
):
logger.info(f"{TellEventValueEnum.NO_PIN_IN_GRIPPER.value}")
return TellEventValueEnum.NO_PIN_IN_GRIPPER
elif event == TellEventTypeEnum.GIPPER_DETECTION.value and value == TellEventValueEnum.PIN_STILL_IN_GRIPPER.value:
@@ -233,6 +236,17 @@ class TellClient:
elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.DRY.value:
logger.info(f"{TellEventValueEnum.DRY.value}")
return TellEventValueEnum.DRY
elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.COLD.value:
logger.info(f"{TellEventValueEnum.COLD.value}")
logger.info("As cold, likely robot is cooling from previous mount/dry "
"DAQ will block until command is complete")
self.check_command_ok(
timeout=wait_timeout, msg=f"Mount {segment}{puck}-{sample}: "
)
return TellEventValueEnum.COLD
elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.UNKNOWN.value:
logger.info(f"{TellEventValueEnum.UNKNOWN.value}")
return TellEventValueEnum.UNKNOWN
elif event == TellEventTypeEnum.MOTION_SYNC.value and value == TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT.value:
logger.info(f"{TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT.value}")
return TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT