From d9558b980ab2a32bf7515f486288d2f9aba6f2d2 Mon Sep 17 00:00:00 2001 From: appleb_m Date: Mon, 11 May 2026 16:09:41 +0200 Subject: [PATCH] DAQ: changed handling and formatting of tell client mount values. and added b ack mount fail handler - to be tested! --- src/aare/daq/daq.py | 224 ++++++++++++++++++++++++-------- src/aare/devices/tell_client.py | 16 ++- 2 files changed, 185 insertions(+), 55 deletions(-) diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index 437dc8df..ccde41d3 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -1,7 +1,8 @@ import copy +import json import secrets import time -from datetime import datetime +from datetime import datetime, timezone from math import ceil, floor from pathlib import Path from typing import List, Tuple, Optional, Callable @@ -12,7 +13,7 @@ from aareDB import SampleEventType from jfjoch_client.exceptions import NotFoundException from jfjoch_client import ScanResult, ScanResultImagesInner -from aare.common.tell_models import TellStateModel +from aare.common.tell_models import TellStateModel, TellPhaseEnum from aare.daq import workflows from aare.daq.aaredb import AareWrapper @@ -97,7 +98,7 @@ class AareDAQ: AUTO_RASTER_MAX_IMAGES = 4500 AUTO_RASTER_MIN_CELL_SIZE_MM = 0.005 AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD = True - + def __init__(self, cfg: BeamlineConfig, bl: MXBeamline): self.last_time = 0.0 self.__cfg = cfg @@ -116,6 +117,7 @@ class AareDAQ: self._automation_total_sample_time_s = 0.0 self._automation_last_sample_name = "" self._automation_samples_in_queue = 0 + self.__reset_mount_failure_counter("DAQ startup") def _is_hardware_failure(self, error: Exception) -> bool: return isinstance( @@ -210,6 +212,77 @@ class AareDAQ: return None raise + @staticmethod + def _parse_iso_timestamp(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value) + except ValueError: + return None + + def _get_tell_events_from_redis(self) -> list[dict]: + try: + redis_client = getattr(self._AareDAQ__cfg, "_BeamlineConfig__client", None) + beamline_key = getattr(self._AareDAQ__cfg, "_BeamlineConfig__bl", None) + + if redis_client is None or beamline_key is None: + return [] + + redis_key = f"{beamline_key}:tell_events" + raw_value = redis_client.get(redis_key) + if raw_value in (None, "", b""): + return [] + + if isinstance(raw_value, bytes): + raw_value = raw_value.decode("utf-8") + + parsed = json.loads(str(raw_value)) + return parsed if isinstance(parsed, list) else [] + except Exception as e: + logger.debug(f"Failed to read tell_events from Redis: {e}") + return [] + + @staticmethod + def _tell_phase_confirms_previous_sample_unmounted(phase: TellPhaseEnum | None) -> bool: + return phase in { + TellPhaseEnum.OLD_SAMPLE_RETURNED, + TellPhaseEnum.PICKING_NEW_SAMPLE, + TellPhaseEnum.PLACING_NEW_SAMPLE, + TellPhaseEnum.FINALIZING, + TellPhaseEnum.COMPLETE, + } + + def _was_previous_sample_unmounted_since(self, started_at: datetime) -> bool: + tell_state = self._safe_tell_state() + if tell_state is not None: + state_ts = self._parse_iso_timestamp(tell_state.last_update_ts) + if ( + state_ts is not None + and state_ts >= started_at + and tell_state.operation == "mount" + and self._tell_phase_confirms_previous_sample_unmounted(tell_state.phase) + ): + return True + + if ( + tell_state.last_event_class == "Motion Sync" + and tell_state.last_event_value == "Sample put on Puck" + ): + if state_ts is not None and state_ts >= started_at: + return True + + for event in reversed(self._get_tell_events_from_redis()): + if ( + event.get("class") == "Motion Sync" + and event.get("event") == "Sample put on Puck" + ): + event_ts = self._parse_iso_timestamp(event.get("timestamp")) + if event_ts is not None and event_ts >= started_at: + return True + + return False + def set_face_detection_progress_callback(self, cb: Callable[[dict], None] | None) -> None: self._face_detection_progress_cb = cb @@ -574,6 +647,9 @@ class AareDAQ: Returns: True if successful, False otherwise """ + previous_sample = None + mount_started_at = datetime.now(timezone.utc) + try: previous_sample = self.sample if previous_sample is None or previous_sample.db_id is None: @@ -610,6 +686,25 @@ class AareDAQ: raise except Exception as e: logger.error(f"Mount failed: {e}") + + previous_sample_unmounted = ( + previous_sample is not None + and previous_sample.db_id is not None + and self._was_previous_sample_unmounted_since(mount_started_at) + ) + + if previous_sample_unmounted: + logger.info( + "Mount failed after TELL confirmed previous sample was unmounted; " + "marking previous sample as unmounted and clearing cached current_sample" + ) + self.__cfg.current_sample = None + self.__aare.send_sample_event( + previous_sample, + SampleEventType.UNMOUNTED, + comment="Auto-unmount succeeded before mount failed", + ) + try: self.__set_state(BeamlineStateEnum.SampleAlignment) except Exception: @@ -622,10 +717,7 @@ class AareDAQ: event_type=SampleEventType.MOUNTFAILED ) - sample_name = self._sample_mount_display_name(sample) - if sample is None: - raise UnmountingFailed(f"Failed to unmount {sample_name}: {e}") from e - raise MountingFailed(f"Failed to mount {sample_name}: {e}") from e + return False def _execute_loop_centering(self, sample: SampleShortInfo) -> bool: """ @@ -1012,17 +1104,6 @@ class AareDAQ: location=mounted_address.puck, ) - def _find_sample_by_mounted_address(self, mounted_address) -> SampleShortInfo | None: - for sample in self.__cfg.spreadsheet.s: - if self._sample_matches_mounted_address(sample, mounted_address): - return sample - - for sample in self.__cfg.reference_tools.s: - if self._sample_matches_mounted_address(sample, mounted_address): - return sample - - return None - @staticmethod def _sample_mount_display_name(sample: SampleShortInfo | None) -> str: if sample is None: @@ -1301,20 +1382,41 @@ class AareDAQ: self.__cfg.state_busy = False raise - def park_and_dry(self): + def _execute_dry(self, park=True): + self.__devs.tell.check_enable_motion() + self.__devs.tell.wait_not_busy() + self.__devs.tell.set_in_mount_position(True) + + if self.sample: + self.__devs.tell.unmount(wait=True, timeout=60.0) + self.__cfg.current_sample = None + + if park: + """set wait_cold to -1 to go to park after drying""" + self.__devs.tell.dry(wait_cold=-1, wait=True) + else: + """use default tell wait cold after drying""" + self.__devs.tell.dry(wait=True) + + def park_and_dry(self, park = True): self.__cfg.try_set_busy(timeout=360) try: - self.__devs.tell.check_enable_motion() - self.__devs.tell.wait_not_busy() - self.__devs.tell.set_in_mount_position(True) - self.__devs.tell.unmount(wait=True, timeout=60.0) - self.__devs.tell.dry(wait_cold=-1, wait=True) - self.__cfg.current_sample = None + self._execute_dry(park=park) self.__cfg.state_busy = False except Exception as e: self.__cfg.state_busy = False logger.error(f"Failed to park and dry: {e}") raise + + def __reset_mount_failure_counter(self, reason: str) -> None: + previous_count = self.__cfg.get_mount_fail_count() + if previous_count > 0: + logger.warning( + f"Resetting mount failure counter from {previous_count} due to: {reason}" + ) + else: + logger.debug(f"Mount failure counter already clear: {reason}") + self.__cfg.record_mount_success() def blower_control(self): try: @@ -1337,38 +1439,42 @@ class AareDAQ: logger.error("Goniometer didn't reach position based on magnet position sensor readout") raise Exception("Goniometer is not in position based on magnet position sensor readout") - def __mount_fail_handler( self, mount_error: bool = False, dry_after_fail_count: int = 3, end_automation_after_fail_count: int = 5 ): - if mount_error: - logger.error("Mount failed") - logger.debug("Mount error: ", extra={"mount_error": mount_error}) - logger.debug("Currently fail handler disabled due to bugs..?") # if mount_error: - # count = self.__cfg.record_mount_failure() - # if count > end_automation_after_fail_count: - # logger.error(f"Mounting continued to fail after stopping automation, contact your local contact") - # logger.debug(f"Clearing mount fail count") - # self.__cfg.record_mount_success() - # raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.") - # elif count == end_automation_after_fail_count: - # logger.error(f"Mounting failed {count} times, stopping automation") - # raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.") - # elif count == dry_after_fail_count: - # logger.warning(f"Mount failed {count} times in a row; drying") - # try: - # self.park_and_dry() - # except Exception as e: - # logger.exception(f"Failed to dry after mount failure: {e}") - # else: - # return - # else: - # logger.debug("Mounting succeeded, resetting mount failure counter") - # self.__cfg.record_mount_success() + # logger.error("Mount failed") + # logger.debug("Mount error: ", extra={"mount_error": mount_error}) + # logger.debug("Currently fail handler disabled due to bugs..?") + if mount_error: + count = self.__cfg.record_mount_failure() + if count > end_automation_after_fail_count: + logger.error(f"Mounting continued to fail after stopping automation, contact your local contact") + logger.debug(f"Clearing mount fail count") + self.__cfg.record_mount_success() + raise CriticalTellException(f"Mounting continued to fail after stopping automation, contact your local contact.") + elif count == end_automation_after_fail_count: + logger.error(f"Mounting failed {count} times, stopping automation") + try: + self._execute_dry(park=True) + except Exception as e: + logger.exception(f"Failed to dry after mount failure: {e}") + raise CriticalTellException(f"Mount failed {count} times in a row, stopping automation.") + elif count == dry_after_fail_count: + #Dry after several fails + logger.warning(f"Mount failed {count} times in a row; drying") + try: + self._execute_dry(park=False) + except Exception as e: + logger.exception(f"Failed to dry after mount failure: {e}") + else: + return + else: + logger.debug("Mounting succeeded, resetting mount failure counter") + self.__cfg.record_mount_success() def __mount_handler(self, target : SampleShortInfo): @@ -1376,17 +1482,29 @@ class AareDAQ: wait=True, timeout=360.0) logger.debug(f"Mount response: {value.value}") if value == TellEventValueEnum.NO_PIN_IN_GRIPPER: + """No Pin in gripper is classed as a non critical mount fail + Unless it happens multiple times in a row, in which case the mount fail handler should dry the gripper + and if it continues stop automation as either there is several missing samples, they are stuck in the dewar + or a problem with the gripper + """ raise MountingFailed("No Pin in Gripper") elif value == TellEventValueEnum.PIN_IS_LOST_GRIPPER: + """Pin is lost gripper is classed as a non critical mount fail + Unless it happens multiple times in a row, in which case the mount fail handler should dry the gripper + asit may be a problem with the gripper or the type of pins. If it continues after this automation should stop. + """ logger.error("Pin is lost") raise MountingFailed("Pin is lost") elif value == TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT or value == TellEventValueEnum.DRY or value == TellEventValueEnum.COLD: + """Robot clear after mount, tell dry or tell cold are not fails, they are events telling the daq to allow + automation to continue as the robot is busy but will not crash if the end station state is changed + this allows us to save a lot of time in autoamtion""" logger.info(f"Robot is {value.value} - freeing beamline for user") else: + """Currently there are no other fail messages from Tell""" logger.info(f"Mount response: {value.value} is not currently handled") return - def __mount(self, target: SampleShortInfo | None): self.__devs.smargon_move_home() self.__devs.aerotech_pos = ABR_POS_MOUNT @@ -1418,14 +1536,12 @@ class AareDAQ: self.__cfg.current_sample = target except CriticalTellException as e: logger.error(f"Critical Tell error: {e}") - self.__mount_fail_handler(mount_error=True) raise except Exception as e: logger.error(f"Mount failed: {e}") self.__mount_fail_handler(mount_error=True) raise - def recovery_unmount_sample(self) -> None: self.__cfg.try_set_busy(timeout=360) try: @@ -2474,7 +2590,7 @@ class AareDAQ: ) return RasterGridRequest( - exp_time_s=0.02, + exp_time_s=0.01, #TODO different settings for different beamlines transmission=1.0, smargon_top_left=SmargonCoordinate(chi_deg=geom.smargon.chi_deg, phi_deg=geom.smargon.phi_deg, diff --git a/src/aare/devices/tell_client.py b/src/aare/devices/tell_client.py index 98fbecf0..11c58f52 100755 --- a/src/aare/devices/tell_client.py +++ b/src/aare/devices/tell_client.py @@ -219,7 +219,10 @@ class TellClient: timeout=wait_timeout, msg=f"Mount {segment}{puck}-{sample}: " ) return value - elif event == TellEventTypeEnum.GIPPER_DETECTION.value and value == TellEventValueEnum.NO_PIN_IN_GRIPPER.value: + elif ( + event == TellEventTypeEnum.GIPPER_DETECTION.value + and value == TellEventValueEnum.NO_PIN_IN_GRIPPER.value + ): logger.info(f"{TellEventValueEnum.NO_PIN_IN_GRIPPER.value}") return TellEventValueEnum.NO_PIN_IN_GRIPPER elif event == TellEventTypeEnum.GIPPER_DETECTION.value and value == TellEventValueEnum.PIN_STILL_IN_GRIPPER.value: @@ -233,6 +236,17 @@ class TellClient: elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.DRY.value: logger.info(f"{TellEventValueEnum.DRY.value}") return TellEventValueEnum.DRY + elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.COLD.value: + logger.info(f"{TellEventValueEnum.COLD.value}") + logger.info("As cold, likely robot is cooling from previous mount/dry " + "DAQ will block until command is complete") + self.check_command_ok( + timeout=wait_timeout, msg=f"Mount {segment}{puck}-{sample}: " + ) + return TellEventValueEnum.COLD + elif event == TellEventTypeEnum.MOTION_TASK.value and value == TellEventValueEnum.UNKNOWN.value: + logger.info(f"{TellEventValueEnum.UNKNOWN.value}") + return TellEventValueEnum.UNKNOWN elif event == TellEventTypeEnum.MOTION_SYNC.value and value == TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT.value: logger.info(f"{TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT.value}") return TellEventValueEnum.ROBOT_CLEAR_AFTER_MOUNT