AareDAQ/src/aare/daq/daq.py

import copy
import json
import secrets
import time
from collections.abc import Callable
from datetime import UTC, datetime
from math import ceil
from pathlib import Path

from aarecommon.config.beamline import cfg_get
from aarecommon.config.logger import setup_logger
from aarecommon.config.logger_events import (
    log_timing,
    merge_log_context,
    raster_request_log_context,
    rotation_request_log_context,
    sample_log_context,
)
from aarecommon.errors.exception_handler import (
    AerotechCommunicationError,
    AutoRasterSampleSkipped,
    BeamlineBusyException,
    BeamlineBusyTimeoutException,
    BECCommunicationError,
    CriticalTellException,
    DataCollectionException,
    JFJochCommunicationError,
    LoopCenteringFailed,
    MagnetPositionSensorError,
    MaintenanceStateException,
    MountingFailed,
    RasterScanException,
    SmargonCommunicationError,
    StateTransitionFailed,
    TellCommunicationError,
    TransformationInvalidException,
    UnmountingFailed,
    WarningTellException,
)
from aarecommon.math.coordinate import AerotechCoordinate, Coordinate, SmargonCoordinate
from aarecommon.math.diffraction_geometry import DiffractionGeometry
from aarecommon.math.sample_geometry import SampleGeometryModel
from aarecommon.models.automation import (
    AutomationProgress,
    StepState,
    StepStatus,
    WorkflowStateKind,
)
from aarecommon.models.beamline import MXBeamline
from aarecommon.models.models import (
    AutofocusSettings,
    BeamlineStatus,
    DAQOperation,
    DAQStatusModel,
    FluorescenceSpectrumOutputModel,
    FluorescenceSpectrumParameterModel,
    PuckLoadedInfo,
    SampleCameraSettings,
    SampleShortInfo,
    SampleShortInfoList,
    SessionStatus,
    SimpleScanParameters,
    ZoomModeEnum,
)
from aarecommon.models.raster_grid import CompletedRasterGrid, RasterGridRequest
from aarecommon.models.rotation_scan import CompletedRotationScan, RotationScanRequest
from aarecommon.models.tell import TellPhaseEnum, TellStateModel
from aareDB import SampleEventType

from aare.daq import workflows
from aare.daq.aaredb import AareWrapper
from aare.daq.config import ABR_POS_MOUNT, BeamlineConfig, BeamlineStateEnum
from aare.daq.config_model import LocalContactConfigModel
from aare.daq.devices import BeamlineDevices
from aare.daq.mlbox import MlBox
from aare.daq.operations.common.ml_bounding_box import get_ml_bounding_box
from aare.daq.operations.common.runtime import DAQRuntimeState
from aare.daq.operations.common.services import (
    DataCollectionPreparer,
    FaceDetectionProgressEmitter,
    OperationServices,
    PredictionProvider,
    SampleEventPublisher,
    ScanIngestionService,
    StateController,
    TraceWriter,
)
from aare.daq.operations.common.simulate_scan_result import build_fake_rotation_result
from aare.daq.operations.face_detection import (
    FaceDetectionContext,
    FaceDetectionResult,
    FaceDetectionService,
)
from aare.daq.operations.face_detection.models import (
    FaceDetectionDependencies,
    FaceDetectionSettings,
)
from aare.daq.operations.loop_centering import LoopCenteringContext, LoopCenteringService
from aare.daq.operations.loop_centering.models import (
    LoopCenteringDependencies,
    LoopCenteringSettings,
)
from aare.daq.operations.mounting.models import (
    MountingContext,
    MountingDependencies,
    MountingResult,
    MountingSettings,
)
from aare.daq.operations.mounting.service import MountingService
from aare.daq.operations.raster.models import RasterContext, RasterDependencies, RasterSettings
from aare.daq.operations.raster.service import RasterService
from aare.daq.operations.rotation.models import (
    RotationContext,
    RotationDependencies,
    RotationSettings,
)
from aare.daq.operations.rotation.service import RotationService
from aare.daq.operations.screenshot.service import ScreenshotService
from aare.devices.area_detector import AutoEnum
from aare.devices.jfjoch import JFJochWrapper

logger = setup_logger("aareDAQ")


class _DAQSampleProvider:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    @property
    def sample(self) -> SampleShortInfo | None:
        return self._daq.sample


class _DAQSampleGeometryProvider:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    @property
    def sample_geometry(self) -> SampleGeometryModel:
        return self._daq.sample_geometry


class _DAQNonCriticalRunner:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def __call__(
        self, action, *, description: str, sample: SampleShortInfo | None = None
    ) -> object | None:
        return self._daq._run_noncritical(action, description=description, sample=sample)


class _DAQScreenshotSampleProvider:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    @property
    def sample(self) -> SampleShortInfo | None:
        return self._daq.sample


class _DAQPGroupProvider:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    @property
    def pgroup(self) -> str | None:
        return self._daq._cfg.pgroup


class _DAQStatusProvider:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    @property
    def status(self) -> DAQStatusModel:
        return self._daq.status


class _DAQStateSetter:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def set_state(self, target: BeamlineStateEnum) -> None:
        self._daq._set_state(target)


class _DAQTraceAppender:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def append_smargon_trace(self, *, sample_id: int | None, event: str) -> None:
        self._daq._append_smargon_trace(sample_id=sample_id, event=event)


class _DAQSampleEventSender:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def send_sample_event(self, sample_id: int, event_type, comment: str | None = None) -> None:
        self._daq._aare.send_sample_event(sample_id, event_type, comment)


class _DAQScanIngestor:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def ingest_scan(self, *, sample, result, geom, beam_mark_pxl) -> None:
        self._daq._aare.ingest_scan(
            sample=sample, result=result, geom=geom, beam_mark_pxl=beam_mark_pxl
        )

    def ingest_gridscan(
        self, *, sample, raster_result, raster_request, geom, com, beam_mark_pxl
    ) -> None:
        self._daq._aare.ingest_gridscan(
            sample=sample,
            raster_result=raster_result,
            raster_request=raster_request,
            geom=geom,
            com=com,
            beam_mark_pxl=beam_mark_pxl,
        )


class _DAQDatacollectionSetupRunner:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def prepare(self, request, screening: bool = False) -> None:
        self._daq._setup_datacollection(request=request, screening=screening)


class _LoopCenteringPredictionGetter:
    def __init__(self, daq: "AareDAQ", settings: LoopCenteringSettings):
        self._daq = daq
        self._settings = settings

    def get_predictions(self):
        return self._daq._mlbox.predict_all_best(
            overlap_with_pin=self._settings.overlap_with_pin,
            confidence_min=self._settings.confidence_min,
            return_image=True,
            return_bundle_meta=True,
        )


class _FaceDetectionProgressReporter:
    def __init__(self, daq: "AareDAQ"):
        self._daq = daq

    def emit_progress(self, payload: dict) -> None:
        self._daq._emit_face_detection_progress(payload)


# TODO tidy up DAQ - migrate functions into different scripts, to reduce size?
# TODO investigate using a state machine within each operation to reduce callbacks?
class AareDAQ:
    """
    Main Data Acquisition class for the Aare system.

    This class orchestrates interactions between various beamline devices,
    the Aare database, ML services, and data collection workflows.
    """

    MIN_SPOTS_LOW_RES_THRESHOLD = 10.0
    AUTOMATION_BUSY_TIMEOUT_S = 600
    MANUAL_RASTER_BUSY_TIMEOUT_S = 600
    AUTO_RASTER_MAX_IMAGES = 4500
    AUTO_RASTER_MIN_CELL_SIZE_MM = 0.005
    AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD = True

    def __init__(self, cfg: BeamlineConfig, bl: MXBeamline):
        self.last_time = 0.0
        self._cfg = cfg
        self._devs = BeamlineDevices(bl)
        self._mlbox = MlBox(bl)
        self._jfjoch = JFJochWrapper(bl)
        self._bl = bl.value.upper()
        self._beamline = bl
        self._aare = AareWrapper(bl)
        self._saved_box = None
        self._smargon_trace_path = Path("/sls/mx/applications/logs") / "smargon_trace.csv"
        self._face_detection_progress_cb: Callable[[dict], None] | None = None
        self._automation_progress_cb: Callable[[AutomationProgress], None] | None = None
        self._last_sample_sync_ts = 0.0
        self._sample_sync_min_interval_s = 2.0
        self._automation_completed_samples = 0
        self._automation_total_sample_time_s = 0.0
        self._automation_last_sample_name = ""
        self._automation_samples_in_queue = 0
        self._last_mount_error_message = ""

        self._screenshot_service = ScreenshotService(
            mlbox=self._mlbox,
            aare=self._aare,
            logger=logger,
            run_noncritical=_DAQNonCriticalRunner(self),
            sample_provider=_DAQScreenshotSampleProvider(self),
            pgroup_provider=_DAQPGroupProvider(self),
        )

    def _cached_detector_metadata(self) -> dict:
        return self._cfg.cached_detector_metadata

    def refresh_detector_metadata_cache(self) -> dict[str, object]:
        det_cfg = self._jfjoch.detector()
        dtz_low = self._cfg._coerce_optional_float(self._devs.dtz_low)
        dtz_high = self._cfg._coerce_optional_float(self._devs.dtz_high)

        payload = self._cfg.set_detector_metadata(
            {
                "detector_description": det_cfg.description,
                "detector_serial_number": det_cfg.serial_number,
                "detector_width": det_cfg.width,
                "detector_height": det_cfg.height,
                "pixel_size_mm": det_cfg.pixel_size_mm,
                "dtz_low": dtz_low,
                "dtz_high": dtz_high,
            }
        )
        logger.info(
            "Refreshed hardware metadata cache",
            extra={
                "detector_description": payload.get("detector_description"),
                "detector_serial_number": payload.get("detector_serial_number"),
                "dtz_low": payload.get("dtz_low"),
                "dtz_high": payload.get("dtz_high"),
            },
        )
        return payload

    def get_runtime_simulation_state(self) -> dict[str, bool]:
        return self._cfg.runtime_simulation_state

    def get_local_contact_links(self) -> dict[str, str | None]:
        return self._cfg.local_contact_links

    def get_local_contact_device_state(self) -> dict[str, dict[str, str | bool | None]]:
        status = self.status
        sim = self.get_runtime_simulation_state()

        return {
            "bec": {"mode": "simulated" if sim.get("bec") else "live", "error": None},
            "detector": {"mode": "simulated" if sim.get("detector") else "live", "error": None},
            "tell": {
                "mode": "simulated" if sim.get("tell") else "live",
                "error": getattr(status, "tell_error", None),
            },
            "aerotech": {
                "mode": "simulated" if sim.get("aerotech") else "live",
                "error": getattr(status, "aerotech_error", None),
            },
            "smargon": {
                "mode": "simulated" if sim.get("smargon") else "live",
                "error": getattr(status, "smargon_error", None),
            },
        }

    def restart_bec_worker(self) -> dict[str, object]:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.restart_bec_worker(simulated=self._cfg.simulate_bec)
            return {"ok": True, "device": "bec", "simulated": self._cfg.simulate_bec}
        finally:
            self._cfg.state_busy = False

    def restart_detector(self) -> dict[str, object]:
        self._cfg.try_set_busy(timeout=360)
        try:
            beamline = MXBeamline.SIMULATED if self._cfg.simulated_detector else self._beamline
            logger.info(f"Restarting JFJoch wrapper with simulated={self._cfg.simulated_detector}")
            self._jfjoch = JFJochWrapper(beamline)
            return {
                "ok": True,
                "device": "detector",
                "simulated": bool(self._cfg.simulated_detector),
            }
        finally:
            self._cfg.state_busy = False

    def restart_tell(self) -> dict[str, object]:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.restart_tell(simulated=self._cfg.simulate_tell)
            return {"ok": True, "device": "tell", "simulated": self._cfg.simulate_tell}
        finally:
            self._cfg.state_busy = False

    def restart_aerotech(self) -> dict[str, object]:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.restart_aerotech(simulated=self._cfg.simulate_aerotech)
            return {"ok": True, "device": "aerotech", "simulated": self._cfg.simulate_aerotech}
        finally:
            self._cfg.state_busy = False

    def restart_smargon(self) -> dict[str, object]:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.restart_smargon(simulated=self._cfg.simulate_smargon)
            return {"ok": True, "device": "smargon", "simulated": self._cfg.simulate_smargon}
        finally:
            self._cfg.state_busy = False

    def set_runtime_simulation(self, device: str, enabled: bool) -> dict[str, object]:
        device = str(device).strip().lower()

        if device == "bec":
            self._cfg.simulate_bec = enabled
            return self.restart_bec_worker()
        elif device == "detector":
            self._cfg.simulated_detector = enabled
            return self.restart_detector()
        elif device == "tell":
            self._cfg.simulate_tell = enabled
            return self.restart_tell()
        elif device == "aerotech":
            self._cfg.simulate_aerotech = enabled
            return self.restart_aerotech()
        elif device == "smargon":
            self._cfg.simulate_smargon = enabled
            return self.restart_smargon()

        raise ValueError(
            "Unknown simulation device. Expected one of: bec, detector, tell, aerotech, smargon"
        )

    def _is_hardware_failure(self, error: Exception) -> bool:
        return isinstance(
            error,
            (
                CriticalTellException,
                TellCommunicationError,
                BECCommunicationError,
                SmargonCommunicationError,
                AerotechCommunicationError,
                JFJochCommunicationError,
                MountingFailed,
                UnmountingFailed,
                MagnetPositionSensorError,
                TransformationInvalidException,
                StateTransitionFailed,
                MaintenanceStateException,
                DataCollectionException,
                RasterScanException,
            ),
        )

    @staticmethod
    def _is_jfjoch_detector_state_error(message: str | None) -> bool:
        text = str(message or "").lower()
        return (
            "daq state error" in text
            or "must be idle to start measurement" in text
            or "must be idle" in text
        )

    def _raise_if_critical_jfjoch_detector_error(self, error: Exception, *, command: str) -> None:
        if not isinstance(error, JFJochCommunicationError):
            return

        if getattr(error, "status_code", None) != 500 and not self._is_jfjoch_detector_state_error(
            str(error)
        ):
            return

        message = (
            f"Critical detector error while running JFJoch command '{command}'. "
            f"Automation has been stopped. "
            f"There is an error with the detector. Please call your local contact. "
            f"Original error: {error}"
        )
        logger.critical(message)
        raise JFJochCommunicationError(
            message,
            operation=getattr(error, "operation", None),
            endpoint=getattr(error, "endpoint", None),
            base_url=getattr(error, "base_url", None),
            status_code=getattr(error, "status_code", None),
            critical=True,
        ) from error

    def _raise_if_critical_bec_error(self, error: Exception, *, command: str) -> None:
        if not isinstance(error, BECCommunicationError):
            return

        message = (
            f"Critical BEC error while running '{command}'. "
            f"Automation has been stopped because the beamline state may be inconsistent. "
            f"Original error: {error}"
        )
        logger.critical(message)
        raise BECCommunicationError(
            message,
            operation=getattr(error, "operation", None),
            endpoint=getattr(error, "endpoint", None),
            base_url=getattr(error, "base_url", None),
            exception=getattr(error, "exception", None),
            critical=True,
        ) from error

    def _ensure_not_in_maintenance(self, *, context: str) -> None:
        if self._cfg.state == BeamlineStateEnum.Maintenance:
            raise MaintenanceStateException(
                f"Automation cannot continue because beamline is in Maintenance during {context}"
            )

    def _validate_automation_state(self, *, context: str) -> None:
        self._ensure_not_in_maintenance(context=context)
        if self._cfg.state == BeamlineStateEnum.Moving:
            raise StateTransitionFailed(
                f"Automation cannot continue because beamline is still in Moving state during {context}"
            )

    def _run_noncritical(
        self,
        action: Callable[[], object],
        *,
        description: str,
        sample: SampleShortInfo | None = None,
    ) -> object | None:
        try:
            return action()
        except Exception as e:
            logger.warning(
                f"Non-critical failure during {description}: {e}",
                extra={
                    "sample_name": getattr(sample, "sample_name", None),
                    "sample_id": getattr(sample, "db_id", None),
                },
                exc_info=True,
            )
            return None

    def _run_critical(
        self,
        action: Callable[[], object],
        *,
        description: str,
        sample: SampleShortInfo | None = None,
    ) -> object:
        try:
            return action()
        except Exception as e:
            if isinstance(e, WarningTellException):
                logger.warning(
                    f"Non-critical warning during {description}: {e}",
                    extra={
                        "sample_name": getattr(sample, "sample_name", None),
                        "sample_id": getattr(sample, "db_id", None),
                    },
                    exc_info=True,
                )
                return None
            raise

    @staticmethod
    def _parse_iso_timestamp(value: str | None) -> datetime | None:
        if not value:
            return None
        try:
            return datetime.fromisoformat(value)
        except ValueError:
            return None

    def _get_tell_events_from_redis(self) -> list[dict]:
        try:
            redis_client = getattr(self._cfg, "_client", None)
            beamline_key = getattr(self._cfg, "_bl", None)

            if redis_client is None or beamline_key is None:
                return []

            redis_key = f"{beamline_key}:tell_events"
            raw_value = redis_client.get(redis_key)
            if raw_value in (None, "", b""):
                return []

            if isinstance(raw_value, bytes):
                raw_value = raw_value.decode("utf-8")

            parsed = json.loads(str(raw_value))
            return parsed if isinstance(parsed, list) else []
        except Exception as e:
            logger.debug(f"Failed to read tell_events from Redis: {e}", exc_info=True)
            return []

    @staticmethod
    def _tell_phase_confirms_previous_sample_unmounted(phase: TellPhaseEnum | None) -> bool:
        return phase in {
            TellPhaseEnum.OLD_SAMPLE_RETURNED,
            TellPhaseEnum.PICKING_NEW_SAMPLE,
            TellPhaseEnum.PLACING_NEW_SAMPLE,
            TellPhaseEnum.FINALIZING,
            TellPhaseEnum.COMPLETE,
        }

    def _was_previous_sample_unmounted_since(self, started_at: datetime) -> bool:
        tell_state = self._safe_tell_state()
        if tell_state is not None:
            state_ts = self._parse_iso_timestamp(tell_state.last_update_ts)
            if (
                state_ts is not None
                and state_ts >= started_at
                and tell_state.operation == "mount"
                and self._tell_phase_confirms_previous_sample_unmounted(tell_state.phase)
            ):
                return True

            if (
                (
                    tell_state.last_event_class == "Motion Sync"
                    and tell_state.last_event_value == "Sample put on Puck"
                )
                and state_ts is not None
                and state_ts >= started_at
            ):
                return True

        for event in reversed(self._get_tell_events_from_redis()):
            if event.get("class") == "Motion Sync" and event.get("event") == "Sample put on Puck":
                event_ts = self._parse_iso_timestamp(event.get("timestamp"))
                if event_ts is not None and event_ts >= started_at:
                    return True

        return False

    def set_face_detection_progress_callback(self, cb: Callable[[dict], None] | None) -> None:
        self._face_detection_progress_cb = cb

    def set_automation_progress_callback(
        self, cb: Callable[[AutomationProgress], None] | None
    ) -> None:
        self._automation_progress_cb = cb

    def _emit_face_detection_progress(self, payload: dict) -> None:
        if self._face_detection_progress_cb is None:
            return
        try:
            self._face_detection_progress_cb(payload)
        except Exception as e:
            logger.warning(f"Failed to emit face detection progress: {e}", exc_info=True)

    def _emit_automation_progress(self, progress: AutomationProgress) -> None:
        if self._automation_progress_cb is None:
            return
        try:
            self._automation_progress_cb(progress)
        except Exception as e:
            logger.warning(f"Failed to emit automation progress: {e}", exc_info=True)

    def _record_best_effort_step_failure(
        self,
        *,
        progress: AutomationProgress,
        step: WorkflowStateKind,
        error: Exception,
        sample: SampleShortInfo | None,
        code: str,
    ) -> None:
        reason = str(error)
        event_context = merge_log_context(
            sample_log_context(sample),
            {
                "step": step.value,
                "code": code,
                "exception_class": error.__class__.__name__,
                "reason": reason,
            },
        )

        self._set_progress_step(progress, step, StepStatus.FAILED, reason)
        progress.append_event(
            level="WARNING",
            code=code,
            exception_class=error.__class__.__name__,
            message=reason,
            sample_id=getattr(sample, "db_id", None),
            context=event_context,
        )
        logger.warning(
            "Best-effort automation step failed; continuing workflow", extra=event_context
        )

    def _new_automation_progress(self) -> AutomationProgress:
        return AutomationProgress(
            current_step=None,
            steps=[
                StepState(step=WorkflowStateKind.MOUNT, status=StepStatus.PENDING),
                StepState(step=WorkflowStateKind.LOOP_CENTRE, status=StepStatus.PENDING),
                StepState(step=WorkflowStateKind.RASTER, status=StepStatus.PENDING),
                StepState(step=WorkflowStateKind.DATA_COLLECTION, status=StepStatus.PENDING),
                StepState(step=WorkflowStateKind.FINAL, status=StepStatus.PENDING),
            ],
            finished=False,
            success=None,
            samples_in_queue=self._automation_samples_in_queue,
            avg_time_per_sample=(
                self._automation_total_sample_time_s / self._automation_completed_samples
                if self._automation_completed_samples > 0
                else 0.0
            ),
            current_sample_name=self._automation_last_sample_name,
        )

    @staticmethod
    def _step_display_name(step: WorkflowStateKind) -> str:
        labels = {
            WorkflowStateKind.MOUNT: "Mount",
            WorkflowStateKind.LOOP_CENTRE: "Center",
            WorkflowStateKind.RASTER: "Raster",
            WorkflowStateKind.DATA_COLLECTION: "Collect",
            WorkflowStateKind.FINAL: "Paused/Finished",
        }
        return labels.get(step, str(step.value))

    def _get_progress_step(
        self, progress: AutomationProgress, step: WorkflowStateKind
    ) -> StepState | None:
        for item in progress.steps:
            if item.step == step:
                return item
        return None

    def _set_progress_context(
        self,
        progress: AutomationProgress,
        *,
        current_sample_name: str | None = None,
        samples_in_queue: int | None = None,
    ) -> None:
        if current_sample_name is not None:
            safe_name = str(current_sample_name or "")
            progress.current_sample_name = safe_name
            self._automation_last_sample_name = safe_name

        if samples_in_queue is not None:
            safe_count = max(0, int(samples_in_queue))
            progress.samples_in_queue = safe_count
            self._automation_samples_in_queue = safe_count
        else:
            progress.samples_in_queue = max(0, int(self._automation_samples_in_queue))

        if self._automation_completed_samples > 0:
            progress.avg_time_per_sample = (
                self._automation_total_sample_time_s / self._automation_completed_samples
            )
        else:
            progress.avg_time_per_sample = 0.0

    def _record_completed_sample_time(self, progress: AutomationProgress, elapsed_s: float) -> None:
        if elapsed_s <= 0:
            return
        self._automation_completed_samples += 1
        self._automation_total_sample_time_s += float(elapsed_s)
        progress.avg_time_per_sample = (
            self._automation_total_sample_time_s / self._automation_completed_samples
        )

    def _set_progress_step(
        self,
        progress: AutomationProgress,
        step: WorkflowStateKind,
        status: StepStatus,
        message: str = "",
        *,
        make_current: bool = False,
        error_code: str | None = None,
    ) -> None:
        now = time.time()

        for item in progress.steps:
            if item.step == step:
                item.status = status
                item.message = message
                item.error_code = error_code

                if status == StepStatus.RUNNING:
                    if item.started_at is None:
                        item.started_at = now
                    item.completed_at = None
                elif status in {
                    StepStatus.SUCCESS,
                    StepStatus.FAILED,
                    StepStatus.SKIPPED,
                    StepStatus.PAUSED,
                }:
                    if item.started_at is None:
                        item.started_at = now
                    item.completed_at = now
                break

        if make_current:
            progress.current_step = self._step_display_name(step)

        self._emit_automation_progress(progress)

    def _mark_progress_running(
        self, progress: AutomationProgress, step: WorkflowStateKind, message: str = ""
    ) -> None:
        self._set_progress_step(progress, step, StepStatus.RUNNING, message, make_current=True)

    def _mark_progress_success(
        self, progress: AutomationProgress, step: WorkflowStateKind, message: str = ""
    ) -> None:
        self._set_progress_step(progress, step, StepStatus.SUCCESS, message)

    def _mark_progress_failed(
        self, progress: AutomationProgress, step: WorkflowStateKind, message: str = ""
    ) -> None:
        self._set_progress_step(
            progress,
            step,
            StepStatus.FAILED,
            message,
            make_current=True,
            error_code=message or None,
        )
        self._set_progress_step(
            progress,
            WorkflowStateKind.FINAL,
            StepStatus.FAILED,
            message,
            error_code=message or None,
        )
        progress.current_step = self._step_display_name(WorkflowStateKind.FINAL)
        progress.finished = True
        progress.success = False
        self._emit_automation_progress(progress)

    def _mark_progress_finished(
        self, progress: AutomationProgress, success: bool, message: str = ""
    ) -> None:
        final_status = StepStatus.SUCCESS if success else StepStatus.FAILED
        self._set_progress_step(progress, WorkflowStateKind.FINAL, final_status, message)
        progress.current_step = self._step_display_name(WorkflowStateKind.FINAL)
        progress.finished = True
        progress.success = success
        self._emit_automation_progress(progress)

    # --------------------------------------------
    #               Operation Services
    # --------------------------------------------

    def _build_runtime_state(self) -> DAQRuntimeState:
        return DAQRuntimeState(
            sample_provider=_DAQSampleProvider(self),
            sample_geometry_provider=_DAQSampleGeometryProvider(self),
            status_provider=_DAQStatusProvider(self),
        )

    def _build_operation_services(self) -> OperationServices:
        return OperationServices(
            screenshots=self._screenshot_service,
            state=StateController(setter=_DAQStateSetter(self)),
            traces=TraceWriter(appender=_DAQTraceAppender(self)),
            events=SampleEventPublisher(sender=_DAQSampleEventSender(self)),
            ingestion=ScanIngestionService(ingestor=_DAQScanIngestor(self)),
            datacollection=DataCollectionPreparer(runner=_DAQDatacollectionSetupRunner(self)),
        )

    def _create_loop_centering_settings(self) -> LoopCenteringSettings:
        return LoopCenteringSettings()

    def _create_loop_centering_service(self) -> LoopCenteringService:
        settings = self._create_loop_centering_settings()
        services = self._build_operation_services()
        services.predictions = PredictionProvider(
            getter=_LoopCenteringPredictionGetter(self, settings)
        )

        return LoopCenteringService(
            context=LoopCenteringContext(
                deps=LoopCenteringDependencies(cfg=self._cfg, devs=self._devs, mlbox=self._mlbox),
                runtime=self._build_runtime_state(),
                services=services,
                settings=settings,
            ),
            logger=logger,
        )

    def _create_face_detection_service(self) -> FaceDetectionService:
        services = self._build_operation_services()
        services.face_detection_progress = FaceDetectionProgressEmitter(
            reporter=_FaceDetectionProgressReporter(self)
        )

        return FaceDetectionService(
            context=FaceDetectionContext(
                deps=FaceDetectionDependencies(cfg=self._cfg, devs=self._devs, mlbox=self._mlbox),
                runtime=self._build_runtime_state(),
                services=services,
                settings=FaceDetectionSettings(),
            ),
            logger=logger,
        )

    def _create_mounting_service(self) -> MountingService:
        return MountingService(
            context=MountingContext(
                deps=MountingDependencies(cfg=self._cfg, devs=self._devs),
                settings=MountingSettings(mount_position=ABR_POS_MOUNT),
            ),
            logger=logger,
        )

    def _create_raster_service(self) -> RasterService:
        return RasterService(
            context=RasterContext(
                deps=RasterDependencies(
                    cfg=self._cfg,
                    devs=self._devs,
                    mlbox=self._mlbox,
                    jfjoch=self._jfjoch,
                    aare=self._aare,
                ),
                runtime=self._build_runtime_state(),
                services=self._build_operation_services(),
                settings=RasterSettings(
                    auto_raster_max_images=self.AUTO_RASTER_MAX_IMAGES,
                    auto_raster_min_cell_size_mm=self.AUTO_RASTER_MIN_CELL_SIZE_MM,
                    auto_raster_skip_if_exceed_max_image_threshold=self.AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD,
                ),
            ),
            logger=logger,
        )

    def _create_rotation_service(self) -> RotationService:
        return RotationService(
            context=RotationContext(
                deps=RotationDependencies(
                    cfg=self._cfg, devs=self._devs, jfjoch=self._jfjoch, aare=self._aare
                ),
                runtime=self._build_runtime_state(),
                services=self._build_operation_services(),
                settings=RotationSettings(),
            ),
            logger=logger,
        )

    # --------------------------------------------
    #               Operation Handlers
    # --------------------------------------------
    # TODO make sure this is implemented currectly
    def _handle_operation_error(
        self,
        operation: DAQOperation,
        sample: SampleShortInfo | None,
        error: Exception,
        event_type: SampleEventType = SampleEventType.FAILED,
        additional_comment: str | None = None,
    ) -> None:
        """
        Centralized databse maessage error handling for all operations.

        Args:
            operation: Name of the operation that failed (e.g., "mount", "alc", "raster")
            sample: The sample being processed
            error: The exception that occurred
            event_type: Type of event to send to database - "mount", "alc", "axc", or "generic"
        """
        sample_name = getattr(sample, "sample_name", None)
        sample_id = getattr(sample, "db_id", None)

        logger.exception(
            "Operation failed",
            extra={
                "operation": operation.value,
                "event_type": str(event_type),
                "sample_name": sample_name,
                "sample_id": sample_id,
                "additional_comment": additional_comment,
            },
        )

        if sample is None or sample_id is None:
            logger.error(f"Error in {operation.value}: {error}")
            return

        comment = f"Error in {operation.value}: {error}"
        if additional_comment is not None:
            comment += f" Additional comment: {additional_comment}"

        try:
            self._aare.send_sample_event(
                sample_id=sample_id, event_type=event_type, comment=comment
            )
        except Exception:
            logger.exception(
                "Failed to report operation error to database",
                extra={
                    "operation": operation.value,
                    "sample_name": sample_name,
                    "sample_id": sample_id,
                },
            )

    # todo make sure these functions are correctly implemented!
    # Operation handlers should handle database communication and beamline state changes,
    # where possible/between operations. For example in raster, we may use XtalSnapshot to take a screenshot, then
    # return to Datacollection.
    # Busy stats are handled by the public function call i.e. sample() or by automation i.e. measure()
    def _execute_mount_and_prepare(self, sample: SampleShortInfo | None) -> bool:
        """
        Operation handler for executing mounting and take screenshot.

        Returns:
            True if successful, False otherwise
        """
        previous_sample = None
        mount_started_at = datetime.now(UTC)
        self._last_mount_error_message = ""

        try:
            previous_sample = self.sample
            if previous_sample is None or previous_sample.db_id is None:
                try:
                    previous_sample = self.sync_current_sample_from_tell(
                        force=True, clear_cached_on_empty=False
                    )
                except Exception as sync_error:
                    logger.warning(
                        f"Failed to reconcile previous sample from TELL before mount: {sync_error}",
                        exc_info=True,
                    )

            if previous_sample is not None and previous_sample.db_id is not None:
                self._aare.send_sample_event(previous_sample.db_id, SampleEventType.UNMOUNTING)

            self._set_state(BeamlineStateEnum.RobotSampleExchange)

            if sample is not None and sample.db_id is not None:
                self._aare.send_sample_event(sample.db_id, SampleEventType.MOUNTING)

            self._devs.tell.blower_on()
            mounting_result: MountingResult = self._create_mounting_service().execute(target=sample)

            if not mounting_result.success:
                raise mounting_result.error or MountingFailed(
                    mounting_result.comment or "Mount failed"
                )

            previous_sample = mounting_result.previous_sample or previous_sample
            mounted_sample = mounting_result.mounted_sample or sample

            if (
                mounting_result.did_unmount_previous
                and previous_sample is not None
                and previous_sample.db_id is not None
            ):
                self._aare.send_sample_event(previous_sample.db_id, SampleEventType.UNMOUNTED)

            self._set_state(BeamlineStateEnum.SampleAlignment)

            if mounted_sample is not None and mounted_sample.db_id is not None:
                self._aare.send_sample_event(mounted_sample.db_id, SampleEventType.MOUNTED)
                self.save_screenshot_db(mounted_sample.db_id, f"{mounted_sample.db_id}_mounted")

            return True

        except TransformationInvalidException as e:
            self._last_mount_error_message = str(e) or "Mount failed"
            logger.error(f"Mount failed due to invalid transformation: {e}")
            self._handle_operation_error(
                operation=DAQOperation.MOUNT if sample is not None else DAQOperation.UNMOUNT,
                sample=sample,
                error=e,
                event_type=SampleEventType.MOUNTFAILED,
            )
            raise

        except TellCommunicationError as e:
            self._last_mount_error_message = str(e) or "Mount failed"
            logger.error(f"Tell communication error occured: {e}")
            self._handle_operation_error(
                operation=DAQOperation.MOUNT if sample is not None else DAQOperation.UNMOUNT,
                sample=sample,
                error=e,
                event_type=SampleEventType.MOUNTFAILED,
            )
            raise

        except Exception as e:
            self._last_mount_error_message = str(e) or "Mount failed"
            logger.exception("Mount failed")

            previous_sample_unmounted = (
                previous_sample is not None
                and previous_sample.db_id is not None
                and self._was_previous_sample_unmounted_since(mount_started_at)
            )

            if previous_sample_unmounted:
                logger.info(
                    "Mount failed after TELL confirmed previous sample was unmounted; "
                    "marking previous sample as unmounted and clearing cached current_sample"
                )
                self._cfg.current_sample = None
                self._aare.send_sample_event(
                    previous_sample.db_id,
                    SampleEventType.UNMOUNTED,
                    comment="Auto-unmount succeeded before mount failed",
                )

            try:
                self._set_state(BeamlineStateEnum.SampleAlignment)
            except Exception:
                logger.exception("Failed to set state to SampleAlignment")
            finally:
                self._handle_operation_error(
                    operation=DAQOperation.MOUNT if sample is not None else DAQOperation.UNMOUNT,
                    sample=sample,
                    error=e,
                    event_type=SampleEventType.MOUNTFAILED,
                )

            return False

    def _execute_loop_centering(self, sample: SampleShortInfo) -> bool:
        """
        Execute loop centering sequence.

        Returns:
            True if successful, False otherwise
        """
        result = None
        additional_comment = None

        if sample is None or sample.db_id is None or sample.db_id < 0:
            logger.error("Loop centering failed: no sample provided")
            return False

        self._set_state(BeamlineStateEnum.SampleAlignment)
        self._devs.lamp_light = 2.5

        try:
            self._aare.send_sample_event(sample.db_id, SampleEventType.CENTERING)

            service = self._create_loop_centering_service()
            result = service.run(sample_id=sample.db_id)

            if not result.success:
                self._handle_operation_error(
                    operation=DAQOperation.LOOP_CENTERING,
                    sample=sample,
                    error=result.error or Exception("Loop centering failed"),
                    event_type=SampleEventType.ALCFAILED,
                    additional_comment=result.comment if result.comment is not None else "",
                )
                return False

            self.save_screenshot_db(sample.db_id, "loop_centering")
            self._aare.send_sample_event(sample.db_id, SampleEventType.CENTERED)
            return True

        except Exception as e:
            logger.exception("Loop centering failed")
            if result is not None and result.error is not None:
                additional_comment = result.comment if result.comment is not None else ""
            self._handle_operation_error(
                operation=DAQOperation.LOOP_CENTERING,
                sample=sample,
                error=e,
                event_type=SampleEventType.ALCFAILED,
                additional_comment=additional_comment,
            )
            return False

    def _execute_face_detection(
        self,
        steps: int = 14,
        step_size: int = 15,
        face_min_ratio: float = 0.3,
        report_error: bool = True,
        sample: SampleShortInfo | None = None,
    ) -> FaceDetectionResult:
        """
        Execute face detection sequence through the face detection service.

        Returns:
            FaceDetectionResult
        """
        self._set_state(BeamlineStateEnum.SampleAlignment)
        result: FaceDetectionResult | None = None

        try:
            if sample is None:
                try:
                    sample = self.sample
                    logger.debug(f"No sample provided, using current sample from DAQ {sample}")
                except Exception:
                    logger.exception("Failed to get current sample")
                    sample = None

            aare = getattr(self, "_aare", None)
            if aare is not None and sample is not None and sample.db_id is not None:
                aare.send_sample_event(sample.db_id, SampleEventType.LOOPFACEDETECTING)

            result = self._create_face_detection_service().run(
                steps=steps, step_size=step_size, face_min_ratio=face_min_ratio
            )

            if not result.success:
                if report_error:
                    self._handle_operation_error(
                        operation=DAQOperation.FACE_CENTERING,
                        sample=sample,
                        error=result.error or Exception("Face detection failed"),
                        event_type=SampleEventType.LOOPFACEDETECTFAILED,
                        additional_comment=result.comment,
                    )
                return result

            if aare is not None:
                aare.send_sample_event(sample.db_id, SampleEventType.LOOPFACEDETECTED)
            return result

        except Exception as e:
            logger.exception("Face detection failed")
            additional_comment = f"{e}" if e is not None else ""

            try:
                sample = self.sample
            except Exception:
                logger.debug("Could not read the current sample for face detection", exc_info=True)
                sample = None

            if report_error:
                self._handle_operation_error(
                    operation=DAQOperation.FACE_CENTERING,
                    sample=sample,
                    error=e,
                    event_type=SampleEventType.LOOPFACEDETECTFAILED,
                    additional_comment=additional_comment,
                )

            payload = (
                result.payload
                if result is not None
                else {"running": False, "samples": [], "height_fit": {}, "area_fit": {}}
            )
            return FaceDetectionResult(
                success=False, payload=payload, error=e, comment=additional_comment
            )

    def _execute_raster_sequence(
        self, grid_request: RasterGridRequest, auto_center: bool = False
    ) -> CompletedRasterGrid | None:
        """
        Execute raster scan with optional auto-centering.

        Args:
            grid_request: Raster grid parameters
            auto_center: If True, use auto-centering; if False, use direct raster

        Returns:
            CompletedRasterGrid result or None if failed
        """
        try:
            logger.info(
                "Starting raster sequence",
                extra=merge_log_context(
                    sample_log_context(self.sample),
                    raster_request_log_context(grid_request),
                    {"auto_center": auto_center},
                ),
            )

            raster_service = self._create_raster_service()

            if auto_center:
                setup_request = copy.deepcopy(grid_request)
                setup_request.smargon_top_left = None
                self._setup_datacollection(request=setup_request)
                result = raster_service.execute_auto_center(grid_request)
                if result is None:
                    logger.error(
                        "Raster sequence returned no result after auto-centering",
                        extra=merge_log_context(
                            sample_log_context(self.sample),
                            raster_request_log_context(grid_request),
                        ),
                    )
            else:
                # Resolve + validate the requested dtz (sets cfg.dtz, requires
                # request.dtz) BEFORE configuring JFJoch, so JFJoch receives the
                # commanded target rather than the live, still-in-transit position.
                self._setup_datacollection(request=grid_request)
                logger.debug(f"Is detector simulated? {self._cfg.simulated_detector}")
                if not self._cfg.simulated_detector:
                    logger.info("initialise detector for raster")
                    status = self.status
                    self._jfjoch.measure_raster(grid_request, status)
                    logger.info("detector initialised")
                else:
                    logger.info("Simulated detector mode enabled; using fake raster result.")

                self._set_state(BeamlineStateEnum.DataCollection)
                raster_result = raster_service.execute(grid_request)
                result = CompletedRasterGrid(r=[raster_result])
                self._set_state(BeamlineStateEnum.SampleAlignment)

            if result is not None:
                logger.info(
                    "Raster sequence completed",
                    extra=merge_log_context(
                        sample_log_context(self.sample),
                        raster_request_log_context(grid_request),
                        {
                            "auto_center": auto_center,
                            "result_count": len(result.r)
                            if hasattr(result, "r") and result.r is not None
                            else None,
                        },
                    ),
                )
            else:
                if self.sample is not None and self.sample.db_id is not None:
                    self._aare.send_sample_event(self.sample.db_id, SampleEventType.RASTERINGFAILED)
                logger.error(
                    "Raster sequence returned no result",
                    extra=merge_log_context(
                        sample_log_context(self.sample), raster_request_log_context(grid_request)
                    ),
                )
            return result

        except JFJochCommunicationError as e:
            logger.exception(
                "Raster sequence failed due to JFJoch communication error",
                extra=merge_log_context(
                    sample_log_context(self.sample), raster_request_log_context(grid_request)
                ),
            )
            self._handle_operation_error(
                operation=DAQOperation.RASTER,
                sample=self.sample,
                error=e,
                event_type=SampleEventType.RASTERINGFAILED,
                additional_comment=f"JFJoch communication error: {e}",
            )
            raise

        except BeamlineBusyTimeoutException as e:
            logger.exception(
                "Raster sequence failed because the beamline busy state timed out",
                extra=merge_log_context(
                    sample_log_context(self.sample),
                    raster_request_log_context(grid_request),
                    {"auto_center": auto_center},
                ),
            )
            self._handle_operation_error(
                operation=DAQOperation.RASTER,
                sample=self.sample,
                error=e,
                event_type=SampleEventType.RASTERINGFAILED,
                additional_comment=(
                    "Beamline busy timeout during raster sequence. "
                    "The operation exceeded the configured busy-state TTL."
                ),
            )
            raise

        except AutoRasterSampleSkipped:
            raise

        except Exception as e:
            logger.exception(
                "Raster sequence failed",
                extra=merge_log_context(
                    sample_log_context(self.sample),
                    raster_request_log_context(grid_request),
                    {"auto_center": auto_center},
                ),
            )
            self._handle_operation_error(
                operation=DAQOperation.RASTER,
                sample=self.sample,
                error=e,
                event_type=SampleEventType.RASTERINGFAILED,
                additional_comment=f"Raster sequence failed: {e}",
            )
            return None

    def _execute_rotation_sequence(
        self, rotation_request: RotationScanRequest
    ) -> CompletedRotationScan | None:
        """
        Execute rotation scan.

        Args:
            rotation_request: Rotation scan parameters

        Returns:
            CompletedRotationScan result or None if failed
        """
        min_exp_time = 0.0011
        if rotation_request.exp_time_s < min_exp_time:
            logger.warning(f"Exposure shorter than default of {min_exp_time} s! Adjusting.")
            rotation_request.exp_time_s = min_exp_time
        try:
            return self._create_rotation_service().run(rotation_request)
        except JFJochCommunicationError as e:
            logger.error(f"Rotation sequence failed due to JFJoch Communication error: {e}")
            self._handle_operation_error(
                operation=DAQOperation.ROTATION,
                sample=self.sample,
                error=e,
                event_type=SampleEventType.COLLECTIONFAILED,
                additional_comment=f"JFJoch communication error: {e}",
            )
            raise
        except Exception as e:
            logger.error(f"Rotation sequence failed: {e}")
            self._handle_operation_error(
                operation=DAQOperation.ROTATION,
                sample=self.sample,
                error=e,
                event_type=SampleEventType.COLLECTIONFAILED,
            )
            raise

    def _append_smargon_trace(self, *, sample_id: int | None, event: str) -> None:
        try:
            path = self._smargon_trace_path
            path.parent.mkdir(parents=True, exist_ok=True)

            is_new_file = not path.exists() or path.stat().st_size == 0
            pos = self.smargon
            sh = pos.sh_mm

            with path.open("a", encoding="utf-8", buffering=1) as f:
                if is_new_file:
                    f.write(
                        "timestamp,event,sample_id,omega_deg,zoom,"
                        "shx_mm,shy_mm,shz_mm,phi_deg,chi_deg\n"
                    )

                f.write(
                    f"{datetime.now().isoformat(timespec='milliseconds')},"
                    f"{event},"
                    f"{'' if sample_id is None else sample_id},"
                    f"{self.omega:.3f},"
                    f"{self.zoom:.3f},"
                    f"{sh.x:.5f},"
                    f"{sh.y:.5f},"
                    f"{sh.z:.5f},"
                    f"{pos.phi_deg:.5f},"
                    f"{pos.chi_deg:.5f}\n"
                )
                f.flush()

        except Exception as e:
            logger.warning(f"Failed to append smargon trace: {e}", exc_info=True)

    def _sample_matches_mounted_address(
        self, sample: SampleShortInfo | None, mounted_address
    ) -> bool:
        if sample is None or sample.location is None or mounted_address is None:
            return False
        return (
            sample.location.segment == mounted_address.puck.segment
            and sample.location.pos == mounted_address.puck.pos
            and sample.pin == mounted_address.pin
        )

    def _find_sample_by_mounted_address(self, mounted_address) -> SampleShortInfo | None:
        for sample in self._cfg.spreadsheet.s:
            if self._sample_matches_mounted_address(sample, mounted_address):
                return sample

        for sample in self._cfg.reference_tools.s:
            if self._sample_matches_mounted_address(sample, mounted_address):
                return sample

        return None

    def _placeholder_sample_from_mounted_address(self, mounted_address) -> SampleShortInfo:
        return SampleShortInfo(
            db_id=-1,
            puck_name="",
            dewar_name="",
            sample_name=f"Mounted sample {mounted_address.puck.segment}{mounted_address.puck.pos}-{mounted_address.pin}",
            run_number=0,
            user="",
            pin=mounted_address.pin,
            location=mounted_address.puck,
        )

    @staticmethod
    def _sample_mount_display_name(sample: SampleShortInfo | None) -> str:
        if sample is None:
            return "current sample"

        sample_name = str(getattr(sample, "sample_name", "") or "").strip()
        if sample_name:
            return sample_name

        return "sample"

    def sync_current_sample_from_tell(
        self, force: bool = False, clear_cached_on_empty: bool = True
    ) -> SampleShortInfo | None:
        current_sample = self._cfg.current_sample

        if current_sample is not None and current_sample.location is None:
            return current_sample

        now = time.monotonic()
        if not force and (now - self._last_sample_sync_ts) < self._sample_sync_min_interval_s:
            return current_sample

        self._last_sample_sync_ts = now
        mounted_address = self._devs.tell.get_mounted_sample()

        if mounted_address is None:
            if current_sample is not None and current_sample.location is not None:
                if clear_cached_on_empty:
                    logger.warning("TELL reports no mounted sample; clearing cached current_sample")
                    self._cfg.current_sample = None
                else:
                    logger.warning(
                        "TELL reports no mounted sample; keeping cached current_sample to avoid losing context"
                    )
            return self._cfg.current_sample

        if self._sample_matches_mounted_address(current_sample, mounted_address):
            return current_sample

        resolved_sample = self._find_sample_by_mounted_address(mounted_address)
        if resolved_sample is None:
            resolved_sample = self._placeholder_sample_from_mounted_address(mounted_address)
            logger.warning(
                "Mounted sample from TELL was not found in known sample lists; using placeholder",
                extra={"mounted_address": str(mounted_address)},
            )
        else:
            logger.info(
                f"Reconciled cached sample from TELL to {resolved_sample.sample_name}",
                extra={"db_id": resolved_sample.db_id},
            )

        self._cfg.current_sample = resolved_sample
        return resolved_sample

    @property
    def state(self) -> BeamlineStateEnum:
        return self._cfg.state

    @property
    def busy(self) -> bool:
        return self._cfg.state_busy

    @state.setter
    def state(self, target: BeamlineStateEnum):
        if target == BeamlineStateEnum.Moving:
            logger.error(
                "Cannot explicitly move to busy state",
                extra={"target": target, "state": self._cfg.state},
            )
            raise RuntimeError("Cannot explicitly move to busy state")

        start = time.perf_counter()
        self._cfg.try_set_busy(timeout=300)
        try:
            self._set_state(target)
        finally:
            self._cfg.state_busy = False

        end = time.perf_counter()

        self.last_time = end - start

    def _expand_macros(self, name: str) -> str:
        name = name.replace("{date}", datetime.now().strftime("%Y%m%d"))
        name = name.replace("{sample}", self.sample.sample_name)
        name = name.replace("{CrystalName}", self.sample.sample_name)
        name = name.replace("{puck}", self.sample.puck_name)
        name = name.replace("{position}", f"{self.sample.pin:02d}")
        name = name.replace("{sample_id}", f"{self.sample.db_id}")
        name = name.replace("{beamline}", f"{self._beamline.value.lower()}")
        name = name.replace(
            "{prefix}", f"{self.sample.puck_name}/{self.sample.pin:02d}/{self.sample.sample_name}"
        )
        # TODO work out why clean_filename is removing slahses
        # name = clean_filename(name)
        return name

    def spreadsheet_params(self) -> tuple[SimpleScanParameters | None, str | None]:
        file_prefix = None

        if self.status.sample is None:
            return None, file_prefix

        logger.debug(f"generate params for sample: {self.status.sample}")
        aaredb_params = (
            self.status.sample.aaredb_params
            if hasattr(self.status.sample, "aaredb_params")
            else None
        )
        if aaredb_params is None:
            return None, file_prefix

        if (
            getattr(aaredb_params, "exposure", None) is None
            and getattr(aaredb_params, "transmission", None) is None
            and getattr(aaredb_params, "oscillation", None) is None
            and getattr(aaredb_params, "totalrange", None) is None
            and getattr(aaredb_params, "targetresolution", None) is None
        ):
            return None, file_prefix

        params = SimpleScanParameters()

        if (exp := getattr(aaredb_params, "exposure", None)) is not None:
            params.exp_time_s = exp

        if (trans := getattr(aaredb_params, "transmission", None)) is not None:
            logger.debug(f"transmission: {trans}")
            params.transmission = trans / 100.0 if trans > 1.0 else trans

        if (res := getattr(aaredb_params, "targetresolution", None)) is not None:
            logger.debug(f"resolution: {res}")
            logger.debug(f"requested dtz: {self.diffraction_geometry.calc_dtz_mm(res)} ")
            new_res = 1 / ((1 / res) + 0.1)
            corrected_dtz = self.diffraction_geometry.calc_dtz_mm(new_res)
            logger.debug(f"corrected dtz: {corrected_dtz}")
            corrected_dtz = max(corrected_dtz, 108)
            params.dtz = round(corrected_dtz)

        osc = getattr(aaredb_params, "oscillation", None)
        total = getattr(aaredb_params, "totalrange", None)

        if osc is not None:
            osc = abs(osc)
            if osc > 0:
                params.incr_omega_deg = osc
                if total is not None and abs(total) > 0:
                    params.steps = round(abs(total) / osc)
                else:
                    params.steps = round(360.0 / osc)
        elif total is not None and abs(total) > 0:
            default_osc = SimpleScanParameters().incr_omega_deg
            params.incr_omega_deg = default_osc
            params.steps = round(abs(total) / default_osc)

        return params, file_prefix

    @property
    def omega(self) -> float:
        return self._devs.aerotech_omega

    def _omega(self, val: float):
        self._saved_box = None
        logger.info(f"Set omega to {val}")
        if -2000 < val < 2000:
            try:
                self._devs.aerotech_omega = val
                self._cfg.state_busy = False
            except Exception:
                logger.exception("Omega move timed out")
                self._cfg.state_busy = False
        else:
            self._cfg.state_busy = False
            logger.error("Omega has to be between -2000 and 2000 degrees")
            raise ValueError("Omega has to be between -2000 and 2000 degrees (for now)")

    @omega.setter
    def omega(self, val: float):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        self._omega(val)

    def omega_rel(self, val: float):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        curr_omega = self._devs.aerotech_omega
        self._omega(curr_omega + val)

    @property
    def zoom(self) -> float:
        return self._devs.zoom

    @zoom.setter
    def zoom(self, val: float):
        self._saved_box = None
        # Decide off the beamline STATE (persisted in Redis) so it is robust to
        # how/when beam-location was entered, rather than a separate flag that
        # must be set exactly on the state transition.
        beam_location = self._cfg.state == BeamlineStateEnum.BeamLocation
        logger.debug(
            f"zoom -> {val} (state={self._cfg.state}, beam_location_presets={beam_location})"
        )
        if beam_location:
            # Auto-exposure is too dark to see the beam at high zoom, so apply
            # the preset per-zoom gain/exposure (interpolated between the stored
            # zoom stops). Every other state keeps auto-exposure.
            if self._cfg.zoom_mode != ZoomModeEnum.BeamLocation:
                self._cfg.zoom_mode = ZoomModeEnum.BeamLocation
            self._devs.zoom = val
            time.sleep(0.2)
            settings = self._cfg.zoom_settings.get_camera_settings(val)
            self._devs.samcam_settings = settings
            logger.debug(
                f"applied beam-location preset for zoom {val}: gain={settings.gain}, exposure={settings.exposure}"
            )
        else:
            self._devs.samcam_auto(AutoEnum.AUTO)
            self._devs.zoom = val
            time.sleep(0.2)
            self._devs.samcam_auto(AutoEnum.ONCE)

    def save_beam_location_camera_setting(self) -> None:
        """Persist the camera's current gain/exposure for the current zoom as
        the beam-location preset (Redis), so it is re-applied on future zooms."""
        zoom_value = self.zoom
        settings = self.samcam_settings
        self._cfg.save_zoom_camera_setting(zoom_value, settings, mode=ZoomModeEnum.BeamLocation)
        logger.info(
            f"Saved beam-location camera setting for zoom {zoom_value}: "
            f"gain={settings.gain}, exposure={settings.exposure}"
        )

    @property
    def front_light(self) -> float:
        val = self._devs.lamp_light
        if val <= 1.0:
            return 0
        elif val >= 2.5:
            return 100.0
        else:
            return (val - 1.0) / 1.5 * 100.0

    @front_light.setter
    def front_light(self, f: float):
        conv = (f / 100.0 * 1.5) + 1.0
        logger.info(f"Light {f} -> {conv}")
        self._devs.lamp_light = conv

    @property
    def back_light(self) -> float:
        val = self._devs.back_light
        if val <= 0.0:
            return 0.0
        elif val <= 3.0:
            return val / 3.0 * 100.0
        else:
            return 100.0

    @back_light.setter
    def back_light(self, f: float):
        conv = f / 100.0 * 3.0
        logger.info(f"Backlight {f} -> {conv}")
        self._devs.back_light = conv

    @property
    def sample(self) -> SampleShortInfo | None:
        return self._cfg.current_sample

    @property
    def samcam_settings(self) -> SampleCameraSettings:
        return self._devs.samcam_settings

    @samcam_settings.setter
    def samcam_settings(self, s: SampleCameraSettings):
        self._devs.samcam_settings = s

    def tweak_abr_meas_pos(self, c: AerotechCoordinate):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            new_meas_pos = AerotechCoordinate(at_mm=self._cfg.abr_meas_pos.at_mm + c.at_mm)
            self._cfg.abr_meas_pos = new_meas_pos
            self._devs.aerotech_pos = new_meas_pos
            self._saved_box = None
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def save_abr_meas_pos(self):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            self._cfg.abr_meas_pos = AerotechCoordinate(at_mm=self._devs.aerotech_pos.at_mm)
            self._devs.bec_worker.save_current_aerotech_position()
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def goto_abr_meas_pos(self):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            self._devs.aerotech_pos = self._cfg.abr_meas_pos
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def create_sample(self, target: SampleShortInfo):

        self._cfg.try_set_busy(timeout=360)

        try:
            curr_sample = self._cfg.current_sample

            if curr_sample is not None and curr_sample.location is not None:
                raise RuntimeError("Sample from TELL is loaded")

            self._aare.create_manual_sample(target)
            self._cfg.current_sample = target
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def check_tell_mount_start_conditions(self) -> None:
        self._devs.tell.validate_mount_start_conditions()

    def _execute_dry(self, park: bool = True, unmount: bool = False):
        self._create_mounting_service().dry(park=park, unmount=unmount)

    def park_and_dry(self, park=True, unmount: bool = False):
        self._cfg.try_set_busy(timeout=360)
        try:
            self._set_state(BeamlineStateEnum.RobotSampleExchange)
        except TransformationInvalidException as e:
            logger.error(f"Failed to go to robot sample exchange: {e}")
            logger.warning("trying to day and park without unmounting first")
        try:
            self._execute_dry(park=park, unmount=unmount)
            self._cfg.state_busy = False
        except Exception as e:
            self._cfg.state_busy = False
            logger.error(f"Failed to park and dry: {e}")
            raise

    def tell_toggle_blower(self):
        try:
            self._devs.tell.toggle_blower()
        except Exception:
            logger.exception("Failed to turn off blower")

    def initialise_smargon(self):
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.smargon_initialize()
            self._cfg.state_busy = False
        except Exception as e:
            self._cfg.state_busy = False
            logger.error(f"Failed to initialise Smargon: {e}")
            raise

    def initialise_detector(self):
        self._cfg.try_set_busy(timeout=360)
        try:
            self._jfjoch.initialize()
            self._cfg.state_busy = False
        except Exception as e:
            self._cfg.state_busy = False
            logger.error(f"Failed to initialise detector: {e}")
            raise

    def recovery_unmount_sample(self) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._set_state(BeamlineStateEnum.RobotSampleExchange)
            self._devs.tell.check_enable_motion()
            self._devs.tell.wait_not_busy()
            self._devs.tell.set_in_mount_position(True)
            self._devs.tell.unmount(wait=True, timeout=360.0)
            self._cfg.current_sample = None
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    @sample.setter
    def sample(self, target: SampleShortInfo | None):
        self._cfg.try_set_busy(timeout=360)
        try:
            logger.debug(f"Mount target {target}")

            if target is None:
                logger.debug("Unmounting sample")
                operation_name = "Unmount"
            else:
                logger.debug(f"Mounting sample: {target}")
                operation_name = "Mount"

            if not self._execute_mount_and_prepare(target):
                current_sample = target or self._cfg.current_sample
                sample_name = self._sample_mount_display_name(current_sample)
                if target is None:
                    raise UnmountingFailed(f"Failed to {operation_name.lower()} {sample_name}")
                raise MountingFailed(f"Failed to {operation_name.lower()} {sample_name}")

            logger.info(f"Sample operation completed: {target}")
            self._cfg.state_busy = False

        except Exception as e:
            self._cfg.state_busy = False
            logger.debug(f"Failed to change mounted sample: {e}")
            raise

    def list_loaded_pucks(self) -> list[PuckLoadedInfo]:
        return []

    def _auto_focus(self, settings: AutofocusSettings, settle_time_s: float = 1.0) -> float:
        # TODO uses old code change
        """
        Scan smargon Z and find the position with maximum focus measure.

        Args:
            settings: AutofocusSettings with center, radius, range, and steps
            settle_time_s: Time to wait after each move before capturing image

        Returns:
            Best Z position (mm) found during the scan
        """
        raise NotImplementedError("Autofocus not implemented yet")

    def auto_focus(self, settings: AutofocusSettings) -> float:
        """
        Public autofocus method. Only allowed in SampleAlignment state.

        Args:
            settings: AutofocusSettings with center, radius, range, and steps

        Returns:
            Best Z position (mm) found during the scan
        """
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            best_z = self._auto_focus(settings)
            self._cfg.state_busy = False
            return best_z
        except Exception as e:
            logger.error(f"Autofocus failed: {e}")
            self._cfg.state_busy = False
            raise

    def auto_exposure(self):
        self._devs.samcam_auto(AutoEnum.ONCE)

    def _setup_datacollection(
        self, request: RasterGridRequest | RotationScanRequest, screening: bool = False
    ):
        request_omega = getattr(request, "omega_deg", None)
        if request_omega is None:
            request_omega = getattr(request, "start_omega_deg", None)

        if request_omega is not None:
            logger.info(f"requesting omega to move to {request_omega}")
            self._devs.aerotech_omega = request_omega

        if request.dtz is not None:
            if request.dtz < self._cfg.cached_dtz_low:
                raise ValueError(
                    f"Requested DTZ {request.dtz} is less than low detector limit {self._cfg.cached_dtz_low}"
                )
            elif request.dtz > self._cfg.cached_dtz_high:
                raise ValueError(
                    f"Requested DTZ {request.dtz} exceeds high detector limit {self._cfg.cached_dtz_high}"
                )
            logger.info(f"requesting dtz to move to {request.dtz}")
            self._cfg.dtz = request.dtz
        else:
            raise ValueError("No DTZ specified")

        if self.sample is not None and self.sample.db_id is not None:
            sample_id = self.sample.db_id
            if screening:
                screenshot_name = f"{sample_id}_before_screening"
            elif type(request) is RasterGridRequest:
                screenshot_name = f"{sample_id}_before_raster"
            else:
                screenshot_name = f"{sample_id}_before_data_collection"
            self.save_screenshot_db(sample_id, screenshot_name)

        if request.transmission is not None and request.transmission != self._devs.transmission:
            logger.info(f"requesting transmission to move to {request.transmission}")
            self._devs.transmission = request.transmission

        start_pos = getattr(request, "start", None)
        if start_pos is not None:
            logger.info(f"requesting smargon to move to {start_pos}")
            self._devs.smargon_pos = start_pos
        else:
            smargon_top_left = getattr(request, "smargon_top_left", None)
            has_valid_smargon_target = (
                smargon_top_left is not None
                and getattr(smargon_top_left, "sh_mm", None) is not None
            )

            is_zero_placeholder = (
                has_valid_smargon_target
                and abs(smargon_top_left.sh_mm.x) < 1e-6
                and abs(smargon_top_left.sh_mm.y) < 1e-6
                and abs(smargon_top_left.sh_mm.z) < 1e-6
                and abs(smargon_top_left.phi_deg) < 1e-6
                and abs(smargon_top_left.chi_deg) < 1e-6
            )

            if has_valid_smargon_target and not is_zero_placeholder:
                logger.info(f"requesting smargon to move to {smargon_top_left}")
                self._devs.set_smargon_pos(
                    SmargonCoordinate(
                        sh_mm=smargon_top_left.sh_mm,
                        phi_deg=smargon_top_left.phi_deg,
                        chi_deg=smargon_top_left.chi_deg,
                    )
                )
            else:
                logger.debug(
                    "No explicit smargon target in request; skipping pre-datacollection smargon move"
                )

        self._devs.smargon_wait(timeout=180)

    def _build_fake_rotation_result(self, request: RotationScanRequest) -> CompletedRotationScan:
        start_angle = 0.0
        try:
            start_angle = float(self.omega)
        except Exception:
            logger.debug("Could not read omega for the fake rotation result", exc_info=True)

        return build_fake_rotation_result(request, start_angle=start_angle)

    def measure_raster(self, request: RasterGridRequest, auto_center: bool) -> CompletedRasterGrid:
        """
        Execute a raster scan.

        Args:
            request: RasterGridRequest parameters for the scan.
            auto_center: Boolean flag indicating if this is part of an automated sequence.

        Returns:
            CompletedRasterGrid result.
        """
        logger.info(
            "Received raster scan request",
            extra=merge_log_context(
                sample_log_context(self.sample), raster_request_log_context(request)
            ),
        )
        self._cfg.try_set_busy(timeout=ceil(360))
        try:
            result = self._execute_raster_sequence(request, auto_center=auto_center)

            if result is None:
                raise RasterScanException("Raster scan failed")
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
            return result
        except Exception:
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
            raise

    def _rotation(self, request: RotationScanRequest) -> CompletedRotationScan:
        omega_start = self.omega
        status = self.status

        if request.exp_time_s < 0.004:
            logger.warning("Exposure time too short for PXII rotation scan")
            request.exp_time_s = 0.004
        total_time = request.exp_time_s * request.steps
        if self.sample is not None and self.sample.db_id is not None:
            self._aare.create_rotation_run(self.sample, request, status)

        if not self._cfg.simulated_detector:
            try:
                self._jfjoch.wait_till_running(timeout=60.0)
            except Exception as e:
                self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_running")
                raise
        try:
            if request.screening:
                self._devs.aerotech.screening_scan(
                    rotation_deg=request.steps * request.incr_omega_deg,
                    wedge_deg=request.wedge_omega_deg,
                    time_sec=total_time,
                    steps=request.steps,
                    run_async=True,
                )
            else:
                self._devs.aerotech.rotation_scan(
                    rotation_deg=request.steps * request.incr_omega_deg,
                    time_sec=total_time,
                    start_pos_deg=request.start_omega_deg,
                    run_async=True,
                )

                # Is this for helical scans...? do we do smargon scans?
                if request.start is not None and request.end is not None:
                    smargon_time_step = request.exp_time_s / float(request.steps)
                    pos_step = (request.end.sh_mm - request.start.sh_mm) * (
                        1.0 / float(request.steps)
                    )

                    for i in range(request.steps):
                        self._devs.smargon.target = SmargonCoordinate(
                            sh_mm=request.start.sh_mm + pos_step * i
                        )
                        time.sleep(smargon_time_step)

            self._devs.aerotech.wait_till_done(timeout=int(round(total_time + 60, 0)))
            self._devs.aerotech_omega = omega_start

            if self._cfg.simulated_detector:
                logger.warning("Detector in simulation mode, returning fake zero rotation result.")
                return self._build_fake_rotation_result(request)
            else:
                try:
                    scan_result = self._jfjoch.wait_till_done(60)
                except Exception as e:
                    self._raise_if_critical_jfjoch_detector_error(e, command="wait_till_done")
                    raise
                return CompletedRotationScan(request=copy.deepcopy(request), result=scan_result)

        except JFJochCommunicationError as e:
            logger.error(f"Exception during rotation scan related to JFJoch: {e}")
            raise
        except Exception as e:
            logger.error(f"Exception during rotation scan: {e}")
            raise

    def measure_rotation(self, request: RotationScanRequest) -> CompletedRotationScan:
        """
        Execute a rotation scan.

        Args:
            request: RotationScanRequest parameters.

        Returns:
            CompletedRotationScan result.
        """
        total_time = request.exp_time_s * request.steps
        logger.info(
            "Received rotation scan request",
            extra=merge_log_context(
                sample_log_context(self.sample),
                rotation_request_log_context(request, total_time_s=total_time),
            ),
        )
        self._cfg.try_set_busy(timeout=ceil(total_time + 360))

        try:
            result = self._execute_rotation_sequence(request)

            if result is None:
                logger.error("Rotation scan failed, no result returned")
                raise DataCollectionException("Rotation scan failed, no result returned")
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
            return result
        finally:
            try:
                if self._cfg.state_busy and self._cfg.state != BeamlineStateEnum.Maintenance:
                    self._set_state(BeamlineStateEnum.SampleAlignment)
            except Exception:
                logger.exception("Failed to restore SampleAlignment after rotation")
            finally:
                self._cfg.state_busy = False

    @property
    def dtz(self) -> float:
        tmp = self._cfg.dtz
        if tmp is None:
            return cfg_get("daq.data_collection_settings.default_raster_scan_settings.dtz", 200)
        else:
            return tmp

    @dtz.setter
    def dtz(self, val: float):
        self._cfg.try_set_busy(timeout=360)
        state = self._cfg.state

        dtz_low = self._cfg.cached_dtz_low
        dtz_high = self._cfg.cached_dtz_high

        if dtz_low is None or dtz_high is None:
            logger.warning("DTZ limits not found in cache, refreshing hardware metadata")
            try:
                self.refresh_detector_metadata_cache()
            except Exception as e:
                self._cfg.state_busy = False
                raise RuntimeError(f"DTZ limits unavailable and refresh failed: {e}") from e
            dtz_low = self._cfg.cached_dtz_low
            dtz_high = self._cfg.cached_dtz_high

        if dtz_low is None or dtz_high is None:
            self._cfg.state_busy = False
            raise RuntimeError("DTZ limits are unavailable")

        if val < dtz_low or val > dtz_high:
            self._cfg.state_busy = False
            raise RuntimeError(f"dtz={val} outside limits {dtz_low} to {dtz_high}")

        if state == BeamlineStateEnum.DataCollection:
            self._cfg.state_busy = False
            raise RuntimeError("Cannot set dtz during data collection")
        elif state == BeamlineStateEnum.SampleAlignment:
            self._devs.set_dtz(val, wait=False)

        self._cfg.dtz = val
        self._cfg.state_busy = False

    @property
    def smargon(self) -> SmargonCoordinate:
        return self._devs.smargon_pos

    @smargon.setter
    def smargon(self, sc: SmargonCoordinate):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            self._saved_box = None
            self._devs.smargon_pos = sc
            self._devs.smargon_wait()
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def mark_beam(self, x_pxl: float, y_pxl: float):
        self._cfg.set_busy(BeamlineStateEnum.BeamLocation)
        try:
            self._cfg.mark_beam(x_pxl, y_pxl, self._devs.zoom)
            self._cfg.state_busy = False
        except Exception:
            self._cfg.state_busy = False
            raise

    def clear_mark_beam(self):
        self._cfg.set_busy(BeamlineStateEnum.BeamLocation)
        self._cfg.clear_mark_beam()
        self._cfg.state_busy = False

    @property
    def sample_geometry(self) -> SampleGeometryModel:
        zoom = self._devs.zoom
        aerotech_pos_ref = self._cfg.abr_meas_pos.at_mm
        aerotech_pos = self._devs.aerotech_pos.at_mm
        sample_geom = SampleGeometryModel(
            beam_location_pxl=self._cfg.beam_mark_coeff.apply(zoom),
            pixel_in_mm=self._cfg.pixel_to_mm(zoom),
            omega_deg=self._devs.aerotech_omega,
            smargon=self._devs.smargon_pos,
            beam_size_mm=self._cfg.beam_size_mm,
            aerotech=aerotech_pos - aerotech_pos_ref,
            aerotech_meas=aerotech_pos,
        )
        return sample_geom

    @property
    def beam_center(self) -> tuple[float, float]:
        return self._cfg.beam_center

    @beam_center.setter
    def beam_center(self, val: tuple[float, float]):
        self._cfg.beam_center = val

    @property
    def beam_size_mm(self) -> Coordinate:
        return self._cfg.beam_size_mm

    @beam_size_mm.setter
    def beam_size_mm(self, val: Coordinate):
        self._cfg.beam_size_mm = val

    def get_beam_mark(self):
        return self._cfg.get_beam_mark(self._devs.zoom)

    def ml_bounding_box(
        self, sample_id: int | None = None, filename: str | None = None
    ) -> RasterGridRequest | None:
        """
        Request an ML-based bounding box for the sample.

        Args:
            sample_id: Optional sample ID.
            filename: Optional filename for saving/logging.

        Returns:
            RasterGridRequest object representing the found bounding box, or None if failed.
        """
        try:
            self._cfg.try_set_busy(timeout=360)
            r = get_ml_bounding_box(
                mlbox=self._mlbox,
                sample=self.sample,
                sample_geometry=self.sample_geometry,
                filename=filename,
                upload_image=self._aare.upload_image,
                logger=logger,
                max_images=self.AUTO_RASTER_MAX_IMAGES,
                min_cell_size_mm=self.AUTO_RASTER_MAX_IMAGES,
                skip_if_exceed_max_image_threshold=self.AUTO_RASTER_SKIP_IF_EXCEED_MAX_IMAGE_THRESHOLD,
            )
            self._cfg.state_busy = False
            return r
        except Exception:
            self._cfg.state_busy = False
            raise

    def face_detection(
        self, steps: int = 14, step_size: int = 15, face_min_ratio: float = 0.3
    ) -> dict:
        """
        Perform a face detection sequence by rotating the sample and using ML to find the flat face.

        Args:
            steps: Number of rotation steps. Default is 14.
            step_size: Size of each rotation step in degrees. Default is 15.
            face_min_ratio: Minimum ratio of loopface count to loop_all count to use loop_face over loop_all.

        Returns:
            Dictionary containing face detection results, including found samples and fits.
        """
        self._cfg.try_set_busy(timeout=360)
        try:
            result = self._execute_face_detection(
                steps=steps, step_size=step_size, face_min_ratio=face_min_ratio, report_error=True
            )
            return result.payload
        finally:
            self._cfg.state_busy = False

    @log_timing(logger, "Auto loop center")
    def auto_loop_center(self, sample: SampleShortInfo | None = None) -> float:
        """
        Automatically center the loop using ML-based detection.
        This performs a multi-step sequence including rotation and centering.

        Args:
            sample: the current SampleShortInfo. If None, the current sample (self.sample) is used.

        Returns:
            Time taken for the centering process (seconds)

        Raises:
            LoopCenteringFailed: If the centering sequence fails
        """
        start = time.perf_counter()
        try:
            self._cfg.try_set_busy(timeout=360)
            if sample is None:
                if self.sample is None:
                    raise LoopCenteringFailed(
                        "No sample available for loop centering."
                        "If you have mounted a manual sample,"
                        "please add it in the manual sample panel."
                    )
                sample = self.sample

            if not self._execute_loop_centering(sample):
                raise LoopCenteringFailed

            self._cfg.state_busy = False

        except Exception:
            self._cfg.zoom_mode = ZoomModeEnum.User
            self._cfg.state_busy = False
            raise

        finally:
            self._cfg.zoom_mode = ZoomModeEnum.User

        end = time.perf_counter()
        return end - start

    def _default_screenshot_message(self, sample_id: int) -> str:
        omega_value = self.omega
        zoom_value = self.zoom
        samcam = self.samcam_settings
        return (
            f"sample_id: {sample_id} "
            f"zoom: {zoom_value} "
            f"exp:{samcam.exposure} "
            f"gain:{samcam.gain} "
            f"omega:{omega_value:.2f}"
        )

    def save_screenshot(self, filename: str, settle_time_s: float = 0.2) -> None:
        """
        Capture a screenshot and write it locally.

        Args:
            filename: Name to give to the uploaded image.
            settle_time_s: float time to wait before taking the screenshot.
        """
        self._screenshot_service.save_local(filename, settle_time_s)

    def save_screenshot_db(self, sample_id: int, filename: str, settle_time_s: float = 0.2):
        """
        Capture a screenshot and upload it to the database for a specific sample.

        Args:
            sample_id: Database ID of the sample.
            filename: Name to give to the uploaded image.
            settle_time_s: float time to wait before taking the screenshot.
        """
        self._screenshot_service.save_to_db(sample_id, filename, settle_time_s)

    def send_screenshot_db(self, filename: str | None = None, message: str | None = None) -> None:
        sample = self.sample
        if sample is None:
            raise ValueError("No sample with a valid sample_id is mounted.")

        self._screenshot_service.send_to_db(
            filename=filename,
            message=message,
            default_message=self._default_screenshot_message(sample.db_id),
        )

    def send_message_db(self, db_id: int, event_type: SampleEventType, comment: str | None = None):
        self._aare.send_sample_event(db_id, event_type, comment)

    @property
    def sample_spreadsheet(self) -> SampleShortInfoList:
        return self._cfg.spreadsheet

    @property
    def reference_tools(self) -> SampleShortInfoList:
        return self._cfg.reference_tools

    def sample_spreadsheet_user(self, pgroup: str) -> SampleShortInfoList:
        # sample = copy.deepcopy(self.sample_spreadsheet) #switch to copy if too heavy!
        # sample.s = list(filter(lambda x: x.user == pgroup, sample.s))
        # return sample
        return SampleShortInfoList(s=[x for x in self.sample_spreadsheet.s if x.user == pgroup])

    def get_beamline_default_raster_params(self) -> SimpleScanParameters:
        default_exp_time_s = cfg_get(
            "daq.data_collection_settings.default_raster_scan_settings.exp_time_s", 0.01
        )
        default_transmission = cfg_get(
            "daq.data_collection_settings.default_raster_scan_settings.transmission", 1.0
        )
        default_dtz = cfg_get("daq.data_collection_settings.default_raster_scan_settings.dtz", 250)
        return SimpleScanParameters(
            dtz=default_dtz, exp_time_s=default_exp_time_s, transmission=default_transmission
        )

    def get_beamline_default_rotation_params(self) -> SimpleScanParameters:
        default_exp_time_s = cfg_get(
            "daq.data_collection_settings.default_rotation_settings.exp_time_s", 0.01
        )
        default_transmission = cfg_get(
            "daq.data_collection_settings.default_rotation_settings.transmission", 1.0
        )
        default_dtz = cfg_get("daq.data_collection_settings.default_rotation_settings.dtz", 250)
        default_start_omega_deg = cfg_get(
            "daq.data_collection_settings.default_rotation_settings.start_omega_deg", 0.0
        )
        default_increment_omega_deg = cfg_get(
            "daq.data_collection_settings.default_rotation_settings.incr_omega_deg", 0.2
        )
        default_steps = cfg_get(
            "daq.data_collection_settings.default_rotation_settings.steps", 1800
        )
        return SimpleScanParameters(
            dtz=default_dtz,
            exp_time_s=default_exp_time_s,
            transmission=default_transmission,
            start_omega_deg=default_start_omega_deg,
            incr_omega_deg=default_increment_omega_deg,
            steps=default_steps,
        )

    def get_auto_raster_params(self) -> SimpleScanParameters:
        default_params = self.get_beamline_default_raster_params()
        if self.status.sample is None:
            return default_params

        aaredb_params = (
            self.status.sample.aaredb_params
            if hasattr(self.status.sample, "aaredb_params")
            else None
        )

        if aaredb_params is None:
            return default_params

        params = default_params

        # Exposure
        exp = getattr(aaredb_params, "exposure", None)
        if exp is not None:
            params.exp_time_s = exp
        else:
            params.exp_time_s = 0.04  # Default

        # Transmission
        trans = getattr(aaredb_params, "transmission", None)
        if trans is not None:
            logger.debug(f"transmission: {trans}")
            params.transmission = trans / 100.0 if trans > 1.0 else trans
        else:
            params.transmission = 1.0  # Default

        # Resolution and DTZ
        res = getattr(aaredb_params, "targetresolution", None)
        if res is not None:
            logger.debug(f"resolution: {res}")
            try:
                params.dtz = self.diffraction_geometry.calc_dtz_mm(res)
                logger.debug(f"dtz: {params.dtz}")
            except Exception:
                logger.exception(f"Failed to calculate dtz for resolution {res}")

            # Adjust exposure time based on resolution
            if res <= 1.5:
                params.exp_time_s = 0.02
            elif 1.5 < res <= 3.0:
                params.exp_time_s = 0.04
            else:
                params.exp_time_s = 0.08

        return params

    def get_collection_params(self, prefer_smart: bool = False) -> tuple[SimpleScanParameters, str]:
        spreadsheet_params, _file_prefix = self.spreadsheet_params()
        logger.debug(f"spreadsheet_params: {spreadsheet_params}")
        smart_params = self._cfg.auto_params
        default_params = SimpleScanParameters(exp_time_s=0.04, dtz=110, incr_omega_deg=0.2)
        if prefer_smart:
            if smart_params:
                return smart_params, "smart_params"
            if spreadsheet_params:
                return spreadsheet_params, "spreadsheet_params"
        else:
            if spreadsheet_params:
                return spreadsheet_params, "spreadsheet_params"
            if smart_params:
                return smart_params, "smart_params"

        return default_params, "defaults"

    def _end_operation(
        self,
        start: float,
        operation: DAQOperation | None = DAQOperation.AUTOMATION,
        error: bool = False,
    ) -> float:
        """
        End an operation and return elapsed time.

        Returns:
            (elapsed_seconds, end_time_perf_counter)
        """
        if operation is DAQOperation.AUTOMATION:
            msg = "Ended automation operation"
        else:
            msg = f"Ended automation operation after {operation.value}"

        if error:
            msg += "with an error"
            logger.error(f"{msg}, time taken {time.perf_counter() - start} seconds.")
            try:
                if self._cfg.state_busy:
                    self._set_state(BeamlineStateEnum.RobotSampleExchange)
                else:
                    logger.warning(
                        "Skipping recovery transition to RobotSampleExchange: "
                        "beamline is no longer busy. The busy key may have expired"
                        "befor recovery could run"
                    )
            except Exception:
                logger.exception(
                    "Failed to transition to RobotSampleExchange during error recovery"
                )

        else:
            msg += " successfully"
            logger.info(f"{msg}, time taken {time.perf_counter() - start} seconds.")
        self._cfg.state_busy = False
        end = time.perf_counter()
        return end - start

    @log_timing(logger, "Measure Sequence")
    def measure(self, sample: SampleShortInfo) -> float:
        """
        Main measurement sequence for a given sample.
        Includes mounting, loop centering, face detection, auto-raster, and data collection.

        Args:
            sample: SampleShortInfo object containing sample details

        Returns:
            Total time taken for the measurement (seconds)
        """
        start = time.perf_counter()
        sample_started_at = time.time()
        progress = self._new_automation_progress()
        self._set_progress_context(
            progress, current_sample_name=getattr(sample, "sample_name", "") or ""
        )
        self._emit_automation_progress(progress)

        formatted_date = datetime.now().strftime("%Y%m%d")
        sample_prefix = f"{formatted_date}/{sample.puck_name}/{sample.pin:02d}/{sample.sample_name}"

        try:
            self._validate_automation_state(context="automation start")
            self._cfg.try_set_busy(timeout=self.AUTOMATION_BUSY_TIMEOUT_S)

            self._validate_automation_state(context="after acquiring automation busy state")
            logger.info("Cancelling any pending jfjoch operations")
            self._jfjoch.cancel()
            self._set_progress_context(
                progress, current_sample_name=getattr(sample, "sample_name", "") or ""
            )

            self._mark_progress_running(progress, WorkflowStateKind.MOUNT, "Mounting sample")
            if not self._execute_mount_and_prepare(sample):
                mount_error_message = self._last_mount_error_message or "Mount failed"
                self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, mount_error_message)
                return self._end_operation(start, DAQOperation.MOUNT, error=True)
            self._validate_automation_state(context="after mount")
            self._mark_progress_success(progress, WorkflowStateKind.MOUNT, "Mount complete")
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._validate_automation_state(context="after transition to SampleAlignment")

            logger.info(f"mounting done at {time.perf_counter() - start}")

            self._mark_progress_running(progress, WorkflowStateKind.LOOP_CENTRE, "Centering sample")

            local_contact_config = self.get_local_contact_config()
            mount_to_center_sleep_s = float(local_contact_config.mount_to_center_sleep_s)

            if mount_to_center_sleep_s > 0:
                logger.info(
                    f"Sleeping {mount_to_center_sleep_s:.2f}s between mount and loop centering"
                )
                sleep_started_at = time.monotonic()
                time.sleep(mount_to_center_sleep_s)
                logger.info(
                    "Finished mount-to-center sleep",
                    extra={
                        "requested_sleep_s": mount_to_center_sleep_s,
                        "actual_sleep_s": time.monotonic() - sleep_started_at,
                    },
                )

            centered = False
            try:
                centered = self._execute_loop_centering(sample)
            except LoopCenteringFailed as e:
                self._record_best_effort_step_failure(
                    progress=progress,
                    step=WorkflowStateKind.LOOP_CENTRE,
                    error=e,
                    sample=sample,
                    code="LOOP_CENTERING_FAILED",
                )
            else:
                if not centered:
                    self._record_best_effort_step_failure(
                        progress=progress,
                        step=WorkflowStateKind.LOOP_CENTRE,
                        error=LoopCenteringFailed("Centering returned no result"),
                        sample=sample,
                        code="LOOP_CENTERING_FAILED",
                    )
            self._validate_automation_state(context="after loop centering")

            logger.info(f"Loop Centering done at {time.perf_counter() - start}")

            face_detection_result = self._execute_face_detection(
                steps=7, step_size=30, face_min_ratio=0.3, report_error=True
            )
            if not face_detection_result.success:
                logger.warning(
                    "Face detection failed during automation; continuing with latest payload",
                    extra=merge_log_context(
                        sample_log_context(sample), {"comment": face_detection_result.comment}
                    ),
                )
            self._validate_automation_state(context="after face detection")
            logger.info(f"Face Detection done at {time.perf_counter() - start}")

            self._mark_progress_success(
                progress, WorkflowStateKind.LOOP_CENTRE, "Centering complete"
            )

            self._mark_progress_running(progress, WorkflowStateKind.RASTER, "Running raster")
            hex_string = secrets.token_hex(3)
            raster_params = self.get_auto_raster_params()
            geom = self.sample_geometry

            if raster_params.filename is not None:
                logger.info(f"Using filename {raster_params.filename}")
                sample_prefix = f"{raster_params.filename}/{sample.sample_name}"

            raster_grid = RasterGridRequest(
                exp_time_s=raster_params.exp_time_s,
                file_prefix=sample_prefix + f"_{hex_string}",
                smargon_top_left=SmargonCoordinate(),
                n_x=1,
                n_y=1,
                dtz=raster_params.dtz,
                grid_size_mm=Coordinate(x=geom.beam_size_mm.x * 0.5, y=geom.beam_size_mm.y * 0.5),
                omega_deg=self.omega,
                transmission=raster_params.transmission,
            )

            try:
                raster_result = self._execute_raster_sequence(raster_grid, auto_center=True)
            except AutoRasterSampleSkipped as e:
                logger.warning(
                    "Skipping sample during automation because auto-raster grid is too large",
                    extra=merge_log_context(
                        sample_log_context(sample),
                        raster_request_log_context(raster_grid),
                        {"reason": str(e)},
                    ),
                )
                self._set_progress_step(
                    progress, WorkflowStateKind.RASTER, StepStatus.SKIPPED, str(e)
                )
                self._set_progress_step(
                    progress,
                    WorkflowStateKind.DATA_COLLECTION,
                    StepStatus.SKIPPED,
                    "Data collection skipped because auto-raster was too large",
                )
                self._mark_progress_finished(
                    progress, True, "Sample skipped: auto-raster too large"
                )
                return self._end_operation(start, DAQOperation.AUTOMATION, error=False)

            if raster_result is None:
                logger.error(
                    "Raster result was None during automation",
                    extra=merge_log_context(
                        sample_log_context(sample), raster_request_log_context(raster_grid)
                    ),
                )
                self._mark_progress_failed(progress, WorkflowStateKind.RASTER, "Raster failed")
                return self._end_operation(start, DAQOperation.RASTER, error=True)

            self._validate_automation_state(context="after raster")
            self._mark_progress_success(progress, WorkflowStateKind.RASTER, "Raster complete")
            self._set_state(BeamlineStateEnum.DataCollection)
            self._validate_automation_state(context="after transition to DataCollection")

            logger.info(f"Raster scans completed at {time.perf_counter() - start}")

            self._mark_progress_running(
                progress, WorkflowStateKind.DATA_COLLECTION, "Collecting data"
            )
            params, source = self.get_collection_params(prefer_smart=False)
            logger.info(f"Using {source} for data collection: {params}")

            self._devs.dtz = params.dtz
            start_omega = self.omega

            update_sample = self.sample
            if (
                update_sample is not None
                and update_sample.db_id is not None
                and update_sample.db_id == sample.db_id
            ):
                logger.info(
                    f"Updating sample info {sample.db_id} old run_number"
                    f" sample.run_number {sample.run_number} new run_number {update_sample.run_number}"
                )

            if params.filename is not None:
                logger.info(f"Using filename {params.filename}")
                sample_prefix = f"{params.filename}/{sample.sample_name}"

            rotation_request = RotationScanRequest(
                start_omega_deg=start_omega,
                dtz=params.dtz,
                file_prefix="data/" + sample_prefix + f"_{hex_string}",
                exp_time_s=params.exp_time_s,
                incr_omega_deg=params.incr_omega_deg,
                steps=params.steps,
                transmission=params.transmission,
            )

            rotation_result = self._execute_rotation_sequence(rotation_request)
            if rotation_result is None:
                logger.error("Rotation result was None")
                self._mark_progress_failed(
                    progress, WorkflowStateKind.DATA_COLLECTION, "Collection failed"
                )
                return self._end_operation(start, DAQOperation.ROTATION, error=True)

            self._validate_automation_state(context="after data collection")
            self._mark_progress_success(
                progress, WorkflowStateKind.DATA_COLLECTION, "Collection complete"
            )
            logger.info(f"Rotation scan done at {time.perf_counter() - start}")

            self._validate_automation_state(context="automation end")

        except BECCommunicationError as e:
            self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
            self._raise_if_critical_bec_error(e, command=getattr(e, "operation", None) or "bec")

        except JFJochCommunicationError as e:
            self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
            self._raise_if_critical_jfjoch_detector_error(e, command=e.endpoint or "unknown")
            raise

        except (
            TransformationInvalidException,
            StateTransitionFailed,
            MaintenanceStateException,
        ) as e:
            logger.error(f"Critical automation state error: {e}")
            if progress.current_step is not None:
                current_kind = next(
                    (
                        item.step
                        for item in progress.steps
                        if self._step_display_name(item.step) == progress.current_step
                    ),
                    WorkflowStateKind.FINAL,
                )
                self._mark_progress_failed(progress, current_kind, str(e))
            else:
                self._mark_progress_finished(progress, False, str(e))

            self._handle_operation_error(
                operation=DAQOperation.AUTOMATION,
                sample=sample,
                error=e,
                event_type=SampleEventType.FAILED,
            )
            self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
            raise

        except (BeamlineBusyTimeoutException, BeamlineBusyException) as e:
            time_of_measure = abs(time.perf_counter() - start)

            if time_of_measure > self.AUTOMATION_BUSY_TIMEOUT_S:
                logger.error(
                    f"Error in measure due to Beamline Busy State timeout:"
                    f"Time of: {time_of_measure} is greater than timeout duration {self.AUTOMATION_BUSY_TIMEOUT_S}"
                    f"Error thrown: {e}"
                )

            else:
                logger.error(f"Error in measure due to Beamline Busy State: {e}")

            if progress.current_step is not None:
                current_kind = next(
                    (
                        item.step
                        for item in progress.steps
                        if self._step_display_name(item.step) == progress.current_step
                    ),
                    WorkflowStateKind.FINAL,
                )
                self._mark_progress_failed(progress, current_kind, str(e))
            else:
                self._mark_progress_finished(progress, False, str(e))

            self._handle_operation_error(
                operation=DAQOperation.AUTOMATION,
                sample=sample,
                error=e,
                event_type=SampleEventType.FAILED,
            )
            self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
            raise

        except MountingFailed as e:
            logger.error(f"Failed to mount sample: {e}")
            if e.critical:
                self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
                raise
            else:
                pass

        except Exception as e:
            logger.error(f"Error in measure: {e}")
            if progress.current_step is not None:
                current_kind = next(
                    (
                        item.step
                        for item in progress.steps
                        if self._step_display_name(item.step) == progress.current_step
                    ),
                    WorkflowStateKind.FINAL,
                )
                self._mark_progress_failed(progress, current_kind, str(e))
            else:
                self._mark_progress_finished(progress, False, str(e))

            self._handle_operation_error(
                operation=DAQOperation.AUTOMATION,
                sample=sample,
                error=e,
                event_type=SampleEventType.FAILED,
            )
            self._end_operation(start, operation=DAQOperation.AUTOMATION, error=True)
            raise RuntimeError(f"Critical Error in automation: {e}") from e

        self._record_completed_sample_time(progress, time.time() - sample_started_at)
        self._mark_progress_finished(progress, True, "Automation complete")
        return self._end_operation(start, DAQOperation.AUTOMATION, error=False)

    @log_timing(logger, "Changing Beamline State")
    def _set_state(self, target: BeamlineStateEnum):
        """__set_state assumes that beamline is already in busy state
        it will apply a proper transformation and change state afterward
        specifically:
           1. If target state is maintenance, just go there
           2. If target state is same as current, nothing will happen
           3. If target state cannot be reached, exception is raised and current state is kept
           4. If exception is raised during transformation, state is set to maintenance
           5. If transformation goes OK, target state is set

        Busy state will be cleared only, if exception is raised."""

        curr_state = self._cfg.state

        if hasattr(curr_state, "value") and hasattr(target, "value"):
            logger.info(
                f"State transition requested: {curr_state} -> {target}",
                extra={"from_state": curr_state, "to_state": target},
            )

        if not self._cfg.state_busy:
            raise RuntimeError("Beamline should be busy")

        if target == BeamlineStateEnum.Maintenance:
            self._cfg.state = BeamlineStateEnum.Maintenance
            logger.warning("Beamline entered Maintenance state during state transition request.")
            return
        elif target == curr_state:
            logger.debug(
                f"State already set to {target}",
                extra={"from_state": curr_state, "to_state": target},
            )
            # TODO unify BeamlineStateEnum and BeamlineState and allow transition from same state to same state to recover motor positions!
            # TODO or just allow BEC thingy
            return
        elif target != curr_state:
            self._cfg.state = BeamlineStateEnum.Moving
            try:
                match curr_state:
                    case BeamlineStateEnum.Maintenance:
                        if target == BeamlineStateEnum.SampleExchange:
                            workflows.m2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleAlignment:
                            workflows.m2sa(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.SampleExchange:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.se2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamLocation:
                            workflows.se2sa(self._devs, self._cfg)
                            workflows.sa2bl(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.DewarTransfer:
                            workflows.common2dh(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.RobotSampleExchange:
                            workflows.common_2rse(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.DewarTransfer:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.dh2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.RobotSampleExchange:
                            workflows.dh2sa(self._devs, self._cfg)
                            workflows.sa2rse(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.dh2sa(self._devs, self._cfg)
                            workflows.sa2se(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.DataCollection:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.dc2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.RobotSampleExchange:
                            workflows.dc2rse(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.XtalSnapshot:
                            workflows.sa2xtal_snapshot(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.BeamLocation:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.bl2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.bl2sa(self._devs, self._cfg)
                            workflows.sa2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamstopAlignment:
                            workflows.bl2ba(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.FluxMeasurement:
                            workflows.bl2flux_measurement(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.BeamstopAlignment:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.ba2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.ba2sa(self._devs, self._cfg)
                            workflows.sa2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.FluxMeasurement:
                            workflows.ba2flux_measurement(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamLocation:
                            workflows.ba2bl(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.bl2sa(self._devs, self._cfg)
                            workflows.sa2se(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.FluxMeasurement:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.flux_measurement2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.flux_measurement2sa(self._devs, self._cfg)
                            workflows.sa2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamLocation:
                            workflows.flux_measurement2bl(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamstopAlignment:
                            workflows.flux_measurement2ba(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.SampleAlignment:
                        if target == BeamlineStateEnum.DewarTransfer:
                            workflows.sa2dh(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.sa2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.RobotSampleExchange:
                            workflows.sa2rse(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.DataCollection:
                            workflows.sa2dc(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.XrayFluorescence:
                            workflows.sa2xrf(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamLocation:
                            workflows.sa2bl(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamstopAlignment:
                            workflows.sa2ba(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.FluxMeasurement:
                            workflows.sa2flux_measurement(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.XtalSnapshot:
                            workflows.sa2xtal_snapshot(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.XrayFluorescence:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.xrf2sa(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.RobotSampleExchange:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.rse2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.rse2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.DewarTransfer:
                            workflows.common2dh(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                    case BeamlineStateEnum.XtalSnapshot:
                        if target == BeamlineStateEnum.SampleAlignment:
                            workflows.xtal_snapshot2sa(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.SampleExchange:
                            workflows.xtal_snapshot2se(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.RobotSampleExchange:
                            workflows.xtal_snapshot2rse(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.DataCollection:
                            workflows.xtal_snapshot2dc(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.XrayFluorescence:
                            workflows.xtal_snapshot2xrf(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.DewarTransfer:
                            workflows.xtal_snapshot2dh(self._devs, self._cfg)
                        elif target == BeamlineStateEnum.BeamLocation:
                            workflows.xtal_snapshot2bl(self._devs, self._cfg)
                        else:
                            raise TransformationInvalidException(
                                f"Cannot go from {curr_state} to {target}, not implemented"
                            )
                self._cfg.state = target
                # Beam-location mode uses preset per-zoom camera settings; any
                # other state goes back to auto-exposure (User zoom mode). Apply
                # the preset for the current zoom on entry.
                if target == BeamlineStateEnum.BeamLocation:
                    self._cfg.zoom_mode = ZoomModeEnum.BeamLocation
                    self._devs.samcam_settings = self._cfg.zoom_settings.get_camera_settings(
                        self._devs.zoom
                    )
                elif self._cfg.zoom_mode == ZoomModeEnum.BeamLocation:
                    self._cfg.zoom_mode = ZoomModeEnum.User
                logger.info(
                    f"State transition completed: {curr_state} -> {target}",
                    extra={"from_state": curr_state, "to_state": target},
                )
            except TransformationInvalidException as e:
                logger.error(
                    f"State transition invali: {curr_state} -> {target}: {e}",
                    extra={"from_state": curr_state, "to_state": target},
                )
                self._cfg.state = curr_state
                raise
            except Exception as e:
                logger.exception(
                    f"Exception during state transition : {curr_state} -> {target}. "
                    "Changing state to maintenance due to error.",
                    extra={"from_state": curr_state, "to_state": target},
                )
                self._cfg.state = BeamlineStateEnum.Maintenance
                self._cfg.state_busy = False
                raise StateTransitionFailed(
                    f"State transition failed: {curr_state} -> {target}. "
                    f"Beamline moved to Maintenance. Original error: {e}"
                ) from e

    @property
    def shutter(self) -> bool:
        return self._devs.shutter

    @shutter.setter
    def shutter(self, v: bool):
        self._devs.shutter = v

    @property
    def diffraction_geometry(self) -> DiffractionGeometry:
        try:
            metadata = self._cached_detector_metadata()
            width = int(metadata.get("detector_width", 1))
            height = int(metadata.get("detector_height", 1))
            pixel_size_mm = float(metadata.get("pixel_size_mm", 0.15))
            detector_description = str(metadata.get("detector_description", "unavailable"))
            detector_serial_number = str(metadata.get("detector_serial_number", "unavailable"))
            energy = self._devs.energy_kev
            dtz = self._devs.dtz
            beam_center = self._cfg.beam_center
            return DiffractionGeometry(
                energy_keV=energy,
                dtz_mm=dtz,
                detector_size_pxl=(width, height),
                pixel_size_mm=pixel_size_mm,
                beam_center_pxl=beam_center,
                detector_description=detector_description,
                detector_serial_number=detector_serial_number,
                poni_rot1_rad=-0.001396263,
                poni_rot2_rad=-0.003839724,
            )
        except Exception as e:
            logger.warning(
                f"Falling back to default diffraction geometry because cached detector metadata is unavailable: {e}",
                exc_info=True,
            )
            energy = self._devs.energy_kev
            dtz = self._devs.dtz
            beam_center = self._cfg.beam_center
            return DiffractionGeometry(
                energy_keV=energy,
                dtz_mm=dtz,
                detector_size_pxl=(1, 1),
                pixel_size_mm=0.15,
                beam_center_pxl=beam_center,
                detector_description="unavailable",
                detector_serial_number="unavailable",
                poni_rot1_rad=-0.001396263,
                poni_rot2_rad=-0.003839724,
            )

    @property
    def beamline_status(self) -> BeamlineStatus:
        try:
            ring_current = self._devs.ring_current
            front_light = self.front_light
            back_light = self.back_light
            cryojet_temp = self._devs.cryojet_temp
            shutter_open = self._devs.shutter
            exp_shutter_open = self._devs.exp_shutter.state()
            pss_prohibited = self._devs.pss.is_prohibited()
            pss_alarm = self._devs.pss.alarm_active()
            flux = self._devs.full_flux
            samcam_settings = self._devs.samcam_settings
            bl = self._bl
            transmission = self._devs.transmission
            zoom = self._devs.zoom
            commisioning_mode = self._cfg.commissioning_mode

            dtz_min = self._cfg.cached_dtz_low
            dtz_max = self._cfg.cached_dtz_high

            if (
                dtz_min is None
                or dtz_max is None
                or (dtz_min > dtz_max)
                or (dtz_min == dtz_max)
                or (dtz_min == 0 and dtz_max == 0)
            ):
                logger.warning(
                    "DTZ limits missing from cache, using conservative defaults in beamline_status"
                )
                dtz_min = cfg_get("daq.hardware.default_detector_distance_minimum", 100)
                dtz_max = cfg_get("daq.hardware.default_detector_distance_maximum", 1000)
                logger.warning(f"using dtz_min {dtz_min} and dtz_max {dtz_max}")

            return BeamlineStatus(
                ring_current_mA=ring_current,
                front_light=front_light,
                back_light=back_light,
                cryojet_K=cryojet_temp,
                shutter_open=shutter_open,
                exp_shutter_open=exp_shutter_open,
                flux_ph_s=flux,
                sample_camera=samcam_settings,
                name=bl,
                transmission=transmission,
                zoom=zoom,
                commissioning_mode=commisioning_mode,
                dtz_min=dtz_min,
                dtz_max=dtz_max,
                pss_prohibited=pss_prohibited,
                pss_alarm=pss_alarm,
            )

        except Exception:
            logger.exception("Failed to retrieve beamline status")
            raise

    def _safe_sample(self) -> tuple[SampleShortInfo | None, bool, str | None]:
        """
        Return (sample, tell_connected, tell_error) without raising.
        """
        try:
            return self.sync_current_sample_from_tell(), True, None
        except TellCommunicationError as e:
            return self._cfg.current_sample, False, str(e)
        except Exception as e:
            logger.warning(f"error in status sample info call: {e}", exc_info=True)
            # Keep status flowing even if Tell code throws something unexpected
            return self._cfg.current_sample, False, f"TELL unavailable: {e}"

    def _aerotech_status(self) -> tuple[bool, str | None]:
        aerotech_ok = True
        aerotech_err: str | None = None
        try:
            _ = self._devs.aerotech.status()
        except Exception as e:
            logger.warning(f"Aerotech error in _aerotech_status: {e}", exc_info=True)
            aerotech_ok = False
            aerotech_err = f"Cannot connect to Aerotech: {e}"
        return aerotech_ok, aerotech_err

    def _safe_geom(self) -> tuple[SampleGeometryModel, bool, str | None, bool, str | None]:
        """
        Return (geom, smargon_connected, smargon_error, aerotech_connected, aerotech_error) without raising.
        Uses a conservative fallback geometry if Smargon access fails.
        """
        aerotech_connected, smargon_connected = True, True
        aerotech_error, smargon_error = None, None
        try:
            return (
                self.sample_geometry,
                smargon_connected,
                smargon_error,
                aerotech_connected,
                aerotech_error,
            )
        except SmargonCommunicationError as e:
            logger.warning(f"Smargon error in _safe_geom: {e}")
            smargon_error = f"Cannot connect to Smargon: {e}"
            smargon_connected = False
        except AerotechCommunicationError as e:
            logger.warning(f"Aeroetch error in _safe_geom: {e}")
            aerotech_error = f"Cannot connect to Aerotech: {e}"
            self._devs.exp_shutter.close()
            aerotech_connected = False
        except Exception as e:
            logger.warning(f"Unexpected error in _safe_geom: {e}", exc_info=True)
            smargon_error = f"Safe geometry failed: {e}"
            aerotech_error = f"Safe geometry failed: {e}"
            aerotech_connected = False
            smargon_connected = False
        zoom = self._devs.zoom
        logger.warning("Safe geometry failed: falling back to default settings")
        fallback = SampleGeometryModel(
            beam_location_pxl=self._cfg.beam_mark_coeff.apply(zoom),
            pixel_in_mm=self._cfg.pixel_to_mm(zoom),
            omega_deg=0.0,
            smargon=SmargonCoordinate(
                sh_mm=Coordinate(x=0.0, y=0.0, z=0.0), phi_deg=0.0, chi_deg=0.0
            ),
            beam_size_mm=self._cfg.beam_size_mm,
            aerotech=Coordinate(x=0.0, y=0.0, z=0.0),
            aerotech_meas=Coordinate(x=0.0, y=0.0, z=0.0),
        )
        return (fallback, smargon_connected, smargon_error, aerotech_connected, aerotech_error)

    def _safe_beamline_status(self) -> BeamlineStatus:
        try:
            return self.beamline_status
        except Exception as e:
            # TODO add error message to send to GUI to say problem
            logger.error(f"Failed to retrieve beamline status: {e!s}")
            raise

    def _safe_tell_state(self) -> TellStateModel | None:
        try:
            redis_client = getattr(self._cfg, "_client", None)
            beamline_key = getattr(self._cfg, "_bl", None)

            if redis_client is None or beamline_key is None:
                return None

            redis_key = f"{beamline_key}:tell_state"
            raw_value = redis_client.get(redis_key)
            if raw_value in (None, "", b""):
                return None

            if isinstance(raw_value, bytes):
                raw_value = raw_value.decode("utf-8")

            return TellStateModel.model_validate_json(str(raw_value))
        except Exception as e:
            logger.warning(f"Failed to read tell_state from Redis: {e}", exc_info=True)
            return None

    def _safe_diffraction_geometry(self) -> DiffractionGeometry:
        try:
            return self.diffraction_geometry
        except Exception as e:
            logger.warning(f"Failed to retrieve diffraction geomtrey: {e!s}", exc_info=True)
            # Must satisfy pydantic constraints in DiffractionGeometry
            return DiffractionGeometry(
                energy_keV=12.4,
                dtz_mm=150.0,
                detector_size_pxl=(1, 1),
                pixel_size_mm=0.15,
                beam_center_pxl=(0.0, 0.0),
                detector_description="unavailable",
                detector_serial_number="unavailable",
                poni_rot1_rad=0.0,
                poni_rot2_rad=0.0,
            )

    @property
    def status(self) -> DAQStatusModel:
        try:
            # og_start = time.perf_counter()
            safe_sample, tell_ok, tell_err = self._safe_sample()
            # logger.debug(f"Safe sample info call took {time.perf_counter() - og_start:.3f}s")
            # start = time.perf_counter()
            safe_geom, smargon_ok, smargon_err, aerotech_ok, aerotech_err = self._safe_geom()
            # logger.debug(f"safe geom call took {time.perf_counter() - start:.3f}s")
            # start = time.perf_counter()
            safe_tell_state = self._safe_tell_state()
            # logger.debug(f"Safe tell call call took {time.perf_counter() - start:.3f}s")
            # start = time.perf_counter()
            safe_beamline_status = self._safe_beamline_status()
            # logger.debug(f"Safe beamline status call took {time.perf_counter() - start:.3f}s")
            # start = time.perf_counter()
            safe_diffraction_geom = self._safe_diffraction_geometry()
            # logger.debug(f"Safe diffraction geometry call took {time.perf_counter() - start:.3f}s")
            # start = time.perf_counter()
            session_status = SessionStatus(
                current_pgroup=self._cfg.pgroup,
                session=self._cfg.session_state(0),  # 0 is dummy session
                staff=False,
            )
            # logger.debug(f"Safe session status call took {time.perf_counter() - start:.3f}s")
            # start = time.perf_counter()
            status = DAQStatusModel(
                state=self.state,
                busy=self.busy,
                geom=safe_geom,
                bl=safe_beamline_status,
                sample=safe_sample,
                session=session_status,
                diffraction=safe_diffraction_geom,
                box=self._saved_box,
                last_best_res=self._cfg.last_best_res,
                last_best_b_factor=self._cfg.last_best_b_factor,
                crystal_size=self._cfg.crystal_size,
                tell_connected=tell_ok,
                tell_error=tell_err,
                tell_state=safe_tell_state,
                smargon_connected=smargon_ok,
                smargon_error=smargon_err,
                aerotech_connected=aerotech_ok,
                aerotech_error=aerotech_err,
            )
            # logger.debug(f"Creating DAQStatusModel took {time.perf_counter() - start:.3f}s")
            # logger.debug(f"returning status call took {time.perf_counter() - og_start:.3f}s")
            return status

        except Exception as e:
            logger.error(f"Failed to retrieve DAQ status: {e}")
            raise

    def cancel(self):
        if self._cfg.state == BeamlineStateEnum.DataCollection:
            self._devs.aerotech.cancel()
            self._jfjoch.cancel()

    def anneal(self, time_s: float):
        self._cfg.set_busy(BeamlineStateEnum.SampleAlignment)
        try:
            self._devs.anneal(time_s)
        finally:
            self._cfg.state_busy = False

    def mono_pitch_scan(self, plot: bool = False) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.mono_pitch_scan_runner(plot=plot)
        finally:
            self._cfg.state_busy = False

    def change_energy(self, value: float, plot: bool = False) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.change_energy(value=value, plot=plot)
        finally:
            self._cfg.state_busy = False

    def bec_load_user_macros(self) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.load_user_macros()
        finally:
            self._cfg.state_busy = False

    def bec_list_all_user_macros(self) -> list[str]:
        macros = self._devs.bec_worker.list_all_user_macros()
        if macros is None:
            return []
        return [str(item) for item in macros]

    def bec_list_all_devices(self) -> list[str]:
        devices = self._devs.bec_worker.list_position_devices()
        if devices is None:
            return []
        return [str(item) for item in devices]

    def bec_reinitialise_planner_and_position_devices(self, method: str = "auto") -> list[str]:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.load_user_macros()
            return self._devs.bec_worker.reinitialise_planner_and_position_devices(method=method)
        finally:
            self._cfg.state_busy = False

    def bec_save_current_bs_pos(self) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.save_current_bs_pos()
        finally:
            self._cfg.state_busy = False

    def bec_save_current_collimator_pos(self) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.save_current_collimator_pos()
        finally:
            self._cfg.state_busy = False

    def bec_save_current_aerotech_position(self) -> None:
        self._cfg.try_set_busy(timeout=360)
        try:
            self._devs.bec_worker.save_current_aerotech_position()
        finally:
            self._cfg.state_busy = False

    def steer_beam_available(self) -> bool:
        return "beam_steering" in self._devs.bec_worker.dev

    def steer_beam(self, x: int | None, y: int | None):
        """Run the routine to move the beam to the sample location. Update the location if provided."""
        if "beam_steering" not in self._devs.bec_worker.dev:
            raise BECCommunicationError("Beam steering device does not exist in the BEC config.")
        self._cfg.try_set_busy(timeout=360)
        try:
            if x is not None:
                self._devs.bec_worker.dev.beam_steering.sample_loc_x_px.set(x).wait()
            if y is not None:
                self._devs.bec_worker.dev.beam_steering.sample_loc_y_px.set(y).wait()
            self._devs.bec_worker.dev.beam_steering.trigger().wait()
        finally:
            self._cfg.state_busy = False

    def fluorimeter_take_spectrum(
        self, fm: FluorescenceSpectrumParameterModel
    ) -> FluorescenceSpectrumOutputModel:
        self._cfg.try_set_busy(timeout=360)

        try:
            self._set_state(BeamlineStateEnum.XrayFluorescence)

            if fm.transmission is not None:
                self._devs.transmission = fm.transmission

            # TODO: Fill

            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
            return None
        except Exception:
            self._set_state(BeamlineStateEnum.SampleAlignment)
            self._cfg.state_busy = False
            raise

    def get_local_contact_config(self) -> LocalContactConfigModel:
        return self._cfg.get_local_contact_config()

    def set_local_contact_config(self, config: LocalContactConfigModel) -> LocalContactConfigModel:
        return self._cfg.set_local_contact_config(config)