DAQ: finish refactor of exception handler, improve mount count fail handling and updated tests

This commit is contained in:
appleb_m committed 2026-06-09 14:25:37 +02:00
1 parent 675549c1c2
commit bd8f66009c
10 files changed
+265 -138

No files matched your search

-63
View File
@@ -352,16 +352,6 @@ class SmargonCommunicationError(SmargonException):
self.base_url = base_url
self.operation = operation
self.status_code = status_code
logger.error(
message,
extra={
"device": "smargon",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
"status_code": status_code,
},
)
def __str__(self) -> str:
return self.message
@@ -389,15 +379,6 @@ class TellCommunicationError(TellException):
self.endpoint = endpoint
self.base_url = base_url
self.operation = operation
logger.error(
message,
extra={
"device": "tell",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
},
)
def __str__(self) -> str:
return self.message
@@ -427,16 +408,6 @@ class JFJochCommunicationError(JFJochException):
self.endpoint = endpoint
self.base_url = base_url
self.status_code = status_code
logger.error(
message,
extra={
"device": "jfjjoch",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
"status_code": status_code,
},
)
def __str__(self) -> str:
return self.message
@@ -464,16 +435,6 @@ class AareDBCommunicationError(AutomationError):
self.endpoint = endpoint
self.base_url = base_url
self.status_code = status_code
logger.error(
message,
extra={
"device": "aaredb",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
"status_code": status_code,
},
)
def __str__(self) -> str:
return self.message
@@ -503,16 +464,6 @@ class AerotechCommunicationError(AerotechException):
self.base_url = base_url
self.operation = operation
self.status_code = status_code
logger.error(
message,
extra={
"device": "aerotech",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
"status_code": status_code,
},
)
def __str__(self) -> str:
return self.message
@@ -524,10 +475,6 @@ class MagnetPositionSensorErorr(AutomationError):
def __init__(self, message: str = "Magnet position sensor error", *, critical: bool | None = None):
super().__init__(message, critical=critical)
self.message = message
logger.error(
message,
extra={"device": "magnet_position_sensor"},
)
def __str__(self) -> str:
return self.message
@@ -552,16 +499,6 @@ class BECCommunicationError(BECException):
self.exception = exception
self.endpoint = endpoint
self.base_url = base_url
logger.error(
message,
extra={
"device": "bec",
"operation": operation,
"endpoint": endpoint,
"base_url": base_url,
"exception": exception,
},
)
def __str__(self) -> str:
return self.message
+12
View File
@@ -87,6 +87,18 @@ def create_default_watchers(
]
def redis_key_to_env_var(key: str) -> str:
"""Adapter from canonical Redis key shape to an env-var-safe name.
``aare:watchers:{bl}:{watcher}:threshold`` →
``AARE_WATCHERS_{BL}_{WATCHER}_THRESHOLD``.
Used by the GUI today since it has no Redis client; the loader's
``get_value`` argument keeps the Redis key shape authoritative so the
DAQ side can later swap in ``cfg.get`` without churn (plan §7)."""
return key.replace(":", "_").upper()
def load_watcher_threshold_overrides(
*,
beamline: str,
+11 -4
View File
@@ -241,8 +241,6 @@ class AareDAQ:
pgroup_provider=_DAQPGroupProvider(self),
)
self._create_mounting_service().reset_mount_failure_counter("DAQ startup")
def get_runtime_simulation_state(self) -> dict[str, bool]:
return self.__cfg.runtime_simulation_state
@@ -2496,9 +2494,9 @@ class AareDAQ:
logger.info(f"mounting done at {time.perf_counter() - start}")
self._mark_progress_running(progress, WorkflowStateKind.LOOP_CENTRE, "Centering sample")
centered = False
try:
if not self._execute_loop_centering(sample):
raise LoopCenteringFailed("Centering failed")
centered = self._execute_loop_centering(sample)
except LoopCenteringFailed as e:
self._record_best_effort_step_failure(
progress=progress,
@@ -2507,6 +2505,15 @@ class AareDAQ:
sample=sample,
code="LOOP_CENTERING_FAILED",
)
else:
if not centered:
self._record_best_effort_step_failure(
progress=progress,
step=WorkflowStateKind.LOOP_CENTRE,
error=LoopCenteringFailed("Centering returned no result"),
sample=sample,
code="LOOP_CENTERING_FAILED",
)
self._validate_automation_state(context="after loop centering")
logger.info(f"Loop Centering done at {time.perf_counter() - start}")
+32 -52
View File
@@ -3,27 +3,31 @@ import time
from aare.common.exception_handler import (
CriticalTellException,
MountingFailed,
TellCommunicationError,
UnmountingFailed,
)
from aare.devices.tell_client import TellEventValueEnum
from aare.daq.operations.common.runtime import with_retry
from aare.daq.operations.mounting.models import MountingContext, MountingResult
DRY_AFTER_FAIL_COUNT = 3
STOP_AFTER_FAIL_COUNT = 5
class MountingService:
def __init__(self, *, context: MountingContext, logger):
self.ctx = context
self.logger = logger
def reset_mount_failure_counter(self, reason: str) -> None:
previous_count = self.ctx.cfg.get_mount_fail_count()
previous_count = self.ctx.cfg.get_mount_failure_streak()
if previous_count > 0:
self.logger.warning(
f"Resetting mount failure counter from {previous_count} due to: {reason}"
)
else:
self.logger.debug(f"Mount failure counter already clear: {reason}")
self.ctx.cfg.record_mount_success()
self.ctx.cfg.reset_mount_failure_streak()
def _magnet_position_sensor_check(self, timeout: float = 1.0) -> None:
if self.ctx.devs.magnet_position_sensor.value != 0:
@@ -40,42 +44,31 @@ class MountingService:
)
raise Exception("Goniometer is not in position based on magnet position sensor readout")
def _on_mount_failure(
self,
*,
dry_after_fail_count: int = 3,
end_automation_after_fail_count: int = 5,
) -> None:
count = self.ctx.cfg.record_mount_failure()
def _handle_consecutive_mount_failure(self) -> None:
count = self.ctx.cfg.increment_mount_failure_streak()
self.logger.warning(f"Consecutive mount failure count is now {count}")
if count > end_automation_after_fail_count:
self.logger.error(
"Mounting continued to fail after stopping automation, contact your local contact"
)
self.logger.debug("Clearing mount fail count")
self.ctx.cfg.record_mount_success()
raise MountingFailed(
"Mounting continued to fail after stopping automation, contact your local contact.",
critical=True,
)
if count == end_automation_after_fail_count:
self.logger.error(f"Mounting failed {count} times, stopping automation")
self.logger.error("Unmounting sample and drying")
try:
self._unmount_current_sample(timeout=60.0)
self.dry(park=True)
except Exception as e:
self.logger.exception(f"Failed to dry after mount failure: {e}")
raise MountingFailed(f"Mount failed {count} times in a row, stopping automation.", critical=True)
if count == dry_after_fail_count:
if count == DRY_AFTER_FAIL_COUNT:
self.logger.warning(f"Mount failed {count} times in a row; drying")
try:
self.dry(park=False)
except Exception as e:
self.logger.exception(f"Failed to dry after mount failure: {e}")
if count >= STOP_AFTER_FAIL_COUNT:
self.logger.error(f"Mount failed {count} times in a row, stopping automation")
self.logger.error("Unmounting sample and drying")
try:
self._unmount_current_sample(timeout=60.0)
self.dry(park=True)
except Exception as e:
self.logger.exception(f"Failed to clean up after repeated mount failure: {e}")
raise MountingFailed(
f"Mount failed {count} times in a row, stopping automation.",
critical=True,
)
def _mount_handler(self, target) -> None:
self._prepare_mount_hardware()
@@ -122,7 +115,6 @@ class MountingService:
raise CriticalTellException(f"Critical error in TELL mount: {message}")
def _prepare_mount_hardware(self) -> None:
#self.ctx.devs.tell.check_command_ok()
self.ctx.devs.smargon_move_home()
self.ctx.devs.aerotech_pos = self.ctx.mount_position
self._magnet_position_sensor_check(timeout=360.0)
@@ -139,7 +131,7 @@ class MountingService:
self.ctx.cfg.current_sample = None
return previous_sample
def dry(self, *, park: bool, unmount:bool=False) -> None:
def dry(self, *, park: bool, unmount: bool = False) -> None:
self.ctx.devs.tell.check_enable_motion()
self.ctx.devs.tell.wait_not_busy()
self.ctx.devs.tell.set_in_mount_position(True)
@@ -154,11 +146,9 @@ class MountingService:
self.ctx.devs.tell.dry(wait=True)
def execute(self, *, target) -> MountingResult:
previous_sample = self.ctx.cfg.current_sample
try:
if target is None:
try:
unmounted_sample = self._unmount_current_sample(timeout=60.0)
@@ -176,24 +166,14 @@ class MountingService:
log_msg = f"Unmounting sample: {previous_sample}\nand {log_msg}"
self.logger.debug(log_msg)
def _attempt_mount() -> None:
try:
self._mount_handler(target)
except Exception:
self._on_mount_failure()
raise
with_retry(
_attempt_mount,
max_attempts=5,
on=(MountingFailed,),
escalate=MountingFailed,
escalate_message="Mount failed 5 times in a row, stopping automation.",
escalate_critical=True,
)
try:
self._mount_handler(target)
except (MountingFailed, TellCommunicationError):
self._handle_consecutive_mount_failure()
raise
self.logger.debug("Mounting succeeded, resetting mount failure counter")
self.ctx.cfg.record_mount_success()
self.ctx.cfg.reset_mount_failure_streak()
self.ctx.cfg.current_sample = target
return MountingResult(
+41 -3
View File
@@ -120,18 +120,40 @@ def register_exception_handlers(app) -> None:
@app.exception_handler(AutomationError)
async def _handle_automation_error(request: Request, exc: AutomationError) -> JSONResponse:
body = _error_body(exc)
log = logger.error if exc.critical else logger.warning
log(
"AutomationError: %s",
body["message"],
extra={
"exception_class": body["exception_class"],
"code": body["code"],
"critical": body["critical"],
**body["context"],
},
)
status = (
api_status.HTTP_503_SERVICE_UNAVAILABLE
if exc.critical
else api_status.HTTP_422_UNPROCESSABLE_ENTITY
)
return JSONResponse(status_code=status, content=_error_body(exc))
return JSONResponse(status_code=status, content=body)
@app.exception_handler(AareUserError)
async def _handle_user_error(request: Request, exc: AareUserError) -> JSONResponse:
body = _error_body(exc)
logger.info(
"AareUserError: %s",
body["message"],
extra={
"exception_class": body["exception_class"],
"code": body["code"],
**body["context"],
},
)
return JSONResponse(
status_code=api_status.HTTP_400_BAD_REQUEST,
content=_error_body(exc),
content=body,
)
@app.exception_handler(AareAuthError)
@@ -149,9 +171,19 @@ def register_exception_handlers(app) -> None:
if isinstance(exc, AuthenticationException)
else api_status.HTTP_403_FORBIDDEN
)
body = _error_body(exc, code_override=code_override)
logger.warning(
"AareAuthError: %s",
body["message"],
extra={
"exception_class": body["exception_class"],
"code": body["code"],
"status_code": status,
},
)
return JSONResponse(
status_code=status,
content=_error_body(exc, code_override=code_override),
content=body,
headers=getattr(exc, "headers", None),
)
@@ -161,6 +193,12 @@ def register_exception_handlers(app) -> None:
it internally (e.g. validation, route-not-found) and routes raise it
manually. Adapt to the new body shape so clients see one schema."""
detail = exc.detail
log = logger.error if exc.status_code >= 500 else logger.warning
log(
"HTTPException %s: %s",
exc.status_code,
detail,
)
# Preserve legacy callers that passed a {code, message} dict
if isinstance(detail, dict) and "code" in detail and "message" in detail:
legacy_code = str(detail.get("code"))
+6 -5
View File
@@ -15,7 +15,7 @@ from jfjoch_client import ScanResult, ScanResultImagesInner
from aare.common.auth_models import BatonStatus
from aare.common.coordinate import SmargonCoordinate, AerotechCoordinate
from aare.common.error_codes import export_error_codes, AuthErrorCode
from aare.common.error_codes import export_error_codes, AareErrorCode, AuthErrorCode
from aare.common.models import (
DAQStatusModel,
SampleShortInfoList,
@@ -39,6 +39,7 @@ from aare.common.recurrence_watcher import (
WatcherTrip,
create_default_watchers,
load_watcher_threshold_overrides,
redis_key_to_env_var,
resolve_exception_class,
)
from aare.common.raster_grid import RasterGridRequest, CompletedRasterGrid, CompletedRasterGridElem
@@ -894,7 +895,7 @@ class DAQWorker(QObject):
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
else:
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
if code == "JFJOCH_UNAVAILABLE" or status == 503:
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
if self._is_critical_detector_failure(status, body_json, err_msg):
@@ -948,7 +949,7 @@ class DAQWorker(QObject):
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
else:
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
if code == "JFJOCH_UNAVAILABLE" or status == 503:
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR or status == 503:
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
if self._is_critical_detector_failure(status, body_json, err_msg):
@@ -1092,7 +1093,7 @@ class DAQWorker(QObject):
overrides = load_watcher_threshold_overrides(
beamline=beamline,
watcher_names=watcher_names,
get_value=os.getenv,
get_value=lambda k: os.getenv(redis_key_to_env_var(k)),
)
return create_default_watchers(overrides)
@@ -1205,7 +1206,7 @@ class DAQWorker(QObject):
if status_int == 500:
return True
if code == "JFJOCH_UNAVAILABLE" and status_int is not None and 500 <= status_int < 600:
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR and status_int is not None and 500 <= status_int < 600:
return True
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
@@ -130,3 +130,41 @@ def test_aerotech_communication_error():
def test_magnet_position_sensor_error():
exc = MagnetPositionSensorErorr()
assert "Magnet position sensor error" in str(exc)
@pytest.mark.parametrize(
"factory",
[
lambda: TransformationInvalidException("t"),
lambda: LoopCenteringFailed("lc"),
lambda: UnmountingFailed("un"),
lambda: MountingFailed("mt"),
lambda: ManualMountException("mm"),
lambda: SmartMagnetFaultException("sm"),
lambda: TellCommandWhileBusyException("tb"),
lambda: TellConnectionException("tc"),
lambda: WarningTellException("wt"),
lambda: CriticalTellException("ct"),
lambda: AXCFailed("ax"),
lambda: BeamlineBusyException("bb"),
lambda: SampleException("se"),
lambda: AuthenticationException("ae"),
lambda: TellCommunicationError("tce", endpoint="/x", operation="GET"),
lambda: SmargonCommunicationError("sce", endpoint="/m", status_code=500),
lambda: JFJochCommunicationError("jce", operation="POST"),
lambda: AareDBCommunicationError("dbce"),
lambda: AerotechCommunicationError("ace"),
lambda: MagnetPositionSensorErorr(),
],
)
def test_exception_construction_emits_no_logs(factory, caplog):
"""Constructing an Aare exception must not emit log records.
Logging is the responsibility of the catch site (server handler,
best-effort wrapper, or callsite that decides to swallow). Emitting
on construction double-logs and pollutes the WARNING-vs-ERROR split
for best-effort flows."""
caplog.clear()
with caplog.at_level("DEBUG"):
factory()
assert caplog.records == []
+26 -1
View File
@@ -8,6 +8,7 @@ from aare.common.recurrence_watcher import (
RecurrenceWatcher,
create_default_watchers,
load_watcher_threshold_overrides,
redis_key_to_env_var,
resolve_exception_class,
)
@@ -54,4 +55,28 @@ def test_create_default_watchers_applies_overrides():
def test_resolve_exception_class_maps_known_classes():
assert resolve_exception_class("MountingFailed") is MountingFailed
assert resolve_exception_class("LoopCenteringFailed") is LoopCenteringFailed
assert resolve_exception_class("UnknownClass") is None
assert resolve_exception_class("UnknownClass") is None
def test_redis_key_to_env_var_converts_colons_and_uppercases():
assert redis_key_to_env_var("aare:watchers:default:tell:threshold") == (
"AARE_WATCHERS_DEFAULT_TELL_THRESHOLD"
)
assert redis_key_to_env_var("aare:watchers:mx:alc:threshold") == (
"AARE_WATCHERS_MX_ALC_THRESHOLD"
)
def test_load_watcher_threshold_overrides_through_env_var_adapter(monkeypatch):
monkeypatch.setenv("AARE_WATCHERS_DEFAULT_TELL_THRESHOLD", "9")
monkeypatch.setenv("AARE_WATCHERS_DEFAULT_ALC_THRESHOLD", "bad")
import os
overrides = load_watcher_threshold_overrides(
beamline="default",
watcher_names=["tell", "alc", "smargon"],
get_value=lambda k: os.getenv(redis_key_to_env_var(k)),
)
assert overrides == {"tell": 9}
@@ -52,11 +52,15 @@ def _make_context(previous_sample=None):
magnet_position_sensor=types.SimpleNamespace(value=0),
)
streak = {"count": 0}
cfg = types.SimpleNamespace(
current_sample=previous_sample,
get_mount_fail_count=lambda: 0,
record_mount_success=lambda: None,
record_mount_failure=lambda: 1,
get_mount_failure_streak=lambda: streak["count"],
increment_mount_failure_streak=lambda: streak.__setitem__("count", streak["count"] + 1) or streak["count"],
reset_mount_failure_streak=lambda: streak.__setitem__("count", 0),
record_mount_failure=lambda: streak.__setitem__("count", streak["count"] + 1) or streak["count"],
record_mount_success=lambda: streak.__setitem__("count", 0),
)
return MountingContext(
@@ -143,17 +147,37 @@ def test_dry_unmounts_current_sample_before_drying(mock_logger):
assert dry_calls == [{"wait_cold": -1, "wait": True}]
def test_execute_mount_escalates_to_critical_after_retry_exhaustion(mock_logger):
def test_execute_mount_triggers_dry_on_third_consecutive_failure(mock_logger):
target_sample = _make_sample(2, "new")
calls = {"count": 0}
def record_mount_failure():
calls["count"] += 1
return calls["count"]
dry_calls = []
ctx = _make_context()
ctx.cfg.record_mount_failure = record_mount_failure
ctx.devs.tell.mount = lambda **kwargs: TellEventValueEnum.NO_PIN_IN_GRIPPER
ctx.devs.tell.dry = lambda **kwargs: dry_calls.append(kwargs)
ctx.cfg.increment_mount_failure_streak()
ctx.cfg.increment_mount_failure_streak()
service = MountingService(context=ctx, logger=mock_logger)
result = service.execute(target=target_sample)
assert result.success is False
assert isinstance(result.error, MountingFailed)
assert result.error.critical is False
assert dry_calls == [{"wait": True}]
def test_execute_mount_stops_automation_on_fifth_consecutive_failure(mock_logger):
target_sample = _make_sample(2, "new")
dry_calls = []
ctx = _make_context()
ctx.devs.tell.mount = lambda **kwargs: TellEventValueEnum.NO_PIN_IN_GRIPPER
ctx.devs.tell.dry = lambda **kwargs: dry_calls.append(kwargs)
ctx.cfg.increment_mount_failure_streak()
ctx.cfg.increment_mount_failure_streak()
ctx.cfg.increment_mount_failure_streak()
ctx.cfg.increment_mount_failure_streak()
service = MountingService(context=ctx, logger=mock_logger)
@@ -163,3 +187,37 @@ def test_execute_mount_escalates_to_critical_after_retry_exhaustion(mock_logger)
assert isinstance(result.error, MountingFailed)
assert result.error.critical is True
assert "Mount failed 5 times in a row" in str(result.error)
assert {"wait_cold": -1, "wait": True} in dry_calls
def test_execute_mount_success_resets_failure_streak(mock_logger):
previous_sample = _make_sample(1, "old")
target_sample = _make_sample(2, "new")
ctx = _make_context(previous_sample=previous_sample)
ctx.cfg.increment_mount_failure_streak()
ctx.cfg.increment_mount_failure_streak()
service = MountingService(context=ctx, logger=mock_logger)
result = service.execute(target=target_sample)
assert result.success is True
assert ctx.cfg.get_mount_failure_streak() == 0
def test_execute_mount_critical_tell_error_does_not_increment_failure_streak(mock_logger):
target_sample = _make_sample(2, "new")
ctx = _make_context()
def mount(**kwargs):
raise CriticalTellException("critical tell problem")
ctx.devs.tell.mount = mount
service = MountingService(context=ctx, logger=mock_logger)
result = service.execute(target=target_sample)
assert result.success is False
assert isinstance(result.error, CriticalTellException)
assert ctx.cfg.get_mount_failure_streak() == 0
@@ -279,6 +279,37 @@ def test_only_four_root_handlers_plus_fallbacks(app):
REQUIRED_BODY_KEYS = {"critical", "code", "exception_class", "message", "context"}
@pytest.mark.asyncio
async def test_automation_error_logs_error_when_critical(app, mock_request, caplog):
exc = MountingFailed("threshold breached", critical=True)
handler = app.exception_handlers[AutomationError]
caplog.clear()
with caplog.at_level("DEBUG"):
await handler(mock_request, exc)
assert any(r.levelname == "ERROR" for r in caplog.records), caplog.records
@pytest.mark.asyncio
async def test_automation_error_logs_warning_when_not_critical(app, mock_request, caplog):
exc = MountingFailed("benign")
handler = app.exception_handlers[AutomationError]
caplog.clear()
with caplog.at_level("DEBUG"):
await handler(mock_request, exc)
assert any(r.levelname == "WARNING" for r in caplog.records), caplog.records
assert not any(r.levelname == "ERROR" for r in caplog.records)
@pytest.mark.asyncio
async def test_user_error_logs_info(app, mock_request, caplog):
exc = ManualMountException("user must intervene")
handler = app.exception_handlers[AareUserError]
caplog.clear()
with caplog.at_level("DEBUG"):
await handler(mock_request, exc)
assert any(r.levelname == "INFO" for r in caplog.records), caplog.records
@pytest.mark.asyncio
async def test_response_body_shape_uniform(app, mock_request):
cases: list[tuple[type, Exception]] = [