DAQ: finish refactor of exception handler, improve mount count fail handling and updated tests
This commit is contained in:
1 parent
675549c1c2
commit
bd8f66009c
10 files changed
+265
-138
No files matched your search
@@ -352,16 +352,6 @@ class SmargonCommunicationError(SmargonException):
|
||||
self.base_url = base_url
|
||||
self.operation = operation
|
||||
self.status_code = status_code
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "smargon",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -389,15 +379,6 @@ class TellCommunicationError(TellException):
|
||||
self.endpoint = endpoint
|
||||
self.base_url = base_url
|
||||
self.operation = operation
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "tell",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -427,16 +408,6 @@ class JFJochCommunicationError(JFJochException):
|
||||
self.endpoint = endpoint
|
||||
self.base_url = base_url
|
||||
self.status_code = status_code
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "jfjjoch",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -464,16 +435,6 @@ class AareDBCommunicationError(AutomationError):
|
||||
self.endpoint = endpoint
|
||||
self.base_url = base_url
|
||||
self.status_code = status_code
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "aaredb",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -503,16 +464,6 @@ class AerotechCommunicationError(AerotechException):
|
||||
self.base_url = base_url
|
||||
self.operation = operation
|
||||
self.status_code = status_code
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "aerotech",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
"status_code": status_code,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -524,10 +475,6 @@ class MagnetPositionSensorErorr(AutomationError):
|
||||
def __init__(self, message: str = "Magnet position sensor error", *, critical: bool | None = None):
|
||||
super().__init__(message, critical=critical)
|
||||
self.message = message
|
||||
logger.error(
|
||||
message,
|
||||
extra={"device": "magnet_position_sensor"},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -552,16 +499,6 @@ class BECCommunicationError(BECException):
|
||||
self.exception = exception
|
||||
self.endpoint = endpoint
|
||||
self.base_url = base_url
|
||||
logger.error(
|
||||
message,
|
||||
extra={
|
||||
"device": "bec",
|
||||
"operation": operation,
|
||||
"endpoint": endpoint,
|
||||
"base_url": base_url,
|
||||
"exception": exception,
|
||||
},
|
||||
)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.message
|
||||
@@ -87,6 +87,18 @@ def create_default_watchers(
|
||||
]
|
||||
|
||||
|
||||
def redis_key_to_env_var(key: str) -> str:
|
||||
"""Adapter from canonical Redis key shape to an env-var-safe name.
|
||||
|
||||
``aare:watchers:{bl}:{watcher}:threshold`` →
|
||||
``AARE_WATCHERS_{BL}_{WATCHER}_THRESHOLD``.
|
||||
|
||||
Used by the GUI today since it has no Redis client; the loader's
|
||||
``get_value`` argument keeps the Redis key shape authoritative so the
|
||||
DAQ side can later swap in ``cfg.get`` without churn (plan §7)."""
|
||||
return key.replace(":", "_").upper()
|
||||
|
||||
|
||||
def load_watcher_threshold_overrides(
|
||||
*,
|
||||
beamline: str,
|
||||
|
||||
+11
-4
@@ -241,8 +241,6 @@ class AareDAQ:
|
||||
pgroup_provider=_DAQPGroupProvider(self),
|
||||
)
|
||||
|
||||
self._create_mounting_service().reset_mount_failure_counter("DAQ startup")
|
||||
|
||||
def get_runtime_simulation_state(self) -> dict[str, bool]:
|
||||
return self.__cfg.runtime_simulation_state
|
||||
|
||||
@@ -2496,9 +2494,9 @@ class AareDAQ:
|
||||
logger.info(f"mounting done at {time.perf_counter() - start}")
|
||||
|
||||
self._mark_progress_running(progress, WorkflowStateKind.LOOP_CENTRE, "Centering sample")
|
||||
centered = False
|
||||
try:
|
||||
if not self._execute_loop_centering(sample):
|
||||
raise LoopCenteringFailed("Centering failed")
|
||||
centered = self._execute_loop_centering(sample)
|
||||
except LoopCenteringFailed as e:
|
||||
self._record_best_effort_step_failure(
|
||||
progress=progress,
|
||||
@@ -2507,6 +2505,15 @@ class AareDAQ:
|
||||
sample=sample,
|
||||
code="LOOP_CENTERING_FAILED",
|
||||
)
|
||||
else:
|
||||
if not centered:
|
||||
self._record_best_effort_step_failure(
|
||||
progress=progress,
|
||||
step=WorkflowStateKind.LOOP_CENTRE,
|
||||
error=LoopCenteringFailed("Centering returned no result"),
|
||||
sample=sample,
|
||||
code="LOOP_CENTERING_FAILED",
|
||||
)
|
||||
self._validate_automation_state(context="after loop centering")
|
||||
|
||||
logger.info(f"Loop Centering done at {time.perf_counter() - start}")
|
||||
|
||||
@@ -3,27 +3,31 @@ import time
|
||||
from aare.common.exception_handler import (
|
||||
CriticalTellException,
|
||||
MountingFailed,
|
||||
TellCommunicationError,
|
||||
UnmountingFailed,
|
||||
)
|
||||
from aare.devices.tell_client import TellEventValueEnum
|
||||
from aare.daq.operations.common.runtime import with_retry
|
||||
from aare.daq.operations.mounting.models import MountingContext, MountingResult
|
||||
|
||||
|
||||
DRY_AFTER_FAIL_COUNT = 3
|
||||
STOP_AFTER_FAIL_COUNT = 5
|
||||
|
||||
|
||||
class MountingService:
|
||||
def __init__(self, *, context: MountingContext, logger):
|
||||
self.ctx = context
|
||||
self.logger = logger
|
||||
|
||||
def reset_mount_failure_counter(self, reason: str) -> None:
|
||||
previous_count = self.ctx.cfg.get_mount_fail_count()
|
||||
previous_count = self.ctx.cfg.get_mount_failure_streak()
|
||||
if previous_count > 0:
|
||||
self.logger.warning(
|
||||
f"Resetting mount failure counter from {previous_count} due to: {reason}"
|
||||
)
|
||||
else:
|
||||
self.logger.debug(f"Mount failure counter already clear: {reason}")
|
||||
self.ctx.cfg.record_mount_success()
|
||||
self.ctx.cfg.reset_mount_failure_streak()
|
||||
|
||||
def _magnet_position_sensor_check(self, timeout: float = 1.0) -> None:
|
||||
if self.ctx.devs.magnet_position_sensor.value != 0:
|
||||
@@ -40,42 +44,31 @@ class MountingService:
|
||||
)
|
||||
raise Exception("Goniometer is not in position based on magnet position sensor readout")
|
||||
|
||||
def _on_mount_failure(
|
||||
self,
|
||||
*,
|
||||
dry_after_fail_count: int = 3,
|
||||
end_automation_after_fail_count: int = 5,
|
||||
) -> None:
|
||||
count = self.ctx.cfg.record_mount_failure()
|
||||
def _handle_consecutive_mount_failure(self) -> None:
|
||||
count = self.ctx.cfg.increment_mount_failure_streak()
|
||||
self.logger.warning(f"Consecutive mount failure count is now {count}")
|
||||
|
||||
if count > end_automation_after_fail_count:
|
||||
self.logger.error(
|
||||
"Mounting continued to fail after stopping automation, contact your local contact"
|
||||
)
|
||||
self.logger.debug("Clearing mount fail count")
|
||||
self.ctx.cfg.record_mount_success()
|
||||
raise MountingFailed(
|
||||
"Mounting continued to fail after stopping automation, contact your local contact.",
|
||||
critical=True,
|
||||
)
|
||||
|
||||
if count == end_automation_after_fail_count:
|
||||
self.logger.error(f"Mounting failed {count} times, stopping automation")
|
||||
self.logger.error("Unmounting sample and drying")
|
||||
try:
|
||||
self._unmount_current_sample(timeout=60.0)
|
||||
self.dry(park=True)
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to dry after mount failure: {e}")
|
||||
raise MountingFailed(f"Mount failed {count} times in a row, stopping automation.", critical=True)
|
||||
|
||||
if count == dry_after_fail_count:
|
||||
if count == DRY_AFTER_FAIL_COUNT:
|
||||
self.logger.warning(f"Mount failed {count} times in a row; drying")
|
||||
try:
|
||||
self.dry(park=False)
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to dry after mount failure: {e}")
|
||||
|
||||
if count >= STOP_AFTER_FAIL_COUNT:
|
||||
self.logger.error(f"Mount failed {count} times in a row, stopping automation")
|
||||
self.logger.error("Unmounting sample and drying")
|
||||
try:
|
||||
self._unmount_current_sample(timeout=60.0)
|
||||
self.dry(park=True)
|
||||
except Exception as e:
|
||||
self.logger.exception(f"Failed to clean up after repeated mount failure: {e}")
|
||||
|
||||
raise MountingFailed(
|
||||
f"Mount failed {count} times in a row, stopping automation.",
|
||||
critical=True,
|
||||
)
|
||||
|
||||
def _mount_handler(self, target) -> None:
|
||||
self._prepare_mount_hardware()
|
||||
|
||||
@@ -122,7 +115,6 @@ class MountingService:
|
||||
raise CriticalTellException(f"Critical error in TELL mount: {message}")
|
||||
|
||||
def _prepare_mount_hardware(self) -> None:
|
||||
#self.ctx.devs.tell.check_command_ok()
|
||||
self.ctx.devs.smargon_move_home()
|
||||
self.ctx.devs.aerotech_pos = self.ctx.mount_position
|
||||
self._magnet_position_sensor_check(timeout=360.0)
|
||||
@@ -139,7 +131,7 @@ class MountingService:
|
||||
self.ctx.cfg.current_sample = None
|
||||
return previous_sample
|
||||
|
||||
def dry(self, *, park: bool, unmount:bool=False) -> None:
|
||||
def dry(self, *, park: bool, unmount: bool = False) -> None:
|
||||
self.ctx.devs.tell.check_enable_motion()
|
||||
self.ctx.devs.tell.wait_not_busy()
|
||||
self.ctx.devs.tell.set_in_mount_position(True)
|
||||
@@ -154,11 +146,9 @@ class MountingService:
|
||||
self.ctx.devs.tell.dry(wait=True)
|
||||
|
||||
def execute(self, *, target) -> MountingResult:
|
||||
|
||||
previous_sample = self.ctx.cfg.current_sample
|
||||
|
||||
try:
|
||||
|
||||
if target is None:
|
||||
try:
|
||||
unmounted_sample = self._unmount_current_sample(timeout=60.0)
|
||||
@@ -176,24 +166,14 @@ class MountingService:
|
||||
log_msg = f"Unmounting sample: {previous_sample}\nand {log_msg}"
|
||||
self.logger.debug(log_msg)
|
||||
|
||||
def _attempt_mount() -> None:
|
||||
try:
|
||||
self._mount_handler(target)
|
||||
except Exception:
|
||||
self._on_mount_failure()
|
||||
raise
|
||||
|
||||
with_retry(
|
||||
_attempt_mount,
|
||||
max_attempts=5,
|
||||
on=(MountingFailed,),
|
||||
escalate=MountingFailed,
|
||||
escalate_message="Mount failed 5 times in a row, stopping automation.",
|
||||
escalate_critical=True,
|
||||
)
|
||||
try:
|
||||
self._mount_handler(target)
|
||||
except (MountingFailed, TellCommunicationError):
|
||||
self._handle_consecutive_mount_failure()
|
||||
raise
|
||||
|
||||
self.logger.debug("Mounting succeeded, resetting mount failure counter")
|
||||
self.ctx.cfg.record_mount_success()
|
||||
self.ctx.cfg.reset_mount_failure_streak()
|
||||
self.ctx.cfg.current_sample = target
|
||||
|
||||
return MountingResult(
|
||||
|
||||
@@ -120,18 +120,40 @@ def register_exception_handlers(app) -> None:
|
||||
|
||||
@app.exception_handler(AutomationError)
|
||||
async def _handle_automation_error(request: Request, exc: AutomationError) -> JSONResponse:
|
||||
body = _error_body(exc)
|
||||
log = logger.error if exc.critical else logger.warning
|
||||
log(
|
||||
"AutomationError: %s",
|
||||
body["message"],
|
||||
extra={
|
||||
"exception_class": body["exception_class"],
|
||||
"code": body["code"],
|
||||
"critical": body["critical"],
|
||||
**body["context"],
|
||||
},
|
||||
)
|
||||
status = (
|
||||
api_status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
if exc.critical
|
||||
else api_status.HTTP_422_UNPROCESSABLE_ENTITY
|
||||
)
|
||||
return JSONResponse(status_code=status, content=_error_body(exc))
|
||||
return JSONResponse(status_code=status, content=body)
|
||||
|
||||
@app.exception_handler(AareUserError)
|
||||
async def _handle_user_error(request: Request, exc: AareUserError) -> JSONResponse:
|
||||
body = _error_body(exc)
|
||||
logger.info(
|
||||
"AareUserError: %s",
|
||||
body["message"],
|
||||
extra={
|
||||
"exception_class": body["exception_class"],
|
||||
"code": body["code"],
|
||||
**body["context"],
|
||||
},
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=api_status.HTTP_400_BAD_REQUEST,
|
||||
content=_error_body(exc),
|
||||
content=body,
|
||||
)
|
||||
|
||||
@app.exception_handler(AareAuthError)
|
||||
@@ -149,9 +171,19 @@ def register_exception_handlers(app) -> None:
|
||||
if isinstance(exc, AuthenticationException)
|
||||
else api_status.HTTP_403_FORBIDDEN
|
||||
)
|
||||
body = _error_body(exc, code_override=code_override)
|
||||
logger.warning(
|
||||
"AareAuthError: %s",
|
||||
body["message"],
|
||||
extra={
|
||||
"exception_class": body["exception_class"],
|
||||
"code": body["code"],
|
||||
"status_code": status,
|
||||
},
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=_error_body(exc, code_override=code_override),
|
||||
content=body,
|
||||
headers=getattr(exc, "headers", None),
|
||||
)
|
||||
|
||||
@@ -161,6 +193,12 @@ def register_exception_handlers(app) -> None:
|
||||
it internally (e.g. validation, route-not-found) and routes raise it
|
||||
manually. Adapt to the new body shape so clients see one schema."""
|
||||
detail = exc.detail
|
||||
log = logger.error if exc.status_code >= 500 else logger.warning
|
||||
log(
|
||||
"HTTPException %s: %s",
|
||||
exc.status_code,
|
||||
detail,
|
||||
)
|
||||
# Preserve legacy callers that passed a {code, message} dict
|
||||
if isinstance(detail, dict) and "code" in detail and "message" in detail:
|
||||
legacy_code = str(detail.get("code"))
|
||||
|
||||
@@ -15,7 +15,7 @@ from jfjoch_client import ScanResult, ScanResultImagesInner
|
||||
|
||||
from aare.common.auth_models import BatonStatus
|
||||
from aare.common.coordinate import SmargonCoordinate, AerotechCoordinate
|
||||
from aare.common.error_codes import export_error_codes, AuthErrorCode
|
||||
from aare.common.error_codes import export_error_codes, AareErrorCode, AuthErrorCode
|
||||
from aare.common.models import (
|
||||
DAQStatusModel,
|
||||
SampleShortInfoList,
|
||||
@@ -39,6 +39,7 @@ from aare.common.recurrence_watcher import (
|
||||
WatcherTrip,
|
||||
create_default_watchers,
|
||||
load_watcher_threshold_overrides,
|
||||
redis_key_to_env_var,
|
||||
resolve_exception_class,
|
||||
)
|
||||
from aare.common.raster_grid import RasterGridRequest, CompletedRasterGrid, CompletedRasterGridElem
|
||||
@@ -894,7 +895,7 @@ class DAQWorker(QObject):
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
else:
|
||||
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
|
||||
if self._is_critical_detector_failure(status, body_json, err_msg):
|
||||
@@ -948,7 +949,7 @@ class DAQWorker(QObject):
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
else:
|
||||
code = body_json.get("code", "") if isinstance(body_json, dict) else ""
|
||||
if code == "JFJOCH_UNAVAILABLE" or status == 503:
|
||||
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR or status == 503:
|
||||
self._emit_detector_message(f"JFJoch: {err_msg}", is_error=True)
|
||||
|
||||
if self._is_critical_detector_failure(status, body_json, err_msg):
|
||||
@@ -1092,7 +1093,7 @@ class DAQWorker(QObject):
|
||||
overrides = load_watcher_threshold_overrides(
|
||||
beamline=beamline,
|
||||
watcher_names=watcher_names,
|
||||
get_value=os.getenv,
|
||||
get_value=lambda k: os.getenv(redis_key_to_env_var(k)),
|
||||
)
|
||||
return create_default_watchers(overrides)
|
||||
|
||||
@@ -1205,7 +1206,7 @@ class DAQWorker(QObject):
|
||||
if status_int == 500:
|
||||
return True
|
||||
|
||||
if code == "JFJOCH_UNAVAILABLE" and status_int is not None and 500 <= status_int < 600:
|
||||
if code == AareErrorCode.JF_JOCH_COMMUNICATION_ERROR and status_int is not None and 500 <= status_int < 600:
|
||||
return True
|
||||
|
||||
if "daq state error" in text or "must be idle to start measurement" in text or "must be idle" in text:
|
||||
|
||||
@@ -130,3 +130,41 @@ def test_aerotech_communication_error():
|
||||
def test_magnet_position_sensor_error():
|
||||
exc = MagnetPositionSensorErorr()
|
||||
assert "Magnet position sensor error" in str(exc)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"factory",
|
||||
[
|
||||
lambda: TransformationInvalidException("t"),
|
||||
lambda: LoopCenteringFailed("lc"),
|
||||
lambda: UnmountingFailed("un"),
|
||||
lambda: MountingFailed("mt"),
|
||||
lambda: ManualMountException("mm"),
|
||||
lambda: SmartMagnetFaultException("sm"),
|
||||
lambda: TellCommandWhileBusyException("tb"),
|
||||
lambda: TellConnectionException("tc"),
|
||||
lambda: WarningTellException("wt"),
|
||||
lambda: CriticalTellException("ct"),
|
||||
lambda: AXCFailed("ax"),
|
||||
lambda: BeamlineBusyException("bb"),
|
||||
lambda: SampleException("se"),
|
||||
lambda: AuthenticationException("ae"),
|
||||
lambda: TellCommunicationError("tce", endpoint="/x", operation="GET"),
|
||||
lambda: SmargonCommunicationError("sce", endpoint="/m", status_code=500),
|
||||
lambda: JFJochCommunicationError("jce", operation="POST"),
|
||||
lambda: AareDBCommunicationError("dbce"),
|
||||
lambda: AerotechCommunicationError("ace"),
|
||||
lambda: MagnetPositionSensorErorr(),
|
||||
],
|
||||
)
|
||||
def test_exception_construction_emits_no_logs(factory, caplog):
|
||||
"""Constructing an Aare exception must not emit log records.
|
||||
|
||||
Logging is the responsibility of the catch site (server handler,
|
||||
best-effort wrapper, or callsite that decides to swallow). Emitting
|
||||
on construction double-logs and pollutes the WARNING-vs-ERROR split
|
||||
for best-effort flows."""
|
||||
caplog.clear()
|
||||
with caplog.at_level("DEBUG"):
|
||||
factory()
|
||||
assert caplog.records == []
|
||||
@@ -8,6 +8,7 @@ from aare.common.recurrence_watcher import (
|
||||
RecurrenceWatcher,
|
||||
create_default_watchers,
|
||||
load_watcher_threshold_overrides,
|
||||
redis_key_to_env_var,
|
||||
resolve_exception_class,
|
||||
)
|
||||
|
||||
@@ -54,4 +55,28 @@ def test_create_default_watchers_applies_overrides():
|
||||
def test_resolve_exception_class_maps_known_classes():
|
||||
assert resolve_exception_class("MountingFailed") is MountingFailed
|
||||
assert resolve_exception_class("LoopCenteringFailed") is LoopCenteringFailed
|
||||
assert resolve_exception_class("UnknownClass") is None
|
||||
assert resolve_exception_class("UnknownClass") is None
|
||||
|
||||
|
||||
def test_redis_key_to_env_var_converts_colons_and_uppercases():
|
||||
assert redis_key_to_env_var("aare:watchers:default:tell:threshold") == (
|
||||
"AARE_WATCHERS_DEFAULT_TELL_THRESHOLD"
|
||||
)
|
||||
assert redis_key_to_env_var("aare:watchers:mx:alc:threshold") == (
|
||||
"AARE_WATCHERS_MX_ALC_THRESHOLD"
|
||||
)
|
||||
|
||||
|
||||
def test_load_watcher_threshold_overrides_through_env_var_adapter(monkeypatch):
|
||||
monkeypatch.setenv("AARE_WATCHERS_DEFAULT_TELL_THRESHOLD", "9")
|
||||
monkeypatch.setenv("AARE_WATCHERS_DEFAULT_ALC_THRESHOLD", "bad")
|
||||
|
||||
import os
|
||||
|
||||
overrides = load_watcher_threshold_overrides(
|
||||
beamline="default",
|
||||
watcher_names=["tell", "alc", "smargon"],
|
||||
get_value=lambda k: os.getenv(redis_key_to_env_var(k)),
|
||||
)
|
||||
|
||||
assert overrides == {"tell": 9}
|
||||
@@ -52,11 +52,15 @@ def _make_context(previous_sample=None):
|
||||
magnet_position_sensor=types.SimpleNamespace(value=0),
|
||||
)
|
||||
|
||||
streak = {"count": 0}
|
||||
|
||||
cfg = types.SimpleNamespace(
|
||||
current_sample=previous_sample,
|
||||
get_mount_fail_count=lambda: 0,
|
||||
record_mount_success=lambda: None,
|
||||
record_mount_failure=lambda: 1,
|
||||
get_mount_failure_streak=lambda: streak["count"],
|
||||
increment_mount_failure_streak=lambda: streak.__setitem__("count", streak["count"] + 1) or streak["count"],
|
||||
reset_mount_failure_streak=lambda: streak.__setitem__("count", 0),
|
||||
record_mount_failure=lambda: streak.__setitem__("count", streak["count"] + 1) or streak["count"],
|
||||
record_mount_success=lambda: streak.__setitem__("count", 0),
|
||||
)
|
||||
|
||||
return MountingContext(
|
||||
@@ -143,17 +147,37 @@ def test_dry_unmounts_current_sample_before_drying(mock_logger):
|
||||
assert dry_calls == [{"wait_cold": -1, "wait": True}]
|
||||
|
||||
|
||||
def test_execute_mount_escalates_to_critical_after_retry_exhaustion(mock_logger):
|
||||
def test_execute_mount_triggers_dry_on_third_consecutive_failure(mock_logger):
|
||||
target_sample = _make_sample(2, "new")
|
||||
calls = {"count": 0}
|
||||
|
||||
def record_mount_failure():
|
||||
calls["count"] += 1
|
||||
return calls["count"]
|
||||
dry_calls = []
|
||||
|
||||
ctx = _make_context()
|
||||
ctx.cfg.record_mount_failure = record_mount_failure
|
||||
ctx.devs.tell.mount = lambda **kwargs: TellEventValueEnum.NO_PIN_IN_GRIPPER
|
||||
ctx.devs.tell.dry = lambda **kwargs: dry_calls.append(kwargs)
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
|
||||
service = MountingService(context=ctx, logger=mock_logger)
|
||||
|
||||
result = service.execute(target=target_sample)
|
||||
|
||||
assert result.success is False
|
||||
assert isinstance(result.error, MountingFailed)
|
||||
assert result.error.critical is False
|
||||
assert dry_calls == [{"wait": True}]
|
||||
|
||||
|
||||
def test_execute_mount_stops_automation_on_fifth_consecutive_failure(mock_logger):
|
||||
target_sample = _make_sample(2, "new")
|
||||
dry_calls = []
|
||||
|
||||
ctx = _make_context()
|
||||
ctx.devs.tell.mount = lambda **kwargs: TellEventValueEnum.NO_PIN_IN_GRIPPER
|
||||
ctx.devs.tell.dry = lambda **kwargs: dry_calls.append(kwargs)
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
|
||||
service = MountingService(context=ctx, logger=mock_logger)
|
||||
|
||||
@@ -163,3 +187,37 @@ def test_execute_mount_escalates_to_critical_after_retry_exhaustion(mock_logger)
|
||||
assert isinstance(result.error, MountingFailed)
|
||||
assert result.error.critical is True
|
||||
assert "Mount failed 5 times in a row" in str(result.error)
|
||||
assert {"wait_cold": -1, "wait": True} in dry_calls
|
||||
|
||||
|
||||
def test_execute_mount_success_resets_failure_streak(mock_logger):
|
||||
previous_sample = _make_sample(1, "old")
|
||||
target_sample = _make_sample(2, "new")
|
||||
ctx = _make_context(previous_sample=previous_sample)
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
ctx.cfg.increment_mount_failure_streak()
|
||||
|
||||
service = MountingService(context=ctx, logger=mock_logger)
|
||||
|
||||
result = service.execute(target=target_sample)
|
||||
|
||||
assert result.success is True
|
||||
assert ctx.cfg.get_mount_failure_streak() == 0
|
||||
|
||||
|
||||
def test_execute_mount_critical_tell_error_does_not_increment_failure_streak(mock_logger):
|
||||
target_sample = _make_sample(2, "new")
|
||||
ctx = _make_context()
|
||||
|
||||
def mount(**kwargs):
|
||||
raise CriticalTellException("critical tell problem")
|
||||
|
||||
ctx.devs.tell.mount = mount
|
||||
|
||||
service = MountingService(context=ctx, logger=mock_logger)
|
||||
|
||||
result = service.execute(target=target_sample)
|
||||
|
||||
assert result.success is False
|
||||
assert isinstance(result.error, CriticalTellException)
|
||||
assert ctx.cfg.get_mount_failure_streak() == 0
|
||||
@@ -279,6 +279,37 @@ def test_only_four_root_handlers_plus_fallbacks(app):
|
||||
REQUIRED_BODY_KEYS = {"critical", "code", "exception_class", "message", "context"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_automation_error_logs_error_when_critical(app, mock_request, caplog):
|
||||
exc = MountingFailed("threshold breached", critical=True)
|
||||
handler = app.exception_handlers[AutomationError]
|
||||
caplog.clear()
|
||||
with caplog.at_level("DEBUG"):
|
||||
await handler(mock_request, exc)
|
||||
assert any(r.levelname == "ERROR" for r in caplog.records), caplog.records
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_automation_error_logs_warning_when_not_critical(app, mock_request, caplog):
|
||||
exc = MountingFailed("benign")
|
||||
handler = app.exception_handlers[AutomationError]
|
||||
caplog.clear()
|
||||
with caplog.at_level("DEBUG"):
|
||||
await handler(mock_request, exc)
|
||||
assert any(r.levelname == "WARNING" for r in caplog.records), caplog.records
|
||||
assert not any(r.levelname == "ERROR" for r in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_error_logs_info(app, mock_request, caplog):
|
||||
exc = ManualMountException("user must intervene")
|
||||
handler = app.exception_handlers[AareUserError]
|
||||
caplog.clear()
|
||||
with caplog.at_level("DEBUG"):
|
||||
await handler(mock_request, exc)
|
||||
assert any(r.levelname == "INFO" for r in caplog.records), caplog.records
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_body_shape_uniform(app, mock_request):
|
||||
cases: list[tuple[type, Exception]] = [
|
||||
|
||||
Reference in new issue
Block a user