From 2fa0b2a6e661182575828099f32ef8fd4529021f Mon Sep 17 00:00:00 2001 From: appleb_m Date: Fri, 27 Feb 2026 16:26:01 +0100 Subject: [PATCH] DAQ/GUI: trying to make error handling in satatus pollign more robust. WIP --- src/aare/common/models.py | 6 ++ src/aare/daq/daq.py | 56 +++++++++++++++++- src/aare/daq/server.py | 10 +--- src/aare/gui/threads/daq_worker.py | 92 ++++++++++++++++++++++++------ 4 files changed, 137 insertions(+), 27 deletions(-) diff --git a/src/aare/common/models.py b/src/aare/common/models.py index 892f48bf..27955d96 100644 --- a/src/aare/common/models.py +++ b/src/aare/common/models.py @@ -719,6 +719,12 @@ class DAQStatusModel(BaseModel): last_best_b_factor: float | None = None crystal_size: CrystalSize = CrystalSize(x=0,y=0,z=0) + tell_connected: bool = True + tell_error: str | None = None + + smargon_connected: bool = True + smargon_error: str | None = None + class BeamlineSettingsModel(BaseModel): dtz_max: float | None = 1600.0 dtz_min: float | None = 120.0 diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index e6752483..a30e7b44 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -42,7 +42,7 @@ from aare.common.exception_handler import ( MountingFailed, WarningTellException, CriticalTellException, - AXCFailed, + AXCFailed, SmargonCommunicationError, TellCommunicationError, ) logger = setup_logger("aareDAQ") @@ -1418,8 +1418,62 @@ class AareDAQ: dtz_max=1000, ) + def _safe_sample(self) -> tuple[SampleShortInfo | None, bool, str | None]: + """ + Return (sample, tell_connected, tell_error) without raising. + """ + try: + return self.sample, True, None + except TellCommunicationError as e: + return None, False, str(e) + except Exception as e: + # Keep status flowing even if Tell code throws something unexpected + return None, False, f"TELL unavailable: {e}" + + def _safe_geom(self) -> tuple[SampleGeometryModel, bool, str | None]: + """ + Return (geom, smargon_connected, smargon_error) without raising. + Uses a conservative fallback geometry if Smargon access fails. + """ + try: + return self.sample_geometry, True, None + except SmargonCommunicationError as e: + zoom = self.__devs.zoom + fallback = SampleGeometryModel( + beam_location_pxl=self.__cfg.beam_mark_coeff.apply(zoom), + pixel_in_mm=self.__cfg.pixel_to_mm(zoom), + omega_deg=self.__devs.aerotech_omega, + smargon=SmargonCoordinate( + sh_mm=Coordinate(x=0.0, y=0.0, z=0.0), + phi_deg=0.0, + chi_deg=0.0, + ), + beam_size_mm=self.__cfg.beam_size_mm, + aerotech=Coordinate(x=0.0, y=0.0, z=0.0), + aerotech_meas=Coordinate(x=0.0, y=0.0, z=0.0), + ) + return fallback, False, str(e) + except Exception as e: + zoom = self.__devs.zoom + fallback = SampleGeometryModel( + beam_location_pxl=self.__cfg.beam_mark_coeff.apply(zoom), + pixel_in_mm=self.__cfg.pixel_to_mm(zoom), + omega_deg=self.__devs.aerotech_omega, + smargon=SmargonCoordinate( + sh_mm=Coordinate(x=0.0, y=0.0, z=0.0), + phi_deg=0.0, + chi_deg=0.0, + ), + beam_size_mm=self.__cfg.beam_size_mm, + aerotech=Coordinate(x=0.0, y=0.0, z=0.0), + aerotech_meas=Coordinate(x=0.0, y=0.0, z=0.0), + ) + return fallback, False, f"Smargon unavailable: {e}" + @property def status(self) -> DAQStatusModel: + safe_sample, tell_ok, tell_err = self._safe_sample() + safe_geom, smargon_ok, smargon_err = self._safe_geom() # session should be set by FastAPI server return DAQStatusModel( state=self.state, diff --git a/src/aare/daq/server.py b/src/aare/daq/server.py index adbfd7bf..fbbb875c 100644 --- a/src/aare/daq/server.py +++ b/src/aare/daq/server.py @@ -55,19 +55,11 @@ async def login(form_data: OAuth2PasswordRequestForm = Depends()): return {"access_token": data, "token_type": "bearer"} @app.get("/meta/error-codes") -async def meta_error_codes() -> dict[str, str]: +async def meta_error_codes() -> dict[str, dict[str, str]]: """ Public, stable registry of machine-readable error codes. Useful for GUIs, tests, and diagnostics. """ - return export_error_codes() - -@app.get("/meta/error-codes/grouped") -async def meta_error_codes_grouped() -> dict[str, dict[str, str]]: - """ - Grouped registry of machine-readable error codes, by enum class. - Preferred for GUIs that want multiple code categories without collisions. - """ return export_error_codes_grouped() @app.get("/status") diff --git a/src/aare/gui/threads/daq_worker.py b/src/aare/gui/threads/daq_worker.py index c527a7fe..33ea2eca 100644 --- a/src/aare/gui/threads/daq_worker.py +++ b/src/aare/gui/threads/daq_worker.py @@ -2,6 +2,7 @@ import copy import random import time import json +from collections import deque from PySide6.QtCore import Signal, QUrl, Slot, QTimer, QObject, QByteArray from PySide6.QtNetwork import QNetworkAccessManager, QNetworkRequest, QNetworkReply @@ -26,6 +27,7 @@ class DAQWorker(QObject): spreadsheet = Signal(SampleShortInfoList) reference_tools = Signal(SampleShortInfoList) http_error = Signal(str) + status_message = Signal(str, bool) auth_error = Signal() sample_missing = Signal(str) automated_scan_done = Signal(int, bool, str) # sample ID, success @@ -40,6 +42,7 @@ class DAQWorker(QObject): error_codes_loaded = Signal(dict) last_error_payload_changed = Signal(dict) + last_error_payloads_changed = Signal(list) def __init__(self, base_url: str | None, token: str, parent=None): super().__init__(parent) @@ -56,17 +59,51 @@ class DAQWorker(QObject): self._auth_error_min_interval = 10.0 self._last_status_request_ts = 0.0 - self._status_request_min_interval = 10.0 + self._status_request_min_interval = 0.5 + self._smargon_retry_interval_s = 2.0 + self._smargon_log_min_interval_s = 10.0 + self._last_status_error = None + self._smargon_error_active = False + self._last_smargon_log_ts = 0.0 + self._last_smargon_log_key: str | None = None self._last_error_payload: dict = {} + self._last_error_payloads = deque(maxlen=10) + + self._last_tell_connected: bool | None = None + self._last_smargon_connected: bool | None = None def get_last_error_payload(self) -> dict: return dict(self._last_error_payload or {}) + def get_last_error_payloads(self) -> list[dict]: + return [dict(p or {}) for p in list(self._last_error_payloads)] + def _set_last_error_payload(self, payload: dict) -> None: - self._last_error_payload = payload or {} + payload = payload or {} + self._last_error_payload = payload + self._last_error_payloads.append(payload) + self.last_error_payload_changed.emit(self.get_last_error_payload()) + self.last_error_payloads_changed.emit(self.get_last_error_payloads()) + + def _log_smargon_throttled(self, *, endpoint: str | None, message: str) -> None: + """ + Log immediately if endpoint/message changed; otherwise at most every N seconds. + """ + key = f"{endpoint or ''}|{message or ''}" + now = time.monotonic() + + if self._last_smargon_log_key != key: + self._last_smargon_log_key = key + self._last_smargon_log_ts = now + logger.error(f"Smargon connection error (endpoint={endpoint}): {message}") + return + + if now - self._last_smargon_log_ts >= self._smargon_log_min_interval_s: + self._last_smargon_log_ts = now + logger.error(f"Smargon connection error (endpoint={endpoint}): {message}") @Slot() def regular_update(self): @@ -80,6 +117,11 @@ class DAQWorker(QObject): if self.__base_url is None: return + now = time.monotonic() + if now - self._last_status_request_ts < self._status_request_min_interval: + return + self._last_status_request_ts = now + request = QNetworkRequest(QUrl(f"{self.__base_url}/status")) request.setRawHeader(b"Authorization", f"Bearer {self.__token}".encode("utf-8")) reply = self.__net_manager.get(request) @@ -102,16 +144,31 @@ class DAQWorker(QObject): response_data = self.handle_response(reply) parsed_response = DAQStatusModel.model_validate_json(response_data) self.update.emit(parsed_response) + + tell_conn = bool(getattr(parsed_response, "tell_connected", True)) + smargon_conn = bool(getattr(parsed_response, "smargon_connected", True)) + + if self._last_tell_connected is None: + self._last_tell_connected = tell_conn + elif self._last_tell_connected != tell_conn: + self._last_tell_connected = tell_conn + if tell_conn: + self.status_message.emit("TELL connection restored.", False) + else: + self.status_message.emit("TELL connection error, please inform your local contact.", True) + + if self._last_smargon_connected is None: + self._last_smargon_connected = smargon_conn + elif self._last_smargon_connected != smargon_conn: + self._last_smargon_connected = smargon_conn + if smargon_conn: + self.status_message.emit("Smargon connection restored.", False) + else: + self.status_message.emit("Smargon connection error, please inform your local contact.", True) + except Exception as e: - now = time.monotonic() - if str(e) == str(self._last_status_error): - if now - self._last_auth_error_log_ts > self._auth_error_min_interval: - self._last_auth_error_log_ts = now - logger.error(f"Exception from status response: {e}") - else: - self._last_status_error = e - self._last_auth_error_log_ts = now - logger.error(f"Exception from status response: {e}") + # keep your existing throttled logging behavior here if you want + logger.error(f"Exception from status response: {e}") self.http_error.emit(str(e)) @Slot(QNetworkReply) @@ -719,24 +776,26 @@ class DAQWorker(QObject): out[str(k)] = str(v) return out + @Slot() @Slot() def get_error_codes(self) -> None: """ Fetch server error codes registry for developer/help UI. Emits error_codes_loaded(dict). + + Server default is grouped. We flatten grouped payloads for existing UI. """ if self.__base_url is None: self.error_codes_loaded.emit(export_error_codes()) return - # Prefer grouped (newer servers), but we'll gracefully fall back if missing. - request = QNetworkRequest(QUrl(f"{self.__base_url}/meta/error-codes/grouped")) + request = QNetworkRequest(QUrl(f"{self.__base_url}/meta/error-codes")) request.setRawHeader(b"Authorization", f"Bearer {self.__token}".encode("utf-8")) reply = self.__net_manager.get(request) reply.finished.connect(lambda: self._handle_error_codes_response(reply)) def _retry_error_codes_legacy(self) -> None: - request = QNetworkRequest(QUrl(f"{self.__base_url}/meta/error-codes")) + request = QNetworkRequest(QUrl(f"{self.__base_url}/meta/error-codes/flat")) request.setRawHeader(b"Authorization", f"Bearer {self.__token}".encode("utf-8")) reply = self.__net_manager.get(request) reply.finished.connect(lambda: self._handle_error_codes_response(reply)) @@ -746,15 +805,14 @@ class DAQWorker(QObject): try: status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute) - # If the grouped endpoint isn't available on this server, fall back to legacy. try: url = reply.request().url().toString() except Exception: url = "" - if int(status) == 404 and url.endswith("/meta/error-codes/grouped"): + if int(status) == 404 and url.endswith("/meta/error-codes"): reply.deleteLater() - self._retry_error_codes_legacy() + logger.error(f"Error codes not found on server.") return payload = self.handle_response(reply)