mx/AareDAQ
0
1
Fork 0
Code Issues 49 Pull Requests Actions Packages Projects Releases 84 Wiki Activity

DAQ: fix bug in TransformationInvalidException handling and pass this to GUI to stop automation if critical error occurs

Browse Source
This commit is contained in:
appleb_m
2026-05-01 14:28:20 +02:00
parent 16fe6e9ef4
commit f4718d396c
6 changed files with 198 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/aare/common/error_codes.py
+6
View File
@@ -35,6 +35,7 @@ class DAQErrorCode(StrEnum):
LOOP_CENTERING_FAILED = "LOOP_CENTERING_FAILED"
AXC_FAILED = "AXC_FAILED"
TRANSFORMATION_INVALID = "TRANSFORMATION_INVALID"
AUTOMATION_CRITICAL = "AUTOMATION_CRITICAL"
SAMPLE_NOT_FOUND = "SAMPLE_NOT_FOUND"
DATA_COLLECTION_FAILED = "DATA_COLLECTION_FAILED"
RASTER_SCAN_FAILED = "RASTER_SCAN_FAILED"
@@ -52,6 +53,7 @@ class DAQErrorCode(StrEnum):
AAREDB_UNAVAILABLE = "AAREDB_UNAVAILABLE"
MAGNET_POSITION_SENSOR_ERROR = "MAGNET_POSITION_SENSOR_ERROR"
_ERROR_CODE_HELP: dict[str, str] = {
# Auth/JWT
AuthErrorCode.AUTHENTICATION_ERROR: (
@@ -110,6 +112,10 @@ _ERROR_CODE_HELP: dict[str, str] = {
DAQErrorCode.TRANSFORMATION_INVALID: (
"The requested beamline state transition is not allowed from the current state."
),
DAQErrorCode.AUTOMATION_CRITICAL: (
"A critical failure occurred during fully automated measurement. The GUI should stop "
"dispatching further queue items and staff/operator recovery may be required."
),
DAQErrorCode.SAMPLE_NOT_FOUND: (
"The requested sample could not be found in the available sample lists."
),
src/aare/daq/daq.py
+37 -5
View File
@@ -461,11 +461,21 @@ class AareDAQ:
self.__mount(sample)
if previous_sample is not None and previous_sample.db_id is not None:
self.__aare.send_sample_event(previous_sample, SampleEventType.UNMOUNTED)
self.__set_state(BeamlineStateEnum.SampleAlignment)
if sample is not None and sample.db_id is not None:
self.__aare.send_sample_event(sample, SampleEventType.MOUNTED)
self.save_screenshot_db(sample.db_id, f"{sample.db_id}_mounted")
self.__set_state(BeamlineStateEnum.SampleAlignment)
self.__set_state(BeamlineStateEnum.SampleAlignment)
return True
except TransformationInvalidException as e:
logger.error(f"Mount failed due to invalid transformation: {e}")
self._handle_operation_error(
operation=DAQOperation.MOUNT,
sample=sample,
error=e,
event_type=SampleEventType.MOUNTFAILED,
)
raise
except Exception as e:
logger.error(f"Mount failed: {e}")
try:
@@ -1273,19 +1283,26 @@ class AareDAQ:
try:
logger.debug(f"Mount target {target}")
if target is None:
logger.debug("Unmounting sample")
type = "Unmount"
else:
logger.debug(f"Mounting sample: {target}")
type = "Mount"
if not self._execute_mount_and_prepare(target):
raise MountingFailed(f"Failed to {type} sample {target or self.__cfg.current_sample}")
logger.info(f"Sample mounted: {target}")
current_sample = target or self.__cfg.current_sample
if target is None:
raise UnmountingFailed(f"Failed to {type} sample {current_sample}")
raise MountingFailed(f"Failed to {type} sample {current_sample}")
logger.info(f"Sample operation completed: {target}")
self.__cfg.state_busy = False
except Exception as e:
self.__cfg.state_busy = False
logger.debug(f"Failed to mount sample: {e}")
logger.debug(f"Failed to change mounted sample: {e}")
raise
@property
@@ -2602,6 +2619,18 @@ class AareDAQ:
if error:
msg += f"with an error"
self.__set_state(BeamlineStateEnum.RobotSampleExchange)
try:
if self.__cfg.state_busy:
self.__set_state(BeamlineStateEnum.RobotSampleExchange)
else:
logger.warning(
"Skipping recovery transition to RobotSampleExchange: "
"beamline is no longer busy (busy was released earlier)."
)
except Exception:
logger.exception(
"Failed to transition to RobotSampleExchange during error recovery"
)
else:
msg += f"successfully"
logger.error(f"{msg}, time taken {time.perf_counter() - start} seconds.")
@@ -2777,6 +2806,9 @@ class AareDAQ:
if target == BeamlineStateEnum.Maintenance:
self.__cfg.state = BeamlineStateEnum.Maintenance
elif target == curr_state:
logger.debug(f"State already set to {target}")
return
elif target != curr_state:
self.__cfg.state = BeamlineStateEnum.Moving
try:
@@ -2890,10 +2922,10 @@ class AareDAQ:
except TransformationInvalidException as e:
logger.error(f"Cannot go from {curr_state} to {target}: {e}")
self.__cfg.state = curr_state
self.__cfg.state_busy = False
raise
except Exception as e:
self.__cfg.state = BeamlineStateEnum.Maintenance
#TODO check if this is the right thing to do
self.__cfg.state_busy = False
raise
src/aare/daq/server.py
+12 -2
View File
@@ -13,7 +13,7 @@ import uvicorn
from aare.common.coordinate import AerotechCoordinate
from aare.common.auth_models import BatonStatus, BatonRequestStatus
from aare.common.coordinate import SmargonCoordinate, Coordinate
from aare.common.error_codes import export_error_codes_grouped
from aare.common.error_codes import export_error_codes_grouped, DAQErrorCode
from aare.common.logger_config import setup_logger, get_uvicorn_logging_config
from aare.common.models import SampleShortInfo, DAQStatusModel, BeamlineStateEnum, BeamlineSettingsModel, \
SampleShortInfoList, SessionStatus, SampleCameraSettings, AutofocusSettings, TokenData, \
@@ -1279,7 +1279,17 @@ async def auto(s: SampleShortInfo, token: str = Depends(oauth2_scheme)):
Formatted string of the total runtime.
"""
auth.check_jwt_rw(cfg, auth.parse_token(token))
runtime = daq.measure(s)
try:
runtime = daq.measure(s)
except Exception as e:
logger.exception("Critical automation failure in /scan/auto")
raise HTTPException(
status_code=api_status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={
"code": DAQErrorCode.AUTOMATION_CRITICAL.value,
"message": str(e) or "Critical automation failure",
},
) from e
return f"{runtime:0.3f}"
src/aare/gui/main_window.py
+84
View File
@@ -620,6 +620,7 @@ class MainWindow(QMainWindow):
self.daq.standard_scan_completed.connect(self.rotation.scan_completed)
self.daq.automated_scan_done.connect(self.job_list_panel.automated_scan_done)
self.daq.automation_critical_failure.connect(self._on_automation_critical_failure)
self.daq.raster_scan_completed.connect(self.raster.grid_scan_completed)
self.manual_sample_panel.sample_manual.connect(self.daq.sample_manual)
@@ -1057,6 +1058,88 @@ class MainWindow(QMainWindow):
self._dev_help_dialog.raise_()
self._dev_help_dialog.activateWindow()
@Slot(str)
def _on_automation_critical_failure(self, message: str) -> None:
"""
Hard-stop handler for fatal automation failures coming from the server
(transformation errors, busy-flag corruption, unhandled 500s on /scan/auto).
Stops the queue, marks the progress widget as failed and surfaces the
beamline recovery dialog (staff) or a critical error message (users).
"""
logger.critical(f"Automation critical failure: {message}")
# 1. Stop the sample queue so we don't keep dispatching new samples
try:
if self.job_list_panel is not None and self.job_list_panel.is_running():
self.job_list_panel.pause_automation()
if self.daq is not None:
self.daq.send_status_request()
except Exception as e:
logger.error(f"Failed to pause automation queue after critical failure: {e}")
# 2. Mark the automation progress widget as finished-with-error so
# _is_automation_active() returns False and idle/close timers behave.
try:
from aare.common.automation_models import (
AutomationProgress,
StepState,
StepStatus,
WorkflowStateKind,
)
progress = getattr(self.automation_progress_panel, "_progress", None)
if progress is None:
progress = AutomationProgress(
current_step=None,
steps=[
StepState(step=WorkflowStateKind.MOUNT, status=StepStatus.PENDING),
StepState(step=WorkflowStateKind.LOOP_CENTRE, status=StepStatus.PENDING),
StepState(step=WorkflowStateKind.RASTER, status=StepStatus.PENDING),
StepState(step=WorkflowStateKind.DATA_COLLECTION, status=StepStatus.PENDING),
StepState(step=WorkflowStateKind.FINAL, status=StepStatus.PENDING),
],
finished=False,
success=None,
)
for step in progress.steps:
if step.status == StepStatus.RUNNING:
step.status = StepStatus.FAILED
step.message = message
if step.step == WorkflowStateKind.FINAL:
step.status = StepStatus.FAILED
step.message = message
progress.finished = True
progress.success = False
self.automation_progress_panel.set_progress(progress)
except Exception as e:
logger.error(f"Failed to update automation progress after critical failure: {e}")
# 3. Banner so the operator sees it immediately
try:
self.alert_banner.show_message(
f"Automation halted: {message}", True, auto_clear_ms=0
)
except Exception:
pass
# 4. Surface recovery UI
try:
if bool(getattr(self.__decoded_token, "staff", False)):
self.show_beamline_recovery()
else:
QMessageBox.critical(
self,
"Automation halted",
(
"A critical error occurred during automation and the "
"beamline could not recover automatically:\n\n"
f"{message}\n\n"
"Please contact your local contact to recover the beamline."
),
)
except Exception as e:
logger.error(f"Failed to surface recovery UI after critical failure: {e}")
def show_beamline_recovery(self) -> None:
if not bool(getattr(self.__decoded_token, "staff", False)):
return
@@ -1075,6 +1158,7 @@ class MainWindow(QMainWindow):
logger.warning(f"Automation: Sample missing: {msg}")
return
QMessageBox.warning(self, "No Sample", f"{msg}")
#TODO tidy up mount and sampel view fucntions
@Slot()
def mount_view(self):
src/aare/gui/panels/sample_queue_panel.py
+9 -3
View File
@@ -243,8 +243,7 @@ class SampleQueuePanel(QFrame):
self._current_db_id = next_item.db_id
self.auto_scan.emit(next_item)
else:
self.pause_automation()
self.unmount.emit()
self._finish_empty_queue()
else:
if reply == "Warning":
@@ -267,9 +266,16 @@ class SampleQueuePanel(QFrame):
"\n If not please continue with new "
"samples.\nOtherwise contact your"
" local contact for support")
self.show_error_dialog(title=reply, msg="Critical Error", info="Repeated mount failuers.\n"
"Please check to see if pins"
" are loaded in these postions."
"\n If not please continue with new "
"samples.\nOtherwise contact your"
" local contact for support")
elif reply == "Authentication Error":
self.pause_automation()
self.show_error_dialog(title=reply,msg="Authentication Error", info="Please take the session to continue")
self.show_error_dialog(title=reply, msg="Authentication Error",
info="Please take the session to continue")
else:
self.table_model.remove_sample(db_id)
self._current_db_id = None
src/aare/gui/threads/daq_worker.py
+50 -5
View File
@@ -11,7 +11,7 @@ from jfjoch_client import ScanResult, ScanResultImagesInner
from aare.common.coordinate import SmargonCoordinate, Coordinate, AerotechCoordinate
from aare.common.auth_models import BatonStatus
from aare.common.coordinate import SmargonCoordinate, Coordinate
from aare.common.error_codes import export_error_codes
from aare.common.error_codes import export_error_codes, DAQErrorCode
from aare.common.exception_handler import JFJochCommunicationError
from aare.common.models import (
DAQStatusModel,
@@ -68,6 +68,7 @@ class DAQWorker(QObject):
fluorimeter_spectrum_update = Signal(FluorescenceSpectrumOutputModel)
sample_resync_completed = Signal(str)
automation_critical_failure = Signal(str)
error_codes_loaded = Signal(dict)
last_error_payload_changed = Signal(dict)
last_error_payloads_changed = Signal(list)
@@ -1024,6 +1025,31 @@ class DAQWorker(QObject):
reply = self.__net_manager.get(request)
reply.finished.connect(lambda: self.handle_reference_tools_response(reply))
@staticmethod
def _is_critical_automation_failure(
status: int | None,
body_json: dict | None = None,
) -> bool:
"""Detect server-side critical automation errors that require operator recovery."""
try:
status_int = int(status) if status is not None else None
except Exception:
status_int = None
code = None
if isinstance(body_json, dict):
code = body_json.get("code")
if code == DAQErrorCode.AUTOMATION_CRITICAL.value:
return True
# Conservative fallback for /scan/auto: an unstructured 5xx means the
# server raised unexpectedly during automation.
if status_int is not None and 500 <= status_int < 600 and code is None:
return True
return False
def handle_auto_scan_response(self, reply, sample_id: int):
if reply.error() == QNetworkReply.NetworkError.NoError:
resp = reply.readAll().data().decode("utf-8")
@@ -1032,12 +1058,18 @@ class DAQWorker(QObject):
else:
status = reply.attribute(QNetworkRequest.Attribute.HttpStatusCodeAttribute)
err_str = reply.errorString()
body_json = None
try:
response_body = reply.readAll().data().decode("utf-8")
if response_body:
body_json = json.loads(response_body)
if "detail" in body_json:
err_str = body_json["detail"]
parsed_body = json.loads(response_body)
if isinstance(parsed_body, dict):
body_json = parsed_body
err_str = (
body_json.get("message")
or body_json.get("detail")
or response_body
)
else:
err_str = response_body
except Exception:
@@ -1056,6 +1088,11 @@ class DAQWorker(QObject):
elif status == 417:
self.sample_missing.emit(err_str)
self.automated_scan_done.emit(sample_id, False, "Critical")
elif self._is_critical_automation_failure(status, body_json):
logger.critical(f"Critical automation failure: {err_str}")
self.http_error.emit(err_str)
self.automated_scan_done.emit(sample_id, False, "Critical")
self.automation_critical_failure.emit(err_str)
else:
logger.error(f"Error in auto scan: {err_str}")
self.http_error.emit(err_str)
@@ -1208,12 +1245,20 @@ class DAQWorker(QObject):
def _parse_automation_progress(progress_payload: dict) -> AutomationProgress:
steps: list[StepState] = []
step_aliases = {
"final": WorkflowStateKind.FINAL,
"Paused/Finished": WorkflowStateKind.FINAL,
}
for raw_step in progress_payload.get("steps", []):
raw_kind = raw_step.get("step")
raw_status = raw_step.get("status", StepStatus.PENDING.value)
raw_message = raw_step.get("message", "")
step_kind = WorkflowStateKind(raw_kind)
step_kind = step_aliases.get(raw_kind)
if step_kind is None:
step_kind = WorkflowStateKind(raw_kind)
step_status = StepStatus(raw_status)
steps.append(
StepState(