diff --git a/src/aare/daq/daq.py b/src/aare/daq/daq.py index b6067e6b..61faa431 100644 --- a/src/aare/daq/daq.py +++ b/src/aare/daq/daq.py @@ -74,7 +74,8 @@ from aare.common.exception_handler import ( from aare.devices.tell_client import TellEventValueEnum logger = setup_logger("aareDAQ") -#TODO tidy up DAQ - migrate functions itno different scripts, to reduce size? +#TODO tidy up DAQ - migrate functions into different scripts, to reduce size? +#TODO investigate using a state machine within each operation to reduce callbacks? class AareDAQ: """ Main Data Acquisition class for the Aare system. @@ -99,6 +100,10 @@ class AareDAQ: self._automation_progress_cb: Callable[[AutomationProgress], None] | None = None self._last_sample_sync_ts = 0.0 self._sample_sync_min_interval_s = 2.0 + self._automation_completed_samples = 0 + self._automation_total_sample_time_s = 0.0 + self._automation_last_sample_name = "" + self._automation_samples_in_queue = 0 def _is_hardware_failure(self, error: Exception) -> bool: return isinstance( @@ -197,6 +202,12 @@ class AareDAQ: ], finished=False, success=None, + samples_in_queue=self._automation_samples_in_queue, + avg_time_per_sample=( + self._automation_total_sample_time_s / self._automation_completed_samples + if self._automation_completed_samples > 0 else 0.0 + ), + current_sample_name=self._automation_last_sample_name, ) @staticmethod @@ -210,6 +221,55 @@ class AareDAQ: } return labels.get(step, str(step.value)) + def _get_progress_step( + self, + progress: AutomationProgress, + step: WorkflowStateKind, + ) -> StepState | None: + for item in progress.steps: + if item.step == step: + return item + return None + + def _set_progress_context( + self, + progress: AutomationProgress, + *, + current_sample_name: str | None = None, + samples_in_queue: int | None = None, + ) -> None: + if current_sample_name is not None: + safe_name = str(current_sample_name or "") + progress.current_sample_name = safe_name + self._automation_last_sample_name = safe_name + + if samples_in_queue is not None: + safe_count = max(0, int(samples_in_queue)) + progress.samples_in_queue = safe_count + self._automation_samples_in_queue = safe_count + else: + progress.samples_in_queue = max(0, int(self._automation_samples_in_queue)) + + if self._automation_completed_samples > 0: + progress.avg_time_per_sample = ( + self._automation_total_sample_time_s / self._automation_completed_samples + ) + else: + progress.avg_time_per_sample = 0.0 + + def _record_completed_sample_time( + self, + progress: AutomationProgress, + elapsed_s: float, + ) -> None: + if elapsed_s <= 0: + return + self._automation_completed_samples += 1 + self._automation_total_sample_time_s += float(elapsed_s) + progress.avg_time_per_sample = ( + self._automation_total_sample_time_s / self._automation_completed_samples + ) + def _set_progress_step( self, progress: AutomationProgress, @@ -218,11 +278,29 @@ class AareDAQ: message: str = "", *, make_current: bool = False, + error_code: str | None = None, ) -> None: + now = time.time() + for item in progress.steps: if item.step == step: item.status = status item.message = message + item.error_code = error_code + + if status == StepStatus.RUNNING: + if item.started_at is None: + item.started_at = now + item.completed_at = None + elif status in { + StepStatus.SUCCESS, + StepStatus.FAILED, + StepStatus.SKIPPED, + StepStatus.PAUSED, + }: + if item.started_at is None: + item.started_at = now + item.completed_at = now break if make_current: @@ -264,8 +342,15 @@ class AareDAQ: StepStatus.FAILED, message, make_current=True, + error_code=message or None, + ) + self._set_progress_step( + progress, + WorkflowStateKind.FINAL, + StepStatus.FAILED, + message, + error_code=message or None, ) - self._set_progress_step(progress, WorkflowStateKind.FINAL, StepStatus.FAILED, message) progress.current_step = self._step_display_name(WorkflowStateKind.FINAL) progress.finished = True progress.success = False @@ -1150,7 +1235,7 @@ class AareDAQ: self.__devs.tell.wait_not_busy() self.__devs.tell.set_in_mount_position(True) self.__devs.tell.unmount(wait=True, timeout=60.0) - self.__devs.tell.dry(wait_cold=-1, wait=True, timeout=360.0) + self.__devs.tell.dry(wait_cold=-1, wait=True) self.__cfg.current_sample = None self.__cfg.state_busy = False except Exception as e: @@ -2670,7 +2755,12 @@ class AareDAQ: Total time taken for the measurement (seconds) """ start = time.perf_counter() + sample_started_at = time.time() progress = self._new_automation_progress() + self._set_progress_context( + progress, + current_sample_name=getattr(sample, "sample_name", "") or "", + ) self._emit_automation_progress(progress) formatted_date = datetime.now().strftime('%Y%m%d') @@ -2686,6 +2776,11 @@ class AareDAQ: self.__cfg.try_set_busy(timeout=360) logger.info(f"set busy at {time.perf_counter() - start}") + self._set_progress_context( + progress, + current_sample_name=getattr(sample, "sample_name", "") or "", + ) + self._mark_progress_running(progress, WorkflowStateKind.MOUNT, "Mounting sample") if not self._execute_mount_and_prepare(sample): self._mark_progress_failed(progress, WorkflowStateKind.MOUNT, "Mount failed") @@ -2803,6 +2898,7 @@ class AareDAQ: raise Exception(f"Critical Error in automation: {e}") from e + self._record_completed_sample_time(progress, time.time() - sample_started_at) self._mark_progress_finished(progress, True, "Automation complete") return self._end_operation(start, DAQOperation.AUTOMATION, error=False) diff --git a/src/aare/gui/main_window.py b/src/aare/gui/main_window.py index 6006b205..81337b70 100644 --- a/src/aare/gui/main_window.py +++ b/src/aare/gui/main_window.py @@ -320,8 +320,14 @@ class MainWindow(QMainWindow): self.tabifyDockWidget(self.manual_sample_dock, self.automation_progress_dock) self.tabifyDockWidget(self.automation_progress_dock, self.log_dock) + self.job_list_panel.samples_in_queue_changed.connect( + self.automation_progress_panel.set_samples_in_queue + ) + self.automation_progress_panel.set_samples_in_queue( + len(self.job_list_panel.table_model.samples) + ) - #smargon trace panel + # smargon trace panel self.smargon_trace_panel = SmargonTracePanel() self.smargon_trace_dock = QDockWidget("Smargon trace", self) self.smargon_trace_dock.setObjectName("smargon_trace_dock") diff --git a/src/aare/gui/panels/automation_panel.py b/src/aare/gui/panels/automation_panel.py index df589c73..bdea5347 100644 --- a/src/aare/gui/panels/automation_panel.py +++ b/src/aare/gui/panels/automation_panel.py @@ -1,15 +1,9 @@ -""" -AutomationProgressWidget - -Displays: -- Current step progress -""" - from __future__ import annotations import time +from datetime import datetime -from PySide6.QtCore import Slot +from PySide6.QtCore import Slot, QTimer from PySide6.QtWidgets import ( QWidget, QVBoxLayout, @@ -29,12 +23,18 @@ logger = setup_logger("aareGUI") class AutomationProgressWidget(QWidget): """Compact fixed-step widget for DAQ automation progress.""" + DEFAULT_SAMPLE_ESTIMATE_S = 150.0 + def __init__(self, parent: QWidget | None = None): super().__init__(parent) self._progress: AutomationProgress | None = None + self._gui_samples_in_queue: int | None = None self._labels: dict[WorkflowStateKind, QLabel] = {} self._title_label: QLabel | None = None self._stats_label: QLabel | None = None + self._refresh_timer = QTimer(self) + self._refresh_timer.setInterval(1000) + self._refresh_timer.timeout.connect(self._refresh_live_metrics) self._setup_ui() self.clear() @@ -67,6 +67,7 @@ class AutomationProgressWidget(QWidget): layout.addStretch() def clear(self) -> None: + self._gui_samples_in_queue = 0 empty = AutomationProgress( current_step=None, steps=[ @@ -79,7 +80,8 @@ class AutomationProgressWidget(QWidget): finished=False, success=None, samples_in_queue=0, - current_sample_name="None" + avg_time_per_sample=0.0, + current_sample_name="", ) self.set_progress(empty) @@ -125,18 +127,112 @@ class AutomationProgressWidget(QWidget): return base + " background-color: #F5F5F5; color: #616161;" return base + " background-color: #F7F7F7; color: #757575;" + @staticmethod + def _format_duration(seconds: float | None) -> str: + if seconds is None or seconds <= 0: + return "0m 00s" + if seconds < 60: + return f"0m {int(round(seconds)):02d}s" + minutes, secs = divmod(int(round(seconds)), 60) + if minutes < 60: + return f"{minutes}m {secs:02d}s" + hours, minutes = divmod(minutes, 60) + return f"{hours}h {minutes:02d}m" + + @staticmethod + def _format_eta_timestamp(epoch_seconds: float | None) -> str: + if epoch_seconds is None: + return "N/A" + return datetime.fromtimestamp(epoch_seconds).strftime("%H:%M:%S") + + def _has_live_timing(self, progress: AutomationProgress) -> bool: + if progress.finished: + return False + return any( + step.status == StepStatus.RUNNING and step.started_at is not None + for step in progress.steps + ) + + def _current_sample_elapsed_seconds(self, progress: AutomationProgress) -> float: + started_times = [ + step.started_at + for step in progress.steps + if step.started_at is not None and step.step != WorkflowStateKind.FINAL + ] + if not started_times: + return 0.0 + return max(0.0, time.time() - min(started_times)) + + def _has_active_sample(self, progress: AutomationProgress) -> bool: + if progress.finished: + return False + if progress.current_sample_name: + return True + return any( + step.status == StepStatus.RUNNING + for step in progress.steps + if step.step != WorkflowStateKind.FINAL + ) + + @Slot() + def _refresh_live_metrics(self) -> None: + if self._progress is None: + return + self.set_progress(self._progress) + + @Slot(int) + def set_samples_in_queue(self, count: int) -> None: + self._gui_samples_in_queue = max(0, int(count)) + if self._progress is not None: + self.set_progress(self._progress) + @Slot(object) def set_progress(self, progress: AutomationProgress) -> None: self._progress = progress - # Update Queue Stats + if self._has_live_timing(progress): + if not self._refresh_timer.isActive(): + self._refresh_timer.start() + else: + self._refresh_timer.stop() + if self._stats_label: - avg_time = f"{progress.avg_time_per_sample:.1f}s" if progress.avg_time_per_sample > 0 else "N/A" - stats_text = ( - f"Current: {progress.current_sample_name or 'None'}
" - f"Queue: {progress.samples_in_queue} samples | Avg: {avg_time}" + measured_avg_seconds = progress.avg_time_per_sample if progress.avg_time_per_sample > 0 else None + displayed_avg_seconds = measured_avg_seconds or self.DEFAULT_SAMPLE_ESTIMATE_S + + current_step = progress.current_step or "Idle" + current_sample = progress.current_sample_name or "None" + samples_left = ( + self._gui_samples_in_queue + if self._gui_samples_in_queue is not None + else max(0, int(progress.samples_in_queue or 0)) ) - self._stats_label.setText(stats_text) + + has_active_sample = self._has_active_sample(progress) + current_sample_elapsed = self._current_sample_elapsed_seconds(progress) + + if progress.finished or not has_active_sample: + current_sample_remaining = 0.0 + queue_remaining = displayed_avg_seconds * samples_left + else: + current_sample_remaining = max(0.0, displayed_avg_seconds - current_sample_elapsed) + if samples_left == 0: + queue_remaining = current_sample_remaining + else: + queue_remaining = current_sample_remaining + (displayed_avg_seconds * samples_left) + + eta_clock = time.time() + queue_remaining if queue_remaining > 0 else None + + stats_lines = [ + f"Current sample: {current_sample}", + f"Current step: {current_step}", + f"Samples left: {samples_left}", + f"Avg sample time: {self._format_duration(displayed_avg_seconds)}", + f"This sample ETA: {self._format_duration(current_sample_remaining)}", + f"Queue remaining: {self._format_duration(queue_remaining)}", + f"Done by: {self._format_eta_timestamp(eta_clock)}", + ] + self._stats_label.setText("
".join(stats_lines)) for step_state in progress.steps: label = self._labels.get(step_state.step) @@ -146,11 +242,10 @@ class AutomationProgressWidget(QWidget): title = self._label_for_step(step_state.step) icon = self._icon_for_status(step_state.status) - # Calculate duration duration_str = "" - if step_state.started_at: + if step_state.started_at is not None: end = step_state.completed_at or time.time() - duration_str = f" ({end - step_state.started_at:.1f}s)" + duration_str = f" ({self._format_duration(end - step_state.started_at)})" error_str = f"
Error: {step_state.error_code}" if step_state.error_code else "" message = f" — {step_state.message}" if step_state.message else "" diff --git a/src/aare/gui/panels/sample_queue_panel.py b/src/aare/gui/panels/sample_queue_panel.py index 73b1b804..79afdbb9 100644 --- a/src/aare/gui/panels/sample_queue_panel.py +++ b/src/aare/gui/panels/sample_queue_panel.py @@ -21,6 +21,7 @@ class SampleQueuePanel(QFrame): unmount = Signal() park_and_dry = Signal() viewer_track_online = Signal() + samples_in_queue_changed = Signal(int) def __init__(self, parent=None, samples: SampleShortInfoList | None = None): super().__init__(parent) @@ -94,6 +95,16 @@ class SampleQueuePanel(QFrame): # Add the button layout to the main layout layout.addLayout(button_layout) + self._emit_samples_in_queue_changed() + + def _samples_left_in_queue(self) -> int: + total = len(self.table_model.samples) + if self.__pause or total == 0: + return total + return max(0, total - 1) + + def _emit_samples_in_queue_changed(self) -> None: + self.samples_in_queue_changed.emit(self._samples_left_in_queue()) def remove_selected_samples(self): """Remove selected samples from the queue.""" @@ -111,12 +122,15 @@ class SampleQueuePanel(QFrame): sample = self.table_model.samples[row] self.table_model.remove_sample(sample.db_id) + self._emit_samples_in_queue_changed() + def ring_current_check(self): if not ring_current_low_check(self, self.ring_current): self.table_model.set_running(False) self.set_to_pause = True self.__pause = True self.play_button.setText("▶ Run") + self._emit_samples_in_queue_changed() return False return True @@ -198,15 +212,18 @@ class SampleQueuePanel(QFrame): self.play_button.setText("⏸ Pause") current = self.table_model.samples[0] self._current_db_id = current.db_id + self._emit_samples_in_queue_changed() self.auto_scan.emit(current) self.viewer_track_online.emit() else: self.__set_to_pause = True self.__pause = True self.play_button.setText("▶ Run") + self._emit_samples_in_queue_changed() def clear(self): self.table_model.clearSamples() + self._emit_samples_in_queue_changed() @Slot(DAQStatusModel) def update_daq_status(self, s: DAQStatusModel): @@ -220,6 +237,7 @@ class SampleQueuePanel(QFrame): self.play_button.setText("▶ Run") if set_id_to_None: self._current_db_id = None + self._emit_samples_in_queue_changed() @Slot(int, bool) def automated_scan_done(self, db_id: int, success: bool, reply:str): @@ -241,9 +259,11 @@ class SampleQueuePanel(QFrame): self.table_model.set_running(True) self.__pause = False self.play_button.setText("⏸ Pause") + self._emit_samples_in_queue_changed() else: logger.debug("No beam, or user quit, ending automation") self.unmount.emit() + self._emit_samples_in_queue_changed() return if success: @@ -253,6 +273,7 @@ class SampleQueuePanel(QFrame): if not self.__pause and len(self.table_model.samples) > 0: next_item = self.table_model.samples[0] self._current_db_id = next_item.db_id + self._emit_samples_in_queue_changed() self.auto_scan.emit(next_item) else: self._finish_empty_queue() @@ -289,6 +310,9 @@ class SampleQueuePanel(QFrame): if not self.__pause and len(self.table_model.samples) > 0: next_item = self.table_model.samples[0] self._current_db_id = next_item.db_id + self._emit_samples_in_queue_changed() self.auto_scan.emit(next_item) else: - self._finish_empty_queue() \ No newline at end of file + self._finish_empty_queue() + + self._emit_samples_in_queue_changed() \ No newline at end of file diff --git a/src/aare/gui/threads/daq_worker.py b/src/aare/gui/threads/daq_worker.py index 0bb11244..edcfc6ad 100644 --- a/src/aare/gui/threads/daq_worker.py +++ b/src/aare/gui/threads/daq_worker.py @@ -1257,6 +1257,9 @@ class DAQWorker(QObject): step=step_kind, status=step_status, message=str(raw_message or ""), + started_at=raw_step.get("started_at"), + completed_at=raw_step.get("completed_at"), + error_code=raw_step.get("error_code"), ) ) @@ -1265,6 +1268,9 @@ class DAQWorker(QObject): steps=steps, finished=bool(progress_payload.get("finished", False)), success=progress_payload.get("success"), + samples_in_queue=int(progress_payload.get("samples_in_queue", 0) or 0), + avg_time_per_sample=float(progress_payload.get("avg_time_per_sample", 0.0) or 0.0), + current_sample_name=str(progress_payload.get("current_sample_name") or ""), ) def _handle_automation_progress_event(self, payload: str) -> None: