new_gui: GUI-driven per-stage automation + queue drag-reorder, add-all, sort

- AutomationRunner (main_window): drives each queued sample mount -> enabled
  protocol stages -> unmount via per-stage daq ops; advances on raster/standard
  completion or busy edge; pausable per step; live beam-rail progress. Replaces
  automated_scan-per-sample. raster=raster_scan_auto, collect/xrf=YAML defaults.
- queue: drag-to-reorder rows (QueueList drop target + QueueRow QDrag handle) +
  AppState.queue_reorder; ▲▼ kept as fallback
- library: Add all button, sort (Name/Puck/Location/Priority), search across
  name/puck/location, shown/total count

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/aare/gui/new_gui/README.md

@@ -106,7 +106,16 @@ Backend contract is documented in the `new-gui-backend-paths` memory.
   else a `status.busy` True→False edge, else ⏭ Skip) and **pausing before any
   stage flagged with a ⚑ breakpoint**; ▶ resumes, ❚❚ pauses after the current
   stage, ⏹ ends. (Automation pauses between samples; per-stage control inside a
-  sample is bounded by the backend's `automated_scan`.)
+  sample is **GUI-driven** — see below.)
+- **Automation** (queue page): **GUI-driven per-stage runner** drives each queued
+  sample through mount → its enabled protocol stages → unmount, firing the
+  per-stage backend ops and advancing on completion (raster/standard signals,
+  else a `busy` edge); **pausable per step** (Pause holds after the current step).
+  The right-hand beam-rail shows live progress. Raster uses `raster_scan_auto`;
+  collect/XRF use YAML defaults (validate collect params against the dose-aware
+  backend path for production). Queue: **drag rows to reorder** (grip handle),
+  per-sample protocol chips, ▲▼ fallback. Library: **search** (name/puck/location),
+  **sort** (Name/Puck/Location/Priority) and **Add all**.
 - **Bookmarks** (mounted): ＋ Bookmark captures the current smargon position
   (shown as numbered markers on the camera, projected via `smargon_to_picture`);
   each has an **editable label**; **Go** returns there (`move_smargon` +

src/aare/gui/new_gui/automation_view.py

@@ -7,8 +7,10 @@ chips are local for now; a pausable protocol-driven automation comes later.
 
 from __future__ import annotations
 
-from PySide6.QtCore import Qt, Signal
+from PySide6.QtCore import QMimeData, Qt, Signal
+from PySide6.QtGui import QDrag
 from PySide6.QtWidgets import (
+    QComboBox,
     QHBoxLayout,
     QLabel,
     QLineEdit,
@@ -18,6 +20,8 @@ from PySide6.QtWidgets import (
     QWidget,
 )
 
+_QUEUE_MIME = "application/x-aare-queue-row"
+
 from aare.gui.new_gui.state import PROTOCOL_KEYS, AppState
 from aare.gui.new_gui.theme import (
     FONT_MONO_FALLBACK,
@@ -103,9 +107,19 @@ class LibraryRow(QWidget):
         lay.addWidget(self._add)
 
 
+# Library sort keys (label -> accessor producing a sort key).
+_SORT_KEYS = {
+    "Name": lambda s: sample_label(s).lower(),
+    "Puck": lambda s: (sample_puck(s), sample_label(s).lower()),
+    "Location": lambda s: sample_pos(s),
+    "Priority": lambda s: -float(getattr(s, "priority", 0) or 0),
+}
+
+
 class LibraryPanel(QWidget):
     add_sample = Signal(object)
     add_puck = Signal(str)
+    add_all = Signal(object)   # list of samples (current filter+sort)
 
     def __init__(self, state: AppState, palette: Palette, parent=None):
         super().__init__(parent)
@@ -124,11 +138,41 @@ class LibraryPanel(QWidget):
         hl = QVBoxLayout(header)
         hl.setContentsMargins(14, 14, 14, 10)
         hl.setSpacing(10)
-        hl.addWidget(section_label("LIBRARY"))
+        title_row = QHBoxLayout()
+        title_row.setSpacing(8)
+        title_row.addWidget(section_label("LIBRARY"))
+        title_row.addStretch(1)
+        self._count = QLabel("")
+        self._count.setStyleSheet(
+            f"font-family:{FONT_MONO_FALLBACK}; font-size:10.5px;"
+            f" color:{palette.text_faint};")
+        title_row.addWidget(self._count)
+        hl.addLayout(title_row)
+
         self._search = QLineEdit()
-        self._search.setPlaceholderText("Search samples…")
+        self._search.setPlaceholderText("Search name / puck / location…")
         self._search.textChanged.connect(self._rebuild)
         hl.addWidget(self._search)
+
+        controls = QHBoxLayout()
+        controls.setSpacing(6)
+        sort_lbl = QLabel("Sort")
+        sort_lbl.setStyleSheet(f"font-size:11px; color:{palette.text_faint};")
+        controls.addWidget(sort_lbl)
+        self._sort = QComboBox()
+        self._sort.addItems(list(_SORT_KEYS.keys()))
+        self._sort.currentIndexChanged.connect(self._rebuild)
+        controls.addWidget(self._sort, 1)
+        self._add_all_btn = QPushButton("Add all")
+        self._add_all_btn.setCursor(Qt.PointingHandCursor)
+        self._add_all_btn.setStyleSheet(
+            f"QPushButton {{ background:{palette.accent}; color:{palette.accent_text};"
+            f" border:none; border-radius:8px; padding:6px 12px; font-size:11.5px;"
+            f" font-weight:600; }}")
+        self._add_all_btn.clicked.connect(
+            lambda: self.add_all.emit(self._sorted_filtered()))
+        controls.addWidget(self._add_all_btn)
+        hl.addLayout(controls)
         outer.addWidget(header)
 
         self._scroll = QScrollArea()
@@ -162,20 +206,36 @@ class LibraryPanel(QWidget):
     def _queued_ids(self) -> set:
         return {item.sample_id for item in self._state.queue}
 
+    def _sorted_filtered(self) -> list:
+        query = self._search.text().strip().lower()
+        out = []
+        for s in self._samples:
+            if query and not any(
+                query in str(f(s)).lower()
+                for f in (sample_label, sample_puck, sample_pos)
+            ):
+                continue
+            out.append(s)
+        key = _SORT_KEYS.get(self._sort.currentText(), _SORT_KEYS["Name"])
+        try:
+            out.sort(key=key)
+        except TypeError:
+            out.sort(key=lambda s: sample_label(s).lower())
+        return out
+
     def _rebuild(self) -> None:
         while self._list_lay.count() > 1:
             w = self._list_lay.takeAt(0).widget()
             if w:
                 w.setParent(None)
                 w.deleteLater()
-        query = self._search.text().strip().lower()
         queued = self._queued_ids()
-        for s in self._samples:
-            if query and query not in sample_label(s).lower():
-                continue
+        shown = self._sorted_filtered()
+        for s in shown:
             row = LibraryRow(s, self._p, added=sample_id(s) in queued)
             row.add_clicked.connect(self.add_sample)
             self._list_lay.insertWidget(self._list_lay.count() - 1, row)
+        self._count.setText(f"{len(shown)}/{len(self._samples)}")
 
     def _rebuild_pucks(self) -> None:
         # clear all but the label
@@ -208,6 +268,7 @@ class QueueRow(QWidget):
     def __init__(self, index: int, item, palette: Palette, parent=None):
         super().__init__(parent)
         self._index = index
+        self._press = None
         p = palette
         self.setAttribute(Qt.WA_StyledBackground, True)
         self.setStyleSheet(
@@ -291,6 +352,76 @@ class QueueRow(QWidget):
         )
         return b
 
+    # ---- mouse drag to reorder ----
+    def mousePressEvent(self, event):  # noqa: N802
+        if event.button() == Qt.LeftButton:
+            self._press = event.position().toPoint()
+        super().mousePressEvent(event)
+
+    def mouseMoveEvent(self, event):  # noqa: N802
+        if self._press is None or not (event.buttons() & Qt.LeftButton):
+            return
+        if (event.position().toPoint() - self._press).manhattanLength() < 12:
+            return
+        drag = QDrag(self)
+        mime = QMimeData()
+        mime.setData(_QUEUE_MIME, str(self._index).encode())
+        drag.setMimeData(mime)
+        pm = self.grab()
+        drag.setPixmap(pm)
+        drag.setHotSpot(event.position().toPoint())
+        self._press = None
+        drag.exec(Qt.MoveAction)
+
+
+class QueueList(QWidget):
+    """Drop target hosting the queue rows; emits reorder on drop."""
+
+    reorder_requested = Signal(int, int)   # src index, dst index (src removed)
+
+    def __init__(self, palette: Palette, parent=None):
+        super().__init__(parent)
+        self.setAcceptDrops(True)
+        self._lay = QVBoxLayout(self)
+        self._lay.setContentsMargins(24, 14, 24, 14)
+        self._lay.setSpacing(8)
+        self._lay.addStretch(1)
+
+    def add_row(self, row: QWidget) -> None:
+        self._lay.insertWidget(self._lay.count() - 1, row)
+
+    def clear_rows(self) -> None:
+        while self._lay.count() > 1:
+            w = self._lay.takeAt(0).widget()
+            if w:
+                w.setParent(None)
+                w.deleteLater()
+
+    def _rows(self) -> list:
+        return [self._lay.itemAt(i).widget() for i in range(self._lay.count() - 1)]
+
+    def dragEnterEvent(self, event):  # noqa: N802
+        if event.mimeData().hasFormat(_QUEUE_MIME):
+            event.acceptProposedAction()
+
+    def dragMoveEvent(self, event):  # noqa: N802
+        if event.mimeData().hasFormat(_QUEUE_MIME):
+            event.acceptProposedAction()
+
+    def dropEvent(self, event):  # noqa: N802
+        if not event.mimeData().hasFormat(_QUEUE_MIME):
+            return
+        src = int(bytes(event.mimeData().data(_QUEUE_MIME)).decode())
+        y = event.position().toPoint().y()
+        dst = 0
+        for i, row in enumerate(self._rows()):
+            if i == src:
+                continue
+            if y > row.y() + row.height() / 2:
+                dst += 1
+        event.acceptProposedAction()
+        self.reorder_requested.emit(src, dst)
+
 
 class AutomationView(QWidget):
     start_requested = Signal()
@@ -311,6 +442,7 @@ class AutomationView(QWidget):
         self.library = LibraryPanel(state, palette)
         self.library.add_sample.connect(self._on_add_sample)
         self.library.add_puck.connect(self._on_add_puck)
+        self.library.add_all.connect(self._state.queue_add_many)
         root.addWidget(self.library)
 
         # centre: queue
@@ -388,12 +520,9 @@ class AutomationView(QWidget):
         self._scroll = QScrollArea()
         self._scroll.setWidgetResizable(True)
         self._scroll.setFrameShape(QScrollArea.NoFrame)
-        self._queue_host = QWidget()
-        self._queue_lay = QVBoxLayout(self._queue_host)
-        self._queue_lay.setContentsMargins(24, 14, 24, 14)
-        self._queue_lay.setSpacing(8)
-        self._queue_lay.addStretch(1)
-        self._scroll.setWidget(self._queue_host)
+        self._queue_list = QueueList(self._p)
+        self._queue_list.reorder_requested.connect(self._state.queue_reorder)
+        self._scroll.setWidget(self._queue_list)
 
         # empty-state overlay label kept simple as a member
         self._empty = QWidget()
@@ -503,18 +632,14 @@ class AutomationView(QWidget):
                 chip.blockSignals(False)
 
         # rebuild queue rows
-        while self._queue_lay.count() > 1:
-            w = self._queue_lay.takeAt(0).widget()
-            if w:
-                w.setParent(None)
-                w.deleteLater()
+        self._queue_list.clear_rows()
         queue = self._state.queue
         for i, item in enumerate(queue):
             row = QueueRow(i, item, self._p)
             row.toggled.connect(self._state.toggle_row_protocol)
             row.move.connect(self._state.queue_move)
             row.remove.connect(self._state.queue_remove)
-            self._queue_lay.insertWidget(self._queue_lay.count() - 1, row)
+            self._queue_list.add_row(row)
 
         empty = len(queue) == 0
         self._empty.setVisible(empty)

src/aare/gui/new_gui/main_window.py

@@ -21,7 +21,7 @@ from aare.common.beamline import cfg_get, mx_beamline
 from aare.common.logger_config import setup_logger
 from aare.gui.new_gui.automation_view import AutomationView
 from aare.gui.new_gui.manual_view import ManualView
-from aare.gui.new_gui.state import AppState
+from aare.gui.new_gui.state import PROTOCOL_KEYS, AppState
 from aare.gui.new_gui.status_bar import StatusBar
 from aare.gui.new_gui.theme import DEFAULT_THEME, THEMES, build_qss
 from aare.gui.new_gui.top_bar import TopBar
@@ -71,10 +71,14 @@ class MainWindow(QWidget):
         self._palette = _saved_palette()
         self._defaults = _collect_defaults()
 
-        # automation run bookkeeping
-        self._auto_running: list = []
-        self._auto_index = 0
+        # automation run bookkeeping (GUI-driven per-stage runner)
+        self._auto_steps: list = []          # [(QueueItem, stage_key)]
+        self._auto_step = -1
+        self._auto_active = False
         self._auto_paused = False
+        self._auto_await: str | None = None  # stage awaiting completion
+        self._auto_seen_busy = False
+        self._auto_done_stages: dict = {}    # id(item) -> set of completed stages
 
         # baton / session bookkeeping
         self._staff = False
@@ -241,7 +245,11 @@ class MainWindow(QWidget):
             self.daq.baton_timeout_checked.connect(self._on_baton_timeout)
         if hasattr(self.daq, "fluorimeter_spectrum_update"):
             self.daq.fluorimeter_spectrum_update.connect(self._on_spectrum)
-        self.daq.automated_scan_done.connect(self._on_auto_done)
+        # GUI-driven automation completion hooks
+        if hasattr(self.daq, "raster_scan_completed"):
+            self.daq.raster_scan_completed.connect(self._auto_raster_done)
+        if hasattr(self.daq, "standard_scan_completed"):
+            self.daq.standard_scan_completed.connect(self._auto_collect_done)
 
         # errors / alerts -> banner (handlers read self.alert dynamically)
         if hasattr(self.daq, "http_error"):
@@ -521,6 +529,7 @@ class MainWindow(QWidget):
     @Slot(object)
     def _on_status(self, s) -> None:
         self._last_status = s
+        self._auto_busy_edge(s)
         session = getattr(s, "session", None)
         staff = bool(getattr(session, "staff", False)) if session else False
         if staff != self._staff:
@@ -742,60 +751,184 @@ class MainWindow(QWidget):
             return
         self.daq.move_smargon(SmargonCoordinate(sh_mm=sh))
 
-    # ---- automation loop (GUI-local; iterates automated_scan, pausable) ----
+    # ---- GUI-driven per-stage automation runner ---------------------------
+    # Drives each queued sample through mount → its enabled protocol stages →
+    # unmount, firing the per-stage backend ops and advancing on completion
+    # (raster/standard signals, else a busy True→False edge). Pausable per
+    # stage. Raster uses raster_scan_auto; collect/XRF use YAML defaults.
+    _AUTO_BUSY_STAGES = ("mount", "center", "xrf", "unmount")
+
     def _start_automation(self) -> None:
         queue = list(self.state.queue)
         if not queue:
             return
-        self._auto_running = queue
-        self._auto_index = 0
+        steps: list = []
+        for item in queue:
+            steps.append((item, "mount"))
+            for k in PROTOCOL_KEYS:
+                if item.protocol.get(k):
+                    steps.append((item, k))
+            steps.append((item, "unmount"))
+        self._auto_steps = steps
+        self._auto_step = -1
+        self._auto_active = True
         self._auto_paused = False
+        self._auto_await = None
+        self._auto_done_stages = {}
         self.automation.set_running(True, paused=False)
-        if hasattr(self.daq, "start_automation_progress_stream"):
-            self.daq.start_automation_progress_stream()
-        self._note(f"Starting automation: {len(queue)} samples queued.")
-        self._run_next_auto()
+        self._note(f"Starting automation: {len(queue)} samples · "
+                   f"{len(steps)} steps.")
+        self._auto_advance()
 
-    def _run_next_auto(self) -> None:
+    def _auto_advance(self) -> None:
+        if not self._auto_active or self._auto_paused:
+            return
+        self._auto_await = None
+        self._auto_step += 1
+        if self._auto_step >= len(self._auto_steps):
+            self._auto_finish()
+            return
+        item, stage = self._auto_steps[self._auto_step]
+        self._auto_fire(item, stage)
+
+    def _auto_fire(self, item, stage: str) -> None:
+        self._auto_await = stage
+        self._auto_seen_busy = False
+        self._auto_push_progress()
+        try:
+            if stage == "mount":
+                self.daq.mount(item.sample)
+            elif stage == "center":
+                self.daq.center_loop()
+            elif stage == "raster":
+                self.daq.raster_scan_auto(self._auto_raster_req(item))
+            elif stage == "xrf":
+                self.daq.fluorimeter_spectrum(self._auto_xrf_req())
+            elif stage == "collect":
+                self.daq.standard_scan(self._auto_collect_req(item))
+            elif stage == "unmount":
+                self.daq.unmount()
+        except Exception as exc:   # pragma: no cover - hardware path
+            self._note(f"Automation step '{stage}' failed: {exc}", error=True)
+            self._on_auto_stop()
+
+    def _auto_step_done(self, stage: str) -> None:
+        if not self._auto_active or self._auto_await != stage:
+            return
+        item = self._auto_steps[self._auto_step][0]
+        self._auto_done_stages.setdefault(id(item), set()).add(stage)
+        self._auto_await = None
+        self._auto_push_progress()
         if self._auto_paused:
             return
-        if self._auto_index >= len(self._auto_running):
-            self._note("Automation queue complete.")
-            self._auto_running = []
-            self.automation.set_running(False)
-            return
-        item = self._auto_running[self._auto_index]
-        self.daq.automated_scan(item.sample)
+        self._auto_advance()
 
-    @Slot(int, bool, str)
-    def _on_auto_done(self, sample_id: int, success: bool, error: str) -> None:
-        if not self._auto_running:
+    def _auto_busy_edge(self, s) -> None:
+        if not self._auto_active or self._auto_await not in self._AUTO_BUSY_STAGES:
             return
-        if not success:
-            self._note(f"Automation stopped: {error}", error=True)
-            self._auto_running = []
-            self.automation.set_running(False)
-            return
-        self._auto_index += 1
-        self._run_next_auto()   # no-op while paused; resumes on _on_auto_pause
+        if bool(getattr(s, "busy", False)):
+            self._auto_seen_busy = True
+        elif self._auto_seen_busy:
+            self._auto_step_done(self._auto_await)
+
+    def _auto_raster_done(self, *_a) -> None:
+        self._auto_step_done("raster")
+
+    def _auto_collect_done(self, *_a) -> None:
+        self._auto_step_done("collect")
 
     def _on_auto_pause(self) -> None:
-        if not self._auto_running:
+        if not self._auto_active:
             return
         self._auto_paused = not self._auto_paused
         self.automation.set_running(True, paused=self._auto_paused)
         if self._auto_paused:
-            self._note("Automation will pause after the current sample.")
+            self._note("Automation will pause after the current step.")
         else:
             self._note("Resuming automation.")
-            self._run_next_auto()
+            if self._auto_await is None:     # between steps → continue now
+                self._auto_advance()
 
     def _on_auto_stop(self) -> None:
-        self._auto_running = []
+        self._auto_active = False
         self._auto_paused = False
+        self._auto_await = None
+        self._auto_steps = []
         self.automation.set_running(False)
         self._note("Automation stopped.")
 
+    def _auto_finish(self) -> None:
+        self._auto_active = False
+        self._auto_await = None
+        self.automation.set_running(False)
+        self._note("Automation queue complete.")
+
+    # ---- per-stage request builders (defaults; raster is auto-centred) ----
+    def _auto_collect_req(self, item):
+        from aare.gui.new_gui import requests
+        d = self._defaults
+        params = {
+            "incr_omega_deg": d.get("increment_omega_deg", 0.1),
+            "steps": d.get("steps", 1800),
+            "dtz": d.get("rotation_dtz", d.get("dtz")),
+            "transmission": d.get("transmission", 1.0),
+        }
+        return requests.build_rotation_request(
+            collect_params=params, status=self.manual.last_status,
+            exp_time_s=float(d.get("rotation_exp_time_s", d.get("exp_time_s", 0.02))),
+            start_omega_deg=float(d.get("start_omega_deg", 0.0)),
+            file_prefix=None)
+
+    def _auto_xrf_req(self):
+        from aare.gui.new_gui import requests
+        return requests.build_xrf_request(1.0, self._defaults.get("transmission", 1.0))
+
+    def _auto_raster_req(self, item):
+        from aare.gui.new_gui import requests
+        d = self._defaults
+        cell = d.get("raster_cell_mm", 0.02)
+        # auto=True → backend auto-centres; a nominal 0.4mm grid seeds n_x/n_y.
+        return requests.build_raster_request(
+            cell_mm=(cell, cell), transmission=d.get("transmission", 1.0),
+            grid_rect_img=(0.0, 0.0, 0.0, 0.0), status=self.manual.last_status,
+            exp_time_s=float(d.get("raster_exp_time_s", d.get("exp_time_s", 0.02))),
+            dtz=d.get("raster_dtz", d.get("dtz")), auto=True, file_prefix=None)
+
+    def _auto_push_progress(self) -> None:
+        """Feed the beam-rail progress panel from the runner's state."""
+        from types import SimpleNamespace
+        if not self._auto_steps:
+            self.automation.progress.set_idle()
+            return
+        idx = min(self._auto_step, len(self._auto_steps) - 1)
+        cur_item, _ = self._auto_steps[idx]
+        # remaining distinct samples (including current)
+        seen, remaining = set(), 0
+        for j in range(idx, len(self._auto_steps)):
+            it = self._auto_steps[j][0]
+            if id(it) not in seen:
+                seen.add(id(it))
+                remaining += 1
+        done = self._auto_done_stages.get(id(cur_item), set())
+        steps = []
+        for it, stage in self._auto_steps:
+            if it is not cur_item:
+                continue
+            if stage in done:
+                status = "SUCCESS"
+            elif self._auto_active and stage == self._auto_steps[idx][1] \
+                    and self._auto_await == stage:
+                status = "RUNNING"
+            else:
+                status = "PENDING"
+            steps.append(SimpleNamespace(step=stage.upper(), status=status,
+                                         started_at=None, message=""))
+        prog = SimpleNamespace(
+            current_sample_name=getattr(cur_item.sample, "sample_name", "—"),
+            samples_in_queue=remaining, avg_time_per_sample=0.0,
+            finished=False, steps=steps)
+        self.automation.progress.update_progress(prog)
+
     def _on_auth_error(self) -> None:
         self._note("Lost the baton / authentication error.", error=True)
 

src/aare/gui/new_gui/state.py

@@ -327,6 +327,15 @@ class AppState(QObject):
             self._queue[index], self._queue[j] = self._queue[j], self._queue[index]
             self.queue_changed.emit()
 
+    def queue_reorder(self, src: int, dst: int) -> None:
+        n = len(self._queue)
+        if not (0 <= src < n) or src == dst:
+            return
+        item = self._queue.pop(src)
+        dst = max(0, min(dst, len(self._queue)))
+        self._queue.insert(dst, item)
+        self.queue_changed.emit()
+
     def queue_clear(self) -> None:
         if self._queue:
             self._queue = []