Mirko Holler
30b5119a2f
Type-tags queue jobs with a "kind" field (back-compat: missing kind defaults to "tomo"), and adds "command" jobs that reconfigure the beamline (currently: absolute moves on an allow-listed device, plus an optimize_idgap stub) between tomograms instead of running a scan. Command jobs are added via tomo_queue_add_command() against a curated named-action registry (_TOMO_QUEUE_ACTIONS) - not arbitrary code - so every queued action is auditable and safe to persist across a kernel restart. Validation is two-layer: schema checks at add time, and each action re-checks its own arguments against live hardware at execution time, which is the actual safety boundary. Crash-resume for a command job re-runs the whole step sequence from the top if every step is idempotent, or prompts the operator otherwise. flomni_webpage_generator.py is guarded against command jobs, which have no "params" key. See AI_docs/TOMO_QUEUE_COMMAND_JOBS_PLAN.md for the full design.
121 lines
4.3 KiB
Python
121 lines
4.3 KiB
Python
"""Shared helpers for driving tomo_queue_execute() against the live sim."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
_SUBPROCESS_SCRIPT = Path(__file__).with_name("_run_queue_subprocess.py")
|
|
|
|
SHORT_SCAN_PARAMS = dict(
|
|
# Single acquisition per angle instead of a full Fermat grid -- makes a
|
|
# queued "tomogram" fast enough to run in a test.
|
|
single_point_instead_of_fermat_scan=True,
|
|
tomo_countingtime=0.05,
|
|
frames_per_trigger=1,
|
|
fovx=2,
|
|
fovy=2,
|
|
tomo_shellstep=1,
|
|
tomo_angle_range=180,
|
|
zero_deg_reference_at_each_subtomo=False,
|
|
)
|
|
|
|
# N = int(tomo_angle_range / tomo_angle_stepsize) = 1 -> 8 projections total,
|
|
# the fastest a real tomogram can be for tests that don't care about the
|
|
# per-job N itself, just that a job runs cleanly.
|
|
FAST_STEPSIZE = 170.0
|
|
|
|
|
|
def short_params(flomni, tomo_angle_stepsize: float = FAST_STEPSIZE) -> dict:
|
|
for name, value in SHORT_SCAN_PARAMS.items():
|
|
setattr(flomni, name, value)
|
|
flomni.tomo_angle_stepsize = tomo_angle_stepsize
|
|
return {name: getattr(flomni, name) for name in flomni._TOMO_SCAN_PARAM_NAMES}
|
|
|
|
|
|
def add_short_job(flomni, label: str, tomo_angle_stepsize: float = FAST_STEPSIZE) -> int:
|
|
short_params(flomni, tomo_angle_stepsize)
|
|
return flomni.tomo_queue_add(label)
|
|
|
|
|
|
class ProgressSampler:
|
|
"""Samples ``flomni.progress[key]`` on a background thread while the
|
|
caller runs something blocking (e.g. ``tomo_queue_execute()``) in the
|
|
foreground.
|
|
|
|
``tomo_queue_execute()`` itself must stay on the main thread:
|
|
BECIPythonClient's live-update machinery installs a SIGINT handler per
|
|
scan request (``ipython_live_updates.process_request``), and Python only
|
|
allows ``signal.signal()`` from the main thread of the main interpreter
|
|
-- calling it from a worker thread raises ``ValueError``. So it's the
|
|
*sampling*, not the execution, that runs in the background here.
|
|
"""
|
|
|
|
def __init__(self, flomni, key: str = "subtomo_total_projections", interval: float = 0.05):
|
|
self._flomni = flomni
|
|
self._key = key
|
|
self._interval = interval
|
|
self._observations: list = []
|
|
self._stop = threading.Event()
|
|
self._thread = threading.Thread(target=self._run, daemon=True)
|
|
|
|
def _run(self):
|
|
while not self._stop.is_set():
|
|
self._observations.append(self._flomni.progress.get(self._key))
|
|
time.sleep(self._interval)
|
|
|
|
def __enter__(self) -> "ProgressSampler":
|
|
self._thread.start()
|
|
return self
|
|
|
|
def __exit__(self, *exc_info) -> None:
|
|
self._stop.set()
|
|
self._thread.join(timeout=5)
|
|
|
|
@property
|
|
def observations(self) -> list:
|
|
return list(self._observations)
|
|
|
|
def first_index_of(self, value) -> int | None:
|
|
try:
|
|
return self._observations.index(value)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def spawn_queue_subprocess(
|
|
services_config_path, log_path, start_index: int = 0
|
|
) -> subprocess.Popen:
|
|
"""Launch _run_queue_subprocess.py as a real OS process against the same
|
|
sim, so it can be SIGKILLed -- or run concurrently with a second client
|
|
editing the queue -- the way a real crashed/second kernel would be.
|
|
|
|
Output goes to ``log_path``, not a PIPE: a killed/long-running subprocess
|
|
whose stdout pipe nobody drains can deadlock once the OS pipe buffer
|
|
fills (tomo_queue_execute() prints a lot -- progress bars per move).
|
|
"""
|
|
with open(log_path, "w") as log_file:
|
|
# Popen dup()s the fd for the child; safe to close our copy right
|
|
# after spawning instead of leaking it for the subprocess's lifetime.
|
|
return subprocess.Popen(
|
|
[sys.executable, str(_SUBPROCESS_SCRIPT), str(services_config_path), str(start_index)],
|
|
stdout=log_file,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
|
|
|
|
def wait_until(predicate, timeout: float, interval: float = 0.1):
|
|
"""Poll ``predicate()`` until it returns a truthy value or ``timeout``
|
|
elapses. Returns the truthy value, or raises TimeoutError.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
time.sleep(interval)
|
|
raise TimeoutError(f"condition not met within {timeout}s")
|