Mirko Holler
30b5119a2f
Type-tags queue jobs with a "kind" field (back-compat: missing kind defaults to "tomo"), and adds "command" jobs that reconfigure the beamline (currently: absolute moves on an allow-listed device, plus an optimize_idgap stub) between tomograms instead of running a scan. Command jobs are added via tomo_queue_add_command() against a curated named-action registry (_TOMO_QUEUE_ACTIONS) - not arbitrary code - so every queued action is auditable and safe to persist across a kernel restart. Validation is two-layer: schema checks at add time, and each action re-checks its own arguments against live hardware at execution time, which is the actual safety boundary. Crash-resume for a command job re-runs the whole step sequence from the top if every step is idempotent, or prompts the operator otherwise. flomni_webpage_generator.py is guarded against command jobs, which have no "params" key. See AI_docs/TOMO_QUEUE_COMMAND_JOBS_PLAN.md for the full design.
90 lines
3.6 KiB
Python
90 lines
3.6 KiB
Python
"""Checklist section B -- sim, crash/interrupt paths. See TOMO_QUEUE_TESTING.md.
|
|
|
|
B5 (real SIGKILL crash-resume) lives in test_tomo_queue_crash_subprocess.py --
|
|
it needs a real OS process to kill, unlike B6/B7 here.
|
|
"""
|
|
|
|
import pytest
|
|
from _queue_helpers import add_short_job
|
|
|
|
|
|
def test_exception_mid_scan_pauses_queue(flomni_sim, monkeypatch):
|
|
"""B6: tomo_scan() raising mid-job must leave that job "incomplete",
|
|
re-raise out of tomo_queue_execute(), and leave later jobs untouched
|
|
("pending"). Re-running afterward must resume the failed job first,
|
|
before the still-pending one.
|
|
"""
|
|
flomni = flomni_sim
|
|
|
|
add_short_job(flomni, "job1")
|
|
add_short_job(flomni, "job2")
|
|
jobs = flomni._tomo_queue_proxy.as_list()
|
|
job1_id, job2_id = jobs[0]["id"], jobs[1]["id"]
|
|
|
|
call_count = {"n": 0}
|
|
original_tomo_scan = type(flomni).tomo_scan
|
|
|
|
def _raise_once(self, *a, **k):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
raise RuntimeError("simulated mid-scan failure")
|
|
return original_tomo_scan(self, *a, **k)
|
|
|
|
monkeypatch.setattr(type(flomni), "tomo_scan", _raise_once)
|
|
|
|
with pytest.raises(RuntimeError, match="simulated mid-scan failure"):
|
|
flomni.tomo_queue_execute()
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[job1_id]["status"] == "incomplete", "failed job must be marked incomplete"
|
|
assert jobs[job2_id]["status"] == "pending", "later job must be untouched by the failure"
|
|
|
|
# tomo_scan() raised before setting progress["tomo_start_time"], so the
|
|
# resume attempt below is itself a clean no-op ("No tomo scan in
|
|
# progress to resume") -- this test is about *selection order*
|
|
# (resume-before-fresh), not resume accuracy (that's B5).
|
|
flomni.tomo_queue_execute()
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[job1_id]["status"] == "done", "the previously-failed job must have been retried"
|
|
assert jobs[job2_id]["status"] == "done", "the pending job must have run after it"
|
|
assert call_count["n"] == 2, "tomo_scan should have been called exactly once per attempt"
|
|
|
|
|
|
def test_resume_before_fresh_explicit(flomni_sim, monkeypatch):
|
|
"""B7: an "incomplete" job below a fresh "pending" job in list order
|
|
must still run first, regardless of position (new pick-next behavior;
|
|
the old strictly-in-order loop couldn't reorder around this at all).
|
|
"""
|
|
flomni = flomni_sim
|
|
|
|
add_short_job(flomni, "fresh_pending")
|
|
add_short_job(flomni, "crashed_earlier")
|
|
jobs = flomni._tomo_queue_proxy.as_list()
|
|
fresh_id, crashed_id = jobs[0]["id"], jobs[1]["id"]
|
|
|
|
# crashed_earlier sits BELOW fresh_pending in list order, but is
|
|
# resumable -- it must be picked first regardless.
|
|
flomni._tomo_queue_proxy.update_by_id(crashed_id, status="incomplete")
|
|
flomni.progress.reset() # so its "resume" is a clean, fast no-op
|
|
|
|
run_order = []
|
|
original_update_by_id = type(flomni._tomo_queue_proxy).update_by_id
|
|
|
|
def _track(self, job_id, **kwargs):
|
|
if kwargs.get("status") == "running":
|
|
run_order.append(job_id)
|
|
return original_update_by_id(self, job_id, **kwargs)
|
|
|
|
monkeypatch.setattr(type(flomni._tomo_queue_proxy), "update_by_id", _track)
|
|
flomni.tomo_queue_execute()
|
|
|
|
assert run_order == [
|
|
crashed_id,
|
|
fresh_id,
|
|
], f"crashed_earlier (resumable) should run before fresh_pending, got order {run_order}"
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[fresh_id]["status"] == "done"
|
|
assert jobs[crashed_id]["status"] == "done"
|