Mirko HollerandClaude Sonnet 5
ff50711dc9
Sections A-C of TOMO_QUEUE_TESTING.md's checklist, run against a live sim session: params-restored-per-job, legacy queue migration, empty/all-done no-ops, start_index semantics, exception/SIGKILL crash-resume, resume- before-fresh ordering, and concurrent queue edits from a second client. All 10 tests pass together with no cross-test interference. The flomni_sim fixture and _bootstrap.py factor out what it takes to construct a live Flomni against the sim (builtins/reload bootstrap, side-effect neutralization, RT-feedback/fsamx setup) for reuse by both in-process tests and the SIGKILL-based subprocess tests. Also adds the AI_docs/ handoff docs (testing checklist results, command- jobs plan with the action-registry decisions) that this work updated. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
90 lines
3.6 KiB
Python
90 lines
3.6 KiB
Python
"""Checklist section B -- sim, crash/interrupt paths. See TOMO_QUEUE_TESTING.md.
|
|
|
|
B5 (real SIGKILL crash-resume) lives in test_tomo_queue_crash_subprocess.py --
|
|
it needs a real OS process to kill, unlike B6/B7 here.
|
|
"""
|
|
|
|
import pytest
|
|
from _queue_helpers import add_short_job
|
|
|
|
|
|
def test_exception_mid_scan_pauses_queue(flomni_sim, monkeypatch):
|
|
"""B6: tomo_scan() raising mid-job must leave that job "incomplete",
|
|
re-raise out of tomo_queue_execute(), and leave later jobs untouched
|
|
("pending"). Re-running afterward must resume the failed job first,
|
|
before the still-pending one.
|
|
"""
|
|
flomni = flomni_sim
|
|
|
|
add_short_job(flomni, "job1")
|
|
add_short_job(flomni, "job2")
|
|
jobs = flomni._tomo_queue_proxy.as_list()
|
|
job1_id, job2_id = jobs[0]["id"], jobs[1]["id"]
|
|
|
|
call_count = {"n": 0}
|
|
original_tomo_scan = type(flomni).tomo_scan
|
|
|
|
def _raise_once(self, *a, **k):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
raise RuntimeError("simulated mid-scan failure")
|
|
return original_tomo_scan(self, *a, **k)
|
|
|
|
monkeypatch.setattr(type(flomni), "tomo_scan", _raise_once)
|
|
|
|
with pytest.raises(RuntimeError, match="simulated mid-scan failure"):
|
|
flomni.tomo_queue_execute()
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[job1_id]["status"] == "incomplete", "failed job must be marked incomplete"
|
|
assert jobs[job2_id]["status"] == "pending", "later job must be untouched by the failure"
|
|
|
|
# tomo_scan() raised before setting progress["tomo_start_time"], so the
|
|
# resume attempt below is itself a clean no-op ("No tomo scan in
|
|
# progress to resume") -- this test is about *selection order*
|
|
# (resume-before-fresh), not resume accuracy (that's B5).
|
|
flomni.tomo_queue_execute()
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[job1_id]["status"] == "done", "the previously-failed job must have been retried"
|
|
assert jobs[job2_id]["status"] == "done", "the pending job must have run after it"
|
|
assert call_count["n"] == 2, "tomo_scan should have been called exactly once per attempt"
|
|
|
|
|
|
def test_resume_before_fresh_explicit(flomni_sim, monkeypatch):
|
|
"""B7: an "incomplete" job below a fresh "pending" job in list order
|
|
must still run first, regardless of position (new pick-next behavior;
|
|
the old strictly-in-order loop couldn't reorder around this at all).
|
|
"""
|
|
flomni = flomni_sim
|
|
|
|
add_short_job(flomni, "fresh_pending")
|
|
add_short_job(flomni, "crashed_earlier")
|
|
jobs = flomni._tomo_queue_proxy.as_list()
|
|
fresh_id, crashed_id = jobs[0]["id"], jobs[1]["id"]
|
|
|
|
# crashed_earlier sits BELOW fresh_pending in list order, but is
|
|
# resumable -- it must be picked first regardless.
|
|
flomni._tomo_queue_proxy.update_by_id(crashed_id, status="incomplete")
|
|
flomni.progress.reset() # so its "resume" is a clean, fast no-op
|
|
|
|
run_order = []
|
|
original_update_by_id = type(flomni._tomo_queue_proxy).update_by_id
|
|
|
|
def _track(self, job_id, **kwargs):
|
|
if kwargs.get("status") == "running":
|
|
run_order.append(job_id)
|
|
return original_update_by_id(self, job_id, **kwargs)
|
|
|
|
monkeypatch.setattr(type(flomni._tomo_queue_proxy), "update_by_id", _track)
|
|
flomni.tomo_queue_execute()
|
|
|
|
assert run_order == [
|
|
crashed_id,
|
|
fresh_id,
|
|
], f"crashed_earlier (resumable) should run before fresh_pending, got order {run_order}"
|
|
|
|
jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()}
|
|
assert jobs[fresh_id]["status"] == "done"
|
|
assert jobs[crashed_id]["status"] == "done"
|