"""Checklist section B -- sim, crash/interrupt paths. See TOMO_QUEUE_TESTING.md. B5 (real SIGKILL crash-resume) lives in test_tomo_queue_crash_subprocess.py -- it needs a real OS process to kill, unlike B6/B7 here. """ import pytest from _queue_helpers import add_short_job def test_exception_mid_scan_pauses_queue(flomni_sim, monkeypatch): """B6: tomo_scan() raising mid-job must leave that job "incomplete", re-raise out of tomo_queue_execute(), and leave later jobs untouched ("pending"). Re-running afterward must resume the failed job first, before the still-pending one. """ flomni = flomni_sim add_short_job(flomni, "job1") add_short_job(flomni, "job2") jobs = flomni._tomo_queue_proxy.as_list() job1_id, job2_id = jobs[0]["id"], jobs[1]["id"] call_count = {"n": 0} original_tomo_scan = type(flomni).tomo_scan def _raise_once(self, *a, **k): call_count["n"] += 1 if call_count["n"] == 1: raise RuntimeError("simulated mid-scan failure") return original_tomo_scan(self, *a, **k) monkeypatch.setattr(type(flomni), "tomo_scan", _raise_once) with pytest.raises(RuntimeError, match="simulated mid-scan failure"): flomni.tomo_queue_execute() jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()} assert jobs[job1_id]["status"] == "incomplete", "failed job must be marked incomplete" assert jobs[job2_id]["status"] == "pending", "later job must be untouched by the failure" # tomo_scan() raised before setting progress["tomo_start_time"], so the # resume attempt below is itself a clean no-op ("No tomo scan in # progress to resume") -- this test is about *selection order* # (resume-before-fresh), not resume accuracy (that's B5). flomni.tomo_queue_execute() jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()} assert jobs[job1_id]["status"] == "done", "the previously-failed job must have been retried" assert jobs[job2_id]["status"] == "done", "the pending job must have run after it" assert call_count["n"] == 2, "tomo_scan should have been called exactly once per attempt" def test_resume_before_fresh_explicit(flomni_sim, monkeypatch): """B7: an "incomplete" job below a fresh "pending" job in list order must still run first, regardless of position (new pick-next behavior; the old strictly-in-order loop couldn't reorder around this at all). """ flomni = flomni_sim add_short_job(flomni, "fresh_pending") add_short_job(flomni, "crashed_earlier") jobs = flomni._tomo_queue_proxy.as_list() fresh_id, crashed_id = jobs[0]["id"], jobs[1]["id"] # crashed_earlier sits BELOW fresh_pending in list order, but is # resumable -- it must be picked first regardless. flomni._tomo_queue_proxy.update_by_id(crashed_id, status="incomplete") flomni.progress.reset() # so its "resume" is a clean, fast no-op run_order = [] original_update_by_id = type(flomni._tomo_queue_proxy).update_by_id def _track(self, job_id, **kwargs): if kwargs.get("status") == "running": run_order.append(job_id) return original_update_by_id(self, job_id, **kwargs) monkeypatch.setattr(type(flomni._tomo_queue_proxy), "update_by_id", _track) flomni.tomo_queue_execute() assert run_order == [ crashed_id, fresh_id, ], f"crashed_earlier (resumable) should run before fresh_pending, got order {run_order}" jobs = {j["id"]: j for j in flomni._tomo_queue_proxy.as_list()} assert jobs[fresh_id]["status"] == "done" assert jobs[crashed_id]["status"] == "done"