csaxs_bec/tests/e2e/test_tomo_queue_concurrency.py

"""Checklist section C -- concurrency, the path the GUI will use. See TOMO_QUEUE_TESTING.md.

Session A (executing the queue) is always a subprocess -- tomo_queue_execute()
must run on a process's real main thread (see _queue_helpers.ProgressSampler).
Session B (editing concurrently) is the pytest process's own flomni_sim
connection: a second, independent client hitting the same redis-backed
tomo_queue global var, exactly like a second operator's kernel or the future
GUI dialog would.
"""

import datetime
import uuid

from _queue_helpers import add_short_job, spawn_queue_subprocess, wait_until


def test_concurrent_queue_edits_from_second_client(
    flomni_sim, bec_services_config_file_path, tmp_path
):
    """C8: while session A executes the queue, session B reorders the
    pending tail, deletes a pending job, and appends a new one, all mid-job1.
    Session A must pick up every change at the next job boundary and write
    statuses to the right jobs -- the real-redis version of what the offline
    mock harness proved in memory (section 2 of the doc).
    """
    flomni = flomni_sim
    bec = flomni.client

    add_short_job(flomni, "job1")
    add_short_job(flomni, "job2")
    add_short_job(flomni, "job3")
    add_short_job(flomni, "job4")
    jobs = flomni._tomo_queue_proxy.as_list()
    ids = {j["label"]: j["id"] for j in jobs}

    proc = spawn_queue_subprocess(bec_services_config_file_path, tmp_path / "sessionA.log")
    try:
        wait_until(lambda: flomni._tomo_queue_proxy.as_list()[0]["status"] == "running", timeout=30)

        # -- session B's edits, all mid-job1: reorder the pending tail
        # (job4 ahead of job2), delete job3, append job5. Deliberately does
        # NOT touch any live tomo_* param global var here (only the
        # tomo_queue list itself) -- job1's own live scan in session A is
        # reading those same params every projection; touching them mid-run
        # would perturb the *running* job, which is a different hazard than
        # the one this test is about (see TOMO_QUEUE_COMMAND_JOBS_PLAN's
        # "Done must be blocked while running" rule for the GUI).
        current = {j["id"]: j for j in bec.get_global_var("tomo_queue")}
        reordered = [
            current[ids["job1"]],  # the running job, left in place
            current[ids["job4"]],
            current[ids["job2"]],
        ]
        bec.set_global_var("tomo_queue", reordered)

        job5 = dict(current[ids["job1"]])  # clone an existing job's params snapshot
        job5.update(
            id=uuid.uuid4().hex,
            label="job5",
            status="pending",
            added_at=datetime.datetime.now().isoformat(),
        )
        flomni._tomo_queue_proxy.append(job5)

        returncode = proc.wait(timeout=420)  # 4 real jobs x ~8 projections each
    finally:
        if proc.poll() is None:
            proc.kill()
            proc.wait(timeout=10)

    log_text = (tmp_path / "sessionA.log").read_text()
    assert returncode == 0, f"session A failed (see log):\n{log_text[-4000:]}"

    jobs = flomni._tomo_queue_proxy.as_list()
    labels_in_order = [j["label"] for j in jobs]
    statuses = {j["label"]: j["status"] for j in jobs}

    assert "job3" not in labels_in_order, "job3 was deleted mid-run and must not reappear"
    assert labels_in_order == [
        "job1",
        "job4",
        "job2",
        "job5",
    ], f"queue should reflect the reorder+append made mid-run, got {labels_in_order}"
    assert statuses == {"job1": "done", "job4": "done", "job2": "done", "job5": "done"}


def test_delete_running_job_mid_run(flomni_sim, bec_services_config_file_path, tmp_path):
    """C9: deleting the *running* job mid-run (unprotected at CLI level --
    the GUI will block it, per section 5 of the doc) must degrade
    gracefully: update_by_id() returns False, job1's scan finishes on its
    own terms, nothing crashes, and no status gets written to the wrong job.
    """
    flomni = flomni_sim

    add_short_job(flomni, "job1")
    add_short_job(flomni, "job2")

    proc = spawn_queue_subprocess(bec_services_config_file_path, tmp_path / "sessionA.log")
    try:
        wait_until(lambda: flomni._tomo_queue_proxy.as_list()[0]["status"] == "running", timeout=30)

        # Delete the running job (index 0) out from under session A.
        flomni.tomo_queue_delete(0)

        returncode = proc.wait(timeout=180)
    finally:
        if proc.poll() is None:
            proc.kill()
            proc.wait(timeout=10)

    log_text = (tmp_path / "sessionA.log").read_text()
    assert (
        returncode == 0
    ), f"session A should degrade gracefully, not crash (see log):\n{log_text[-4000:]}"

    jobs = flomni._tomo_queue_proxy.as_list()
    labels = [j["label"] for j in jobs]
    assert labels == ["job2"], f"deleted job1 must not reappear, got {labels}"
    assert jobs[0]["status"] == "done", "job2 should still have run after job1's scan finished"