training loop: run ledger, studio export importer with an image pool, train_run with gold gate and auto-promotion, SLURM template, CI

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_import_studio_export.py

@@ -0,0 +1,66 @@
+"""tools/import_studio_export.py: images once in the pool, symlinked per dataset."""
+import json
+from pathlib import Path
+
+import pytest
+
+from tools import import_studio_export as imp
+
+YAML = "path: /x\ntrain: train/images\nval: val/images\n\nnames:\n  0: loop_all\n  1: pin\n"
+
+
+def make_export(root: Path, name: str, train=("tpi_1", "tpi_2"), val=("tpi_3",), missing_label=()):
+    exp = root / name
+    for split, keys in (("train", train), ("val", val)):
+        (exp / split / "images").mkdir(parents=True)
+        (exp / split / "labels").mkdir(parents=True)
+        for k in keys:
+            (exp / split / "images" / f"{k}.jpg").write_bytes(b"\xff\xd8" + k.encode() * 20)
+            if k not in missing_label:
+                (exp / split / "labels" / f"{k}.txt").write_text("0 0.1 0.1 0.2 0.1 0.2 0.2\n" if k != "tpi_2" else "")
+    (exp / "dataset.yaml").write_text(YAML)
+    (exp / "manifest.csv").write_text("key,tpid,split\n" + "\n".join(f"{k},{k[4:]},train" for k in train) + "\n")
+    return exp
+
+
+def test_import_builds_pool_and_symlinks(tmp_path):
+    repo = tmp_path / "repo"; repo.mkdir()
+    exp = make_export(tmp_path, "export_a", missing_label=("tpi_3",))
+    src = imp.import_export(exp, "2026-10-02_alc", root=repo)
+    ds = repo / "data" / "datasets" / "2026-10-02_alc"
+    pool = repo / "data" / "pool" / "images"
+    assert sorted(p.name for p in pool.iterdir()) == ["tpi_1.jpg", "tpi_2.jpg", "tpi_3.jpg"]
+    link = ds / "images" / "train" / "tpi_1.jpg"
+    assert link.is_symlink() and link.resolve() == (pool / "tpi_1.jpg").resolve()
+    assert (ds / "labels" / "train" / "tpi_1.txt").read_text().startswith("0 0.1")
+    assert (ds / "labels" / "val" / "tpi_3.txt").read_text() == ""      # missing label -> background
+    assert src["counts"]["train"] == {"images": 2, "labels": 2, "empty": 1}
+    assert src["counts"]["val"] == {"images": 1, "labels": 0, "empty": 1}
+    assert src["new_pool_images"] == 3 and src["classes"] == {0: "loop_all", 1: "pin"}
+    assert "path: ." in (ds / "dataset.yaml").read_text() and (ds / "manifest.csv").exists()
+    assert json.loads((ds / "source.json").read_text())["name"] == "2026-10-02_alc"
+
+
+def test_second_dataset_reuses_the_pool_and_names_are_immutable(tmp_path):
+    repo = tmp_path / "repo"; repo.mkdir()
+    imp.import_export(make_export(tmp_path, "a"), "2026-10-02_alc", root=repo)
+    src = imp.import_export(make_export(tmp_path, "b", train=("tpi_1", "tpi_2", "tpi_9"), val=("tpi_3",)),
+                            "2026-10-09_alc", root=repo)
+    assert src["new_pool_images"] == 1
+    assert len(list((repo / "data" / "pool" / "images").iterdir())) == 4
+    with pytest.raises(SystemExit):
+        imp.import_export(make_export(tmp_path, "c"), "2026-10-02_alc", root=repo)
+
+
+def test_gold_import(tmp_path):
+    repo = tmp_path / "repo"; repo.mkdir()
+    gold = tmp_path / "gold"
+    (gold / "images").mkdir(parents=True); (gold / "labels").mkdir()
+    (gold / "images" / "tpi_7.jpg").write_bytes(b"\xff\xd8gold")
+    (gold / "labels" / "tpi_7.txt").write_text("1 0.5 0.5 0.6 0.5 0.6 0.6\n")
+    (gold / "gold_manifest.json").write_text(json.dumps({"keys": ["tpi_7"]}))
+    src = imp.import_export(make_export(tmp_path, "a"), "2026-10-02_alc", root=repo, gold_dir=gold)
+    assert src["gold"]["images"] == 1 and src["gold"]["manifest_sha256"]
+    assert (repo / "data" / "gold" / "images" / "tpi_7.jpg").is_symlink()
+    assert (repo / "data" / "gold" / "labels" / "tpi_7.txt").exists()
+    assert (repo / "data" / "pool" / "images" / "tpi_7.jpg").exists()

tests/test_ledger.py

@@ -0,0 +1,48 @@
+"""tools/ledger.py: runs as git-tracked JSON with fixed states."""
+import json
+
+import pytest
+
+from tools import ledger
+
+
+def test_new_set_metrics_render_check(tmp_path):
+    run = ledger.new_run("dataset/2026-10-02_alc", "yolo26n-seg.pt", "alc", note="first", root=tmp_path)
+    assert run["status"] == "exported" and run["id"].endswith("_alc")
+    assert (tmp_path / "runs" / run["id"] / "run.json").exists()
+
+    ledger.set_status(run["id"], "training", kv={"host": "spark"}, root=tmp_path)
+    again = ledger.load(run["id"], root=tmp_path)
+    assert again["status"] == "training" and again["host"] == "spark" and len(again["history"]) == 2
+
+    m = tmp_path / "m.json"
+    m.write_text(json.dumps({"overall": {"f1": 0.91, "mean_mask_iou": 0.8}, "gate": {"pass": True, "baseline": "b"}}))
+    ledger.record_metrics(run["id"], m, root=tmp_path)
+    ledger.set_status(run["id"], "gate_passed", root=tmp_path)
+    again = ledger.load(run["id"], root=tmp_path)
+    assert again["metrics"]["f1"] == 0.91 and again["metrics"]["gate_pass"] is True
+    assert (tmp_path / "runs" / run["id"] / "metrics.json").exists()
+
+    readme = ledger.render(root=tmp_path)
+    text = readme.read_text()
+    assert run["id"] in text and "gate_passed" in text and "0.910" in text
+    assert ledger.check(root=tmp_path) == []
+    assert run["id"] in ledger.table(root=tmp_path)
+
+
+def test_second_run_same_day_gets_a_suffix(tmp_path):
+    a = ledger.new_run("dataset/x", "yolo26n-seg.pt", "alc", root=tmp_path)
+    b = ledger.new_run("dataset/x", "yolo26n-seg.pt", "alc", root=tmp_path)
+    assert b["id"] == a["id"] + "-2"
+
+
+def test_unknown_state_and_check_problems(tmp_path):
+    run = ledger.new_run("dataset/x", "yolo26n-seg.pt", "t", root=tmp_path)
+    with pytest.raises(SystemExit):
+        ledger.set_status(run["id"], "done", root=tmp_path)
+    # measured without a metrics.json is a problem the CI check reports
+    p = tmp_path / "runs" / run["id"] / "run.json"
+    d = json.loads(p.read_text()); d["status"] = "measured"; p.write_text(json.dumps(d))
+    problems = ledger.check(root=tmp_path)
+    assert any("no metrics.json" in x for x in problems)
+    assert ledger.main(["--root", str(tmp_path), "check"]) == 1