noise-aware gate, frozen core of unseen frames, and a cycle script for the Spark

A paired bootstrap over the 149 unseen gold frames (2026-10-02) showed that a
difference between two models carries about +-0.023 of noise in F1, +-0.018 in
the mask score and +-0.10 in crystal F1, while the gate's tolerance was 0.005.

- tools/gate_stats.py: per-frame counts and a paired bootstrap (pure Python).
- train_run.py: when the baseline's gold predictions exist, the gate requires
  the candidate to beat it on F1 or the mask score in 90 % of the draws and
  not to lose the other in more than 90 %; the point rule is the fallback and
  is always reported (pass_point_rule).
- tools/compare_unseen.py replaces the Spark's ~/fair_gate_all.py: all models
  on the frames none trained on, on the frozen core (data/gold/unseen_core.json,
  149 frames) and on all unseen frames once the gold set grows, with the
  interval and win share against the active model.
- measure_weights.py keeps a version's gate verdict when it re-measures it.
- tools/spark_cycle.sh: gold growth, export, re-measure, train, gate, compare,
  push, with what cycles 3 and 4 taught built in.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

data/gold/unseen_core.json

@@ -0,0 +1,211 @@
+{
+ "frozen": "2026-10-02",
+ "note": "core = gold frames no model had trained on when the set was frozen; seen = gold frames the April beamline model trained on (or a dataset used)",
+ "core": [
+  "tpi_10130",
+  "tpi_10697",
+  "tpi_1119",
+  "tpi_14517",
+  "tpi_14688",
+  "tpi_14689",
+  "tpi_14698",
+  "tpi_14709",
+  "tpi_14711",
+  "tpi_14715",
+  "tpi_14718",
+  "tpi_15087",
+  "tpi_16223",
+  "tpi_16235",
+  "tpi_16237",
+  "tpi_16252",
+  "tpi_16278",
+  "tpi_16292",
+  "tpi_16319",
+  "tpi_16321",
+  "tpi_16325",
+  "tpi_16357",
+  "tpi_16361",
+  "tpi_16747",
+  "tpi_16753",
+  "tpi_16766",
+  "tpi_16770",
+  "tpi_16956",
+  "tpi_16963",
+  "tpi_17889",
+  "tpi_20221",
+  "tpi_21693",
+  "tpi_31669",
+  "tpi_31928",
+  "tpi_31929",
+  "tpi_32762",
+  "tpi_33849",
+  "tpi_34038",
+  "tpi_34389",
+  "tpi_36563",
+  "tpi_4053",
+  "tpi_45649",
+  "tpi_52402",
+  "tpi_55303",
+  "tpi_55371",
+  "tpi_55383",
+  "tpi_55408",
+  "tpi_55416",
+  "tpi_55565",
+  "tpi_55896",
+  "tpi_55912",
+  "tpi_55941",
+  "tpi_55958",
+  "tpi_56012",
+  "tpi_56096",
+  "tpi_56104",
+  "tpi_56715",
+  "tpi_56717",
+  "tpi_56840",
+  "tpi_57070",
+  "tpi_57242",
+  "tpi_57300",
+  "tpi_57342",
+  "tpi_57654",
+  "tpi_57684",
+  "tpi_57702",
+  "tpi_57991",
+  "tpi_58097",
+  "tpi_58229",
+  "tpi_58417",
+  "tpi_58588",
+  "tpi_58595",
+  "tpi_58842",
+  "tpi_59004",
+  "tpi_59011",
+  "tpi_59120",
+  "tpi_59182",
+  "tpi_59312",
+  "tpi_59412",
+  "tpi_59496",
+  "tpi_59564",
+  "tpi_59570",
+  "tpi_59911",
+  "tpi_59988",
+  "tpi_60052",
+  "tpi_60625",
+  "tpi_61066",
+  "tpi_61069",
+  "tpi_61087",
+  "tpi_61317",
+  "tpi_61327",
+  "tpi_61334",
+  "tpi_61340",
+  "tpi_61343",
+  "tpi_61344",
+  "tpi_61350",
+  "tpi_61358",
+  "tpi_61361",
+  "tpi_61364",
+  "tpi_61372",
+  "tpi_61374",
+  "tpi_61379",
+  "tpi_61380",
+  "tpi_61381",
+  "tpi_61385",
+  "tpi_61392",
+  "tpi_61411",
+  "tpi_61419",
+  "tpi_61428",
+  "tpi_61430",
+  "tpi_61436",
+  "tpi_61437",
+  "tpi_61439",
+  "tpi_61441",
+  "tpi_61442",
+  "tpi_61443",
+  "tpi_61445",
+  "tpi_61447",
+  "tpi_61449",
+  "tpi_61450",
+  "tpi_61452",
+  "tpi_61457",
+  "tpi_61565",
+  "tpi_61572",
+  "tpi_61579",
+  "tpi_61581",
+  "tpi_61594",
+  "tpi_61610",
+  "tpi_61613",
+  "tpi_61618",
+  "tpi_61627",
+  "tpi_61629",
+  "tpi_61633",
+  "tpi_61635",
+  "tpi_61651",
+  "tpi_61653",
+  "tpi_61658",
+  "tpi_61670",
+  "tpi_61686",
+  "tpi_61702",
+  "tpi_61769",
+  "tpi_61834",
+  "tpi_61839",
+  "tpi_61865",
+  "tpi_6203",
+  "tpi_7667",
+  "tpi_8507",
+  "tpi_8744",
+  "tpi_9752"
+ ],
+ "seen": [
+  "tpi_10036",
+  "tpi_10948",
+  "tpi_11106",
+  "tpi_11133",
+  "tpi_11523",
+  "tpi_12381",
+  "tpi_12548",
+  "tpi_12567",
+  "tpi_12580",
+  "tpi_1265",
+  "tpi_12804",
+  "tpi_14167",
+  "tpi_14524",
+  "tpi_14530",
+  "tpi_14552",
+  "tpi_14642",
+  "tpi_14863",
+  "tpi_2014",
+  "tpi_2640",
+  "tpi_3077",
+  "tpi_3466",
+  "tpi_3550",
+  "tpi_4296",
+  "tpi_4313",
+  "tpi_4528",
+  "tpi_5066",
+  "tpi_5178",
+  "tpi_5225",
+  "tpi_5387",
+  "tpi_5425",
+  "tpi_5725",
+  "tpi_5956",
+  "tpi_6202",
+  "tpi_6209",
+  "tpi_6730",
+  "tpi_6737",
+  "tpi_6787",
+  "tpi_7072",
+  "tpi_7302",
+  "tpi_7348",
+  "tpi_7532",
+  "tpi_7658",
+  "tpi_7683",
+  "tpi_7751",
+  "tpi_7810",
+  "tpi_7836",
+  "tpi_7879",
+  "tpi_8142",
+  "tpi_8196",
+  "tpi_8348",
+  "tpi_8400",
+  "tpi_8715",
+  "tpi_8855",
+  "tpi_9805"
+ ]
+}

tests/test_gate.py

@@ -31,3 +31,49 @@ def test_gate_refuses_a_guarded_class_regression(tmp_path):
 
 def test_no_baseline_means_no_gate():
     assert train_run.gate(_report(0.8, 0.9, 1, 0), None) is None
+
+
+def _frames(per_frame_tp, fp=0, objects=2, iou=0.9):
+    """One class, ``objects`` gold objects per frame, ``tp`` of them found."""
+    return {f"f{i}": {0: (float(tp), float(fp), float(objects - tp), iou * tp)} for i, tp in enumerate(per_frame_tp)}
+
+
+def test_paired_bootstrap_separates_a_real_gain_from_a_tie():
+    from tools import gate_stats
+    base = _frames([1] * 60)
+    better = _frames([2] * 40 + [1] * 20)            # finds more on two thirds of the frames
+    same = _frames([1] * 60)
+    r = gate_stats.paired_bootstrap(better, base, draws=400)
+    assert r["frames"] == 60 and r["f1"]["win"] > 0.99 and r["f1"]["lo"] > 0 and r["mask_score"]["win"] > 0.99
+    t = gate_stats.paired_bootstrap(same, base, draws=400)
+    assert t["f1"]["delta"] == 0 and t["f1"]["win"] == 0.0     # never strictly better
+
+
+def test_paired_bootstrap_pairs_only_on_common_frames():
+    from tools import gate_stats
+    cand = _frames([2] * 10); base = _frames([1] * 6)
+    assert gate_stats.paired_bootstrap(cand, base, draws=50)["frames"] == 6
+    assert gate_stats.paired_bootstrap(cand, {}, draws=50) == {"frames": 0, "draws": 0}
+
+
+def test_gate_with_bootstrap_needs_a_clear_win(tmp_path):
+    base = tmp_path / "base.json"
+    base.write_text(json.dumps(_report(0.80, 0.90, 80, 20, {"loop_all": {"f1": 0.90}, "pin": {"f1": 0.90}})))
+    new = _report(0.806, 0.90, 81, 19, {"loop_all": {"f1": 0.90}, "pin": {"f1": 0.90}})
+    noisy = {"frames": 150, "draws": 2000, "f1": {"delta": 0.006, "lo": -0.02, "hi": 0.03, "win": 0.66},
+             "mask_score": {"delta": 0.004, "lo": -0.02, "hi": 0.03, "win": 0.60}}
+    g = train_run.gate(new, base, noisy)
+    assert g["pass_point_rule"] is True and g["pass"] is False and g["rule"] == "paired bootstrap"
+    clear = {"frames": 150, "draws": 2000, "f1": {"delta": 0.05, "lo": 0.02, "hi": 0.08, "win": 1.0},
+             "mask_score": {"delta": 0.04, "lo": 0.01, "hi": 0.07, "win": 0.99}}
+    assert train_run.gate(new, base, clear)["pass"] is True
+    lopsided = {"frames": 150, "draws": 2000, "f1": {"delta": 0.03, "lo": 0.0, "hi": 0.06, "win": 0.97},
+                "mask_score": {"delta": -0.04, "lo": -0.07, "hi": -0.01, "win": 0.01}}
+    assert train_run.gate(new, base, lopsided)["pass"] is False      # wins F1 but clearly loses the masks
+
+
+def test_gate_without_predictions_keeps_the_point_rule(tmp_path):
+    base = tmp_path / "base.json"
+    base.write_text(json.dumps(_report(0.69, 0.903, 60, 40)))
+    g = train_run.gate(_report(0.85, 0.883, 85, 15), base)
+    assert g["rule"] == "point" and g["pass"] is True and g["pass_point_rule"] is True

tools/compare_unseen.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+"""Every measured model on the gold frames none of them trained on, with the noise made visible.
+
+    python tools/compare_unseen.py            # table, runs/clean_gold_comparison.json
+
+Two sets of frames:
+
+* the **core**: the 149 frames that were unseen by every model on 2026-10-02
+  (``data/gold/unseen_core.json``). It never changes, so the progression of the models over time
+  stays comparable;
+* **all unseen** frames: the core plus every gold frame added since that no dataset has used.
+  It grows with the gold set and is the one to judge a candidate on.
+
+A model is listed when its predictions on the current gold set exist under
+``runs/_measure_<version>/gold_predictions`` (``train_run.py`` keeps them for a new run; after
+the gold set grew, re-measure the older versions or they are compared on fewer frames).
+For every model the script also reports, against the active model, the paired-bootstrap
+interval of the difference in F1 and mask score and the share of draws it wins (tools/gate_stats.py).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import gate_stats  # noqa: E402
+
+NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"]
+BEAMLINE_PREFIX = "yolo26n-seg-overlap-false"   # the model trained before the loop
+
+
+def class_report(c) -> Dict[str, float]:
+    tp, fp, fn, iou = c
+    precision = tp / (tp + fp) if (tp + fp) else 0.0
+    recall = tp / (tp + fn) if (tp + fn) else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
+    return {"tp": int(tp), "fp": int(fp), "fn": int(fn), "precision": precision, "recall": recall, "f1": f1,
+            "mean_mask_iou": iou / tp if tp else 0.0, "n_matched": int(tp)}
+
+
+def report(frames: Dict[str, gate_stats.FrameCounts]) -> Dict:
+    rows = list(frames.values())
+    per_class = {}
+    for cid, name in enumerate(NAMES):
+        c = gate_stats._total(rows, cid)
+        if c[0] or c[1] or c[2]:
+            per_class[name] = class_report(c)
+    return {"per_class": per_class, "overall": class_report(gate_stats._total(rows, None)),
+            "dataset": {"gold_images": len(rows)}}
+
+
+def mask_score(o: Dict) -> float:
+    return o["mean_mask_iou"] * (o["tp"] / (o["tp"] + o["fn"]) if (o["tp"] + o["fn"]) else 0.0)
+
+
+def table(title: str, reps: Dict[str, Dict]) -> None:
+    print(title)
+    print(f"{'run':46} {'frames':>6} {'F1':>6} {'recall':>7} {'IoU matched':>12} {'mask score':>11}   loop/pin/crystal/face/ice F1")
+    for k, r in reps.items():
+        o, pc = r["overall"], r["per_class"]
+        print(f"{k:46} {r['dataset']['gold_images']:6d} {o['f1']:6.3f} {o['recall']:7.2f} {o['mean_mask_iou']:12.3f} {mask_score(o):11.3f}   "
+              + "/".join(f"{pc.get(c, {}).get('f1', 0):.2f}" for c in NAMES[:5]))
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--root", default=str(HERE.parent), help="AareLC_train checkout (default: this one)")
+    ap.add_argument("--draws", type=int, default=2000)
+    args = ap.parse_args(argv)
+    root = Path(args.root).expanduser()
+    sys.path.insert(0, str(root))
+    gold = root / "data" / "gold" / "labels"
+    core_file = root / "data" / "gold" / "unseen_core.json"
+    if not core_file.exists():
+        print(f"{core_file} is missing: it lists the frozen core and the gold frames an older model trained on")
+        return 2
+    cj = json.loads(core_file.read_text(encoding="utf-8"))
+    core, seen = set(cj["core"]), set(cj.get("seen", []))
+    trained_on = set()
+    for m in (root / "data" / "datasets").glob("*/manifest.csv"):
+        with open(m, encoding="utf-8") as fh:
+            next(fh, None)
+            for line in fh:
+                key = line.split(",")[0].strip()
+                if key.startswith("tpi_"):
+                    trained_on.add(key)
+    stems = sorted(p.stem for p in gold.glob("*.txt"))
+    key_of = lambda s: "tpi_" + re.search(r"tpi_(\d+)", s).group(1)
+    unseen = [s for s in stems if s not in seen and key_of(s) not in trained_on]
+    core_now = [s for s in unseen if s in core]
+
+    ver2run = {}
+    for rj in (root / "runs").glob("*/run.json"):
+        r = json.loads(rj.read_text(encoding="utf-8"))
+        if r.get("version"):
+            ver2run[r["version"]] = r["id"]
+    active = None
+    ap_file = root / "models" / "active" / "production.json"
+    if ap_file.exists():
+        active = json.loads(ap_file.read_text(encoding="utf-8")).get("version")
+
+    frames: Dict[str, Dict[str, gate_stats.FrameCounts]] = {}
+    label_of: Dict[str, str] = {}
+    for d in sorted((root / "runs").glob("_measure_*/gold_predictions")):
+        ver = d.parent.name[len("_measure_"):]
+        label = ("beamline " + ver) if ver.startswith(BEAMLINE_PREFIX) else ver2run.get(ver, ver)
+        frames[label] = gate_stats.per_frame(d, gold, stems=unseen)
+        label_of[ver] = label
+    order = sorted(frames, key=lambda k: (not k.startswith("beamline"), k))
+    reps = {k: report(frames[k]) for k in order}
+    core_reps = {k: report({s: f for s, f in frames[k].items() if s in core}) for k in order}
+
+    print(f"clean gold frames: {len(unseen)}" + (f" (core {len(core_now)})" if len(unseen) != len(core_now) else ""))
+    table("", reps)
+    short = [k for k in order if reps[k]["dataset"]["gold_images"] < len(unseen)]
+    if short:
+        print(f"!! measured on fewer frames than the current unseen set (re-measure them): {', '.join(short)}")
+    if len(unseen) != len(core_now):
+        table(f"\ncore, the {len(core_now)} frames frozen on {cj.get('frozen', '?')}:", core_reps)
+
+    boot = {}
+    base_label = label_of.get(active) if active else None
+    if base_label and base_label in frames:
+        print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have: difference [95 % interval], share of draws won")
+        classes = {"crystal": NAMES.index("crystal")}
+        for k in order:
+            if k == base_label:
+                continue
+            b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes)
+            if not b.get("frames"):
+                continue
+            boot[k] = b
+            f, m, c = b["f1"], b["mask_score"], b["per_class"]["crystal"]["f1"]
+            print(f"{k:46} F1 {f['delta']:+.3f} [{f['lo']:+.3f}, {f['hi']:+.3f}] {f['win']:4.0%}   mask {m['delta']:+.3f} [{m['lo']:+.3f}, {m['hi']:+.3f}] {m['win']:4.0%}   crystal {c['delta']:+.3f} [{c['lo']:+.3f}, {c['hi']:+.3f}] {c['win']:4.0%}")
+
+    out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"),
+           "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot}
+    (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/gate_stats.py

@@ -0,0 +1,102 @@
+"""How sure is a difference between two models measured on the same gold frames?
+
+The gold set is small (149 unseen frames on 2026-10-02: 40 crystals, 10 ice objects), so a
+difference of one or two points in F1 between two models can be the luck of which frames are
+gold. A paired bootstrap answers that: draw the gold frames again with replacement, the SAME
+draw for both models, and look at the spread of the difference. ``win`` is the share of draws
+in which the candidate beats the baseline.
+
+Pure Python on purpose (the gate's tests run without numpy); only ``per_frame`` needs the
+evaluation code, and imports it when called.
+"""
+from __future__ import annotations
+
+import random
+from pathlib import Path
+from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+Counts = Tuple[float, float, float, float]          # tp, fp, fn, sum of matched mask IoU
+FrameCounts = Dict[int, Counts]                      # class id -> counts, for one frame
+
+
+def per_frame(pred_dir, gold_dir, iou_thr: float = 0.5, stems: Optional[Iterable[str]] = None) -> Dict[str, FrameCounts]:
+    """Per gold frame and class: tp, fp, fn and the summed mask IoU of the matches.
+
+    ``stems`` restricts the frames; a gold frame without a prediction file is left out
+    (the caller pairs on the frames both models have)."""
+    from src.eval.metrics import evaluate_image
+    from src.eval.yolo_labels import read_label_file
+
+    pred_dir, gold_dir = Path(pred_dir), Path(gold_dir)
+    wanted = set(stems) if stems is not None else None
+    out: Dict[str, FrameCounts] = {}
+    for g in sorted(gold_dir.glob("*.txt")):
+        if wanted is not None and g.stem not in wanted:
+            continue
+        p = pred_dir / g.name
+        if not p.exists():
+            continue
+        frame: FrameCounts = {}
+        for c, cc in evaluate_image(read_label_file(g), read_label_file(p), iou_thr).items():
+            frame[int(c)] = (float(cc.tp), float(cc.fp), float(cc.fn), float(sum(cc.mask_ious)))
+        out[g.stem] = frame
+    return out
+
+
+def _total(frames: Sequence[FrameCounts], class_id: Optional[int]) -> Counts:
+    tp = fp = fn = iou = 0.0
+    for f in frames:
+        for c, (a, b, d, e) in f.items():
+            if class_id is None or c == class_id:
+                tp += a; fp += b; fn += d; iou += e
+    return tp, fp, fn, iou
+
+
+def f1_of(c: Counts) -> float:
+    tp, fp, fn, _ = c
+    return 2 * tp / (2 * tp + fp + fn) if tp else 0.0
+
+
+def mask_score_of(c: Counts) -> float:
+    """Mean mask IoU over every gold object, a missed object counting as zero."""
+    tp, _, fn, iou = c
+    return iou / (tp + fn) if (tp + fn) else 0.0
+
+
+def _summary(point: float, deltas: List[float]) -> Dict[str, float]:
+    d = sorted(deltas)
+    n = len(d)
+    lo, hi = d[int(0.025 * (n - 1))], d[int(0.975 * (n - 1))]
+    return {"delta": round(point, 4), "lo": round(lo, 4), "hi": round(hi, 4),
+            "win": round(sum(1 for x in d if x > 0) / n, 3)}
+
+
+def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCounts], *, draws: int = 2000,
+                     seed: int = 0, classes: Optional[Mapping[str, int]] = None) -> Dict:
+    """Candidate minus baseline on the frames both have: point difference, 95 % interval and win share
+    for F1 and the mask score, overall and for each class in ``classes`` (name -> class id)."""
+    keys = sorted(set(cand) & set(base))
+    n = len(keys)
+    if n == 0:
+        return {"frames": 0, "draws": 0}
+    a = [cand[k] for k in keys]; b = [base[k] for k in keys]
+    targets: List[Tuple[str, Optional[int]]] = [("overall", None)] + [(name, cid) for name, cid in (classes or {}).items()]
+    point = {}
+    for name, cid in targets:
+        ca, cb = _total(a, cid), _total(b, cid)
+        point[name] = (f1_of(ca) - f1_of(cb), mask_score_of(ca) - mask_score_of(cb))
+    rng = random.Random(seed)
+    d_f1 = {name: [] for name, _ in targets}; d_ms = {name: [] for name, _ in targets}
+    for _ in range(draws):
+        idx = [rng.randrange(n) for _ in range(n)]
+        sa = [a[i] for i in idx]; sb = [b[i] for i in idx]
+        for name, cid in targets:
+            ca, cb = _total(sa, cid), _total(sb, cid)
+            d_f1[name].append(f1_of(ca) - f1_of(cb)); d_ms[name].append(mask_score_of(ca) - mask_score_of(cb))
+    out = {"frames": n, "draws": draws,
+           "f1": _summary(point["overall"][0], d_f1["overall"]),
+           "mask_score": _summary(point["overall"][1], d_ms["overall"])}
+    if classes:
+        out["per_class"] = {name: {"f1": _summary(point[name][0], d_f1[name]), "mask_score": _summary(point[name][1], d_ms[name])}
+                            for name in classes}
+    return out

tools/measure_weights.py

@@ -38,6 +38,15 @@ def main(argv=None) -> int:
     report["weights"] = str(Path(args.weights).resolve())
     report["measured_at"] = datetime.now().isoformat(timespec="seconds")
     out = reg / "gold_report.json"
+    if out.exists():
+        # a re-measure on a grown gold set must not erase the verdict the gate gave this version
+        try:
+            old = json.loads(out.read_text(encoding="utf-8"))
+            if old.get("gate") is not None:
+                report["gate"] = old["gate"]
+                report["gate_measured_on"] = old.get("gate_measured_on") or old.get("dataset", {}).get("gold_images")
+        except ValueError:
+            pass
     out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
     meta = reg / "metadata.json"
     if not meta.exists():

tools/spark_cycle.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# One training cycle on the Spark, start to finish, unattended.
+#
+#   nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 &
+#   bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick      # also the fast settings-A run first
+#
+# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold`
+# stage: gold batches, share, floors); export the reviewed pool as dataset <tag> into this
+# repo; stop vLLM; re-measure every registry model on the grown gold set so the gate and
+# the comparison stand on the same frames; train the promoted recipe from the base weights;
+# gate (paired bootstrap against the active model), promote on a clear win; compare every
+# model on the unseen frames; push; start vLLM again.
+#
+# What earlier cycles taught, built in:
+#   - a failed gate is a verdict, not a crash: train_run.py is called once per run;
+#   - a full-resolution-mask training does not fit next to vLLM in 121 GB: vLLM is stopped
+#     while it runs and started again on exit, whatever happens;
+#   - torch's native JIT wants Python headers the Spark lacks: switched off, headers on CPATH;
+#   - a grown gold set invalidates stored predictions: everything is re-measured first.
+set -eo pipefail
+TAG="${1:?usage: spark_cycle.sh <dataset tag, e.g. v5> [--quick]}"; QUICK="${2:-}"
+export PATH=$HOME/bin:$PATH
+export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12
+export TORCH_DISABLE_NATIVE_JIT=1
+PY=$HOME/AareLC_studio/.venv/bin/python
+STUDIO=$HOME/AareLC_studio; TRAIN=$HOME/AareLC_train; CFG=pipeline.prod.json
+say() { echo "== $(date +%H:%M) $*"; }
+
+while pgrep -f "[p]ipeline.run auto" >/dev/null; do sleep 30; done   # the reviewers' batch goes first
+cd "$STUDIO"; set -a; . ~/.aaredb_env; set +a
+say "gold: growing by rule"
+$PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true
+say "export + import into AareLC_train (dataset $TAG)"
+$PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true
+cd "$TRAIN"
+DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; }
+DSDIR="data/datasets/${DS#dataset/}"
+say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames"
+
+vllm_back() {
+  for i in 1 2 3; do
+    docker start vllm >> ~/vllm_restart.log 2>&1 || true; sleep 60
+    if docker ps --format "{{.Names}}" | grep -q "^vllm$"; then echo "$(date) vllm up (try $i)" >> ~/vllm_restart.log; return 0; fi
+  done
+  echo "$(date) vllm did NOT come back" >> ~/vllm_restart.log
+}
+trap vllm_back EXIT
+docker stop vllm >/dev/null 2>&1 || true
+say "vLLM stopped, $(free -g | awk 'NR==2{print $7}') GB available"
+
+say "re-measuring every registry model on the $(ls data/gold/labels | wc -l) gold frames"
+for d in models/registry/*/; do
+  v=$(basename "$d"); [ -f "$d/weights.pt" ] || continue
+  $PY tools/measure_weights.py "$d/weights.pt" --version "$v" 2>&1 | tail -n 1
+done
+BEAMLINE_PT=$STUDIO/models/best_yolo26n-seg-overlap-false_2026-04-12.pt   # the model before the loop lives in the studio
+[ -f "$BEAMLINE_PT" ] && $PY tools/measure_weights.py "$BEAMLINE_PT" --version yolo26n-seg-overlap-false-2026-04-12 2>&1 | tail -n 1
+git add models/registry data/gold && git commit -q -m "gold grown to $(ls data/gold/labels | wc -l) frames; every registry model re-measured on it" || true
+
+run() {  # tag suffix, note, extra train_run.py args...
+  local suffix="$1" note="$2"; shift 2
+  local id; id=$(python3 tools/ledger.py new --dataset "$DS" --tag "$TAG-$suffix" --base models/base/yolo26n-seg.pt --note "$note")
+  git add runs && git commit -q -m "run $id: exported on $DS" || true
+  say "training $id"
+  set +e
+  $PY tools/train_run.py "$id" --device 0 --workers 8 "$@" > "runs/$id/train.log" 2>&1 < /dev/null
+  local rc=$?            # 0 = gate passed (or no baseline), 2 = gate failed: both are verdicts
+  set -e
+  [ $rc -eq 0 ] || [ $rc -eq 2 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)"
+  local v; v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')")
+  if [ -n "$v" ] && [ -d "runs/$id/gold_predictions" ]; then mkdir -p "runs/_measure_$v" && cp -r "runs/$id/gold_predictions" "runs/_measure_$v/"; fi
+  python3 -c "import json; m=json.load(open('runs/$id/metrics.json')); g=m.get('gate') or {}; b=g.get('bootstrap') or {}; print('   gate:', g.get('pass'), '| rule:', g.get('rule'), '| F1', g.get('f1_delta'), 'mask', round(g.get('mask_score',0)-g.get('mask_score_baseline',0),4), '| win F1', (b.get('f1') or {}).get('win'), 'win mask', (b.get('mask_score') or {}).get('win'))" 2>/dev/null || true
+  python3 tools/ledger.py render >/dev/null 2>&1 || true
+  git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
+}
+[ "$QUICK" = "--quick" ] && run recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4
+run noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1
+
+say "every model on the unseen gold frames"
+$PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true
+git add runs && git commit -q -m "unseen-frame comparison after cycle $TAG" || true
+say "active model now: $(python3 -c 'import json;print(json.load(open("models/active/production.json"))["version"])')"
+bash ~/sync_push.sh 2>&1 | tail -2
+say "cycle $TAG done"

tools/train_run.py

@@ -38,6 +38,12 @@ AUGMENT = dict(scale=0.7, translate=0.15, mosaic=0.5, close_mosaic=10, fliplr=0.
                shear=5.0, perspective=0.0, mask_ratio=1)
 GATE_TOL = 0.005
 CLASS_TOL = 0.02
+# Since 2026-10-02 the gate knows how noisy the gold set is: when the baseline's predictions are
+# at hand, a paired bootstrap over the gold frames decides. The candidate must beat the baseline
+# on F1 or on the mask score in WIN_CLEAR of the draws, and must not lose the other one in more
+# than 1 - WIN_FLOOR of them. The 0.005 point rule stays as the fallback and is always reported.
+WIN_CLEAR = 0.90
+WIN_FLOOR = 0.10
 GUARDED_CLASSES = ("loop_all", "pin")
 
 
@@ -119,7 +125,7 @@ def mask_score(overall: Dict[str, Any]) -> float:
     return float(overall.get("mean_mask_iou", 0.0)) * (tp / (tp + fn) if tp + fn else 0.0)
 
 
-def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
+def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
     if not baseline or not Path(baseline).exists():
         return None
     base = json.loads(Path(baseline).read_text(encoding="utf-8"))
@@ -132,12 +138,53 @@ def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str,
         a, b = per_class.get(c) or {}, base_classes.get(c) or {}
         if "f1" in a and "f1" in b:
             class_deltas[c] = float(a["f1"]) - float(b["f1"])
-    ok = (f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0)
-          and all(d >= -CLASS_TOL for d in class_deltas.values()))
-    return {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
-            "mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
-            "matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
-            "class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass": bool(ok)}
+    guarded = all(d >= -CLASS_TOL for d in class_deltas.values())
+    ok_point = f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0) and guarded
+    out = {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
+           "mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
+           "matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
+           "class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass_point_rule": bool(ok_point)}
+    if paired and paired.get("frames"):
+        f, m = paired["f1"], paired["mask_score"]
+        improved = f["win"] >= WIN_CLEAR or m["win"] >= WIN_CLEAR
+        not_worse = f["win"] >= WIN_FLOOR and m["win"] >= WIN_FLOOR
+        out.update({"rule": "paired bootstrap", "bootstrap": paired, "pass": bool(improved and not_worse and guarded)})
+    else:
+        out.update({"rule": "point", "pass": bool(ok_point)})
+    return out
+
+
+def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]:
+    """The baseline model's predictions on the gold frames, when they were kept: the re-measure
+    directory of its version, or the gold predictions of the run that produced it."""
+    if not baseline:
+        return None
+    version = Path(baseline).parent.name
+    d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions"
+    if d.is_dir():
+        return d
+    for rj in sorted((ROOT / "runs").glob("*/run.json")):
+        try:
+            if json.loads(rj.read_text(encoding="utf-8")).get("version") == version and (rj.parent / "gold_predictions").is_dir():
+                return rj.parent / "gold_predictions"
+        except (OSError, ValueError):
+            continue
+    return None
+
+
+def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
+    """Paired bootstrap of this run against the baseline on the gold frames both were measured on."""
+    base_pred = baseline_predictions(baseline)
+    if base_pred is None:
+        return None
+    from tools import gate_stats
+    gold = ROOT / "data" / "gold" / "labels"
+    cand = gate_stats.per_frame(run_dir / "gold_predictions", gold)
+    base = gate_stats.per_frame(base_pred, gold)
+    missing = len(cand) - len(set(cand) & set(base))
+    if missing:
+        print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest")
+    return gate_stats.paired_bootstrap(cand, base)
 
 
 def active_version() -> Optional[str]:
@@ -231,7 +278,12 @@ def main(argv=None) -> int:
         av = active_version()
         if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists():
             baseline = ROOT / "models" / "registry" / av / "gold_report.json"
-    g = gate(report, baseline)
+    paired = None
+    try:
+        paired = paired_with_baseline(run_dir, baseline)
+    except Exception as e:  # the point rule still stands
+        print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule")
+    g = gate(report, baseline, paired)
     report["gate"] = g
     (reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
     metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),