From 831367d35daa92bd2af2b366cbedc77c6eb01499 Mon Sep 17 00:00:00 2001 From: GotthardG <51994228+GotthardG@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:51:50 +0200 Subject: [PATCH] noise-aware gate, frozen core of unseen frames, and a cycle script for the Spark A paired bootstrap over the 149 unseen gold frames (2026-10-02) showed that a difference between two models carries about +-0.023 of noise in F1, +-0.018 in the mask score and +-0.10 in crystal F1, while the gate's tolerance was 0.005. - tools/gate_stats.py: per-frame counts and a paired bootstrap (pure Python). - train_run.py: when the baseline's gold predictions exist, the gate requires the candidate to beat it on F1 or the mask score in 90 % of the draws and not to lose the other in more than 90 %; the point rule is the fallback and is always reported (pass_point_rule). - tools/compare_unseen.py replaces the Spark's ~/fair_gate_all.py: all models on the frames none trained on, on the frozen core (data/gold/unseen_core.json, 149 frames) and on all unseen frames once the gold set grows, with the interval and win share against the active model. - measure_weights.py keeps a version's gate verdict when it re-measures it. - tools/spark_cycle.sh: gold growth, export, re-measure, train, gate, compare, push, with what cycles 3 and 4 taught built in. Co-Authored-By: Claude Fable 5.1 --- data/gold/unseen_core.json | 211 +++++++++++++++++++++++++++++++++++++ tests/test_gate.py | 46 ++++++++ tools/compare_unseen.py | 148 ++++++++++++++++++++++++++ tools/gate_stats.py | 102 ++++++++++++++++++ tools/measure_weights.py | 9 ++ tools/spark_cycle.sh | 84 +++++++++++++++ tools/train_run.py | 68 ++++++++++-- 7 files changed, 660 insertions(+), 8 deletions(-) create mode 100644 data/gold/unseen_core.json create mode 100644 tools/compare_unseen.py create mode 100644 tools/gate_stats.py create mode 100755 tools/spark_cycle.sh diff --git a/data/gold/unseen_core.json b/data/gold/unseen_core.json new file mode 100644 index 0000000..75cd6d7 --- /dev/null +++ b/data/gold/unseen_core.json @@ -0,0 +1,211 @@ +{ + "frozen": "2026-10-02", + "note": "core = gold frames no model had trained on when the set was frozen; seen = gold frames the April beamline model trained on (or a dataset used)", + "core": [ + "tpi_10130", + "tpi_10697", + "tpi_1119", + "tpi_14517", + "tpi_14688", + "tpi_14689", + "tpi_14698", + "tpi_14709", + "tpi_14711", + "tpi_14715", + "tpi_14718", + "tpi_15087", + "tpi_16223", + "tpi_16235", + "tpi_16237", + "tpi_16252", + "tpi_16278", + "tpi_16292", + "tpi_16319", + "tpi_16321", + "tpi_16325", + "tpi_16357", + "tpi_16361", + "tpi_16747", + "tpi_16753", + "tpi_16766", + "tpi_16770", + "tpi_16956", + "tpi_16963", + "tpi_17889", + "tpi_20221", + "tpi_21693", + "tpi_31669", + "tpi_31928", + "tpi_31929", + "tpi_32762", + "tpi_33849", + "tpi_34038", + "tpi_34389", + "tpi_36563", + "tpi_4053", + "tpi_45649", + "tpi_52402", + "tpi_55303", + "tpi_55371", + "tpi_55383", + "tpi_55408", + "tpi_55416", + "tpi_55565", + "tpi_55896", + "tpi_55912", + "tpi_55941", + "tpi_55958", + "tpi_56012", + "tpi_56096", + "tpi_56104", + "tpi_56715", + "tpi_56717", + "tpi_56840", + "tpi_57070", + "tpi_57242", + "tpi_57300", + "tpi_57342", + "tpi_57654", + "tpi_57684", + "tpi_57702", + "tpi_57991", + "tpi_58097", + "tpi_58229", + "tpi_58417", + "tpi_58588", + "tpi_58595", + "tpi_58842", + "tpi_59004", + "tpi_59011", + "tpi_59120", + "tpi_59182", + "tpi_59312", + "tpi_59412", + "tpi_59496", + "tpi_59564", + "tpi_59570", + "tpi_59911", + "tpi_59988", + "tpi_60052", + "tpi_60625", + "tpi_61066", + "tpi_61069", + "tpi_61087", + "tpi_61317", + "tpi_61327", + "tpi_61334", + "tpi_61340", + "tpi_61343", + "tpi_61344", + "tpi_61350", + "tpi_61358", + "tpi_61361", + "tpi_61364", + "tpi_61372", + "tpi_61374", + "tpi_61379", + "tpi_61380", + "tpi_61381", + "tpi_61385", + "tpi_61392", + "tpi_61411", + "tpi_61419", + "tpi_61428", + "tpi_61430", + "tpi_61436", + "tpi_61437", + "tpi_61439", + "tpi_61441", + "tpi_61442", + "tpi_61443", + "tpi_61445", + "tpi_61447", + "tpi_61449", + "tpi_61450", + "tpi_61452", + "tpi_61457", + "tpi_61565", + "tpi_61572", + "tpi_61579", + "tpi_61581", + "tpi_61594", + "tpi_61610", + "tpi_61613", + "tpi_61618", + "tpi_61627", + "tpi_61629", + "tpi_61633", + "tpi_61635", + "tpi_61651", + "tpi_61653", + "tpi_61658", + "tpi_61670", + "tpi_61686", + "tpi_61702", + "tpi_61769", + "tpi_61834", + "tpi_61839", + "tpi_61865", + "tpi_6203", + "tpi_7667", + "tpi_8507", + "tpi_8744", + "tpi_9752" + ], + "seen": [ + "tpi_10036", + "tpi_10948", + "tpi_11106", + "tpi_11133", + "tpi_11523", + "tpi_12381", + "tpi_12548", + "tpi_12567", + "tpi_12580", + "tpi_1265", + "tpi_12804", + "tpi_14167", + "tpi_14524", + "tpi_14530", + "tpi_14552", + "tpi_14642", + "tpi_14863", + "tpi_2014", + "tpi_2640", + "tpi_3077", + "tpi_3466", + "tpi_3550", + "tpi_4296", + "tpi_4313", + "tpi_4528", + "tpi_5066", + "tpi_5178", + "tpi_5225", + "tpi_5387", + "tpi_5425", + "tpi_5725", + "tpi_5956", + "tpi_6202", + "tpi_6209", + "tpi_6730", + "tpi_6737", + "tpi_6787", + "tpi_7072", + "tpi_7302", + "tpi_7348", + "tpi_7532", + "tpi_7658", + "tpi_7683", + "tpi_7751", + "tpi_7810", + "tpi_7836", + "tpi_7879", + "tpi_8142", + "tpi_8196", + "tpi_8348", + "tpi_8400", + "tpi_8715", + "tpi_8855", + "tpi_9805" + ] +} \ No newline at end of file diff --git a/tests/test_gate.py b/tests/test_gate.py index a01d64e..8e10543 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -31,3 +31,49 @@ def test_gate_refuses_a_guarded_class_regression(tmp_path): def test_no_baseline_means_no_gate(): assert train_run.gate(_report(0.8, 0.9, 1, 0), None) is None + + +def _frames(per_frame_tp, fp=0, objects=2, iou=0.9): + """One class, ``objects`` gold objects per frame, ``tp`` of them found.""" + return {f"f{i}": {0: (float(tp), float(fp), float(objects - tp), iou * tp)} for i, tp in enumerate(per_frame_tp)} + + +def test_paired_bootstrap_separates_a_real_gain_from_a_tie(): + from tools import gate_stats + base = _frames([1] * 60) + better = _frames([2] * 40 + [1] * 20) # finds more on two thirds of the frames + same = _frames([1] * 60) + r = gate_stats.paired_bootstrap(better, base, draws=400) + assert r["frames"] == 60 and r["f1"]["win"] > 0.99 and r["f1"]["lo"] > 0 and r["mask_score"]["win"] > 0.99 + t = gate_stats.paired_bootstrap(same, base, draws=400) + assert t["f1"]["delta"] == 0 and t["f1"]["win"] == 0.0 # never strictly better + + +def test_paired_bootstrap_pairs_only_on_common_frames(): + from tools import gate_stats + cand = _frames([2] * 10); base = _frames([1] * 6) + assert gate_stats.paired_bootstrap(cand, base, draws=50)["frames"] == 6 + assert gate_stats.paired_bootstrap(cand, {}, draws=50) == {"frames": 0, "draws": 0} + + +def test_gate_with_bootstrap_needs_a_clear_win(tmp_path): + base = tmp_path / "base.json" + base.write_text(json.dumps(_report(0.80, 0.90, 80, 20, {"loop_all": {"f1": 0.90}, "pin": {"f1": 0.90}}))) + new = _report(0.806, 0.90, 81, 19, {"loop_all": {"f1": 0.90}, "pin": {"f1": 0.90}}) + noisy = {"frames": 150, "draws": 2000, "f1": {"delta": 0.006, "lo": -0.02, "hi": 0.03, "win": 0.66}, + "mask_score": {"delta": 0.004, "lo": -0.02, "hi": 0.03, "win": 0.60}} + g = train_run.gate(new, base, noisy) + assert g["pass_point_rule"] is True and g["pass"] is False and g["rule"] == "paired bootstrap" + clear = {"frames": 150, "draws": 2000, "f1": {"delta": 0.05, "lo": 0.02, "hi": 0.08, "win": 1.0}, + "mask_score": {"delta": 0.04, "lo": 0.01, "hi": 0.07, "win": 0.99}} + assert train_run.gate(new, base, clear)["pass"] is True + lopsided = {"frames": 150, "draws": 2000, "f1": {"delta": 0.03, "lo": 0.0, "hi": 0.06, "win": 0.97}, + "mask_score": {"delta": -0.04, "lo": -0.07, "hi": -0.01, "win": 0.01}} + assert train_run.gate(new, base, lopsided)["pass"] is False # wins F1 but clearly loses the masks + + +def test_gate_without_predictions_keeps_the_point_rule(tmp_path): + base = tmp_path / "base.json" + base.write_text(json.dumps(_report(0.69, 0.903, 60, 40))) + g = train_run.gate(_report(0.85, 0.883, 85, 15), base) + assert g["rule"] == "point" and g["pass"] is True and g["pass_point_rule"] is True diff --git a/tools/compare_unseen.py b/tools/compare_unseen.py new file mode 100644 index 0000000..1743f71 --- /dev/null +++ b/tools/compare_unseen.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Every measured model on the gold frames none of them trained on, with the noise made visible. + + python tools/compare_unseen.py # table, runs/clean_gold_comparison.json + +Two sets of frames: + +* the **core**: the 149 frames that were unseen by every model on 2026-10-02 + (``data/gold/unseen_core.json``). It never changes, so the progression of the models over time + stays comparable; +* **all unseen** frames: the core plus every gold frame added since that no dataset has used. + It grows with the gold set and is the one to judge a candidate on. + +A model is listed when its predictions on the current gold set exist under +``runs/_measure_/gold_predictions`` (``train_run.py`` keeps them for a new run; after +the gold set grew, re-measure the older versions or they are compared on fewer frames). +For every model the script also reports, against the active model, the paired-bootstrap +interval of the difference in F1 and mask score and the share of draws it wins (tools/gate_stats.py). +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Dict, List, Optional + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import gate_stats # noqa: E402 + +NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"] +BEAMLINE_PREFIX = "yolo26n-seg-overlap-false" # the model trained before the loop + + +def class_report(c) -> Dict[str, float]: + tp, fp, fn, iou = c + precision = tp / (tp + fp) if (tp + fp) else 0.0 + recall = tp / (tp + fn) if (tp + fn) else 0.0 + f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0 + return {"tp": int(tp), "fp": int(fp), "fn": int(fn), "precision": precision, "recall": recall, "f1": f1, + "mean_mask_iou": iou / tp if tp else 0.0, "n_matched": int(tp)} + + +def report(frames: Dict[str, gate_stats.FrameCounts]) -> Dict: + rows = list(frames.values()) + per_class = {} + for cid, name in enumerate(NAMES): + c = gate_stats._total(rows, cid) + if c[0] or c[1] or c[2]: + per_class[name] = class_report(c) + return {"per_class": per_class, "overall": class_report(gate_stats._total(rows, None)), + "dataset": {"gold_images": len(rows)}} + + +def mask_score(o: Dict) -> float: + return o["mean_mask_iou"] * (o["tp"] / (o["tp"] + o["fn"]) if (o["tp"] + o["fn"]) else 0.0) + + +def table(title: str, reps: Dict[str, Dict]) -> None: + print(title) + print(f"{'run':46} {'frames':>6} {'F1':>6} {'recall':>7} {'IoU matched':>12} {'mask score':>11} loop/pin/crystal/face/ice F1") + for k, r in reps.items(): + o, pc = r["overall"], r["per_class"] + print(f"{k:46} {r['dataset']['gold_images']:6d} {o['f1']:6.3f} {o['recall']:7.2f} {o['mean_mask_iou']:12.3f} {mask_score(o):11.3f} " + + "/".join(f"{pc.get(c, {}).get('f1', 0):.2f}" for c in NAMES[:5])) + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", default=str(HERE.parent), help="AareLC_train checkout (default: this one)") + ap.add_argument("--draws", type=int, default=2000) + args = ap.parse_args(argv) + root = Path(args.root).expanduser() + sys.path.insert(0, str(root)) + gold = root / "data" / "gold" / "labels" + core_file = root / "data" / "gold" / "unseen_core.json" + if not core_file.exists(): + print(f"{core_file} is missing: it lists the frozen core and the gold frames an older model trained on") + return 2 + cj = json.loads(core_file.read_text(encoding="utf-8")) + core, seen = set(cj["core"]), set(cj.get("seen", [])) + trained_on = set() + for m in (root / "data" / "datasets").glob("*/manifest.csv"): + with open(m, encoding="utf-8") as fh: + next(fh, None) + for line in fh: + key = line.split(",")[0].strip() + if key.startswith("tpi_"): + trained_on.add(key) + stems = sorted(p.stem for p in gold.glob("*.txt")) + key_of = lambda s: "tpi_" + re.search(r"tpi_(\d+)", s).group(1) + unseen = [s for s in stems if s not in seen and key_of(s) not in trained_on] + core_now = [s for s in unseen if s in core] + + ver2run = {} + for rj in (root / "runs").glob("*/run.json"): + r = json.loads(rj.read_text(encoding="utf-8")) + if r.get("version"): + ver2run[r["version"]] = r["id"] + active = None + ap_file = root / "models" / "active" / "production.json" + if ap_file.exists(): + active = json.loads(ap_file.read_text(encoding="utf-8")).get("version") + + frames: Dict[str, Dict[str, gate_stats.FrameCounts]] = {} + label_of: Dict[str, str] = {} + for d in sorted((root / "runs").glob("_measure_*/gold_predictions")): + ver = d.parent.name[len("_measure_"):] + label = ("beamline " + ver) if ver.startswith(BEAMLINE_PREFIX) else ver2run.get(ver, ver) + frames[label] = gate_stats.per_frame(d, gold, stems=unseen) + label_of[ver] = label + order = sorted(frames, key=lambda k: (not k.startswith("beamline"), k)) + reps = {k: report(frames[k]) for k in order} + core_reps = {k: report({s: f for s, f in frames[k].items() if s in core}) for k in order} + + print(f"clean gold frames: {len(unseen)}" + (f" (core {len(core_now)})" if len(unseen) != len(core_now) else "")) + table("", reps) + short = [k for k in order if reps[k]["dataset"]["gold_images"] < len(unseen)] + if short: + print(f"!! measured on fewer frames than the current unseen set (re-measure them): {', '.join(short)}") + if len(unseen) != len(core_now): + table(f"\ncore, the {len(core_now)} frames frozen on {cj.get('frozen', '?')}:", core_reps) + + boot = {} + base_label = label_of.get(active) if active else None + if base_label and base_label in frames: + print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have: difference [95 % interval], share of draws won") + classes = {"crystal": NAMES.index("crystal")} + for k in order: + if k == base_label: + continue + b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes) + if not b.get("frames"): + continue + boot[k] = b + f, m, c = b["f1"], b["mask_score"], b["per_class"]["crystal"]["f1"] + print(f"{k:46} F1 {f['delta']:+.3f} [{f['lo']:+.3f}, {f['hi']:+.3f}] {f['win']:4.0%} mask {m['delta']:+.3f} [{m['lo']:+.3f}, {m['hi']:+.3f}] {m['win']:4.0%} crystal {c['delta']:+.3f} [{c['lo']:+.3f}, {c['hi']:+.3f}] {c['win']:4.0%}") + + out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"), + "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot} + (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/gate_stats.py b/tools/gate_stats.py new file mode 100644 index 0000000..96bb51e --- /dev/null +++ b/tools/gate_stats.py @@ -0,0 +1,102 @@ +"""How sure is a difference between two models measured on the same gold frames? + +The gold set is small (149 unseen frames on 2026-10-02: 40 crystals, 10 ice objects), so a +difference of one or two points in F1 between two models can be the luck of which frames are +gold. A paired bootstrap answers that: draw the gold frames again with replacement, the SAME +draw for both models, and look at the spread of the difference. ``win`` is the share of draws +in which the candidate beats the baseline. + +Pure Python on purpose (the gate's tests run without numpy); only ``per_frame`` needs the +evaluation code, and imports it when called. +""" +from __future__ import annotations + +import random +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple + +Counts = Tuple[float, float, float, float] # tp, fp, fn, sum of matched mask IoU +FrameCounts = Dict[int, Counts] # class id -> counts, for one frame + + +def per_frame(pred_dir, gold_dir, iou_thr: float = 0.5, stems: Optional[Iterable[str]] = None) -> Dict[str, FrameCounts]: + """Per gold frame and class: tp, fp, fn and the summed mask IoU of the matches. + + ``stems`` restricts the frames; a gold frame without a prediction file is left out + (the caller pairs on the frames both models have).""" + from src.eval.metrics import evaluate_image + from src.eval.yolo_labels import read_label_file + + pred_dir, gold_dir = Path(pred_dir), Path(gold_dir) + wanted = set(stems) if stems is not None else None + out: Dict[str, FrameCounts] = {} + for g in sorted(gold_dir.glob("*.txt")): + if wanted is not None and g.stem not in wanted: + continue + p = pred_dir / g.name + if not p.exists(): + continue + frame: FrameCounts = {} + for c, cc in evaluate_image(read_label_file(g), read_label_file(p), iou_thr).items(): + frame[int(c)] = (float(cc.tp), float(cc.fp), float(cc.fn), float(sum(cc.mask_ious))) + out[g.stem] = frame + return out + + +def _total(frames: Sequence[FrameCounts], class_id: Optional[int]) -> Counts: + tp = fp = fn = iou = 0.0 + for f in frames: + for c, (a, b, d, e) in f.items(): + if class_id is None or c == class_id: + tp += a; fp += b; fn += d; iou += e + return tp, fp, fn, iou + + +def f1_of(c: Counts) -> float: + tp, fp, fn, _ = c + return 2 * tp / (2 * tp + fp + fn) if tp else 0.0 + + +def mask_score_of(c: Counts) -> float: + """Mean mask IoU over every gold object, a missed object counting as zero.""" + tp, _, fn, iou = c + return iou / (tp + fn) if (tp + fn) else 0.0 + + +def _summary(point: float, deltas: List[float]) -> Dict[str, float]: + d = sorted(deltas) + n = len(d) + lo, hi = d[int(0.025 * (n - 1))], d[int(0.975 * (n - 1))] + return {"delta": round(point, 4), "lo": round(lo, 4), "hi": round(hi, 4), + "win": round(sum(1 for x in d if x > 0) / n, 3)} + + +def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCounts], *, draws: int = 2000, + seed: int = 0, classes: Optional[Mapping[str, int]] = None) -> Dict: + """Candidate minus baseline on the frames both have: point difference, 95 % interval and win share + for F1 and the mask score, overall and for each class in ``classes`` (name -> class id).""" + keys = sorted(set(cand) & set(base)) + n = len(keys) + if n == 0: + return {"frames": 0, "draws": 0} + a = [cand[k] for k in keys]; b = [base[k] for k in keys] + targets: List[Tuple[str, Optional[int]]] = [("overall", None)] + [(name, cid) for name, cid in (classes or {}).items()] + point = {} + for name, cid in targets: + ca, cb = _total(a, cid), _total(b, cid) + point[name] = (f1_of(ca) - f1_of(cb), mask_score_of(ca) - mask_score_of(cb)) + rng = random.Random(seed) + d_f1 = {name: [] for name, _ in targets}; d_ms = {name: [] for name, _ in targets} + for _ in range(draws): + idx = [rng.randrange(n) for _ in range(n)] + sa = [a[i] for i in idx]; sb = [b[i] for i in idx] + for name, cid in targets: + ca, cb = _total(sa, cid), _total(sb, cid) + d_f1[name].append(f1_of(ca) - f1_of(cb)); d_ms[name].append(mask_score_of(ca) - mask_score_of(cb)) + out = {"frames": n, "draws": draws, + "f1": _summary(point["overall"][0], d_f1["overall"]), + "mask_score": _summary(point["overall"][1], d_ms["overall"])} + if classes: + out["per_class"] = {name: {"f1": _summary(point[name][0], d_f1[name]), "mask_score": _summary(point[name][1], d_ms[name])} + for name in classes} + return out diff --git a/tools/measure_weights.py b/tools/measure_weights.py index 183ce28..d12f798 100644 --- a/tools/measure_weights.py +++ b/tools/measure_weights.py @@ -38,6 +38,15 @@ def main(argv=None) -> int: report["weights"] = str(Path(args.weights).resolve()) report["measured_at"] = datetime.now().isoformat(timespec="seconds") out = reg / "gold_report.json" + if out.exists(): + # a re-measure on a grown gold set must not erase the verdict the gate gave this version + try: + old = json.loads(out.read_text(encoding="utf-8")) + if old.get("gate") is not None: + report["gate"] = old["gate"] + report["gate_measured_on"] = old.get("gate_measured_on") or old.get("dataset", {}).get("gold_images") + except ValueError: + pass out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") meta = reg / "metadata.json" if not meta.exists(): diff --git a/tools/spark_cycle.sh b/tools/spark_cycle.sh new file mode 100755 index 0000000..4dc5749 --- /dev/null +++ b/tools/spark_cycle.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# One training cycle on the Spark, start to finish, unattended. +# +# nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 & +# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first +# +# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold` +# stage: gold batches, share, floors); export the reviewed pool as dataset into this +# repo; stop vLLM; re-measure every registry model on the grown gold set so the gate and +# the comparison stand on the same frames; train the promoted recipe from the base weights; +# gate (paired bootstrap against the active model), promote on a clear win; compare every +# model on the unseen frames; push; start vLLM again. +# +# What earlier cycles taught, built in: +# - a failed gate is a verdict, not a crash: train_run.py is called once per run; +# - a full-resolution-mask training does not fit next to vLLM in 121 GB: vLLM is stopped +# while it runs and started again on exit, whatever happens; +# - torch's native JIT wants Python headers the Spark lacks: switched off, headers on CPATH; +# - a grown gold set invalidates stored predictions: everything is re-measured first. +set -eo pipefail +TAG="${1:?usage: spark_cycle.sh [--quick]}"; QUICK="${2:-}" +export PATH=$HOME/bin:$PATH +export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12 +export TORCH_DISABLE_NATIVE_JIT=1 +PY=$HOME/AareLC_studio/.venv/bin/python +STUDIO=$HOME/AareLC_studio; TRAIN=$HOME/AareLC_train; CFG=pipeline.prod.json +say() { echo "== $(date +%H:%M) $*"; } + +while pgrep -f "[p]ipeline.run auto" >/dev/null; do sleep 30; done # the reviewers' batch goes first +cd "$STUDIO"; set -a; . ~/.aaredb_env; set +a +say "gold: growing by rule" +$PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true +say "export + import into AareLC_train (dataset $TAG)" +$PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true +cd "$TRAIN" +DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; } +DSDIR="data/datasets/${DS#dataset/}" +say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames" + +vllm_back() { + for i in 1 2 3; do + docker start vllm >> ~/vllm_restart.log 2>&1 || true; sleep 60 + if docker ps --format "{{.Names}}" | grep -q "^vllm$"; then echo "$(date) vllm up (try $i)" >> ~/vllm_restart.log; return 0; fi + done + echo "$(date) vllm did NOT come back" >> ~/vllm_restart.log +} +trap vllm_back EXIT +docker stop vllm >/dev/null 2>&1 || true +say "vLLM stopped, $(free -g | awk 'NR==2{print $7}') GB available" + +say "re-measuring every registry model on the $(ls data/gold/labels | wc -l) gold frames" +for d in models/registry/*/; do + v=$(basename "$d"); [ -f "$d/weights.pt" ] || continue + $PY tools/measure_weights.py "$d/weights.pt" --version "$v" 2>&1 | tail -n 1 +done +BEAMLINE_PT=$STUDIO/models/best_yolo26n-seg-overlap-false_2026-04-12.pt # the model before the loop lives in the studio +[ -f "$BEAMLINE_PT" ] && $PY tools/measure_weights.py "$BEAMLINE_PT" --version yolo26n-seg-overlap-false-2026-04-12 2>&1 | tail -n 1 +git add models/registry data/gold && git commit -q -m "gold grown to $(ls data/gold/labels | wc -l) frames; every registry model re-measured on it" || true + +run() { # tag suffix, note, extra train_run.py args... + local suffix="$1" note="$2"; shift 2 + local id; id=$(python3 tools/ledger.py new --dataset "$DS" --tag "$TAG-$suffix" --base models/base/yolo26n-seg.pt --note "$note") + git add runs && git commit -q -m "run $id: exported on $DS" || true + say "training $id" + set +e + $PY tools/train_run.py "$id" --device 0 --workers 8 "$@" > "runs/$id/train.log" 2>&1 < /dev/null + local rc=$? # 0 = gate passed (or no baseline), 2 = gate failed: both are verdicts + set -e + [ $rc -eq 0 ] || [ $rc -eq 2 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)" + local v; v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')") + if [ -n "$v" ] && [ -d "runs/$id/gold_predictions" ]; then mkdir -p "runs/_measure_$v" && cp -r "runs/$id/gold_predictions" "runs/_measure_$v/"; fi + python3 -c "import json; m=json.load(open('runs/$id/metrics.json')); g=m.get('gate') or {}; b=g.get('bootstrap') or {}; print(' gate:', g.get('pass'), '| rule:', g.get('rule'), '| F1', g.get('f1_delta'), 'mask', round(g.get('mask_score',0)-g.get('mask_score_baseline',0),4), '| win F1', (b.get('f1') or {}).get('win'), 'win mask', (b.get('mask_score') or {}).get('win'))" 2>/dev/null || true + python3 tools/ledger.py render >/dev/null 2>&1 || true + git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true +} +[ "$QUICK" = "--quick" ] && run recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4 +run noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1 + +say "every model on the unseen gold frames" +$PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true +git add runs && git commit -q -m "unseen-frame comparison after cycle $TAG" || true +say "active model now: $(python3 -c 'import json;print(json.load(open("models/active/production.json"))["version"])')" +bash ~/sync_push.sh 2>&1 | tail -2 +say "cycle $TAG done" diff --git a/tools/train_run.py b/tools/train_run.py index f53e261..aaf9551 100644 --- a/tools/train_run.py +++ b/tools/train_run.py @@ -38,6 +38,12 @@ AUGMENT = dict(scale=0.7, translate=0.15, mosaic=0.5, close_mosaic=10, fliplr=0. shear=5.0, perspective=0.0, mask_ratio=1) GATE_TOL = 0.005 CLASS_TOL = 0.02 +# Since 2026-10-02 the gate knows how noisy the gold set is: when the baseline's predictions are +# at hand, a paired bootstrap over the gold frames decides. The candidate must beat the baseline +# on F1 or on the mask score in WIN_CLEAR of the draws, and must not lose the other one in more +# than 1 - WIN_FLOOR of them. The 0.005 point rule stays as the fallback and is always reported. +WIN_CLEAR = 0.90 +WIN_FLOOR = 0.10 GUARDED_CLASSES = ("loop_all", "pin") @@ -119,7 +125,7 @@ def mask_score(overall: Dict[str, Any]) -> float: return float(overall.get("mean_mask_iou", 0.0)) * (tp / (tp + fn) if tp + fn else 0.0) -def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str, Any]]: +def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]: if not baseline or not Path(baseline).exists(): return None base = json.loads(Path(baseline).read_text(encoding="utf-8")) @@ -132,12 +138,53 @@ def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str, a, b = per_class.get(c) or {}, base_classes.get(c) or {} if "f1" in a and "f1" in b: class_deltas[c] = float(a["f1"]) - float(b["f1"]) - ok = (f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0) - and all(d >= -CLASS_TOL for d in class_deltas.values())) - return {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4), - "mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4), - "matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4), - "class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass": bool(ok)} + guarded = all(d >= -CLASS_TOL for d in class_deltas.values()) + ok_point = f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0) and guarded + out = {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4), + "mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4), + "matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4), + "class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass_point_rule": bool(ok_point)} + if paired and paired.get("frames"): + f, m = paired["f1"], paired["mask_score"] + improved = f["win"] >= WIN_CLEAR or m["win"] >= WIN_CLEAR + not_worse = f["win"] >= WIN_FLOOR and m["win"] >= WIN_FLOOR + out.update({"rule": "paired bootstrap", "bootstrap": paired, "pass": bool(improved and not_worse and guarded)}) + else: + out.update({"rule": "point", "pass": bool(ok_point)}) + return out + + +def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]: + """The baseline model's predictions on the gold frames, when they were kept: the re-measure + directory of its version, or the gold predictions of the run that produced it.""" + if not baseline: + return None + version = Path(baseline).parent.name + d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions" + if d.is_dir(): + return d + for rj in sorted((ROOT / "runs").glob("*/run.json")): + try: + if json.loads(rj.read_text(encoding="utf-8")).get("version") == version and (rj.parent / "gold_predictions").is_dir(): + return rj.parent / "gold_predictions" + except (OSError, ValueError): + continue + return None + + +def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Dict[str, Any]]: + """Paired bootstrap of this run against the baseline on the gold frames both were measured on.""" + base_pred = baseline_predictions(baseline) + if base_pred is None: + return None + from tools import gate_stats + gold = ROOT / "data" / "gold" / "labels" + cand = gate_stats.per_frame(run_dir / "gold_predictions", gold) + base = gate_stats.per_frame(base_pred, gold) + missing = len(cand) - len(set(cand) & set(base)) + if missing: + print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest") + return gate_stats.paired_bootstrap(cand, base) def active_version() -> Optional[str]: @@ -231,7 +278,12 @@ def main(argv=None) -> int: av = active_version() if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists(): baseline = ROOT / "models" / "registry" / av / "gold_report.json" - g = gate(report, baseline) + paired = None + try: + paired = paired_with_baseline(run_dir, baseline) + except Exception as e: # the point rule still stands + print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule") + g = gate(report, baseline, paired) report["gate"] = g (reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),