compare_unseen: per-frame counts for head-to-head comparisons of any two models

Written to runs/_unseen_per_frame.json (not tracked): overall and crystal
tp/fp/fn and matched IoU per unseen gold frame and model, which lets the
monitor page bootstrap the difference between any pair.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

.gitignore

@@ -438,6 +438,7 @@ runs/*/gold_predictions/
 runs/*/dataset.resolved.yaml
 runs/slurm-*.out
 runs/_measure_*/
+runs/_unseen_per_frame.json
 data/datasets/*/labels/*.cache
 !models/base/yolo26n-seg.pt
 !models/base/yolo26s-seg.pt

tools/compare_unseen.py

@@ -138,6 +138,17 @@ def main(argv: Optional[List[str]] = None) -> int:
             f, m, c = b["f1"], b["mask_score"], b["per_class"]["crystal"]["f1"]
             print(f"{k:46} F1 {f['delta']:+.3f} [{f['lo']:+.3f}, {f['hi']:+.3f}] {f['win']:4.0%}   mask {m['delta']:+.3f} [{m['lo']:+.3f}, {m['hi']:+.3f}] {m['win']:4.0%}   crystal {c['delta']:+.3f} [{c['lo']:+.3f}, {c['hi']:+.3f}] {c['win']:4.0%}")
 
+    # per frame and model: overall and crystal counts, so any two models can be compared head to head
+    # with their noise interval (the monitor page runs the paired bootstrap in the browser)
+    crystal = NAMES.index("crystal")
+
+    def row(f):
+        o = gate_stats._total([f], None); c = gate_stats._total([f], crystal)
+        return [int(o[0]), int(o[1]), int(o[2]), round(o[3], 3), int(c[0]), int(c[1]), int(c[2]), round(c[3], 3)]
+    per_frame = {"stems": unseen, "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"],
+                 "models": {k: [row(frames[k][st]) if st in frames[k] else None for st in unseen] for k in order}}
+    (root / "runs" / "_unseen_per_frame.json").write_text(json.dumps(per_frame, separators=(",", ":")), encoding="utf-8")
+
     out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"),
            "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot}
     (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8")