diff --git a/.gitignore b/.gitignore index 3a66f890..1e69aa12 100644 --- a/.gitignore +++ b/.gitignore @@ -438,6 +438,7 @@ runs/*/gold_predictions/ runs/*/dataset.resolved.yaml runs/slurm-*.out runs/_measure_*/ +runs/_unseen_per_frame.json data/datasets/*/labels/*.cache !models/base/yolo26n-seg.pt !models/base/yolo26s-seg.pt diff --git a/tools/compare_unseen.py b/tools/compare_unseen.py index 1743f713..b9d08859 100644 --- a/tools/compare_unseen.py +++ b/tools/compare_unseen.py @@ -138,6 +138,17 @@ def main(argv: Optional[List[str]] = None) -> int: f, m, c = b["f1"], b["mask_score"], b["per_class"]["crystal"]["f1"] print(f"{k:46} F1 {f['delta']:+.3f} [{f['lo']:+.3f}, {f['hi']:+.3f}] {f['win']:4.0%} mask {m['delta']:+.3f} [{m['lo']:+.3f}, {m['hi']:+.3f}] {m['win']:4.0%} crystal {c['delta']:+.3f} [{c['lo']:+.3f}, {c['hi']:+.3f}] {c['win']:4.0%}") + # per frame and model: overall and crystal counts, so any two models can be compared head to head + # with their noise interval (the monitor page runs the paired bootstrap in the browser) + crystal = NAMES.index("crystal") + + def row(f): + o = gate_stats._total([f], None); c = gate_stats._total([f], crystal) + return [int(o[0]), int(o[1]), int(o[2]), round(o[3], 3), int(c[0]), int(c[1]), int(c[2]), round(c[3], 3)] + per_frame = {"stems": unseen, "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"], + "models": {k: [row(frames[k][st]) if st in frames[k] else None for st in unseen] for k in order}} + (root / "runs" / "_unseen_per_frame.json").write_text(json.dumps(per_frame, separators=(",", ":")), encoding="utf-8") + out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"), "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot} (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8")