training loop: run ledger, studio export importer with an image pool, train_run with gold gate and auto-promotion, SLURM template, CI

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/eval/evaluate.py

@@ -0,0 +1,88 @@
+"""Eval harness: compare predicted YOLO-seg labels against a gold set.
+
+Reports detection precision/recall/F1 per class (bbox IoU >= threshold to match)
+and mean mask IoU over matched instances. Operates on label files only (no source
+images needed), so it can score any pre-labeler / model against the frozen gold set.
+
+    python -m src.pipeline.evaluate --pred OUT/labels --gold GOLD/labels [--names dataset.yaml]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Dict, List
+
+from src.eval.metrics import aggregate, evaluate_image, full_report
+from src.eval.yolo_labels import load_class_names, read_label_file
+
+
+def _stems(label_dir: Path) -> Dict[str, Path]:
+    return {p.stem: p for p in label_dir.glob("*.txt")}
+
+
+def evaluate_dirs(pred_dir, gold_dir, iou_thr: float = 0.5,
+                  names: List[str] | None = None) -> Dict:
+    pred_dir, gold_dir = Path(pred_dir), Path(gold_dir)
+    gold_files = _stems(gold_dir)
+    pred_files = _stems(pred_dir)
+
+    # Gold defines the evaluation set; a missing prediction counts as all-FN.
+    per_image = []
+    missing_pred = 0
+    for stem, gpath in sorted(gold_files.items()):
+        gold = read_label_file(gpath)
+        ppath = pred_files.get(stem)
+        pred = read_label_file(ppath) if ppath else []
+        if ppath is None:
+            missing_pred += 1
+        per_image.append(evaluate_image(gold, pred, iou_thr))
+
+    report = full_report(aggregate(per_image), names)
+    report["dataset"] = {
+        "gold_images": len(gold_files),
+        "pred_images": len(pred_files),
+        "missing_predictions": missing_pred,
+        "iou_threshold": iou_thr,
+    }
+    return report
+
+
+def _print_report(report: Dict) -> None:
+    ds = report["dataset"]
+    print(f"\nGold images: {ds['gold_images']}  |  predictions: {ds['pred_images']}  "
+          f"|  missing: {ds['missing_predictions']}  |  match IoU >= {ds['iou_threshold']}")
+    header = f"{'class':<14}{'P':>7}{'R':>7}{'F1':>7}{'maskIoU':>9}{'TP':>6}{'FP':>6}{'FN':>6}"
+    print("\n" + header)
+    print("-" * len(header))
+
+    def row(name, r):
+        print(f"{name:<14}{r['precision']:>7.3f}{r['recall']:>7.3f}{r['f1']:>7.3f}"
+              f"{r['mean_mask_iou']:>9.3f}{r['tp']:>6}{r['fp']:>6}{r['fn']:>6}")
+
+    for name, r in report["per_class"].items():
+        row(name, r)
+    print("-" * len(header))
+    row("OVERALL", report["overall"])
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Evaluate predicted YOLO-seg labels vs. a gold set")
+    ap.add_argument("--pred", required=True, help="Directory of predicted .txt labels")
+    ap.add_argument("--gold", required=True, help="Directory of gold .txt labels")
+    ap.add_argument("--iou", type=float, default=0.5, help="bbox IoU threshold to match instances")
+    ap.add_argument("--names", default=None, help="Optional dataset.yaml for class names")
+    ap.add_argument("--report", default=None, help="Optional path to write the JSON report")
+    args = ap.parse_args()
+
+    names = load_class_names(args.names) if args.names else []
+    report = evaluate_dirs(args.pred, args.gold, iou_thr=args.iou, names=names)
+    _print_report(report)
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"\nWrote {args.report}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())