jungfrauandClaude Fable 5.1
63ee07fe61
CI / ledger-and-tools (push) Successful in 19s
454 of the 1 029 gold outlines are a proposal kept unchanged (loop 159 of 317, pin 140 of
224, face 140 of 295, crystal 14 of 149): the pre-labelling model's own masks. The
evaluation now ignores them object by object for every model, like a crowd region: they
leave the gold, and an unmatched prediction on one of them is not a false alarm.
tools/gold_kept.py finds them against the studio's batch proposals (mask IoU >= 0.98) and
writes data/gold/kept_machine.json; evaluate_image, evaluate_dirs and gate_stats.per_frame
read it. With it, the frame-level exclusion of 845bd87 is no longer applied (it stays as
the fallback without the file), so the gold set can grow with frames the active model
pre-labelled. Every registry model is measured again on this basis and the three v5 runs
are re-gated, without promotion.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013uu3Bxsu5JqFhaWa9Ftu6z
95 lines
3.6 KiB
Python
95 lines
3.6 KiB
Python
"""Eval harness: compare predicted YOLO-seg labels against a gold set.
|
|
|
|
Reports detection precision/recall/F1 per class (bbox IoU >= threshold to match)
|
|
and mean mask IoU over matched instances. Operates on label files only (no source
|
|
images needed), so it can score any pre-labeler / model against the frozen gold set.
|
|
|
|
python -m src.pipeline.evaluate --pred OUT/labels --gold GOLD/labels [--names dataset.yaml]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Dict, List
|
|
|
|
from src.eval.metrics import aggregate, evaluate_image, full_report
|
|
from src.eval.yolo_labels import load_class_names, read_kept_outlines, read_label_file
|
|
|
|
|
|
def _stems(label_dir: Path) -> Dict[str, Path]:
|
|
return {p.stem: p for p in label_dir.glob("*.txt")}
|
|
|
|
|
|
def evaluate_dirs(pred_dir, gold_dir, iou_thr: float = 0.5,
|
|
names: List[str] | None = None) -> Dict:
|
|
pred_dir, gold_dir = Path(pred_dir), Path(gold_dir)
|
|
gold_files = _stems(gold_dir)
|
|
pred_files = _stems(pred_dir)
|
|
# outlines a model proposed and the reviewer kept are not judged (see evaluate_image)
|
|
kept = read_kept_outlines(gold_dir.parent)
|
|
|
|
# Gold defines the evaluation set; a missing prediction counts as all-FN.
|
|
per_image = []
|
|
missing_pred = 0
|
|
ignored = 0
|
|
for stem, gpath in sorted(gold_files.items()):
|
|
gold = read_label_file(gpath)
|
|
ppath = pred_files.get(stem)
|
|
pred = read_label_file(ppath) if ppath else []
|
|
if ppath is None:
|
|
missing_pred += 1
|
|
ign = kept.get(stem, ())
|
|
ignored += len(ign)
|
|
per_image.append(evaluate_image(gold, pred, iou_thr, ignore=ign))
|
|
|
|
report = full_report(aggregate(per_image), names)
|
|
report["dataset"] = {
|
|
"gold_images": len(gold_files),
|
|
"pred_images": len(pred_files),
|
|
"missing_predictions": missing_pred,
|
|
"iou_threshold": iou_thr,
|
|
"ignored_objects": ignored,
|
|
}
|
|
return report
|
|
|
|
|
|
def _print_report(report: Dict) -> None:
|
|
ds = report["dataset"]
|
|
print(f"\nGold images: {ds['gold_images']} | predictions: {ds['pred_images']} "
|
|
f"| missing: {ds['missing_predictions']} | match IoU >= {ds['iou_threshold']}")
|
|
header = f"{'class':<14}{'P':>7}{'R':>7}{'F1':>7}{'maskIoU':>9}{'TP':>6}{'FP':>6}{'FN':>6}"
|
|
print("\n" + header)
|
|
print("-" * len(header))
|
|
|
|
def row(name, r):
|
|
print(f"{name:<14}{r['precision']:>7.3f}{r['recall']:>7.3f}{r['f1']:>7.3f}"
|
|
f"{r['mean_mask_iou']:>9.3f}{r['tp']:>6}{r['fp']:>6}{r['fn']:>6}")
|
|
|
|
for name, r in report["per_class"].items():
|
|
row(name, r)
|
|
print("-" * len(header))
|
|
row("OVERALL", report["overall"])
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description="Evaluate predicted YOLO-seg labels vs. a gold set")
|
|
ap.add_argument("--pred", required=True, help="Directory of predicted .txt labels")
|
|
ap.add_argument("--gold", required=True, help="Directory of gold .txt labels")
|
|
ap.add_argument("--iou", type=float, default=0.5, help="bbox IoU threshold to match instances")
|
|
ap.add_argument("--names", default=None, help="Optional dataset.yaml for class names")
|
|
ap.add_argument("--report", default=None, help="Optional path to write the JSON report")
|
|
args = ap.parse_args()
|
|
|
|
names = load_class_names(args.names) if args.names else []
|
|
report = evaluate_dirs(args.pred, args.gold, iou_thr=args.iou, names=names)
|
|
_print_report(report)
|
|
if args.report:
|
|
Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
print(f"\nWrote {args.report}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|