jungfrauandClaude Fable 5.1
63ee07fe61
CI / ledger-and-tools (push) Successful in 19s
454 of the 1 029 gold outlines are a proposal kept unchanged (loop 159 of 317, pin 140 of
224, face 140 of 295, crystal 14 of 149): the pre-labelling model's own masks. The
evaluation now ignores them object by object for every model, like a crowd region: they
leave the gold, and an unmatched prediction on one of them is not a false alarm.
tools/gold_kept.py finds them against the studio's batch proposals (mask IoU >= 0.98) and
writes data/gold/kept_machine.json; evaluate_image, evaluate_dirs and gate_stats.per_frame
read it. With it, the frame-level exclusion of 845bd87 is no longer applied (it stays as
the fallback without the file), so the gold set can grow with frames the active model
pre-labelled. Every registry model is measured again on this basis and the three v5 runs
are re-gated, without promotion.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013uu3Bxsu5JqFhaWa9Ftu6z
102 lines
4.0 KiB
Python
102 lines
4.0 KiB
Python
"""Which gold outlines are a model's proposal that the reviewer kept unchanged.
|
|
|
|
Written to ``data/gold/kept_machine.json`` as ``{"frames": {stem: [gold object indices]}}``: a
|
|
gold object is kept-machine when a proposed object of the same class in the studio's batch
|
|
annotation overlaps it with mask IoU >= 0.98. Such an outline is the pre-labelling model's own
|
|
mask; the evaluation ignores it for every model (src/eval/metrics.evaluate_image), so a model
|
|
is never scored against itself and the gold set can grow with any reviewed frame.
|
|
|
|
python tools/gold_kept.py [--studio ~/AareLC_studio/data/pipeline_prod] [--iou 0.98]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import re
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import cv2
|
|
import numpy as np
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"]
|
|
GRID = 512
|
|
|
|
|
|
def gold_objects(path: Path):
|
|
out = []
|
|
for line in path.read_text(encoding="utf-8").split("\n"):
|
|
p = line.split()
|
|
if len(p) < 7:
|
|
continue
|
|
out.append((int(p[0]), np.array([float(v) for v in p[1:]], dtype=np.float64).reshape(-1, 2)))
|
|
return out
|
|
|
|
|
|
def raster(pts_norm) -> np.ndarray:
|
|
m = np.zeros((GRID, GRID), np.uint8)
|
|
cv2.fillPoly(m, [np.round(pts_norm * GRID).astype(np.int32).reshape(-1, 1, 2)], 1)
|
|
return m.astype(bool)
|
|
|
|
|
|
def iou(a, b) -> float:
|
|
u = int((a | b).sum())
|
|
return int((a & b).sum()) / u if u else 0.0
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--root", default=str(HERE.parent))
|
|
ap.add_argument("--studio", default=str(Path.home() / "AareLC_studio" / "data" / "pipeline_prod"))
|
|
ap.add_argument("--iou", type=float, default=0.98, help="mask IoU from which a gold outline counts as the proposal kept")
|
|
args = ap.parse_args()
|
|
root, studio = Path(args.root).expanduser(), Path(args.studio).expanduser()
|
|
labels = root / "data" / "gold" / "labels"
|
|
proposals = {}
|
|
for f in glob.glob(str(studio / "20*/annotations/tpi_*.json")):
|
|
proposals.setdefault(Path(f).stem, f)
|
|
frames, kept, total = {}, Counter(), Counter()
|
|
with_proposal = 0
|
|
for g in sorted(labels.glob("*.txt")):
|
|
m = re.search(r"tpi_(\d+)", g.stem)
|
|
objs = gold_objects(g)
|
|
for c, _ in objs:
|
|
total[NAMES[c]] += 1
|
|
f = proposals.get("tpi_" + m.group(1)) if m else None
|
|
if not f:
|
|
continue
|
|
with_proposal += 1
|
|
a = json.loads(Path(f).read_text(encoding="utf-8"))
|
|
W, H = float(a.get("width") or 2040), float(a.get("height") or 2040)
|
|
props = []
|
|
for o in a.get("objects") or []:
|
|
poly = (o.get("polygons") or [None])[0]
|
|
if poly and len(poly) >= 3:
|
|
props.append((o.get("class_id"), raster(np.array(poly, dtype=np.float64) / [W, H])))
|
|
idx = []
|
|
for i, (c, pts) in enumerate(objs):
|
|
rm = raster(pts)
|
|
if any(pc == c and iou(rm, pm) >= args.iou for pc, pm in props):
|
|
kept[NAMES[c]] += 1
|
|
idx.append(i)
|
|
if idx:
|
|
frames[g.stem] = idx
|
|
out = {"built": datetime.now().isoformat(timespec="seconds"), "source": str(studio), "iou": args.iou,
|
|
"gold_frames": sum(1 for _ in labels.glob("*.txt")), "with_a_proposal": with_proposal,
|
|
"kept_by_class": dict(kept), "gold_by_class": dict(total), "frames": frames}
|
|
dest = root / "data" / "gold" / "kept_machine.json"
|
|
dest.write_text(json.dumps(out, indent=1, sort_keys=True) + "\n", encoding="utf-8")
|
|
n = sum(len(v) for v in frames.values())
|
|
print(f"{n} of {sum(total.values())} gold outlines are a proposal kept unchanged, on {len(frames)} frames -> {dest}")
|
|
for c in NAMES:
|
|
if total[c]:
|
|
print(f" {c:10} {kept[c]:4} of {total[c]:4}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|