GotthardGandClaude Fable 5.1
831367d35d
A paired bootstrap over the 149 unseen gold frames (2026-10-02) showed that a difference between two models carries about +-0.023 of noise in F1, +-0.018 in the mask score and +-0.10 in crystal F1, while the gate's tolerance was 0.005. - tools/gate_stats.py: per-frame counts and a paired bootstrap (pure Python). - train_run.py: when the baseline's gold predictions exist, the gate requires the candidate to beat it on F1 or the mask score in 90 % of the draws and not to lose the other in more than 90 %; the point rule is the fallback and is always reported (pass_point_rule). - tools/compare_unseen.py replaces the Spark's ~/fair_gate_all.py: all models on the frames none trained on, on the frozen core (data/gold/unseen_core.json, 149 frames) and on all unseen frames once the gold set grows, with the interval and win share against the active model. - measure_weights.py keeps a version's gate verdict when it re-measures it. - tools/spark_cycle.sh: gold growth, export, re-measure, train, gate, compare, push, with what cycles 3 and 4 taught built in. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure any weights on the gold set and store the report in the registry.
|
|
|
|
Used once to give the deployed model its ``gold_report.json``, the baseline the
|
|
gate compares every new run against:
|
|
|
|
python tools/measure_weights.py /path/to/best.pt --version yolo26n-seg-overlap-false-2026-04-12
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from tools.train_run import measure # noqa: E402
|
|
|
|
DEFAULT_NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"]
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("weights")
|
|
ap.add_argument("--version", required=True, help="registry directory name to write gold_report.json into")
|
|
ap.add_argument("--conf", type=float, default=0.25)
|
|
ap.add_argument("--names", default=",".join(DEFAULT_NAMES))
|
|
args = ap.parse_args(argv)
|
|
reg = ROOT / "models" / "registry" / args.version
|
|
reg.mkdir(parents=True, exist_ok=True)
|
|
work = ROOT / "runs" / f"_measure_{args.version}"
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
report = measure(Path(args.weights), work, args.names.split(","), conf=args.conf)
|
|
report["weights"] = str(Path(args.weights).resolve())
|
|
report["measured_at"] = datetime.now().isoformat(timespec="seconds")
|
|
out = reg / "gold_report.json"
|
|
if out.exists():
|
|
# a re-measure on a grown gold set must not erase the verdict the gate gave this version
|
|
try:
|
|
old = json.loads(out.read_text(encoding="utf-8"))
|
|
if old.get("gate") is not None:
|
|
report["gate"] = old["gate"]
|
|
report["gate_measured_on"] = old.get("gate_measured_on") or old.get("dataset", {}).get("gold_images")
|
|
except ValueError:
|
|
pass
|
|
out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
meta = reg / "metadata.json"
|
|
if not meta.exists():
|
|
meta.write_text(json.dumps({"model_id": args.version, "task": "segmentation",
|
|
"metrics": report["overall"], "note": "metadata written by measure_weights.py"},
|
|
indent=2) + "\n", encoding="utf-8")
|
|
o = report["overall"]
|
|
print(f"{args.version}: f1 {o['f1']:.3f} mask IoU {o['mean_mask_iou']:.3f} -> {out}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|