GotthardGandClaude Fable 5.1
62f568b0f6
The cluster's first run was gated against a baseline measured on the Spark. The same weights score lower there (F1 0.840 vs 0.861, mask score 0.706 vs 0.757 on the 203 gold frames), so the gate failed a model that beats the active one on F1 in 98.7 % of bootstrap draws when both are measured on one machine. Gold reports now record the environment (ultralytics, torch, host). When the active model's report comes from other versions, train_run.py measures its weights again in the job and gates against that; the predictions feed the paired bootstrap. Without the weights there is no verdict and no promotion. The SLURM scripts pull the active model's weights. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure any weights on the gold set and store the report in the registry.
|
|
|
|
Used once to give the deployed model its ``gold_report.json``, the baseline the
|
|
gate compares every new run against:
|
|
|
|
python tools/measure_weights.py /path/to/best.pt --version yolo26n-seg-overlap-false-2026-04-12
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from tools.train_run import environment, measure # noqa: E402
|
|
|
|
DEFAULT_NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"]
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("weights")
|
|
ap.add_argument("--version", required=True, help="registry directory name to write gold_report.json into")
|
|
ap.add_argument("--conf", type=float, default=0.25)
|
|
ap.add_argument("--names", default=",".join(DEFAULT_NAMES))
|
|
args = ap.parse_args(argv)
|
|
reg = ROOT / "models" / "registry" / args.version
|
|
reg.mkdir(parents=True, exist_ok=True)
|
|
work = ROOT / "runs" / f"_measure_{args.version}"
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
report = measure(Path(args.weights), work, args.names.split(","), conf=args.conf)
|
|
report["weights"] = str(Path(args.weights).resolve())
|
|
report["environment"] = environment()
|
|
report["measured_at"] = datetime.now().isoformat(timespec="seconds")
|
|
out = reg / "gold_report.json"
|
|
if out.exists():
|
|
# a re-measure on a grown gold set must not erase the verdict the gate gave this version
|
|
try:
|
|
old = json.loads(out.read_text(encoding="utf-8"))
|
|
if old.get("gate") is not None:
|
|
report["gate"] = old["gate"]
|
|
report["gate_measured_on"] = old.get("gate_measured_on") or old.get("dataset", {}).get("gold_images")
|
|
except ValueError:
|
|
pass
|
|
out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
meta = reg / "metadata.json"
|
|
if not meta.exists():
|
|
meta.write_text(json.dumps({"model_id": args.version, "task": "segmentation",
|
|
"metrics": report["overall"], "note": "metadata written by measure_weights.py"},
|
|
indent=2) + "\n", encoding="utf-8")
|
|
o = report["overall"]
|
|
print(f"{args.version}: f1 {o['f1']:.3f} mask IoU {o['mean_mask_iou']:.3f} -> {out}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|