train_run.py --no-gate registers weights the gold cannot score on their own
The loop-head crop model of the studio second pass is scored by the studio measure --crop-weights, both passes together; its own gold measure on whole frames would mean nothing. The run ends in the new ledger state registered. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JPZJw4gJgQmPzTa6ix7eGp
This commit is contained in:
1 parent
9c498d8249
commit
815aec62e9
2 files changed
+23
-1
No files matched your search
@@ -33,6 +33,7 @@ STATES = [
|
||||
"queued", # job submitted (Spark or cluster)
|
||||
"training", # epochs running, train.log grows
|
||||
"measured", # gold measure done, metrics.json written
|
||||
"registered", # weights in the registry without a gold gate (train_run.py --no-gate)
|
||||
"gate_passed", # measure beats the deployed model's baseline
|
||||
"gate_failed",
|
||||
"promoted", # models/active/production.json points at this run's weights
|
||||
|
||||
+22
-1
@@ -290,6 +290,9 @@ def main(argv=None) -> int:
|
||||
ap.add_argument("--no-promote", action="store_true", help="gate, but leave production.json alone")
|
||||
ap.add_argument("--skip-train", action="store_true", help="reuse runs/<id>/ultralytics/train/weights/best.pt")
|
||||
ap.add_argument("--force", action="store_true", help="train again a run that already has a gate verdict")
|
||||
ap.add_argument("--no-gate", action="store_true",
|
||||
help="register the weights and stop: no gold measure, no gate, for a model the gold cannot "
|
||||
"score on its own (the loop-head crop model, which the studio's measure --crop-weights scores)")
|
||||
ap.add_argument("--set", nargs="*", default=[], metavar="KEY=VALUE",
|
||||
help="override any Ultralytics train argument, e.g. fliplr=0 mask_ratio=1 overlap_mask=true; recorded in the ledger")
|
||||
args = ap.parse_args(argv)
|
||||
@@ -298,7 +301,7 @@ def main(argv=None) -> int:
|
||||
run_dir = ledger.run_path(args.run_id).parent
|
||||
# A gate verdict is an outcome, not a crash: the exit code 2 of a failed gate must not make a
|
||||
# wrapper's retry train the whole run a second time (cycle 4 lost two hours that way).
|
||||
if run.get("status") in ("gate_failed", "gate_passed", "promoted", "deployed") and not (args.skip_train or args.force):
|
||||
if run.get("status") in ("gate_failed", "gate_passed", "promoted", "deployed", "registered") and not (args.skip_train or args.force):
|
||||
print(f"{args.run_id} is already {run['status']}: not training it again (--force to retrain, --skip-train to re-measure)")
|
||||
return 0
|
||||
ds = dataset_dir(run)
|
||||
@@ -340,6 +343,24 @@ def main(argv=None) -> int:
|
||||
reg.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(best, reg / "weights.pt")
|
||||
|
||||
if args.no_gate:
|
||||
# a model the gold cannot score on its own: registered, not gated (the crop model of the
|
||||
# loop-head second pass is scored by the studio's measure --crop-weights, both passes together)
|
||||
metrics = {"overall": None, "gate": {"rule": "none", "note": "registered with --no-gate"},
|
||||
"training": results_summary(run_dir / "ultralytics" / "train"),
|
||||
"version": version, "measured_at": None}
|
||||
(run_dir / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8")
|
||||
(reg / "metadata.json").write_text(json.dumps({
|
||||
"model_id": version, "family": family, "task": "segmentation",
|
||||
"training_date": datetime.now().strftime("%Y-%m-%d"), "run": args.run_id,
|
||||
"dataset": run["dataset"], "base_weights": run["base_weights"], "hyperparams": hyper,
|
||||
"export_formats": ["pt"], "metrics": None, "gate": metrics["gate"],
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
ledger.record_metrics(args.run_id, run_dir / "metrics.json")
|
||||
ledger.set_status(args.run_id, "registered", kv={"version": version}, note="--no-gate: no gold measure")
|
||||
print(f"registered {version} without a gold gate")
|
||||
return 0
|
||||
|
||||
report = measure(reg / "weights.pt", run_dir, names, conf=args.conf)
|
||||
report["environment"] = environment()
|
||||
baseline = Path(args.baseline) if args.baseline else None
|
||||
|
||||
Reference in new issue
Block a user