From 62f568b0f696bbceb4ae884c2e826f08d730a798 Mon Sep 17 00:00:00 2001 From: GotthardG <51994228+GotthardG@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:02:39 +0200 Subject: [PATCH] gate: baseline and candidate measured in the same environment The cluster's first run was gated against a baseline measured on the Spark. The same weights score lower there (F1 0.840 vs 0.861, mask score 0.706 vs 0.757 on the 203 gold frames), so the gate failed a model that beats the active one on F1 in 98.7 % of bootstrap draws when both are measured on one machine. Gold reports now record the environment (ultralytics, torch, host). When the active model's report comes from other versions, train_run.py measures its weights again in the job and gates against that; the predictions feed the paired bootstrap. Without the weights there is no verdict and no promotion. The SLURM scripts pull the active model's weights. Co-Authored-By: Claude Fable 5.1 --- slurm/README.md | 12 +++++++ slurm/train.sbatch | 4 ++- slurm/train.sbatch.merlin | 4 ++- tests/test_gate.py | 31 ++++++++++++++++++ tools/measure_weights.py | 3 +- tools/train_run.py | 67 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 3 deletions(-) diff --git a/slurm/README.md b/slurm/README.md index 91d3137..4b6d861 100644 --- a/slurm/README.md +++ b/slurm/README.md @@ -74,3 +74,15 @@ registry entry locally. Pushing needs a key on the cluster that Gitea accepts: u `git push origin main` from the login node by hand, or the commit stays there for someone to push. Latency at the beamline must be measured again before a 1024 px or a small-model engine is deployed: the live server scales frames to 640 px today. + +## One environment per verdict + +The gate compares two models only when both were measured with the same library versions. +The first cluster run (`2026-09-30_v3-s-640px`, 2026-10-01) was gated against a baseline report +written on the Spark: the same weights scored F1 0.840 and mask score 0.706 on the cluster and +0.861 and 0.757 on the Spark, and the gate failed a model that passes it in one environment. +Since then every gold report records `environment` (ultralytics, torch, host); when the active +model's report comes from other versions, `train_run.py` measures the active model again inside +the job, which is why the job pulls `models/registry//weights.pt`. If those weights are +missing the run ends as `gate_failed` with an `environment_note` and nothing is promoted. +Keeping the cluster's `ultralytics` at the Spark's version removes the difference at its source. diff --git a/slurm/train.sbatch b/slurm/train.sbatch index 830ef14..e26a0e0 100644 --- a/slurm/train.sbatch +++ b/slurm/train.sbatch @@ -29,7 +29,9 @@ git fetch -q --tags git pull -q --ff-only DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")" git checkout -q "$DATASET_TAG" -- data/datasets # the dataset exactly as tagged -git lfs pull -q -I "data/pool/images/**,data/gold/**,models/base/**" # the labelled pool, the gold set, the base weights +# the active model's weights too: the gate measures the baseline again in this job's own environment +ACTIVE="$(python -c "import json;print(json.load(open('models/active/production.json'))['version'])" 2>/dev/null || true)" +git lfs pull -q -I "data/pool/images/**,data/gold/**,models/base/**,models/registry/$ACTIVE/weights.pt" # the labelled pool, the gold set, the base weights python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION" $APPTAINER python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log" diff --git a/slurm/train.sbatch.merlin b/slurm/train.sbatch.merlin index 25d3697..7637f55 100644 --- a/slurm/train.sbatch.merlin +++ b/slurm/train.sbatch.merlin @@ -31,7 +31,9 @@ git fetch -q --tags git pull -q --ff-only DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")" git checkout -q "$DATASET_TAG" -- data/datasets # the dataset exactly as tagged -git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**" >/dev/null +# the active model's weights too: the gate measures the baseline again in this job's own environment +ACTIVE="$(python -c "import json;print(json.load(open('models/active/production.json'))['version'])" 2>/dev/null || true)" +git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**,models/registry/$ACTIVE/weights.pt" >/dev/null python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION" python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log" diff --git a/tests/test_gate.py b/tests/test_gate.py index 8e10543..8df3279 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -77,3 +77,34 @@ def test_gate_without_predictions_keeps_the_point_rule(tmp_path): base.write_text(json.dumps(_report(0.69, 0.903, 60, 40))) g = train_run.gate(_report(0.85, 0.883, 85, 15), base) assert g["rule"] == "point" and g["pass"] is True and g["pass_point_rule"] is True + + +def test_same_environment_trusts_legacy_reports_only_on_the_spark(): + here = {"host": "gpu102", "ultralytics": "8.4.37", "torch": "2.5.1"} + assert train_run.same_environment({"ultralytics": "8.4.37", "torch": "2.5.1", "host": "mx-spark-01"}, here) is True + assert train_run.same_environment({"ultralytics": "8.4.160", "torch": "2.5.1"}, here) is False + assert train_run.same_environment(None, here) is False # legacy report, read on the cluster + assert train_run.same_environment(None, {"host": "mx-spark-01", "ultralytics": "x", "torch": "y"}) is True + + +def test_baseline_from_another_environment_is_measured_again_or_refused(tmp_path, monkeypatch): + reg = tmp_path / "models" / "registry" / "v-old"; reg.mkdir(parents=True) + stored = reg / "gold_report.json" + stored.write_text(json.dumps(dict(_report(0.84, 0.865, 80, 20), environment={"ultralytics": "8.4.160", "torch": "2.14.0"}))) + monkeypatch.setattr(train_run, "ROOT", tmp_path) + monkeypatch.setattr(train_run, "environment", lambda: {"host": "gpu102", "ultralytics": "8.4.37", "torch": "2.5.1"}) + # no weights here: no baseline to gate against, and a reason + path, note = train_run.baseline_here(stored, ["loop_all"], 0.25) + assert path is None and "weights are not here" in note + # an lfs pointer is not weights either + (reg / "weights.pt").write_text("version https://git-lfs.github.com/spec/v1") + assert train_run.baseline_here(stored, ["loop_all"], 0.25)[0] is None + # real weights: measured again here, the local report is the baseline + (reg / "weights.pt").write_bytes(b"0" * 1_100_000) + monkeypatch.setattr(train_run, "measure", lambda w, work, names, conf=0.25: _report(0.86, 0.886, 85, 15)) + path, note = train_run.baseline_here(stored, ["loop_all"], 0.25) + assert path.name == "gold_report.local.json" and json.loads(path.read_text())["overall"]["f1"] == 0.86 + assert "measured again" in note + # same environment: the stored report is used as it is + monkeypatch.setattr(train_run, "environment", lambda: {"host": "x", "ultralytics": "8.4.160", "torch": "2.14.0"}) + assert train_run.baseline_here(stored, ["loop_all"], 0.25) == (stored, None) diff --git a/tools/measure_weights.py b/tools/measure_weights.py index d12f798..1ed6d7d 100644 --- a/tools/measure_weights.py +++ b/tools/measure_weights.py @@ -18,7 +18,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from tools.train_run import measure # noqa: E402 +from tools.train_run import environment, measure # noqa: E402 DEFAULT_NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"] @@ -36,6 +36,7 @@ def main(argv=None) -> int: work.mkdir(parents=True, exist_ok=True) report = measure(Path(args.weights), work, args.names.split(","), conf=args.conf) report["weights"] = str(Path(args.weights).resolve()) + report["environment"] = environment() report["measured_at"] = datetime.now().isoformat(timespec="seconds") out = reg / "gold_report.json" if out.exists(): diff --git a/tools/train_run.py b/tools/train_run.py index aaf9551..93d78ed 100644 --- a/tools/train_run.py +++ b/tools/train_run.py @@ -154,11 +154,67 @@ def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict return out +def environment() -> Dict[str, Any]: + """What a measurement depends on besides the weights: the library versions decide how masks are + decoded and filtered, so two machines can score the same weights differently.""" + env: Dict[str, Any] = {"host": socket.gethostname()} + for mod in ("ultralytics", "torch"): + try: + env[mod] = __import__(mod).__version__ + except Exception: + env[mod] = None + return env + + +def same_environment(report_env: Optional[Dict[str, Any]], here: Dict[str, Any]) -> bool: + """Was a stored report measured with the libraries of this machine? Reports from before + 2026-10-02 carry no environment: they were all measured on the Spark.""" + if not report_env: + return str(here.get("host", "")).startswith("mx-spark") + return all(report_env.get(k) == here.get(k) for k in ("ultralytics", "torch")) + + +def baseline_here(baseline: Optional[Path], names: list, conf: float) -> tuple: + """The baseline report to gate against, measured in THIS environment. + + On 2026-10-02 a cluster run was gated against a baseline measured on the Spark: the same + weights scored F1 0.840 and mask score 0.706 on the cluster, 0.861 and 0.757 on the Spark, + and the gate failed a model that passes it in a single environment. So when the stored + baseline report comes from other library versions, the baseline weights are measured again + here (predictions kept under runs/_measure_, where the paired bootstrap finds + them). Returns (report path or None, note or None); a baseline that cannot be measured here + gives (None, why) and the caller refuses to promote. + """ + if not baseline or not Path(baseline).exists(): + return baseline, None + base = json.loads(Path(baseline).read_text(encoding="utf-8")) + here = environment() + if same_environment(base.get("environment"), here): + return Path(baseline), None + version = Path(baseline).parent.name + weights = Path(baseline).parent / "weights.pt" + if not weights.exists() or weights.stat().st_size < 1_000_000: # absent, or a git-lfs pointer + return None, (f"the baseline {version} was measured with {base.get('environment') or 'the Spark environment'}, this machine has " + f"{here}; its weights are not here to measure again (git lfs pull -I 'models/registry/{version}/weights.pt')") + work = ROOT / "runs" / f"_measure_{version}" + work.mkdir(parents=True, exist_ok=True) + rep = measure(weights, work, names, conf=conf) + rep["environment"] = here + local = work / "gold_report.local.json" + local.write_text(json.dumps(rep, indent=2) + "\n", encoding="utf-8") + print(f"gate: baseline {version} measured again in this environment (ultralytics {here.get('ultralytics')}, torch {here.get('torch')}): " + f"F1 {rep['overall']['f1']:.4f} (stored report: {base['overall']['f1']:.4f})") + return local, f"baseline measured again here; stored report came from {base.get('environment') or 'the Spark'}" + + def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]: """The baseline model's predictions on the gold frames, when they were kept: the re-measure directory of its version, or the gold predictions of the run that produced it.""" if not baseline: return None + if Path(baseline).name == "gold_report.local.json": # measured again here: predictions sit beside it + d = Path(baseline).parent / "gold_predictions" + return d if d.is_dir() else None version = Path(baseline).parent.name d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions" if d.is_dir(): @@ -273,17 +329,28 @@ def main(argv=None) -> int: shutil.copy2(best, reg / "weights.pt") report = measure(reg / "weights.pt", run_dir, names, conf=args.conf) + report["environment"] = environment() baseline = Path(args.baseline) if args.baseline else None if baseline is None: av = active_version() if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists(): baseline = ROOT / "models" / "registry" / av / "gold_report.json" + stored_baseline = baseline + baseline, env_note = baseline_here(baseline, names, args.conf) paired = None try: paired = paired_with_baseline(run_dir, baseline) except Exception as e: # the point rule still stands print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule") g = gate(report, baseline, paired) + if g is not None: + g["baseline"] = str(stored_baseline) + if env_note: + g["environment_note"] = env_note + elif stored_baseline is not None and env_note: + # there is a baseline, but not one this machine can be compared with: no verdict, no promotion + g = {"baseline": str(stored_baseline), "pass": False, "rule": "none", "environment_note": env_note} + print(f"gate: no verdict: {env_note}") report["gate"] = g (reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),