gate: baseline and candidate measured in the same environment

The cluster's first run was gated against a baseline measured on the Spark.
The same weights score lower there (F1 0.840 vs 0.861, mask score 0.706 vs
0.757 on the 203 gold frames), so the gate failed a model that beats the
active one on F1 in 98.7 % of bootstrap draws when both are measured on one
machine.

Gold reports now record the environment (ultralytics, torch, host). When the
active model's report comes from other versions, train_run.py measures its
weights again in the job and gates against that; the predictions feed the
paired bootstrap. Without the weights there is no verdict and no promotion.
The SLURM scripts pull the active model's weights.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

slurm/README.md

@@ -74,3 +74,15 @@ registry entry locally. Pushing needs a key on the cluster that Gitea accepts: u
 `git push origin main` from the login node by hand, or the commit stays there for someone
 to push. Latency at the beamline must be measured again before a 1024 px or a small-model
 engine is deployed: the live server scales frames to 640 px today.
+
+## One environment per verdict
+
+The gate compares two models only when both were measured with the same library versions.
+The first cluster run (`2026-09-30_v3-s-640px`, 2026-10-01) was gated against a baseline report
+written on the Spark: the same weights scored F1 0.840 and mask score 0.706 on the cluster and
+0.861 and 0.757 on the Spark, and the gate failed a model that passes it in one environment.
+Since then every gold report records `environment` (ultralytics, torch, host); when the active
+model's report comes from other versions, `train_run.py` measures the active model again inside
+the job, which is why the job pulls `models/registry/<active>/weights.pt`. If those weights are
+missing the run ends as `gate_failed` with an `environment_note` and nothing is promoted.
+Keeping the cluster's `ultralytics` at the Spark's version removes the difference at its source.

slurm/train.sbatch

@@ -29,7 +29,9 @@ git fetch -q --tags
 git pull -q --ff-only
 DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")"
 git checkout -q "$DATASET_TAG" -- data/datasets                # the dataset exactly as tagged
-git lfs pull -q -I "data/pool/images/**,data/gold/**,models/base/**"   # the labelled pool, the gold set, the base weights
+# the active model's weights too: the gate measures the baseline again in this job's own environment
+ACTIVE="$(python -c "import json;print(json.load(open('models/active/production.json'))['version'])" 2>/dev/null || true)"
+git lfs pull -q -I "data/pool/images/**,data/gold/**,models/base/**,models/registry/$ACTIVE/weights.pt"   # the labelled pool, the gold set, the base weights
 
 python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION"
 $APPTAINER python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log"

slurm/train.sbatch.merlin

@@ -31,7 +31,9 @@ git fetch -q --tags
 git pull -q --ff-only
 DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")"
 git checkout -q "$DATASET_TAG" -- data/datasets                # the dataset exactly as tagged
-git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**" >/dev/null
+# the active model's weights too: the gate measures the baseline again in this job's own environment
+ACTIVE="$(python -c "import json;print(json.load(open('models/active/production.json'))['version'])" 2>/dev/null || true)"
+git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**,models/registry/$ACTIVE/weights.pt" >/dev/null
 
 python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION"
 python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log"

tests/test_gate.py

@@ -77,3 +77,34 @@ def test_gate_without_predictions_keeps_the_point_rule(tmp_path):
     base.write_text(json.dumps(_report(0.69, 0.903, 60, 40)))
     g = train_run.gate(_report(0.85, 0.883, 85, 15), base)
     assert g["rule"] == "point" and g["pass"] is True and g["pass_point_rule"] is True
+
+
+def test_same_environment_trusts_legacy_reports_only_on_the_spark():
+    here = {"host": "gpu102", "ultralytics": "8.4.37", "torch": "2.5.1"}
+    assert train_run.same_environment({"ultralytics": "8.4.37", "torch": "2.5.1", "host": "mx-spark-01"}, here) is True
+    assert train_run.same_environment({"ultralytics": "8.4.160", "torch": "2.5.1"}, here) is False
+    assert train_run.same_environment(None, here) is False                                   # legacy report, read on the cluster
+    assert train_run.same_environment(None, {"host": "mx-spark-01", "ultralytics": "x", "torch": "y"}) is True
+
+
+def test_baseline_from_another_environment_is_measured_again_or_refused(tmp_path, monkeypatch):
+    reg = tmp_path / "models" / "registry" / "v-old"; reg.mkdir(parents=True)
+    stored = reg / "gold_report.json"
+    stored.write_text(json.dumps(dict(_report(0.84, 0.865, 80, 20), environment={"ultralytics": "8.4.160", "torch": "2.14.0"})))
+    monkeypatch.setattr(train_run, "ROOT", tmp_path)
+    monkeypatch.setattr(train_run, "environment", lambda: {"host": "gpu102", "ultralytics": "8.4.37", "torch": "2.5.1"})
+    # no weights here: no baseline to gate against, and a reason
+    path, note = train_run.baseline_here(stored, ["loop_all"], 0.25)
+    assert path is None and "weights are not here" in note
+    # an lfs pointer is not weights either
+    (reg / "weights.pt").write_text("version https://git-lfs.github.com/spec/v1")
+    assert train_run.baseline_here(stored, ["loop_all"], 0.25)[0] is None
+    # real weights: measured again here, the local report is the baseline
+    (reg / "weights.pt").write_bytes(b"0" * 1_100_000)
+    monkeypatch.setattr(train_run, "measure", lambda w, work, names, conf=0.25: _report(0.86, 0.886, 85, 15))
+    path, note = train_run.baseline_here(stored, ["loop_all"], 0.25)
+    assert path.name == "gold_report.local.json" and json.loads(path.read_text())["overall"]["f1"] == 0.86
+    assert "measured again" in note
+    # same environment: the stored report is used as it is
+    monkeypatch.setattr(train_run, "environment", lambda: {"host": "x", "ultralytics": "8.4.160", "torch": "2.14.0"})
+    assert train_run.baseline_here(stored, ["loop_all"], 0.25) == (stored, None)

tools/measure_weights.py

@@ -18,7 +18,7 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
 
-from tools.train_run import measure  # noqa: E402
+from tools.train_run import environment, measure  # noqa: E402
 
 DEFAULT_NAMES = ["loop_all", "pin", "crystal", "loop_face", "ice", "needle"]
 
@@ -36,6 +36,7 @@ def main(argv=None) -> int:
     work.mkdir(parents=True, exist_ok=True)
     report = measure(Path(args.weights), work, args.names.split(","), conf=args.conf)
     report["weights"] = str(Path(args.weights).resolve())
+    report["environment"] = environment()
     report["measured_at"] = datetime.now().isoformat(timespec="seconds")
     out = reg / "gold_report.json"
     if out.exists():

tools/train_run.py

@@ -154,11 +154,67 @@ def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict
     return out
 
 
+def environment() -> Dict[str, Any]:
+    """What a measurement depends on besides the weights: the library versions decide how masks are
+    decoded and filtered, so two machines can score the same weights differently."""
+    env: Dict[str, Any] = {"host": socket.gethostname()}
+    for mod in ("ultralytics", "torch"):
+        try:
+            env[mod] = __import__(mod).__version__
+        except Exception:
+            env[mod] = None
+    return env
+
+
+def same_environment(report_env: Optional[Dict[str, Any]], here: Dict[str, Any]) -> bool:
+    """Was a stored report measured with the libraries of this machine? Reports from before
+    2026-10-02 carry no environment: they were all measured on the Spark."""
+    if not report_env:
+        return str(here.get("host", "")).startswith("mx-spark")
+    return all(report_env.get(k) == here.get(k) for k in ("ultralytics", "torch"))
+
+
+def baseline_here(baseline: Optional[Path], names: list, conf: float) -> tuple:
+    """The baseline report to gate against, measured in THIS environment.
+
+    On 2026-10-02 a cluster run was gated against a baseline measured on the Spark: the same
+    weights scored F1 0.840 and mask score 0.706 on the cluster, 0.861 and 0.757 on the Spark,
+    and the gate failed a model that passes it in a single environment. So when the stored
+    baseline report comes from other library versions, the baseline weights are measured again
+    here (predictions kept under runs/_measure_<version>, where the paired bootstrap finds
+    them). Returns (report path or None, note or None); a baseline that cannot be measured here
+    gives (None, why) and the caller refuses to promote.
+    """
+    if not baseline or not Path(baseline).exists():
+        return baseline, None
+    base = json.loads(Path(baseline).read_text(encoding="utf-8"))
+    here = environment()
+    if same_environment(base.get("environment"), here):
+        return Path(baseline), None
+    version = Path(baseline).parent.name
+    weights = Path(baseline).parent / "weights.pt"
+    if not weights.exists() or weights.stat().st_size < 1_000_000:      # absent, or a git-lfs pointer
+        return None, (f"the baseline {version} was measured with {base.get('environment') or 'the Spark environment'}, this machine has "
+                      f"{here}; its weights are not here to measure again (git lfs pull -I 'models/registry/{version}/weights.pt')")
+    work = ROOT / "runs" / f"_measure_{version}"
+    work.mkdir(parents=True, exist_ok=True)
+    rep = measure(weights, work, names, conf=conf)
+    rep["environment"] = here
+    local = work / "gold_report.local.json"
+    local.write_text(json.dumps(rep, indent=2) + "\n", encoding="utf-8")
+    print(f"gate: baseline {version} measured again in this environment (ultralytics {here.get('ultralytics')}, torch {here.get('torch')}): "
+          f"F1 {rep['overall']['f1']:.4f} (stored report: {base['overall']['f1']:.4f})")
+    return local, f"baseline measured again here; stored report came from {base.get('environment') or 'the Spark'}"
+
+
 def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]:
     """The baseline model's predictions on the gold frames, when they were kept: the re-measure
     directory of its version, or the gold predictions of the run that produced it."""
     if not baseline:
         return None
+    if Path(baseline).name == "gold_report.local.json":               # measured again here: predictions sit beside it
+        d = Path(baseline).parent / "gold_predictions"
+        return d if d.is_dir() else None
     version = Path(baseline).parent.name
     d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions"
     if d.is_dir():
@@ -273,17 +329,28 @@ def main(argv=None) -> int:
     shutil.copy2(best, reg / "weights.pt")
 
     report = measure(reg / "weights.pt", run_dir, names, conf=args.conf)
+    report["environment"] = environment()
     baseline = Path(args.baseline) if args.baseline else None
     if baseline is None:
         av = active_version()
         if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists():
             baseline = ROOT / "models" / "registry" / av / "gold_report.json"
+    stored_baseline = baseline
+    baseline, env_note = baseline_here(baseline, names, args.conf)
     paired = None
     try:
         paired = paired_with_baseline(run_dir, baseline)
     except Exception as e:  # the point rule still stands
         print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule")
     g = gate(report, baseline, paired)
+    if g is not None:
+        g["baseline"] = str(stored_baseline)
+        if env_note:
+            g["environment_note"] = env_note
+    elif stored_baseline is not None and env_note:
+        # there is a baseline, but not one this machine can be compared with: no verdict, no promotion
+        g = {"baseline": str(stored_baseline), "pass": False, "rule": "none", "environment_note": env_note}
+        print(f"gate: no verdict: {env_note}")
     report["gate"] = g
     (reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
     metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),