noise-aware gate, frozen core of unseen frames, and a cycle script for the Spark

A paired bootstrap over the 149 unseen gold frames (2026-10-02) showed that a
difference between two models carries about +-0.023 of noise in F1, +-0.018 in
the mask score and +-0.10 in crystal F1, while the gate's tolerance was 0.005.

- tools/gate_stats.py: per-frame counts and a paired bootstrap (pure Python).
- train_run.py: when the baseline's gold predictions exist, the gate requires
  the candidate to beat it on F1 or the mask score in 90 % of the draws and
  not to lose the other in more than 90 %; the point rule is the fallback and
  is always reported (pass_point_rule).
- tools/compare_unseen.py replaces the Spark's ~/fair_gate_all.py: all models
  on the frames none trained on, on the frozen core (data/gold/unseen_core.json,
  149 frames) and on all unseen frames once the gold set grows, with the
  interval and win share against the active model.
- measure_weights.py keeps a version's gate verdict when it re-measures it.
- tools/spark_cycle.sh: gold growth, export, re-measure, train, gate, compare,
  push, with what cycles 3 and 4 taught built in.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
GotthardGandClaude Fable 5.1 authored and jungfrau committed 2026-10-02 13:25:45 +02:00
1 parent 591f7b0b31
commit 831367d35d
7 files changed
+660 -8

No files matched your search

+60 -8
View File
@@ -38,6 +38,12 @@ AUGMENT = dict(scale=0.7, translate=0.15, mosaic=0.5, close_mosaic=10, fliplr=0.
shear=5.0, perspective=0.0, mask_ratio=1)
GATE_TOL = 0.005
CLASS_TOL = 0.02
# Since 2026-10-02 the gate knows how noisy the gold set is: when the baseline's predictions are
# at hand, a paired bootstrap over the gold frames decides. The candidate must beat the baseline
# on F1 or on the mask score in WIN_CLEAR of the draws, and must not lose the other one in more
# than 1 - WIN_FLOOR of them. The 0.005 point rule stays as the fallback and is always reported.
WIN_CLEAR = 0.90
WIN_FLOOR = 0.10
GUARDED_CLASSES = ("loop_all", "pin")
@@ -119,7 +125,7 @@ def mask_score(overall: Dict[str, Any]) -> float:
return float(overall.get("mean_mask_iou", 0.0)) * (tp / (tp + fn) if tp + fn else 0.0)
def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
if not baseline or not Path(baseline).exists():
return None
base = json.loads(Path(baseline).read_text(encoding="utf-8"))
@@ -132,12 +138,53 @@ def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str,
a, b = per_class.get(c) or {}, base_classes.get(c) or {}
if "f1" in a and "f1" in b:
class_deltas[c] = float(a["f1"]) - float(b["f1"])
ok = (f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0)
and all(d >= -CLASS_TOL for d in class_deltas.values()))
return {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
"mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
"matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
"class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass": bool(ok)}
guarded = all(d >= -CLASS_TOL for d in class_deltas.values())
ok_point = f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0) and guarded
out = {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
"mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
"matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
"class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass_point_rule": bool(ok_point)}
if paired and paired.get("frames"):
f, m = paired["f1"], paired["mask_score"]
improved = f["win"] >= WIN_CLEAR or m["win"] >= WIN_CLEAR
not_worse = f["win"] >= WIN_FLOOR and m["win"] >= WIN_FLOOR
out.update({"rule": "paired bootstrap", "bootstrap": paired, "pass": bool(improved and not_worse and guarded)})
else:
out.update({"rule": "point", "pass": bool(ok_point)})
return out
def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]:
"""The baseline model's predictions on the gold frames, when they were kept: the re-measure
directory of its version, or the gold predictions of the run that produced it."""
if not baseline:
return None
version = Path(baseline).parent.name
d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions"
if d.is_dir():
return d
for rj in sorted((ROOT / "runs").glob("*/run.json")):
try:
if json.loads(rj.read_text(encoding="utf-8")).get("version") == version and (rj.parent / "gold_predictions").is_dir():
return rj.parent / "gold_predictions"
except (OSError, ValueError):
continue
return None
def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
"""Paired bootstrap of this run against the baseline on the gold frames both were measured on."""
base_pred = baseline_predictions(baseline)
if base_pred is None:
return None
from tools import gate_stats
gold = ROOT / "data" / "gold" / "labels"
cand = gate_stats.per_frame(run_dir / "gold_predictions", gold)
base = gate_stats.per_frame(base_pred, gold)
missing = len(cand) - len(set(cand) & set(base))
if missing:
print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest")
return gate_stats.paired_bootstrap(cand, base)
def active_version() -> Optional[str]:
@@ -231,7 +278,12 @@ def main(argv=None) -> int:
av = active_version()
if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists():
baseline = ROOT / "models" / "registry" / av / "gold_report.json"
g = gate(report, baseline)
paired = None
try:
paired = paired_with_baseline(run_dir, baseline)
except Exception as e: # the point rule still stands
print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule")
g = gate(report, baseline, paired)
report["gate"] = g
(reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),