noise-aware gate, frozen core of unseen frames, and a cycle script for the Spark
A paired bootstrap over the 149 unseen gold frames (2026-10-02) showed that a difference between two models carries about +-0.023 of noise in F1, +-0.018 in the mask score and +-0.10 in crystal F1, while the gate's tolerance was 0.005. - tools/gate_stats.py: per-frame counts and a paired bootstrap (pure Python). - train_run.py: when the baseline's gold predictions exist, the gate requires the candidate to beat it on F1 or the mask score in 90 % of the draws and not to lose the other in more than 90 %; the point rule is the fallback and is always reported (pass_point_rule). - tools/compare_unseen.py replaces the Spark's ~/fair_gate_all.py: all models on the frames none trained on, on the frozen core (data/gold/unseen_core.json, 149 frames) and on all unseen frames once the gold set grows, with the interval and win share against the active model. - measure_weights.py keeps a version's gate verdict when it re-measures it. - tools/spark_cycle.sh: gold growth, export, re-measure, train, gate, compare, push, with what cycles 3 and 4 taught built in. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
591f7b0b31
commit
831367d35d
7 files changed
+660
-8
No files matched your search
+60
-8
@@ -38,6 +38,12 @@ AUGMENT = dict(scale=0.7, translate=0.15, mosaic=0.5, close_mosaic=10, fliplr=0.
|
||||
shear=5.0, perspective=0.0, mask_ratio=1)
|
||||
GATE_TOL = 0.005
|
||||
CLASS_TOL = 0.02
|
||||
# Since 2026-10-02 the gate knows how noisy the gold set is: when the baseline's predictions are
|
||||
# at hand, a paired bootstrap over the gold frames decides. The candidate must beat the baseline
|
||||
# on F1 or on the mask score in WIN_CLEAR of the draws, and must not lose the other one in more
|
||||
# than 1 - WIN_FLOOR of them. The 0.005 point rule stays as the fallback and is always reported.
|
||||
WIN_CLEAR = 0.90
|
||||
WIN_FLOOR = 0.10
|
||||
GUARDED_CLASSES = ("loop_all", "pin")
|
||||
|
||||
|
||||
@@ -119,7 +125,7 @@ def mask_score(overall: Dict[str, Any]) -> float:
|
||||
return float(overall.get("mean_mask_iou", 0.0)) * (tp / (tp + fn) if tp + fn else 0.0)
|
||||
|
||||
|
||||
def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
|
||||
def gate(report: Dict[str, Any], baseline: Optional[Path], paired: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
|
||||
if not baseline or not Path(baseline).exists():
|
||||
return None
|
||||
base = json.loads(Path(baseline).read_text(encoding="utf-8"))
|
||||
@@ -132,12 +138,53 @@ def gate(report: Dict[str, Any], baseline: Optional[Path]) -> Optional[Dict[str,
|
||||
a, b = per_class.get(c) or {}, base_classes.get(c) or {}
|
||||
if "f1" in a and "f1" in b:
|
||||
class_deltas[c] = float(a["f1"]) - float(b["f1"])
|
||||
ok = (f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0)
|
||||
and all(d >= -CLASS_TOL for d in class_deltas.values()))
|
||||
return {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
|
||||
"mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
|
||||
"matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
|
||||
"class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass": bool(ok)}
|
||||
guarded = all(d >= -CLASS_TOL for d in class_deltas.values())
|
||||
ok_point = f1_delta >= -GATE_TOL and iou_delta >= -GATE_TOL and (f1_delta > 0 or iou_delta > 0) and guarded
|
||||
out = {"baseline": str(baseline), "f1_delta": round(f1_delta, 4), "mask_iou_delta": round(iou_delta, 4),
|
||||
"mask_score": round(mask_score(report["overall"]), 4), "mask_score_baseline": round(mask_score(base["overall"]), 4),
|
||||
"matched_only_iou_delta": round(float(report["overall"]["mean_mask_iou"]) - float(base["overall"]["mean_mask_iou"]), 4),
|
||||
"class_f1_delta": {k: round(v, 4) for k, v in class_deltas.items()}, "pass_point_rule": bool(ok_point)}
|
||||
if paired and paired.get("frames"):
|
||||
f, m = paired["f1"], paired["mask_score"]
|
||||
improved = f["win"] >= WIN_CLEAR or m["win"] >= WIN_CLEAR
|
||||
not_worse = f["win"] >= WIN_FLOOR and m["win"] >= WIN_FLOOR
|
||||
out.update({"rule": "paired bootstrap", "bootstrap": paired, "pass": bool(improved and not_worse and guarded)})
|
||||
else:
|
||||
out.update({"rule": "point", "pass": bool(ok_point)})
|
||||
return out
|
||||
|
||||
|
||||
def baseline_predictions(baseline: Optional[Path]) -> Optional[Path]:
|
||||
"""The baseline model's predictions on the gold frames, when they were kept: the re-measure
|
||||
directory of its version, or the gold predictions of the run that produced it."""
|
||||
if not baseline:
|
||||
return None
|
||||
version = Path(baseline).parent.name
|
||||
d = ROOT / "runs" / f"_measure_{version}" / "gold_predictions"
|
||||
if d.is_dir():
|
||||
return d
|
||||
for rj in sorted((ROOT / "runs").glob("*/run.json")):
|
||||
try:
|
||||
if json.loads(rj.read_text(encoding="utf-8")).get("version") == version and (rj.parent / "gold_predictions").is_dir():
|
||||
return rj.parent / "gold_predictions"
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Dict[str, Any]]:
|
||||
"""Paired bootstrap of this run against the baseline on the gold frames both were measured on."""
|
||||
base_pred = baseline_predictions(baseline)
|
||||
if base_pred is None:
|
||||
return None
|
||||
from tools import gate_stats
|
||||
gold = ROOT / "data" / "gold" / "labels"
|
||||
cand = gate_stats.per_frame(run_dir / "gold_predictions", gold)
|
||||
base = gate_stats.per_frame(base_pred, gold)
|
||||
missing = len(cand) - len(set(cand) & set(base))
|
||||
if missing:
|
||||
print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest")
|
||||
return gate_stats.paired_bootstrap(cand, base)
|
||||
|
||||
|
||||
def active_version() -> Optional[str]:
|
||||
@@ -231,7 +278,12 @@ def main(argv=None) -> int:
|
||||
av = active_version()
|
||||
if av and (ROOT / "models" / "registry" / av / "gold_report.json").exists():
|
||||
baseline = ROOT / "models" / "registry" / av / "gold_report.json"
|
||||
g = gate(report, baseline)
|
||||
paired = None
|
||||
try:
|
||||
paired = paired_with_baseline(run_dir, baseline)
|
||||
except Exception as e: # the point rule still stands
|
||||
print(f"gate: paired bootstrap unavailable ({e}); falling back to the point rule")
|
||||
g = gate(report, baseline, paired)
|
||||
report["gate"] = g
|
||||
(reg / "gold_report.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
metrics = {"overall": report["overall"], "gate": g, "training": results_summary(run_dir / "ultralytics" / "train"),
|
||||
|
||||
Reference in new issue
Block a user