GotthardGandClaude Fable 5.1
3794330fc3
CI / ledger-and-tools (push) Successful in 18s
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
34 lines
1.4 KiB
Python
34 lines
1.4 KiB
Python
"""tools/train_run.py gate: recall-aware mask score and the pass rule."""
|
|
import json
|
|
|
|
from tools import train_run
|
|
|
|
|
|
def _report(f1, iou, tp, fn, classes=None):
|
|
return {"overall": {"f1": f1, "mean_mask_iou": iou, "tp": tp, "fn": fn, "fp": 0, "precision": 1.0, "recall": tp / (tp + fn)},
|
|
"per_class": classes or {}}
|
|
|
|
|
|
def test_mask_score_weights_by_recall():
|
|
assert abs(train_run.mask_score({"mean_mask_iou": 0.9, "tp": 60, "fn": 40}) - 0.54) < 1e-9
|
|
assert train_run.mask_score({"mean_mask_iou": 0.9, "tp": 0, "fn": 0}) == 0.0
|
|
|
|
|
|
def test_gate_prefers_finding_more_with_slightly_looser_masks(tmp_path):
|
|
base = tmp_path / "base.json"
|
|
base.write_text(json.dumps(_report(0.69, 0.903, 60, 40, {"loop_all": {"f1": 0.71}, "pin": {"f1": 0.84}})))
|
|
new = _report(0.85, 0.883, 85, 15, {"loop_all": {"f1": 0.90}, "pin": {"f1": 0.92}})
|
|
g = train_run.gate(new, base)
|
|
assert g["pass"] is True and g["mask_iou_delta"] > 0 and g["matched_only_iou_delta"] < 0
|
|
|
|
|
|
def test_gate_refuses_a_guarded_class_regression(tmp_path):
|
|
base = tmp_path / "base.json"
|
|
base.write_text(json.dumps(_report(0.70, 0.90, 60, 40, {"loop_all": {"f1": 0.80}, "pin": {"f1": 0.84}})))
|
|
new = _report(0.80, 0.90, 80, 20, {"loop_all": {"f1": 0.75}, "pin": {"f1": 0.90}})
|
|
assert train_run.gate(new, base)["pass"] is False
|
|
|
|
|
|
def test_no_baseline_means_no_gate():
|
|
assert train_run.gate(_report(0.8, 0.9, 1, 0), None) is None
|