gate: bootstrap by mount, light states and excluded frames of the gold set

Frames of one mount are the same loop seen again, so the paired bootstrap now draws
mounts (data/gold/groups.json, built by tools/gold_groups.py from the mount ids the
studio writes in the gold manifest). The comparison also reports the unseen core without
the saturated frames, and frames the studio dropped from the gold set are pruned.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tests/test_gate.py

@@ -108,3 +108,44 @@ def test_baseline_from_another_environment_is_measured_again_or_refused(tmp_path
     # same environment: the stored report is used as it is
     monkeypatch.setattr(train_run, "environment", lambda: {"host": "x", "ultralytics": "8.4.160", "torch": "2.14.0"})
     assert train_run.baseline_here(stored, ["loop_all"], 0.25) == (stored, None)
+
+
+def test_bootstrap_by_mount_widens_the_interval_of_repeated_frames():
+    """Ten frames of one mount all won by the candidate: frame-wise draws call it a sure win,
+    mount-wise draws see one lucky mount among eleven."""
+    from tools import gate_stats
+    base = _frames([1] * 20)
+    cand = _frames([2] * 10 + [1] * 10)
+    groups = {f"f{i}": "mount-a" for i in range(10)}
+    by_frame = gate_stats.paired_bootstrap(cand, base, draws=800)
+    by_mount = gate_stats.paired_bootstrap(cand, base, draws=800, groups=groups)
+    assert by_frame["mounts"] == 20 and by_mount["mounts"] == 11
+    assert by_frame["f1"]["delta"] == by_mount["f1"]["delta"]          # the point estimate does not move
+    assert by_frame["f1"]["win"] > 0.99 and by_mount["f1"]["win"] < 0.75
+    assert by_mount["f1"]["lo"] <= 0 < by_frame["f1"]["lo"]
+
+
+def test_gold_groups_lists_shared_mounts_and_dropped_frames(tmp_path):
+    from tools import gate_stats, gold_groups
+    gold = tmp_path / "data" / "gold"
+    (gold / "labels").mkdir(parents=True); (gold / "images").mkdir()
+    manifest = {"images": {"tpi_1": {"tpid": 1, "mount": "PXII:s6416"}, "tpi_2": {"tpid": 2, "mount": "PXII:s6416"},
+                           "tpi_3": {"tpid": 3, "mount": "PXIII:s7"}, "tpi_4": {"tpid": 4}},
+                "dropped": {"tpi_9": "same image as tpi_1"}}
+    (gold / "gold_manifest.json").write_text(json.dumps(manifest))
+    for stem in ("tpi_1", "tpi_9"):
+        (gold / "labels" / f"{stem}.txt").write_text("")
+    assert gold_groups.prune(gold) == 1 and not (gold / "labels" / "tpi_9.txt").exists()
+    g = gold_groups.build(gold, with_light=False)
+    assert g["mount"] == {"tpi_1": "PXII:s6416", "tpi_2": "PXII:s6416"}      # single-frame mounts are not listed
+    assert g["excluded"] == {"tpi_9": "same image as tpi_1"}
+    (gold / "groups.json").write_text(json.dumps(g))
+    assert gate_stats.load_groups(gold)["mount"]["tpi_2"] == "PXII:s6416"
+    assert gate_stats.load_groups(tmp_path)["mount"] == {}                   # no file: every frame its own mount
+
+
+def test_light_state_thresholds():
+    from tools import gold_groups
+    assert gold_groups.light_state(0.6, 200) == "saturated"
+    assert gold_groups.light_state(0.0, 12) == "dark"
+    assert gold_groups.light_state(0.1, 120) == "ok"

tools/compare_unseen.py

@@ -89,7 +89,8 @@ def main(argv: Optional[List[str]] = None) -> int:
                 key = line.split(",")[0].strip()
                 if key.startswith("tpi_"):
                     trained_on.add(key)
-    stems = sorted(p.stem for p in gold.glob("*.txt"))
+    groups = gate_stats.load_groups(gold.parent)
+    stems = sorted(p.stem for p in gold.glob("*.txt") if p.stem not in groups["excluded"])
     key_of = lambda s: "tpi_" + re.search(r"tpi_(\d+)", s).group(1)
     unseen = [s for s in stems if s not in seen and key_of(s) not in trained_on]
     core_now = [s for s in unseen if s in core]
@@ -114,6 +115,10 @@ def main(argv: Optional[List[str]] = None) -> int:
     order = sorted(frames, key=lambda k: (not k.startswith("beamline"), k))
     reps = {k: report(frames[k]) for k in order}
     core_reps = {k: report({s: f for s, f in frames[k].items() if s in core}) for k in order}
+    # frames whose light makes them unusable (saturated) are part of the measure; this is the same
+    # table without them, to tell a model that got worse from a set that holds more bad light
+    bad_light = {s for s, state in groups["light"].items() if state == "saturated"}
+    usable_reps = {k: report({s: f for s, f in frames[k].items() if s in core and s not in bad_light}) for k in order}
 
     print(f"clean gold frames: {len(unseen)}" + (f" (core {len(core_now)})" if len(unseen) != len(core_now) else ""))
     table("", reps)
@@ -126,12 +131,12 @@ def main(argv: Optional[List[str]] = None) -> int:
     boot = {}
     base_label = label_of.get(active) if active else None
     if base_label and base_label in frames:
-        print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have: difference [95 % interval], share of draws won")
+        print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have, drawn by mount: difference [95 % interval], share of draws won")
         classes = {"crystal": NAMES.index("crystal")}
         for k in order:
             if k == base_label:
                 continue
-            b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes)
+            b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes, groups=groups["mount"])
             if not b.get("frames"):
                 continue
             boot[k] = b
@@ -145,12 +150,15 @@ def main(argv: Optional[List[str]] = None) -> int:
     def row(f):
         o = gate_stats._total([f], None); c = gate_stats._total([f], crystal)
         return [int(o[0]), int(o[1]), int(o[2]), round(o[3], 3), int(c[0]), int(c[1]), int(c[2]), round(c[3], 3)]
-    per_frame = {"stems": unseen, "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"],
+    per_frame = {"stems": unseen, "mount": [groups["mount"].get(st) or st for st in unseen],
+                 "light": [groups["light"].get(st, "ok") for st in unseen], "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"],
                  "models": {k: [row(frames[k][st]) if st in frames[k] else None for st in unseen] for k in order}}
     (root / "runs" / "_unseen_per_frame.json").write_text(json.dumps(per_frame, separators=(",", ":")), encoding="utf-8")
 
     out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"),
-           "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot}
+           "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot,
+           "core_reports_usable_light": usable_reps, "saturated_frames": sorted(s for s in bad_light if s in core),
+           "mounts": len({groups["mount"].get(s) or s for s in unseen}), "excluded": groups["excluded"]}
     (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8")
     return 0
 

tools/gate_stats.py

@@ -6,11 +6,16 @@ gold. A paired bootstrap answers that: draw the gold frames again with replaceme
 draw for both models, and look at the spread of the difference. ``win`` is the share of draws
 in which the candidate beats the baseline.
 
+Frames of one mount are not independent (the same loop at another angle, 24 of the 203 gold
+frames on 2026-10-02), so the draw is by mount when ``data/gold/groups.json`` names them: a
+mount is drawn with all its frames. Frame-wise draws would call the interval narrower than it is.
+
 Pure Python on purpose (the gate's tests run without numpy); only ``per_frame`` needs the
 evaluation code, and imports it when called.
 """
 from __future__ import annotations
 
+import json
 import random
 from pathlib import Path
 from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
@@ -19,6 +24,15 @@ Counts = Tuple[float, float, float, float]          # tp, fp, fn, sum of matched
 FrameCounts = Dict[int, Counts]                      # class id -> counts, for one frame
 
 
+def load_groups(gold_root) -> Dict[str, Dict[str, str]]:
+    """``data/gold/groups.json`` (tools/gold_groups.py): ``mount`` = frame -> mount id (a frame not
+    listed is its own mount), ``light`` = frame -> state for the frames whose light is not usable,
+    ``excluded`` = frame -> why it is left out of every measurement. Empty when the file is missing."""
+    p = Path(gold_root) / "groups.json"
+    g = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {}
+    return {"mount": dict(g.get("mount") or {}), "light": dict(g.get("light") or {}), "excluded": dict(g.get("excluded") or {})}
+
+
 def per_frame(pred_dir, gold_dir, iou_thr: float = 0.5, stems: Optional[Iterable[str]] = None) -> Dict[str, FrameCounts]:
     """Per gold frame and class: tp, fp, fn and the summed mask IoU of the matches.
 
@@ -72,14 +86,21 @@ def _summary(point: float, deltas: List[float]) -> Dict[str, float]:
 
 
 def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCounts], *, draws: int = 2000,
-                     seed: int = 0, classes: Optional[Mapping[str, int]] = None) -> Dict:
+                     seed: int = 0, classes: Optional[Mapping[str, int]] = None,
+                     groups: Optional[Mapping[str, str]] = None) -> Dict:
     """Candidate minus baseline on the frames both have: point difference, 95 % interval and win share
-    for F1 and the mask score, overall and for each class in ``classes`` (name -> class id)."""
+    for F1 and the mask score, overall and for each class in ``classes`` (name -> class id).
+
+    ``groups`` (frame -> mount id) makes the draw by mount: the frames of a mount come and go together."""
     keys = sorted(set(cand) & set(base))
     n = len(keys)
     if n == 0:
         return {"frames": 0, "draws": 0}
     a = [cand[k] for k in keys]; b = [base[k] for k in keys]
+    members: Dict[str, List[int]] = {}
+    for i, k in enumerate(keys):
+        members.setdefault((groups or {}).get(k) or f"frame:{k}", []).append(i)
+    units = [members[g] for g in sorted(members)]
     targets: List[Tuple[str, Optional[int]]] = [("overall", None)] + [(name, cid) for name, cid in (classes or {}).items()]
     point = {}
     for name, cid in targets:
@@ -88,12 +109,12 @@ def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCo
     rng = random.Random(seed)
     d_f1 = {name: [] for name, _ in targets}; d_ms = {name: [] for name, _ in targets}
     for _ in range(draws):
-        idx = [rng.randrange(n) for _ in range(n)]
+        idx = [i for _ in range(len(units)) for i in units[rng.randrange(len(units))]]
         sa = [a[i] for i in idx]; sb = [b[i] for i in idx]
         for name, cid in targets:
             ca, cb = _total(sa, cid), _total(sb, cid)
             d_f1[name].append(f1_of(ca) - f1_of(cb)); d_ms[name].append(mask_score_of(ca) - mask_score_of(cb))
-    out = {"frames": n, "draws": draws,
+    out = {"frames": n, "mounts": len(units), "draws": draws,
            "f1": _summary(point["overall"][0], d_f1["overall"]),
            "mask_score": _summary(point["overall"][1], d_ms["overall"])}
     if classes:

tools/gold_groups.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Which gold frames belong together, and which have unusable light -> ``data/gold/groups.json``.
+
+    python tools/gold_groups.py                      # rebuild groups.json from data/gold
+    python tools/gold_groups.py --sync-from <studio gold dir>   # take the studio's gold set first
+
+* ``mount``: the studio writes a ``mount`` id on every gold frame of ``gold_manifest.json`` (same
+  sample, or consecutive uploads of one centring). Frames of a mount are the same loop seen again,
+  so the gate's bootstrap draws mounts, not frames (tools/gate_stats.py). Only mounts with more
+  than one gold frame are listed.
+* ``light``: ``saturated`` when more than half of the frame is blown out, ``dark`` when its mean
+  grey level is under 30. Saturated frames stay in the measure (the beamline produces them); the
+  comparison also reports the result without them.
+* ``excluded``: the manifest's ``dropped`` frames (an exact duplicate of another gold frame, say).
+  They are removed from data/gold and never measured.
+
+On 2026-10-02: 203 frames, 7 mounts with 24 frames, 6 saturated, 29 dark, 1 dropped
+(tpi_14688, the same image as tpi_14689 with a different label).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Optional
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path.insert(0, str(ROOT))
+IMG_EXT = (".jpg", ".jpeg", ".png")
+
+SATURATED_SHARE = 0.5      # share of pixels at or above 250
+DARK_MEAN = 30.0           # mean grey level
+
+
+def light_state(saturated_share: float, mean: float) -> str:
+    if saturated_share > SATURATED_SHARE:
+        return "saturated"
+    if mean < DARK_MEAN:
+        return "dark"
+    return "ok"
+
+
+def image_light(path: Path) -> str:
+    import cv2
+    g = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
+    if g is None:
+        return "ok"
+    g = cv2.resize(g, (510, 510), interpolation=cv2.INTER_AREA)
+    return light_state(float((g >= 250).mean()), float(g.mean()))
+
+
+def build(gold: Path, *, with_light: bool = True) -> Dict:
+    manifest = json.loads((gold / "gold_manifest.json").read_text(encoding="utf-8"))
+    images = manifest.get("images") or {}
+    by_mount: Dict[str, list] = {}
+    for stem, info in images.items():
+        if info.get("mount"):
+            by_mount.setdefault(str(info["mount"]), []).append(stem)
+    mount = {stem: m for m, stems in sorted(by_mount.items()) if len(stems) > 1 for stem in sorted(stems)}
+    light: Dict[str, str] = {}
+    if with_light:
+        for stem in sorted(images):
+            img = next((gold / "images" / f"{stem}{e}" for e in IMG_EXT if (gold / "images" / f"{stem}{e}").exists()), None)
+            if img is not None:
+                state = image_light(img)
+                if state != "ok":
+                    light[stem] = state
+    return {
+        "note": "mount: gold frames of one mount (drawn together by the gate's bootstrap); light: frames whose light is not "
+                "usable; excluded: frames removed from the gold set. Built by tools/gold_groups.py",
+        "updated": datetime.now().isoformat(timespec="seconds"),
+        "frames": len(images), "mounts_with_several_frames": len({m for m in mount.values()}),
+        "mount": mount, "light": light, "excluded": dict(manifest.get("dropped") or {}),
+    }
+
+
+def prune(gold: Path) -> int:
+    """Remove the label and image of every frame the manifest no longer lists."""
+    keep = set(json.loads((gold / "gold_manifest.json").read_text(encoding="utf-8")).get("images") or {})
+    n = 0
+    for sub in ("labels", "images"):
+        for p in sorted((gold / sub).glob("*")) if (gold / sub).exists() else []:
+            if p.stem.startswith("tpi_") and p.stem not in keep:
+                p.unlink()
+                n += 1
+    return n
+
+
+def main(argv: Optional[list] = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--root", default=str(ROOT))
+    ap.add_argument("--sync-from", default=None, help="studio gold directory (images/, labels/, gold_manifest.json) to import first")
+    ap.add_argument("--no-light", action="store_true", help="skip reading the images")
+    args = ap.parse_args(argv)
+    root = Path(args.root).expanduser()
+    gold = root / "data" / "gold"
+    if args.sync_from:
+        from tools.import_studio_export import import_gold
+        info = import_gold(Path(args.sync_from).expanduser(), root=root)
+        print(f"gold: {info['images']} images taken from {args.sync_from}")
+    removed = prune(gold)
+    if removed:
+        print(f"gold: removed {removed} files of frames no longer in the manifest")
+    g = build(gold, with_light=not args.no_light)
+    (gold / "groups.json").write_text(json.dumps(g, indent=1), encoding="utf-8")
+    states = {}
+    for s in g["light"].values():
+        states[s] = states.get(s, 0) + 1
+    print(f"gold: {g['frames']} frames, {g['mounts_with_several_frames']} mounts holding {len(g['mount'])} of them, "
+          f"light {states or 'all ok'}, excluded {len(g['excluded'])} -> {gold / 'groups.json'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/train_run.py

@@ -240,7 +240,7 @@ def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Di
     missing = len(cand) - len(set(cand) & set(base))
     if missing:
         print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest")
-    return gate_stats.paired_bootstrap(cand, base)
+    return gate_stats.paired_bootstrap(cand, base, groups=gate_stats.load_groups(gold.parent)["mount"])
 
 
 def active_version() -> Optional[str]: