diff --git a/tests/test_gate.py b/tests/test_gate.py index 8df3279..d04a468 100644 --- a/tests/test_gate.py +++ b/tests/test_gate.py @@ -108,3 +108,44 @@ def test_baseline_from_another_environment_is_measured_again_or_refused(tmp_path # same environment: the stored report is used as it is monkeypatch.setattr(train_run, "environment", lambda: {"host": "x", "ultralytics": "8.4.160", "torch": "2.14.0"}) assert train_run.baseline_here(stored, ["loop_all"], 0.25) == (stored, None) + + +def test_bootstrap_by_mount_widens_the_interval_of_repeated_frames(): + """Ten frames of one mount all won by the candidate: frame-wise draws call it a sure win, + mount-wise draws see one lucky mount among eleven.""" + from tools import gate_stats + base = _frames([1] * 20) + cand = _frames([2] * 10 + [1] * 10) + groups = {f"f{i}": "mount-a" for i in range(10)} + by_frame = gate_stats.paired_bootstrap(cand, base, draws=800) + by_mount = gate_stats.paired_bootstrap(cand, base, draws=800, groups=groups) + assert by_frame["mounts"] == 20 and by_mount["mounts"] == 11 + assert by_frame["f1"]["delta"] == by_mount["f1"]["delta"] # the point estimate does not move + assert by_frame["f1"]["win"] > 0.99 and by_mount["f1"]["win"] < 0.75 + assert by_mount["f1"]["lo"] <= 0 < by_frame["f1"]["lo"] + + +def test_gold_groups_lists_shared_mounts_and_dropped_frames(tmp_path): + from tools import gate_stats, gold_groups + gold = tmp_path / "data" / "gold" + (gold / "labels").mkdir(parents=True); (gold / "images").mkdir() + manifest = {"images": {"tpi_1": {"tpid": 1, "mount": "PXII:s6416"}, "tpi_2": {"tpid": 2, "mount": "PXII:s6416"}, + "tpi_3": {"tpid": 3, "mount": "PXIII:s7"}, "tpi_4": {"tpid": 4}}, + "dropped": {"tpi_9": "same image as tpi_1"}} + (gold / "gold_manifest.json").write_text(json.dumps(manifest)) + for stem in ("tpi_1", "tpi_9"): + (gold / "labels" / f"{stem}.txt").write_text("") + assert gold_groups.prune(gold) == 1 and not (gold / "labels" / "tpi_9.txt").exists() + g = gold_groups.build(gold, with_light=False) + assert g["mount"] == {"tpi_1": "PXII:s6416", "tpi_2": "PXII:s6416"} # single-frame mounts are not listed + assert g["excluded"] == {"tpi_9": "same image as tpi_1"} + (gold / "groups.json").write_text(json.dumps(g)) + assert gate_stats.load_groups(gold)["mount"]["tpi_2"] == "PXII:s6416" + assert gate_stats.load_groups(tmp_path)["mount"] == {} # no file: every frame its own mount + + +def test_light_state_thresholds(): + from tools import gold_groups + assert gold_groups.light_state(0.6, 200) == "saturated" + assert gold_groups.light_state(0.0, 12) == "dark" + assert gold_groups.light_state(0.1, 120) == "ok" diff --git a/tools/compare_unseen.py b/tools/compare_unseen.py index b9d0885..a014dae 100644 --- a/tools/compare_unseen.py +++ b/tools/compare_unseen.py @@ -89,7 +89,8 @@ def main(argv: Optional[List[str]] = None) -> int: key = line.split(",")[0].strip() if key.startswith("tpi_"): trained_on.add(key) - stems = sorted(p.stem for p in gold.glob("*.txt")) + groups = gate_stats.load_groups(gold.parent) + stems = sorted(p.stem for p in gold.glob("*.txt") if p.stem not in groups["excluded"]) key_of = lambda s: "tpi_" + re.search(r"tpi_(\d+)", s).group(1) unseen = [s for s in stems if s not in seen and key_of(s) not in trained_on] core_now = [s for s in unseen if s in core] @@ -114,6 +115,10 @@ def main(argv: Optional[List[str]] = None) -> int: order = sorted(frames, key=lambda k: (not k.startswith("beamline"), k)) reps = {k: report(frames[k]) for k in order} core_reps = {k: report({s: f for s, f in frames[k].items() if s in core}) for k in order} + # frames whose light makes them unusable (saturated) are part of the measure; this is the same + # table without them, to tell a model that got worse from a set that holds more bad light + bad_light = {s for s, state in groups["light"].items() if state == "saturated"} + usable_reps = {k: report({s: f for s, f in frames[k].items() if s in core and s not in bad_light}) for k in order} print(f"clean gold frames: {len(unseen)}" + (f" (core {len(core_now)})" if len(unseen) != len(core_now) else "")) table("", reps) @@ -126,12 +131,12 @@ def main(argv: Optional[List[str]] = None) -> int: boot = {} base_label = label_of.get(active) if active else None if base_label and base_label in frames: - print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have: difference [95 % interval], share of draws won") + print(f"\nagainst the active model ({base_label}), paired bootstrap over the frames both have, drawn by mount: difference [95 % interval], share of draws won") classes = {"crystal": NAMES.index("crystal")} for k in order: if k == base_label: continue - b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes) + b = gate_stats.paired_bootstrap(frames[k], frames[base_label], draws=args.draws, classes=classes, groups=groups["mount"]) if not b.get("frames"): continue boot[k] = b @@ -145,12 +150,15 @@ def main(argv: Optional[List[str]] = None) -> int: def row(f): o = gate_stats._total([f], None); c = gate_stats._total([f], crystal) return [int(o[0]), int(o[1]), int(o[2]), round(o[3], 3), int(c[0]), int(c[1]), int(c[2]), round(c[3], 3)] - per_frame = {"stems": unseen, "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"], + per_frame = {"stems": unseen, "mount": [groups["mount"].get(st) or st for st in unseen], + "light": [groups["light"].get(st, "ok") for st in unseen], "fields": ["tp", "fp", "fn", "iou_sum", "crystal_tp", "crystal_fp", "crystal_fn", "crystal_iou_sum"], "models": {k: [row(frames[k][st]) if st in frames[k] else None for st in unseen] for k in order}} (root / "runs" / "_unseen_per_frame.json").write_text(json.dumps(per_frame, separators=(",", ":")), encoding="utf-8") out = {"clean_frames": len(unseen), "reports": reps, "core_frames": len(core_now), "core_frozen": cj.get("frozen"), - "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot} + "core_reports": core_reps, "active": active, "bootstrap_vs_active": boot, + "core_reports_usable_light": usable_reps, "saturated_frames": sorted(s for s in bad_light if s in core), + "mounts": len({groups["mount"].get(s) or s for s in unseen}), "excluded": groups["excluded"]} (root / "runs" / "clean_gold_comparison.json").write_text(json.dumps(out, indent=2), encoding="utf-8") return 0 diff --git a/tools/gate_stats.py b/tools/gate_stats.py index 96bb51e..b9dbd79 100644 --- a/tools/gate_stats.py +++ b/tools/gate_stats.py @@ -6,11 +6,16 @@ gold. A paired bootstrap answers that: draw the gold frames again with replaceme draw for both models, and look at the spread of the difference. ``win`` is the share of draws in which the candidate beats the baseline. +Frames of one mount are not independent (the same loop at another angle, 24 of the 203 gold +frames on 2026-10-02), so the draw is by mount when ``data/gold/groups.json`` names them: a +mount is drawn with all its frames. Frame-wise draws would call the interval narrower than it is. + Pure Python on purpose (the gate's tests run without numpy); only ``per_frame`` needs the evaluation code, and imports it when called. """ from __future__ import annotations +import json import random from pathlib import Path from typing import Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -19,6 +24,15 @@ Counts = Tuple[float, float, float, float] # tp, fp, fn, sum of matched FrameCounts = Dict[int, Counts] # class id -> counts, for one frame +def load_groups(gold_root) -> Dict[str, Dict[str, str]]: + """``data/gold/groups.json`` (tools/gold_groups.py): ``mount`` = frame -> mount id (a frame not + listed is its own mount), ``light`` = frame -> state for the frames whose light is not usable, + ``excluded`` = frame -> why it is left out of every measurement. Empty when the file is missing.""" + p = Path(gold_root) / "groups.json" + g = json.loads(p.read_text(encoding="utf-8")) if p.exists() else {} + return {"mount": dict(g.get("mount") or {}), "light": dict(g.get("light") or {}), "excluded": dict(g.get("excluded") or {})} + + def per_frame(pred_dir, gold_dir, iou_thr: float = 0.5, stems: Optional[Iterable[str]] = None) -> Dict[str, FrameCounts]: """Per gold frame and class: tp, fp, fn and the summed mask IoU of the matches. @@ -72,14 +86,21 @@ def _summary(point: float, deltas: List[float]) -> Dict[str, float]: def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCounts], *, draws: int = 2000, - seed: int = 0, classes: Optional[Mapping[str, int]] = None) -> Dict: + seed: int = 0, classes: Optional[Mapping[str, int]] = None, + groups: Optional[Mapping[str, str]] = None) -> Dict: """Candidate minus baseline on the frames both have: point difference, 95 % interval and win share - for F1 and the mask score, overall and for each class in ``classes`` (name -> class id).""" + for F1 and the mask score, overall and for each class in ``classes`` (name -> class id). + + ``groups`` (frame -> mount id) makes the draw by mount: the frames of a mount come and go together.""" keys = sorted(set(cand) & set(base)) n = len(keys) if n == 0: return {"frames": 0, "draws": 0} a = [cand[k] for k in keys]; b = [base[k] for k in keys] + members: Dict[str, List[int]] = {} + for i, k in enumerate(keys): + members.setdefault((groups or {}).get(k) or f"frame:{k}", []).append(i) + units = [members[g] for g in sorted(members)] targets: List[Tuple[str, Optional[int]]] = [("overall", None)] + [(name, cid) for name, cid in (classes or {}).items()] point = {} for name, cid in targets: @@ -88,12 +109,12 @@ def paired_bootstrap(cand: Mapping[str, FrameCounts], base: Mapping[str, FrameCo rng = random.Random(seed) d_f1 = {name: [] for name, _ in targets}; d_ms = {name: [] for name, _ in targets} for _ in range(draws): - idx = [rng.randrange(n) for _ in range(n)] + idx = [i for _ in range(len(units)) for i in units[rng.randrange(len(units))]] sa = [a[i] for i in idx]; sb = [b[i] for i in idx] for name, cid in targets: ca, cb = _total(sa, cid), _total(sb, cid) d_f1[name].append(f1_of(ca) - f1_of(cb)); d_ms[name].append(mask_score_of(ca) - mask_score_of(cb)) - out = {"frames": n, "draws": draws, + out = {"frames": n, "mounts": len(units), "draws": draws, "f1": _summary(point["overall"][0], d_f1["overall"]), "mask_score": _summary(point["overall"][1], d_ms["overall"])} if classes: diff --git a/tools/gold_groups.py b/tools/gold_groups.py new file mode 100644 index 0000000..a4a012d --- /dev/null +++ b/tools/gold_groups.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Which gold frames belong together, and which have unusable light -> ``data/gold/groups.json``. + + python tools/gold_groups.py # rebuild groups.json from data/gold + python tools/gold_groups.py --sync-from # take the studio's gold set first + +* ``mount``: the studio writes a ``mount`` id on every gold frame of ``gold_manifest.json`` (same + sample, or consecutive uploads of one centring). Frames of a mount are the same loop seen again, + so the gate's bootstrap draws mounts, not frames (tools/gate_stats.py). Only mounts with more + than one gold frame are listed. +* ``light``: ``saturated`` when more than half of the frame is blown out, ``dark`` when its mean + grey level is under 30. Saturated frames stay in the measure (the beamline produces them); the + comparison also reports the result without them. +* ``excluded``: the manifest's ``dropped`` frames (an exact duplicate of another gold frame, say). + They are removed from data/gold and never measured. + +On 2026-10-02: 203 frames, 7 mounts with 24 frames, 6 saturated, 29 dark, 1 dropped +(tpi_14688, the same image as tpi_14689 with a different label). +""" +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime +from pathlib import Path +from typing import Dict, Optional + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT)) +IMG_EXT = (".jpg", ".jpeg", ".png") + +SATURATED_SHARE = 0.5 # share of pixels at or above 250 +DARK_MEAN = 30.0 # mean grey level + + +def light_state(saturated_share: float, mean: float) -> str: + if saturated_share > SATURATED_SHARE: + return "saturated" + if mean < DARK_MEAN: + return "dark" + return "ok" + + +def image_light(path: Path) -> str: + import cv2 + g = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) + if g is None: + return "ok" + g = cv2.resize(g, (510, 510), interpolation=cv2.INTER_AREA) + return light_state(float((g >= 250).mean()), float(g.mean())) + + +def build(gold: Path, *, with_light: bool = True) -> Dict: + manifest = json.loads((gold / "gold_manifest.json").read_text(encoding="utf-8")) + images = manifest.get("images") or {} + by_mount: Dict[str, list] = {} + for stem, info in images.items(): + if info.get("mount"): + by_mount.setdefault(str(info["mount"]), []).append(stem) + mount = {stem: m for m, stems in sorted(by_mount.items()) if len(stems) > 1 for stem in sorted(stems)} + light: Dict[str, str] = {} + if with_light: + for stem in sorted(images): + img = next((gold / "images" / f"{stem}{e}" for e in IMG_EXT if (gold / "images" / f"{stem}{e}").exists()), None) + if img is not None: + state = image_light(img) + if state != "ok": + light[stem] = state + return { + "note": "mount: gold frames of one mount (drawn together by the gate's bootstrap); light: frames whose light is not " + "usable; excluded: frames removed from the gold set. Built by tools/gold_groups.py", + "updated": datetime.now().isoformat(timespec="seconds"), + "frames": len(images), "mounts_with_several_frames": len({m for m in mount.values()}), + "mount": mount, "light": light, "excluded": dict(manifest.get("dropped") or {}), + } + + +def prune(gold: Path) -> int: + """Remove the label and image of every frame the manifest no longer lists.""" + keep = set(json.loads((gold / "gold_manifest.json").read_text(encoding="utf-8")).get("images") or {}) + n = 0 + for sub in ("labels", "images"): + for p in sorted((gold / sub).glob("*")) if (gold / sub).exists() else []: + if p.stem.startswith("tpi_") and p.stem not in keep: + p.unlink() + n += 1 + return n + + +def main(argv: Optional[list] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--root", default=str(ROOT)) + ap.add_argument("--sync-from", default=None, help="studio gold directory (images/, labels/, gold_manifest.json) to import first") + ap.add_argument("--no-light", action="store_true", help="skip reading the images") + args = ap.parse_args(argv) + root = Path(args.root).expanduser() + gold = root / "data" / "gold" + if args.sync_from: + from tools.import_studio_export import import_gold + info = import_gold(Path(args.sync_from).expanduser(), root=root) + print(f"gold: {info['images']} images taken from {args.sync_from}") + removed = prune(gold) + if removed: + print(f"gold: removed {removed} files of frames no longer in the manifest") + g = build(gold, with_light=not args.no_light) + (gold / "groups.json").write_text(json.dumps(g, indent=1), encoding="utf-8") + states = {} + for s in g["light"].values(): + states[s] = states.get(s, 0) + 1 + print(f"gold: {g['frames']} frames, {g['mounts_with_several_frames']} mounts holding {len(g['mount'])} of them, " + f"light {states or 'all ok'}, excluded {len(g['excluded'])} -> {gold / 'groups.json'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/train_run.py b/tools/train_run.py index 93d78ed..f5225c2 100644 --- a/tools/train_run.py +++ b/tools/train_run.py @@ -240,7 +240,7 @@ def paired_with_baseline(run_dir: Path, baseline: Optional[Path]) -> Optional[Di missing = len(cand) - len(set(cand) & set(base)) if missing: print(f"gate: the baseline has no prediction for {missing} gold frames (gold grew since it was measured); pairing on the rest") - return gate_stats.paired_bootstrap(cand, base) + return gate_stats.paired_bootstrap(cand, base, groups=gate_stats.load_groups(gold.parent)["mount"]) def active_version() -> Optional[str]: