training loop: run ledger, studio export importer with an image pool, train_run with gold gate and auto-promotion, SLURM template, CI

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/eval/__init__.py

@@ -0,0 +1,4 @@
+"""Gold-set evaluation, vendored from AareLC_studio/src/pipeline (metrics, yolo_labels, evaluate) on 2026-09-28.
+
+Kept identical on purpose so a measure here and in the studio agree; update by re-copying.
+"""

src/eval/evaluate.py

@@ -0,0 +1,88 @@
+"""Eval harness: compare predicted YOLO-seg labels against a gold set.
+
+Reports detection precision/recall/F1 per class (bbox IoU >= threshold to match)
+and mean mask IoU over matched instances. Operates on label files only (no source
+images needed), so it can score any pre-labeler / model against the frozen gold set.
+
+    python -m src.pipeline.evaluate --pred OUT/labels --gold GOLD/labels [--names dataset.yaml]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Dict, List
+
+from src.eval.metrics import aggregate, evaluate_image, full_report
+from src.eval.yolo_labels import load_class_names, read_label_file
+
+
+def _stems(label_dir: Path) -> Dict[str, Path]:
+    return {p.stem: p for p in label_dir.glob("*.txt")}
+
+
+def evaluate_dirs(pred_dir, gold_dir, iou_thr: float = 0.5,
+                  names: List[str] | None = None) -> Dict:
+    pred_dir, gold_dir = Path(pred_dir), Path(gold_dir)
+    gold_files = _stems(gold_dir)
+    pred_files = _stems(pred_dir)
+
+    # Gold defines the evaluation set; a missing prediction counts as all-FN.
+    per_image = []
+    missing_pred = 0
+    for stem, gpath in sorted(gold_files.items()):
+        gold = read_label_file(gpath)
+        ppath = pred_files.get(stem)
+        pred = read_label_file(ppath) if ppath else []
+        if ppath is None:
+            missing_pred += 1
+        per_image.append(evaluate_image(gold, pred, iou_thr))
+
+    report = full_report(aggregate(per_image), names)
+    report["dataset"] = {
+        "gold_images": len(gold_files),
+        "pred_images": len(pred_files),
+        "missing_predictions": missing_pred,
+        "iou_threshold": iou_thr,
+    }
+    return report
+
+
+def _print_report(report: Dict) -> None:
+    ds = report["dataset"]
+    print(f"\nGold images: {ds['gold_images']}  |  predictions: {ds['pred_images']}  "
+          f"|  missing: {ds['missing_predictions']}  |  match IoU >= {ds['iou_threshold']}")
+    header = f"{'class':<14}{'P':>7}{'R':>7}{'F1':>7}{'maskIoU':>9}{'TP':>6}{'FP':>6}{'FN':>6}"
+    print("\n" + header)
+    print("-" * len(header))
+
+    def row(name, r):
+        print(f"{name:<14}{r['precision']:>7.3f}{r['recall']:>7.3f}{r['f1']:>7.3f}"
+              f"{r['mean_mask_iou']:>9.3f}{r['tp']:>6}{r['fp']:>6}{r['fn']:>6}")
+
+    for name, r in report["per_class"].items():
+        row(name, r)
+    print("-" * len(header))
+    row("OVERALL", report["overall"])
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="Evaluate predicted YOLO-seg labels vs. a gold set")
+    ap.add_argument("--pred", required=True, help="Directory of predicted .txt labels")
+    ap.add_argument("--gold", required=True, help="Directory of gold .txt labels")
+    ap.add_argument("--iou", type=float, default=0.5, help="bbox IoU threshold to match instances")
+    ap.add_argument("--names", default=None, help="Optional dataset.yaml for class names")
+    ap.add_argument("--report", default=None, help="Optional path to write the JSON report")
+    args = ap.parse_args()
+
+    names = load_class_names(args.names) if args.names else []
+    report = evaluate_dirs(args.pred, args.gold, iou_thr=args.iou, names=names)
+    _print_report(report)
+    if args.report:
+        Path(args.report).write_text(json.dumps(report, indent=2), encoding="utf-8")
+        print(f"\nWrote {args.report}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/eval/metrics.py

@@ -0,0 +1,136 @@
+"""Quality metrics for auto-annotation: detection P/R/F1 + per-instance mask IoU.
+
+Works entirely on normalized polygons (YOLO-seg form), so no source images are
+needed at eval time: masks are rasterized into a fixed grid for IoU. Matching is
+greedy within a class by bbox IoU; mask IoU is reported for matched pairs.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Sequence, Tuple
+
+import cv2
+import numpy as np
+
+from src.eval.yolo_labels import PolygonInstance
+
+RASTER = 512  # grid resolution for mask IoU
+
+
+def bbox_iou(a: Sequence[float], b: Sequence[float]) -> float:
+    """IoU of two [x1, y1, x2, y2] boxes (any consistent units)."""
+    ax1, ay1, ax2, ay2 = a
+    bx1, by1, bx2, by2 = b
+    ix1, iy1 = max(ax1, bx1), max(ay1, by1)
+    ix2, iy2 = min(ax2, bx2), min(ay2, by2)
+    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)
+    inter = iw * ih
+    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)
+    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)
+    union = area_a + area_b - inter
+    return float(inter / union) if union > 0 else 0.0
+
+
+def _raster(points_norm, size: int = RASTER) -> np.ndarray:
+    m = np.zeros((size, size), dtype=np.uint8)
+    if len(points_norm) >= 3:
+        pts = np.array([(x * size, y * size) for x, y in points_norm], dtype=np.int32).reshape(-1, 1, 2)
+        cv2.fillPoly(m, [pts], 1)
+    return m
+
+
+def mask_iou(points_a_norm, points_b_norm, size: int = RASTER) -> float:
+    """IoU of two normalized polygons, via rasterization into a fixed grid."""
+    if len(points_a_norm) < 3 or len(points_b_norm) < 3:
+        return 0.0
+    ma, mb = _raster(points_a_norm, size), _raster(points_b_norm, size)
+    inter = int(np.logical_and(ma, mb).sum())
+    union = int(np.logical_or(ma, mb).sum())
+    return float(inter / union) if union > 0 else 0.0
+
+
+@dataclass
+class ClassCount:
+    tp: int = 0
+    fp: int = 0
+    fn: int = 0
+    mask_ious: List[float] = field(default_factory=list)
+
+    def merge(self, other: "ClassCount") -> None:
+        self.tp += other.tp
+        self.fp += other.fp
+        self.fn += other.fn
+        self.mask_ious.extend(other.mask_ious)
+
+
+def _match_class(gold: List[PolygonInstance], pred: List[PolygonInstance],
+                 iou_thr: float) -> ClassCount:
+    """Greedy bbox-IoU matching within a single class."""
+    cc = ClassCount()
+    gold_boxes = [g.bbox() for g in gold]
+    pred_boxes = [p.bbox() for p in pred]
+
+    pairs: List[Tuple[float, int, int]] = []
+    for gi, gb in enumerate(gold_boxes):
+        for pi, pb in enumerate(pred_boxes):
+            iou = bbox_iou(gb, pb)
+            if iou >= iou_thr:
+                pairs.append((iou, gi, pi))
+    pairs.sort(reverse=True)
+
+    used_g, used_p = set(), set()
+    for iou, gi, pi in pairs:
+        if gi in used_g or pi in used_p:
+            continue
+        used_g.add(gi)
+        used_p.add(pi)
+        cc.tp += 1
+        cc.mask_ious.append(mask_iou(gold[gi].points, pred[pi].points))
+
+    cc.fn = len(gold) - len(used_g)
+    cc.fp = len(pred) - len(used_p)
+    return cc
+
+
+def evaluate_image(gold: List[PolygonInstance], pred: List[PolygonInstance],
+                   iou_thr: float = 0.5) -> Dict[int, ClassCount]:
+    """Per-class counts for one image."""
+    by_class: Dict[int, ClassCount] = {}
+    classes = {i.class_id for i in gold} | {i.class_id for i in pred}
+    for c in classes:
+        g = [i for i in gold if i.class_id == c]
+        p = [i for i in pred if i.class_id == c]
+        by_class[c] = _match_class(g, p, iou_thr)
+    return by_class
+
+
+def aggregate(per_image: List[Dict[int, ClassCount]]) -> Dict[int, ClassCount]:
+    """Sum per-image per-class counts across the dataset."""
+    total: Dict[int, ClassCount] = {}
+    for img in per_image:
+        for c, cc in img.items():
+            total.setdefault(c, ClassCount()).merge(cc)
+    return total
+
+
+def class_report(cc: ClassCount) -> Dict[str, float]:
+    precision = cc.tp / (cc.tp + cc.fp) if (cc.tp + cc.fp) else 0.0
+    recall = cc.tp / (cc.tp + cc.fn) if (cc.tp + cc.fn) else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
+    mean_iou = float(np.mean(cc.mask_ious)) if cc.mask_ious else 0.0
+    return {
+        "tp": cc.tp, "fp": cc.fp, "fn": cc.fn,
+        "precision": precision, "recall": recall, "f1": f1,
+        "mean_mask_iou": mean_iou, "n_matched": len(cc.mask_ious),
+    }
+
+
+def full_report(total: Dict[int, ClassCount], names: List[str] | None = None) -> Dict:
+    """Per-class + micro-averaged overall report."""
+    per_class = {}
+    overall = ClassCount()
+    for c, cc in sorted(total.items()):
+        name = names[c] if names and 0 <= c < len(names) else str(c)
+        per_class[name] = class_report(cc)
+        overall.merge(cc)
+    return {"per_class": per_class, "overall": class_report(overall)}

src/eval/yolo_labels.py

@@ -0,0 +1,123 @@
+"""Read/write Ultralytics YOLO-segmentation label files.
+
+A YOLO-seg label line is::
+
+    <class_id> x1 y1 x2 y2 ... xn yn
+
+with every coordinate normalized to [0, 1] (x / image_width, y / image_height).
+This module keeps points in normalized form (matching the on-disk format) and
+offers helpers to convert to/from absolute pixels.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+Point = Tuple[float, float]
+
+
+@dataclass
+class PolygonInstance:
+    """One segmented object: a class id and a single normalized polygon ring."""
+
+    class_id: int
+    points: List[Point]              # normalized (x, y) in [0, 1]
+    class_name: Optional[str] = None
+
+    def bbox(self) -> Tuple[float, float, float, float]:
+        """Axis-aligned bounding box (normalized) of the polygon."""
+        xs = [p[0] for p in self.points]
+        ys = [p[1] for p in self.points]
+        return min(xs), min(ys), max(xs), max(ys)
+
+
+def to_yolo_seg_line(inst: PolygonInstance) -> str:
+    coords = " ".join(f"{c:.6f}" for p in inst.points for c in p)
+    return f"{inst.class_id} {coords}".strip()
+
+
+def parse_yolo_seg_line(line: str) -> Optional[PolygonInstance]:
+    parts = line.split()
+    if len(parts) < 7:  # class id + at least 3 (x, y) pairs
+        return None
+    try:
+        class_id = int(float(parts[0]))
+        nums = [float(x) for x in parts[1:]]
+    except ValueError:
+        return None
+    pts = [(nums[i], nums[i + 1]) for i in range(0, len(nums) - 1, 2)]
+    if len(pts) < 3:
+        return None
+    return PolygonInstance(class_id=class_id, points=pts)
+
+
+def read_label_file(path) -> List[PolygonInstance]:
+    """Parse a YOLO-seg ``.txt`` file into polygon instances (empty if missing)."""
+    p = Path(path)
+    if not p.exists():
+        return []
+    out: List[PolygonInstance] = []
+    for line in p.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        inst = parse_yolo_seg_line(line)
+        if inst is not None:
+            out.append(inst)
+    return out
+
+
+def write_label_file(path, instances: List[PolygonInstance]) -> None:
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    body = "\n".join(to_yolo_seg_line(i) for i in instances)
+    p.write_text(body + ("\n" if instances else ""), encoding="utf-8")
+
+
+def abs_to_norm(points_abs, img_w: int, img_h: int) -> List[Point]:
+    return [(x / img_w, y / img_h) for (x, y) in points_abs]
+
+
+def norm_to_abs(points_norm, img_w: int, img_h: int) -> List[Point]:
+    return [(x * img_w, y * img_h) for (x, y) in points_norm]
+
+
+def load_class_names(dataset_yaml) -> List[str]:
+    """Read class names from a dataset.yaml ``names:`` block (best effort).
+
+    Supports both the ``names: {0: a, 1: b}`` mapping and ``names: [a, b]`` list
+    forms without requiring PyYAML.
+    """
+    p = Path(dataset_yaml)
+    if not p.exists():
+        return []
+    try:
+        import yaml  # optional
+        data = yaml.safe_load(p.read_text(encoding="utf-8"))
+        names = data.get("names") if isinstance(data, dict) else None
+        if isinstance(names, dict):
+            return [names[k] for k in sorted(names)]
+        if isinstance(names, list):
+            return list(names)
+    except Exception:
+        pass
+
+    # Minimal fallback parser for "  0: loop_all" style lines.
+    names_map = {}
+    in_names = False
+    for raw in p.read_text(encoding="utf-8").splitlines():
+        if raw.strip().startswith("names:"):
+            in_names = True
+            continue
+        if in_names:
+            stripped = raw.strip()
+            if not stripped or (raw[:1] not in (" ", "\t") and ":" in stripped):
+                break
+            if ":" in stripped:
+                key, _, val = stripped.partition(":")
+                try:
+                    names_map[int(key.strip().lstrip("- "))] = val.strip().strip("'\"")
+                except ValueError:
+                    continue
+    return [names_map[k] for k in sorted(names_map)] if names_map else []