From 95cae4012ed59420898d7adb73a381920571a6bc Mon Sep 17 00:00:00 2001 From: jungfrau Date: Wed, 4 Nov 2026 18:14:37 +0100 Subject: [PATCH] import: a corrected loop-head crop gets a content-addressed pool name, the gold import prunes dropped and released frames; spark_cycle.sh RESUME=1 starts after the gold growth and the frame export Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JPZJw4gJgQmPzTa6ix7eGp --- tools/import_studio_export.py | 35 +++++++++++++++++++++++++++-------- tools/spark_cycle.sh | 25 ++++++++++++++++++------- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/tools/import_studio_export.py b/tools/import_studio_export.py index e510203d..569d166d 100755 --- a/tools/import_studio_export.py +++ b/tools/import_studio_export.py @@ -74,20 +74,31 @@ def write_dataset_yaml(dest: Path, names: Dict[int, str]) -> Path: def add_to_pool(image: Path, pool: Path) -> tuple[Path, bool]: - """Copy ``image`` into the pool unless an identical file is already there.""" + """Copy ``image`` into the pool unless an identical file is already there. + + A camera frame (tpi_N.jpg) never changes, so a same-named file of another size is an error. A + derived image (a loop-head crop, tpi_N_head.jpg) is cut from the label and changes when the label + is corrected: it is kept next to the old one under a content-addressed name (tpi_N_head..jpg); + the dataset links to it under the plain name, so the label file still matches.""" pool.mkdir(parents=True, exist_ok=True) dest = pool / image.name - if dest.exists(): - if dest.stat().st_size == image.stat().st_size: - return dest, False + if not dest.exists(): + shutil.copy2(image, dest) + return dest, True + if dest.stat().st_size == image.stat().st_size: + return dest, False + if "_head" not in image.stem: raise SystemExit(f"{dest} exists with a different size than {image}; a pool frame never changes") + dest = pool / f"{image.stem}.{_sha256(image)[:8]}{image.suffix}" + if dest.exists(): + return dest, False shutil.copy2(image, dest) return dest, True -def link_into(dataset_images: Path, pool_file: Path) -> Path: +def link_into(dataset_images: Path, pool_file: Path, name: Optional[str] = None) -> Path: dataset_images.mkdir(parents=True, exist_ok=True) - link = dataset_images / pool_file.name + link = dataset_images / (name or pool_file.name) rel = os.path.relpath(pool_file, dataset_images) if link.is_symlink() or link.exists(): link.unlink() @@ -113,7 +124,7 @@ def import_export(export_dir: Path, name: str, *, root: Path = ROOT, gold_dir: O for img in sorted(p for p in src_images.glob("*") if p.suffix.lower() in IMG_EXT): pool_file, new = add_to_pool(img, pool) added += int(new) - link_into(dest / "images" / split, pool_file) + link_into(dest / "images" / split, pool_file, name=img.name) lbl = src_labels / f"{img.stem}.txt" out_lbl = dest / "labels" / split / f"{img.stem}.txt" out_lbl.parent.mkdir(parents=True, exist_ok=True) @@ -161,10 +172,18 @@ def import_gold(gold_dir: Path, *, root: Path = ROOT) -> Dict: shutil.copy2(lbl, dest / "labels" / lbl.name) n += 1 manifest = gold_dir / "gold_manifest.json" - info = {"images": n, "manifest_sha256": None} + info = {"images": n, "manifest_sha256": None, "pruned": 0} if manifest.exists(): shutil.copy2(manifest, dest / "gold_manifest.json") info["manifest_sha256"] = _sha256(manifest) + # The copy above only adds: a frame the studio dropped or released must leave this gold too, + # or the gate measures on frames the pool trains on (6 released frames sat here on 4 Nov 2026). + keep = set((json.loads(manifest.read_text(encoding="utf-8")).get("images") or {})) + for sub in ("images", "labels"): + for p in sorted((dest / sub).glob("tpi_*")) if (dest / sub).exists() else []: + if p.name.split(".")[0] not in keep: + p.unlink() + info["pruned"] += 1 return info diff --git a/tools/spark_cycle.sh b/tools/spark_cycle.sh index 5b79b0c8..6e34238c 100755 --- a/tools/spark_cycle.sh +++ b/tools/spark_cycle.sh @@ -5,6 +5,8 @@ # bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first # MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes # MODELS="s n" CROP_MODELS="s m" ORDER="s crop-s n crop-m" bash ~/AareLC_train/tools/spark_cycle.sh v7 +# RESUME=1 ... bash ~/AareLC_train/tools/spark_cycle.sh v7 # gold grown and dataset v7 exported by an +# # earlier run that stopped later: start at the crop export (skipped too if its tag exists) # # also the loop-head crop models (studio second pass) on crops of the same pool, in that order # # Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold` @@ -47,19 +49,28 @@ avail_gb() { free -g | awk 'NR==2{print $7}'; } while pgrep -f "[p]ipeline.run auto" >/dev/null; do sleep 30; done # the reviewers' batch goes first cd "$STUDIO"; set -a; . ~/.aaredb_env; set +a -say "gold: growing by rule" -$PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true -say "export + import into AareLC_train (dataset $TAG)" -$PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true +if [ -z "${RESUME:-}" ]; then + say "gold: growing by rule" + $PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true + say "export + import into AareLC_train (dataset $TAG)" + $PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true +else + say "resume: the gold is grown and dataset $TAG exported by an earlier run of this cycle (the gold rule is not idempotent, so it is not run again)" +fi cd "$TRAIN" DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; } DSDIR="data/datasets/${DS#dataset/}" say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames" CDS="" if [ -n "$CROP_MODELS" ]; then - say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)" - (cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop" | cut -c1-220 || true) - CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; } + CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1) + if [ -n "$CDS" ]; then + say "crop dataset $CDS exists already" + else + say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)" + (cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop|exists" | cut -c1-220 || true) + CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; } + fi CDSDIR="data/datasets/${CDS#dataset/}" say "crop dataset $CDS: $(ls $CDSDIR/images/train | wc -l) train / $(ls $CDSDIR/images/val | wc -l) val" fi