import: a corrected loop-head crop gets a content-addressed pool name, the gold import prunes dropped and released frames; spark_cycle.sh RESUME=1 starts after the gold growth and the frame export

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JPZJw4gJgQmPzTa6ix7eGp
This commit is contained in:
jungfrauandClaude Fable 5.1 committed 2026-11-04 18:14:37 +01:00
1 parent ed4e20a50b
commit 95cae4012e
2 files changed
+45 -15

No files matched your search

+27 -8
View File
@@ -74,20 +74,31 @@ def write_dataset_yaml(dest: Path, names: Dict[int, str]) -> Path:
def add_to_pool(image: Path, pool: Path) -> tuple[Path, bool]:
"""Copy ``image`` into the pool unless an identical file is already there."""
"""Copy ``image`` into the pool unless an identical file is already there.
A camera frame (tpi_N.jpg) never changes, so a same-named file of another size is an error. A
derived image (a loop-head crop, tpi_N_head.jpg) is cut from the label and changes when the label
is corrected: it is kept next to the old one under a content-addressed name (tpi_N_head.<sha8>.jpg);
the dataset links to it under the plain name, so the label file still matches."""
pool.mkdir(parents=True, exist_ok=True)
dest = pool / image.name
if dest.exists():
if dest.stat().st_size == image.stat().st_size:
return dest, False
if not dest.exists():
shutil.copy2(image, dest)
return dest, True
if dest.stat().st_size == image.stat().st_size:
return dest, False
if "_head" not in image.stem:
raise SystemExit(f"{dest} exists with a different size than {image}; a pool frame never changes")
dest = pool / f"{image.stem}.{_sha256(image)[:8]}{image.suffix}"
if dest.exists():
return dest, False
shutil.copy2(image, dest)
return dest, True
def link_into(dataset_images: Path, pool_file: Path) -> Path:
def link_into(dataset_images: Path, pool_file: Path, name: Optional[str] = None) -> Path:
dataset_images.mkdir(parents=True, exist_ok=True)
link = dataset_images / pool_file.name
link = dataset_images / (name or pool_file.name)
rel = os.path.relpath(pool_file, dataset_images)
if link.is_symlink() or link.exists():
link.unlink()
@@ -113,7 +124,7 @@ def import_export(export_dir: Path, name: str, *, root: Path = ROOT, gold_dir: O
for img in sorted(p for p in src_images.glob("*") if p.suffix.lower() in IMG_EXT):
pool_file, new = add_to_pool(img, pool)
added += int(new)
link_into(dest / "images" / split, pool_file)
link_into(dest / "images" / split, pool_file, name=img.name)
lbl = src_labels / f"{img.stem}.txt"
out_lbl = dest / "labels" / split / f"{img.stem}.txt"
out_lbl.parent.mkdir(parents=True, exist_ok=True)
@@ -161,10 +172,18 @@ def import_gold(gold_dir: Path, *, root: Path = ROOT) -> Dict:
shutil.copy2(lbl, dest / "labels" / lbl.name)
n += 1
manifest = gold_dir / "gold_manifest.json"
info = {"images": n, "manifest_sha256": None}
info = {"images": n, "manifest_sha256": None, "pruned": 0}
if manifest.exists():
shutil.copy2(manifest, dest / "gold_manifest.json")
info["manifest_sha256"] = _sha256(manifest)
# The copy above only adds: a frame the studio dropped or released must leave this gold too,
# or the gate measures on frames the pool trains on (6 released frames sat here on 4 Nov 2026).
keep = set((json.loads(manifest.read_text(encoding="utf-8")).get("images") or {}))
for sub in ("images", "labels"):
for p in sorted((dest / sub).glob("tpi_*")) if (dest / sub).exists() else []:
if p.name.split(".")[0] not in keep:
p.unlink()
info["pruned"] += 1
return info
+18 -7
View File
@@ -5,6 +5,8 @@
# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first
# MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes
# MODELS="s n" CROP_MODELS="s m" ORDER="s crop-s n crop-m" bash ~/AareLC_train/tools/spark_cycle.sh v7
# RESUME=1 ... bash ~/AareLC_train/tools/spark_cycle.sh v7 # gold grown and dataset v7 exported by an
# # earlier run that stopped later: start at the crop export (skipped too if its tag exists)
# # also the loop-head crop models (studio second pass) on crops of the same pool, in that order
#
# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold`
@@ -47,19 +49,28 @@ avail_gb() { free -g | awk 'NR==2{print $7}'; }
while pgrep -f "[p]ipeline.run auto" >/dev/null; do sleep 30; done # the reviewers' batch goes first
cd "$STUDIO"; set -a; . ~/.aaredb_env; set +a
say "gold: growing by rule"
$PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true
say "export + import into AareLC_train (dataset $TAG)"
$PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true
if [ -z "${RESUME:-}" ]; then
say "gold: growing by rule"
$PY -m src.pipeline.run gold --config $CFG 2>&1 | grep -E "^gold" | cut -c1-220 || true
say "export + import into AareLC_train (dataset $TAG)"
$PY -m src.pipeline.run export --config $CFG --dest "data/pipeline_prod/dataset_$(date +%F)_$TAG" --train-repo "$TRAIN" --train-tag "$TAG" 2>&1 | grep -E "^export|dataset |committed|tagged|gold" | cut -c1-220 || true
else
say "resume: the gold is grown and dataset $TAG exported by an earlier run of this cycle (the gold rule is not idempotent, so it is not run again)"
fi
cd "$TRAIN"
DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; }
DSDIR="data/datasets/${DS#dataset/}"
say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames"
CDS=""
if [ -n "$CROP_MODELS" ]; then
say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)"
(cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop" | cut -c1-220 || true)
CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; }
CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1)
if [ -n "$CDS" ]; then
say "crop dataset $CDS exists already"
else
say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)"
(cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop|exists" | cut -c1-220 || true)
CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; }
fi
CDSDIR="data/datasets/${CDS#dataset/}"
say "crop dataset $CDS: $(ls $CDSDIR/images/train | wc -l) train / $(ls $CDSDIR/images/val | wc -l) val"
fi