spark_cycle.sh: crop second-pass models (CROP_MODELS), a training order (ORDER) and the two-pass measure on the gold

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JPZJw4gJgQmPzTa6ix7eGp
This commit is contained in:
jungfrauandClaude Fable 5.1 committed 2026-11-04 17:56:24 +01:00
1 parent 67a694c9e7
commit f8e7da9732
1 file changed
+74 -7
+74 -7
View File
@@ -4,6 +4,8 @@
# nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 &
# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first
# MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes
# MODELS="s n" CROP_MODELS="s m" ORDER="s crop-s n crop-m" bash ~/AareLC_train/tools/spark_cycle.sh v7
# # also the loop-head crop models (studio second pass) on crops of the same pool, in that order
#
# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold`
# stage: gold batches, share, floors); export the reviewed pool as dataset <tag> into this
@@ -13,6 +15,10 @@
# only the nano run may promote itself, a bigger model is a candidate until its speed at the
# beamline is known (its latency on this machine is measured and kept next to the run);
# compare every model on the unseen frames; push; start vLLM again if it was running.
# With CROP_MODELS the pool is also exported as loop-head crops (dataset <tag>crop) and a crop
# model is trained per size without a gate (the gold cannot score a crop model alone); at the
# end every crop model is measured as a two-pass on the gold, behind each first-pass model of
# this cycle and behind the active model. ORDER interleaves the trainings ("s crop-s n crop-m").
#
# What earlier cycles taught, built in:
# - a failed gate is a verdict, not a crash: train_run.py is called once per run;
@@ -26,6 +32,9 @@
set -eo pipefail
TAG="${1:?usage: spark_cycle.sh <dataset tag, e.g. v5> [--quick]}"; QUICK="${2:-}"
MODELS="${MODELS:-n}" # yolo26 sizes to train, e.g. "n s m l"
CROP_MODELS="${CROP_MODELS:-}" # loop-head crop model sizes for the studio second pass, e.g. "s m"
ORDER="${ORDER:-}" # training order, e.g. "s crop-s n crop-m" (default: MODELS, then the crop models)
FIRST_RUNS=""; CROP_RUNS=""
MEM_FLOOR_GB="${MEM_FLOOR_GB:-40}" # a training starts only with this much memory available
export PATH=$HOME/bin:$PATH
export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12
@@ -46,9 +55,17 @@ cd "$TRAIN"
DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; }
DSDIR="data/datasets/${DS#dataset/}"
say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames"
CDS=""
if [ -n "$CROP_MODELS" ]; then
say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)"
(cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop" | cut -c1-220 || true)
CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; }
CDSDIR="data/datasets/${CDS#dataset/}"
say "crop dataset $CDS: $(ls $CDSDIR/images/train | wc -l) train / $(ls $CDSDIR/images/val | wc -l) val"
fi
# Base weights for every size asked for (Ultralytics fetches a missing one from its releases).
for size in $MODELS; do
for size in $MODELS $CROP_MODELS; do
base="models/base/yolo26${size}-seg.pt"
[ -f "$base" ] && continue
say "fetching $base"
@@ -112,16 +129,66 @@ run() { # size, tag suffix, note, extra train_run.py args...
fi
python3 tools/ledger.py render >/dev/null 2>&1 || true
git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
FIRST_RUNS="$FIRST_RUNS $id"
}
run_crop() { # size: the loop-head crop model for the studio second pass, trained on the crop dataset, no gold gate
local size="$1"
local base="models/base/yolo26${size}-seg.pt"
local id; id=$(python3 tools/ledger.py new --dataset "$CDS" --tag "$TAG-crop-$size" --base "$base" --note "loop-head crop model (yolo26$size) for the studio second pass on the $TAG pool: crops 1.6x the face, min 512 px, head classes only; promoted recipe; no gold gate, measured as a two-pass behind the first-pass models, Spark")
git add runs && git commit -q -m "run $id: exported on $CDS" || true
wait_for_memory
say "training $id from $base ($(avail_gb) GB available)"
set +e
$PY tools/train_run.py "$id" --device 0 --workers 8 --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null
local rc=$?
if [ $rc -ne 0 ] && grep -aqi "out of memory" "runs/$id/train.log"; then
say "$id ran out of memory at batch 16; once more at batch 8"
sleep 60
$PY tools/train_run.py "$id" --device 0 --workers 8 --batch 8 --force --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null
rc=$?
fi
set -e
[ $rc -eq 0 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)"
local best="runs/$id/ultralytics/train/weights/best.pt"
if [ -f "$best" ]; then
$PY tools/bench_latency.py "$best" --out "runs/$id/latency.json" 2>/dev/null | sed 's/^/ latency: /' | cut -c1-200 || true
fi
python3 tools/ledger.py render >/dev/null 2>&1 || true
git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
CROP_RUNS="$CROP_RUNS $id"
}
[ "$QUICK" = "--quick" ] && run n recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4
for size in $MODELS; do
if [ "$size" = "n" ]; then
run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1
else
run "$size" "${size}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${size} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1
fi
if [ -z "$ORDER" ]; then
ORDER="$MODELS"; for size in $CROP_MODELS; do ORDER="$ORDER crop-$size"; done
fi
for job in $ORDER; do
case "$job" in
crop-*) run_crop "${job#crop-}" ;;
n) run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1 ;;
*) run "$job" "${job}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${job} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1 ;;
esac
done
if [ -n "$CROP_RUNS" ]; then
# Every crop model as a two-pass on the gold (crop objects of crystal and loop_face replace the
# first pass's), behind the active model and behind each first-pass model of this cycle.
say "two-pass measure on the gold frames (replace crystal+loop_face)"
python3 -c "import json; c=json.load(open('$STUDIO/$CFG')); c['headcrop']={'classes':['crystal','loop_face']}; json.dump(c, open('$STUDIO/pipeline.prod.cf.json','w'), indent=1)"
FIRSTS="$(python3 -c 'import json;print(json.load(open("models/active/production.json"))["version"])')"
for id in $FIRST_RUNS; do
v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')")
[ -n "$v" ] && [ -f "models/registry/$v/weights.pt" ] && FIRSTS="$FIRSTS $v"
done
for cid in $CROP_RUNS; do
cv=$(python3 -c "import json;print(json.load(open('runs/$cid/run.json')).get('version') or '')")
[ -n "$cv" ] && [ -f "models/registry/$cv/weights.pt" ] || { echo " $cid left no registry weights, not measured"; continue; }
for fv in $FIRSTS; do
say "two-pass $fv + $cv"
(cd "$STUDIO" && $PY -m src.pipeline.run measure --config pipeline.prod.cf.json --weights "$TRAIN/models/registry/$fv/weights.pt" --crop-weights "$TRAIN/models/registry/$cv/weights.pt" --out "data/pipeline_prod/gold/predictions/two_pass_${fv}+${cv}_cf" 2>&1 | tail -14 | cut -c1-220 || true)
done
done
fi
say "every model on the unseen gold frames"
$PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true
git add runs && git commit -q -m "unseen-frame comparison after cycle $TAG" || true