diff --git a/tools/spark_cycle.sh b/tools/spark_cycle.sh index 2b8ebf7b..5b79b0c8 100755 --- a/tools/spark_cycle.sh +++ b/tools/spark_cycle.sh @@ -4,6 +4,8 @@ # nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 & # bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first # MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes +# MODELS="s n" CROP_MODELS="s m" ORDER="s crop-s n crop-m" bash ~/AareLC_train/tools/spark_cycle.sh v7 +# # also the loop-head crop models (studio second pass) on crops of the same pool, in that order # # Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold` # stage: gold batches, share, floors); export the reviewed pool as dataset into this @@ -13,6 +15,10 @@ # only the nano run may promote itself, a bigger model is a candidate until its speed at the # beamline is known (its latency on this machine is measured and kept next to the run); # compare every model on the unseen frames; push; start vLLM again if it was running. +# With CROP_MODELS the pool is also exported as loop-head crops (dataset crop) and a crop +# model is trained per size without a gate (the gold cannot score a crop model alone); at the +# end every crop model is measured as a two-pass on the gold, behind each first-pass model of +# this cycle and behind the active model. ORDER interleaves the trainings ("s crop-s n crop-m"). # # What earlier cycles taught, built in: # - a failed gate is a verdict, not a crash: train_run.py is called once per run; @@ -26,6 +32,9 @@ set -eo pipefail TAG="${1:?usage: spark_cycle.sh [--quick]}"; QUICK="${2:-}" MODELS="${MODELS:-n}" # yolo26 sizes to train, e.g. "n s m l" +CROP_MODELS="${CROP_MODELS:-}" # loop-head crop model sizes for the studio second pass, e.g. "s m" +ORDER="${ORDER:-}" # training order, e.g. "s crop-s n crop-m" (default: MODELS, then the crop models) +FIRST_RUNS=""; CROP_RUNS="" MEM_FLOOR_GB="${MEM_FLOOR_GB:-40}" # a training starts only with this much memory available export PATH=$HOME/bin:$PATH export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12 @@ -46,9 +55,17 @@ cd "$TRAIN" DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; } DSDIR="data/datasets/${DS#dataset/}" say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames" +CDS="" +if [ -n "$CROP_MODELS" ]; then + say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)" + (cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop" | cut -c1-220 || true) + CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; } + CDSDIR="data/datasets/${CDS#dataset/}" + say "crop dataset $CDS: $(ls $CDSDIR/images/train | wc -l) train / $(ls $CDSDIR/images/val | wc -l) val" +fi # Base weights for every size asked for (Ultralytics fetches a missing one from its releases). -for size in $MODELS; do +for size in $MODELS $CROP_MODELS; do base="models/base/yolo26${size}-seg.pt" [ -f "$base" ] && continue say "fetching $base" @@ -112,16 +129,66 @@ run() { # size, tag suffix, note, extra train_run.py args... fi python3 tools/ledger.py render >/dev/null 2>&1 || true git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true + FIRST_RUNS="$FIRST_RUNS $id" +} +run_crop() { # size: the loop-head crop model for the studio second pass, trained on the crop dataset, no gold gate + local size="$1" + local base="models/base/yolo26${size}-seg.pt" + local id; id=$(python3 tools/ledger.py new --dataset "$CDS" --tag "$TAG-crop-$size" --base "$base" --note "loop-head crop model (yolo26$size) for the studio second pass on the $TAG pool: crops 1.6x the face, min 512 px, head classes only; promoted recipe; no gold gate, measured as a two-pass behind the first-pass models, Spark") + git add runs && git commit -q -m "run $id: exported on $CDS" || true + wait_for_memory + say "training $id from $base ($(avail_gb) GB available)" + set +e + $PY tools/train_run.py "$id" --device 0 --workers 8 --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null + local rc=$? + if [ $rc -ne 0 ] && grep -aqi "out of memory" "runs/$id/train.log"; then + say "$id ran out of memory at batch 16; once more at batch 8" + sleep 60 + $PY tools/train_run.py "$id" --device 0 --workers 8 --batch 8 --force --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null + rc=$? + fi + set -e + [ $rc -eq 0 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)" + local best="runs/$id/ultralytics/train/weights/best.pt" + if [ -f "$best" ]; then + $PY tools/bench_latency.py "$best" --out "runs/$id/latency.json" 2>/dev/null | sed 's/^/ latency: /' | cut -c1-200 || true + fi + python3 tools/ledger.py render >/dev/null 2>&1 || true + git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true + CROP_RUNS="$CROP_RUNS $id" } [ "$QUICK" = "--quick" ] && run n recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4 -for size in $MODELS; do - if [ "$size" = "n" ]; then - run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1 - else - run "$size" "${size}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${size} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1 - fi +if [ -z "$ORDER" ]; then + ORDER="$MODELS"; for size in $CROP_MODELS; do ORDER="$ORDER crop-$size"; done +fi +for job in $ORDER; do + case "$job" in + crop-*) run_crop "${job#crop-}" ;; + n) run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1 ;; + *) run "$job" "${job}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${job} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1 ;; + esac done +if [ -n "$CROP_RUNS" ]; then + # Every crop model as a two-pass on the gold (crop objects of crystal and loop_face replace the + # first pass's), behind the active model and behind each first-pass model of this cycle. + say "two-pass measure on the gold frames (replace crystal+loop_face)" + python3 -c "import json; c=json.load(open('$STUDIO/$CFG')); c['headcrop']={'classes':['crystal','loop_face']}; json.dump(c, open('$STUDIO/pipeline.prod.cf.json','w'), indent=1)" + FIRSTS="$(python3 -c 'import json;print(json.load(open("models/active/production.json"))["version"])')" + for id in $FIRST_RUNS; do + v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')") + [ -n "$v" ] && [ -f "models/registry/$v/weights.pt" ] && FIRSTS="$FIRSTS $v" + done + for cid in $CROP_RUNS; do + cv=$(python3 -c "import json;print(json.load(open('runs/$cid/run.json')).get('version') or '')") + [ -n "$cv" ] && [ -f "models/registry/$cv/weights.pt" ] || { echo " $cid left no registry weights, not measured"; continue; } + for fv in $FIRSTS; do + say "two-pass $fv + $cv" + (cd "$STUDIO" && $PY -m src.pipeline.run measure --config pipeline.prod.cf.json --weights "$TRAIN/models/registry/$fv/weights.pt" --crop-weights "$TRAIN/models/registry/$cv/weights.pt" --out "data/pipeline_prod/gold/predictions/two_pass_${fv}+${cv}_cf" 2>&1 | tail -14 | cut -c1-220 || true) + done + done +fi + say "every model on the unseen gold frames" $PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true git add runs && git commit -q -m "unseen-frame comparison after cycle $TAG" || true