spark_cycle.sh: crop second-pass models (CROP_MODELS), a training order (ORDER) and the two-pass measure on the gold
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01JPZJw4gJgQmPzTa6ix7eGp
This commit is contained in:
1 parent
67a694c9e7
commit
f8e7da9732
1 file changed
+74
-7
+74
-7
@@ -4,6 +4,8 @@
|
||||
# nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 &
|
||||
# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first
|
||||
# MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes
|
||||
# MODELS="s n" CROP_MODELS="s m" ORDER="s crop-s n crop-m" bash ~/AareLC_train/tools/spark_cycle.sh v7
|
||||
# # also the loop-head crop models (studio second pass) on crops of the same pool, in that order
|
||||
#
|
||||
# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold`
|
||||
# stage: gold batches, share, floors); export the reviewed pool as dataset <tag> into this
|
||||
@@ -13,6 +15,10 @@
|
||||
# only the nano run may promote itself, a bigger model is a candidate until its speed at the
|
||||
# beamline is known (its latency on this machine is measured and kept next to the run);
|
||||
# compare every model on the unseen frames; push; start vLLM again if it was running.
|
||||
# With CROP_MODELS the pool is also exported as loop-head crops (dataset <tag>crop) and a crop
|
||||
# model is trained per size without a gate (the gold cannot score a crop model alone); at the
|
||||
# end every crop model is measured as a two-pass on the gold, behind each first-pass model of
|
||||
# this cycle and behind the active model. ORDER interleaves the trainings ("s crop-s n crop-m").
|
||||
#
|
||||
# What earlier cycles taught, built in:
|
||||
# - a failed gate is a verdict, not a crash: train_run.py is called once per run;
|
||||
@@ -26,6 +32,9 @@
|
||||
set -eo pipefail
|
||||
TAG="${1:?usage: spark_cycle.sh <dataset tag, e.g. v5> [--quick]}"; QUICK="${2:-}"
|
||||
MODELS="${MODELS:-n}" # yolo26 sizes to train, e.g. "n s m l"
|
||||
CROP_MODELS="${CROP_MODELS:-}" # loop-head crop model sizes for the studio second pass, e.g. "s m"
|
||||
ORDER="${ORDER:-}" # training order, e.g. "s crop-s n crop-m" (default: MODELS, then the crop models)
|
||||
FIRST_RUNS=""; CROP_RUNS=""
|
||||
MEM_FLOOR_GB="${MEM_FLOOR_GB:-40}" # a training starts only with this much memory available
|
||||
export PATH=$HOME/bin:$PATH
|
||||
export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12
|
||||
@@ -46,9 +55,17 @@ cd "$TRAIN"
|
||||
DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset tag for $TAG"; exit 1; }
|
||||
DSDIR="data/datasets/${DS#dataset/}"
|
||||
say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames"
|
||||
CDS=""
|
||||
if [ -n "$CROP_MODELS" ]; then
|
||||
say "export --crop + import into AareLC_train (dataset ${TAG}crop: crops around the labelled heads, gold and held-out frames excluded)"
|
||||
(cd "$STUDIO" && $PY -m src.pipeline.run export --config $CFG --crop --dest "data/pipeline_prod/dataset_crop_$(date +%F)_${TAG}crop" --train-repo "$TRAIN" --train-tag "${TAG}crop" 2>&1 | grep -E "^export|dataset |committed|tagged|crop" | cut -c1-220 || true)
|
||||
CDS=$(git tag -l "dataset/*_${TAG}crop" | tail -1); [ -n "$CDS" ] || { echo "no dataset tag for ${TAG}crop"; exit 1; }
|
||||
CDSDIR="data/datasets/${CDS#dataset/}"
|
||||
say "crop dataset $CDS: $(ls $CDSDIR/images/train | wc -l) train / $(ls $CDSDIR/images/val | wc -l) val"
|
||||
fi
|
||||
|
||||
# Base weights for every size asked for (Ultralytics fetches a missing one from its releases).
|
||||
for size in $MODELS; do
|
||||
for size in $MODELS $CROP_MODELS; do
|
||||
base="models/base/yolo26${size}-seg.pt"
|
||||
[ -f "$base" ] && continue
|
||||
say "fetching $base"
|
||||
@@ -112,16 +129,66 @@ run() { # size, tag suffix, note, extra train_run.py args...
|
||||
fi
|
||||
python3 tools/ledger.py render >/dev/null 2>&1 || true
|
||||
git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
|
||||
FIRST_RUNS="$FIRST_RUNS $id"
|
||||
}
|
||||
run_crop() { # size: the loop-head crop model for the studio second pass, trained on the crop dataset, no gold gate
|
||||
local size="$1"
|
||||
local base="models/base/yolo26${size}-seg.pt"
|
||||
local id; id=$(python3 tools/ledger.py new --dataset "$CDS" --tag "$TAG-crop-$size" --base "$base" --note "loop-head crop model (yolo26$size) for the studio second pass on the $TAG pool: crops 1.6x the face, min 512 px, head classes only; promoted recipe; no gold gate, measured as a two-pass behind the first-pass models, Spark")
|
||||
git add runs && git commit -q -m "run $id: exported on $CDS" || true
|
||||
wait_for_memory
|
||||
say "training $id from $base ($(avail_gb) GB available)"
|
||||
set +e
|
||||
$PY tools/train_run.py "$id" --device 0 --workers 8 --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null
|
||||
local rc=$?
|
||||
if [ $rc -ne 0 ] && grep -aqi "out of memory" "runs/$id/train.log"; then
|
||||
say "$id ran out of memory at batch 16; once more at batch 8"
|
||||
sleep 60
|
||||
$PY tools/train_run.py "$id" --device 0 --workers 8 --batch 8 --force --no-gate --set fliplr=0 mask_ratio=1 > "runs/$id/train.log" 2>&1 < /dev/null
|
||||
rc=$?
|
||||
fi
|
||||
set -e
|
||||
[ $rc -eq 0 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)"
|
||||
local best="runs/$id/ultralytics/train/weights/best.pt"
|
||||
if [ -f "$best" ]; then
|
||||
$PY tools/bench_latency.py "$best" --out "runs/$id/latency.json" 2>/dev/null | sed 's/^/ latency: /' | cut -c1-200 || true
|
||||
fi
|
||||
python3 tools/ledger.py render >/dev/null 2>&1 || true
|
||||
git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
|
||||
CROP_RUNS="$CROP_RUNS $id"
|
||||
}
|
||||
[ "$QUICK" = "--quick" ] && run n recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4
|
||||
for size in $MODELS; do
|
||||
if [ "$size" = "n" ]; then
|
||||
run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1
|
||||
else
|
||||
run "$size" "${size}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${size} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1
|
||||
fi
|
||||
if [ -z "$ORDER" ]; then
|
||||
ORDER="$MODELS"; for size in $CROP_MODELS; do ORDER="$ORDER crop-$size"; done
|
||||
fi
|
||||
for job in $ORDER; do
|
||||
case "$job" in
|
||||
crop-*) run_crop "${job#crop-}" ;;
|
||||
n) run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1 ;;
|
||||
*) run "$job" "${job}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${job} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -n "$CROP_RUNS" ]; then
|
||||
# Every crop model as a two-pass on the gold (crop objects of crystal and loop_face replace the
|
||||
# first pass's), behind the active model and behind each first-pass model of this cycle.
|
||||
say "two-pass measure on the gold frames (replace crystal+loop_face)"
|
||||
python3 -c "import json; c=json.load(open('$STUDIO/$CFG')); c['headcrop']={'classes':['crystal','loop_face']}; json.dump(c, open('$STUDIO/pipeline.prod.cf.json','w'), indent=1)"
|
||||
FIRSTS="$(python3 -c 'import json;print(json.load(open("models/active/production.json"))["version"])')"
|
||||
for id in $FIRST_RUNS; do
|
||||
v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')")
|
||||
[ -n "$v" ] && [ -f "models/registry/$v/weights.pt" ] && FIRSTS="$FIRSTS $v"
|
||||
done
|
||||
for cid in $CROP_RUNS; do
|
||||
cv=$(python3 -c "import json;print(json.load(open('runs/$cid/run.json')).get('version') or '')")
|
||||
[ -n "$cv" ] && [ -f "models/registry/$cv/weights.pt" ] || { echo " $cid left no registry weights, not measured"; continue; }
|
||||
for fv in $FIRSTS; do
|
||||
say "two-pass $fv + $cv"
|
||||
(cd "$STUDIO" && $PY -m src.pipeline.run measure --config pipeline.prod.cf.json --weights "$TRAIN/models/registry/$fv/weights.pt" --crop-weights "$TRAIN/models/registry/$cv/weights.pt" --out "data/pipeline_prod/gold/predictions/two_pass_${fv}+${cv}_cf" 2>&1 | tail -14 | cut -c1-220 || true)
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
say "every model on the unseen gold frames"
|
||||
$PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true
|
||||
git add runs && git commit -q -m "unseen-frame comparison after cycle $TAG" || true
|
||||
|
||||
Reference in new issue
Block a user