spark cycle trains several model sizes in turn and times each one the way the server calls it
MODELS="n s m l" runs the promoted recipe with yolo26n, s, m and l one after the other; only the nano run may promote itself, the others are candidates until their speed at the beamline is known. Each training waits for MEM_FLOOR_GB of free memory, an out-of-memory run is tried again at batch 8, and vLLM is restarted at the end only if it was running. tools/bench_latency.py measures the answer time per frame as the live server asks for it (frame scaled by infer_scale, letterboxed to 640, PyTorch fp16) and writes runs/<id>/latency.json. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
jungfrauandClaude Fable 5.1 committed
1 parent
ddaa5a99e6
commit
20fb1d03fb
2 files changed
+143
-12
No files matched your search
@@ -0,0 +1,85 @@
|
||||
"""How fast does a model answer the way the live server asks it?
|
||||
|
||||
The server scales the 2040 px camera frame by ``infer_scale`` (0.5 in production) and
|
||||
hands it to the model, which letterboxes it to ``imgsz`` 640. This times exactly that call
|
||||
in PyTorch on this machine (the beamline runs a TensorRT engine on its own GPU, so the
|
||||
absolute number differs there; the ratio between models carries over):
|
||||
|
||||
python tools/bench_latency.py runs/<id>/ultralytics/train/weights/best.pt --out runs/<id>/latency.json
|
||||
|
||||
Prints and writes one JSON line: median and p90 milliseconds per frame, frames per second.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("weights")
|
||||
ap.add_argument("--images", default=str(ROOT / "data" / "gold" / "images"), help="frames to time on")
|
||||
ap.add_argument("--frames", type=int, default=150)
|
||||
ap.add_argument("--warmup", type=int, default=20)
|
||||
ap.add_argument("--imgsz", type=int, default=640)
|
||||
ap.add_argument("--infer-scale", type=float, default=0.5)
|
||||
ap.add_argument("--device", default="0")
|
||||
ap.add_argument("--half", action="store_true", default=True)
|
||||
ap.add_argument("--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
from ultralytics import YOLO
|
||||
from ultralytics.utils import LOGGER
|
||||
LOGGER.setLevel("ERROR") # 'half' is deprecated in 8.4 but still does the job; no warning per frame
|
||||
|
||||
paths = sorted(Path(args.images).glob("*.jpg"))[: args.frames + args.warmup]
|
||||
if len(paths) < args.warmup + 10:
|
||||
sys.exit(f"not enough frames under {args.images}")
|
||||
frames = []
|
||||
for p in paths:
|
||||
img = cv2.imread(str(p))
|
||||
if img is None:
|
||||
continue
|
||||
if args.infer_scale < 1.0:
|
||||
img = cv2.resize(img, (max(1, int(img.shape[1] * args.infer_scale)), max(1, int(img.shape[0] * args.infer_scale))),
|
||||
interpolation=cv2.INTER_AREA)
|
||||
frames.append(img)
|
||||
model = YOLO(args.weights)
|
||||
kw = dict(imgsz=args.imgsz, device=args.device, half=args.half, conf=0.25, verbose=False)
|
||||
for img in frames[: args.warmup]:
|
||||
model.predict(img, **kw)
|
||||
times = []
|
||||
for img in frames[args.warmup:]:
|
||||
t = time.perf_counter()
|
||||
model.predict(img, **kw)
|
||||
times.append((time.perf_counter() - t) * 1000.0)
|
||||
times.sort()
|
||||
med = statistics.median(times)
|
||||
out = {"weights": str(args.weights), "frames": len(times), "imgsz": args.imgsz, "infer_scale": args.infer_scale,
|
||||
"half": bool(args.half), "ms_median": round(med, 2), "ms_p90": round(times[int(0.9 * (len(times) - 1))], 2),
|
||||
"ms_max": round(times[-1], 2), "fps_median": round(1000.0 / med, 1), "device": args.device,
|
||||
"backend": "pytorch"}
|
||||
try:
|
||||
import torch
|
||||
out["gpu"] = torch.cuda.get_device_name(0)
|
||||
import ultralytics
|
||||
out["ultralytics"] = ultralytics.__version__
|
||||
out["torch"] = torch.__version__
|
||||
except Exception:
|
||||
pass
|
||||
line = json.dumps(out)
|
||||
print(line)
|
||||
if args.out:
|
||||
Path(args.out).write_text(line + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+58
-12
@@ -2,29 +2,39 @@
|
||||
# One training cycle on the Spark, start to finish, unattended.
|
||||
#
|
||||
# nohup bash ~/AareLC_train/tools/spark_cycle.sh v5 > ~/cycle_v5.log 2>&1 &
|
||||
# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first
|
||||
# bash ~/AareLC_train/tools/spark_cycle.sh v5 --quick # also the fast settings-A run first
|
||||
# MODELS="n s m l" bash ~/AareLC_train/tools/spark_cycle.sh v5 # the promoted recipe at several model sizes
|
||||
#
|
||||
# Steps: wait for a running pipeline batch; grow the gold set by its rule (studio `gold`
|
||||
# stage: gold batches, share, floors); export the reviewed pool as dataset <tag> into this
|
||||
# repo; stop vLLM; re-measure every registry model on the grown gold set so the gate and
|
||||
# the comparison stand on the same frames; train the promoted recipe from the base weights;
|
||||
# gate (paired bootstrap against the active model), promote on a clear win; compare every
|
||||
# model on the unseen frames; push; start vLLM again.
|
||||
# the comparison stand on the same frames; train the promoted recipe from the base weights,
|
||||
# one model size after the other; gate each (paired bootstrap against the active model);
|
||||
# only the nano run may promote itself, a bigger model is a candidate until its speed at the
|
||||
# beamline is known (its latency on this machine is measured and kept next to the run);
|
||||
# compare every model on the unseen frames; push; start vLLM again if it was running.
|
||||
#
|
||||
# What earlier cycles taught, built in:
|
||||
# - a failed gate is a verdict, not a crash: train_run.py is called once per run;
|
||||
# - a full-resolution-mask training does not fit next to vLLM in 121 GB: vLLM is stopped
|
||||
# while it runs and started again on exit, whatever happens;
|
||||
# while the trainings run and started again on exit if it was up before;
|
||||
# - the trainings run one at a time and each waits for MEM_FLOOR_GB of free memory, so the
|
||||
# studio server, the inference workers and a reviewers' batch keep their room;
|
||||
# - an out-of-memory training is tried once more at half the batch size;
|
||||
# - torch's native JIT wants Python headers the Spark lacks: switched off, headers on CPATH;
|
||||
# - a grown gold set invalidates stored predictions: everything is re-measured first.
|
||||
set -eo pipefail
|
||||
TAG="${1:?usage: spark_cycle.sh <dataset tag, e.g. v5> [--quick]}"; QUICK="${2:-}"
|
||||
MODELS="${MODELS:-n}" # yolo26 sizes to train, e.g. "n s m l"
|
||||
MEM_FLOOR_GB="${MEM_FLOOR_GB:-40}" # a training starts only with this much memory available
|
||||
export PATH=$HOME/bin:$PATH
|
||||
export CPATH=$HOME/local/root/usr/include:$HOME/local/root/usr/include/python3.12
|
||||
export TORCH_DISABLE_NATIVE_JIT=1
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
PY=$HOME/AareLC_studio/.venv/bin/python
|
||||
STUDIO=$HOME/AareLC_studio; TRAIN=$HOME/AareLC_train; CFG=pipeline.prod.json
|
||||
say() { echo "== $(date +%H:%M) $*"; }
|
||||
avail_gb() { free -g | awk 'NR==2{print $7}'; }
|
||||
|
||||
while pgrep -f "[p]ipeline.run auto" >/dev/null; do sleep 30; done # the reviewers' batch goes first
|
||||
cd "$STUDIO"; set -a; . ~/.aaredb_env; set +a
|
||||
@@ -37,7 +47,19 @@ DS=$(git tag -l "dataset/*_$TAG" | tail -1); [ -n "$DS" ] || { echo "no dataset
|
||||
DSDIR="data/datasets/${DS#dataset/}"
|
||||
say "dataset $DS: $(ls $DSDIR/images/train | wc -l) train / $(ls $DSDIR/images/val | wc -l) val; gold $(ls data/gold/labels | wc -l) frames"
|
||||
|
||||
# Base weights for every size asked for (Ultralytics fetches a missing one from its releases).
|
||||
for size in $MODELS; do
|
||||
base="models/base/yolo26${size}-seg.pt"
|
||||
[ -f "$base" ] && continue
|
||||
say "fetching $base"
|
||||
(cd models/base && $PY -c "from ultralytics import YOLO; YOLO('yolo26${size}-seg.pt')" >/dev/null 2>&1) || { echo "!! could not fetch $base"; exit 1; }
|
||||
git add "$base" && git commit -q -m "base weights yolo26${size}-seg" || true
|
||||
done
|
||||
|
||||
VLLM_WAS_UP=0
|
||||
docker ps --format "{{.Names}}" | grep -q "^vllm$" && VLLM_WAS_UP=1
|
||||
vllm_back() {
|
||||
[ "$VLLM_WAS_UP" = 1 ] || { echo "$(date) vllm was not running before the cycle: left stopped" >> ~/vllm_restart.log; return 0; }
|
||||
for i in 1 2 3; do
|
||||
docker start vllm >> ~/vllm_restart.log 2>&1 || true; sleep 60
|
||||
if docker ps --format "{{.Names}}" | grep -q "^vllm$"; then echo "$(date) vllm up (try $i)" >> ~/vllm_restart.log; return 0; fi
|
||||
@@ -46,7 +68,7 @@ vllm_back() {
|
||||
}
|
||||
trap vllm_back EXIT
|
||||
docker stop vllm >/dev/null 2>&1 || true
|
||||
say "vLLM stopped, $(free -g | awk 'NR==2{print $7}') GB available"
|
||||
say "vLLM stopped (was up: $VLLM_WAS_UP), $(avail_gb) GB available"
|
||||
|
||||
say "re-measuring every registry model on the $(ls data/gold/labels | wc -l) gold frames"
|
||||
for d in models/registry/*/; do
|
||||
@@ -57,24 +79,48 @@ BEAMLINE_PT=$STUDIO/models/best_yolo26n-seg-overlap-false_2026-04-12.pt # the
|
||||
[ -f "$BEAMLINE_PT" ] && $PY tools/measure_weights.py "$BEAMLINE_PT" --version yolo26n-seg-overlap-false-2026-04-12 2>&1 | tail -n 1
|
||||
git add models/registry data/gold && git commit -q -m "gold grown to $(ls data/gold/labels | wc -l) frames; every registry model re-measured on it" || true
|
||||
|
||||
run() { # tag suffix, note, extra train_run.py args...
|
||||
local suffix="$1" note="$2"; shift 2
|
||||
local id; id=$(python3 tools/ledger.py new --dataset "$DS" --tag "$TAG-$suffix" --base models/base/yolo26n-seg.pt --note "$note")
|
||||
wait_for_memory() { # up to two hours for the floor, then go anyway and say so
|
||||
local waited=0
|
||||
while [ "$(avail_gb)" -lt "$MEM_FLOOR_GB" ] && [ $waited -lt 7200 ]; do sleep 60; waited=$((waited + 60)); done
|
||||
[ "$(avail_gb)" -ge "$MEM_FLOOR_GB" ] || echo " only $(avail_gb) GB available after two hours; starting anyway"
|
||||
}
|
||||
run() { # size, tag suffix, note, extra train_run.py args...
|
||||
local size="$1" suffix="$2" note="$3"; shift 3
|
||||
local base="models/base/yolo26${size}-seg.pt"
|
||||
local id; id=$(python3 tools/ledger.py new --dataset "$DS" --tag "$TAG-$suffix" --base "$base" --note "$note")
|
||||
git add runs && git commit -q -m "run $id: exported on $DS" || true
|
||||
say "training $id"
|
||||
wait_for_memory
|
||||
say "training $id from $base ($(avail_gb) GB available)"
|
||||
set +e
|
||||
$PY tools/train_run.py "$id" --device 0 --workers 8 "$@" > "runs/$id/train.log" 2>&1 < /dev/null
|
||||
local rc=$? # 0 = gate passed (or no baseline), 2 = gate failed: both are verdicts
|
||||
if [ $rc -ne 0 ] && [ $rc -ne 2 ] && grep -aqi "out of memory" "runs/$id/train.log"; then
|
||||
say "$id ran out of memory at batch 16; once more at batch 8"
|
||||
sleep 60
|
||||
$PY tools/train_run.py "$id" --device 0 --workers 8 --batch 8 --force "$@" > "runs/$id/train.log" 2>&1 < /dev/null
|
||||
rc=$?
|
||||
fi
|
||||
set -e
|
||||
[ $rc -eq 0 ] || [ $rc -eq 2 ] || echo "!! $id ended with exit code $rc: $(grep -aiE "error|out of memory" runs/$id/train.log | tail -1 | cut -c1-160)"
|
||||
local v; v=$(python3 -c "import json;print(json.load(open('runs/$id/run.json')).get('version') or '')")
|
||||
if [ -n "$v" ] && [ -d "runs/$id/gold_predictions" ]; then mkdir -p "runs/_measure_$v" && cp -r "runs/$id/gold_predictions" "runs/_measure_$v/"; fi
|
||||
python3 -c "import json; m=json.load(open('runs/$id/metrics.json')); g=m.get('gate') or {}; b=g.get('bootstrap') or {}; print(' gate:', g.get('pass'), '| rule:', g.get('rule'), '| F1', g.get('f1_delta'), 'mask', round(g.get('mask_score',0)-g.get('mask_score_baseline',0),4), '| win F1', (b.get('f1') or {}).get('win'), 'win mask', (b.get('mask_score') or {}).get('win'))" 2>/dev/null || true
|
||||
# the answer time the live server would see, in PyTorch on this machine (TensorRT at the beamline is faster)
|
||||
local best="runs/$id/ultralytics/train/weights/best.pt"
|
||||
if [ -f "$best" ]; then
|
||||
$PY tools/bench_latency.py "$best" --out "runs/$id/latency.json" 2>/dev/null | sed 's/^/ latency: /' | cut -c1-200 || true
|
||||
fi
|
||||
python3 tools/ledger.py render >/dev/null 2>&1 || true
|
||||
git add runs models && git commit -q -m "run $id: $(python3 -c "import json;print(json.load(open('runs/$id/run.json'))['status'])")" || true
|
||||
}
|
||||
[ "$QUICK" = "--quick" ] && run recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4
|
||||
run noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1
|
||||
[ "$QUICK" = "--quick" ] && run n recipeA "settings A (fliplr 0.5, mask_ratio 4) on dataset $TAG: quick signal, gate only, Spark" --no-promote --set fliplr=0.5 mask_ratio=4
|
||||
for size in $MODELS; do
|
||||
if [ "$size" = "n" ]; then
|
||||
run n noflip-mask1 "promoted recipe (fliplr 0, mask_ratio 1) on dataset $TAG: candidate for the beamlines, Spark" --set fliplr=0 mask_ratio=1
|
||||
else
|
||||
run "$size" "${size}-noflip-mask1" "promoted recipe (fliplr 0, mask_ratio 1) with yolo26${size} on dataset $TAG: gate only until its beamline speed is known, Spark" --no-promote --set fliplr=0 mask_ratio=1
|
||||
fi
|
||||
done
|
||||
|
||||
say "every model on the unseen gold frames"
|
||||
$PY tools/compare_unseen.py 2>&1 | cut -c1-200 || true
|
||||
|
||||
Reference in new issue
Block a user