diff --git a/slurm/train.sbatch.merlin b/slurm/train.sbatch.merlin new file mode 100644 index 0000000..25d3697 --- /dev/null +++ b/slurm/train.sbatch.merlin @@ -0,0 +1,46 @@ +#!/bin/bash +# Train one ledger run on the GPU cluster. Placeholders __LIKE_THIS__ are filled in +# once the cluster facts are known (see slurm/README.md). +# +# sbatch slurm/train.sbatch [extra train_run.py args] # run created with: python tools/ledger.py new ... +# +#SBATCH --job-name=aarelc-train +#SBATCH --cluster=gmerlin7 +#SBATCH --partition=a100-general # mx +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=128G +#SBATCH --time=3:00:00 # e.g. 08:00:00 +#SBATCH --output=/data/user/duan_j/AareLC_train/runs/slurm-%j.out +set -euo pipefail + +RUN_ID="${1:?usage: sbatch slurm/train.sbatch [train_run.py args, e.g. --batch 8]}" +shift +REPO="${AARELC_TRAIN_REPO:-/data/user/duan_j/AareLC_train}" # a checkout on the shared filesystem + +# --- environment: keep exactly one of the three --------------------------------- +# module load __MODULES__ # e.g. cuda/12 python/3.12 +module purge +module load git-lfs +source /data/user/duan_j/.venv/dino-yolo/bin/activate # venv with ultralytics + opencv +# APPTAINER="apptainer exec --nv __CONTAINER__" # container with CUDA + ultralytics +#APPTAINER="${APPTAINER:-}" + +cd "$REPO" +git fetch -q --tags +git pull -q --ff-only +DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")" +git checkout -q "$DATASET_TAG" -- data/datasets # the dataset exactly as tagged +git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**" >/dev/null + +python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION" +python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log" +STATUS=${PIPESTATUS[0]} + +# --- report back: commit the run, the registry entry and, on a pass, the promotion -- +# git add "runs/$RUN_ID" models +if ! git diff --cached --quiet; then + git commit -q -m "run $RUN_ID: $(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['status'])")" +fi +# git push -q origin HEAD:main # needs a deploy key on the cluster (__DEPLOY_KEY__) +exit "$STATUS"