add ra train script

slurm/ra.train.sbatch

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Train one ledger run on the GPU cluster. Placeholders __LIKE_THIS__ are filled in
+# once the cluster facts are known (see slurm/README.md).
+#
+#   sbatch slurm/train.sbatch <run_id> [extra train_run.py args]   # run created with: python tools/ledger.py new ...
+#
+#SBATCH --job-name=aarelc-train
+#SBATCH --partition=gpu
+#SBATCH --gres=gpu:1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=128G
+#SBATCH --time=3:00:00                 # e.g. 08:00:00
+#SBATCH --output=/das/home/duan_j/p21515/AareLC_train/runs/slurm-%j.out
+set -euo pipefail
+
+RUN_ID="${1:?usage: sbatch slurm/train.sbatch <run_id> [train_run.py args, e.g. --batch 8]}"
+shift
+REPO="${AARELC_TRAIN_REPO:-/das/home/duan_j/p21515/AareLC_train}"   # a checkout on the shared filesystem
+
+# --- environment: keep exactly one of the three ---------------------------------
+# module load __MODULES__                                   # e.g. cuda/12 python/3.12
+source /das/work/p21/p21515/AareLC_train/.venv/bin/activate
+# APPTAINER="apptainer exec --nv __CONTAINER__"             # container with CUDA + ultralytics
+#APPTAINER="${APPTAINER:-}"
+
+cd "$REPO"
+git fetch -q --tags
+git pull -q --ff-only
+DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")"
+git checkout -q "$DATASET_TAG" -- data/datasets                # the dataset exactly as tagged
+git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**" >/dev/null
+
+python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION"
+python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" "$@" 2>&1 | tee "runs/$RUN_ID/train.log"
+STATUS=${PIPESTATUS[0]}
+
+# --- report back: commit the run, the registry entry and, on a pass, the promotion --
+# git add "runs/$RUN_ID" models
+if ! git diff --cached --quiet; then
+    git commit -q -m "run $RUN_ID: $(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['status'])")"
+fi
+# git push -q origin HEAD:main   # needs a deploy key on the cluster (__DEPLOY_KEY__)
+exit "$STATUS"