From acaf09483ef70c65edd24ba9580f391b81a77b97 Mon Sep 17 00:00:00 2001
From: GotthardG <51994228+GotthardG@users.noreply.github.com>
Date: Tue, 29 Sep 2026 16:53:44 +0200
Subject: [PATCH] slurm: base weights via LFS, and the cluster run
 2026-09-29_v2-noflip-mask1 spelled out

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 slurm/README.md    | 33 +++++++++++++++++++++++++++++++++
 slurm/train.sbatch |  2 +-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/slurm/README.md b/slurm/README.md
index 985c54a6..e716766a 100644
--- a/slurm/README.md
+++ b/slurm/README.md
@@ -26,3 +26,36 @@ git commit -am "run <id>: exported" && git push
 sbatch slurm/train.sbatch <id>
 python tools/ledger.py show
 ```
+
+## The run that is waiting for the cluster
+
+`2026-09-29_v2-noflip-mask1` — dataset `dataset/2026-09-29_v2` (the whole reviewed pool as of
+2026-09-29, gold grown to 200 frames), default recipe: no horizontal flip, full-resolution
+masks, base weights `models/base/yolo26n-seg.pt` (in the repository under LFS). Its twin with
+the older settings A trains on the Spark for the comparison.
+
+Once, on the login node (fill the placeholders in `train.sbatch` first):
+
+```bash
+cd <shared filesystem>
+git clone git@gitea.psi.ch:mx/AareLC_train.git        # or https, with a token
+cd AareLC_train
+git lfs install
+git lfs pull -I "data/pool/images/**,data/gold/**,models/base/**"
+python -m venv .venv && . .venv/bin/activate && pip install ultralytics opencv-python-headless numpy
+```
+
+Then, for this run and every later one:
+
+```bash
+sbatch slurm/train.sbatch 2026-09-29_v2-noflip-mask1
+squeue -u $USER                                        # the job
+tail -f runs/2026-09-29_v2-noflip-mask1/train.log      # the training
+python tools/ledger.py show                            # the state
+```
+
+The job trains, measures the weights on the gold set, gates against the active model and
+promotes on a pass (`models/active/production.json`); it commits the run directory and the
+registry entry locally. Pushing needs a key on the cluster that Gitea accepts: until then,
+`git push origin main` from the login node by hand, or the commit stays there for someone
+to push.
diff --git a/slurm/train.sbatch b/slurm/train.sbatch
index 99686360..f75ea08f 100644
--- a/slurm/train.sbatch
+++ b/slurm/train.sbatch
@@ -28,7 +28,7 @@ git fetch -q --tags
 git pull -q --ff-only
 DATASET_TAG="$(python -c "import json;print(json.load(open('runs/$RUN_ID/run.json'))['dataset'])")"
 git checkout -q "$DATASET_TAG" -- data/datasets                # the dataset exactly as tagged
-git lfs pull -q -I "data/pool/images/**,data/gold/**"          # only the labelled pool and the gold set
+git lfs pull -q -I "data/pool/images/**,data/gold/**,models/base/**"   # the labelled pool, the gold set, the base weights
 
 python tools/ledger.py set "$RUN_ID" queued --kv "job=$SLURM_JOB_ID" "host=$(hostname)" "partition=$SLURM_JOB_PARTITION"
 $APPTAINER python tools/train_run.py "$RUN_ID" --device 0 --workers "$SLURM_CPUS_PER_TASK" 2>&1 | tee "runs/$RUN_ID/train.log"