diff --git a/.gitea/workflows/battery.yml b/.gitea/workflows/battery.yml new file mode 100644 index 000000000..86b04567c --- /dev/null +++ b/.gitea/workflows/battery.yml @@ -0,0 +1,142 @@ +name: Validation battery + +# Every push runs the ci tier of tools/battery (25 sets, about 10 minutes of rugnux on the RTX 5080, +# plus the build); the full battery (open + inhouse arms, about 2 hours of rugnux) runs only when +# started by hand. Both run on the one machine that holds the battery data, one at a time (the +# concurrency group below), under their own runner label. The private arm is never mounted or run. +# +# The job fails when a ci-tier set fails on lattice or space group, crashes or has no input - in any +# run that includes those sets, so a manual full run fails on them too. Every other failure and every +# metric is reported, not judged: read the table at the end of the battery step and the report. +# Only each run's report.md and battery.log are kept: in /data/battery/ci/reports and as the job's +# artifact; the run directory itself is scratch. +# +# Runner side (/opt/gitea/config.yaml on that machine, then restart the runner): +# runner.labels: + "jfjoch_battery:docker://gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609" +# runner.timeout: 12h (the default 3h kills a full battery) +# container.valid_volumes: ["/ssd/data/open", "/ssd/data/inhouse", "/data/battery/ci", +# "/data/battery/pdb_cache", "/data/battery/cod_cache", +# "/data/battery/full_run.lock", "/opt/xtal"] +# mkdir -p /data/battery/ci (scratch run directories and the kept reports) +# CCP4 at /opt/xtal/ccp4-9 (SHELXL for the small-molecule check, REFMAC for --model-check) + +on: + push: + branches: + - '**' + workflow_dispatch: + inputs: + subset: + description: 'Empty = full battery (open + inhouse); or e.g. "--tier ci", "--tier smoke", "--only 5reo,lyso_x06da_ref"' + required: false + default: '' + label: + description: 'Run label (part of the run directory name)' + required: false + default: 'manual' + +env: + # Same flags as the build workflow: CPU-side timings depend on them (see CLAUDE.md). + LINUX_CMAKE_FLAGS: '-DCMAKE_CXX_FLAGS="-march=x86-64-v3 -flto=auto" -DCMAKE_C_FLAGS="-march=x86-64-v3 -flto=auto"' + +concurrency: + group: battery + cancel-in-progress: false + +jobs: + battery: + name: Validation battery + runs-on: jfjoch_battery + container: + # The rocky8 image: its CUDA 12.9 runs on the machine's driver (580, CUDA 13.0); the rocky9 + # image's CUDA 13.3 would need a newer one. + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609 + # nvidia-smi inside the container still counts every process on the GPU (host and other + # containers), so the battery's gpu_others / "timing NOT a reference" marking keeps working. + options: --gpus all + # The data at the paths the site config below names; the lock file is the host's, so a full + # run here and one on the host cannot overlap. + volumes: + - /ssd/data/open:/ssd/data/open:ro + - /ssd/data/inhouse:/ssd/data/inhouse:ro + - /data/battery/ci:/data/battery/ci + - /data/battery/pdb_cache:/data/battery/pdb_cache + - /data/battery/cod_cache:/data/battery/cod_cache + - /data/battery/full_run.lock:/data/battery/full_run.lock + - /opt/xtal:/opt/xtal:ro + timeout-minutes: 720 + env: + # A push runs the ci tier; a manual run takes what it was given (empty = full battery). + SUBSET: ${{ github.event_name == 'push' && '--tier ci' || github.event.inputs.subset }} + LABEL: ${{ github.event_name == 'push' && 'ci' || github.event.inputs.label }} + steps: + - uses: actions/checkout@v4 + - name: Build rugnux + shell: bash + run: | + rm -rf build && mkdir build && cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. + ninja -j16 rugnux + - name: Python for the scorer + shell: bash + run: | + # the image's python3 is 3.6, too old for gemmi + dnf -y -q install python3.12 python3.12-pip + python3.12 -m venv /tmp/bvenv + /tmp/bvenv/bin/pip install -q gemmi h5py + - name: Site config + shell: bash + run: | + cat > /tmp/site.json <&1 | tee battery.log + - name: Keep the report + if: always() + shell: bash + run: | + run=$(sed -n 's/^run directory //p' battery.log | head -1) + [ -n "$run" ] && [ -f "$run/report.md" ] || { echo "no report"; exit 1; } + mkdir -p battery_out /data/battery/ci/reports + cp "$run/report.md" "battery_out/$(basename "$run").md" + cp battery.log "battery_out/$(basename "$run").log" + cp "$run/report.md" "/data/battery/ci/reports/$(basename "$run").md" + # the report's own per-arm summary and failures, into the job log + awk '/^## Summary/{p=1} /^### By population/{p=0} p' "$run/report.md" + awk '/^### Failures/{p=1;print;next} /^### /{p=0} p' "$run/report.md" + - name: Upload report + if: always() + uses: actions/upload-artifact@v3 + with: + name: battery-report + path: battery_out/ + - name: Delete the run directory + if: always() + shell: bash + run: | + # a finished run directory is read-only; the container runs as root, which deletes it anyway + rm -rf "/data/battery/ci/scratch/${{ github.run_id }}" + # files written by root in the container go back to the owner of the battery tree + chown -R "$(stat -c %u:%g /data/battery/ci)" /data/battery/ci /data/battery/pdb_cache /data/battery/cod_cache 2>/dev/null || true