diff --git a/.gitea/workflows/battery.yml b/.gitea/workflows/battery.yml new file mode 100644 index 000000000..398babefc --- /dev/null +++ b/.gitea/workflows/battery.yml @@ -0,0 +1,136 @@ +name: Validation battery + +# Every push runs the ci tier of tools/battery (25 sets, about 10 minutes of rugnux on an RTX 5080, +# plus the build); the full battery (open + inhouse arms, about 2 hours of rugnux there) runs only +# when started by hand. Both run on the one machine that holds the battery data, one at a time (the +# concurrency group below), under their own runner label, with the data mounted read-only and +# /data/battery (scratch, kept reports, PDB/COD caches, the full-run lock) read-write. The site +# config below names only the open and inhouse arms; the private arm is never run here. +# +# The job fails when a ci-tier set fails on lattice or space group, crashes or has no input - in any +# run that includes those sets, so a manual full run fails on them too. Every other failure and every +# metric is reported, not judged: read the table at the end of the battery step and the report. +# Only each run's report.md and battery.log are kept: report.md in /data/battery/ci_reports, both as +# the job's artifact; the run directory itself is scratch. +# +# Runner side (/opt/gitea/config.yaml on that machine, then restart the runner): +# runner.labels: + "jfjoch_battery:docker://gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609" +# runner.timeout: 12h (the default 3h kills a full battery) +# container.valid_volumes: ["/data/open", "/data/inhouse", "/data/battery", "/opt/xtal"] +# CCP4 (SHELXL for the small-molecule check, REFMAC for --model-check) installed at /opt/xtal/ccp4-9 + +on: + push: + branches: + - '**' + workflow_dispatch: + inputs: + subset: + description: 'Empty = full battery (open + inhouse); or e.g. "--tier ci", "--tier smoke", "--only 5reo,lyso_x06da_ref"' + required: false + default: '' + label: + description: 'Run label (part of the run directory name)' + required: false + default: 'manual' + +env: + # Same flags as the build workflow: CPU-side timings depend on them (see CLAUDE.md). + LINUX_CMAKE_FLAGS: '-DCMAKE_CXX_FLAGS="-march=x86-64-v3 -flto=auto" -DCMAKE_C_FLAGS="-march=x86-64-v3 -flto=auto"' + +concurrency: + group: battery + cancel-in-progress: false + +jobs: + battery: + name: Validation battery + runs-on: jfjoch_battery + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609 + # nvidia-smi inside the container still counts every process on the GPU (host and other + # containers), so the battery's gpu_others / "timing NOT a reference" marking keeps working. + options: --gpus all + volumes: + - /data/open:/data/open:ro + - /data/inhouse:/data/inhouse:ro + - /data/battery:/data/battery + - /opt/xtal:/opt/xtal:ro + timeout-minutes: 720 + env: + # A push runs the ci tier; a manual run takes what it was given (empty = full battery). + SUBSET: ${{ github.event_name == 'push' && '--tier ci' || github.event.inputs.subset }} + LABEL: ${{ github.event_name == 'push' && 'ci' || github.event.inputs.label }} + steps: + - uses: actions/checkout@v4 + - name: Build rugnux + shell: bash + run: | + rm -rf build && mkdir build && cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release ${{ env.LINUX_CMAKE_FLAGS }} .. + ninja -j16 rugnux + - name: Python for the scorer + shell: bash + run: | + # the image's python3 is 3.6, too old for gemmi + dnf -y -q install python3.12 python3.12-pip + python3.12 -m venv /tmp/bvenv + /tmp/bvenv/bin/pip install -q gemmi h5py + - name: Site config + shell: bash + run: | + # The run directory (merged files, logs, results.json) is scratch, deleted after the job; + # only report.md is kept, in /data/battery/ci_reports and as the job's artifact. + cat > /tmp/site.json <&1 | tee battery.log + - name: Keep the report + if: always() + shell: bash + run: | + run=$(sed -n 's/^run directory //p' battery.log | head -1) + [ -n "$run" ] && [ -f "$run/report.md" ] || { echo "no report"; exit 1; } + mkdir -p battery_out /data/battery/ci_reports + cp "$run/report.md" "battery_out/$(basename "$run").md" + cp battery.log "battery_out/$(basename "$run").log" + cp "$run/report.md" "/data/battery/ci_reports/$(basename "$run").md" + # the report's own per-arm summary and failures, into the job log + awk '/^## Summary/{p=1} /^### By population/{p=0} p' "$run/report.md" + awk '/^### Failures/{p=1;print;next} /^### /{p=0} p' "$run/report.md" + - name: Upload report + if: always() + uses: actions/upload-artifact@v3 + with: + name: battery-report + path: battery_out/ + - name: Delete the run directory + if: always() + shell: bash + run: | + # a finished run directory is read-only; the container runs as root, which deletes it anyway + rm -rf "/data/battery/scratch/${{ github.run_id }}" + # files written by root in the container go back to the owner of the battery tree + chown -R "$(stat -c %u:%g /data/battery)" /data/battery/ci_reports /data/battery/pdb_cache /data/battery/cod_cache 2>/dev/null || true diff --git a/tools/battery/README.md b/tools/battery/README.md index d7763accb..4ab385405 100644 --- a/tools/battery/README.md +++ b/tools/battery/README.md @@ -134,6 +134,9 @@ the reference: **Timing.** Each set runs once, so every time includes reading the images from disk (unless they were still in the page cache from before the run). +The in-house standard proteins run with `--model` since rc174, so their times include placing +the model; an earlier run (rc173 and before) did not, and a time delta on that arm against one is +partly that, not a change of speed. Compare in-house times between runs on the same side of it. The private arm is never mixed with the others. A private run gets its own run directory (suffix `_private`) and its own report (`report_PRIVATE.*`), and it cannot be the baseline of a @@ -225,7 +228,8 @@ B=tools/battery/battery.py R=build/rugnux/rugnux $B run --rugnux $R # full battery: open + inhouse -$B run --rugnux $R --tier smoke # 22 sets covering readers, symmetry, known hard cases +$B run --rugnux $R --tier smoke # 21 sets covering readers, symmetry, known hard cases +$B run --rugnux $R --tier ci # 25 sets every recent release passed (CI, below) $B run --rugnux $R --only 5reo,lyso_ref # named sets $B run --rugnux $R --arm private # the private arm, on its own ``` @@ -239,8 +243,34 @@ Useful options: - `--baseline RUN|none` sets the run the report is compared against. - `--model-check` adds the REFMAC check to the open arm's rows (off by default, see below). -A run prints one line per set and ends by writing its report. The **smoke** tier is marked in the -manifests (`"tiers": {"smoke": ""}`). +A run prints one line per set as it goes and ends by writing its report and printing a table, one +line per set, each column rugnux / reference: the space group with ✓ where the set passed by the +scorer's rule and ✗ where it failed (lattice, symmetry, crash, no input), d_min, ISa, CC1/2 overall +and in the highest shell, R_meas of the lowest shell, R-free of the model (rugnux's placement / +the model's published) and rugnux's wall time. On the open arm only the space group, d_min and +R-free have a reference. `$B table RUN` prints it again for any finished run. + +**Tiers** are marked in the manifests (`"tiers": {"": ""}`): + +- **smoke** covers the readers, the symmetry classes and some known hard cases; several of its + sets fail on purpose. +- **ci** is the routine regression check: 25 sets that passed with the same space group and steady + d_min and ISa in every full run of rc174 and rc175, with no warning that questions the lattice or + the group (supercell, harmonic, twinning, untested centring). It spans HDF5, miniCBF, cbf.gz, + marCCD (both formats) and SMV; every crystal system, centred and high-symmetry lattices, small + molecules, anomalous and long-wavelength data, pink beam, a twinnable P31 and one large set + (8qaw, about 2 minutes) - about 10 minutes of rugnux in all on the RTX 5080. A set that starts + failing for a known, accepted reason leaves the tier; one that is merely slow stays. + +**Exit status.** `run` exits with status 1 when a ci-tier set in the run failed - a wrong lattice +or space group, a crash, a timeout or a missing input - and names those sets. This holds in any +run that includes them (`--tier ci`, a full run, `--only` naming one), so a red CI job means the +same thing whichever run it started. Nothing else fails a run: other sets' failures and every +metric are reported, not judged. + +**CI.** `.gitea/workflows/battery.yml` builds rugnux with the CI flags and runs the ci tier on every +push, or by hand any subset (empty: the full open + inhouse battery), on the machine that holds the +data. The job keeps report.md and the log as its artifact; its site config names no private arm. **Full runs.** A run without `--only` or `--tier` takes the site lock (`flock`, non-blocking), so two full runs cannot overlap. Do not start one without the owner's go-ahead: it takes hours, and @@ -433,6 +463,7 @@ was run `--unforced`), so its forced XDS-arm rows are left out the same way. | `refres_range`, `refres_shells_past_limit`, `refres_unique_reflections`, `refres_completeness`, `refres_multiplicity`, `refres_i_over_sigma`, `refres_r_meas`, `refres_cc_half`, `refres_isa` | XDS arms: the REFRES_* keys, the same merge over the reference range | | `isa_ratio`, `r_meas_ratio` | refres_isa / isa_ref and refres_r_meas / r_meas_ref | | `cc_half_noise_ratio` | XDS arms: (1/refres_cc_half - 1) / (1/(cc_half_ref - 0.0005) - 1), reported, not scored | +| `cc_half_high`, `cc_half_high_ref` | CC1/2 of the highest-resolution shell: rugnux's own table, XDS's (`cc_half_high` in the manifest's `ref`), each at its own limit | | `lowres_d`, `lowres_r_meas`, `refres_lowres_d`, `refres_lowres_r_meas`, `lowres_d_ref`, `lowres_r_meas_ref`, `lowres_r_meas_ratio` | the lowest-resolution shell's high-resolution limit and R_meas: rugnux's own table, the reference-range table, XDS's; and refres_lowres_r_meas / lowres_r_meas_ref | | `model`, `model_note` | open arm, and in-house sets with a `model`: the coordinates given to `--model`, or why there were none | | `rfree`, `rwork`, `cc_model`, `model_fit`, `rfree_deposited`, `rfree_ratio` | any row with a model: R_FREE, R_WORK, CC_MODEL_OVERALL and MODEL_FIT as rugnux reports them (placement-only, own free set: trend fields), the published R-free, and rfree / rfree_deposited | diff --git a/tools/battery/battery.py b/tools/battery/battery.py index 98511e6c3..8760b81cc 100644 --- a/tools/battery/battery.py +++ b/tools/battery/battery.py @@ -11,12 +11,13 @@ Arms and where their truth comes from: site config) and a private run gets its own run directory and report. Usage (README.md in this directory is the full guide): - battery.py run --rugnux BIN [--label L] [--arm open,inhouse|private] [--only a,b | --tier smoke] + battery.py run --rugnux BIN [--label L] [--arm open,inhouse|private] [--only a,b | --tier smoke|ci] [--build-dir DIR] [--gpulock] [--threads N] [--model-check] [--baseline RUN|none] battery.py list runs under the runs root, with their state battery.py compare RUN_A RUN_B [--all] [--rerun-changed] battery.py report RUN --out DIR [--baseline RUN] re-render a run's report into DIR/.{md,html} + battery.py table RUN the one-line-per-set table a run ends with battery.py abort RUN [--reason TEXT] mark an unfinished run as aborted battery.py recheck RUN [--only a,b] [--jobs N] rerun the model checks of a finished run's open arm on its p.mtz files; recorded in the manifest @@ -37,6 +38,7 @@ A run writes /__