From 01017c4bb7b5e411696afa2610b8d0a2fcd14054 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 06:11:40 +0100 Subject: [PATCH 01/19] FFTW removed: every CPU FFT is pocketfft's (common/CpuFFT) FFTW is no longer fetched, built or linked - not even by jfjoch_test. The JFJOCH_USE_FFTW define and its #ifdefs are gone: the CPU FFT indexer, the viewer's FFT chart and the rest are always built. The test that checked pocketfft against FFTW now checks it against a direct double-precision DFT (forward) and a round trip (backward), on the same shapes. The user-facing name stays: -X FFTW, the OpenAPI enum value, the CBOR and HDF5 strings - the CPU FFT indexer is still called FFTW, only the implementation changed; the CLI page says so. Under MSVC pocketfft is told explicitly to run scalar (it already decides so itself: its vector code needs the GCC/Clang vector extension). Licences: licenses/fftw.txt, its COLLECT.sh line, the THIRD_PARTY_NOTICES row and note, and the viewer's embedded copy removed; docs/THIRD_PARTY_NOTICES.md regenerated with update_version.sh's own sed block. CLAUDE.md and the docs that named FFTW as the library now name pocketfft. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- CLAUDE.md | 11 +- CMakeLists.txt | 39 -- THIRD_PARTY_NOTICES.md | 2 - common/CpuFFT.cpp | 5 + common/Logger.cpp | 2 - docker/ubuntu2404/Dockerfile | 2 +- docker/ubuntu2404/aarch64-sbsa.cmake | 2 +- docs/CPU_DATA_ANALYSIS_DECISIONS.md | 2 +- docs/CPU_DATA_ANALYSIS_IMAGE.md | 2 +- docs/GPU_DATA_ANALYSIS.md | 8 +- docs/RUGNUX_CLI.md | 2 +- docs/RUGNUX_INSTALL.md | 4 +- docs/SOFTWARE.md | 2 +- docs/THIRD_PARTY_NOTICES.md | 2 - .../geom_refinement/BeamCenterFFTEngine.h | 2 +- image_analysis/indexing/FFTIndexer.cpp | 10 +- image_analysis/indexing/IndexerFactory.cpp | 4 - image_analysis/indexing/IndexerThreadPool.cpp | 13 +- .../scale_merge/TranslationalNCS.cpp | 4 +- .../ModelStructureFactorsGPU.h | 2 +- .../MolecularReplacementGPU.h | 2 +- .../structure_refinement/RigidBodyGPU.h | 2 +- licenses/COLLECT.sh | 1 - licenses/fftw.txt | 340 ----------------- rugnux/rugnux_cli.cpp | 2 +- tests/CMakeLists.txt | 2 +- tests/CpuFFTTest.cpp | 135 ++++--- tests/IndexingUnitTest.cpp | 2 - tests/ModelValidationTest.cpp | 10 +- tests/PortableTest.cpp | 2 +- viewer/charts/JFJochDatasetInfoChartView.cpp | 12 - viewer/charts/JFJochDatasetInfoChartView.h | 2 - viewer/resources/third_party_licenses.html | 343 ------------------ 33 files changed, 124 insertions(+), 851 deletions(-) delete mode 100644 licenses/fftw.txt diff --git a/CLAUDE.md b/CLAUDE.md index d90043cb0..d3ecfc9d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ changing CLI behaviour, update the parser, that page and, for a basic option, th ## Build Out-of-source CMake build, C++20, heavy use of `FetchContent` (spdlog, zstd, HDF5, -slsDetectorPackage, Catch2, cpp-httplib, libzmq, libtiff, FFTW, Ceres (with abseil), zlib-ng and Eigen are +slsDetectorPackage, Catch2, cpp-httplib, libzmq, libtiff, Ceres (with abseil), zlib-ng and Eigen are downloaded and statically linked — the **first configure needs network access** and is slow; the fast-feedback indexer behind FFBIDX is vendored in `image_analysis/indexing/fast-feedback-indexer/`). **No library has to come from the host**: `zlib-devel`/`eigen3-devel` are no longer needed. @@ -38,8 +38,11 @@ make -j$(nproc) jfjoch_broker # the main service; build other targets by na Key CMake options: - `JFJOCH_USE_CUDA` (default ON) — GPU path. Needs CUDA ≥ 12.8 (older is warned about and ignored). Provides the `ffbidx` and `fft` GPU indexers; without it the indexers are the CPU `fftw` - and a CPU port of the fast-feedback indexer (`FFBIDXIndexerCPU`, always built). FFTW is fetched and - `JFJOCH_USE_FFTW` defined unconditionally, so `fftw` is always there. CUDA absence is not a build + and a CPU port of the fast-feedback indexer (`FFBIDXIndexerCPU`, always built). Every CPU FFT (the + `fftw` indexer, beam-centre search, tNCS, model maps, MR, the viewer's chart) goes through + `common/CpuFFT` on the pocketfft header vendored with gemmi - FFTW is no longer a dependency, the + `fftw`/`FFTW` name is kept for the CPU indexer only for back compatibility. pocketfft vectorises + with GCC/Clang vector extensions and runs scalar under MSVC. CUDA absence is not a build error — nvcc is looked for on `PATH`, `CUDA_PATH`, `CUDA_HOME`, `CUDA_ROOT`, `/usr/local/cuda` and `/opt/cuda`. - `JFJOCH_WRITER_ONLY` (default OFF) — builds only the HDF5 writer (with the detector control, @@ -393,7 +396,7 @@ CXX_THREADS). The indexer is chosen with `-X` (`Auto|FFT|cuFFT|FFTW|FFBIDX|None`, default `Auto`, which resolves to a GPU indexer when one is present and `fftw` otherwise): `ffbidx` wants a known cell (`-C`) and suits sparse serial stills; `fft` indexes de novo and suits strong rotation data - it is the algorithm, run -with cuFFT on a GPU and with FFTW without one (or under `--no-gpu`), while `cufft`/`fftw` name one +with cuFFT on a GPU and with pocketfft without one (or under `--no-gpu`), while `cufft`/`fftw` name one implementation each. Only rugnux's CLI spells it this way: internally (and in the broker's OpenAPI) `FFT` is the GPU implementation, which the CLI's `cufft` maps onto. diff --git a/CMakeLists.txt b/CMakeLists.txt index eae9a9287..88e03e81b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -474,45 +474,6 @@ FetchContent_Declare(tiff EXCLUDE_FROM_ALL) FetchContent_MakeAvailable(tiff) -# FFTW single precision (target fftw3f): no longer used by the programs, whose CPU FFTs are pocketfft's -# (common/CpuFFT) - linked only by jfjoch_test, which checks pocketfft against it. JFJOCH_USE_FFTW still -# means "the CPU FFT indexer is built". Built from the release tarball -- the git repo ships no pre-generated codelets (needs the -# OCaml genfft). -SET(ENABLE_FLOAT ON CACHE BOOL "" FORCE) -SET(BUILD_TESTS OFF CACHE BOOL "" FORCE) -# SIMD codelets are runtime-dispatched (cpuid), so enabling AVX2 does NOT require an AVX2 CPU -# -- FFTW falls back at runtime. The codelet sets are arch-specific, so guard by processor: -# x86 gets SSE2/AVX/AVX2, aarch64 (Apple Silicon, Grace on DGX Spark) gets NEON - added below the -# FetchContent, as FFTW's CMake build has no switch for it; other arches build scalar. -IF (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|amd64") - SET(ENABLE_SSE2 ON CACHE BOOL "" FORCE) - SET(ENABLE_AVX ON CACHE BOOL "" FORCE) - SET(ENABLE_AVX2 ON CACHE BOOL "" FORCE) -ENDIF() -FetchContent_Declare(fftw - URL https://www.fftw.org/fftw-3.3.10.tar.gz - URL_HASH SHA256=56c932549852cddcfafdab3820b0200c7742675be92179e59e6215b340e26467 - EXCLUDE_FROM_ALL) -FetchContent_MakeAvailable(fftw) -# FFTW exposes fftw3.h only via $, so consuming fftw3f straight from -# the build tree (FetchContent) leaves the header unreachable. On Linux a system-installed -# fftw3.h masks this; on Windows it does not. Add the source-tree api/ dir for build-tree use. -TARGET_INCLUDE_DIRECTORIES(fftw3f INTERFACE $) -# NEON: only FFTW's autotools build knows it (--enable-neon); its CMake build silently builds scalar -# code on ARM (and ignores an ENABLE_NEON, as this file once set). So the NEON codelets are added -# the way its CMakeLists adds the SSE2/AVX ones: the sources, and HAVE_NEON so that dft/conf.c and -# rdft/conf.c register them - as a definition, because its config.h template carries HAVE_NEON only -# as a comment. NEON is part of every aarch64 CPU, so no compiler flag is needed. It is single -# precision, which is all that is built here. Measured on Apple Silicon against the scalar build -# (same results to 1e-8): the indexer's batched 1D r2c 1.8x faster at a power-of-two length and 1.3x -# at 730, the 2D r2c of the beam-centre search 1.9x, a 128^3 c2r map 2.6x. -IF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") - FILE(GLOB _fftw_neon_sources ${fftw_SOURCE_DIR}/dft/simd/neon/*.c ${fftw_SOURCE_DIR}/rdft/simd/neon/*.c) - TARGET_SOURCES(fftw3f PRIVATE ${_fftw_neon_sources}) - TARGET_COMPILE_DEFINITIONS(fftw3f PRIVATE HAVE_NEON=1) -ENDIF() -ADD_COMPILE_DEFINITIONS(JFJOCH_USE_FFTW) - # Eigen (header-only), consumed by the analysis libraries (the vendored fast-feedback indexer among them) # directly and by Ceres through its own find_package(Eigen3). The headers are fetched here and a two-file config package for # them is written into the build tree; Eigen3_DIR then points at it, so every find_package(Eigen3) diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index eb27cc3b6..00edbbc6a 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -26,7 +26,6 @@ linked into the Jungfraujoch executables. | [cpp-httplib](https://github.com/yhirose/cpp-httplib) | 0.56.0 | Yuji Hirose | MIT | [cpp-httplib.txt](licenses/cpp-httplib.txt) | | [libzmq (ZeroMQ)](https://github.com/zeromq/libzmq) | 4.3.5 | iMatix and contributors | MPL-2.0 | [libzmq.txt](licenses/libzmq.txt) | | [libtiff](https://gitlab.com/libtiff/libtiff) | 4.7.2 | Sam Leffler; SGI | libtiff (BSD-like) | [libtiff.txt](licenses/libtiff.txt) | -| [FFTW](https://www.fftw.org/) | 3.3.10 | Matteo Frigo; MIT | GPL-2.0-or-later | [fftw.txt](licenses/fftw.txt) | | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) | commit 0c70ed3a1a2d | Google Inc. and contributors | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) | | [Abseil](https://github.com/abseil/abseil-cpp) | 20250127 (required by Ceres) | Google Inc. | Apache-2.0 | [abseil.txt](licenses/abseil.txt) | | [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) (with TF-PSA-Crypto) | 4.2.0 | The Mbed TLS Contributors | Apache-2.0 OR GPL-2.0-or-later (used under Apache-2.0) | [mbedtls.txt](licenses/mbedtls.txt), [tf-psa-crypto.txt](licenses/tf-psa-crypto.txt) | @@ -163,7 +162,6 @@ served frontend, so the shipped web UI carries its own attribution. * **slsDetectorPackage's bundled code**: the slsDetectorPackage sources compiled into the binaries include an MD5 implementation taken from OpenSSL (Apache-2.0) and the RapidJSON headers (MIT); their `msinttypes/` headers (BSD-3-Clause) are for old MSVC compilers only. -* **FFTW** is GPL-2.0-or-later — compatible with, and absorbed by, this project's GPL-3.0 license. * **Apache-2.0** components: where upstream ships a `NOTICE` file, it is reproduced in the corresponding `licenses/` text. * **CCP4 Monomer Library (LGPL-3.0)** is data, not code: the restraint dictionaries of the compounds diff --git a/common/CpuFFT.cpp b/common/CpuFFT.cpp index 6085fc0d3..cef90287a 100644 --- a/common/CpuFFT.cpp +++ b/common/CpuFFT.cpp @@ -6,6 +6,11 @@ // Under a name of its own: gemmi includes the same header with POCKETFFT_NO_MULTITHREADING, and its // templates and ours must not be merged by the linker into one (single-threaded or not) copy. #define pocketfft jfjoch_pocketfft +// MSVC: the scalar code (pocketfft vectorises only with the GCC/Clang vector extension, and decides so +// itself; said here so that it stays so). +#ifdef _MSC_VER +#define POCKETFFT_NO_VECTORS +#endif #include "../gemmi_gph/gemmi/third_party/pocketfft_hdronly.h" namespace cpu_fft { diff --git a/common/Logger.cpp b/common/Logger.cpp index cccee71f4..52c95c625 100644 --- a/common/Logger.cpp +++ b/common/Logger.cpp @@ -53,9 +53,7 @@ Logger::Logger(const std::string &service_name, const std::string &file_name) { #ifdef JFJOCH_USE_CUDA variant += "C"; #endif -#ifdef JFJOCH_USE_FFTW variant += "F"; -#endif std::vector sinks; sinks.push_back(std::make_shared()); diff --git a/docker/ubuntu2404/Dockerfile b/docker/ubuntu2404/Dockerfile index c84fc9d38..046e6f304 100644 --- a/docker/ubuntu2404/Dockerfile +++ b/docker/ubuntu2404/Dockerfile @@ -162,7 +162,7 @@ ENV CMAKE_PREFIX_PATH=/opt/qt-${QT_VERSION}-static:/opt/dbus-${DBUS_VERSION}-sta # qemu-user-static HDF5 runs two try_run() probes (config/ConfigureChecks.cmake) that # need CMAKE_CROSSCOMPILING_EMULATOR. libzmq guards its own run-checks # with NOT CMAKE_CROSSCOMPILING; Ceres, libtiff, zstd, Catch2, spdlog, -# cpp-httplib, FFTW and ffbidx have no run-checks at all. +# cpp-httplib and ffbidx have no run-checks at all. # binutils-aarch64-linux-gnu reading aarch64 ELF on the build host. # :arm64 libraries ZLIB is one of the project's two external dependencies (Eigen, the # other, is header-only and installed above); OpenSSL is needed by the diff --git a/docker/ubuntu2404/aarch64-sbsa.cmake b/docker/ubuntu2404/aarch64-sbsa.cmake index 3c4f76841..cd4b90095 100644 --- a/docker/ubuntu2404/aarch64-sbsa.cmake +++ b/docker/ubuntu2404/aarch64-sbsa.cmake @@ -9,7 +9,7 @@ # /usr/lib/aarch64-linux-gnu and headers are shared in /usr/include, so CMAKE_LIBRARY_ARCHITECTURE # is what steers find_library() rather than CMAKE_SYSROOT. set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_PROCESSOR aarch64) # also selects FFTW's NEON codelets in CMakeLists.txt +set(CMAKE_SYSTEM_PROCESSOR aarch64) set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) diff --git a/docs/CPU_DATA_ANALYSIS_DECISIONS.md b/docs/CPU_DATA_ANALYSIS_DECISIONS.md index fae331bb5..574bbdcee 100644 --- a/docs/CPU_DATA_ANALYSIS_DECISIONS.md +++ b/docs/CPU_DATA_ANALYSIS_DECISIONS.md @@ -275,7 +275,7 @@ The refinement walks a coarse-to-fine ladder, 6 Å → 4.5 Å → 3.5 Å, each z The refinement sees **only the working reflections**. The step is then committed only if it lowers **R-free**, computed at full resolution on the free set it never saw; otherwise the model is put back exactly where it was read and the maps are the ones it would have given. The whole addition costs about two seconds. Note that the origin is a gauge in some space groups — free in all three directions in $P1$, and along the unique axis in a polar group — so those components of the translation are undetermined; nothing is done about that beyond the Levenberg–Marquardt damping and the R-free gate, which between them make an undetermined direction harmless rather than unstable. -Where a CUDA GPU is present, the refinement's target is evaluated on it: the same density, the same composition of $F_\mathrm{calc}$ from one copy of the model, the same bulk-solvent mask with its islands removed, the same scale fit and the same Jacobian, computed on the device, where a fit takes a fraction of a second rather than several seconds. The device is chosen once per validation, before anything is computed, and never revisited - the real fit and every replicate of the null of §14.5 on the same one. What goes on the GPU is decided for all parts at once, on one budget: 80% of the card's total memory (judged on the total, so the same input on the same machine always takes the same path), less what a merge still at work holds there. Every engine is reserved up front and kept to the end, so it is their sum that has to fit: the structure factors to the data's resolution (which also make the maps), one rigid-body engine, the null's structure factors to its coarser limit and one engine for the scale fits of §14.2 each take their place in that order where they still fit, and more rigid-body engines and then more scale-fit engines are added only within half of the budget: they change only how many fits run at once, and on a large cell, where one fit already fills the card, they buy little. A part that does not fit, or a cell too small for the GPU's gridding, runs on the CPU, as everything does where there is no GPU. A CUDA failure ends the validation, not the run: the validation reports why it did not finish, decides nothing, and the reflection files are those of a run without a model. The two agree to rounding and not bit for bit: the device measures distances in single precision and transforms with cuFFT instead of FFTW, and GEMMI's scale fit, whose stopping rule leaves it unconverged at about $10^{-4}$ of $|F|$, can respond to that rounding with a step of its own. A rigid-body step accepted or rejected on an R-free difference below about $5\cdot10^{-4}$ can therefore go either way between the two; every kernel is deterministic, so either one gives the same answer on every run. +Where a CUDA GPU is present, the refinement's target is evaluated on it: the same density, the same composition of $F_\mathrm{calc}$ from one copy of the model, the same bulk-solvent mask with its islands removed, the same scale fit and the same Jacobian, computed on the device, where a fit takes a fraction of a second rather than several seconds. The device is chosen once per validation, before anything is computed, and never revisited - the real fit and every replicate of the null of §14.5 on the same one. What goes on the GPU is decided for all parts at once, on one budget: 80% of the card's total memory (judged on the total, so the same input on the same machine always takes the same path), less what a merge still at work holds there. Every engine is reserved up front and kept to the end, so it is their sum that has to fit: the structure factors to the data's resolution (which also make the maps), one rigid-body engine, the null's structure factors to its coarser limit and one engine for the scale fits of §14.2 each take their place in that order where they still fit, and more rigid-body engines and then more scale-fit engines are added only within half of the budget: they change only how many fits run at once, and on a large cell, where one fit already fills the card, they buy little. A part that does not fit, or a cell too small for the GPU's gridding, runs on the CPU, as everything does where there is no GPU. A CUDA failure ends the validation, not the run: the validation reports why it did not finish, decides nothing, and the reflection files are those of a run without a model. The two agree to rounding and not bit for bit: the device measures distances in single precision and transforms with cuFFT instead of pocketfft, and GEMMI's scale fit, whose stopping rule leaves it unconverged at about $10^{-4}$ of $|F|$, can respond to that rounding with a step of its own. A rigid-body step accepted or rejected on an R-free difference below about $5\cdot10^{-4}$ can therefore go either way between the two; every kernel is deterministic, so either one gives the same answer on every run. The rotation and translation actually taken are reported (`RIGID_BODY_ROTATION_DEG`, `RIGID_BODY_SHIFT_A`), together with the R-free before it (`R_FREE_BEFORE_RIGID_BODY`), so what the placement bought is visible. diff --git a/docs/CPU_DATA_ANALYSIS_IMAGE.md b/docs/CPU_DATA_ANALYSIS_IMAGE.md index ca528af07..ddac0f395 100644 --- a/docs/CPU_DATA_ANALYSIS_IMAGE.md +++ b/docs/CPU_DATA_ANALYSIS_IMAGE.md @@ -182,7 +182,7 @@ margin, not a centre: the surface can be locally flat over tens of pixels, and t peak-to-runner-up margin says so. A **local walk** is then seeded at the capture and refines it; where it declines there it is started again from the file's centre, and only where neither start gives it something to fit does the capture stand alone. The transforms run on the GPU where one is present and -they fit half of the card's total memory, and on FFTW otherwise, with everything that decides anything +they fit half of the card's total memory, and on the CPU FFT (pocketfft) otherwise, with everything that decides anything shared between the two. The run log then carries: diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md index 82343dbdb..af375c978 100644 --- a/docs/GPU_DATA_ANALYSIS.md +++ b/docs/GPU_DATA_ANALYSIS.md @@ -12,7 +12,7 @@ computes. The methods themselves are described once, in the (`JFJOCH_USE_CUDA=OFF`) runs the whole pipeline on it, the `ffbidx` indexer included (an in-tree CPU port of the library's kernels). How closely the two agree is stated engine by engine below and in the source: where the arithmetic is integer, or the order of every sum is fixed to -the host's, the result is the same to the bit; where the GPU uses float distances, cuFFT for FFTW, or a +the host's, the result is the same to the bit; where the GPU uses float distances, cuFFT for pocketfft, or a different (fixed) summation order, the two agree to rounding. Every engine is deterministic on its own - no floating-point atomics decide a result - so the same input gives the same output on every run of a GPU build. @@ -74,7 +74,7 @@ nothing. stays on the host. [§1.4](CPU_DATA_ANALYSIS_IMAGE.md#14-the-beam-centre) - **Beam centre by FFT** (`BeamCenterFFTGPU.cu`) - the self-convolutions that score every candidate centre at once, their combination into one score surface, and the shortlist of its peaks, with only - the shortlist brought back. Runs on FFTW on the host instead where it would need more than half of + the shortlist brought back. Runs on the CPU FFT on the host instead where it would need more than half of the card's total memory, decided before the card is touched and logged. [§1.4](CPU_DATA_ANALYSIS_IMAGE.md#14-the-beam-centre) @@ -158,7 +158,7 @@ total and not on what is free, the same input on the same card always takes the The model's structure factors and the rigid body run on the card where there is one, every part placed on it or on the CPU up front against one budget, 80% of the card's total memory ([§14.8](CPU_DATA_ANALYSIS_DECISIONS.md#148-rigid-body-placement-of-the-model)); the GPU and CPU -agree to rounding (float distances, cuFFT for FFTW, the crystal's symmetry composed in reciprocal +agree to rounding (float distances, cuFFT for pocketfft, the crystal's symmetry composed in reciprocal space instead of by symmetrizing the grid). On a machine with several cards, the rigid-body engines the first card has no room for go on the others, each against its own budget. @@ -217,7 +217,7 @@ reflections, the grids, the symmetry and the restraints resident, and per evalua positions up and one value and one gradient per atom back. Everything that depends on the cell, the group and the reflections - the grids, the operators, the adjoint's lists - is worked out once on the host with gemmi when the engine is made. The GPU and CPU engines compute the same target to rounding (float distances and densities, -cuFFT for FFTW, summation order); the GPU's result is the same from run to run, as no sum of a result is a +cuFFT for pocketfft, summation order); the GPU's result is the same from run to run, as no sum of a result is a floating-point atomic: every per-point, per-reflection and per-atom sum is a gather in a fixed order, and every total a `cub::DeviceReduce` tree, whose order for a given size is fixed. diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md index 47f6ede82..6a5237e7a 100644 --- a/docs/RUGNUX_CLI.md +++ b/docs/RUGNUX_CLI.md @@ -280,7 +280,7 @@ For very long axes the direction grid's angular resolution binds well below 500 | Option | Description | | --- | --- | | `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation; several masters are then read as one stills run | -| `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each; `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise. `FFBIDX` without a cell (`-C`, or one from `-z`) resolves to `None` and indexes nothing, with no error. On rotation data `-X` is not consulted: the FFT search runs on the GPU where there is one and on the CPU otherwise, so `Auto`, `FFT`, `cuFFT` (with a GPU) and `FFTW` (without one, or under `--no-gpu`) are accepted and any other value stops the run at indexing; `FFBIDX` and `None` are for stills | +| `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each (`FFTW` keeps its old name: the CPU transforms are pocketfft's); `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise. `FFBIDX` without a cell (`-C`, or one from `-z`) resolves to `None` and indexes nothing, with no error. On rotation data `-X` is not consulted: the FFT search runs on the GPU where there is one and on the CPU otherwise, so `Auto`, `FFT`, `cuFFT` (with a GPU) and `FFTW` (without one, or under `--no-gpu`) are accepted and any other value stops the run at indexing; `FFBIDX` and `None` are for stills | | `-F` | Short for `-X FFT`, kept for old command lines | | `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with `-S P1`. The lattice is indexed in that system, never above or below it, and only the space group within it is searched | | `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10; accepted 5-40). A candidate with a shorter axis is discarded — but on rotation data a row shorter than the floor, down to 3 Å, is recognised under the multiple of it the search reports past the floor, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given. On stills a crystal below the floor cannot be indexed unless this is lowered | diff --git a/docs/RUGNUX_INSTALL.md b/docs/RUGNUX_INSTALL.md index 7d7ac0af9..cfa2f529d 100644 --- a/docs/RUGNUX_INSTALL.md +++ b/docs/RUGNUX_INSTALL.md @@ -84,8 +84,8 @@ The released Linux and Windows archives are CUDA builds (the macOS one is CPU-on an NVIDIA **driver** on the host — 525.60.13 or newer for the CUDA 12 archive, 580.65.06 or newer for the CUDA 13 ones — and no CUDA toolkit, because everything CUDA is linked statically. With no GPU or no driver, `rugnux` reports zero CUDA -devices and falls back to the CPU path, which works but is far slower: the FFT search runs on FFTW -and `ffbidx` on its CPU implementation, and the refinement of a placed model takes minutes rather than +devices and falls back to the CPU path, which works but is far slower: the FFT search runs on the CPU FFT +(pocketfft) and `ffbidx` on its CPU implementation, and the refinement of a placed model takes minutes rather than seconds. A **V100 needs the CUDA 12 archive**; which generations each build covers is in [Release contents ▸ GPU generations and the NVIDIA driver](RELEASE_CONTENTS.md#gpu-generations-and-the-nvidia-driver). diff --git a/docs/SOFTWARE.md b/docs/SOFTWARE.md index 94f23e61b..db59cbfc8 100644 --- a/docs/SOFTWARE.md +++ b/docs/SOFTWARE.md @@ -35,7 +35,7 @@ Optional: * Qt version 6 (for jfjoch_viewer) Many further dependencies (spdlog, Zstandard, HDF5, slsDetectorPackage, libzmq, libtiff, -FFTW, Ceres with abseil, the fast feedback indexer, Catch2, ...) are downloaded automatically +Ceres with abseil, the fast feedback indexer, Catch2, ...) are downloaded automatically by CMake and statically linked; building therefore requires network access on the first configure. zlib (as zlib-ng in its zlib-compatible mode) and Eigen are among them: a copy already on the machine is used instead if the configure is given `-DZLIB_ROOT=` or `-DEigen3_DIR=`. diff --git a/docs/THIRD_PARTY_NOTICES.md b/docs/THIRD_PARTY_NOTICES.md index 13990f7b0..83aff2794 100644 --- a/docs/THIRD_PARTY_NOTICES.md +++ b/docs/THIRD_PARTY_NOTICES.md @@ -31,7 +31,6 @@ linked into the Jungfraujoch executables. | [cpp-httplib](https://github.com/yhirose/cpp-httplib) | 0.56.0 | Yuji Hirose | MIT | [cpp-httplib.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/cpp-httplib.txt) | | [libzmq (ZeroMQ)](https://github.com/zeromq/libzmq) | 4.3.5 | iMatix and contributors | MPL-2.0 | [libzmq.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/libzmq.txt) | | [libtiff](https://gitlab.com/libtiff/libtiff) | 4.7.2 | Sam Leffler; SGI | libtiff (BSD-like) | [libtiff.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/libtiff.txt) | -| [FFTW](https://www.fftw.org/) | 3.3.10 | Matteo Frigo; MIT | GPL-2.0-or-later | [fftw.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/fftw.txt) | | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) | commit 0c70ed3a1a2d | Google Inc. and contributors | BSD-3-Clause | [ceres-solver.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/ceres-solver.txt) | | [Abseil](https://github.com/abseil/abseil-cpp) | 20250127 (required by Ceres) | Google Inc. | Apache-2.0 | [abseil.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/abseil.txt) | | [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) (with TF-PSA-Crypto) | 4.2.0 | The Mbed TLS Contributors | Apache-2.0 OR GPL-2.0-or-later (used under Apache-2.0) | [mbedtls.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/mbedtls.txt), [tf-psa-crypto.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/tf-psa-crypto.txt) | @@ -168,7 +167,6 @@ served frontend, so the shipped web UI carries its own attribution. * **slsDetectorPackage's bundled code**: the slsDetectorPackage sources compiled into the binaries include an MD5 implementation taken from OpenSSL (Apache-2.0) and the RapidJSON headers (MIT); their `msinttypes/` headers (BSD-3-Clause) are for old MSVC compilers only. -* **FFTW** is GPL-2.0-or-later — compatible with, and absorbed by, this project's GPL-3.0 license. * **Apache-2.0** components: where upstream ships a `NOTICE` file, it is reproduced in the corresponding `licenses/` text. * **CCP4 Monomer Library (LGPL-3.0)** is data, not code: the restraint dictionaries of the compounds diff --git a/image_analysis/geom_refinement/BeamCenterFFTEngine.h b/image_analysis/geom_refinement/BeamCenterFFTEngine.h index 226adbf00..6de55cfde 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTEngine.h +++ b/image_analysis/geom_refinement/BeamCenterFFTEngine.h @@ -71,7 +71,7 @@ constexpr double BEAM_CENTER_VARIANCE_FLOOR = 1e-4; // Smallest 2-3-5-7-smooth transform length >= n. Both engines pad to it, so both score the same // grid. A smooth length is not a micro-optimisation here: the naive next-power-of-two pad of a -// 16 Mpixel detector costs 3x the time of the smooth one on both FFTW and cuFFT, and 2.5 GB of +// 16 Mpixel detector costs 3x the time of the smooth one on both the CPU FFT and cuFFT, and 2.5 GB of // cuFFT plan memory against a few hundred MB (measured). [[nodiscard]] int64_t BeamCenterFFTPadSize(int64_t n); diff --git a/image_analysis/indexing/FFTIndexer.cpp b/image_analysis/indexing/FFTIndexer.cpp index ab666f957..cd76beb53 100644 --- a/image_analysis/indexing/FFTIndexer.cpp +++ b/image_analysis/indexing/FFTIndexer.cpp @@ -43,15 +43,15 @@ FFTIndexer::FFTIndexer(const IndexingSettings &settings) if (max_length_A <= settings.GetFFT_HighResolution_A()) throw std::invalid_argument("Largest unit cell cannot be smaller than resolution"); if (nDirections <= 1) - throw std::invalid_argument("FFTWIndexer: number of directions must be > 1"); + throw std::invalid_argument("FFTIndexer: number of directions must be > 1"); if (!(max_length_A > 0.f)) - throw std::invalid_argument("FFTWIndexer: max_length_A must be > 0"); + throw std::invalid_argument("FFTIndexer: max_length_A must be > 0"); if (!(settings.GetFFT_HighResolution_A() > 0.f)) - throw std::invalid_argument("FFTWIndexer: high resolution must be > 0"); + throw std::invalid_argument("FFTIndexer: high resolution must be > 0"); if (histogram_size < 1) - throw std::invalid_argument("FFTWIndexer: histogram_size must be >= 1"); + throw std::invalid_argument("FFTIndexer: histogram_size must be >= 1"); if (histogram_size > 1000000) - throw std::invalid_argument("FFTWIndexer: histogram_size too large"); + throw std::invalid_argument("FFTIndexer: histogram_size too large"); SetupDirectionVectors(); } diff --git a/image_analysis/indexing/IndexerFactory.cpp b/image_analysis/indexing/IndexerFactory.cpp index 52be026d3..18994fa47 100644 --- a/image_analysis/indexing/IndexerFactory.cpp +++ b/image_analysis/indexing/IndexerFactory.cpp @@ -10,9 +10,7 @@ #include "FFTIndexerGPU.h" #endif -#ifdef JFJOCH_USE_FFTW #include "FFTIndexerCPU.h" -#endif std::unique_ptr CreateIndexer(const DiffractionExperiment& experiment) { std::unique_ptr indexer; @@ -27,10 +25,8 @@ std::unique_ptr CreateIndexer(const DiffractionExperiment& experiment) indexer->Setup(experiment); break; case IndexingAlgorithmEnum::FFTW: -#ifdef JFJOCH_USE_FFTW indexer = std::make_unique(experiment.GetIndexingSettings()); indexer->Setup(experiment); -#endif break; case IndexingAlgorithmEnum::FFT: #ifdef JFJOCH_USE_CUDA diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 013ea4e51..4a99f4d28 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -18,9 +18,7 @@ #include "FFTIndexerGPU.h" #endif -#ifdef JFJOCH_USE_FFTW #include "FFTIndexerCPU.h" -#endif double WarmUpCuFFT() { #ifdef JFJOCH_USE_CUDA @@ -189,10 +187,8 @@ static std::unique_ptr MakeIndexer(IndexingAlgorithmEnum algorithm, con #endif if (algorithm == IndexingAlgorithmEnum::FFBIDX) return std::make_unique(); -#ifdef JFJOCH_USE_FFTW if (algorithm == IndexingAlgorithmEnum::FFTW) return std::make_unique(settings); -#endif return nullptr; } @@ -468,17 +464,12 @@ IndexerResult IndexerThreadPool::Run(const DiffractionExperiment &experiment, co // available for the resolved algorithm" from deep inside a worker. const auto requested = experiment.GetIndexingSettings().GetAlgorithm(); const bool have_gpu = get_gpu_count() > 0; -#ifdef JFJOCH_USE_FFTW - constexpr bool fftw_built = true; -#else - constexpr bool fftw_built = false; -#endif const bool servable = (algorithm == IndexingAlgorithmEnum::FFT && have_gpu && (requested == IndexingAlgorithmEnum::Auto || requested == IndexingAlgorithmEnum::FFT)) || (algorithm == IndexingAlgorithmEnum::FFBIDX && (requested == IndexingAlgorithmEnum::Auto || requested == IndexingAlgorithmEnum::FFBIDX)) || - (algorithm == IndexingAlgorithmEnum::FFTW && fftw_built && + (algorithm == IndexingAlgorithmEnum::FFTW && ((requested == IndexingAlgorithmEnum::Auto && !have_gpu) || requested == IndexingAlgorithmEnum::FFTW)); if (!servable) { std::string msg; @@ -487,8 +478,6 @@ IndexerResult IndexerThreadPool::Run(const DiffractionExperiment &experiment, co "always uses the GPU FFT indexer here; select FFT or Auto, or run FFTW on a CPU-only node."; else if (algorithm == IndexingAlgorithmEnum::FFT && !have_gpu) msg = "FFT is the GPU indexer but no GPU is available. Select FFTW or Auto for CPU indexing."; - else if (algorithm == IndexingAlgorithmEnum::FFTW) - msg = "FFTW (CPU) indexing was requested but this build has no FFTW indexer."; else msg = "the requested indexing algorithm resolved to one with no indexer available on this host."; throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, diff --git a/image_analysis/scale_merge/TranslationalNCS.cpp b/image_analysis/scale_merge/TranslationalNCS.cpp index cf9e1f486..02aa3c97d 100644 --- a/image_analysis/scale_merge/TranslationalNCS.cpp +++ b/image_analysis/scale_merge/TranslationalNCS.cpp @@ -115,8 +115,8 @@ namespace { return out; } - // Round up to the next even number whose only prime factors are 2, 3 and 5 - the sizes FFTW has - // codelets for. + // Round up to the next even number whose only prime factors are 2, 3 and 5 - the sizes the FFT + // factorises fastest. int SmoothSize(int v) { v = std::max(2, v); for (;; ++v) { diff --git a/image_analysis/structure_refinement/ModelStructureFactorsGPU.h b/image_analysis/structure_refinement/ModelStructureFactorsGPU.h index 12349b56a..43114fb1f 100644 --- a/image_analysis/structure_refinement/ModelStructureFactorsGPU.h +++ b/image_analysis/structure_refinement/ModelStructureFactorsGPU.h @@ -9,7 +9,7 @@ // bulk-solvent mask (gemmi's SolventMasker with the Refmac radii), each transformed and read off as // prepare_asu_data() lists them; and the inverse transform of a set of map coefficients, as // get_f_phi_on_grid() and MapFromFPhi() make it - moved to the device. The two agree to rounding, not -// bit for bit: float distances and cuFFT for FFTW, and the crystal's symmetry is composed in reciprocal +// bit for bit: float distances and cuFFT for pocketfft, and the crystal's symmetry is composed in reciprocal // space (SymmetryComposition, RigidBodyRefine.h) instead of by symmetrizing the grid. The GPU is // deterministic on its own. // diff --git a/image_analysis/structure_refinement/MolecularReplacementGPU.h b/image_analysis/structure_refinement/MolecularReplacementGPU.h index 47e323fc8..ee50f53ba 100644 --- a/image_analysis/structure_refinement/MolecularReplacementGPU.h +++ b/image_analysis/structure_refinement/MolecularReplacementGPU.h @@ -6,7 +6,7 @@ // The molecular-replacement search on a GPU (CUDA builds only; the header itself needs no CUDA): the // rotation function and the translation function of MolecularReplacement.h, the same functions as the // CPU computes - the same trilinear interpolation of the same molecular transform, the same bins, the -// same coefficients on the same grid - in float on the device, with cuFFT for FFTW. The two agree to +// same coefficients on the same grid - in float on the device, with cuFFT for pocketfft. The two agree to // rounding, not bit for bit. Every reduction runs in a fixed order and nothing is summed with an // atomic, so the GPU is deterministic on its own. diff --git a/image_analysis/structure_refinement/RigidBodyGPU.h b/image_analysis/structure_refinement/RigidBodyGPU.h index d99ad72e7..52a476f5d 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.h +++ b/image_analysis/structure_refinement/RigidBodyGPU.h @@ -7,7 +7,7 @@ // It is the function RigidBodyTarget computes on the CPU - the same density, the same composition of // Fcalc from one copy, the same bulk-solvent mask, scale fit, residuals and Jacobian - moved to the // device so that a fit takes a fraction of a second instead of several seconds. The two agree to -// rounding, not bit for bit: float distances, cuFFT for FFTW. The GPU is deterministic on its own. +// rounding, not bit for bit: float distances, cuFFT for pocketfft. The GPU is deterministic on its own. #include #include diff --git a/licenses/COLLECT.sh b/licenses/COLLECT.sh index ad8408608..31e69767f 100644 --- a/licenses/COLLECT.sh +++ b/licenses/COLLECT.sh @@ -50,7 +50,6 @@ copy_license catch2.txt "$(find_dep catch2-src/LICENSE.txt)" copy_license cpp-httplib.txt "$(find_dep httplib-src/LICENSE)" copy_license libzmq.txt "$(find_dep libzmq-src/LICENSE)" copy_license libtiff.txt "$(find_dep tiff-src/LICENSE.md)" -copy_license fftw.txt "$(find_dep fftw-src/COPYING)" copy_license ceres-solver.txt "$(find_dep ceres-src/LICENSE)" copy_license abseil.txt "$(find_dep ceres-src/third_party/abseil-cpp/LICENSE)" copy_license eigen.txt "$(find_dep eigen3-src/COPYING.MPL2)" diff --git a/licenses/fftw.txt b/licenses/fftw.txt deleted file mode 100644 index 623b6258a..000000000 --- a/licenses/fftw.txt +++ /dev/null @@ -1,340 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index bbbb2f184..701de869d 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1203,7 +1203,7 @@ int RugnuxCli::Parse(int argc, char **argv) { if (alg == "ffbidx") indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; - else if (alg == "fft") // the algorithm: on the GPU where there is one, else with FFTW + else if (alg == "fft") // the algorithm: on the GPU where there is one, else on the CPU indexing_algorithm = get_gpu_count() > 0 ? IndexingAlgorithmEnum::FFT : IndexingAlgorithmEnum::FFTW; else if (alg == "cufft") indexing_algorithm = IndexingAlgorithmEnum::FFT; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 17b290a3e..84112652d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -247,7 +247,7 @@ ADD_EXECUTABLE(jfjoch_test target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochStreamWriter JFJochRugnuxOutput JFJochRugnux JFJochModelWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview - jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib fftw3f) + jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib) target_include_directories(jfjoch_test PRIVATE .) IF (JFJOCH_CUDA_AVAILABLE) TARGET_COMPILE_DEFINITIONS(jfjoch_test PRIVATE FFBIDX_SIMPLE_DATA_DIR="${FFBIDX_SIMPLE_DATA_DIR}") diff --git a/tests/CpuFFTTest.cpp b/tests/CpuFFTTest.cpp index 2046231d6..98cdcbf7d 100644 --- a/tests/CpuFFTTest.cpp +++ b/tests/CpuFFTTest.cpp @@ -4,17 +4,17 @@ #include #include +#include #include #include #include -#include - #include "../common/CpuFFT.h" #include "../image_analysis/structure_refinement/ModelFFT.h" -// pocketfft (common/CpuFFT) against FFTW on the shapes and layouts the call sites use: the same -// transform to float rounding. FFTW is linked into the test binary only, as the reference. +// pocketfft (common/CpuFFT) on the shapes and layouts the call sites use, against a direct DFT in double +// precision: the forward transforms to float rounding, the backward ones by a round trip (an unnormalised +// forward + backward pair gives the input times the number of points). namespace { @@ -28,78 +28,103 @@ std::vector Random(size_t n, unsigned seed) { } // Largest difference relative to the largest value of the reference. -template -double RelativeError(const std::vector &a, const std::vector &ref) { +template +double RelativeError(const std::vector &a, const std::vector &ref) { REQUIRE(a.size() == ref.size()); double diff = 0, scale = 0; for (size_t i = 0; i < a.size(); i++) { - diff = std::max(diff, static_cast(std::abs(a[i] - ref[i]))); - scale = std::max(scale, static_cast(std::abs(ref[i]))); + diff = std::max(diff, std::abs(std::complex(a[i]) - std::complex(ref[i]))); + scale = std::max(scale, std::abs(std::complex(ref[i]))); } return diff / scale; } +// Forward DFT of a row-major array along every axis, computed directly, in double precision. +std::vector> DirectDFT(const std::vector &shape, const std::vector &in) { + std::vector> a(in.begin(), in.end()); + size_t stride = a.size(); + for (size_t n : shape) { + stride /= n; + std::vector> line(n); + for (size_t base = 0; base < a.size(); base++) { + if ((base / stride) % n != 0) + continue; // not the first element of a line along this axis + for (size_t k = 0; k < n; k++) { + std::complex s = 0; + for (size_t j = 0; j < n; j++) + s += a[base + j * stride] * std::polar(1.0, -2.0 * M_PI * static_cast(j * k % n) / n); + line[k] = s; + } + for (size_t k = 0; k < n; k++) + a[base + k * stride] = line[k]; + } + } + return a; +} + +// The half (n/2 + 1 values) of the last axis that a real transform keeps. +std::vector> HalfLastAxis(const std::vector &shape, + const std::vector> &full) { + const size_t n = shape.back(), nc = n / 2 + 1; + std::vector> half; + for (size_t i = 0; i < full.size(); i++) + if (i % n < nc) + half.push_back(full[i]); + return half; +} + +std::vector Scaled(const std::vector &v, double s) { + std::vector r(v.size()); + for (size_t i = 0; i < v.size(); i++) + r[i] = static_cast(v[i] * s); + return r; +} + } // namespace -TEST_CASE("CpuFFT_BatchedR2CMatchesFFTW", "[fft]") { +TEST_CASE("CpuFFT_BatchedR2CMatchesDirectDFT", "[fft]") { // The FFT indexer's histograms: one transform per direction; 3142 = 2 x 1571 is a real length. - for (int n : {3142, 3150, 1024}) { - const int howmany = 37, nc = n / 2 + 1; - std::vector in = Random(static_cast(howmany) * n, n); - std::vector> ours(static_cast(howmany) * nc), ref(ours.size()); - cpu_fft::R2C({static_cast(howmany), static_cast(n)}, {1}, in.data(), ours.data()); - std::vector work = in; - fftwf_plan p = fftwf_plan_many_dft_r2c(1, &n, howmany, work.data(), nullptr, 1, n, - reinterpret_cast(ref.data()), nullptr, 1, nc, - FFTW_ESTIMATE); - fftwf_execute(p); - fftwf_destroy_plan(p); + for (size_t n : {3142, 3150, 1024}) { + const size_t howmany = 5; + const std::vector in = Random(howmany * n, n); + std::vector> ours(howmany * (n / 2 + 1)); + cpu_fft::R2C({howmany, n}, {1}, in.data(), ours.data()); + // DirectDFT over the batch axis too would mix the lines: transform each line on its own. + std::vector> ref; + for (size_t i = 0; i < howmany; i++) { + const std::vector line(in.begin() + i * n, in.begin() + (i + 1) * n); + const auto r = HalfLastAxis({n}, DirectDFT({n}, line)); + ref.insert(ref.end(), r.begin(), r.end()); + } CHECK(RelativeError(ours, ref) < 1e-5); } } -TEST_CASE("CpuFFT_2DRoundTripMatchesFFTW", "[fft]") { +TEST_CASE("CpuFFT_2DRoundTrip", "[fft]") { // The beam-centre search: a 2D r2c and c2r on a 2-3-5-7-smooth padded size. - const int ny = 210, nx = 336, nxc = nx / 2 + 1; - std::vector in = Random(static_cast(ny) * nx, 1); - std::vector> ours(static_cast(ny) * nxc), ref(ours.size()); - cpu_fft::R2C({static_cast(ny), static_cast(nx)}, {0, 1}, in.data(), ours.data()); - std::vector work = in; - fftwf_plan p = fftwf_plan_dft_r2c_2d(ny, nx, work.data(), reinterpret_cast(ref.data()), - FFTW_ESTIMATE); - fftwf_execute(p); - fftwf_destroy_plan(p); - CHECK(RelativeError(ours, ref) < 1e-5); + const std::vector shape = {210, 336}; + const std::vector in = Random(shape[0] * shape[1], 1); + std::vector> ours(shape[0] * (shape[1] / 2 + 1)); + cpu_fft::R2C(shape, {0, 1}, in.data(), ours.data()); + CHECK(RelativeError(ours, HalfLastAxis(shape, DirectDFT(shape, in))) < 1e-5); - std::vector back_ours(in.size()), back_ref(in.size()); - cpu_fft::C2R({static_cast(ny), static_cast(nx)}, {0, 1}, ours.data(), back_ours.data()); - p = fftwf_plan_dft_c2r_2d(ny, nx, reinterpret_cast(ref.data()), back_ref.data(), FFTW_ESTIMATE); - fftwf_execute(p); - fftwf_destroy_plan(p); - CHECK(RelativeError(back_ours, back_ref) < 1e-5); + std::vector back(in.size()); + cpu_fft::C2R(shape, {0, 1}, ours.data(), back.data()); + CHECK(RelativeError(back, Scaled(in, static_cast(in.size()))) < 1e-5); } -TEST_CASE("CpuFFT_3DMatchesFFTW", "[fft]") { +TEST_CASE("CpuFFT_3DRoundTrip", "[fft]") { // The model maps (u fastest, w halved) and the tNCS Patterson (row-major, last axis halved) are both a // row-major 3D transform halving the last axis; an odd length and a factor 7 included. - const int n0 = 70, n1 = 45, n2 = 54, nc = static_cast(static_cast(n0) * n1 * (n2 / 2 + 1)); - const std::vector in = Random(static_cast(n0) * n1 * n2, 2); - std::vector> ours(nc), ref(nc); - cpu_fft::R2C({70, 45, 54}, {0, 1, 2}, in.data(), ours.data()); - std::vector work = in; - fftwf_plan p = fftwf_plan_dft_r2c_3d(n0, n1, n2, work.data(), reinterpret_cast(ref.data()), - FFTW_ESTIMATE); - fftwf_execute(p); - fftwf_destroy_plan(p); - CHECK(RelativeError(ours, ref) < 1e-5); + const std::vector shape = {70, 45, 54}; + const std::vector in = Random(shape[0] * shape[1] * shape[2], 2); + std::vector> ours(shape[0] * shape[1] * (shape[2] / 2 + 1)); + cpu_fft::R2C(shape, {0, 1, 2}, in.data(), ours.data()); + CHECK(RelativeError(ours, HalfLastAxis(shape, DirectDFT(shape, in))) < 1e-5); - std::vector back_ours(in.size()), back_ref(in.size()); - cpu_fft::C2R({70, 45, 54}, {0, 1, 2}, ours.data(), back_ours.data()); - p = fftwf_plan_dft_c2r_3d(n0, n1, n2, reinterpret_cast(ref.data()), back_ref.data(), - FFTW_ESTIMATE); - fftwf_execute(p); - fftwf_destroy_plan(p); - CHECK(RelativeError(back_ours, back_ref) < 1e-5); + std::vector back(in.size()); + cpu_fft::C2R(shape, {0, 1, 2}, ours.data(), back.data()); + CHECK(RelativeError(back, Scaled(in, static_cast(in.size()))) < 1e-5); } TEST_CASE("CpuFFT_MapToFPhiInParallelMatchesMapToFPhi", "[fft]") { diff --git a/tests/IndexingUnitTest.cpp b/tests/IndexingUnitTest.cpp index cc56abdbc..a913d9a9a 100644 --- a/tests/IndexingUnitTest.cpp +++ b/tests/IndexingUnitTest.cpp @@ -710,9 +710,7 @@ namespace { #ifdef JFJOCH_USE_CUDA ret.push_back(IndexingAlgorithmEnum::FFT); #endif -#ifdef JFJOCH_USE_FFTW ret.push_back(IndexingAlgorithmEnum::FFTW); -#endif return ret; } diff --git a/tests/ModelValidationTest.cpp b/tests/ModelValidationTest.cpp index d5bd688c6..522d7caa5 100644 --- a/tests/ModelValidationTest.cpp +++ b/tests/ModelValidationTest.cpp @@ -1023,7 +1023,7 @@ TEST_CASE("ModelValidation_WarmUpChangesNothing", "[ModelValidation]") { std::filesystem::remove(path); } -// The model path's structure factors come from FFTW; they must be gemmi's own transform to float +// The model path's structure factors come from the CPU FFT (pocketfft); they must be gemmi's own transform to float // precision, in the same layout, so prepare_asu_data() reads the same reflections from either. // A wrong model may come in a group that does not keep the data's metric - a cubic group on an @@ -1161,7 +1161,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat std::filesystem::remove(wrong_path); } TEST_CASE("ModelValidation_MapToFPhiMatchesGemmi", "[ModelValidation]") { - // An even grid, and an odd one on every axis (FFTW and pocketfft split odd lengths differently). + // An even grid, and an odd one on every axis (FFT libraries split odd lengths differently). const auto size = GENERATE(std::array{20, 24, 30}, std::array{15, 21, 27}); gemmi::Grid map; map.unit_cell.set(40.0, 50.0, 60.0, 90.0, 95.0, 90.0); @@ -1193,7 +1193,7 @@ TEST_CASE("ModelValidation_MapToFPhiMatchesGemmi", "[ModelValidation]") { CHECK(a.v[i].hkl == b.v[i].hkl); } -// The output maps come from FFTW too; each must be gemmi's own map to float precision, point for point +// The output maps come from the CPU FFT too; each must be gemmi's own map to float precision, point for point // in the same layout. Coefficients with arbitrary phases, negative indices, expanded by symmetry and // Friedel, on grids that are odd along u and v (w is even by construction of a half-l grid). TEST_CASE("ModelValidation_MapFromFPhiMatchesGemmi", "[ModelValidation]") { @@ -1849,7 +1849,7 @@ TEST_CASE("RigidBodyGPU_AxisBrickBoundCoversWrappedBoxes", "[ModelValidation][gp // The GPU target is the CPU target's function, computed on the device: at the same placement the two // give the same residuals and the same Jacobian to rounding (float distances on the device, cuFFT for -// FFTW), on every group of the composition tests, both zones, anisotropic atoms included. +// pocketfft), on every group of the composition tests, both zones, anisotropic atoms included. TEST_CASE("RigidBodyGPU_MatchesCPU", "[ModelValidation][gpu]") { if (get_gpu_count() == 0) SKIP("No GPU"); @@ -2021,7 +2021,7 @@ TEST_CASE("RigidBodyGPU_CallsLeaveTheCallerOnItsCard", "[ModelValidation][gpu]") } // The model's structure factors on the GPU against the CPU path model validation takes - its density -// (PutModelDensityOnGrid) and gemmi's mask on the symmetrized grid, transformed by FFTW and read off by prepare_asu_data() - in the five groups, +// (PutModelDensityOnGrid) and gemmi's mask on the symmetrized grid, transformed by the CPU FFT and read off by prepare_asu_data() - in the five groups, // at a resolution where the mask has a grid of its own (SolventMaskGrid) and at one where it is on the // density's. The same reflections // in the same order; F_calc to rounding; F_mask to the few grid points at an atom's mask radius that float diff --git a/tests/PortableTest.cpp b/tests/PortableTest.cpp index 3c67eed15..214445ada 100644 --- a/tests/PortableTest.cpp +++ b/tests/PortableTest.cpp @@ -115,7 +115,7 @@ TEST_CASE("Portable_BitShuffleBlock_MatchesReference", "[portable][compression]" } } -// The CPU indexer (FFTW, NEON codelets on arm64) on a synthetic orthorhombic lattice. Named +// The CPU FFT indexer (FFTW, on pocketfft) on a synthetic orthorhombic lattice. Named // explicitly rather than through Auto, so a CUDA build on a runner without a GPU tests it too. TEST_CASE("Portable_FFTWIndexer_SyntheticLattice", "[portable][Indexing]") { const UnitCell uc(39, 45, 78, 90, 90, 90); diff --git a/viewer/charts/JFJochDatasetInfoChartView.cpp b/viewer/charts/JFJochDatasetInfoChartView.cpp index 03b8ce9fb..c8ff0b8d6 100644 --- a/viewer/charts/JFJochDatasetInfoChartView.cpp +++ b/viewer/charts/JFJochDatasetInfoChartView.cpp @@ -360,14 +360,10 @@ void JFJochDatasetInfoChartView::updateChart() { delete item; band_items_.clear(); -#ifdef JFJOCH_USE_FFTW if (m_showFFT) buildFFTChart(); else buildTimeDomainChart(); -#else - buildTimeDomainChart(); -#endif // Light red behind the images the merge rejected, light yellow behind the ones it down-weighted. // Above the plot background, below the grid and the curves. @@ -825,13 +821,11 @@ void JFJochDatasetInfoChartView::contextMenuEvent(QContextMenuEvent *event) { binActions.push_back(act); } -#ifdef JFJOCH_USE_FFTW QAction *actShowFFT = menu.addAction("Show FFT (amplitude vs Hz)"); actShowFFT->setCheckable(true); actShowFFT->setChecked(m_showFFT); // Require valid sampling interval actShowFFT->setEnabled(!values.empty() && image_time_s > 0.0); -#endif menu.addSeparator(); const bool has_range = sel_first_ >= 0; @@ -872,11 +866,9 @@ void JFJochDatasetInfoChartView::contextMenuEvent(QContextMenuEvent *event) { if (ok && v >= 1) { setBinning(v); } -#ifdef JFJOCH_USE_FFTW } else if (chosen == actShowFFT) { m_showFFT = !m_showFFT; updateChart(); -#endif } } @@ -897,7 +889,6 @@ void JFJochDatasetInfoChartView::mouseMoveEvent(QMouseEvent *event) { } } -#ifdef JFJOCH_USE_FFTW if (m_showFFT && !m_fftFrequenciesHz.empty()) { // FFT mode: x is frequency in Hz const QPointF chartPos = chart()->mapToValue(event->pos(), series); @@ -948,7 +939,6 @@ void JFJochDatasetInfoChartView::mouseMoveEvent(QMouseEvent *event) { m_hoverPendingIdx = -1; return; } -#endif if (values.empty()) return; @@ -1102,7 +1092,6 @@ void JFJochDatasetInfoChartView::onHoverLoadTimeout() { } } -#ifdef JFJOCH_USE_FFTW void JFJochDatasetInfoChartView::buildFFTChart() { const size_t N = values.size(); @@ -1179,5 +1168,4 @@ void JFJochDatasetInfoChartView::buildFFTChart() { } StyleChartAxes(chart()); } -#endif diff --git a/viewer/charts/JFJochDatasetInfoChartView.h b/viewer/charts/JFJochDatasetInfoChartView.h index e6c1cfb40..9b32db8e1 100644 --- a/viewer/charts/JFJochDatasetInfoChartView.h +++ b/viewer/charts/JFJochDatasetInfoChartView.h @@ -105,12 +105,10 @@ private: // FFT view toggle bool m_showFFT = false; -#ifdef JFJOCH_USE_FFTW // Cached FFT data for hover/status bar etc. void buildFFTChart(); std::vector m_fftMagnitudes; std::vector m_fftFrequenciesHz; -#endif signals: void imageSelected(int64_t number); diff --git a/viewer/resources/third_party_licenses.html b/viewer/resources/third_party_licenses.html index 32455f252..c2f2cf019 100644 --- a/viewer/resources/third_party_licenses.html +++ b/viewer/resources/third_party_licenses.html @@ -22,7 +22,6 @@ manifest with copyright holders is in THIRD_PARTY_NOTICES.md in the source distr
  • eigen
  • Fast feedback indexer
  • fast-float
  • -
  • fftw
  • gemmi
  • half
  • hdf5
  • @@ -1934,348 +1933,6 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    -

    fftw

    -
    		    GNU GENERAL PUBLIC LICENSE
    -		       Version 2, June 1991
    -
    - Copyright (C) 1989, 1991 Free Software Foundation, Inc.
    -     51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
    - Everyone is permitted to copy and distribute verbatim copies
    - of this license document, but changing it is not allowed.
    -
    -			    Preamble
    -
    -  The licenses for most software are designed to take away your
    -freedom to share and change it.  By contrast, the GNU General Public
    -License is intended to guarantee your freedom to share and change free
    -software--to make sure the software is free for all its users.  This
    -General Public License applies to most of the Free Software
    -Foundation's software and to any other program whose authors commit to
    -using it.  (Some other Free Software Foundation software is covered by
    -the GNU Library General Public License instead.)  You can apply it to
    -your programs, too.
    -
    -  When we speak of free software, we are referring to freedom, not
    -price.  Our General Public Licenses are designed to make sure that you
    -have the freedom to distribute copies of free software (and charge for
    -this service if you wish), that you receive source code or can get it
    -if you want it, that you can change the software or use pieces of it
    -in new free programs; and that you know you can do these things.
    -
    -  To protect your rights, we need to make restrictions that forbid
    -anyone to deny you these rights or to ask you to surrender the rights.
    -These restrictions translate to certain responsibilities for you if you
    -distribute copies of the software, or if you modify it.
    -
    -  For example, if you distribute copies of such a program, whether
    -gratis or for a fee, you must give the recipients all the rights that
    -you have.  You must make sure that they, too, receive or can get the
    -source code.  And you must show them these terms so they know their
    -rights.
    -
    -  We protect your rights with two steps: (1) copyright the software, and
    -(2) offer you this license which gives you legal permission to copy,
    -distribute and/or modify the software.
    -
    -  Also, for each author's protection and ours, we want to make certain
    -that everyone understands that there is no warranty for this free
    -software.  If the software is modified by someone else and passed on, we
    -want its recipients to know that what they have is not the original, so
    -that any problems introduced by others will not reflect on the original
    -authors' reputations.
    -
    -  Finally, any free program is threatened constantly by software
    -patents.  We wish to avoid the danger that redistributors of a free
    -program will individually obtain patent licenses, in effect making the
    -program proprietary.  To prevent this, we have made it clear that any
    -patent must be licensed for everyone's free use or not licensed at all.
    -
    -  The precise terms and conditions for copying, distribution and
    -modification follow.
    -
    -		    GNU GENERAL PUBLIC LICENSE
    -   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
    -
    -  0. This License applies to any program or other work which contains
    -a notice placed by the copyright holder saying it may be distributed
    -under the terms of this General Public License.  The "Program", below,
    -refers to any such program or work, and a "work based on the Program"
    -means either the Program or any derivative work under copyright law:
    -that is to say, a work containing the Program or a portion of it,
    -either verbatim or with modifications and/or translated into another
    -language.  (Hereinafter, translation is included without limitation in
    -the term "modification".)  Each licensee is addressed as "you".
    -
    -Activities other than copying, distribution and modification are not
    -covered by this License; they are outside its scope.  The act of
    -running the Program is not restricted, and the output from the Program
    -is covered only if its contents constitute a work based on the
    -Program (independent of having been made by running the Program).
    -Whether that is true depends on what the Program does.
    -
    -  1. You may copy and distribute verbatim copies of the Program's
    -source code as you receive it, in any medium, provided that you
    -conspicuously and appropriately publish on each copy an appropriate
    -copyright notice and disclaimer of warranty; keep intact all the
    -notices that refer to this License and to the absence of any warranty;
    -and give any other recipients of the Program a copy of this License
    -along with the Program.
    -
    -You may charge a fee for the physical act of transferring a copy, and
    -you may at your option offer warranty protection in exchange for a fee.
    -
    -  2. You may modify your copy or copies of the Program or any portion
    -of it, thus forming a work based on the Program, and copy and
    -distribute such modifications or work under the terms of Section 1
    -above, provided that you also meet all of these conditions:
    -
    -    a) You must cause the modified files to carry prominent notices
    -    stating that you changed the files and the date of any change.
    -
    -    b) You must cause any work that you distribute or publish, that in
    -    whole or in part contains or is derived from the Program or any
    -    part thereof, to be licensed as a whole at no charge to all third
    -    parties under the terms of this License.
    -
    -    c) If the modified program normally reads commands interactively
    -    when run, you must cause it, when started running for such
    -    interactive use in the most ordinary way, to print or display an
    -    announcement including an appropriate copyright notice and a
    -    notice that there is no warranty (or else, saying that you provide
    -    a warranty) and that users may redistribute the program under
    -    these conditions, and telling the user how to view a copy of this
    -    License.  (Exception: if the Program itself is interactive but
    -    does not normally print such an announcement, your work based on
    -    the Program is not required to print an announcement.)
    -
    -These requirements apply to the modified work as a whole.  If
    -identifiable sections of that work are not derived from the Program,
    -and can be reasonably considered independent and separate works in
    -themselves, then this License, and its terms, do not apply to those
    -sections when you distribute them as separate works.  But when you
    -distribute the same sections as part of a whole which is a work based
    -on the Program, the distribution of the whole must be on the terms of
    -this License, whose permissions for other licensees extend to the
    -entire whole, and thus to each and every part regardless of who wrote it.
    -
    -Thus, it is not the intent of this section to claim rights or contest
    -your rights to work written entirely by you; rather, the intent is to
    -exercise the right to control the distribution of derivative or
    -collective works based on the Program.
    -
    -In addition, mere aggregation of another work not based on the Program
    -with the Program (or with a work based on the Program) on a volume of
    -a storage or distribution medium does not bring the other work under
    -the scope of this License.
    -
    -  3. You may copy and distribute the Program (or a work based on it,
    -under Section 2) in object code or executable form under the terms of
    -Sections 1 and 2 above provided that you also do one of the following:
    -
    -    a) Accompany it with the complete corresponding machine-readable
    -    source code, which must be distributed under the terms of Sections
    -    1 and 2 above on a medium customarily used for software interchange; or,
    -
    -    b) Accompany it with a written offer, valid for at least three
    -    years, to give any third party, for a charge no more than your
    -    cost of physically performing source distribution, a complete
    -    machine-readable copy of the corresponding source code, to be
    -    distributed under the terms of Sections 1 and 2 above on a medium
    -    customarily used for software interchange; or,
    -
    -    c) Accompany it with the information you received as to the offer
    -    to distribute corresponding source code.  (This alternative is
    -    allowed only for noncommercial distribution and only if you
    -    received the program in object code or executable form with such
    -    an offer, in accord with Subsection b above.)
    -
    -The source code for a work means the preferred form of the work for
    -making modifications to it.  For an executable work, complete source
    -code means all the source code for all modules it contains, plus any
    -associated interface definition files, plus the scripts used to
    -control compilation and installation of the executable.  However, as a
    -special exception, the source code distributed need not include
    -anything that is normally distributed (in either source or binary
    -form) with the major components (compiler, kernel, and so on) of the
    -operating system on which the executable runs, unless that component
    -itself accompanies the executable.
    -
    -If distribution of executable or object code is made by offering
    -access to copy from a designated place, then offering equivalent
    -access to copy the source code from the same place counts as
    -distribution of the source code, even though third parties are not
    -compelled to copy the source along with the object code.
    -
    -  4. You may not copy, modify, sublicense, or distribute the Program
    -except as expressly provided under this License.  Any attempt
    -otherwise to copy, modify, sublicense or distribute the Program is
    -void, and will automatically terminate your rights under this License.
    -However, parties who have received copies, or rights, from you under
    -this License will not have their licenses terminated so long as such
    -parties remain in full compliance.
    -
    -  5. You are not required to accept this License, since you have not
    -signed it.  However, nothing else grants you permission to modify or
    -distribute the Program or its derivative works.  These actions are
    -prohibited by law if you do not accept this License.  Therefore, by
    -modifying or distributing the Program (or any work based on the
    -Program), you indicate your acceptance of this License to do so, and
    -all its terms and conditions for copying, distributing or modifying
    -the Program or works based on it.
    -
    -  6. Each time you redistribute the Program (or any work based on the
    -Program), the recipient automatically receives a license from the
    -original licensor to copy, distribute or modify the Program subject to
    -these terms and conditions.  You may not impose any further
    -restrictions on the recipients' exercise of the rights granted herein.
    -You are not responsible for enforcing compliance by third parties to
    -this License.
    -
    -  7. If, as a consequence of a court judgment or allegation of patent
    -infringement or for any other reason (not limited to patent issues),
    -conditions are imposed on you (whether by court order, agreement or
    -otherwise) that contradict the conditions of this License, they do not
    -excuse you from the conditions of this License.  If you cannot
    -distribute so as to satisfy simultaneously your obligations under this
    -License and any other pertinent obligations, then as a consequence you
    -may not distribute the Program at all.  For example, if a patent
    -license would not permit royalty-free redistribution of the Program by
    -all those who receive copies directly or indirectly through you, then
    -the only way you could satisfy both it and this License would be to
    -refrain entirely from distribution of the Program.
    -
    -If any portion of this section is held invalid or unenforceable under
    -any particular circumstance, the balance of the section is intended to
    -apply and the section as a whole is intended to apply in other
    -circumstances.
    -
    -It is not the purpose of this section to induce you to infringe any
    -patents or other property right claims or to contest validity of any
    -such claims; this section has the sole purpose of protecting the
    -integrity of the free software distribution system, which is
    -implemented by public license practices.  Many people have made
    -generous contributions to the wide range of software distributed
    -through that system in reliance on consistent application of that
    -system; it is up to the author/donor to decide if he or she is willing
    -to distribute software through any other system and a licensee cannot
    -impose that choice.
    -
    -This section is intended to make thoroughly clear what is believed to
    -be a consequence of the rest of this License.
    -
    -  8. If the distribution and/or use of the Program is restricted in
    -certain countries either by patents or by copyrighted interfaces, the
    -original copyright holder who places the Program under this License
    -may add an explicit geographical distribution limitation excluding
    -those countries, so that distribution is permitted only in or among
    -countries not thus excluded.  In such case, this License incorporates
    -the limitation as if written in the body of this License.
    -
    -  9. The Free Software Foundation may publish revised and/or new versions
    -of the General Public License from time to time.  Such new versions will
    -be similar in spirit to the present version, but may differ in detail to
    -address new problems or concerns.
    -
    -Each version is given a distinguishing version number.  If the Program
    -specifies a version number of this License which applies to it and "any
    -later version", you have the option of following the terms and conditions
    -either of that version or of any later version published by the Free
    -Software Foundation.  If the Program does not specify a version number of
    -this License, you may choose any version ever published by the Free Software
    -Foundation.
    -
    -  10. If you wish to incorporate parts of the Program into other free
    -programs whose distribution conditions are different, write to the author
    -to ask for permission.  For software which is copyrighted by the Free
    -Software Foundation, write to the Free Software Foundation; we sometimes
    -make exceptions for this.  Our decision will be guided by the two goals
    -of preserving the free status of all derivatives of our free software and
    -of promoting the sharing and reuse of software generally.
    -
    -			    NO WARRANTY
    -
    -  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
    -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
    -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
    -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
    -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
    -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
    -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
    -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
    -REPAIR OR CORRECTION.
    -
    -  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
    -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
    -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
    -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
    -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
    -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
    -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
    -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
    -POSSIBILITY OF SUCH DAMAGES.
    -
    -		     END OF TERMS AND CONDITIONS
    -
    -	    How to Apply These Terms to Your New Programs
    -
    -  If you develop a new program, and you want it to be of the greatest
    -possible use to the public, the best way to achieve this is to make it
    -free software which everyone can redistribute and change under these terms.
    -
    -  To do so, attach the following notices to the program.  It is safest
    -to attach them to the start of each source file to most effectively
    -convey the exclusion of warranty; and each file should have at least
    -the "copyright" line and a pointer to where the full notice is found.
    -
    -    <one line to give the program's name and a brief idea of what it does.>
    -    Copyright (C) <year>  <name of author>
    -
    -    This program is free software; you can redistribute it and/or modify
    -    it under the terms of the GNU General Public License as published by
    -    the Free Software Foundation; either version 2 of the License, or
    -    (at your option) any later version.
    -
    -    This program is distributed in the hope that it will be useful,
    -    but WITHOUT ANY WARRANTY; without even the implied warranty of
    -    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    -    GNU General Public License for more details.
    -
    -    You should have received a copy of the GNU General Public License
    -    along with this program; if not, write to the Free Software
    -    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
    -
    -
    -Also add information on how to contact you by electronic and paper mail.
    -
    -If the program is interactive, make it output a short notice like this
    -when it starts in an interactive mode:
    -
    -    Gnomovision version 69, Copyright (C) year  name of author
    -    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
    -    This is free software, and you are welcome to redistribute it
    -    under certain conditions; type `show c' for details.
    -
    -The hypothetical commands `show w' and `show c' should show the appropriate
    -parts of the General Public License.  Of course, the commands you use may
    -be called something other than `show w' and `show c'; they could even be
    -mouse-clicks or menu items--whatever suits your program.
    -
    -You should also get your employer (if you work as a programmer) or your
    -school, if any, to sign a "copyright disclaimer" for the program, if
    -necessary.  Here is a sample; alter the names:
    -
    -  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
    -  `Gnomovision' (which makes passes at compilers) written by James Hacker.
    -
    -  <signature of Ty Coon>, 1 April 1989
    -  Ty Coon, President of Vice
    -
    -This General Public License does not permit incorporating your program into
    -proprietary programs.  If your program is a subroutine library, you may
    -consider it more useful to permit linking proprietary applications with the
    -library.  If this is what you want to do, use the GNU Library General
    -Public License instead of this License.
    -

    gemmi

    Mozilla Public License Version 2.0
     ==================================
    
    From cfe15f2343bba2cb44e4f8024d298d04943747a1 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 06:11:40 +0100
    Subject: [PATCH 02/19] tNCS test: the near-extinct synthetic clear of the
     gate, and its counterpart
    
    "A near-extinct class is reported as a sub-lattice" built its crystal with the
    weak class at 2% of the strong, which reads a modulation of 19.996x against
    the 20x gate: FFTW's rounding of the Patterson put it over, pocketfft's under.
    Swept on the same synthetic: depth 0.90/0.95/0.96/0.97/0.98 read
    12.0/18.0/20.0/22.8/26.2x. The test now uses the weak class at 1% (26x), and
    also checks the other side of the gate, which it did not before: a deep but
    genuine pseudo-symmetry (weak class at 5%, 12x) is detected and NOT reported
    as near-extinct.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     tests/TranslationalNCSTest.cpp | 10 +++++++++-
     1 file changed, 9 insertions(+), 1 deletion(-)
    
    diff --git a/tests/TranslationalNCSTest.cpp b/tests/TranslationalNCSTest.cpp
    index 0ca30d949..1e478ba88 100644
    --- a/tests/TranslationalNCSTest.cpp
    +++ b/tests/TranslationalNCSTest.cpp
    @@ -254,13 +254,21 @@ TEST_CASE("A near-extinct class is reported as a sub-lattice", "[tncs]") {
         const gemmi::UnitCell cell = Cell(70.0, 85.0, 110.0);
         const gemmi::SpaceGroup *sg = gemmi::find_spacegroup_by_name("P 1 2 1");
         const std::array u{0.0, 0.425, 0.5};   // no p/q with q <= 6 fits the whole vector
    -    const auto merged = Synthetic(cell, "P 1 2 1", &u, 0.96);   // weak class at 2% of the strong
    +    // The weak class at 1% of the strong reads a modulation of 26x, clear of the 20x gate; at 2% it
    +    // read 20.0x, on the gate itself, and the FFT's rounding decided the outcome.
    +    const auto merged = Synthetic(cell, "P 1 2 1", &u, 0.98);
         const auto r = AnalyzeTranslationalNCS(merged, cell, sg);
         REQUIRE(r.measurable);
         REQUIRE(r.detected);
         CHECK_FALSE(r.commensurate);
         CHECK(r.near_extinct_class);
         CHECK(TranslationalNCSToText(r).find("supercell candidate") != std::string::npos);
    +
    +    // A deep but genuine pseudo-symmetry - the weak class at 5% of the strong, a 12x modulation - is
    +    // not one.
    +    const auto deep = AnalyzeTranslationalNCS(Synthetic(cell, "P 1 2 1", &u, 0.90), cell, sg);
    +    REQUIRE(deep.detected);
    +    CHECK_FALSE(deep.near_extinct_class);
     }
     
     // The vector is not only reported: the L-test picks its partner steps with it and the twin-immune
    
    From 6f6e57b364de1e292fee86e90387f150e8c70fd1 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:11:11 +0100
    Subject: [PATCH 03/19] FFT indexer: histogram padded to the next
     2-3-5-7-smooth length
    
    The histogram length is ceil(2 pi * 2 * max_cell / d_high): 3142 = 2 x 1571
    for rugnux's defaults (500 A, 2 A), 1571 (prime) for the broker's (250 A,
    2 A), and almost never smooth otherwise - 217 of 222 lengths over the cell
    bounds 100-1200 A at 2 and 3.5 A. pocketfft does such a length by Bluestein.
    It is padded on to the next length whose factors are 2, 3, 5 and 7 (3142 ->
    3150); the bin spacing is unchanged, only the zero padding grows by 0.25%.
    
    The indexer's transform (16384 directions, one thread), on a loaded box
    (load 11): 3142 0.75 s, 3150 0.07 s - per FFT-indexed image. Rotation sets,
    --no-refine, CPU -N 16: 47.2 vs 47.2 s and 39.9 vs 39.3 s (few images are
    indexed); GPU 9.6 vs 9.3 s and 14.1 vs 14.7 s. Results: one lysozyme set
    bit-identical on CPU and GPU, the other the same merge to rounding (same SG,
    cell, I/sigma, R_meas, CC1/2, ISa; the pre-pass lattice moves by 0.2 A).
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     image_analysis/indexing/FFTIndexer.cpp | 11 +++++++++++
     1 file changed, 11 insertions(+)
    
    diff --git a/image_analysis/indexing/FFTIndexer.cpp b/image_analysis/indexing/FFTIndexer.cpp
    index cd76beb53..e895689b4 100644
    --- a/image_analysis/indexing/FFTIndexer.cpp
    +++ b/image_analysis/indexing/FFTIndexer.cpp
    @@ -35,7 +35,18 @@ FFTIndexer::FFTIndexer(const IndexingSettings &settings)
         const float one_over_d_max = 1.0f / settings.GetFFT_HighResolution_A();
     
         histogram_spacing = 1.0f / (2.0f * max_length_A);
    +    // Then zero-padded a little further, to the next length whose prime factors are all 2, 3, 5 or 7:
    +    // the default (500 A, 2 A) is 3142 = 2 x 1571, which both FFT libraries can only do the slow way.
         histogram_size = std::ceil(oversampling * one_over_d_max / histogram_spacing);
    +    for (bool smooth = false; !smooth; ) {
    +        int64_t v = histogram_size;
    +        for (int p : {2, 3, 5, 7})
    +            while (v % p == 0)
    +                v /= p;
    +        smooth = (v == 1);
    +        if (!smooth)
    +            ++histogram_size;
    +    }
     
         input_size = histogram_size * nDirections;
         output_size = (histogram_size / 2 + 1) * nDirections;
    
    From 88330f395f34535575fcaa8ff85f8a6cc6259454 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:21:12 +0100
    Subject: [PATCH 04/19] Refinement gradient on the Apple GPU: the sums over the
     derivative map in Metal
    
    RESULT-CHANGING (ulp level; owner approved for Metal ports where exactness costs speed)
    
    AtomGradientsFromMap() takes a `gpu` flag; the CPU engine sets it, and on macOS the sums of every
    atom's box against the FFT method's map then run on the Apple GPU (AtomGradientMetal.mm/.metal):
    one SIMD group of 32 threads per atom, the lanes taking the box's points in turn, as
    RefinementGPU.cu's atom_gradient_kernel does under CUDA. Each atom's box, radius and Gaussians are
    the CPU's (built on the host as the CPU loops build them), and a point's place and the radius test
    are spelled as GatherMapPoints() spells them with fast math and contraction off, so the GPU takes
    the same points; the sums are in single precision and in another order, so the gradient is not the
    CPU's bit for bit. Where the GPU does not take it (rugnux --no-gpu, no device, a box wider than the
    cell) or fails (one warning, then the CPU from there on) the CPU loops run as before.
    
    Parity (AtomGradientMetal_MatchesCPU, [portable][Metal], 2000 atoms, a quarter anisotropic, P1
    triclinic and P212121): worst |GPU - CPU| over the largest |CPU| of that derivative 0.6-1.0e-6 for
    x, y, z, B and 2.1-7.6e-6 for U; held to 1e-5 and 5e-5.
    
    M6 Mac mini, rugnux --mode refine, an open-arm set (9zlo, its merged MTZ and refined model):
    90.6 s -> 45.8 s wall; R-work 0.2697 both, R-free 0.2936 -> 0.2938. With --no-gpu the refined model
    is byte-identical to the build before (115 s: --no-gpu also takes the density gather off the GPU).
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     THIRD_PARTY_NOTICES.md                        |   6 +-
     docs/GPU_DATA_ANALYSIS.md                     |   6 +-
     .../structure_refinement/AtomGradientMetal.h  |  31 ++++
     .../AtomGradientMetal.metal                   | 134 +++++++++++++++++
     .../structure_refinement/AtomGradientMetal.mm | 136 ++++++++++++++++++
     .../structure_refinement/CMakeLists.txt       |  13 ++
     .../structure_refinement/RefinementXray.cpp   | 102 ++++++++++++-
     .../structure_refinement/RefinementXray.h     |   6 +-
     tests/AtomGradientMetalTest.cpp               | 102 +++++++++++++
     tests/CMakeLists.txt                          |   1 +
     10 files changed, 529 insertions(+), 8 deletions(-)
     create mode 100644 image_analysis/structure_refinement/AtomGradientMetal.h
     create mode 100644 image_analysis/structure_refinement/AtomGradientMetal.metal
     create mode 100644 image_analysis/structure_refinement/AtomGradientMetal.mm
     create mode 100644 tests/AtomGradientMetalTest.cpp
    
    diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
    index eb27cc3b6..807d735a5 100644
    --- a/THIRD_PARTY_NOTICES.md
    +++ b/THIRD_PARTY_NOTICES.md
    @@ -80,7 +80,7 @@ These live in the source tree (see the path) rather than being fetched; traccc,
     | [traccc (ACTS)](https://github.com/acts-project/traccc) | `image_analysis/spot_finding/StrongPixelSet.cpp`, `SpotExtractorGPU.cu` | CERN, for the benefit of the ACTS project | MPL-2.0 | [traccc.txt](licenses/traccc.txt) |
     | [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (adapted) | `image_analysis/indexing/FFBIDXIndexerCPU.cpp` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](licenses/fast-feedback-indexer.txt) |
     | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) (adapted) | `image_analysis/geom_refinement/LMSolver.h`, `LMSolver.cpp` | Google Inc. | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) |
    -| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) |
    +| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal`, `AtomGradientMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) |
     | [Durin](https://github.com/DiamondLightSource/durin) (adapted) | `xds-plugin/plugin.h`, `plugin.cpp` | Diamond Light Source Ltd.; Global Phasing Ltd. | BSD-3-Clause | [durin.txt](licenses/durin.txt) |
     | [xbflash.qspi](https://github.com/Xilinx/XRT) | `tools/xbflash.qspi/` | Xilinx / AMD | Apache-2.0 | [xbflash-qspi.txt](licenses/xbflash-qspi.txt) |
     | [wingetopt](https://github.com/alex85k/wingetopt) | `tools/wingetopt/` | Todd C. Miller; The NetBSD Foundation | ISC AND BSD-2-Clause | [wingetopt.txt](licenses/wingetopt.txt) |
    @@ -154,8 +154,8 @@ served frontend, so the shipped web UI carries its own attribution.
       `FFBIDXIndexerCPU.cpp` ports its CUDA kernels to the CPU. The file names the origin at the top and
       is covered by `licenses/fast-feedback-indexer.txt`.
     * **GEMMI** is also vendored (table above); separately, `ModelScaling.cpp`, `ModelMaskGPU.cu`,
    -  `ModelStructureFactorsGPU.cpp` and `ModelDensityMetal.metal` in `image_analysis/structure_refinement/`
    -  follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
    +  `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` and `AtomGradientMetal.metal` in
    +  `image_analysis/structure_refinement/` follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
       Each names the GEMMI source it follows and is covered by `licenses/gemmi.txt`.
     * **Durin** (`xds-plugin/plugin.h`/`.cpp`): the XDS plugin's entry points are based on the Durin plugin
       of Diamond Light Source and Global Phasing; both files name the origin at the top and are covered by
    diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md
    index 82343dbdb..60f8ddf21 100644
    --- a/docs/GPU_DATA_ANALYSIS.md
    +++ b/docs/GPU_DATA_ANALYSIS.md
    @@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato
     
     ## Apple GPUs (Metal)
     
    -A macOS build has no CUDA, and three engines run on the Apple GPU instead (none under
    +A macOS build has no CUDA, and four engines run on the Apple GPU instead (none under
     `rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning):
     
     - **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its
    @@ -358,6 +358,10 @@ A macOS build has no CUDA, and three engines run on the Apple GPU instead (none
       for model validation and molecular replacement, as a gather over bricks of the grid as under CUDA;
       the same grid as the CPU's **to the bit**.
       [§14.1](CPU_DATA_ANALYSIS_DECISIONS.md#141-model-structure-factors)
    +- **Refinement gradient** (`AtomGradientMetal.mm`, `.metal`) - the sums of the CPU refinement
    +  engine's gradient over the FFT method's map, each atom's box walked by one SIMD group as under CUDA;
    +  the CPU's points, summed in single precision, so the gradient is the CPU's to a few parts in 1e5
    +  (a macOS CI step checks it against the CPU's). It halves the time of a refinement on the CPU engine.
     - **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the
       rotation function's trial orientations scored from a copy of the molecular transform; equal to the
       CPU's to rounding.
    diff --git a/image_analysis/structure_refinement/AtomGradientMetal.h b/image_analysis/structure_refinement/AtomGradientMetal.h
    new file mode 100644
    index 000000000..0e1ce706a
    --- /dev/null
    +++ b/image_analysis/structure_refinement/AtomGradientMetal.h
    @@ -0,0 +1,31 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#pragma once
    +
    +// AtomGradientsFromMap()'s sums (RefinementXray.cpp) on an Apple GPU (macOS only): every atom's box walked against
    +// the FFT method's map, one SIMD group of 32 threads per atom, as RefinementGPU.cu's atom_gradient_kernel does under
    +// CUDA. The points, Gaussians and arithmetic are the CPU's, but the sums are in single precision and in another
    +// order, so the gradient is the CPU's to a few parts in 1e5, not bit for bit. tests/AtomGradientMetalTest.cpp holds
    +// it to the CPU's.
    +
    +#include 
    +#include 
    +
    +#include "ModelDensityMetal.h"   // ModelDensityMetalAtom, ModelDensityMetalGrid
    +
    +// Per atom the nine sums of AtomGradientsFromMap() before occupancy and the constants: an isotropic atom's
    +// sum v dr2 x, y, z and sum v db (s[0..3]); an anisotropic atom's sum e (M r)_t (s[0..2]) and
    +// sum e (M + 2 (M r)(M r)^T) as u11 u22 u33 u12 u13 u23 (s[3..8]).
    +struct AtomGradientMetalSums {
    +    float s[9];
    +};
    +
    +// The sums of `atoms` (b with the addend 0, as the gradient takes them) over `map` (nu nv nw floats, u fastest) into
    +// `out`, one per atom. False where the GPU does not take it - no Metal device the process may use (rugnux --no-gpu
    +// hides it), or an atom whose box is wider than the cell; `out` is then untouched. Throws JFJochException on a
    +// Metal failure.
    +bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector &atoms,
    +                       const float *map, std::vector &out);
    +// How many gradients the GPU has summed in this process.
    +size_t AtomGradientMetalCalls();
    diff --git a/image_analysis/structure_refinement/AtomGradientMetal.metal b/image_analysis/structure_refinement/AtomGradientMetal.metal
    new file mode 100644
    index 000000000..d2ffc4ed6
    --- /dev/null
    +++ b/image_analysis/structure_refinement/AtomGradientMetal.metal
    @@ -0,0 +1,134 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only AND MPL-2.0
    +
    +// The refinement gradient's sums over the map (AtomGradientMetal.h), compiled when it is first used from this source,
    +// which CMakeLists.txt copies into the binary. The place of a point and the test against the radius are spelled as
    +// GatherMapPoints() (RefinementXray.cpp) spells them, with no a*b+c contracted, so that the GPU takes the CPU's points.
    +
    +#include 
    +using namespace metal;
    +
    +#pragma clang fp contract(off)
    +
    +struct Atom {   // ModelDensityMetalAtom
    +    float nctr[3];
    +    int c[3], d[3];
    +    float max_r2;
    +    float occ;
    +    int aniso;
    +    float a[5];
    +    float b[5][6];
    +};
    +
    +struct Grid {   // ModelDensityMetalGrid
    +    int nu, nv, nw;
    +    float m00, m01, m02, m11, m12, m22;
    +};
    +
    +struct Sums {   // AtomGradientMetalSums
    +    float s[9];
    +};
    +
    +// gemmi's unsafe_expapprox() (formfact.hpp, MPL-2.0; https://github.com/project-gemmi/gemmi), as in
    +// ModelDensityMetal.metal.
    +static float expapprox(float x) {
    +    const float val = 12102203.1615614f * x + 1065353216.f;
    +    const int vali = static_cast(val);
    +    const int xu1 = vali & 0x7F800000;
    +    const int xu2 = (vali & 0x7FFFFF) | 0x3F800000;
    +    const float a = as_type(xu1);
    +    const float b = as_type(xu2);
    +    return a * (0.509871020f + b * (0.312146713f + b * (0.166617139f + b *
    +            (-2.190619930e-3f + b * 1.3555747234e-2f))));
    +}
    +
    +static int wrap(int i, int n) {
    +    const int m = i % n;
    +    return m < 0 ? m + n : m;
    +}
    +
    +// One SIMD group of 32 threads per atom: the lanes take the box's points in turn, u fastest, each summing its own,
    +// and the group's sums are added at the end.
    +kernel void atom_gradient(device const Atom *atoms [[buffer(0)]], constant Grid &g [[buffer(1)]],
    +                          device const float *map [[buffer(2)]], device Sums *out [[buffer(3)]],
    +                          constant uint &n [[buffer(4)]],
    +                          uint group [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]],
    +                          uint groups_per_tg [[simdgroups_per_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
    +    const uint i = group * groups_per_tg + sg;
    +    if (i >= n)
    +        return;
    +    device const Atom &at = atoms[i];
    +    const int span[3] = {2 * at.d[0] + 1, 2 * at.d[1] + 1, 2 * at.d[2] + 1};
    +    const int lo[3] = {at.c[0] - at.d[0], at.c[1] - at.d[1], at.c[2] - at.d[2]};
    +    const int total = span[0] * span[1] * span[2];
    +    float s[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
    +    // the lane's point, as offsets in the box, moved on by 32 points at a time
    +    int iu = int(lane), iv = 0, iw = 0;
    +    while (iu >= span[0]) {
    +        iu -= span[0];
    +        if (++iv == span[1]) {
    +            iv = 0;
    +            iw++;
    +        }
    +    }
    +    for (int t = int(lane); t < total; t += 32) {
    +        const int u = lo[0] + iu, v = lo[1] + iv, w = lo[2] + iw;
    +        iu += 32;
    +        while (iu >= span[0]) {
    +            iu -= span[0];
    +            if (++iv == span[1]) {
    +                iv = 0;
    +                iw++;
    +            }
    +        }
    +        // GatherMapPoints()'s place and test
    +        const float fz = at.nctr[2] - float(w);
    +        const float z = g.m22 * fz;
    +        const float fy = at.nctr[1] - float(v);
    +        const float y = g.m11 * fy + g.m12 * fz;
    +        const float yz = y * y + z * z;
    +        const float x_yz = g.m01 * fy + g.m02 * fz;
    +        const float x = g.m00 * (at.nctr[0] - float(u)) + x_yz;
    +        const float r2 = yz + x * x;
    +        if (r2 > at.max_r2)
    +            continue;
    +        const float value = map[uint(wrap(u, g.nu)) + uint(g.nu) * (uint(wrap(v, g.nv)) + uint(g.nv) * uint(wrap(w, g.nw)))];
    +        if (at.aniso == 0) {
    +            float dr2 = 0, db = 0;
    +            for (int q = 0; q < 5; q++) {
    +                const float b = at.b[q][0];
    +                const float e = at.a[q] * expapprox(max(b * r2, -88.f));
    +                dr2 += e * b;
    +                db += e * b * (1.5f + b * r2);
    +            }
    +            s[0] += value * dr2 * x;
    +            s[1] += value * dr2 * y;
    +            s[2] += value * dr2 * z;
    +            s[3] += value * db;
    +        } else {
    +            for (int q = 0; q < 5; q++) {
    +                device const float *m = at.b[q];   // u11 u22 u33 u12 u13 u23
    +                // gemmi's SMat33::r_u_r(), in its order
    +                const float rur = x * x * m[0] + y * y * m[1] + z * z * m[2] + 2 * (x * y * m[3] + x * z * m[4] + y * z * m[5]);
    +                const float e = value * at.a[q] * expapprox(max(rur, -88.f));
    +                const float mr0 = m[0] * x + m[3] * y + m[4] * z;
    +                const float mr1 = m[3] * x + m[1] * y + m[5] * z;
    +                const float mr2 = m[4] * x + m[5] * y + m[2] * z;
    +                s[0] += e * mr0;
    +                s[1] += e * mr1;
    +                s[2] += e * mr2;
    +                s[3] += e * (m[0] + 2 * mr0 * mr0);
    +                s[4] += e * (m[1] + 2 * mr1 * mr1);
    +                s[5] += e * (m[2] + 2 * mr2 * mr2);
    +                s[6] += e * (m[3] + 2 * mr0 * mr1);
    +                s[7] += e * (m[4] + 2 * mr0 * mr2);
    +                s[8] += e * (m[5] + 2 * mr1 * mr2);
    +            }
    +        }
    +    }
    +    for (int k = 0; k < 9; k++)
    +        s[k] = simd_sum(s[k]);
    +    if (lane == 0)
    +        for (int k = 0; k < 9; k++)
    +            out[i].s[k] = s[k];
    +}
    diff --git a/image_analysis/structure_refinement/AtomGradientMetal.mm b/image_analysis/structure_refinement/AtomGradientMetal.mm
    new file mode 100644
    index 000000000..780187a2d
    --- /dev/null
    +++ b/image_analysis/structure_refinement/AtomGradientMetal.mm
    @@ -0,0 +1,136 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#import 
    +
    +#include "AtomGradientMetal.h"
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +#include "../../common/CUDAWrapper.h"   // gpus_are_hidden
    +#include "../../common/JFJochException.h"
    +
    +// The kernel's source, AtomGradientMetal.metal, copied in at configure time (CMakeLists.txt).
    +extern const char *const atom_gradient_metal_source;
    +
    +namespace {
    +
    +constexpr int ATOMS_PER_THREADGROUP = 8;   // SIMD groups of 32 threads
    +
    +std::atomic calls_on_gpu{0};
    +
    +static_assert(sizeof(AtomGradientMetalSums) == 9 * 4);
    +
    +struct Kernel {
    +    id device = nil;   // nil: no Metal device
    +    id pipeline = nil;
    +    id queue = nil;
    +};
    +
    +// Compiled once per process, the first time it is asked for.
    +const Kernel &GetKernel() {
    +    static const Kernel kernel = [] {
    +        Kernel k;
    +        k.device = MTLCreateSystemDefaultDevice();
    +        if (k.device == nil)
    +            return k;
    +        MTLCompileOptions *options = [MTLCompileOptions new];
    +        // Fast math off, so that a point's place rounds as on the CPU and the GPU takes the CPU's points.
    +        options.fastMathEnabled = NO;
    +        options.languageVersion = MTLLanguageVersion3_0;
    +        NSError *error = nil;
    +        id library = [k.device newLibraryWithSource:@(atom_gradient_metal_source) options:options
    +                                                          error:&error];
    +        if (library == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("atom gradient kernel: ") + error.localizedDescription.UTF8String);
    +        id function = [library newFunctionWithName:@"atom_gradient"];
    +        k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
    +        if (k.pipeline == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("atom_gradient: ") + error.localizedDescription.UTF8String);
    +        k.queue = [k.device newCommandQueue];
    +        return k;
    +    }();
    +    return kernel;
    +}
    +
    +id NewBuffer(id device, size_t bytes) {
    +    id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared];
    +    if (b == nil)
    +        throw JFJochException(JFJochExceptionCategory::GPUMetalError, "atom gradient: out of GPU memory");
    +    return b;
    +}
    +
    +}  // namespace
    +
    +bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector &atoms,
    +                       const float *map, std::vector &out) {
    +    // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
    +    @autoreleasepool {
    +        const size_t points = static_cast(grid.nu) * grid.nv * grid.nw;
    +        if (gpus_are_hidden() || points > std::numeric_limits::max() ||
    +            atoms.size() > std::numeric_limits::max())
    +            return false;
    +        const int n3[3] = {grid.nu, grid.nv, grid.nw};
    +        for (const ModelDensityMetalAtom &a : atoms)
    +            for (int k = 0; k < 3; k++)
    +                if (2 * a.d[k] + 1 > n3[k])
    +                    return false;   // a box wider than the cell: the CPU's walk adds both images of a point
    +        const Kernel &k = GetKernel();
    +        if (k.device == nil)
    +            return false;
    +        if (atoms.empty()) {
    +            out.clear();
    +            return true;
    +        }
    +
    +        // The map as it is where its pages can be handed to the GPU, a copy otherwise.
    +        const size_t page = static_cast(getpagesize());
    +        const size_t bytes = points * sizeof(float);
    +        id d_map = nil;
    +        if (reinterpret_cast(map) % page == 0)
    +            d_map = [k.device newBufferWithBytesNoCopy:const_cast(map) length:(bytes + page - 1) / page * page
    +                                               options:MTLResourceStorageModeShared deallocator:nil];
    +        if (d_map == nil) {
    +            d_map = NewBuffer(k.device, bytes);
    +            std::memcpy(d_map.contents, map, bytes);
    +        }
    +        id d_atoms = NewBuffer(k.device, atoms.size() * sizeof(ModelDensityMetalAtom));
    +        std::memcpy(d_atoms.contents, atoms.data(), atoms.size() * sizeof(ModelDensityMetalAtom));
    +        id d_out = NewBuffer(k.device, atoms.size() * sizeof(AtomGradientMetalSums));
    +        const uint32_t n = static_cast(atoms.size());
    +
    +        id cb = [k.queue commandBuffer];
    +        id e = [cb computeCommandEncoder];
    +        [e setComputePipelineState:k.pipeline];
    +        [e setBuffer:d_atoms offset:0 atIndex:0];
    +        [e setBytes:&grid length:sizeof(grid) atIndex:1];
    +        [e setBuffer:d_map offset:0 atIndex:2];
    +        [e setBuffer:d_out offset:0 atIndex:3];
    +        [e setBytes:&n length:sizeof(n) atIndex:4];
    +        [e dispatchThreadgroups:MTLSizeMake((n + ATOMS_PER_THREADGROUP - 1) / ATOMS_PER_THREADGROUP, 1, 1)
    +          threadsPerThreadgroup:MTLSizeMake(32 * ATOMS_PER_THREADGROUP, 1, 1)];
    +        [e endEncoding];
    +        [cb commit];
    +        [cb waitUntilCompleted];
    +        if (cb.status != MTLCommandBufferStatusCompleted)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  cb.error ? cb.error.localizedDescription.UTF8String
    +                                           : "atom gradient: command buffer failed");
    +        out.resize(atoms.size());
    +        std::memcpy(out.data(), d_out.contents, atoms.size() * sizeof(AtomGradientMetalSums));
    +        calls_on_gpu++;
    +        return true;
    +    }
    +}
    +
    +size_t AtomGradientMetalCalls() {
    +    return calls_on_gpu;
    +}
    diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt
    index 54e66051f..f513855cf 100644
    --- a/image_analysis/structure_refinement/CMakeLists.txt
    +++ b/image_analysis/structure_refinement/CMakeLists.txt
    @@ -108,5 +108,18 @@ IF (APPLE)
                 ModelDensityMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/ModelDensityMetalSource.cpp)
         SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
         SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
    +    # The refinement gradient's sums over the map (AtomGradientMetal.h), carried in the same way.
    +    FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal text)
    +    SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal)
    +    FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp
    +         "// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
    +         "extern const char *const atom_gradient_metal_source;\n"
    +         "const char *const atom_gradient_metal_source = R\"GRADIENT_METAL(\n${text})GRADIENT_METAL\";\n")
    +    CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp
    +                   ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp COPYONLY)
    +    TARGET_SOURCES(JFJochStructureRefinement PRIVATE AtomGradientMetal.mm AtomGradientMetal.h
    +            AtomGradientMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp)
    +    SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
    +    SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
         TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation")
     ENDIF()
    diff --git a/image_analysis/structure_refinement/RefinementXray.cpp b/image_analysis/structure_refinement/RefinementXray.cpp
    index 3544710fb..956d7c9ae 100644
    --- a/image_analysis/structure_refinement/RefinementXray.cpp
    +++ b/image_analysis/structure_refinement/RefinementXray.cpp
    @@ -16,12 +16,19 @@
     #include "gemmi/math.hpp"
     #include "gemmi/twin.hpp"
     
    +#include "../../common/JFJochException.h"
     #include "../../common/Logger.h"
     #include "../../common/ParallelFor.h"
     #include "ModelFFT.h"
     #include "ModelGrid.h"
     #include "RefinementMath.h"
     
    +#ifdef __APPLE__
    +#include 
    +
    +#include "AtomGradientMetal.h"
    +#endif
    +
     namespace {
     
     using Table = gemmi::IT92;
    @@ -230,7 +237,7 @@ public:
                     map_.data[i] *= norm;
             });
             const bool aniso = AnyAniso();
    -        AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr);
    +        AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr, true);
             if (!aniso)
                 grad_u_.clear();
             return sum;
    @@ -919,10 +926,97 @@ void AnisoGradient(const gemmi::Grid &map, const gemmi::DensityCalculator
             *grad_u = {occ * gu[0], occ * gu[1], occ * gu[2], 2 * occ * gu[3], 2 * occ * gu[4], 2 * occ * gu[5]};
     }
     
    +#ifdef __APPLE__
    +// AtomGradientsFromMap() with the sums over the map on the Apple GPU (AtomGradientMetal.h): each atom's box, radius and
    +// Gaussians as the CPU's loops below take them, then the sums turned into the derivatives as there. False where the
    +// GPU does not take it - and from the first time it fails on, with one warning, as the CPU then does the same work.
    +std::atomic gradient_metal_failed{false};
    +
    +bool AtomGradientsFromMapMetal(const gemmi::Grid &map, const gemmi::DensityCalculator &dc,
    +                               const std::vector &atoms, double blur, size_t nthreads,
    +                               std::vector> &grad, std::vector> *grad_u) {
    +    if (gradient_metal_failed)
    +        return false;
    +    using CReal = Table::Coef::coef_type;
    +    const gemmi::UpperTriangularMat33 &om = map.orth_n;
    +    const ModelDensityMetalGrid g{map.nu, map.nv, map.nw, static_cast(om.a11), static_cast(om.a12),
    +                                  static_cast(om.a13), static_cast(om.a22), static_cast(om.a23),
    +                                  static_cast(om.a33)};
    +    const int n = static_cast(atoms.size());
    +    std::vector in(n);
    +    ParallelChunks(n, nthreads, [&](int lo, int hi) {
    +        for (int i = lo; i < hi; i++) {
    +            const gemmi::Atom &atom = *atoms[i];
    +            ModelDensityMetalAtom &a = in[i];
    +            const auto &coef = IonFormFactor(atom.element.elem, atom.charge);
    +            double radius;
    +            a.aniso = atom.aniso.nonzero() ? 1 : 0;
    +            for (auto &b : a.b)
    +                std::fill(std::begin(b), std::end(b), 0.0f);
    +            if (!a.aniso) {
    +                const float b = static_cast(atom.b_iso + blur);
    +                const auto precal = coef.precalculate_density_iso(b, 0.0f);
    +                radius = dc.estimate_radius(precal, b);
    +                for (int q = 0; q < 5; q++) {
    +                    a.a[q] = precal.a[q];
    +                    a.b[q][0] = precal.b[q];
    +                }
    +            } else {
    +                const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
    +                const CReal b_max = AnisoRadiusB(aniso_b);
    +                radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
    +                const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
    +                for (int q = 0; q < 5; q++) {
    +                    const gemmi::SMat33 &m = precal.b[q];
    +                    a.a[q] = precal.a[q];
    +                    const float row[6] = {m.u11, m.u22, m.u33, m.u12, m.u13, m.u23};
    +                    std::copy(std::begin(row), std::end(row), a.b[q]);
    +                }
    +            }
    +            const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
    +            const double n3[3] = {fpos.x * map.nu, fpos.y * map.nv, fpos.z * map.nw};
    +            for (int k = 0; k < 3; k++) {
    +                a.nctr[k] = static_cast(n3[k]);
    +                a.c[k] = gemmi::iround(n3[k]);
    +            }
    +            a.d[0] = std::min(static_cast(std::ceil(radius / map.spacing[0])), map.nu - 1);
    +            a.d[1] = std::min(static_cast(std::ceil(radius / map.spacing[1])), map.nv - 1);
    +            a.d[2] = std::min(static_cast(std::ceil(radius / map.spacing[2])), map.nw - 1);
    +            a.max_r2 = static_cast(radius * radius);
    +            a.occ = atom.occ;
    +        }
    +    });
    +    std::vector sums;
    +    try {
    +        if (!AtomGradientMetal(g, in, map.data.data(), sums))
    +            return false;
    +    } catch (const JFJochException &e) {
    +        if (!gradient_metal_failed.exchange(true))
    +            Logger("Rugnux").Warning("Refinement gradient: the Apple GPU failed, on the CPU from here on: {}", e.what());
    +        return false;
    +    }
    +    constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
    +    for (int i = 0; i < n; i++) {
    +        const float *s = sums[i].s;
    +        const double occ = atoms[i]->occ;
    +        if (!in[i].aniso) {
    +            grad[i] = {2.0 * occ * s[0], 2.0 * occ * s[1], 2.0 * occ * s[2], occ * s[3] / FOUR_PI2};
    +            continue;
    +        }
    +        grad[i] = {2 * occ * s[0], 2 * occ * s[1], 2 * occ * s[2],
    +                   occ * (double(s[3]) + s[4] + s[5]) / gemmi::u_to_b()};
    +        if (grad_u != nullptr)
    +            (*grad_u)[i] = {occ * s[3], occ * s[4], occ * s[5], 2 * occ * s[6], 2 * occ * s[7], 2 * occ * s[8]};
    +    }
    +    return true;
    +}
    +#endif
    +
     }  // namespace
     
     void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads,
    -                          std::vector> &grad, std::vector> *grad_u) {
    +                          std::vector> &grad, std::vector> *grad_u,
    +                          [[maybe_unused]] bool gpu) {
         const gemmi::DensityCalculator dc = Calculator(map.unit_cell, *map.spacegroup, 1.0, blur);
         std::vector atoms;
         for (const gemmi::Chain &ch : model.chains)
    @@ -932,6 +1026,10 @@ void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &mod
         grad.assign(atoms.size(), {0, 0, 0, 0});
         if (grad_u != nullptr)
             grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0});
    +#ifdef __APPLE__
    +    if (gpu && AtomGradientsFromMapMetal(map, dc, atoms, blur, nthreads, grad, grad_u))
    +        return;
    +#endif
         const int nu = map.nu, nv = map.nv, nw = map.nw;
         constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
         // in blocks of atoms taken as the threads come free: an anisotropic atom, or one of high B, costs several times
    diff --git a/image_analysis/structure_refinement/RefinementXray.h b/image_analysis/structure_refinement/RefinementXray.h
    index 66bf1dd44..3294fcc19 100644
    --- a/image_analysis/structure_refinement/RefinementXray.h
    +++ b/image_analysis/structure_refinement/RefinementXray.h
    @@ -288,7 +288,9 @@ std::unique_ptr MakeRefinementEngineCPU(const gemmi::UnitCell
     // derivatives: dT/d(x, y, z, B) per atom. `map` is u fastest on the grid of RigidBodyZoneGrid(), the atoms
     // gridded with `blur`. Exposed for the tests. An atom with a nonzero aniso is gridded anisotropically, as
     // PutModelDensityOnGrid() grids it: its B derivative is that along U + dB / (8 pi^2) I, and with `grad_u` (resized
    -// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23.
    +// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23. With `gpu`, on the Apple GPU
    +// where there is one (AtomGradientMetal.h, macOS): the same points summed in single precision, the CPU's gradient to
    +// a few parts in 1e5 rather than bit for bit.
     void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads,
                               std::vector> &grad,
    -                          std::vector> *grad_u = nullptr);
    +                          std::vector> *grad_u = nullptr, bool gpu = false);
    diff --git a/tests/AtomGradientMetalTest.cpp b/tests/AtomGradientMetalTest.cpp
    new file mode 100644
    index 000000000..0ed3040dc
    --- /dev/null
    +++ b/tests/AtomGradientMetalTest.cpp
    @@ -0,0 +1,102 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#include 
    +
    +#ifdef __APPLE__
    +
    +#include 
    +#include 
    +#include 
    +
    +#include 
    +
    +#include "../image_analysis/structure_refinement/AtomGradientMetal.h"
    +#include "../image_analysis/structure_refinement/RefinementXray.h"
    +#include "../image_analysis/structure_refinement/RigidBodyRefine.h"
    +
    +// The refinement gradient's sums on the Apple GPU (AtomGradientMetal.h) against the CPU's: the same points, summed
    +// in single precision on the GPU and in double on the CPU. A Mac without a Metal device fails rather than skips, as
    +// in ModelDensityMetalTest.cpp.
    +
    +namespace {
    +    // Atoms scattered over the whole cell - so that boxes wrap at every face - of a few elements, a quarter of them
    +    // anisotropic, with occupancies below one.
    +    gemmi::Model random_atoms(const gemmi::UnitCell &cell) {
    +        std::mt19937 rng(17);
    +        std::uniform_real_distribution frac(0.0, 1.0), b(8.0, 60.0), occ(0.3, 1.0), u(0.05, 0.4);
    +        const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S};
    +        gemmi::Model model(1);
    +        gemmi::Chain chain("A");
    +        gemmi::Residue res;
    +        res.name = "UNK";
    +        res.seqid = gemmi::SeqId(1, ' ');
    +        for (int i = 0; i < 2000; i++) {
    +            gemmi::Atom a;
    +            a.name = "X";
    +            a.element = elements[i % 4];
    +            a.pos = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng)));
    +            a.b_iso = static_cast(b(rng));
    +            a.occ = static_cast(occ(rng));
    +            if (i % 4 == 3) {
    +                const float d1 = static_cast(u(rng)), d2 = static_cast(u(rng)),
    +                            d3 = static_cast(u(rng));
    +                a.aniso = {d1, d2, d3, 0.3f * std::min(d1, d2), -0.2f * std::min(d1, d3), 0.1f * std::min(d2, d3)};
    +            }
    +            res.atoms.push_back(a);
    +        }
    +        chain.residues.push_back(res);
    +        model.chains.push_back(chain);
    +        return model;
    +    }
    +
    +    // The largest difference of a derivative between the two, over the largest of that derivative on the CPU.
    +    template 
    +    double worst(const std::vector> &cpu, const std::vector> &gpu) {
    +        double out = 0;
    +        for (size_t c = 0; c < N; c++) {
    +            double max_g = 0, max_d = 0;
    +            for (size_t i = 0; i < cpu.size(); i++) {
    +                max_g = std::max(max_g, std::fabs(cpu[i][c]));
    +                max_d = std::max(max_d, std::fabs(gpu[i][c] - cpu[i][c]));
    +            }
    +            out = std::max(out, max_d / max_g);
    +        }
    +        return out;
    +    }
    +
    +    void check_gradient(const gemmi::UnitCell &cell, const char *group, double d_min) {
    +        const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group);
    +        gemmi::Grid map = RigidBodyZoneGrid(cell, sg, d_min);
    +        map.data.resize(map.point_count());
    +        // a smooth map with both signs, as the FFT method's derivative map is
    +        for (int w = 0; w < map.nw; w++)
    +            for (int v = 0; v < map.nv; v++)
    +                for (int u = 0; u < map.nu; u++)
    +                    map.data[map.index_q(u, v, w)] = static_cast(
    +                        std::sin(2 * M_PI * (3.0 * u / map.nu + 1.0 * v / map.nv)) +
    +                        std::cos(2 * M_PI * (2.0 * v / map.nv - 4.0 * w / map.nw)) +
    +                        0.5 * std::sin(2 * M_PI * (5.0 * u / map.nu + 3.0 * w / map.nw)));
    +        const gemmi::Model model = random_atoms(cell);
    +        const double blur = 10.0;
    +        std::vector> gc, gg;
    +        std::vector> uc, ug;
    +        AtomGradientsFromMap(map, model, blur, 4, gc, &uc, false);
    +        const size_t before = AtomGradientMetalCalls();
    +        AtomGradientsFromMap(map, model, blur, 4, gg, &ug, true);
    +        REQUIRE(AtomGradientMetalCalls() == before + 1);   // the GPU summed it
    +        REQUIRE(gg.size() == gc.size());
    +        const double g = worst(gc, gg), u = worst(uc, ug);
    +        CAPTURE(group, g, u);
    +        CHECK(g < 1e-5);
    +        CHECK(u < 5e-5);
    +    }
    +}
    +
    +TEST_CASE("AtomGradientMetal_MatchesCPU", "[portable][Metal]") {
    +    // Triclinic, so that every term of the orthogonalization is there; then an orthorhombic group.
    +    check_gradient(gemmi::UnitCell(41, 46, 52, 81, 96, 107), "P 1", 2.0);
    +    check_gradient(gemmi::UnitCell(50, 56, 63, 90, 90, 90), "P 21 21 21", 1.6);
    +}
    +
    +#endif
    diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
    index 17b290a3e..ce0055c8f 100644
    --- a/tests/CMakeLists.txt
    +++ b/tests/CMakeLists.txt
    @@ -35,6 +35,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
             BraggIntegrationEngineMetalTest.cpp
             MolecularReplacementMetalTest.cpp
             ModelDensityMetalTest.cpp
    +        AtomGradientMetalTest.cpp
             WriteReflectionsTest.cpp
             LoadReferenceMtzFreeFlagsTest.cpp
             LoadReferenceSfCifTest.cpp
    
    From d3f537aac69e232ed10397c08bb6753be8b1fdb4 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 05/19] -S with -C: the cell goes onto the given axes by the
     nearest relabelling
    
    CellMappingOperators admits every relabelling within 5% / 3 deg of the given cell, so on a
    cell with two lengths that close more than one maps; ops.front() took the first in
    enumeration order (the identity where it qualifies) even where another put the cell on the
    typed axes better, e.g. a pure 2-fold given on the shorter of two near-equal axes. The
    nearest one is now taken (the OnGivenAxes distance), the identity on a tie.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     rugnux/RugnuxScaleMerge.cpp | 19 ++++++++++++++++---
     1 file changed, 16 insertions(+), 3 deletions(-)
    
    diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp
    index a48b88e5f..d703e1579 100644
    --- a/rugnux/RugnuxScaleMerge.cpp
    +++ b/rugnux/RugnuxScaleMerge.cpp
    @@ -1200,11 +1200,24 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
             for (const auto &[hkl, sum] : summed)
                 axial.push_back({hkl, sum.first / std::sqrt(sum.second)});
             // With the cell given too, the setting is the user's: the cell goes onto the given cell's axes.
    +        // Where two lengths are within the 5% CellsCorrespond allows, more than one relabelling maps the
    +        // cell onto the given one, and the given axes are those of the nearest (the identity on a tie).
             gemmi::Op cob = gemmi::Op::identity();
             if (user_cell_) {
    -            const auto ops = CellMappingOperators(gemmi::UnitCell(*result.consensus_cell), gemmi::UnitCell(*user_cell_));
    -            if (!ops.empty())
    -                cob = ops.front();
    +            const UnitCell &given = *user_cell_;
    +            double best = std::numeric_limits::infinity();
    +            for (const gemmi::Op &op : CellMappingOperators(gemmi::UnitCell(*result.consensus_cell),
    +                                                            gemmi::UnitCell(given))) {
    +                const UnitCell m = CellInBasis(*result.consensus_cell, op);
    +                const double d = std::fabs(m.alpha - given.alpha) + std::fabs(m.beta - given.beta)
    +                                 + std::fabs(m.gamma - given.gamma)
    +                                 + 100.0 * (std::fabs(m.a - given.a) / given.a + std::fabs(m.b - given.b) / given.b
    +                                            + std::fabs(m.c - given.c) / given.c);
    +                if (d < best) {
    +                    best = d;
    +                    cob = op;
    +                }
    +            }
             } else
                 cob = SeatGroupByAbsences(*result.consensus_cell, *user_fixed_sg_, axial);
             if (!(cob == gemmi::Op::identity())) {
    
    From 5d95974f37e4aec09f0764a43a1016476285b3ea Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 06/19] LatticeSearchForSystem: ask only for the system's own
     Bravais centrings
    
    The table has no character of an A/B-centred class, nor of a C/F/R one outside its systems,
    so 6 of the 7 searches were empty on a triclinic -C cell and 3-5 elsewhere. Same result: a
    randomized probe over P/C/I/F/R lattices of every system (3000 cells, rotated, near-equal
    lengths) never found a class outside the new lists.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     image_analysis/lattice_search/LatticeSearch.cpp | 9 ++++++++-
     1 file changed, 8 insertions(+), 1 deletion(-)
    
    diff --git a/image_analysis/lattice_search/LatticeSearch.cpp b/image_analysis/lattice_search/LatticeSearch.cpp
    index dd22b9fc8..f51e3e22d 100644
    --- a/image_analysis/lattice_search/LatticeSearch.cpp
    +++ b/image_analysis/lattice_search/LatticeSearch.cpp
    @@ -569,8 +569,15 @@ std::optional LatticeSearchForGroup(const CrystalLattice &L
     
     std::optional LatticeSearchForSystem(const CrystalLattice &L, gemmi::CrystalSystem system) {
         const bool hexagonal_family = system == gemmi::CrystalSystem::Hexagonal || system == gemmi::CrystalSystem::Trigonal;
    +    // The system's Bravais lattices: the table above has characters of no other centring.
    +    const std::string centrings = system == gemmi::CrystalSystem::Triclinic ? "P"
    +                                  : system == gemmi::CrystalSystem::Monoclinic ? "PCI"
    +                                  : system == gemmi::CrystalSystem::Orthorhombic ? "PCIF"
    +                                  : system == gemmi::CrystalSystem::Tetragonal ? "PI"
    +                                  : system == gemmi::CrystalSystem::Cubic ? "PIF"
    +                                  : "PR";
         std::optional best;
    -    for (const char c : {'P', 'C', 'A', 'B', 'I', 'F', 'R'}) {
    +    for (const char c : centrings) {
             const auto s = !hexagonal_family ? system
                            : c == 'R' ? gemmi::CrystalSystem::Trigonal : gemmi::CrystalSystem::Hexagonal;
             if (auto r = LatticeSearchForClass(L, s, c); r && (!best || MetricDeviation(*r) < MetricDeviation(*best)))
    
    From cebf90af1089880ef6b4061bb65537b7f26e6dfb Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 07/19] LatticeSearch.h: the LatticePointGroupOrder comment
     back above its declaration
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     image_analysis/lattice_search/LatticeSearch.h | 4 ++--
     1 file changed, 2 insertions(+), 2 deletions(-)
    
    diff --git a/image_analysis/lattice_search/LatticeSearch.h b/image_analysis/lattice_search/LatticeSearch.h
    index 28419d6fe..1ddb1afdd 100644
    --- a/image_analysis/lattice_search/LatticeSearch.h
    +++ b/image_analysis/lattice_search/LatticeSearch.h
    @@ -48,8 +48,6 @@ std::optional LatticeSearchForClass(const CrystalLattice& L
                                                              double dist_tolerance = LATTICE_SEARCH_DIST_TOLERANCE,
                                                              double angle_tolerance_deg = LATTICE_SEARCH_ANGLE_TOLERANCE_DEG);
     
    -// Order of the lattice point group, so "lower symmetry" is a well-defined comparison
    -// (gemmi's enum orders Trigonal after Tetragonal, which have 6 and 8 rotations).
     // The lattice of a space group the user fixed, searched for in L's metric: L is put in the setting of
     // the group's own Bravais class and never promoted past it (a P 1 run stays triclinic on a metrically
     // monoclinic cell). A trigonal-P group sits on a hexagonal-P lattice; a primitive group the lattice
    @@ -64,6 +62,8 @@ std::optional LatticeSearchForGroup(const CrystalLattice &L
     // The hexagonal family takes a rhombohedral lattice as well. Empty where the metric carries none.
     std::optional LatticeSearchForSystem(const CrystalLattice &L, gemmi::CrystalSystem system);
     
    +// Order of the lattice point group, so "lower symmetry" is a well-defined comparison
    +// (gemmi's enum orders Trigonal after Tetragonal, which have 6 and 8 rotations).
     int LatticePointGroupOrder(gemmi::CrystalSystem system);
     
     // Whether the conventional metric of this class asserts an equality between cell LENGTHS (a = b, and
    
    From b3e20dd62bb42498d905a8f95b26ae2524aeafda Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 08/19] ProperSubgroup: the translations count, on some origin
     of the subgroup
    
    Comparing rotations only passed a model group with a pure 2-fold where the data's group has a
    2(1) (P 1 2 1 against P 21 21 2): the model was then refined in its own group, its copies
    overlaid where the data have none, and the twin arm switched off. Each operator must now be
    one of the larger group's once the subgroup's origin is moved by some p (an operator (R, t)
    reads (R, t + (R - 1) p)), so P 1 21 1 in P 21 21 21 (origin z + 1/4) still passes.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     .../structure_refinement/Refinement.cpp       | 26 ++++++++++++++++++-
     .../structure_refinement/Refinement.h         |  3 ++-
     tests/RefinementTest.cpp                      |  5 ++++
     3 files changed, 32 insertions(+), 2 deletions(-)
    
    diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp
    index 924ed0803..a5c173e54 100644
    --- a/image_analysis/structure_refinement/Refinement.cpp
    +++ b/image_analysis/structure_refinement/Refinement.cpp
    @@ -4150,7 +4150,31 @@ bool ProperSubgroup(const gemmi::GroupOps &sub, const gemmi::GroupOps &super) {
         for (const gemmi::Op::Tran &cen : sub.cen_ops)
             if (std::find(super.cen_ops.begin(), super.cen_ops.end(), cen) == super.cen_ops.end())
                 return false;
    -    return true;
    +    // The translations too, on some origin of sub: moved by p, its operator (R, t) reads (R, t + (R - 1) p). The
    +    // rotations alone also pass a 2 where super has a 2(1) - P 1 2 1 is a subgroup of P 21 21 2 on no origin, and a
    +    // model refined under it would overlay copies where the data have none.
    +    constexpr int D = gemmi::Op::DEN;
    +    for (int px = 0; px < D; ++px)
    +        for (int py = 0; py < D; ++py)
    +            for (int pz = 0; pz < D; ++pz) {
    +                const int p[3] = {px, py, pz};
    +                bool all = true;
    +                for (const gemmi::Op &op : sub.sym_ops) {
    +                    gemmi::Op moved = op;
    +                    for (int i = 0; i < 3; ++i)
    +                        moved.tran[i] += (op.rot[i][0] * px + op.rot[i][1] * py + op.rot[i][2] * pz) / D - p[i];
    +                    moved.wrap();
    +                    const gemmi::Op &s = *super.find_by_rotation(op.rot);
    +                    if (std::none_of(super.cen_ops.begin(), super.cen_ops.end(),
    +                                     [&](const gemmi::Op::Tran &cen) { return s.add_centering(cen) == moved; })) {
    +                        all = false;
    +                        break;
    +                    }
    +                }
    +                if (all)
    +                    return true;
    +            }
    +    return false;
     }
     
     std::vector ExpandToSubgroup(const std::vector &merged, const gemmi::SpaceGroup &from,
    diff --git a/image_analysis/structure_refinement/Refinement.h b/image_analysis/structure_refinement/Refinement.h
    index 484001fd5..08861f2a7 100644
    --- a/image_analysis/structure_refinement/Refinement.h
    +++ b/image_analysis/structure_refinement/Refinement.h
    @@ -284,7 +284,8 @@ struct RefinementResult {
     // As REFMAC5 keeps twin-related reflections in one set (Roversi et al. (2012) Acta Cryst. D68, 418-424)
     int CloseFreeSet(std::vector &merged, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg);
     
    -// Whether every operation of `sub` is one of `super`'s (its rotation, and its centring), `super` the larger group:
    +// Whether every operation of `sub` is one of `super`'s (its rotation, its translation on some origin of `sub`, and its
    +// centring), `super` the larger group:
     // `sub` is then a group the data merged in `super` could not tell from it (a P2(1) crystal whose copies are
     // pseudo-symmetric, merged in P4(1)2(1)2).
     bool ProperSubgroup(const gemmi::GroupOps &sub, const gemmi::GroupOps &super);
    diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp
    index 7f84cf8f4..cfe691ecc 100644
    --- a/tests/RefinementTest.cpp
    +++ b/tests/RefinementTest.cpp
    @@ -1281,6 +1281,11 @@ TEST_CASE("Refinement_model_in_a_subgroup_of_the_data", "[Refinement]") {
         CHECK(!ProperSubgroup(super.operations(), sub.operations()));
         CHECK(!ProperSubgroup(sub.operations(), sub.operations()));
         CHECK(!ProperSubgroup(gemmi::find_spacegroup_by_name("C 1 2 1")->operations(), super.operations()));
    +    // the translations count, on any origin of the subgroup
    +    const gemmi::GroupOps p21212 = gemmi::find_spacegroup_by_name("P 21 21 2")->operations();
    +    CHECK(ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 21 1")->operations(), p21212));
    +    CHECK(ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 1 2")->operations(), p21212));
    +    CHECK(!ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 2 1")->operations(), p21212));
         gemmi::Structure truth = residue_crystal({"TRP"}, cell, "P 1 21 1", 5);
         {
             // the residue's mate by the 2-fold along a, exact: the crystal is then P 2 2(1) 1 (pseudo-P2(1)2(1)2(1))
    
    From b9a1cde246e603f9a5543015277e63277098625d Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 09/19] Frontend: plotly factory interop, shell plots sized on
     their own box, downloads null check
    
    - Plot.jsx: (factory.default ?? factory)(Plotly), so an interop that unwraps the CommonJS
      default also works.
    - RugnuxPage: the square shell plots took their side from the whole right panel's height, so
      with the checks box (up to 40%) above them they overflowed on height-limited windows; the
      side now comes from the plots' own division.
    - jfjoch_downloads.js: return where the version element is missing.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/_static/jfjoch_downloads.js       |  5 +++--
     frontend/src/components/Plot.jsx       |  5 +++--
     frontend/src/components/RugnuxPage.tsx | 15 +++++++++------
     3 files changed, 15 insertions(+), 10 deletions(-)
    
    diff --git a/docs/_static/jfjoch_downloads.js b/docs/_static/jfjoch_downloads.js
    index ed4c16457..f42df731f 100644
    --- a/docs/_static/jfjoch_downloads.js
    +++ b/docs/_static/jfjoch_downloads.js
    @@ -16,8 +16,9 @@
             return;
         }
         var blocks = document.querySelectorAll(".jfj-dl");
    -    if (!blocks.length || !window.fetch) return;
    -    var version = blocks[0].querySelector("[data-jfj=version]").textContent;
    +    var versionElement = blocks.length ? blocks[0].querySelector("[data-jfj=version]") : null;
    +    if (!versionElement || !window.fetch) return;
    +    var version = versionElement.textContent;
         var API = "https://gitea.psi.ch/api/v1/repos/mx/jungfraujoch/releases/tags/" + encodeURIComponent(version);
     
         function megabytes(bytes) {
    diff --git a/frontend/src/components/Plot.jsx b/frontend/src/components/Plot.jsx
    index 15fdb8bb8..10c34f2aa 100644
    --- a/frontend/src/components/Plot.jsx
    +++ b/frontend/src/components/Plot.jsx
    @@ -9,8 +9,9 @@ import reactPlotlyFactory from "react-plotly.js/factory";
     import Plotly from "plotly.js-cartesian-dist-min";
     
     // react-plotly.js is CommonJS (exports.default); in a "type": "module" package Vite 8 gives the default
    -// import as its module.exports, as Node does, so the factory is the .default of that.
    -const PlotlyPlot = reactPlotlyFactory.default(Plotly);
    +// import as its module.exports, as Node does, so the factory is the .default of that - and the import
    +// itself where an interop has already unwrapped it.
    +const PlotlyPlot = (reactPlotlyFactory.default ?? reactPlotlyFactory)(Plotly);
     
     // Plotly's `responsive` config follows the window only; a plot's own box also changes when the
     // drawer opens or closes, so the plot is redrawn to its box whenever that resizes.
    diff --git a/frontend/src/components/RugnuxPage.tsx b/frontend/src/components/RugnuxPage.tsx
    index 52c464c7e..cf0cd0d22 100644
    --- a/frontend/src/components/RugnuxPage.tsx
    +++ b/frontend/src/components/RugnuxPage.tsx
    @@ -222,14 +222,15 @@ function ShellPlotsOrLater({s, height, row}: { s: rugnux_live_status, height: nu
     
     // The right panel: the checks, and under them the shell plots, small and square, one above the other
     // (side: their side in px; sx: how the plots' division takes its height - on the stage, what the checks leave)
    -function RightPanel({s, pal, side, sx}: { s: rugnux_live_status, pal: LivePalette, side: number, sx?: object }) {
    +function RightPanel({s, pal, side, sx, plotsRef}: { s: rugnux_live_status, pal: LivePalette, side: number, sx?: object,
    +    plotsRef?: React.Ref }) {
         return (
             <>
                  *": {flexShrink: 0}}}>
                     
                 
                 By resolution{s.stills && s.shells_merge && ` · ${s.shells_merge}`}} sx={sx}>
    -                
    +                
                         
                     
                 
    @@ -255,9 +256,11 @@ const AXIS_TITLE = 18;   // what the lower of two stacked shell plots adds for t
     function StageDashboard({s, name, givenCell, deadline_s, modelLoaded, measuring, pal}: DashboardProps) {
         const [stripRef, , stripHeight] = useSize();
         const [imageRef, , imageHeight] = useSize();
    -    const [rightRef, rightWidth, rightHeight] = useSize();
    -    // the shell plots, square, one above the other in the right panel: as large as its width or height allows
    -    const side = Math.max(120, Math.min(rightWidth, (rightHeight - AXIS_TITLE) / 2));
    +    const [rightRef, rightWidth] = useSize();
    +    const [plotsRef, , plotsHeight] = useSize();
    +    // the shell plots, square, one above the other in the right panel: as large as its width or the height the
    +    // checks leave them allows
    +    const side = Math.max(120, Math.min(rightWidth, (plotsHeight - AXIS_TITLE) / 2));
         const hairline = {borderLeft: "1px solid", borderColor: "divider", pl: 2.5};
         return (
             
    @@ -280,7 +283,7 @@ function StageDashboard({s, name, givenCell, deadline_s, modelLoaded, measuring,
                     
                     {/* the checks over the shell plots, small and square - a point a shell */}
                     
    -                    
    +                    
                     
                     {/* the collection image by image along the bottom, wide - a point an image */}
                     
    
    From 8bceabbde2450980fc898ef36fca9fb29cab3b71 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:48:55 +0100
    Subject: [PATCH 10/19] rugnux docs/usage: -C semantics in the usage, -z with
     several stills files and in --mode refine
    
    - usage: -C without -S fixes the crystal system from the angles as typed, on the typed axes;
      RUGNUX_CLI.md says the axes as typed too.
    - RUGNUX_ADVANCED.md: a reference -z lifts the group check of several _process.h5 files as
      well as the cell check (it fixes the space group too, rugnux_cli.cpp).
    - RUGNUX_CLI.md: --mode refine refines against the input reflections; -z is not used there and
      a reflection file given only with -z does not select the mode.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/RUGNUX_ADVANCED.md | 4 ++--
     docs/RUGNUX_CLI.md      | 5 +++--
     rugnux/rugnux_cli.cpp   | 4 +++-
     3 files changed, 8 insertions(+), 5 deletions(-)
    
    diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md
    index 586f316f0..53ec4405c 100644
    --- a/docs/RUGNUX_ADVANCED.md
    +++ b/docs/RUGNUX_ADVANCED.md
    @@ -329,8 +329,8 @@ rocking curve merged with are the first file's; the ice-ring gate pools the fram
     
     The files must be of one crystal: each file's cell the first's lattice (in its setting or another) and
     the same space group. A run stops with a message naming both cells, or both groups, where they are not -
    -unless the cell is given (`-C`, or a reference `-z`) and, for the group, `-S`: then everything is merged
    -in what is given, and the crystals of a file not on that cell are left out. A mix of stills and rotation
    +unless the cell is given (`-C`, or a reference `-z`) and, for the group, `-S` (or the group of a reference
    +`-z` that states one): then everything is merged in what is given, and the crystals of a file not on that cell are left out. A mix of stills and rotation
     files is refused, and so are several rotation files (`--mode scale` merges one sweep at a time; several
     sweeps of one crystal are merged together from their masters by `--mode mx`). Under `--force-still` a
     rotation file counts as stills. `-s`/`-e` select images of one file and are refused with several. The
    diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md
    index 6a5237e7a..ee0e18498 100644
    --- a/docs/RUGNUX_CLI.md
    +++ b/docs/RUGNUX_CLI.md
    @@ -61,7 +61,8 @@ Refines a model against merged data that already exist - an MTZ or a structure-f
     reads no images. The model is required, given among the inputs or with `--model`. Without `--mode`,
     rugnux takes `--mode refine` when the inputs hold reflections and no images and a model is given -
     among the inputs or with `--model` (`rugnux -o refined data.mtz model.cif`,
    -`rugnux -o refined --model model.cif data.mtz`). A restraint dictionary for a ligand can be given among the
    +`rugnux -o refined --model model.cif data.mtz`); the reflections are an input then, not `-z`, which `--mode refine`
    +does not use (a reflection file given only with `-z` does not select it). A restraint dictionary for a ligand can be given among the
     inputs as well (`--restraints`)
     ([Refining a model against structure factors](RUGNUX_MODEL_AND_REFERENCE.md#refining-a-model-against-structure-factors-rugnux---mode-refine)).
     
    @@ -282,7 +283,7 @@ For very long axes the direction grid's angular resolution binds well below 500
     | `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation; several masters are then read as one stills run |
     | `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each (`FFTW` keeps its old name: the CPU transforms are pocketfft's); `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise. `FFBIDX` without a cell (`-C`, or one from `-z`) resolves to `None` and indexes nothing, with no error. On rotation data `-X` is not consulted: the FFT search runs on the GPU where there is one and on the CPU otherwise, so `Auto`, `FFT`, `cuFFT` (with a GPU) and `FFTW` (without one, or under `--no-gpu`) are accepted and any other value stops the run at indexing; `FFBIDX` and `None` are for stills |
     | `-F` | Short for `-X FFT`, kept for old command lines |
    -| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with `-S P1`. The lattice is indexed in that system, never above or below it, and only the space group within it is searched |
    +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with `-S P1`. The lattice is indexed in that system, never above or below it, on the axes of the cell as typed (an F- or R-centred one in its conventional setting), and only the space group within it is searched |
     | `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10; accepted 5-40). A candidate with a shorter axis is discarded — but on rotation data a row shorter than the floor, down to 3 Å, is recognised under the multiple of it the search reports past the floor, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given. On stills a crystal below the floor cannot be indexed unless this is lowered |
     | `--min-indexed-spots ` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
     | `-S, --space-group ` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). A fixed group fixes the lattice: indexing puts the crystal in the group's own Bravais class and nothing promotes, demotes, reduces or doubles it afterwards. A primitive group on a lattice with no primitive cell of its system (P2 on a C-centred monoclinic crystal, P3 on a rhombohedral one) is merged on the centred cell, and the positions the centring extinguishes are kept (they read about zero). With `-C` as well, the cell is written on the given cell's axes |
    diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp
    index 701de869d..f78afddc3 100644
    --- a/rugnux/rugnux_cli.cpp
    +++ b/rugnux/rugnux_cli.cpp
    @@ -184,7 +184,9 @@ Re-merging (--mode scale)
     
     Basic options
       -o, --output-prefix       Prefix of every output file (default: output)
    -  -C, --unit-cell          Known unit cell a,b,c,alpha,beta,gamma (A, deg); helps stills
    +  -C, --unit-cell          Known unit cell a,b,c,alpha,beta,gamma (A, deg); helps stills.
    +                                 Without -S its angles, read as typed (90 means exactly 90), fix
    +                                 the crystal system, kept on the axes as typed
       -S, --space-group          Known space group, number or symbol: 96 or P43212
       -A, --anomalous                Merge Friedel pairs apart (anomalous statistics)
       --scaling-high-resolution   High-resolution limit (default: from the CC1/2 fall-off)
    
    From 1f5427008d5eba60e013fbcd9e9e8565f2ec723f Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Sun, 15 Nov 2026 21:46:38 +0100
    Subject: [PATCH 11/19] Auto mode: P 1 on a metric-constrained lattice is
     decided against the lattice held triclinic
    
    A lattice is promoted to a centred or higher class on its metric alone, and the geometry and
    integration are held to that class. Where the space-group search then finds no symmetry (P 1),
    the constraint has nothing behind it. A stored rotation run in that state is now made once more
    from the start with the lattice held triclinic (IndexingSettings::FixedLatticeSystem, the hold a
    triclinic -C gives), and the merge whose CC1/2, averaged per shell on Fisher's z, is higher is the
    answer.
    
    Measured on a triclinic small-molecule set indexed C-centred monoclinic (angles held at 90 deg,
    0.05 deg inside tolerance; the C lattice took 45% of the spots against 63% free, and a 27 deg
    segment indexed in the other hand of the pseudo-2-fold): bare run CC1/2 0.203, R_meas 41.3%,
    I/sigma 11.1 (UNUSABLE) -> CC1/2 0.987, R_meas 10.5%, I/sigma 12.7, identical to -S P1; one extra
    run (~3 s). No set of the rc186/rc187 full, private, multisweep or private-multisweep battery runs
    ends in P 1 on a lattice other than triclinic P, so no battery set reaches the new path.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/CPU_DATA_ANALYSIS_INDEXING.md |  7 ++++
     rugnux/Rugnux.cpp                  | 54 ++++++++++++++++++++++++++++++
     2 files changed, 61 insertions(+)
    
    diff --git a/docs/CPU_DATA_ANALYSIS_INDEXING.md b/docs/CPU_DATA_ANALYSIS_INDEXING.md
    index aa83b6b9a..7fac4ebb2 100644
    --- a/docs/CPU_DATA_ANALYSIS_INDEXING.md
    +++ b/docs/CPU_DATA_ANALYSIS_INDEXING.md
    @@ -295,6 +295,13 @@ re-checks, about a second each. The checks taken on the
     pre-pass frames - the geometry walk's refits, the metric-symmetry arm and the pre-pass spot-crowding count - stay on those frames
     in both modes, and their near ties are re-checked once every frame is in on both.
     
    +A lattice is promoted to a centred or higher class on its metric alone, and the geometry and
    +integration are then held to that class. Where the space-group search afterwards finds no symmetry
    +at all (P 1) on such a lattice, the constraint has nothing left behind it, so a stored rotation run
    +is made once more with the lattice held triclinic from its first pass (as `-S P1` holds it), and of
    +the two merges the one whose CC1/2, averaged per resolution shell on Fisher's $z$, is higher is the
    +answer.
    +
     **A first pass that ends with no usable lattice tries a leaner, shallower one.** A crystal sitting in
     a crystalline powder floods the pass with spots that are not its own — measured, 2112 spots a frame
     against the 80 a clean crystal on the same beamline gives — and the FFT then takes its cell out of the
    diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp
    index 9a43c9272..6b6b899c3 100644
    --- a/rugnux/Rugnux.cpp
    +++ b/rugnux/Rugnux.cpp
    @@ -16,6 +16,7 @@
     #include 
     #include 
     #include 
    +#include 
     #include 
     #include 
     #include 
    @@ -471,6 +472,27 @@ void RaiseFFTBoundForKnownCell(IndexingSettings &settings, const DiffractionExpe
                                   "(shortest axis {:.1f} A)", floor_A, shortest);
         }
     }
    +
    +    // The merged CC1/2 per shell on Fisher's z, averaged: what two merges of one sweep are judged on.
    +    double MergedShellZ(const ProcessResult &r) {
    +        double z = 0;
    +        int n = 0;
    +        for (const auto &sh : r.merge_statistics.shells)
    +            if (std::isfinite(sh.cc_half) && std::fabs(sh.cc_half) < 1.0) {
    +                z += std::atanh(sh.cc_half);
    +                n++;
    +            }
    +        return n > 0 ? z / n : -std::numeric_limits::infinity();
    +    }
    +
    +    // The space-group search ran and found no symmetry, on a lattice indexed under a metric constraint -
    +    // a centring, or angles held at a higher system's values - that the intensities then gave no support.
    +    bool TriclinicOnConstrainedLattice(const ProcessResult &r) {
    +        return r.has_merge_statistics && r.space_group_search && r.space_group && r.space_group->number == 1
    +               && r.rotation_lattice_type
    +               && (r.rotation_lattice_type->centering != 'P'
    +                   || r.rotation_lattice_type->crystal_system != gemmi::CrystalSystem::Triclinic);
    +    }
     }
     
     
    @@ -509,6 +531,13 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) {
         looser->near_tie_rechecks_ = std::make_shared();
         looser->looser_spots_ = true;
         looser->config_.spot_finding.false_pixels_per_frame *= 10.0f;
    +    // And as it starts with the lattice held triclinic, for a run whose space-group search ends in P 1 on
    +    // a lattice it indexed under a metric constraint: the intensities said there is no symmetry, so the
    +    // constraint the geometry and integration were held to has nothing behind it.
    +    const auto free_triclinic = std::make_shared(*this);
    +    free_triclinic->near_tie_rechecks_ = std::make_shared();
    +    free_triclinic->experiment_.ImportIndexingSettings(
    +            experiment_.GetIndexingSettings().FixedLatticeSystem(gemmi::CrystalSystem::Triclinic));
         const SpotFindingSettings spot_finding = config_.spot_finding;
         Rugnux *run = this;
         ProcessResult result;
    @@ -531,6 +560,31 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) {
                 result = again->RunAllPasses(observer);
                 run = again.get();
             }
    +        // Two hypotheses about the lattice, decided on the merges: the constrained one as integrated, and
    +        // the free triclinic one integrated again from its own first pass.
    +        if (!result.cancelled && !config_.live && !user_cell_ && TriclinicOnConstrainedLattice(result)) {
    +            Logger logger("Rugnux");
    +            logger.Info("P 1 on a {}-centred {} lattice: the run is made again with the lattice held triclinic",
    +                        result.rotation_lattice_type->centering,
    +                        gemmi::crystal_system_str(result.rotation_lattice_type->crystal_system));
    +            if (observer)
    +                observer->OnStep("Triclinic lattice hypothesis (P 1 on a constrained lattice)");
    +            try {
    +                ProcessResult free = free_triclinic->RunAllPasses(observer);
    +                const double z_constrained = MergedShellZ(result);
    +                const double z_free = free.has_merge_statistics ? MergedShellZ(free)
    +                                                                : -std::numeric_limits::infinity();
    +                logger.Info("Lattice held as indexed or triclinic: merged CC1/2 per shell on Fisher's z {:+.4f} "
    +                            "as indexed, {:+.4f} triclinic => {}", z_constrained, z_free,
    +                            z_free > z_constrained ? "triclinic" : "as indexed");
    +                if (z_free > z_constrained) {
    +                    result = std::move(free);
    +                    run = free_triclinic.get();
    +                }
    +            } catch (const NoLatticeFound &) {
    +                logger.Info("Lattice held triclinic: no lattice found - the lattice as indexed stands");
    +            }
    +        }
         } catch (NoLatticeFound &e) {
             if (!looser_spots_ && config_.spot_finding.adaptive_threshold && !config_.live && !cancelled_) {
                 Logger("Rugnux").Info("No lattice: the run is made again with the spot finder's operating point ten "
    
    From 1f8f0c8a6fc5a10c426d90367c4ff0468cd7772f Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Sun, 15 Nov 2026 23:01:51 +0100
    Subject: [PATCH 12/19] -S without -C fixes only the space group: the cell is
     still found and reduced
    
    A space group given alone no longer fixes the lattice. The first pass asks the
    sub-lattice / twin-composite hypotheses and the long-axis hypothesis as in a
    run without -S, and a lattice they adopt is then put in the fixed group's own
    Bravais class (LatticeSearchForGroup), as the indexer does with the cell it
    finds. Only a cell given with -C (with or without -S) fixes the lattice and
    its setting; those runs are unchanged (.hkl byte-identical on the checks).
    
    On the private triclinic small-molecule set, -S P1 landed on 2x/4x supercells;
    it now gives the primitive cell on 5 of 7 datasets (I/sigma up to 2-5x higher).
    -S P12/c1 on dnba, refused in rc187 for a metric violation, now merges.
    The doubled-cell hypothesis stays off under -S: without the space-group
    search there is nothing to re-express its doubled cell on its centred lattice,
    and adopting it doubled the primitive cell.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/RUGNUX_CLI.md                |  4 ++--
     rugnux/RugnuxFirstPassRescues.cpp | 25 ++++++++++++++++++++-----
     2 files changed, 22 insertions(+), 7 deletions(-)
    
    diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md
    index ee0e18498..c86ece294 100644
    --- a/docs/RUGNUX_CLI.md
    +++ b/docs/RUGNUX_CLI.md
    @@ -283,10 +283,10 @@ For very long axes the direction grid's angular resolution binds well below 500
     | `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation; several masters are then read as one stills run |
     | `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each (`FFTW` keeps its old name: the CPU transforms are pocketfft's); `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise. `FFBIDX` without a cell (`-C`, or one from `-z`) resolves to `None` and indexes nothing, with no error. On rotation data `-X` is not consulted: the FFT search runs on the GPU where there is one and on the CPU otherwise, so `Auto`, `FFT`, `cuFFT` (with a GPU) and `FFTW` (without one, or under `--no-gpu`) are accepted and any other value stops the run at indexing; `FFBIDX` and `None` are for stills |
     | `-F` | Short for `-X FFT`, kept for old command lines |
    -| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with `-S P1`. The lattice is indexed in that system, never above or below it, on the axes of the cell as typed (an F- or R-centred one in its conventional setting), and only the space group within it is searched |
    +| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it. On stills the run always uses it: the geometry refinement takes it as a prior (1 % on the lengths, 1° on the angles the crystal system leaves free), and the log warns - with both numbers - where the frames refine its shape more than 3 % / 3° away, or where a sample of frames indexed de novo puts materially more spots on the lattice than the cell given ([§7.7](CPU_DATA_ANALYSIS_INDEXING.md#77-stills-a-given-cell-and-the-detector-modules)). Without `-S` the cell fixes the crystal system, read from the angles as typed: an angle is 90° or 120° only where it is typed so (`90` or `90.00`, not `90.03`). Three 90s: orthorhombic, or tetragonal / cubic where the lengths are equal within tolerance; two 90s and a 120: hexagonal (or rhombohedral in hexagonal axes); two 90s: monoclinic; three equal angles: rhombohedral; anything else: triclinic, and the run behaves as with this cell and `-S P1`. The lattice is indexed in that system, never above or below it, on the axes of the cell as typed (an F- or R-centred one in its conventional setting), and only the space group within it is searched |
     | `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10; accepted 5-40). A candidate with a shorter axis is discarded — but on rotation data a row shorter than the floor, down to 3 Å, is recognised under the multiple of it the search reports past the floor, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given. On stills a crystal below the floor cannot be indexed unless this is lowered |
     | `--min-indexed-spots ` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
    -| `-S, --space-group ` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). A fixed group fixes the lattice: indexing puts the crystal in the group's own Bravais class and nothing promotes, demotes, reduces or doubles it afterwards. A primitive group on a lattice with no primitive cell of its system (P2 on a C-centred monoclinic crystal, P3 on a rhombohedral one) is merged on the centred cell, and the positions the centring extinguishes are kept (they read about zero). With `-C` as well, the cell is written on the given cell's axes |
    +| `-S, --space-group ` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). Without `-C` the group fixes only the symmetry, not the cell: the lattice is found and reduced as in a run without `-S` (a supercell the indexer returns is still stepped down to the crystal's cell, a long axis is still looked for), indexing puts the crystal in the group's own Bravais class, nothing promotes or demotes it past that class, and the group is applied to the cell found, in its conventional setting. With `-C` as well, the lattice and its setting are the ones given and nothing reduces or doubles the cell. A primitive group on a lattice with no primitive cell of its system (P2 on a C-centred monoclinic crystal, P3 on a rhombohedral one) is merged on the centred cell, and the positions the centring extinguishes are kept (they read about zero). With `-C`, the cell is written on the given cell's axes |
     | `-r, --geom-refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default; on rotation data restrained toward the sweep's geometry, [§7.4](CPU_DATA_ANALYSIS_INDEXING.md#74-multi-stage-tightening-of-inlier-tolerance)) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) |
     | `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) |
     | `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass |
    diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp
    index ce07efff3..1b868ee85 100644
    --- a/rugnux/RugnuxFirstPassRescues.cpp
    +++ b/rugnux/RugnuxFirstPassRescues.cpp
    @@ -212,7 +212,7 @@ void Rugnux::FirstPassRun::StartLongAxisSearch(std::unique_ptr indexing_bound_A))
    @@ -273,7 +273,7 @@ void Rugnux::FirstPassRun::JoinLongAxisSpeculation() {
     // Not when a cell was given: the standard pass was already sized for it (RaiseFFTBoundForKnownCell).
     auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass {
         JoinLongAxisSpeculation();
    -    if (rx.cancelled_ || rx.experiment_.GetUnitCell().has_value() || rx.user_fixed_sg_)
    +    if (rx.cancelled_ || rx.experiment_.GetUnitCell().has_value())
             return standing;
         const auto start = std::chrono::steady_clock::now();
         const LongAxisReach reach = LongAxisBound();
    @@ -961,8 +961,9 @@ namespace {
     }
     
     auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass {
    -    // A space group or a cell the user fixed fixes the lattice, its volume included.
    -    if (rx.user_fixed_sg_ || rx.experiment_.GetIndexingSettings().GetFixedLatticeSystem())
    +    // A cell the user fixed with -C fixes the lattice, its volume included. A space group alone does
    +    // not: the cell is still found and reduced, and the group is applied to it.
    +    if ((rx.user_cell_ && rx.user_fixed_sg_) || rx.experiment_.GetIndexingSettings().GetFixedLatticeSystem())
             return best;
         // Asked again of each sub-lattice it adopts: a cell doubled along two or three axes is a sub-lattice
         // of index 4 or 8, reached one halving at a time. Each cell on the way is first asked whether it is
    @@ -970,9 +971,12 @@ auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass {
         // halving of their composite, and a halving can split the composite and lose one domain. A domain's
         // cell can itself be a multiple of the true one, and is halved on as any other; the overlap it
         // carries is a vector of every finer lattice in its orientation, so it holds through the halving.
    +    bool changed = false;
         for (int level = 0;; level++) {
    -        if (best.result.has_value() && !best.result->twin_overlap.has_value())
    +        if (best.result.has_value() && !best.result->twin_overlap.has_value()) {
                 best = TwinCompositeHypothesis(std::move(best));
    +            changed |= best.result.has_value() && best.result->twin_overlap.has_value();
    +        }
             if (level == 3 || rx.cancelled_ || !best.result.has_value() || best.score <= 0)
                 break;
             const CrystalLattice prim = best.result->lattice
    @@ -1084,6 +1088,17 @@ auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass {
             if (!adopt)
                 break;
             best = std::move(*sub_best);
    +        changed = true;
    +    }
    +    // A space group the user fixed is applied to the cell found here as the indexer applies it to the
    +    // cell it finds: the lattice is put in the group's own Bravais class (LatticeSearchForGroup).
    +    if (changed && rx.user_fixed_sg_ && best.result.has_value()) {
    +        RotationIndexerResult &r = *best.result;
    +        if (const auto fixed = LatticeSearchForGroup(r.lattice.ToPrimitive(r.search_result.centering),
    +                                                     *rx.user_fixed_sg_)) {
    +            r.search_result = *fixed;
    +            r.lattice = fixed->conventional;
    +        }
         }
         return best;
     }
    
    From 3a4dcba3bd92b8e66c0edfdd3e4a9290e831a1f0 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 07:56:15 +0100
    Subject: [PATCH 13/19] Side-chain field on the Apple GPU: the states' direct
     sums in Metal, bit for bit
    
    RESULT-CHANGING on the CPU (ulp level): the B factor's exp in EvaluatorCPU::Contribution is
    BraggExp (bragg_integration/BraggExp.h, at most 0.53 ulp) instead of the libm expf, so that the
    Apple GPU, whose exp no host function reproduces, computes the same bits.
    
    MakeSideChainFieldEvaluatorCPU takes a `gpu` flag (the refinement passes settings.gpu); on macOS the
    evaluator's Contribution() and the per-state sums of Evaluate() then run on the Apple GPU
    (SideChainFieldMetal.mm/.metal): one thread per reflection and state, as SideChainFieldGPU.cu's
    contribution_kernel, the evaluator's tables uploaded once, states in batches of 32 so that the
    GPU's buffer stays small beside the states' own. Fast math and contraction off - also over the
    BraggExp prepended to the kernel source, which a first version left contracted (91 of 60000
    values 1-2 ulp off). The likelihood over the combinations stays on the CPU (double). Where the
    GPU is hidden (--no-gpu) or fails (one warning) the CPU does the same sums.
    
    SideChainFieldMetal_BitIdenticalToCPU ([portable][Metal]): 20000 reflections, P212121, 70
    six-atom states (three batches): every contribution the same bits, every combination's -ln L equal.
    
    M6 Mac mini, rugnux --mode refine of an open-arm set's merged MTZ and refined model, on top of the
    Metal gradient (previous commit):
      9s02 (20355 atoms, 306298 reflections, 5 NCS copies): side-chain field 149.0 -> 30.4 s, refinement
           309 -> 184 s wall; R-work 0.2254 / R-free 0.2390 both.
      9zlo: field 7.6 -> 2.4 s, 45.8 -> 41.3 s; R unchanged.
      --no-gpu on 9zlo: the refined model byte-identical to the build before (the field moved nothing).
    Not run here: SideChainFieldTest.cpp's [Refinement] cases (jfjoch_test is not built on macOS).
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/GPU_DATA_ANALYSIS.md                     |   5 +-
     .../structure_refinement/CMakeLists.txt       |  16 ++
     .../structure_refinement/Refinement.cpp       |   2 +-
     .../structure_refinement/SideChainField.cpp   |  63 ++++++-
     .../structure_refinement/SideChainField.h     |   4 +-
     .../SideChainFieldMetal.h                     |  38 +++++
     .../SideChainFieldMetal.metal                 |  58 +++++++
     .../SideChainFieldMetal.mm                    | 156 ++++++++++++++++++
     tests/CMakeLists.txt                          |   1 +
     tests/SideChainFieldMetalTest.cpp             |  77 +++++++++
     10 files changed, 411 insertions(+), 9 deletions(-)
     create mode 100644 image_analysis/structure_refinement/SideChainFieldMetal.h
     create mode 100644 image_analysis/structure_refinement/SideChainFieldMetal.metal
     create mode 100644 image_analysis/structure_refinement/SideChainFieldMetal.mm
     create mode 100644 tests/SideChainFieldMetalTest.cpp
    
    diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md
    index 60f8ddf21..af83088ac 100644
    --- a/docs/GPU_DATA_ANALYSIS.md
    +++ b/docs/GPU_DATA_ANALYSIS.md
    @@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato
     
     ## Apple GPUs (Metal)
     
    -A macOS build has no CUDA, and four engines run on the Apple GPU instead (none under
    +A macOS build has no CUDA, and five engines run on the Apple GPU instead (none under
     `rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning):
     
     - **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its
    @@ -362,6 +362,9 @@ A macOS build has no CUDA, and four engines run on the Apple GPU instead (none u
       engine's gradient over the FFT method's map, each atom's box walked by one SIMD group as under CUDA;
       the CPU's points, summed in single precision, so the gradient is the CPU's to a few parts in 1e5
       (a macOS CI step checks it against the CPU's). It halves the time of a refinement on the CPU engine.
    +- **Side-chain field** (`SideChainFieldMetal.mm`, `.metal`) - the direct sums of every side-chain state's
    +  structure factors over the reflections, one thread per reflection and state; the CPU's sums **to the bit**
    +  (the B factor's exp is `BraggExp.h` on both), which a macOS CI step checks.
     - **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the
       rotation function's trial orientations scored from a copy of the molecular transform; equal to the
       CPU's to rounding.
    diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt
    index f513855cf..f3012ba4f 100644
    --- a/image_analysis/structure_refinement/CMakeLists.txt
    +++ b/image_analysis/structure_refinement/CMakeLists.txt
    @@ -121,5 +121,21 @@ IF (APPLE)
                 AtomGradientMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp)
         SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
         SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
    +    # The side-chain field's direct sums (SideChainFieldMetal.h), carried in the same way, after the exp the CPU's
    +    # sums take too (bragg_integration/BraggExp.h).
    +    FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h exp_text)
    +    FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal text)
    +    SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal
    +                 ${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h)
    +    FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp
    +         "// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
    +         "extern const char *const side_chain_field_metal_source;\n"
    +         "const char *const side_chain_field_metal_source = R\"FIELD_METAL(\n#include \nusing namespace metal;\n#pragma clang fp contract(off)\n${exp_text}\n${text})FIELD_METAL\";\n")
    +    CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp
    +                   ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp COPYONLY)
    +    TARGET_SOURCES(JFJochStructureRefinement PRIVATE SideChainFieldMetal.mm SideChainFieldMetal.h
    +            SideChainFieldMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp)
    +    SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
    +    SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
         TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation")
     ENDIF()
    diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp
    index 924ed0803..bef6fc53f 100644
    --- a/image_analysis/structure_refinement/Refinement.cpp
    +++ b/image_analysis/structure_refinement/Refinement.cpp
    @@ -3444,7 +3444,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector> f_total(ctx.fcalc.begin(), ctx.fcalc.begin() + ctx.refl.Size());
                 evaluator->SetTotal(f_total);
                 // the copies of each residue, from the NCS anchors (corresponding C-alpha atoms): per residue, numbered
    diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp
    index 8c3dca856..84a518fad 100644
    --- a/image_analysis/structure_refinement/SideChainField.cpp
    +++ b/image_analysis/structure_refinement/SideChainField.cpp
    @@ -13,9 +13,16 @@
     #include 
     
     #include "../../common/ParallelFor.h"
    +#include "../bragg_integration/BraggExp.h"
     #include "RefinementMath.h"
     #include "SideChainRotamers.h"
     
    +#ifdef __APPLE__
    +#include "../../common/JFJochException.h"
    +#include "../../common/Logger.h"
    +#include "SideChainFieldMetal.h"
    +#endif
    +
     namespace {
     
     constexpr double REPULSION_SIGMA = 0.2;   // A, the refinement's (RefinementRestraints.cpp)
    @@ -100,7 +107,7 @@ inline void SinCos2Pi(float v, float &sn, float &cs) {
     class EvaluatorCPU : public SideChainFieldEvaluator {
     public:
         EvaluatorCPU(const FieldReflections &refl, const std::vector> &ops,
    -                 const std::vector &elements, size_t nthreads)
    +                 const std::vector &elements, size_t nthreads, [[maybe_unused]] bool gpu)
             : refl_(refl), elements_(elements), nthreads_(nthreads), n_ops_(ops.size()) {
             // per operator and row (k * rows + m): R^T h and h . t, so that the phase of an atom at x is
             // 2 pi (R^T h . x + h . t)
    @@ -126,14 +133,29 @@ public:
                         f += ff[i] * std::exp(-ff[4 + i] * refl.stol2[m]);
                     f0_[e * rows + m] = f;
                 }
    +#ifdef __APPLE__
    +        if (gpu)
    +            try {
    +                metal_ = SideChainFieldMetal::Make(rows, n_ops_, hx_, hy_, hz_, ht_, f0_, refl.stol2);
    +            } catch (const JFJochException &e) {
    +                MetalFailed(e);
    +            }
    +#endif
         }
     
         void SetTotal(const std::vector> &f) override { total_ = f; }
     
         // In single precision, as the GPU's contribution_kernel: per atom of the state its form factor at every row of
         // a block, then per operator the phase's sine and cosine (SinCos2Pi) - loops over the rows that the compiler
    -    // vectorises.
    +    // vectorises. The B factor's exp is BraggExp, which an Apple GPU computes to the same bits.
         std::vector> Contribution(const FieldStates &states, int s) override {
    +#ifdef __APPLE__
    +        if (metal_) {
    +            std::vector>> one(1);
    +            if (ContributionsMetal(states, s, one))
    +                return one[0];
    +        }
    +#endif
             const size_t rows = refl_.Size();
             std::vector> f(rows);
             constexpr size_t BLOCK = 256;
    @@ -145,7 +167,7 @@ public:
                     const FieldAtom &at = states.atoms[a];
                     const float *f0 = &f0_[at.element * rows + lo];
                     for (size_t m = 0; m < n; m++)
    -                    fa[m] = f0[m] * at.occ * std::exp(-at.b * refl_.stol2[lo + m]);
    +                    fa[m] = f0[m] * at.occ * BraggExp(-at.b * refl_.stol2[lo + m]);
                     for (size_t k = 0; k < n_ops_; k++) {
                         const size_t o = k * rows + lo;
                         for (size_t m = 0; m < n; m++)
    @@ -204,10 +226,39 @@ public:
     private:
         std::vector>> All(const FieldStates &states) {
             std::vector>> df;
    -        for (size_t s = 0; s < states.Count(); s++)
    +#ifdef __APPLE__
    +        // on the GPU in batches of states, so that its buffer stays small beside the states' own
    +        constexpr size_t BATCH = 32;
    +        for (size_t s = 0; metal_ && s < states.Count(); s += BATCH) {
    +            std::vector>> batch(std::min(BATCH, states.Count() - s));
    +            if (!ContributionsMetal(states, static_cast(s), batch))
    +                break;
    +            for (auto &f : batch)
    +                df.push_back(std::move(f));
    +        }
    +#endif
    +        for (size_t s = df.size(); s < states.Count(); s++)
                 df.push_back(Contribution(states, static_cast(s)));
             return df;
         }
    +#ifdef __APPLE__
    +    // Contribution() of the states from `first` on, one per element of `out`, on the Apple GPU (SideChainFieldMetal.h):
    +    // the same values, bit for bit. False where it failed: the CPU from then on, with one warning.
    +    bool ContributionsMetal(const FieldStates &states, int first, std::vector>> &out) {
    +        try {
    +            metal_->Contributions(states, first, out);
    +            return true;
    +        } catch (const JFJochException &e) {
    +            MetalFailed(e);
    +            return false;
    +        }
    +    }
    +    void MetalFailed(const JFJochException &e) {
    +        Logger("Rugnux").Warning("Side-chain field: the Apple GPU failed, on the CPU from here on: {}", e.what());
    +        metal_.reset();
    +    }
    +    std::unique_ptr metal_;
    +#endif
         double Term(size_t m, std::complex f_atoms) const {
             const double fc = std::abs(refl_.k_total[m] * (f_atoms + refl_.fmask[m]));
             double dfc = 0;
    @@ -322,8 +373,8 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) {
     std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl,
                                                                             const std::vector> &ops,
                                                                             const std::vector &elements,
    -                                                                        size_t nthreads) {
    -    return std::make_unique(refl, ops, elements, nthreads);
    +                                                                        size_t nthreads, bool gpu) {
    +    return std::make_unique(refl, ops, elements, nthreads, gpu);
     }
     
     std::vector FieldAtomsOf(const gemmi::Residue &res, const SideChain &sc, const std::vector &p,
    diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h
    index be098f2c7..409b3a896 100644
    --- a/image_analysis/structure_refinement/SideChainField.h
    +++ b/image_analysis/structure_refinement/SideChainField.h
    @@ -91,10 +91,12 @@ public:
     // The group's operators as the evaluator takes them: rotation (row-major) and translation, fractional.
     std::vector> FieldOperators(const gemmi::SpaceGroup &sg);
     
    +// With `gpu`, the direct sums on the Apple GPU where there is one (SideChainFieldMetal.h, macOS): the same values,
    +// bit for bit.
     std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl,
                                                                             const std::vector> &ops,
                                                                             const std::vector &elements,
    -                                                                        size_t nthreads);
    +                                                                        size_t nthreads, bool gpu = false);
     #ifdef JFJOCH_USE_CUDA
     // The device memory the GPU evaluator takes at most for `rows` reflections under `n_ops` operators, bytes.
     size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops);
    diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.h b/image_analysis/structure_refinement/SideChainFieldMetal.h
    new file mode 100644
    index 000000000..4eea7ad92
    --- /dev/null
    +++ b/image_analysis/structure_refinement/SideChainFieldMetal.h
    @@ -0,0 +1,38 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#pragma once
    +
    +// The side-chain field's direct sums (SideChainField.cpp, EvaluatorCPU::Contribution) on an Apple GPU (macOS only):
    +// one thread per reflection and state, as SideChainFieldGPU.cu's contribution_kernel does under CUDA, with the CPU's
    +// single-precision arithmetic in its order (SideChainFieldMetal.metal spells it out). tests/SideChainFieldMetalTest.cpp
    +// holds it to the CPU's.
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +#include "SideChainField.h"   // FieldStates
    +
    +class SideChainFieldMetal {
    +public:
    +    // The evaluator's tables, kept on the GPU: per operator and row (k * rows + m) R^T h and h . t, per element and
    +    // row the form factor, per row stol2. Null where there is no Metal device the process may use (rugnux --no-gpu
    +    // hides it). Throws JFJochException on a Metal failure.
    +    static std::unique_ptr Make(size_t rows, size_t n_ops, const std::vector &hx,
    +                                                     const std::vector &hy, const std::vector &hz,
    +                                                     const std::vector &ht, const std::vector &f0,
    +                                                     const std::vector &stol2);
    +    ~SideChainFieldMetal();
    +    // The contributions of states first .. first + out.size() - 1, each `rows` values into its vector.
    +    void Contributions(const FieldStates &states, int first, std::vector>> &out);
    +    // How many states the GPU has summed in this process.
    +    static size_t States();
    +
    +    struct Impl;
    +
    +private:
    +    explicit SideChainFieldMetal(std::unique_ptr impl);
    +    std::unique_ptr impl_;
    +};
    diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.metal b/image_analysis/structure_refinement/SideChainFieldMetal.metal
    new file mode 100644
    index 000000000..7621a884f
    --- /dev/null
    +++ b/image_analysis/structure_refinement/SideChainFieldMetal.metal
    @@ -0,0 +1,58 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +// The side-chain field's direct sums (SideChainFieldMetal.h), compiled when first used from this source, which
    +// CMakeLists.txt copies into the binary after bragg_integration/BraggExp.h - with fast math off and no a*b+c
    +// contracted, as on the host, so that every operation below rounds as EvaluatorCPU::Contribution's does.
    +
    +#include 
    +using namespace metal;
    +
    +#pragma clang fp contract(off)
    +
    +struct Atom {   // FieldAtom
    +    float x, y, z, b, occ;
    +    int element;
    +};
    +
    +struct Params {
    +    uint rows, n_ops, first;
    +};
    +
    +// SideChainField.cpp's SinCos2Pi(): sin and cos of 2 pi v, v taken to [-1/2, 1/2] by its nearest integer, the half
    +// angle by its Taylor series, then the double-angle formulas.
    +static void sincos_2pi(float v, thread float &sn, thread float &cs) {
    +    const float x = 3.14159265f * (v - rint(v));
    +    const float x2 = x * x;
    +    const float s = x * (1 + x2 * (-1.f / 6 + x2 * (1.f / 120 + x2 * (-1.f / 5040 + x2 * (1.f / 362880 + x2 * (-1.f / 39916800))))));
    +    const float c = 1 + x2 * (-1.f / 2 + x2 * (1.f / 24 + x2 * (-1.f / 720 + x2 * (1.f / 40320 + x2 * (-1.f / 3628800 + x2 * (1.f / 479001600))))));
    +    sn = 2 * s * c;
    +    cs = c * c - s * s;
    +}
    +
    +// One thread per row m and state first + s: the state's atoms in order, each over the operators in order.
    +kernel void contribution(device const float *hx [[buffer(0)]], device const float *hy [[buffer(1)]],
    +                         device const float *hz [[buffer(2)]], device const float *ht [[buffer(3)]],
    +                         device const float *f0 [[buffer(4)]], device const float *stol2 [[buffer(5)]],
    +                         device const Atom *atoms [[buffer(6)]], device const int *start [[buffer(7)]],
    +                         constant Params &p [[buffer(8)]], device float2 *out [[buffer(9)]],
    +                         uint2 gid [[thread_position_in_grid]]) {
    +    const uint m = gid.x, s = gid.y;
    +    if (m >= p.rows)
    +        return;
    +    const float st = stol2[m];
    +    float re = 0, im = 0;
    +    for (int a = start[p.first + s]; a < start[p.first + s + 1]; a++) {
    +        const Atom at = atoms[a];
    +        const float fa = f0[uint(at.element) * p.rows + m] * at.occ * BraggExp(-at.b * st);
    +        for (uint k = 0; k < p.n_ops; k++) {
    +            const uint o = k * p.rows + m;
    +            const float v = hx[o] * at.x + hy[o] * at.y + hz[o] * at.z + ht[o];
    +            float sn, cs;
    +            sincos_2pi(v, sn, cs);
    +            re += fa * cs;
    +            im += fa * sn;
    +        }
    +    }
    +    out[s * p.rows + m] = float2(re, im);
    +}
    diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.mm b/image_analysis/structure_refinement/SideChainFieldMetal.mm
    new file mode 100644
    index 000000000..b8cfd41ee
    --- /dev/null
    +++ b/image_analysis/structure_refinement/SideChainFieldMetal.mm
    @@ -0,0 +1,156 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#import 
    +
    +#include "SideChainFieldMetal.h"
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +#include "../../common/CUDAWrapper.h"   // gpus_are_hidden
    +#include "../../common/JFJochException.h"
    +
    +// The kernel's source, SideChainFieldMetal.metal, copied in at configure time (CMakeLists.txt).
    +extern const char *const side_chain_field_metal_source;
    +
    +namespace {
    +
    +std::atomic states_on_gpu{0};
    +
    +static_assert(sizeof(FieldAtom) == 6 * 4, "the kernel's Atom is six words");
    +
    +struct Kernel {
    +    id device = nil;   // nil: no Metal device
    +    id pipeline = nil;
    +    id queue = nil;
    +};
    +
    +// Compiled once per process, the first time it is asked for.
    +const Kernel &GetKernel() {
    +    static const Kernel kernel = [] {
    +        Kernel k;
    +        k.device = MTLCreateSystemDefaultDevice();
    +        if (k.device == nil)
    +            return k;
    +        MTLCompileOptions *options = [MTLCompileOptions new];
    +        // Fast math off: nothing reassociated or approximated, as on the CPU.
    +        options.fastMathEnabled = NO;
    +        options.languageVersion = MTLLanguageVersion3_0;
    +        NSError *error = nil;
    +        id library = [k.device newLibraryWithSource:@(side_chain_field_metal_source) options:options
    +                                                          error:&error];
    +        if (library == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("side-chain field kernel: ") + error.localizedDescription.UTF8String);
    +        id function = [library newFunctionWithName:@"contribution"];
    +        k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
    +        if (k.pipeline == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("contribution: ") + error.localizedDescription.UTF8String);
    +        k.queue = [k.device newCommandQueue];
    +        return k;
    +    }();
    +    return kernel;
    +}
    +
    +id NewBuffer(id device, size_t bytes) {
    +    id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared];
    +    if (b == nil)
    +        throw JFJochException(JFJochExceptionCategory::GPUMetalError, "side-chain field: out of GPU memory");
    +    return b;
    +}
    +
    +id Upload(id device, const std::vector &v) {
    +    id b = NewBuffer(device, v.size() * sizeof(float));
    +    std::memcpy(b.contents, v.data(), v.size() * sizeof(float));
    +    return b;
    +}
    +
    +struct Params {
    +    uint32_t rows, n_ops, first;
    +};
    +
    +}  // namespace
    +
    +struct SideChainFieldMetal::Impl {
    +    uint32_t rows, n_ops;
    +    id hx, hy, hz, ht, f0, stol2;
    +};
    +
    +SideChainFieldMetal::SideChainFieldMetal(std::unique_ptr impl) : impl_(std::move(impl)) {}
    +
    +SideChainFieldMetal::~SideChainFieldMetal() = default;
    +
    +std::unique_ptr SideChainFieldMetal::Make(size_t rows, size_t n_ops, const std::vector &hx,
    +                                                               const std::vector &hy,
    +                                                               const std::vector &hz,
    +                                                               const std::vector &ht,
    +                                                               const std::vector &f0,
    +                                                               const std::vector &stol2) {
    +    @autoreleasepool {
    +        if (gpus_are_hidden() || rows * std::max(n_ops, 1) > std::numeric_limits::max())
    +            return nullptr;
    +        const Kernel &k = GetKernel();
    +        if (k.device == nil)
    +            return nullptr;
    +        auto impl = std::make_unique();
    +        impl->rows = static_cast(rows);
    +        impl->n_ops = static_cast(n_ops);
    +        impl->hx = Upload(k.device, hx);
    +        impl->hy = Upload(k.device, hy);
    +        impl->hz = Upload(k.device, hz);
    +        impl->ht = Upload(k.device, ht);
    +        impl->f0 = Upload(k.device, f0);
    +        impl->stol2 = Upload(k.device, stol2);
    +        return std::unique_ptr(new SideChainFieldMetal(std::move(impl)));
    +    }
    +}
    +
    +void SideChainFieldMetal::Contributions(const FieldStates &states, int first,
    +                                        std::vector>> &out) {
    +    // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
    +    @autoreleasepool {
    +        const Kernel &k = GetKernel();
    +        const size_t rows = impl_->rows, count = out.size();
    +        if (count == 0)
    +            return;
    +        id d_atoms = NewBuffer(k.device, states.atoms.size() * sizeof(FieldAtom));
    +        std::memcpy(d_atoms.contents, states.atoms.data(), states.atoms.size() * sizeof(FieldAtom));
    +        id d_start = NewBuffer(k.device, states.start.size() * sizeof(int));
    +        std::memcpy(d_start.contents, states.start.data(), states.start.size() * sizeof(int));
    +        id d_out = NewBuffer(k.device, count * rows * sizeof(std::complex));
    +        const Params p{impl_->rows, impl_->n_ops, static_cast(first)};
    +
    +        id cb = [k.queue commandBuffer];
    +        id e = [cb computeCommandEncoder];
    +        [e setComputePipelineState:k.pipeline];
    +        id tables[] = {impl_->hx, impl_->hy, impl_->hz, impl_->ht, impl_->f0, impl_->stol2};
    +        for (int i = 0; i < 6; i++)
    +            [e setBuffer:tables[i] offset:0 atIndex:i];
    +        [e setBuffer:d_atoms offset:0 atIndex:6];
    +        [e setBuffer:d_start offset:0 atIndex:7];
    +        [e setBytes:&p length:sizeof(p) atIndex:8];
    +        [e setBuffer:d_out offset:0 atIndex:9];
    +        [e dispatchThreads:MTLSizeMake(rows, count, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
    +        [e endEncoding];
    +        [cb commit];
    +        [cb waitUntilCompleted];
    +        if (cb.status != MTLCommandBufferStatusCompleted)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  cb.error ? cb.error.localizedDescription.UTF8String
    +                                           : "side-chain field: command buffer failed");
    +        const auto *f = static_cast *>(d_out.contents);
    +        for (size_t s = 0; s < count; s++)
    +            out[s].assign(f + s * rows, f + (s + 1) * rows);
    +        states_on_gpu += count;
    +    }
    +}
    +
    +size_t SideChainFieldMetal::States() {
    +    return states_on_gpu;
    +}
    diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
    index ce0055c8f..860a589a1 100644
    --- a/tests/CMakeLists.txt
    +++ b/tests/CMakeLists.txt
    @@ -36,6 +36,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
             MolecularReplacementMetalTest.cpp
             ModelDensityMetalTest.cpp
             AtomGradientMetalTest.cpp
    +        SideChainFieldMetalTest.cpp
             WriteReflectionsTest.cpp
             LoadReferenceMtzFreeFlagsTest.cpp
             LoadReferenceSfCifTest.cpp
    diff --git a/tests/SideChainFieldMetalTest.cpp b/tests/SideChainFieldMetalTest.cpp
    new file mode 100644
    index 000000000..8266114aa
    --- /dev/null
    +++ b/tests/SideChainFieldMetalTest.cpp
    @@ -0,0 +1,77 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#include 
    +
    +#ifdef __APPLE__
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +#include "gemmi/it92.hpp"
    +#include "gemmi/symmetry.hpp"
    +#include "../image_analysis/structure_refinement/SideChainField.h"
    +#include "../image_analysis/structure_refinement/SideChainFieldMetal.h"
    +
    +// The side-chain field's direct sums on the Apple GPU (SideChainFieldMetal.h) are the CPU's, bit for bit, so the
    +// contributions are compared as bit patterns and the energies exactly. A Mac without a Metal device fails rather than
    +// skips, as in ModelDensityMetalTest.cpp.
    +
    +TEST_CASE("SideChainFieldMetal_BitIdenticalToCPU", "[portable][Metal]") {
    +    std::mt19937 rng(23);
    +    std::uniform_real_distribution unit(0.0f, 1.0f);
    +    std::uniform_int_distribution index(-25, 25);
    +    // reflections of a 50 A cell to about 2 A, in no particular order, a tenth of them free
    +    FieldReflections fr;
    +    for (int i = 0; i < 20000; i++) {
    +        const float h = static_cast(index(rng)), k = static_cast(index(rng)), l = static_cast(index(rng));
    +        fr.hkl.push_back({h, k, l});
    +        fr.stol2.push_back((h * h + k * k + l * l) / (4.0f * 50 * 50));
    +        fr.fo.push_back(100 * unit(rng));
    +        fr.d.push_back(0.9f);
    +        fr.var.push_back(50 + 100 * unit(rng));
    +        fr.centric.push_back(i % 7 == 0);
    +        fr.work.push_back(i % 10 != 0);
    +        fr.k_total.push_back(1.0f);
    +        fr.fmask.push_back({10 * unit(rng), 10 * unit(rng)});
    +    }
    +    const std::vector elements = {gemmi::IT92::get(gemmi::El::C, 0).coefs,
    +                                              gemmi::IT92::get(gemmi::El::N, 0).coefs,
    +                                              gemmi::IT92::get(gemmi::El::O, 0).coefs};
    +    const auto ops = FieldOperators(*gemmi::find_spacegroup_by_name("P 21 21 21"));
    +    // 70 states of six atoms each - more than one batch - and combinations of one and of two states
    +    FieldStates states;
    +    for (int s = 0; s < 70; s++) {
    +        std::vector atoms;
    +        for (int a = 0; a < 6; a++)
    +            atoms.push_back({unit(rng), unit(rng), unit(rng), 10 + 50 * unit(rng), 0.5f + 0.5f * unit(rng), a % 3});
    +        states.Add(atoms);
    +    }
    +    states.current = {0};
    +    for (int c = 0; c < 69; c++)
    +        states.AddCombo(c % 2 == 0 ? std::vector{c} : std::vector{c, c + 1});
    +
    +    auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, false);
    +    auto gpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, true);
    +    const size_t before = SideChainFieldMetal::States();
    +    size_t different = 0;
    +    for (int s : {0, 33, 69}) {
    +        const std::vector> c = cpu->Contribution(states, s), g = gpu->Contribution(states, s);
    +        REQUIRE(c.size() == g.size());
    +        for (size_t m = 0; m < c.size(); m++)
    +            different += std::memcmp(&c[m], &g[m], sizeof(c[m])) != 0;
    +    }
    +    REQUIRE(SideChainFieldMetal::States() == before + 3);   // the GPU summed them
    +    CHECK(different == 0);
    +
    +    const std::vector> total = cpu->Contribution(states, 0);
    +    cpu->SetTotal(total);
    +    gpu->SetTotal(total);
    +    const std::vector ec = cpu->Evaluate(states), eg = gpu->Evaluate(states);
    +    REQUIRE(SideChainFieldMetal::States() == before + 3 + 70);
    +    CHECK(ec == eg);
    +}
    +
    +#endif
    
    From 1178452466df86a5417f692a33767616f0fe6b2c Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 08:23:53 +0100
    Subject: [PATCH 14/19] Bulk-solvent mask shrink on the Apple GPU, bit for bit
    
    PutMaskOnGrid() takes a `gpu` flag (default on, as PutModelDensityOnGrid's): on macOS the shrink by
    the 0.8 A radius (ShrinkMask) runs on the Apple GPU (ShrinkMaskMetal.mm/.metal), one thread per point
    applying the CPU's rule - a masked point becomes solvent where a near offset is solvent, or a far
    offset is a solvent point with a masked near offset - read from the mask as it was, with the CPU's
    offsets; the mask and its copy handed to the GPU in place where page aligned. The atoms, the
    symmetrize and gemmi's island removal stay on the CPU. --no-gpu, no device, or a failure (one
    warning) keep the CPU's loop.
    
    ShrinkMaskMetal_BitIdenticalToCPU ([portable][Metal]): clustered atoms, 0.4 A mask grids of a
    triclinic P1 and a P212121 cell - every point the same bits.
    
    M6 Mac mini, rugnux --mode refine (open-arm sets' merged MTZ and refined model), on top of the two
    previous Metal commits: 9s02 184.2 -> 175.6 s, 9zlo 41.3 -> 39.0 s; the refined models
    byte-identical to the build before on both.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/GPU_DATA_ANALYSIS.md                     |   4 +-
     .../structure_refinement/CMakeLists.txt       |  13 ++
     .../structure_refinement/ModelGrid.cpp        |  25 +++-
     .../structure_refinement/ModelGrid.h          |   5 +-
     .../structure_refinement/ShrinkMaskMetal.h    |  20 +++
     .../ShrinkMaskMetal.metal                     |  52 +++++++
     .../structure_refinement/ShrinkMaskMetal.mm   | 138 ++++++++++++++++++
     tests/CMakeLists.txt                          |   1 +
     tests/ShrinkMaskMetalTest.cpp                 |  80 ++++++++++
     9 files changed, 332 insertions(+), 6 deletions(-)
     create mode 100644 image_analysis/structure_refinement/ShrinkMaskMetal.h
     create mode 100644 image_analysis/structure_refinement/ShrinkMaskMetal.metal
     create mode 100644 image_analysis/structure_refinement/ShrinkMaskMetal.mm
     create mode 100644 tests/ShrinkMaskMetalTest.cpp
    
    diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md
    index af83088ac..b235e06ef 100644
    --- a/docs/GPU_DATA_ANALYSIS.md
    +++ b/docs/GPU_DATA_ANALYSIS.md
    @@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato
     
     ## Apple GPUs (Metal)
     
    -A macOS build has no CUDA, and five engines run on the Apple GPU instead (none under
    +A macOS build has no CUDA, and six engines run on the Apple GPU instead (none under
     `rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning):
     
     - **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its
    @@ -365,6 +365,8 @@ A macOS build has no CUDA, and five engines run on the Apple GPU instead (none u
     - **Side-chain field** (`SideChainFieldMetal.mm`, `.metal`) - the direct sums of every side-chain state's
       structure factors over the reflections, one thread per reflection and state; the CPU's sums **to the bit**
       (the B factor's exp is `BraggExp.h` on both), which a macOS CI step checks.
    +- **Bulk-solvent mask shrink** (`ShrinkMaskMetal.mm`, `.metal`) - the mask's shrink by the 0.8 Å radius,
    +  one thread per grid point; the same mask as the CPU's **to the bit**, which a macOS CI step checks.
     - **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the
       rotation function's trial orientations scored from a copy of the molecular transform; equal to the
       CPU's to rounding.
    diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt
    index f3012ba4f..873099f93 100644
    --- a/image_analysis/structure_refinement/CMakeLists.txt
    +++ b/image_analysis/structure_refinement/CMakeLists.txt
    @@ -137,5 +137,18 @@ IF (APPLE)
                 SideChainFieldMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp)
         SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
         SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
    +    # The bulk-solvent mask's shrink (ShrinkMaskMetal.h), carried in the same way.
    +    FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/ShrinkMaskMetal.metal text)
    +    SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ShrinkMaskMetal.metal)
    +    FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp.tmp
    +         "// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
    +         "extern const char *const shrink_mask_metal_source;\n"
    +         "const char *const shrink_mask_metal_source = R\"SHRINK_METAL(\n${text})SHRINK_METAL\";\n")
    +    CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp.tmp
    +                   ${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp COPYONLY)
    +    TARGET_SOURCES(JFJochStructureRefinement PRIVATE ShrinkMaskMetal.mm ShrinkMaskMetal.h
    +            ShrinkMaskMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp)
    +    SET_SOURCE_FILES_PROPERTIES(ShrinkMaskMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
    +    SET_SOURCE_FILES_PROPERTIES(ShrinkMaskMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
         TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation")
     ENDIF()
    diff --git a/image_analysis/structure_refinement/ModelGrid.cpp b/image_analysis/structure_refinement/ModelGrid.cpp
    index 09f41f1ff..65468c364 100644
    --- a/image_analysis/structure_refinement/ModelGrid.cpp
    +++ b/image_analysis/structure_refinement/ModelGrid.cpp
    @@ -18,6 +18,7 @@
     #include "../../common/JFJochException.h"
     #include "../../common/Logger.h"
     #include "ModelDensityMetal.h"
    +#include "ShrinkMaskMetal.h"
     #endif
     
     namespace {
    @@ -469,6 +470,11 @@ void PutModelDensityOnGrid(gemmi::DensityCalculator &dc, const gem
     
     namespace {
     
    +#ifdef __APPLE__
    +// From the first time the Apple GPU fails to shrink a mask on, with one warning, the CPU does it.
    +std::atomic shrink_metal_failed{false};
    +#endif
    +
     // gemmi's SolventMasker::shrink() - set_margin_around(r, 1, -1), then the margin set to 1 - with the planes of the
     // grid spread over the threads, the same mask bit for bit. gemmi walks the unmasked points and marks the masked
     // points within r of each: those among the nearest neighbours, and, where one of those was masked, the rest.
    @@ -476,7 +482,7 @@ namespace {
     // of the marks does not matter: here each masked point reads, from the mask as it was, whether gemmi would have
     // marked it - an unmasked nearest neighbour, or an unmasked point further out within r that has a masked nearest
     // neighbour of its own (the offsets are symmetric, so the points that reach q are the points q reaches).
    -void ShrinkMask(gemmi::Grid &grid, double r, size_t nthreads) {
    +void ShrinkMask(gemmi::Grid &grid, double r, size_t nthreads, [[maybe_unused]] bool gpu) {
         // the two rings of offsets, as set_margin_around() builds them: a positive offset stored wrapped (u - nu), so
         // that a point's coordinate plus an offset stays within [-n, n), which is what index_near_zero() takes
         const int du = static_cast(std::floor(r / grid.spacing[0])), dv = static_cast(std::floor(r / grid.spacing[1])),
    @@ -499,6 +505,19 @@ void ShrinkMask(gemmi::Grid &grid, double r, size_t nthreads) {
             return in[grid.index_near_zero(u + d[0], v + d[1], w + d[2])] >= 1.0f;
         };
         std::vector out(in.size());
    +#ifdef __APPLE__
    +    if (gpu && !shrink_metal_failed) {
    +        try {
    +            if (ShrinkMaskMetal(grid.nu, grid.nv, grid.nw, near, far, in.data(), out.data())) {
    +                grid.data = std::move(out);
    +                return;
    +            }
    +        } catch (const JFJochException &e) {
    +            if (!shrink_metal_failed.exchange(true))
    +                Logger("Rugnux").Warning("Solvent mask: the Apple GPU failed, on the CPU from here on: {}", e.what());
    +        }
    +    }
    +#endif
         ParallelChunks(grid.nw, nthreads, [&](int lo, int hi) {
             for (int w = lo; w < hi; ++w)
                 for (int v = 0; v < grid.nv; ++v)
    @@ -532,7 +551,7 @@ void ShrinkMask(gemmi::Grid &grid, double r, size_t nthreads) {
     // rshrink of each point (set_margin_around), and on a grid whose spacing is coarser than rshrink along all
     // three axes there are none - which is most rigid-body zones - yet gemmi's still walks the whole grid.
     void PutMaskOnGrid(gemmi::Grid &grid, const gemmi::Model &model, const std::vector &orbit_leaders,
    -                   size_t nthreads) {
    +                   size_t nthreads, bool gpu) {
         const gemmi::SolventMasker masker(gemmi::AtomicRadiiSet::Refmac);
         masker.clear(grid);
         std::vector boxes;
    @@ -556,7 +575,7 @@ void PutMaskOnGrid(gemmi::Grid &grid, const gemmi::Model &model, const st
         for (int i = 0; i < 3; i++)
             shrink_has_offsets = shrink_has_offsets || static_cast(std::floor(masker.rshrink / grid.spacing[i])) > 0;
         if (shrink_has_offsets)
    -        ShrinkMask(grid, masker.rshrink, nthreads);
    +        ShrinkMask(grid, masker.rshrink, nthreads, gpu);
     }
     
     gemmi::Grid SolventMaskGrid(const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min,
    diff --git a/image_analysis/structure_refinement/ModelGrid.h b/image_analysis/structure_refinement/ModelGrid.h
    index f6f58a9de..d658f41fa 100644
    --- a/image_analysis/structure_refinement/ModelGrid.h
    +++ b/image_analysis/structure_refinement/ModelGrid.h
    @@ -57,9 +57,10 @@ void PutModelDensityOnGrid(gemmi::DensityCalculator, float> &
                                const std::vector &orbit_leaders, size_t nthreads, bool gpu = true,
                                bool ion_charges = false);
     
    -// `grid` sized, with its cell and group set.
    +// `grid` sized, with its cell and group set. On macOS the Apple GPU shrinks the mask (ShrinkMaskMetal.h), to the
    +// same bits; `gpu` false keeps it on the CPU.
     void PutMaskOnGrid(gemmi::Grid &grid, const gemmi::Model &model, const std::vector &orbit_leaders,
    -                   size_t nthreads);
    +                   size_t nthreads, bool gpu = true);
     
     // The step a bulk-solvent mask's grid is held below. The mask's shrink (0.8 A, the Refmac radii) reaches no grid point
     // on a coarser grid and is skipped, and the macromolecule keeps the whole 1.0 A probe: some 5-10% of the cell's solvent
    diff --git a/image_analysis/structure_refinement/ShrinkMaskMetal.h b/image_analysis/structure_refinement/ShrinkMaskMetal.h
    new file mode 100644
    index 000000000..e04bde667
    --- /dev/null
    +++ b/image_analysis/structure_refinement/ShrinkMaskMetal.h
    @@ -0,0 +1,20 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#pragma once
    +
    +// The bulk-solvent mask's shrink (ModelGrid.cpp, ShrinkMask) on an Apple GPU (macOS only): one thread per point, the
    +// CPU's rule read from the mask as it was - the same mask, bit for bit. tests/ShrinkMaskMetalTest.cpp holds it to the
    +// CPU's.
    +
    +#include 
    +#include 
    +#include 
    +
    +// `in` shrunk into `out` (nu nv nw floats each, u fastest), with the CPU's offsets: `near`, then `far`, each stored
    +// wrapped (a positive offset less the grid's size). False where the GPU does not take it - no Metal device the process
    +// may use (rugnux --no-gpu hides it); `out` is then untouched. Throws JFJochException on a Metal failure.
    +bool ShrinkMaskMetal(int nu, int nv, int nw, const std::vector> &near,
    +                     const std::vector> &far, const float *in, float *out);
    +// How many masks the GPU has shrunk in this process.
    +size_t ShrinkMaskMetalCalls();
    diff --git a/image_analysis/structure_refinement/ShrinkMaskMetal.metal b/image_analysis/structure_refinement/ShrinkMaskMetal.metal
    new file mode 100644
    index 000000000..2a13b5bb9
    --- /dev/null
    +++ b/image_analysis/structure_refinement/ShrinkMaskMetal.metal
    @@ -0,0 +1,52 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +// The bulk-solvent mask's shrink (ShrinkMaskMetal.h), compiled when first used from this source, which
    +// CMakeLists.txt copies into the binary.
    +
    +#include 
    +using namespace metal;
    +
    +struct Params {
    +    int nu, nv, nw;
    +    int n_near, n_far;
    +};
    +
    +// gemmi's Grid::index_near_zero(): a coordinate in [-n, n) taken into the grid.
    +static uint index_near_zero(int u, int v, int w, constant Params &p) {
    +    u += u < 0 ? p.nu : 0;
    +    v += v < 0 ? p.nv : 0;
    +    w += w < 0 ? p.nw : 0;
    +    return uint(u) + uint(p.nu) * (uint(v) + uint(p.nv) * uint(w));
    +}
    +
    +// One thread per point, ModelGrid.cpp's ShrinkMask() rule read from the mask as it was: a masked point becomes
    +// solvent where a near offset is solvent, or where a far offset is a solvent point with a masked near offset of its
    +// own. `offsets` holds the near ones, then the far ones, each stored wrapped as the CPU stores it.
    +kernel void shrink_mask(device const float *in [[buffer(0)]], device float *out [[buffer(1)]],
    +                        device const int3 *offsets [[buffer(2)]], constant Params &p [[buffer(3)]],
    +                        uint i [[thread_position_in_grid]]) {
    +    if (i >= uint(p.nu) * uint(p.nv) * uint(p.nw))
    +        return;
    +    const float x = in[i];
    +    if (x >= 1.0f) {
    +        out[i] = x;
    +        return;
    +    }
    +    const int u = int(i % uint(p.nu)), v = int(i / uint(p.nu) % uint(p.nv)), w = int(i / (uint(p.nu) * uint(p.nv)));
    +    bool margin = false;
    +    for (int k = 0; k < p.n_near && !margin; k++)
    +        margin = in[index_near_zero(u + offsets[k].x, v + offsets[k].y, w + offsets[k].z, p)] >= 1.0f;
    +    for (int k = p.n_near; k < p.n_near + p.n_far && !margin; k++) {
    +        const int3 d = offsets[k];
    +        if (!(in[index_near_zero(u + d.x, v + d.y, w + d.z, p)] >= 1.0f))
    +            continue;
    +        int pu = u + d.x, pv = v + d.y, pw = w + d.z;   // back into [0, n)
    +        pu += pu < 0 ? p.nu : 0;
    +        pv += pv < 0 ? p.nv : 0;
    +        pw += pw < 0 ? p.nw : 0;
    +        for (int j = 0; j < p.n_near && !margin; j++)
    +            margin = !(in[index_near_zero(pu + offsets[j].x, pv + offsets[j].y, pw + offsets[j].z, p)] >= 1.0f);
    +    }
    +    out[i] = margin ? 1.0f : x;
    +}
    diff --git a/image_analysis/structure_refinement/ShrinkMaskMetal.mm b/image_analysis/structure_refinement/ShrinkMaskMetal.mm
    new file mode 100644
    index 000000000..b9fce2385
    --- /dev/null
    +++ b/image_analysis/structure_refinement/ShrinkMaskMetal.mm
    @@ -0,0 +1,138 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#import 
    +
    +#include "ShrinkMaskMetal.h"
    +
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +#include 
    +
    +#include "../../common/CUDAWrapper.h"   // gpus_are_hidden
    +#include "../../common/JFJochException.h"
    +
    +// The kernel's source, ShrinkMaskMetal.metal, copied in at configure time (CMakeLists.txt).
    +extern const char *const shrink_mask_metal_source;
    +
    +namespace {
    +
    +std::atomic calls_on_gpu{0};
    +
    +struct Kernel {
    +    id device = nil;   // nil: no Metal device
    +    id pipeline = nil;
    +    id queue = nil;
    +};
    +
    +// Compiled once per process, the first time it is asked for.
    +const Kernel &GetKernel() {
    +    static const Kernel kernel = [] {
    +        Kernel k;
    +        k.device = MTLCreateSystemDefaultDevice();
    +        if (k.device == nil)
    +            return k;
    +        MTLCompileOptions *options = [MTLCompileOptions new];
    +        options.languageVersion = MTLLanguageVersion3_0;
    +        NSError *error = nil;
    +        id library = [k.device newLibraryWithSource:@(shrink_mask_metal_source) options:options
    +                                                          error:&error];
    +        if (library == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("mask shrink kernel: ") + error.localizedDescription.UTF8String);
    +        id function = [library newFunctionWithName:@"shrink_mask"];
    +        k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
    +        if (k.pipeline == nil)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  std::string("shrink_mask: ") + error.localizedDescription.UTF8String);
    +        k.queue = [k.device newCommandQueue];
    +        return k;
    +    }();
    +    return kernel;
    +}
    +
    +id NewBuffer(id device, size_t bytes) {
    +    id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared];
    +    if (b == nil)
    +        throw JFJochException(JFJochExceptionCategory::GPUMetalError, "mask shrink: out of GPU memory");
    +    return b;
    +}
    +
    +// `data` as it is where its pages can be handed to the GPU, nil otherwise.
    +id InPlace(id device, const float *data, size_t bytes) {
    +    const size_t page = static_cast(getpagesize());
    +    if (reinterpret_cast(data) % page != 0)
    +        return nil;
    +    return [device newBufferWithBytesNoCopy:const_cast(data) length:(bytes + page - 1) / page * page
    +                                    options:MTLResourceStorageModeShared deallocator:nil];
    +}
    +
    +struct Params {
    +    int32_t nu, nv, nw, n_near, n_far;
    +};
    +
    +}  // namespace
    +
    +bool ShrinkMaskMetal(int nu, int nv, int nw, const std::vector> &near,
    +                     const std::vector> &far, const float *in, float *out) {
    +    // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
    +    @autoreleasepool {
    +        const size_t points = static_cast(nu) * nv * nw;
    +        if (gpus_are_hidden() || points > std::numeric_limits::max())
    +            return false;
    +        const Kernel &k = GetKernel();
    +        if (k.device == nil)
    +            return false;
    +        const size_t bytes = points * sizeof(float);
    +        id d_in = InPlace(k.device, in, bytes);
    +        if (d_in == nil) {
    +            d_in = NewBuffer(k.device, bytes);
    +            std::memcpy(d_in.contents, in, bytes);
    +        }
    +        id d_out = InPlace(k.device, out, bytes);
    +        const bool out_in_place = d_out != nil;
    +        if (!out_in_place)
    +            d_out = NewBuffer(k.device, bytes);
    +        // the offsets as int3, which Metal pads to four ints
    +        id d_offsets = NewBuffer(k.device, (near.size() + far.size()) * 4 * sizeof(int32_t));
    +        auto *o = static_cast(d_offsets.contents);
    +        size_t n = 0;
    +        for (const auto *v : {&near, &far})
    +            for (const std::array &d : *v) {
    +                o[4 * n] = d[0];
    +                o[4 * n + 1] = d[1];
    +                o[4 * n + 2] = d[2];
    +                o[4 * n + 3] = 0;
    +                n++;
    +            }
    +        const Params p{nu, nv, nw, static_cast(near.size()), static_cast(far.size())};
    +
    +        id cb = [k.queue commandBuffer];
    +        id e = [cb computeCommandEncoder];
    +        [e setComputePipelineState:k.pipeline];
    +        [e setBuffer:d_in offset:0 atIndex:0];
    +        [e setBuffer:d_out offset:0 atIndex:1];
    +        [e setBuffer:d_offsets offset:0 atIndex:2];
    +        [e setBytes:&p length:sizeof(p) atIndex:3];
    +        [e dispatchThreads:MTLSizeMake(points, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
    +        [e endEncoding];
    +        [cb commit];
    +        [cb waitUntilCompleted];
    +        if (cb.status != MTLCommandBufferStatusCompleted)
    +            throw JFJochException(JFJochExceptionCategory::GPUMetalError,
    +                                  cb.error ? cb.error.localizedDescription.UTF8String
    +                                           : "mask shrink: command buffer failed");
    +        if (!out_in_place)
    +            std::memcpy(out, d_out.contents, bytes);
    +        calls_on_gpu++;
    +        return true;
    +    }
    +}
    +
    +size_t ShrinkMaskMetalCalls() {
    +    return calls_on_gpu;
    +}
    diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
    index 860a589a1..4ce32fe1e 100644
    --- a/tests/CMakeLists.txt
    +++ b/tests/CMakeLists.txt
    @@ -37,6 +37,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
             ModelDensityMetalTest.cpp
             AtomGradientMetalTest.cpp
             SideChainFieldMetalTest.cpp
    +        ShrinkMaskMetalTest.cpp
             WriteReflectionsTest.cpp
             LoadReferenceMtzFreeFlagsTest.cpp
             LoadReferenceSfCifTest.cpp
    diff --git a/tests/ShrinkMaskMetalTest.cpp b/tests/ShrinkMaskMetalTest.cpp
    new file mode 100644
    index 000000000..ca779f15d
    --- /dev/null
    +++ b/tests/ShrinkMaskMetalTest.cpp
    @@ -0,0 +1,80 @@
    +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute 
    +// SPDX-License-Identifier: GPL-3.0-only
    +
    +#include 
    +
    +#ifdef __APPLE__
    +
    +#include 
    +#include 
    +#include 
    +
    +#include 
    +
    +#include "../image_analysis/structure_refinement/ModelGrid.h"
    +#include "../image_analysis/structure_refinement/RefinementXray.h"
    +#include "../image_analysis/structure_refinement/ShrinkMaskMetal.h"
    +
    +// The bulk-solvent mask shrunk on the Apple GPU (ShrinkMaskMetal.h) is the CPU's, bit for bit. A Mac without a
    +// Metal device fails rather than skips, as in ModelDensityMetalTest.cpp.
    +
    +namespace {
    +    // Atoms in clusters over the cell - so that the mask has an edge to shrink everywhere and boxes wrap at every face.
    +    gemmi::Model clustered_atoms(const gemmi::UnitCell &cell) {
    +        std::mt19937 rng(29);
    +        std::uniform_real_distribution frac(0.0, 1.0);
    +        std::normal_distribution spread(0.0, 2.5);
    +        const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S};
    +        gemmi::Model model(1);
    +        gemmi::Chain chain("A");
    +        gemmi::Residue res;
    +        res.name = "UNK";
    +        res.seqid = gemmi::SeqId(1, ' ');
    +        for (int c = 0; c < 40; c++) {
    +            const gemmi::Position centre = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng)));
    +            for (int i = 0; i < 50; i++) {
    +                gemmi::Atom a;
    +                a.name = "X";
    +                a.element = elements[i % 4];
    +                a.pos = centre + gemmi::Position(spread(rng), spread(rng), spread(rng));
    +                a.b_iso = 20.0f;
    +                a.occ = 1.0f;
    +                res.atoms.push_back(a);
    +            }
    +        }
    +        chain.residues.push_back(res);
    +        model.chains.push_back(chain);
    +        return model;
    +    }
    +
    +    void check_same_mask(const gemmi::UnitCell &cell, const char *group, double d_min) {
    +        const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group);
    +        gemmi::Grid cpu = BulkSolventMask::MaskGrid(cell, sg, d_min);
    +        cpu.data.resize(cpu.point_count());
    +        gemmi::Grid gpu = cpu;
    +        const std::vector leaders = OrbitLeaders(cpu, 4);
    +        const gemmi::Model model = clustered_atoms(cell);
    +        PutMaskOnGrid(cpu, model, leaders, 4, false);
    +        const size_t before = ShrinkMaskMetalCalls();
    +        PutMaskOnGrid(gpu, model, leaders, 4, true);
    +        REQUIRE(ShrinkMaskMetalCalls() == before + 1);   // the GPU shrank it
    +        REQUIRE(gpu.data.size() == cpu.data.size());
    +        size_t different = 0, solvent = 0;
    +        for (size_t i = 0; i < cpu.data.size(); i++) {
    +            different += std::memcmp(&cpu.data[i], &gpu.data[i], sizeof(float)) != 0;
    +            solvent += cpu.data[i] >= 1.0f;
    +        }
    +        CAPTURE(group, solvent, cpu.data.size());
    +        CHECK(solvent > cpu.data.size() / 10);
    +        CHECK(solvent < cpu.data.size() * 9 / 10);
    +        CHECK(different == 0);
    +    }
    +}
    +
    +TEST_CASE("ShrinkMaskMetal_BitIdenticalToCPU", "[portable][Metal]") {
    +    // Triclinic, so that the offsets are not symmetric in the axes; then a group with symmetry.
    +    check_same_mask(gemmi::UnitCell(41, 46, 52, 81, 96, 107), "P 1", 2.0);
    +    check_same_mask(gemmi::UnitCell(50, 56, 63, 90, 90, 90), "P 21 21 21", 1.6);
    +}
    +
    +#endif
    
    From 9073d6bf4b92759309e308db51f130993ae4a019 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 08:50:34 +0100
    Subject: [PATCH 15/19] Revert "RugnuxCommandLine test: --refine given back
     where the config refines without a GPU"
    
    This reverts commit aaafc03ee699ea9330d4f03658f8991c881ab7a1.
    ---
     tests/RugnuxTest.cpp | 5 -----
     1 file changed, 5 deletions(-)
    
    diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp
    index f96204636..e96d5175f 100644
    --- a/tests/RugnuxTest.cpp
    +++ b/tests/RugnuxTest.cpp
    @@ -448,11 +448,6 @@ TEST_CASE("RugnuxCommandLine_Full", "[process]") {
         CHECK(RugnuxCommandLine(config, x, "in.h5", "", false).find("--no-refine") != std::string::npos);
         config.refine = true;
         CHECK(RugnuxCommandLine(config, x, "in.h5", "", false).find("--no-refine") == std::string::npos);
    -    // Refined without a GPU only where asked: --refine given back.
    -    CHECK(RugnuxCommandLine(config, x, "in.h5", "", false).find("--refine ") == std::string::npos);
    -    config.refine_without_gpu = true;
    -    CHECK(RugnuxCommandLine(config, x, "in.h5", "", false).find("--refine ") != std::string::npos);
    -    config.refine_without_gpu = false;
         // The solvent build is off by default: only --refine-solvent is given back.
         CHECK(RugnuxCommandLine(config, x, "in.h5", "", false).find("--refine-solvent") == std::string::npos);
         config.refine_solvent = true;
    
    From fdaa16b22a3c1c01ec07d349fb11f01f6030c9a5 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 08:50:34 +0100
    Subject: [PATCH 16/19] Revert "without a GPU a placed model is not refined
     unless --refine asks for it"
    
    The CPU refinement protocol (no population, one round) brings CPU refinement past a
    10x speed-up, so refinement stays on without a GPU, the viewer included.
    
    This reverts commit 5d61bc3bd.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/RUGNUX_CLI.md                 |  3 +--
     docs/RUGNUX_MODEL_AND_REFERENCE.md |  3 +--
     rugnux/Rugnux.h                    |  3 ---
     rugnux/RugnuxCommandLine.cpp       |  2 --
     rugnux/RugnuxScaleMerge.cpp        | 12 +-----------
     rugnux/rugnux_cli.cpp              |  9 ---------
     6 files changed, 3 insertions(+), 29 deletions(-)
    
    diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md
    index c86ece294..a85be37f6 100644
    --- a/docs/RUGNUX_CLI.md
    +++ b/docs/RUGNUX_CLI.md
    @@ -332,10 +332,9 @@ serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rota
     | `-z, --reference ` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity), hands over the R-free set and reports CCref. Not a scale anchor |
     | `--free-flags ` | Take only the R-free test set of this MTZ or structure-factor mmCIF - e.g. the deposited reflections of the `--model` - for the validation, the refinement and the merged files; nothing else of the file is used, so unlike `-z` it fixes no space group, cell or frame. Taken on the model's axes where the merge matches the file's intensities (`R_FREE_FLAGS=`, see [Free set and model bias](RUGNUX_MODEL_AND_REFERENCE.md#free-set-and-model-bias)) |
     | `--reference-column   High-resolution limit (default: from the CC1/2 fall-off)
       --model                  Atomic model (PDB/mmCIF) to place, refine and validate with
       --no-refine                    Place and score the model, but do not refine it
    -  --refine                       Refine the placed model also without a GPU (on the CPU: minutes, not seconds)
       --refine-solvent               Also build waters and relabel ions in the refined model (default off)
       --restraints             Restraint dictionary (.cif) for a ligand the built-in one lacks; repeatable
       -z, --reference          Reference reflections of the same crystal form (MTZ/mmCIF)
    @@ -261,7 +260,6 @@ enum {
         OPT_CRYSTFEL_GEOMETRY,
         OPT_MR,
         OPT_NO_REFINE,
    -    OPT_REFINE,
         OPT_REFINE_SOLVENT,
         OPT_NO_REFINE_SOLVENT,
         OPT_RESTRAINTS,
    @@ -351,7 +349,6 @@ static option long_options[] = {
         {"crystfel-geometry", required_argument, nullptr, OPT_CRYSTFEL_GEOMETRY},
         {"mr", required_argument, nullptr, OPT_MR},
         {"no-refine", no_argument, nullptr, OPT_NO_REFINE},
    -    {"refine", no_argument, nullptr, OPT_REFINE},
         {"refine-solvent", no_argument, nullptr, OPT_REFINE_SOLVENT},
         {"no-refine-solvent", no_argument, nullptr, OPT_NO_REFINE_SOLVENT},
         {"restraints", required_argument, nullptr, OPT_RESTRAINTS},
    @@ -944,7 +941,6 @@ struct RugnuxCli {
         std::string model_pdb;           // --model: PDB to validate merged intensities against (R-free + maps)
         std::string mr_model;            // --mr: --model, placed by molecular replacement straight away
         bool refine_model = true;        // --no-refine: a placed model is refined unless it is given
    -    bool refine_without_gpu = false; // --refine: refined also without a GPU (RugnuxConfig::refine_without_gpu)
         bool refine_solvent = false;     // --refine-solvent: the refinement builds waters and relabels ions
         std::vector restraint_files;   // --restraints and the restraint files among the inputs
         RestraintDictionary restraints;             // read from them, over the embedded dictionary
    @@ -1278,10 +1274,6 @@ int RugnuxCli::Parse(int argc, char **argv) {
             case OPT_NO_REFINE:
                 refine_model = false;
                 break;
    -        case OPT_REFINE:
    -            refine_model = true;
    -            refine_without_gpu = true;
    -            break;
             case OPT_REFINE_SOLVENT:
                 refine_solvent = true;
                 break;
    @@ -3869,7 +3861,6 @@ int RugnuxInputRun::PrepareRugnuxInput() {
         config.model = model;
         config.force_mr = !mr_model.empty();
         config.refine = refine_model;   // a placed model is refined unless --no-refine
    -    config.refine_without_gpu = refine_without_gpu;
         config.refine_solvent = refine_solvent;
         config.restraints = restraints;
         // When merging, the merged reflections (.mtz/.cif) are the wanted output; skip the large
    
    From 3ffc853fcb1b219e6928f9d3f8ddb3b623cadaae Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 08:58:45 +0100
    Subject: [PATCH 17/19] docs/THIRD_PARTY_NOTICES.md regenerated from the root
     notices (the Metal gradient's GEMMI row)
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/THIRD_PARTY_NOTICES.md | 6 +++---
     1 file changed, 3 insertions(+), 3 deletions(-)
    
    diff --git a/docs/THIRD_PARTY_NOTICES.md b/docs/THIRD_PARTY_NOTICES.md
    index 83aff2794..7aa288f6c 100644
    --- a/docs/THIRD_PARTY_NOTICES.md
    +++ b/docs/THIRD_PARTY_NOTICES.md
    @@ -84,7 +84,7 @@ These live in the source tree (see the path) rather than being fetched; traccc,
     | [traccc (ACTS)](https://github.com/acts-project/traccc) | `image_analysis/spot_finding/StrongPixelSet.cpp`, `SpotExtractorGPU.cu` | CERN, for the benefit of the ACTS project | MPL-2.0 | [traccc.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/traccc.txt) |
     | [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (adapted) | `image_analysis/indexing/FFBIDXIndexerCPU.cpp` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/fast-feedback-indexer.txt) |
     | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) (adapted) | `image_analysis/geom_refinement/LMSolver.h`, `LMSolver.cpp` | Google Inc. | BSD-3-Clause | [ceres-solver.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/ceres-solver.txt) |
    -| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/gemmi.txt) |
    +| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal`, `AtomGradientMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/gemmi.txt) |
     | [Durin](https://github.com/DiamondLightSource/durin) (adapted) | `xds-plugin/plugin.h`, `plugin.cpp` | Diamond Light Source Ltd.; Global Phasing Ltd. | BSD-3-Clause | [durin.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/durin.txt) |
     | [xbflash.qspi](https://github.com/Xilinx/XRT) | `tools/xbflash.qspi/` | Xilinx / AMD | Apache-2.0 | [xbflash-qspi.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/xbflash-qspi.txt) |
     | [wingetopt](https://github.com/alex85k/wingetopt) | `tools/wingetopt/` | Todd C. Miller; The NetBSD Foundation | ISC AND BSD-2-Clause | [wingetopt.txt](https://gitea.psi.ch/mx/jungfraujoch/src/branch/main/licenses/wingetopt.txt) |
    @@ -158,8 +158,8 @@ served frontend, so the shipped web UI carries its own attribution.
       `FFBIDXIndexerCPU.cpp` ports its CUDA kernels to the CPU. The file names the origin at the top and
       is covered by `licenses/fast-feedback-indexer.txt`.
     * **GEMMI** is also vendored (table above); separately, `ModelScaling.cpp`, `ModelMaskGPU.cu`,
    -  `ModelStructureFactorsGPU.cpp` and `ModelDensityMetal.metal` in `image_analysis/structure_refinement/`
    -  follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
    +  `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` and `AtomGradientMetal.metal` in
    +  `image_analysis/structure_refinement/` follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
       Each names the GEMMI source it follows and is covered by `licenses/gemmi.txt`.
     * **Durin** (`xds-plugin/plugin.h`/`.cpp`): the XDS plugin's entry points are based on the Durin plugin
       of Diamond Light Source and Global Phasing; both files name the origin at the top and are covered by
    
    From c4e4dc88c9761c2d1794f52f91d692fa325f3637 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 09:29:41 +0100
    Subject: [PATCH 18/19] rugnux: _plot.txt gets dataset and image columns for
     runs of several inputs
    
    The per-image table numbered its rows by the run's processed-image ordinal
    only, which is unusable for a run made of several files. It now carries
    three numbering columns: ordinal (the run's row, as before), dataset (the
    input, from 1 in the order given) and image (the image within that input).
    The header describes each column before the legend line.
    
    - Several stills masters read as one: rows are renumbered per master.
    - Joint merge of sweeps: the table covered only as many rows as the
      reference sweep's file held, so the other sweeps' frames were missing;
      it now has one row per frame of every merged sweep, with each sweep's own
      angle and per-image analysis (bkg, resolution, spots) from its own run
      and the scaling columns from the joint merge.
    - The angle is a per-row value now, so each sweep reads on its own goniometer.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/RUGNUX_REPORT.md          |   2 +-
     docs/RUGNUX_TUTORIAL.md        |  15 ++--
     rugnux/DiagnosticOutput.cpp    |  47 +++++++---
     rugnux/DiagnosticOutput.h      |  16 +++-
     rugnux/Rugnux.cpp              |  15 +++-
     rugnux/RugnuxJointSweeps.h     |   2 +
     rugnux/rugnux_cli.cpp          |  19 ++++
     tests/DiagnosticOutputTest.cpp | 160 +++++++++++++++++++++++----------
     8 files changed, 207 insertions(+), 69 deletions(-)
    
    diff --git a/docs/RUGNUX_REPORT.md b/docs/RUGNUX_REPORT.md
    index b6f193e05..0199f8615 100644
    --- a/docs/RUGNUX_REPORT.md
    +++ b/docs/RUGNUX_REPORT.md
    @@ -599,7 +599,7 @@ The vocabulary is **closed and stable**: a code is never renamed, and never reus
     meaning. New codes are only ever added, and adding one moves `REPORT_VERSION` at the next release.
     
     The columns are: `FIRST_IMAGE`/`LAST_IMAGE` — inclusive, in processed-image ordinals (the numbering
    -of `_plot.txt` and of every other per-image array `rugnux` writes; with `-s`/`--stride` the
    +of the `ordinal` column of `_plot.txt` and of every other per-image array `rugnux` writes; with `-s`/`--stride` the
     source image is `start + ordinal * stride`); `ROTATION` — the width of the range in degrees;
     `SEVERITY` — the fraction of the run's typical diffracting power missing over the range, 0 (as good
     as the run) to 1 (nothing at all); `SCALE` and `CC` — the range's mean per-image scale and
    diff --git a/docs/RUGNUX_TUTORIAL.md b/docs/RUGNUX_TUTORIAL.md
    index 2fd07d6c6..335b3d043 100644
    --- a/docs/RUGNUX_TUTORIAL.md
    +++ b/docs/RUGNUX_TUTORIAL.md
    @@ -363,13 +363,15 @@ Absorption is corrected only by the empirical surface a rotation run fits and ke
       JSON also by the broker's `/start`, merged into its dataset settings; see
       [Module geometry](RUGNUX_FORMATS.md#module-geometry)).
     - `_plot.txt` — **the per-image table**, one row per processed image, whitespace-separated,
    -  with a single `#` legend line over its own columns so gnuplot plots it as it stands
    -  (`plot '_plot.txt' using 1:4`). The columns are:
    +  under a `#` header - one line per column saying what it is, then a legend line over the columns -
    +  so gnuplot plots it as it stands (`plot '_plot.txt' using 1:5`). The columns are:
     
       | Column | Meaning |
       |--------|---------|
    -  | `image` | Processed-image ordinal, the numbering every per-image array `rugnux` writes uses; with `-s`/`--stride` the source image is `start + ordinal * stride`. |
    -  | `angle_deg` | Spindle angle at mid-exposure. |
    +  | `ordinal` | Processed-image ordinal: the row of the run, the numbering every per-image array `rugnux` writes uses (and the report's `FIRST_IMAGE`/`LAST_IMAGE`). |
    +  | `dataset` | The input the image is from, counted from 1 in the order the inputs were given: the sweep of a joint merge of several (`--sweeps joint`), the file of several read as one stills run. 1 on a run of one input. |
    +  | `image` | The image's number in that input, from 0, as `-s`/`-e` count it; with `-s`/`--stride` it is `start + ordinal * stride`. |
    +  | `angle_deg` | Spindle angle at mid-exposure, on that input's own goniometer. |
       | `bkg` | Background estimate from the azimuthal profile (the `bkgEstimate` of the HDF5 file). |
       | `resolution_A` | How far the image's spots reach, from spot finding alone. |
       | `spots` | Spots found. |
    @@ -382,7 +384,10 @@ Absorption is corrected only by the empirical surface a rotation run fits and ke
     
       A quantity nothing measured for an image is written `nan`, which gnuplot and numpy both skip; the
       columns never shift and a row is never left out. A stills run writes the table too, with `nan` in
    -  the columns a sweep would fill; `--no-merge` writes it with the scaling columns `nan`.
    +  the columns a sweep would fill; `--no-merge` writes it with the scaling columns `nan`. A joint merge of
    +  sweeps writes one row per frame of the sweeps it merged, sweep after sweep in input order:
    +  each row's per-image analysis is its own sweep's, the scaling columns the joint
    +  merge's; a sweep left out of the merge has no rows.
     
       **Reading `sigma_M_deg`.** It is fitted per image — a maximum-likelihood Gaussian rocking width
       over that image's own indexed spots — but it is then **smoothed in frame order** over the same
    diff --git a/rugnux/DiagnosticOutput.cpp b/rugnux/DiagnosticOutput.cpp
    index 9d6940405..476ea4ef0 100644
    --- a/rugnux/DiagnosticOutput.cpp
    +++ b/rugnux/DiagnosticOutput.cpp
    @@ -109,16 +109,43 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
         return WriteJPEGToMem(rgb_image, 85);
     }
     
    +void FillJointPlotRows(PerImagePlot &joint, size_t first_row, int dataset, const PerImagePlot &own) {
    +    for (auto *v : {&joint.angle_deg, &joint.bkg_estimate, &joint.resolution_A, &joint.spot_count})
    +        v->resize(joint.image_count, NAN);
    +    for (size_t f = 0; f < own.image_count; f++) {
    +        const size_t row = first_row + f;
    +        joint.dataset[row] = dataset;
    +        joint.image[row] = own.image[f];
    +        joint.angle_deg[row] = Element(own.angle_deg, f);
    +        joint.bkg_estimate[row] = Element(own.bkg_estimate, f);
    +        joint.resolution_A[row] = Element(own.resolution_A, f);
    +        joint.spot_count[row] = Element(own.spot_count, f);
    +    }
    +}
    +
     std::string PerImagePlotText(const PerImagePlot &plot) {
    -    const std::optional &goniometer = plot.goniometer;
         std::string file;
     
    -    // One comment line, its names over their own columns, and nothing else a plotting program has
    -    // to be told to skip. The '#' takes the first character of the image-number field.
    -    file += "#" + fmt::format("{:>5} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
    +    // What each column is, then one comment line with the names over their own columns - nothing a
    +    // plotting program has to be told to skip, as gnuplot and numpy pass over '#' lines. The '#' takes
    +    // the first character of the ordinal field.
    +    file += "# ordinal: row of the run, the numbering of every per-image array rugnux writes\n"
    +            "# dataset: input file or sweep, from 1 in the order given\n"
    +            "# image: image of that input, from 0 (with -s/--stride: start + ordinal * stride)\n"
    +            "# angle_deg: spindle angle at mid-exposure\n"
    +            "# bkg: background estimate from the azimuthal profile\n"
    +            "# resolution_A: how far the image's spots reach\n"
    +            "# spots: spots found\n"
    +            "# scale_G: per-image scale factor fitted by scaling\n"
    +            "# sigma_M_deg: mosaicity, the rocking-curve standard deviation\n"
    +            "# cc_to_merge: correlation of the image's observations with the merged data\n"
    +            "# cc_n: observations that correlation was taken over\n"
    +            "# merged: 1 where the image's observations are in the merged data, 0 where not\n"
    +            "# disposition: 1 merged, 0.5 downgraded, 0 rejected\n";
    +    file += "#" + fmt::format("{:>7} {:>7} {:>7} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
                                    " {:>7} {:>11}\n",
    -                               "image", "angle_deg", "bkg", "resolution_A", "spots", "scale_G",
    -                               "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition");
    +                               "ordinal", "dataset", "image", "angle_deg", "bkg", "resolution_A", "spots",
    +                               "scale_G", "sigma_M_deg", "cc_to_merge", "cc_n", "merged", "disposition");
     
         // Whether "merged" has anything to answer at all. With no merge behind the run - --no-merge, or
         // a sweep nothing indexed on - a column of zeros would read as a sweep every frame of which was
    @@ -128,10 +155,6 @@ std::string PerImagePlotText(const PerImagePlot &plot) {
                                               [](float g) { return std::isfinite(g); });
     
         for (size_t i = 0; i < plot.image_count; i++) {
    -        const float angle = goniometer
    -                                ? goniometer->GetAngle_deg(static_cast(i))
    -                                      + goniometer->GetWedge_deg() / 2.0f
    -                                : NAN;
             const float scale_g = Element(plot.scale_g, i);
             // Did this image's observations reach the merged data? The sweep-quality diagnostic says so
             // outright; where it did not run, an image with no fitted scale contributed nothing, which
    @@ -154,9 +177,9 @@ std::string PerImagePlotText(const PerImagePlot &plot) {
             } else if (merge_ran && std::isfinite(scale_g))
                 disposition = 1.0f;
     
    -        file += fmt::format("{:>6} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}"
    +        file += fmt::format("{:>8} {:>7} {:>7} {:>10.4f} {:>12.4g} {:>12.4g} {:>8.0f} {:>12.5g} {:>13.4f}"
                                 " {:>12.4f} {:>8} {:>7} {:>11}\n",
    -                            i, angle, Element(plot.bkg_estimate, i), Element(plot.resolution_A, i),
    +                            i, plot.dataset[i], plot.image[i], Element(plot.angle_deg, i), Element(plot.bkg_estimate, i), Element(plot.resolution_A, i),
                                 Element(plot.spot_count, i), scale_g, Element(plot.mosaicity_deg, i),
                                 Element(plot.cc_to_merge, i), Element(plot.cc_n, i),
                                 merge_ran ? std::to_string(merged) : "nan", disposition);
    diff --git a/rugnux/DiagnosticOutput.h b/rugnux/DiagnosticOutput.h
    index 04d41d4e2..9001eff98 100644
    --- a/rugnux/DiagnosticOutput.h
    +++ b/rugnux/DiagnosticOutput.h
    @@ -29,9 +29,14 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
     
     // The columns of _plot.txt. Every vector is indexed by the processed-image ordinal - the
     // numbering of every per-image array rugnux writes - and a value nothing measured is NAN.
    +// dataset and image say which input (from 1, in the order given) and which of its own images a row is,
    +// so a run made of several files - stills masters read as one, a joint merge of sweeps - plots per file.
     // frame_disposition holds FrameDisposition (Merge.h) and is empty when the sweep-quality diagnostic
     // did not run.
     struct PerImagePlot {
    +    std::vector dataset;
    +    std::vector image;
    +    std::vector angle_deg;               // at mid-exposure; NAN without a goniometer
         std::vector bkg_estimate;
         std::vector resolution_A;
         std::vector spot_count;
    @@ -40,10 +45,15 @@ struct PerImagePlot {
         std::vector cc_to_merge;
         std::vector cc_n;
         std::vector frame_disposition;
    -    std::optional goniometer;   // the angle column; none: nan
         size_t image_count = 0;                     // the rows
     };
     
    -// The text of _plot.txt: one '#' legend line naming the columns, then one whitespace-separated
    -// row per image, so `plot '' using 1:4` works with nothing in between.
    +// The rows a joint merge of sweeps (--sweeps joint) holds of one sweep, from `first_row` on: which input
    +// and image each is, its angle and its per-image analysis, all from that sweep's own run (`own`). The
    +// scaling columns stay the joint merge's.
    +void FillJointPlotRows(PerImagePlot &joint, size_t first_row, int dataset, const PerImagePlot &own);
    +
    +// The text of _plot.txt: '#' lines saying what each column is, the last of them naming the
    +// columns over their own fields, then one whitespace-separated row per image, so
    +// `plot '' using 1:5` works with nothing in between.
     std::string PerImagePlotText(const PerImagePlot &plot);
    diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp
    index 9a43c9272..270f722c2 100644
    --- a/rugnux/Rugnux.cpp
    +++ b/rugnux/Rugnux.cpp
    @@ -1420,8 +1420,19 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             }
             if (result.has_merge_statistics)
                 plot.frame_disposition = result.merge_statistics.sweep_quality.frame_disposition;
    -        plot.goniometer = experiment_.GetGoniometer();
    -        plot.image_count = static_cast(images_to_process);
    +        // One input as far as this run knows: the caller renumbers a run of several (rugnux_cli.cpp).
    +        // The goniometer is already shifted to the processed range, so ordinal i is at its angle i.
    +        const auto goniometer = experiment_.GetGoniometer();
    +        // A joint merge's rows are the frames of all its sweeps, more than the reference sweep's file holds.
    +        plot.image_count = joint ? indexer->GetIntegrationOutcome().size() : static_cast(images_to_process);
    +        plot.dataset.assign(plot.image_count, 1);
    +        plot.image.resize(plot.image_count);
    +        plot.angle_deg.assign(plot.image_count, NAN);
    +        for (size_t i = 0; i < plot.image_count; i++) {
    +            plot.image[i] = start_image + static_cast(i) * config_.stride;
    +            if (goniometer)
    +                plot.angle_deg[i] = goniometer->GetAngle_deg(static_cast(i)) + goniometer->GetWedge_deg() / 2.0f;
    +        }
             result.outputs->plot = std::move(plot);
         }
     
    diff --git a/rugnux/RugnuxJointSweeps.h b/rugnux/RugnuxJointSweeps.h
    index 45164d653..98ac2d72c 100644
    --- a/rugnux/RugnuxJointSweeps.h
    +++ b/rugnux/RugnuxJointSweeps.h
    @@ -14,6 +14,7 @@
     #include "../gemmi_gph/gemmi/symmetry.hpp"
     #include "../image_analysis/IntegrationOutcome.h"
     #include "../image_analysis/geom_refinement/CellEsd.h"
    +#include "DiagnosticOutput.h"
     
     // Several sweeps of one crystal merged as one (--sweeps joint), in two levels: each sweep is processed on
     // its own - indexed, integrated, its partials combined into fulls and scaled - and the fulls of all of them
    @@ -31,6 +32,7 @@ struct JointMemberSweep {
         std::vector merged;
         std::vector fulls;                   // all its fulls, unmerged, on the same axes
         std::vector batches;            // the batch headers of its own unmerged MTZ
    +    PerImagePlot plot;                               // its own per-image table, for the joint one
         double cc_half = NAN, completeness = NAN, d_min = NAN;   // its merge on its own
         double isa = NAN;                                // ... and its ISa, NaN where b was not resolved
         double distance_mm = NAN;                        // the detector distance its run ended at
    diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp
    index f78afddc3..787c55b79 100644
    --- a/rugnux/rugnux_cli.cpp
    +++ b/rugnux/rugnux_cli.cpp
    @@ -3974,6 +3974,18 @@ int RugnuxInputRun::RunPreparedInput() {
             }
         }
     
    +    // The table of a run read from several inputs: each row in its own input's numbering.
    +    if (multi_reader && result.outputs && result.outputs->plot) {
    +        PerImagePlot &plot = *result.outputs->plot;
    +        for (size_t i = 0; i < plot.image_count; i++) {
    +            size_t k = multi_reader->GetNumberOfInputs() - 1;
    +            while (plot.image[i] < multi_reader->FirstImage(k))
    +                k--;
    +            plot.dataset[i] = static_cast(k) + 1;
    +            plot.image[i] -= multi_reader->FirstImage(k);
    +        }
    +    }
    +
         // Every file the run made, from what it returned.
         if (!output_prefix.empty()) {
             try {
    @@ -4322,6 +4334,8 @@ int RugnuxCli::RunJointSweeps(const RugnuxSweepPlan &plan) {
                 m.fulls = merged.scaled_fulls;
                 if (sweep.member_result->outputs->unmerged)
                     m.batches = sweep.member_result->outputs->unmerged->batches;
    +            if (sweep.member_result->outputs->plot)
    +                m.plot = *sweep.member_result->outputs->plot;
                 m.cc_half = merged.statistics.overall.cc_half;
                 m.completeness = merged.statistics.overall.possible_unique_reflections > 0
                                  ? 100.0 * merged.statistics.overall.unique_reflections
    @@ -4543,6 +4557,11 @@ int RugnuxCli::RunJointSweeps(const RugnuxSweepPlan &plan) {
                                                                         : relation.sweeps[k].why_left_out)});
                     result.joint_sweeps.push_back(std::move(w));
                 }
    +            // The joint table's rows are the sweeps' frames end to end: each says which sweep and frame it is.
    +            if (result.outputs && result.outputs->plot)
    +                for (size_t seg = 0; seg < input.segments.size(); ++seg)
    +                    FillJointPlotRows(*result.outputs->plot, input.segments[seg].first_frame,
    +                                      input.segment_sweep[seg] + 1, members[input.segment_sweep[seg]].plot);
                 if (result.outputs && result.outputs->merged && export_unmerged) {
                     const MergedOutput &merged = *result.outputs->merged;
                     std::vector history{"From Rugnux " + jfjoch_version() + ", data reduction",
    diff --git a/tests/DiagnosticOutputTest.cpp b/tests/DiagnosticOutputTest.cpp
    index 257349645..8993cb6d8 100644
    --- a/tests/DiagnosticOutputTest.cpp
    +++ b/tests/DiagnosticOutputTest.cpp
    @@ -25,6 +25,24 @@ namespace {
             return ret;
         }
     
    +    // One input, its images in order: what a run of one file writes.
    +    void OneDataset(PerImagePlot &plot, size_t n) {
    +        plot.image_count = n;
    +        plot.dataset.assign(n, 1);
    +        plot.image.resize(n);
    +        for (size_t i = 0; i < n; i++)
    +            plot.image[i] = static_cast(i);
    +    }
    +
    +    // The table's rows: the lines after its '#' header.
    +    std::vector Rows(const std::vector &lines) {
    +        std::vector ret;
    +        for (const auto &line : lines)
    +            if (line[0] != '#')
    +                ret.push_back(line);
    +        return ret;
    +    }
    +
         std::vector ReadLines(const std::string &path) {
             std::ifstream f(path);
             std::vector ret;
    @@ -93,6 +111,9 @@ TEST_CASE("DiagnosticPlot_Format") {
         const std::string prefix = "diagnostic_plot_format_test";
     
         PerImagePlot plot;
    +    OneDataset(plot, 4);
    +    plot.image = {10, 12, 14, 16};
    +    plot.angle_deg = {10.25f, 10.75f, 11.25f, 11.75f};
         plot.bkg_estimate = {1.0f, 2.0f, 3.0f, 4.0f};
         plot.resolution_A = {2.5f, 2.4f, NAN, 2.6f};
         plot.spot_count = {100.0f, 120.0f, 0.0f, 90.0f};
    @@ -104,57 +125,66 @@ TEST_CASE("DiagnosticPlot_Format") {
                                   static_cast(FrameDisposition::Downgraded),
                                   static_cast(FrameDisposition::Rejected),
                                   static_cast(FrameDisposition::Rejected)};
    -
    -    const GoniometerAxis goniometer("omega", 10.0f, 0.5f, {0, 1, 0}, {});
    -    plot.goniometer = goniometer;
    -    plot.image_count = 4;
         WritePerImagePlot(prefix, plot);
     
    -    const auto lines = ReadLines(prefix + "_plot.txt");
    +    const auto all = ReadLines(prefix + "_plot.txt");
    +    // The header first, all of it comments, the last line of it the legend; no '#' after it.
    +    size_t header = 0;
    +    while (header < all.size() && all[header][0] == '#')
    +        header++;
    +    REQUIRE(header > 1);
    +    const auto lines = std::vector(all.begin() + static_cast(header) - 1, all.end());
         REQUIRE(lines.size() == 5);
    -
    -    // Exactly one legend row, and it is a comment.
    -    REQUIRE(lines[0][0] == '#');
         for (size_t i = 1; i < lines.size(); i++)
             CHECK(lines[i].find('#') == std::string::npos);
     
    +    // Every column is described in the header.
         const auto legend = Columns(lines[0]);
    -    REQUIRE(legend.size() == 11);
    -    CHECK(legend[0] == "#image");
    -    CHECK(legend[1] == "angle_deg");
    -    CHECK(legend[6] == "sigma_M_deg");
    -    CHECK(legend[8] == "cc_n");
    -    CHECK(legend[9] == "merged");
    -    CHECK(legend[10] == "disposition");
    +    REQUIRE(legend.size() == 13);
    +    for (size_t c = 1; c < legend.size(); c++) {
    +        bool described = false;
    +        for (size_t l = 0; l + 1 < header; l++)
    +            described |= all[l].starts_with("# " + legend[c] + ":");
    +        CHECK(described);
    +    }
    +    CHECK(legend[0] == "#ordinal");
    +    CHECK(legend[1] == "dataset");
    +    CHECK(legend[2] == "image");
    +    CHECK(legend[3] == "angle_deg");
    +    CHECK(legend[8] == "sigma_M_deg");
    +    CHECK(legend[10] == "cc_n");
    +    CHECK(legend[11] == "merged");
    +    CHECK(legend[12] == "disposition");
     
         for (size_t i = 1; i < lines.size(); i++) {
             const auto row = Columns(lines[i]);
             REQUIRE(row.size() == legend.size());
             CHECK(row[0] == std::to_string(i - 1));
    +        CHECK(row[1] == "1");
    +        CHECK(row[2] == std::to_string(10 + 2 * (i - 1)));
         }
     
    -    // Mid-exposure angle: start + ordinal * increment + increment / 2.
    -    CHECK(std::stof(Columns(lines[1])[1]) == 10.25f);
    -    CHECK(std::stof(Columns(lines[3])[1]) == 11.25f);
    +    CHECK(std::stof(Columns(lines[1])[3]) == 10.25f);
    +    CHECK(std::stof(Columns(lines[3])[3]) == 11.25f);
     
         // Merged and downgraded frames are in the merged data; a rejected one is not.
    -    CHECK(Columns(lines[1])[9] == "1");
    -    CHECK(Columns(lines[2])[9] == "1");
    -    CHECK(Columns(lines[3])[9] == "0");
    -    CHECK(Columns(lines[4])[9] == "0");
    +    CHECK(Columns(lines[1])[11] == "1");
    +    CHECK(Columns(lines[2])[11] == "1");
    +    CHECK(Columns(lines[3])[11] == "0");
    +    CHECK(Columns(lines[4])[11] == "0");
     
         // Graded: merged 1, downgraded 0.5, rejected 0 - and a rejected frame that never got a scale
         // never reached the merge, so nothing is known of it.
    -    CHECK(Columns(lines[1])[10] == "1");
    -    CHECK(Columns(lines[2])[10] == "0.5");
    -    CHECK(Columns(lines[3])[10] == "nan");
    -    CHECK(Columns(lines[4])[10] == "0");
    +    CHECK(Columns(lines[1])[12] == "1");
    +    CHECK(Columns(lines[2])[12] == "0.5");
    +    CHECK(Columns(lines[3])[12] == "nan");
    +    CHECK(Columns(lines[4])[12] == "0");
     
         // The count the correlation beside it was taken over.
    -    CHECK(Columns(lines[1])[8] == "500");
    +    CHECK(Columns(lines[1])[10] == "500");
     
         // A frame nothing measured leaves its own columns unplottable, not the row missing.
    -    CHECK(Columns(lines[3])[3] == "nan");
    +    CHECK(Columns(lines[3])[5] == "nan");
     }
     
     TEST_CASE("DiagnosticPlot_NoDisposition") {
    @@ -162,22 +192,22 @@ TEST_CASE("DiagnosticPlot_NoDisposition") {
     
         // Without the sweep-quality diagnostic, a frame counts as merged when it was scaled at all.
         PerImagePlot plot;
    +    OneDataset(plot, 3);
         plot.scale_g = {1.0f, NAN, 0.8f};
    -    plot.image_count = 3;
         WritePerImagePlot(prefix, plot);
     
    -    const auto lines = ReadLines(prefix + "_plot.txt");
    -    REQUIRE(lines.size() == 4);
    -    CHECK(Columns(lines[1])[9] == "1");
    -    CHECK(Columns(lines[2])[9] == "0");
    -    CHECK(Columns(lines[3])[9] == "1");
    -    CHECK(Columns(lines[1])[10] == "1");
    -    CHECK(Columns(lines[2])[10] == "nan");
    -    CHECK(Columns(lines[3])[10] == "1");
    +    const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
    +    REQUIRE(lines.size() == 3);
    +    CHECK(Columns(lines[0])[11] == "1");
    +    CHECK(Columns(lines[1])[11] == "0");
    +    CHECK(Columns(lines[2])[11] == "1");
    +    CHECK(Columns(lines[0])[12] == "1");
    +    CHECK(Columns(lines[1])[12] == "nan");
    +    CHECK(Columns(lines[2])[12] == "1");
     
         // No goniometer: the angle column is present and unplottable, so the columns never shift.
    -    CHECK(Columns(lines[1]).size() == 11);
    -    CHECK(Columns(lines[1])[1] == "nan");
    +    CHECK(Columns(lines[0]).size() == 13);
    +    CHECK(Columns(lines[0])[3] == "nan");
     }
     
     TEST_CASE("DiagnosticPlot_NoMerge") {
    @@ -186,16 +216,54 @@ TEST_CASE("DiagnosticPlot_NoMerge") {
         // Nothing was merged - --no-merge, or a sweep nothing indexed on - so "merged" has no answer to
         // give and must not read as a sweep every frame of which was thrown out.
         PerImagePlot plot;
    +    OneDataset(plot, 2);
         plot.bkg_estimate = {12.0f, 13.0f};
         plot.scale_g = {NAN, NAN};
    -    plot.image_count = 2;
         WritePerImagePlot(prefix, plot);
     
    -    const auto lines = ReadLines(prefix + "_plot.txt");
    -    REQUIRE(lines.size() == 3);
    -    CHECK(Columns(lines[1])[9] == "nan");
    -    CHECK(Columns(lines[2])[9] == "nan");
    -    CHECK(Columns(lines[1])[10] == "nan");
    +    const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
    +    REQUIRE(lines.size() == 2);
    +    CHECK(Columns(lines[0])[11] == "nan");
    +    CHECK(Columns(lines[1])[11] == "nan");
    +    CHECK(Columns(lines[0])[12] == "nan");
         // The columns a run without a merge still fills are there.
    -    CHECK(std::stof(Columns(lines[1])[2]) == 12.0f);
    +    CHECK(std::stof(Columns(lines[0])[4]) == 12.0f);
    +}
    +
    +TEST_CASE("DiagnosticPlot_JointSweeps") {
    +    const std::string prefix = "diagnostic_plot_joint_test";
    +
    +    // Two sweeps of 2 and 3 frames, as their own runs measured them.
    +    PerImagePlot a, b;
    +    OneDataset(a, 2);
    +    a.angle_deg = {0.05f, 0.15f};
    +    a.bkg_estimate = {5.0f, 6.0f};
    +    OneDataset(b, 3);
    +    b.angle_deg = {90.05f, 90.15f, 90.25f};
    +    b.spot_count = {7.0f, 8.0f, 9.0f};
    +
    +    // The joint merge's table: its rows are the sweeps' frames end to end, its scale the joint merge's.
    +    PerImagePlot joint;
    +    OneDataset(joint, 5);
    +    joint.scale_g = {1.0f, 1.0f, 2.0f, 2.0f, 2.0f};
    +    FillJointPlotRows(joint, 0, 1, a);
    +    FillJointPlotRows(joint, 2, 2, b);
    +    WritePerImagePlot(prefix, joint);
    +
    +    const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
    +    REQUIRE(lines.size() == 5);
    +    const std::vector dataset = {"1", "1", "2", "2", "2"};
    +    const std::vector image = {"0", "1", "0", "1", "2"};
    +    for (size_t i = 0; i < lines.size(); i++) {
    +        const auto row = Columns(lines[i]);
    +        CHECK(row[0] == std::to_string(i));
    +        CHECK(row[1] == dataset[i]);
    +        CHECK(row[2] == image[i]);
    +    }
    +    CHECK(std::stof(Columns(lines[0])[3]) == 0.05f);
    +    CHECK(std::stof(Columns(lines[4])[3]) == 90.25f);
    +    CHECK(std::stof(Columns(lines[1])[4]) == 6.0f);
    +    CHECK(Columns(lines[2])[4] == "nan");
    +    CHECK(std::stof(Columns(lines[3])[6]) == 8.0f);
    +    CHECK(std::stof(Columns(lines[2])[7]) == 2.0f);
     }
    
    From cbc9fa9f30eab2775886dc70899813316e8ac340 Mon Sep 17 00:00:00 2001
    From: Filip Leonarski 
    Date: Mon, 16 Nov 2026 09:55:34 +0100
    Subject: [PATCH 19/19] rugnux: name each dataset's file in _plot.txt and the
     report
    
    The dataset column of _plot.txt is now tied to its file in the file itself:
    the header has one "# dataset k: " line per input, numbered
    as the column. A joint merge lists every sweep given (joined or not), so k
    is the report's SWEEP_k_; a run of one input names its one file.
    
    A run of several masters read as one stills run lists each input in report
    section 1 as DATASET_nn =   , nn the plot's
    dataset number. (INPUT_nn is not reused: its rows carry merged/left-out
    counts of --mode scale that a stills run does not have per input.) The
    per-input indexing log lines are now made from the same counts.
    
    Co-Authored-By: Claude Opus 5.5 
    Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
    ---
     docs/RUGNUX_REPORT.md          |  3 +-
     docs/RUGNUX_TUTORIAL.md        |  6 +--
     rugnux/DiagnosticOutput.cpp    |  2 +
     rugnux/DiagnosticOutput.h      |  1 +
     rugnux/ResultReport.cpp        |  6 +++
     rugnux/Rugnux.h                |  7 ++++
     rugnux/rugnux_cli.cpp          | 74 +++++++++++++++++++++-------------
     tests/DiagnosticOutputTest.cpp |  9 ++++-
     8 files changed, 74 insertions(+), 34 deletions(-)
    
    diff --git a/docs/RUGNUX_REPORT.md b/docs/RUGNUX_REPORT.md
    index 0199f8615..e98c3e4ff 100644
    --- a/docs/RUGNUX_REPORT.md
    +++ b/docs/RUGNUX_REPORT.md
    @@ -239,6 +239,7 @@ The keys most scripts read, by section; the rest of this page explains the ones
     | `EXPERIMENT_TYPE` | `ROTATION` or `STILLS` (section 1) |
     | `IMAGES_PROCESSED`, `WAVELENGTH` | Images the run read, and the wavelength (Å) |
     | `INPUT_nn` | `--mode scale` of several `_process.h5` files: one row per file, in the order given - its images, the crystals indexed in it, those of them merged and those left out (off the run's cell form, a collapsed scale, `--min-image-cc`), and the file ([Several files](RUGNUX_ADVANCED.md#several-files)) |
    +| `DATASET_nn` | Several inputs read as one stills run: one row per input, in the order given - its images processed, those of them indexed, and the file (absolute path). `nn` is the `dataset` column of `_plot.txt` (a joint merge of sweeps numbers them `SWEEP_k_` instead, with the same `k`) |
     | `OSCILLATION_RANGE`, `STARTING_ANGLE`, `ROTATION_AXIS` | Rotation only: the per-image rotation the run integrated at (deg: the file's, scaled where the run adopted a goniometer rotation scale), the first image's angle and the axis |
     | `DETECTOR_DISTANCE`, `BEAM_CENTRE`, `DETECTOR_TILT` | The geometry the run integrated at (mm; the point of normal incidence in pixels; the three PONI rotations in degrees) - on a rotation run the post-refined one, not the header's |
     | `DIRECT_BEAM` | Where the direct beam meets the detector under that tilt (pixels) |
    @@ -371,7 +372,7 @@ A run of several sweeps merged as one (`--sweeps joint`, the default with severa
       broken by the higher ISa - and **`SWEEP_DECAY_MODEL=`** how the decay over the
       sweeps was modelled: `dose-linear` (a B rising along the dose order), `per-sweep B` (one B a sweep) or
       `none`, whichever predicted held-out frames better.
    -- Per sweep `k`: **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep
    +- Per sweep `k` (every input given, joined or not, in the order given - the `dataset` column of `_plot.txt`): **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep
       gave processed on its own - **`SWEEP_k_SPACE_GROUP_ALONE=`**, **`SWEEP_k_CELL=`**,
       **`SWEEP_k_CC_HALF_ALONE=`**, **`SWEEP_k_COMPLETENESS_ALONE=`**, **`SWEEP_k_RESOLUTION_ALONE=`**;
       **`SWEEP_k_ISOMORPHISM_RATIO=`** / **`SWEEP_k_ISOMORPHISM_RATIO_SE=`**, whether it measures the reference
    diff --git a/docs/RUGNUX_TUTORIAL.md b/docs/RUGNUX_TUTORIAL.md
    index 335b3d043..65b98bce0 100644
    --- a/docs/RUGNUX_TUTORIAL.md
    +++ b/docs/RUGNUX_TUTORIAL.md
    @@ -363,13 +363,13 @@ Absorption is corrected only by the empirical surface a rotation run fits and ke
       JSON also by the broker's `/start`, merged into its dataset settings; see
       [Module geometry](RUGNUX_FORMATS.md#module-geometry)).
     - `_plot.txt` — **the per-image table**, one row per processed image, whitespace-separated,
    -  under a `#` header - one line per column saying what it is, then a legend line over the columns -
    -  so gnuplot plots it as it stands (`plot '_plot.txt' using 1:5`). The columns are:
    +  under a `#` header - one line per column saying what it is, one `# dataset k: ` line per input
    +  (its absolute path), then a legend line over the columns - so gnuplot plots it as it stands (`plot '_plot.txt' using 1:5`). The columns are:
     
       | Column | Meaning |
       |--------|---------|
       | `ordinal` | Processed-image ordinal: the row of the run, the numbering every per-image array `rugnux` writes uses (and the report's `FIRST_IMAGE`/`LAST_IMAGE`). |
    -  | `dataset` | The input the image is from, counted from 1 in the order the inputs were given: the sweep of a joint merge of several (`--sweeps joint`), the file of several read as one stills run. 1 on a run of one input. |
    +  | `dataset` | The input the image is from, counted from 1 in the order the inputs were given: the sweep of a joint merge of several (`--sweeps joint`; the report's `SWEEP_k_`), the file of several read as one stills run (the report's `DATASET_nn`). 1 on a run of one input. The header names the file of each. |
       | `image` | The image's number in that input, from 0, as `-s`/`-e` count it; with `-s`/`--stride` it is `start + ordinal * stride`. |
       | `angle_deg` | Spindle angle at mid-exposure, on that input's own goniometer. |
       | `bkg` | Background estimate from the azimuthal profile (the `bkgEstimate` of the HDF5 file). |
    diff --git a/rugnux/DiagnosticOutput.cpp b/rugnux/DiagnosticOutput.cpp
    index 476ea4ef0..25e104ae7 100644
    --- a/rugnux/DiagnosticOutput.cpp
    +++ b/rugnux/DiagnosticOutput.cpp
    @@ -142,6 +142,8 @@ std::string PerImagePlotText(const PerImagePlot &plot) {
                 "# cc_n: observations that correlation was taken over\n"
                 "# merged: 1 where the image's observations are in the merged data, 0 where not\n"
                 "# disposition: 1 merged, 0.5 downgraded, 0 rejected\n";
    +    for (size_t k = 0; k < plot.dataset_files.size(); k++)
    +        file += fmt::format("# dataset {}: {}\n", k + 1, plot.dataset_files[k]);
         file += "#" + fmt::format("{:>7} {:>7} {:>7} {:>10} {:>12} {:>12} {:>8} {:>12} {:>13} {:>12} {:>8}"
                                    " {:>7} {:>11}\n",
                                    "ordinal", "dataset", "image", "angle_deg", "bkg", "resolution_A", "spots",
    diff --git a/rugnux/DiagnosticOutput.h b/rugnux/DiagnosticOutput.h
    index 9001eff98..4da07f2dd 100644
    --- a/rugnux/DiagnosticOutput.h
    +++ b/rugnux/DiagnosticOutput.h
    @@ -34,6 +34,7 @@ std::string RenderDetectorDiagnostic(const DiffractionExperiment &experiment,
     // frame_disposition holds FrameDisposition (Merge.h) and is empty when the sweep-quality diagnostic
     // did not run.
     struct PerImagePlot {
    +    std::vector dataset_files;     // the file of dataset k + 1, for the header
         std::vector dataset;
         std::vector image;
         std::vector angle_deg;               // at mid-exposure; NAN without a goniometer
    diff --git a/rugnux/ResultReport.cpp b/rugnux/ResultReport.cpp
    index c76dc838f..9b751a52a 100644
    --- a/rugnux/ResultReport.cpp
    +++ b/rugnux/ResultReport.cpp
    @@ -972,6 +972,12 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
                                fmt::format("{} {} {} {} {}", in.images, in.indexed, in.merged, in.indexed - in.merged,
                                            in.file)));
             }
    +        // Several inputs read as one stills run: images, indexed, file - k as the _plot.txt dataset column counts.
    +        for (size_t i = 0; i < result.dataset_inputs.size(); ++i) {
    +            const auto &in = result.dataset_inputs[i];
    +            Add(s, KeyText(fmt::format("DATASET_{:02}", i + 1).c_str(),
    +                           fmt::format("{} {} {}", in.images, in.indexed, in.file)));
    +        }
             Add(s, KeyReal("WAVELENGTH", experiment.GetWavelength_A(), "{:.5f}"));
             if (const auto gonio = experiment.GetGoniometer()) {
                 // The oscillation the run integrated at: the file's, scaled where the run adopted a
    diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h
    index 3953fa7ee..366199b52 100644
    --- a/rugnux/Rugnux.h
    +++ b/rugnux/Rugnux.h
    @@ -595,10 +595,17 @@ struct ScaleInputCount {
         int64_t images = 0, indexed = 0, merged = 0;
     };
     
    +// One of several inputs read as one stills run (JFJochMultiReader): its images processed, and those indexed.
    +struct DatasetInputCount {
    +    std::string file;
    +    int64_t images = 0, indexed = 0;
    +};
    +
     struct ProcessResult {
         bool cancelled = false;
         uint64_t images_processed = 0;
         std::vector scale_inputs;   // --mode scale of several files, in the order given
    +    std::vector dataset_inputs;   // several inputs read as one stills run, in the order given
         // The file image of processed-image ordinal f is first_image + f * image_stride (-s, --stride).
         int first_image = 0;
         int image_stride = 1;
    diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp
    index 787c55b79..b4d501eab 100644
    --- a/rugnux/rugnux_cli.cpp
    +++ b/rugnux/rugnux_cli.cpp
    @@ -3974,18 +3974,43 @@ int RugnuxInputRun::RunPreparedInput() {
             }
         }
     
    -    // The table of a run read from several inputs: each row in its own input's numbering.
    -    if (multi_reader && result.outputs && result.outputs->plot) {
    -        PerImagePlot &plot = *result.outputs->plot;
    -        for (size_t i = 0; i < plot.image_count; i++) {
    -            size_t k = multi_reader->GetNumberOfInputs() - 1;
    -            while (plot.image[i] < multi_reader->FirstImage(k))
    -                k--;
    -            plot.dataset[i] = static_cast(k) + 1;
    -            plot.image[i] -= multi_reader->FirstImage(k);
    +    // Per input of a run read from several, so a collection that indexed worse than the others shows - in the
    +    // report, and in the log below.
    +    if (multi_reader) {
    +        const auto &indexed = result.image_indexed;
    +        for (size_t k = 0; k < multi_reader->GetNumberOfInputs(); k++) {
    +            const int64_t first = multi_reader->FirstImage(k);
    +            const int64_t last = first + static_cast(multi_reader->ImagesInInput(k));
    +            DatasetInputCount count{.file = std::filesystem::absolute(input_files[k]).lexically_normal().string()};
    +            for (size_t o = 0; o < indexed.size(); o++) {
    +                const int64_t image = start_image + static_cast(o) * image_stride;
    +                if (image >= first && image < last) {
    +                    count.images++;
    +                    count.indexed += indexed[o] > 0.0f;
    +                }
    +            }
    +            result.dataset_inputs.push_back(std::move(count));
             }
         }
     
    +    // The table names its inputs; a joint merge's names its sweeps (joint_finish). A run read from several
    +    // has each row in its own input's numbering.
    +    if (result.outputs && result.outputs->plot) {
    +        PerImagePlot &plot = *result.outputs->plot;
    +        if (multi_reader) {
    +            for (const DatasetInputCount &in : result.dataset_inputs)
    +                plot.dataset_files.push_back(in.file);
    +            for (size_t i = 0; i < plot.image_count; i++) {
    +                size_t k = multi_reader->GetNumberOfInputs() - 1;
    +                while (plot.image[i] < multi_reader->FirstImage(k))
    +                    k--;
    +                plot.dataset[i] = static_cast(k) + 1;
    +                plot.image[i] -= multi_reader->FirstImage(k);
    +            }
    +        } else if (!joint_finish)
    +            plot.dataset_files = {std::filesystem::absolute(input_file).lexically_normal().string()};
    +    }
    +
         // Every file the run made, from what it returned.
         if (!output_prefix.empty()) {
             try {
    @@ -4028,24 +4053,11 @@ int RugnuxInputRun::RunPreparedInput() {
         logger.Info("Total throughput:{:.2f} MB/s", result.throughput_MBs);
         if (result.indexing_rate.has_value())
             logger.Info("Indexing rate:   {:.2f}%", result.indexing_rate.value() * 100.0);
    -    // Per input of a run read from several, so a collection that indexed worse than the others shows.
    -    if (multi_reader) {
    -        const auto &indexed = result.image_indexed;
    -        for (size_t k = 0; k < multi_reader->GetNumberOfInputs(); k++) {
    -            const int64_t first = multi_reader->FirstImage(k);
    -            const int64_t last = first + static_cast(multi_reader->ImagesInInput(k));
    -            int64_t processed = 0, n_indexed = 0;
    -            for (size_t o = 0; o < indexed.size(); o++) {
    -                const int64_t image = start_image + static_cast(o) * image_stride;
    -                if (image >= first && image < last) {
    -                    processed++;
    -                    n_indexed += indexed[o] > 0.0f;
    -                }
    -            }
    -            logger.Info("  input {}:       {:.2f}% of {} images ({})", k + 1,
    -                        processed > 0 ? 100.0 * static_cast(n_indexed) / static_cast(processed) : 0.0,
    -                        processed, input_files[k]);
    -        }
    +    for (size_t k = 0; k < result.dataset_inputs.size(); k++) {
    +        const DatasetInputCount &in = result.dataset_inputs[k];
    +        logger.Info("  input {}:       {:.2f}% of {} images ({})", k + 1,
    +                    in.images > 0 ? 100.0 * static_cast(in.indexed) / static_cast(in.images) : 0.0,
    +                    in.images, input_files[k]);
         }
         // Final one-line summary of the adopted crystal (whether de-novo determined or fixed with -S),
         // so it is not buried in the space-group-search block (which is de-novo only) or only in the mmCIF.
    @@ -4557,11 +4569,15 @@ int RugnuxCli::RunJointSweeps(const RugnuxSweepPlan &plan) {
                                                                         : relation.sweeps[k].why_left_out)});
                     result.joint_sweeps.push_back(std::move(w));
                 }
    -            // The joint table's rows are the sweeps' frames end to end: each says which sweep and frame it is.
    -            if (result.outputs && result.outputs->plot)
    +            // The joint table's rows are the sweeps' frames end to end: each says which sweep and frame it is,
    +            // the sweep numbered as SWEEP_k_ in the report (every input given, joined or not).
    +            if (result.outputs && result.outputs->plot) {
    +                for (const std::string &in : inputs)
    +                    result.outputs->plot->dataset_files.push_back(std::filesystem::absolute(in).lexically_normal().string());
                     for (size_t seg = 0; seg < input.segments.size(); ++seg)
                         FillJointPlotRows(*result.outputs->plot, input.segments[seg].first_frame,
                                           input.segment_sweep[seg] + 1, members[input.segment_sweep[seg]].plot);
    +            }
                 if (result.outputs && result.outputs->merged && export_unmerged) {
                     const MergedOutput &merged = *result.outputs->merged;
                     std::vector history{"From Rugnux " + jfjoch_version() + ", data reduction",
    diff --git a/tests/DiagnosticOutputTest.cpp b/tests/DiagnosticOutputTest.cpp
    index 8993cb6d8..885c8671c 100644
    --- a/tests/DiagnosticOutputTest.cpp
    +++ b/tests/DiagnosticOutputTest.cpp
    @@ -3,6 +3,7 @@
     
     #include 
     
    +#include 
     #include 
     #include 
     #include 
    @@ -245,12 +246,18 @@ TEST_CASE("DiagnosticPlot_JointSweeps") {
         // The joint merge's table: its rows are the sweeps' frames end to end, its scale the joint merge's.
         PerImagePlot joint;
         OneDataset(joint, 5);
    +    joint.dataset_files = {"/data/a_master.h5", "/data/b_master.h5"};
         joint.scale_g = {1.0f, 1.0f, 2.0f, 2.0f, 2.0f};
         FillJointPlotRows(joint, 0, 1, a);
         FillJointPlotRows(joint, 2, 2, b);
         WritePerImagePlot(prefix, joint);
     
    -    const auto lines = Rows(ReadLines(prefix + "_plot.txt"));
    +    // The header names the file of each dataset, numbered as the dataset column.
    +    const auto all = ReadLines(prefix + "_plot.txt");
    +    CHECK(std::count(all.begin(), all.end(), "# dataset 1: /data/a_master.h5") == 1);
    +    CHECK(std::count(all.begin(), all.end(), "# dataset 2: /data/b_master.h5") == 1);
    +
    +    const auto lines = Rows(all);
         REQUIRE(lines.size() == 5);
         const std::vector dataset = {"1", "1", "2", "2", "2"};
         const std::vector image = {"0", "1", "0", "1", "2"};