diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index 738fc2278..e428b70b9 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -49,6 +49,68 @@ on: default: 'manual' jobs: + documentation: + name: Build documentation + # Uploads to the release, so it waits for the job that creates it. First in the file, so that + # it is handed out ahead of the long Linux builds: build-windows and build-macos wait for it. + needs: create-release + runs-on: jfjoch_rocky8 + container: + image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609 + steps: + - uses: actions/checkout@v4 + - name: Build documentation + shell: bash + run: bash make_doc.sh + # Built once here for the Windows and macOS rugnux packages too (the Windows runner has no + # Python new enough for Sphinx). The sphinx-build doctree cache in public/.doctrees is not shipped. + - name: Upload the HTML for the Windows and macOS packages + uses: actions/upload-artifact@v3 + with: + name: docs-html + path: | + public/ + !public/.doctrees/ + # The Rugnux pages site is the gitea-pages branch of rugnux/gitea-pages, updated with a release. + # The HTML is pushed from a copy, so the checkout the release upload below works on is left as it was. + - name: Upload documentation to rugnux/gitea-pages + if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) + shell: bash + env: + PAGES_TOKEN: ${{ secrets.RUGNUX_PAGES_TOKEN || secrets.PIP_REPOSITORY_API_TOKEN }} + PAGES_TOKEN_NAME: ${{ secrets.RUGNUX_PAGES_TOKEN && 'RUGNUX_PAGES_TOKEN' || 'PIP_REPOSITORY_API_TOKEN' }} + run: | + echo "Pushing with secret ${PAGES_TOKEN_NAME}" + rm -rf /tmp/rugnux-pages + mkdir /tmp/rugnux-pages + cp -a public/. /tmp/rugnux-pages/ + cd /tmp/rugnux-pages + git init -q + git checkout -q -b gitea-pages + git config user.email "filip.leonarski@psi.ch" + git config user.name "Filip Leonarski (Gitea)" + git add . + git commit -q -m "Deploy site from ${GITHUB_REF_NAME} ${GITHUB_SHA::9}" + git push -f "https://${PAGES_TOKEN}@gitea.psi.ch/rugnux/gitea-pages.git" gitea-pages + - name: Upload documentation + if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) + shell: bash + run: | + git config --global user.email "filip.leonarski@psi.ch" + git config --global user.name "Filip Leonarski (Gitea)" + git checkout --orphan gitea-pages + git reset --hard + # Remove everything except .git and public + git clean -fdx -e public -e .git + + # Move public contents to root and remove the now-empty public dir + shopt -s dotglob nullglob + mv public/* . + rm -rf public + git add . + git commit -m "Deploy site" + git push -f https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git gitea-pages + # ---- Portable Linux artifacts: viewer, rugnux, XDS plugin ----------------------------------------- # One viewer-only build per variant packs three tarballs, one program each (see CPack in # CMakeLists.txt): the viewer build compiles everything rugnux and the plugin need anyway. @@ -92,19 +154,17 @@ jobs: run: | cd build ninja -j16 + ninja rugnux_docs # the HTML documentation, shipped in the rugnux tarball cpack - - name: Verify one libjpeg in the viewer + - name: Verify no Qt JPEG/TIFF plugin in the viewer shell: bash run: | set -euo pipefail - # With the static Qt, Qt's JPEG plugin brings its own libjpeg-turbo (another ABI) and the - # linker silently mixes it with ours (see viewer/CMakeLists.txt). Files, not pipes into - # `grep -q`, for the SIGPIPE reason given in the rugnux check below. + # The viewer excludes them (see viewer/CMakeLists.txt). Files, not pipes into `grep -q`, + # for the SIGPIPE reason given in the rugnux check below. strings build/viewer/jfjoch_viewer > /tmp/jfjv.strings if grep -qE 'QJpegHandler|QTiffHandler' /tmp/jfjv.strings; then echo "a Qt JPEG/TIFF plugin is linked"; exit 1; fi - grep -c 'libjpeg-turbo version' /tmp/jfjv.strings > /tmp/jfjv.njpeg || true - if [ "$(cat /tmp/jfjv.njpeg)" != 1 ]; then echo "expected one libjpeg-turbo, found $(cat /tmp/jfjv.njpeg)"; exit 1; fi - echo "one libjpeg-turbo, no Qt JPEG/TIFF plugin" + echo "no Qt JPEG/TIFF plugin" - name: Verify rugnux is self-contained shell: bash run: | @@ -112,6 +172,7 @@ jobs: rm -rf /tmp/rgx && mkdir -p /tmp/rgx tar xf build/rugnux-*.tar.gz -C /tmp/rgx bin=$(find /tmp/rgx -name rugnux -type f) + test -f "$(dirname "$bin")/../share/doc/jfjoch_rugnux/html/RUGNUX_CLI.html" # Each producer writes to a file and the greps read files, rather than piping into # `grep -q`: grep exits at the first match, the producer then dies of SIGPIPE, and under # `set -o pipefail` that becomes the pipeline's status -- which a leading `!` would turn @@ -120,6 +181,13 @@ jobs: if grep -q cufft /tmp/rgx.ldd; then echo "cuFFT is linked dynamically"; exit 1; fi find /tmp/rgx -name 'libcufft*' > /tmp/rgx.cufft if [ -s /tmp/rgx.cufft ]; then echo "a libcufft is shipped in the tarball"; exit 1; fi + # Nothing but glibc and the GCC runtime: every other library (zstd, zlib, HDF5, ...) is built + # and linked statically, and a host copy found by some dependency's configure is a second, + # different version resolved at runtime. + awk '/=>|ld-linux/ {print $1}' /tmp/rgx.ldd > /tmp/rgx.libs + grep -vE '^(/lib64/)?(linux-vdso|ld-linux-x86-64|libc|libm|libmvec|libdl|libpthread|librt|libstdc\+\+|libgcc_s)\.so' \ + /tmp/rgx.libs > /tmp/rgx.extra || true + if [ -s /tmp/rgx.extra ]; then echo "rugnux links non-system libraries:"; cat /tmp/rgx.extra; exit 1; fi echo "self-contained: static cuFFT, nothing to ship beside it" - name: Verify the XDS plugin exports shell: bash @@ -164,7 +232,7 @@ jobs: build-windows: name: build:portable:windows-x86_64:${{ matrix.variant }} # Uploads to the release, so it waits for the job that creates it. - needs: create-release + needs: [create-release, documentation] runs-on: windows-11-cuda-qt timeout-minutes: 120 strategy: @@ -202,6 +270,14 @@ jobs: run: | cd build\tests jfjoch_portable_test.exe "[portable]" + # The HTML documentation for the rugnux package, built by the documentation job, where the + # OPTIONAL install rule for rugnux_docs looks for it. download-artifact comes from the + # mx/download-artifact mirror (the actions org has none); v4 refuses any server but github.com. + - name: Download the HTML documentation + uses: https://gitea.psi.ch/mx/download-artifact@v3 + with: + name: docs-html + path: build/docs_html - name: Build installer (NSIS) and rugnux zip shell: cmd run: | @@ -219,7 +295,7 @@ jobs: $names = [System.IO.Compression.ZipFile]::OpenRead($zip[0].FullName).Entries | ForEach-Object { $_.FullName } $names if (-not ($names -match 'rugnux\.exe$')) { throw 'rugnux.exe missing from the zip' } - if ($names -match 'jfjoch_viewer') { throw 'the viewer is in the rugnux zip' } + if ($names -match 'jfjoch_viewer\.exe$') { throw 'the viewer is in the rugnux zip' } if ('${{ matrix.use_cuda }}' -eq 'ON' -and -not ($names -match 'cufft64_.*\.dll$')) { throw 'cuFFT DLL missing from the CUDA zip' } - name: Upload installer and rugnux zip to release if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) @@ -239,7 +315,7 @@ jobs: build-macos: name: build:portable:macos-arm64:nocuda # Uploads to the release, so it waits for the job that creates it. - needs: create-release + needs: [create-release, documentation] # A Mac mini running act_runner in host mode (no containers on macOS): the job runs as the runner's # user with that user's tools - Xcode command line tools, CMake.app and Qt under ~/Qt - so the # paths below are this machine's, like the Windows runner's C:/Qt. @@ -288,6 +364,14 @@ jobs: run: cd build-macos/tests && ./jfjoch_portable_test "[Metal]" # One build, two artifacts: the viewer .dmg and rugnux as .tar.gz (CPACK_PROJECT_CONFIG_FILE # in CMakeLists.txt gives each generator its components). The XDS plugin is Linux-only. + # The HTML documentation for the rugnux package, built by the documentation job, where the + # OPTIONAL install rule for rugnux_docs looks for it. download-artifact comes from the + # mx/download-artifact mirror (the actions org has none); v4 refuses any server but github.com. + - name: Download the HTML documentation + uses: https://gitea.psi.ch/mx/download-artifact@v3 + with: + name: docs-html + path: build-macos/docs_html - name: Build viewer DMG and rugnux tgz shell: bash run: | @@ -415,7 +499,7 @@ jobs: file "$bin" > /tmp/tc.bin cat /tmp/tc.bin if ! grep -q "ARM aarch64" /tmp/tc.bin; then echo "not an aarch64 binary"; exit 1; fi - # nothing x86 may have leaked in (libjpeg-turbo's ExternalProject is the classic culprit) + # nothing x86 may have leaked in (an ExternalProject not told the target is the classic culprit) find /tmp/tgzcheck -type f -exec file {} + > /tmp/tc.arch if grep -q "x86-64" /tmp/tc.arch; then echo "an x86-64 file leaked into the tarball"; exit 1; fi # cuFFT must be linked statically: nothing to ship beside the binary, nothing to find @@ -754,6 +838,20 @@ jobs: cd build ninja frontend ninja -j16 package + - name: Verify the services link no non-system libraries + shell: bash + run: | + set -euo pipefail + # As for rugnux in build-portable-tgz: glibc and the GCC runtime only, plus cuFFT from the + # CUDA toolkit in a CUDA build. Files, not pipes into grep, for the SIGPIPE reason given there. + for bin in build/broker/jfjoch_broker build/rugnux/rugnux build/writer/jfjoch_writer; do + ldd "$bin" > /tmp/svc.ldd 2>&1 || true + awk '/=>|ld-linux/ {print $1}' /tmp/svc.ldd > /tmp/svc.libs + grep -vE '^(/lib64/)?(linux-vdso|ld-linux-x86-64|libc|libm|libmvec|libdl|libpthread|librt|libstdc\+\+|libgcc_s|libcufft)\.so' \ + /tmp/svc.libs > /tmp/svc.extra || true + if [ -s /tmp/svc.extra ]; then echo "$bin links non-system libraries:"; cat /tmp/svc.extra; exit 1; fi + done + echo "only system libraries linked" - name: Upload packages if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) shell: bash @@ -937,58 +1035,6 @@ jobs: twine upload -u __token__ -p ${{ secrets.CI_PYPI_TOKEN }} --skip-existing dist/* twine upload -u __token__ -p ${{ secrets.PIP_REPOSITORY_API_TOKEN }} --repository-url https://gitea.psi.ch/api/packages/mx/pypi dist/* - documentation: - name: Build documentation - # Uploads to the release, so it waits for the job that creates it. - needs: create-release - runs-on: jfjoch_rocky8 - container: - image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609 - steps: - - uses: actions/checkout@v4 - - name: Build documentation - shell: bash - run: bash make_doc.sh - # The Rugnux pages site is the gitea-pages branch of rugnux/gitea-pages, updated with a release. - # The HTML is pushed from a copy, so the checkout the release upload below works on is left as it was. - - name: Upload documentation to rugnux/gitea-pages - if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) - shell: bash - env: - PAGES_TOKEN: ${{ secrets.RUGNUX_PAGES_TOKEN || secrets.PIP_REPOSITORY_API_TOKEN }} - PAGES_TOKEN_NAME: ${{ secrets.RUGNUX_PAGES_TOKEN && 'RUGNUX_PAGES_TOKEN' || 'PIP_REPOSITORY_API_TOKEN' }} - run: | - echo "Pushing with secret ${PAGES_TOKEN_NAME}" - rm -rf /tmp/rugnux-pages - mkdir /tmp/rugnux-pages - cp -a public/. /tmp/rugnux-pages/ - cd /tmp/rugnux-pages - git init -q - git checkout -q -b gitea-pages - git config user.email "filip.leonarski@psi.ch" - git config user.name "Filip Leonarski (Gitea)" - git add . - git commit -q -m "Deploy site from ${GITHUB_REF_NAME} ${GITHUB_SHA::9}" - git push -f "https://${PAGES_TOKEN}@gitea.psi.ch/rugnux/gitea-pages.git" gitea-pages - - name: Upload documentation - if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true) - shell: bash - run: | - git config --global user.email "filip.leonarski@psi.ch" - git config --global user.name "Filip Leonarski (Gitea)" - git checkout --orphan gitea-pages - git reset --hard - # Remove everything except .git and public - git clean -fdx -e public -e .git - - # Move public contents to root and remove the now-empty public dir - shopt -s dotglob nullglob - mv public/* . - rm -rf public - git add . - git commit -m "Deploy site" - git push -f https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git gitea-pages - # ---- Release ------------------------------------------------------------------------------------- # Runs on every event, but only creates something when asked. Keeping it unconditional is what # lets every other job depend on it with a plain `needs:` -- a skipped dependency would otherwise diff --git a/CLAUDE.md b/CLAUDE.md index f717306ec..20a2a96e8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,8 +9,9 @@ X-ray detectors. It receives detector data, runs it through an FPGA-accelerated (spot finding, azimuthal/ROI integration, compression), streams images out over ZeroMQ for writing to HDF5, and runs crystallographic analysis (indexing, integration, scaling/merging). Most authoritative documentation lives in `docs/` and on Read The Docs -(https://jungfraujoch.readthedocs.io). When changing CLI behaviour, the program's own -usage message is the source of truth, not the docs. +(https://jungfraujoch.readthedocs.io). rugnux's options are all described in +`docs/RUGNUX_CLI.md` (the program's own usage shows only the modes and the basic options): when +changing CLI behaviour, update the parser, that page and, for a basic option, the usage together. ## Build @@ -25,8 +26,8 @@ the build tree is what every `find_package(Eigen3)` resolves — ours, Ceres' an already on the machine is used instead if the configure is given `-DZLIB_ROOT=` / `-DEigen3_DIR=`. Do **not** turn either of these into `FetchContent ... OVERRIDE_FIND_PACKAGE` — see the Eigen note in `CMakeLists.txt` for the crash that rules it out. -libjpeg-turbo is built via `ExternalProject` in `preview/`; Mbed TLS (the viewer's HTTPS -backend for cpp-httplib) is fetched only for viewer builds. +JPEG is encoded by the vendored single-header `stb_image_write` (`preview/stb/`); Mbed TLS +(the viewer's HTTPS backend for cpp-httplib) is fetched only for viewer builds. ``` mkdir build && cd build @@ -360,7 +361,8 @@ generated API model and internal types. - `jfjoch_viewer` — interactive Qt desktop (`viewer/`), results not persisted. - `rugnux` (`rugnux/rugnux_cli.cpp`, built on the `JFJochRugnux` library in the same directory) — offline batch over a stored HDF5, invoked as `rugnux {} ` (it has **no `--help`**; run - it with no arguments to print the usage, which is the authority on its flags). Rotation vs stills + it with no arguments to print a short usage: one section per mode and the basic options; every + option is listed in `docs/RUGNUX_CLI.md`, which must be kept in step with the parser). Rotation vs stills is auto-detected from the goniometer axis. Merging is on by default (`--no-merge` to disable); merging writes `.mtz`/`.cif`/`.hkl` and skips the bulky `_process.h5` unless `--write-process-h5`, while `--no-merge` writes only `_process.h5`. `--mode` picks what a run does: `mx` (the full @@ -417,8 +419,8 @@ portable; don't constrain it for portability's sake. intended Windows configuration. `JFJOCH_VIEWER_ONLY` is forced ON on Windows and macOS, so a plain configure there already builds -just the portable subset. libtiff, libjpeg-turbo (via `ExternalProject` in `preview/`) and Mbed TLS -(viewer builds only) are now brought in by the build itself, so they need nothing from the host. +just the portable subset. libtiff and Mbed TLS (viewer builds only) are now brought in by the build +itself, so they need nothing from the host. **ZLIB** and **Eigen** (header-only; needed by Ceres, by the analysis libs directly, and by `ffbidx` under CUDA) are brought in by the build too, so on Windows only Qt still has to be installed first. Both are still reached through an ordinary `find_package`, and that is the point: **vendoring Eigen diff --git a/CMakeLists.txt b/CMakeLists.txt index aaff6cbdc..e5cb7c592 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,13 +425,18 @@ SET(CMAKE_POLICY_VERSION_MINIMUM 3.5) # libzmq must be made available BEFORE sls_detector_package for the override above to take effect. FetchContent_MakeAvailable(libzmq) +# httplib links zstd::libzstd when that target exists, and otherwise looks for a system libzstd +# (pkg-config) - which put a second, older zstd beside our static one, and the BSHUF_ZSTD calls +# resolved to the system copy at runtime. Point it at our static build. +FetchContent_MakeAvailable(zstd) +add_library(zstd::libzstd ALIAS libzstd_static) IF (JFJOCH_PORTABLE_ONLY) - # A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2 + # A portable build still needs hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2 # for jfjoch_portable_test. Only sls_detector_package (detector) is not built here -- it does # not configure under MSVC. - FetchContent_MakeAvailable(zstd catch2 hdf5 spdlog httplib) + FetchContent_MakeAvailable(catch2 hdf5 spdlog httplib) ELSE() - FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib) + FetchContent_MakeAvailable(sls_detector_package catch2 hdf5 spdlog httplib) ENDIF() # libtiff (used by JFJochPreview in every build mode): build it ourselves. Its C++ binding @@ -582,7 +587,7 @@ SET(JFJOCH_NOTICE_FILES IF (JFJOCH_RUGNUX_ONLY) # The viewer-only subtree minus viewer/ and tools/: just rugnux and what it links. broker/ is # here for JFJochAPI alone (JFJochReader links it); its service targets are gated out. preview/ - # arrives through JFJochWriter -> JFJochPreview, so libjpeg-turbo and libtiff are built too. + # arrives through JFJochWriter -> JFJochPreview, so libtiff is built too. ADD_SUBDIRECTORY(jungfrau) ADD_SUBDIRECTORY(compression) ADD_SUBDIRECTORY(common) @@ -944,6 +949,31 @@ else() endif() +# The Sphinx HTML documentation, shipped with rugnux: its usage points at +# share/doc/jfjoch_rugnux/html/RUGNUX_CLI.html in the release archive. Never part of the default +# build - `make rugnux_docs` builds it in a venv of its own from docs/requirements.txt (so it needs +# Python 3.10+ and pip's network, not an installed Sphinx), and the OPTIONAL install rule ships +# whatever it left in the build tree; without it the packages are made as before, with no HTML. +SET(JFJOCH_DOCS_HTML_DIR ${CMAKE_BINARY_DIR}/docs_html) +FIND_PACKAGE(Python3 3.10 COMPONENTS Interpreter) +IF (Python3_FOUND) + IF (WIN32) + SET(_docs_python ${CMAKE_BINARY_DIR}/docs_venv/Scripts/python.exe) + ELSE() + SET(_docs_python ${CMAKE_BINARY_DIR}/docs_venv/bin/python) + ENDIF() + ADD_CUSTOM_TARGET(rugnux_docs + COMMAND ${Python3_EXECUTABLE} -m venv ${CMAKE_BINARY_DIR}/docs_venv + COMMAND ${_docs_python} -m pip install -q -r ${CMAKE_SOURCE_DIR}/docs/requirements.txt + COMMAND ${_docs_python} -m sphinx -q -b html -d ${CMAKE_BINARY_DIR}/docs_doctrees + ${CMAKE_SOURCE_DIR}/docs ${JFJOCH_DOCS_HTML_DIR} + COMMENT "Building the HTML documentation into ${JFJOCH_DOCS_HTML_DIR}") +ELSE() + MESSAGE(STATUS "No Python 3.10 or newer: no rugnux_docs target, rugnux is packaged without the HTML documentation") +ENDIF() +INSTALL(DIRECTORY ${JFJOCH_DOCS_HTML_DIR}/ DESTINATION share/doc/jfjoch_rugnux/html COMPONENT rugnux + OPTIONAL PATTERN ".buildinfo" EXCLUDE) + # Ship the notices (top-level LICENSE, the third-party manifest, the verbatim license texts, and # the acknowledgements the programs refer a user to) with every package variant. Installed per # component so the files land in whichever component package(s) the current build mode produces. diff --git a/README.md b/README.md index 38dd65832..dfa306534 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,17 @@ and small-molecule - from images to merged reflections. All documentation is now placed in [docs/](docs/) subdirectory and for the current version hosted on [Jungfraujoch Read The Docs page](https://jungfraujoch.readthedocs.io). + +## How to cite + +- **Jungfraujoch**: F. Leonarski et al., "Jungfraujoch: hardware-accelerated data-acquisition system for + kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30, 227-234 + [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268); for real-time serial + crystallography and live indexing feedback also J. Duan et al., "Jungfraujoch: 38 GB s⁻¹ real-time + serial crystallography – data acquisition and live indexing feedback" (2026), J. Appl. Cryst. 59, + 1668-1677 [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083). +- **Rugnux**: F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons + and XFELs in one program" (2026), bioRxiv + [doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). + +The full list is in [docs/PUBLICATIONS.md](docs/PUBLICATIONS.md). diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 6b3a66af2..6e81df442 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -28,7 +28,6 @@ linked into the Jungfraujoch executables. | [FFTW](https://www.fftw.org/) | 3.3.10 | Matteo Frigo; MIT | GPL-2.0-or-later | [fftw.txt](licenses/fftw.txt) | | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) | (pinned) | Google Inc. and contributors | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) | | [Abseil](https://github.com/abseil/abseil-cpp) | 20250127 (required by Ceres) | Google Inc. | Apache-2.0 | [abseil.txt](licenses/abseil.txt) | -| [libjpeg-turbo](https://github.com/libjpeg-turbo/libjpeg-turbo) | 3.2.0 | D. R. Commander and others; IJG | IJG + BSD-3-Clause + Zlib | [libjpeg-turbo.txt](licenses/libjpeg-turbo.txt) | | [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) (with TF-PSA-Crypto) | 4.2.0 | The Mbed TLS Contributors | Apache-2.0 OR GPL-2.0-or-later (used under Apache-2.0) | [mbedtls.txt](licenses/mbedtls.txt), [tf-psa-crypto.txt](licenses/tf-psa-crypto.txt) | | [Catch2](https://github.com/catchorg/Catch2) | 3.16.0 | Catch2 Authors | BSL-1.0 | [catch2.txt](licenses/catch2.txt) | | [zlib-ng](https://github.com/zlib-ng/zlib-ng) | 2.3.3 | Jean-loup Gailly and Mark Adler; the zlib-ng contributors | Zlib | [zlib.txt](licenses/zlib.txt) | @@ -58,10 +57,12 @@ These live in the source tree (see the path) rather than being fetched; traccc a | [Bitshuffle](https://github.com/kiyo-masui/bitshuffle) | `compression/bitshuffle/` | Kiyoshi Masui | MIT | [bitshuffle.txt](licenses/bitshuffle.txt) | | [Bitshuffle (h-perf)](https://github.com/kalcutter/bitshuffle) | `compression/bitshuffle_hperf/` | Kal Conley | Apache-2.0 | [bitshuffle-hperf.txt](licenses/bitshuffle-hperf.txt) | | [LZ4](https://github.com/lz4/lz4) | `compression/lz4/` | Yann Collet | BSD-2-Clause | [lz4.txt](licenses/lz4.txt) | +| [stb_image_write](https://github.com/nothings/stb) (v1.16, commit 31c1ad3) | `preview/stb/` | Sean Barrett | MIT OR Unlicense | [stb_image_write.txt](licenses/stb_image_write.txt) | | [HLS arbitrary-precision types](https://github.com/Xilinx/HLS_arbitrary_Precision_Types) | `fpga/include/` | Xilinx, Inc. | Apache-2.0 | [xilinx-hls-headers.txt](licenses/xilinx-hls-headers.txt) | | [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (commit a94fd4f) | `image_analysis/indexing/fast-feedback-indexer/` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](licenses/fast-feedback-indexer.txt) | | [GEMMI](https://github.com/project-gemmi/gemmi) | `gemmi_gph/` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) | | [CCP4 Monomer Library](https://github.com/MonomerLibrary/monomers) (subset: restraint dictionary data) | `image_analysis/structure_refinement/monomers/` | CCP4 / the Monomer Library authors (Vagin et al.) | LGPL-3.0-only | [ccp4-monomer-library.txt](licenses/ccp4-monomer-library.txt) | +| [Larch](https://github.com/xraypy/xraylarch) (its Cromer-Liberman routines, as GEMMI ports them) | `gemmi_gph/fprime.cpp` | Matthew Newville, The University of Chicago, et al. | MIT | [larch.txt](licenses/larch.txt) | | [PEGTL](https://github.com/taocpp/PEGTL) | `gemmi_gph/gemmi/third_party/tao/` | Dr. Colin Hirsch and Daniel Frey | MIT | [pegtl.txt](licenses/pegtl.txt) | | [sajson](https://github.com/chadaustin/sajson) | `gemmi_gph/gemmi/third_party/sajson.h` | Chad Austin | MIT | [sajson.txt](licenses/sajson.txt) | | [fast_float](https://github.com/fastfloat/fast_float) | `gemmi_gph/gemmi/third_party/fast_float.h` | The fast_float authors (Daniel Lemire et al.) | Apache-2.0 OR MIT OR BSL-1.0 | [fast-float.txt](licenses/fast-float.txt) | diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 3c311adcd..171dc380d 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -83,6 +83,11 @@ GPUWorkBeside::~GPUWorkBeside() { remove(gpu_work_beside_n); } +bool gpu_work_fits_beside_merges(size_t bytes) { + std::lock_guard lock(gpu_work_m); + return gpu_merges_n == 0 || fits_beside_merges(bytes); +} + bool wait_for_gpu_work_beside(std::chrono::seconds timeout) { std::unique_lock lock(gpu_work_m); return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; }); diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index 537864081..80aa9d967 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -165,6 +165,12 @@ public: void Holds(size_t bytes); }; +// Whether work holding `bytes` of the card would start beside the main line's merges now (GPUWorkBeside with +// after_merges): no merge is held, or what they said they hold (GPUMerge::Holds) and `bytes` fit within 80% of +// the first card's total. For a main-line caller that is about to wait for such work, and must not where that +// work waits for its merge. Build-independent - it only counts. +bool gpu_work_fits_beside_merges(size_t bytes); + // Whether the card is too small for a probe pass beside the main line's image pass (12 GB, decided on the // first card's total memory), so that they take turns (GPUWorkBeside). Build-independent. bool gpu_takes_turns(); diff --git a/common/DetectorModules.cpp b/common/DetectorModules.cpp index 7b313ed65..464e40b7f 100644 --- a/common/DetectorModules.cpp +++ b/common/DetectorModules.cpp @@ -4,6 +4,7 @@ #include "DetectorModules.h" #include "DetectorGeometry.h" #include "Definitions.h" +#include "PixelMask.h" #include #include @@ -75,9 +76,14 @@ const std::vector &DetectorModules::Get() const { } bool DetectorModules::Moved() const { + // Not an exact comparison: a module read back from a file went through a length and a unit vector + // in single precision, which puts a module as imaged a few 1e-4 px off its box when it is far from + // pixel (0, 0) on both axes. + constexpr double tolerance = 1e-3; for (const auto &m: modules) - if (m.ox != static_cast(m.x0) || m.oy != static_cast(m.y0) - || m.fx != 1.0 || m.fy != 0.0 || m.sx != 0.0 || m.sy != 1.0) + if (std::fabs(m.ox - static_cast(m.x0)) > tolerance || std::fabs(m.oy - static_cast(m.y0)) > tolerance + || std::fabs(m.fx - 1.0) > tolerance / CONVERTED_MODULE_COLS || std::fabs(m.fy) > tolerance / CONVERTED_MODULE_COLS + || std::fabs(m.sx) > tolerance / CONVERTED_MODULE_COLS || std::fabs(m.sy - 1.0) > tolerance / CONVERTED_MODULE_COLS) return true; return false; } @@ -193,3 +199,39 @@ DetectorModules ModulesAsImaged(const DetectorGeometry &geom) { } return DetectorModules(out); } + +DetectorModules ModulesFromGapMask(const std::vector &mask, int64_t width, int64_t height) { + if (width <= 0 || height <= 0 || mask.size() != static_cast(width * height)) + return {}; + const auto clear = [&](int64_t x, int64_t y) { + return x >= 0 && y >= 0 && x < width && y < height + && (mask[y * width + x] & (1u << PixelMask::ModuleGapPixelBit)) == 0; + }; + std::vector taken(mask.size(), 0); + std::vector out; + for (int64_t y = 0; y < height; y++) + for (int64_t x = 0; x < width; x++) { + if (!clear(x, y) || taken[y * width + x]) + continue; + // The first clear pixel of a region not yet taken is a module's corner: the module runs to the + // gap along the row and down the column, and has to be whole and bounded by gap all round. + int64_t w = 0, h = 0; + while (clear(x + w, y)) w++; + while (clear(x, y + h)) h++; + if (!((w == CONVERTED_MODULE_COLS && h == CONVERTED_MODULE_LINES) + || (w == CONVERTED_MODULE_LINES && h == CONVERTED_MODULE_COLS))) + return {}; + for (int64_t yy = y - 1; yy <= y + h; yy++) + for (int64_t xx = x - 1; xx <= x + w; xx++) { + const bool inside = xx >= x && xx < x + w && yy >= y && yy < y + h; + if (clear(xx, yy) != inside) + return {}; + if (inside) + taken[yy * width + xx] = 1; + } + out.push_back(DetectorModule::AsImaged(x, y, w, h)); + } + if (out.size() < 2) + return {}; + return DetectorModules(out); +} diff --git a/common/DetectorModules.h b/common/DetectorModules.h index d38649d4f..4f979b952 100644 --- a/common/DetectorModules.h +++ b/common/DetectorModules.h @@ -121,3 +121,9 @@ public: // CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES turned the way the geometry turns it, placed as imaged. // Empty for a detector of one module. DetectorModules ModulesAsImaged(const DetectorGeometry &geom); + +// The modules of a JUNGFRAU image where only its pixel mask says where they are: every region of pixels +// clear of the module-gap bit (PixelMask::ModuleGapPixelBit) has to be one whole module, +// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES either way round, and is taken as placed as imaged. +// Empty where any region is not, or where there are fewer than two. The mask is row-major, width x height. +DetectorModules ModulesFromGapMask(const std::vector &mask, int64_t width, int64_t height); diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 488a8f1fb..d8bb68ff4 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -653,6 +653,10 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() { void DiffractionExperiment::FillMessage(StartMessage &message) const { message.images_per_file = GetImagesPerFile(); message.detector_modules = GetModules().Get(); + // A JUNGFRAU image assembled here is made of modules this experiment places itself: they go into + // the file one by one, so that what reads it back knows them. + if (message.detector_modules.empty() && GetDetectorType() == DetectorType::JUNGFRAU && IsGeometryTransformed()) + message.detector_modules = ModulesAsImaged(detector.GetGeometry()).Get(); message.beam_center_x = GetBeamX_pxl(); message.beam_center_y = GetBeamY_pxl(); const auto direct_beam = GetDiffractionGeometry().GetDirectBeam_pxl(); diff --git a/common/HostMemory.cpp b/common/HostMemory.cpp index 54c4a8b66..4c6771082 100644 --- a/common/HostMemory.cpp +++ b/common/HostMemory.cpp @@ -3,6 +3,8 @@ #include "HostMemory.h" +#include + #if defined(_WIN32) #ifndef NOMINMAX #define NOMINMAX @@ -47,6 +49,7 @@ HostWorkerPool &HostWorkerPool::Instance() { } HostWorkerPool::HostWorkerPool() { + SetBudget(0); SetSize(0); } @@ -57,14 +60,24 @@ void HostWorkerPool::SetSize(size_t bytes) { cv.notify_all(); } +void HostWorkerPool::SetBudget(size_t bytes) { + std::lock_guard lock(m); + budget = bytes > 0 ? bytes : DefaultHostMemoryBudget(); + cv.notify_all(); +} + +size_t HostWorkerPool::available() const { + return std::min(size, budget > held ? budget - held : 0); +} + size_t HostWorkerPool::Size() { std::lock_guard lock(m); - return size; + return available(); } HostWorkerPool::Reservation::Reservation(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(bytes) { std::unique_lock lock(pool.m); - pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.size; }); + pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.available(); }); pool.reserved += bytes; } @@ -75,3 +88,20 @@ HostWorkerPool::Reservation::~Reservation() { } pool.cv.notify_all(); } + +HostWorkerPool::Held::Held(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(0) { + Set(bytes); +} + +HostWorkerPool::Held::~Held() { + Set(0); +} + +void HostWorkerPool::Held::Set(size_t new_bytes) { + { + std::lock_guard lock(pool.m); + pool.held = pool.held - bytes + new_bytes; + bytes = new_bytes; + } + pool.cv.notify_all(); +} diff --git a/common/HostMemory.h b/common/HostMemory.h index 07dae951b..312d1f999 100644 --- a/common/HostMemory.h +++ b/common/HostMemory.h @@ -24,20 +24,26 @@ size_t DefaultHostMemoryBudget(); // its reflections, its merge, the detector-sized arrays. constexpr double HOST_WORKER_POOL_FRACTION = 0.5; -// One pool for those buffers, process-wide. A worker reserves its bytes before it makes them and hands -// them back when it is done; one that would take the pool past its size waits until others have handed -// theirs back - except that a reservation made while nothing is reserved is always let through, however -// large. So how many such workers run at once is decided by their bytes, not by the thread count, and -// however the steps holding them nest or run side by side; what each worker computes, and where its -// result goes, does not change. A worker must not reserve again while it holds a reservation, and none -// may reserve on ParallelFor's pool: a holder's own parallel passes need those threads. Work spread over -// that pool is bounded by its count instead (Size() / bytes each). +// One pool for those buffers, process-wide, held in one ledger with what the run holds beside them. The run +// says what it holds - its reflections, its merge engine, a validation's own grids - as Held bytes, and the +// workers may hold at once the pool's size or what the budget leaves beside the Held bytes, whichever is +// less: so the two together never claim more than the budget, however large the run's own part is. A worker +// reserves its bytes before it makes them and hands them back when it is done; one that would take the pool +// past that waits until others have handed theirs back - except that a reservation made while no worker holds +// one is always let through, however large. So how many such workers run at once is decided by their bytes, +// not by the thread count, and however the steps holding them nest or run side by side; what each worker +// computes, and where its result goes, does not change. A worker must not reserve again while it holds a +// reservation, and none may reserve on ParallelFor's pool: a holder's own parallel passes need those threads. +// Work spread over that pool is bounded by its count instead (Size() / bytes each). class HostWorkerPool { public: static HostWorkerPool &Instance(); // The pool's size in bytes; 0 sets it back to HOST_WORKER_POOL_FRACTION of DefaultHostMemoryBudget(). void SetSize(size_t bytes); + // The budget the pool and the Held bytes share; 0 sets it back to DefaultHostMemoryBudget(). + void SetBudget(size_t bytes); + // What the workers may hold at once now: the pool's size, or the budget less the Held bytes if less. [[nodiscard]] size_t Size(); class Reservation { @@ -51,10 +57,26 @@ public: size_t bytes; }; + // Bytes the run holds beside the workers for as long as this lives; never waits. + class Held { + public: + Held(HostWorkerPool &pool, size_t bytes); + ~Held(); + Held(const Held &) = delete; + Held &operator=(const Held &) = delete; + void Set(size_t bytes); // what it holds from now on + private: + HostWorkerPool &pool; + size_t bytes; + }; + private: HostWorkerPool(); + size_t available() const; // Size(), with the lock held std::mutex m; std::condition_variable cv; size_t size = 0; + size_t budget = 0; + size_t held = 0; size_t reserved = 0; }; diff --git a/common/IndexingSettings.cpp b/common/IndexingSettings.cpp index 8d35f57b1..d8dfd1db6 100644 --- a/common/IndexingSettings.cpp +++ b/common/IndexingSettings.cpp @@ -53,7 +53,7 @@ IndexingSettings &IndexingSettings::Algorithm(IndexingAlgorithmEnum input) { // only recover a basis vector up to this length, so the ceiling is exactly the longest cell the // indexer can ever find; at 500 it excluded 1091 of the PDB's 206950 X-ray entries (0.53%) outright. // 1200 A leaves 8. Nothing pays for the wider range: the histogram is sized from the VALUE in use, -// and only a caller that asks for more - a given cell that needs it, or the long-axis rescue - gets +// and only a caller that asks for more - a given cell that needs it, or the long-axis hypothesis - gets // a longer transform. IndexingSettings &IndexingSettings::FFT_MaxUnitCell_A(float input) { check_finite("FFT indexing max unit cell (A)", input); diff --git a/common/IndexingSettings.h b/common/IndexingSettings.h index a779fda7b..3b0e4db1d 100644 --- a/common/IndexingSettings.h +++ b/common/IndexingSettings.h @@ -29,7 +29,7 @@ class IndexingSettings { public: // The longest cell the FFT search can be asked to reach, and so the longest it can ever find: // FFTIndexer sizes its histogram from fft_max_unit_cell_A and the transform's last usable bin IS - // that length. Callers that widen the bound (a given cell, the long-axis rescue) must clamp to + // that length. Callers that widen the bound (a given cell, the long-axis hypothesis) must clamp to // this rather than let the setter throw - a rescue that recovers an implausible axis must not // take the whole run down with it. static constexpr float fft_max_unit_cell_limit_A = 1200.0; diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 390707bd1..0aa2ee4fb 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -473,8 +473,10 @@ struct EndMessage { // setting the merge works in - is only chosen afterwards, so the two settings are not always the // same one. Absent means they are, i.e. the identity. std::optional> reindex_matrix; - // Stills: the rocking curve the partialities were computed from (and the frames predicted through). + // Stills: the rocking curve the frames were predicted through (and the partialities computed from). std::optional stills_run_width; + // Stills: the cell given on the command line, whose population of frames the merge keeps. + std::optional stills_given_cell; // Geometry refined by the offline analysis. When present these overwrite the master-file geometry // at finalization (rugnux writes the refined values; the broker leaves them empty and keeps the diff --git a/common/Reflection.h b/common/Reflection.h index 1b8b5caec..6f2d45b37 100644 --- a/common/Reflection.h +++ b/common/Reflection.h @@ -24,7 +24,7 @@ struct Reflection { float bkg; float var_bkg; // non-signal (background) part of sigma^2, carried to the merge float sigma; - float dist_ewald; + float dist_ewald; // stills: signed excitation error |S| - 1/lambda (1/A), positive outside the sphere // The reciprocal Lorentz factor (rotation only - a still's Lorentz factor is one) times the // reciprocal polarization factor, and nothing else. This is what LP means everywhere in the // field, and it is what the CBOR key "rlp" and the HDF5 dataset "lp" store the reciprocal of. @@ -53,7 +53,7 @@ struct Reflection { float zeta; float image_scale_corr; // I_true = image_scale_corr * I; = prescaling_corr * qe_corr * flight_corr / (partiality * image_scale) bool observed = false; - bool on_ice_ring = false; // sits on a hexagonal-ice powder ring: excluded from scaling, kept for merging + bool on_ice_ring = false; // sits on a hexagonal-ice powder ring: excluded from scaling; merged on stills, ring by ring on rotation (DecideIceRings) // The signal disk lost pixels to the mask or to saturation, so the intensity is the profile's // estimate over what remained. Such a measurement may be low, and the merge does not let it // testify against a larger observation of the same reflection (WilsonOutliers.h). @@ -68,13 +68,25 @@ struct Reflection { // measurement, every correction and the per-frame scale applied, its sigma under the error model the // merge weighted it with - but NOT averaged with its symmetry mates. h k l are the index it was // measured at, not its ASU representative. Written as the SHELX HKLF 4 file, so that the program -// reading it sees the equivalents and computes Rint itself. +// reading it sees the equivalents and computes Rint itself, and as the intensities of the unmerged MTZ. struct ScaledFull { int32_t h = 0; int32_t k = 0; int32_t l = 0; float I = NAN; float sigma = NAN; + float image_number = NAN; // the frame of the event's peak partial, which tells two events of one index apart + // The full's counting variance as the merge rebuilds it at an intensity I, before its error model: + // var_bkg + var_per_I * I + (capture * I)^2, all at the intensity's scale (what a later merge rebuilds + // the full from). NaN where not exported. + float var_bkg = NAN; + float var_per_I = NAN; + float capture = NAN; + float d = NAN; // resolution, A + // The diffracted beam's direction in the crystal: its components along the real-space cell vectors + // (a.s1, b.s1, c.s1), so that s1 = sum_i (a_i.s1) a*_i - reindexed like h, k, l. What a joint merge of + // several sweeps fits one absorption surface over. NaN where not known. + float s1_cell[3] = {NAN, NAN, NAN}; }; struct MergedReflection { @@ -92,6 +104,8 @@ struct MergedReflection { // observation at the run's typical frame scale, over the precision they have. 1 on a sweep // without a weak stretch; small for a reflection measured only where the crystal barely // diffracted, whose scaled-up noise would otherwise count as much as a well-measured pair. + // Internal, not reported: it enters the CC1/2 the processing decides on (the automatic resolution + // cut, MergeStatisticsShell::cc_half_internal), never the reported CC1/2, which is the plain Pearson. float cc_weight = 1.0f; float d = 0.0; // Any observation of this reflection sat on an ice ring. The intensity is still merged and diff --git a/common/ScalingSettings.cpp b/common/ScalingSettings.cpp index 8b70c6ac9..8fc2cd2c9 100644 --- a/common/ScalingSettings.cpp +++ b/common/ScalingSettings.cpp @@ -315,3 +315,12 @@ ScalingSettings& ScalingSettings::ReportResolutionRange_A(std::optional ScalingSettings::GetReportResolutionRange_A() const { return report_resolution_range; } + +ScalingSettings& ScalingSettings::SweepSegments(std::vector input) { + sweep_segments = std::move(input); + return *this; +} + +const std::vector &ScalingSettings::GetSweepSegments() const { + return sweep_segments; +} diff --git a/common/ScalingSettings.h b/common/ScalingSettings.h index 6e3d0b8e7..d1b3af5da 100644 --- a/common/ScalingSettings.h +++ b/common/ScalingSettings.h @@ -3,7 +3,9 @@ #pragma once +#include #include +#include #include "JFJochException.h" // How the high-resolution cutoff for the written reflections and the reported shell table is chosen @@ -17,8 +19,20 @@ struct ReportResolutionRange { double d_max = 0.0; }; +// One sweep of several whose frames are laid end to end in one rotation merge (the joint merge of several +// sweeps): frames [first_frame, first_frame + n_frames). Every observation of a segment's frames is a full +// already combined in its own sweep (partiality 1, filed under the frame of its peak), so the merge +// combines nothing again, recomputes nothing from the rocking geometry, and lets nothing that runs along +// the frames - the per-frame scale's smoothness above all - reach across from one sweep into the next +// (RotationScaleMerge). +struct SweepSegment { + int32_t first_frame = 0; + int32_t n_frames = 0; +}; + class ScalingSettings { bool refine_wedge = false; + std::vector sweep_segments; bool merge_friedel = true; std::optional high_resolution_limit_A; @@ -150,6 +164,8 @@ public: ScalingSettings& ResolutionCCTarget(double input); ScalingSettings& ReportShellCount(int input); ScalingSettings& ReportResolutionRange_A(std::optional input); + // The sweeps a rotation merge's frames are made of; empty (the default): one sweep of partials. + ScalingSettings& SweepSegments(std::vector input); [[nodiscard]] bool GetRefineWedge() const; @@ -189,4 +205,5 @@ public: [[nodiscard]] double GetResolutionCCTarget() const; [[nodiscard]] int GetReportShellCount() const; [[nodiscard]] std::optional GetReportResolutionRange_A() const; + [[nodiscard]] const std::vector &GetSweepSegments() const; }; diff --git a/docker/common/build-qt.sh b/docker/common/build-qt.sh index cdda7c521..9f20b4ee6 100755 --- a/docker/common/build-qt.sh +++ b/docker/common/build-qt.sh @@ -4,9 +4,8 @@ # Shared by all four build images (docker/*/Dockerfile). The build context is docker/, so each # Dockerfile reaches this with COPY common/. Version inputs come from that Dockerfile's ARGs. # -# No JPEG or TIFF in Qt: the viewer has its own libjpeg-turbo and libtiff (preview/), and a static -# Qt's JPEG plugin brings a second libjpeg-turbo of another ABI that the linker silently mixes with -# ours (see viewer/CMakeLists.txt). +# No JPEG or TIFF in Qt: the viewer has its own JPEG encoder and libtiff (preview/), and nothing +# goes through QImage for these formats (see viewer/CMakeLists.txt). set -eux diff --git a/docker/rocky8/Dockerfile b/docker/rocky8/Dockerfile index d349de053..4ce078255 100644 --- a/docker/rocky8/Dockerfile +++ b/docker/rocky8/Dockerfile @@ -6,9 +6,8 @@ ARG QT_VERSION=6.11.1 ARG DBUS_VERSION=1.14.10 ARG NODE_MAJOR=22 ARG EIGEN_VERSION=3.4.0 -# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent / -# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source -# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). +# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external +# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). # Update base and install common utilities RUN dnf -y install epel-release && \ @@ -22,7 +21,6 @@ RUN dnf -y install epel-release && \ tar \ xz \ make \ - nasm \ gcc-toolset-13 \ gcc-toolset-13-gcc \ gcc-toolset-13-gcc-c++ \ diff --git a/docker/rocky9/Dockerfile b/docker/rocky9/Dockerfile index c46ef8613..5c6ff1bed 100644 --- a/docker/rocky9/Dockerfile +++ b/docker/rocky9/Dockerfile @@ -6,9 +6,8 @@ ARG QT_VERSION=6.11.1 ARG DBUS_VERSION=1.14.10 ARG NODE_MAJOR=22 ARG EIGEN_VERSION=3.4.0 -# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent / -# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source -# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). +# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external +# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). ARG DIALS_VERSION=3.27.0 ARG DIALS_TARBALL_URL=https://github.com/dials/dials/releases/download/v3.27.0/dials-v3-27-0-linux-x86_64.tar.xz ARG DIALS_PREFIX=/opt @@ -39,7 +38,6 @@ RUN dnf -y update && \ python3 \ python3.12 \ perl \ - nasm \ pkgconf-pkg-config \ xcb-util-devel \ xcb-util-*-devel \ diff --git a/docker/ubuntu2204/Dockerfile b/docker/ubuntu2204/Dockerfile index f2216d01e..509b068b4 100644 --- a/docker/ubuntu2204/Dockerfile +++ b/docker/ubuntu2204/Dockerfile @@ -9,9 +9,8 @@ ARG QT_VERSION=6.11.1 ARG DBUS_VERSION=1.14.10 ARG NODE_MAJOR=22 ARG EIGEN_VERSION=3.4.0 -# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent / -# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source -# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). +# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external +# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). # Update base and install toolchain + Qt dependencies # Notes: @@ -35,7 +34,6 @@ RUN set -eux; \ pkg-config \ gdb \ file \ - nasm \ python3.11 \ python3.11-venv \ dpkg-dev \ diff --git a/docker/ubuntu2404/Dockerfile b/docker/ubuntu2404/Dockerfile index 65c8051a8..c84fc9d38 100644 --- a/docker/ubuntu2404/Dockerfile +++ b/docker/ubuntu2404/Dockerfile @@ -9,10 +9,8 @@ ARG QT_VERSION=6.11.1 ARG DBUS_VERSION=1.14.10 ARG NODE_MAJOR=22 ARG EIGEN_VERSION=3.4.0 -# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent / -# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source -# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). nasm is required by the -# project's libjpeg-turbo ExternalProject for SIMD-accelerated JPEG. +# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external +# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). # Update base and install toolchain + static-Qt build dependencies # (XCB/X11/OpenGL/EGL/DBus/fontconfig/freetype/zlib headers needed to build static Qt). @@ -48,7 +46,6 @@ RUN set -eux; \ pkg-config \ gdb \ file \ - nasm \ dpkg-dev \ fakeroot \ libnuma-dev \ diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 064924338..504e01a88 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -1,6 +1,19 @@ # Acknowledgements -Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). +How to cite Jungfraujoch and Rugnux, and the papers that describe them, is on the +[Publications](PUBLICATIONS.md) page. In short: + +- **Rugnux**: F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons + and XFELs in one program" (2026), bioRxiv + [doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). +- **Jungfraujoch**: F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, + Z. Matěj, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: + hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), + J. Synchrotron Rad. 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268); + for real-time serial crystallography and live indexing feedback also J. Duan, H.-C. Stadler, + J. Nan et al., "Jungfraujoch: 38 GB s⁻¹ real-time serial crystallography – data acquisition and + live indexing feedback" (2026), J. Appl. Cryst. 59, 1668-1677 + [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083). ## Funding and support @@ -44,7 +57,7 @@ Cambridge, United Kingdom: Global Phasing Ltd. ## Crystallographic methods adopted from other packages -The analysis pipeline reimplements methods first published, and in most cases first implemented, by +The analysis pipeline builds on methods first published, and in most cases first implemented, by other crystallographic software. The code below is Jungfraujoch's own; the methods are theirs, and are acknowledged here. Where a package's source was consulted this is said explicitly. Most of these packages are neither linked nor vendored; the three that are - GEMMI, traccc and @@ -65,6 +78,16 @@ high-throughput reduction and analysis of serial femtosecond X-ray diffraction d J. Appl. Cryst. 47, 1118-1131 [doi:10.1107/S1600576714007626](https://doi.org/10.1107/S1600576714007626). +**Per-crystal post-refinement of serial stills** - refining, for each still, the partiality +parameters the spot positions cannot pin (a small reorientation, the crystal's own rocking width and the +radial scale of its reciprocal lattice, which a shot's photon energy or a crystal's cell moves) against +the merged intensities follows the post-refinement of PRIME and the mosaic-block model of Sauter et al.: +M. Uervirojnangkoorn, O. B. Zeldin, A. Y. Lyubimov, J. Hattne, A. S. Brewster, N. K. Sauter, A. T. Brunger +and W. I. Weis, "Enabling X-ray free electron laser crystallography for challenging biological systems +from a limited number of crystals" (2015), eLife 4, e05421; N. K. Sauter, J. Hattne, A. S. Brewster, +N. Echols, P. H. Zwart and P. D. Adams, "Improved crystal orientation and physical properties from +single-shot XFEL stills" (2014), Acta Cryst. D70, 3299-3309 (DOIs not verified here, so not given). + Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP @@ -229,6 +252,28 @@ H = |I(h) − I(Th)|/(I(h) + I(Th)) of the pairs a law relates is uniform on [0, gives the fraction. T. O. Yeates, "Detecting and overcoming crystal twinning" (1997), Methods Enzymol. 276, 344-358 [doi:10.1016/S0076-6879(97)76068-3](https://doi.org/10.1016/S0076-6879%2897%2976068-3). +**The symmetry hypotheses a refused promotion leaves** - the adopted group and every other index-2 subgroup +of the refused group perfectly twinned by a law of its coset, all merging alike, carried to the model and +decided by placing and refining it in each - follow the advice to refine in the candidate groups and compare, +rather than decide from intensity statistics, that pseudo-symmetry and twinning call for: +P. H. Zwart, R. W. Grosse-Kunstleve, A. A. Lebedev, G. N. Murshudov and P. D. Adams, "Surprises and pitfalls +arising from (pseudo)symmetry" (2008), Acta Cryst. D64, 99-107 +[doi:10.1107/S090744490705531X](https://doi.org/10.1107/S090744490705531X); A. A. Lebedev and M. N. Isupov, +"Space-group and origin ambiguity in macromolecular structures with pseudo-symmetry and its treatment with the +program Zanuda" (2014), Acta Cryst. D70, 2430-2443 +[doi:10.1107/S1399004714014795](https://doi.org/10.1107/S1399004714014795). + +**Twin refinement** (§15.12) refines the model against Yeates's twinned intensity, (1−α) I(h) + α I(Th), with the +twin fraction refined beside it as SHELXL refines BASF: R. Herbst-Irmer and G. M. Sheldrick, "Refinement of twinned +structures with SHELXL97" (1998), Acta Cryst. B54, 443-449 +[doi:10.1107/S0108768197018454](https://doi.org/10.1107/S0108768197018454). The twin laws come from the lattice's +metric as A. A. Lebedev, A. A. Vagin and G. N. Murshudov describe, "Intensity statistics in twinned crystals with +examples from the PDB" (2006), Acta Cryst. D62, 83-95 +[doi:10.1107/S0907444905036759](https://doi.org/10.1107/S0907444905036759), and the fractions are refined by +least squares at every cycle, as REFMAC5 refines them (Murshudov et al. 2011, Acta Cryst. D67, 355-367, cited +above). REFMAC5's target is a twinned-intensity likelihood; ours keeps the amplitude likelihood with the twinned +model amplitude in it, a simplification of that. + **A first-pass cell that is the coincidence lattice of two twin domains** (twinning by reticular merohedry of index 2 or 3, e.g. obverse/reverse rhombohedral domains or a cubic crystal twinned about [111]) is recognised by the nodes that belong to neither domain being empty. The twin index is @@ -247,7 +292,10 @@ noise floor of the statistic depends strongly on how many reflections a dataset modulation is what the axial systematic-absence test scores against, so that a reflection class a pseudo-translation merely suppresses is not read as extinct and does not buy a screw axis; the estimate of its depth there is our own, measured per axial row from the merged intensities rather -than from the Patterson vector. R. J. Read, +than from the Patterson vector. Where a symmetry operator or a twin law carries the vector to one +that is not ±itself, the modulation is also read on classes averaged over the lattice's rotations - +their modulation model summed over the copy pairs each rotation relates, as the measured intensity +sums them; that averaging is our own construction. R. J. Read, P. D. Adams and A. J. McCoy, "Intensity statistics in the presence of translational noncrystallographic symmetry" (2013), Acta Cryst. D69, 176-183 [doi:10.1107/S0907444912045374](https://doi.org/10.1107/S0907444912045374). @@ -319,7 +367,13 @@ reporting of I/sigma(I). K. Diederichs and P. A. Karplus, "Improved R-factors for diffraction data analysis in macromolecular crystallography" (1997), Nat. Struct. Biol. 4, 269-275 [doi:10.1038/nsb0497-269](https://doi.org/10.1038/nsb0497-269); P. A. Karplus and K. Diederichs, "Linking crystallographic model and data quality" (2012), Science 336, 1030-1033 -[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231); M. S. Weiss, "Global +[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231) (CC\* is also what a joint merge of +several sweeps tests their isomorphism against: CC_AB / (CC\*_A CC\*_B) is the cosine of the angle between +two data sets, their systematic difference apart from their noise, as K. Diederichs, "Dissecting random and +systematic differences between noisy composite data sets" (2017), Acta Cryst. D73, 286-293 +[doi:10.1107/S2059798317000699](https://doi.org/10.1107/S2059798317000699) shows - which is why the join is +decided on the size of that ratio, and a doubtful sweep by the merge with and without it, the idea of +delta-CC1/2 by Assmann, Brehm and Diederichs (2016), below); M. S. Weiss, "Global indicators of X-ray data quality" (2001), J. Appl. Cryst. 34, 130-135 [doi:10.1107/S0021889800018227](https://doi.org/10.1107/S0021889800018227) (R_pim beside R_meas and R_merge); K. Diederichs and @@ -359,6 +413,15 @@ J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in t package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399 [doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198). +**Several sweeps merged as one** - each sweep's partials combined into fulls on their own, and the +fulls of all the sweeps then scaled together, every sweep with its own frame scales and a relative B +against the joint merge, is how XSCALE merges several data sets (W. Kabsch, "Integration, scaling, +space-group assignment and post-refinement" (2010), above). Reading the radiation dose a sweep was taken +at from its relative B, which orders the sweeps by dose from the data rather than from the order they +were given in, follows K. Diederichs, "Some aspects of quantitative analysis and correction of radiation +damage" (2006), Acta Cryst. D62, 96-101 +[doi:10.1107/S0907444905031537](https://doi.org/10.1107/S0907444905031537). + **Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature: D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography: Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75 @@ -378,10 +441,21 @@ symmetric than its crystals and no reference or model is given, each crystal is which its intensities agree best with those of all the other crystals, crystal by crystal until none changes. The objective - the summed agreement between every pair of crystals - and the iteration that maximises it are taken from Brehm and Diederichs, whose method CrystFEL's `ambigator` also implements; -rugnux measures the agreement as a product of standardised intensities rather than a correlation, which -lets the sum over all partners be taken against one running sum instead of pair by pair. +rugnux measures the agreement as a product of standardised amplitudes (square roots of the intensities) +rather than a correlation, which lets the sum over all partners be taken against one running sum instead +of pair by pair. W. Brehm and K. Diederichs, "Breaking the indexing ambiguity in serial crystallography" (2014), Acta Cryst. D70, 101-109 [doi:10.1107/S1399004713025431](https://doi.org/10.1107/S1399004713025431). +The same objective settles which twin law puts each sweep of a joint merge of several sweeps on the +reference's indexing - decided over all pairs of sweeps at once, as dials.cosym does for its data sets: +R. J. Gildea and G. Winter, "Determination of Patterson group symmetry from sparse multi-crystal data sets +in the presence of an indexing ambiguity" (2018), Acta Cryst. D74, 405-410 +[doi:10.1107/S2059798318002978](https://doi.org/10.1107/S2059798318002978). + +**Reference sweep of a joint merge** — the sweep the others are put on the axes of is the one whose own +merge reaches the finest resolution, as KAMO takes the data set of the smallest Wilson B as its reference: +K. Yamashita, K. Hirata and M. Yamamoto, "KAMO: towards automated data processing for microcrystals" +(2018), Acta Cryst. D74, 441-449 [doi:10.1107/S2059798318004576](https://doi.org/10.1107/S2059798318004576). ### Physical corrections and calibration @@ -595,12 +669,114 @@ predicted distances around metal sites in proteins" (2006), Acta Cryst. D62, 678 [doi:10.1107/S0907444906014594](https://doi.org/10.1107/S0907444906014594), and, for Mg²⁺, Na⁺ and K⁺ at nucleic acids, F. Leonarski, L. D'Ascenzo and P. Auffinger, "Mg2+ ions: do they bind to nucleobase nitrogens?" (2017), Nucleic Acids Res. 45, 987-1004 [doi:10.1093/nar/gkw1175](https://doi.org/10.1093/nar/gkw1175). +The anisotropic displacements: a TLS group's U is that of a rigid body's translation, libration and their +correlation, U = T + A L A^T + A S + S^T A^T, of V. Schomaker and K. N. Trueblood, "On the rigid-body motion of +molecules in crystals" (1968), Acta Cryst. B24, 63-76 +[doi:10.1107/S0567740868001718](https://doi.org/10.1107/S0567740868001718), refined against the likelihood with a +residual B per atom, and a model's B column read as the total or as the residual to its TLS groups, as in +M. D. Winn, M. N. Isupov and G. N. Murshudov, "Use of TLS parameters to model anisotropic displacements in +macromolecular refinement" (2001), Acta Cryst. D57, 122-133 +[doi:10.1107/S0907444900014736](https://doi.org/10.1107/S0907444900014736). The chains are cut into TLS groups as +TLSMD cuts them - contiguous segments, by dynamic programming over the fit of an isotropic TLS model to the +refined B factors: J. Painter and E. A. Merritt, "Optimal description of a protein structure in terms of multiple +groups undergoing TLS motion" (2006), Acta Cryst. D62, 439-450 +[doi:10.1107/S0907444906005270](https://doi.org/10.1107/S0907444906005270). Atoms refined with a U of their own are +restrained by the rigid-bond postulate of F. L. Hirshfeld, "Can X-ray data distinguish bonding effects from +vibrational smearing?" (1976), Acta Cryst. A32, 239-244 +[doi:10.1107/S0567739476000533](https://doi.org/10.1107/S0567739476000533), in the form of SHELXL's RIGU - the +components of U along the bond and across it in the bond's plane: A. Thorn, B. Dittrich and G. M. Sheldrick, +"Enhanced rigid-bond restraints" (2012), Acta Cryst. A68, 448-451 +[doi:10.1107/S0108767312014535](https://doi.org/10.1107/S0108767312014535) - with SHELXL's similarity (SIMU) and +approximate-isotropy (ISOR) restraints as G. M. Sheldrick, "Crystal structure refinement with SHELXL" (2015), Acta +Cryst. C71, 3-8 [doi:10.1107/S2053229614024218](https://doi.org/10.1107/S2053229614024218) documents them. Which +displacement model a refinement keeps - isotropic, TLS, or a U per atom - is decided on the free set, as a test of +what the data support rather than a rule by resolution: E. A. Merritt, "To B or not to B: a question of +resolution?" (2012), Acta Cryst. D68, 468-477 [doi:10.1107/S0907444911028320](https://doi.org/10.1107/S0907444911028320). + +**NCS restraints** — copies of a molecule in the model are found by sequence alignment with REFMAC5's criterion +(more than 15 residues aligned, 80% identity - here of the longer chain - and 2.5 Å rms after superposition; +Murshudov et al. 2011, above) and restrained locally, on pairs of +corresponding interatomic distances under the Geman-McClure robust loss, as REFMAC5's local NCS restraints are, with +the distance range of R. A. Nicholls, F. Long and G. N. Murshudov, "Low-resolution refinement tools in REFMAC5" +(2012), Acta Cryst. D68, 404-417 [doi:10.1107/S090744491105606X](https://doi.org/10.1107/S090744491105606X), and +the bonds and angles left out as they leave them out. The case for local restraints over a superposition, and for +deciding them by R-free, is G. J. Kleywegt's, "Use of non-crystallographic symmetry in protein structure refinement" +(1996), Acta Cryst. D52, 842-857 [doi:10.1107/S0907444995016477](https://doi.org/10.1107/S0907444995016477); I. Usón, +E. Pohl, T. R. Schneider, Z. Dauter, A. Schmidt, H.-J. Fritz and G. M. Sheldrick, "1.7 Å structure of the stabilized +REIv mutant T39K. Application of local NCS restraints" (1999), Acta Cryst. D55, 1158-1167 +[doi:10.1107/S0907444999003972](https://doi.org/10.1107/S0907444999003972); and J. J. Headd, N. Echols, +P. V. Afonine, N. W. Moriarty, R. J. Gildea and P. D. Adams, "Flexible torsion-angle noncrystallographic symmetry +restraints for improved macromolecular structure refinement" (2014), Acta Cryst. D70, 1346-1356 +[doi:10.1107/S1399004714003277](https://doi.org/10.1107/S1399004714003277), whose comparison of global, torsion and +no NCS is why the copies' B factors are refined independently here. **Unmodelled density** — the blobs the refinement lists are the connected regions of the difference map above a cutoff, found by flood fill as Coot's "unmodelled blobs" finds them, in GEMMI's implementation (`find_blobs_by_flood_fill`): P. Emsley, B. Lohkamp, W. G. Scott and K. Cowtan, "Features and development of Coot" (2010), Acta Cryst. D66, 486-501 [doi:10.1107/S0907444910007493](https://doi.org/10.1107/S0907444910007493). +**Special positions** — an atom on a crystallographic special position is placed on it and its shifts are +confined to the subspace its site symmetry leaves unchanged, as R. W. Grosse-Kunstleve and P. D. Adams treat +special positions, "Algorithms for deriving crystallographic space-group information. II. Treatment of special +positions" (2002), Acta Cryst. A58, 60-65 [doi:10.1107/S0108767301016658](https://doi.org/10.1107/S0108767301016658). + +**Water build** — the waters the refinement adds and removes follow the documented ordered-solvent defaults of +phenix.refine: difference-map peaks from 3 σ, hydrogen-bond partners to 3.2 Å and picking only to 2.8 Å, with +the waters added, filtered and refined between macro-cycles: +Afonine et al. (2012, above); +the distances to the model that Coot's Find Waters keeps (2.4-3.2 Å), from the Coot manual and Emsley et al. +(2010, above); and the removal of a water in negative difference density at -3 σ, the cut Echols et al. (2014, +below) use to exclude a site. + +**Solvent audit** — the waters and ions of the refined model are scored by their surroundings with the +stereochemical criteria of the surveys of ions at nucleic acids by P. Auffinger's group: the coordination bands +1.9-2.3 / 2.3-2.6 / 2.6-3.2 Å, the Mg²⁺ exclusion zone and the clash shells of N. Naleem, A. Henning-Knechtel, +S. Kirmizialtin and P. Auffinger, "Cat_Wiz: a stereochemistry-guided toolkit for locating, diagnosing, and +annotating Mg2+ ions in RNA structures" (2026), Nucleic Acids Res. 54, gkag120 +[doi:10.1093/nar/gkag120](https://doi.org/10.1093/nar/gkag120); the Mg-O and Mg-N distances and the +2.3-3.4 Å exclusion zone of F. Leonarski, A. Henning-Knechtel, S. Kirmizialtin, E. Ennifar and P. Auffinger, +"Principles of ion binding to RNA inferred from the analysis of a 1.55 Å resolution bacterial ribosome +structure – Part I: Mg2+" (2025), Nucleic Acids Res. 53, gkae1148 +[doi:10.1093/nar/gkae1148](https://doi.org/10.1093/nar/gkae1148); the Na⁺, K⁺ and water distances and +coordination numbers of F. Leonarski, L. D'Ascenzo and P. Auffinger, "Nucleobase carbonyl groups are poor +Mg2+ inner-sphere binders but excellent monovalent ion binders—a critical PDB survey" (2019), RNA 25, 173-192 +[doi:10.1261/rna.068437.118](https://doi.org/10.1261/rna.068437.118), and the distance bands of their +2017 survey (above). The ligand sphere of the bond-valence sums and the reading of halides by the donors +around them follow N. Echols, N. Morshed, P. V. Afonine, A. J. McCoy, M. D. Miller, R. J. Read, +J. S. Richardson, T. C. Terwilliger and P. D. Adams, "Automated identification of elemental ions in +macromolecular crystal structures" (2014), Acta Cryst. D70, 1104-1114 +[doi:10.1107/S1399004714001308](https://doi.org/10.1107/S1399004714001308), and the halide's positive and +negative neighbours N. Morshed, N. Echols and P. D. Adams, "Using support vector machines to improve elemental +ion identification in macromolecular crystal structures" (2015), Acta Cryst. D71, 1147-1158 +[doi:10.1107/S1399004715004241](https://doi.org/10.1107/S1399004715004241); the chloride's shell of donors +at up to 3.4 Å is O. Carugo's, "Buried chloride stereochemistry in the Protein Data Bank" (2014), BMC +Struct. Biol. 14, 19 [doi:10.1186/s12900-014-0019-8](https://doi.org/10.1186/s12900-014-0019-8). The +bond-valence sums are I. D. Brown and D. Altermatt's, "Bond-valence parameters obtained from a systematic +analysis of the Inorganic Crystal Structure Database" (1985), Acta Cryst. B41, 244-247 +[doi:10.1107/S0108768185002063](https://doi.org/10.1107/S0108768185002063), with the R₀ of N. E. Brese and +M. O'Keeffe, "Bond-valence parameters for solids" (1991), Acta Cryst. B47, 192-197 +[doi:10.1107/S0108768190011041](https://doi.org/10.1107/S0108768190011041), as tabulated by H. Zheng, +K. M. Langner, G. P. Shields, J. Hou, M. Kowiel, F. H. Allen, G. Murshudov and W. Minor, "Data mining of +iron(II) and iron(III) bond-valence parameters, and their relevance for macromolecular crystallography" +(2017), Acta Cryst. D73, 316-325 [doi:10.1107/S2059798317000584](https://doi.org/10.1107/S2059798317000584), +weighted by occupancy as P. Müller, S. Köpke and G. M. Sheldrick weight them, "Is the bond-valence method +able to identify metal atoms in protein structures?" (2003), Acta Cryst. D59, 32-37 +[doi:10.1107/S0907444902018000](https://doi.org/10.1107/S0907444902018000); their normalised vector sum and +its outlier bound are CheckMyMetal's: H. Zheng, D. R. Cooper, P. J. Porebski, I. G. Shabalin, K. B. Handing +and W. Minor, "CheckMyMetal: a macromolecular metal-binding validation tool" (2017), Acta Cryst. D73, 223-233 +[doi:10.1107/S2059798317001061](https://doi.org/10.1107/S2059798317001061). The score that combines them is +our own. The divalent ion's place on a purine N7 - in the base's plane, 3.5-3.9 Å from its O6 or N6 - is the test of +Leonarski et al. (2017, above). A light ion is told from a water by a density above the waters', as Echols et al. +(2014, above) flag one; a halide's few or no donors are allowed as Z. Dauter and M. Dauter describe halides binding +also at hydrophobic surfaces, "Entering a new phase: using solvent halide ions in protein structure determination" +(2001), Structure 9, R21-R26 [doi:10.1016/S0969-2126(01)00565-2](https://doi.org/10.1016/S0969-2126(01)00565-2). +The anomalous scattering expected of each species is f″ at the data's wavelength by D. T. Cromer and D. A. +Liberman, "Anomalous dispersion calculations near to and on the long-wavelength side of an absorption edge" +(1981), Acta Cryst. A37, 267-268 [doi:10.1107/S0567739481000600](https://doi.org/10.1107/S0567739481000600), +with the corrections of L. Kissel and R. H. Pratt, "Corrections to tabulated anomalous-scattering factors" +(1990), Acta Cryst. A46, 170-175 [doi:10.1107/S0108767389010718](https://doi.org/10.1107/S0108767389010718), as +GEMMI computes them (its port of the Larch routines). + ## Software and computing methods Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading @@ -625,6 +801,13 @@ iterations" (2011), SIAM J. Numer. Anal. 49, 1715-1735 procedures for nonlinear integral equations" (1965), J. ACM 12, 547-560 [doi:10.1145/321296.321305](https://doi.org/10.1145/321296.321305). +The interval the report gives for ISa comes from a bootstrap over the unique reflections: +B. Efron, "Bootstrap methods: another look at the jackknife" (1979), Ann. Statist. 7, 1-26 +[doi:10.1214/aos/1176344552](https://doi.org/10.1214/aos/1176344552). Its draws are Poisson weights +hashed from the replicate and the reflection with the SplitMix64 finaliser of G. L. Steele Jr., +D. Lea and C. H. Flood, "Fast splittable pseudorandom number generators" (2014), Proc. OOPSLA 2014, +453-472 [doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). + The single-precision exp the Bragg integration's profile fit shares between the CPU and an Apple GPU (`BraggExp.h`) is built from textbook error-free floating-point arithmetic: the argument reduction of W. J. Cody and W. Waite, "Software Manual for the Elementary Functions" (1980), Prentice-Hall; the diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6807fa598..e36ba63d8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,16 @@ # Changelog ## 1.0.0 +### 1.0.0-rc.181 + +* Rugnux: Several rotation sweeps of one crystal are merged as one dataset (`rugnux a_master.h5 b_master.h5`), on a common indexing and cell, with a non-isomorphous sweep left out and reported; `--sweeps separate` processes each sweep on its own. Bruker `.sfrm` frames and imgCIF files with a full goniometer axis chain are read. +* Rugnux: More robust rotation processing - a retry with a more sensitive spot finder when no lattice is found, a self-consistent detector distance, long axes and doubled cells tested as hypotheses, a `-C` supercell multiple stepped down, and each ice ring merged, down-weighted or rejected on its own. +* Rugnux: Serial stills are merged with a partiality-aware error model, reported as `ERROR_MODEL_A/B/C` with no ISa or R_merge/R_meas/R_pim; runs without `-S` have an overall intensity scale about 0.86-0.88 times that of earlier versions, with unchanged statistics. A second crystal form is left out of the merge and reported, the indexing ambiguity is resolved more reliably, and `--model` no longer influences processing. +* Rugnux: The reported CC1/2 is the standard unweighted one, which can be lower than before on sweeps with weak frames; ISa comes with a 95% interval (`ISA_LOW`/`ISA_HIGH`). +* Rugnux: Model refinement adds anisotropic/TLS, local NCS and twin refinement where R-free supports them; building waters and relabelling ions is now off by default (`--refine-solvent` turns it on). +* NXmx: A detector of several modules is written as `module_0` ... `module_N` only, so DIALS reads each module as its own panel. +* Other: JPEG previews use a bundled encoder (libjpeg-turbo no longer needed), Rugnux prints how to cite it, and the release archives contain the HTML documentation. + ### 1.0.0-rc.180 * Rugnux: A placed model is refined by default - restrained maximum-likelihood refinement on the GPU (CPU without one), writing `_refined_model.cif`, `_refined_maps.mtz` and the difference-map blobs in `_blobs.json`; `--no-refine` skips it. diff --git a/docs/CPU_DATA_ANALYSIS_DECISIONS.md b/docs/CPU_DATA_ANALYSIS_DECISIONS.md index 7943fe4b2..abb73189b 100644 --- a/docs/CPU_DATA_ANALYSIS_DECISIONS.md +++ b/docs/CPU_DATA_ANALYSIS_DECISIONS.md @@ -82,7 +82,9 @@ The median needs three observations, so a reflection measured once or twice — By default the reported/written high-resolution limit is trimmed where $\mathrm{CC}_{1/2}$ falls off: a logistic is fitted to $\mathrm{CC}_{1/2}(s)$, and the limit is set **one reported-shell width past** the point where the fit crosses 0.30 — deliberately "one shell too far", so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. `--scaling-high-resolution` overrides the limit and `--resolution-cutoff off` disables it. -**Ice sits out of this fit, and only this fit.** A powder ring is reproducible: past the crystal's own limit the ice is still there and still the same in both half-sets, so the two halves agree *about the ring*, and a Pearson $\mathrm{CC}_{1/2}$ cannot tell that agreement from diffraction. Shells have been measured at $\mathrm{CC}_{1/2}=0.765$ where $\langle I/\sigma\rangle$ is $-0.1$ and $R_\mathrm{meas}$ is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, still written and still counted in the shell table (§10.10), and what changes is only that the resolution **decision** now reads the same curve that scaling, the error model and the space-group search already read. +**The fit runs on an internal, weighted $\mathrm{CC}_{1/2}$, not on the reported one.** An observation from a frame at scale $G$ below the run's typical scale $G_\mathrm{ref}$ was scaled up by $G_\mathrm{ref}/G$ and carries $(G_\mathrm{ref}/G)^2$ more counting variance than the same observation at the typical scale. On rotation each reflection's half-set pair therefore enters the fit's Pearson sums with a weight in $(0, 1]$: the variance its two half-set means would have had with every observation at the typical scale, over the variance they have. It is 1 on a sweep without a weak stretch, and small for a reflection measured only where the crystal barely diffracted, whose scaled-up noise would otherwise pull the curve down as much as a well-measured pair holds it up. The same per-frame factor sets the information each observation carries in the ΔCC1/2 frame ledger (§ frame disposition in [integration](CPU_DATA_ANALYSIS_INTEGRATION.md)), and the two-pass and beam-centre guards read this weighted form too. The **reported** $\mathrm{CC}_{1/2}$ — `CC_HALF`, `CC_STAR`, the shell tables and the mmCIF — is the standard one of XDS, Aimless and DIALS: the plain Pearson correlation of the $1/\sigma^2$-weighted half-set means, every reflection counted alike. Stills weight nothing in either. + +**Ice sits out of this fit, and only this fit.** A powder ring is reproducible: past the crystal's own limit the ice is still there and still the same in both half-sets, so the two halves agree *about the ring*, and a Pearson $\mathrm{CC}_{1/2}$ cannot tell that agreement from diffraction. Shells have been measured at $\mathrm{CC}_{1/2}=0.765$ where $\langle I/\sigma\rangle$ is $-0.1$ and $R_\mathrm{meas}$ is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, written and counted in the shell table wherever the per-ring decision of §10.10 keeps them, and what changes is only that the resolution **decision** now reads the same curve that scaling, the error model and the space-group search already read. **A cut the shell table refutes is not quoted.** Two things keep the number inside what the bins show. The fitted crossing has to lie **inside the fitted bins**: on a real fall-off it sits between the last bin at the target and the first bin below it, with the extension bins to spare, and a crossing past the last fitted bin is an extrapolation of a fall-off these data never showed — the crossing is then read off the bins themselves, between the two that straddle the target. And no single resolution is quoted at all where the curve is not a fall-off: where $\mathrm{CC}_{1/2}$ never reaches the target in any shell, where the fit reads finer than the finest shell whose $\mathrm{CC}_{1/2}$ still reaches it, or where $\mathrm{CC}_{1/2}$ **climbs back over the target** after falling below it — a noise shell that climbs back is not the edge of the data. The report then says which of the three happened and points at the shell table instead. @@ -132,7 +134,7 @@ The model electron density is sampled on a grid (IT92 X-ray form factors, with a ### 14.2 Bulk solvent and scaling -A flat bulk-solvent mask around the model is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$: +A flat bulk-solvent mask around the model (the REFMAC radii, on the density's grid, or on one of its own at 0.75 Å where that grid is too coarse for the mask's 0.8 Å shrink to act) is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$: $ F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2. diff --git a/docs/CPU_DATA_ANALYSIS_IMAGE.md b/docs/CPU_DATA_ANALYSIS_IMAGE.md index a670a355c..feedeaf40 100644 --- a/docs/CPU_DATA_ANALYSIS_IMAGE.md +++ b/docs/CPU_DATA_ANALYSIS_IMAGE.md @@ -277,7 +277,7 @@ wrong one does not. where its lattice also beats chance on the pooled spots (§4.1), so that one frame against none on a sparse pattern cannot flip it. The decision holds for the rest of the run, and it flips the **axis**, not the angles, because prediction reads the axis too. It runs before the second first - pass above and before the long-axis rescue, since with the sign wrong every candidate lattice is + pass above and before the long-axis hypothesis, since with the sign wrong every candidate lattice is wrong. **Post-refinement.** On a two-pass rotation run, whatever centre the first pass ends on is refined diff --git a/docs/CPU_DATA_ANALYSIS_INDEXING.md b/docs/CPU_DATA_ANALYSIS_INDEXING.md index 614c49d2d..673f0731c 100644 --- a/docs/CPU_DATA_ANALYSIS_INDEXING.md +++ b/docs/CPU_DATA_ANALYSIS_INDEXING.md @@ -109,7 +109,7 @@ For each direction, the FFT magnitude spectrum is computed; peaks correspond to The running-mean background window keeps a **constant width** and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel's running sum is still valid. -The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. +The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames. Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. @@ -422,6 +422,19 @@ geometry refinement fits on the same population: its sampled frames are filtered and the module placement are formed, the population told from the strong frames (at least 20 indexed spots), whose cells show the valley between two forms where a weak frame's cell spreads across it. +Without a given cell the same question is asked of the frames on the run's own cell (within the +indexing tolerance of it), whose two forms can lie inside that tolerance. Each frame's cell is put on +the run's axes (of the orders of its axes, the one nearest the run's cell, so two near-equal axes indexed +the other way round compare alike), described by its edges as logarithms, and projected on the direction +the cells spread along most, which is the direction between two forms where there are two. The angles are +left out: a still's refined angles carry the indexing more than the crystal - one form comes out with its +monoclinic angle on one axis or another, or held at 90° - and spread more than two forms' angles differ. +The populations of that coordinate are found as above; the largest is merged, and the next, where it +holds at least a tenth of the frames and its mean shape is at least 1 % from the major one's (closer +groups are one form whose cell drifts), is left out of the merge and reported (`CELL_FORMS`, see the +report), with its cell to process it with `-C`. The cell written is the merged frames' own, with or +without a given cell. + **Detector modules.** Where the detector is described module by module ([Module geometry](RUGNUX_FORMATS.md#module-geometry)), the same refinement places each module with enough spots on its frames' lattices as a rigid block - a shift and an in-plane turn - where that @@ -431,4 +444,5 @@ gauge holds their spot-weighted mean shift, turn and outward push, and with the also the two tilt-like gradients across the detector, at zero, so the beam centre, distance, tilt and crystal orientations carry what all modules share. A coarse search of each module's position comes first, scored like the beam centre by the spots that land on a lattice. The placement is used from -there on for a foreign multi-panel detector; for a JUNGFRAU master it is measured and reported only. +there on for a foreign multi-panel detector and for modules given as a CrystFEL `.geom`; for a JUNGFRAU +master it is measured and reported only. diff --git a/docs/CPU_DATA_ANALYSIS_INTEGRATION.md b/docs/CPU_DATA_ANALYSIS_INTEGRATION.md index 1bde68271..b5da3d702 100644 --- a/docs/CPU_DATA_ANALYSIS_INTEGRATION.md +++ b/docs/CPU_DATA_ANALYSIS_INTEGRATION.md @@ -158,7 +158,7 @@ The curve $\bar B(r)$ is the adaptive spot finder's peak-excluded background per **When it is applied.** `--background-radial` takes `on`, `off` or `auto`; in Rugnux it is **`auto` by default** (the broker keeps it off), and auto is decided **per run, in the pre-scan, on empty apertures** — the same measurement that, on stills, also chooses how far the ring reaches (the $r_3$, up to the one the run starts from, whose ring predicts the empty disk best). On a grid of apertures away from every found spot, the ring's prediction of the empty disk is taken twice: plain, and with the correction applied exactly as the integrator applies it (the clip following the curve, and the curve's mean over the ring pixels kept minus its mean over the disk). With nothing in the aperture each difference is the error the ring makes in a reflection's background, noise plus bias. Rotation measures every stencil the spot width can give it (and the one it starts from, which the geometry pre-pass and the starvation guard return to) and reads the one it gets. - **Stills** have no other correction, so it is applied to every image of a run where it lowers the apertures' mean square error at all. -- **Rotation** has one: under auto, each image whose peak-excluded ice score (§3.3) reaches `--ice-min-score` is corrected with the curve of its reflections' rings. The frame's curve on every image replaces that only where it takes **at least 10 %** of the apertures' mean square error away. On photon-counting sweeps the apertures gain 0.1–5 % (statistically significant, $z\approx3$–4), and correcting every image there was measured to cost ISa and anomalous signal; where it pays — the rings of home-source CCD frames, a smooth ice ring — they gain 13–31 %. A run below the margin keeps the per-image ice correction, bit for bit. +- **Rotation** has one: under auto, each image whose peak-excluded ice score (§3.3) reaches `--ice-min-score` is corrected, with the frame's curve as on stills. (It used the curve of the reflections' own rings until it was measured to miss the narrow peak of a smooth ice ring: on a crystal with smooth ice the ring cores stayed 4–9 standard errors high and their flanks low, against the off-ring reflections, and the frame's curve brought all three low-resolution rings within 3.) Correcting **every** image instead is chosen only where it takes **at least 10 %** of the apertures' mean square error away. On photon-counting sweeps the apertures gain 0.1–5 % (statistically significant, $z\approx3$–4), and correcting every image there was measured to cost ISa and anomalous signal; where it pays — the rings of home-source CCD frames, a smooth ice ring — they gain 13–31 %. A run below the margin keeps the per-image ice correction, bit for bit. The model is a function of **radius alone**, so it is applied only where that is true of the background. The per-image ice gate rests on this: smooth powder ice *is* a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes **43 % of the bands' excess amplitude**, and the improvement is **7× larger inside the bands than outside**, which is its stated mechanism; on a crystal whose ice is textured the same correction *increased* the excess amplitude by half; on a clean crystal it is inert to four decimal places. A strongly polarised beam or panels of different response make the background non-radial too; the apertures see that as a ring the correction predicts *worse*, and a run where they do keeps it off. Auto engages only where a peak-excluded background exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off. @@ -341,7 +341,7 @@ The partiality applied is fixed by the data type and scaling stage, not chosen f 3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma(d^*)$ — the run's rocking width, measured once on the intensity-weighted excitation errors and shared by every crystal. It is measured per group of frames of similar prediction window (four per shell), each against its own window, the shell taking the median of the groups whose curve is narrower than their window, so the frames with the widest windows do not set it everywhere; and it is measured in eight shells of equal width in $d^*$, made non-decreasing (pool-adjacent-violators, weighted by each shell's summed signal) and interpolated linearly in $d^*$ as a table. The three-term form $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius, the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature — is still fitted, sets the prediction window (§8.2) and stands in for the table where fewer than three shells are measurable. The curve is centred on the run's **radial scale** of the reciprocal lattice — a wavelength or cell a little off, which the spot positions cannot see since they fix only $\lambda d^*$ — measured from where the curve's centre lies against $d^{*2}$ and applied to $\mathbf{q}$. Where the frames were predicted through the measured window (§8.2), the post-refinement uses that same curve rather than measuring it again; the curve is written to `_process.h5` (`/entry/MX/stillsRockingWidth*`, `stillsRadialScale`), so `--mode scale` merges with it too. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times; the refined partiality is adopted on every crystal it was refined on. The per-crystal fits of $G$ and of the tilt measure each residual against its expected variance at the model, background + counting + $(0.3\,I_\mathrm{model})^2$: a still's strong reflections scatter about the model by tens of per cent, and in counting sigmas alone every strong reflection would be an outlier and the scale would be set by the weak ones. -Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. +Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. On stills this holds for the merges the scale and the partiality are refined against. The final merge, which has the fitted error model, takes every observation: the model's partiality term $c\,((1-p)/p)\,\langle I\rangle$ (§10.5) weights a low-partiality observation by what it is worth. A reflection - each Bijvoet hand under `-A` - is reported only if at least one of its observations reaches $p \ge 0.2$; one seen only far out on its rocking curve rests on the partiality model alone. The error model is fitted on the observations at $p \ge 0.2$. ### 10.3 Smoothing of per-frame scales @@ -388,9 +388,9 @@ Per-shell and overall merging statistics are computed on corrected intensities, itself count as missing instead of leaving the denominator along with the data, - the anomalous signal-to-noise $\mathrm{SigAno}$ and the half-set anomalous correlation $\mathrm{CC}_\mathrm{anom}$ (below). -The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly. +The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path adds a partiality term, $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2 + c^2\left(\frac{1-p}{p}\right)^2\langle I\rangle^2$, and reports **no ISa**: there $1/b$ is the $I/\sigma$ ceiling of one fully recorded still under the stills model - the same frames processed as a rotation sweep read several times higher - which describes the processing model more than the instrument and the crystal. It reports $a$, $b$ and $c$ instead (`ERROR_MODEL_A/B/C`, `_reflns.jfjoch_error_model_a/b/c`); $1/b$ is still used inside, by the space-group search. -$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. +$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$ (and on stills $c_\mathrm{XDS} = c^2/a$). Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. **Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ($\mathrm{CC}_\mathrm{anom}$) is reported beside it: $\Delta I$ is formed once per half-set and the two are correlated over the acentric pairs where **both** hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic $\sigma$ cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS's `CCanom` and `phenix.merging_statistics`' `cc_anom`; XDS's `Anomal Corr` is a **different quantity and is not comparable with it** — on the same observations it reads two to three times higher in the low shells. $\mathrm{CC}_\mathrm{anom}$ is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the `CCanom` column of the printed merge-statistics table and as `CC_ANOM=` in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints `-` and the results report writes no key, which is not the same statement as a value measured to be zero. @@ -432,7 +432,7 @@ Each surface is **cross-validated** on the half-set agreement it is meant to imp Each batch's $B$ is fitted on **resolution-shell means**, not on single observations: $\ln(I_\mathrm{ref}/I_\mathrm{obs})$ of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs $I_\mathrm{obs} > 0$, which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely *dimmer* than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as **absent** rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report ([the results report](RUGNUX_REPORT.md)) to name. -**Frame disposition (rotation).** After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures **ΔCC1/2** — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run) with each reflection's error variance taken from the **observed** scatter of its own observations rather than from the error model's σ's: a bad stretch claims the same σ's as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group's own $(n, \sum I, \sum I^2)$ from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic's own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, $(1-CC^2)/\sqrt{n_\mathrm{refl}-3}$, and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is **both** several standard errors below zero (Fisher-transformed) **and** well below the run's own per-batch distribution (median − 3 robust σ, measured once before anything is removed), **and** where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame `merged` / `downgraded` / `rejected` ledger is described in [the results report](RUGNUX_REPORT.md). +**Frame disposition (rotation).** After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures **ΔCC1/2** — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run), with each observation carrying the information $\min(1, (G/G_\mathrm{ref})^2)$ of its frame's scale against the typical one — the internal weighting of the automatic resolution cut ([decisions](CPU_DATA_ANALYSIS_DECISIONS.md) §13.4), not part of the reported $\mathrm{CC}_{1/2}$ — and with each reflection's error variance taken from the **observed** scatter of its own observations rather than from the error model's σ's: a bad stretch claims the same σ's as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group's own $(n, \sum I, \sum I^2)$ from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic's own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, $(1-CC^2)/\sqrt{n_\mathrm{refl}-3}$, and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is **both** several standard errors below zero (Fisher-transformed) **and** well below the run's own per-batch distribution (median − 3 robust σ, measured once before anything is removed), **and** where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame `merged` / `downgraded` / `rejected` ledger is described in [the results report](RUGNUX_REPORT.md). ### 10.7 R-free test-set flags @@ -475,15 +475,13 @@ A reference dataset (`--reference`, MTZ or SF-mmCIF) supplies known intensities **Resolve the indexing (merohedral) ambiguity.** When the lattice symmetry is higher than the crystal's Laue symmetry (e.g. $P3$, $P4$, $P6$, $C2$), more than one indexing of the same lattice is geometrically valid, and the two solutions produce *different* merged intensities that a self-consistent scale cannot tell apart — which of them is the reference's own, only an external reference can say. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation $\mathrm{CC}_\mathrm{ref}$ of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is **metric-preserving** — only the $hkl$ labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined, and the whole merge is then repeated in the chosen indexing. For stills, where each crystal is indexed independently and a merge of mixed indexings would average reflections that are not symmetry mates, it is done **per image**, at integration time: a run with a reference resolves the ambiguity image by image (the image's partiality/Lorentz-corrected intensities are correlated with the reference under each candidate operator, which is scale-invariant, and the best-correlating one is adopted; a crystal whose own cell decides the indexing keeps it, as described below). In neither workflow is the reference a **scale** target: both scale against their own data (§10.2), so $\mathrm{ISa}$ and the merging statistics come from the data alone and no cross-dataset systematic is imported. Because the stills choice is made at integration time, a later re-merge of stored reflections (`--mode scale`) cannot repair a dataset integrated without a reference; the data-only resolution below runs in the full pipeline only. Where there is no reference dataset but there is a **model** (`--model`), the reference intensities are computed from it instead - $|F_\mathrm{model}|^2$ from the atomic structure factors with a flat bulk-solvent contribution at the standard constants ($k_\mathrm{sol}=0.35$, $B_\mathrm{sol}=46$ Å$^2$), which are not fitted because there are no observations yet. Nothing is scaled against them; they serve only to rank the candidate indexings, and the correlation that does the ranking is scale-invariant. This needs the cell and space group up front (`-C` / `-S`, as serial indexing wants anyway); on rotation data the same job is done after the merge, in §14.5, where a merge exists to fit the model to properly. -**Serial stills without a reference or model.** The data cannot say which indexing is the *true* one, but they can say which crystals are in the *same* one, and that is all a merge needs. Where the space group is fixed (`-S`) and its lattice admits alternative indexings, the crystals are made consistent with each other before anything is scaled, following Brehm & Diederichs (2014). Each crystal's partiality- and Lorentz-corrected intensities are divided by the run's mean intensity in their resolution shell (20 shells of equal reciprocal volume), standardised within the crystal so that its unknown scale and offset drop out, and weighted by $1/(1+(\sigma/s)^2)$, $s$ being the crystal's spread, so that a weak or barely recorded reflection counts for little. The agreement of two crystals is the sum of the products of these values over the reflections they share, and the objective is that agreement summed over every pair of crystals, each in the indexing assigned to it. That sum is $(|M|^2 - \mathrm{const})/2$ with $M$ the per-reflection sum over all crystals, so a crystal's agreement with all the others is one dot product with $M$ less its own contribution: each crystal in turn takes the indexing in which that dot product is largest, $M$ is updated, and the passes repeat until no more than one crystal in a thousand moves. A pass costs one read of the reflections, not one per pair of crystals, which is what keeps tens of thousands of crystals to seconds. The objective only grows, so it converges; it is started four times (once from the indexing as found, three times from a seeded random assignment) and the start that ends highest is kept, and of the equally consistent answers related by an overall relabelling the one that moves the fewest crystals is taken. The order of every floating-point sum is fixed (crystals in image order, sequential passes), so the result does not depend on the thread count. The run reports how clearly the crystals separated: for each crystal the correlation with the mean of all the others in its chosen indexing and in the best other one, compared by Fisher's $z$ in units of its standard error ($\sqrt{2/(n-3)}$ for $n$ shared reflections) — the median of that margin and the share of crystals above $3\sigma$. +**Serial stills without a reference or model.** The data cannot say which indexing is the *true* one, but they can say which crystals are in the *same* one, and that is all a merge needs. Where the space group is fixed (`-S`) and its lattice admits alternative indexings, the crystals are made consistent with each other before anything is scaled, following Brehm & Diederichs (2014). Each crystal is compared on the square roots of its partiality- and Lorentz-corrected intensities, the sign kept: intensities follow the Wilson distribution, whose long tail lets a few very strong reflections dominate every product of two crystals, so that the strongest crystals settle among themselves and a group of them can hold one another in the wrong indexing against the weaker majority (on a large-cell serial run this left a fifth of the crystals in the wrong indexing); amplitudes have no such tail. They are put on the run's mean and spread in their resolution shell (20 shells of equal reciprocal volume; a shell past the resolution the run reaches holds only noise and may have a mean of zero or below, so it is not divided by its mean), standardised within the crystal so that its unknown scale and offset drop out, and weighted by $1/(1+(\sigma/s)^2)$, $s$ being the crystal's spread, so that a weak or barely recorded reflection counts for little. The agreement of two crystals is the sum of the products of these values over the reflections they share, and the objective is that agreement summed over every pair of crystals, each in the indexing assigned to it. That sum is $(|M|^2 - \mathrm{const})/2$ with $M$ the per-reflection sum over all crystals, so a crystal's agreement with all the others is one dot product with $M$ less its own contribution: each crystal in turn takes the indexing in which that dot product is largest, $M$ is updated, and the passes repeat until no more than one crystal in a thousand moves. A pass costs one read of the reflections, not one per pair of crystals, which is what keeps tens of thousands of crystals to seconds. The objective only grows, so it converges; it is started four times (once from the indexing as found, three times from a seeded random assignment) and the start that ends highest is kept, and of the equally consistent answers related by an overall relabelling the one that moves the fewest crystals is taken. The order of every floating-point sum is fixed (crystals in image order, sequential passes), so the result does not depend on the thread count. The run reports how clearly the crystals separated: for each crystal the correlation with the mean of all the others in its chosen indexing and in the best other one, compared by Fisher's $z$ in units of its standard error ($\sqrt{2/(n-3)}$ for $n$ shared reflections) — the median of that margin and the share of crystals above $3\sigma$. A twin law is only an ambiguity for crystals whose own lattice cannot tell the indexings apart. On a pseudo-merohedral cell whose axes differ by a few per cent, each crystal's refined cell already says which axis is which, and the indexer has used that; letting the intensities overrule it there moves crystals that were right. So for each law the share of crystals whose refined cell fits the run's cell better as indexed than after the law is measured (a tie counting half — on an exactly merohedral lattice the share is about one half). Where it is at least 0.9 the law is decided by the cells: a crystal whose cell prefers its indexing stays there, as an anchor, and only the crystals whose cell does not are decided by the intensities, against the anchors. With a reference or a model the per-image choice above stands except where the cells decide: once all images are integrated, a crystal its own cell anchors goes back to that indexing, and the reference chooses only among the indexings left open to the others; the same data-only resolution is then run only to measure it: the report gives the number of crystals on which the data alone would have chosen otherwise. ### 10.10 Ice rings at the scale and merge stages -Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q = 2\pi/d$ (§1.2) of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are also left out of the **resolution-cut fit** (§13.4), which is the one consumer that reads the *merged* reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge's own accumulator (host and device alike) as well as on the stills one. They are otherwise **kept in the final merge**, which is also what the established scaling programs do by default, so the affected shells keep their completeness. - -Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge's own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged $\mathrm{CC}_{1/2}$ and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell. +Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q = 2\pi/d$ (§1.2) of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are also left out of the **resolution-cut fit** (§13.4), which is the one consumer that reads the *merged* reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge's own accumulator (host and device alike) as well as on the stills one. The stills merge keeps them in the final merge. The **rotation merge decides ring by ring**, because what a ring does to its reflections differs between rings and between runs: it can bias them (up on the ring, where its powder intensity is under the spot; down on the flanks, where it is in the background ring), only add scatter, or leave them alone. Deleting every band unconditionally was implemented once and removed: on a crystal whose band was harmless it removed better-than-average reflections and 6 to 8 points of completeness in the affected shell for no measurable gain. So each ring is split into its **core** ($|q-q_\mathrm{ring}|<0.01$ Å$^{-1}$) and its two **flanks** (out to $w$), and every band is compared with the off-ring merged reflections within $0.02$ Å$^{-2}$ of the ring in $1/d^2$. Both are read against a log-linear **Wilson fit** of the mean intensity over those off-ring reflections (the fit behind the reported Wilson $B$; over all off-ring reflections where the window cannot be fitted), as $t=(I-I_\mathrm{exp})/\sqrt{\sigma^2+I_\mathrm{exp}^2}$, which has unit scatter whether a reflection is signal- or noise-dominated. A band whose mean $t$ departs from its neighbours' by more than 3 standard errors is **rejected** — a bias, which no larger $\sigma$ would remove; one whose variance of $t$ exceeds the neighbours' by more than 3 standard errors of the log ratio (taken from the samples' own kurtosis: Wilson intensities are far from Gaussian) is **down-weighted**, its excess variance added to each reflection's own; the rest is **merged** as measured. A band of fewer than 20 reflections, or a ring with fewer than 50 neighbours, is merged. The decision is applied to the merge's accumulators, so the written reflections, the shell table and the half-set $\mathrm{CC}$ all see the same thing, and each band's verdict is logged and reported (`ICE_RING_BAND_nn`, see the results report). It complements the radial background correction (§9), which removes most of the bias at its source: on 58 rotation crystals with ice and a model, it lowered the model's R-free against merging every band on 26 and raised it on one, by 0.003. --- diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index e63f0e730..252707360 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -27,8 +27,12 @@ statistics and its map coefficients go out; the restraint dictionary is compiled ### 15.1 What is refined, and in what order The model is taken as the refinement can use it: the first model of the file only, its hydrogens removed, -every anisotropic B replaced by its isotropic equivalent, and every B clamped to the range the refinement -allows (2 Ų up to twice the model's largest B, within 100-300 Ų). Occupancies are kept as they are, and +and every B clamped to the range the refinement allows (2 Ų up to twice the model's largest B, within +100-300 Ų). An atom's anisotropic U (ANISOU, `_atom_site_anisotrop`) is kept. Where the model has TLS +groups (`_pdbx_refine_tls*`, PDB `REMARK 3`) whose atoms bring no U of their own, their U is made from the +groups, and the B column is read either as the total B (as phenix.refine writes it) or as the residual to the +groups (as REFMAC writes it) - whichever of the two, and of the isotropic B as it stands, scores the lowest +R-free on the model as given (`REFINE_ADP_INPUT`). Occupancies are kept as they are, and so are alternative conformations, each refined with its own restraints. Two stages follow. @@ -68,7 +72,10 @@ $$ F_\mathrm{model} = k\,e^{-\mathbf{s}^\top\mathbf{B}\mathbf{s}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}s^2/4}\,F_\mathrm{mask}\right), $$ -with the atoms' $F_\mathrm{calc}$ computed with the dictionary's riding hydrogens included (§15.4). The +with the atoms' $F_\mathrm{calc}$ computed with the dictionary's riding hydrogens included (§15.4), from the IT92 +form factors: a single-atom ion (a metal or a halide the dictionary describes) as the ion its dictionary charge +makes it - Mg²⁺ and Na⁺ 10 electrons, K⁺, Ca²⁺ and Cl⁻ 18 - every other atom as the neutral atom, whatever formal +charge the file gives it (a carboxylate's O⁻, a lysine's N⁺); the refined model is written with those charges. The scale, the anisotropic B, the bulk solvent and the likelihood's parameters are fitted at the start of each **macro-cycle** (§15.3) and held fixed while the minimiser runs; the mask is drawn for the model as it is at that point. @@ -86,7 +93,8 @@ gradient gather, whatever the number of atoms. The derivative of a riding hydrog **Scale and solvent.** The flat bulk-solvent mask is drawn with the REFMAC radii (probe 1.0 Å, shrink 0.8 Å) on a grid of its own, with a spacing of $d_\mathrm{min}/3$ or 0.4 Å, whichever is finer (coarser only where -a cell would need more than $2^{26}$ points). On the atoms' density grid the shrink is lost at low +a cell would need more than $2^{26}$ points, and then never coarser than 0.75 Å nor than the density grid, +which a cell that large uses for its mask). On the atoms' density grid the shrink is lost at low resolution: at 3 Å its 1 Å step leaves no grid point within the shrink radius. $k_\mathrm{sol}$ and $B_\mathrm{sol}$ are searched over their physical range on the working reflections, and the overall scale and anisotropic B are fitted at each point of that search (§14.2) - at the points of its fine pass by @@ -166,19 +174,53 @@ analytic derivatives: above it, $d_0$ the sum of the energy types' van der Waals radii, 2.6 Å where both atoms can hydrogen-bond and 1.9 Å between a metal and N, O or S. The list is rebuilt at every macro-cycle. +**Special positions.** An atom closer than 0.5 Å to one of its own symmetry images (the depositions put theirs +within 0.03 Å of the site) is on a special position: the symmetry operations, each with its lattice translation, +that take it onto itself are its site symmetry (Grosse-Kunstleve & Adams 2002). It is placed exactly on the site - +the mean of its position and its images under the site symmetry - and its shifts are confined to the subspace +those operations leave unchanged: the gradient is projected by the mean of their rotations (along a 2-fold or +4-fold axis, nowhere on a centre of inversion). Its occupancy is kept as given, the coinciding copies each +counting it (0.25 on a 4-fold axis), it makes no contact with its own copies, each of its contacts is listed +once rather than once per copy, and it moves with no rigid body. + **Metal sites.** A metal - an ion, or a metal atom of a ligand such as a heme's iron - is restrained to the donor atoms (O, N or S) that the model's LINK / `_struct_conn` records bind it to. Where the library has a link for the pair - its ZN-CYS, ZN-HISND, ZN-HISNE, FE-CYS, FE-HISND and FE-HISNE - that link is used as it is, with the modification it makes to the donor's residue (the proton the bond replaces). Every other record becomes a link of one bond at the distance of the library's own statistics of metal sites (its `metals.json`, carried as `monomers/metals.cif`) -for the metal, the donor element and the site's coordination number - the number of such records the metal -has, plus the bonds its own residue's dictionary gives it (a heme iron's four pyrrole nitrogens) - interpolated linearly between two tabulated coordination numbers (rows resting on at least 20 sites +for the metal, the donor element and the site's coordination number - but only where the model makes the +contact an inner-sphere one: no further than that distance plus 0.33 Å (Mg-O 2.07 + 0.33 = 2.4 Å, the +inner-sphere bound of Leonarski et al. 2017, the same margin for every metal). A record at an outer-sphere +distance - deposited models carry many, a Mg "bonded" to a phosphate O at 2.9 Å, a Na to a water at 3.0 Å - +would pull the ion onto that atom; it stays a contact instead. The coordination number is the number of +records the metal has within its inner sphere, plus the bonds its own residue's dictionary gives it (a heme iron's four pyrrole nitrogens) - interpolated linearly between two tabulated coordination numbers (rows resting on at least 20 sites count; σ is the row's standard deviation, at least 0.02 Å). Only the bond is restrained, not the angles at the metal. A contact without a record stays a repulsion, and a record to a symmetry mate is -left out, as every bond to a symmetry mate is. The rmsd of these bonds - with the dictionary's own bonds of +left out, as every bond to a symmetry mate is. Which records stay contacts is decided once, on the model as given: one topology serves the whole refinement, its +solvent build included, so a record is not judged again on a distance the refinement has moved (a record at 2.50 Å +refused, the ion slid to 2.39 Å, would have been taken). The records are written out with the refined model as they were +read, without the names of the links made here. The rmsd of these bonds - with the dictionary's own bonds of a metal within a ligand, a heme iron's to its porphyrin - is reported apart from the covalent bonds' (`REFINE_RMSD_METAL_BONDS`). +**Anisotropic atoms.** An atom's U is held as its equivalent isotropic B and a traceless anisotropy, +$U = U_\mathrm{dev} + B/(8\pi^2)\,I$, so that everything that refines, restrains and bounds B works on an anisotropic +atom as on an isotropic one, its shape held. Both engines grid it with its full U (an anisotropic Gaussian per +form-factor term, cut at the radius of its widest axis) and give the target's derivatives in its six U elements by the +same FFT method: for $\rho_q = a_q \exp(\mathbf{r}^T M_q \mathbf{r})$, $M_q = -4\pi^2(8\pi^2 U + B_q + B_\mathrm{blur})^{-1}$, +$\partial\rho_q/\partial U = \rho_q\,(M_q + 2 M_q\mathbf{r}\,\mathbf{r}^T M_q)$. Every U stays positive definite: its +smallest eigenvalue (as B) at least the cycle's floor, its largest at most the ceiling of §15.1. + +A **TLS group** gives each of its atoms $U = T + A L A^T + A S + S^T A^T$ at $\mathbf{r}$ from its origin, +$A = [[0, z, -y], [-z, 0, x], [y, -x, 0]]$ (Schomaker and Trueblood), plus a residual isotropic B per atom; its 20 +parameters (the trace of S does not enter U) are refined against the X-ray target alone, the residual B held, with +the derivatives by the chain rule from the atoms' U derivatives ($G$ for T, $A^T G A$ for L, $2 A^T G$ for S, $G$ an +atom's symmetric matrix of derivatives). An atom refined with a **U of its own** is restrained, for each pair of +bonded or angle-related atoms, by the rigid bond $|(U_i - U_j)\mathbf{n}|^2/\sigma^2$, $\sigma$ = 0.004 Ų along the +line $\mathbf{n}$ between them (the components SHELXL's RIGU restrains), and by the similarity +$|U_i - U_j|^2/\sigma^2$, $\sigma$ = 0.04 Ų (SIMU); an atom in no pair (a water, an ion) by +$|U - \mathrm{tr}(U)/3\,I|^2/\sigma^2$, $\sigma$ = 0.1 Ų (ISOR). + **B restraints.** The B factors of two atoms are restrained to each other in proportion to their size, by the symmetric Kullback-Leibler divergence of two isotropic Gaussians of variances $B_i$ and $B_j$: @@ -192,6 +234,28 @@ common atom, 0.2 and 0.3 for the side chains and the rest, and 0.5 for atoms wit among them, symmetry mates included) that no bond or angle relates. No atom's B goes below 2 Ų or above the ceiling of §15.1. +**NCS restraints.** Where the model holds several copies of one molecule, they are found from the model alone: two +polymer chains of one kind (protein, or nucleic acid) are copies where their sequences align with more than 15 +residues identical and identical in 80% of the longer chain or more, and their C-alpha (P) atoms lie within 2.5 Å +rms of each other after superposition - REFMAC5's criterion, the identity taken on the longer chain. The same +sequence in a conformation further away than that, such as a quasi-equivalent subunit of a virus capsid, is another +molecule. In the residues aligned to a residue of the same name, the atoms of the same name correspond - not a +hydrogen, not an alternative conformation, both refinable. Each copy is paired with the next of its molecule (and the +last with the first, where there are three or more), and each pair is restrained **locally**, on its interatomic +distances rather than on a superposition: every distance within a chain shorter than 4.2 Å in either copy, except those a bond or an angle of the dictionary already fixes, against the same +distance in the other copy, with the robust loss of Geman and McClure, + +$$ +E = rac{(\Delta/\sigma)^2}{1 + (\Delta/0.5\, ext{Å})^2},\qquad \Delta = d_\mathrm{copy\,1} - d_\mathrm{copy\,2}, +$$ + +the squared normalised difference where the two distances agree and never more than $(0.5\, ext{Å}/\sigma)^2$ where +they do not. A region in which the copies really differ - a loop in a crystal contact, a side chain in another +rotamer, a domain turned on a hinge - lets itself go as the refinement moves it, without a rule deciding in advance +which residues to exclude, and rigid motions of whole regions cost nothing, since only distances within 4.2 Å are +compared. Waters and ligands are not restrained, nor are B factors: the copies' B factors are refined independently. +Whether the restraints are used at all, and how tightly ($\sigma$), the weight trials decide (§15.7). + ### 15.6 The minimiser Positions and B factors are minimised in turn, each by an L-BFGS of rugnux's own (`RefinementMinimiser`), @@ -241,7 +305,41 @@ are: stays within 0.001 of the best so far, and the trajectory ends at its best step. On a deposited model, whose free set the depositor's refinement has already fitted, R-work falls while R-free rises, and the trajectory ends where it began. -3. **The final ladder**: the trajectory's weights times $\{0.5, 1, 2\}$, kept where one displaces it. +3. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved + was never kept on 55 refinements, and taking it away changed no choice). +4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U + held), each trial as long as a step on the positions where the weights ended, held - only the displacement + parameters refine, so the trials differ in the displacement model alone; the model chosen then takes a step with + the positions refined, kept where it displaces its trial (where the geometry is out of bounds there, which held + positions cannot mend, each trial refines the positions too, a macro-cycle longer than a step): TLS groups with a residual B - the model's own, one per chain + of at least 50 refinable atoms, and the chains cut into 2, 4, 8 and 16 contiguous segments of at least 10 residues + as TLSMD cuts them (by dynamic programming over the least-squares fit of an isotropic TLS model, a quadratic B(r), + to the refined B factors), each cut where it gives more groups than the one before; and a U per atom, at the U + weight balanced as $w_{B,0}$ is (on the U shaken by 20% of its B) times $\{1/4, 1\}$. The cuts between one group + per chain and the finest run only where the finest fits the free set better (lower $-\ln L$) than one per chain, + and the U at its balanced weight only where the U at a quarter of it lowers R-free or $-\ln L$ of the free + reflections from where the weights ended; on 49 refinements of 33 structures neither the cuts' rule nor leaving + out the U weight $\times 4$ changed a choice taken. A + trial displaces the one before it (the first: where the weights ended) only where its geometry is within bounds + and both its R-free (beyond the tie) and the $-\ln L$ of the free reflections are lower: the free set tests the + parameters a model adds, so the choice is made on the evidence of each run, not at a resolution (Merritt 2012). + The refined U are written as ANISOU and the TLS groups as `_pdbx_refine_tls*` and `REMARK 3` (`REFINE_ADP_MODEL`, + `REFINE_ANISO_ATOMS`, `REFINE_TLS_GROUPS`). It is decided after the solvent build (§15.10), on the model with its + solvent: a water a round put in refines a U of its own in the per-atom trial as every refinable atom does, and no + TLS group holds a water. +5. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the + rule every trial is held to (its geometry within bounds and R-free lower beyond the tie), the model is written as + it came, scaled to the data, and the solvent build's edits are undone (`REFINE_AS_GIVEN`). What comes out is never + worse on the free set than what went in. This holds only where the free set is free to the model as given: a model + placed as given on a free set drawn by the run (`MODEL_FREE_SET`; in `--mode refine`, a file without free flags) has + most likely been refined against those reflections, and its R-free as given does not compete. + +Where the model has copies (§15.5), the NCS restraints are one more weight: every rung of the first ladder is run +without them, and then with them at $\sigma = 0.1$ Å at the B factors' weight the rungs without chose (NCS moves the +X-ray weight a rung up as often as not, the B factors' choice it never changed on 39 refinements), and every step of +the trajectory also tries them changed - switched on where they are off, and in turn tightened ($\sigma$ halved) and +loosened ($\sigma$ doubled, off beyond 0.4 Å) where they are on. Listed after their twin without, the restraints are +kept only where they lower R-free by more than the tie. A model without copies runs exactly the trials it ran before. $w$ never exceeds $w_0/8$. The trials of one set run side by side - on one GPU, on several, or on the CPU's threads - each from the same start and taking nothing from the one before it, so the result does not depend @@ -281,7 +379,132 @@ in the report. A blob read as `LIGAND` is density of ligand size where the model molecule or an unmodelled side chain is that as well as the compound that was soaked. The readings point a person at the density; they do not identify a compound. -### 15.10 GPU and CPU +### 15.10 The solvent audit + +Every water and every single-atom ion of the refined model (and, with `--refine-score-only`, of the model as +given) is measured in its surroundings and its density and scored as each of water, Mg²⁺, Na⁺, K⁺, Ca²⁺ and Cl⁻ +(NH₄⁺ is scored too, and only reported). The audit changes nothing; the solvent build below reads it to relabel +sites. Both run only with `--refine-solvent` (off by default for now). + +**What is measured.** The model's N and O atoms around the site, symmetry mates included, counted in the +coordination bands of Cat_Wiz - 1.9-2.3 Å (Mg-like), 2.3-2.6 Å (Na, Ca), 2.6-3.2 Å (K, water) - and in the +clash shells 3.2-3.4 and 3.4-3.6 Å beyond them. Each neighbour is typed from the dictionary's energy type +(donor, acceptor or both) and from the chemistry of its residue: Lys NZ, Arg NE/NH1/NH2 and a polypeptide's +N-terminal N are positive; Asp/Glu carboxylate O, a C-terminal carboxylate and a nucleotide's phosphate O are +negative; His ring N and the Asn/Gln amide ends are both donor and acceptor, as the model's orientation of +them is a guess. Also measured: the shortest distance between two neighbours closer than 2.6 Å (the +octahedron's cis O…O), the nearest positive and negative group, the other ions within 4.2 and 6 Å, the +bond-valence sum $V = \sum_j q_j \exp((R_0 - d_j)/0.37)$ over the N and O within 3.5 Å for each cation +($R_0$ of Brese & O'Keeffe, $q_j$ the neighbour's occupancy) with its normalised vector sum (CheckMyMetal's +nVECSUM), and the 2mFo-DFc, mFo-DFc and anomalous maps at the site, in their rms. + +**The electrons a site shows.** The site's own density is fitted to 2mFo-DFc with its neighbours' held - a local +real-space fit of one amplitude at a B tied to its surroundings: the map at points 0.3 Å apart within 1.0 Å of the +site, put on the model's scale, less the density the model's other atoms put there (each atom's IT92 density to +the data's resolution at its B, to 4.5 Å), against a water's density (O with its two H) at the environment's B +(the mean B of the N and O within 3.5 Å, as CheckMyMetal compares a metal's with its ligands'), least squares. +The amplitude, in waters, is the electrons it shows, and counted apart from the neighbours' that merge with it at +moderate resolution. The map's scale per model unit is the median, over the macromolecule's C, N and O within +6 Å (or all of them where fewer than three are), of each atom's height in the map over what the model puts there - +not the waters', whose occupancy is the model's guess (the depositions' waters show half the density their model +gives). An **ion** - whose label's restraints may hold it off its density - is read where that fit puts it, +within 0.6 Å of its modelled position (its shell, bands and bond valences are measured from there); a water where +it is modelled. On the refine and open arms the depositions' ions read Mg 11, Na 10, Cl 19, K 21-23, Ca 22-26 and +Zn 33 (median), the waters 4-6. + +**The anomalous signal a site would show.** For each species, f″ at the data's wavelength (Cromer-Liberman, with +the corrections of Kissel & Pratt, as GEMMI computes it) at the environment's B, put on the scale of the anomalous +map by least squares on the model's S and P (at least three); without an anomalous map, or nothing to scale it +by, the term is left out. + +**The score.** For each species, a plain sum of terms in log units: + +| Term | What it is | +|---|---| +| first shell | the neighbours in the species' band against the coordination number it holds (Mg 6, Na 5-6, K 6-10, Ca 6-8, Cl 3-5, NH₄⁺ 4), quadratic; for water only more than four, or none at all, costs; for Mg, Na, Ca and Cl a missing ligand costs little (0.5), as the model may simply not have that water - and a halide binds at hydrophobic surfaces too, with few donors or none (Dauter & Dauter 2001), where its electrons and its anomalous signal tell it from a water | +| distances | for Mg, each neighbour in the band against Mg-O 2.07 ± 0.04 Å, Mg-N 2.19 ± 0.08 Å; for the others, whose distances are spread over their bands, the band is all there is | +| contacts it does not make | 2 per N or O closer than the species' shell (closer than 2.9 Å for Cl), and for Mg per N or O in its exclusion zone 2.3-3.4 Å; for Mg and Ca 2 per first-shell purine N7 the site is more than 1.0 Å out of the plane of, or not 3.5-3.9 Å from the base's O6 or N6 (Leonarski et al. 2017) | +| chemistry | a cation: 2 per donor-only neighbour in its shell and per positive group within 3.5 Å; Cl: 2 per acceptor-only neighbour and per negative group within 3.2 Å; water: 2 per hydrogen bond beyond two donated or two accepted (neighbours at 2.6-3.4 Å) | +| bond valence | for a cation, $V$ against its charge $z$, Gaussian with a spread of $0.25z$ widened by the neighbours' distance errors, and the vector sum against CheckMyMetal's outlier bound 0.23 | +| electrons | the electrons the site shows against those the species would (its density's height over a water's at the environment's B: Mg²⁺ and Na⁺ about 11, K⁺, Ca²⁺ and Cl⁻ about 20) - an ion at full occupancy, a water as the model's waters show (their median) - log-normal with the spread of the waters' (an ion's occupancy and order vary as much), at least that of the map's calibration. A site as dense as a full 10-electron scatterer is so more than a typical water, as Echols et al. flag a light ion by a 2mFo-DFc above the waters' mean; where the density is weak the term tells little, and the shell decides | +| anomalous | the anomalous map at the site against what the species would show, Gaussian in the map's rms | + +A site bound to another cation (closer than 2.6 Å) shares that cation's other ligands as neighbours - the +cis waters of a Mg(H₂O)₆ are 2.9 Å apart - and those are left out of its score. **Coordinate error**: each +atom's is taken as its rms displacement $\sqrt{B/8\pi^2}$ times $d_\mathrm{min}/10$ Å (about 0.1 Å at B = 20 Ų +and 2 Å); a neighbour's band is widened by the error of the distance and its distance term is widened by it +too, so that where the coordinates are poor the species' shells overlap and the score stops telling them +apart instead of telling them apart wrongly. The best of water, Mg, Na, K, Ca and Cl is reported, and +`AMBIGUOUS` where the runner-up is within 2. The audit counts a site as disagreeing with its label where +the best species is another one, not ambiguously. + +**Caveats.** The refined model's metal-site distances are restrained (§15.5) where its LINK records name +the bonds, which pulls an ion's shell towards the label it carries - the first pitfall of every survey of +ions in the PDB; the local fit reads the ion where its density puts it, but its ligands stay where the restraints +held them. `--mode refine --refine-score-only` audits the model as given instead. The species are the six above: +a site denser than any of them (a cacodylate's As, a heavier metal) is called the heaviest that fits worst least. +Per site, everything above is in `_solvent.json` (with `electrons`, `b_env`, `peak_shift_A`, +`anomalous_expected`, `n7_ligands`, `n7_off_plane`); the counts are the `REFINE_SOLVENT_` keys of the report. + +**The solvent build.** Within the refinement, and to 2.8 Å (phenix.refine's ordered-solvent limit), waters are built +and removed and waters and ions relabelled, as phenix.refine builds waters within its macro-cycles. Where the weight +trajectory (§15.7) ends, a round reads the maps and the audit of that model and changes its solvent; then the solvent +alone - its waters and single-atom ions, the rest held - is refined at the weights the trajectory ended at (its B +too, at its balanced weight where the trajectory held B, and 10 iterations a macro-cycle), and the round is kept +where that lowers R-free; where it does not but the round as built does (refining the solvent alone can lose what its +sites gained: on a deposited model 41 waters lowered R-free by 0.0016 as built and raised it by 0.0036 refined), it +is kept as built; otherwise it is undone. The rest of the model goes on with its B factors held or refined as the +trajectory had them. The rounds stop at one that changes nothing, at one undone, or after two (a third gained at most +0.0004 in R-free where it was kept, on twelve sets, and was refused on the others); the final ladder of §15.7 then +refines what they kept with the rest. The atom list of a refinement is fixed - its engines are made for it - so the +waters a round may add are in the model from the start as placeholders: one water each (as many as a fifteenth of the +model's heavy atoms, at least 64) at occupancy 0, which puts no density, no mask and no contact anywhere and is not +refined until a round puts one at a peak; a water taken out goes to occupancy 0 the same way, and a relabelled site +keeps its place. A round thus costs a map, an audit and a refinement of the solvent: on twelve refine-arm and +open-arm depositions the build adds 0.2-0.9 times the refinement's time (median 0.7), where a refinement of its own +per round added 1-3.7 times (median 2.3), at an R-free within 0.001 of that on eight of them (two 0.0016-0.0028 +higher, one 0.0016 lower). `--refine-score-only` builds nothing, and the build and the audit run only with `--refine-solvent` +(off by default for now). + +- *Relabelled* is a water or a single-atom ion (no alternative conformer) whose best-supported species is another + one, not ambiguously - a water becomes an ion, an ion water or another ion - once per site. It takes the + species' residue name, element and atom name, and scatters as its ion (§15.2); its metal records go, and a metal + gets a record for each N, O and S of its inner sphere in the model's own copy (within the library's distance plus + 0.33 Å, at the coordination of those within an octahedral site's), which the next refinement restrains as it + restrains the model's own (§15.5). + +- *Candidates* are the local maxima of mFo-DFc from 3 σ (phenix.refine's peak cutoff), each symmetry-equivalent + set once, moved to the copy nearest the model. A peak closer than 1.2 Å to a model atom is that atom's own + residual and is passed over. +- *Not solvent* - reported in `_solvent.json` with the reason, never filled: a peak 1.2-1.9 Å from a model + atom (a missing or displaced atom); one beside a model atom (within 2.6 Å) that sits in mFo-DFc of -3 σ or + lower (the atom is displaced); one within 3 Å of where a truncated residue's missing atoms would be (an atom the + dictionary bonds to an atom the model lacks); the centre of three or more other peaks at 1.3-1.7 Å (a + tetrahedral group, sulfate or phosphate) or any peak with another 1.1-1.9 Å away (atoms of one molecule); + elongated density (the rms length of the density above 3 σ within 1.6 Å along its longest axis over its shortest + above 3); and 2mFo-DFc above every S and P of the model (a heavier atom). +- *A water* is placed where the peak has a hydrogen-bond partner - an N or O that can donate or accept at + 2.2-3.4 Å (phenix.refine's 3.2 Å, widened for the coordinate error), a metal within 2.6 Å (its ligand), or a + halide at 2.9-3.4 Å - and nothing closer than 2.2 Å to an N or O, nor than 3.0 Å to any other atom (Coot keeps + waters 2.4-3.2 Å from the model; a little wider here for the peak's position), strongest first, none within + 2.2 Å of one already placed. It goes into the chain of its nearest atom, numbered on from that chain, with + occupancy 1 and the mean B of its partners. A peak on a special position (§15.5) is placed exactly on it with + occupancy 1 over the site's multiplicity, its coinciding copies one water (on the refine and open arms this was + better or equal in R-free on all five sets with such peaks, against refusing them); one 0.5-2.2 Å from its own + copy, which it would clash with, is refused. +- *Removed* is any water whose mFo-DFc at its centre is -3 σ or lower (the cut Echols et al. use), and a water the + build itself added that has no N, O or ion within 3.4 Å any more or whose B reached the refinement's ceiling. + phenix.refine's other filter, 2mFo-DFc below 1 σ at a refined water, is not used: on the refine arm it raised + R-free wherever it was tried, on the deposited waters (five sets of five) and on the waters built here, and so + did taking out deposited waters without a partner. A weak water still explains density, and a deposited one + carries its depositor's judgement. + +`REFINE_SOLVENT_ADDED` and `REFINE_SOLVENT_REMOVED` count the waters added and still there and those of the model +as given taken out, `REFINE_SOLVENT_RELABELLED` the sites relabelled (each, with its scores, in the report and the +JSON); `REFINE_SOLVENT_NOT_SOLVENT` the not-solvent peaks of the final model's difference map; +`REFINE_SOLVENT_BUILD_ROUNDS` the rounds kept. The model as given (`REFINE_R_FREE_START`, the `REFINE_SCORE_` terms) and the placement stage are the first refinement's; the refined model, its R-factors, geometry, maps and blobs are the last kept round's. + +### 15.11 GPU and CPU On a CUDA build the target, the gradient and the restraints are evaluated on the card - the density gather, cuFFT, the composition of the symmetry and the target per reflection, the adjoint through a CSR list, the inverse transform and one warp per atom for its gradient; the restraints in one kernel whose @@ -292,19 +515,98 @@ the card's budget, or with `--no-gpu`, or on a build without CUDA (macOS include Metal path), the CPU engine computes the same target, to rounding, several times more slowly (seconds on a GPU, typically 2-15 minutes on the CPU). -### 15.11 Not yet +### 15.12 Twinned data + +Where the lattice admits a **twin law** for the group the data are merged in - a rotation of the lattice's metric +symmetry (Le Page two-folds within 3° of obliquity, the search the free set is grouped by, §10.7) that the group +lacks, one per coset of the group, with its square in the group so that it exchanges two domains - the placed +model is refined a second time, against the data as twinned by the law, and the two refinements are compared. +`rugnux ` does this on its own; nothing has to be given. + +**The target.** A crystal twinned by law $T$ at fraction $\alpha$ records $(1-\alpha)I(\mathbf{h}) + \alpha I(T\mathbf{h})$ +(Yeates 1997), so the model an observation is compared with is + +$$ +|F_\mathrm{twin}(\mathbf{h})| = k\,e^{-\mathbf{s}^\top\mathbf{B}\mathbf{s}/4}\sqrt{(1-\alpha)\,|F_\mathrm{calc}(\mathbf{h}) + F_\mathrm{sol}(\mathbf{h})|^2 + \alpha\,|F_\mathrm{calc}(T\mathbf{h}) + F_\mathrm{sol}(T\mathbf{h})|^2}, +$$ + +and the target is the likelihood of §15.2 with $|F_\mathrm{model}|$ replaced by $|F_\mathrm{twin}|$, $D$ and $\Sigma$ +fitted against it (§15.3). At $\alpha = 0$ it is the untwinned target exactly. It is an approximation: a twinned +amplitude is treated as Rice-distributed about $D|F_\mathrm{twin}|$, while the distribution of a sum of two +intensities is narrower; $\Sigma$, fitted to the free set, absorbs that, and REFMAC5's twinned-intensity likelihood +(Murshudov et al. 2011) is the treatment this one simplifies. The mates $T\mathbf{h}$ that were not observed are +composed as rows of their own, with no observation: their structure factors come from the same transform, so a +twinned evaluation costs no extra FFT, only one more kernel per evaluation on the GPU (the twinned term per +reflection and the gather of its mate's derivative, the law being an involution on the rows). + +**The fraction, the scale and the solvent** are fitted inside the twin sum at every macro-cycle: the scale and the +solvent as in §15.3, then $\alpha$, then the scale and the solvent again against $F_o$ detwinned by the model as it +stands, $F_o|F_\mathrm{model}(\mathbf{h})|/|F_\mathrm{twin}(\mathbf{h})|$ (at which scale the twinned model meets $F_o$ +as the untwinned one meets the detwinned $F_o$), then $\alpha$ again. $\alpha \in [0, 1/2]$ comes from the regression +of $F_o^2$, with an intercept, on the model's intensities at $\mathbf{h}$ and at $T\mathbf{h}$, all normalised in +resolution shells of 500 reflections: $\alpha = b_{T\mathbf{h}}/(b_\mathbf{h} + b_{T\mathbf{h}})$, over the working +reflections that are not their own mate. The expectation of $F_o^2$ given the model, $D^2|F_\mathrm{model}|^2 + \Sigma$, +is linear in the two intensities, so an unrelated partner takes a coefficient of 0 however poor the model; a +least-squares fit of the twinned amplitudes instead took a fraction of 0.16 for unrelated partners of one model at +R-free 0.31, the average of two predictions being a better prediction than either. + +**The decision.** The twinned refinement is kept where its fraction is above 0 and the law beats its own **null** +on the free reflections by at least three standard errors - the null being the same twinned model at the same +fraction with every reflection's mate replaced by an unrelated reflection of the same resolution and of the same +set (a free reflection's by a free one, as a law's are): what the averaging of two intensities buys whatever they +are (it lifts a weak reflection's model amplitude off zero). The comparison is paired, reflection by reflection, +$|F_o - |F_\mathrm{null}|| - |F_o - |F_\mathrm{twin}||$ over the free set, summed over each free twin pair (the two +mates share their two $F_\mathrm{calc}$, so they are one term, not two independent ones), its mean against its +standard error (`REFINE_TWIN_NULL_Z`). The R-free of the twinned refinement against the untwinned one's does not +enter: two refinements of one model differ in R-free by a few thousandths from their trajectories alone, and any +twin target lowers R by its averaging, so that comparison favours the twinned one with or without a twin. A pseudo-symmetry the model already carries (non-crystallographic +symmetry parallel to the law) predicts $I(T\mathbf{h}) \approx I(\mathbf{h})$, so the law changes nothing and is not +taken for a twin. + +**The data's own evidence.** A lead of two to three standard errors is enough where the merge's intensity statistics +read twinned as well - the L-test itself below 0.44 (`TWINNING_VERDICT` `INDICATED` with +`ESTIMATED_TWIN_FRACTION_FROM` `L_TEST`; not where only the second moment reads low, nor where a pseudo-translation +leaves the L-test unreadable): two independent pieces of evidence, the data's and the model's, then agree. +`REFINE_TWIN_RULE` says which rule decided. The data-side test is not made a gate - it misses twins a model sees, a +pseudo-merohedral law or a near-perfect twin among them - only further evidence where the null alone is short. +`--mode refine` has no merge statistics, so there the null alone decides. + +A law is tested only where the free set keeps its twin pairs together, and is otherwise reported +(`REFINE_TWIN` `FREE_SET_SPLIT`) rather than tested. The free set rugnux draws (§10.7) keeps together the pairs of +every law that is a rotation of the cell's own basis vectors; a law of a centred lattice that is not (its matrix +fractional in the cell's basis) can be split by it, and so can a free set `--mode refine` reads from a file. Of several laws, the one the placed model asks the largest fraction of is refined. Each of the +two refinements is a whole one - its solvent built (§15.10) and its displacement model chosen (§15.7) against its +own target - and the decision is taken where they ended. `--refine-score-only` makes the same test on the model +as given: it is scored against the data untwinned and twinned by the law, and the twinned score is taken by the +same rule. + +**Maps.** Against twinned data the map coefficients use $F_o$ detwinned in proportion to the model's own share of +the twinned intensity, $F_o|F_\mathrm{model}(\mathbf{h})|/|F_\mathrm{twin}(\mathbf{h})|$, and $DF_c$ of the model's own +$|F_\mathrm{model}(\mathbf{h})|$; `FP` in `_refined_maps.mtz` is the detwinned amplitude. + +The keys are `REFINE_TWIN` (`NO_TWIN_LAW`, `FREE_SET_SPLIT`, `ZERO_FRACTION`, `UNTWINNED`, `TWINNED`), +`REFINE_TWIN_LAW`, `REFINE_TWIN_FRACTION`, `REFINE_TWIN_R_WORK`/`_R_FREE`, `REFINE_TWIN_R_FREE_NULL`, `REFINE_TWIN_NULL_Z` and +`REFINE_UNTWINNED_R_WORK`/`_R_FREE`, the last five of the two refinements where they ended. Where `REFINE_TWIN` is `TWINNED`, every +other `REFINE_` key, `REFINE_R_FREE_START` and the `REFINE_SCORE_` terms included, is of the twinned refinement. + +### 15.13 Not yet What the refinement does not do yet - each a known gap, not a design choice: - **Custom ligands and other monomer libraries.** Only the 84 compounds of §15.4 are restrained. Any other residue is held fixed (it stays in $F_\mathrm{calc}$, the mask and the repulsions); no restraints are generated for a new ligand, and neither a user's restraint CIF nor an external monomer library can be given. -- **TLS** refinement. -- **Anisotropic ADPs**: an anisotropic B is replaced by its isotropic equivalent, and only isotropic B is - refined. -- **NCS restraints**: copies are refined independently. -- **Twin refinement**: the target is not a twinned one, also where the twin hypothesis' merge was written. -- **Water building**: waters are neither picked nor removed; the blob finder only points at candidates. +- **TLS restraints**: a TLS group's T and L are not held positive semi-definite (only every atom's total U is kept + positive definite); the TLS groups are contiguous segments of one chain, never a domain made of several. +- **Global NCS and NCS of B factors**: copies are restrained locally, on their interatomic distances (§15.5), + never by a superposition, and their B factors are refined independently; ligands and waters are not restrained + between copies. +- **More than two twin domains** (tetartohedral twinning): one law at a time is refined, and laws whose square is + not in the group are not considered. +- **Ion identification beyond the six species**: waters and ions are relabelled among water, Mg²⁺, Na⁺, K⁺, Ca²⁺ + and Cl⁻ (§15.10); NH₄⁺ is only reported, transition metals (Zn, Mn, ...) are measured but never relabelled, and no + ion is placed at a peak that is not a water first. A site's hypotheses are compared by a local fit of its + density, not by refining each in turn. - **Alternative conformations and occupancies**: existing conformers are refined in place; no occupancy is refined and no conformer is added. - **Rotamer and peptide-flip search**, and **Ramachandran restraints**. @@ -317,21 +619,36 @@ What the refinement does not do yet - each a known gap, not a design choice: - R. C. Agarwal, "A new least-squares refinement technique based on the fast Fourier transform algorithm", *Acta Cryst.* **A34** (1978), 791-809 (the gradient by the FFT method, §15.2). - N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and its figure of merit, §15.2, §15.8). -- G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5). +- G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5; the twinned refinement §15.12 simplifies, its twin laws pruned by fraction and its fractions refined by least squares). - R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with errors", *Acta Cryst.* **A42** (1986), 140-149 (the m, D weighting of the map coefficients, §15.8). +- T. O. Yeates, "Detecting and overcoming crystal twinning", *Methods Enzymol.* **276** (1997), 344-358 (the twinned intensity, §15.12). +- A. A. Lebedev, A. A. Vagin & G. N. Murshudov, "Intensity statistics in twinned crystals with examples from the PDB", *Acta Cryst.* **D62** (2006), 83-95 (twin laws from the lattice's metric, §15.12). +- R. Herbst-Irmer & G. M. Sheldrick, "Refinement of twinned structures with SHELXL97", *Acta Cryst.* **B54** (1998), 443-449 (twinned refinement with the fraction refined beside the model, §15.12). - V. Y. Lunin & T. P. Skovoroda, "R-free likelihood-based estimates of errors for phases calculated from atomic models", *Acta Cryst.* **A51** (1995), 880-887, and V. Y. Lunin, P. V. Afonine & A. G. Urzhumtsev, "Likelihood-based refinement. I. Irremovable model errors", *Acta Cryst.* **A58** (2002), 270-282 (the likelihood's parameters from the free set, §15.3). - J.-S. Jiang & A. T. Brünger, "Protein hydration observed by X-ray diffraction: solvation properties of penicillopepsin and neuraminidase crystal structures", *J. Mol. Biol.* **243** (1994), 100-115, and A. Fokine & A. Urzhumtsev, "Flat bulk-solvent model: obtaining optimal parameters", *Acta Cryst.* **D58** (2002), 1387-1392 (the flat bulk-solvent mask and its parameters, §15.3). - P. V. Afonine, R. W. Grosse-Kunstleve & P. D. Adams, "A robust bulk-solvent correction and anisotropic scaling procedure", *Acta Cryst.* **D61** (2005), 850-855 (the grid search over the solvent parameters, §15.3). - P. J. Huber, "Robust estimation of a location parameter", *Ann. Math. Statist.* **35** (1964), 73-101, and P. Charbonnier, L. Blanc-Féraud, G. Aubert & M. Barlaud, "Deterministic edge-preserving regularization in computed imaging", *IEEE Trans. Image Process.* **6** (1997), 298-311 (the robust scale fit and its smooth, pseudo-Huber loss, §15.3). - A. A. Vagin, R. A. Steiner, A. A. Lebedev, L. Potterton, S. McNicholas, F. Long & G. N. Murshudov, "REFMAC5 dictionary: organization of prior chemical knowledge and guidelines for its use", *Acta Cryst.* **D60** (2004), 2184-2195 (the restraint dictionary, §15.4). - M. Wojdyr, "GEMMI: A library for structural biology", *J. Open Source Softw.* **7** (2022), 4200 (the topology, the riding hydrogens and the blob search, §15.4, §15.9). +- V. Schomaker & K. N. Trueblood, *Acta Cryst.* **B24** (1968), 63-76, and M. D. Winn, M. N. Isupov & G. N. Murshudov, *Acta Cryst.* **D57** (2001), 122-133 (TLS, and the B column total or residual to it, §15.1, §15.5); J. Painter & E. A. Merritt, *Acta Cryst.* **D62** (2006), 439-450 (TLSMD's segments, §15.7). +- F. L. Hirshfeld, *Acta Cryst.* **A32** (1976), 239-244, A. Thorn, B. Dittrich & G. M. Sheldrick, *Acta Cryst.* **A68** (2012), 448-451, and G. M. Sheldrick, *Acta Cryst.* **C71** (2015), 3-8 (the rigid-bond, similarity and isotropy restraints of anisotropic atoms, §15.5); E. A. Merritt, *Acta Cryst.* **D68** (2012), 468-477 (the displacement model decided by test, §15.7). - A. Blondel & M. Karplus, "New formulation for derivatives of torsion angles and improper torsion angles in molecular mechanics: elimination of singularities", *J. Comput. Chem.* **17** (1996), 1132-1141 (the torsion derivatives, §15.5). - M. M. Harding, "Geometry of metal-ligand interactions in proteins", *Acta Cryst.* **D57** (2001), 401-411, and "Small revisions to predicted distances around metal sites in proteins", *Acta Cryst.* **D62** (2006), 678-682; F. Leonarski, L. D'Ascenzo & P. Auffinger, "Mg2+ ions: do they bind to nucleobase nitrogens?", *Nucleic Acids Res.* **45** (2017), 987-1004 (the metal-site distances, §15.5). +- R. W. Grosse-Kunstleve & P. D. Adams, "Algorithms for deriving crystallographic space-group information. II. Treatment of special positions", *Acta Cryst.* **A58** (2002), 60-65 (the site symmetry and the constraint of an atom on a special position, §15.5). +- D. T. Cromer & D. A. Liberman, "Anomalous dispersion calculations near to and on the long-wavelength side of an absorption edge", *Acta Cryst.* **A37** (1981), 267-268, and L. Kissel & R. H. Pratt, "Corrections to tabulated anomalous-scattering factors", *Acta Cryst.* **A46** (1990), 170-175 (f″ at the data's wavelength, §15.10). +- Z. Dauter & M. Dauter, "Entering a new phase: using solvent halide ions in protein structure determination", *Structure* **9** (2001), R21-R26 (halides at sites with few donors, §15.10). +- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6). - A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7). - P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7). - A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free). +- G. J. Kleywegt, "Use of non-crystallographic symmetry in protein structure refinement", *Acta Cryst.* **D52** (1996), 842-857; I. Usón, E. Pohl, T. R. Schneider, Z. Dauter, A. Schmidt, H.-J. Fritz & G. M. Sheldrick, "1.7 Å structure of the stabilized REIv mutant T39K. Application of local NCS restraints", *Acta Cryst.* **D55** (1999), 1158-1167; R. A. Nicholls, F. Long & G. N. Murshudov, "Low-resolution refinement tools in REFMAC5", *Acta Cryst.* **D68** (2012), 404-417; J. J. Headd et al., "Flexible torsion-angle noncrystallographic symmetry restraints for improved macromolecular structure refinement", *Acta Cryst.* **D70** (2014), 1346-1356 (local NCS restraints with a robust loss, decided by R-free, and Murshudov et al. 2011 above for the copies found by alignment, §15.5, §15.7). - P. Emsley, B. Lohkamp, W. G. Scott & K. Cowtan, "Features and development of Coot", *Acta Cryst.* **D66** (2010), 486-501 (unmodelled blobs, §15.9). +- N. Naleem, A. Henning-Knechtel, S. Kirmizialtin & P. Auffinger, "Cat_Wiz: a stereochemistry-guided toolkit for locating, diagnosing, and annotating Mg2+ ions in RNA structures", *Nucleic Acids Res.* **54** (2026), gkag120; F. Leonarski, L. D'Ascenzo & P. Auffinger, *Nucleic Acids Res.* **45** (2017), 987-1004 and *RNA* **25** (2019), 173-192; F. Leonarski, A. Henning-Knechtel, S. Kirmizialtin, E. Ennifar & P. Auffinger, *Nucleic Acids Res.* **53** (2025), gkae1148 (the coordination bands, distances and exclusion zone of the solvent audit, §15.10). +- P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367, and P. Emsley, B. Lohkamp, W. G. Scott & K. Cowtan, "Features and development of Coot", *Acta Cryst.* **D66** (2010), 486-501 (the peak cutoff, the partner and clash distances and the resolution limit of the water build, §15.10). +- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114, and N. Morshed, N. Echols & P. D. Adams, "Using support vector machines to improve elemental ion identification in macromolecular crystal structures", *Acta Cryst.* **D71** (2015), 1147-1158 (the ligand sphere and the halide rules, §15.10). +- H. Zheng et al., "CheckMyMetal: a macromolecular metal-binding validation tool", *Acta Cryst.* **D73** (2017), 223-233 (the vector sum, §15.10); O. Carugo, "Buried chloride stereochemistry in the Protein Data Bank", *BMC Struct. Biol.* **14** (2014), 19 (the chloride's shell, §15.10). +- I. D. Brown & D. Altermatt, *Acta Cryst.* **B41** (1985), 244-247, and N. E. Brese & M. O'Keeffe, "Bond-valence parameters for solids", *Acta Cryst.* **B47** (1991), 192-197 (the bond-valence sums, §15.10). - M. Abramowitz & I. A. Stegun, *Handbook of Mathematical Functions*, National Bureau of Standards (1964), 9.8.1-9.8.4 (the Bessel functions of the target, §15.2). The full citations, with DOIs, are in [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md). diff --git a/docs/EXTERNAL_TEST_DATA.md b/docs/EXTERNAL_TEST_DATA.md index f374959ed..67828355b 100644 --- a/docs/EXTERNAL_TEST_DATA.md +++ b/docs/EXTERNAL_TEST_DATA.md @@ -4,7 +4,7 @@ Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeli ever sees its own detectors is not tested. The datasets below were collected by other people, on detectors and in file formats we do not produce ourselves, and are used here to check that `rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected -at other facilities, eleven on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone +at other facilities, thirteen on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone else's detector and someone else's acquisition system. Their authors published all of these for exactly this kind of reuse, and this page is where we credit them. @@ -101,6 +101,8 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the | [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC | | [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset | | [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor | +| [6OQZ](https://www.rcsb.org/structure/6OQZ) | IRRMC [10.18430/m3.irrmc.5324](https://doi.org/10.18430/m3.irrmc.5324) | Home source, Bruker rotating anode | 1.60 | I 2 2 2 | 92.9 97.9 102.7 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of glucose isomerase from non-merohedrally twinned crystals | +| [6OR0](https://www.rcsb.org/structure/6OR0) | IRRMC [10.18430/m3.irrmc.5325](https://doi.org/10.18430/m3.irrmc.5325) | Home source, Bruker rotating anode | 1.55 | I 21 3 | 78.0 78.0 78.0 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of insulin from non-merohedrally twinned crystals | | [6OWW](https://www.rcsb.org/structure/6OWW) | Zenodo [10.5281/zenodo.2943248](https://doi.org/10.5281/zenodo.2943248) | ALS 8.3.1 | 3.84 | P 1 21 1 | 85.8 86.0 214.3 90.0 89.9 90.0 | PILATUS3 6M | Crystal structure of a Human Cardiac Calsequestrin Filament Complexed with Ytterbium | | [6P8J](https://www.rcsb.org/structure/6P8J) | SBGrid [10.15785/sbgrid/671](https://doi.org/10.15785/sbgrid/671) | APS 24-ID-E | 1.47 | P 1 21 1 | 46.5 152.2 88.3 90.0 90.1 90.0 | Dectris Eiger 16M | Structure of P. aeruginosa ATCC27853 CdnD D62N/D64N mutant bound to ATP | | [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 | @@ -150,6 +152,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the | [7Q6J](https://www.rcsb.org/structure/7Q6J) | Zenodo [10.5281/zenodo.4686876](https://doi.org/10.5281/zenodo.4686876) | PETRA III, DESY P11 | 2.20 | P 21 21 21 | 72.7 115.9 116.2 90.0 90.0 90.0 | PILATUS 6MF | Crystal structure of the human GDAP1 CMT2 mutant-H123R | | [7QIJ](https://www.rcsb.org/structure/7QIJ) | SBGrid [10.15785/sbgrid/907](https://doi.org/10.15785/sbgrid/907) | PETRA III, EMBL c/o DESY P13 (MX1) | 4.10 | P 21 21 21 | 143.5 324.9 369.4 90.0 90.0 90.0 | PILATUS 6M-F | Complex of the Yersinia enterocolitica Type III secretion export gate YscV with substrate:chaperone complex YscX:YscY | | [7QIS](https://www.rcsb.org/structure/7QIS) | IRRMC [10.18430/M37QIS](https://doi.org/10.18430/M37QIS) | BESSY 14.2 | 1.83 | P 61 | 100.3 100.3 206.2 90.0 90.0 120.0 | PILATUS3 2M | CRYSTAL STRUCTURE OF THE P1 difluoroethylglycine (DfeGly) BPTI MUTANT- BOVINE CHYMOTRYPSIN COMPLEX | +| [7QT9](https://www.rcsb.org/structure/7QT9) | Zenodo [10.5281/zenodo.5836055](https://doi.org/10.5281/zenodo.5836055) | Diamond I24 | 2.43 | I 1 2 1 | 45.2 54.5 115.7 90.0 101.1 90.0 | PILATUS3 6M | Room temperature In-situ SARS-CoV-2 MPRO with bound Z4439011584 | | [7RAA](https://www.rcsb.org/structure/7RAA) | SBGrid [10.15785/sbgrid/881](https://doi.org/10.15785/sbgrid/881) | SSRL BL12-2 | 2.69 | P 43 21 2 | 66.4 66.4 298.3 90.0 90.0 90.0 | PILATUS 6M | Designed StabIL-2 seq15 | | [7RIS](https://www.rcsb.org/structure/7RIS) | IRRMC [10.18430/M37RIS](https://doi.org/10.18430/M37RIS) | APS 21-ID-D | 1.72 | P 32 2 1 | 44.5 44.5 189.9 90.0 90.0 120.0 | Dectris Eiger 9M | Crystal structure of RPA3624, a beta-propeller lactonase from Rhodopseudomonas palustris, with active-site bound phosphate | | [7RJI](https://www.rcsb.org/structure/7RJI) | IRRMC [10.18430/M37RJI](https://doi.org/10.18430/M37RJI) | LNLS W01B-MX2 | 1.71 | H 3 2 | 83.0 83.0 124.8 90.0 90.0 120.0 | PILATUS 2M | BthTX-II variant b, from Bothrops jararacussu venom, complexed with stearic acid | @@ -244,7 +247,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the | [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 | | [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) | -The table holds 201 PDB entries. Six more datasets, small-molecule / chemical-crystallography +The table holds 204 PDB entries. Eight more datasets, small-molecule / chemical-crystallography sets kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero detector 2θ, have no PDB entry and are listed [below](#datasets-with-no-pdb-entry). @@ -313,7 +316,7 @@ the rest were deleted, so a run over the data directory sees a single collection | 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å | | 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** | -**Thirteen further archives hold more than one collection.** Their layout was read from the +**Seventeen further archives hold more than one collection.** Their layout was read from the image files and repository listings; one sweep is kept for a run over the data directory unless noted. @@ -329,6 +332,10 @@ noted. | lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run | | 9E2T | one continuous sweep plus screening images | the 2700-frame sweep | | 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) | +| 6OR0 | four 900 × 0.2° Bruker `.sfrm` runs of one twinned crystal at 10 cm: ω scans at 2θ = 0° (`01`) and 35° (`02`), φ scans on the fixed-χ (54.74°) platform at 2θ = 0° (`03`) and 35° (`04`) | the `01` run (2θ = 0°): the header distance of these SMART 6000 frames is ~4% short of what the data give, which the distance refinement corrects on a 2θ = 0° run and not yet on a swung one | +| 6OQZ | eleven `.sfrm` ω runs of 472-900 × 0.2° at 18 cm, 2θ = 10°, 25° and 45° (low, medium, high resolution) at different φ, plus two 5-frame runs at 2θ = 0° and the orientation-matrix runs `gdri_MATRIX_*` | the low-resolution `01` run (2θ = 10°), for the reason given for 6OR0; three crystals, the test set for multi-lattice integration | +| laspartic | seven Bruker-converted full-imgCIF kappa runs, φ (`01`, 1200 × 0.3°) and ω (`02`-`07`, 292-394 frames) at 2θ = -26.7° | the 1200-frame `01` φ run | +| onitroaniline | the same seven-run kappa layout (`01` 1200 × 0.3° φ, `02`-`07` ω), deposited twice: as Bruker `.sfrm` and as APEX's full-imgCIF conversion of the same frames | the `01` run, read from the `.sfrm` files | | 7D98 | one 250 × 0.5° miniCBF sweep (frames 11-260), plus four HDF5 masters that repeat frames 61-260 in 60-frame pieces | the miniCBF sweep; the HDF5 files stay beside it and the row is pinned | | 6H2P | two sweeps of one crystal, 1800 × 0.1° at 0.918 Å and 3600 × 0.1° at 1.893 Å (the Mn K edge) | **both**, as two battery rows, `6h2p_native` and `6h2p_1p89A`; the long-wavelength row has no deposited resolution to compare with | | 8C3E | four 0.5° ω scans of one crystal at different (φ, χ) settings, 56, 150, 150 and 50 frames (`lcb2_8_newt_001` to `_004`) | the 150-frame `_003` scan | @@ -347,9 +354,38 @@ noted. frames (`-images-1-180.zip`, `-images-181-360.zip`, or `2wnq-181-360.zip`); both were extracted into one directory, which holds the one 360 × 0.5° sweep. +## Multi-sweep battery rows + +The opt-in `multisweep` arm of the battery (`tools/battery/multisweep.json`, see the +[battery README](../tools/battery/README.md#multi-sweep-rows-the-multisweep-arm-opt-in)) gives +rugnux several sweeps of the archives above at once and scores the one merge against the base +row's reference (7QT9, which has no single-sweep row, carries its own), plus which sweeps joined. +It uses only sweeps that are on disk: for 6UKF, 7DKP, +9PBB, 8U0I and 36GK the second collection was deleted (above), so they have no row until it is +downloaded again. Two-wavelength pairs (6IU8, 6H2P, 7RIS) are refused by rugnux as one dataset; +6IU8 is the row that checks it. + +| Row | Base row | Sweeps | What it covers | Expected | +|---|---|---|---|---| +| `6or0_sweeps` | open/6or0 | 4 | one twinned crystal: omega and phi scans at 2theta 0 and 35 deg (swung detector, header distance ~4% short) | all join | +| `6oqz_sweeps` | open/6oqz | 4 | one crystal at 2theta 10, 25, 45 and 45 deg; multi-lattice; the header distance makes the low-2theta runs' cell ~3% short | all join | +| `lalanine_sweeps` | open/lalanine | 4 | small molecule, four runs of one crystal at 2theta 20 deg | all join | +| `laspartic_sweeps` | open/laspartic | 7 | small molecule, seven kappa runs of one crystal (phi + six omega), sparse, twinned | all join | +| `cytidine_omega` | open/cytidine | 3 | small molecule, three omega scans at phi 0/120/240 at 2theta 30 deg | all join | +| `8c3e_scans` | open/8c3e | 4 | home source, four omega scans of one twinned crystal at different (phi, chi) | all join | +| `8dyz_wedges` | open/8dyz | 4 | one crystal, four of its eight 50 deg wedges, translated between wedges, room temperature, 1.27 A | all join | +| `8dz7_crystals` | open/8dz7 | 4 | two crystals, two 50 deg wedges each, room temperature, 1.34 A | all join | +| `6o2h_crystals` | open/6o2h | 4 | four triclinic crystals, one 50 deg wedge each, room temperature, 1.21 A | all join | +| `7qt9_crystals` | open data, own reference | 20 | in-situ crystals at room temperature, twenty 20 deg wedges (the depositors merged them as 16 crystals), none complete alone | more than half join | +| `7l84_helical` | open/7l84 | 2 | two ~720 deg helical sweeps of one crystal, S-SAD at 1.892 A, room temperature | all join | +| `6fid_sweeps` | open/6fid | 2 | two 360 deg sweeps of one crystal (aligned and random orientation), S-SAD at 2.066 A | all join | +| `6iu8_mad` | open/6iu8 | 2 | keep separate: a two-wavelength MAD pair of one crystal - one dataset per wavelength | refused (keep separate) | +| `lyso_x10sa_90deg_pair` | inhouse/lyso_x10sa_90deg_1 | 2 | two sweeps of one crystal at the same orientation, 42 s apart | all join | +| `lyso_x10sa_two_crystals` | inhouse/lyso_x10sa_90deg_1 | 2 | keep separate: two lysozyme crystals of one form at one wavelength whose intensities differ (cells 1 % apart) | one left out (keep separate) | + ## Datasets published as Raw Data Letters -Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a +Five of the datasets - 6R72, 8QQ7, 6RLR, laspartic and onitroaniline - were published as IUCrData Raw Data Letters, a format whose purpose is to make raw images citable and re-processable in their own right. The letters describe the collections and the difficulties in them, and are the reference for what the data are: @@ -364,6 +400,15 @@ data are: IUCrData 7, x220852 [doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR. +- M. Lutz and L. Kroon-Batenburg, "Accurate intensity integration in the twinned γ-form of + *o*-nitroaniline" (2022), IUCrData 7 + [doi:10.1107/S2414314622010598](https://doi.org/10.1107/S2414314622010598) - covers + onitroaniline. +- M. Lutz, "Twinned L-aspartic acid" (2025), IUCrData 10 + [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) - covers laspartic: + a twofold rotation about [100] with twin index 2 and obliquity 0.74°, whose coincidence lattice + is the doubled cell first reported for the compound (Bernal, 1931). + The authors of the second letter also published their own reciprocal-space reconstruction of the 6RLR data as a separate Zenodo record, [10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763). @@ -437,8 +482,8 @@ The PDB entry names the detector as a Rigaku HyPix-6000HE. ## Deposited models and structure factors -All 201 PDB entries in the table are released, and RCSB reports released structure factors -(`status_code_sf = REL`) for every one of them; the six small-molecule datasets have none. A merged +All 204 PDB entries in the table are released, and RCSB reports released structure factors +(`status_code_sf = REL`) for every one of them; the eight small-molecule datasets have none. A merged result from this pipeline can therefore be checked against the deposited model or against the deposited intensities. @@ -591,6 +636,7 @@ should be read as a statement that a deposited model is wrong. | PDB | Why | |---|---| +| 7QT9 | Twenty 20° wedges (200 × 0.1°) of in-situ crystals at room temperature, none complete on its own: the dataset is a multi-sweep row (`7qt9_crystals`, below), not a single-sweep one. | | 9VX7 | The uploaded sweep does not index in the deposited cell, with `rugnux` or with DIALS or XDS given that cell, while XDS indexes it de novo in a different cell; the deposited values cannot be scored against these images, so the row is not in the battery. | ## An archive that ships placeholder images @@ -609,8 +655,10 @@ reader that globs `*.img` will pick them up, so they are named here rather than | `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition | | `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition | | `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition | +| `laspartic` | Zenodo record 10.5281/zenodo.15432050 | a small-molecule dataset, not a PDB deposition | +| `onitroaniline` | Zenodo record 10.5281/zenodo.7193538 | a small-molecule dataset, not a PDB deposition | -Five of the six small-molecule sets have a published structure to check a run against. These are +Six of the eight small-molecule sets have a published structure to check a run against. These are reference values from the literature, not results obtained here. | Dataset | Space group | Cell (A, deg) | T | Reference | @@ -620,6 +668,10 @@ reference values from the literature, not results obtained here. | `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) | | `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) | | `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) | +| `laspartic` | `P 1 21 1` (4) | 5.1237 6.9197 7.6006 / 90 100.442 90 | 150 K | twin fractions 0.537/0.463; COD 1577465 - M. Lutz, IUCrData **10** (2025) [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) | + +`onitroaniline` has no cell here: the letter describing it could not be retrieved when the row was +added, so a run on it is not scored against a reference. `cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**, 064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no @@ -632,20 +684,20 @@ symmetries rather than to be easy to process. The counts below describe where it like everything else on this page, they are metadata about the depositions and their files, not measurements. -- **Repository**, over the 207 datasets on this page (the 201 rows of the table and the six - small-molecule sets): IRRMC 95, Zenodo 42, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5, +- **Repository**, over the 212 datasets on this page (the 204 rows of the table and the eight + small-molecule sets): IRRMC 97, Zenodo 45, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5, ESRF 3, UQ eSpace 1. - **Facility** - counted from the facility part of the Facility / beamline column, the beamline - ignored so that entries deposited with and without one count the same, over the 190 rows that - name one: APS 29, Diamond 21, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9, + ignored so that entries deposited with and without one count the same, over the 191 rows that + name one: APS 29, Diamond 22, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9, SLS 8, SPring-8 8, SOLEIL 7, SSRF 7, PAL/PLS 5, CHESS 4, ALBA 3, Australian Synchrotron 3, CLSI 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, Photon Factory 2, and one each from NSRRC, - RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other eleven - rows were collected on laboratory sources: nine on rotating anodes, one on a liquid-metal jet + RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other thirteen + rows were collected on laboratory sources: eleven on rotating anodes, one on a liquid-metal jet and one on a sealed tube. -- **Crystal system, from the deposited space group of the 201 rows:** monoclinic 53, - orthorhombic 50, tetragonal 30, trigonal 22, hexagonal 20, cubic 13, triclinic 13. -- **Pink beam:** none of these datasets was collected with pink beam. All 201 rows are deposited +- **Crystal system, from the deposited space group of the 204 rows:** monoclinic 54, + orthorhombic 51, tetragonal 30, trigonal 22, hexagonal 20, cubic 14, triclinic 13. +- **Pink beam:** none of these datasets was collected with pink beam. All 204 rows are deposited as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all but 5REO, which leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` `M`); 9Q41 is the one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The @@ -654,7 +706,7 @@ measurements. - **Long cell axes:** sixteen rows have a deposited cell axis longer than 320 Å - 4P8R, 4MTK, 7D98, 8V4O, 9ZMU, 9V16, 9Z72, 4NWV, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL. -The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats +The marCCD, SMV, Bruker `.sfrm`, full-imgCIF and gzip-compressed miniCBF datasets are the reason rugnux reads those formats natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with. ## Licences diff --git a/docs/EXTERNAL_TEST_DATA_STILLS.md b/docs/EXTERNAL_TEST_DATA_STILLS.md index b68d40972..859a752a9 100644 --- a/docs/EXTERNAL_TEST_DATA_STILLS.md +++ b/docs/EXTERNAL_TEST_DATA_STILLS.md @@ -30,8 +30,8 @@ the paper of the entry. | `lcls242_s`, `lcls242_t` | [242](https://cxidb.org/id-242.html), [10.11577/3374214](https://doi.org/10.11577/3374214) | LCLS MFX | Rayonix MX340-HS; LCLS-I XTC | [9O4S](https://www.rcsb.org/structure/9O4S) (small cell), [9O4T](https://www.rcsb.org/structure/9O4T) (large cell) | P 1 21 1, 1.95 | Wolff et al., bioRxiv (2026) | | `euxfel83` | [83](https://cxidb.org/id-83.html), [10.11577/1464102](https://doi.org/10.11577/1464102) | European XFEL SPB/SFX | AGIPD 1M, 16 modules of 512 x 128, 200 µm; Cheetah CXI | [6GTH](https://www.rcsb.org/structure/6GTH) | P 32 2 1, 1.69 | Wiedorn et al., Nat. Commun. (2018) | | `euxfel152` | [152](https://cxidb.org/id-152.html), [10.11577/1660970](https://doi.org/10.11577/1660970) | European XFEL SPB/SFX | AGIPD 1M; Cheetah CXI | [6U57](https://www.rcsb.org/structure/6U57) | I 2 3, 2.80 | Echelmeier et al., Nat. Commun. (2020) | -| `alvra103`, `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) | -| `alvra213`, `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) | +| `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) | +| `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) | Every set of the manifest is in the table and every set in the table is in the manifest, with the same CXIDB entry, PDB entry, space group and dmin; `tools/battery/doc_check.py` checks @@ -63,8 +63,8 @@ broken download resumes with `curl -C -`. | `lcls242_*` | `lcls_242/xtc/`, `lcls_242/xtc_more/`: `xtc/mfxl1013721-r0256` to `r0267` streams (the two crystal forms are in every run; each set keeps its own form) | ~90,000 | **subset**: runs 256-267 (44 of about 220 stream files; ~610 GB of ~3 TB) | ~700 GB | | `euxfel83` | `euxfel_83/raw/`: `XFEL-r0243-c00.cxi`, `XFEL-r0278-c00.cxi`, `XFEL-r0280-c00.cxi` (Cheetah hits, from the entry's `cheetah-calib-mar-v1/hdf5/` tree on the NERSC archive); `euxfel_83/ref/agipd_mar18_v10.geom` from the entry | 2,790 | **subset**: 3 files; the deposited stream has 14,445 frames | 4.7 GB | | `euxfel152` | `euxfel_152/raw/`: runs r0108, 0110, 0112, 0114, 0115, 0116, 0118, 0123, 0125 extracted from `XFEL2042-runs-cxi.tar` (range requests on the tar members); `euxfel_152/ref/kdo_embedded.geom`, the geometry embedded in the header of the entry's `KDO8PS.stream` | 3,801 | **subset**: 9 runs; the deposited stream has 37,006 frames | 5.7 GB | -| `alvra103` | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB | -| `alvra213` | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB | +| `alvra103_int16` (source) | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB | +| `alvra213_int16` (source) | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB | **The masks of the SwissFEL geometry files.** rugnux takes the pixel masks and bad regions a CrystFEL geometry file names. The SwissFEL files name their mask with an absolute path on the facility's disk, @@ -78,9 +78,9 @@ that line differs. **The `_int16` sets.** The SwissFEL files store converted JUNGFRAU data as uncompressed float32 keV, 68 MB (JF16M) or 17 MB (JF4M) a frame, which caps reading at about 74 or 295 frames/s at 5 GB/s. The `_int16` sets are copies with the same layout in which `/data/data` holds whole photons (keV divided -by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller; the battery runs -them with the same arguments as the float32 originals (rugnux measures the gain from the frames), and -they merge to the same numbers. Make them with +by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller. The battery runs only the +copies: they merge to the same numbers as the float32 originals (rugnux measures the gain from the +frames), and the originals are kept only as the source of the conversion and as a test of the format. Make them with `tools/battery/still/float32_to_int16.py ` (h5py and hdf5plugin), into `swissfel_103/conv/` (4.57 keV) and `swissfel_213/conv/` (12.037 keV, keeping the tarball's directory tree), named `_int16.h5`. One MbCO dark run (`run_000238`) holds no frames and has no copy. diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md index 24a57b695..9c0e93706 100644 --- a/docs/GPU_DATA_ANALYSIS.md +++ b/docs/GPU_DATA_ANALYSIS.md @@ -166,7 +166,8 @@ the first card has no room for go on the others, each against its own budget. as a gather over bricks of the grid, bit-identical from run to run. [§14.1](CPU_DATA_ANALYSIS_DECISIONS.md#141-model-structure-factors) - **Bulk-solvent mask** (`ModelMaskGPU.cu`) - the solvent mask from every symmetry image of every - atom, with island removal and the shrink, as gemmi computes it. + atom, with island removal and the shrink, as gemmi computes it - on the density's grid, or on one of its + own at 0.75 A where the density's is too coarse for the 0.8 A shrink to act. [§14.2](CPU_DATA_ANALYSIS_DECISIONS.md#142-bulk-solvent-and-scaling) - **Structure factors and maps** (`ModelStructureFactorsGPU.cu`) - F_calc and F_mask from the two grids by cuFFT, composed over the group's operators and read off at the data's reflections; and the @@ -269,7 +270,8 @@ and both gradients come back into pinned buffers. The minimiser (L-BFGS) stays o Once per macro-cycle ([§15.3](CPU_DATA_ANALYSIS_REFINEMENT.md#153-scale-bulk-solvent-and-the-likelihood-parameters)): - **F_calc and F_mask** - the forward path above without the target, and the bulk-solvent mask drawn on the card - (`ModelMaskGPU.cu`) on **its own fine grid** (d_min/3 or 0.4 A, whichever is finer, at most 2^26 points), + (`ModelMaskGPU.cu`) on **its own fine grid** (d_min/3 or 0.4 A, whichever is finer, at most 2^26 points but never coarser than 0.75 A nor than the + density's grid), with a grid, spectrum and cuFFT plan of its own where that grid is not the density's: one block per (atom, symmetry image) sets the points within the atom's REFMAC radius, the islands are removed by a union-find labelling (integer `atomicCAS`, rooted at each component's lowest index, so the partition does not @@ -284,28 +286,45 @@ Once per macro-cycle ([§15.3](CPU_DATA_ANALYSIS_REFINEMENT.md#153-scale-bulk-so of the first round found **on the card** by a radix select (`residual_median`: the residuals are not negative, so their order is that of their bits, found a byte at a time in four passes of integer counts - the element `std::nth_element` puts at n/2). After a trial's first macro-cycle the fit starts from the fine pass around the - previous winner, skipping the coarse one. Its stream has the card's highest priority, so that the fit's short, + previous winner, skipping the coarse one. The fits go in batches of at most 32 (the coarse pass's 88 in three, in + grid order; a fit's sums do not depend on the others in its batch), and the robust weights take the residuals' + place, so the engine holds about 300 bytes per reflection. Its stream has the card's highest priority, so that the fit's short, host-waited launches do not queue behind the long kernels of the other trials on the card. - **D and Σ**, the likelihood's parameters, are fitted on the host (bin by bin, on all of the run's threads). **Trials, cards and waits.** The weight trials of one set ([§15.7](CPU_DATA_ANALYSIS_REFINEMENT.md#157-the-weights-decided-by-trial)) run side by side, each on a slot of -its own - a copy of the model, its topology, a refinement engine and a scale-fit engine - as many slots as the +its own - a copy of the model, its topology and a refinement engine - as many slots as the largest set of trials has (six at the defaults), the ones beside the main one built on their own threads while the placement runs. The slots' engines go on the cards in -turn from the main one's, each card taking engines while they fit its budget; on a machine with several cards -the trials are spread over all of them. A slot whose engine cannot be made is left out rather than moved to the +turn from the main one's, each card taking engines while they fit its budget, so the first slots are on cards of +their own and on a machine with several cards the trials are spread over all of them. The engines on one card +share what does not change - the setup worked out on the host, the reflections' tables, the adjoint's lists and +the phase table - and the bulk-solvent mask's buffers (`RefinementGPUCard`). The working buffers of a transform - +the grid, its spectrum, cuFFT's work area, the mask's own grid and spectrum - come in lanes (`RefinementGPULane`): +a slot has a lane of its own where the card holds one more, and shares the least used lane otherwise, its engine +taking the lane's lock from its launches to their wait (an evaluation, F_calc, F_mask), so the lane runs one +evaluation after another while its other slots do their host work - the minimiser, the gradients, the restraint +lists. Lanes side by side keep a card busier (two evaluations at once on a T4 cost 9% more each than one); slots +sharing a lane overlap more host work. The slots on a card share its scale-fit engine too, a fit waiting for the +one before it. A slot whose engine cannot be made is left out rather than moved to the CPU, and the outcome does not depend on how many trials run at once or on which card. The engine waits on its stream through an event made with `cudaEventBlockingSync` (`StreamWaiter`), so a waiting trial sleeps instead of spinning a core - with six engines on one card the spinning took half of the refinement's CPU time. **Memory.** `RefinementGPUEngine::DeviceBytes` counts the engine's reserve before anything is allocated: per atom -(positions, boxes, densities, mask atoms, gradients), the grid, its spectrum and cuFFT's work area, the mask's -own grid and spectrum, per reflection (indices, scales, F_o, the row model, F_calc, F_mask, coefficients, -terms), the adjoint's lists, the gather's pair reserve - sized for every atom at the largest B a refinement may -reach with the widest blur - and the restraints at about 30 records and 60 slots per atom. The engine goes on the -card when that total, plus what is already resident there (a merge still at work), is within 80% of the card's -total memory; the trial slots are counted against the same budget, card by card. +(positions, boxes, densities, mask atoms, gradients), per reflection (scales, the row model, F_calc, F_mask, +coefficients, terms), the gather's pair reserve - sized for every atom at the largest B a refinement may +reach with the widest blur - and the restraints: the records and gradient slots of the model's two restraint sets +as the refinement starts (the geometry with its repulsions and NCS restraints, and the B restraints), a quarter more +for the contacts that come and go. `RefinementGPUEngine::CardBytes` counts the part the engines on a card share +(the tables, the adjoint's lists, the phases and the mask's buffers), `LaneBytes` a lane. The engine goes on the +card when the two, the scale-fit engine and what is already resident there (a merge still at work), are within +80% of the card's total memory (with a lane); the trial slots are counted against the same budget, card by card - +each one its engine's own part and a lane where both fit, else its engine's part alone, the first one on another +card also the shared part and a scale-fit engine. On a large model (510k atoms with riding hydrogens, 3.1M +reflections to 2.1 Å) an engine's own part is 2.0 GB, a lane 1.5 GB, the shared part 1.7 GB and the scale-fit +engine 0.9 GB: a 16 GB card takes three slots (three lanes; two where 80% of it is 12.5 GB), a 24 GB card five. **Falling back.** Where the GPU cannot take the case (an atom's box wider than the cell, more than 192 mask operators), where the engine is over the budget, or where CUDA fails while the diff --git a/docs/HDF5.md b/docs/HDF5.md index e6908eab6..b93e578e3 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data. This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data` -VDS still resolves to the original images when they are available. `rugnux` can also process +VDS still resolves to the original images when they are available. Where the input is anything but a +single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no +`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. ## 3. NXmx-standard content @@ -252,6 +254,11 @@ so it is a measurement. A user handing either to XDS should know which of the tw ### `/entry/instrument/detector/module` (NXdetector_module) +Written for a detector whose image is one flat panel (e.g. an EIGER). A detector of several modules +(a JUNGFRAU image assembled by Jungfraujoch, or an input whose modules are known) is written as +`module_0`, `module_1`, ... instead (next section), with no whole-image `module` beside them: NXmx +readers such as DIALS/dxtbx take every `NXdetector_module` as one panel. + `data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). The two pixel-direction vectors carry the discrete image orientation (mirror in Y, multiples of 90° about the @@ -260,10 +267,14 @@ beam); for a detector that this system assembled itself, they are the McStas for ### `/entry/instrument/detector/module_` (NXdetector_module, one per module) -The master files written here keep the single `module` group above. Rugnux writes the placement of a -detector's modules, where a stills run placed them, into `_modules.h5`: one group per module in -the NeXus `NXdetector_module` layout that NXmx uses, and reads the same groups back (`--module-geometry`, -or a master that states its modules one by one): +A master of a multi-module detector states its modules one by one, in the order the `/entry/MX/peakPanel` +indices refer to. *n* is zero-padded to the width of the last index (`module_0` ... `module_7`, but +`module_00` ... `module_15`), so that the groups sort by name in module order and DIALS panel *n* is +module *n*; for an image Jungfraujoch assembled, each module is +its box of the image, as imaged. Rugnux writes the placement of a detector's modules, where a stills run +placed them, into `_modules.h5` in the same layout, and reads the same groups back +(`--module-geometry`, or a master that states its modules one by one). Masters written by releases up to +1.0.0-rc.181 may carry the whole-image `module` beside `module_`; they read back the same way. | Field | Units / notes | |-------|-------| @@ -397,7 +408,8 @@ variants. | `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below | | `stillsRockingWidthFormula` | 1/Å, rad, relative | stills: the run's rocking width σ² = γ₀² + (γₑ d\*)² + (σ_bw λ d\*²/2)², as `[γ₀, γₑ, σ_bw]`; offline (Rugnux) only | | `stillsRadialScale` | | stills: the scale of the reciprocal lattice the rocking curve is centred on | -| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\*, which the partialities use (held beyond the first and last entry; the formula above is what the prediction window uses); `--mode scale` merges with formula and table instead of measuring another | +| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\* before the image pass, which the frames were predicted through (held beyond the first and last entry; the formula above is what the prediction window uses); the partiality post-refinement balances it against the merge, and `--mode scale` starts from this same curve instead of measuring another | +| `stillsGivenUnitCell` | Å, deg | stills: the cell given with `-C`, whose population of frames the merge kept (`[6]`); `--mode scale` leaves out the same frames | **Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index 9e48cedb8..63847ebf1 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -186,7 +186,9 @@ The results - the merged data, the map coefficients and, with a model, the place refinement - travel to the writer in the end message, which writes them beside the master file as `.mtz`, `_maps.mtz`, `_model.cif` and `_model.pdb`, and `_refined_model.cif` (and `.pdb`), `_refined_maps.mtz` and -`_blobs.json`, as rugnux names them. An end message +`_blobs.json`, as rugnux names them. These files name both programs as the software that +made them: the MTZ files in a HISTORY record beside Rugnux's, the models in their `_software` loop +(Rugnux, and Jungfraujoch for the data collection). An end message that would be too large for the pushers' buffer (256 MiB) goes without them, with an error in the log and a warning in the live status; the collection itself is not affected. diff --git a/docs/JUNGFRAUJOCH.md b/docs/JUNGFRAUJOCH.md index 6b4a8bb6a..b2a1a6ebf 100644 --- a/docs/JUNGFRAUJOCH.md +++ b/docs/JUNGFRAUJOCH.md @@ -49,3 +49,15 @@ The broker streams images as CBOR messages compatible with the DECTRIS Stream2 p ([HDF5 / NeXus data format](HDF5.md)). Which downstream programs read which of the three HDF5 layouts, and the XDS plugin, are in [Integration with MX software](SOFTWARE_INTEGRATION.md). The command-line tools shipped beside the services are listed in [Tools](TOOLS.md). + +## How to cite + +The primary citation for Jungfraujoch is F. Leonarski et al., "Jungfraujoch: hardware-accelerated +data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30, +227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). For real-time +serial crystallography and live indexing feedback cite also J. Duan et al., "Jungfraujoch: +38 GB s⁻¹ real-time serial crystallography – data acquisition and live indexing feedback" (2026), +J. Appl. Cryst. 59, 1668-1677 [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083). +The files `jfjoch_writer` writes from Rugnux live mode name both Jungfraujoch and Rugnux; cite the +[Rugnux paper](PUBLICATIONS.md#rugnux) too when you use them. All papers are on the +[Publications](PUBLICATIONS.md) page. diff --git a/docs/PUBLICATIONS.md b/docs/PUBLICATIONS.md new file mode 100644 index 000000000..a73883ecf --- /dev/null +++ b/docs/PUBLICATIONS.md @@ -0,0 +1,64 @@ +# Publications + +## How to cite + +**Rugnux** — if you find Rugnux useful, please cite: + +F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons and XFELs in +one program" (2026), bioRxiv [doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). + +**Jungfraujoch** — the primary citation is: + +F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matěj, A. Castellane, +B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated data-acquisition +system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30, 227-234 +[doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). + +Work that relies on real-time serial crystallography, live indexing feedback or the system's +full data rate should also cite: + +J. Duan, H.-C. Stadler, J. Nan, Z. Matěj, G. Gotthard, F. Dworkowski, M. Bjelčić, O. Aurelius, +A. Mozzanica, S. Grimm, M. Burian, T. Ursby, M. Wang and F. Leonarski, "Jungfraujoch: 38 GB s⁻¹ +real-time serial crystallography – data acquisition and live indexing feedback" (2026), +J. Appl. Cryst. 59, 1668-1677 +[doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083). + +Data collected with Jungfraujoch and processed with Rugnux in the broker (Rugnux live mode) come out +of both programs, and the files `jfjoch_writer` writes from them name both: please cite both. + +## Rugnux + +- F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons and XFELs + in one program" (2026), bioRxiv + [doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). + +## Jungfraujoch + +- J. Duan, H.-C. Stadler, J. Nan, Z. Matěj, G. Gotthard, F. Dworkowski, M. Bjelčić, O. Aurelius, + A. Mozzanica, S. Grimm, M. Burian, T. Ursby, M. Wang and F. Leonarski, "Jungfraujoch: 38 GB s⁻¹ + real-time serial crystallography – data acquisition and live indexing feedback" (2026), + J. Appl. Cryst. 59, 1668-1677 + [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083) — the system at its + full data rate, with live indexing feedback during serial data collection. +- P. Gasparotto, L. Barba, H.-C. Stadler, G. Assmann, H. Mendonça, A. W. Ashton, M. Janousch, + F. Leonarski and B. Béjar, "TORO Indexer: a PyTorch-based indexing algorithm for kilohertz serial + crystallography" (2024), J. Appl. Cryst. 57, 931-944 + [doi:10.1107/S1600576724003182](https://doi.org/10.1107/S1600576724003182) — the GPU indexing + algorithm behind Jungfraujoch's real-time indexing. +- F. Leonarski, J. Nan, Z. Matěj, Q. Bertrand, A. Furrer, I. Gorgisyan, M. Bjelčić, M. Kepa, + H. Glover, V. Hinger, T. Eriksson, A. Cehovin, M. Eguiraun, P. Gasparotto, A. Mozzanica, + T. Weinert, A. Gonzalez, J. Standfuss, M. Wang, T. Ursby and F. Dworkowski, "Kilohertz serial + crystallography with the JUNGFRAU detector at a fourth-generation synchrotron source" (2023), + IUCrJ 10, 729-737 [doi:10.1107/S2052252523008618](https://doi.org/10.1107/S2052252523008618) — + serial crystallography at 2 kHz with a JUNGFRAU 4M and Jungfraujoch at the BioMAX beamline of + MAX IV. +- F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matěj, A. Castellane, + B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated + data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. + 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268) — the + primary citation: the FPGA-accelerated acquisition system. +- F. Leonarski, A. Mozzanica, M. Brückner, C. Lopez-Cuenca, S. Redford, L. Sala, A. Babic, + H. Billich, O. Bunk, B. Schmitt and M. Wang, "JUNGFRAU detector for brighter x-ray sources: + Solutions for IT and data science challenges in macromolecular crystallography" (2020), + Struct. Dyn. 7, 014305 [doi:10.1063/1.5143480](https://doi.org/10.1063/1.5143480) — the data + handling problem of the JUNGFRAU detector that Jungfraujoch was designed to solve. diff --git a/docs/RUGNUX.md b/docs/RUGNUX.md index 07aea9754..8a816a105 100644 --- a/docs/RUGNUX.md +++ b/docs/RUGNUX.md @@ -39,8 +39,8 @@ the whole sweep gives. This is for rotation data and `--mode mx`; the collection is ignored. > **Note.** `rugnux` is under very active development. This page describes the tool and -> its options at a high level; the authoritative, always-current list of options is the program's -> own usage message — run `rugnux` with no arguments. +> its options at a high level. Run `rugnux` with no arguments for its usage - how to run each mode +> and the basic options; every option is on [Command-line options](RUGNUX_CLI.md). ```{contents} On this page :local: @@ -56,7 +56,7 @@ the data, and scaling and merging run by default. `-o` names the output files (d **We expect no need to tune anything else.** The defaults are meant to be the answer, and a run that needed a flag to do well is one we want to hear about. The other options exist for unusual cases and -for diagnosis; they are listed under [Command-line options](RUGNUX_ADVANCED.md#command-line-options). +for diagnosis; they are listed under [Command-line options](RUGNUX_CLI.md). ### Rotation data @@ -99,8 +99,10 @@ rugnux -o myrun --model model.pdb dataset_master.h5 # or with --model A model is told from the images by its contents (PDB, or mmCIF with atoms), gzipped or not. After merging it is **placed** — as a rigid body, or by molecular replacement where that does not place it or the model is a prediction (AlphaFold, ModelCIF) — then **refined** against the merge (restrained, -maximum likelihood; seconds on a GPU, typically minutes on the CPU; `--no-refine` skips it), and the -run writes the maps (σA-weighted 2mFo−DFc / mFo−DFc), the placed and refined model, and +maximum likelihood; seconds on a GPU, typically minutes on the CPU; `--no-refine` skips it) with +its solvent as given; `--refine-solvent` (off by default for now, also with `--mode refine`) also rebuilds +the solvent - waters added at difference peaks and removed from negative density, ions relabelled +where their surroundings say so, each site audited - and the run writes the maps (σA-weighted 2mFo−DFc / mFo−DFc), the placed and refined model, and the blobs of difference density the model does not explain. R-work / R-free are in the report. The model **does not change processing decisions**: the merge, its space group and resolution are @@ -121,7 +123,7 @@ myrun.mtz merged intensities + French-Wilson amplitudes, for CCP4 / p myrun.cif the same, as mmCIF - the self-describing format, and what to deposit myrun.hkl the scaled observations unmerged, as SHELX HKLF 4 - for SHELXL, SHELXC / ANODE myrun.ins the SHELX instructions to go with it: CELL, ZERR (the cell's e.s.d.s), LATT, SYMM -myrun_unmerged.mtz every observation before scaling, for pointless / aimless / careless - +myrun_unmerged.mtz the observations unmerged, scaled by the merge, for pointless / aimless - the largest file of the run (--no-export-unmerged skips it) myrun_report.txt what the run determined: cell, space group, statistics, warnings myrun_plot.txt one row per image, for plotting how the crystal behaved over the sweep @@ -137,6 +139,7 @@ myrun_model.cif / .pdb the model as placed, in the cell and space group of my myrun_refined_model.cif the refined model (and .pdb) myrun_refined_maps.mtz map coefficients on the refined model myrun_blobs.json the unexplained difference-density blobs +myrun_solvent.json with --refine-solvent: the refined model's waters and ions, each scored by its surroundings, and the waters the build added and removed ``` A stills run writes the same merged files (its `.hkl` holds the merged reflections) and, on a @@ -246,7 +249,7 @@ A few things worth knowing before reaching for more flags: with `Killed` ran out of memory ([When host memory runs out](RUGNUX_INSTALL.md#when-host-memory-runs-out)). With any question about a run, send `myrun_report.txt` and `myrun_log.txt`. - Everything else is in [Running Rugnux](RUGNUX_TUTORIAL.md#running-rugnux) and the full - [Command-line options](RUGNUX_ADVANCED.md#command-line-options). + [Command-line options](RUGNUX_CLI.md). ## The rest of the manual @@ -269,8 +272,10 @@ One page per job, so the answer needed is near the top of a short page: Phaser, SHELXC/D/E and, for small molecules, SHELXT / SHELXL. - [The results report](RUGNUX_REPORT.md) — the `KEY= value` interface, sweep quality and the anisotropy section. +- [Command-line options](RUGNUX_CLI.md) — how to run each mode, and every option with what it + does. - [Advanced usage](RUGNUX_ADVANCED.md) — reference data and the indexing ambiguity, model - validation, re-merging, and the full command-line option tables. + validation and re-merging. - [Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md) — every model and reference option, and decision by decision what a model can and cannot change. - [Detector calibration](RUGNUX_CALIBRATION.md) — the geometry from a calibrant's powder rings @@ -290,3 +295,11 @@ One page per job, so the answer needed is near the top of a short page: Use `rugnux` to re-analyse data after acquisition, to experiment with processing parameters, or to produce merged intensities for downstream structure solution. + +## How to cite + +If you find Rugnux useful, please cite F. Leonarski and M. Wang, "Rugnux: from diffraction images to +structure at synchrotrons and XFELs in one program" (2026), bioRxiv +[doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). Rugnux prints this at +startup and at the head of its report; the other papers are on the [Publications](PUBLICATIONS.md) +page. diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md index 0981c315b..ee006c39a 100644 --- a/docs/RUGNUX_ADVANCED.md +++ b/docs/RUGNUX_ADVANCED.md @@ -1,7 +1,7 @@ # Advanced Rugnux The machinery behind a run that needs more than the defaults: external reference data, model -validation, re-merging without re-integration, and the full option tables. +validation and re-merging without re-integration; the option tables are on [Command-line options](RUGNUX_CLI.md). ```{contents} On this page :local: @@ -46,10 +46,10 @@ against a foreign dataset injects that dataset's systematics — so `-z` never p errors into the intensities. `--reference-column` picks the column to read where the automatic choice (`F-model`, else `IMEAN`/`I`, else `FP`/`FOBS`/`F`) is not the right one. -For the indexing ambiguity an atomic model does as well: `--model` computes the intensities it -needs from the structure (a model that fits also sets the hand label and the setting, see -[Validating against a model](#validating-against-a-model-rugnux---model)). Where a reference dataset exists, prefer it; -where only a model does, it resolves the ambiguity just the same. +An atomic model does not take part in the processing: `--model` is placed and validated after the merge, +and a run with one merges exactly what a run without one does. A model that fits then sets the frame the +merge is written in - the hand label, the setting and the indexing (see +[Validating against a model](#validating-against-a-model-rugnux---model)) - by relabelling it, never by re-merging. ### The indexing ambiguity @@ -102,16 +102,15 @@ How to resolve it: | **Rotation**, a reference dataset exists | `-z reference.mtz`. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the *hkl* labels change; the cell does not | | **Serial stills**, a reference dataset exists | `-z reference.mtz`. Resolved **per image**, at integration time, by correlating each crystal's intensities with the reference — so the merge never mixes hands in the first place. Where the crystals' own cells tell the indexings apart (pseudo-merohedry), a crystal its cell decides keeps that indexing and the reference decides the rest. The data-only resolution below runs as a check: `AMBIGUITY_DATA_DISAGREES=` | | **Rotation**, only a model | `--model model.pdb`. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (`MODEL_FIT= ACCEPTED`) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (`MODEL_INDEXING_MARGIN_SIGMA`). It is the **data** that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model's indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand | -| **Serial stills**, only a model | `--model model.pdb`, with the cell and space group given (`-C` / `-S`). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be, the crystals their own cell decides again excepted. The data-only resolution below is then run only as a check: `AMBIGUITY_DATA_DISAGREES=` counts the crystals on which it would have chosen otherwise | -| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A model or reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing | +| **Serial stills**, only a model, space group given (`-S`) | The crystals are made consistent with each other from the data alone, as in the row below - the model plays no part in the processing. Validation then fits the merge to the model in each candidate indexing, as for rotation data, and relabels the written reflections into the model's indexing where it wins | +| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing | | Neither | The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame; for stills whose space group the run determined, a merge of mixed indexings, which looks like a perfect twin and can pass for higher symmetry than the crystal has. Run again with `-S` | Two things to know about where the choice lands: -- **`--mode scale` cannot repair a stills run after the fact.** The per-image test happens at - integration time, so a `_process.h5` whose images were integrated without a reference — an MTZ or a - model — has already lost the distinction, and no re-merge brings it back; the data-only resolution - runs in the full pipeline, not in `--mode scale`. On rotation data, where +- **`--mode scale` resolves it as the full run does.** On stills with no reference MTZ it makes the + crystals consistent from the data alone before merging; the per-image test against a reference + happens at integration time and is kept in the stored reflections. On rotation data, where the ambiguity is one choice for the whole dataset, `--mode scale --model model.pdb` does resolve it. - **The choice reaches the written reflections**, not only the R-factors and the maps: the merged `.mtz` / `.cif` / `.hkl` (and `_unmerged.mtz`, where it is asked for) are written in the indexing @@ -291,9 +290,13 @@ declines it and the run stops with a message saying so. `--model` works here as that the model is placed by the rigid body only: a model it does not place, or a prediction, is not searched for by molecular replacement, `--mr` is refused, and the placed model is not refined. On rotation data, where the [indexing ambiguity](#the-indexing-ambiguity) is one choice for the whole dataset, it can settle the -enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. What no re-merge can -repair is a **stills** `_process.h5` integrated without a reference: there the ambiguity was -decided per image, at integration time, and the distinction is gone from the stored reflections. +enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. On +stills the re-merge is the full run's own merge: the same frames (a given cell's population, as +`/entry/MX/stillsGivenUnitCell` records it), the same rocking curve, the same data-only resolution of +the indexing ambiguity and the same scaling, so a +`_process.h5` merged again with the options it was written with reproduces the full run's reflections +(with a reference MTZ, the full run also lets the crystals' own cells overrule the reference's indexing +before merging, which `--mode scale` does not). Where the full run re-seated the lattice — the space group it settled on is in a different setting from the one each image was indexed in — the file records the change of basis as @@ -312,10 +315,10 @@ describes them - the cell and space group; `_software` with Rugnux named twice, beamline and its general class, the detector's make and model in `_diffrn_detector.type` and its class in `_diffrn_detector.detector`, and the collection date from the NXmx start time); and the merging statistics overall (`_reflns`) and per shell (`_reflns_shell`): completeness, multiplicity, -, R_merge, R_meas, R_pim, CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries +, R_merge, R_meas, R_pim (`?` on a stills merge, where they mean nothing), CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries its e.s.d.s (`_cell.length_a_esd`, …). A quantity the input does not state is left out rather than guessed. The unmerged observations are in -the `_unmerged.mtz` (unscaled) and the `.hkl` (scaled), not in the `.cif`. +the `_unmerged.mtz` and the `.hkl` (both scaled on a rotation merge), not in the `.cif`. Items PDBx does not define — ISa, the error model, the twinning and anisotropy diagnostics, the refined distance and beam centre — are written under the `jfjoch_` local prefix, which a dictionary @@ -338,204 +341,4 @@ the broker is not a file here, and has none. ## Command-line options -General: - -| Option | Description | -| --- | --- | -| `-o, --output-prefix ` | Output file prefix (default: `output`) | -| `-N, --threads ` | Number of worker threads (default, and for any value ≤ 0: every CPU the run may use - its affinity and cgroup quota). Some stages take fewer, because past a point more workers make them slower: the per-image loop of `--mode mx` uses at most 16 per GPU unless `-N` was given a positive value, and first-pass spot finding and the beam-stop pre-scan have ceilings of their own that `-N` does not lift. On a small card the two ceilings, and the per-image loop's 16 where `-N` was left at its default, come down to as many workers as keep their buffers (about 24 bytes per detector pixel each) within half of the card's total memory. Scaling, merging and the space-group search use the full count | -| `-s, --start-image ` | First image to process (default: 0) | -| `-e, --end-image ` | End image number, exclusive: `-s 0 -e 100` processes images 0-99 (default: all) | -| `-t, --stride ` | Process every *n*-th image (default: 1) | -| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `_log.txt` is written either way | -| `--simulate-live ` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only | -| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse | -| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) | - -Mode — `--mode ` (default `mx`): - -| Value | Description | -| --- | --- | -| `mx` | Full analysis — spot finding, indexing, integration and merging | -| `azint` | Only azimuthal integration (no spot finding/indexing); writes `_process.h5` | -| `scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) | -| `calibration` | Determine the detector geometry from powder rings; writes `.poni` and `.json` | -| `refine` | Refine the `--model` against merged structure factors (an MTZ or a structure-factor mmCIF) and process nothing; see [Refining a model against structure factors](RUGNUX_MODEL_AND_REFERENCE.md#refining-a-model-against-structure-factors-rugnux---mode-refine) (`--refine-score-only`, `--refine-d-min`) | - -Refinement (`--mode refine`): the columns are found, not named - amplitudes with their sigmas (F/SIGF, -FP/SIGFP, ...) where the file has them, else intensities (IMEAN/SIGIMEAN, I/SIGI, ...) made into amplitudes by -French-Wilson, else the mean of the anomalous pairs; the free set from a FreeR_flag-like column (0 the test set, -or the minority value where 0 is the majority, as phenix writes it), drawn by rugnux where the file has none. -The model is taken as placed in the data's cell, and refined in the data's space group. - -| Option | Description | -| --- | --- | -| `--refine-score-only` | Score the model as given and refine nothing: R-work/R-free, the scale, the bulk solvent, the likelihood and the geometry, each term under `REFINE_SCORE_` (`--developer` prints them all) | -| `--refine-d-min ` | Leave out reflections finer than this (Å) | - -Calibration (`--mode calibration`): - -| Option | Description | -| --- | --- | -| `--calibrant ` | Powder standard: `lab6` \| `agbh` \| `ceo2` \| `si` \| `ice` (default `lab6`, case-insensitive). A cell given with `-C` overrides it: that cell is the standard, its rings enumerated from the cell, with the absences of `-S` where given | -| `--calibration ` | How the rings are measured: `rings` \| `spots` (default `rings`; see above). `rings` defaults `--azim-phi-bins` to 32 | -| `--no-refine-tilt` | Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance, for a calibration handed to a program that cannot express a tilted detector (XDS). On stills it holds the tilt in the geometry refinement too | - -Detector mask: - -| Option | Description | -| --- | --- | -| `--detect-beam-stop[=N\|off]` | Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. **On by default** (60 images); `=off` disables. A reflection whose signal disk reaches into the shadow is not integrated: the rest of such a disk lies in the stop's half-shadow and reads low. Pixels the finder reports that hold the detector's no-data mark (a PILATUS gap) are masked as dead instead | - -Geometry: - -| Option | Description | -| --- | --- | -| `--beam-center-check[=off]` | Measure the beam centre from the isotropy of the scattered background on **every** run, report how far the file's value is from it (against how right this geometry needs it to be), and on rotation data index a **second first pass** at the measured centre. The fit reads the projection `--detect-beam-stop` already builds, so it costs no extra frames. **On by default**; `=off` disables it, and with it the measured centre's part in the post-refinement bound. The measured centre is adopted where the file's centre indexes nothing and the measured one indexes a majority; where the two return cells related by an integer volume factor (2 to 4) and the measured centre's cell, larger or smaller, carries materially more of the pooled validation spots against its own chance level; and, on a two-pass run, where the first pass is run at both centres and the measured one merges better — done where the two give the same cell in different metric symmetries, where they agree only once less of each frame is read, and where they agree but lie further apart than the geometry absorbs and the file's centre merges inconsistently in the lowest-resolution shell. Otherwise the file's centre is kept. On stills, where there is no first pass, the file's centre, the measured one and the point-symmetry capture's are each indexed on a sample of frames and the one putting materially more spots on the lattice is taken; with `-C` the detector distance is searched too (30-3000 mm, the file's value one hypothesis). Still runs when `--beam-x`/`--beam-y` are given. See [§1.4](CPU_DATA_ANALYSIS_IMAGE.md) | - -Spot finding: - -| Option | Description | -| --- | --- | -| `--spot-sigma ` | Noise sigma level for spot finding (default: 4.0) | -| `--spot-threshold ` | Photon-count threshold for spot finding (default: 10) | -| `--adaptive-spots` | Self-calibrating detection (**default**, stills and rotation alike): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed `--spot-threshold`, so one setting adapts across datasets (no per-dataset `--spot-threshold`/`--spot-sigma` tuning) | -| `--no-adaptive-spots` | Turn adaptive detection off and use the fixed `--spot-threshold` / `--spot-sigma` finder | -| `--spot-false-pixels ` | Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies `--adaptive-spots`) | -| `--spot-high-resolution ` | High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills | -| `--spot-low-resolution ` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit) | -| `--min-pix-per-spot ` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is scored at min-pix 2/3/1 (one lattice search per frame, refined on each list) and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. | -| `--max-spots ` | Maximum spots kept per image (the strongest ones) and handed to indexing. **If omitted, the budget is measured** on rotation data: the first pass reads how deep into an image's spot list its spots still lie on the lattice it found, and the run keeps that many (never more than 1000). Give a value to pin it. Stills always use the fixed 1000. | -| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** | - -Azimuthal integration (the radial profile behind the per-image ice-ring score). Every *q* here is -*q* = 2π/*d*, in Å⁻¹: - -| Option | Description | -| --- | --- | -| `-q, --azim-q-spacing ` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) | -| `--azim-min-q ` | Minimum Q, 1/Å | -| `--azim-max-q ` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see | -| `--azim-phi-bins ` | Number of azimuthal (phi) bins (default: 1) | -| `--polarization-correction ` | Enable/disable the azimuthal polarization correction | -| `--solid-angle-correction ` | Enable/disable the azimuthal solid-angle correction | - -Indexing: - -A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation -indexing) by default; a dataset without one is processed as independent stills. `--force-still` -overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request -rotation explicitly and pick the pass or lattice. - -The FFT search looks for cell axes between 10 Å (`--fft-min-unit-cell`) and a default **longest -axis of 500 Å**, which has no flag of its own — a reference cell (`-C`) moves both bounds to cover -the cell it names. A de-novo rotation run also tries a second first-pass hypothesis with the short -end lowered to 5 Å, so a small-molecule cell below the 10 Å floor is indexed on its true axes -rather than as a supercell of them; the standard pass's answer stands unless that evidence takes it. An axis beyond the search's reach is **not refused**: the run returns a -plausible shorter sub-cell or an axis harmonic and processes it happily, so a cell that comes out -at a half or a third of the expected long axis should be read as this limit, not as the crystal. -For very long axes the direction grid's angular resolution binds well below 500 Å — see the -[analysis reference](CPU_DATA_ANALYSIS_INDEXING.md#5-fft-indexing-unknown-unit-cell) on FFT indexing. - -| Option | Description | -| --- | --- | -| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | -| `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each; `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise; rotation data always use the FFT search | -| `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it | -| `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered | -| `--min-indexed-spots ` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it | -| `-S, --space-group ` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](#the-indexing-ambiguity)) | -| `-r, --geom-refine ` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) | -| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) | -| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) | -| `--force-rotation-lattice ` | Force rotation lattice (9 floats, Å), skipping the first pass | -| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see the rotation section) | -| `--prepass-fraction ` | Rotation: the two-pass geometry pre-pass integrates the first fraction `f` of the sweep, but at least 120° of it (0.05-1, default **0.5**; `1` integrates the whole sweep twice). The second pass always integrates every frame. The pre-scan and the first-pass indexing sample the same part of the sweep | -| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), a stationary goniometer axis counting as stills, `=off` disables | -| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) | - -Indexer choice in brief: `ffbidx` (GPU or CPU) refines toward a **known cell** and is best for sparse -serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the -[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS_INDEXING.md) for the algorithms. - -Scaling and merging: - -| Option | Description | -| --- | --- | -| `--no-merge` | Skip scaling and merging, which are on by default; the per-image `_process.h5` is then written instead of the merged files (`_unmerged.mtz` and the results report are still written) | -| `-A, --anomalous` | Anomalous mode: merge each Bijvoet hand as its own unique reflection, so multiplicity, completeness, ⟨I/σ⟩ and the error model are counted anomalously. A default rotation merge already writes `I(+)`/`I(-)` (see [Reflection-file conventions](RUGNUX_INTEGRATION.md#reflection-file-conventions)); `-A` changes the counting basis, not whether the anomalous signal is in the file | -| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills | -| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) | -| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below) | -| `--relative-b[=deg]` | rot3d: fit a per-batch relative-*B* beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise) | -| `--simple-stills` | Stills: treat every reflection as a full (*p* = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement | -| `--no-expected-variance-merge` | Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting | -| `--capture-uncertainty ` | Systematic sigma num·(1−captured fraction)·⟨I⟩: on rot3d fulls the share of the rocking curve not captured, on stills the share of each partial its partiality does not see (default: 1.0 for rotation, 0 otherwise) | -| `--min-captured-fraction ` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) | -| `--scaling-high-resolution ` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) | -| `--scaling-low-resolution ` | Low-resolution limit of the scale fit, Å (default: 50, which is also XDS's own default; 0 removes the limit). Reflections coarser than it are merged and written but kept out of every fit of the scale model: on a large cell a few dozen of them, measured on a background the beam stop depresses, bent the model enough to move the merged 10–50 Å intensities by tens of percent. Which of them are measured at all is the beam-stop mask's decision - a reflection whose signal disk reaches the shadow is not integrated (`--detect-beam-stop`) | -| `--resolution-cutoff ` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) | -| `--resolution-cc-target ` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) | -| `--resolution-shells ` | Number of resolution shells in the reported statistics table (default: 9). The bins are equal steps in 1/d² between the lowest- and highest-resolution reflection merged, which is XDS's rule, and 9 is XDS's count — so at the same resolution limits the two tables have the same shells and can be read row for row | -| `--report-resolution [,]` | Also report the merging statistics over this resolution range, Å (either order; `dmax` defaults to the run's own low-resolution limit), as a second table beside the run's own — the `REFRES_*` keys of the [results report](RUGNUX_REPORT.md#the-reference-range-table) — binned from the same merged reflections, so a run can be read against another program's table at that program's range. Report-only: the cut, the scaling, the error model and every decision stay the run's own, and shells finer than the run's own limit are reported as not merged rather than filled from data the run did not keep | -| `--min-partiality ` | Minimum partiality to accept a reflection (default: 0.02 for rotation, 0.5 for stills) | -| `--ice-min-score ` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness | -| `--ice-min-spot-ratio ` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) | -| `--reject-outliers ` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rotation, 10 for stills; 0 = off) | -| `--min-image-cc ` | Per-image CC limit, percent (default: no limit) | -| `--search-min-zeta ` | De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry \|ζ\| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences | -| `--mosaicity ` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed | -| `--scaling-iterations ` | Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle, and a merge that hits the cap is flagged `SCALING_NOT_CONVERGED` (default: 100) | -| `-z, --reference ` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](#the-indexing-ambiguity), hands over the R-free set and reports CCref. Not a scale anchor | -| `--reference-column