mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 89 Wiki Activity

v1.0.0-rc.181 #92

Merged
leonarski_f merged 330 commits from rc181 into main 2026-11-02 12:05:21 +01:00
Conversation 0 Commits 330 Files Changed 266
+30502 -3716
266 changed files with 30502 additions and 3716 deletions
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitea/workflows/build_and_test.yml
+109 -63
View File
@@ -49,6 +49,68 @@ on:
default: 'manual'
jobs:
documentation:
name: Build documentation
# Uploads to the release, so it waits for the job that creates it. First in the file, so that
# it is handed out ahead of the long Linux builds: build-windows and build-macos wait for it.
needs: create-release
runs-on: jfjoch_rocky8
container:
image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609
steps:
- uses: actions/checkout@v4
- name: Build documentation
shell: bash
run: bash make_doc.sh
# Built once here for the Windows and macOS rugnux packages too (the Windows runner has no
# Python new enough for Sphinx). The sphinx-build doctree cache in public/.doctrees is not shipped.
- name: Upload the HTML for the Windows and macOS packages
uses: actions/upload-artifact@v3
with:
name: docs-html
path: |
public/
!public/.doctrees/
# The Rugnux pages site is the gitea-pages branch of rugnux/gitea-pages, updated with a release.
# The HTML is pushed from a copy, so the checkout the release upload below works on is left as it was.
- name: Upload documentation to rugnux/gitea-pages
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
shell: bash
env:
PAGES_TOKEN: ${{ secrets.RUGNUX_PAGES_TOKEN || secrets.PIP_REPOSITORY_API_TOKEN }}
PAGES_TOKEN_NAME: ${{ secrets.RUGNUX_PAGES_TOKEN && 'RUGNUX_PAGES_TOKEN' || 'PIP_REPOSITORY_API_TOKEN' }}
run: |
echo "Pushing with secret ${PAGES_TOKEN_NAME}"
rm -rf /tmp/rugnux-pages
mkdir /tmp/rugnux-pages
cp -a public/. /tmp/rugnux-pages/
cd /tmp/rugnux-pages
git init -q
git checkout -q -b gitea-pages
git config user.email "filip.leonarski@psi.ch"
git config user.name "Filip Leonarski (Gitea)"
git add .
git commit -q -m "Deploy site from ${GITHUB_REF_NAME} ${GITHUB_SHA::9}"
git push -f "https://${PAGES_TOKEN}@gitea.psi.ch/rugnux/gitea-pages.git" gitea-pages
- name: Upload documentation
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
shell: bash
run: |
git config --global user.email "filip.leonarski@psi.ch"
git config --global user.name "Filip Leonarski (Gitea)"
git checkout --orphan gitea-pages
git reset --hard
# Remove everything except .git and public
git clean -fdx -e public -e .git
# Move public contents to root and remove the now-empty public dir
shopt -s dotglob nullglob
mv public/* .
rm -rf public
git add .
git commit -m "Deploy site"
git push -f https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git gitea-pages
# ---- Portable Linux artifacts: viewer, rugnux, XDS plugin -----------------------------------------
# One viewer-only build per variant packs three tarballs, one program each (see CPack in
# CMakeLists.txt): the viewer build compiles everything rugnux and the plugin need anyway.
@@ -92,19 +154,17 @@ jobs:
run: |
cd build
ninja -j16
ninja rugnux_docs # the HTML documentation, shipped in the rugnux tarball
cpack
- name: Verify one libjpeg in the viewer
- name: Verify no Qt JPEG/TIFF plugin in the viewer
shell: bash
run: |
set -euo pipefail
# With the static Qt, Qt's JPEG plugin brings its own libjpeg-turbo (another ABI) and the
# linker silently mixes it with ours (see viewer/CMakeLists.txt). Files, not pipes into
# `grep -q`, for the SIGPIPE reason given in the rugnux check below.
# The viewer excludes them (see viewer/CMakeLists.txt). Files, not pipes into `grep -q`,
# for the SIGPIPE reason given in the rugnux check below.
strings build/viewer/jfjoch_viewer > /tmp/jfjv.strings
if grep -qE 'QJpegHandler|QTiffHandler' /tmp/jfjv.strings; then echo "a Qt JPEG/TIFF plugin is linked"; exit 1; fi
grep -c 'libjpeg-turbo version' /tmp/jfjv.strings > /tmp/jfjv.njpeg || true
if [ "$(cat /tmp/jfjv.njpeg)" != 1 ]; then echo "expected one libjpeg-turbo, found $(cat /tmp/jfjv.njpeg)"; exit 1; fi
echo "one libjpeg-turbo, no Qt JPEG/TIFF plugin"
echo "no Qt JPEG/TIFF plugin"
- name: Verify rugnux is self-contained
shell: bash
run: |
@@ -112,6 +172,7 @@ jobs:
rm -rf /tmp/rgx && mkdir -p /tmp/rgx
tar xf build/rugnux-*.tar.gz -C /tmp/rgx
bin=$(find /tmp/rgx -name rugnux -type f)
test -f "$(dirname "$bin")/../share/doc/jfjoch_rugnux/html/RUGNUX_CLI.html"
# Each producer writes to a file and the greps read files, rather than piping into
# `grep -q`: grep exits at the first match, the producer then dies of SIGPIPE, and under
# `set -o pipefail` that becomes the pipeline's status -- which a leading `!` would turn
@@ -120,6 +181,13 @@ jobs:
if grep -q cufft /tmp/rgx.ldd; then echo "cuFFT is linked dynamically"; exit 1; fi
find /tmp/rgx -name 'libcufft*' > /tmp/rgx.cufft
if [ -s /tmp/rgx.cufft ]; then echo "a libcufft is shipped in the tarball"; exit 1; fi
# Nothing but glibc and the GCC runtime: every other library (zstd, zlib, HDF5, ...) is built
# and linked statically, and a host copy found by some dependency's configure is a second,
# different version resolved at runtime.
awk '/=>|ld-linux/ {print $1}' /tmp/rgx.ldd > /tmp/rgx.libs
grep -vE '^(/lib64/)?(linux-vdso|ld-linux-x86-64|libc|libm|libmvec|libdl|libpthread|librt|libstdc\+\+|libgcc_s)\.so' \
/tmp/rgx.libs > /tmp/rgx.extra || true
if [ -s /tmp/rgx.extra ]; then echo "rugnux links non-system libraries:"; cat /tmp/rgx.extra; exit 1; fi
echo "self-contained: static cuFFT, nothing to ship beside it"
- name: Verify the XDS plugin exports
shell: bash
@@ -164,7 +232,7 @@ jobs:
build-windows:
name: build:portable:windows-x86_64:${{ matrix.variant }}
# Uploads to the release, so it waits for the job that creates it.
needs: create-release
needs: [create-release, documentation]
runs-on: windows-11-cuda-qt
timeout-minutes: 120
strategy:
@@ -202,6 +270,14 @@ jobs:
run: |
cd build\tests
jfjoch_portable_test.exe "[portable]"
# The HTML documentation for the rugnux package, built by the documentation job, where the
# OPTIONAL install rule for rugnux_docs looks for it. download-artifact comes from the
# mx/download-artifact mirror (the actions org has none); v4 refuses any server but github.com.
- name: Download the HTML documentation
uses: https://gitea.psi.ch/mx/download-artifact@v3
with:
name: docs-html
path: build/docs_html
- name: Build installer (NSIS) and rugnux zip
shell: cmd
run: |
@@ -219,7 +295,7 @@ jobs:
$names = [System.IO.Compression.ZipFile]::OpenRead($zip[0].FullName).Entries | ForEach-Object { $_.FullName }
$names
if (-not ($names -match 'rugnux\.exe$')) { throw 'rugnux.exe missing from the zip' }
if ($names -match 'jfjoch_viewer') { throw 'the viewer is in the rugnux zip' }
if ($names -match 'jfjoch_viewer\.exe$') { throw 'the viewer is in the rugnux zip' }
if ('${{ matrix.use_cuda }}' -eq 'ON' -and -not ($names -match 'cufft64_.*\.dll$')) { throw 'cuFFT DLL missing from the CUDA zip' }
- name: Upload installer and rugnux zip to release
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
@@ -239,7 +315,7 @@ jobs:
build-macos:
name: build:portable:macos-arm64:nocuda
# Uploads to the release, so it waits for the job that creates it.
needs: create-release
needs: [create-release, documentation]
# A Mac mini running act_runner in host mode (no containers on macOS): the job runs as the runner's
# user with that user's tools - Xcode command line tools, CMake.app and Qt under ~/Qt - so the
# paths below are this machine's, like the Windows runner's C:/Qt.
@@ -288,6 +364,14 @@ jobs:
run: cd build-macos/tests && ./jfjoch_portable_test "[Metal]"
# One build, two artifacts: the viewer .dmg and rugnux as .tar.gz (CPACK_PROJECT_CONFIG_FILE
# in CMakeLists.txt gives each generator its components). The XDS plugin is Linux-only.
# The HTML documentation for the rugnux package, built by the documentation job, where the
# OPTIONAL install rule for rugnux_docs looks for it. download-artifact comes from the
# mx/download-artifact mirror (the actions org has none); v4 refuses any server but github.com.
- name: Download the HTML documentation
uses: https://gitea.psi.ch/mx/download-artifact@v3
with:
name: docs-html
path: build-macos/docs_html
- name: Build viewer DMG and rugnux tgz
shell: bash
run: |
@@ -415,7 +499,7 @@ jobs:
file "$bin" > /tmp/tc.bin
cat /tmp/tc.bin
if ! grep -q "ARM aarch64" /tmp/tc.bin; then echo "not an aarch64 binary"; exit 1; fi
# nothing x86 may have leaked in (libjpeg-turbo's ExternalProject is the classic culprit)
# nothing x86 may have leaked in (an ExternalProject not told the target is the classic culprit)
find /tmp/tgzcheck -type f -exec file {} + > /tmp/tc.arch
if grep -q "x86-64" /tmp/tc.arch; then echo "an x86-64 file leaked into the tarball"; exit 1; fi
# cuFFT must be linked statically: nothing to ship beside the binary, nothing to find
@@ -754,6 +838,20 @@ jobs:
cd build
ninja frontend
ninja -j16 package
- name: Verify the services link no non-system libraries
shell: bash
run: |
set -euo pipefail
# As for rugnux in build-portable-tgz: glibc and the GCC runtime only, plus cuFFT from the
# CUDA toolkit in a CUDA build. Files, not pipes into grep, for the SIGPIPE reason given there.
for bin in build/broker/jfjoch_broker build/rugnux/rugnux build/writer/jfjoch_writer; do
ldd "$bin" > /tmp/svc.ldd 2>&1 || true
awk '/=>|ld-linux/ {print $1}' /tmp/svc.ldd > /tmp/svc.libs
grep -vE '^(/lib64/)?(linux-vdso|ld-linux-x86-64|libc|libm|libmvec|libdl|libpthread|librt|libstdc\+\+|libgcc_s|libcufft)\.so' \
/tmp/svc.libs > /tmp/svc.extra || true
if [ -s /tmp/svc.extra ]; then echo "$bin links non-system libraries:"; cat /tmp/svc.extra; exit 1; fi
done
echo "only system libraries linked"
- name: Upload packages
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
shell: bash
@@ -937,58 +1035,6 @@ jobs:
twine upload -u __token__ -p ${{ secrets.CI_PYPI_TOKEN }} --skip-existing dist/*
twine upload -u __token__ -p ${{ secrets.PIP_REPOSITORY_API_TOKEN }} --repository-url https://gitea.psi.ch/api/packages/mx/pypi dist/*
documentation:
name: Build documentation
# Uploads to the release, so it waits for the job that creates it.
needs: create-release
runs-on: jfjoch_rocky8
container:
image: gitea.psi.ch/leonarski_f/jfjoch_rocky8:2609
steps:
- uses: actions/checkout@v4
- name: Build documentation
shell: bash
run: bash make_doc.sh
# The Rugnux pages site is the gitea-pages branch of rugnux/gitea-pages, updated with a release.
# The HTML is pushed from a copy, so the checkout the release upload below works on is left as it was.
- name: Upload documentation to rugnux/gitea-pages
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
shell: bash
env:
PAGES_TOKEN: ${{ secrets.RUGNUX_PAGES_TOKEN || secrets.PIP_REPOSITORY_API_TOKEN }}
PAGES_TOKEN_NAME: ${{ secrets.RUGNUX_PAGES_TOKEN && 'RUGNUX_PAGES_TOKEN' || 'PIP_REPOSITORY_API_TOKEN' }}
run: |
echo "Pushing with secret ${PAGES_TOKEN_NAME}"
rm -rf /tmp/rugnux-pages
mkdir /tmp/rugnux-pages
cp -a public/. /tmp/rugnux-pages/
cd /tmp/rugnux-pages
git init -q
git checkout -q -b gitea-pages
git config user.email "filip.leonarski@psi.ch"
git config user.name "Filip Leonarski (Gitea)"
git add .
git commit -q -m "Deploy site from ${GITHUB_REF_NAME} ${GITHUB_SHA::9}"
git push -f "https://${PAGES_TOKEN}@gitea.psi.ch/rugnux/gitea-pages.git" gitea-pages
- name: Upload documentation
if: github.event_name == 'workflow_dispatch' && (github.event.inputs.create_release == 'true' || github.event.inputs.create_release == true)
shell: bash
run: |
git config --global user.email "filip.leonarski@psi.ch"
git config --global user.name "Filip Leonarski (Gitea)"
git checkout --orphan gitea-pages
git reset --hard
# Remove everything except .git and public
git clean -fdx -e public -e .git
# Move public contents to root and remove the now-empty public dir
shopt -s dotglob nullglob
mv public/* .
rm -rf public
git add .
git commit -m "Deploy site"
git push -f https://${{secrets.GITHUB_TOKEN}}@gitea.psi.ch/${{ github.repository }}.git gitea-pages
# ---- Release -------------------------------------------------------------------------------------
# Runs on every event, but only creates something when asked. Keeping it unconditional is what
# lets every other job depend on it with a plain `needs:` -- a skipped dependency would otherwise
CLAUDE.md
+9 -7
View File
@@ -9,8 +9,9 @@ X-ray detectors. It receives detector data, runs it through an FPGA-accelerated
(spot finding, azimuthal/ROI integration, compression), streams images out over ZeroMQ for
writing to HDF5, and runs crystallographic analysis (indexing, integration, scaling/merging).
Most authoritative documentation lives in `docs/` and on Read The Docs
(https://jungfraujoch.readthedocs.io). When changing CLI behaviour, the program's own
usage message is the source of truth, not the docs.
(https://jungfraujoch.readthedocs.io). rugnux's options are all described in
`docs/RUGNUX_CLI.md` (the program's own usage shows only the modes and the basic options): when
changing CLI behaviour, update the parser, that page and, for a basic option, the usage together.
## Build
@@ -25,8 +26,8 @@ the build tree is what every `find_package(Eigen3)` resolves — ours, Ceres' an
already on the machine is used instead if the configure is given `-DZLIB_ROOT=` / `-DEigen3_DIR=`.
Do **not** turn either of these into `FetchContent ... OVERRIDE_FIND_PACKAGE` — see the Eigen note in
`CMakeLists.txt` for the crash that rules it out.
libjpeg-turbo is built via `ExternalProject` in `preview/`; Mbed TLS (the viewer's HTTPS
backend for cpp-httplib) is fetched only for viewer builds.
JPEG is encoded by the vendored single-header `stb_image_write` (`preview/stb/`); Mbed TLS
(the viewer's HTTPS backend for cpp-httplib) is fetched only for viewer builds.
```
mkdir build && cd build
@@ -360,7 +361,8 @@ generated API model and internal types.
- `jfjoch_viewer` — interactive Qt desktop (`viewer/`), results not persisted.
- `rugnux` (`rugnux/rugnux_cli.cpp`, built on the `JFJochRugnux` library in the same directory) — offline
batch over a stored HDF5, invoked as `rugnux {<options>} <input.h5>` (it has **no `--help`**; run
it with no arguments to print the usage, which is the authority on its flags). Rotation vs stills
it with no arguments to print a short usage: one section per mode and the basic options; every
option is listed in `docs/RUGNUX_CLI.md`, which must be kept in step with the parser). Rotation vs stills
is auto-detected from the goniometer axis. Merging is on by default (`--no-merge` to disable);
merging writes `.mtz`/`.cif`/`.hkl` and skips the bulky `_process.h5` unless `--write-process-h5`,
while `--no-merge` writes only `_process.h5`. `--mode` picks what a run does: `mx` (the full
@@ -417,8 +419,8 @@ portable; don't constrain it for portability's sake.
intended Windows configuration.
`JFJOCH_VIEWER_ONLY` is forced ON on Windows and macOS, so a plain configure there already builds
just the portable subset. libtiff, libjpeg-turbo (via `ExternalProject` in `preview/`) and Mbed TLS
(viewer builds only) are now brought in by the build itself, so they need nothing from the host.
just the portable subset. libtiff and Mbed TLS (viewer builds only) are now brought in by the build
itself, so they need nothing from the host.
**ZLIB** and **Eigen** (header-only; needed by Ceres, by the analysis libs directly, and by `ffbidx`
under CUDA) are brought in by the build too, so on Windows only Qt still has to be installed first.
Both are still reached through an ordinary `find_package`, and that is the point: **vendoring Eigen
CMakeLists.txt
+34 -4
View File
@@ -425,13 +425,18 @@ SET(CMAKE_POLICY_VERSION_MINIMUM 3.5)
# libzmq must be made available BEFORE sls_detector_package for the override above to take effect.
FetchContent_MakeAvailable(libzmq)
# httplib links zstd::libzstd when that target exists, and otherwise looks for a system libzstd
# (pkg-config) - which put a second, older zstd beside our static one, and the BSHUF_ZSTD calls
# resolved to the system copy at runtime. Point it at our static build.
FetchContent_MakeAvailable(zstd)
add_library(zstd::libzstd ALIAS libzstd_static)
IF (JFJOCH_PORTABLE_ONLY)
# A portable build still needs zstd/hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2
# A portable build still needs hdf5/spdlog/httplib (JFJochReader uses httplib), and catch2
# for jfjoch_portable_test. Only sls_detector_package (detector) is not built here -- it does
# not configure under MSVC.
FetchContent_MakeAvailable(zstd catch2 hdf5 spdlog httplib)
FetchContent_MakeAvailable(catch2 hdf5 spdlog httplib)
ELSE()
FetchContent_MakeAvailable(zstd sls_detector_package catch2 hdf5 spdlog httplib)
FetchContent_MakeAvailable(sls_detector_package catch2 hdf5 spdlog httplib)
ENDIF()
# libtiff (used by JFJochPreview in every build mode): build it ourselves. Its C++ binding
@@ -582,7 +587,7 @@ SET(JFJOCH_NOTICE_FILES
IF (JFJOCH_RUGNUX_ONLY)
# The viewer-only subtree minus viewer/ and tools/: just rugnux and what it links. broker/ is
# here for JFJochAPI alone (JFJochReader links it); its service targets are gated out. preview/
# arrives through JFJochWriter -> JFJochPreview, so libjpeg-turbo and libtiff are built too.
# arrives through JFJochWriter -> JFJochPreview, so libtiff is built too.
ADD_SUBDIRECTORY(jungfrau)
ADD_SUBDIRECTORY(compression)
ADD_SUBDIRECTORY(common)
@@ -944,6 +949,31 @@ else()
endif()
# The Sphinx HTML documentation, shipped with rugnux: its usage points at
# share/doc/jfjoch_rugnux/html/RUGNUX_CLI.html in the release archive. Never part of the default
# build - `make rugnux_docs` builds it in a venv of its own from docs/requirements.txt (so it needs
# Python 3.10+ and pip's network, not an installed Sphinx), and the OPTIONAL install rule ships
# whatever it left in the build tree; without it the packages are made as before, with no HTML.
SET(JFJOCH_DOCS_HTML_DIR ${CMAKE_BINARY_DIR}/docs_html)
FIND_PACKAGE(Python3 3.10 COMPONENTS Interpreter)
IF (Python3_FOUND)
IF (WIN32)
SET(_docs_python ${CMAKE_BINARY_DIR}/docs_venv/Scripts/python.exe)
ELSE()
SET(_docs_python ${CMAKE_BINARY_DIR}/docs_venv/bin/python)
ENDIF()
ADD_CUSTOM_TARGET(rugnux_docs
COMMAND ${Python3_EXECUTABLE} -m venv ${CMAKE_BINARY_DIR}/docs_venv
COMMAND ${_docs_python} -m pip install -q -r ${CMAKE_SOURCE_DIR}/docs/requirements.txt
COMMAND ${_docs_python} -m sphinx -q -b html -d ${CMAKE_BINARY_DIR}/docs_doctrees
${CMAKE_SOURCE_DIR}/docs ${JFJOCH_DOCS_HTML_DIR}
COMMENT "Building the HTML documentation into ${JFJOCH_DOCS_HTML_DIR}")
ELSE()
MESSAGE(STATUS "No Python 3.10 or newer: no rugnux_docs target, rugnux is packaged without the HTML documentation")
ENDIF()
INSTALL(DIRECTORY ${JFJOCH_DOCS_HTML_DIR}/ DESTINATION share/doc/jfjoch_rugnux/html COMPONENT rugnux
OPTIONAL PATTERN ".buildinfo" EXCLUDE)
# Ship the notices (top-level LICENSE, the third-party manifest, the verbatim license texts, and
# the acknowledgements the programs refer a user to) with every package variant. Installed per
# component so the files land in whichever component package(s) the current build mode produces.
README.md
+14
View File
@@ -6,3 +6,17 @@ and small-molecule - from images to merged reflections.
All documentation is now placed in [docs/](docs/) subdirectory and for the current version hosted on
[Jungfraujoch Read The Docs page](https://jungfraujoch.readthedocs.io).
## How to cite
- **Jungfraujoch**: F. Leonarski et al., "Jungfraujoch: hardware-accelerated data-acquisition system for
kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30, 227-234
[doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268); for real-time serial
crystallography and live indexing feedback also J. Duan et al., "Jungfraujoch: 38 GB s⁻¹ real-time
serial crystallography – data acquisition and live indexing feedback" (2026), J. Appl. Cryst. 59,
1668-1677 [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083).
- **Rugnux**: F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons
and XFELs in one program" (2026), bioRxiv
[doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522).
The full list is in [docs/PUBLICATIONS.md](docs/PUBLICATIONS.md).
THIRD_PARTY_NOTICES.md
+2 -1
View File
@@ -28,7 +28,6 @@ linked into the Jungfraujoch executables.
| [FFTW](https://www.fftw.org/) | 3.3.10 | Matteo Frigo; MIT | GPL-2.0-or-later | [fftw.txt](licenses/fftw.txt) |
| [Ceres Solver](https://github.com/ceres-solver/ceres-solver) | (pinned) | Google Inc. and contributors | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) |
| [Abseil](https://github.com/abseil/abseil-cpp) | 20250127 (required by Ceres) | Google Inc. | Apache-2.0 | [abseil.txt](licenses/abseil.txt) |
| [libjpeg-turbo](https://github.com/libjpeg-turbo/libjpeg-turbo) | 3.2.0 | D. R. Commander and others; IJG | IJG + BSD-3-Clause + Zlib | [libjpeg-turbo.txt](licenses/libjpeg-turbo.txt) |
| [Mbed TLS](https://github.com/Mbed-TLS/mbedtls) (with TF-PSA-Crypto) | 4.2.0 | The Mbed TLS Contributors | Apache-2.0 OR GPL-2.0-or-later (used under Apache-2.0) | [mbedtls.txt](licenses/mbedtls.txt), [tf-psa-crypto.txt](licenses/tf-psa-crypto.txt) |
| [Catch2](https://github.com/catchorg/Catch2) | 3.16.0 | Catch2 Authors | BSL-1.0 | [catch2.txt](licenses/catch2.txt) |
| [zlib-ng](https://github.com/zlib-ng/zlib-ng) | 2.3.3 | Jean-loup Gailly and Mark Adler; the zlib-ng contributors | Zlib | [zlib.txt](licenses/zlib.txt) |
@@ -58,10 +57,12 @@ These live in the source tree (see the path) rather than being fetched; traccc a
| [Bitshuffle](https://github.com/kiyo-masui/bitshuffle) | `compression/bitshuffle/` | Kiyoshi Masui | MIT | [bitshuffle.txt](licenses/bitshuffle.txt) |
| [Bitshuffle (h-perf)](https://github.com/kalcutter/bitshuffle) | `compression/bitshuffle_hperf/` | Kal Conley | Apache-2.0 | [bitshuffle-hperf.txt](licenses/bitshuffle-hperf.txt) |
| [LZ4](https://github.com/lz4/lz4) | `compression/lz4/` | Yann Collet | BSD-2-Clause | [lz4.txt](licenses/lz4.txt) |
| [stb_image_write](https://github.com/nothings/stb) (v1.16, commit 31c1ad3) | `preview/stb/` | Sean Barrett | MIT OR Unlicense | [stb_image_write.txt](licenses/stb_image_write.txt) |
| [HLS arbitrary-precision types](https://github.com/Xilinx/HLS_arbitrary_Precision_Types) | `fpga/include/` | Xilinx, Inc. | Apache-2.0 | [xilinx-hls-headers.txt](licenses/xilinx-hls-headers.txt) |
| [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (commit a94fd4f) | `image_analysis/indexing/fast-feedback-indexer/` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](licenses/fast-feedback-indexer.txt) |
| [GEMMI](https://github.com/project-gemmi/gemmi) | `gemmi_gph/` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) |
| [CCP4 Monomer Library](https://github.com/MonomerLibrary/monomers) (subset: restraint dictionary data) | `image_analysis/structure_refinement/monomers/` | CCP4 / the Monomer Library authors (Vagin et al.) | LGPL-3.0-only | [ccp4-monomer-library.txt](licenses/ccp4-monomer-library.txt) |
| [Larch](https://github.com/xraypy/xraylarch) (its Cromer-Liberman routines, as GEMMI ports them) | `gemmi_gph/fprime.cpp` | Matthew Newville, The University of Chicago, et al. | MIT | [larch.txt](licenses/larch.txt) |
| [PEGTL](https://github.com/taocpp/PEGTL) | `gemmi_gph/gemmi/third_party/tao/` | Dr. Colin Hirsch and Daniel Frey | MIT | [pegtl.txt](licenses/pegtl.txt) |
| [sajson](https://github.com/chadaustin/sajson) | `gemmi_gph/gemmi/third_party/sajson.h` | Chad Austin | MIT | [sajson.txt](licenses/sajson.txt) |
| [fast_float](https://github.com/fastfloat/fast_float) | `gemmi_gph/gemmi/third_party/fast_float.h` | The fast_float authors (Daniel Lemire et al.) | Apache-2.0 OR MIT OR BSL-1.0 | [fast-float.txt](licenses/fast-float.txt) |
common/CUDAWrapper.cpp
+5
View File
@@ -83,6 +83,11 @@ GPUWorkBeside::~GPUWorkBeside() {
remove(gpu_work_beside_n);
}
bool gpu_work_fits_beside_merges(size_t bytes) {
std::lock_guard lock(gpu_work_m);
return gpu_merges_n == 0 || fits_beside_merges(bytes);
}
bool wait_for_gpu_work_beside(std::chrono::seconds timeout) {
std::unique_lock lock(gpu_work_m);
return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; });
common/CUDAWrapper.h
+6
View File
@@ -165,6 +165,12 @@ public:
void Holds(size_t bytes);
};
// Whether work holding `bytes` of the card would start beside the main line's merges now (GPUWorkBeside with
// after_merges): no merge is held, or what they said they hold (GPUMerge::Holds) and `bytes` fit within 80% of
// the first card's total. For a main-line caller that is about to wait for such work, and must not where that
// work waits for its merge. Build-independent - it only counts.
bool gpu_work_fits_beside_merges(size_t bytes);
// Whether the card is too small for a probe pass beside the main line's image pass (12 GB, decided on the
// first card's total memory), so that they take turns (GPUWorkBeside). Build-independent.
bool gpu_takes_turns();
common/DetectorModules.cpp
+44 -2
View File
@@ -4,6 +4,7 @@
#include "DetectorModules.h"
#include "DetectorGeometry.h"
#include "Definitions.h"
#include "PixelMask.h"
#include <algorithm>
#include <cmath>
@@ -75,9 +76,14 @@ const std::vector<DetectorModule> &DetectorModules::Get() const {
}
bool DetectorModules::Moved() const {
// Not an exact comparison: a module read back from a file went through a length and a unit vector
// in single precision, which puts a module as imaged a few 1e-4 px off its box when it is far from
// pixel (0, 0) on both axes.
constexpr double tolerance = 1e-3;
for (const auto &m: modules)
if (m.ox != static_cast<double>(m.x0) || m.oy != static_cast<double>(m.y0)
|| m.fx != 1.0 || m.fy != 0.0 || m.sx != 0.0 || m.sy != 1.0)
if (std::fabs(m.ox - static_cast<double>(m.x0)) > tolerance || std::fabs(m.oy - static_cast<double>(m.y0)) > tolerance
|| std::fabs(m.fx - 1.0) > tolerance / CONVERTED_MODULE_COLS || std::fabs(m.fy) > tolerance / CONVERTED_MODULE_COLS
|| std::fabs(m.sx) > tolerance / CONVERTED_MODULE_COLS || std::fabs(m.sy - 1.0) > tolerance / CONVERTED_MODULE_COLS)
return true;
return false;
}
@@ -193,3 +199,39 @@ DetectorModules ModulesAsImaged(const DetectorGeometry &geom) {
}
return DetectorModules(out);
}
DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height) {
if (width <= 0 || height <= 0 || mask.size() != static_cast<size_t>(width * height))
return {};
const auto clear = [&](int64_t x, int64_t y) {
return x >= 0 && y >= 0 && x < width && y < height
&& (mask[y * width + x] & (1u << PixelMask::ModuleGapPixelBit)) == 0;
};
std::vector<uint8_t> taken(mask.size(), 0);
std::vector<DetectorModule> out;
for (int64_t y = 0; y < height; y++)
for (int64_t x = 0; x < width; x++) {
if (!clear(x, y) || taken[y * width + x])
continue;
// The first clear pixel of a region not yet taken is a module's corner: the module runs to the
// gap along the row and down the column, and has to be whole and bounded by gap all round.
int64_t w = 0, h = 0;
while (clear(x + w, y)) w++;
while (clear(x, y + h)) h++;
if (!((w == CONVERTED_MODULE_COLS && h == CONVERTED_MODULE_LINES)
|| (w == CONVERTED_MODULE_LINES && h == CONVERTED_MODULE_COLS)))
return {};
for (int64_t yy = y - 1; yy <= y + h; yy++)
for (int64_t xx = x - 1; xx <= x + w; xx++) {
const bool inside = xx >= x && xx < x + w && yy >= y && yy < y + h;
if (clear(xx, yy) != inside)
return {};
if (inside)
taken[yy * width + xx] = 1;
}
out.push_back(DetectorModule::AsImaged(x, y, w, h));
}
if (out.size() < 2)
return {};
return DetectorModules(out);
}
common/DetectorModules.h
+6
View File
@@ -121,3 +121,9 @@ public:
// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES turned the way the geometry turns it, placed as imaged.
// Empty for a detector of one module.
DetectorModules ModulesAsImaged(const DetectorGeometry &geom);
// The modules of a JUNGFRAU image where only its pixel mask says where they are: every region of pixels
// clear of the module-gap bit (PixelMask::ModuleGapPixelBit) has to be one whole module,
// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES either way round, and is taken as placed as imaged.
// Empty where any region is not, or where there are fewer than two. The mask is row-major, width x height.
DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height);
common/DiffractionExperiment.cpp
+4
View File
@@ -653,6 +653,10 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() {
void DiffractionExperiment::FillMessage(StartMessage &message) const {
message.images_per_file = GetImagesPerFile();
message.detector_modules = GetModules().Get();
// A JUNGFRAU image assembled here is made of modules this experiment places itself: they go into
// the file one by one, so that what reads it back knows them.
if (message.detector_modules.empty() && GetDetectorType() == DetectorType::JUNGFRAU && IsGeometryTransformed())
message.detector_modules = ModulesAsImaged(detector.GetGeometry()).Get();
message.beam_center_x = GetBeamX_pxl();
message.beam_center_y = GetBeamY_pxl();
const auto direct_beam = GetDiffractionGeometry().GetDirectBeam_pxl();
common/HostMemory.cpp
+32 -2
View File
@@ -3,6 +3,8 @@
#include "HostMemory.h"
#include <algorithm>
#if defined(_WIN32)
#ifndef NOMINMAX
#define NOMINMAX
@@ -47,6 +49,7 @@ HostWorkerPool &HostWorkerPool::Instance() {
}
HostWorkerPool::HostWorkerPool() {
SetBudget(0);
SetSize(0);
}
@@ -57,14 +60,24 @@ void HostWorkerPool::SetSize(size_t bytes) {
cv.notify_all();
}
void HostWorkerPool::SetBudget(size_t bytes) {
std::lock_guard lock(m);
budget = bytes > 0 ? bytes : DefaultHostMemoryBudget();
cv.notify_all();
}
size_t HostWorkerPool::available() const {
return std::min(size, budget > held ? budget - held : 0);
}
size_t HostWorkerPool::Size() {
std::lock_guard lock(m);
return size;
return available();
}
HostWorkerPool::Reservation::Reservation(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(bytes) {
std::unique_lock lock(pool.m);
pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.size; });
pool.cv.wait(lock, [&] { return pool.reserved == 0 || pool.reserved + bytes <= pool.available(); });
pool.reserved += bytes;
}
@@ -75,3 +88,20 @@ HostWorkerPool::Reservation::~Reservation() {
}
pool.cv.notify_all();
}
HostWorkerPool::Held::Held(HostWorkerPool &pool, size_t bytes) : pool(pool), bytes(0) {
Set(bytes);
}
HostWorkerPool::Held::~Held() {
Set(0);
}
void HostWorkerPool::Held::Set(size_t new_bytes) {
{
std::lock_guard lock(pool.m);
pool.held = pool.held - bytes + new_bytes;
bytes = new_bytes;
}
pool.cv.notify_all();
}
common/HostMemory.h
+30 -8
View File
@@ -24,20 +24,26 @@ size_t DefaultHostMemoryBudget();
// its reflections, its merge, the detector-sized arrays.
constexpr double HOST_WORKER_POOL_FRACTION = 0.5;
// One pool for those buffers, process-wide. A worker reserves its bytes before it makes them and hands
// them back when it is done; one that would take the pool past its size waits until others have handed
// theirs back - except that a reservation made while nothing is reserved is always let through, however
// large. So how many such workers run at once is decided by their bytes, not by the thread count, and
// however the steps holding them nest or run side by side; what each worker computes, and where its
// result goes, does not change. A worker must not reserve again while it holds a reservation, and none
// may reserve on ParallelFor's pool: a holder's own parallel passes need those threads. Work spread over
// that pool is bounded by its count instead (Size() / bytes each).
// One pool for those buffers, process-wide, held in one ledger with what the run holds beside them. The run
// says what it holds - its reflections, its merge engine, a validation's own grids - as Held bytes, and the
// workers may hold at once the pool's size or what the budget leaves beside the Held bytes, whichever is
// less: so the two together never claim more than the budget, however large the run's own part is. A worker
// reserves its bytes before it makes them and hands them back when it is done; one that would take the pool
// past that waits until others have handed theirs back - except that a reservation made while no worker holds
// one is always let through, however large. So how many such workers run at once is decided by their bytes,
// not by the thread count, and however the steps holding them nest or run side by side; what each worker
// computes, and where its result goes, does not change. A worker must not reserve again while it holds a
// reservation, and none may reserve on ParallelFor's pool: a holder's own parallel passes need those threads.
// Work spread over that pool is bounded by its count instead (Size() / bytes each).
class HostWorkerPool {
public:
static HostWorkerPool &Instance();
// The pool's size in bytes; 0 sets it back to HOST_WORKER_POOL_FRACTION of DefaultHostMemoryBudget().
void SetSize(size_t bytes);
// The budget the pool and the Held bytes share; 0 sets it back to DefaultHostMemoryBudget().
void SetBudget(size_t bytes);
// What the workers may hold at once now: the pool's size, or the budget less the Held bytes if less.
[[nodiscard]] size_t Size();
class Reservation {
@@ -51,10 +57,26 @@ public:
size_t bytes;
};
// Bytes the run holds beside the workers for as long as this lives; never waits.
class Held {
public:
Held(HostWorkerPool &pool, size_t bytes);
~Held();
Held(const Held &) = delete;
Held &operator=(const Held &) = delete;
void Set(size_t bytes); // what it holds from now on
private:
HostWorkerPool &pool;
size_t bytes;
};
private:
HostWorkerPool();
size_t available() const; // Size(), with the lock held
std::mutex m;
std::condition_variable cv;
size_t size = 0;
size_t budget = 0;
size_t held = 0;
size_t reserved = 0;
};
common/IndexingSettings.cpp
+1 -1
View File
@@ -53,7 +53,7 @@ IndexingSettings &IndexingSettings::Algorithm(IndexingAlgorithmEnum input) {
// only recover a basis vector up to this length, so the ceiling is exactly the longest cell the
// indexer can ever find; at 500 it excluded 1091 of the PDB's 206950 X-ray entries (0.53%) outright.
// 1200 A leaves 8. Nothing pays for the wider range: the histogram is sized from the VALUE in use,
// and only a caller that asks for more - a given cell that needs it, or the long-axis rescue - gets
// and only a caller that asks for more - a given cell that needs it, or the long-axis hypothesis - gets
// a longer transform.
IndexingSettings &IndexingSettings::FFT_MaxUnitCell_A(float input) {
check_finite("FFT indexing max unit cell (A)", input);
common/IndexingSettings.h
+1 -1
View File
@@ -29,7 +29,7 @@ class IndexingSettings {
public:
// The longest cell the FFT search can be asked to reach, and so the longest it can ever find:
// FFTIndexer sizes its histogram from fft_max_unit_cell_A and the transform's last usable bin IS
// that length. Callers that widen the bound (a given cell, the long-axis rescue) must clamp to
// that length. Callers that widen the bound (a given cell, the long-axis hypothesis) must clamp to
// this rather than let the setter throw - a rescue that recovers an implausible axis must not
// take the whole run down with it.
static constexpr float fft_max_unit_cell_limit_A = 1200.0;
common/JFJochMessages.h
+3 -1
View File
@@ -473,8 +473,10 @@ struct EndMessage {
// setting the merge works in - is only chosen afterwards, so the two settings are not always the
// same one. Absent means they are, i.e. the identity.
std::optional<std::array<int32_t, 9>> reindex_matrix;
// Stills: the rocking curve the partialities were computed from (and the frames predicted through).
// Stills: the rocking curve the frames were predicted through (and the partialities computed from).
std::optional<StillsRunWidth> stills_run_width;
// Stills: the cell given on the command line, whose population of frames the merge keeps.
std::optional<UnitCell> stills_given_cell;
// Geometry refined by the offline analysis. When present these overwrite the master-file geometry
// at finalization (rugnux writes the refined values; the broker leaves them empty and keeps the
common/Reflection.h
+17 -3
View File
@@ -24,7 +24,7 @@ struct Reflection {
float bkg;
float var_bkg; // non-signal (background) part of sigma^2, carried to the merge
float sigma;
float dist_ewald;
float dist_ewald; // stills: signed excitation error |S| - 1/lambda (1/A), positive outside the sphere
// The reciprocal Lorentz factor (rotation only - a still's Lorentz factor is one) times the
// reciprocal polarization factor, and nothing else. This is what LP means everywhere in the
// field, and it is what the CBOR key "rlp" and the HDF5 dataset "lp" store the reciprocal of.
@@ -53,7 +53,7 @@ struct Reflection {
float zeta;
float image_scale_corr; // I_true = image_scale_corr * I; = prescaling_corr * qe_corr * flight_corr / (partiality * image_scale)
bool observed = false;
bool on_ice_ring = false; // sits on a hexagonal-ice powder ring: excluded from scaling, kept for merging
bool on_ice_ring = false; // sits on a hexagonal-ice powder ring: excluded from scaling; merged on stills, ring by ring on rotation (DecideIceRings)
// The signal disk lost pixels to the mask or to saturation, so the intensity is the profile's
// estimate over what remained. Such a measurement may be low, and the merge does not let it
// testify against a larger observation of the same reflection (WilsonOutliers.h).
@@ -68,13 +68,25 @@ struct Reflection {
// measurement, every correction and the per-frame scale applied, its sigma under the error model the
// merge weighted it with - but NOT averaged with its symmetry mates. h k l are the index it was
// measured at, not its ASU representative. Written as the SHELX HKLF 4 file, so that the program
// reading it sees the equivalents and computes Rint itself.
// reading it sees the equivalents and computes Rint itself, and as the intensities of the unmerged MTZ.
struct ScaledFull {
int32_t h = 0;
int32_t k = 0;
int32_t l = 0;
float I = NAN;
float sigma = NAN;
float image_number = NAN; // the frame of the event's peak partial, which tells two events of one index apart
// The full's counting variance as the merge rebuilds it at an intensity I, before its error model:
// var_bkg + var_per_I * I + (capture * I)^2, all at the intensity's scale (what a later merge rebuilds
// the full from). NaN where not exported.
float var_bkg = NAN;
float var_per_I = NAN;
float capture = NAN;
float d = NAN; // resolution, A
// The diffracted beam's direction in the crystal: its components along the real-space cell vectors
// (a.s1, b.s1, c.s1), so that s1 = sum_i (a_i.s1) a*_i - reindexed like h, k, l. What a joint merge of
// several sweeps fits one absorption surface over. NaN where not known.
float s1_cell[3] = {NAN, NAN, NAN};
};
struct MergedReflection {
@@ -92,6 +104,8 @@ struct MergedReflection {
// observation at the run's typical frame scale, over the precision they have. 1 on a sweep
// without a weak stretch; small for a reflection measured only where the crystal barely
// diffracted, whose scaled-up noise would otherwise count as much as a well-measured pair.
// Internal, not reported: it enters the CC1/2 the processing decides on (the automatic resolution
// cut, MergeStatisticsShell::cc_half_internal), never the reported CC1/2, which is the plain Pearson.
float cc_weight = 1.0f;
float d = 0.0;
// Any observation of this reflection sat on an ice ring. The intensity is still merged and
common/ScalingSettings.cpp
+9
View File
@@ -315,3 +315,12 @@ ScalingSettings& ScalingSettings::ReportResolutionRange_A(std::optional<ReportRe
std::optional<ReportResolutionRange> ScalingSettings::GetReportResolutionRange_A() const {
return report_resolution_range;
}
ScalingSettings& ScalingSettings::SweepSegments(std::vector<SweepSegment> input) {
sweep_segments = std::move(input);
return *this;
}
const std::vector<SweepSegment> &ScalingSettings::GetSweepSegments() const {
return sweep_segments;
}
common/ScalingSettings.h
+17
View File
@@ -3,7 +3,9 @@
#pragma once
#include <cstdint>
#include <optional>
#include <vector>
#include "JFJochException.h"
// How the high-resolution cutoff for the written reflections and the reported shell table is chosen
@@ -17,8 +19,20 @@ struct ReportResolutionRange {
double d_max = 0.0;
};
// One sweep of several whose frames are laid end to end in one rotation merge (the joint merge of several
// sweeps): frames [first_frame, first_frame + n_frames). Every observation of a segment's frames is a full
// already combined in its own sweep (partiality 1, filed under the frame of its peak), so the merge
// combines nothing again, recomputes nothing from the rocking geometry, and lets nothing that runs along
// the frames - the per-frame scale's smoothness above all - reach across from one sweep into the next
// (RotationScaleMerge).
struct SweepSegment {
int32_t first_frame = 0;
int32_t n_frames = 0;
};
class ScalingSettings {
bool refine_wedge = false;
std::vector<SweepSegment> sweep_segments;
bool merge_friedel = true;
std::optional<double> high_resolution_limit_A;
@@ -150,6 +164,8 @@ public:
ScalingSettings& ResolutionCCTarget(double input);
ScalingSettings& ReportShellCount(int input);
ScalingSettings& ReportResolutionRange_A(std::optional<ReportResolutionRange> input);
// The sweeps a rotation merge's frames are made of; empty (the default): one sweep of partials.
ScalingSettings& SweepSegments(std::vector<SweepSegment> input);
[[nodiscard]] bool GetRefineWedge() const;
@@ -189,4 +205,5 @@ public:
[[nodiscard]] double GetResolutionCCTarget() const;
[[nodiscard]] int GetReportShellCount() const;
[[nodiscard]] std::optional<ReportResolutionRange> GetReportResolutionRange_A() const;
[[nodiscard]] const std::vector<SweepSegment> &GetSweepSegments() const;
};
docker/common/build-qt.sh
+2 -3
View File
@@ -4,9 +4,8 @@
# Shared by all four build images (docker/*/Dockerfile). The build context is docker/, so each
# Dockerfile reaches this with COPY common/. Version inputs come from that Dockerfile's ARGs.
#
# No JPEG or TIFF in Qt: the viewer has its own libjpeg-turbo and libtiff (preview/), and a static
# Qt's JPEG plugin brings a second libjpeg-turbo of another ABI that the linker silently mixes with
# ours (see viewer/CMakeLists.txt).
# No JPEG or TIFF in Qt: the viewer has its own JPEG encoder and libtiff (preview/), and nothing
# goes through QImage for these formats (see viewer/CMakeLists.txt).
set -eux
docker/rocky8/Dockerfile
+2 -4
View File
@@ -6,9 +6,8 @@ ARG QT_VERSION=6.11.1
ARG DBUS_VERSION=1.14.10
ARG NODE_MAJOR=22
ARG EIGEN_VERSION=3.4.0
# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent /
# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source
# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external
# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# Update base and install common utilities
RUN dnf -y install epel-release && \
@@ -22,7 +21,6 @@ RUN dnf -y install epel-release && \
tar \
xz \
make \
nasm \
gcc-toolset-13 \
gcc-toolset-13-gcc \
gcc-toolset-13-gcc-c++ \
docker/rocky9/Dockerfile
+2 -4
View File
@@ -6,9 +6,8 @@ ARG QT_VERSION=6.11.1
ARG DBUS_VERSION=1.14.10
ARG NODE_MAJOR=22
ARG EIGEN_VERSION=3.4.0
# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent /
# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source
# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external
# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
ARG DIALS_VERSION=3.27.0
ARG DIALS_TARBALL_URL=https://github.com/dials/dials/releases/download/v3.27.0/dials-v3-27-0-linux-x86_64.tar.xz
ARG DIALS_PREFIX=/opt
@@ -39,7 +38,6 @@ RUN dnf -y update && \
python3 \
python3.12 \
perl \
nasm \
pkgconf-pkg-config \
xcb-util-devel \
xcb-util-*-devel \
docker/ubuntu2204/Dockerfile
+2 -4
View File
@@ -9,9 +9,8 @@ ARG QT_VERSION=6.11.1
ARG DBUS_VERSION=1.14.10
ARG NODE_MAJOR=22
ARG EIGEN_VERSION=3.4.0
# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent /
# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source
# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external
# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# Update base and install toolchain + Qt dependencies
# Notes:
@@ -35,7 +34,6 @@ RUN set -eux; \
pkg-config \
gdb \
file \
nasm \
python3.11 \
python3.11-venv \
dpkg-dev \
docker/ubuntu2404/Dockerfile
+2 -5
View File
@@ -9,10 +9,8 @@ ARG QT_VERSION=6.11.1
ARG DBUS_VERSION=1.14.10
ARG NODE_MAJOR=22
ARG EIGEN_VERSION=3.4.0
# HDF5, libtiff and libjpeg-turbo are built by the jungfraujoch CMake itself (FetchContent /
# ExternalProject). Eigen is an external find_package(Eigen3 3.4) dependency, installed from source
# below (3.4.x is required; Eigen's same-major-version rule rejects 5.x). nasm is required by the
# project's libjpeg-turbo ExternalProject for SIMD-accelerated JPEG.
# HDF5 and libtiff are built by the jungfraujoch CMake itself (FetchContent). Eigen is an external
# find_package(Eigen3 3.4) dependency, installed from source below (3.4.x is required; Eigen's same-major-version rule rejects 5.x).
# Update base and install toolchain + static-Qt build dependencies
# (XCB/X11/OpenGL/EGL/DBus/fontconfig/freetype/zlib headers needed to build static Qt).
@@ -48,7 +46,6 @@ RUN set -eux; \
pkg-config \
gdb \
file \
nasm \
dpkg-dev \
fakeroot \
libnuma-dev \
docs/ACKNOWLEDGEMENT.md
+189 -6
View File
@@ -1,6 +1,19 @@
# Acknowledgements
Citation: F. Leonarski, M. Bruckner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matej, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad., 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268).
How to cite Jungfraujoch and Rugnux, and the papers that describe them, is on the
[Publications](PUBLICATIONS.md) page. In short:
- **Rugnux**: F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons
and XFELs in one program" (2026), bioRxiv
[doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522).
- **Jungfraujoch**: F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler,
Z. Matěj, A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch:
hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors" (2023),
J. Synchrotron Rad. 30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268);
for real-time serial crystallography and live indexing feedback also J. Duan, H.-C. Stadler,
J. Nan et al., "Jungfraujoch: 38 GB s⁻¹ real-time serial crystallography – data acquisition and
live indexing feedback" (2026), J. Appl. Cryst. 59, 1668-1677
[doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083).
## Funding and support
@@ -44,7 +57,7 @@ Cambridge, United Kingdom: Global Phasing Ltd.
## Crystallographic methods adopted from other packages
The analysis pipeline reimplements methods first published, and in most cases first implemented, by
The analysis pipeline builds on methods first published, and in most cases first implemented, by
other crystallographic software. The code below is Jungfraujoch's own; the methods are theirs, and
are acknowledged here. Where a package's source was consulted this is said explicitly. Most of these
packages are neither linked nor vendored; the three that are - GEMMI, traccc and
@@ -65,6 +78,16 @@ high-throughput reduction and analysis of serial femtosecond X-ray diffraction d
J. Appl. Cryst. 47, 1118-1131
[doi:10.1107/S1600576714007626](https://doi.org/10.1107/S1600576714007626).
**Per-crystal post-refinement of serial stills** - refining, for each still, the partiality
parameters the spot positions cannot pin (a small reorientation, the crystal's own rocking width and the
radial scale of its reciprocal lattice, which a shot's photon energy or a crystal's cell moves) against
the merged intensities follows the post-refinement of PRIME and the mosaic-block model of Sauter et al.:
M. Uervirojnangkoorn, O. B. Zeldin, A. Y. Lyubimov, J. Hattne, A. S. Brewster, N. K. Sauter, A. T. Brunger
and W. I. Weis, "Enabling X-ray free electron laser crystallography for challenging biological systems
from a limited number of crystals" (2015), eLife 4, e05421; N. K. Sauter, J. Hattne, A. S. Brewster,
N. Echols, P. H. Zwart and P. D. Adams, "Improved crystal orientation and physical properties from
single-shot XFEL stills" (2014), Acta Cryst. D70, 3299-3309 (DOIs not verified here, so not given).
Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the
ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann,
A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP
@@ -229,6 +252,28 @@ H = |I(h) − I(Th)|/(I(h) + I(Th)) of the pairs a law relates is uniform on [0,
gives the fraction. T. O. Yeates, "Detecting and overcoming crystal twinning" (1997), Methods Enzymol.
276, 344-358 [doi:10.1016/S0076-6879(97)76068-3](https://doi.org/10.1016/S0076-6879%2897%2976068-3).
**The symmetry hypotheses a refused promotion leaves** - the adopted group and every other index-2 subgroup
of the refused group perfectly twinned by a law of its coset, all merging alike, carried to the model and
decided by placing and refining it in each - follow the advice to refine in the candidate groups and compare,
rather than decide from intensity statistics, that pseudo-symmetry and twinning call for:
P. H. Zwart, R. W. Grosse-Kunstleve, A. A. Lebedev, G. N. Murshudov and P. D. Adams, "Surprises and pitfalls
arising from (pseudo)symmetry" (2008), Acta Cryst. D64, 99-107
[doi:10.1107/S090744490705531X](https://doi.org/10.1107/S090744490705531X); A. A. Lebedev and M. N. Isupov,
"Space-group and origin ambiguity in macromolecular structures with pseudo-symmetry and its treatment with the
program Zanuda" (2014), Acta Cryst. D70, 2430-2443
[doi:10.1107/S1399004714014795](https://doi.org/10.1107/S1399004714014795).
**Twin refinement** (§15.12) refines the model against Yeates's twinned intensity, (1−α) I(h) + α I(Th), with the
twin fraction refined beside it as SHELXL refines BASF: R. Herbst-Irmer and G. M. Sheldrick, "Refinement of twinned
structures with SHELXL97" (1998), Acta Cryst. B54, 443-449
[doi:10.1107/S0108768197018454](https://doi.org/10.1107/S0108768197018454). The twin laws come from the lattice's
metric as A. A. Lebedev, A. A. Vagin and G. N. Murshudov describe, "Intensity statistics in twinned crystals with
examples from the PDB" (2006), Acta Cryst. D62, 83-95
[doi:10.1107/S0907444905036759](https://doi.org/10.1107/S0907444905036759), and the fractions are refined by
least squares at every cycle, as REFMAC5 refines them (Murshudov et al. 2011, Acta Cryst. D67, 355-367, cited
above). REFMAC5's target is a twinned-intensity likelihood; ours keeps the amplitude likelihood with the twinned
model amplitude in it, a simplification of that.
**A first-pass cell that is the coincidence lattice of two twin domains** (twinning by reticular
merohedry of index 2 or 3, e.g. obverse/reverse rhombohedral domains or a cubic crystal twinned about
[111]) is recognised by the nodes that belong to neither domain being empty. The twin index is
@@ -247,7 +292,10 @@ noise floor of the statistic depends strongly on how many reflections a dataset
modulation is what the axial systematic-absence test scores against, so that a reflection class a
pseudo-translation merely suppresses is not read as extinct and does not buy a screw axis; the
estimate of its depth there is our own, measured per axial row from the merged intensities rather
than from the Patterson vector. R. J. Read,
than from the Patterson vector. Where a symmetry operator or a twin law carries the vector to one
that is not ±itself, the modulation is also read on classes averaged over the lattice's rotations -
their modulation model summed over the copy pairs each rotation relates, as the measured intensity
sums them; that averaging is our own construction. R. J. Read,
P. D. Adams and A. J. McCoy, "Intensity statistics in the presence of translational
noncrystallographic symmetry" (2013), Acta Cryst. D69, 176-183
[doi:10.1107/S0907444912045374](https://doi.org/10.1107/S0907444912045374).
@@ -319,7 +367,13 @@ reporting of I/sigma(I). K. Diederichs and P. A. Karplus, "Improved
R-factors for diffraction data analysis in macromolecular crystallography" (1997), Nat. Struct. Biol.
4, 269-275 [doi:10.1038/nsb0497-269](https://doi.org/10.1038/nsb0497-269); P. A. Karplus and
K. Diederichs, "Linking crystallographic model and data quality" (2012), Science 336, 1030-1033
[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231); M. S. Weiss, "Global
[doi:10.1126/science.1218231](https://doi.org/10.1126/science.1218231) (CC\* is also what a joint merge of
several sweeps tests their isomorphism against: CC_AB / (CC\*_A CC\*_B) is the cosine of the angle between
two data sets, their systematic difference apart from their noise, as K. Diederichs, "Dissecting random and
systematic differences between noisy composite data sets" (2017), Acta Cryst. D73, 286-293
[doi:10.1107/S2059798317000699](https://doi.org/10.1107/S2059798317000699) shows - which is why the join is
decided on the size of that ratio, and a doubtful sweep by the merge with and without it, the idea of
delta-CC1/2 by Assmann, Brehm and Diederichs (2016), below); M. S. Weiss, "Global
indicators of X-ray data quality" (2001), J. Appl. Cryst. 34, 130-135
[doi:10.1107/S0021889800018227](https://doi.org/10.1107/S0021889800018227) (R_pim beside R_meas and
R_merge); K. Diederichs and
@@ -359,6 +413,15 @@ J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in t
package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399
[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198).
**Several sweeps merged as one** - each sweep's partials combined into fulls on their own, and the
fulls of all the sweeps then scaled together, every sweep with its own frame scales and a relative B
against the joint merge, is how XSCALE merges several data sets (W. Kabsch, "Integration, scaling,
space-group assignment and post-refinement" (2010), above). Reading the radiation dose a sweep was taken
at from its relative B, which orders the sweeps by dose from the data rather than from the order they
were given in, follows K. Diederichs, "Some aspects of quantitative analysis and correction of radiation
damage" (2006), Acta Cryst. D62, 96-101
[doi:10.1107/S0907444905031537](https://doi.org/10.1107/S0907444905031537).
**Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature:
D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography:
Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75
@@ -378,10 +441,21 @@ symmetric than its crystals and no reference or model is given, each crystal is
which its intensities agree best with those of all the other crystals, crystal by crystal until none
changes. The objective - the summed agreement between every pair of crystals - and the iteration that
maximises it are taken from Brehm and Diederichs, whose method CrystFEL's `ambigator` also implements;
rugnux measures the agreement as a product of standardised intensities rather than a correlation, which
lets the sum over all partners be taken against one running sum instead of pair by pair.
rugnux measures the agreement as a product of standardised amplitudes (square roots of the intensities)
rather than a correlation, which lets the sum over all partners be taken against one running sum instead
of pair by pair.
W. Brehm and K. Diederichs, "Breaking the indexing ambiguity in serial crystallography" (2014),
Acta Cryst. D70, 101-109 [doi:10.1107/S1399004713025431](https://doi.org/10.1107/S1399004713025431).
The same objective settles which twin law puts each sweep of a joint merge of several sweeps on the
reference's indexing - decided over all pairs of sweeps at once, as dials.cosym does for its data sets:
R. J. Gildea and G. Winter, "Determination of Patterson group symmetry from sparse multi-crystal data sets
in the presence of an indexing ambiguity" (2018), Acta Cryst. D74, 405-410
[doi:10.1107/S2059798318002978](https://doi.org/10.1107/S2059798318002978).
**Reference sweep of a joint merge** — the sweep the others are put on the axes of is the one whose own
merge reaches the finest resolution, as KAMO takes the data set of the smallest Wilson B as its reference:
K. Yamashita, K. Hirata and M. Yamamoto, "KAMO: towards automated data processing for microcrystals"
(2018), Acta Cryst. D74, 441-449 [doi:10.1107/S2059798318004576](https://doi.org/10.1107/S2059798318004576).
### Physical corrections and calibration
@@ -595,12 +669,114 @@ predicted distances around metal sites in proteins" (2006), Acta Cryst. D62, 678
[doi:10.1107/S0907444906014594](https://doi.org/10.1107/S0907444906014594), and, for Mg²⁺, Na⁺ and K⁺ at
nucleic acids, F. Leonarski, L. D'Ascenzo and P. Auffinger, "Mg2+ ions: do they bind to nucleobase
nitrogens?" (2017), Nucleic Acids Res. 45, 987-1004 [doi:10.1093/nar/gkw1175](https://doi.org/10.1093/nar/gkw1175).
The anisotropic displacements: a TLS group's U is that of a rigid body's translation, libration and their
correlation, U = T + A L A^T + A S + S^T A^T, of V. Schomaker and K. N. Trueblood, "On the rigid-body motion of
molecules in crystals" (1968), Acta Cryst. B24, 63-76
[doi:10.1107/S0567740868001718](https://doi.org/10.1107/S0567740868001718), refined against the likelihood with a
residual B per atom, and a model's B column read as the total or as the residual to its TLS groups, as in
M. D. Winn, M. N. Isupov and G. N. Murshudov, "Use of TLS parameters to model anisotropic displacements in
macromolecular refinement" (2001), Acta Cryst. D57, 122-133
[doi:10.1107/S0907444900014736](https://doi.org/10.1107/S0907444900014736). The chains are cut into TLS groups as
TLSMD cuts them - contiguous segments, by dynamic programming over the fit of an isotropic TLS model to the
refined B factors: J. Painter and E. A. Merritt, "Optimal description of a protein structure in terms of multiple
groups undergoing TLS motion" (2006), Acta Cryst. D62, 439-450
[doi:10.1107/S0907444906005270](https://doi.org/10.1107/S0907444906005270). Atoms refined with a U of their own are
restrained by the rigid-bond postulate of F. L. Hirshfeld, "Can X-ray data distinguish bonding effects from
vibrational smearing?" (1976), Acta Cryst. A32, 239-244
[doi:10.1107/S0567739476000533](https://doi.org/10.1107/S0567739476000533), in the form of SHELXL's RIGU - the
components of U along the bond and across it in the bond's plane: A. Thorn, B. Dittrich and G. M. Sheldrick,
"Enhanced rigid-bond restraints" (2012), Acta Cryst. A68, 448-451
[doi:10.1107/S0108767312014535](https://doi.org/10.1107/S0108767312014535) - with SHELXL's similarity (SIMU) and
approximate-isotropy (ISOR) restraints as G. M. Sheldrick, "Crystal structure refinement with SHELXL" (2015), Acta
Cryst. C71, 3-8 [doi:10.1107/S2053229614024218](https://doi.org/10.1107/S2053229614024218) documents them. Which
displacement model a refinement keeps - isotropic, TLS, or a U per atom - is decided on the free set, as a test of
what the data support rather than a rule by resolution: E. A. Merritt, "To B or not to B: a question of
resolution?" (2012), Acta Cryst. D68, 468-477 [doi:10.1107/S0907444911028320](https://doi.org/10.1107/S0907444911028320).
**NCS restraints** — copies of a molecule in the model are found by sequence alignment with REFMAC5's criterion
(more than 15 residues aligned, 80% identity - here of the longer chain - and 2.5 Å rms after superposition;
Murshudov et al. 2011, above) and restrained locally, on pairs of
corresponding interatomic distances under the Geman-McClure robust loss, as REFMAC5's local NCS restraints are, with
the distance range of R. A. Nicholls, F. Long and G. N. Murshudov, "Low-resolution refinement tools in REFMAC5"
(2012), Acta Cryst. D68, 404-417 [doi:10.1107/S090744491105606X](https://doi.org/10.1107/S090744491105606X), and
the bonds and angles left out as they leave them out. The case for local restraints over a superposition, and for
deciding them by R-free, is G. J. Kleywegt's, "Use of non-crystallographic symmetry in protein structure refinement"
(1996), Acta Cryst. D52, 842-857 [doi:10.1107/S0907444995016477](https://doi.org/10.1107/S0907444995016477); I. Usón,
E. Pohl, T. R. Schneider, Z. Dauter, A. Schmidt, H.-J. Fritz and G. M. Sheldrick, "1.7 Å structure of the stabilized
REIv mutant T39K. Application of local NCS restraints" (1999), Acta Cryst. D55, 1158-1167
[doi:10.1107/S0907444999003972](https://doi.org/10.1107/S0907444999003972); and J. J. Headd, N. Echols,
P. V. Afonine, N. W. Moriarty, R. J. Gildea and P. D. Adams, "Flexible torsion-angle noncrystallographic symmetry
restraints for improved macromolecular structure refinement" (2014), Acta Cryst. D70, 1346-1356
[doi:10.1107/S1399004714003277](https://doi.org/10.1107/S1399004714003277), whose comparison of global, torsion and
no NCS is why the copies' B factors are refined independently here.
**Unmodelled density** — the blobs the refinement lists are the connected regions of the difference map above
a cutoff, found by flood fill as Coot's "unmodelled blobs" finds them, in GEMMI's implementation
(`find_blobs_by_flood_fill`): P. Emsley, B. Lohkamp, W. G. Scott and K. Cowtan, "Features and development
of Coot" (2010), Acta Cryst. D66, 486-501 [doi:10.1107/S0907444910007493](https://doi.org/10.1107/S0907444910007493).
**Special positions** — an atom on a crystallographic special position is placed on it and its shifts are
confined to the subspace its site symmetry leaves unchanged, as R. W. Grosse-Kunstleve and P. D. Adams treat
special positions, "Algorithms for deriving crystallographic space-group information. II. Treatment of special
positions" (2002), Acta Cryst. A58, 60-65 [doi:10.1107/S0108767301016658](https://doi.org/10.1107/S0108767301016658).
**Water build** — the waters the refinement adds and removes follow the documented ordered-solvent defaults of
phenix.refine: difference-map peaks from 3 σ, hydrogen-bond partners to 3.2 Å and picking only to 2.8 Å, with
the waters added, filtered and refined between macro-cycles:
Afonine et al. (2012, above);
the distances to the model that Coot's Find Waters keeps (2.4-3.2 Å), from the Coot manual and Emsley et al.
(2010, above); and the removal of a water in negative difference density at -3 σ, the cut Echols et al. (2014,
below) use to exclude a site.
**Solvent audit** — the waters and ions of the refined model are scored by their surroundings with the
stereochemical criteria of the surveys of ions at nucleic acids by P. Auffinger's group: the coordination bands
1.9-2.3 / 2.3-2.6 / 2.6-3.2 Å, the Mg²⁺ exclusion zone and the clash shells of N. Naleem, A. Henning-Knechtel,
S. Kirmizialtin and P. Auffinger, "Cat_Wiz: a stereochemistry-guided toolkit for locating, diagnosing, and
annotating Mg2+ ions in RNA structures" (2026), Nucleic Acids Res. 54, gkag120
[doi:10.1093/nar/gkag120](https://doi.org/10.1093/nar/gkag120); the Mg-O and Mg-N distances and the
2.3-3.4 Å exclusion zone of F. Leonarski, A. Henning-Knechtel, S. Kirmizialtin, E. Ennifar and P. Auffinger,
"Principles of ion binding to RNA inferred from the analysis of a 1.55 Å resolution bacterial ribosome
structure – Part I: Mg2+" (2025), Nucleic Acids Res. 53, gkae1148
[doi:10.1093/nar/gkae1148](https://doi.org/10.1093/nar/gkae1148); the Na⁺, K⁺ and water distances and
coordination numbers of F. Leonarski, L. D'Ascenzo and P. Auffinger, "Nucleobase carbonyl groups are poor
Mg2+ inner-sphere binders but excellent monovalent ion binders—a critical PDB survey" (2019), RNA 25, 173-192
[doi:10.1261/rna.068437.118](https://doi.org/10.1261/rna.068437.118), and the distance bands of their
2017 survey (above). The ligand sphere of the bond-valence sums and the reading of halides by the donors
around them follow N. Echols, N. Morshed, P. V. Afonine, A. J. McCoy, M. D. Miller, R. J. Read,
J. S. Richardson, T. C. Terwilliger and P. D. Adams, "Automated identification of elemental ions in
macromolecular crystal structures" (2014), Acta Cryst. D70, 1104-1114
[doi:10.1107/S1399004714001308](https://doi.org/10.1107/S1399004714001308), and the halide's positive and
negative neighbours N. Morshed, N. Echols and P. D. Adams, "Using support vector machines to improve elemental
ion identification in macromolecular crystal structures" (2015), Acta Cryst. D71, 1147-1158
[doi:10.1107/S1399004715004241](https://doi.org/10.1107/S1399004715004241); the chloride's shell of donors
at up to 3.4 Å is O. Carugo's, "Buried chloride stereochemistry in the Protein Data Bank" (2014), BMC
Struct. Biol. 14, 19 [doi:10.1186/s12900-014-0019-8](https://doi.org/10.1186/s12900-014-0019-8). The
bond-valence sums are I. D. Brown and D. Altermatt's, "Bond-valence parameters obtained from a systematic
analysis of the Inorganic Crystal Structure Database" (1985), Acta Cryst. B41, 244-247
[doi:10.1107/S0108768185002063](https://doi.org/10.1107/S0108768185002063), with the R₀ of N. E. Brese and
M. O'Keeffe, "Bond-valence parameters for solids" (1991), Acta Cryst. B47, 192-197
[doi:10.1107/S0108768190011041](https://doi.org/10.1107/S0108768190011041), as tabulated by H. Zheng,
K. M. Langner, G. P. Shields, J. Hou, M. Kowiel, F. H. Allen, G. Murshudov and W. Minor, "Data mining of
iron(II) and iron(III) bond-valence parameters, and their relevance for macromolecular crystallography"
(2017), Acta Cryst. D73, 316-325 [doi:10.1107/S2059798317000584](https://doi.org/10.1107/S2059798317000584),
weighted by occupancy as P. Müller, S. Köpke and G. M. Sheldrick weight them, "Is the bond-valence method
able to identify metal atoms in protein structures?" (2003), Acta Cryst. D59, 32-37
[doi:10.1107/S0907444902018000](https://doi.org/10.1107/S0907444902018000); their normalised vector sum and
its outlier bound are CheckMyMetal's: H. Zheng, D. R. Cooper, P. J. Porebski, I. G. Shabalin, K. B. Handing
and W. Minor, "CheckMyMetal: a macromolecular metal-binding validation tool" (2017), Acta Cryst. D73, 223-233
[doi:10.1107/S2059798317001061](https://doi.org/10.1107/S2059798317001061). The score that combines them is
our own. The divalent ion's place on a purine N7 - in the base's plane, 3.5-3.9 Å from its O6 or N6 - is the test of
Leonarski et al. (2017, above). A light ion is told from a water by a density above the waters', as Echols et al.
(2014, above) flag one; a halide's few or no donors are allowed as Z. Dauter and M. Dauter describe halides binding
also at hydrophobic surfaces, "Entering a new phase: using solvent halide ions in protein structure determination"
(2001), Structure 9, R21-R26 [doi:10.1016/S0969-2126(01)00565-2](https://doi.org/10.1016/S0969-2126(01)00565-2).
The anomalous scattering expected of each species is f″ at the data's wavelength by D. T. Cromer and D. A.
Liberman, "Anomalous dispersion calculations near to and on the long-wavelength side of an absorption edge"
(1981), Acta Cryst. A37, 267-268 [doi:10.1107/S0567739481000600](https://doi.org/10.1107/S0567739481000600),
with the corrections of L. Kissel and R. H. Pratt, "Corrections to tabulated anomalous-scattering factors"
(1990), Acta Cryst. A46, 170-175 [doi:10.1107/S0108767389010718](https://doi.org/10.1107/S0108767389010718), as
GEMMI computes them (its port of the Larch routines).
## Software and computing methods
Decoding bitshuffle+LZ4 images on the GPU, rather than decompressing them on the host and uploading
@@ -625,6 +801,13 @@ iterations" (2011), SIAM J. Numer. Anal. 49, 1715-1735
procedures for nonlinear integral equations" (1965), J. ACM 12, 547-560
[doi:10.1145/321296.321305](https://doi.org/10.1145/321296.321305).
The interval the report gives for ISa comes from a bootstrap over the unique reflections:
B. Efron, "Bootstrap methods: another look at the jackknife" (1979), Ann. Statist. 7, 1-26
[doi:10.1214/aos/1176344552](https://doi.org/10.1214/aos/1176344552). Its draws are Poisson weights
hashed from the replicate and the reflection with the SplitMix64 finaliser of G. L. Steele Jr.,
D. Lea and C. H. Flood, "Fast splittable pseudorandom number generators" (2014), Proc. OOPSLA 2014,
453-472 [doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
The single-precision exp the Bragg integration's profile fit shares between the CPU and an Apple GPU
(`BraggExp.h`) is built from textbook error-free floating-point arithmetic: the argument reduction of
W. J. Cody and W. Waite, "Software Manual for the Elementary Functions" (1980), Prentice-Hall; the
docs/CHANGELOG.md
+10
View File
@@ -1,6 +1,16 @@
# Changelog
## 1.0.0
### 1.0.0-rc.181
* Rugnux: Several rotation sweeps of one crystal are merged as one dataset (`rugnux a_master.h5 b_master.h5`), on a common indexing and cell, with a non-isomorphous sweep left out and reported; `--sweeps separate` processes each sweep on its own. Bruker `.sfrm` frames and imgCIF files with a full goniometer axis chain are read.
* Rugnux: More robust rotation processing - a retry with a more sensitive spot finder when no lattice is found, a self-consistent detector distance, long axes and doubled cells tested as hypotheses, a `-C` supercell multiple stepped down, and each ice ring merged, down-weighted or rejected on its own.
* Rugnux: Serial stills are merged with a partiality-aware error model, reported as `ERROR_MODEL_A/B/C` with no ISa or R_merge/R_meas/R_pim; runs without `-S` have an overall intensity scale about 0.86-0.88 times that of earlier versions, with unchanged statistics. A second crystal form is left out of the merge and reported, the indexing ambiguity is resolved more reliably, and `--model` no longer influences processing.
* Rugnux: The reported CC1/2 is the standard unweighted one, which can be lower than before on sweeps with weak frames; ISa comes with a 95% interval (`ISA_LOW`/`ISA_HIGH`).
* Rugnux: Model refinement adds anisotropic/TLS, local NCS and twin refinement where R-free supports them; building waters and relabelling ions is now off by default (`--refine-solvent` turns it on).
* NXmx: A detector of several modules is written as `module_0` ... `module_N` only, so DIALS reads each module as its own panel.
* Other: JPEG previews use a bundled encoder (libjpeg-turbo no longer needed), Rugnux prints how to cite it, and the release archives contain the HTML documentation.
### 1.0.0-rc.180
* Rugnux: A placed model is refined by default - restrained maximum-likelihood refinement on the GPU (CPU without one), writing `<prefix>_refined_model.cif`, `<prefix>_refined_maps.mtz` and the difference-map blobs in `<prefix>_blobs.json`; `--no-refine` skips it.
docs/CPU_DATA_ANALYSIS_DECISIONS.md
+4 -2
View File
@@ -82,7 +82,9 @@ The median needs three observations, so a reflection measured once or twice —
By default the reported/written high-resolution limit is trimmed where $\mathrm{CC}_{1/2}$ falls off: a logistic is fitted to $\mathrm{CC}_{1/2}(s)$, and the limit is set **one reported-shell width past** the point where the fit crosses 0.30 — deliberately "one shell too far", so weak-but-real data below the crossing are kept rather than discarded. The extension is measured over the range that is actually kept, not the full measured range, so a detector reaching far past where the crystal diffracts cannot inflate it. `--scaling-high-resolution` overrides the limit and `--resolution-cutoff off` disables it.
**Ice sits out of this fit, and only this fit.** A powder ring is reproducible: past the crystal's own limit the ice is still there and still the same in both half-sets, so the two halves agree *about the ring*, and a Pearson $\mathrm{CC}_{1/2}$ cannot tell that agreement from diffraction. Shells have been measured at $\mathrm{CC}_{1/2}=0.765$ where $\langle I/\sigma\rangle$ is $-0.1$ and $R_\mathrm{meas}$ is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, still written and still counted in the shell table (§10.10), and what changes is only that the resolution **decision** now reads the same curve that scaling, the error model and the space-group search already read.
**The fit runs on an internal, weighted $\mathrm{CC}_{1/2}$, not on the reported one.** An observation from a frame at scale $G$ below the run's typical scale $G_\mathrm{ref}$ was scaled up by $G_\mathrm{ref}/G$ and carries $(G_\mathrm{ref}/G)^2$ more counting variance than the same observation at the typical scale. On rotation each reflection's half-set pair therefore enters the fit's Pearson sums with a weight in $(0, 1]$: the variance its two half-set means would have had with every observation at the typical scale, over the variance they have. It is 1 on a sweep without a weak stretch, and small for a reflection measured only where the crystal barely diffracted, whose scaled-up noise would otherwise pull the curve down as much as a well-measured pair holds it up. The same per-frame factor sets the information each observation carries in the ΔCC1/2 frame ledger (§ frame disposition in [integration](CPU_DATA_ANALYSIS_INTEGRATION.md)), and the two-pass and beam-centre guards read this weighted form too. The **reported** $\mathrm{CC}_{1/2}$ — `CC_HALF`, `CC_STAR`, the shell tables and the mmCIF — is the standard one of XDS, Aimless and DIALS: the plain Pearson correlation of the $1/\sigma^2$-weighted half-set means, every reflection counted alike. Stills weight nothing in either.
**Ice sits out of this fit, and only this fit.** A powder ring is reproducible: past the crystal's own limit the ice is still there and still the same in both half-sets, so the two halves agree *about the ring*, and a Pearson $\mathrm{CC}_{1/2}$ cannot tell that agreement from diffraction. Shells have been measured at $\mathrm{CC}_{1/2}=0.765$ where $\langle I/\sigma\rangle$ is $-0.1$ and $R_\mathrm{meas}$ is 470 %, inside the written range because the logistic was fitted through them; against archived references, runs whose frames carry ice were written a mean twelve per cent finer than the reference where runs without ice were written four per cent finer, and every over-claim past twenty per cent has rings. The same ring drags the curve the other way where the data are good — one shell of forty falling to 0.51 because the two strongest ice lines cross it, with the shells either side at 0.99 — which is a real defect of those reflections but narrower than the shell it defames. Both signs are the same cause and both leave the fit: the reflections flagged in §3.3 are still merged, written and counted in the shell table wherever the per-ring decision of §10.10 keeps them, and what changes is only that the resolution **decision** now reads the same curve that scaling, the error model and the space-group search already read.
**A cut the shell table refutes is not quoted.** Two things keep the number inside what the bins show. The fitted crossing has to lie **inside the fitted bins**: on a real fall-off it sits between the last bin at the target and the first bin below it, with the extension bins to spare, and a crossing past the last fitted bin is an extrapolation of a fall-off these data never showed — the crossing is then read off the bins themselves, between the two that straddle the target. And no single resolution is quoted at all where the curve is not a fall-off: where $\mathrm{CC}_{1/2}$ never reaches the target in any shell, where the fit reads finer than the finest shell whose $\mathrm{CC}_{1/2}$ still reaches it, or where $\mathrm{CC}_{1/2}$ **climbs back over the target** after falling below it — a noise shell that climbs back is not the edge of the data. The report then says which of the three happened and points at the shell table instead.
@@ -132,7 +134,7 @@ The model electron density is sampled on a grid (IT92 X-ray form factors, with a
### 14.2 Bulk solvent and scaling
A flat bulk-solvent mask around the model is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$:
A flat bulk-solvent mask around the model (the REFMAC radii, on the density's grid, or on one of its own at 0.75 Å where that grid is too coarse for the mask's 0.8 Å shrink to act) is transformed to $F_\mathrm{mask}$, and the model is scaled to the observed amplitudes by an overall least-squares fit of a scale $k$, an anisotropic $B$, and the flat-solvent parameters $k_\mathrm{sol}, B_\mathrm{sol}$:
$
F_\mathrm{model} = k\,e^{-\mathbf{h}^\top \mathbf{B}\,\mathbf{h}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}\,s^2}\,F_\mathrm{mask}\right),\quad s^2 = 1/4d^2.
docs/CPU_DATA_ANALYSIS_IMAGE.md
+1 -1
View File
@@ -277,7 +277,7 @@ wrong one does not.
where its lattice also beats chance on the pooled spots (§4.1), so that one frame against none on a
sparse pattern cannot flip it. The decision holds for the rest of the run, and it flips the
**axis**, not the angles, because prediction reads the axis too. It runs before the second first
pass above and before the long-axis rescue, since with the sign wrong every candidate lattice is
pass above and before the long-axis hypothesis, since with the sign wrong every candidate lattice is
wrong.
**Post-refinement.** On a two-pass rotation run, whatever centre the first pass ends on is refined
docs/CPU_DATA_ANALYSIS_INDEXING.md
+16 -2
View File
@@ -109,7 +109,7 @@ For each direction, the FFT magnitude spectrum is computed; peaks correspond to
The running-mean background window keeps a **constant width** and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel's running sum is still valid.
The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it.
The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames.
Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$.
@@ -422,6 +422,19 @@ geometry refinement fits on the same population: its sampled frames are filtered
and the module placement are formed, the population told from the strong frames (at least 20 indexed
spots), whose cells show the valley between two forms where a weak frame's cell spreads across it.
Without a given cell the same question is asked of the frames on the run's own cell (within the
indexing tolerance of it), whose two forms can lie inside that tolerance. Each frame's cell is put on
the run's axes (of the orders of its axes, the one nearest the run's cell, so two near-equal axes indexed
the other way round compare alike), described by its edges as logarithms, and projected on the direction
the cells spread along most, which is the direction between two forms where there are two. The angles are
left out: a still's refined angles carry the indexing more than the crystal - one form comes out with its
monoclinic angle on one axis or another, or held at 90° - and spread more than two forms' angles differ.
The populations of that coordinate are found as above; the largest is merged, and the next, where it
holds at least a tenth of the frames and its mean shape is at least 1 % from the major one's (closer
groups are one form whose cell drifts), is left out of the merge and reported (`CELL_FORMS`, see the
report), with its cell to process it with `-C`. The cell written is the merged frames' own, with or
without a given cell.
**Detector modules.** Where the detector is described module by module
([Module geometry](RUGNUX_FORMATS.md#module-geometry)), the same refinement places each module with
enough spots on its frames' lattices as a rigid block - a shift and an in-plane turn - where that
@@ -431,4 +444,5 @@ gauge holds their spot-weighted mean shift, turn and outward push, and with the
also the two tilt-like gradients across the detector, at zero, so the beam centre, distance, tilt and
crystal orientations carry what all modules share. A coarse search of each module's position comes
first, scored like the beam centre by the spots that land on a lattice. The placement is used from
there on for a foreign multi-panel detector; for a JUNGFRAU master it is measured and reported only.
there on for a foreign multi-panel detector and for modules given as a CrystFEL `.geom`; for a JUNGFRAU
master it is measured and reported only.
docs/CPU_DATA_ANALYSIS_INTEGRATION.md
+7 -9
View File
@@ -158,7 +158,7 @@ The curve $\bar B(r)$ is the adaptive spot finder's peak-excluded background per
**When it is applied.** `--background-radial` takes `on`, `off` or `auto`; in Rugnux it is **`auto` by default** (the broker keeps it off), and auto is decided **per run, in the pre-scan, on empty apertures** — the same measurement that, on stills, also chooses how far the ring reaches (the $r_3$, up to the one the run starts from, whose ring predicts the empty disk best). On a grid of apertures away from every found spot, the ring's prediction of the empty disk is taken twice: plain, and with the correction applied exactly as the integrator applies it (the clip following the curve, and the curve's mean over the ring pixels kept minus its mean over the disk). With nothing in the aperture each difference is the error the ring makes in a reflection's background, noise plus bias. Rotation measures every stencil the spot width can give it (and the one it starts from, which the geometry pre-pass and the starvation guard return to) and reads the one it gets.
- **Stills** have no other correction, so it is applied to every image of a run where it lowers the apertures' mean square error at all.
- **Rotation** has one: under auto, each image whose peak-excluded ice score (§3.3) reaches `--ice-min-score` is corrected with the curve of its reflections' rings. The frame's curve on every image replaces that only where it takes **at least 10 %** of the apertures' mean square error away. On photon-counting sweeps the apertures gain 0.1–5 % (statistically significant, $z\approx3$–4), and correcting every image there was measured to cost ISa and anomalous signal; where it pays — the rings of home-source CCD frames, a smooth ice ring — they gain 13–31 %. A run below the margin keeps the per-image ice correction, bit for bit.
- **Rotation** has one: under auto, each image whose peak-excluded ice score (§3.3) reaches `--ice-min-score` is corrected, with the frame's curve as on stills. (It used the curve of the reflections' own rings until it was measured to miss the narrow peak of a smooth ice ring: on a crystal with smooth ice the ring cores stayed 4–9 standard errors high and their flanks low, against the off-ring reflections, and the frame's curve brought all three low-resolution rings within 3.) Correcting **every** image instead is chosen only where it takes **at least 10 %** of the apertures' mean square error away. On photon-counting sweeps the apertures gain 0.1–5 % (statistically significant, $z\approx3$–4), and correcting every image there was measured to cost ISa and anomalous signal; where it pays — the rings of home-source CCD frames, a smooth ice ring — they gain 13–31 %. A run below the margin keeps the per-image ice correction, bit for bit.
The model is a function of **radius alone**, so it is applied only where that is true of the background. The per-image ice gate rests on this: smooth powder ice *is* a radial feature and is corrected; ice made of discrete crystallite spots — which the profile channel is blind to and the spot channel catches — leaves no smooth ring to model, and correcting it makes matters worse. Measured against a fixed atomic model, comparing ice bands with resolution-matched decoy bands carrying no ice: on a crystal with pure smooth ice the correction removes **43 % of the bands' excess amplitude**, and the improvement is **7× larger inside the bands than outside**, which is its stated mechanism; on a crystal whose ice is textured the same correction *increased* the excess amplitude by half; on a clean crystal it is inert to four decimal places. A strongly polarised beam or panels of different response make the background non-radial too; the apertures see that as a ring the correction predicts *worse*, and a run where they do keeps it off. Auto engages only where a peak-excluded background exists (adaptive spot finding, §3.2) — a plain azimuthal profile carries the Bragg peaks and cannot support an absolute threshold, so without one auto leaves the correction off.
@@ -341,7 +341,7 @@ The partiality applied is fixed by the data type and scaling stage, not chosen f
3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma(d^*)$ — the run's rocking width, measured once on the intensity-weighted excitation errors and shared by every crystal. It is measured per group of frames of similar prediction window (four per shell), each against its own window, the shell taking the median of the groups whose curve is narrower than their window, so the frames with the widest windows do not set it everywhere; and it is measured in eight shells of equal width in $d^*$, made non-decreasing (pool-adjacent-violators, weighted by each shell's summed signal) and interpolated linearly in $d^*$ as a table. The three-term form $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius, the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature — is still fitted, sets the prediction window (§8.2) and stands in for the table where fewer than three shells are measurable. The curve is centred on the run's **radial scale** of the reciprocal lattice — a wavelength or cell a little off, which the spot positions cannot see since they fix only $\lambda d^*$ — measured from where the curve's centre lies against $d^{*2}$ and applied to $\mathbf{q}$. Where the frames were predicted through the measured window (§8.2), the post-refinement uses that same curve rather than measuring it again; the curve is written to `_process.h5` (`/entry/MX/stillsRockingWidth*`, `stillsRadialScale`), so `--mode scale` merges with it too. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times; the refined partiality is adopted on every crystal it was refined on. The per-crystal fits of $G$ and of the tilt measure each residual against its expected variance at the model, background + counting + $(0.3\,I_\mathrm{model})^2$: a still's strong reflections scatter about the model by tens of per cent, and in counting sigmas alone every strong reflection would be an outlier and the scale would be set by the weak ones.
Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy.
Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. On stills this holds for the merges the scale and the partiality are refined against. The final merge, which has the fitted error model, takes every observation: the model's partiality term $c\,((1-p)/p)\,\langle I\rangle$ (§10.5) weights a low-partiality observation by what it is worth. A reflection - each Bijvoet hand under `-A` - is reported only if at least one of its observations reaches $p \ge 0.2$; one seen only far out on its rocking curve rests on the partiality model alone. The error model is fitted on the observations at $p \ge 0.2$.
### 10.3 Smoothing of per-frame scales
@@ -388,9 +388,9 @@ Per-shell and overall merging statistics are computed on corrected intensities,
itself count as missing instead of leaving the denominator along with the data,
- the anomalous signal-to-noise $\mathrm{SigAno}$ and the half-set anomalous correlation $\mathrm{CC}_\mathrm{anom}$ (below).
The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly.
The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path adds a partiality term, $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2 + c^2\left(\frac{1-p}{p}\right)^2\langle I\rangle^2$, and reports **no ISa**: there $1/b$ is the $I/\sigma$ ceiling of one fully recorded still under the stills model - the same frames processed as a rotation sweep read several times higher - which describes the processing model more than the instrument and the crystal. It reports $a$, $b$ and $c$ instead (`ERROR_MODEL_A/B/C`, `_reflns.jfjoch_error_model_a/b/c`); $1/b$ is still used inside, by the space-group search.
$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention.
$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$ (and on stills $c_\mathrm{XDS} = c^2/a$). Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention.
**Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ($\mathrm{CC}_\mathrm{anom}$) is reported beside it: $\Delta I$ is formed once per half-set and the two are correlated over the acentric pairs where **both** hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic $\sigma$ cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS's `CCanom` and `phenix.merging_statistics`' `cc_anom`; XDS's `Anomal Corr` is a **different quantity and is not comparable with it** — on the same observations it reads two to three times higher in the low shells. $\mathrm{CC}_\mathrm{anom}$ is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the `CCanom` column of the printed merge-statistics table and as `CC_ANOM=` in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints `-` and the results report writes no key, which is not the same statement as a value measured to be zero.
@@ -432,7 +432,7 @@ Each surface is **cross-validated** on the half-set agreement it is meant to imp
Each batch's $B$ is fitted on **resolution-shell means**, not on single observations: $\ln(I_\mathrm{ref}/I_\mathrm{obs})$ of one weak observation is unbounded and biased downwards — the observation appears in the response and in its own weight, and the logarithm needs $I_\mathrm{obs} > 0$, which keeps only the upward half of the noise — and on decayed data that bias grows with dose until it reverses the sign of the answer. The shells are laid inside the range the run actually diffracted to, and the fit carries an intercept as well as a slope, so a batch that is merely *dimmer* than the run (an attenuated beam, a mis-fitted frame scale) is not reported as damage. A batch whose shells are too weak to fit, or whose solved value reaches the bound the smoothing solve clamps to, is reported as **absent** rather than as a number. The first→last headline is reported only where a straight line describes the curve: radiation damage is progressive, so a curve that dips and recovers is a disturbance rather than dose, and is left to the sweep-quality report ([the results report](RUGNUX_REPORT.md)) to name.
**Frame disposition (rotation).** After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures **ΔCC1/2** — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run) with each reflection's error variance taken from the **observed** scatter of its own observations rather than from the error model's σ's: a bad stretch claims the same σ's as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group's own $(n, \sum I, \sum I^2)$ from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic's own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, $(1-CC^2)/\sqrt{n_\mathrm{refl}-3}$, and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is **both** several standard errors below zero (Fisher-transformed) **and** well below the run's own per-batch distribution (median − 3 robust σ, measured once before anything is removed), **and** where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame `merged` / `downgraded` / `rejected` ledger is described in [the results report](RUGNUX_REPORT.md).
**Frame disposition (rotation).** After the correction surfaces are fitted, and on the corrected fulls, Rugnux measures **ΔCC1/2** — the overall CC1/2 of the merged data with a group of images minus the CC1/2 without it, over the reflections that group touches — for each 10° batch of the sweep and for each stretch the sweep-quality diagnostic flagged. It is computed in the σ-τ form (no random half-dataset split, so the answer is the same every run), with each observation carrying the information $\min(1, (G/G_\mathrm{ref})^2)$ of its frame's scale against the typical one — the internal weighting of the automatic resolution cut ([decisions](CPU_DATA_ANALYSIS_DECISIONS.md) §13.4), not part of the reported $\mathrm{CC}_{1/2}$ — and with each reflection's error variance taken from the **observed** scatter of its own observations rather than from the error model's σ's: a bad stretch claims the same σ's as a good one, so an error-model estimate would read a batch that adds noise as a batch that adds precision, inverting the sign of the measurement. The leave-one-out is a subtraction of the group's own $(n, \sum I, \sum I^2)$ from the per-reflection totals, so measuring a group costs one pass over its observations rather than a re-merge, and reflections the group holds the only observations of are excluded from both sides — the published statistic's own restriction. Its standard error is reported beside it as that of a single CC1/2 on the same reflection count, $(1-CC^2)/\sqrt{n_\mathrm{refl}-3}$, and a ΔCC1/2 smaller than that says nothing; a group whose ΔCC1/2 is positive or near zero is not evidence of harm. A batch is removed only where its ΔCC1/2 is **both** several standard errors below zero (Fisher-transformed) **and** well below the run's own per-batch distribution (median − 3 robust σ, measured once before anything is removed), **and** where the per-image CC to the merge — an independent channel, measured on the partials one frame at a time — also says the stretch agrees with the run worse than a typical frame does, because selecting frames by their disagreement with the merge and then reporting that the merge improved is circular; its edges are then slid frame by frame with the whole stretch re-measured at each position — so the range is located finely while the count of reflections the verdict rests on stays that of the stretch — the worst range is removed, the reference is re-formed and the scan repeats, never past a quarter of the sweep and never over a stretch narrower than one rocking event. The resulting per-frame `merged` / `downgraded` / `rejected` ledger is described in [the results report](RUGNUX_REPORT.md).
### 10.7 R-free test-set flags
@@ -475,15 +475,13 @@ A reference dataset (`--reference`, MTZ or SF-mmCIF) supplies known intensities
**Resolve the indexing (merohedral) ambiguity.** When the lattice symmetry is higher than the crystal's Laue symmetry (e.g. $P3$, $P4$, $P6$, $C2$), more than one indexing of the same lattice is geometrically valid, and the two solutions produce *different* merged intensities that a self-consistent scale cannot tell apart — which of them is the reference's own, only an external reference can say. The candidate reindexings are the identity together with the twin-law cosets of the metric symmetry (from the unit-cell metric and the Laue group); each is scored by the intensity correlation $\mathrm{CC}_\mathrm{ref}$ of the reindexed merge against the reference, and the data are re-merged in the best-correlating indexing. The reindex is **metric-preserving** — only the $hkl$ labels change, the cell is unchanged — and it is a no-op for a holohedral crystal, which has no twin laws (the lattice and Laue symmetry coincide). For rotation data this is done once, after the space group is determined, and the whole merge is then repeated in the chosen indexing. For stills, where each crystal is indexed independently and a merge of mixed indexings would average reflections that are not symmetry mates, it is done **per image**, at integration time: a run with a reference resolves the ambiguity image by image (the image's partiality/Lorentz-corrected intensities are correlated with the reference under each candidate operator, which is scale-invariant, and the best-correlating one is adopted; a crystal whose own cell decides the indexing keeps it, as described below). In neither workflow is the reference a **scale** target: both scale against their own data (§10.2), so $\mathrm{ISa}$ and the merging statistics come from the data alone and no cross-dataset systematic is imported. Because the stills choice is made at integration time, a later re-merge of stored reflections (`--mode scale`) cannot repair a dataset integrated without a reference; the data-only resolution below runs in the full pipeline only. Where there is no reference dataset but there is a **model** (`--model`), the reference intensities are computed from it instead - $|F_\mathrm{model}|^2$ from the atomic structure factors with a flat bulk-solvent contribution at the standard constants ($k_\mathrm{sol}=0.35$, $B_\mathrm{sol}=46$ Å$^2$), which are not fitted because there are no observations yet. Nothing is scaled against them; they serve only to rank the candidate indexings, and the correlation that does the ranking is scale-invariant. This needs the cell and space group up front (`-C` / `-S`, as serial indexing wants anyway); on rotation data the same job is done after the merge, in §14.5, where a merge exists to fit the model to properly.
**Serial stills without a reference or model.** The data cannot say which indexing is the *true* one, but they can say which crystals are in the *same* one, and that is all a merge needs. Where the space group is fixed (`-S`) and its lattice admits alternative indexings, the crystals are made consistent with each other before anything is scaled, following Brehm & Diederichs (2014). Each crystal's partiality- and Lorentz-corrected intensities are divided by the run's mean intensity in their resolution shell (20 shells of equal reciprocal volume), standardised within the crystal so that its unknown scale and offset drop out, and weighted by $1/(1+(\sigma/s)^2)$, $s$ being the crystal's spread, so that a weak or barely recorded reflection counts for little. The agreement of two crystals is the sum of the products of these values over the reflections they share, and the objective is that agreement summed over every pair of crystals, each in the indexing assigned to it. That sum is $(|M|^2 - \mathrm{const})/2$ with $M$ the per-reflection sum over all crystals, so a crystal's agreement with all the others is one dot product with $M$ less its own contribution: each crystal in turn takes the indexing in which that dot product is largest, $M$ is updated, and the passes repeat until no more than one crystal in a thousand moves. A pass costs one read of the reflections, not one per pair of crystals, which is what keeps tens of thousands of crystals to seconds. The objective only grows, so it converges; it is started four times (once from the indexing as found, three times from a seeded random assignment) and the start that ends highest is kept, and of the equally consistent answers related by an overall relabelling the one that moves the fewest crystals is taken. The order of every floating-point sum is fixed (crystals in image order, sequential passes), so the result does not depend on the thread count. The run reports how clearly the crystals separated: for each crystal the correlation with the mean of all the others in its chosen indexing and in the best other one, compared by Fisher's $z$ in units of its standard error ($\sqrt{2/(n-3)}$ for $n$ shared reflections) — the median of that margin and the share of crystals above $3\sigma$.
**Serial stills without a reference or model.** The data cannot say which indexing is the *true* one, but they can say which crystals are in the *same* one, and that is all a merge needs. Where the space group is fixed (`-S`) and its lattice admits alternative indexings, the crystals are made consistent with each other before anything is scaled, following Brehm & Diederichs (2014). Each crystal is compared on the square roots of its partiality- and Lorentz-corrected intensities, the sign kept: intensities follow the Wilson distribution, whose long tail lets a few very strong reflections dominate every product of two crystals, so that the strongest crystals settle among themselves and a group of them can hold one another in the wrong indexing against the weaker majority (on a large-cell serial run this left a fifth of the crystals in the wrong indexing); amplitudes have no such tail. They are put on the run's mean and spread in their resolution shell (20 shells of equal reciprocal volume; a shell past the resolution the run reaches holds only noise and may have a mean of zero or below, so it is not divided by its mean), standardised within the crystal so that its unknown scale and offset drop out, and weighted by $1/(1+(\sigma/s)^2)$, $s$ being the crystal's spread, so that a weak or barely recorded reflection counts for little. The agreement of two crystals is the sum of the products of these values over the reflections they share, and the objective is that agreement summed over every pair of crystals, each in the indexing assigned to it. That sum is $(|M|^2 - \mathrm{const})/2$ with $M$ the per-reflection sum over all crystals, so a crystal's agreement with all the others is one dot product with $M$ less its own contribution: each crystal in turn takes the indexing in which that dot product is largest, $M$ is updated, and the passes repeat until no more than one crystal in a thousand moves. A pass costs one read of the reflections, not one per pair of crystals, which is what keeps tens of thousands of crystals to seconds. The objective only grows, so it converges; it is started four times (once from the indexing as found, three times from a seeded random assignment) and the start that ends highest is kept, and of the equally consistent answers related by an overall relabelling the one that moves the fewest crystals is taken. The order of every floating-point sum is fixed (crystals in image order, sequential passes), so the result does not depend on the thread count. The run reports how clearly the crystals separated: for each crystal the correlation with the mean of all the others in its chosen indexing and in the best other one, compared by Fisher's $z$ in units of its standard error ($\sqrt{2/(n-3)}$ for $n$ shared reflections) — the median of that margin and the share of crystals above $3\sigma$.
A twin law is only an ambiguity for crystals whose own lattice cannot tell the indexings apart. On a pseudo-merohedral cell whose axes differ by a few per cent, each crystal's refined cell already says which axis is which, and the indexer has used that; letting the intensities overrule it there moves crystals that were right. So for each law the share of crystals whose refined cell fits the run's cell better as indexed than after the law is measured (a tie counting half — on an exactly merohedral lattice the share is about one half). Where it is at least 0.9 the law is decided by the cells: a crystal whose cell prefers its indexing stays there, as an anchor, and only the crystals whose cell does not are decided by the intensities, against the anchors. With a reference or a model the per-image choice above stands except where the cells decide: once all images are integrated, a crystal its own cell anchors goes back to that indexing, and the reference chooses only among the indexings left open to the others; the same data-only resolution is then run only to measure it: the report gives the number of crystals on which the data alone would have chosen otherwise.
### 10.10 Ice rings at the scale and merge stages
Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q = 2\pi/d$ (§1.2) of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are also left out of the **resolution-cut fit** (§13.4), which is the one consumer that reads the *merged* reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge's own accumulator (host and device alike) as well as on the stills one. They are otherwise **kept in the final merge**, which is also what the established scaling programs do by default, so the affected shells keep their completeness.
Nothing on an ice band is deleted from the merged output. Deleting the bands was implemented, measured against an external arbiter rather than against the merge's own statistics, and removed: on the one rotation-battery crystal where a band was both dead by its own merged $\mathrm{CC}_{1/2}$ and scorable by anomalous peak height, dropping it changed the mean anomalous density at the known sites by $-0.001\pm0.018\,\sigma$ — about 2 % of the site height — while removing 1149 unique reflections whose mean $I/\sigma$ was 3.62 against the dataset's own 3.05, i.e. better-than-average data, and costing 6 to 8 points of completeness in the affected shell.
Where the gate of §3.3 has found ice, reflections falling within $\pm w$ in $q = 2\pi/d$ (§1.2) of a hexagonal-ice band ($w=0.03$ Å$^{-1}$ offline, about the measured ring half-width) are marked. Marked reflections are **excluded where a model is fitted** — the per-frame scale $G$, the per-image correlation, and the $P1$ merge the space-group search runs on — because ice contamination is a *positive bias*, not extra scatter, and a least-squares scale absorbs it into $G$ and into the error-model $b$, where it damages every other reflection on the same frame. They are also left out of the **resolution-cut fit** (§13.4), which is the one consumer that reads the *merged* reflections rather than the observations: an ice-flagged observation marks its merged reflection, on the rotation merge's own accumulator (host and device alike) as well as on the stills one. The stills merge keeps them in the final merge. The **rotation merge decides ring by ring**, because what a ring does to its reflections differs between rings and between runs: it can bias them (up on the ring, where its powder intensity is under the spot; down on the flanks, where it is in the background ring), only add scatter, or leave them alone. Deleting every band unconditionally was implemented once and removed: on a crystal whose band was harmless it removed better-than-average reflections and 6 to 8 points of completeness in the affected shell for no measurable gain. So each ring is split into its **core** ($|q-q_\mathrm{ring}|<0.01$ Å$^{-1}$) and its two **flanks** (out to $w$), and every band is compared with the off-ring merged reflections within $0.02$ Å$^{-2}$ of the ring in $1/d^2$. Both are read against a log-linear **Wilson fit** of the mean intensity over those off-ring reflections (the fit behind the reported Wilson $B$; over all off-ring reflections where the window cannot be fitted), as $t=(I-I_\mathrm{exp})/\sqrt{\sigma^2+I_\mathrm{exp}^2}$, which has unit scatter whether a reflection is signal- or noise-dominated. A band whose mean $t$ departs from its neighbours' by more than 3 standard errors is **rejected** — a bias, which no larger $\sigma$ would remove; one whose variance of $t$ exceeds the neighbours' by more than 3 standard errors of the log ratio (taken from the samples' own kurtosis: Wilson intensities are far from Gaussian) is **down-weighted**, its excess variance added to each reflection's own; the rest is **merged** as measured. A band of fewer than 20 reflections, or a ring with fewer than 50 neighbours, is merged. The decision is applied to the merge's accumulators, so the written reflections, the shell table and the half-set $\mathrm{CC}$ all see the same thing, and each band's verdict is logged and reported (`ICE_RING_BAND_nn`, see the results report). It complements the radial background correction (§9), which removes most of the bias at its source: on 58 rotation crystals with ice and a model, it lowered the model's R-free against merging every band on 26 and raised it on one, by 0.003.
---
docs/CPU_DATA_ANALYSIS_REFINEMENT.md
+334 -17
View File
@@ -27,8 +27,12 @@ statistics and its map coefficients go out; the restraint dictionary is compiled
### 15.1 What is refined, and in what order
The model is taken as the refinement can use it: the first model of the file only, its hydrogens removed,
every anisotropic B replaced by its isotropic equivalent, and every B clamped to the range the refinement
allows (2 Ų up to twice the model's largest B, within 100-300 Ų). Occupancies are kept as they are, and
and every B clamped to the range the refinement allows (2 Ų up to twice the model's largest B, within
100-300 Ų). An atom's anisotropic U (ANISOU, `_atom_site_anisotrop`) is kept. Where the model has TLS
groups (`_pdbx_refine_tls*`, PDB `REMARK 3`) whose atoms bring no U of their own, their U is made from the
groups, and the B column is read either as the total B (as phenix.refine writes it) or as the residual to the
groups (as REFMAC writes it) - whichever of the two, and of the isotropic B as it stands, scores the lowest
R-free on the model as given (`REFINE_ADP_INPUT`). Occupancies are kept as they are, and
so are alternative conformations, each refined with its own restraints.
Two stages follow.
@@ -68,7 +72,10 @@ $$
F_\mathrm{model} = k\,e^{-\mathbf{s}^\top\mathbf{B}\mathbf{s}/4}\left(F_\mathrm{calc} + k_\mathrm{sol}\,e^{-B_\mathrm{sol}s^2/4}\,F_\mathrm{mask}\right),
$$
with the atoms' $F_\mathrm{calc}$ computed with the dictionary's riding hydrogens included (§15.4). The
with the atoms' $F_\mathrm{calc}$ computed with the dictionary's riding hydrogens included (§15.4), from the IT92
form factors: a single-atom ion (a metal or a halide the dictionary describes) as the ion its dictionary charge
makes it - Mg²⁺ and Na⁺ 10 electrons, K⁺, Ca²⁺ and Cl⁻ 18 - every other atom as the neutral atom, whatever formal
charge the file gives it (a carboxylate's O⁻, a lysine's N⁺); the refined model is written with those charges. The
scale, the anisotropic B, the bulk solvent and the likelihood's parameters are fitted at the start of each
**macro-cycle** (§15.3) and held fixed while the minimiser runs; the mask is drawn for the model as it is
at that point.
@@ -86,7 +93,8 @@ gradient gather, whatever the number of atoms. The derivative of a riding hydrog
**Scale and solvent.** The flat bulk-solvent mask is drawn with the REFMAC radii (probe 1.0 Å, shrink 0.8 Å)
on a grid of its own, with a spacing of $d_\mathrm{min}/3$ or 0.4 Å, whichever is finer (coarser only where
a cell would need more than $2^{26}$ points). On the atoms' density grid the shrink is lost at low
a cell would need more than $2^{26}$ points, and then never coarser than 0.75 Å nor than the density grid,
which a cell that large uses for its mask). On the atoms' density grid the shrink is lost at low
resolution: at 3 Å its 1 Å step leaves no grid point within the shrink radius. $k_\mathrm{sol}$ and
$B_\mathrm{sol}$ are searched over their physical range on the working reflections, and the overall scale
and anisotropic B are fitted at each point of that search (§14.2) - at the points of its fine pass by
@@ -166,19 +174,53 @@ analytic derivatives:
above it, $d_0$ the sum of the energy types' van der Waals radii, 2.6 Å where both atoms can hydrogen-bond
and 1.9 Å between a metal and N, O or S. The list is rebuilt at every macro-cycle.
**Special positions.** An atom closer than 0.5 Å to one of its own symmetry images (the depositions put theirs
within 0.03 Å of the site) is on a special position: the symmetry operations, each with its lattice translation,
that take it onto itself are its site symmetry (Grosse-Kunstleve & Adams 2002). It is placed exactly on the site -
the mean of its position and its images under the site symmetry - and its shifts are confined to the subspace
those operations leave unchanged: the gradient is projected by the mean of their rotations (along a 2-fold or
4-fold axis, nowhere on a centre of inversion). Its occupancy is kept as given, the coinciding copies each
counting it (0.25 on a 4-fold axis), it makes no contact with its own copies, each of its contacts is listed
once rather than once per copy, and it moves with no rigid body.
**Metal sites.** A metal - an ion, or a metal atom of a ligand such as a heme's iron - is restrained to the
donor atoms (O, N or S) that the model's LINK / `_struct_conn` records bind it to. Where the library has a link for the pair - its ZN-CYS, ZN-HISND,
ZN-HISNE, FE-CYS, FE-HISND and FE-HISNE - that link is used as it is, with the modification it makes to
the donor's residue (the proton the bond replaces). Every other record becomes a link of one bond at the
distance of the library's own statistics of metal sites (its `metals.json`, carried as `monomers/metals.cif`)
for the metal, the donor element and the site's coordination number - the number of such records the metal
has, plus the bonds its own residue's dictionary gives it (a heme iron's four pyrrole nitrogens) - interpolated linearly between two tabulated coordination numbers (rows resting on at least 20 sites
for the metal, the donor element and the site's coordination number - but only where the model makes the
contact an inner-sphere one: no further than that distance plus 0.33 Å (Mg-O 2.07 + 0.33 = 2.4 Å, the
inner-sphere bound of Leonarski et al. 2017, the same margin for every metal). A record at an outer-sphere
distance - deposited models carry many, a Mg "bonded" to a phosphate O at 2.9 Å, a Na to a water at 3.0 Å -
would pull the ion onto that atom; it stays a contact instead. The coordination number is the number of
records the metal has within its inner sphere, plus the bonds its own residue's dictionary gives it (a heme iron's four pyrrole nitrogens) - interpolated linearly between two tabulated coordination numbers (rows resting on at least 20 sites
count; σ is the row's standard deviation, at least 0.02 Å). Only the bond is restrained, not
the angles at the metal. A contact without a record stays a repulsion, and a record to a symmetry mate is
left out, as every bond to a symmetry mate is. The rmsd of these bonds - with the dictionary's own bonds of
left out, as every bond to a symmetry mate is. Which records stay contacts is decided once, on the model as given: one topology serves the whole refinement, its
solvent build included, so a record is not judged again on a distance the refinement has moved (a record at 2.50 Å
refused, the ion slid to 2.39 Å, would have been taken). The records are written out with the refined model as they were
read, without the names of the links made here. The rmsd of these bonds - with the dictionary's own bonds of
a metal within a ligand, a heme iron's to its porphyrin - is reported apart from the covalent bonds'
(`REFINE_RMSD_METAL_BONDS`).
**Anisotropic atoms.** An atom's U is held as its equivalent isotropic B and a traceless anisotropy,
$U = U_\mathrm{dev} + B/(8\pi^2)\,I$, so that everything that refines, restrains and bounds B works on an anisotropic
atom as on an isotropic one, its shape held. Both engines grid it with its full U (an anisotropic Gaussian per
form-factor term, cut at the radius of its widest axis) and give the target's derivatives in its six U elements by the
same FFT method: for $\rho_q = a_q \exp(\mathbf{r}^T M_q \mathbf{r})$, $M_q = -4\pi^2(8\pi^2 U + B_q + B_\mathrm{blur})^{-1}$,
$\partial\rho_q/\partial U = \rho_q\,(M_q + 2 M_q\mathbf{r}\,\mathbf{r}^T M_q)$. Every U stays positive definite: its
smallest eigenvalue (as B) at least the cycle's floor, its largest at most the ceiling of §15.1.
A **TLS group** gives each of its atoms $U = T + A L A^T + A S + S^T A^T$ at $\mathbf{r}$ from its origin,
$A = [[0, z, -y], [-z, 0, x], [y, -x, 0]]$ (Schomaker and Trueblood), plus a residual isotropic B per atom; its 20
parameters (the trace of S does not enter U) are refined against the X-ray target alone, the residual B held, with
the derivatives by the chain rule from the atoms' U derivatives ($G$ for T, $A^T G A$ for L, $2 A^T G$ for S, $G$ an
atom's symmetric matrix of derivatives). An atom refined with a **U of its own** is restrained, for each pair of
bonded or angle-related atoms, by the rigid bond $|(U_i - U_j)\mathbf{n}|^2/\sigma^2$, $\sigma$ = 0.004 Ų along the
line $\mathbf{n}$ between them (the components SHELXL's RIGU restrains), and by the similarity
$|U_i - U_j|^2/\sigma^2$, $\sigma$ = 0.04 Ų (SIMU); an atom in no pair (a water, an ion) by
$|U - \mathrm{tr}(U)/3\,I|^2/\sigma^2$, $\sigma$ = 0.1 Ų (ISOR).
**B restraints.** The B factors of two atoms are restrained to each other in proportion to their size, by
the symmetric Kullback-Leibler divergence of two isotropic Gaussians of variances $B_i$ and $B_j$:
@@ -192,6 +234,28 @@ common atom, 0.2 and 0.3 for the side chains and the rest, and 0.5 for atoms wit
among them, symmetry mates included) that no bond or angle relates. No atom's B goes below 2 Ų or above
the ceiling of §15.1.
**NCS restraints.** Where the model holds several copies of one molecule, they are found from the model alone: two
polymer chains of one kind (protein, or nucleic acid) are copies where their sequences align with more than 15
residues identical and identical in 80% of the longer chain or more, and their C-alpha (P) atoms lie within 2.5 Å
rms of each other after superposition - REFMAC5's criterion, the identity taken on the longer chain. The same
sequence in a conformation further away than that, such as a quasi-equivalent subunit of a virus capsid, is another
molecule. In the residues aligned to a residue of the same name, the atoms of the same name correspond - not a
hydrogen, not an alternative conformation, both refinable. Each copy is paired with the next of its molecule (and the
last with the first, where there are three or more), and each pair is restrained **locally**, on its interatomic
distances rather than on a superposition: every distance within a chain shorter than 4.2 Å in either copy, except those a bond or an angle of the dictionary already fixes, against the same
distance in the other copy, with the robust loss of Geman and McClure,
$$
E = rac{(\Delta/\sigma)^2}{1 + (\Delta/0.5\, ext{Å})^2},\qquad \Delta = d_\mathrm{copy\,1} - d_\mathrm{copy\,2},
$$
the squared normalised difference where the two distances agree and never more than $(0.5\, ext{Å}/\sigma)^2$ where
they do not. A region in which the copies really differ - a loop in a crystal contact, a side chain in another
rotamer, a domain turned on a hinge - lets itself go as the refinement moves it, without a rule deciding in advance
which residues to exclude, and rigid motions of whole regions cost nothing, since only distances within 4.2 Å are
compared. Waters and ligands are not restrained, nor are B factors: the copies' B factors are refined independently.
Whether the restraints are used at all, and how tightly ($\sigma$), the weight trials decide (§15.7).
### 15.6 The minimiser
Positions and B factors are minimised in turn, each by an L-BFGS of rugnux's own (`RefinementMinimiser`),
@@ -241,7 +305,41 @@ are:
stays within 0.001 of the best so far, and the trajectory ends at its best step. On a deposited model,
whose free set the depositor's refinement has already fitted, R-work falls while R-free rises, and the
trajectory ends where it began.
3. **The final ladder**: the trajectory's weights times $\{0.5, 1, 2\}$, kept where one displaces it.
3. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved
was never kept on 55 refinements, and taking it away changed no choice).
4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U
held), each trial as long as a step on the positions where the weights ended, held - only the displacement
parameters refine, so the trials differ in the displacement model alone; the model chosen then takes a step with
the positions refined, kept where it displaces its trial (where the geometry is out of bounds there, which held
positions cannot mend, each trial refines the positions too, a macro-cycle longer than a step): TLS groups with a residual B - the model's own, one per chain
of at least 50 refinable atoms, and the chains cut into 2, 4, 8 and 16 contiguous segments of at least 10 residues
as TLSMD cuts them (by dynamic programming over the least-squares fit of an isotropic TLS model, a quadratic B(r),
to the refined B factors), each cut where it gives more groups than the one before; and a U per atom, at the U
weight balanced as $w_{B,0}$ is (on the U shaken by 20% of its B) times $\{1/4, 1\}$. The cuts between one group
per chain and the finest run only where the finest fits the free set better (lower $-\ln L$) than one per chain,
and the U at its balanced weight only where the U at a quarter of it lowers R-free or $-\ln L$ of the free
reflections from where the weights ended; on 49 refinements of 33 structures neither the cuts' rule nor leaving
out the U weight $\times 4$ changed a choice taken. A
trial displaces the one before it (the first: where the weights ended) only where its geometry is within bounds
and both its R-free (beyond the tie) and the $-\ln L$ of the free reflections are lower: the free set tests the
parameters a model adds, so the choice is made on the evidence of each run, not at a resolution (Merritt 2012).
The refined U are written as ANISOU and the TLS groups as `_pdbx_refine_tls*` and `REMARK 3` (`REFINE_ADP_MODEL`,
`REFINE_ANISO_ATOMS`, `REFINE_TLS_GROUPS`). It is decided after the solvent build (§15.10), on the model with its
solvent: a water a round put in refines a U of its own in the per-atom trial as every refinable atom does, and no
TLS group holds a water.
5. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the
rule every trial is held to (its geometry within bounds and R-free lower beyond the tie), the model is written as
it came, scaled to the data, and the solvent build's edits are undone (`REFINE_AS_GIVEN`). What comes out is never
worse on the free set than what went in. This holds only where the free set is free to the model as given: a model
placed as given on a free set drawn by the run (`MODEL_FREE_SET`; in `--mode refine`, a file without free flags) has
most likely been refined against those reflections, and its R-free as given does not compete.
Where the model has copies (§15.5), the NCS restraints are one more weight: every rung of the first ladder is run
without them, and then with them at $\sigma = 0.1$ Å at the B factors' weight the rungs without chose (NCS moves the
X-ray weight a rung up as often as not, the B factors' choice it never changed on 39 refinements), and every step of
the trajectory also tries them changed - switched on where they are off, and in turn tightened ($\sigma$ halved) and
loosened ($\sigma$ doubled, off beyond 0.4 Å) where they are on. Listed after their twin without, the restraints are
kept only where they lower R-free by more than the tie. A model without copies runs exactly the trials it ran before.
$w$ never exceeds $w_0/8$. The trials of one set run side by side - on one GPU, on several, or on the CPU's
threads - each from the same start and taking nothing from the one before it, so the result does not depend
@@ -281,7 +379,132 @@ in the report. A blob read as `LIGAND` is density of ligand size where the model
molecule or an unmodelled side chain is that as well as the compound that was soaked. The readings point a
person at the density; they do not identify a compound.
### 15.10 GPU and CPU
### 15.10 The solvent audit
Every water and every single-atom ion of the refined model (and, with `--refine-score-only`, of the model as
given) is measured in its surroundings and its density and scored as each of water, Mg²⁺, Na⁺, K⁺, Ca²⁺ and Cl⁻
(NH₄⁺ is scored too, and only reported). The audit changes nothing; the solvent build below reads it to relabel
sites. Both run only with `--refine-solvent` (off by default for now).
**What is measured.** The model's N and O atoms around the site, symmetry mates included, counted in the
coordination bands of Cat_Wiz - 1.9-2.3 Å (Mg-like), 2.3-2.6 Å (Na, Ca), 2.6-3.2 Å (K, water) - and in the
clash shells 3.2-3.4 and 3.4-3.6 Å beyond them. Each neighbour is typed from the dictionary's energy type
(donor, acceptor or both) and from the chemistry of its residue: Lys NZ, Arg NE/NH1/NH2 and a polypeptide's
N-terminal N are positive; Asp/Glu carboxylate O, a C-terminal carboxylate and a nucleotide's phosphate O are
negative; His ring N and the Asn/Gln amide ends are both donor and acceptor, as the model's orientation of
them is a guess. Also measured: the shortest distance between two neighbours closer than 2.6 Å (the
octahedron's cis O…O), the nearest positive and negative group, the other ions within 4.2 and 6 Å, the
bond-valence sum $V = \sum_j q_j \exp((R_0 - d_j)/0.37)$ over the N and O within 3.5 Å for each cation
($R_0$ of Brese & O'Keeffe, $q_j$ the neighbour's occupancy) with its normalised vector sum (CheckMyMetal's
nVECSUM), and the 2mFo-DFc, mFo-DFc and anomalous maps at the site, in their rms.
**The electrons a site shows.** The site's own density is fitted to 2mFo-DFc with its neighbours' held - a local
real-space fit of one amplitude at a B tied to its surroundings: the map at points 0.3 Å apart within 1.0 Å of the
site, put on the model's scale, less the density the model's other atoms put there (each atom's IT92 density to
the data's resolution at its B, to 4.5 Å), against a water's density (O with its two H) at the environment's B
(the mean B of the N and O within 3.5 Å, as CheckMyMetal compares a metal's with its ligands'), least squares.
The amplitude, in waters, is the electrons it shows, and counted apart from the neighbours' that merge with it at
moderate resolution. The map's scale per model unit is the median, over the macromolecule's C, N and O within
6 Å (or all of them where fewer than three are), of each atom's height in the map over what the model puts there -
not the waters', whose occupancy is the model's guess (the depositions' waters show half the density their model
gives). An **ion** - whose label's restraints may hold it off its density - is read where that fit puts it,
within 0.6 Å of its modelled position (its shell, bands and bond valences are measured from there); a water where
it is modelled. On the refine and open arms the depositions' ions read Mg 11, Na 10, Cl 19, K 21-23, Ca 22-26 and
Zn 33 (median), the waters 4-6.
**The anomalous signal a site would show.** For each species, f″ at the data's wavelength (Cromer-Liberman, with
the corrections of Kissel & Pratt, as GEMMI computes it) at the environment's B, put on the scale of the anomalous
map by least squares on the model's S and P (at least three); without an anomalous map, or nothing to scale it
by, the term is left out.
**The score.** For each species, a plain sum of terms in log units:
| Term | What it is |
|---|---|
| first shell | the neighbours in the species' band against the coordination number it holds (Mg 6, Na 5-6, K 6-10, Ca 6-8, Cl 3-5, NH₄⁺ 4), quadratic; for water only more than four, or none at all, costs; for Mg, Na, Ca and Cl a missing ligand costs little (0.5), as the model may simply not have that water - and a halide binds at hydrophobic surfaces too, with few donors or none (Dauter & Dauter 2001), where its electrons and its anomalous signal tell it from a water |
| distances | for Mg, each neighbour in the band against Mg-O 2.07 ± 0.04 Å, Mg-N 2.19 ± 0.08 Å; for the others, whose distances are spread over their bands, the band is all there is |
| contacts it does not make | 2 per N or O closer than the species' shell (closer than 2.9 Å for Cl), and for Mg per N or O in its exclusion zone 2.3-3.4 Å; for Mg and Ca 2 per first-shell purine N7 the site is more than 1.0 Å out of the plane of, or not 3.5-3.9 Å from the base's O6 or N6 (Leonarski et al. 2017) |
| chemistry | a cation: 2 per donor-only neighbour in its shell and per positive group within 3.5 Å; Cl: 2 per acceptor-only neighbour and per negative group within 3.2 Å; water: 2 per hydrogen bond beyond two donated or two accepted (neighbours at 2.6-3.4 Å) |
| bond valence | for a cation, $V$ against its charge $z$, Gaussian with a spread of $0.25z$ widened by the neighbours' distance errors, and the vector sum against CheckMyMetal's outlier bound 0.23 |
| electrons | the electrons the site shows against those the species would (its density's height over a water's at the environment's B: Mg²⁺ and Na⁺ about 11, K⁺, Ca²⁺ and Cl⁻ about 20) - an ion at full occupancy, a water as the model's waters show (their median) - log-normal with the spread of the waters' (an ion's occupancy and order vary as much), at least that of the map's calibration. A site as dense as a full 10-electron scatterer is so more than a typical water, as Echols et al. flag a light ion by a 2mFo-DFc above the waters' mean; where the density is weak the term tells little, and the shell decides |
| anomalous | the anomalous map at the site against what the species would show, Gaussian in the map's rms |
A site bound to another cation (closer than 2.6 Å) shares that cation's other ligands as neighbours - the
cis waters of a Mg(H₂O)₆ are 2.9 Å apart - and those are left out of its score. **Coordinate error**: each
atom's is taken as its rms displacement $\sqrt{B/8\pi^2}$ times $d_\mathrm{min}/10$ Š(about 0.1 Šat B = 20 Ų
and 2 Å); a neighbour's band is widened by the error of the distance and its distance term is widened by it
too, so that where the coordinates are poor the species' shells overlap and the score stops telling them
apart instead of telling them apart wrongly. The best of water, Mg, Na, K, Ca and Cl is reported, and
`AMBIGUOUS` where the runner-up is within 2. The audit counts a site as disagreeing with its label where
the best species is another one, not ambiguously.
**Caveats.** The refined model's metal-site distances are restrained (§15.5) where its LINK records name
the bonds, which pulls an ion's shell towards the label it carries - the first pitfall of every survey of
ions in the PDB; the local fit reads the ion where its density puts it, but its ligands stay where the restraints
held them. `--mode refine --refine-score-only` audits the model as given instead. The species are the six above:
a site denser than any of them (a cacodylate's As, a heavier metal) is called the heaviest that fits worst least.
Per site, everything above is in `<prefix>_solvent.json` (with `electrons`, `b_env`, `peak_shift_A`,
`anomalous_expected`, `n7_ligands`, `n7_off_plane`); the counts are the `REFINE_SOLVENT_` keys of the report.
**The solvent build.** Within the refinement, and to 2.8 Å (phenix.refine's ordered-solvent limit), waters are built
and removed and waters and ions relabelled, as phenix.refine builds waters within its macro-cycles. Where the weight
trajectory (§15.7) ends, a round reads the maps and the audit of that model and changes its solvent; then the solvent
alone - its waters and single-atom ions, the rest held - is refined at the weights the trajectory ended at (its B
too, at its balanced weight where the trajectory held B, and 10 iterations a macro-cycle), and the round is kept
where that lowers R-free; where it does not but the round as built does (refining the solvent alone can lose what its
sites gained: on a deposited model 41 waters lowered R-free by 0.0016 as built and raised it by 0.0036 refined), it
is kept as built; otherwise it is undone. The rest of the model goes on with its B factors held or refined as the
trajectory had them. The rounds stop at one that changes nothing, at one undone, or after two (a third gained at most
0.0004 in R-free where it was kept, on twelve sets, and was refused on the others); the final ladder of §15.7 then
refines what they kept with the rest. The atom list of a refinement is fixed - its engines are made for it - so the
waters a round may add are in the model from the start as placeholders: one water each (as many as a fifteenth of the
model's heavy atoms, at least 64) at occupancy 0, which puts no density, no mask and no contact anywhere and is not
refined until a round puts one at a peak; a water taken out goes to occupancy 0 the same way, and a relabelled site
keeps its place. A round thus costs a map, an audit and a refinement of the solvent: on twelve refine-arm and
open-arm depositions the build adds 0.2-0.9 times the refinement's time (median 0.7), where a refinement of its own
per round added 1-3.7 times (median 2.3), at an R-free within 0.001 of that on eight of them (two 0.0016-0.0028
higher, one 0.0016 lower). `--refine-score-only` builds nothing, and the build and the audit run only with `--refine-solvent`
(off by default for now).
- *Relabelled* is a water or a single-atom ion (no alternative conformer) whose best-supported species is another
one, not ambiguously - a water becomes an ion, an ion water or another ion - once per site. It takes the
species' residue name, element and atom name, and scatters as its ion (§15.2); its metal records go, and a metal
gets a record for each N, O and S of its inner sphere in the model's own copy (within the library's distance plus
0.33 Å, at the coordination of those within an octahedral site's), which the next refinement restrains as it
restrains the model's own (§15.5).
- *Candidates* are the local maxima of mFo-DFc from 3 σ (phenix.refine's peak cutoff), each symmetry-equivalent
set once, moved to the copy nearest the model. A peak closer than 1.2 Å to a model atom is that atom's own
residual and is passed over.
- *Not solvent* - reported in `<prefix>_solvent.json` with the reason, never filled: a peak 1.2-1.9 Å from a model
atom (a missing or displaced atom); one beside a model atom (within 2.6 Å) that sits in mFo-DFc of -3 σ or
lower (the atom is displaced); one within 3 Å of where a truncated residue's missing atoms would be (an atom the
dictionary bonds to an atom the model lacks); the centre of three or more other peaks at 1.3-1.7 Å (a
tetrahedral group, sulfate or phosphate) or any peak with another 1.1-1.9 Å away (atoms of one molecule);
elongated density (the rms length of the density above 3 σ within 1.6 Å along its longest axis over its shortest
above 3); and 2mFo-DFc above every S and P of the model (a heavier atom).
- *A water* is placed where the peak has a hydrogen-bond partner - an N or O that can donate or accept at
2.2-3.4 Å (phenix.refine's 3.2 Å, widened for the coordinate error), a metal within 2.6 Å (its ligand), or a
halide at 2.9-3.4 Å - and nothing closer than 2.2 Å to an N or O, nor than 3.0 Å to any other atom (Coot keeps
waters 2.4-3.2 Å from the model; a little wider here for the peak's position), strongest first, none within
2.2 Å of one already placed. It goes into the chain of its nearest atom, numbered on from that chain, with
occupancy 1 and the mean B of its partners. A peak on a special position (§15.5) is placed exactly on it with
occupancy 1 over the site's multiplicity, its coinciding copies one water (on the refine and open arms this was
better or equal in R-free on all five sets with such peaks, against refusing them); one 0.5-2.2 Å from its own
copy, which it would clash with, is refused.
- *Removed* is any water whose mFo-DFc at its centre is -3 σ or lower (the cut Echols et al. use), and a water the
build itself added that has no N, O or ion within 3.4 Å any more or whose B reached the refinement's ceiling.
phenix.refine's other filter, 2mFo-DFc below 1 σ at a refined water, is not used: on the refine arm it raised
R-free wherever it was tried, on the deposited waters (five sets of five) and on the waters built here, and so
did taking out deposited waters without a partner. A weak water still explains density, and a deposited one
carries its depositor's judgement.
`REFINE_SOLVENT_ADDED` and `REFINE_SOLVENT_REMOVED` count the waters added and still there and those of the model
as given taken out, `REFINE_SOLVENT_RELABELLED` the sites relabelled (each, with its scores, in the report and the
JSON); `REFINE_SOLVENT_NOT_SOLVENT` the not-solvent peaks of the final model's difference map;
`REFINE_SOLVENT_BUILD_ROUNDS` the rounds kept. The model as given (`REFINE_R_FREE_START`, the `REFINE_SCORE_` terms) and the placement stage are the first refinement's; the refined model, its R-factors, geometry, maps and blobs are the last kept round's.
### 15.11 GPU and CPU
On a CUDA build the target, the gradient and the restraints are evaluated on the card - the density gather,
cuFFT, the composition of the symmetry and the target per reflection, the adjoint through a CSR list, the inverse transform and one warp per atom for its gradient; the restraints in one kernel whose
@@ -292,19 +515,98 @@ the card's budget, or with `--no-gpu`, or on a build without CUDA (macOS include
Metal path), the CPU engine computes the same target, to rounding, several times more slowly (seconds on a
GPU, typically 2-15 minutes on the CPU).
### 15.11 Not yet
### 15.12 Twinned data
Where the lattice admits a **twin law** for the group the data are merged in - a rotation of the lattice's metric
symmetry (Le Page two-folds within 3° of obliquity, the search the free set is grouped by, §10.7) that the group
lacks, one per coset of the group, with its square in the group so that it exchanges two domains - the placed
model is refined a second time, against the data as twinned by the law, and the two refinements are compared.
`rugnux <file>` does this on its own; nothing has to be given.
**The target.** A crystal twinned by law $T$ at fraction $\alpha$ records $(1-\alpha)I(\mathbf{h}) + \alpha I(T\mathbf{h})$
(Yeates 1997), so the model an observation is compared with is
$$
|F_\mathrm{twin}(\mathbf{h})| = k\,e^{-\mathbf{s}^\top\mathbf{B}\mathbf{s}/4}\sqrt{(1-\alpha)\,|F_\mathrm{calc}(\mathbf{h}) + F_\mathrm{sol}(\mathbf{h})|^2 + \alpha\,|F_\mathrm{calc}(T\mathbf{h}) + F_\mathrm{sol}(T\mathbf{h})|^2},
$$
and the target is the likelihood of §15.2 with $|F_\mathrm{model}|$ replaced by $|F_\mathrm{twin}|$, $D$ and $\Sigma$
fitted against it (§15.3). At $\alpha = 0$ it is the untwinned target exactly. It is an approximation: a twinned
amplitude is treated as Rice-distributed about $D|F_\mathrm{twin}|$, while the distribution of a sum of two
intensities is narrower; $\Sigma$, fitted to the free set, absorbs that, and REFMAC5's twinned-intensity likelihood
(Murshudov et al. 2011) is the treatment this one simplifies. The mates $T\mathbf{h}$ that were not observed are
composed as rows of their own, with no observation: their structure factors come from the same transform, so a
twinned evaluation costs no extra FFT, only one more kernel per evaluation on the GPU (the twinned term per
reflection and the gather of its mate's derivative, the law being an involution on the rows).
**The fraction, the scale and the solvent** are fitted inside the twin sum at every macro-cycle: the scale and the
solvent as in §15.3, then $\alpha$, then the scale and the solvent again against $F_o$ detwinned by the model as it
stands, $F_o|F_\mathrm{model}(\mathbf{h})|/|F_\mathrm{twin}(\mathbf{h})|$ (at which scale the twinned model meets $F_o$
as the untwinned one meets the detwinned $F_o$), then $\alpha$ again. $\alpha \in [0, 1/2]$ comes from the regression
of $F_o^2$, with an intercept, on the model's intensities at $\mathbf{h}$ and at $T\mathbf{h}$, all normalised in
resolution shells of 500 reflections: $\alpha = b_{T\mathbf{h}}/(b_\mathbf{h} + b_{T\mathbf{h}})$, over the working
reflections that are not their own mate. The expectation of $F_o^2$ given the model, $D^2|F_\mathrm{model}|^2 + \Sigma$,
is linear in the two intensities, so an unrelated partner takes a coefficient of 0 however poor the model; a
least-squares fit of the twinned amplitudes instead took a fraction of 0.16 for unrelated partners of one model at
R-free 0.31, the average of two predictions being a better prediction than either.
**The decision.** The twinned refinement is kept where its fraction is above 0 and the law beats its own **null**
on the free reflections by at least three standard errors - the null being the same twinned model at the same
fraction with every reflection's mate replaced by an unrelated reflection of the same resolution and of the same
set (a free reflection's by a free one, as a law's are): what the averaging of two intensities buys whatever they
are (it lifts a weak reflection's model amplitude off zero). The comparison is paired, reflection by reflection,
$|F_o - |F_\mathrm{null}|| - |F_o - |F_\mathrm{twin}||$ over the free set, summed over each free twin pair (the two
mates share their two $F_\mathrm{calc}$, so they are one term, not two independent ones), its mean against its
standard error (`REFINE_TWIN_NULL_Z`). The R-free of the twinned refinement against the untwinned one's does not
enter: two refinements of one model differ in R-free by a few thousandths from their trajectories alone, and any
twin target lowers R by its averaging, so that comparison favours the twinned one with or without a twin. A pseudo-symmetry the model already carries (non-crystallographic
symmetry parallel to the law) predicts $I(T\mathbf{h}) \approx I(\mathbf{h})$, so the law changes nothing and is not
taken for a twin.
**The data's own evidence.** A lead of two to three standard errors is enough where the merge's intensity statistics
read twinned as well - the L-test itself below 0.44 (`TWINNING_VERDICT` `INDICATED` with
`ESTIMATED_TWIN_FRACTION_FROM` `L_TEST`; not where only the second moment reads low, nor where a pseudo-translation
leaves the L-test unreadable): two independent pieces of evidence, the data's and the model's, then agree.
`REFINE_TWIN_RULE` says which rule decided. The data-side test is not made a gate - it misses twins a model sees, a
pseudo-merohedral law or a near-perfect twin among them - only further evidence where the null alone is short.
`--mode refine` has no merge statistics, so there the null alone decides.
A law is tested only where the free set keeps its twin pairs together, and is otherwise reported
(`REFINE_TWIN` `FREE_SET_SPLIT`) rather than tested. The free set rugnux draws (§10.7) keeps together the pairs of
every law that is a rotation of the cell's own basis vectors; a law of a centred lattice that is not (its matrix
fractional in the cell's basis) can be split by it, and so can a free set `--mode refine` reads from a file. Of several laws, the one the placed model asks the largest fraction of is refined. Each of the
two refinements is a whole one - its solvent built (§15.10) and its displacement model chosen (§15.7) against its
own target - and the decision is taken where they ended. `--refine-score-only` makes the same test on the model
as given: it is scored against the data untwinned and twinned by the law, and the twinned score is taken by the
same rule.
**Maps.** Against twinned data the map coefficients use $F_o$ detwinned in proportion to the model's own share of
the twinned intensity, $F_o|F_\mathrm{model}(\mathbf{h})|/|F_\mathrm{twin}(\mathbf{h})|$, and $DF_c$ of the model's own
$|F_\mathrm{model}(\mathbf{h})|$; `FP` in `<prefix>_refined_maps.mtz` is the detwinned amplitude.
The keys are `REFINE_TWIN` (`NO_TWIN_LAW`, `FREE_SET_SPLIT`, `ZERO_FRACTION`, `UNTWINNED`, `TWINNED`),
`REFINE_TWIN_LAW`, `REFINE_TWIN_FRACTION`, `REFINE_TWIN_R_WORK`/`_R_FREE`, `REFINE_TWIN_R_FREE_NULL`, `REFINE_TWIN_NULL_Z` and
`REFINE_UNTWINNED_R_WORK`/`_R_FREE`, the last five of the two refinements where they ended. Where `REFINE_TWIN` is `TWINNED`, every
other `REFINE_` key, `REFINE_R_FREE_START` and the `REFINE_SCORE_` terms included, is of the twinned refinement.
### 15.13 Not yet
What the refinement does not do yet - each a known gap, not a design choice:
- **Custom ligands and other monomer libraries.** Only the 84 compounds of §15.4 are restrained. Any other
residue is held fixed (it stays in $F_\mathrm{calc}$, the mask and the repulsions); no restraints are
generated for a new ligand, and neither a user's restraint CIF nor an external monomer library can be given.
- **TLS** refinement.
- **Anisotropic ADPs**: an anisotropic B is replaced by its isotropic equivalent, and only isotropic B is
refined.
- **NCS restraints**: copies are refined independently.
- **Twin refinement**: the target is not a twinned one, also where the twin hypothesis' merge was written.
- **Water building**: waters are neither picked nor removed; the blob finder only points at candidates.
- **TLS restraints**: a TLS group's T and L are not held positive semi-definite (only every atom's total U is kept
positive definite); the TLS groups are contiguous segments of one chain, never a domain made of several.
- **Global NCS and NCS of B factors**: copies are restrained locally, on their interatomic distances (§15.5),
never by a superposition, and their B factors are refined independently; ligands and waters are not restrained
between copies.
- **More than two twin domains** (tetartohedral twinning): one law at a time is refined, and laws whose square is
not in the group are not considered.
- **Ion identification beyond the six species**: waters and ions are relabelled among water, Mg²⁺, Na⁺, K⁺, Ca²⁺
and Cl⁻ (§15.10); NH₄⁺ is only reported, transition metals (Zn, Mn, ...) are measured but never relabelled, and no
ion is placed at a peak that is not a water first. A site's hypotheses are compared by a local fit of its
density, not by refining each in turn.
- **Alternative conformations and occupancies**: existing conformers are refined in place; no occupancy is
refined and no conformer is added.
- **Rotamer and peptide-flip search**, and **Ramachandran restraints**.
@@ -317,21 +619,36 @@ What the refinement does not do yet - each a known gap, not a design choice:
- R. C. Agarwal, "A new least-squares refinement technique based on the fast Fourier transform algorithm", *Acta Cryst.* **A34** (1978), 791-809 (the gradient by the FFT method, §15.2).
- N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and its figure of merit, §15.2, §15.8).
- G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5).
- G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5; the twinned refinement §15.12 simplifies, its twin laws pruned by fraction and its fractions refined by least squares).
- R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with errors", *Acta Cryst.* **A42** (1986), 140-149 (the m, D weighting of the map coefficients, §15.8).
- T. O. Yeates, "Detecting and overcoming crystal twinning", *Methods Enzymol.* **276** (1997), 344-358 (the twinned intensity, §15.12).
- A. A. Lebedev, A. A. Vagin & G. N. Murshudov, "Intensity statistics in twinned crystals with examples from the PDB", *Acta Cryst.* **D62** (2006), 83-95 (twin laws from the lattice's metric, §15.12).
- R. Herbst-Irmer & G. M. Sheldrick, "Refinement of twinned structures with SHELXL97", *Acta Cryst.* **B54** (1998), 443-449 (twinned refinement with the fraction refined beside the model, §15.12).
- V. Y. Lunin & T. P. Skovoroda, "R-free likelihood-based estimates of errors for phases calculated from atomic models", *Acta Cryst.* **A51** (1995), 880-887, and V. Y. Lunin, P. V. Afonine & A. G. Urzhumtsev, "Likelihood-based refinement. I. Irremovable model errors", *Acta Cryst.* **A58** (2002), 270-282 (the likelihood's parameters from the free set, §15.3).
- J.-S. Jiang & A. T. Brünger, "Protein hydration observed by X-ray diffraction: solvation properties of penicillopepsin and neuraminidase crystal structures", *J. Mol. Biol.* **243** (1994), 100-115, and A. Fokine & A. Urzhumtsev, "Flat bulk-solvent model: obtaining optimal parameters", *Acta Cryst.* **D58** (2002), 1387-1392 (the flat bulk-solvent mask and its parameters, §15.3).
- P. V. Afonine, R. W. Grosse-Kunstleve & P. D. Adams, "A robust bulk-solvent correction and anisotropic scaling procedure", *Acta Cryst.* **D61** (2005), 850-855 (the grid search over the solvent parameters, §15.3).
- P. J. Huber, "Robust estimation of a location parameter", *Ann. Math. Statist.* **35** (1964), 73-101, and P. Charbonnier, L. Blanc-Féraud, G. Aubert & M. Barlaud, "Deterministic edge-preserving regularization in computed imaging", *IEEE Trans. Image Process.* **6** (1997), 298-311 (the robust scale fit and its smooth, pseudo-Huber loss, §15.3).
- A. A. Vagin, R. A. Steiner, A. A. Lebedev, L. Potterton, S. McNicholas, F. Long & G. N. Murshudov, "REFMAC5 dictionary: organization of prior chemical knowledge and guidelines for its use", *Acta Cryst.* **D60** (2004), 2184-2195 (the restraint dictionary, §15.4).
- M. Wojdyr, "GEMMI: A library for structural biology", *J. Open Source Softw.* **7** (2022), 4200 (the topology, the riding hydrogens and the blob search, §15.4, §15.9).
- V. Schomaker & K. N. Trueblood, *Acta Cryst.* **B24** (1968), 63-76, and M. D. Winn, M. N. Isupov & G. N. Murshudov, *Acta Cryst.* **D57** (2001), 122-133 (TLS, and the B column total or residual to it, §15.1, §15.5); J. Painter & E. A. Merritt, *Acta Cryst.* **D62** (2006), 439-450 (TLSMD's segments, §15.7).
- F. L. Hirshfeld, *Acta Cryst.* **A32** (1976), 239-244, A. Thorn, B. Dittrich & G. M. Sheldrick, *Acta Cryst.* **A68** (2012), 448-451, and G. M. Sheldrick, *Acta Cryst.* **C71** (2015), 3-8 (the rigid-bond, similarity and isotropy restraints of anisotropic atoms, §15.5); E. A. Merritt, *Acta Cryst.* **D68** (2012), 468-477 (the displacement model decided by test, §15.7).
- A. Blondel & M. Karplus, "New formulation for derivatives of torsion angles and improper torsion angles in molecular mechanics: elimination of singularities", *J. Comput. Chem.* **17** (1996), 1132-1141 (the torsion derivatives, §15.5).
- M. M. Harding, "Geometry of metal-ligand interactions in proteins", *Acta Cryst.* **D57** (2001), 401-411, and "Small revisions to predicted distances around metal sites in proteins", *Acta Cryst.* **D62** (2006), 678-682; F. Leonarski, L. D'Ascenzo & P. Auffinger, "Mg2+ ions: do they bind to nucleobase nitrogens?", *Nucleic Acids Res.* **45** (2017), 987-1004 (the metal-site distances, §15.5).
- R. W. Grosse-Kunstleve & P. D. Adams, "Algorithms for deriving crystallographic space-group information. II. Treatment of special positions", *Acta Cryst.* **A58** (2002), 60-65 (the site symmetry and the constraint of an atom on a special position, §15.5).
- D. T. Cromer & D. A. Liberman, "Anomalous dispersion calculations near to and on the long-wavelength side of an absorption edge", *Acta Cryst.* **A37** (1981), 267-268, and L. Kissel & R. H. Pratt, "Corrections to tabulated anomalous-scattering factors", *Acta Cryst.* **A46** (1990), 170-175 (f″ at the data's wavelength, §15.10).
- Z. Dauter & M. Dauter, "Entering a new phase: using solvent halide ions in protein structure determination", *Structure* **9** (2001), R21-R26 (halides at sites with few donors, §15.10).
- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10).
- J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6).
- A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7).
- P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7).
- A. T. Brünger, "Free R value: a novel statistical quantity for assessing the accuracy of crystal structures", *Nature* **355** (1992), 472-475 (R-free).
- G. J. Kleywegt, "Use of non-crystallographic symmetry in protein structure refinement", *Acta Cryst.* **D52** (1996), 842-857; I. Usón, E. Pohl, T. R. Schneider, Z. Dauter, A. Schmidt, H.-J. Fritz & G. M. Sheldrick, "1.7 Å structure of the stabilized REIv mutant T39K. Application of local NCS restraints", *Acta Cryst.* **D55** (1999), 1158-1167; R. A. Nicholls, F. Long & G. N. Murshudov, "Low-resolution refinement tools in REFMAC5", *Acta Cryst.* **D68** (2012), 404-417; J. J. Headd et al., "Flexible torsion-angle noncrystallographic symmetry restraints for improved macromolecular structure refinement", *Acta Cryst.* **D70** (2014), 1346-1356 (local NCS restraints with a robust loss, decided by R-free, and Murshudov et al. 2011 above for the copies found by alignment, §15.5, §15.7).
- P. Emsley, B. Lohkamp, W. G. Scott & K. Cowtan, "Features and development of Coot", *Acta Cryst.* **D66** (2010), 486-501 (unmodelled blobs, §15.9).
- N. Naleem, A. Henning-Knechtel, S. Kirmizialtin & P. Auffinger, "Cat_Wiz: a stereochemistry-guided toolkit for locating, diagnosing, and annotating Mg2+ ions in RNA structures", *Nucleic Acids Res.* **54** (2026), gkag120; F. Leonarski, L. D'Ascenzo & P. Auffinger, *Nucleic Acids Res.* **45** (2017), 987-1004 and *RNA* **25** (2019), 173-192; F. Leonarski, A. Henning-Knechtel, S. Kirmizialtin, E. Ennifar & P. Auffinger, *Nucleic Acids Res.* **53** (2025), gkae1148 (the coordination bands, distances and exclusion zone of the solvent audit, §15.10).
- P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367, and P. Emsley, B. Lohkamp, W. G. Scott & K. Cowtan, "Features and development of Coot", *Acta Cryst.* **D66** (2010), 486-501 (the peak cutoff, the partner and clash distances and the resolution limit of the water build, §15.10).
- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114, and N. Morshed, N. Echols & P. D. Adams, "Using support vector machines to improve elemental ion identification in macromolecular crystal structures", *Acta Cryst.* **D71** (2015), 1147-1158 (the ligand sphere and the halide rules, §15.10).
- H. Zheng et al., "CheckMyMetal: a macromolecular metal-binding validation tool", *Acta Cryst.* **D73** (2017), 223-233 (the vector sum, §15.10); O. Carugo, "Buried chloride stereochemistry in the Protein Data Bank", *BMC Struct. Biol.* **14** (2014), 19 (the chloride's shell, §15.10).
- I. D. Brown & D. Altermatt, *Acta Cryst.* **B41** (1985), 244-247, and N. E. Brese & M. O'Keeffe, "Bond-valence parameters for solids", *Acta Cryst.* **B47** (1991), 192-197 (the bond-valence sums, §15.10).
- M. Abramowitz & I. A. Stegun, *Handbook of Mathematical Functions*, National Bureau of Standards (1964), 9.8.1-9.8.4 (the Bessel functions of the target, §15.2).
The full citations, with DOIs, are in [ACKNOWLEDGEMENT.md](ACKNOWLEDGEMENT.md).
docs/EXTERNAL_TEST_DATA.md
+69 -17
View File
@@ -4,7 +4,7 @@ Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeli
ever sees its own detectors is not tested. The datasets below were collected by other people,
on detectors and in file formats we do not produce ourselves, and are used here to check that
`rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected
at other facilities, eleven on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
at other facilities, thirteen on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
else's detector and someone else's acquisition system. Their authors published all of these for
exactly this kind of reuse, and this page is where we credit them.
@@ -101,6 +101,8 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
| [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC |
| [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset |
| [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor |
| [6OQZ](https://www.rcsb.org/structure/6OQZ) | IRRMC [10.18430/m3.irrmc.5324](https://doi.org/10.18430/m3.irrmc.5324) | Home source, Bruker rotating anode | 1.60 | I 2 2 2 | 92.9 97.9 102.7 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of glucose isomerase from non-merohedrally twinned crystals |
| [6OR0](https://www.rcsb.org/structure/6OR0) | IRRMC [10.18430/m3.irrmc.5325](https://doi.org/10.18430/m3.irrmc.5325) | Home source, Bruker rotating anode | 1.55 | I 21 3 | 78.0 78.0 78.0 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of insulin from non-merohedrally twinned crystals |
| [6OWW](https://www.rcsb.org/structure/6OWW) | Zenodo [10.5281/zenodo.2943248](https://doi.org/10.5281/zenodo.2943248) | ALS 8.3.1 | 3.84 | P 1 21 1 | 85.8 86.0 214.3 90.0 89.9 90.0 | PILATUS3 6M | Crystal structure of a Human Cardiac Calsequestrin Filament Complexed with Ytterbium |
| [6P8J](https://www.rcsb.org/structure/6P8J) | SBGrid [10.15785/sbgrid/671](https://doi.org/10.15785/sbgrid/671) | APS 24-ID-E | 1.47 | P 1 21 1 | 46.5 152.2 88.3 90.0 90.1 90.0 | Dectris Eiger 16M | Structure of P. aeruginosa ATCC27853 CdnD D62N/D64N mutant bound to ATP |
| [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 |
@@ -150,6 +152,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
| [7Q6J](https://www.rcsb.org/structure/7Q6J) | Zenodo [10.5281/zenodo.4686876](https://doi.org/10.5281/zenodo.4686876) | PETRA III, DESY P11 | 2.20 | P 21 21 21 | 72.7 115.9 116.2 90.0 90.0 90.0 | PILATUS 6MF | Crystal structure of the human GDAP1 CMT2 mutant-H123R |
| [7QIJ](https://www.rcsb.org/structure/7QIJ) | SBGrid [10.15785/sbgrid/907](https://doi.org/10.15785/sbgrid/907) | PETRA III, EMBL c/o DESY P13 (MX1) | 4.10 | P 21 21 21 | 143.5 324.9 369.4 90.0 90.0 90.0 | PILATUS 6M-F | Complex of the Yersinia enterocolitica Type III secretion export gate YscV with substrate:chaperone complex YscX:YscY |
| [7QIS](https://www.rcsb.org/structure/7QIS) | IRRMC [10.18430/M37QIS](https://doi.org/10.18430/M37QIS) | BESSY 14.2 | 1.83 | P 61 | 100.3 100.3 206.2 90.0 90.0 120.0 | PILATUS3 2M | CRYSTAL STRUCTURE OF THE P1 difluoroethylglycine (DfeGly) BPTI MUTANT- BOVINE CHYMOTRYPSIN COMPLEX |
| [7QT9](https://www.rcsb.org/structure/7QT9) | Zenodo [10.5281/zenodo.5836055](https://doi.org/10.5281/zenodo.5836055) | Diamond I24 | 2.43 | I 1 2 1 | 45.2 54.5 115.7 90.0 101.1 90.0 | PILATUS3 6M | Room temperature In-situ SARS-CoV-2 MPRO with bound Z4439011584 |
| [7RAA](https://www.rcsb.org/structure/7RAA) | SBGrid [10.15785/sbgrid/881](https://doi.org/10.15785/sbgrid/881) | SSRL BL12-2 | 2.69 | P 43 21 2 | 66.4 66.4 298.3 90.0 90.0 90.0 | PILATUS 6M | Designed StabIL-2 seq15 |
| [7RIS](https://www.rcsb.org/structure/7RIS) | IRRMC [10.18430/M37RIS](https://doi.org/10.18430/M37RIS) | APS 21-ID-D | 1.72 | P 32 2 1 | 44.5 44.5 189.9 90.0 90.0 120.0 | Dectris Eiger 9M | Crystal structure of RPA3624, a beta-propeller lactonase from Rhodopseudomonas palustris, with active-site bound phosphate |
| [7RJI](https://www.rcsb.org/structure/7RJI) | IRRMC [10.18430/M37RJI](https://doi.org/10.18430/M37RJI) | LNLS W01B-MX2 | 1.71 | H 3 2 | 83.0 83.0 124.8 90.0 90.0 120.0 | PILATUS 2M | BthTX-II variant b, from Bothrops jararacussu venom, complexed with stearic acid |
@@ -244,7 +247,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
| [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 |
| [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) |
The table holds 201 PDB entries. Six more datasets, small-molecule / chemical-crystallography
The table holds 204 PDB entries. Eight more datasets, small-molecule / chemical-crystallography
sets kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero
detector 2θ, have no PDB entry and are listed [below](#datasets-with-no-pdb-entry).
@@ -313,7 +316,7 @@ the rest were deleted, so a run over the data directory sees a single collection
| 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å |
| 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** |
**Thirteen further archives hold more than one collection.** Their layout was read from the
**Seventeen further archives hold more than one collection.** Their layout was read from the
image files and repository listings; one sweep is kept for a run over the data directory unless
noted.
@@ -329,6 +332,10 @@ noted.
| lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run |
| 9E2T | one continuous sweep plus screening images | the 2700-frame sweep |
| 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) |
| 6OR0 | four 900 × 0.2° Bruker `.sfrm` runs of one twinned crystal at 10 cm: ω scans at 2θ = 0° (`01`) and 35° (`02`), φ scans on the fixed-χ (54.74°) platform at 2θ = 0° (`03`) and 35° (`04`) | the `01` run (2θ = 0°): the header distance of these SMART 6000 frames is ~4% short of what the data give, which the distance refinement corrects on a 2θ = 0° run and not yet on a swung one |
| 6OQZ | eleven `.sfrm` ω runs of 472-900 × 0.2° at 18 cm, 2θ = 10°, 25° and 45° (low, medium, high resolution) at different φ, plus two 5-frame runs at 2θ = 0° and the orientation-matrix runs `gdri_MATRIX_*` | the low-resolution `01` run (2θ = 10°), for the reason given for 6OR0; three crystals, the test set for multi-lattice integration |
| laspartic | seven Bruker-converted full-imgCIF kappa runs, φ (`01`, 1200 × 0.3°) and ω (`02`-`07`, 292-394 frames) at 2θ = -26.7° | the 1200-frame `01` φ run |
| onitroaniline | the same seven-run kappa layout (`01` 1200 × 0.3° φ, `02`-`07` ω), deposited twice: as Bruker `.sfrm` and as APEX's full-imgCIF conversion of the same frames | the `01` run, read from the `.sfrm` files |
| 7D98 | one 250 × 0.5° miniCBF sweep (frames 11-260), plus four HDF5 masters that repeat frames 61-260 in 60-frame pieces | the miniCBF sweep; the HDF5 files stay beside it and the row is pinned |
| 6H2P | two sweeps of one crystal, 1800 × 0.1° at 0.918 Å and 3600 × 0.1° at 1.893 Å (the Mn K edge) | **both**, as two battery rows, `6h2p_native` and `6h2p_1p89A`; the long-wavelength row has no deposited resolution to compare with |
| 8C3E | four 0.5° ω scans of one crystal at different (φ, χ) settings, 56, 150, 150 and 50 frames (`lcb2_8_newt_001` to `_004`) | the 150-frame `_003` scan |
@@ -347,9 +354,38 @@ noted.
frames (`<pdb>-images-1-180.zip`, `<pdb>-images-181-360.zip`, or `2wnq-181-360.zip`); both were
extracted into one directory, which holds the one 360 × 0.5° sweep.
## Multi-sweep battery rows
The opt-in `multisweep` arm of the battery (`tools/battery/multisweep.json`, see the
[battery README](../tools/battery/README.md#multi-sweep-rows-the-multisweep-arm-opt-in)) gives
rugnux several sweeps of the archives above at once and scores the one merge against the base
row's reference (7QT9, which has no single-sweep row, carries its own), plus which sweeps joined.
It uses only sweeps that are on disk: for 6UKF, 7DKP,
9PBB, 8U0I and 36GK the second collection was deleted (above), so they have no row until it is
downloaded again. Two-wavelength pairs (6IU8, 6H2P, 7RIS) are refused by rugnux as one dataset;
6IU8 is the row that checks it.
| Row | Base row | Sweeps | What it covers | Expected |
|---|---|---|---|---|
| `6or0_sweeps` | open/6or0 | 4 | one twinned crystal: omega and phi scans at 2theta 0 and 35 deg (swung detector, header distance ~4% short) | all join |
| `6oqz_sweeps` | open/6oqz | 4 | one crystal at 2theta 10, 25, 45 and 45 deg; multi-lattice; the header distance makes the low-2theta runs' cell ~3% short | all join |
| `lalanine_sweeps` | open/lalanine | 4 | small molecule, four runs of one crystal at 2theta 20 deg | all join |
| `laspartic_sweeps` | open/laspartic | 7 | small molecule, seven kappa runs of one crystal (phi + six omega), sparse, twinned | all join |
| `cytidine_omega` | open/cytidine | 3 | small molecule, three omega scans at phi 0/120/240 at 2theta 30 deg | all join |
| `8c3e_scans` | open/8c3e | 4 | home source, four omega scans of one twinned crystal at different (phi, chi) | all join |
| `8dyz_wedges` | open/8dyz | 4 | one crystal, four of its eight 50 deg wedges, translated between wedges, room temperature, 1.27 A | all join |
| `8dz7_crystals` | open/8dz7 | 4 | two crystals, two 50 deg wedges each, room temperature, 1.34 A | all join |
| `6o2h_crystals` | open/6o2h | 4 | four triclinic crystals, one 50 deg wedge each, room temperature, 1.21 A | all join |
| `7qt9_crystals` | open data, own reference | 20 | in-situ crystals at room temperature, twenty 20 deg wedges (the depositors merged them as 16 crystals), none complete alone | more than half join |
| `7l84_helical` | open/7l84 | 2 | two ~720 deg helical sweeps of one crystal, S-SAD at 1.892 A, room temperature | all join |
| `6fid_sweeps` | open/6fid | 2 | two 360 deg sweeps of one crystal (aligned and random orientation), S-SAD at 2.066 A | all join |
| `6iu8_mad` | open/6iu8 | 2 | keep separate: a two-wavelength MAD pair of one crystal - one dataset per wavelength | refused (keep separate) |
| `lyso_x10sa_90deg_pair` | inhouse/lyso_x10sa_90deg_1 | 2 | two sweeps of one crystal at the same orientation, 42 s apart | all join |
| `lyso_x10sa_two_crystals` | inhouse/lyso_x10sa_90deg_1 | 2 | keep separate: two lysozyme crystals of one form at one wavelength whose intensities differ (cells 1 % apart) | one left out (keep separate) |
## Datasets published as Raw Data Letters
Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a
Five of the datasets - 6R72, 8QQ7, 6RLR, laspartic and onitroaniline - were published as IUCrData Raw Data Letters, a
format whose purpose is to make raw images citable and re-processable in their own right. The
letters describe the collections and the difficulties in them, and are the reference for what the
data are:
@@ -364,6 +400,15 @@ data are:
IUCrData 7, x220852
[doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR.
- M. Lutz and L. Kroon-Batenburg, "Accurate intensity integration in the twinned γ-form of
*o*-nitroaniline" (2022), IUCrData 7
[doi:10.1107/S2414314622010598](https://doi.org/10.1107/S2414314622010598) - covers
onitroaniline.
- M. Lutz, "Twinned L-aspartic acid" (2025), IUCrData 10
[doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) - covers laspartic:
a twofold rotation about [100] with twin index 2 and obliquity 0.74°, whose coincidence lattice
is the doubled cell first reported for the compound (Bernal, 1931).
The authors of the second letter also published their own reciprocal-space reconstruction of the
6RLR data as a separate Zenodo record,
[10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763).
@@ -437,8 +482,8 @@ The PDB entry names the detector as a Rigaku HyPix-6000HE.
## Deposited models and structure factors
All 201 PDB entries in the table are released, and RCSB reports released structure factors
(`status_code_sf = REL`) for every one of them; the six small-molecule datasets have none. A merged
All 204 PDB entries in the table are released, and RCSB reports released structure factors
(`status_code_sf = REL`) for every one of them; the eight small-molecule datasets have none. A merged
result from this pipeline can therefore be checked against the deposited model or against the
deposited intensities.
@@ -591,6 +636,7 @@ should be read as a statement that a deposited model is wrong.
| PDB | Why |
|---|---|
| 7QT9 | Twenty 20° wedges (200 × 0.1°) of in-situ crystals at room temperature, none complete on its own: the dataset is a multi-sweep row (`7qt9_crystals`, below), not a single-sweep one. |
| 9VX7 | The uploaded sweep does not index in the deposited cell, with `rugnux` or with DIALS or XDS given that cell, while XDS indexes it de novo in a different cell; the deposited values cannot be scored against these images, so the row is not in the battery. |
## An archive that ships placeholder images
@@ -609,8 +655,10 @@ reader that globs `*.img` will pick them up, so they are named here rather than
| `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition |
| `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition |
| `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition |
| `laspartic` | Zenodo record 10.5281/zenodo.15432050 | a small-molecule dataset, not a PDB deposition |
| `onitroaniline` | Zenodo record 10.5281/zenodo.7193538 | a small-molecule dataset, not a PDB deposition |
Five of the six small-molecule sets have a published structure to check a run against. These are
Six of the eight small-molecule sets have a published structure to check a run against. These are
reference values from the literature, not results obtained here.
| Dataset | Space group | Cell (A, deg) | T | Reference |
@@ -620,6 +668,10 @@ reference values from the literature, not results obtained here.
| `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) |
| `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) |
| `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) |
| `laspartic` | `P 1 21 1` (4) | 5.1237 6.9197 7.6006 / 90 100.442 90 | 150 K | twin fractions 0.537/0.463; COD 1577465 - M. Lutz, IUCrData **10** (2025) [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) |
`onitroaniline` has no cell here: the letter describing it could not be retrieved when the row was
added, so a run on it is not scored against a reference.
`cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**,
064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no
@@ -632,20 +684,20 @@ symmetries rather than to be easy to process. The counts below describe where it
like everything else on this page, they are metadata about the depositions and their files, not
measurements.
- **Repository**, over the 207 datasets on this page (the 201 rows of the table and the six
small-molecule sets): IRRMC 95, Zenodo 42, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
- **Repository**, over the 212 datasets on this page (the 204 rows of the table and the eight
small-molecule sets): IRRMC 97, Zenodo 45, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
ESRF 3, UQ eSpace 1.
- **Facility** - counted from the facility part of the Facility / beamline column, the beamline
ignored so that entries deposited with and without one count the same, over the 190 rows that
name one: APS 29, Diamond 21, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9,
ignored so that entries deposited with and without one count the same, over the 191 rows that
name one: APS 29, Diamond 22, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9,
SLS 8, SPring-8 8, SOLEIL 7, SSRF 7, PAL/PLS 5, CHESS 4, ALBA 3, Australian Synchrotron 3,
CLSI 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, Photon Factory 2, and one each from NSRRC,
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other eleven
rows were collected on laboratory sources: nine on rotating anodes, one on a liquid-metal jet
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other thirteen
rows were collected on laboratory sources: eleven on rotating anodes, one on a liquid-metal jet
and one on a sealed tube.
- **Crystal system, from the deposited space group of the 201 rows:** monoclinic 53,
orthorhombic 50, tetragonal 30, trigonal 22, hexagonal 20, cubic 13, triclinic 13.
- **Pink beam:** none of these datasets was collected with pink beam. All 201 rows are deposited
- **Crystal system, from the deposited space group of the 204 rows:** monoclinic 54,
orthorhombic 51, tetragonal 30, trigonal 22, hexagonal 20, cubic 14, triclinic 13.
- **Pink beam:** none of these datasets was collected with pink beam. All 204 rows are deposited
as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all but 5REO, which
leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` `M`); 9Q41 is the
one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The
@@ -654,7 +706,7 @@ measurements.
- **Long cell axes:** sixteen rows have a deposited cell axis longer than 320 Å - 4P8R,
4MTK, 7D98, 8V4O, 9ZMU, 9V16, 9Z72, 4NWV, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL.
The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
The marCCD, SMV, Bruker `.sfrm`, full-imgCIF and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with.
## Licences
docs/EXTERNAL_TEST_DATA_STILLS.md
+7 -7
View File
@@ -30,8 +30,8 @@ the paper of the entry.
| `lcls242_s`, `lcls242_t` | [242](https://cxidb.org/id-242.html), [10.11577/3374214](https://doi.org/10.11577/3374214) | LCLS MFX | Rayonix MX340-HS; LCLS-I XTC | [9O4S](https://www.rcsb.org/structure/9O4S) (small cell), [9O4T](https://www.rcsb.org/structure/9O4T) (large cell) | P 1 21 1, 1.95 | Wolff et al., bioRxiv (2026) |
| `euxfel83` | [83](https://cxidb.org/id-83.html), [10.11577/1464102](https://doi.org/10.11577/1464102) | European XFEL SPB/SFX | AGIPD 1M, 16 modules of 512 x 128, 200 µm; Cheetah CXI | [6GTH](https://www.rcsb.org/structure/6GTH) | P 32 2 1, 1.69 | Wiedorn et al., Nat. Commun. (2018) |
| `euxfel152` | [152](https://cxidb.org/id-152.html), [10.11577/1660970](https://doi.org/10.11577/1660970) | European XFEL SPB/SFX | AGIPD 1M; Cheetah CXI | [6U57](https://www.rcsb.org/structure/6U57) | I 2 3, 2.80 | Echelmeier et al., Nat. Commun. (2020) |
| `alvra103`, `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) |
| `alvra213`, `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) |
| `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) |
| `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) |
Every set of the manifest is in the table and every set in the table is in the manifest, with the
same CXIDB entry, PDB entry, space group and d<sub>min</sub>; `tools/battery/doc_check.py` checks
@@ -63,8 +63,8 @@ broken download resumes with `curl -C -`.
| `lcls242_*` | `lcls_242/xtc/`, `lcls_242/xtc_more/`: `xtc/mfxl1013721-r0256` to `r0267` streams (the two crystal forms are in every run; each set keeps its own form) | ~90,000 | **subset**: runs 256-267 (44 of about 220 stream files; ~610 GB of ~3 TB) | ~700 GB |
| `euxfel83` | `euxfel_83/raw/`: `XFEL-r0243-c00.cxi`, `XFEL-r0278-c00.cxi`, `XFEL-r0280-c00.cxi` (Cheetah hits, from the entry's `cheetah-calib-mar-v1/hdf5/` tree on the NERSC archive); `euxfel_83/ref/agipd_mar18_v10.geom` from the entry | 2,790 | **subset**: 3 files; the deposited stream has 14,445 frames | 4.7 GB |
| `euxfel152` | `euxfel_152/raw/`: runs r0108, 0110, 0112, 0114, 0115, 0116, 0118, 0123, 0125 extracted from `XFEL2042-runs-cxi.tar` (range requests on the tar members); `euxfel_152/ref/kdo_embedded.geom`, the geometry embedded in the header of the entry's `KDO8PS.stream` | 3,801 | **subset**: 9 runs; the deposited stream has 37,006 frames | 5.7 GB |
| `alvra103` | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB |
| `alvra213` | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB |
| `alvra103_int16` (source) | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB |
| `alvra213_int16` (source) | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB |
**The masks of the SwissFEL geometry files.** rugnux takes the pixel masks and bad regions a CrystFEL
geometry file names. The SwissFEL files name their mask with an absolute path on the facility's disk,
@@ -78,9 +78,9 @@ that line differs.
**The `_int16` sets.** The SwissFEL files store converted JUNGFRAU data as uncompressed float32 keV,
68 MB (JF16M) or 17 MB (JF4M) a frame, which caps reading at about 74 or 295 frames/s at 5 GB/s. The
`_int16` sets are copies with the same layout in which `/data/data` holds whole photons (keV divided
by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller; the battery runs
them with the same arguments as the float32 originals (rugnux measures the gain from the frames), and
they merge to the same numbers. Make them with
by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller. The battery runs only the
copies: they merge to the same numbers as the float32 originals (rugnux measures the gain from the
frames), and the originals are kept only as the source of the conversion and as a test of the format. Make them with
`tools/battery/still/float32_to_int16.py <in.h5> <out.h5> <photon energy keV>` (h5py and hdf5plugin),
into `swissfel_103/conv/` (4.57 keV) and `swissfel_213/conv/` (12.037 keV, keeping the tarball's
directory tree), named `<input>_int16.h5`. One MbCO dark run (`run_000238`) holds no frames and has no copy.
docs/GPU_DATA_ANALYSIS.md
+31 -12
View File
@@ -166,7 +166,8 @@ the first card has no room for go on the others, each against its own budget.
as a gather over bricks of the grid, bit-identical from run to run.
[§14.1](CPU_DATA_ANALYSIS_DECISIONS.md#141-model-structure-factors)
- **Bulk-solvent mask** (`ModelMaskGPU.cu`) - the solvent mask from every symmetry image of every
atom, with island removal and the shrink, as gemmi computes it.
atom, with island removal and the shrink, as gemmi computes it - on the density's grid, or on one of its
own at 0.75 A where the density's is too coarse for the 0.8 A shrink to act.
[§14.2](CPU_DATA_ANALYSIS_DECISIONS.md#142-bulk-solvent-and-scaling)
- **Structure factors and maps** (`ModelStructureFactorsGPU.cu`) - F_calc and F_mask from the two grids
by cuFFT, composed over the group's operators and read off at the data's reflections; and the
@@ -269,7 +270,8 @@ and both gradients come back into pinned buffers. The minimiser (L-BFGS) stays o
Once per macro-cycle ([§15.3](CPU_DATA_ANALYSIS_REFINEMENT.md#153-scale-bulk-solvent-and-the-likelihood-parameters)):
- **F_calc and F_mask** - the forward path above without the target, and the bulk-solvent mask drawn on the card
(`ModelMaskGPU.cu`) on **its own fine grid** (d_min/3 or 0.4 A, whichever is finer, at most 2^26 points),
(`ModelMaskGPU.cu`) on **its own fine grid** (d_min/3 or 0.4 A, whichever is finer, at most 2^26 points but never coarser than 0.75 A nor than the
density's grid),
with a grid, spectrum and cuFFT plan of its own where that grid is not the density's: one block per
(atom, symmetry image) sets the points within the atom's REFMAC radius, the islands are removed by a
union-find labelling (integer `atomicCAS`, rooted at each component's lowest index, so the partition does not
@@ -284,28 +286,45 @@ Once per macro-cycle ([§15.3](CPU_DATA_ANALYSIS_REFINEMENT.md#153-scale-bulk-so
of the first round found **on the card** by a radix select (`residual_median`: the residuals are not negative,
so their order is that of their bits, found a byte at a time in four passes of integer counts - the element
`std::nth_element` puts at n/2). After a trial's first macro-cycle the fit starts from the fine pass around the
previous winner, skipping the coarse one. Its stream has the card's highest priority, so that the fit's short,
previous winner, skipping the coarse one. The fits go in batches of at most 32 (the coarse pass's 88 in three, in
grid order; a fit's sums do not depend on the others in its batch), and the robust weights take the residuals'
place, so the engine holds about 300 bytes per reflection. Its stream has the card's highest priority, so that the fit's short,
host-waited launches do not queue behind the long kernels of the other trials on the card.
- **D and Σ**, the likelihood's parameters, are fitted on the host (bin by bin, on all of the run's threads).
**Trials, cards and waits.** The weight trials of one set
([§15.7](CPU_DATA_ANALYSIS_REFINEMENT.md#157-the-weights-decided-by-trial)) run side by side, each on a slot of
its own - a copy of the model, its topology, a refinement engine and a scale-fit engine - as many slots as the
its own - a copy of the model, its topology and a refinement engine - as many slots as the
largest set of trials has (six at the defaults), the ones beside the main one built on
their own threads while the placement runs. The slots' engines go on the cards in
turn from the main one's, each card taking engines while they fit its budget; on a machine with several cards
the trials are spread over all of them. A slot whose engine cannot be made is left out rather than moved to the
turn from the main one's, each card taking engines while they fit its budget, so the first slots are on cards of
their own and on a machine with several cards the trials are spread over all of them. The engines on one card
share what does not change - the setup worked out on the host, the reflections' tables, the adjoint's lists and
the phase table - and the bulk-solvent mask's buffers (`RefinementGPUCard`). The working buffers of a transform -
the grid, its spectrum, cuFFT's work area, the mask's own grid and spectrum - come in lanes (`RefinementGPULane`):
a slot has a lane of its own where the card holds one more, and shares the least used lane otherwise, its engine
taking the lane's lock from its launches to their wait (an evaluation, F_calc, F_mask), so the lane runs one
evaluation after another while its other slots do their host work - the minimiser, the gradients, the restraint
lists. Lanes side by side keep a card busier (two evaluations at once on a T4 cost 9% more each than one); slots
sharing a lane overlap more host work. The slots on a card share its scale-fit engine too, a fit waiting for the
one before it. A slot whose engine cannot be made is left out rather than moved to the
CPU, and the outcome does not depend on how many trials run at once or on which card. The engine waits on its
stream through an event made with `cudaEventBlockingSync` (`StreamWaiter`), so a waiting trial sleeps instead of
spinning a core - with six engines on one card the spinning took half of the refinement's CPU time.
**Memory.** `RefinementGPUEngine::DeviceBytes` counts the engine's reserve before anything is allocated: per atom
(positions, boxes, densities, mask atoms, gradients), the grid, its spectrum and cuFFT's work area, the mask's
own grid and spectrum, per reflection (indices, scales, F_o, the row model, F_calc, F_mask, coefficients,
terms), the adjoint's lists, the gather's pair reserve - sized for every atom at the largest B a refinement may
reach with the widest blur - and the restraints at about 30 records and 60 slots per atom. The engine goes on the
card when that total, plus what is already resident there (a merge still at work), is within 80% of the card's
total memory; the trial slots are counted against the same budget, card by card.
(positions, boxes, densities, mask atoms, gradients), per reflection (scales, the row model, F_calc, F_mask,
coefficients, terms), the gather's pair reserve - sized for every atom at the largest B a refinement may
reach with the widest blur - and the restraints: the records and gradient slots of the model's two restraint sets
as the refinement starts (the geometry with its repulsions and NCS restraints, and the B restraints), a quarter more
for the contacts that come and go. `RefinementGPUEngine::CardBytes` counts the part the engines on a card share
(the tables, the adjoint's lists, the phases and the mask's buffers), `LaneBytes` a lane. The engine goes on the
card when the two, the scale-fit engine and what is already resident there (a merge still at work), are within
80% of the card's total memory (with a lane); the trial slots are counted against the same budget, card by card -
each one its engine's own part and a lane where both fit, else its engine's part alone, the first one on another
card also the shared part and a scale-fit engine. On a large model (510k atoms with riding hydrogens, 3.1M
reflections to 2.1 Å) an engine's own part is 2.0 GB, a lane 1.5 GB, the shared part 1.7 GB and the scale-fit
engine 0.9 GB: a 16 GB card takes three slots (three lanes; two where 80% of it is 12.5 GB), a 24 GB card five.
**Falling back.** Where the GPU cannot take the case (an atom's box wider than the cell, more than 192 mask
operators), where the engine is over the budget, or where CUDA fails while the
docs/HDF5.md
+18 -6
View File
@@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data.
This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead
of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data`
VDS still resolves to the original images when they are available. `rugnux` can also process
VDS still resolves to the original images when they are available. Where the input is anything but a
single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no
`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process
an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set.
## 3. NXmx-standard content
@@ -252,6 +254,11 @@ so it is a measurement. A user handing either to XDS should know which of the tw
### `/entry/instrument/detector/module` (NXdetector_module)
Written for a detector whose image is one flat panel (e.g. an EIGER). A detector of several modules
(a JUNGFRAU image assembled by Jungfraujoch, or an input whose modules are known) is written as
`module_0`, `module_1`, ... instead (next section), with no whole-image `module` beside them: NXmx
readers such as DIALS/dxtbx take every `NXdetector_module` as one panel.
`data_origin`, `data_size`, `fast_pixel_direction`, `slow_pixel_direction`, `module_offset` — all
NXmx (`fast/slow_pixel_direction` and `module_offset` carry transformation attributes). The two
pixel-direction vectors carry the discrete image orientation (mirror in Y, multiples of 90° about the
@@ -260,10 +267,14 @@ beam); for a detector that this system assembled itself, they are the McStas for
### `/entry/instrument/detector/module_<n>` (NXdetector_module, one per module)
The master files written here keep the single `module` group above. Rugnux writes the placement of a
detector's modules, where a stills run placed them, into `<prefix>_modules.h5`: one group per module in
the NeXus `NXdetector_module` layout that NXmx uses, and reads the same groups back (`--module-geometry`,
or a master that states its modules one by one):
A master of a multi-module detector states its modules one by one, in the order the `/entry/MX/peakPanel`
indices refer to. *n* is zero-padded to the width of the last index (`module_0` ... `module_7`, but
`module_00` ... `module_15`), so that the groups sort by name in module order and DIALS panel *n* is
module *n*; for an image Jungfraujoch assembled, each module is
its box of the image, as imaged. Rugnux writes the placement of a detector's modules, where a stills run
placed them, into `<prefix>_modules.h5` in the same layout, and reads the same groups back
(`--module-geometry`, or a master that states its modules one by one). Masters written by releases up to
1.0.0-rc.181 may carry the whole-image `module` beside `module_<n>`; they read back the same way.
| Field | Units / notes |
|-------|-------|
@@ -397,7 +408,8 @@ variants.
| `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below |
| `stillsRockingWidthFormula` | 1/Å, rad, relative | stills: the run's rocking width σ² = γ₀² + (γₑ d\*)² + (σ_bw λ d\*²/2)², as `[γ₀, γₑ, σ_bw]`; offline (Rugnux) only |
| `stillsRadialScale` | | stills: the scale of the reciprocal lattice the rocking curve is centred on |
| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\*, which the partialities use (held beyond the first and last entry; the formula above is what the prediction window uses); `--mode scale` merges with formula and table instead of measuring another |
| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\* before the image pass, which the frames were predicted through (held beyond the first and last entry; the formula above is what the prediction window uses); the partiality post-refinement balances it against the merge, and `--mode scale` starts from this same curve instead of measuring another |
| `stillsGivenUnitCell` | Å, deg | stills: the cell given with `-C`, whose population of frames the merge kept (`[6]`); `--mode scale` leaves out the same frames |
**Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is
processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by
docs/JFJOCH_BROKER.md
+3 -1
View File
@@ -186,7 +186,9 @@ The results - the merged data, the map coefficients and, with a model, the place
refinement - travel to the writer in the end message, which writes them beside the master file as
`<file_prefix>.mtz`, `<file_prefix>_maps.mtz`, `<file_prefix>_model.cif` and `<file_prefix>_model.pdb`,
and `<file_prefix>_refined_model.cif` (and `.pdb`), `<file_prefix>_refined_maps.mtz` and
`<file_prefix>_blobs.json`, as rugnux names them. An end message
`<file_prefix>_blobs.json`, as rugnux names them. These files name both programs as the software that
made them: the MTZ files in a HISTORY record beside Rugnux's, the models in their `_software` loop
(Rugnux, and Jungfraujoch for the data collection). An end message
that would be too large for the pushers' buffer (256 MiB) goes without them, with an error in the log
and a warning in the live status; the collection itself is not affected.
docs/JUNGFRAUJOCH.md
+12
View File
@@ -49,3 +49,15 @@ The broker streams images as CBOR messages compatible with the DECTRIS Stream2 p
([HDF5 / NeXus data format](HDF5.md)). Which downstream programs read which of the three HDF5
layouts, and the XDS plugin, are in [Integration with MX software](SOFTWARE_INTEGRATION.md). The
command-line tools shipped beside the services are listed in [Tools](TOOLS.md).
## How to cite
The primary citation for Jungfraujoch is F. Leonarski et al., "Jungfraujoch: hardware-accelerated
data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30,
227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268). For real-time
serial crystallography and live indexing feedback cite also J. Duan et al., "Jungfraujoch:
38 GB s⁻¹ real-time serial crystallography – data acquisition and live indexing feedback" (2026),
J. Appl. Cryst. 59, 1668-1677 [doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083).
The files `jfjoch_writer` writes from Rugnux live mode name both Jungfraujoch and Rugnux; cite the
[Rugnux paper](PUBLICATIONS.md#rugnux) too when you use them. All papers are on the
[Publications](PUBLICATIONS.md) page.
docs/PUBLICATIONS.md
+64
View File
@@ -0,0 +1,64 @@
# Publications
## How to cite
**Rugnux** — if you find Rugnux useful, please cite:
F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons and XFELs in
one program" (2026), bioRxiv [doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522).
**Jungfraujoch** — the primary citation is:
F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matěj, A. Castellane,
B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated data-acquisition
system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad. 30, 227-234
[doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268).
Work that relies on real-time serial crystallography, live indexing feedback or the system's
full data rate should also cite:
J. Duan, H.-C. Stadler, J. Nan, Z. Matěj, G. Gotthard, F. Dworkowski, M. Bjelčić, O. Aurelius,
A. Mozzanica, S. Grimm, M. Burian, T. Ursby, M. Wang and F. Leonarski, "Jungfraujoch: 38 GB s⁻¹
real-time serial crystallography – data acquisition and live indexing feedback" (2026),
J. Appl. Cryst. 59, 1668-1677
[doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083).
Data collected with Jungfraujoch and processed with Rugnux in the broker (Rugnux live mode) come out
of both programs, and the files `jfjoch_writer` writes from them name both: please cite both.
## Rugnux
- F. Leonarski and M. Wang, "Rugnux: from diffraction images to structure at synchrotrons and XFELs
in one program" (2026), bioRxiv
[doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522).
## Jungfraujoch
- J. Duan, H.-C. Stadler, J. Nan, Z. Matěj, G. Gotthard, F. Dworkowski, M. Bjelčić, O. Aurelius,
A. Mozzanica, S. Grimm, M. Burian, T. Ursby, M. Wang and F. Leonarski, "Jungfraujoch: 38 GB s⁻¹
real-time serial crystallography – data acquisition and live indexing feedback" (2026),
J. Appl. Cryst. 59, 1668-1677
[doi:10.1107/S1600576726008083](https://doi.org/10.1107/S1600576726008083) — the system at its
full data rate, with live indexing feedback during serial data collection.
- P. Gasparotto, L. Barba, H.-C. Stadler, G. Assmann, H. Mendonça, A. W. Ashton, M. Janousch,
F. Leonarski and B. Béjar, "TORO Indexer: a PyTorch-based indexing algorithm for kilohertz serial
crystallography" (2024), J. Appl. Cryst. 57, 931-944
[doi:10.1107/S1600576724003182](https://doi.org/10.1107/S1600576724003182) — the GPU indexing
algorithm behind Jungfraujoch's real-time indexing.
- F. Leonarski, J. Nan, Z. Matěj, Q. Bertrand, A. Furrer, I. Gorgisyan, M. Bjelčić, M. Kepa,
H. Glover, V. Hinger, T. Eriksson, A. Cehovin, M. Eguiraun, P. Gasparotto, A. Mozzanica,
T. Weinert, A. Gonzalez, J. Standfuss, M. Wang, T. Ursby and F. Dworkowski, "Kilohertz serial
crystallography with the JUNGFRAU detector at a fourth-generation synchrotron source" (2023),
IUCrJ 10, 729-737 [doi:10.1107/S2052252523008618](https://doi.org/10.1107/S2052252523008618) —
serial crystallography at 2 kHz with a JUNGFRAU 4M and Jungfraujoch at the BioMAX beamline of
MAX IV.
- F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matěj, A. Castellane,
B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang, "Jungfraujoch: hardware-accelerated
data-acquisition system for kilohertz pixel-array X-ray detectors" (2023), J. Synchrotron Rad.
30, 227-234 [doi:10.1107/S1600577522010268](https://doi.org/10.1107/S1600577522010268) — the
primary citation: the FPGA-accelerated acquisition system.
- F. Leonarski, A. Mozzanica, M. Brückner, C. Lopez-Cuenca, S. Redford, L. Sala, A. Babic,
H. Billich, O. Bunk, B. Schmitt and M. Wang, "JUNGFRAU detector for brighter x-ray sources:
Solutions for IT and data science challenges in macromolecular crystallography" (2020),
Struct. Dyn. 7, 014305 [doi:10.1063/1.5143480](https://doi.org/10.1063/1.5143480) — the data
handling problem of the JUNGFRAU detector that Jungfraujoch was designed to solve.
docs/RUGNUX.md
+21 -8
View File
@@ -39,8 +39,8 @@ the whole sweep gives. This is for rotation data and `--mode mx`; the collection
is ignored.
> **Note.** `rugnux` is under very active development. This page describes the tool and
> its options at a high level; the authoritative, always-current list of options is the program's
> own usage message — run `rugnux` with no arguments.
> its options at a high level. Run `rugnux` with no arguments for its usage - how to run each mode
> and the basic options; every option is on [Command-line options](RUGNUX_CLI.md).
```{contents} On this page
:local:
@@ -56,7 +56,7 @@ the data, and scaling and merging run by default. `-o` names the output files (d
**We expect no need to tune anything else.** The defaults are meant to be the answer, and a run that
needed a flag to do well is one we want to hear about. The other options exist for unusual cases and
for diagnosis; they are listed under [Command-line options](RUGNUX_ADVANCED.md#command-line-options).
for diagnosis; they are listed under [Command-line options](RUGNUX_CLI.md).
### Rotation data
@@ -99,8 +99,10 @@ rugnux -o myrun --model model.pdb dataset_master.h5 # or with --model
A model is told from the images by its contents (PDB, or mmCIF with atoms), gzipped or not. After
merging it is **placed** — as a rigid body, or by molecular replacement where that does not place it or
the model is a prediction (AlphaFold, ModelCIF) — then **refined** against the merge (restrained,
maximum likelihood; seconds on a GPU, typically minutes on the CPU; `--no-refine` skips it), and the
run writes the maps (σ<sub>A</sub>-weighted 2mFo−DFc / mFo−DFc), the placed and refined model, and
maximum likelihood; seconds on a GPU, typically minutes on the CPU; `--no-refine` skips it) with
its solvent as given; `--refine-solvent` (off by default for now, also with `--mode refine`) also rebuilds
the solvent - waters added at difference peaks and removed from negative density, ions relabelled
where their surroundings say so, each site audited - and the run writes the maps (σ<sub>A</sub>-weighted 2mFo−DFc / mFo−DFc), the placed and refined model, and
the blobs of difference density the model does not explain. R-work / R-free are in the report.
The model **does not change processing decisions**: the merge, its space group and resolution are
@@ -121,7 +123,7 @@ myrun.mtz merged intensities + French-Wilson amplitudes, for CCP4 / p
myrun.cif the same, as mmCIF - the self-describing format, and what to deposit
myrun.hkl the scaled observations unmerged, as SHELX HKLF 4 - for SHELXL, SHELXC / ANODE
myrun.ins the SHELX instructions to go with it: CELL, ZERR (the cell's e.s.d.s), LATT, SYMM
myrun_unmerged.mtz every observation before scaling, for pointless / aimless / careless -
myrun_unmerged.mtz the observations unmerged, scaled by the merge, for pointless / aimless -
the largest file of the run (--no-export-unmerged skips it)
myrun_report.txt what the run determined: cell, space group, statistics, warnings
myrun_plot.txt one row per image, for plotting how the crystal behaved over the sweep
@@ -137,6 +139,7 @@ myrun_model.cif / .pdb the model as placed, in the cell and space group of my
myrun_refined_model.cif the refined model (and .pdb)
myrun_refined_maps.mtz map coefficients on the refined model
myrun_blobs.json the unexplained difference-density blobs
myrun_solvent.json with --refine-solvent: the refined model's waters and ions, each scored by its surroundings, and the waters the build added and removed
```
A stills run writes the same merged files (its `.hkl` holds the merged reflections) and, on a
@@ -246,7 +249,7 @@ A few things worth knowing before reaching for more flags:
with `Killed` ran out of memory ([When host memory runs out](RUGNUX_INSTALL.md#when-host-memory-runs-out)).
With any question about a run, send `myrun_report.txt` and `myrun_log.txt`.
- Everything else is in [Running Rugnux](RUGNUX_TUTORIAL.md#running-rugnux) and the full
[Command-line options](RUGNUX_ADVANCED.md#command-line-options).
[Command-line options](RUGNUX_CLI.md).
## The rest of the manual
@@ -269,8 +272,10 @@ One page per job, so the answer needed is near the top of a short page:
Phaser, SHELXC/D/E and, for small molecules, SHELXT / SHELXL.
- [The results report](RUGNUX_REPORT.md) — the `KEY= value` interface, sweep quality and the
anisotropy section.
- [Command-line options](RUGNUX_CLI.md) — how to run each mode, and every option with what it
does.
- [Advanced usage](RUGNUX_ADVANCED.md) — reference data and the indexing ambiguity, model
validation, re-merging, and the full command-line option tables.
validation and re-merging.
- [Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md) — every model and reference option,
and decision by decision what a model can and cannot change.
- [Detector calibration](RUGNUX_CALIBRATION.md) — the geometry from a calibrant's powder rings
@@ -290,3 +295,11 @@ One page per job, so the answer needed is near the top of a short page:
Use `rugnux` to re-analyse data after acquisition, to experiment with processing
parameters, or to produce merged intensities for downstream structure solution.
## How to cite
If you find Rugnux useful, please cite F. Leonarski and M. Wang, "Rugnux: from diffraction images to
structure at synchrotrons and XFELs in one program" (2026), bioRxiv
[doi:10.64898/2026.10.24.761522](https://doi.org/10.64898/2026.10.24.761522). Rugnux prints this at
startup and at the head of its report; the other papers are on the [Publications](PUBLICATIONS.md)
page.
docs/RUGNUX_ADVANCED.md
+20 -217
View File
@@ -1,7 +1,7 @@
# Advanced Rugnux
The machinery behind a run that needs more than the defaults: external reference data, model
validation, re-merging without re-integration, and the full option tables.
validation and re-merging without re-integration; the option tables are on [Command-line options](RUGNUX_CLI.md).
```{contents} On this page
:local:
@@ -46,10 +46,10 @@ against a foreign dataset injects that dataset's systematics — so `-z` never p
errors into the intensities. `--reference-column` picks the column to read where the automatic
choice (`F-model`, else `IMEAN`/`I`, else `FP`/`FOBS`/`F`) is not the right one.
For the indexing ambiguity an atomic model does as well: `--model` computes the intensities it
needs from the structure (a model that fits also sets the hand label and the setting, see
[Validating against a model](#validating-against-a-model-rugnux---model)). Where a reference dataset exists, prefer it;
where only a model does, it resolves the ambiguity just the same.
An atomic model does not take part in the processing: `--model` is placed and validated after the merge,
and a run with one merges exactly what a run without one does. A model that fits then sets the frame the
merge is written in - the hand label, the setting and the indexing (see
[Validating against a model](#validating-against-a-model-rugnux---model)) - by relabelling it, never by re-merging.
### The indexing ambiguity
@@ -102,16 +102,15 @@ How to resolve it:
| **Rotation**, a reference dataset exists | `-z reference.mtz`. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the *hkl* labels change; the cell does not |
| **Serial stills**, a reference dataset exists | `-z reference.mtz`. Resolved **per image**, at integration time, by correlating each crystal's intensities with the reference — so the merge never mixes hands in the first place. Where the crystals' own cells tell the indexings apart (pseudo-merohedry), a crystal its cell decides keeps that indexing and the reference decides the rest. The data-only resolution below runs as a check: `AMBIGUITY_DATA_DISAGREES=` |
| **Rotation**, only a model | `--model model.pdb`. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (`MODEL_FIT= ACCEPTED`) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (`MODEL_INDEXING_MARGIN_SIGMA`). It is the **data** that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model's indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand |
| **Serial stills**, only a model | `--model model.pdb`, with the cell and space group given (`-C` / `-S`). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be, the crystals their own cell decides again excepted. The data-only resolution below is then run only as a check: `AMBIGUITY_DATA_DISAGREES=` counts the crystals on which it would have chosen otherwise |
| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A model or reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing |
| **Serial stills**, only a model, space group given (`-S`) | The crystals are made consistent with each other from the data alone, as in the row below - the model plays no part in the processing. Validation then fits the merge to the model in each candidate indexing, as for rotation data, and relabels the written reflections into the model's indexing where it wins |
| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing |
| Neither | The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame; for stills whose space group the run determined, a merge of mixed indexings, which looks like a perfect twin and can pass for higher symmetry than the crystal has. Run again with `-S` |
Two things to know about where the choice lands:
- **`--mode scale` cannot repair a stills run after the fact.** The per-image test happens at
integration time, so a `_process.h5` whose images were integrated without a reference — an MTZ or a
model — has already lost the distinction, and no re-merge brings it back; the data-only resolution
runs in the full pipeline, not in `--mode scale`. On rotation data, where
- **`--mode scale` resolves it as the full run does.** On stills with no reference MTZ it makes the
crystals consistent from the data alone before merging; the per-image test against a reference
happens at integration time and is kept in the stored reflections. On rotation data, where
the ambiguity is one choice for the whole dataset, `--mode scale --model model.pdb` does resolve it.
- **The choice reaches the written reflections**, not only the R-factors and the maps: the merged
`.mtz` / `.cif` / `.hkl` (and `_unmerged.mtz`, where it is asked for) are written in the indexing
@@ -291,9 +290,13 @@ declines it and the run stops with a message saying so. `--model` works here as
that the model is placed by the rigid body only: a model it does not place, or a prediction, is not
searched for by molecular replacement, `--mr` is refused, and the placed model is not refined. On rotation data, where the
[indexing ambiguity](#the-indexing-ambiguity) is one choice for the whole dataset, it can settle the
enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. What no re-merge can
repair is a **stills** `_process.h5` integrated without a reference: there the ambiguity was
decided per image, at integration time, and the distinction is gone from the stored reflections.
enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. On
stills the re-merge is the full run's own merge: the same frames (a given cell's population, as
`/entry/MX/stillsGivenUnitCell` records it), the same rocking curve, the same data-only resolution of
the indexing ambiguity and the same scaling, so a
`_process.h5` merged again with the options it was written with reproduces the full run's reflections
(with a reference MTZ, the full run also lets the crystals' own cells overrule the reference's indexing
before merging, which `--mode scale` does not).
Where the full run re-seated the lattice — the space group it settled on is in a different setting
from the one each image was indexed in — the file records the change of basis as
@@ -312,10 +315,10 @@ describes them - the cell and space group; `_software` with Rugnux named twice,
beamline and its general class, the detector's make and model in `_diffrn_detector.type` and its
class in `_diffrn_detector.detector`, and the collection date from the NXmx start time); and the
merging statistics overall (`_reflns`) and per shell (`_reflns_shell`): completeness, multiplicity,
<I/σ(I)>, R_merge, R_meas, R_pim, CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries
<I/σ(I)>, R_merge, R_meas, R_pim (`?` on a stills merge, where they mean nothing), CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries
its e.s.d.s (`_cell.length_a_esd`, …). A
quantity the input does not state is left out rather than guessed. The unmerged observations are in
the `_unmerged.mtz` (unscaled) and the `.hkl` (scaled), not in the `.cif`.
the `_unmerged.mtz` and the `.hkl` (both scaled on a rotation merge), not in the `.cif`.
Items PDBx does not define — ISa, the error model, the twinning and anisotropy diagnostics, the
refined distance and beam centre — are written under the `jfjoch_` local prefix, which a dictionary
@@ -338,204 +341,4 @@ the broker is not a file here, and has none.
## Command-line options
General:
| Option | Description |
| --- | --- |
| `-o, --output-prefix <txt>` | Output file prefix (default: `output`) |
| `-N, --threads <num>` | Number of worker threads (default, and for any value ≤ 0: every CPU the run may use - its affinity and cgroup quota). Some stages take fewer, because past a point more workers make them slower: the per-image loop of `--mode mx` uses at most 16 per GPU unless `-N` was given a positive value, and first-pass spot finding and the beam-stop pre-scan have ceilings of their own that `-N` does not lift. On a small card the two ceilings, and the per-image loop's 16 where `-N` was left at its default, come down to as many workers as keep their buffers (about 24 bytes per detector pixel each) within half of the card's total memory. Scaling, merging and the space-group search use the full count |
| `-s, --start-image <num>` | First image to process (default: 0) |
| `-e, --end-image <num>` | End image number, exclusive: `-s 0 -e 100` processes images 0-99 (default: all) |
| `-t, --stride <num>` | Process every *n*-th image (default: 1) |
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
Mode — `--mode <name>` (default `mx`):
| Value | Description |
| --- | --- |
| `mx` | Full analysis — spot finding, indexing, integration and merging |
| `azint` | Only azimuthal integration (no spot finding/indexing); writes `<prefix>_process.h5` |
| `scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) |
| `calibration` | Determine the detector geometry from powder rings; writes `<prefix>.poni` and `<prefix>.json` |
| `refine` | Refine the `--model` against merged structure factors (an MTZ or a structure-factor mmCIF) and process nothing; see [Refining a model against structure factors](RUGNUX_MODEL_AND_REFERENCE.md#refining-a-model-against-structure-factors-rugnux---mode-refine) (`--refine-score-only`, `--refine-d-min`) |
Refinement (`--mode refine`): the columns are found, not named - amplitudes with their sigmas (F/SIGF,
FP/SIGFP, ...) where the file has them, else intensities (IMEAN/SIGIMEAN, I/SIGI, ...) made into amplitudes by
French-Wilson, else the mean of the anomalous pairs; the free set from a FreeR_flag-like column (0 the test set,
or the minority value where 0 is the majority, as phenix writes it), drawn by rugnux where the file has none.
The model is taken as placed in the data's cell, and refined in the data's space group.
| Option | Description |
| --- | --- |
| `--refine-score-only` | Score the model as given and refine nothing: R-work/R-free, the scale, the bulk solvent, the likelihood and the geometry, each term under `REFINE_SCORE_` (`--developer` prints them all) |
| `--refine-d-min <num>` | Leave out reflections finer than this (Å) |
Calibration (`--mode calibration`):
| Option | Description |
| --- | --- |
| `--calibrant <name>` | Powder standard: `lab6` \| `agbh` \| `ceo2` \| `si` \| `ice` (default `lab6`, case-insensitive). A cell given with `-C` overrides it: that cell is the standard, its rings enumerated from the cell, with the absences of `-S` where given |
| `--calibration <txt>` | How the rings are measured: `rings` \| `spots` (default `rings`; see above). `rings` defaults `--azim-phi-bins` to 32 |
| `--no-refine-tilt` | Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance, for a calibration handed to a program that cannot express a tilted detector (XDS). On stills it holds the tilt in the geometry refinement too |
Detector mask:
| Option | Description |
| --- | --- |
| `--detect-beam-stop[=N\|off]` | Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. **On by default** (60 images); `=off` disables. A reflection whose signal disk reaches into the shadow is not integrated: the rest of such a disk lies in the stop's half-shadow and reads low. Pixels the finder reports that hold the detector's no-data mark (a PILATUS gap) are masked as dead instead |
Geometry:
| Option | Description |
| --- | --- |
| `--beam-center-check[=off]` | Measure the beam centre from the isotropy of the scattered background on **every** run, report how far the file's value is from it (against how right this geometry needs it to be), and on rotation data index a **second first pass** at the measured centre. The fit reads the projection `--detect-beam-stop` already builds, so it costs no extra frames. **On by default**; `=off` disables it, and with it the measured centre's part in the post-refinement bound. The measured centre is adopted where the file's centre indexes nothing and the measured one indexes a majority; where the two return cells related by an integer volume factor (2 to 4) and the measured centre's cell, larger or smaller, carries materially more of the pooled validation spots against its own chance level; and, on a two-pass run, where the first pass is run at both centres and the measured one merges better — done where the two give the same cell in different metric symmetries, where they agree only once less of each frame is read, and where they agree but lie further apart than the geometry absorbs and the file's centre merges inconsistently in the lowest-resolution shell. Otherwise the file's centre is kept. On stills, where there is no first pass, the file's centre, the measured one and the point-symmetry capture's are each indexed on a sample of frames and the one putting materially more spots on the lattice is taken; with `-C` the detector distance is searched too (30-3000 mm, the file's value one hypothesis). Still runs when `--beam-x`/`--beam-y` are given. See [§1.4](CPU_DATA_ANALYSIS_IMAGE.md) |
Spot finding:
| Option | Description |
| --- | --- |
| `--spot-sigma <num>` | Noise sigma level for spot finding (default: 4.0) |
| `--spot-threshold <num>` | Photon-count threshold for spot finding (default: 10) |
| `--adaptive-spots` | Self-calibrating detection (**default**, stills and rotation alike): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed `--spot-threshold`, so one setting adapts across datasets (no per-dataset `--spot-threshold`/`--spot-sigma` tuning) |
| `--no-adaptive-spots` | Turn adaptive detection off and use the fixed `--spot-threshold` / `--spot-sigma` finder |
| `--spot-false-pixels <num>` | Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies `--adaptive-spots`) |
| `--spot-high-resolution <num>` | High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills |
| `--spot-low-resolution <num>` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit) |
| `--min-pix-per-spot <num>` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is scored at min-pix 2/3/1 (one lattice search per frame, refined on each list) and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. |
| `--max-spots <num>` | Maximum spots kept per image (the strongest ones) and handed to indexing. **If omitted, the budget is measured** on rotation data: the first pass reads how deep into an image's spot list its spots still lie on the lattice it found, and the run keeps that many (never more than 1000). Give a value to pin it. Stills always use the fixed 1000. |
| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** |
Azimuthal integration (the radial profile behind the per-image ice-ring score). Every *q* here is
*q* = 2π/*d*, in Å⁻¹:
| Option | Description |
| --- | --- |
| `-q, --azim-q-spacing <num>` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) |
| `--azim-min-q <num>` | Minimum Q, 1/Å |
| `--azim-max-q <num>` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see |
| `--azim-phi-bins <num>` | Number of azimuthal (phi) bins (default: 1) |
| `--polarization-correction <on\|off>` | Enable/disable the azimuthal polarization correction |
| `--solid-angle-correction <on\|off>` | Enable/disable the azimuthal solid-angle correction |
Indexing:
A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation
indexing) by default; a dataset without one is processed as independent stills. `--force-still`
overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request
rotation explicitly and pick the pass or lattice.
The FFT search looks for cell axes between 10 Å (`--fft-min-unit-cell`) and a default **longest
axis of 500 Å**, which has no flag of its own — a reference cell (`-C`) moves both bounds to cover
the cell it names. A de-novo rotation run also tries a second first-pass hypothesis with the short
end lowered to 5 Å, so a small-molecule cell below the 10 Å floor is indexed on its true axes
rather than as a supercell of them; the standard pass's answer stands unless that evidence takes it. An axis beyond the search's reach is **not refused**: the run returns a
plausible shorter sub-cell or an axis harmonic and processes it happily, so a cell that comes out
at a half or a third of the expected long axis should be read as this limit, not as the crystal.
For very long axes the direction grid's angular resolution binds well below 500 Å — see the
[analysis reference](CPU_DATA_ANALYSIS_INDEXING.md#5-fft-indexing-unknown-unit-cell) on FFT indexing.
| Option | Description |
| --- | --- |
| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation |
| `-X, --indexing-algorithm <txt>` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each; `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise; rotation data always use the FFT search |
| `-C, --unit-cell <cell>` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it |
| `--fft-min-unit-cell <num>` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered |
| `--min-indexed-spots <num>` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
| `-S, --space-group <num\|symbol>` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](#the-indexing-ambiguity)) |
| `-r, --geom-refine <txt>` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) |
| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) |
| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) |
| `--force-rotation-lattice <vec>` | Force rotation lattice (9 floats, Å), skipping the first pass |
| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see the rotation section) |
| `--prepass-fraction <f>` | Rotation: the two-pass geometry pre-pass integrates the first fraction `f` of the sweep, but at least 120° of it (0.05-1, default **0.5**; `1` integrates the whole sweep twice). The second pass always integrates every frame. The pre-scan and the first-pass indexing sample the same part of the sweep |
| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), a stationary goniometer axis counting as stills, `=off` disables |
| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) |
Indexer choice in brief: `ffbidx` (GPU or CPU) refines toward a **known cell** and is best for sparse
serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the
[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS_INDEXING.md) for the algorithms.
Scaling and merging:
| Option | Description |
| --- | --- |
| `--no-merge` | Skip scaling and merging, which are on by default; the per-image `_process.h5` is then written instead of the merged files (`_unmerged.mtz` and the results report are still written) |
| `-A, --anomalous` | Anomalous mode: merge each Bijvoet hand as its own unique reflection, so multiplicity, completeness, ⟨I/σ⟩ and the error model are counted anomalously. A default rotation merge already writes `I(+)`/`I(-)` (see [Reflection-file conventions](RUGNUX_INTEGRATION.md#reflection-file-conventions)); `-A` changes the counting basis, not whether the anomalous signal is in the file |
| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills |
| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) |
| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls (see below) |
| `--relative-b[=deg]` | rot3d: fit a per-batch relative-*B* beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise) |
| `--simple-stills` | Stills: treat every reflection as a full (*p* = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement |
| `--no-expected-variance-merge` | Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting |
| `--capture-uncertainty <num>` | Systematic sigma num·(1−captured fraction)·⟨I⟩: on rot3d fulls the share of the rocking curve not captured, on stills the share of each partial its partiality does not see (default: 1.0 for rotation, 0 otherwise) |
| `--min-captured-fraction <num>` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) |
| `--scaling-high-resolution <num>` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) |
| `--scaling-low-resolution <num>` | Low-resolution limit of the scale fit, Å (default: 50, which is also XDS's own default; 0 removes the limit). Reflections coarser than it are merged and written but kept out of every fit of the scale model: on a large cell a few dozen of them, measured on a background the beam stop depresses, bent the model enough to move the merged 10–50 Å intensities by tens of percent. Which of them are measured at all is the beam-stop mask's decision - a reflection whose signal disk reaches the shadow is not integrated (`--detect-beam-stop`) |
| `--resolution-cutoff <txt>` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) |
| `--resolution-cc-target <num>` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) |
| `--resolution-shells <num>` | Number of resolution shells in the reported statistics table (default: 9). The bins are equal steps in 1/d² between the lowest- and highest-resolution reflection merged, which is XDS's rule, and 9 is XDS's count — so at the same resolution limits the two tables have the same shells and can be read row for row |
| `--report-resolution <dmin>[,<dmax>]` | Also report the merging statistics over this resolution range, Å (either order; `dmax` defaults to the run's own low-resolution limit), as a second table beside the run's own — the `REFRES_*` keys of the [results report](RUGNUX_REPORT.md#the-reference-range-table) — binned from the same merged reflections, so a run can be read against another program's table at that program's range. Report-only: the cut, the scaling, the error model and every decision stay the run's own, and shells finer than the run's own limit are reported as not merged rather than filled from data the run did not keep |
| `--min-partiality <num>` | Minimum partiality to accept a reflection (default: 0.02 for rotation, 0.5 for stills) |
| `--ice-min-score <num>` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging and the exclusion from scaling (default: 1.5; 0 = no gate). The fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness |
| `--ice-min-spot-ratio <num>` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) |
| `--reject-outliers <num>` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rotation, 10 for stills; 0 = off) |
| `--min-image-cc <num>` | Per-image CC limit, percent (default: no limit) |
| `--search-min-zeta <num>` | De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry \|ζ\| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences |
| `--mosaicity <num>` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed |
| `--scaling-iterations <num>` | Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle, and a merge that hits the cap is flagged `SCALING_NOT_CONVERGED` (default: 100) |
| `-z, --reference <file>` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](#the-indexing-ambiguity), hands over the R-free set and reports CC<sub>ref</sub>. Not a scale anchor |
| `--reference-column <label>` | Reference column to use, by MTZ label — an mmCIF under the labels `gemmi cif2mtz` gives it (default: auto — F-model, else IMEAN/I/…, else FP/FOBS/F) |
| `--model <file>` | Validate the merged intensities against this atomic model (PDB or mmCIF, gzipped or not; the format is taken from the file's content) — R-work / R-free and maps (see [Validating against a model](#validating-against-a-model-rugnux---model)). An experimental model is placed as a rigid body, and by molecular replacement where the rigid body does not place it; a prediction (AlphaFold or ModelCIF) goes to molecular replacement straight away (`MODEL_PLACEMENT=`). It also settles the frame the reflections are written in: the enantiomorph, and the [indexing ambiguity](#the-indexing-ambiguity) where no `-z` did; another space group or the twin hypothesis' subgroup it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif`. For serial stills given `-C` / `-S`, the model's structure factors become the per-image reference. A placed model is then [refined](RUGNUX_MODEL_AND_REFERENCE.md#refinement) against the merge it was placed on (`<prefix>_refined_model.cif`, `<prefix>_refined_maps.mtz`, `<prefix>_blobs.json`, `REFINE_` keys) - seconds on a GPU, typically 2-15 minutes on the CPU - which decides nothing about the processing |
| `--mr <file>` | As `--model`, but placed by molecular replacement straight away, without trying the rigid body first. `--mode mx` only; not with `--model` |
| `--no-refine` | Do not refine the placed model (for debugging): by default a model that `--model` or `--mr` places is refined |
| `--write-process-h5` | Also write the (large) `_process.h5` when merging (default: only `.mtz`/`.cif`) |
| `--developer` | Write the full `<prefix>_report.txt`: the pipeline-internal keys and the long explanations the default report leaves out. Nothing is computed differently — the same report, rendered in full (see [The results report](RUGNUX_REPORT.md#the-developer-report)) |
| `--finalist-ledger` | Report the full-resolution evidence for each space group the search considered, not only the one it adopted, and the twin-immune zone evidence of the adopted group's 2-folds (report-only; the decision is unchanged). Both are read on a merge of the observations in P1 that only this option makes, so it costs one extra merge; rotation only, and not with a fixed centred `-S` |
| `--export-unmerged` | Write `<prefix>_unmerged.mtz`, an unmerged MTZ (POINTLESS column layout) of the integrated observations, for aimless / pointless / careless. **On by default** whenever there is an output prefix. Rotation partials are summed into one full per reflection. Intensities carry the deterministic per-reflection corrections and nothing else — Lorentz and polarization in `LP`, sensor efficiency in `QE`, flight path in `FLIGHT` — while the partiality is not divided out and the per-image scale is not applied. Written in `--mode mx` and `--mode scale`, and with `--no-merge`. See [The unmerged export](RUGNUX_INTEGRATION.md#the-unmerged-export) |
| `--no-export-unmerged` | Do not write `<prefix>_unmerged.mtz`. It is the largest file a run produces, so a run whose observations are not going to another scaling program can skip it |
| `--export-unmerged-partials` | Write `<prefix>_unmerged_partials.mtz`, the same observations with each partial as its own row (one batch per image) for the reading program to sum. Off by default, and independent of `--export-unmerged` |
Integration:
| Option | Description |
| --- | --- |
| `--integrator <txt>` | Spot integrator: `gaussian` (profile-fit, default) \| `empirical` \| `boxsum` (classical fallback) |
| `--integration-radius <r>` | Signal-box radius `r1`, or `r1,r2,r3` (px). One value ⇒ `r2=r1+2`, `r3=r1+4` |
| `--adaptive-integration-radius[=on\|off]` | Set the signal radius `r1` from how wide this crystal's spots actually are (default: **on**). On stills it measures instead how far the background ring should reach: on empty apertures of the pre-scan frames, the `r3` (up to 14) whose ring best predicts the background under the signal disk. `r1` is not the integration domain — that is the profile-fit grid — but it *is* the aperture the profile **width** is learned over, and a second moment over a disk of radius `a` saturates at `a²/4`, so at the shipped `r1 = 4` the learned σ can never exceed 2 px and a broader spot is fitted with a profile the model cannot represent. The pre-scan reads `r80`, the radius holding 80 % of a spot's flux, off isolated strong spots over a fixed 14 px aperture that owes nothing to `r1`, fits it against `1/d` and evaluates it at 5 Å; then `r1 = clamp(round(2·r80), 4, 6)`, `r2 = r1 + 2`, and `r3` is taken so the `r2..r3` background ring keeps the area it has at the default `4,6,13`. The ceiling of 6 is pattern density: `r2` also drives the neighbour-ownership radius and the ring's inner edge, and past it a dense pattern starts losing reflections whose ring falls below six clean pixels. Ignored when `--integration-radius` is given. The widened radius applies to the **final** integration pass only — the two-pass geometry pre-pass keeps the radius the run started with, because the post-refinement fits the detector distance and beam to the observed reflection positions and those move with the signal disk (33 µm and 0.02 px between `r1 = 4` and `r1 = 6` on one crystal, enough for the second pass's de-novo lattice search to settle on a different lattice and index a fifth fewer frames). And where the widened radius leaves more than 1.1 % of the predicted reflections without a background ring — a pattern too dense for it — the final pass is re-integrated at the fixed 4 px radius, and the log says so. Over the rotation regression battery it moves 12 of 38 crystals and leaves the merged intensities of the other 26 unchanged; where it moves them, per-shell ⟨I/σ⟩ improves by up to 31 % and R_meas by up to 24 % |
| `--integration-stencil <k>` | Push the `r2..r3` background ring out by `k` times the beam's radial streak `bandwidth·R_px`, per reflection (default `0` = the fixed circular ring). A fixed ring otherwise ends up on a streaked reflection's own tails at high resolution and measures them as background. Only the ring moves, and only radially — the `r1` signal box stays a circle — and the growth is capped at `2·r3`. The neighbour exclusion grows with it, so on a crowded pattern a few reflections can be left with too little background and dropped. On a monochromatic beam the streak is zero and this does nothing |
| `--background-clip <n>` | High-side clip of the background ring at `mean + n·√mean` (default 4, whatever the bandwidth; 0 = off). The default background estimator — it rejects neighbour cores and zingers without the symmetric trim's Poisson skew bias. Ignored by `--integrator boxsum` |
| `--background-trim <f>` | Use the old symmetric trimmed mean for the background ring instead of the clip, 0≤f<0.5 (`0.10` was the former default). Switches `--background-clip` off. A symmetric trim is biased low on Poisson data and adds ~5 counts to every partial, so this is for back compatibility only; `0` = plain ring mean. Rings holding more than 512 pixels fall back to the plain mean (the GPU sorts the ring in shared memory and the CPU now matches it), which the default radii never reach but wide ones do |
| `--background-radial[=on\|off\|auto]` | Correct the background ring for the **curvature** of the radial background (default **auto**). Disk and ring are concentric, so a background linear in position cancels between them and only curvature survives — which on a smooth ice ring reaches +26 counts on a single reflection. `auto` decides per run in the pre-scan, on empty apertures of the pre-scan frames: on stills it is applied to every image where it makes the ring predict the background of empty apertures better at all; on rotation data to every image where it takes at least 10 % of that error away, and otherwise only to images whose ice score shows a *smooth* powder ring (ice made of discrete crystallite spots leaves no such ring, and the correction makes the bias worse). Ignored by `--integrator boxsum` (no clip pass to take the curve from) |
| `--integration-high-resolution <num>` | High-resolution limit for prediction and integration. Omitted (or 0) means integration extends as far as the detector reaches — which is what the predictor can place on the detector anyway, since it rejects reflections that miss it. Set a value to integrate less than the detector offers |
| `--max-hkl <n>` | Predict reflections with \|h\|,\|k\|,\|l\| ≤ `n` (max 511). By default this is derived per crystal from the refined cell as `ceil(max(a,b,c)/d_min) + 1`, which is the exact bound: the predictor keeps only \|q\| ≤ 1/d_min and `h = a·q`, so no reflection can lie outside it and no candidate inside it is wasted on a shorter axis. Set it only to override that |
| `--bandwidth <num>` | Relative X-ray bandwidth FWHM (e.g. `0.01` for a 1% DMM). Default: the file's value; where the file states none, the one the pre-scan measures from how much longer strong spots are along their radius than across it, used where it is significant; otherwise 0 (monochromatic). `--bandwidth 0` forces a monochromatic beam |
| `--overlap <txt>` | What to do where two predicted reflections share signal pixels: `off` \| `reject` \| `exclude` (default `exclude`). A shared pixel belongs to the nearer centre; without this a crowded reflection reads high on a dense pattern. `exclude` drops the shared pixels from the profile fit, which renormalises itself, and keeps the reflection; `reject` instead drops the whole reflection when too little of its profile is cleanly its own. `--integrator boxsum` has no profile to renormalise, so only `reject` acts there |
| `--overlap-minpk <f>` | Least fraction of a reflection's expected profile that must be usable for it to be kept (default 0.75, XDS `MINPK`). Governs both the fraction that must be **readable** — not masked, untrusted, in a gap or overloaded — in every profile mode, and, under `--overlap reject`, the fraction that must be cleanly its own. Under `--integrator boxsum` any unreadable pixel discards the disk outright and the reject fraction goes by disk *area*, which cuts harder |
| `--flight-path <txt>` | What the diffracted beam crosses between the sample and the detector: `air` (default) \| `helium` \| `vacuum`. A reflection arriving at angle α to the detector normal flies `D/cos α` instead of `D`, so it crosses more of the medium and reads low; the factor is `exp(D/L·(1/cos α − 1))`, with `L` the attenuation length of the medium from the NIST tabulation and `D` and λ as stated by the file — nothing in it is fitted. **Nothing in NXmx, and no field of any master file Rugnux reads, describes the medium**, so it cannot be auto-detected and is assumed; the report states which medium was assumed and what it was worth (`FLIGHT_PATH`, `FLIGHT_PATH_WILSON_B`). Nor can it be inferred from the implied transmission — in this corpus a confirmed helium station sits at 51 % implied air transmission and a confirmed air station at 63 %. In air the term is +1.5 % at α = 55° for 18 keV over 160 mm and several-fold below 5 keV, where stations use helium precisely because air is unusable; helium attenuates ~1/600 of air, and vacuum leaves intensities untouched |
| `--prediction-mosaicity <num>` | Diagnostic: fix the rocking width (deg) the **prediction** window opens to, leaving partiality on the per-image σ_M. The two are one number by default, so a σ_M that moves takes the integrated reflection population with it |
Geometry overrides (defaults are taken from the input file; override them to reprocess with a corrected geometry):
| Option | Description |
| --- | --- |
| `--beam-x <num>` | Beam centre X (pixel). `--beam-center-check` still runs and can adopt its measured centre ([§1.4](CPU_DATA_ANALYSIS_IMAGE.md)) |
| `--beam-y <num>` | Beam centre Y (pixel). `--beam-center-check` still runs and can adopt its measured centre ([§1.4](CPU_DATA_ANALYSIS_IMAGE.md)) |
| `--detector-distance <num>` | Detector distance (mm). On stills it is then held: neither searched nor refined. Without it, stills with a reference cell (`-C`) search the distance and refine it with the cell |
| `--wavelength <num>` | Wavelength (Å) |
| `--pixel-size <num>` | Pixel size (µm), for a file that does not state it (an XTC stream of an unknown camera) |
| `--gain <num>` | Detector units per photon, for input in detector units (SACLA, CXI, XTC, SMV, marCCD and `--crystfel-geometry` input). Where the background resolves single photons (SACLA, CXI and `--crystfel-geometry` input) the frames are counted in whole photons as they are read; otherwise they are analysed in their own units and the gain enters where a photon count is assumed. The integrated intensities come out in photons either way. Default: measured on the background of a few frames ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--saturation <num>` | Pixel value from which a pixel is overloaded, in the units the frames are read in. Replaces the file's limit and the one stills read off the frames |
| `--crystfel-geometry <file>` | Place the panels of an HDF5/CXI input by a CrystFEL geometry file instead of the file's own metadata: module geometry, camera length, mask arrays and bad regions (optional; [Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--module-geometry <file>` | Where the detector's modules are: an NXmx file with one NXdetector_module per module (what a run writes as `<prefix>_modules.h5`) or a CrystFEL geometry file whose panels are laid on the image. Overrides the input's own |
| `--refine-modules[=on\|off\|diagnose]` | Stills: place each module as a rigid block in the geometry bundle adjustment, where that predicts held-out spots better (default: on). Used for a foreign multi-panel detector, measured and reported only for a JUNGFRAU; the placement is written to `<prefix>_modules.h5` |
| `--mask-good [<file>:]<dataset>[@<bits>]` / `--mask-bad [<file>:]<dataset>[@<bits>]` | HDF5/CXI stills: a mask array in the frames' stored layout, in `<file>` or in each data file; usable only where non-zero (or with all `<bits>`) / unusable where non-zero (or with any `<bits>`). Repeatable ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--rot1 <num>` | PONI detector rotation 1 (rad). On stills, `--rot1` or `--rot2` holds the tilt (not refined); without them stills refine the tilt where held-out frames agree |
| `--rot2 <num>` | PONI detector rotation 2 (rad) |
| `--rot3 <num>` | PONI detector rotation 3, about the beam (rad) — the one `rugnux --mode calibration` writes into its `.poni`, so a refined third rotation can be given straight back |
| `--detector-mirror-y` | The stored image is mirrored in Y relative to the frame the PONI angles are stated in |
| `--detector-quarter-turns <0-3>` | The stored image is turned by this many multiples of 90° about the beam relative to that same frame |
| `--polarization <num>` | Degree of polarization *p* (XDS `FRACTION_OF_POLARIZATION` = (1+*p*)/2), signed: positive for a horizontally polarized source, negative for a vertically polarized one; where the beam-stop pre-scan finds the background clearly suppressed in the other plane, the degree is applied in that plane instead. Default: what a miniCBF `# Polarization` line or a d*TREK `SOURCE_POLARZ` states (a fraction *f*, read as *p* = 2*f* − 1, so a laboratory tube's 0.5 is unpolarized), and otherwise 0.99, an undulator value; bending-magnet and wiggler sources are typically lower (0.8–0.95), and neither ADSC SMV, marCCD nor NXmx files state it, so pass the beamline's known value here |
| `--rotation-scale <k>` | Goniometer rotation scale: the stage turned `k` times the angle stored in the file (the commanded one). Applied to both passes, and overrides the scale Rugnux fits for itself |
Every option, with what it does, is on its own page: [Command-line options](RUGNUX_CLI.md).
docs/RUGNUX_CLI.md
+372
View File
@@ -0,0 +1,372 @@
# Command-line options
`rugnux` run with no arguments prints a short usage: how to run each mode and the options most runs
need. This page is the full reference behind it - every option the program accepts, with what it
does. The program's own usage message is the authority on behaviour; this page is kept consistent
with it.
The same pages ship with the release archive, as `share/doc/jfjoch_rugnux/html/` beside `bin/`, so
they can be read without a network; this one is `share/doc/jfjoch_rugnux/html/RUGNUX_CLI.html`.
```
rugnux [options] <input> [<input> ...] [<broker URL>]
```
```{contents} On this page
:local:
:depth: 2
```
## Running each mode
What a run does is chosen with `--mode` (default `mx`). Within `mx`, rotation and stills are not
chosen by an option: a dataset whose goniometer axis turns is processed as a rotation sweep, one
without a goniometer axis - or with an axis that stands still - as serial stills.
### Rotation data (`--mode mx`, the default)
```
rugnux -o myrun dataset_master.h5 # NXmx / EIGER / Jungfraujoch master file
rugnux -o myrun sweep_00001.cbf # CBF, marCCD or SMV sweep: name one frame
rugnux -o myrun dataset_master.h5 model.cif # with a model (or --model model.cif)
rugnux -o myrun x_master.h5 http://host:5232/ # follow a broker's collection while it is written
rugnux -o myrun a_master.h5 b_master.h5 # several sweeps of one crystal, merged as one
```
Chosen when the file's goniometer axis turns. Several sweeps can be given (below, Inputs). A model is placed (rigid body, or
molecular replacement), refined and validated after merging; it does not change the processing
decisions ([Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md)).
### Serial stills (`--mode mx`)
```
rugnux -o serial *_master.h5 # several NXmx masters of one setup
rugnux -o serial -C 79,79,38,90,90,90 -S P43212 run*.h5 # SACLA (Cheetah HDF5)
rugnux -o serial -C 79,79,38,90,90,90 -S P43212 *.cxi # Cheetah CXI
rugnux -o serial -C 79,79,38,90,90,90 -S P43212 *.xtc # LCLS-I XTC streams
```
Chosen when there is no goniometer axis or it does not turn; `--force-still` processes a rotation
sweep as stills. Giving the known cell (`-C`) and space group (`-S`) is recommended for stills. Several inputs are one dataset, read in the order given. Without `-C` the stills
are indexed de novo; a model is given as for rotation data
([Still / serial data](RUGNUX_TUTORIAL.md#still-serial-data)).
### Refinement (`--mode refine`)
```
rugnux --mode refine -o refined data.mtz model.cif
```
Refines a model against merged data that already exist - an MTZ or a structure-factor mmCIF - and
reads no images. The model is required, given among the inputs or with `--model`
([Refining a model against structure factors](RUGNUX_MODEL_AND_REFERENCE.md#refining-a-model-against-structure-factors-rugnux---mode-refine)).
### Detector calibration (`--mode calibration`)
```
rugnux --mode calibration --calibrant lab6 -o det LaB6_master.h5
```
The detector geometry from a powder standard's rings, written as `det.poni` (pyFAI) and `det.json`
([Detector calibration](RUGNUX_CALIBRATION.md)).
### Azimuthal integration (`--mode azint`)
```
rugnux --mode azint -o profiles dataset_master.h5
```
Azimuthal integration only - no spot finding, indexing or merging; writes `profiles_process.h5`.
The binning is set by the [azimuthal-integration options](#azimuthal-integration).
### Re-merging (`--mode scale`)
```
rugnux -o myrun --write-process-h5 dataset_master.h5 # integrate and merge once
rugnux --mode scale -o remerged -A myrun_process.h5 # re-merge, here anomalously
```
Re-scales and merges the reflections already integrated into a `_process.h5`, without re-running
spot finding or integration ([Re-scaling and re-merging](RUGNUX_ADVANCED.md#re-scaling-and-re-merging-rugnux---mode-scale)).
## Inputs
Inputs are files, never a directory, read in the order given (a shell glob gives them sorted by
name). Several inputs are one stills dataset in `--mode mx`, or several rotation sweeps (below): all the files of
one dataset - `rugnux run*.h5` (SACLA), `*.cxi`, `*.xtc` (LCLS) - or several Jungfraujoch/NXmx
masters of stills, read as one run: their images one after another, merged together, with the
indexing rate reported per master as well. Such masters must share the detector, wavelength,
distance and beam centre. Several inputs must be one format; any `--mode` other than `mx`, and
following a broker, take one.
Several rotation sweeps are told apart from the files: one sweep per CBF/marCCD/SMV/SFRM file-name
template (the name with its frame number taken out - any frame of a sweep names the whole sweep, so
`rugnux a_*.cbf` is the one sweep `a`), one per rotation master. Only the inputs given are processed;
other sweeps beside them are named in the log. A mix of rotation and stills masters is refused, and
so are sweeps at different wavelengths, `-s`/`-e` and `--force-still` with several sweeps.
By default (`--sweeps joint`) the sweeps are one dataset, merged in two levels as XSCALE merges the
data sets XDS made of each sweep: each sweep is processed on its own first - indexed, integrated, its
partials combined and scaled, its space group determined - writing only its log `<prefix>_s<k>_log.txt`;
the sweeps are then put on the axes of the best of them (the finest resolution of its own merge, ties: the higher ISa) - the change
of basis, an axis permutation or a twin law of the lattice, under which their merged intensities correlate
best; where the lattice allows more than one indexing, which one each sweep takes is settled over all pairs
of sweeps at once - and each is judged by its isomorphism ratio with it (see [RUGNUX_REPORT](RUGNUX_REPORT.md)): clearly
about 1 it joins, clearly below it (another crystal) it is left out, and in between the merge decides - it
joins if it helps - adding it lowers the held-out residual of the others' merge, or it adds at least 1%
to their unique reflections while that residual is not worse beyond its noise (a sweep left out
is named in the log). The crystal has one cell: a joined sweep whose own cell is more than 0.3% off that of
the joined sweep reaching the finest resolution (where the cell is measured rather than the distance) is
processed a second time, from the distance that would make the two cells agree, and that run replaces the
first where its intensities agree better with that sweep's (a lower held-out residual). Their fulls are
scaled and merged together: a per-frame scale within each sweep, the sweeps' dose order read from
their relative B factors (not from the input order or the header times), a decay along that order or
one B per sweep, whichever predicts held-out frames better, then the space-group search, the
resolution cut and everything after it on the joint data, into `<prefix>.*`. The detector-dependent
corrections (absorption and detector surfaces) are each sweep's own. The unmerged MTZ holds the joint
merge's scaled observations in the batches of their own sweeps (sweep `s` of the merge: batches
`s * base + frame + 1`, `base` the power of ten past the longest sweep), under the batch headers that
sweep's own run wrote; the report names each sweep in section `1b. SWEEPS`. `--sweeps separate` processes each sweep alone instead, as if it were the only
input, into `<prefix>_s<k>_*` (k = 1, 2, ... in the order given).
What each format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read
without a geometry file where their own metadata places the detector
([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)).
An atomic model and a reference reflection file can be given among the inputs, in any order,
instead of `--model` and `-z`. Each file is told by its contents, not its name: an MTZ (the MTZ
magic) or an mmCIF with a `_refln` or `_diffrn_refln` loop is reflections (`-z`; the input itself in
`--mode refine`), a PDB file or an mmCIF with an `_atom_site` loop is a model (`--model`), anything
else is images. An mmCIF holding both coordinates and reflections has to be given with `--model` or
`-z`. At most one of each, also counting `--model`/`--mr` and `-z`; gzipped or not.
With a broker URL (`http://host:port/`) as the last argument and no input on disk yet, rugnux
follows the broker's current collection while it is written: the input must be the path its master
file will have, the data files are read next to it as the broker reports them closed, and the run
ends with the images the collection delivered. Rotation data, `--mode mx` only. The collection's
token is read from the environment variable `JUNGFRAUJOCH_HTTP_TOKEN` and sent only to the broker.
With the input already on disk, the URL is ignored.
## All options
### General
| Option | Description |
| --- | --- |
| `-o, --output-prefix <txt>` | Output file prefix (default: `output`) |
| `-N, --threads <num>` | Number of worker threads (default, and for any value ≤ 0: every CPU the run may use - its affinity and cgroup quota). Some stages take fewer, because past a point more workers make them slower: the per-image loop of `--mode mx` uses at most 16 per GPU unless `-N` was given a positive value, and first-pass spot finding and the beam-stop pre-scan have ceilings of their own that `-N` does not lift. On a small card the two ceilings, and the per-image loop's 16 where `-N` was left at its default, come down to as many workers as keep their buffers (about 24 bytes per detector pixel each) within half of the card's total memory. Scaling, merging and the space-group search use the full count |
| `-s, --start-image <num>` | First image to process (default: 0) |
| `-e, --end-image <num>` | End image number, exclusive: `-s 0 -e 100` processes images 0-99 (default: all) |
| `-t, --stride <num>` | Process every *n*-th image (default: 1) |
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `<prefix>.*`; `separate` processes each alone into `<prefix>_s<k>_*` |
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
### Mode
`--mode <name>` (default `mx`):
| Value | Description |
| --- | --- |
| `mx` | Full analysis — spot finding, indexing, integration and merging |
| `azint` | Only azimuthal integration (no spot finding/indexing); writes `<prefix>_process.h5` |
| `scale` | Only re-scale/merge the already-integrated reflections in the input `_process.h5` (no re-integration) |
| `calibration` | Determine the detector geometry from powder rings; writes `<prefix>.poni` and `<prefix>.json` |
| `refine` | Refine the `--model` against merged structure factors (an MTZ or a structure-factor mmCIF) and process nothing; see [Refining a model against structure factors](RUGNUX_MODEL_AND_REFERENCE.md#refining-a-model-against-structure-factors-rugnux---mode-refine) (`--refine-score-only`, `--refine-d-min`) |
### Refinement (`--mode refine`)
The columns are found, not named - amplitudes with their sigmas (F/SIGF,
FP/SIGFP, ...) where the file has them, else intensities (IMEAN/SIGIMEAN, I/SIGI, ...) made into amplitudes by
French-Wilson, else the mean of the anomalous pairs; the free set from a FreeR_flag-like column (0 the test set,
or the minority value where 0 is the majority, as phenix writes it), drawn by rugnux where the file has none.
The model is taken as placed in the data's cell, and refined in the data's space group.
| Option | Description |
| --- | --- |
| `--refine-score-only` | Score the model as given and refine nothing: R-work/R-free, the scale, the bulk solvent, the likelihood and the geometry, each term under `REFINE_SCORE_` (`--developer` prints them all); the twin test is the refinement's (`REFINE_TWIN`) |
| `--refine-d-min <num>` | Leave out reflections finer than this (Å) |
### Calibration (`--mode calibration`)
| Option | Description |
| --- | --- |
| `--calibrant <name>` | Powder standard: `lab6` \| `agbh` \| `ceo2` \| `si` \| `ice` (default `lab6`, case-insensitive). A cell given with `-C` overrides it: that cell is the standard, its rings enumerated from the cell, with the absences of `-S` where given |
| `--calibration <txt>` | How the rings are measured: `rings` \| `spots` (default `rings`; see [Detector calibration](RUGNUX_CALIBRATION.md)). `rings` defaults `--azim-phi-bins` to 32 |
| `--no-refine-tilt` | Do not refine the detector tilt: hold rot1/rot2 at the header value and fit only the beam centre and the distance, for a calibration handed to a program that cannot express a tilted detector (XDS). On stills it holds the tilt in the geometry refinement too |
### Detector mask
| Option | Description |
| --- | --- |
| `--detect-beam-stop[=N\|off]` | Find the beam stop and its holder in a projection of N images and add them to the pixel mask as bit 9, so nothing shadowed by them is integrated. **On by default** (60 images); `=off` disables. A reflection whose signal disk reaches into the shadow is not integrated: the rest of such a disk lies in the stop's half-shadow and reads low. Pixels the finder reports that hold the detector's no-data mark (a PILATUS gap) are masked as dead instead |
### Beam centre
| Option | Description |
| --- | --- |
| `--beam-center-check[=off]` | Measure the beam centre from the isotropy of the scattered background on **every** run, report how far the file's value is from it (against how right this geometry needs it to be), and on rotation data index a **second first pass** at the measured centre. The fit reads the projection `--detect-beam-stop` already builds, so it costs no extra frames. **On by default**; `=off` disables it, and with it the measured centre's part in the post-refinement bound. The measured centre is adopted where the file's centre indexes nothing and the measured one indexes a majority; where the two return cells related by an integer volume factor (2 to 4) and the measured centre's cell, larger or smaller, carries materially more of the pooled validation spots against its own chance level; and, on a two-pass run, where the first pass is run at both centres and the measured one merges better — done where the two give the same cell in different metric symmetries, where they agree only once less of each frame is read, and where they agree but lie further apart than the geometry absorbs and the file's centre merges inconsistently in the lowest-resolution shell. Otherwise the file's centre is kept. On stills, where there is no first pass, the file's centre, the measured one and the point-symmetry capture's are each indexed on a sample of frames and the one putting materially more spots on the lattice is taken; with `-C` the detector distance is searched too (30-3000 mm, the file's value one hypothesis). Still runs when `--beam-x`/`--beam-y` are given. See [§1.4](CPU_DATA_ANALYSIS_IMAGE.md) |
### Spot finding
| Option | Description |
| --- | --- |
| `--spot-sigma <num>` | Noise sigma level for spot finding (default: 4.0) |
| `--spot-threshold <num>` | Photon-count threshold for spot finding (default: 10) |
| `--adaptive-spots` | Self-calibrating detection (**default**, stills and rotation alike): the strong-pixel threshold comes from each image's own per-resolution-ring noise instead of the fixed `--spot-threshold`, so one setting adapts across datasets (no per-dataset `--spot-threshold`/`--spot-sigma` tuning) |
| `--no-adaptive-spots` | Turn adaptive detection off and use the fixed `--spot-threshold` / `--spot-sigma` finder |
| `--spot-false-pixels <num>` | Adaptive-detection operating point: expected noise pixels tolerated per frame (default: 100; implies `--adaptive-spots`). A run that finds no lattice at it is made again once at ten times it (`SPOT_FALSE_PIXELS_LOOSENED` in the report) |
| `--spot-high-resolution <num>` | High-resolution limit for spot finding, Å. Omitted (or 0): no resolution clipping — spot finding extends as far as the detector reaches, for rotation data as well as stills |
| `--spot-low-resolution <num>` | Low-resolution limit for spot finding, Å (default: 50; lower it, e.g. 24, to exclude the direct-beam halo on weak serial data; 0 removes the limit) |
| `--min-pix-per-spot <num>` | Minimum connected strong pixels per spot. **If omitted, min-pix is chosen per image** (stills indexing): the frame is scored at min-pix 2/3/1 (one lattice search per frame, refined on each list) and the one maximising indexed-spot count × indexed fraction is kept. Give an explicit value to force a fixed min-pix instead. |
| `--max-spots <num>` | Maximum spots kept per image (the strongest ones) and handed to indexing. **If omitted, the budget is measured** on rotation data: the first pass reads how deep into an image's spot list its spots still lie on the lattice it found, and the run keeps that many (never more than 1000). Give a value to pin it. Stills always use the fixed 1000. |
| `--detect-ice-rings[=on\|off]` | Flag ice-ring spots (de-prioritised in indexing) and exclude ice-ring reflections from scaling; on rotation data the merge then decides ring by ring whether to merge, down-weight or reject them. Default: the master file's `detect_ice_rings`, or — where the file carries no such key — **on for rotation and off for stills** |
### Azimuthal integration
The radial profile behind the per-image ice-ring score, and what `--mode azint` writes. Every *q* here is
*q* = 2π/*d*, in Å⁻¹:
| Option | Description |
| --- | --- |
| `-q, --azim-q-spacing <num>` | Q bin spacing, 1/Å (default: 0.01; finer resolves the narrow ice rings) |
| `--azim-min-q <num>` | Minimum Q, 1/Å |
| `--azim-max-q <num>` | Maximum Q, 1/Å. Omitted: integration extends to the highest Q the detector reaches. The adaptive spot finder shares these Q bins, so this also sets how far self-calibrating detection can see |
| `--azim-phi-bins <num>` | Number of azimuthal (phi) bins (default: 1) |
| `--polarization-correction <on\|off>` | Enable/disable the azimuthal polarization correction |
| `--solid-angle-correction <on\|off>` | Enable/disable the azimuthal solid-angle correction |
### Indexing
A dataset with a **rotation goniometer axis** is processed as rotation data (two-pass rotation
indexing) by default; a dataset without one is processed as independent stills. `--force-still`
overrides the former; the `-R` / `--single-pass-rotation` / `--force-rotation-lattice` flags request
rotation explicitly and pick the pass or lattice.
The FFT search looks for cell axes between 10 Å (`--fft-min-unit-cell`) and a default **longest
axis of 500 Å**, which has no flag of its own — a reference cell (`-C`) moves both bounds to cover
the cell it names. A de-novo rotation run also tries a second first-pass hypothesis with the short
end lowered to 5 Å, so a small-molecule cell below the 10 Å floor is indexed on its true axes
rather than as a supercell of them; the standard pass's answer stands unless that evidence takes it.
The long end is extended the same way: wherever the detector separates the lattice rows of an axis
longer than 500 Å (distance × wavelength / (pixel × spot width)), a coarse search to that length,
up to 1200 Å, offers its lattices as a further hypothesis, kept only where it puts more of the
pooled validation spots on itself beyond chance. An axis beyond the search's reach is **not refused**: the run returns a
plausible shorter sub-cell or an axis harmonic and processes it happily, so a cell that comes out
at a half or a third of the expected long axis should be read as this limit, not as the crystal.
For very long axes the direction grid's angular resolution binds well below 500 Å — see the
[analysis reference](CPU_DATA_ANALYSIS_INDEXING.md#5-fft-indexing-unknown-unit-cell) on FFT indexing.
| Option | Description |
| --- | --- |
| `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation |
| `-X, --indexing-algorithm <txt>` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each; `cuFFT` needs a GPU and is refused without one. `FFBIDX` (known cell, `-C`) runs on the GPU where there is one and otherwise on the CPU. `Auto` takes `FFBIDX` on stills with a known cell, and `FFT` otherwise; rotation data always use the FFT search |
| `-F` | Short for `-X FFT`, kept for old command lines |
| `-C, --unit-cell <cell>` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it. A candidate that is an integer supercell of the cell given (a twin's coincidence lattice, a row taken at a multiple of its length) is stepped down to it |
| `--fft-min-unit-cell <num>` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered |
| `--min-indexed-spots <num>` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it |
| `-S, --space-group <num\|symbol>` | Space group number (`96`) or Hermann-Mauguin symbol (`P43212`) — for indexing and scaling. On serial stills whose group admits two indexings, it also lets the crystals be put on one from the data ([the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)) |
| `-r, --geom-refine <txt>` | Geometry refinement: `none` \| `orientation` \| `beam_and_lattice` (default) \| `flex` (try all three per image, keep whichever indexes the most spots; alias `multi`) |
| `-R, --two-pass-rotation[=num]` | Two-pass offline rotation indexing (default for goniometer data; optional first-pass image count, default 100) |
| `--single-pass-rotation[=num]` | Online-like single-pass rotation indexing (optional min angular range, deg) |
| `--force-rotation-lattice <vec>` | Force rotation lattice (9 floats, Å), skipping the first pass |
| `--rotation-no-postrefine` | Rotation: disable the default-on two-pass geometry post-refine (see [Rotation data](RUGNUX_TUTORIAL.md#rotation-data)) |
| `--prepass-fraction <f>` | Rotation: the two-pass geometry pre-pass integrates the first fraction `f` of the sweep, but at least 120° of it (0.05-1, default **0.5**; `1` integrates the whole sweep twice). The second pass always integrates every frame. The pre-scan and the first-pass indexing sample the same part of the sweep |
| `--refine-geometry[=N\|off]` | Stills: extra first pass that bundle-adjusts the shared beam/distance/cell from N strongly-indexed frames (default 200) then re-indexes; default ON for stills with a reference cell (`-C` / `-z`), a stationary goniometer axis counting as stills, `=off` disables |
| `--index-ice-rings[=on\|off]` | Index on the spots flagged as sitting on an ice ring too, instead of setting them aside (default: **off**; no effect without `--detect-ice-rings`, which does the flagging) |
Indexer choice in brief: `ffbidx` (GPU or CPU) refines toward a **known cell** and is best for sparse
serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rotation data. See the
[CPU/GPU data-analysis reference](CPU_DATA_ANALYSIS_INDEXING.md) for the algorithms.
### Scaling and merging
| Option | Description |
| --- | --- |
| `--no-merge` | Skip scaling and merging, which are on by default; the per-image `_process.h5` is then written instead of the merged files (`_unmerged.mtz` and the results report are still written) |
| `-A, --anomalous` | Anomalous mode: merge each Bijvoet hand as its own unique reflection, so multiplicity, completeness, ⟨I/σ⟩ and the error model are counted anomalously. A default rotation merge already writes `I(+)`/`I(-)` (see [Reflection-file conventions](RUGNUX_INTEGRATION.md#reflection-file-conventions)); `-A` changes the counting basis, not whether the anomalous signal is in the file |
| `--scale-fulls` / `--no-scale-fulls` | rot3d: refit a per-frame scale on the combined fulls (XDS order, Unity model); on by default for rotation data, off for stills |
| `--smooth-g[=deg]` | rot3d: smooth the per-frame scale *G* over a degree range before the 3D combine (XDS DELPHI-like; default 5° for rotation, 0 = off) |
| `--no-scaling-corrections` | rot3d: disable the default-on decay + absorption + modulation correction surfaces fitted on the fulls after scale-fulls |
| `--relative-b[=deg]` | rot3d: fit a per-batch relative-*B* beyond the single decay slope over deg-degree batches, cross-validated (default 10° when bare; off otherwise) |
| `--simple-stills` | Stills: treat every reflection as a full (*p* = 1, single-pass scale/merge) — disables the default-on physical partiality post-refinement |
| `--no-expected-variance-merge` | Stills: disable the default expected-variance merge weighting (which rebuilds each weak observation's signal variance at the reflection mean to de-bias the inverse-variance merge); restores observed-sigma weighting |
| `--capture-uncertainty <num>` | Systematic sigma num·(1−captured fraction)·⟨I⟩: on rot3d fulls the share of the rocking curve not captured, on stills the share of each partial its partiality does not see (default: 1.0 for rotation, 0 otherwise) |
| `--min-captured-fraction <num>` | rot3d: drop a combined full whose rocking curve was captured below this fraction — edge-of-sweep truncated fulls (default: 0.7 for rotation, 0 otherwise; 0 = off) |
| `--scaling-high-resolution <num>` | High-resolution limit for scaling, Å — manual override (default: no limit; disables the automatic cutoff below) |
| `--scaling-low-resolution <num>` | Low-resolution limit of the scale fit, Å (default: 50, which is also XDS's own default; 0 removes the limit). Reflections coarser than it are merged and written but kept out of every fit of the scale model: on a large cell a few dozen of them, measured on a background the beam stop depresses, bent the model enough to move the merged 10–50 Å intensities by tens of percent. Which of them are measured at all is the beam-stop mask's decision - a reflection whose signal disk reaches the shadow is not integrated (`--detect-beam-stop`) |
| `--resolution-cutoff <txt>` | Automatic high-resolution cutoff for the written reflections and reported shells: `cc-logistic` \| `off` (default: `cc-logistic`; ignored when `--scaling-high-resolution` is set) |
| `--resolution-cc-target <num>` | CC1/2 target defining the `cc-logistic` fall-off (default: 0.30) |
| `--resolution-shells <num>` | Number of resolution shells in the reported statistics table (default: 9). The bins are equal steps in 1/d² between the lowest- and highest-resolution reflection merged, which is XDS's rule, and 9 is XDS's count — so at the same resolution limits the two tables have the same shells and can be read row for row |
| `--report-resolution <dmin>[,<dmax>]` | Also report the merging statistics over this resolution range, Å (either order; `dmax` defaults to the run's own low-resolution limit), as a second table beside the run's own — the `REFRES_*` keys of the [results report](RUGNUX_REPORT.md#the-reference-range-table) — binned from the same merged reflections, so a run can be read against another program's table at that program's range. Report-only: the cut, the scaling, the error model and every decision stay the run's own, and shells finer than the run's own limit are reported as not merged rather than filled from data the run did not keep |
| `--min-partiality <num>` | Minimum partiality to accept a reflection (default: 0.02 for rotation, 0.5 for stills) |
| `--ice-min-score <num>` | Ice-presence gate: the measured per-run ice score (1 = no ice) a dataset must reach before **any** ice handling is applied — the flagging, the exclusion from scaling and the per-ring merge decision (default: 1.5; 0 = no gate). The fixed hexagonal bands cover 16–26 % of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal only costs completeness |
| `--ice-min-spot-ratio <num>` | The second ice-presence channel: found **spots** on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so `--ice-min-score` alone is blind to it (default: 2.0; 0 disables this channel) |
| `--reject-outliers <num>` | Per-observation outlier rejection, N σ from the per-reflection median (default: 6 for rotation, 10 for stills; 0 = off) |
| `--min-image-cc <num>` | Per-image CC limit, percent (default: no limit) |
| `--search-min-zeta <num>` | De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry \|ζ\| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law. Where the two searches disagree, the merge of all the observations decides — as it always has for the systematic absences |
| `--mosaicity <num>` | Diagnostic: fix the scaling mosaicity (°) instead of using the per-image seed |
| `--scaling-iterations <num>` | Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle, and a merge that hits the cap is flagged `SCALING_NOT_CONVERGED` (default: 100) |
| `-z, --reference <file>` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity), hands over the R-free set and reports CC<sub>ref</sub>. Not a scale anchor |
| `--free-flags <file>` | Take only the R-free test set of this MTZ or structure-factor mmCIF - e.g. the deposited reflections of the `--model` - for the validation, the refinement and the merged files; nothing else of the file is used, so unlike `-z` it fixes no space group, cell or frame. Taken on the model's axes where the merge matches the file's intensities (`R_FREE_FLAGS=`, see [Free set and model bias](RUGNUX_MODEL_AND_REFERENCE.md#free-set-and-model-bias)) |
| `--reference-column <label>` | Reference column to use, by MTZ label — an mmCIF under the labels `gemmi cif2mtz` gives it (default: auto — F-model, else IMEAN/I/…, else FP/FOBS/F) |
| `--model <file>` | Validate the merged intensities against this atomic model (PDB or mmCIF, gzipped or not; the format is taken from the file's content) — R-work / R-free and maps (see [Validating against a model](RUGNUX_ADVANCED.md#validating-against-a-model-rugnux---model)). An experimental model is placed as a rigid body, and by molecular replacement where the rigid body does not place it; a prediction (AlphaFold or ModelCIF) goes to molecular replacement straight away (`MODEL_PLACEMENT=`). It also settles the frame the reflections are written in: the enantiomorph, and the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity) where no `-z` did; another space group or the twin hypothesis' subgroup it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif`. It does not change the processing: the merge is the same with and without it. A placed model is then [refined](RUGNUX_MODEL_AND_REFERENCE.md#refinement) against the merge it was placed on (`<prefix>_refined_model.cif`, `<prefix>_refined_maps.mtz`, `<prefix>_blobs.json`, `REFINE_` keys) - seconds on a GPU, typically 2-15 minutes on the CPU - which decides nothing about the processing |
| `--mr <file>` | As `--model`, but placed by molecular replacement straight away, without trying the rigid body first: as many copies as the cell holds, in every space group the merge cannot tell apart and in the twin hypothesis' subgroup. A prediction is trimmed to the residues of pLDDT 70 and above, and where the AlphaFold DB's predicted-aligned-error JSON sits beside it under its own name it is cut into rigid domains; any other PDB or mmCIF is taken as an experimental structure, its polymer only (no waters, ligands, hydrogens or alternative conformers). It may relabel the enantiomorph of the answer, and a different space group it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif` with the placed model `<prefix>_mr_model.pdb`. `--mode mx` only; not with `--model` |
| `--no-refine` | Do not refine the placed model (for debugging): by default a model that `--model` or `--mr` places is refined |
| `--refine-solvent` | Also build the solvent of the refined model: waters added at difference-map peaks and removed where the maps no longer hold them, waters and ions relabelled, and each site audited (the `REFINE_SOLVENT_` keys and `<prefix>_solvent.json`). Off by default for now: the model is refined with its solvent as given, and the report says `REFINE_SOLVENT=NOT_PERFORMED`. Also with `--mode refine` |
| `--no-refine-solvent` | The default: no solvent build or audit (turns off an earlier `--refine-solvent`) |
| `--write-process-h5` | Also write the (large) `_process.h5` when merging (default: only `.mtz`/`.cif`) |
| `--developer` | Write the full `<prefix>_report.txt`: the pipeline-internal keys and the long explanations the default report leaves out. Nothing is computed differently — the same report, rendered in full (see [The results report](RUGNUX_REPORT.md#the-developer-report)) |
| `--finalist-ledger` | Report the full-resolution evidence for each space group the search considered, not only the one it adopted, and the twin-immune zone evidence of the adopted group's 2-folds (report-only; the decision is unchanged). Both are read on a merge of the observations in P1 that only this option makes, so it costs one extra merge; rotation only, and not with a fixed centred `-S` |
| `--export-unmerged` | Write `<prefix>_unmerged.mtz`, an unmerged MTZ (POINTLESS column layout), for aimless / pointless. **On by default** whenever there is an output prefix. Rotation partials are summed into one full per reflection, and on a rotation merge `I`/`SIGI` are the merge's own scaled fulls, those it rejected left out. Otherwise intensities carry the deterministic per-reflection corrections and nothing else — Lorentz and polarization in `LP`, sensor efficiency in `QE`, flight path in `FLIGHT` — while the partiality is not divided out and the per-image scale is not applied. Written in `--mode mx` and `--mode scale`, and with `--no-merge`. See [The unmerged export](RUGNUX_INTEGRATION.md#the-unmerged-export) |
| `--no-export-unmerged` | Do not write `<prefix>_unmerged.mtz`. It is the largest file a run produces, so a run whose observations are not going to another scaling program can skip it |
| `--export-unmerged-partials` | Write `<prefix>_unmerged_partials.mtz`, the integrated observations unscaled, each partial as its own row (one batch per image), for a reading program that sums and scales them itself (careless). Off by default, and independent of `--export-unmerged` |
| `--export-observations` | Stills diagnostic: write `<prefix>_obs.mtz`, every integrated observation of the written merge as one row - its signed excitation error, partiality, the run's rocking width, raw I/σ and background, corrections, scale, the σ the merge weighted it with, the frame's cell and orientation, and a bit for each merge test it fails (observations the merge drops included). Processing is unchanged |
| `--dump-observations <file>` | Diagnostic: write the unmerged fulls of every rotation 3D combine to `<file>` as text, for a comparison with XDS |
### Integration
| Option | Description |
| --- | --- |
| `--integrator <txt>` | Spot integrator: `gaussian` (profile-fit, default) \| `empirical` \| `boxsum` (classical fallback) |
| `--integration-radius <r>` | Signal-box radius `r1`, or `r1,r2,r3` (px). One value ⇒ `r2=r1+2`, `r3=r1+4` |
| `--adaptive-integration-radius[=on\|off]` | Set the signal radius `r1` from how wide this crystal's spots actually are (default: **on**). On stills it measures instead how far the background ring should reach: on empty apertures of the pre-scan frames, the `r3` (up to 14) whose ring best predicts the background under the signal disk. `r1` is not the integration domain — that is the profile-fit grid — but it *is* the aperture the profile **width** is learned over, and a second moment over a disk of radius `a` saturates at `a²/4`, so at the shipped `r1 = 4` the learned σ can never exceed 2 px and a broader spot is fitted with a profile the model cannot represent. The pre-scan reads `r80`, the radius holding 80 % of a spot's flux, off isolated strong spots over a fixed 14 px aperture that owes nothing to `r1`, fits it against `1/d` and evaluates it at 5 Å; then `r1 = clamp(round(2·r80), 4, 6)`, `r2 = r1 + 2`, and `r3` is taken so the `r2..r3` background ring keeps the area it has at the default `4,6,13`. The ceiling of 6 is pattern density: `r2` also drives the neighbour-ownership radius and the ring's inner edge, and past it a dense pattern starts losing reflections whose ring falls below six clean pixels. Ignored when `--integration-radius` is given. The widened radius applies to the **final** integration pass only — the two-pass geometry pre-pass keeps the radius the run started with, because the post-refinement fits the detector distance and beam to the observed reflection positions and those move with the signal disk (33 µm and 0.02 px between `r1 = 4` and `r1 = 6` on one crystal, enough for the second pass's de-novo lattice search to settle on a different lattice and index a fifth fewer frames). And where the widened radius leaves more than 1.1 % of the predicted reflections without a background ring — a pattern too dense for it — the final pass is re-integrated at the fixed 4 px radius, and the log says so. Over the rotation regression battery it moves 12 of 38 crystals and leaves the merged intensities of the other 26 unchanged; where it moves them, per-shell ⟨I/σ⟩ improves by up to 31 % and R_meas by up to 24 % |
| `--integration-stencil <k>` | Push the `r2..r3` background ring out by `k` times the beam's radial streak `bandwidth·R_px`, per reflection (default `0` = the fixed circular ring). A fixed ring otherwise ends up on a streaked reflection's own tails at high resolution and measures them as background. Only the ring moves, and only radially — the `r1` signal box stays a circle — and the growth is capped at `2·r3`. The neighbour exclusion grows with it, so on a crowded pattern a few reflections can be left with too little background and dropped. On a monochromatic beam the streak is zero and this does nothing |
| `--background-clip <n>` | High-side clip of the background ring at `mean + n·√mean` (default 4, whatever the bandwidth; 0 = off). The default background estimator — it rejects neighbour cores and zingers without the symmetric trim's Poisson skew bias. Ignored by `--integrator boxsum` |
| `--background-trim <f>` | Use the old symmetric trimmed mean for the background ring instead of the clip, 0≤f<0.5 (`0.10` was the former default). Switches `--background-clip` off. A symmetric trim is biased low on Poisson data and adds ~5 counts to every partial, so this is for back compatibility only; `0` = plain ring mean. Rings holding more than 512 pixels fall back to the plain mean (the GPU sorts the ring in shared memory and the CPU now matches it), which the default radii never reach but wide ones do |
| `--background-radial[=on\|off\|auto]` | Correct the background ring for the **curvature** of the radial background (default **auto**). Disk and ring are concentric, so a background linear in position cancels between them and only curvature survives — which on a smooth ice ring reaches +26 counts on a single reflection. `auto` decides per run in the pre-scan, on empty apertures of the pre-scan frames: on stills it is applied to every image where it makes the ring predict the background of empty apertures better at all; on rotation data to every image where it takes at least 10 % of that error away, and otherwise only to images whose ice score shows a *smooth* powder ring (ice made of discrete crystallite spots leaves no such ring, and the correction makes the bias worse). Ignored by `--integrator boxsum` (no clip pass to take the curve from) |
| `--integration-high-resolution <num>` | High-resolution limit for prediction and integration. Omitted (or 0) means integration extends as far as the detector reaches — which is what the predictor can place on the detector anyway, since it rejects reflections that miss it. Set a value to integrate less than the detector offers |
| `--max-hkl <n>` | Predict reflections with \|h\|,\|k\|,\|l\| ≤ `n` (max 511). By default this is derived per crystal from the refined cell as `ceil(max(a,b,c)/d_min) + 1`, which is the exact bound: the predictor keeps only \|q\| ≤ 1/d_min and `h = a·q`, so no reflection can lie outside it and no candidate inside it is wasted on a shorter axis. Set it only to override that |
| `--bandwidth <num>` | Relative X-ray bandwidth FWHM (e.g. `0.01` for a 1% DMM). Default: the file's value; where the file states none, the one the pre-scan measures from how much longer strong spots are along their radius than across it, used where it is significant; otherwise 0 (monochromatic). `--bandwidth 0` forces a monochromatic beam |
| `--overlap <txt>` | What to do where two predicted reflections share signal pixels: `off` \| `reject` \| `exclude` (default `exclude`). A shared pixel belongs to the nearer centre; without this a crowded reflection reads high on a dense pattern. `exclude` drops the shared pixels from the profile fit, which renormalises itself, and keeps the reflection; `reject` instead drops the whole reflection when too little of its profile is cleanly its own. `--integrator boxsum` has no profile to renormalise, so only `reject` acts there |
| `--overlap-minpk <f>` | Least fraction of a reflection's expected profile that must be usable for it to be kept (default 0.75, XDS `MINPK`). Governs both the fraction that must be **readable** — not masked, untrusted, in a gap or overloaded — in every profile mode, and, under `--overlap reject`, the fraction that must be cleanly its own. Under `--integrator boxsum` any unreadable pixel discards the disk outright and the reject fraction goes by disk *area*, which cuts harder |
| `--flight-path <txt>` | What the diffracted beam crosses between the sample and the detector: `air` (default) \| `helium` \| `vacuum`. A reflection arriving at angle α to the detector normal flies `D/cos α` instead of `D`, so it crosses more of the medium and reads low; the factor is `exp(D/L·(1/cos α − 1))`, with `L` the attenuation length of the medium from the NIST tabulation and `D` and λ as stated by the file — nothing in it is fitted. **Nothing in NXmx, and no field of any master file Rugnux reads, describes the medium**, so it cannot be auto-detected and is assumed; the report states which medium was assumed and what it was worth (`FLIGHT_PATH`, `FLIGHT_PATH_WILSON_B`). Nor can it be inferred from the implied transmission — in this corpus a confirmed helium station sits at 51 % implied air transmission and a confirmed air station at 63 %. In air the term is +1.5 % at α = 55° for 18 keV over 160 mm and several-fold below 5 keV, where stations use helium precisely because air is unusable; helium attenuates ~1/600 of air, and vacuum leaves intensities untouched |
| `--stills-window-sigmas <k>` | Stills: half-width of the prediction window in units of the run's measured rocking width - every frame predicts the reflections within `k` rocking widths of the Ewald sphere (default 2) |
| `--prediction-mosaicity <num>` | Diagnostic: fix the rocking width (deg) the **prediction** window opens to, leaving partiality on the per-image σ_M. The two are one number by default, so a σ_M that moves takes the integrated reflection population with it |
### Geometry overrides
Defaults are taken from the input file; override them to reprocess with a corrected geometry.
| Option | Description |
| --- | --- |
| `--beam-x <num>` | Beam centre X (pixel). `--beam-center-check` still runs and can adopt its measured centre ([§1.4](CPU_DATA_ANALYSIS_IMAGE.md)) |
| `--beam-y <num>` | Beam centre Y (pixel). `--beam-center-check` still runs and can adopt its measured centre ([§1.4](CPU_DATA_ANALYSIS_IMAGE.md)) |
| `--detector-distance <num>` | Detector distance (mm). On stills it is then held: neither searched nor refined. Without it, stills with a reference cell (`-C`) search the distance and refine it with the cell |
| `--wavelength <num>` | Wavelength (Å) |
| `--pixel-size <num>` | Pixel size (µm), for a file that does not state it (an XTC stream of an unknown camera) |
| `--gain <num>` | Detector units per photon, for input in detector units (SACLA, CXI, XTC, SMV, marCCD and `--crystfel-geometry` input). Where the background resolves single photons (SACLA, CXI and `--crystfel-geometry` input) the frames are counted in whole photons as they are read; otherwise they are analysed in their own units and the gain enters where a photon count is assumed. The integrated intensities come out in photons either way. Default: measured on the background of a few frames ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--saturation <num>` | Pixel value from which a pixel is overloaded, in the units the frames are read in. Replaces the file's limit and the one stills read off the frames |
| `--crystfel-geometry <file>` | Place the panels of an HDF5/CXI input by a CrystFEL geometry file instead of the file's own metadata: module geometry, camera length, mask arrays and bad regions (optional; [Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)). Photon energy, camera length and - where the geometry names no mask array - the masks otherwise come from the data files (Cheetah CXI, SACLA) or the command line; gain and saturation are measured. Tilted panels, `dimN` layouts, per-panel data and detector shifts are refused |
| `--module-geometry <file>` | Where the detector's modules are: an NXmx file with one NXdetector_module per module (what a run writes as `<prefix>_modules.h5`) or a CrystFEL geometry file whose panels are laid on the image. Overrides the input's own |
| `--refine-modules[=on\|off\|diagnose]` | Stills: place each module as a rigid block in the geometry bundle adjustment, where that predicts held-out spots better (default: on). Used for a foreign multi-panel detector and for modules from a CrystFEL `.geom` (`--crystfel-geometry`, `--module-geometry`), measured and reported only for a JUNGFRAU; the placement is written to `<prefix>_modules.h5`. `=diagnose` also runs a set of tests on the placement and writes them to the log |
| `--mask-good [<file>:]<dataset>[@<bits>]` / `--mask-bad [<file>:]<dataset>[@<bits>]` | HDF5/CXI stills: a mask array in the frames' stored layout, in `<file>` or in each data file; usable only where non-zero (or with all `<bits>`) / unusable where non-zero (or with any `<bits>`). Repeatable ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--rot1 <num>` | PONI detector rotation 1 (rad). On stills, `--rot1` or `--rot2` holds the tilt (not refined); without them stills refine the tilt where held-out frames agree |
| `--rot2 <num>` | PONI detector rotation 2 (rad) |
| `--rot3 <num>` | PONI detector rotation 3, about the beam (rad) — the one `rugnux --mode calibration` writes into its `.poni`, so a refined third rotation can be given straight back |
| `--detector-mirror-y` | The stored image is mirrored in Y relative to the frame the PONI angles are stated in |
| `--detector-quarter-turns <0-3>` | The stored image is turned by this many multiples of 90° about the beam relative to that same frame |
| `--polarization <num>` | Degree of polarization *p* (XDS `FRACTION_OF_POLARIZATION` = (1+*p*)/2), signed: positive for a horizontally polarized source, negative for a vertically polarized one; where the beam-stop pre-scan finds the background clearly suppressed in the other plane, the degree is applied in that plane instead. Default: what a miniCBF `# Polarization` line or a d*TREK `SOURCE_POLARZ` states (a fraction *f*, read as *p* = 2*f* − 1, so a laboratory tube's 0.5 is unpolarized), and otherwise 0.99, an undulator value; bending-magnet and wiggler sources are typically lower (0.8–0.95), and neither ADSC SMV, marCCD nor NXmx files state it, so pass the beamline's known value here |
| `--rotation-scale <k>` | Goniometer rotation scale: the stage turned `k` times the angle stored in the file (the commanded one). Applied to both passes, and overrides the scale Rugnux fits for itself |
docs/RUGNUX_FORMATS.md
+5 -4
View File
@@ -139,8 +139,8 @@ rugnux reads, and where they really are on the detector plane. The modules come
geometry, never from the image:
* an NXmx file with one `NXdetector_module` group per module (`/entry/instrument/detector/module_0`,
`module_1`, ...: what a run writes as `<prefix>_modules.h5`; the masters Jungfraujoch writes still state
the image as one module);
`module_1`, ...: what a run writes as `<prefix>_modules.h5`, and how the masters Jungfraujoch writes
state a multi-module JUNGFRAU);
* the panels the XFEL readers know (the SACLA sensors from `/metadata`, the panels of a
`--crystfel-geometry` file); each panel is put on the image whole, at a whole pixel, and its fraction of
a pixel and small turn are its placement;
@@ -155,8 +155,9 @@ places each module with enough spots on its frames' lattices as a rigid block (s
`--refine-modules`, on by default) where that predicts held-out spots better; modules move only relative
to one another, the beam centre, distance and crystal orientations carrying what all of them share. The
placement is used from there on for the panels of a foreign multi-panel detector (the sensors of a
panel-table file, the panels of a `--crystfel-geometry` file); for a JUNGFRAU (an NXmx master) it is
measured and reported only. Either way it is written to `<prefix>_modules.h5`, which a later run reads
panel-table file, the panels of a `--crystfel-geometry` file or of a CrystFEL `.geom` given with
`--module-geometry`); for a JUNGFRAU (an NXmx master, with the modules the file gives) it is measured and
reported only. Either way it is written to `<prefix>_modules.h5`, which a later run reads
back with `--module-geometry`.
Spots are always found by `rugnux` itself, including for the two-pass rotation first pass — the
docs/RUGNUX_INTEGRATION.md
+18 -6
View File
@@ -24,9 +24,10 @@ ignores them, and one that does can find them without guessing.
| item | meaning |
|---|---|
| `_reflns.jfjoch_diffrn_ISa` | Asymptotic I/σ in **XDS's sense**: the whole-range `1/√(a·b)` of the error model, so it can be read directly against a `CORRECT.LP` |
| `_reflns.jfjoch_diffrn_ISa` | Asymptotic I/σ in **XDS's sense**: the whole-range `1/√(a·b)` of the error model, so it can be read directly against a `CORRECT.LP`. Rotation only: a stills merge writes the error-model terms instead |
| `_reflns.jfjoch_diffrn_ISa_asymptotic` | The **strong-reflection** tier — the counting-subtracted scatter of well-measured groups. XDS has no equivalent, and it can only ever be the more optimistic of the two. Rotation path only |
| `_reflns.jfjoch_error_model_a`, `_b` | The error model in XDS's convention, `σ² = a(σ₀² + b·I²)`, so the ISa above is re-derivable from the file rather than taken on trust |
| `_reflns.jfjoch_error_model_c` | Stills only: the partiality term, `σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²)` with p the observation's partiality |
| `_reflns.jfjoch_second_moment_I` | Twinning second moment ⟨I²⟩/⟨I⟩² — 2.00 untwinned, 1.50 for a perfect twin |
| `_reflns.jfjoch_L_test_mean_abs_L`, `_L_test_mean_L_squared` | Padilla–Yeates L-test. ⟨\|L\|⟩ is 0.500 untwinned / 0.375 for a perfect twin; ⟨L²⟩ is 0.333 / 0.200. Written only when the test found pairs |
| `_reflns.jfjoch_radiation_damage_relative_B` | Relative *B* from the first to the last rotation batch (Ų); positive is the usual direction, high-resolution intensity fading with dose |
@@ -98,7 +99,7 @@ ctruncate -hklin myrun.mtz -hklout myrun_ct.mtz \
## The unmerged export
`<prefix>_unmerged.mtz` holds every integrated observation, before scaling and merging, in the column
`<prefix>_unmerged.mtz` holds the observations unmerged, in the column
layout POINTLESS writes and **aimless**, **pointless**, **careless** and `iotbx.merging_statistics`
read. It is written by default, in `--mode mx` and `--mode scale` alike and with `--no-merge` as
well, and it replaces nothing — Rugnux still writes its own merged files in the same run. It needs an
@@ -131,7 +132,18 @@ measured is recoverable from the index as stored.
leaves an unmerged file in, and the order AIMLESS requires of its input — so both programs take the
file directly.
**What has been applied to the intensities, and what has not.** `I` and `SIGI` carry the
**Scaled, on a rotation merge.** Where the merge ran on a rotation sweep, `I` and `SIGI` are those of
the merge's own fulls - every correction, the partiality and the scale applied, and the sigma the merge
weighted the full by - as a scaling program's unmerged output carries them, and a full the merge did not
keep (an outlier, or past the resolution cut) is not written; `FLAG` is then 0 throughout. A program that
merges this file reproduces Rugnux's merge, and one that scales it again starts from the scaled data.
The file's title says `scaled`. Each row's other columns are those of the event it was measured in, so the
paragraphs below - which describe the unscaled file written otherwise (stills, `--no-merge`, and
`--export-unmerged-partials`) - still hold for them, but not for `I` and `SIGI`. No `SCALEUSED` column is
written: a full is a weighted combination of its partials, each on its own frame's scale, so no one factor
takes it back to their sum.
**What has been applied to the intensities, and what has not (unscaled).** `I` and `SIGI` carry the
**deterministic per-reflection corrections and nothing else**. Three columns record them: `LP` is
Lorentz x polarization; `QE` is the sensor's quantum efficiency at the angle the diffracted beam
meets the detector; and `FLIGHT` is the attenuation in the medium the reflection crossed on its way
@@ -258,9 +270,9 @@ aimless HKLIN sorted.mtz HKLOUT scaled.mtz
Several sweeps of one crystal form go in as separate `HKLIN`s to the same pointless run — that is
how sweeps Rugnux does not combine itself are merged.
**careless** wants exactly what the unmerged export is — unmerged, unscaled intensities carrying
only the deterministic per-reflection corrections, with the partiality reported and not divided
out. Against its published examples, two renames: `BG`/`SIGBG` are called `BGMEAN`/`BGVAR` here
**careless** wants unmerged, unscaled intensities carrying only the deterministic per-reflection
corrections, with the partiality reported and not divided out - on a rotation merge that is
`--export-unmerged-partials` (`<prefix>_unmerged_partials.mtz`), as `<prefix>_unmerged.mtz` is scaled there. Against its published examples, two renames: `BG`/`SIGBG` are called `BGMEAN`/`BGVAR` here
**and `BGVAR` is a variance, not a sigma**. A `QE` column is present, as in DIALS output
(normalised to 1 at normal incidence where DIALS stores the absorbed fraction — a per-dataset
overall scale). `Hobs`/`Kobs`/`Lobs` are reconstructed from `M/ISYM` by reciprocalspaceship, and
docs/RUGNUX_MODEL_AND_REFERENCE.md
+25 -4
View File
@@ -56,6 +56,7 @@ the rest walks through a run step by step.
| PAE file beside a prediction | `AF-<id>-F1-predicted_aligned_error_v<n>.json` next to `AF-<id>-F1-model_v<n>.cif` (read automatically, no option) | Cuts the prediction into rigid domains for molecular replacement | No (a domain alone can only confirm the answer's group) |
| `-z, --reference <file>` (`--reference-mtz`) | Merged data of the same crystal form: an MTZ, or a PDB structure-factor mmCIF (`-sf.cif`) | Fixes the space group (in its own setting) and the cell (softly), puts the merge on its axes, resolves the indexing ambiguity, hands over its R-free set, reports CC<sub>ref</sub> on stills. Never a scale anchor | **Yes**: the space group, the axes, the indexing and the free set are the reference's |
| `--reference-column <label>` | An MTZ column label | Which column of the reference is read | Through `-z` only |
| `--free-flags <file>` | Merged data carrying an R-free set: an MTZ, or a structure-factor mmCIF - typically the deposited reflections of the `--model` | Its R-free set and nothing else: the model is validated and refined against it, on the model's axes, where the merge matches the file's intensities (`R_FREE_FLAGS= FILE`) | The free flags only |
| `-S, --space-group` | A space group | Fixes it: no space-group search | **Yes**: the group is yours (a fitting model may still relabel its hand) |
| `-C, --unit-cell` | A cell | Indexing looks for this cell (within tolerance); the FFT search range is widened to cover it; on stills it turns on the shared-geometry refinement and, with `-S` and `--model`, the per-image use of the model | **Yes**: the cell, and the axis order the files are written in |
| `--force-rotation-lattice` | Nine numbers: a lattice | Skips the first-pass indexing and uses this lattice | **Yes**: the lattice |
@@ -151,7 +152,9 @@ these letters.
5. **What an accepted model may do** (class b), applied to the reflections before they are written:
- **hand** — where the data's group is the enantiomorph of the model's, the files take the
model's group as their label. No reflection moves and I(+)/I(-) stay as measured. Vetoed where
the anomalous difference map read at the model's atoms is inverted (warning `MODEL_HAND`);
the anomalous difference map read at the model's atoms is inverted (warning `MODEL_HAND`, raised only for a placed model). Where the
group was fixed with `-S` (or by the reference), the model's hand replacing it is warned about
(`MODEL_SPACE_GROUP`);
- **indexing** — where the lattice admits alternative indexings and no `-z` already settled it,
the candidate with the lowest R-free, *if* its lead over the runner-up beats the lead a random
placement of the same model takes (`MODEL_INDEXING_MARGIN_SIGMA=` ≥ 3). The data are
@@ -275,7 +278,8 @@ lists them. The dictionary's hydrogens are added and ride on their parents. Meta
restrained to the donor atoms (O, N, S) the model's LINK / `_struct_conn` records bind them to: by the dictionary's own Zn and Fe
links to Cys and His where they apply, otherwise at the library's typical distance for the metal, the donor
element and the number of such records of the ion (Mg-O 2.07 A, Na-O 2.42 A, K-O 2.82 A at their
usual coordination). A contact without a record, or one to a symmetry mate, is not restrained as a bond. The B factors of bonded atoms and of
usual coordination), where the model's distance is within 0.33 A of that (Mg-O up to 2.4 A); a record at an
outer-sphere distance, a contact without a record, or one to a symmetry mate, is not restrained as a bond. The B factors of bonded atoms and of
atoms in contact (waters included) are restrained to each other in proportion to their size. The weights of
the X-ray term against the geometry and against that B restraint are chosen by R-free among short
refinements whose bond and angle rmsd stay within 0.02 A and 2 degrees: three weights from the placed
@@ -335,7 +339,13 @@ columns are found, not named: amplitudes with their sigmas (F/SIGF, FP/SIGFP, FO
map coefficients such as FC, FWT, DELFWT are passed over) where the file has them, else intensities (IMEAN/SIGIMEAN,
I/SIGI, ...) made into amplitudes by French-Wilson, else the mean of the anomalous pairs. The free set is read from
a FreeR_flag-like column (flag 0 is the test set; where 0 marks the majority, as phenix and CNS write it, the
minority value is); a file without one gets rugnux's own 5 % set. The refinement is the one run after a merge on a placed model, and
minority value is; in a structure-factor mmCIF, `_refln.status` 'f' first); a file without one gets rugnux's own 5 % set.
Where a structure-factor mmCIF's flags are classes (`_refln.pdbx_r_free_flag` 0-19), the class the file marks is not
always the one the depositors held out, so the model as given is scored on each class: the mark is kept where the model
fits that class worst or the test-set count the model file records (REMARK 3, `_refine.ls_number_reflns_R_free`) is
nearest its count; otherwise the class the model fits worst is the test set where that count is nearest it, or where
it is above the marked class by more than twice the scatter of the other classes' R (`REFINE_FREE_SET_SOURCE=
STATUS|REMARK3_COUNT|WORST_FIT`, `REFINE_FREE_SET_CLASS`). The refinement is the one run after a merge on a placed model, and
writes the same files - `<prefix>_refined_model.cif`/`.pdb`, `<prefix>_refined_maps.mtz`, `<prefix>_blobs.json` - with
the `REFINE_` keys in `<prefix>_report.txt`, beside `REFINE_FRENCH_WILSON=` and `REFINE_FREE_FLAGS= FILE|GENERATED`
saying what had to be made of the input.
@@ -343,7 +353,7 @@ saying what had to be made of the input.
| Option | Description |
| --- | --- |
| `--model <file>` | The model (required; it can also be given among the inputs, told by content) |
| `--refine-score-only` | Score the model as given and refine nothing; every term of the score - R-work and R-free overall and in 20 resolution shells, the overall scale, B and anisotropic B, k<sub>sol</sub> and B<sub>sol</sub>, the likelihood over the working and the free reflections and its figure of merit over the working reflections, and the geometry's rmsd and rmsZ - under `REFINE_SCORE_` keys (`--developer` prints them all) |
| `--refine-score-only` | Score the model as given and refine nothing; every term of the score - R-work and R-free overall and in 20 resolution shells, the overall scale, B and anisotropic B, k<sub>sol</sub> and B<sub>sol</sub>, the likelihood over the working and the free reflections and its figure of merit over the working reflections, and the geometry's rmsd and rmsZ - under `REFINE_SCORE_` keys (`--developer` prints them all). The twin test is the refinement's: the model is scored against the data twinned by each law too, and the twinned score taken by the same rule (`REFINE_TWIN`) |
| `--refine-d-min <A>` | Leave out reflections finer than this |
## Free set and model bias
@@ -359,6 +369,17 @@ the new axes by the same rule; an indexing-ambiguity reindex leaves it unchanged
mates share a flag. With `-z`, the reference's flags are inherited where the merge matches it
(`REFERENCE_FREE_FLAGS_INHERITED=`).
**A model refined against these reflections.** A set drawn here is free only for a model that was never refined
against the crystal's reflections. A deposited model, or one refined against an earlier dataset of the same
crystal, has been fitted to every reflection, so its R-free on a fresh set reads close to its R-work - below it,
sometimes - and a refinement from it reads no cross-validation at all. `--free-flags <file>` takes the test set of
the reflections it was refined against (the deposited `-sf.cif`, or an MTZ) and nothing else from them: unlike
`-z` it fixes no group, cell or frame and steers no processing. The flags are put on the merge once it is on the
model's axes, where the merge matches the file's intensities as `-z` requires (CC at least 0.5 over at least half
of its reflections); the validation is then made again on them, and the refinement and the merged files carry
them. `R_FREE_FLAGS=` says which set `R_FREE` and the refinement used: `DRAWN`, `REFERENCE` (`-z`) or `FILE`. A
model placed as given and scored on a set drawn here is warned about (`MODEL_FREE_SET`).
**What is fitted on which reflections.** Everything fitted to place and scale the model — the
overall scale, the anisotropic *B*, the bulk solvent, the rigid body — is fitted on the working
reflections only. The free set is then used for a few discrete *selections*: whether the rigid-body
docs/RUGNUX_OVERVIEW.md
+1 -1
View File
@@ -35,7 +35,7 @@ shortest accepted axis lowered from 10 to 5 Å, for small-molecule cells. The fi
second time at the beam centre the pre-scan measured; the file's centre is kept unless it indexes
nothing, gives an axis harmonic of the measured centre's lattice, or loses to it when both first passes
are merged ([§1.4](CPU_DATA_ANALYSIS_IMAGE.md)). A failed pass triggers the discrete rescues — the
rotation-axis sign, the first-pass depth ladder, the long-axis rescue — before anything is given up on.
rotation-axis sign, the first-pass depth ladder, the long-axis hypothesis — before anything is given up on.
**First integration pass.** At the geometry in the file, every frame of the first half of the sweep, at least 120° of it
(`--prepass-fraction`) is predicted
docs/RUGNUX_REPORT.md
+215 -18
View File
@@ -60,6 +60,15 @@ scaled-up counting error gives it, and is in `R_MEAS_WEIGHTED=` at that weight t
the number with its noise; a large gap between the two says weak frames are kept at low weight.
`MULTIPLICITY=` stays a count of observations.
**`CC_HALF=`** is the standard CC1/2, as XDS, Aimless and DIALS report it and as the mmCIF item
`_reflns.pdbx_CC_half` defines it: the observations of each reflection are dealt into two half-sets,
each half is averaged with 1/sigma^2 weights, and CC1/2 is the plain Pearson correlation of the two
half-set means, every reflection counted alike. The same holds for `REFRES_CC_HALF=`, the CC1/2 and CC\*
columns of every shell table and the mmCIF. The automatic resolution cut is fitted to a different,
internal form of CC1/2, in which a reflection measured only on frames the crystal barely diffracted on
counts less (`FITTED_RESOLUTION_INTERNAL`); `FITTED_RESOLUTION` is read off the standard one. On a sweep
without a weak stretch the two agree.
**`R_MERGE=`** and **`R_PIM=`** are the other two merging R-factors over the same observations as
`R_MEAS=`: R_merge without the multiplicity correction, and the precision-indicating R_pim (Weiss,
2001), which falls as the multiplicity rises. **`CC_STAR=`** is CC\* = sqrt(2 CC1/2 / (1 + CC1/2))
@@ -73,8 +82,13 @@ half-sets - as CrystFEL's partialator deals out crystals - and R_split = (1/sqrt
(1/2 sum (I_1 + I_2)) (White et al., 2012) and CC1/2 are taken over the reflections both halves
measured, so both compare directly with CrystFEL's `compare_hkl`. `CC_HALF=` deals each reflection's
observations out evenly instead, as XDS and cctbx do, and is the one the processing decides on; the two
agree overall and can differ in a single shell. On stills `R_MEAS=` measures the scatter of partial
observations, not the error of the merged intensity, and is not the number to quote.
agree overall and can differ in a single shell.
**`R_MERGE=`, `R_MEAS=`, `R_PIM=` and `R_MEAS_WEIGHTED=` are rotation statistics** and a stills report
has none of them, nor their `REFRES_` forms or their columns in the shell tables (the mmCIF writes `?`
for them). On stills they would measure the scatter of partial observations scaled to full
equivalents - hundreds of percent - not the error of the merged intensity. `R_SPLIT=`, `CC_HALF=`,
`CC_STAR=`, `CC_ANOM=`, `MULTIPLICITY=` and `I_OVER_SIGMA=` are reported as on rotation.
**`OBSERVATIONS_REJECTED=`** counts the observations the merge's outlier tests removed, and
**`OBSERVATIONS_REJECTED_WILSON=`** the part of them the Wilson test took (each listed in the developer
@@ -83,12 +97,13 @@ out because a pixel of the spot was overloaded on one of its frames, as XDS leav
reflection. Outlier rejection takes observations out of `R_MEAS=` and `CC_HALF=` too, so a run that
rejects much scores better on both; read the counts beside them.
The last blocks before `END OF REPORT` are the authorship and the acknowledgement: who wrote
The first blocks of the report are the authorship, the acknowledgement and the citation: who wrote
rugnux, its licence (GPLv3 — free to use for academic institutions and commercial companies alike)
and where releases are published, then the credit to the X-ray research community whose methods
rugnux implements and the open-source projects it is built on — both credited in
`ACKNOWLEDGEMENT.md` beside `LICENSE` and `THIRD_PARTY_NOTICES.md` in the installed package.
rugnux prints the acknowledgement at startup as well.
`ACKNOWLEDGEMENT.md` beside `LICENSE` and `THIRD_PARTY_NOTICES.md` in the installed package, and
the paper to cite when rugnux was useful (see [Publications](PUBLICATIONS.md)). rugnux prints the
acknowledgement and the citation at startup as well.
`REPORT_VERSION=` is the format's own version. Key names, table columns and the reason vocabulary
below are an interface other software may depend on: they do not change without that number moving.
@@ -120,7 +135,8 @@ The header block above section 1 records **how the result was produced**: `RUGNU
Rates, per-image costs and progress remain on stdout only.
Sections, in order: the `SUMMARY` ([below](#the-summary-and-what-the-run-decided)), then
`1. DATA SET AND GEOMETRY`, `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
`1. DATA SET AND GEOMETRY`, on a joint merge of several sweeps `1b. SWEEPS`
([below](#several-sweeps-merged-as-one)), `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
was given (`--model`, `--mr`, or a model file among the inputs), `5. MODEL VALIDATION` - which also
holds the keys of [molecular replacement](#molecular-replacement) and of the
[refinement](#refinement-refine_) where they ran. A `--mode refine` report has one section,
@@ -135,6 +151,11 @@ prediction, good to about 0.2 Å on rotation data; nothing is cut on it. It is *
the detector records: where it reads finer than the high-resolution end of `INCLUDE_RESOLUTION_RANGE`,
the crystal diffracts past the corner and the run was detector-limited.
**`SPOT_FALSE_PIXELS_LOOSENED=`** in section 1 is there only on a run that found no lattice at the spot
finder's operating point (`--spot-false-pixels`, noise pixels tolerated a frame) and was made again at ten
times it: the operating point the result was found at. On a weak sweep the default finds a handful of
spots a frame, too few to index; ten times the noise pixels still leaves the spots mostly the crystal's.
**`JFJOCH_DATASET_SETTINGS=`** in section 1 is the geometry the run integrated at — on a rotation run
the post-refined one — written as the object `jfjoch_broker` takes it in: the four required properties
of `dataset_settings` in `broker/jfjoch_api.yaml`, joined by the three `poni_rot*_rad` angles whenever
@@ -221,16 +242,33 @@ The keys most scripts read, by section; the rest of this page explains the ones
| `DETECTOR_DISTANCE`, `BEAM_CENTRE`, `DETECTOR_TILT` | The geometry the run integrated at (mm; the point of normal incidence in pixels; the three PONI rotations in degrees) - on a rotation run the post-refined one, not the header's |
| `DIRECT_BEAM` | Where the direct beam meets the detector under that tilt (pixels) |
| `LATTICE_FOUND`, `UNIT_CELL_CONSTANTS`, `CELL_ESD` | Section 2: whether a lattice was found, the cell (a b c α β γ, Å and degrees) and its e.s.d.s ([below](#cell-esds)) |
| `CELL_FORMS`, `OTHER_CELL_FORM_*` | Stills: whether the frames on the run's cell were one crystal form or two, and the other one, left out of the merge ([below](#two-crystal-forms-stills)) |
| `SPACE_GROUP_NAME`, `SPACE_GROUP_NUMBER` | The space group the files are written in ([below](#the-space-group-and-the-sohncke-answer-beside-it)) |
| `MOSAICITY_DEG` | Rotation only ([below](#mosaicity)) |
| `MERGE` | Section 3: `PERFORMED` or `NOT_PERFORMED` (`--no-merge`) |
| `INCLUDE_RESOLUTION_RANGE` | `dmax dmin` of the written reflections (Å), as XDS's keyword |
| `FITTED_RESOLUTION` | Where the CC1/2 fall-off crosses the `--resolution-cc-target`; absent where the curve is not a fall-off (a `RESOLUTION_FIT` warning says why) |
| `FITTED_RESOLUTION` | Where the fall-off of the standard CC1/2 (the `CC_HALF` of the shell table) crosses the `--resolution-cc-target` - the number to quote; absent where the curve is not a fall-off (a `RESOLUTION_FIT` warning says why) |
| `FITTED_RESOLUTION_INTERNAL` | Rotation only, internal: the crossing of the automatic cut's own fit, which weights each reflection by the scale of the frames it was measured on so that a weak stretch of the sweep does not drag it in. The written range (`INCLUDE_RESOLUTION_RANGE`) is cut from this one; on a sweep without a weak stretch it equals `FITTED_RESOLUTION` |
| `RESOLUTION_CUT_PLAIN` | Diagnostic, rotation only: the limit the cut would have written from the fit on the standard CC1/2. Nothing is cut at it |
| `UNIQUE_REFLECTIONS`, `TOTAL_OBSERVATIONS`, `COMPLETENESS`, `MULTIPLICITY` | Over the written range; completeness in percent |
| `I_OVER_SIGMA`, `R_MEAS`, `CC_HALF`, `WILSON_B` | The overall merging statistics; the shell table beside them is in the comment lines |
| `I_OVER_SIGMA`, `R_MEAS`, `CC_HALF`, `WILSON_B` | The overall merging statistics; the shell table beside them is in the comment lines. `R_MEAS` (and `R_MERGE`, `R_PIM`) rotation only, `R_SPLIT` stills only |
| `SIGANO`, `CC_ANOM`, `FRIEDELS_LAW` | The anomalous signal, and whether Friedel pairs were merged (`FALSE` with `-A`) |
| `ERROR_MODEL_A`, `ERROR_MODEL_B`, `ISA` | The error model in XDS's convention and ISa = 1/sqrt(a·b); `ISA= undetermined` where b is zero within its error, so 1/b would be noise |
| `ERROR_MODEL_A`, `ERROR_MODEL_B`, `ISA` | The error model in XDS's convention and ISa = 1/sqrt(a·b); `ISA= undetermined` where b is zero within its error, so 1/b would be noise. Rotation only for `ISA` |
| `ISA_LOW`, `ISA_HIGH` | Rotation data: the 95% interval of `ISA` from a bootstrap over the reflections (100 replicates, fixed draws). With `ISA= undetermined`, `ISA_LOW` alone is a lower bound: b reaches zero inside its interval, so only ISa ≥ `ISA_LOW` is measured. Both absent where there is no fit to resample (b out of reach of the data, or a on its bound) and on stills, which write no `ISA`. `ISA_ASYMPTOTIC` (`--developer`) is the ISa of the strong-reflection tier alone |
| `ERROR_MODEL_C` | Stills only: the partiality term of the stills error model, σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²) in the same convention. A stills report writes no `ISA` ([below](#the-stills-error-model)), and `ERROR_MODEL_B= undetermined` where b is zero within its error |
| `R_WORK`, `R_FREE` | Section 5: the model as placed against these data ([below](#model-validation)); the refined model's are `REFINE_R_WORK`/`REFINE_R_FREE` |
| `R_FREE_FLAGS` | Section 5: the free set both are read on - `DRAWN` by this run, inherited from the `-z` `REFERENCE`, or from the `--free-flags` `FILE` |
### The stills error model
A stills merge reports its error model as its terms, `ERROR_MODEL_A`, `ERROR_MODEL_B` and
`ERROR_MODEL_C`, with σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²), p being the partiality of the observation:
b is the systematic error of a fully recorded still and c the error a partial one adds. It writes no
`ISA`. On stills 1/sqrt(a·b) is the I/σ ceiling of a single still observation under the stills model -
the same frames processed as a rotation sweep read several times higher - so it describes the
processing model more than the instrument and the crystal, and read on `CORRECT.LP`'s scale it would
call good serial data useless. The serial figures of merit are `R_SPLIT=`, `CC_HALF=` and `CC_STAR=`.
The mmCIF follows: `_reflns.jfjoch_error_model_c` instead of `_reflns.jfjoch_diffrn_ISa`.
## The summary, and what the run decided
@@ -249,9 +287,9 @@ and a run that produced a textbook data set read identically for their first thr
consumer switches on a code rather than parsing a sentence: `NO_LATTICE`, `INDEXING_AMBIGUITY`,
`SYMMETRY_AMBIGUITY`, `CENTERING_UNTESTED`, `UNUSABLE_MERGE`, `LOW_COMPLETENESS`, `SWEEP_GAPS`,
`GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`,
`LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
`LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
`FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`, `SCALING_NOT_CONVERGED`, `HARMONIC_CONTAMINATION`,
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`. `NONE` when nothing fired. A code appears if and only if its
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `CELL_FORMS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`, `SWEEP_LEFT_OUT`. `NONE` when nothing fired. A code appears if and only if its
warning fired, so the flags and the `WARNING:` lines are two renderings of one list. Thresholds are
deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on
data that turn out fine — the closed
@@ -285,6 +323,46 @@ and a run that produced a textbook data set read identically for their first thr
Then the numbered sections: **1. DATA SET AND GEOMETRY**, **2. CRYSTAL**,
**3. MERGED DATA**, **4. DIAGNOSTICS**, and **5. MODEL VALIDATION** only where a model was given.
## Several sweeps merged as one
A run of several sweeps merged as one (`--sweeps joint`, the default with several rotation inputs, see
[Command-line options](RUGNUX_CLI.md)) describes the joint merge everywhere, and adds section
**1b. SWEEPS**; a run of one input has no such section and no such keys.
- **`SWEEP_COUNT=`** the sweeps given, **`SWEEP_REFERENCE=`** the one (1-based) the others were put on
the axes of - the one with the most reflections - and **`SWEEP_DECAY_MODEL=`** how the decay over the
sweeps was modelled: `dose-linear` (a B rising along the dose order), `per-sweep B` (one B a sweep) or
`none`, whichever predicted held-out frames better.
- Per sweep `k`: **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep
gave processed on its own - **`SWEEP_k_SPACE_GROUP_ALONE=`**, **`SWEEP_k_CELL=`**,
**`SWEEP_k_CC_HALF_ALONE=`**, **`SWEEP_k_COMPLETENESS_ALONE=`**, **`SWEEP_k_RESOLUTION_ALONE=`**;
**`SWEEP_k_ISOMORPHISM_RATIO=`** / **`SWEEP_k_ISOMORPHISM_RATIO_SE=`**, whether it measures the reference
sweep's intensities: their correlation over what it would be for one set of intensities given each sweep's
own CC1/2 (CC_AB / (CC*_A CC*_B), Karplus & Diederichs 2012), on the reflections both have in the coarser
half of their common range, with a bootstrap error - 0.997-1.001 for sweeps of one crystal or isomorphous
crystals, 0.95-0.96 for non-isomorphous ones. **`SWEEP_k_DECISION=`** says what became of the sweep:
`reference` (the sweep whose own merge reaches the finest resolution, ties: the higher ISa, whose axes and cell the joint merge takes),
`isomorphous` (ratio - 2 se >= 0.99: joined), `not isomorphous` (ratio + 2 se < 0.97: left out),
`doubtful: kept` / `doubtful: left out` (in between, or too few common reflections for a ratio: decided
on the merge), `no axes` (it could not be put on the reference's axes) or `no merge`. For a doubtful sweep,
**`SWEEP_k_HELD_OUT_CHANGE=`** / **`SWEEP_k_HELD_OUT_CHANGE_SE=`**: the median relative change of the held-out
residual of the merge of the joined sweeps - each half of their observations predicted from the other
half, without and with this sweep's added, over the reflections it has, after a scale and B factor onto
the reference - with its bootstrap error, and **`SWEEP_k_HELD_OUT_ADDED=`**, the unique reflections it adds to
that merge; the sweep joins where it helps: the change is negative, or it adds at least 1% to the merge's
unique reflections and the change is not above twice its error; **`SWEEP_k_DISTANCE=`**, the detector
distance its run ended at, and **`SWEEP_k_REPROCESSED_FROM=`** where the sweep was processed a second time
(see [RUGNUX_CLI](RUGNUX_CLI.md)): the distance its first run ended at; and,
for a sweep in the joint merge, **`SWEEP_k_REINDEX_OP=`** (onto the reference's axes) with
**`SWEEP_k_REINDEX_CC=`** (the merged-intensity correlation under it), **`SWEEP_k_BATCH_FIRST=`** /
**`SWEEP_k_BATCH_LAST=`** (its batches in the unmerged MTZ: sweep `s` of the merge is
`s * base + frame + 1`, `base` the power of ten past the longest sweep), **`SWEEP_k_SCALE=`** (the
median scale of its frames in the joint merge), **`SWEEP_k_RELATIVE_B=`** (its B against the joint
merge, the least damaged sweep at 0), **`SWEEP_k_ISA=`** (the ISa of its own error model in the joint merge: each sweep's observations are weighted under an error model fitted on their own scatter about the joint means, as XSCALE does per data set) and **`SWEEP_k_DOSE_RANK=`** (its place, from 1, in the dose
order those B factors decide - not the order the sweeps were given in, nor their header times).
- A sweep given but not merged - it gave no merge, could not be put on the reference's axes, is not
isomorphous, or was doubtful and did not help the merge - is named by a `SWEEP_LEFT_OUT` warning.
## The reference-range table
`--report-resolution <dmin>[,<dmax>]` adds a second block of merging statistics to section 3, the
@@ -599,13 +677,17 @@ summary's `Powder` line gives both. A warning under the `ICE_RINGS` flag is give
at least 5 % of the spots, and one under `POWDER_RINGS` where the other rings do.
Hexagonal ice is also measured by the merge itself, which leaves reflections on the ice rings out of
scaling (and keeps them in the merge) where it finds ice:
scaling where it finds ice and, on rotation data, decides ring by ring what the merge does with them:
```
ICE_RINGS_DETECTED= TRUE
ICE_RING_SCORE= 2.69
ICE_SPOT_RATIO= 1.08
ICE_REFLECTIONS_ON_RINGS_PCT= 25.7
ICE_RING_BAND_01= 3.895 LOW_FLANK MERGE 42 -1.1 0.81
ICE_RING_BAND_02= 3.895 CORE MERGE 52 +1.0 2.59
ICE_RING_BAND_03= 3.895 HIGH_FLANK MERGE 43 -2.0 0.94
ICE_RING_BAND_04= 2.249 CORE REJECT 135 -5.4 7.83
```
**`ICE_RING_SCORE=`** is the strongest ice ring over the smooth radial background (fine-grained ice
@@ -613,7 +695,13 @@ that makes a smooth ring), **`ICE_SPOT_RATIO=`** the pile-up of found spots on t
against the ice-free flanks beside them (ice in large crystallites); 1 is no ice on either.
**`ICE_RINGS_DETECTED=`** is `TRUE` where either passes its gate (1.5 and 2.0), and
**`ICE_REFLECTIONS_ON_RINGS_PCT=`** is then the share of the integrated reflections set aside from
scaling. The summary has an `Ice` line, and where the merge found ice an `ICE_RINGS` warning prompts a
scaling. **`ICE_RING_BAND_nn=`** (rotation only) is one line per band of a ring - its core, within
0.01 Å$^{-1}$ in $q$ of the ring, and its low- and high-$q$ flanks out to the flag width: the ring's
$d$ (Å), the band, what the merge did with its reflections (`MERGE`, `DOWN_WEIGHT`, `REJECT`), how
many merged reflections it holds, their bias against the off-ring reflections around the ring in
standard errors, and the ratio of their variance to those reflections' (both on intensities
normalised by a local Wilson fit). A band is rejected where it is biased beyond 3 standard errors,
down-weighted where it is only noisier beyond 3, and merged otherwise. The summary has an `Ice` line, and where the merge found ice an `ICE_RINGS` warning prompts a
look at the ice-ring shells (one warning line, whichever of the two measurements raised it). The measurement is described in
[CPU/GPU data analysis ▸ Resolution and ice-ring handling](CPU_DATA_ANALYSIS_IMAGE.md#33-resolution-and-ice-ring-handling).
@@ -698,6 +786,13 @@ small offset, fitted over the class - modulates at least half of the added class
shell, so the two halves of the cell are one content displaced - and where the doubled merge is not more
symmetric than the half cell's, which a superstructure of it cannot be. A class that is there whatever the
direction (copies that differ in content, twin domains, neighbouring reflections) keeps the half cell.
On a sweep too weak for the probe's 60 frames to measure the lattice's own reflections in any shell, the
probe cannot read the rise; there the class only has to stand clear of the six empty ones in two shells,
the doubled cell is merged, and the rise is read on the doubled merge instead - the added class against
the lattice's own mean intensity, shell by shell, with the same test - before the same symmetry test and
the translation test decide. Such a sweep has too few added reflections to read the translation shell by
shell (the probe arm needs at least 1000), so there the share it modulates is read over all shells at once
and taken at its lower bound: it has to be at least half two standard errors down. **`DOUBLED_CELL_RISE_READ_ON=`** says which (`PROBE` or `MERGE`).
**`DOUBLED_CELL_ADOPTED=`** says which was written as the answer and **`DOUBLED_CELL_DECISION=`** why;
**`DOUBLED_CELL_OTHER_FILE=`** names the other hypothesis's merged file (`<prefix>_doubled.mtz`, or
`<prefix>_halfcell.mtz` where the doubled cell was adopted, with the rest of its files under the same
@@ -748,6 +843,27 @@ as the main one can still give data as good as XDS's, since the domains' reflect
where they overlap the main lattice's. Look at the crystal, and at the result. The summary at the top of
the report has a `Multiple lattices` line, `none found` where nothing was listed.
A warning under the **`LATTICE_EXPLAINS_FEW_SPOTS`** flag fires where `MAIN_LATTICE_SPOTS_PCT=` is under 50 and a
listed lattice other than a domain (`RELATED_CELL` or `FOREIGN`) shares two of the main cell's axes to 2 %. That is
what the census of a cell with one axis wrong looks like - its spots along the missed row split into several
leftover lattices with the same two axes and a different third - so check the cell. It also fires on correct cells
whose leftover spots happen to line up along two of the axes.
## Two crystal forms (stills)
A serial sample can hold two crystal forms of one protein whose cells are close enough to index on one
cell. Merging both mixes two structures, so the merge keeps one form and leaves the other's frames out
([how the forms are told apart](CPU_DATA_ANALYSIS_INDEXING.md)): without `-C` the major form, with `-C`
the given cell's. Section 2 then reports, on every stills run:
- **`CELL_FORMS=`** — `1` or `2`.
- **`OTHER_CELL_FORM_FRAMES=`**, **`OTHER_CELL_FORM_FRACTION=`** — where there are two: the frames of the
other form left out, and their share of the indexed frames on the run's cell.
- **`OTHER_CELL_FORM_CELL=`** — their mean cell, on the axes of the merged form's cell.
A warning under the **`CELL_FORMS`** flag, the summary's `Crystal forms` line and `VERDICT_TEXT` say so,
with the `-C` to process the other form. `UNIT_CELL_CONSTANTS=` is the merged form's own cell.
## Cell e.s.d.s
**`CELL_ESD=`** (section 2, rotation only) is the standard uncertainty of each of the six numbers of
@@ -823,6 +939,14 @@ replacement needs it.
frac(**h**·**u**), with **u** the refined peak vector (**`TNCS_VECTOR=`**, fractional, and
**`TNCS_VECTOR_LENGTH=`** in Å). **`TNCS_MODULATION_NULL=`** is the same search started from random
vectors, so the contrast the search itself can manufacture is measured rather than assumed.
- **`TNCS_MODULATION_AVERAGED=`** / **`TNCS_MODULATION_AVERAGED_NULL=`** — written only where the
Patterson peak is significant but the vector's own modulation is not: the same test with each
reflection's class the mean of cos 2π**h**·R**u** over the lattice's rotations R (its symmetry and
every twin law the metric admits), against the same permutation null. A symmetry operator or twin
law that carries **u** to a vector other than ±**u** averages the vector's own classes - a perfect
twin does so by construction - and this is the test that still sees the modulation then.
**`TNCS_CLASSES=`** (`VECTOR` or `LATTICE_AVERAGED`) says which of the two a detection stands on; the
L-test partner steps then keep the class of every image R**u**.
The vector is good to about 0.05 fractional. It is a starting point for a program that refines it,
not a refined result: the refinement maximises the modulation, not the accuracy of the vector.
@@ -964,7 +1088,7 @@ tried (a prediction, `--mr`) or its null could not be built.
| key | meaning |
|---|---|
| `MODEL_VALIDATION` | `PERFORMED` \| `NOT_PERFORMED` (with `MODEL_VALIDATION_REASON`, and no R-factors) |
| `MODEL_VALIDATION` | `PERFORMED` \| `NOT_PERFORMED` \| `CELL_MISMATCH` (both with `MODEL_VALIDATION_REASON`, and no R-factors). `CELL_MISMATCH`: no change of basis (an axis permutation, a sign flip, I- against C-centred) carries the model's cell onto the data's within 5% in each length and 3 deg in each angle, and the two primitive cells differ in volume by more than that allows, so the model is of another lattice - a cell of another volume, an axis of another length - and is not put into the data's cell to be scored (a centred description of the same primitive cell is not caught here); the reason names both cells. It is reported here only, not as a warning: it is about the model given, and the data's `VERDICT`, `PATHOLOGY_FLAGS` and `WARNING_COUNT` do not depend on it. Molecular replacement is still tried, as for any model that is not placed as given |
| `MODEL_PLACEMENT` | `RIGID_BODY` (the model fits against its null in the answer's merge or the twin hypothesis' one) \| `MR` (placed by molecular replacement) \| `NOT_PLACED` |
| `MODEL_FIT` | `ACCEPTED` \| `REJECTED` — the rigid body's verdict: whether the model may decide anything. Absent where no rigid body was tried |
| `MODEL_FIT_STATISTIC` | What the verdict was taken on; `R_WORK` |
@@ -988,6 +1112,33 @@ twin hypothesis' merge is also made where a metric re-ask promoted the indexed l
the model's group has fewer rotations than the adopted one; made for that reason alone, it is written and
reported only where the model chooses it.
**Symmetry hypotheses of a refused promotion.** Where the search refused a higher point group, the
adopted group is one of several hypotheses the merged intensities cannot separate: the adopted group
untwinned or twinned by a law of the refused group outside it, and each *other* index-2 subgroup of the
refused group perfectly twinned by a law of its coset - each of them merges into the refused group alike.
The report lists them, on every run, as `SYMMETRY_HYPOTHESES=` (their number) and
`SYMMETRY_HYPOTHESIS_<n>= <group> | <law or NONE> | ADOPTED / ADOPTED_TWINNED / SIBLING_TWINNED |
TESTED_BY_MODEL / UNTESTED` (`TESTED_BY_MODEL` only where the model was placed, `MODEL_PLACEMENT=` `RIGID_BODY` or `MR`: a rejected model leaves these lines as a run without one writes them); a sibling's group is named by its rotations alone (symmorphic), its screws
left to the model. They cost nothing without a model. With `--model`, where the model's own group is a
sibling's (it has the sibling's rotations on the data's axes), that sibling's merge is the twin hypothesis
the model is scored on, exactly as a subgroup's above, and the model is refined in its own group there.
With a model that has no group of its own (a prediction, or `--mr` with coordinates without a cell), every
sibling's merge is searched by molecular replacement beside the answer's.
The placed model is refined as the merge's group holds it: in the model's own group where that has the
merge's rotations (so the screws the merge's label leaves open are the model's), and expanded into the
merge's group where the model's group has more rotations (each symmetry image the merge's group lacks
becomes an independent copy). Where the model fits both the answer's merge and the twin hypothesis' within
0.03 in R-free before refinement, and each merge's group holds the model whole that way, **both are
refined**, each with its own twin decision (below), and the refined R-free decides
which the model chooses: the rival replaces the validation's choice only where it refines better by 0.01,
what two refinements of one model differ by from their paths alone. `MODEL_CHOICE_DECIDED_BY=` is
`VALIDATION` or `REFINEMENT`; the rival's `RIVAL_REFINE_SPACE_GROUP`, `RIVAL_REFINE_MERGE`,
`RIVAL_REFINE_R_WORK`, `RIVAL_REFINE_R_FREE`, `RIVAL_REFINE_TWIN` (with `RIVAL_REFINE_TWIN_LAW`) and
`RIVAL_REFINE_REPLACED` stand beside the `REFINE_` keys of the one kept. `MODEL_CHOICE_TWIN_LAW=` is the law
the model's choice file carries (`NONE` without one). The model's choice is refined against its own files
by either route, molecular replacement's included.
The null's own numbers — `MODEL_FIT_STATISTIC`, `MODEL_FIT_VALUE`, the `MODEL_FIT_NULL_*` keys and
the `MODEL_INDEXING_MARGIN*` keys — are written with `--developer`; the default report carries the
verdict (`MODEL_FIT`, `MODEL_FIT_SIGMA`, `MODEL_DECISIONS_TAKEN` and the two decisions).
@@ -1188,27 +1339,70 @@ Where a model was placed and refined ([Refinement](RUGNUX_MODEL_AND_REFERENCE.md
| `REFINE` | `PERFORMED` \| `NOT_PERFORMED` (with `REFINE_REASON`) |
| `REFINE_DEVICE`, `REFINE_SECONDS` | *dev*. `GPU` or `CPU`, and the time the refinement took |
| `REFINE_D_MIN` | The finest reflection refined against, Å |
| `REFINE_R_WORK_START`, `REFINE_R_FREE_START` | The model as given, scored as the refinement scores (riding hydrogens, B isotropic, the fine solvent mask, the robust scale): REFMAC's 0-cycle R-factors to about 0.002 on the same model and data. Not `R_WORK`/`R_FREE`, which are the validation's own scoring of the placed model |
| `REFINE_ADP_INPUT` | How the model's displacements were read ([§15.1](CPU_DATA_ANALYSIS_REFINEMENT.md#151-what-is-refined-and-in-what-order)): `isotropic`, `ANISOU` (its atoms' own U), or `TLS, B total` / `TLS, B residual` (U from its TLS groups, its B column the total or the residual) |
| `REFINE_R_WORK_START`, `REFINE_R_FREE_START` | The model as given, scored as the refinement scores (riding hydrogens, its U as read, the fine solvent mask, the robust scale): REFMAC's 0-cycle R-factors to about 0.002 on the same model and data. Not `R_WORK`/`R_FREE`, which are the validation's own scoring of the placed model |
| `REFINE_RIGID_BODIES` | The chains moved as rigid bodies, where that step was kept (0: not kept) |
| `REFINE_R_WORK_PLACEMENT`, `REFINE_R_FREE_PLACEMENT` | After the placement stage (rigid bodies, then one B per residue, each kept only where it lowers R-free) |
| `REFINE_R_WORK`, `REFINE_R_FREE` | After the restrained stage: the refined model, the one written as `<prefix>_refined_model.cif` |
| `REFINE_AS_GIVEN` | `TRUE` where no refinement lowered R-free beyond the tie of the model as given, which is then the model written (as given, scaled to the data) |
| `REFINE_RMSD_BONDS`, `REFINE_RMSD_ANGLES` | The refined model's rmsd from the dictionary's bonds (Å) and angles (°); the weight is chosen within 0.02 Å and 2° |
| `REFINE_RMSD_METAL_BONDS` | The rmsd of the metal-donor bonds made from the model's LINK records, apart from the covalent bonds. Present where there are any |
| `REFINE_RMSD_METAL_BONDS` | The rmsd of the metal-donor bonds made from the model's LINK records (those within the metal's inner sphere), apart from the covalent bonds. Present where there are any |
| `REFINE_RMSD_BONDS_START`, `REFINE_RMSD_ANGLES_START` | *dev*. The same for the model as given |
| `REFINE_CLASHES` | Pairs of atoms in the refined model more than 0.4 Å closer than their contact distance (symmetry mates included) |
| `REFINE_XRAY_WEIGHT` | *dev*. The X-ray weight of the positions that was kept |
| `REFINE_NCS_GROUPS` | Molecules present more than once in the model (copies found by sequence alignment, §15.5 of the refinement reference). Present only where there are any, as are the other `REFINE_NCS_` keys |
| `REFINE_NCS_COPIES` | The chains of those molecules |
| `REFINE_NCS_CHAINS` | The copies by chain name, `=` between copies of one molecule, e.g. `A=B=C D=E` |
| `REFINE_NCS_RESTRAINTS` | *dev*. The local distance restraints between pairs of copies |
| `REFINE_NCS_SIGMA` | The NCS restraints' sigma (Å) the weight trials kept; 0: they were not kept, and the copies were refined independently |
| `REFINE_NCS_RMSD_START` | The copies' C-alpha (P) rmsd after superposition (Å), averaged over the pairs restrained, before the restrained stage |
| `REFINE_NCS_RMSD` | The same, refined |
| `REFINE_NCS_RELEASED` | Residues of the refined model in which the copies differ: more than half of their restraints beyond the release distance (0.5 Å) |
| `REFINE_ADP_MODEL` | The displacement model the free set chose ([§15.7](CPU_DATA_ANALYSIS_REFINEMENT.md#157-the-weights-decided-by-trial)): `isotropic`, `the model's U held`, `TLS, <n> groups (...)` or `anisotropic` |
| `REFINE_ANISO_ATOMS`, `REFINE_TLS_GROUPS` | The atoms of the refined model with an anisotropic U (ANISOU), and its TLS groups |
| `REFINE_ATOMS`, `REFINE_ATOMS_REFINED` | The model's atoms (hydrogens not counted) and those refined: the atoms of the residues the dictionary describes, with an occupancy above zero |
| `REFINE_RESIDUES_FIXED`, `REFINE_RESIDUES_FIXED_NAMES` | The residues the embedded dictionary does not describe, held fixed but kept in the structure factors, the solvent mask and the clash terms, and their names |
| `REFINE_BULK_SOLVENT_K_SOL`, `REFINE_BULK_SOLVENT_B_SOL` | *dev*. The flat bulk solvent of the refined model |
| `REFINE_TWIN` | The twin hypothesis ([§15.12](CPU_DATA_ANALYSIS_REFINEMENT.md#1512-twinned-data)): `NO_TWIN_LAW` (the lattice admits none for the group), `FREE_SET_SPLIT` (the free set splits the law's twin pairs; not tested), `ZERO_FRACTION`, `UNTWINNED` (tested, not better), `TWINNED` (the refined model and every other `REFINE_` key are of the refinement against the twinned data), `NOT_TESTED` |
| `REFINE_TWIN_LAW`, `REFINE_TWIN_FRACTION` | The twin law tested (an hkl operator in the data's basis) and the fraction it refined to |
| `REFINE_TWIN_SPLIT_PAIRS` | Where `FREE_SET_SPLIT`: the twin pairs with one member in the working and one in the free set |
| `REFINE_TWIN_R_WORK`, `REFINE_TWIN_R_FREE`, `REFINE_UNTWINNED_R_WORK`, `REFINE_UNTWINNED_R_FREE` | The twinned and the untwinned refinement, each where it ended (its solvent built) |
| `REFINE_TWIN_R_FREE_NULL` | The twinned model's R-free at its fraction with every mate replaced by an unrelated reflection of the same resolution and set: what averaging alone buys, and the like-for-like comparison for `REFINE_TWIN_R_FREE`. `REFINE_TWIN_FRACTION_NULL` (*dev*) is the fraction those unrelated mates ask for, about 0 |
| `REFINE_TWIN_NULL_Z` | The law's lead over that null on the free reflections, paired reflection by reflection (a free twin pair as one term), in standard errors; `TWINNED` needs 3, or 2 where the data read twinned (`REFINE_TWIN_RULE`) |
| `REFINE_TWIN_RULE` | Where `UNTWINNED` or `TWINNED`, the rule that decided ([§15.12](CPU_DATA_ANALYSIS_REFINEMENT.md#1512-twinned-data)): `NULL` (the null alone, at 3 standard errors) or `NULL+LTEST` (taken at 2-3 standard errors because the merge's L-test reads twinned as well: `TWINNING_VERDICT` `INDICATED` with `ESTIMATED_TWIN_FRACTION_FROM` `L_TEST`). In `--mode refine` there are no merge statistics, and it is always `NULL` |
| `REFINE_BLOBS` | The blobs of the refined mFo-DFc map (above 3 σ, peaking above 4 σ) |
| `REFINE_BLOBS_LIGAND`, `REFINE_BLOBS_WATER`, `REFINE_BLOBS_ION`, `REFINE_BLOBS_MODEL_ERROR` | How many of them are read as each ([§15.9](CPU_DATA_ANALYSIS_REFINEMENT.md#159-unexplained-density-blobs)); the rest are unassigned |
| `REFINE_SOLVENT` | `PERFORMED` \| `NOT_PERFORMED`: the solvent build and audit of the refined model, off unless `--refine-solvent` is given; the other `REFINE_SOLVENT_` keys and `<prefix>_solvent.json` only where `PERFORMED` |
| `REFINE_SOLVENT_SITES` | The waters and single-atom ions of the refined model the solvent audit scored ([§15.10](CPU_DATA_ANALYSIS_REFINEMENT.md#1510-the-solvent-audit)) |
| `REFINE_SOLVENT_LABELS`, `REFINE_SOLVENT_BEST` | How many of them carry each label (`HOH 265 MG 42`), and how many each species is the best-supported one for |
| `REFINE_SOLVENT_DISAGREE`, `REFINE_SOLVENT_AMBIGUOUS` | The sites whose best-supported species is not their label (water, Mg, Na, K, Ca or Cl), not ambiguously; the sites where the runner-up is within 2 of the best |
| `REFINE_SOLVENT_IONS`, `REFINE_SOLVENT_IONS_DISAGREE` | The same for the sites not labelled water |
| `REFINE_SOLVENT_ADDED` | Waters the water build added and still in the refined model (CPU_DATA_ANALYSIS_REFINEMENT §15.10). Present where the build ran: `--refine-solvent`, a restrained refinement to 2.8 A, not `--refine-score-only` |
| `REFINE_SOLVENT_REMOVED` | Waters of the model as given that the build took out (mFo-DFc at -3 sigma or lower at the water) |
| `REFINE_SOLVENT_RELABELLED` | Waters and single-atom ions the build relabelled as another species (water, Mg, Na, K, Ca or Cl) where their surroundings and density support it, not ambiguously; each, with the scores, in the report's prose and in `<prefix>_solvent.json`. Not with `--no-refine-solvent` |
| `REFINE_SOLVENT_NOT_SOLVENT` | Difference-map peaks of the final model that cannot be a single solvent atom (bonded to the model, beside a displaced atom, a truncated side chain, a cluster or elongated density, denser than S and P): reported, not filled; each with its reason in `<prefix>_solvent.json` |
| `REFINE_SOLVENT_BUILD_ROUNDS` | The build's rounds kept (at most two, within the weight search), each kept where refining the solvent it changed lowered R-free |
| `REFINE_SOLVENT_BUILD_SECONDS` | The build's time: its maps, audits and refinements of the solvent (REFINE_SECONDS includes it too) |
A prose paragraph after them lists the five strongest blobs with their reading, peak, volume, position and
A prose paragraph after the blob keys lists the five strongest blobs with their reading, peak, volume, position and
nearest atom; all of them are in `<prefix>_blobs.json`, strongest first, one object per blob: `kind`,
`centroid` (Å, Cartesian, the copy nearest the model), `peak_sigma`, `volume_A3`, `score_sigma_A3` (the
density above the cutoff integrated), `nearest_atom`, `nearest_atom_distance_A`, `polar_contacts`,
`metal_contacts` and `anomalous_sigma`.
A second paragraph, after the solvent keys, lists the ions, label → best-supported species, their N and O in the bands
1.9-2.3/2.3-2.6/2.6-3.2 Å and their neighbours closer than 3.4 Å. Every site is in `<prefix>_solvent.json`,
in model order, one object per site: `label`, `chain`, `seqid`, `altloc`, `xyz`, `occupancy`, `b_iso`,
`sigma_xyz_A` (the coordinate error assumed), `bands` (N and O by distance), `donors`, `acceptors`,
`donor_acceptors` (closer than 3.4 Å), `min_ligand_distance_A`, `nearest_positive_A`, `nearest_negative_A`,
`positive_within_3.5`, `negative_within_3.2`, `nearest_ion_A`, `ions_within_4.2`, `ions_within_6`, `bvs` and
`nvecsum` (per cation: `MG`, `NA`, `K`, `CA`), `map_sigma`, `difference_sigma`, `anomalous_sigma` (null where
there is no such map), `score` (per species), `best`, `ambiguous`, `agrees` and `neighbours`. Where the water
build ran, a `build` object follows: `rounds`, `added`, `removed`, `not_solvent`, `stop` (why it stopped),
`seconds`, and `edits`, one per water added or removed in the rounds kept and per not-solvent peak of the final
model: `kind` (`ADDED`, `REMOVED`, `NOT_SOLVENT`), `round`, `site` (the water's name, empty for a peak),
`xyz`, `difference_sigma`, `map_sigma`, `b_iso`, `elongation` and `reason`.
**The score of the model as given** follows, every term under `REFINE_SCORE_`: `REFINE_SCORE_R_WORK`,
`REFINE_SCORE_R_FREE`, `REFINE_SCORE_N_WORK` and `REFINE_SCORE_N_FREE` by default, and with `--developer` the
overall scale (`REFINE_SCORE_K_OVERALL`), its isotropic and anisotropic B (`REFINE_SCORE_B_ISO`,
@@ -1224,4 +1418,7 @@ resolution shells. `rugnux --mode refine --refine-score-only` writes this score
In `--mode refine` the input's keys come first: `REFINE_REFLECTIONS` and `REFINE_FREE_REFLECTIONS` (those read,
after any `--refine-d-min`), `REFINE_FRENCH_WILSON` (`TRUE` where the file had intensities only and the amplitudes
were made by French-Wilson), `REFINE_FREE_FLAGS` (`FILE`, or `GENERATED` where the file had no free set and
rugnux drew its own) and `REFINE_D_MIN_LIMIT` (the `--refine-d-min` given).
rugnux drew its own), `REFINE_FREE_SET_SOURCE` and `REFINE_FREE_SET_CLASS` (where a structure-factor mmCIF's
flags are classes: the class taken as the test set, and whether the file's own mark (`STATUS`), the model file's
test-set count with the worst fit (`REMARK3_COUNT`) or the worst fit alone (`WORST_FIT`) chose it) and
`REFINE_D_MIN_LIMIT` (the `--refine-d-min` given).
docs/RUGNUX_TUTORIAL.md
+3 -3
View File
@@ -189,9 +189,9 @@ A crystal form with an [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambi
relatives — needs the space group on the run that integrates: every crystal is indexed in one of the
alternative indexings at random, and they are averaged together in the merge unless the crystals are put
into one first. With `-S` given, the run does that from the data alone, making the crystals consistent
with each other (`INDEXING_AMBIGUITY_RESOLVED= data`); a reference (`-z`), or `--model model.pdb`
together with `-C` and `-S`, puts each image into the reference's indexing instead, so that the merge
also comes out in it. Without `-S` and without either, the mixed merge can look more symmetric than the
with each other (`INDEXING_AMBIGUITY_RESOLVED= data`); a reference (`-z`) puts each image into the
reference's indexing instead, so that the merge also comes out in it (a `--model` relabels the finished
merge into its indexing and changes nothing else). Without `-S` and without a reference, the mixed merge can look more symmetric than the
crystal is, and the run warns.
`ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills; the
docs/SOFTWARE.md
+2 -2
View File
@@ -25,7 +25,7 @@ Required:
* C++20 compiler and C++20 standard library; recommended GCC 11+ or clang 14+ (Intel OneAPI and AMD AOCC also work)
* CMake version 3.26 or newer + a build tool (GNU make or Ninja)
HDF5, libtiff, libjpeg-turbo, zlib and Eigen used to be required system packages; they are now
HDF5, libtiff, zlib and Eigen used to be required system packages; they are now
downloaded and built automatically by CMake (see the note below), so none of them needs to be
installed. Beyond the optional dependencies listed below, no third-party library has to be provided by the host any more.
@@ -36,7 +36,7 @@ Optional:
* Qt version 6 (for jfjoch_viewer)
Many further dependencies (spdlog, Zstandard, HDF5, slsDetectorPackage, libzmq, libtiff,
libjpeg-turbo, Ceres, the fast feedback indexer, Catch2, ...) are downloaded automatically
Ceres, the fast feedback indexer, Catch2, ...) are downloaded automatically
by CMake and statically linked; building therefore requires network access on the first configure.
zlib (as zlib-ng in its zlib-compatible mode) and Eigen are among them: a copy already on the
machine is used instead if the configure is given `-DZLIB_ROOT=<prefix>` or `-DEigen3_DIR=<dir>`.
docs/_ext/jfjoch_downloads.py
+2 -1
View File
@@ -22,7 +22,8 @@ RELEASES = "https://gitea.psi.ch/mx/jungfraujoch/releases"
# docs/landing/: the front page, rendered from its template into the output root as index.html at
# the end of the build (write_landing below), with the same download blocks as the Sphinx pages.
LANDING_FILES = ("landing.css", "landing.js", "hero_live_light.webp", "hero_live_dark.webp", "psi.png")
LANDING_FILES = ("landing.css", "landing.js", "hero_live_light.webp", "hero_live_dark.webp", "psi.png",
"paper_rugnux_fig.webp", "paper_jsr.webp", "paper_jac.webp")
# program -> [(platform, requirements, [(label, asset name with {v}, regex for the live lookup)])]
# The first button of a platform is the recommended one and is drawn filled.
docs/documentation.rst
+4 -1
View File
@@ -73,7 +73,8 @@ All pages
Everything a user of Rugnux or the viewer needs comes first; the :doc:`technical reference <TECHNICAL>`
holds the algorithms, the file formats and streams, the interfaces and the FPGA design. The names
are explained in :doc:`Naming <NAMING>`, the releases in the :doc:`changelog <CHANGELOG>`.
are explained in :doc:`Naming <NAMING>`, the releases in the :doc:`changelog <CHANGELOG>`, and how to
cite the software in :doc:`Publications <PUBLICATIONS>`.
.. raw:: html
@@ -89,6 +90,7 @@ are explained in :doc:`Naming <NAMING>`, the releases in the :doc:`changelog <CH
RUGNUX_INSTALL
RUGNUX_FORMATS
RUGNUX_TUTORIAL
RUGNUX_CLI
RUGNUX_REPORT
RUGNUX_INTEGRATION
RUGNUX_ADVANCED
@@ -127,6 +129,7 @@ are explained in :doc:`Naming <NAMING>`, the releases in the :doc:`changelog <CH
:maxdepth: 1
:caption: Project
PUBLICATIONS
ACKNOWLEDGEMENT
LICENSE
THIRD_PARTY_NOTICES
docs/landing/index.html
+67 -5
View File
@@ -23,6 +23,7 @@ try { var t = localStorage.getItem("jfj-theme"); if (t) document.documentElement
<nav class="top__nav" aria-label="Sections">
<a href="#programs">Programs</a>
<a href="#download">Download</a>
<a href="#publications">Publications</a>
<a href="documentation.html">Documentation</a>
<a href="https://gitea.psi.ch/mx/jungfraujoch">Source</a>
</nav>
@@ -134,6 +135,66 @@ try { var t = localStorage.getItem("jfj-theme"); if (t) document.documentElement
</div>
</section>
<section class="papers" id="publications">
<div class="papers__head">
<h2>Publications</h2>
<p>The papers that describe the software, and the references to cite for it. All are open
access. <a class="papers__all" href="PUBLICATIONS.html">All publications &rarr;</a></p>
</div>
<div class="papers__grid">
<article class="paper">
<figure class="paper__page">
<a href="https://doi.org/10.64898/2026.10.24.761522" aria-hidden="true" tabindex="-1"><img src="paper_rugnux_fig.webp" width="200" height="261" loading="lazy" alt=""></a>
<figcaption>Fig. 2a &middot; <a href="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a></figcaption>
</figure>
<div class="paper__text">
<p class="paper__kicker">Rugnux &middot; preprint</p>
<h3><a href="https://doi.org/10.64898/2026.10.24.761522">Rugnux: from diffraction images to structure at synchrotrons and XFELs in one program</a></h3>
<p class="paper__authors">F. Leonarski and M. Wang</p>
<p class="paper__venue">bioRxiv (2026), posted 29 October 2026 &middot;
<a href="https://doi.org/10.64898/2026.10.24.761522">doi:10.64898/2026.10.24.761522</a></p>
<p class="paper__cite">The reference to cite for Rugnux.</p>
</div>
</article>
<article class="paper">
<figure class="paper__page">
<a href="https://doi.org/10.1107/S1600577522010268" aria-hidden="true" tabindex="-1"><img src="paper_jsr.webp" width="200" height="261" loading="lazy" alt=""></a>
<figcaption>First page &middot; <a href="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a></figcaption>
</figure>
<div class="paper__text">
<p class="paper__kicker">Jungfraujoch &middot; primary reference</p>
<h3><a href="https://doi.org/10.1107/S1600577522010268">Jungfraujoch: hardware-accelerated data-acquisition system for kilohertz pixel-array X-ray detectors</a></h3>
<p class="paper__authors">F. Leonarski, M. Brückner, C. Lopez-Cuenca, A. Mozzanica, H.-C. Stadler, Z. Matěj,
A. Castellane, B. Mesnet, J. Wojdyla, B. Schmitt and M. Wang</p>
<p class="paper__venue">J. Synchrotron Rad. <strong>30</strong>, 227&ndash;234 (2023) &middot;
<a href="https://doi.org/10.1107/S1600577522010268">doi:10.1107/S1600577522010268</a></p>
<p class="paper__cite">The reference to cite for Jungfraujoch.</p>
</div>
</article>
<article class="paper">
<figure class="paper__page">
<a href="https://doi.org/10.1107/S1600576726008083" aria-hidden="true" tabindex="-1"><img src="paper_jac.webp" width="200" height="261" loading="lazy" alt=""></a>
<figcaption>First page &middot; <a href="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0</a></figcaption>
</figure>
<div class="paper__text">
<p class="paper__kicker">Jungfraujoch &middot; live serial crystallography</p>
<h3><a href="https://doi.org/10.1107/S1600576726008083">Jungfraujoch: 38&nbsp;GB&nbsp;s<sup>&minus;1</sup> real-time serial crystallography &ndash; data acquisition and live indexing feedback</a></h3>
<p class="paper__authors">J. Duan, H.-C. Stadler, J. Nan, Z. Matěj, G. Gotthard, F. Dworkowski, M. Bjelčić, O. Aurelius,
A. Mozzanica, S. Grimm, M. Burian, T. Ursby, M. Wang and F. Leonarski</p>
<p class="paper__venue">J. Appl. Cryst. <strong>59</strong>, 1668&ndash;1677 (2026) &middot;
<a href="https://doi.org/10.1107/S1600576726008083">doi:10.1107/S1600576726008083</a></p>
<p class="paper__cite">To cite as well for the 38&nbsp;GB&nbsp;s<sup>&minus;1</sup> acquisition and the live
indexing feedback.</p>
</div>
</article>
</div>
</section>
<section class="licence">
<h2>Free to use, in academia and industry</h2>
<p>Rugnux, the viewer and the Jungfraujoch services are free software under the
@@ -143,17 +204,18 @@ try { var t = localStorage.getItem("jfj-theme"); if (t) document.documentElement
included; modified versions that are distributed must be released under the same licence. The released binaries bundle third-party components under GPL-compatible licences, and the
CUDA builds the NVIDIA CUDA runtime under NVIDIA's EULA: see the
<a href="THIRD_PARTY_NOTICES.html">third-party notices</a>.</p>
<p>The software builds on many existing algorithms, reimplemented (see the
<a href="ACKNOWLEDGEMENT.html">acknowledgements</a>), many of them moved to the GPU and made faster
across a range of computing configurations; it was developed with the help of generative AI
(Claude Opus, Anthropic).</p>
<p>The software builds on decades of developments in the crystallographic community (see the
<a href="ACKNOWLEDGEMENT.html">acknowledgements</a>), taken further in one GPU-accelerated program
whose automated decisions are validated against the diffraction data; it was developed with the
help of generative AI (Claude Opus, Anthropic).</p>
</section>
</main>
<footer class="foot">
<p>Documentation for Jungfraujoch {{release}} &middot; <a href="CHANGELOG.html">changelog</a> &middot;
<a href="documentation.html">documentation</a> &middot; <a href="NAMING.html">why the names</a> &middot;
<a href="documentation.html">documentation</a> &middot; <a href="PUBLICATIONS.html">publications</a> &middot;
<a href="NAMING.html">why the names</a> &middot;
<a href="ACKNOWLEDGEMENT.html">acknowledgements</a> &middot; <a href="https://gitea.psi.ch/mx/jungfraujoch">source</a></p>
<p>&copy; Paul Scherrer Institute.</p>
</footer>
docs/landing/landing.css
+78 -1
View File
@@ -354,6 +354,78 @@ a.jfj-dl__btn--missing, a.jfj-dl__btn--missing:hover {
.jfj-dl__note { margin: 10px 0 0; font-size: 0.82em; color: var(--fg-muted); }
.jfj-dl__note:empty { display: none; }
/* ---- publications ---------------------------------------------------------------------------- */
.papers {
padding-top: 48px;
padding-bottom: 48px;
border-top: 1px solid var(--line);
}
.papers__head { max-width: 640px; margin-bottom: 28px; }
.papers__head h2 { font-size: 2em; font-weight: 700; margin-bottom: 10px; }
.papers__head p { margin: 0; color: var(--fg-muted); font-size: 1.05em; }
.papers__all { font-weight: 600; white-space: nowrap; }
.papers__grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(min(100%, 300px), 1fr));
gap: 20px;
}
.paper {
display: grid;
grid-template-columns: minmax(0, 1fr);
align-content: start;
gap: 20px;
padding: 22px;
border: 1px solid var(--line);
border-radius: 14px;
background: var(--bg-raised);
box-shadow: var(--shadow);
}
/* The paper's first page, or a figure of it, as published (CC BY 4.0), credited under it. */
.paper__page { margin: 0; width: 200px; }
.paper__page > a { display: block; transition: transform 0.12s; }
.paper__page > a:hover { transform: translateY(-2px); }
.paper__page img {
display: block;
width: 100%;
height: auto;
/* One page shape for all three; a figure of another shape sits in it on white. */
aspect-ratio: 200 / 261;
object-fit: contain;
background: #fff;
border: 1px solid var(--line);
border-radius: 3px;
box-shadow: var(--shadow);
}
.paper__page figcaption { margin-top: 6px; font-size: 0.72em; color: var(--fg-muted); }
.paper__text { min-width: 0; }
.paper__kicker {
margin: 0 0 8px;
font-size: 0.74em;
font-weight: 700;
letter-spacing: 0.1em;
text-transform: uppercase;
color: var(--primary);
}
.paper h3 { font-size: 1.2em; font-weight: 650; margin-bottom: 10px; }
.paper h3 a { color: var(--fg); }
.paper h3 a:hover { color: var(--primary); }
.paper h3 sup, .paper__cite sup { font-size: 0.7em; line-height: 0; }
.paper__authors { margin: 0 0 6px; font-size: 0.92em; }
.paper__venue { margin: 0 0 12px; font-size: 0.92em; color: var(--fg-muted); }
.paper__venue a { overflow-wrap: anywhere; }
.paper__cite {
margin: 0;
padding-top: 12px;
border-top: 1px solid var(--line);
font-size: 0.85em;
color: var(--fg-muted);
}
/* ---- licence, footer ------------------------------------------------------------------------- */
.licence {
@@ -378,7 +450,7 @@ a.jfj-dl__btn--missing, a.jfj-dl__btn--missing:hover {
@media (max-width: 760px) {
body { font-size: 15px; }
.top { gap: 14px; flex-wrap: wrap; }
.top__nav { gap: 14px; margin-left: 0; order: 3; width: 100%; font-size: 0.95em; }
.top__nav { gap: 8px 14px; flex-wrap: wrap; margin-left: 0; order: 3; width: 100%; font-size: 0.95em; }
.theme-toggle { margin-left: auto; }
.hero { grid-template-columns: 1fr; padding-top: 28px; padding-bottom: 40px; }
.hero__art { grid-column: 1; grid-row: 1; max-width: 360px; justify-self: center; }
@@ -386,3 +458,8 @@ a.jfj-dl__btn--missing, a.jfj-dl__btn--missing:hover {
.hero__text { grid-row: 2; }
.programs__grid { grid-template-columns: 1fr; }
}
@media (max-width: 520px) {
.paper { gap: 18px; padding: 18px; }
.paper__page { width: 180px; }
}
docs/landing/paper_jac.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 26 KiB

docs/landing/paper_jsr.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 28 KiB

docs/landing/paper_rugnux_fig.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

gemmi_gph/CMakeLists.txt
+4 -1
View File
@@ -12,6 +12,9 @@
#
# assembly.cpp expands a deposited model's strict-NCS operators into the full asymmetric unit.
#
# fprime is GEMMI's Cromer-Liberman f'/f'' (its port of the Larch routines, credited in its header), for the
# anomalous signal the refinement's solvent sites are expected to show at the data's wavelength.
#
# monlib/topo/riding_h are the restraint side: a monomer library read from memory (MonLib) and the
# topology of a model built from it (prepare_topology), for rugnux's restrained refinement.
ADD_LIBRARY(gemmi STATIC
@@ -19,7 +22,7 @@ ADD_LIBRARY(gemmi STATIC
pdb.cpp resinfo.cpp polyheur.cpp calculate.cpp eig3.cpp ccp4.cpp
read_cif.cpp mmcif.cpp json.cpp mmread_gz.cpp
to_mmcif.cpp to_pdb.cpp assembly.cpp
monlib.cpp topo.cpp riding_h.cpp
monlib.cpp topo.cpp riding_h.cpp fprime.cpp
gemmi/cellred.hpp
gemmi/symmetry.hpp
gemmi/fail.hpp
gemmi_gph/LICENSE-larch.txt
+25
View File
@@ -0,0 +1,25 @@
The MIT License
Copyright (c) 2010-2026 Matthew Newville, The University of Chicago
Mauro Rovezzi, University Grenoble Alpes, CNRS
Bruce Ravel, National Institute of Science and Technology
Margaret Koker, Lawrence University
Ryuichi Shimogawa, Stony Brook University
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXP80RESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE qUSE OR OTHER DEALINGS IN THE
SOFTWARE.
gemmi_gph/fprime.cpp
+2961
View File
File diff suppressed because it is too large. Load diff
image_analysis/IndexAndRefine.cpp
+41 -1
View File
@@ -167,6 +167,20 @@ IndexAndRefine::IndexingOutcome IndexAndRefine::DetermineLatticeAndSymmetryRotat
static_cast<int>(std::lround(6.0f * (tw->t * l.Bstar()))),
static_cast<int>(std::lround(6.0f * (tw->t * l.Cstar())))};
outcome.twin_overlap_n = 6 * tw->n;
// The other domain's reflection at the node of h, read in this domain's basis: the index of
// R^-1 q, q the node, R the rotation from this domain to the other.
if (tw->rotation) {
const Coord a[3] = {l.Vec0(), l.Vec1(), l.Vec2()}, as[3] = {l.Astar(), l.Bstar(), l.Cstar()};
std::array<float, 9> p{};
for (int i = 0; i < 3; i++)
for (int j = 0; j < 3; j++)
p[3 * i + j] = (*tw->rotation * a[i]) * as[j];
outcome.twin_partner = p;
outcome.twin_domain_share = 1.0f - tw->fraction;
const auto right = [](const Coord &x, const Coord &y) { return std::abs(angle_deg(x, y) - 90.0f) < 0.5f; };
for (int i = 0; i < 3; i++)
outcome.twin_axis_kept[i] = right(a[i], a[(i + 1) % 3]) && right(a[i], a[(i + 2) % 3]);
}
}
}
return outcome;
@@ -642,6 +656,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg,
settings_prediction.window_ang = stills_window_->ang;
settings_prediction.window_bw = stills_window_->bw;
settings_prediction.window_q_scale = stills_window_->q_scale;
settings_prediction.window_radial = stills_window_->radial;
}
// Online is bounded by what the image-buffer slot can carry, offline by the prediction limit; both
@@ -663,9 +678,34 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg,
std::vector<Reflection> integrated = integrate(prediction.GetReflections(), nrefl, msg.number);
if (const int n = outcome.twin_overlap_n; n > 0) {
const auto &w = outcome.twin_overlap_w;
const auto &p = outcome.twin_partner;
// Kept only where the coincidence is exact: a twin by reticular PSEUDO-merohedry puts the
// other domain's node beside this one's rather than on it, by an offset that grows with the
// index, so a partner that rounds to the reflection itself proves nothing in general. Along an
// axis perpendicular to the other two it does - the twin takes that axis onto itself or its
// reverse exactly - and those are the rows a screw is read on.
const auto exact = [&](const Reflection &r) {
const int i_axis = r.h != 0 ? 0 : r.k != 0 ? 1 : 2;
const bool axial = (r.h != 0) + (r.k != 0) + (r.l != 0) == 1;
if (!p || !axial || !outcome.twin_axis_kept[i_axis])
return false;
const auto row = [&](int i) {
return std::lround((*p)[3 * i] * r.h + (*p)[3 * i + 1] * r.k + (*p)[3 * i + 2] * r.l);
};
const long h = row(0), k = row(1), l = row(2);
return (h == r.h && k == r.k && l == r.l) || (h == -r.h && k == -r.k && l == -r.l);
};
std::erase_if(integrated, [&](const Reflection &r) {
return (r.h * w[0] + r.k * w[1] + r.l * w[2]) % n == 0;
return (r.h * w[0] + r.k * w[1] + r.l * w[2]) % n == 0 && !exact(r);
});
// Such a reflection holds the whole crystal's intensity, every other one this domain's share of
// it: scaled by that share, it is on the others' scale.
for (auto &r : integrated)
if ((r.h * w[0] + r.k * w[1] + r.l * w[2]) % n == 0) {
r.I *= outcome.twin_domain_share;
r.sigma *= outcome.twin_domain_share;
r.var_bkg *= outcome.twin_domain_share * outcome.twin_domain_share;
}
}
msg.integrated_reflections = integrated.size();
auto integration_end_time = std::chrono::steady_clock::now();
image_analysis/IndexAndRefine.h
+8
View File
@@ -97,6 +97,7 @@ struct StillsWindow {
float ang = 0.0f;
float bw = 0.0f;
float q_scale = 1.0f; // centred on the excitation error of the reciprocal lattice scaled by this
float radial = 0.0f; // where an off-sphere reflection is placed (BraggPredictionSettings::window_radial)
};
class IndexAndRefine {
@@ -131,6 +132,13 @@ class IndexAndRefine {
// overlap's index) are left out of integration.
std::array<int, 3> twin_overlap_w{};
int twin_overlap_n = 0;
// Where the twin rotation is known: the twin partner of reflection h, in the candidate's basis,
// is the nearest integer to twin_partner * h (row-major 3x3); twin_axis_kept[i] where axis i is
// perpendicular to the other two, so that a reflection along it whose partner is itself or its
// Friedel mate coincides with that partner exactly.
std::optional<std::array<float, 9>> twin_partner;
std::array<bool, 3> twin_axis_kept{};
float twin_domain_share = 1.0f; // this domain's share of the crystal (1 - the twin fraction)
explicit IndexingOutcome(const DiffractionExperiment& experiment_ref)
: experiment(experiment_ref) {}
image_analysis/MXAnalysisWithoutFPGA.cpp
+10 -11
View File
@@ -237,13 +237,13 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output,
// Where no such background exists (no adaptive finder), auto leaves the correction off: the
// plain profile carries the Bragg peaks and cannot support an absolute threshold.
//
// On stills - and wherever the correction is set for the whole run rather than left to auto, as
// rugnux's pre-scan sets it on a rotation run whose empty apertures ask for it - the correction
// takes its CURVE from that background, and on stills auto applies it to every image that has one
// (the pre-scan turns it off for a run whose empty apertures say the radial model does not hold
// there). A serial frame has a few dozen predicted reflections, so their own
// ring pixels - where the curve comes from otherwise - sample the radial background sparsely,
// while the finder's rings hold every pixel of the frame. And the correction is the measured
// The correction takes its CURVE from that background wherever there is one, and on stills auto
// applies it to every image that has one (the pre-scan turns it off for a run whose empty
// apertures say the radial model does not hold there). The predicted reflections' own ring
// pixels - where the curve comes from otherwise - are a poor sample of it: on a serial frame
// there are a few dozen of them, and on a rotation frame the clipped ring pixels lose the
// narrow peak of an ice ring, so the reflections on the ring come out high and those beside it
// low. The finder's rings hold every pixel of the frame. And the correction is the measured
// curvature itself: where the background has none it is zero, so there is nothing for a gate
// to decide. The ice score is the wrong gate there: a smooth hump that is not ice - the
// solvent or a carrier medium - curves the background as much as an ice ring does, and the
@@ -256,10 +256,9 @@ void MXAnalysisWithoutFPGA::Analyze(DataMessage &output,
const std::vector<float> &ring_bkg = spot_finding_settings.adaptive_threshold
? adaptiveSpotFinder->GetRingBackground() : no_rings;
const bool stills = !experiment.IsRotationIndexing();
if (stills || !bragg_engine->IsBackgroundRadialAuto())
bragg_engine->RadialBackgroundFromRings(ring_bkg, integration.GetQBinCount(),
integration.Settings().GetLowQ_recipA(),
integration.Settings().GetQSpacing_recipA());
bragg_engine->RadialBackgroundFromRings(ring_bkg, integration.GetQBinCount(),
integration.Settings().GetLowQ_recipA(),
integration.Settings().GetQSpacing_recipA());
if (!bragg_engine->IsBackgroundRadialAuto() || ring_bkg.empty())
return;
output.ice_ring_score = AzimuthalIntegrationProfile::IceRingScore(
image_analysis/WriteReflections.cpp
+161 -8
View File
@@ -428,8 +428,9 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
// ISa in XDS's sense: the whole-range 1/sqrt(a*b) of the error model below, so this item can
// be read straight against a CORRECT.LP. The strong-reflection asymptote - a tier XDS does not
// have, and always the more optimistic of the two - is written separately rather than here.
out << "_reflns.jfjoch_diffrn_ISa " << CifStr(error_model.isa)
<< " # 1/sqrt(a*b), the XDS convention\n";
if (!statistics.stills)
out << "_reflns.jfjoch_diffrn_ISa " << CifStr(error_model.isa)
<< " # 1/sqrt(a*b), the XDS convention\n";
if (!error_model.isa_asymptotic.empty())
out << "_reflns.jfjoch_diffrn_ISa_asymptotic " << CifStr(error_model.isa_asymptotic)
<< " # strong-reflection asymptote (Diederichs); rotation path only\n";
@@ -438,6 +439,9 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
<< " # sigma^2 = a*(sigma0^2 + b*I^2), XDS convention\n";
if (!error_model.b.empty())
out << "_reflns.jfjoch_error_model_b " << CifStr(error_model.b) << "\n";
if (!error_model.c.empty())
out << "_reflns.jfjoch_error_model_c " << CifStr(error_model.c)
<< " # stills partiality term: + c*((1-p)/p)^2*I^2\n";
// Dataset-wide isotropic Wilson B-factor estimate (standard PDBx item), analogous to XDS's
// "WILSON LINE ... B=". Emitted only when the log-linear fit succeeded.
if (std::isfinite(statistics.wilson_b) && statistics.wilson_b > 0.0)
@@ -1009,7 +1013,8 @@ MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const DiffractionExperiment &experiment,
bool sum_partials,
const OutputProvenance &provenance,
size_t nthreads) {
size_t nthreads,
const std::vector<ScaledFull> *scaled_fulls) {
const gemmi::SpaceGroup *spacegroup = &experiment.GetSpaceGroupOrP1();
MtzTable mtz;
mtz.space_group_name = spacegroup->xhm();
@@ -1071,7 +1076,9 @@ MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
// flight-path medium (both divisors, hence the multiplies).
// None of the three is a scale, and a program that reads this file has no way to recompute them. The
// partiality is NOT divided out - that IS a scale, FRACTIONCALC carries it, and every program
// this file is for wants to handle it its own way.
// this file is for wants to handle it its own way. All of this is of the unscaled file: given the merge's
// scaled fulls, I and SIGI are theirs (below), every correction and scale applied, and the columns
// beside them describe the event they were measured in.
gemmi::UnmergedHklMover hkl_mover(spacegroup);
// The batch headers are written for every image the observations span, not only for the images
// that produced one. AIMLESS starts a new run wherever the phi series jumps, so an image that
@@ -1127,6 +1134,46 @@ MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
add_row(r);
}
// The merge's own observations, where it handed them over: each event takes the full of its index whose
// peak frame is nearest its centroid (within the bridge of RockingEventFrameGap and a frame of rounding),
// and one the merge did not keep - an outlier, past the resolution cut - is not written. Unscaled
// intensities fitted again by a reading program's own scale model lose the merge's corrections; these
// are the intensities the merged file was made from.
std::vector<const ScaledFull *> scaled;
if (scanning && sum_partials && scaled_fulls != nullptr && !scaled_fulls->empty()) {
std::vector<const ScaledFull *> by_hkl(scaled_fulls->size());
for (size_t i = 0; i < by_hkl.size(); ++i)
by_hkl[i] = &(*scaled_fulls)[i];
const auto key = [](const ScaledFull *f) { return std::tie(f->h, f->k, f->l, f->image_number); };
ParallelSort(by_hkl.begin(), by_hkl.end(), nthreads,
[&](const ScaledFull *a, const ScaledFull *b) { return key(a) < key(b); });
const float max_gap = RockingEventFrameGap(wedge_deg) + 1.0f;
scaled.resize(rows.size(), nullptr);
ParallelChunks(static_cast<int>(rows.size()), ThreadsForWork(rows.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
const Reflection &r = *rows[i];
auto it = std::lower_bound(by_hkl.begin(), by_hkl.end(), r, [](const ScaledFull *f, const Reflection &x) {
return std::tie(f->h, f->k, f->l) < std::tie(x.h, x.k, x.l);
});
for (; it != by_hkl.end() && (*it)->h == r.h && (*it)->k == r.k && (*it)->l == r.l; ++it)
if (std::fabs((*it)->image_number - r.image_number) <= max_gap
&& (scaled[i] == nullptr
|| std::fabs((*it)->image_number - r.image_number)
< std::fabs(scaled[i]->image_number - r.image_number)))
scaled[i] = *it;
}
});
size_t kept = 0;
for (size_t i = 0; i < rows.size(); ++i)
if (scaled[i] != nullptr) {
rows[kept] = rows[i];
scaled[kept++] = scaled[i];
}
rows.resize(kept);
scaled.resize(kept);
mtz.title = "Rugnux unmerged observations, scaled";
}
// Every row has its own slot in each column, so they are filled on several workers.
const size_t w = mtz.columns.size();
for (auto &c : mtz.columns)
@@ -1143,8 +1190,8 @@ MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
row[3] = static_cast<float>((partials ? 256 : 0) + isym);
row[4] = static_cast<float>(1 + static_cast<int>(std::lround(r.image_number))); // BATCH, as above
const float corr = r.prescaling_corr * r.qe_corr * r.flight_corr;
row[5] = r.I * corr;
row[6] = r.sigma * corr;
row[5] = scaled.empty() ? r.I * corr : scaled[i]->I;
row[6] = scaled.empty() ? r.sigma * corr : scaled[i]->sigma;
row[7] = r.partiality;
row[8] = DetectorX(r);
row[9] = DetectorY(r);
@@ -1285,8 +1332,9 @@ void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes
bool sum_partials,
const OutputProvenance &provenance,
const std::string &filename,
size_t nthreads) {
MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads);
size_t nthreads,
const std::vector<ScaledFull> *scaled_fulls) {
MtzTable mtz = UnmergedMtzTable(outcomes, unitCell, experiment, sum_partials, provenance, nthreads, scaled_fulls);
SetUnmergedMtzMosaicity(mtz, outcomes);
WriteMtz(mtz, filename);
}
@@ -1316,3 +1364,108 @@ void WriteReflections(const std::vector<MergedReflection> &reflections,
WriteShelxIns(unitCell, nullptr, experiment, std::filesystem::path(filename).filename().string(),
filename + ".ins");
}
MtzTable ObservationMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const std::vector<std::vector<ObservationVerdict>> &verdicts,
std::optional<double> written_d_min,
const std::optional<StillsRunWidth> &run_width,
const UnitCell &unitCell,
const DiffractionExperiment &experiment) {
constexpr uint32_t OBS_BEYOND_DMIN = 4096, OBS_NO_VERDICT = 2048;
MtzTable mtz;
mtz.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
mtz.unit_cell = unitCell;
mtz.title = "Rugnux stills observations (diagnostic)";
mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
{"obs", experiment.GetSampleName(), "obs", experiment.GetWavelength_A()}};
if (run_width) {
std::string t = "RUNWIDTH gamma0 " + std::to_string(run_width->gamma0) + " gamma_e " + std::to_string(run_width->gamma_e)
+ " bw " + std::to_string(run_width->bandwidth_sigma) + " radial_scale "
+ std::to_string(run_width->radial_scale);
mtz.history.push_back(t);
std::string x = "RUNWIDTH_DSTAR", y = "RUNWIDTH_S";
for (size_t i = 0; i < run_width->table_dstar.size(); ++i) {
x += " " + std::to_string(run_width->table_dstar[i]);
y += " " + std::to_string(run_width->table_s[i]);
}
mtz.history.push_back(x);
mtz.history.push_back(y);
}
if (written_d_min)
mtz.history.push_back("WRITTEN_DMIN " + std::to_string(*written_d_min));
const std::vector<std::string> labels = {
"H", "K", "L", "FRAME", "IMG", "XPRED", "YPRED", "XOBS", "YOBS", "PANEL", "D",
"EPS", "EPSR", "EPSABS", "QX", "QY", "QZ", "PART", "ROCKSIG", "ZETA", "DPHI",
"I", "SIGI", "BKG", "VARBKG", "LP", "QE", "FLIGHT", "SCALECORR", "ISCALED", "SIGSCALED", "SIGMERGE",
"G", "CCIMG", "MOSAIC", "ASX", "ASY", "ASZ", "BSX", "BSY", "BSZ", "CSX", "CSY", "CSZ",
"CELLA", "CELLB", "CELLC", "CELLAL", "CELLBE", "CELLGA", "FLAGS", "USED"};
for (const auto &l : labels)
mtz.columns.push_back({l, (l == "H" || l == "K" || l == "L") ? 'H' : (l == "FLAGS" || l == "USED" ? 'I' : 'R'), 0, {}});
// Predictions are on the detector plane, so the panel is the one whose placed footprint holds the
// point (-1 for a detector without panels, -2 in a gap or off every panel).
const auto panel_of = [&](float x, float y) {
const auto p = experiment.GetModules().PlaneToPanel(x, y);
return p ? static_cast<float>(p->panel) : -2.0f;
};
// The run's rocking width at d*, as StillsPartialityRefine evaluates it: its table, linear in d* and held
// beyond the end shells, else the three-term formula.
const auto rock_sigma = [&](double dstar, double qz) {
if (!run_width)
return NAN;
const auto &x = run_width->table_dstar;
const auto &y = run_width->table_s;
if (!x.empty()) {
if (dstar <= x.front()) return static_cast<float>(y.front());
if (dstar >= x.back()) return static_cast<float>(y.back());
const size_t i = std::upper_bound(x.begin(), x.end(), dstar) - x.begin();
return static_cast<float>(y[i - 1] + (dstar - x[i - 1]) / (x[i] - x[i - 1]) * (y[i] - y[i - 1]));
}
const double ang = run_width->gamma_e * dstar, bw = run_width->bandwidth_sigma * qz;
return static_cast<float>(std::sqrt(run_width->gamma0 * run_width->gamma0 + ang * ang + bw * bw));
};
const double radial = run_width ? run_width->radial_scale : 1.0;
std::vector<float> row(labels.size());
const auto push = [&]() {
for (size_t c = 0; c < row.size(); ++c)
mtz.columns[c].values.push_back(row[c]);
};
for (size_t i = 0; i < outcomes.size(); ++i) {
const auto &o = outcomes[i];
if (o.reflections.empty() || o.latt.CalcVolume() <= 1.0f) // an unindexed frame: no lattice
continue;
const bool have_verdicts = i < verdicts.size() && verdicts[i].size() == o.reflections.size();
const Coord A = o.latt.Astar(), B = o.latt.Bstar(), C = o.latt.Cstar();
const Coord S0 = o.geom.GetScatteringVector();
const double inv_lambda = 1.0 / o.geom.GetWavelength_A();
const UnitCell cell = o.latt.GetUnitCell();
for (size_t j = 0; j < o.reflections.size(); ++j) {
const Reflection &r = o.reflections[j];
const Coord q = A * static_cast<float>(r.h) + B * static_cast<float>(r.k) + C * static_cast<float>(r.l);
const auto eps_at = [&](double s) {
const double x = s * q.x + S0.x, y = s * q.y + S0.y, z = s * q.z + S0.z;
return static_cast<float>(std::sqrt(x * x + y * y + z * z) - inv_lambda);
};
const double dstar = std::sqrt(q.x * q.x + q.y * q.y + q.z * q.z);
uint32_t flags = have_verdicts ? verdicts[i][j].flags : OBS_NO_VERDICT;
if (written_d_min && r.d < *written_d_min)
flags |= OBS_BEYOND_DMIN;
row = {static_cast<float>(r.h), static_cast<float>(r.k), static_cast<float>(r.l), r.image_number,
static_cast<float>(i), r.predicted_x, r.predicted_y, r.observed_x, r.observed_y,
panel_of(r.predicted_x, r.predicted_y), r.d,
eps_at(1.0), eps_at(radial), r.dist_ewald, q.x, q.y, q.z, r.partiality,
rock_sigma(radial * dstar, std::fabs(radial * q.z)), r.zeta, r.delta_phi_deg,
r.I, r.sigma, r.bkg, r.var_bkg, r.prescaling_corr, r.qe_corr, r.flight_corr,
r.image_scale_corr, r.I * r.image_scale_corr, r.sigma * r.image_scale_corr,
have_verdicts ? verdicts[i][j].sigma : NAN,
o.image_scale_g.value_or(NAN), o.image_scale_cc.value_or(NAN), o.mosaicity_deg.value_or(NAN),
A.x, A.y, A.z, B.x, B.y, B.z, C.x, C.y, C.z,
cell.a, cell.b, cell.c, cell.alpha, cell.beta, cell.gamma,
static_cast<float>(flags), flags == 0 ? 1.0f : 0.0f};
push();
}
}
return mtz;
}
image_analysis/WriteReflections.h
+29 -5
View File
@@ -11,6 +11,8 @@
#include "../common/DiffractionExperiment.h"
#include "../common/MtzTable.h"
#include "IntegrationOutcome.h"
#include "scale_merge/Merge.h"
#include "../common/StillsRunWidth.h"
struct MergeStatistics;
struct TwinningAnalysisResult;
@@ -19,12 +21,15 @@ struct CellEsd; // geom_refinement/CellEsd.h
// The error model as it is reported, already formatted. `isa` is the whole-range 1/sqrt(a*b), the
// same quantity XDS's ISa denotes, so a file written here is directly comparable with a CORRECT.LP;
// `isa_asymptotic` is the strong-reflection tier, which only the rotation path has. `a` and `b` are
// in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2). Empty strings are written as unknown.
// in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2), and `c` is the stills partiality term in the same
// convention, sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2). Empty strings are written as unknown.
// A stills merge writes no ISa (MergeStatistics::stills).
struct ErrorModelReport {
std::string isa;
std::string isa_asymptotic;
std::string a;
std::string b;
std::string c;
};
// What the reflection files record about where they came from, beyond the experiment itself and the
@@ -99,27 +104,46 @@ void WriteShelxIns(const UnitCell &cell, const CellEsd *esd, const DiffractionEx
// The intensities carry the deterministic per-reflection corrections and nothing else - Lorentz
// and polarization in LP, the sensor's angle-dependent efficiency in QE, and the attenuation of the
// flight-path medium in FLIGHT - so raw counts are I / LP * QE * FLIGHT. The partiality and the
// per-image scale are left for the reading program to fit, since every program this file is for
// fits a scale model of its own.
// per-image scale are left for the reading program to fit.
// Given the merge's scaled fulls (rotation, summed partials), the file holds the merge's observations
// instead, as a scaling program's unmerged output does: each summed event's I and SIGI are those of the
// full the merge kept for it - scaled, and with the sigma the merge weighted it by - and an event the
// merge rejected or cut is left out.
void WriteUnmergedMtzReflections(const std::vector<IntegrationOutcome> &outcomes,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
bool sum_partials,
const OutputProvenance &provenance,
const std::string &filename,
size_t nthreads = 1);
size_t nthreads = 1,
const std::vector<ScaledFull> *scaled_fulls = nullptr);
// The same file as its table, in two steps, for a caller that builds it while the outcomes' per-image
// scale fields are still being written: UnmergedMtzTable reads everything but mosaicity_deg, which
// SetUnmergedMtzMosaicity then copies into the batch headers.
// scaled_fulls: the merge's, on the indices the outcomes carry (see above); null or empty writes the
// unscaled events.
MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
bool sum_partials,
const OutputProvenance &provenance,
size_t nthreads = 1);
size_t nthreads = 1,
const std::vector<ScaledFull> *scaled_fulls = nullptr);
void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes);
// Diagnostic (--export-observations, stills): every integrated observation of the merged run with what the
// merge made of it - one row per reflection of every outcome, the ones the merge left out included, with a
// FLAGS bit for each test it failed (ObservationFlag; BEYOND_DMIN 4096 = past the written resolution cut).
// verdicts: MergeOnTheFly::Verdicts of the written merge, in the outcomes' order. Columns are listed in the
// code and documented with the campaign's loader, not here; H K L are the indices as measured.
MtzTable ObservationMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const std::vector<std::vector<ObservationVerdict>> &verdicts,
std::optional<double> written_d_min,
const std::optional<StillsRunWidth> &run_width,
const UnitCell &unitCell,
const DiffractionExperiment &experiment);
void WriteReflections(const std::vector<MergedReflection> &reflections,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
image_analysis/beam_stop/SHADOW_FINDER.md
+12
View File
@@ -119,6 +119,18 @@ own radius**, not to a global threshold.
read as one enormous transmitting piece. The model is capped at 1, so it only ever removes dim
pixels and a sweep without such a piece still keeps its mask bit for bit.
12. **Outer shadow (`FindOuterShadow`, stills).** An aperture or cone in front of the detector can dim
everything beyond some radius. Beyond the largest ring the detector holds whole, a ring can then lie
wholly in the dimmed corners, its median is the dimmed level, and step 4 sees nothing. So there each
pixel's pooled background is compared with what the one `OUTER_STEP_PX` nearer the beam along its ray,
carried on by the ray's own fall-off, leads up to; a significant drop below `OUTER_STEP_RATIO` marks the
shadow's edge (pieces narrower than 2 `OUTER_OPENING_PX` or smaller than `OUTER_MIN_PIXELS` are dropped).
In each of `OUTER_RAYS` directions the shadow starts at the innermost edge pixel where the ray's own
background, pooled over the ray, shows the same step and stays below `OUTER_STEP_RATIO` of the
background before it out to the detector's edge, and it runs to that edge. `rugnux` runs it once the stills
beam centre is decided, on the pre-scan's projection, grows it by R3 - R1 (so an observation whose
background ring reaches the shadow is dropped with it, `DropShadowClipped`) and adds it to bit 9.
Three findings are baked in: the **mean** ratio (not the max) does the detecting — the max
projection is a great *picture* but a noisier detector; **size and counting statistics**, not a
position, are what distinguish hardware from a stray low patch; and **a lit pixel is never
image_analysis/beam_stop/ShadowFinder.cpp
+165
View File
@@ -1196,6 +1196,171 @@ std::vector<uint32_t> ShadowFinder::MaskOn(const Projection &p, const std::vecto
return mask;
}
namespace {
// FindOuterShadow on a W x H raster, with its known-masked pixels and the direct beam (cx, cy) on it.
std::vector<uint32_t> OuterShadowOn(const std::vector<float> &mean, const std::vector<uint32_t> &known_mask,
int W, int H, float cx, float cy, uint32_t frames, float gain,
int margin_px, size_t nthreads) {
const int n_pixels = W * H;
// The background pooled over the same box as in GetMask.
Plane<char> valid(n_pixels);
Plane<double> num(n_pixels);
Plane<int32_t> den(n_pixels);
ParallelChunks(n_pixels, nthreads, [&](int lo, int hi) {
for (int i = lo; i < hi; i++) {
valid[i] = known_mask[i] == 0 && std::isfinite(mean[i]);
num[i] = valid[i] ? mean[i] : 0.0;
den[i] = valid[i] ? 1 : 0;
}
});
const auto pooled_sum = box_sum(num, W, H, POOL_PX, nthreads);
const auto pooled_count = box_sum(den, W, H, POOL_PX, nthreads);
// Inside the largest ring the detector holds whole, GetMask's comparison along the ring works.
const float r_whole = std::min({cx, static_cast<float>(W - 1) - cx, cy, static_cast<float>(H - 1) - cy});
// The edge: pixels whose background stepped down from what the ray leads up to, by enough photons to be
// believed. What it leads up to is the background OUTER_STEP_PX nearer the beam, carried on by the fall-off
// over the OUTER_STEP_PX before that: a background can fall steeply with radius on its own (to 60 % over
// OUTER_STEP_PX at high angle on one panel detector), and a step is a drop against that trend. A background
// below zero is a detector's offset, not a shadow.
const auto pooled_at = [&](float x, float y) -> std::optional<double> {
const int xi = static_cast<int>(std::lround(x)), yi = static_cast<int>(std::lround(y));
if (xi < 0 || xi >= W || yi < 0 || yi >= H || pooled_count[yi * W + xi] == 0)
return std::nullopt;
return pooled_sum[yi * W + xi] / pooled_count[yi * W + xi];
};
Plane<char> edge(n_pixels);
ParallelChunks(H, nthreads, [&](int ylo, int yhi) {
for (int y = ylo; y < yhi; y++)
for (int x = 0; x < W; x++) {
const int i = y * W + x;
edge[i] = 0;
const float dx = static_cast<float>(x) - cx, dy = static_cast<float>(y) - cy;
const float r = std::hypot(dx, dy);
if (!valid[i] || r <= r_whole || r <= 2 * OUTER_STEP_PX)
continue;
const float s1 = 1.0f - OUTER_STEP_PX / r, s2 = 1.0f - 2 * OUTER_STEP_PX / r;
const auto inner = pooled_at(cx + dx * s1, cy + dy * s1);
const auto further_in = pooled_at(cx + dx * s2, cy + dy * s2);
if (!inner || !further_in || *inner <= 0.0 || *further_in <= 0.0)
continue;
const double here = pooled_sum[i] / pooled_count[i];
const double expected = *inner * std::min(1.0, *inner / *further_in);
const double photons = static_cast<double>(frames) * pooled_count[i] / gain;
edge[i] = here > 0.0 && here < OUTER_STEP_RATIO * expected
&& poisson_deficit_sigma(here * photons, expected * photons) > MIN_DEFICIT_SIGMA;
}
});
// Only pieces of the edge wide and large enough to have been cast by something.
const Plane<char> opened = dilate(erode(edge, W, H, OUTER_OPENING_PX, nthreads), W, H, OUTER_OPENING_PX, nthreads);
const auto pieces = label_components(opened, W, H, nthreads);
std::vector<int> piece_size(pieces.count, 0);
for (int i = 0; i < n_pixels; i++)
if (pieces.id[i] >= 0)
piece_size[pieces.id[i]]++;
// Along each ray, the shadow starts at the innermost edge pixel where the ray's own background - pooled over
// the ray, OUTER_STEP_PX at a time - steps down against its trend as well, and stays below OUTER_STEP_RATIO
// of the background before the step all the way out to the detector's edge; it runs to that edge. A single
// pixel's step can be the noise of a detector that is not Poisson; a step whose far side recovers further
// out is not an obstruction, but a dim panel edge or a patch of weak pixels.
const auto ray = [cx, cy](int x, int y) {
const double phi = std::atan2(static_cast<double>(y) - cy, static_cast<double>(x) - cx) + std::numbers::pi;
return std::min(OUTER_RAYS - 1, static_cast<int>(phi / (2.0 * std::numbers::pi) * OUTER_RAYS));
};
const auto radius = [cx, cy](int x, int y) {
return static_cast<int>(std::lround(std::hypot(static_cast<float>(x) - cx, static_cast<float>(y) - cy)));
};
const int n_r = std::max({radius(0, 0), radius(W - 1, 0), radius(0, H - 1), radius(W - 1, H - 1)}) + 1;
std::vector<double> sum(static_cast<size_t>(OUTER_RAYS) * n_r, 0.0);
std::vector<int> count(sum.size(), 0);
std::vector<char> at_edge(sum.size(), 0);
for (int i = 0; i < n_pixels; i++) {
const int x = i % W, y = i / W;
const size_t k = static_cast<size_t>(ray(x, y)) * n_r + radius(x, y);
if (valid[i]) {
sum[k] += pooled_sum[i] / pooled_count[i];
count[k]++;
}
if (pieces.id[i] >= 0 && piece_size[pieces.id[i]] >= OUTER_MIN_PIXELS)
at_edge[k] = 1;
}
std::vector<float> starts_at(OUTER_RAYS, std::numeric_limits<float>::infinity());
for (int k = 0; k < OUTER_RAYS; k++) {
const size_t o = static_cast<size_t>(k) * n_r;
std::vector<double> beyond_sum(n_r + 1, 0.0);
std::vector<int> beyond_count(n_r + 1, 0);
for (int r = n_r - 1; r >= 0; r--) {
beyond_sum[r] = beyond_sum[r + 1] + sum[o + r];
beyond_count[r] = beyond_count[r + 1] + count[o + r];
}
// The ray's mean over the radii [lo, hi).
const auto band = [&](int lo, int hi) -> std::optional<double> {
const int n = beyond_count[lo] - beyond_count[hi];
if (n == 0)
return std::nullopt;
return (beyond_sum[lo] - beyond_sum[hi]) / n;
};
for (int r = 2 * OUTER_STEP_PX; r + OUTER_STEP_PX <= n_r; r++) {
if (!at_edge[o + r])
continue;
const auto further_in = band(r - 2 * OUTER_STEP_PX, r - OUTER_STEP_PX);
const auto inner = band(r - OUTER_STEP_PX, r);
const auto outer = band(r, r + OUTER_STEP_PX);
const auto beyond = band(r, n_r);
if (!further_in || !inner || !outer || *further_in <= 0.0 || *inner <= 0.0)
continue;
const double expected = *inner * std::min(1.0, *inner / *further_in);
if (*outer < OUTER_STEP_RATIO * expected && *beyond < OUTER_STEP_RATIO * *inner) {
starts_at[k] = static_cast<float>(r);
break;
}
}
}
Plane<char> shadow(n_pixels);
ParallelChunks(H, nthreads, [&](int ylo, int yhi) {
for (int y = ylo; y < yhi; y++)
for (int x = 0; x < W; x++)
shadow[y * W + x] = static_cast<float>(radius(x, y)) >= starts_at[ray(x, y)];
});
shadow = dilate(shadow, W, H, margin_px, nthreads);
return {shadow.begin(), shadow.end()};
}
} // namespace
std::vector<uint32_t> FindOuterShadow(const DiffractionExperiment &experiment, const PixelMask &mask,
const std::vector<float> &mean, uint32_t frames, int margin_px,
size_t nthreads) {
if (nthreads == 0)
nthreads = AvailableCpus();
const int width = static_cast<int>(experiment.GetXPixelsNumConv());
const int height = static_cast<int>(experiment.GetYPixelsNumConv());
const float gain = experiment.GetBraggIntegrationSettings().GetPixelGain();
const auto [beam_x, beam_y] = experiment.GetDiffractionGeometry().GetDirectBeam_pxl();
// What the beam-stop shadow already masks is read too: a shadow it reached part of is found whole.
std::vector<uint32_t> known = mask.GetMask(experiment);
for (auto &v : known)
v &= ~(1u << PixelMask::BeamStopPixelBit);
const auto &modules = experiment.GetModules();
if (modules.empty())
return OuterShadowOn(mean, known, width, height, beam_x, beam_y, frames, gain, margin_px, nthreads);
// Panels: the rays run on the detector plane, as in GetMask.
const PlaneRaster raster = PlaneRasterOf(modules, width, height);
const auto shadow = OuterShadowOn(AssembleToPlane(modules, mean, width, height, raster, NAN),
AssembleToPlane(modules, known, width, height, raster,
uint32_t{1u << PixelMask::ModuleGapPixelBit}),
static_cast<int>(raster.width), static_cast<int>(raster.height),
beam_x - static_cast<float>(raster.x0), beam_y - static_cast<float>(raster.y0),
frames, gain, margin_px, nthreads);
return GatherFromPlane(modules, shadow, raster, width, height, uint32_t{0});
}
namespace shadow_finder {
int BlockedOutTo(const std::vector<float> &baseline, const std::vector<int> &ring_pixels) {
image_analysis/beam_stop/ShadowFinder.h
+13
View File
@@ -182,3 +182,16 @@ public:
[[nodiscard]] uint32_t GetFrameCount() const;
};
// The shadow of an obstruction at the outer edge of the beam path - an aperture or a cone in front of the
// detector that dims everything beyond some radius - which ShadowFinder cannot see: beyond the largest ring
// the detector holds whole, a ring can lie wholly in the corners, and where those are all dimmed the ring's
// median is the dimmed level, and nothing on the ring is below it. So there each pixel is compared along its
// own ray instead, with the background nearer the beam: a sharp, significant step down against the ray's own
// fall-off marks the edge of the shadow, and where the ray stays dim from there out to the detector's edge,
// everything beyond that edge on the ray is shadow. Returns 1 for shadow, 0 to keep, of the converted
// pixel count, grown by `margin_px` so that what lies that close to the shadow goes with it.
// `mean` is ShadowFinder::GetMeanProjection over `frames` frames; the rays start at the experiment's direct beam.
[[nodiscard]] std::vector<uint32_t> FindOuterShadow(const DiffractionExperiment &experiment, const PixelMask &mask,
const std::vector<float> &mean, uint32_t frames,
int margin_px, size_t nthreads = 0);
image_analysis/beam_stop/ShadowFinderInternal.h
+12
View File
@@ -111,6 +111,18 @@ inline constexpr int MIN_RING_PIXELS = 32;
// its own and does not follow SHADOW_RATIO.
inline constexpr float BLOCKED_RING_RATIO = 0.35f;
// FindOuterShadow: a pixel is at the edge of an outer shadow when its background is below
// OUTER_STEP_RATIO of what the background OUTER_STEP_PX nearer the beam along its ray, carried on by its
// own fall-off, leads up to. Measured on Rayonix stills, such a shadow keeps 45-60 % of the background. The
// edge is a band OUTER_STEP_PX wide; a piece of it narrower than 2 OUTER_OPENING_PX or with fewer than
// OUTER_MIN_PIXELS pixels is not believed (a dim outermost pixel row, the noise). The rays are binned in
// OUTER_RAYS directions.
inline constexpr int OUTER_STEP_PX = 20;
inline constexpr float OUTER_STEP_RATIO = 0.75f;
inline constexpr int OUTER_OPENING_PX = 2;
inline constexpr int OUTER_MIN_PIXELS = 200;
inline constexpr int OUTER_RAYS = 1440;
// The rings that lie wholly inside the stop: walking outward, every judgeable ring (at least
// MIN_RING_PIXELS pixels) before the first whose baseline reaches BLOCKED_RING_RATIO of the typical
// background. -1 when there is none. See GetMask.
image_analysis/bragg_integration/CalcISigma.cpp
+2
View File
@@ -223,6 +223,8 @@ GlobalWilsonB CalcGlobalWilsonB(const std::vector<MergedReflection> &merged) {
const auto reg = regression(x, y, x.size());
const double b = -2.0 * reg.slope; // <I> ~ exp(-2 B s^2), s^2 = 1/(4 d^2), x = 1/d^2
out.n_shells = static_cast<int>(x.size());
out.intercept = reg.intercept;
out.slope = reg.slope;
out.correlation = std::sqrt(std::clamp(static_cast<double>(reg.r_square), 0.0, 1.0));
if (std::isfinite(b) && b > 0.0)
out.b = b;
image_analysis/bragg_integration/CalcISigma.h
+3
View File
@@ -28,6 +28,9 @@ struct GlobalWilsonB {
int n_shells = 0; // resolution shells actually used
double d_low_A = std::numeric_limits<double>::quiet_NaN(); // the fit window, low-resolution end
double d_high_A = std::numeric_limits<double>::quiet_NaN(); // and high-resolution end (A)
// The fitted line itself, ln<I> = intercept + slope / d^2, wherever the fit ran (n_shells >= 3).
double intercept = std::numeric_limits<double>::quiet_NaN();
double slope = std::numeric_limits<double>::quiet_NaN();
};
GlobalWilsonB CalcGlobalWilsonB(const std::vector<MergedReflection> &merged);
image_analysis/bragg_prediction/BraggPrediction.cpp
+13 -3
View File
@@ -45,7 +45,8 @@ int BraggPrediction::TruncateToOutput(int count) {
reflections.begin() + count,
[](const Reflection &a, const Reflection &b) {
if (a.partiality != b.partiality) return a.partiality > b.partiality;
if (a.dist_ewald != b.dist_ewald) return a.dist_ewald < b.dist_ewald;
if (std::fabs(a.dist_ewald) != std::fabs(b.dist_ewald))
return std::fabs(a.dist_ewald) < std::fabs(b.dist_ewald);
if (a.h != b.h) return a.h < b.h;
if (a.k != b.k) return a.k < b.k;
return a.l < b.l;
@@ -142,7 +143,8 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
float S_y = recip_y + S0.y;
float S_z = recip_z + S0.z;
float S_len = sqrtf(S_x * S_x + S_y * S_y + S_z * S_z);
float dist_ewald_sphere = std::fabs(S_len - one_over_wavelength);
// Signed excitation error, |S| - 1/lambda: positive outside the sphere. The window tests its size.
float dist_ewald_sphere = S_len - one_over_wavelength;
// Energy bandwidth thickens the Ewald shell radially: at the
// diffraction condition |S|-1/λ shifts by recip_z·(Δλ/λ), i.e.
@@ -154,7 +156,7 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
const float bw_tol = kBandwidthCutoffSigmas * settings.bandwidth_sigma * std::fabs(recip_z);
radial_cutoff = std::sqrt(radial_cutoff * radial_cutoff + bw_tol * bw_tol);
}
float window_dist = dist_ewald_sphere;
float window_dist = std::fabs(dist_ewald_sphere);
if (settings.window_c0 > 0.0f || settings.window_ang > 0.0f || settings.window_bw > 0.0f) {
const float wb = settings.window_bw * std::fabs(recip_z);
radial_cutoff = std::sqrt(settings.window_c0 * settings.window_c0
@@ -165,6 +167,14 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
}
if (window_dist <= radial_cutoff) {
if (settings.window_radial > 0.0f) {
const float az = std::fabs(recip_z);
const float f = dist_ewald_sphere * std::fmin(settings.window_radial * az / (radial_cutoff * radial_cutoff),
1.0f / az);
S_x += f * S0.x;
S_y += f * S0.y;
S_z += f * S0.z;
}
const float s0_p0 = S0.x * recip_x + S0.y * recip_y + S0.z * recip_z;
const float val = s0_sq * recip_sq - s0_p0 * s0_p0;
image_analysis/bragg_prediction/BraggPrediction.h
+6
View File
@@ -45,6 +45,12 @@ struct BraggPredictionSettings {
// The window is centred where the rocking curve is: on the excitation error of the reciprocal lattice
// scaled by this (StillsPartialityRefine::MeasureRadialScale). dist_ewald stays the lattice's own.
float window_q_scale = 1.0f;
// Stills, with the window: a reflection off the Ewald sphere is placed where the radial part of its
// smeared node diffracts (StillsPartialityRefine::MeasureRadialSpread) - its S moved along S0 by
// dk / k = window_radial |q_z| eps / W^2, W the window at the reflection, but never past the change that
// puts the node on the sphere (eps / |q_z|). window_radial = s^2 times the window's width in sigmas
// squared, s the run's relative radial spread. 0 leaves the positions as they were.
float window_radial = 0.0f;
};
class BraggPrediction {
image_analysis/bragg_prediction/BraggPredictionGPU.cu
+11 -2
View File
@@ -106,7 +106,7 @@ namespace {
float Sy = recip_y + C.S0.y;
float Sz = recip_z + C.S0.z;
float S_len = sqrtf(Sx * Sx + Sy * Sy + Sz * Sz);
float dist_ewald = fabsf(S_len - C.one_over_wavelength);
float dist_ewald = S_len - C.one_over_wavelength; // signed, as on the CPU path
// Energy bandwidth thickens the Ewald shell radially: σ_bw = |recip_z|·(Δλ/λ)
// (= bλ/2d²). Broaden the acceptance window in quadrature (see CPU path).
float radial_cutoff = C.ewald_cutoff;
@@ -114,7 +114,7 @@ namespace {
const float bw_tol = kBandwidthCutoffSigmas * C.bandwidth_sigma * fabsf(recip_z);
radial_cutoff = sqrtf(radial_cutoff * radial_cutoff + bw_tol * bw_tol);
}
float window_dist = dist_ewald;
float window_dist = fabsf(dist_ewald);
if (C.window_c0 > 0.0f || C.window_ang > 0.0f || C.window_bw > 0.0f) {
const float wb = C.window_bw * fabsf(recip_z);
radial_cutoff = sqrtf(C.window_c0 * C.window_c0 + C.window_ang * C.window_ang * recip_sq + wb * wb);
@@ -123,6 +123,14 @@ namespace {
window_dist = fabsf(sqrtf(Scx * Scx + Scy * Scy + Scz * Scz) - C.one_over_wavelength);
}
if (window_dist > radial_cutoff) return false;
if (C.window_radial > 0.0f) { // see the CPU path
const float az = fabsf(recip_z);
const float f = dist_ewald * fminf(C.window_radial * az / (radial_cutoff * radial_cutoff), 1.0f / az);
Sx += f * C.S0.x;
Sy += f * C.S0.y;
Sz += f * C.S0.z;
S_len = sqrtf(Sx * Sx + Sy * Sy + Sz * Sz);
}
float Srx = C.rot[0] * Sx + C.rot[1] * Sy + C.rot[2] * Sz;
float Sry = C.rot[3] * Sx + C.rot[4] * Sy + C.rot[5] * Sz;
float Srz = C.rot[6] * Sx + C.rot[7] * Sy + C.rot[8] * Sz;
@@ -253,6 +261,7 @@ int BraggPredictionGPU::Calc(const DiffractionExperiment &experiment,
hK.window_ang = settings.window_ang;
hK.window_bw = settings.window_bw;
hK.window_q_scale = settings.window_q_scale;
hK.window_radial = settings.window_radial;
cudaMemcpyAsync(dK, &hK, sizeof(KernelConsts), cudaMemcpyHostToDevice, stream);
cudaMemsetAsync(d_count, 0, sizeof(int), stream);
image_analysis/bragg_prediction/BraggPredictionGPU.h
+1
View File
@@ -18,6 +18,7 @@ struct KernelConsts {
float one_over_dmax_sq;
float ewald_cutoff;
float window_c0, window_ang, window_bw, window_q_scale; // the stills window curve; all zero: ewald_cutoff (see BraggPredictionSettings)
float window_radial; // where an off-sphere reflection is placed (see BraggPredictionSettings)
float bandwidth_sigma; // relative Δλ/λ (sigma); 0 = monochromatic
// Angle-dependent sensor efficiency, reduced to the two per-dataset numbers the kernel needs:
// the optical thickness at normal incidence and the absorbed fraction there. qe_a0 = 0 leaves
image_analysis/geom_refinement/PostRefine.cpp
+36 -10
View File
@@ -53,7 +53,7 @@ constexpr double ANGLE_BOUND_DEG = 1.0;
// solve reaches at most this far, and a fit that wants more takes another step from there. A move of
// more than one step is then ratified by re-indexing at it, which is what actually tells a second
// lattice from a wrong header (see PostRefineResult::large_move).
constexpr double STEP_BOUND = 0.01;
constexpr double STEP_BOUND = POSTREFINE_STEP_BOUND;
// ... and the walk is bounded, so a fit that never settles still terminates. Five steps is the +-5 %
// box the solver was always given on the distance, i.e. as far as this fit has ever been trusted.
@@ -424,6 +424,20 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
const UnitCell r0 = reference_latt.GetUnitCell();
const double beam_x0 = nominal_geom.GetBeamX_pxl(), beam_y0 = nominal_geom.GetBeamY_pxl();
const double dist0 = nominal_geom.GetDetectorDistance_mm();
// The beam the fit refines is the PONI, the foot of the normal from the sample. On a detector
// swung out on a 2theta arm the direct beam lies a distance-proportional way from it, so at a
// fixed direct beam the PONI moves with the distance (7 px per mm at 35 deg on a 0.1 mm pixel),
// and a bound on the PONI caps the distance instead of the beam. What the bound is about is
// where the beam hits the detector, which is also what a measured centre measures, so it is
// applied there: direct beam = PONI + beam_per_mm * distance. Zero on an untilted detector.
double beam_per_mm[2] = {0.0, 0.0};
{
const auto [dbx, dby] = nominal_geom.GetDirectBeam_pxl();
if (std::isfinite(dbx) && std::isfinite(dby)) {
beam_per_mm[0] = (dbx - beam_x0) / dist0;
beam_per_mm[1] = (dby - beam_y0) / dist0;
}
}
// Deterministic split of the reflections into a fit half and a held-out half. Avalanche-mix the
// hkl hash so the split bit is decorrelated from the LSB - a plain h+k+l parity collides with the
@@ -670,11 +684,15 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
// the bound and is then refused for being there.
p.SetParameterLowerBound(ds, 0, wide ? dist0 * 0.95 : ds_seed * (1.0 - STEP_BOUND));
p.SetParameterUpperBound(ds, 0, wide ? dist0 * 1.05 : ds_seed * (1.0 + STEP_BOUND));
// The PONI box reaches every PONI whose direct beam the gate below can accept, at any
// distance the walk can reach (MAX_STEPS steps).
for (int j = 0; j < 2; ++j) {
const double b0 = (j == 0) ? beam_x0 : beam_y0;
const double m = settings.measured_beam_px ? (*settings.measured_beam_px)[j] : b0;
p.SetParameterLowerBound(bm, j, std::min(b0, m) - BEAM_BOUND_PXL);
p.SetParameterUpperBound(bm, j, std::max(b0, m) + BEAM_BOUND_PXL);
const double m = settings.measured_beam_px
? (*settings.measured_beam_px)[j] - beam_per_mm[j] * dist0 : b0;
const double reach = std::fabs(beam_per_mm[j]) * dist0 * MAX_STEPS * STEP_BOUND;
p.SetParameterLowerBound(bm, j, std::min(b0, m) - BEAM_BOUND_PXL - reach);
p.SetParameterUpperBound(bm, j, std::max(b0, m) + BEAM_BOUND_PXL + reach);
}
for (int j = 0; j < 3; ++j) {
p.SetParameterLowerBound(rv, j, ax0[j] - 0.05);
@@ -770,6 +788,11 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
bool convJ = solve_joint(FIT, /*wide=*/true, /*fix_distance=*/false, bm_f, ds_f, rv_f,
q0_f, q1_f, q2_f, distance_corr);
const bool held_converged = held_fit.valid() && held_fit.get();
if (convJ) {
result.distance_free_mm = ds_f[0];
result.beam_x_free_px = bm_f[0];
result.beam_y_free_px = bm_f[1];
}
double exc_ref_se = NAN;
double cv_ref = convJ ? joint_cost(VAL, bm_f, ds_f, rv_f, q0_f, q1_f, q2_f,
pos_ref, exc_ref, nullptr, &exc_ref_se)
@@ -876,16 +899,19 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
// step is ratified by re-indexing at where it arrived (result.large_move), which is the
// discriminator the size of the move was standing in for. Names the test it failed, or
// nullptr.
const auto out_of_bounds = [&](const double bm[2], const double q2c[3]) -> const char * {
const auto out_of_bounds = [&](const double bm[2], const double ds[1],
const double q2c[3]) -> const char * {
double ang_shift = 0.0;
for (int j = 0; j < 3; ++j)
ang_shift = std::max(ang_shift, std::fabs(q2c[j] - p2_0[j]));
if (ang_shift >= ANGLE_BOUND_DEG * PI / 180.0)
return "a cell angle moved more than 1 deg";
const double from_nominal = std::hypot(bm[0] - beam_x0, bm[1] - beam_y0);
const double db[2] = {bm[0] + beam_per_mm[0] * ds[0], bm[1] + beam_per_mm[1] * ds[0]};
const double from_nominal = std::hypot(db[0] - beam_x0 - beam_per_mm[0] * dist0,
db[1] - beam_y0 - beam_per_mm[1] * dist0);
const double from_measured = settings.measured_beam_px
? std::hypot(bm[0] - (*settings.measured_beam_px)[0],
bm[1] - (*settings.measured_beam_px)[1])
? std::hypot(db[0] - (*settings.measured_beam_px)[0],
db[1] - (*settings.measured_beam_px)[1])
: std::numeric_limits<double>::infinity();
if (std::min(from_nominal, from_measured) >= BEAM_BOUND_PXL)
return "the beam moved further than the bound from every centre anything believes";
@@ -949,7 +975,7 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
? "the held-out positional residual did not fall by more than its noise"
: !(change.exc_n < 2 || change.exc <= change.exc_se)
? "the held-out excitation residual rose by more than its noise"
: out_of_bounds(bm_f, q2_f);
: out_of_bounds(bm_f, ds_f, q2_f);
// What the half that was held out earned the right to fit is re-fitted on all of it, and
// that second solve moves the geometry - so the bounds are asked again of what will
@@ -971,7 +997,7 @@ PostRefineResult PostRefineRotationGeometry(PostRefineObservations observations,
solve_joint(ALL, !walked, hold_distance, beam, dist, axv, p0, p1, p2); // commit: re-fit on all data
ds_log[0] = dist[0]; bm_log[0] = beam[0]; bm_log[1] = beam[1];
for (int j = 0; j < 3; ++j) { q1_log[j] = p1[j]; q2_log[j] = p2[j]; }
refused = out_of_bounds(beam, p2);
refused = out_of_bounds(beam, dist, p2);
if (refused != nullptr) {
beam[0] = beam_x0; beam[1] = beam_y0; dist[0] = dist0;
for (int j = 0; j < 3; ++j) {
image_analysis/geom_refinement/PostRefine.h
+9
View File
@@ -31,6 +31,10 @@
// Where there are more reflections than the fit's caps, the strongest by I/sigma are the ones kept;
// inside the fit every reflection carries the same weight.
// How far one step of the joint fit may move the distance or a cell length, relative (see STEP_BOUND in
// PostRefine.cpp, where it is explained).
constexpr double POSTREFINE_STEP_BOUND = 0.01;
struct PostRefineResult {
bool ok = false;
DiffractionGeometry geom; // refined (distance / beam left as configured)
@@ -53,6 +57,11 @@ struct PostRefineResult {
// ever - because freeing it did not lower the held-out excitation residual by more than that
// residual's noise (see the joint fit in PostRefine.cpp). The report names it.
bool distance_held = false;
// Where the fit with the distance free went, held or not (NaN where it did not converge): the
// detector move these data ask for at the geometry they were integrated at.
double distance_free_mm = std::numeric_limits<double>::quiet_NaN();
double beam_x_free_px = std::numeric_limits<double>::quiet_NaN();
double beam_y_free_px = std::numeric_limits<double>::quiet_NaN();
// The held-out residual at the geometry the fit STARTED from - what the integration this fit was
// handed actually realises there, before the fit moves anything - and its standard error over the
// held-out values. Two passes at two geometries are compared on this, not on the fit's own
image_analysis/hot_pixels/HotPixels.cpp
+15 -2
View File
@@ -83,6 +83,7 @@ HotPixelFinder::HotPixelFinder(const DiffractionExperiment &experiment, const Pi
key(std::make_unique_for_overwrite<int32_t[]>(width * height)),
n_lit(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
n_error(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
n_neg(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
sum_value(std::make_unique_for_overwrite<int64_t[]>(width * height)),
n_error_ring_ok(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
error_level_sum(std::make_unique_for_overwrite<int64_t[]>(width * height)) {
@@ -91,7 +92,7 @@ HotPixelFinder::HotPixelFinder(const DiffractionExperiment &experiment, const Pi
const float ring_rad = RING_WIDTH_PX * geometry.GetPixelSize_mm() / geometry.GetDetectorDistance_mm();
ParallelChunks(static_cast<int>(height), nthreads, [&](int y0, int y1) {
for (size_t i = y0 * width; i < y1 * width; i++) {
n_lit[i] = n_error[i] = n_error_ring_ok[i] = 0;
n_lit[i] = n_error[i] = n_neg[i] = n_error_ring_ok[i] = 0;
sum_value[i] = error_level_sum[i] = 0;
key[i] = -1;
}
@@ -217,6 +218,8 @@ void HotPixelFinder::AddImage(const int32_t *image, std::vector<int32_t> &scratc
sum_value[i] += v;
if (static_cast<float>(v) > threshold[k])
n_lit[i]++;
if (static_cast<float>(v) < NegativeLimit(level[k], gain))
n_neg[i]++;
}
}
}
@@ -270,7 +273,9 @@ void HotPixelFinder::PrepareDevice() {
set_gpu(c);
return std::make_unique<HotPixelFinderGPU>(key.get(), width * height, key_begin, nrings, SECTORS,
HotPixelLevelRules{MIN_SECTOR_PIXELS, MIN_RING_PIXELS,
LIT_NSIGMA, LIT_OFFSET * gain, gain});
LIT_NSIGMA, LIT_OFFSET * gain, gain,
NEG_NSIGMA, NEG_OFFSET * gain,
NEG_FRACTION_INV, NEG_MIN_FRAMES});
}));
for (auto &b : built)
gpus.push_back(b.get());
@@ -368,6 +373,7 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
const size_t i = c.index[j];
sums_known[i] = 1;
n_lit[i] = c.n_lit[j];
n_neg[i] = c.n_neg[j];
n_error[i] = c.n_error[j];
n_error_ring_ok[i] = c.n_error_ring_ok[j];
sum_value[i] = c.sum_value[j];
@@ -412,6 +418,12 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
ret.mask[i] = 2;
continue;
}
// Unstable: far below zero on a twentieth of its frames, which no count of photons does.
if (sums_known[i] && n_valid(i) >= min_valid && n_neg[i] >= NEG_MIN_FRAMES
&& NEG_FRACTION_INV * n_neg[i] >= n_valid(i)) {
ret.mask[i] = 4;
continue;
}
// Dead: the mirror of hot. A pixel reading at most half of its own sector's level, by
// more than the bound a hot pixel's excess has to clear, is not recording the light that
// reaches it, and every reflection whose disk covers it is integrated short by that
@@ -440,6 +452,7 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
ret.hot += v == 1;
ret.error += v == 2;
ret.dead += v == 3;
ret.unstable += v == 4;
}
return ret;
}
image_analysis/hot_pixels/HotPixels.h
+23 -2
View File
@@ -42,11 +42,20 @@
// covers it is integrated short by the deficit. The detector's outermost row and column get no rule of
// their own - a dead edge row, or a hot pixel there, is caught like any other.
//
// An UNSTABLE pixel reads far below zero - ten times the Poisson noise of its ring's level and ten photons
// more under it - on at least a twentieth of its frames (and on two). No photon can make a count
// negative, and a reflection only adds counts, so such a pixel is broken whatever it reads in between:
// measured, a column of a JUNGFRAU 9M swinging between -2500 and +1000 counts from frame to frame,
// with a normal median. Its negative frames drag the background ring of every reflection near it far
// below zero, and a reflection whose disk then sums to nothing is integrated huge with a one-photon sigma.
//
// A CCD is read with an offset and is not Poisson, but neither test needs it to be: a dead CCD pixel
// reads its offset, far below a ring with any background on it, and a hot one ten times its ring.
// =============================================================================
#include <atomic>
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <memory>
#include <mutex>
@@ -82,14 +91,21 @@ public:
// strength at which the pixels behind the merged outliers were found, and well clear of a pixel
// that is merely a little over-responding.
static constexpr double STRONG_RATIO = 10.0;
// Unstable: below -(NEG_NSIGMA * noise of the level + NEG_OFFSET photons) on at least one frame in
// NEG_FRACTION_INV, and on NEG_MIN_FRAMES.
static constexpr float NEG_NSIGMA = 10.0f;
static constexpr float NEG_OFFSET = 10.0f;
static constexpr int NEG_FRACTION_INV = 20;
static constexpr int NEG_MIN_FRAMES = 2;
// The most frames a finder can be given: its per-pixel counters are 16-bit.
static constexpr int MAX_SAMPLED_FRAMES = UINT16_MAX;
struct Result {
std::vector<uint32_t> mask; // non-zero = masked (1 hot, 2 error value, 3 dead), converted geometry
std::vector<uint32_t> mask; // non-zero = masked (1 hot, 2 error value, 3 dead, 4 unstable), converted geometry
size_t hot = 0; // persistent above the ring
size_t error = 0; // holding the error value or the saturation limit on most frames
size_t dead = 0; // reading at most half of its sector's level
size_t unstable = 0; // far below zero on a twentieth of its frames
uint32_t frames = 0;
};
@@ -139,7 +155,7 @@ private:
std::atomic<size_t> next_band{0};
mutable std::mutex m;
uint32_t frames = 0;
std::unique_ptr<uint16_t[]> n_lit, n_error;
std::unique_ptr<uint16_t[]> n_lit, n_error, n_neg;
std::unique_ptr<int64_t[]> sum_value;
// A pixel's valid frames and the sum of its ring-sector level over them, kept per ring-sector - the
// frames its ring had a level on, and the sum of the level over those - less what the pixel's own
@@ -167,6 +183,11 @@ private:
const std::vector<float> &ring_spread, const std::vector<char> &ring_ok,
const std::vector<int32_t> &local_level,
std::vector<int32_t> &level, std::vector<float> &threshold);
// The count below which a pixel of level `level` is unstable on a frame - one rounding for the fma, as
// on the device (HotPixelsGPU.cu).
[[nodiscard]] static float NegativeLimit(int32_t level, float gain) {
return -std::fma(NEG_NSIGMA, std::sqrt(gain * std::max(static_cast<float>(level), gain)), NEG_OFFSET * gain);
}
[[nodiscard]] int n_valid(size_t i) const { return key_frames[key[i]] - n_error_ring_ok[i]; }
[[nodiscard]] int64_t sum_level(size_t i) const { return key_level_sum[key[i]] - error_level_sum[i]; }
};
image_analysis/hot_pixels/HotPixelsGPU.cu
+27 -13
View File
@@ -123,11 +123,11 @@ __global__ void ring_kernel(const int32_t *__restrict__ image, const uint32_t *_
// HotPixelFinder::AddImage's per-pixel loop, line for line.
__global__ void accumulate_kernel(const int32_t *__restrict__ image, const int32_t *__restrict__ key,
size_t npixels, int sectors, const int32_t *__restrict__ level,
size_t npixels, int sectors, HotPixelLevelRules rules, const int32_t *__restrict__ level,
const float *__restrict__ threshold, const char *__restrict__ ring_ok,
uint16_t *__restrict__ n_lit, uint16_t *__restrict__ n_error,
int64_t *__restrict__ sum_value, uint16_t *__restrict__ n_error_ring_ok,
int64_t *__restrict__ error_level_sum) {
int64_t *__restrict__ error_level_sum, uint16_t *__restrict__ n_neg) {
for (size_t i = blockIdx.x * static_cast<size_t>(blockDim.x) + threadIdx.x; i < npixels;
i += static_cast<size_t>(blockDim.x) * gridDim.x) {
const int32_t k = key[i], v = image[i];
@@ -144,6 +144,12 @@ __global__ void accumulate_kernel(const int32_t *__restrict__ image, const int32
sum_value[i] += v;
if (static_cast<float>(v) > threshold[k])
n_lit[i]++;
// HotPixelFinder::NegativeLimit, with its roundings
const float neg = -__fmaf_rn(rules.neg_nsigma,
__fsqrt_rn(__fmul_rn(rules.gain, fmaxf(static_cast<float>(level[k]), rules.gain))),
rules.neg_offset);
if (static_cast<float>(v) < neg)
n_neg[i]++;
}
}
@@ -202,12 +208,12 @@ __global__ void chance_kernel(size_t npixels, int sectors, const int32_t *__rest
}
// Writes the candidates' indices from `out` on when `out` is given, and counts them either way.
__global__ void candidate_kernel(size_t npixels, int sectors, uint32_t frames, int min_valid, bool spacing_ok,
__global__ void candidate_kernel(size_t npixels, int sectors, HotPixelLevelRules rules, uint32_t frames, int min_valid, bool spacing_ok,
const int32_t *__restrict__ key, const uint32_t *__restrict__ key_frames,
const uint16_t *__restrict__ n_lit, const uint16_t *__restrict__ n_error,
const uint16_t *__restrict__ n_error_ring_ok, const int64_t *__restrict__ sum_value,
const int64_t *__restrict__ key_local_level_sum, const int *__restrict__ k_chance,
uint32_t *__restrict__ count, uint32_t *__restrict__ out) {
const uint16_t *__restrict__ n_neg, uint32_t *__restrict__ count, uint32_t *__restrict__ out) {
const size_t i = blockIdx.x * static_cast<size_t>(blockDim.x) + threadIdx.x;
if (i >= npixels || key[i] < 0) return;
const int nv = valid_frames(key, key_frames, n_error_ring_ok, i);
@@ -217,7 +223,10 @@ __global__ void candidate_kernel(size_t npixels, int sectors, uint32_t frames, i
&& lit >= min(max(2, k_chance[key[i] / sectors]), nv);
// Can be dead (HotPixelFinder::GetMask): at most half of its sector's level, no error frames.
const bool dark = n_error[i] == 0 && nv >= min_valid && 2 * sum_value[i] <= key_local_level_sum[key[i]];
if (!error && !persistent && !dark) return;
// Can be unstable (HotPixelFinder::GetMask): far below zero on a twentieth of its valid frames.
const bool unstable = nv >= min_valid && n_neg[i] >= rules.neg_min_frames
&& rules.neg_fraction_inv * static_cast<int>(n_neg[i]) >= nv;
if (!error && !persistent && !dark && !unstable) return;
const uint32_t slot = atomicAdd(count, 1u);
if (out) out[slot] = static_cast<uint32_t>(i);
}
@@ -256,7 +265,7 @@ HotPixelFinderGPU::HotPixelFinderGPU(const int32_t *host_key, size_t npixels,
sectors(sectors), rules(rules),
key(npixels, ALLOC), pixels_by_key(std::max<size_t>(host_key_begin.back(), 1), ALLOC),
key_begin(host_key_begin.size(), ALLOC),
n_lit(npixels, ALLOC), n_error(npixels, ALLOC), n_error_ring_ok(npixels, ALLOC),
n_lit(npixels, ALLOC), n_error(npixels, ALLOC), n_error_ring_ok(npixels, ALLOC), n_neg(npixels, ALLOC),
sum_value(npixels, ALLOC), error_level_sum(npixels, ALLOC),
key_frames(std::max<size_t>(nkeys, 1), ALLOC), key_level_sum(std::max<size_t>(nkeys, 1), ALLOC),
key_local_level_sum(std::max<size_t>(nkeys, 1), ALLOC) {
@@ -269,6 +278,7 @@ HotPixelFinderGPU::HotPixelFinderGPU(const int32_t *host_key, size_t npixels,
cuda_err(cudaMemsetAsync(n_lit, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_error, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_error_ring_ok, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_neg, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(sum_value, 0, npixels * sizeof(int64_t), stream));
cuda_err(cudaMemsetAsync(error_level_sum, 0, npixels * sizeof(int64_t), stream));
cuda_err(cudaMemsetAsync(key_frames, 0, std::max<size_t>(nkeys, 1) * sizeof(uint32_t), stream));
@@ -326,8 +336,8 @@ void HotPixelFinderGPU::Add(const int32_t *device_image, Frame &frame) {
std::lock_guard lock(accumulate_mutex);
cuda_err(cudaStreamWaitEvent(stream, last_accumulate, 0));
accumulate_kernel<<<static_cast<unsigned>((npixels + THREADS - 1) / THREADS), THREADS, 0, stream>>>(
device_image, key, npixels, sectors, frame.level, frame.threshold, frame.ring_ok,
n_lit, n_error, sum_value, n_error_ring_ok, error_level_sum);
device_image, key, npixels, sectors, rules, frame.level, frame.threshold, frame.ring_ok,
n_lit, n_error, sum_value, n_error_ring_ok, error_level_sum, n_neg);
cuda_err(cudaGetLastError());
cuda_err(cudaEventRecord(last_accumulate, stream));
}
@@ -348,6 +358,7 @@ void HotPixelFinderGPU::Fold(HotPixelFinderGPU &other) {
add(n_lit.get(), other.n_lit.get(), npixels);
add(n_error.get(), other.n_error.get(), npixels);
add(n_error_ring_ok.get(), other.n_error_ring_ok.get(), npixels);
add(n_neg.get(), other.n_neg.get(), npixels);
add(sum_value.get(), other.sum_value.get(), npixels);
add(error_level_sum.get(), other.error_level_sum.get(), npixels);
add(key_frames.get(), other.key_frames.get(), nkeys);
@@ -385,9 +396,9 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
cuda_err(cudaMemcpyAsync(d_k_chance, k_chance.data(), k_chance.size() * sizeof(int), cudaMemcpyHostToDevice,
stream));
cuda_err(cudaMemsetAsync(d_count, 0, sizeof(uint32_t), stream));
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, frames, min_valid, spacing_ok, key, key_frames,
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, rules, frames, min_valid, spacing_ok, key, key_frames,
n_lit, n_error, n_error_ring_ok, sum_value, key_local_level_sum,
d_k_chance, d_count, nullptr);
d_k_chance, n_neg, d_count, nullptr);
cuda_err(cudaGetLastError());
uint32_t n = 0;
cuda_err(cudaMemcpyAsync(&n, d_count, sizeof(uint32_t), cudaMemcpyDeviceToHost, stream));
@@ -407,17 +418,18 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
}
if (n > 0) {
CudaDevicePtr<uint32_t> index(n, ALLOC);
CudaDevicePtr<uint16_t> g_lit(n, ALLOC), g_error(n, ALLOC), g_error_ring_ok(n, ALLOC);
CudaDevicePtr<uint16_t> g_lit(n, ALLOC), g_error(n, ALLOC), g_error_ring_ok(n, ALLOC), g_neg(n, ALLOC);
CudaDevicePtr<int64_t> g_sum(n, ALLOC), g_error_level(n, ALLOC);
cuda_err(cudaMemsetAsync(d_count, 0, sizeof(uint32_t), stream));
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, frames, min_valid, spacing_ok, key,
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, rules, frames, min_valid, spacing_ok, key,
key_frames, n_lit, n_error, n_error_ring_ok, sum_value,
key_local_level_sum, d_k_chance, d_count, index);
key_local_level_sum, d_k_chance, n_neg, d_count, index);
cuda_err(cudaGetLastError());
const unsigned gb = (n + THREADS - 1) / THREADS;
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_lit.get(), g_lit.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_error.get(), g_error.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_error_ring_ok.get(), g_error_ring_ok.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_neg.get(), g_neg.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, sum_value.get(), g_sum.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, error_level_sum.get(), g_error_level.get());
cuda_err(cudaGetLastError());
@@ -425,6 +437,7 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
c.n_lit.resize(n);
c.n_error.resize(n);
c.n_error_ring_ok.resize(n);
c.n_neg.resize(n);
c.sum_value.resize(n);
c.error_level_sum.resize(n);
cuda_err(cudaMemcpyAsync(c.index.data(), index, n * sizeof(uint32_t), cudaMemcpyDeviceToHost, stream));
@@ -432,6 +445,7 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
cuda_err(cudaMemcpyAsync(c.n_error.data(), g_error, n * sizeof(uint16_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.n_error_ring_ok.data(), g_error_ring_ok, n * sizeof(uint16_t),
cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.n_neg.data(), g_neg, n * sizeof(uint16_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.sum_value.data(), g_sum, n * sizeof(int64_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.error_level_sum.data(), g_error_level, n * sizeof(int64_t),
cudaMemcpyDeviceToHost, stream));
image_analysis/hot_pixels/HotPixelsGPU.h
+6 -2
View File
@@ -18,6 +18,10 @@ struct HotPixelLevelRules {
float lit_nsigma;
float lit_offset; // in the frames' units
float gain; // the frames' units per photon
float neg_nsigma; // HotPixelFinder::NegativeLimit
float neg_offset; // in the frames' units
int neg_fraction_inv;
int neg_min_frames;
};
// The device half of HotPixelFinder, for frames already preprocessed on the GPU. Everything a frame adds
@@ -41,7 +45,7 @@ class HotPixelFinderGPU {
CudaDevicePtr<uint32_t> key_begin; // where each key's pixels start in pixels_by_key
// The per-pixel and per-key sums, exactly those of HotPixelFinder.
CudaDevicePtr<uint16_t> n_lit, n_error, n_error_ring_ok;
CudaDevicePtr<uint16_t> n_lit, n_error, n_error_ring_ok, n_neg;
CudaDevicePtr<int64_t> sum_value, error_level_sum;
CudaDevicePtr<uint32_t> key_frames;
CudaDevicePtr<int64_t> key_level_sum, key_local_level_sum;
@@ -99,7 +103,7 @@ public:
// per-pixel sums; and every key's sums.
struct Candidates {
std::vector<uint32_t> index;
std::vector<uint16_t> n_lit, n_error, n_error_ring_ok;
std::vector<uint16_t> n_lit, n_error, n_error_ring_ok, n_neg;
std::vector<int64_t> sum_value, error_level_sum;
std::vector<uint32_t> key_frames;
std::vector<int64_t> key_level_sum, key_local_level_sum;
image_analysis/indexing/FFTIndexer.cpp
+1
View File
@@ -463,6 +463,7 @@ std::vector<CrystalLattice> FFTIndexer::ReduceAndRefine(const std::vector<Coord>
RefineParameters parameters{
.viable_cell_min_spots = viable_cell_min_spots,
.dist_tolerance_vs_reference = dist_tolerance_vs_reference,
.step_down_to_reference = step_down_to_reference,
.reference_unit_cell = reference_unit_cell,
.min_length_A = min_length_A,
// A row DeAlias unfolded is longer than max_length_A by construction, and shorter than twice it.
image_analysis/indexing/Indexer.cpp
+1
View File
@@ -7,6 +7,7 @@ void Indexer::Setup(const DiffractionExperiment& experiment) {
geom = experiment.GetDiffractionGeometry();
indexing_tolerance = experiment.GetIndexingSettings().GetTolerance();
dist_tolerance_vs_reference = experiment.GetIndexingSettings().GetUnitCellDistTolerance();
step_down_to_reference = experiment.GetIndexingSettings().GetRotationIndexing();
viable_cell_min_spots = experiment.GetIndexingSettings().GetViableCellMinSpots();
index_ice_rings = experiment.GetIndexingSettings().GetIndexIceRings();
wavelength_A = experiment.GetWavelength_A();
image_analysis/indexing/Indexer.h
+5
View File
@@ -35,6 +35,11 @@ protected:
int64_t viable_cell_min_spots = 9;
float indexing_tolerance = 0.1;
float dist_tolerance_vs_reference = 0.05;
// Rotation only: a candidate that is an integer supercell of the given cell is stepped down to it
// (see StepDownToReference). On stills it is not: a still is one frame, the given cell picks one
// crystal form among several in the stream, and a sub-lattice of a frame's supercell matches a
// NEIGHBOURING form within the tolerance as readily as the given one.
bool step_down_to_reference = false;
bool index_ice_rings = true;
DiffractionGeometry geom;
image_analysis/indexing/PostIndexingRefinement.cpp
+112 -35
View File
@@ -7,6 +7,7 @@
#include <iostream>
#include <limits>
#include <future>
#include <optional>
namespace {
struct config_ifssr final {
@@ -142,6 +143,106 @@ namespace {
cells.block(3u * j, 0u, 3u, 3u) = cell.transpose();
}
}
// The reference test, on a candidate's three rows: sorted lengths within the relative tolerance and
// sorted angles within REFINE_ANGLE_TOLERANCE_VS_REFERENCE_DEG of any of the reference cells. Each
// angle is folded to its acute complement (min(x,180-x)) so the obtuse/acute setting choice is
// irrelevant. Guards against a right-edges/wrong-angle cell (a pseudo-symmetric near-metric, e.g. a
// monoclinic beta refined to the wrong value) passing on lengths.
bool MatchesReference(const Eigen::Matrix3f &cell_rows, const std::vector<UnitCell> &reference_cells,
float dist_tolerance) {
std::array<float, 3> obs = {cell_rows.row(0).norm(), cell_rows.row(1).norm(), cell_rows.row(2).norm()};
std::sort(obs.begin(), obs.end());
auto fold = [](float deg) { return std::min(deg, 180.0f - deg); };
auto row_angle = [&](int i, int j) {
return std::acos(std::clamp(cell_rows.row(i).normalized().dot(cell_rows.row(j).normalized()),
-1.0f, 1.0f)) * 180.0f / PI;
};
std::array<float, 3> obs_ang = {fold(row_angle(1, 2)), fold(row_angle(0, 2)), fold(row_angle(0, 1))};
std::sort(obs_ang.begin(), obs_ang.end());
auto matches = [&](const UnitCell &reference) {
std::array<float, 3> ref = {reference.a, reference.b, reference.c};
std::sort(ref.begin(), ref.end());
for (int k = 0; k < 3; ++k) {
const float denom = std::max(ref[k], REFINE_MIN_REFERENCE_LENGTH_EPSILON);
if (std::abs(obs[k] - ref[k]) / denom > dist_tolerance)
return false;
}
std::array<float, 3> ref_ang = {fold(reference.alpha), fold(reference.beta), fold(reference.gamma)};
std::sort(ref_ang.begin(), ref_ang.end());
for (int k = 0; k < 3; ++k) {
if (std::abs(obs_ang[k] - ref_ang[k]) > REFINE_ANGLE_TOLERANCE_VS_REFERENCE_DEG)
return false;
}
return true;
};
return std::any_of(reference_cells.begin(), reference_cells.end(), matches);
}
// A candidate that is an integer SUPERCELL of the reference lattice: the FFT builds one from rows it
// reports at a multiple of their length, and on a twin by reticular merohedry the cell indexing both
// domains is their coincidence lattice, a supercell of either. Every node of the candidate is then a
// node of the reference lattice, so each reference basis vector is a combination of the candidate's
// rows with coefficients in (1/n)Z, n the volume ratio. Those combinations are enumerated, and of the
// triples that match the reference, span its volume and contain the candidate's lattice, the one
// that indexes the most spots is returned (on a twin, the stronger domain).
std::optional<Eigen::Matrix3f> StepDownToReference(const Eigen::Matrix3f &cell_rows,
const std::vector<UnitCell> &reference_cells,
float dist_tolerance,
const Eigen::Ref<const Eigen::MatrixX3<float>> &spots,
float indexing_tolerance) {
constexpr int MAX_INDEX = 10;
const float vol = std::abs(cell_rows.determinant());
std::optional<Eigen::Matrix3f> best;
int64_t best_count = -1;
for (const auto &ref: reference_cells) {
const float ref_vol = CrystalLattice(ref).CalcVolume();
if (!(ref_vol > 0.0f))
continue;
const float ratio = vol / ref_vol;
const int n = static_cast<int>(std::lround(ratio));
if (n < 2 || n > MAX_INDEX || std::abs(ratio - static_cast<float>(n)) > 0.15f * static_cast<float>(n))
continue;
const std::array<float, 3> lengths = {ref.a, ref.b, ref.c};
std::array<std::vector<Eigen::RowVector3f>, 3> near;
for (int i = -n; i <= n; i++)
for (int j = -n; j <= n; j++)
for (int k = -n; k <= n; k++) {
const Eigen::RowVector3f v = (static_cast<float>(i) * cell_rows.row(0)
+ static_cast<float>(j) * cell_rows.row(1)
+ static_cast<float>(k) * cell_rows.row(2))
/ static_cast<float>(n);
for (int a = 0; a < 3; a++)
if (std::abs(v.norm() - lengths[a]) <= dist_tolerance * lengths[a])
near[a].push_back(v);
}
for (const auto &v0: near[0])
for (const auto &v1: near[1])
for (const auto &v2: near[2]) {
Eigen::Matrix3f sub;
sub << v0, v1, v2;
if (std::abs(std::abs(sub.determinant()) - ref_vol) > 0.15f * ref_vol)
continue;
const Eigen::Matrix3f m = cell_rows * sub.inverse();
if ((m - m.array().round().matrix()).cwiseAbs().maxCoeff() > 0.1f)
continue;
if (!MatchesReference(sub, {ref}, dist_tolerance))
continue;
int64_t count = 0;
ComputeIndexedMask(spots, sub.transpose(), indexing_tolerance, count);
if (count > best_count) {
best_count = count;
best = sub;
}
}
}
return best;
}
}
std::vector<CrystalLattice> Refine(const std::vector<Coord> &in_spots,
@@ -216,41 +317,17 @@ std::vector<CrystalLattice> Refine(const std::vector<Coord> &in_spots,
Eigen::Vector3f row_norms = cell_rows.rowwise().norm();
if (!reference_cells.empty()) {
std::array<float, 3> obs = {row_norms(0), row_norms(1), row_norms(2)};
std::sort(obs.begin(), obs.end());
// The angles are compared as well as the lengths. Each is folded to its acute complement
// (min(x,180-x)) so the obtuse/acute setting choice is irrelevant, then the sorted
// triples are compared. Guards against a right-edges/wrong-angle cell (a pseudo-symmetric
// near-metric, e.g. a monoclinic beta refined to the wrong value) passing on lengths.
auto fold = [](float deg) { return std::min(deg, 180.0f - deg); };
auto row_angle = [&](int i, int j) {
return std::acos(std::clamp(cell_rows.row(i).normalized().dot(cell_rows.row(j).normalized()),
-1.0f, 1.0f)) * 180.0f / PI;
};
std::array<float, 3> obs_ang = {fold(row_angle(1, 2)), fold(row_angle(0, 2)), fold(row_angle(0, 1))};
std::sort(obs_ang.begin(), obs_ang.end());
auto matches = [&](const UnitCell &reference) {
std::array<float, 3> ref = {reference.a, reference.b, reference.c};
std::sort(ref.begin(), ref.end());
for (int k = 0; k < 3; ++k) {
const float denom = std::max(ref[k], REFINE_MIN_REFERENCE_LENGTH_EPSILON);
if (std::abs(obs[k] - ref[k]) / denom > p.dist_tolerance_vs_reference)
return false;
}
std::array<float, 3> ref_ang = {fold(reference.alpha), fold(reference.beta), fold(reference.gamma)};
std::sort(ref_ang.begin(), ref_ang.end());
for (int k = 0; k < 3; ++k) {
if (std::abs(obs_ang[k] - ref_ang[k]) > REFINE_ANGLE_TOLERANCE_VS_REFERENCE_DEG)
return false;
}
return true;
};
if (std::none_of(reference_cells.begin(), reference_cells.end(), matches))
continue;
if (!MatchesReference(cell_rows, reference_cells, p.dist_tolerance_vs_reference)) {
if (!p.step_down_to_reference)
continue;
const auto sub = StepDownToReference(cell_rows, reference_cells, p.dist_tolerance_vs_reference,
spots.topRows(nspots), p.indexing_tolerance);
if (!sub)
continue;
cell_rows = *sub;
cell_cols = cell_rows.transpose();
row_norms = cell_rows.rowwise().norm();
}
} else {
if (row_norms.minCoeff() < p.min_length_A || row_norms.maxCoeff() > p.max_length_A)
continue;
image_analysis/indexing/PostIndexingRefinement.h
+1
View File
@@ -24,6 +24,7 @@ constexpr float REFINE_ANGLE_TOLERANCE_VS_REFERENCE_DEG = 10.0f;
struct RefineParameters {
int64_t viable_cell_min_spots;
float dist_tolerance_vs_reference;
bool step_down_to_reference = false;
std::optional<UnitCell> reference_unit_cell;
float min_length_A;
float max_length_A;
image_analysis/rotation_indexer/RotationIndexer.h
+6 -1
View File
@@ -40,10 +40,15 @@ struct RotationIndexerResult {
// Twinning by reticular merohedry (rugnux's twin-composite step): a second domain whose lattice
// shares the reflections h of this one with h . t divisible by n, t a vector of this lattice (real
// space, in the frame of `lattice`, so it holds through any change of basis). Those reflections
// carry both domains' intensity, and are not integrated.
// carry both domains' intensity, and are not integrated - but for the axial reflections whose twin
// partner is the reflection itself or its Friedel mate, which carry this domain's intensity alone
// (IndexAndRefine) - scaled by this domain's share. `rotation` takes this domain's lattice onto the
// other's (same frame as `lattice`), where it is known.
struct TwinOverlap {
Coord t;
int n = 0;
std::optional<RotMatrix> rotation;
float fraction = 0.0f; // the other domain's share of the crystal (the twin fraction)
};
std::optional<TwinOverlap> twin_overlap;
};
image_analysis/scale_merge/CMakeLists.txt
+4 -2
View File
@@ -44,8 +44,10 @@ ADD_LIBRARY(JFJochScaleMerge
AndersonMixing.cpp
AndersonMixing.h
SublatticeRelabel.cpp
SublatticeRelabel.h)
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon fftw3f)
SublatticeRelabel.h
IceRingVerdict.cpp
IceRingVerdict.h)
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration fftw3f)
IF (JFJOCH_CUDA_AVAILABLE)
TARGET_SOURCES(JFJochScaleMerge PRIVATE ../indexing/CUDAMemHelpers.h
image_analysis/scale_merge/CellPopulation.cpp
+105 -13
View File
@@ -4,10 +4,13 @@
#include "CellPopulation.h"
#include <algorithm>
#include <array>
#include <cmath>
#include <limits>
std::pair<double, double> GivenCellPopulation(const std::vector<double> &relative_volume) {
#include <Eigen/Dense>
std::vector<CellPopulationRange> CellPopulations(const std::vector<double> &relative_volume) {
constexpr double everything = std::numeric_limits<double>::infinity();
// Below this many frames a histogram has no valleys to speak of.
constexpr size_t MIN_FRAMES = 50;
@@ -19,7 +22,7 @@ std::pair<double, double> GivenCellPopulation(const std::vector<double> &relativ
const size_t n = relative_volume.size();
if (n < MIN_FRAMES)
return {-everything, everything};
return {};
std::vector<double> v = relative_volume;
std::sort(v.begin(), v.end());
@@ -32,7 +35,7 @@ std::pair<double, double> GivenCellPopulation(const std::vector<double> &relativ
std::nth_element(dev.begin(), dev.begin() + n / 2, dev.end());
const double bin = 2.0 * 1.4826 * dev[n / 2] / std::cbrt(static_cast<double>(n));
if (!(bin > 0))
return {-everything, everything};
return {};
const double lo = v.front();
const int nbins = static_cast<int>((v.back() - lo) / bin) + 1;
@@ -66,10 +69,9 @@ std::pair<double, double> GivenCellPopulation(const std::vector<double> &relativ
return -1;
};
// Every population: a local maximum of the histogram and the valleys about it. Of those that hold
// a real share of the frames, the one centred nearest the given cell is kept - not merely the peak
// nearest it, which can be a few stray frames when the given cell is a percent off the crystal.
double best_low = -everything, best_high = everything, best_distance = everything;
// Every population: a local maximum of the histogram and the valleys about it, if it holds a real share
// of the frames.
std::vector<CellPopulationRange> populations;
for (int i = 0; i < nbins; i++) {
if (smooth[i] <= 0 || (i > 0 && smooth[i - 1] > smooth[i]) || (i + 1 < nbins && smooth[i + 1] >= smooth[i]))
continue;
@@ -94,12 +96,102 @@ std::pair<double, double> GivenCellPopulation(const std::vector<double> &relativ
}
if (count < MIN_SHARE * static_cast<double>(n))
continue;
const double distance = std::fabs(sum / static_cast<double>(count));
if (distance < best_distance) {
best_distance = distance;
best_low = low;
best_high = high;
}
populations.push_back({low, high, count, sum / static_cast<double>(count)});
}
return populations;
}
std::pair<double, double> GivenCellPopulation(const std::vector<double> &relative_volume) {
constexpr double everything = std::numeric_limits<double>::infinity();
// The population centred nearest the given cell is kept - not merely the peak nearest it, which can be
// a few stray frames when the given cell is a percent off the crystal.
double best_low = -everything, best_high = everything, best_distance = everything;
for (const auto &p : CellPopulations(relative_volume))
if (std::fabs(p.mean) < best_distance) {
best_distance = std::fabs(p.mean);
best_low = p.low;
best_high = p.high;
}
return {best_low, best_high};
}
namespace {
// A cell's shape, for telling forms apart: its edges as logarithms, so a percent weighs the same on every
// edge. Not its angles: a still's refined angles carry the indexing more than the crystal - the same form
// comes out with its monoclinic angle on one axis or another, obtuse or acute, or held at 90 degrees - and
// those spread more than two forms' angles differ.
Eigen::Vector3d Shape(const UnitCell &c) {
return {std::log(c.a), std::log(c.b), std::log(c.c)};
}
}
UnitCell AlignedCell(const UnitCell &cell, const UnitCell &reference) {
const std::array<float, 3> edge = {cell.a, cell.b, cell.c}, angle = {cell.alpha, cell.beta, cell.gamma};
const Eigen::Vector3d ref = Shape(reference);
std::array<int, 3> p = {0, 1, 2};
UnitCell best = cell;
double best_distance = std::numeric_limits<double>::infinity();
do {
const UnitCell c{edge[p[0]], edge[p[1]], edge[p[2]], angle[p[0]], angle[p[1]], angle[p[2]]};
const double distance = (Shape(c) - ref).squaredNorm();
if (distance < best_distance) {
best_distance = distance;
best = c;
}
} while (std::next_permutation(p.begin(), p.end()));
return best;
}
CellForms SeparateCellForms(const std::vector<UnitCell> &cells, const UnitCell &reference) {
CellForms out;
out.form.assign(cells.size(), 0);
const size_t n = cells.size();
if (n == 0)
return out;
std::vector<Eigen::Vector3d> f(n);
Eigen::Vector3d mean = Eigen::Vector3d::Zero();
for (size_t i = 0; i < n; i++) {
f[i] = Shape(AlignedCell(cells[i], reference));
mean += f[i];
}
mean /= static_cast<double>(n);
Eigen::Matrix3d cov = Eigen::Matrix3d::Zero();
for (const auto &x : f)
cov += (x - mean) * (x - mean).transpose();
// The direction the cells spread along most: between two forms, where there are two, as the spread
// between them adds to the spread of each.
const Eigen::SelfAdjointEigenSolver<Eigen::Matrix3d> eig(cov);
const Eigen::Vector3d axis = eig.eigenvectors().col(2);
out.coordinate.resize(n);
for (size_t i = 0; i < n; i++)
out.coordinate[i] = (f[i] - mean).dot(axis);
auto populations = CellPopulations(out.coordinate);
if (populations.size() < 2)
return out;
std::sort(populations.begin(), populations.end(),
[](const CellPopulationRange &x, const CellPopulationRange &y) { return x.count > y.count; });
const auto &major = populations[0], &minor = populations[1];
const auto in = [](const CellPopulationRange &p, double x) { return x >= p.low && x <= p.high; };
Eigen::Vector3d mean_major = Eigen::Vector3d::Zero(), mean_minor = mean_major;
std::vector<UnitCell> minor_cells;
for (size_t i = 0; i < n; i++) {
if (in(major, out.coordinate[i]))
mean_major += f[i];
else if (in(minor, out.coordinate[i])) {
mean_minor += f[i];
minor_cells.push_back(AlignedCell(cells[i], reference));
}
}
mean_major /= static_cast<double>(major.count);
mean_minor /= static_cast<double>(minor.count);
out.separation = (mean_minor - mean_major).norm();
if (out.separation < MIN_FORM_SEPARATION)
return out;
for (size_t i = 0; i < n; i++)
if (in(minor, out.coordinate[i]))
out.form[i] = 1;
out.minor_frames = minor_cells.size();
out.minor_cell = MeanUnitCell(minor_cells);
return out;
}
image_analysis/scale_merge/CellPopulation.h
+39
View File
@@ -3,9 +3,28 @@
#pragma once
#include <optional>
#include <utility>
#include <vector>
#include "../../common/UnitCell.h"
// Two groups of frames whose cells differ by less than this (relative: in volume for a given cell, in the
// cell's shape for SeparateCellForms) are one crystal form - a cell drifting with dose, or refined a little
// differently in different orientations, falls into groups too.
constexpr double MIN_FORM_SEPARATION = 0.01;
// The populations of a histogram of one value per frame: each a peak out to the valley on either side that
// separates it from the next (an infinite bound where none does), holding at least a tenth of the frames.
// The bin follows the spread of the data, so neither the width of a population nor its distance to the next
// is a fixed number. None for too few frames to tell.
struct CellPopulationRange {
double low, high;
size_t count;
double mean;
};
std::vector<CellPopulationRange> CellPopulations(const std::vector<double> &relative_volume);
// Serial stills given a cell: which frames belong to that crystal. A sample can hold two crystal forms
// (a polymorph, a different hydration state), and a known-cell indexer fits both to the given cell's
// orientation search - the refined cells then fall into two volume populations, and merging both mixes
@@ -17,3 +36,23 @@
//
// relative_volume: v / v_given - 1 per indexed frame. Returns the [low, high] range of it to keep.
std::pair<double, double> GivenCellPopulation(const std::vector<double> &relative_volume);
// Serial stills with the cell found by the run: whether the frames merged on its cell are two crystal forms.
// A sample can hold two forms whose cells are within the tolerance that puts a frame on the run's cell, and
// merging both mixes two structures. Each frame's cell is put on the axes of `reference` (AlignedCell) and
// described by its edges, as logarithms, which are projected on the direction they spread along most, where
// two forms lie apart; the populations of that coordinate
// (CellPopulations) are the forms. The largest is the major form; the next, where its mean shape is
// MIN_FORM_SEPARATION away, the minor one.
struct CellForms {
std::vector<int> form; // per cell: 1 in the minor form, 0 otherwise
std::vector<double> coordinate; // per cell, along the direction of the largest spread
size_t minor_frames = 0;
std::optional<UnitCell> minor_cell; // its mean, on the reference's axes
double separation = 0.0; // between the two forms' mean shapes
};
CellForms SeparateCellForms(const std::vector<UnitCell> &cells, const UnitCell &reference);
// A cell with its axes (each with the angle opposite it) in the order whose edges are nearest `reference`'s:
// a frame indexed with two near-equal axes the other way round, described as the reference is.
UnitCell AlignedCell(const UnitCell &cell, const UnitCell &reference);
image_analysis/scale_merge/ErrorModel.cpp
+186
View File
@@ -8,6 +8,8 @@
#include <functional>
#include <future>
#include <Eigen/Dense>
#include "../../common/ParallelFor.h"
#include "../../common/ThreadAffinity.h"
@@ -122,6 +124,7 @@ bool ErrorModelUpdate(const std::vector<double> &bs2, const std::vector<double>
a_new = std::clamp(Bs / Ass, 0.25, 100.0);
b2_new = 0.0;
fit.b_resolved = false;
fit.a_at_bound = a_new != Bs / Ass;
} else {
if (!(det > 1e-10 * Ass * AII)) return false;
const double a_fit = (Bs * AII - BI * AsI) / det;
@@ -143,6 +146,7 @@ bool ErrorModelUpdate(const std::vector<double> &bs2, const std::vector<double>
chi += r * r;
}
fit.b_resolved = a_new == a_fit && b2_fit >= 2.0 * std::sqrt(chi / (n_bins - 2) * Ass / det);
fit.a_at_bound = a_new != a_fit;
}
const bool settled = std::fabs(a_new - a) <= 1e-4 * a && std::fabs(b2_new - b2) <= 1e-4 * b2;
a = a_new;
@@ -152,3 +156,185 @@ bool ErrorModelUpdate(const std::vector<double> &bs2, const std::vector<double>
fit.b2 = b2;
return !settled;
}
namespace {
// SplitMix64 as a hash of (replicate, group) to draw from.
// Following Steele, Lea & Flood (2014) OOPSLA 2014, 453-472
uint64_t SplitMix64(uint64_t x) {
x += 0x9E3779B97F4A7C15ull;
x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ull;
x = (x ^ (x >> 27)) * 0x94D049BB133111EBull;
return x ^ (x >> 31);
}
// A Poisson count by inversion of its CDF from a uniform in [0, 1), p0 = exp(-lambda). The cap stops a
// u that the rounded CDF never passes; for lambda <= 1, past 20 the probability is below 1e-19.
int Poisson(double lambda, double p0, double u) {
double p = p0, cdf = p0;
int k = 0;
while (u >= cdf && k < 20) {
++k;
p *= lambda / k;
cdf += p;
}
return k;
}
}
std::optional<ErrorModelB2Interval> BootstrapErrorModelB2(const std::vector<ErrorModelSample> &pool, double b2,
int replicates, size_t max_samples, size_t nthreads) {
// The bootstrap over groups with Poisson weights in place of a multinomial draw: Efron (1979) Ann.
// Statist. 7, 1-26. A pool larger than max_samples is resampled thinned, each group drawn
// Poisson(lambda) times, and the replicates' spread about b2 shrunk by sqrt(lambda) to the size of the
// pool - the m-out-of-n bootstrap, for a statistic whose error goes as one over the root of its count.
const double lambda = std::min(1.0, static_cast<double>(max_samples) / static_cast<double>(pool.size()));
const double p0 = std::exp(-lambda);
// One replicate per task, each fitted on one thread: every replicate's draws and fit are its own, so
// the interval is the same on any number of threads.
std::vector<double> b2_r(replicates, NAN);
ParallelChunks(replicates, nthreads, [&](int lo, int hi) {
std::vector<ErrorModelSample> replicate;
std::vector<ErrorModelBinned> scratch;
for (int r = lo; r < hi; ++r) {
replicate.clear();
for (const auto &s : pool) {
const uint64_t h = SplitMix64((static_cast<uint64_t>(r) << 32) | static_cast<uint32_t>(s.group));
const int copies = Poisson(lambda, p0, static_cast<double>(h >> 11) * 0x1.0p-53);
for (int c = 0; c < copies; ++c) replicate.push_back(s);
}
const auto fit = FitErrorModel(replicate, scratch, 1);
if (fit.active) b2_r[r] = fit.b2;
}
});
std::erase_if(b2_r, [](double v) { return std::isnan(v); });
if (b2_r.empty())
return std::nullopt;
std::sort(b2_r.begin(), b2_r.end());
const double last = static_cast<double>(b2_r.size() - 1);
const double q_low = b2_r[static_cast<size_t>(std::floor(0.025 * last))];
const double q_high = b2_r[static_cast<size_t>(std::ceil(0.975 * last))];
// A replicate's b^2 held at zero is b reaching zero, which no shrinking moves away from it.
const double shrink = std::sqrt(lambda);
return ErrorModelB2Interval{q_low > 0.0 ? std::max(0.0, b2 + shrink * (q_low - b2)) : 0.0,
std::max(0.0, b2 + shrink * (q_high - b2))};
}
StillsErrorModelFit FitStillsErrorModel(const std::vector<StillsErrorModelSample> &pool, size_t nthreads,
const StillsErrorModelFit &start) {
constexpr int n_bins = ERROR_MODEL_BINS;
constexpr int n_sub = 4;
StillsErrorModelFit fit;
if (pool.size() < static_cast<size_t>(8 * n_bins * n_sub))
return fit;
// Bins of counting (I/sigma)^2, each split by the partiality term's share of the intensity term. The
// boundaries are put in place with nth_element on (key, position in the pool) - a total order, so which
// sample lands in which bin depends on the samples and the order the caller gave them in alone. The keys
// are computed once and carried in the comparison, never recomputed inside it.
struct Key { double v; size_t i; };
const auto by_key = [](const Key &p, const Key &q) { return p.v < q.v || (p.v == q.v && p.i < q.i); };
std::vector<Key> key(pool.size());
ParallelChunks(static_cast<int>(pool.size()), ThreadsForWork(pool.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
key[i] = {pool[i].I2 / pool[i].s2, static_cast<size_t>(i)};
});
const size_t per = pool.size() / n_bins;
// The split runs down the middle, and the two halves of a split are disjoint ranges, so they run side
// by side while the range is still worth a thread (as in FitErrorModel).
constexpr size_t SPLIT_MIN_PARALLEL = 1u << 17;
const std::function<void(size_t, size_t, int, int, int)> split = [&](size_t lo, size_t hi, int b0, int b1, int depth) {
if (b0 >= b1) return;
const int mid = (b0 + b1) / 2;
const size_t k = static_cast<size_t>(mid) * per;
std::nth_element(key.begin() + lo, key.begin() + k, key.begin() + hi, by_key);
if (depth > 0 && hi - lo > SPLIT_MIN_PARALLEL) {
auto left = std::async(std::launch::async, [&, lo, k, b0, mid, depth] {
RestoreThreadAffinity();
split(lo, k, b0, mid, depth - 1);
});
split(k, hi, mid + 1, b1, depth - 1);
left.get();
} else {
split(lo, k, b0, mid, 0);
split(k, hi, mid + 1, b1, 0);
}
};
split(0, key.size(), 1, n_bins, nthreads > 1 ? 3 : 0);
// b only where the strongest bin reaches past the counting term, as in FitErrorModel.
std::vector<double> top;
for (size_t j = (n_bins - 1) * per; j < key.size(); ++j)
top.push_back(key[j].v);
std::nth_element(top.begin(), top.begin() + top.size() / 2, top.end());
const bool b_measured = top[top.size() / 2] > ERROR_MODEL_B_LEVER_MIN;
// Each bin in four by the share, and the samples copied out in that order.
std::vector<std::pair<size_t, size_t>> groups(n_bins * n_sub);
std::vector<StillsErrorModelSample> x(pool.size());
ParallelFor(n_bins, nthreads, [&](int bin) {
const size_t lo = bin * per, hi = bin == n_bins - 1 ? key.size() : (bin + 1) * per;
for (size_t j = lo; j < hi; ++j) {
const auto &o = pool[key[j].i];
key[j].v = o.I2 > 0.0 ? o.P2 / o.I2 : 0.0;
}
const size_t sub = (hi - lo) / n_sub;
for (int q = 1; q < n_sub; ++q)
std::nth_element(key.begin() + lo + (q - 1) * sub, key.begin() + lo + q * sub, key.begin() + hi, by_key);
for (int q = 0; q < n_sub; ++q)
groups[bin * n_sub + q] = {lo + q * sub, q == n_sub - 1 ? hi : lo + (q + 1) * sub};
for (size_t j = lo; j < hi; ++j)
x[j] = pool[key[j].i];
});
key = {};
const int ng = static_cast<int>(groups.size());
std::vector<double> z2(pool.size());
Eigen::MatrixXd X(ng, 3);
Eigen::VectorXd y(ng);
double a = start.a, b2 = start.b2, c2 = start.c2;
for (int iter = 0; iter < 30; ++iter) {
// Per group: the three terms divided by the current variance, and the median normalised deviation
// expressed as the sum it stands for.
ParallelFor(ng, ThreadsForWork(pool.size(), nthreads), [&](int g) {
const auto [lo, hi] = groups[g];
double s = 0.0, I = 0.0, P = 0.0;
for (size_t i = lo; i < hi; ++i) {
const double v = a * x[i].s2 + b2 * x[i].I2 + c2 * x[i].P2;
s += x[i].s2 / v; I += x[i].I2 / v; P += x[i].P2 / v;
z2[i] = x[i].dev2 / v;
}
const auto mid = z2.begin() + lo + (hi - lo) / 2;
std::nth_element(z2.begin() + lo, mid, z2.begin() + hi);
X.row(g) << s, I, P;
y(g) = *mid / ERROR_MODEL_CHI2_1_MEDIAN * static_cast<double>(hi - lo);
});
// Every group weighs the same in relative terms; a negative term is dropped and the rest refitted.
const Eigen::VectorXd w = y.array().square().inverse();
std::vector<int> terms = b_measured ? std::vector<int>{0, 1, 2} : std::vector<int>{0, 2};
Eigen::VectorXd sol = Eigen::VectorXd::Zero(3);
while (true) {
Eigen::MatrixXd Xs(ng, terms.size());
for (size_t j = 0; j < terms.size(); ++j)
Xs.col(j) = X.col(terms[j]);
const Eigen::MatrixXd A = Xs.transpose() * w.asDiagonal() * Xs;
const Eigen::VectorXd s = A.ldlt().solve(Xs.transpose() * w.asDiagonal() * y);
sol.setZero();
for (size_t j = 0; j < terms.size(); ++j)
sol(terms[j]) = s(j);
const auto neg = std::find_if(terms.begin() + 1, terms.end(), [&](int t) { return sol(t) < 0.0; });
if (neg == terms.end())
break;
terms.erase(neg);
}
if (!std::isfinite(sol(0)) || !std::isfinite(sol(1)) || !std::isfinite(sol(2)))
break;
const double a_new = std::clamp(sol(0), 0.25, 100.0), b2_new = sol(1), c2_new = sol(2);
const bool settled = std::fabs(a_new - a) <= 1e-4 * a && std::fabs(b2_new - b2) <= 1e-4 * b2
&& std::fabs(c2_new - c2) <= 1e-4 * c2;
a = a_new; b2 = b2_new; c2 = c2_new;
fit.active = true;
fit.b_resolved = b_measured && b2 > 0.0 && a == sol(0);
if (settled)
break;
}
fit.a = a; fit.b2 = b2; fit.c2 = c2;
return fit;
}
image_analysis/scale_merge/ErrorModel.h
+44 -1
View File
@@ -4,6 +4,8 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <vector>
// The error model of the rotation merge: var = a * s2 + b^2 * <I>^2, fitted so that the scatter of
@@ -29,6 +31,7 @@
struct ErrorModelSample {
double s2, I2, dev2;
float d;
int32_t group = 0; // the unique reflection it belongs to; only the bootstrap below reads it
};
struct ErrorModelFit {
@@ -36,7 +39,8 @@ struct ErrorModelFit {
double a = 1.0, b2 = 0.0;
bool b_measured = true; // false: no bin reached where b could be seen; b is held at 0
bool b_resolved = true; // b^2 stood clear of two of its standard errors, with a inside its bounds
// and b measured; the only state in which 1/b is reported as ISa
// and b measured
bool a_at_bound = false; // a stopped on its clamp, so the b^2 beside it is not the solution of any fit
};
// Scratch: the partitioned copy of the pool (reused between calls to save the allocation), with the rank
@@ -60,3 +64,42 @@ ErrorModelFit FitErrorModel(const std::vector<ErrorModelSample> &pool, std::vect
// the new (a, b^2) into a, b2 and fit. False when the fit stops, settled or no longer determined.
bool ErrorModelUpdate(const std::vector<double> &bs2, const std::vector<double> &bI2, const std::vector<double> &bd2,
double &a, double &b2, ErrorModelFit &fit);
// The 95% interval of b^2 (b2 being the pool's own fit) by a bootstrap over the reflections: each of the
// replicates refits the pool with every group drawn a Poisson number of times, its samples going or
// staying together, since symmetry mates are deviations from one shared mean. A pool over max_samples is
// resampled thinned to about that size, which bounds the cost. The draws are a hash of (replicate,
// group), so the interval does not depend on the threads or on the order of the pool's groups.
// Report-only: nothing fitted or merged reads it. Empty if no replicate gave a fit.
struct ErrorModelB2Interval {
double low, high;
};
std::optional<ErrorModelB2Interval> BootstrapErrorModelB2(const std::vector<ErrorModelSample> &pool, double b2,
int replicates, size_t max_samples, size_t nthreads);
// The error model of the stills merge: the rotation model plus a partiality term,
// var = a * s2 + b^2 * <I>^2 + c^2 * ((1 - p) / p)^2 * <I>^2,
// p the partiality the observation is divided by. A still records one sample of its rocking curve, and the
// partiality model's error, carried through the division by p, grows like 1/p in the tail; without the term
// the observations below p ~ 0.5 cannot be merged at all, with it they add to the merge. Measured on four
// serial and rotation-as-serial sets, a term growing like 1/p fits the scatter better than the one a
// Gaussian rocking curve propagates (|eps| sigma_eps / sigma^2, which grows only like sqrt(-ln p)) and gives
// the stronger anomalous map; the curve's real tail is heavier than its Gaussian core.
// Fitted as FitErrorModel fits the rotation model - equal-count bins of counting I/sigma, the median
// normalised deviation of each, every bin relative, iterated - with each bin split in four by
// ((1 - p) / p)^2, so that c is told apart from b by observations of one strength and different partiality.
struct StillsErrorModelSample {
double s2, I2, P2, dev2; // P2 = ((1 - p) / p)^2 * I2
};
struct StillsErrorModelFit {
bool active = false;
double a = 1.0, b2 = 0.0, c2 = 0.0;
bool b_resolved = false; // as ErrorModelFit::b_resolved
};
// start: where the iteration begins - identity by default, or the fit of a previous round on a pool that
// has moved little since.
StillsErrorModelFit FitStillsErrorModel(const std::vector<StillsErrorModelSample> &pool, size_t nthreads = 1,
const StillsErrorModelFit &start = {});
image_analysis/scale_merge/IceRingVerdict.cpp
+130
View File
@@ -0,0 +1,130 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "IceRingVerdict.h"
#include <cmath>
#include "../../common/Definitions.h"
#include "../../common/JFJochMath.h"
#include "../bragg_integration/CalcISigma.h"
namespace {
struct Moments {
int n = 0;
double sum = 0.0, sum2 = 0.0, sum3 = 0.0, sum4 = 0.0;
void Add(double t) { ++n; sum += t; sum2 += t * t; sum3 += t * t * t; sum4 += t * t * t * t; }
[[nodiscard]] double Mean() const { return sum / n; }
[[nodiscard]] double Var() const { return (sum2 - sum * sum / n) / (n - 1); }
// The variance of ln(Var()): (kurtosis - 1) / n. Wilson intensities are far from Gaussian (an
// acentric E^2 has kurtosis 9, not 3), so the Gaussian 2 / n would call ordinary scatter excess.
[[nodiscard]] double VarLogVar() const {
const double m = Mean();
const double m2 = sum2 / n - m * m;
const double m4 = sum4 / n - 4.0 * m * sum3 / n + 6.0 * m * m * sum2 / n - 3.0 * m * m * m * m;
return (m4 / (m2 * m2) - 1.0) / n;
}
};
double OneOverD2(float d) { return 1.0 / (static_cast<double>(d) * d); }
}
const char *IceRingActionName(IceRingAction a) {
switch (a) {
case IceRingAction::DownWeight: return "down-weight";
case IceRingAction::Reject: return "reject";
default: return "merge";
}
}
IceRingDecision DecideIceRings(const std::vector<MergedReflection> &merged) {
IceRingDecision out;
out.weight_factor.assign(merged.size(), 1.0);
std::vector<MergedReflection> off_ring;
for (const auto &r : merged)
if (!r.on_ice_ring)
off_ring.push_back(r);
const GlobalWilsonB global = CalcGlobalWilsonB(off_ring);
// The ring each flagged reflection belongs to: the nearest in q.
std::vector<int> ring_of(merged.size(), -1);
std::vector<double> dq_of(merged.size(), 0.0);
for (size_t i = 0; i < merged.size(); ++i) {
if (!merged[i].on_ice_ring || !(merged[i].d > 0.0f)) continue;
const double q = 2.0 * PI / merged[i].d;
for (int k = 0; k < static_cast<int>(ICE_RING_RES_A.size()); ++k) {
const double dq = q - 2.0 * PI / ICE_RING_RES_A[k];
if (ring_of[i] < 0 || std::fabs(dq) < std::fabs(dq_of[i])) {
ring_of[i] = k;
dq_of[i] = dq;
}
}
}
for (int k = 0; k < static_cast<int>(ICE_RING_RES_A.size()); ++k) {
const double s2_ring = OneOverD2(ICE_RING_RES_A[k]);
bool any = false;
for (int r : ring_of) any |= r == k;
if (!any) continue;
std::vector<MergedReflection> window;
for (const auto &r : off_ring)
if (r.d > 0.0f && std::fabs(OneOverD2(r.d) - s2_ring) < ICE_VERDICT_WINDOW_S2)
window.push_back(r);
const GlobalWilsonB local = CalcGlobalWilsonB(window);
const GlobalWilsonB &wilson = local.n_shells >= 3 ? local : global;
auto t_of = [&](const MergedReflection &r) {
const double I_exp = std::exp(wilson.intercept + wilson.slope * OneOverD2(r.d));
return (r.I - I_exp) / std::sqrt(static_cast<double>(r.sigma) * r.sigma + I_exp * I_exp);
};
Moments nb;
if (wilson.n_shells >= 3)
for (const auto &r : window)
nb.Add(t_of(r));
Moments zone[3];
for (size_t i = 0; i < merged.size(); ++i)
if (ring_of[i] == k && nb.n > 0)
zone[dq_of[i] < -ICE_VERDICT_CORE_Q ? 0 : (dq_of[i] > ICE_VERDICT_CORE_Q ? 2 : 1)].Add(t_of(merged[i]));
IceRingZoneVerdict v[3];
for (int z = 0; z < 3; ++z) {
v[z].ring_d = ICE_RING_RES_A[k];
v[z].zone = z - 1;
v[z].n = zone[z].n;
v[z].n_neighbours = nb.n;
if (zone[z].n < ICE_VERDICT_MIN_ZONE || nb.n < ICE_VERDICT_MIN_NEIGHBOURS)
continue;
v[z].bias_z = (zone[z].Mean() - nb.Mean())
/ std::sqrt(zone[z].Var() / zone[z].n + nb.Var() / nb.n);
v[z].var_ratio = zone[z].Var() / nb.Var();
v[z].var_ratio_z = std::log(v[z].var_ratio) / std::sqrt(zone[z].VarLogVar() + nb.VarLogVar());
if (std::fabs(v[z].bias_z) > ICE_VERDICT_SIGMA)
v[z].action = IceRingAction::Reject;
else if (v[z].var_ratio_z > ICE_VERDICT_SIGMA)
v[z].action = IceRingAction::DownWeight;
}
for (size_t i = 0; i < merged.size(); ++i) {
if (ring_of[i] != k) continue;
const IceRingZoneVerdict &zv = v[dq_of[i] < -ICE_VERDICT_CORE_Q ? 0 : (dq_of[i] > ICE_VERDICT_CORE_Q ? 2 : 1)];
if (zv.action == IceRingAction::Reject)
out.weight_factor[i] = 0.0;
else if (zv.action == IceRingAction::DownWeight) {
// The band's excess variance of t, in the reflection's own units.
const auto &r = merged[i];
const double I_exp = std::exp(wilson.intercept + wilson.slope * OneOverD2(r.d));
const double var = static_cast<double>(r.sigma) * r.sigma;
const double excess = (zone[zv.zone + 1].Var() - nb.Var()) * (var + I_exp * I_exp);
out.weight_factor[i] = var / (var + excess);
}
}
for (const auto &zv : v)
if (zv.n > 0)
out.zones.push_back(zv);
}
return out;
}
image_analysis/scale_merge/IceRingVerdict.h
+57
View File
@@ -0,0 +1,57 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <vector>
#include "../../common/Reflection.h"
// What to do with the merged reflections that sit on an ice ring, decided ring by ring from the data.
//
// A powder ring adds its own intensity to the reflections on it, and to the background ring of the
// reflections beside it, so a ring band can be biased (up on the ring, down on its flanks) or merely
// noisier than the crystal's own reflections - or untouched, where the ring is weak. Neither CC1/2 nor
// a scatter test sees a bias: the ring is the same in both half-sets. So each band is compared with the
// off-ring reflections around it in resolution, through the Wilson curve: the expected intensity comes
// from a log-linear Wilson fit (CalcGlobalWilsonB) over the off-ring reflections within
// ICE_VERDICT_WINDOW_S2 of the ring, or over all of them where that window cannot be fitted, and each
// reflection is read as t = (I - I_exp) / sqrt(sigma^2 + I_exp^2) - unit scatter whether the reflection
// is signal- or noise-dominated. Per band (the core of the ring and each flank):
// * the band's mean t differs from its neighbours' by more than ICE_VERDICT_SIGMA standard errors:
// biased - REJECT, since a larger sigma would not remove a bias;
// * otherwise its variance of t exceeds the neighbours' by more than ICE_VERDICT_SIGMA standard errors
// of the log ratio (taken from the samples' own kurtosis): noisier - DOWN-WEIGHT, the excess variance
// added to each reflection's own;
// * otherwise MERGE as it is.
// A band with too few reflections, or a ring with too few neighbours, to measure is merged.
enum class IceRingAction { Merge, DownWeight, Reject };
struct IceRingZoneVerdict {
float ring_d = 0.0f; // the ice ring (A)
int zone = 0; // -1 low-q flank, 0 core, +1 high-q flank
int n = 0; // merged reflections in the band
int n_neighbours = 0; // off-ring reflections they are compared with
double bias_z = 0.0; // (mean t band - mean t neighbours) / its standard error
double var_ratio = 1.0; // var t band / var t neighbours
double var_ratio_z = 0.0; // ln(var_ratio) / its standard error
IceRingAction action = IceRingAction::Merge;
};
struct IceRingDecision {
std::vector<IceRingZoneVerdict> zones;
// Per input reflection: the factor its weight (1/sigma^2) is multiplied by - 1 merged as it is,
// 0 rejected, between them down-weighted.
std::vector<double> weight_factor;
};
constexpr double ICE_VERDICT_SIGMA = 3.0;
constexpr double ICE_VERDICT_CORE_Q = 0.01; // |q - q_ring| below this is the core (A^-1)
constexpr double ICE_VERDICT_WINDOW_S2 = 0.02; // neighbours within this of the ring in 1/d^2 (A^-2)
constexpr int ICE_VERDICT_MIN_ZONE = 20;
constexpr int ICE_VERDICT_MIN_NEIGHBOURS = 50;
// `merged` needs I, sigma, d and on_ice_ring.
IceRingDecision DecideIceRings(const std::vector<MergedReflection> &merged);
const char *IceRingActionName(IceRingAction a);
image_analysis/scale_merge/Merge.cpp
+367 -296
View File
@@ -6,7 +6,11 @@
#include <algorithm>
#include <cmath>
#include <limits>
#include <map>
#include <span>
#include <tuple>
#include <unordered_map>
#include <unordered_set>
#include <spdlog/fmt/fmt.h>
@@ -20,6 +24,7 @@
#include "RfreeFlags.h"
#include "FrenchWilson.h"
#include "OutlierBand.h"
#include "ErrorModel.h"
namespace {
// Which half-set an observation prefers: a splitmix64 bit-mix of the image's stable index, so the
@@ -69,11 +74,11 @@ MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x)
? std::optional<double>(scaling_settings.GetMinCCForImage())
: std::nullopt),
min_partiality(scaling_settings.GetMinPartiality()),
existence_min_partiality(std::min(min_partiality, EXISTENCE_MIN_PARTIALITY)),
capture_coeff(scaling_settings.GetCaptureUncertaintyCoeff()),
generator(scaling_settings.GetMergeFriedel(), space_group),
reject_outliers(scaling_settings.GetOutlierRejectNsigma() > 0.0),
reject_nsigma(scaling_settings.GetOutlierRejectNsigma()),
stills(!x.IsRotationIndexing()) {
reject_nsigma(scaling_settings.GetOutlierRejectNsigma()) {
}
MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional<UnitCell> &cell) {
@@ -93,16 +98,14 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
continue;
if (exclude_ice_rings && r.on_ice_ring)
continue;
if (r.partiality < min_partiality)
if (BelowMergeFloor(r.partiality))
continue;
const float I_corr = r.I * r.image_scale_corr;
float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0)
continue;
auto hkl = generator(r);
auto hkl_key = hkl.pack();
sigma_corr = CorrectedSigma(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality, hkl_key);
const auto [I_corr, w] = Weigh(r, hkl_key);
if (w == 0.0)
continue;
// Robust outlier rejection: drop this observation if it sits outside the band about the
// reflection's median that the error model allows (OutlierBand.h). Needs the active error model
@@ -112,8 +115,7 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
const auto mit = reject_median_I.find(hkl_key);
if (mit != reject_median_I.end()) {
const double I_b = error_model_mean_I.at(hkl_key);
if (OutsideOutlierBand(I_corr, mit->second, reject_nsigma,
static_cast<double>(sigma_corr) * sigma_corr, I_b,
if (OutsideOutlierBand(I_corr, mit->second, reject_nsigma, 1.0 / w, I_b,
error_model_b * I_b, reject_up, reject_down)) {
++rejected;
continue;
@@ -121,7 +123,6 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
}
}
const float w = 1.0f / (sigma_corr * sigma_corr);
out.push_back(Contribution{
.key = hkl_key,
.h = hkl.plus ? hkl.h : -hkl.h,
@@ -130,12 +131,55 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
.w = w,
.wI = w * I_corr,
.d = r.d,
.on_ice_ring = r.on_ice_ring
.on_ice_ring = r.on_ice_ring,
.exists = r.partiality >= existence_min_partiality,
.far_tail = r.partiality < FAR_TAIL_PARTIALITY
});
}
return rejected;
}
// The tests of Contributions and Mask, each on its own rather than up to the first that fails.
std::vector<std::vector<ObservationVerdict>> MergeOnTheFly::Verdicts(const std::vector<IntegrationOutcome> &outcomes) const {
std::vector<std::vector<ObservationVerdict>> out(outcomes.size());
for (size_t i = 0; i < outcomes.size(); ++i) {
const uint32_t image_flags = Mask(outcomes[i]) ? OBS_IMAGE_MASKED : 0;
for (const auto &r: outcomes[i].reflections) {
ObservationVerdict v{.flags = image_flags};
if (generator.IsSystematicallyAbsent(r))
v.flags |= OBS_ABSENT;
const bool scaled = r.image_scale_corr > 0.0 && std::isfinite(r.image_scale_corr);
if (!scaled)
v.flags |= OBS_NO_SCALE;
if (!AcceptReflection(r, high_resolution_limit, std::nullopt))
v.flags |= OBS_RESOLUTION;
if (exclude_ice_rings && r.on_ice_ring)
v.flags |= OBS_ICE_EXCLUDED;
if (BelowMergeFloor(r.partiality))
v.flags |= OBS_LOW_PARTIALITY;
const uint64_t hkl_key = generator(r).pack();
const auto [I_corr, w] = scaled ? Weigh(r, hkl_key) : Weighted{};
if (w == 0.0) {
if (scaled)
v.flags |= OBS_BAD_VALUE;
out[i].push_back(v);
continue;
}
v.sigma = static_cast<float>(std::sqrt(1.0 / w));
if (reject_outliers && error_model_active) {
const auto mit = reject_median_I.find(hkl_key);
const auto bit = error_model_mean_I.find(hkl_key);
if (mit != reject_median_I.end() && bit != error_model_mean_I.end()
&& OutsideOutlierBand(I_corr, mit->second, reject_nsigma, 1.0 / w,
bit->second, error_model_b * bit->second, reject_up, reject_down))
v.flags |= OBS_OUTLIER;
}
out[i].push_back(v);
}
}
return out;
}
void MergeOnTheFly::Accumulate(const std::vector<Contribution> &contributions, int64_t image_id) {
const int prefer = HalfForImage(image_id);
@@ -158,6 +202,13 @@ void MergeOnTheFly::Accumulate(const std::vector<Contribution> &contributions, i
it->second.n_half[half]++;
it->second.sum_wI_split[prefer] += c.wI;
it->second.sum_w_split[prefer] += c.w;
if (c.exists) {
it->second.exists = true;
it->second.exists_half[half] = true;
it->second.exists_split[prefer] = true;
}
if (c.far_tail)
++far_tail_count;
if (c.on_ice_ring)
it->second.on_ice_ring = true;
@@ -193,57 +244,65 @@ void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
});
}
float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const {
if (!error_model_active)
return sigma_corr;
// Intensity for the (b*I)^2 term: the reflection's mean (constant over its
// observations), falling back to this observation only if the mean is unknown.
const auto it = error_model_mean_I.find(hkl_key);
const double I_for_b = (it != error_model_mean_I.end()) ? it->second : I_corr;
// Base variance for the a*sigma^2 term. A weak observation's sigma^2 = a background/read part plus a
// Poisson signal part proportional to its OWN intensity; weighting the merge by 1/sigma^2 with that
// per-observation sigma biases the inverse-variance mean low at <1 photon (an up-fluctuated observation
// gets a larger sigma and is over-downweighted, so the weighted mean drifts below <I>). Rebuild the
// signal part at the reflection's EXPECTED intensity <I> instead - decompose out the background part
// and re-add corr*<I> - so the weight no longer correlates with the observation's own fluctuation. This
// mirrors the rotation combine in RotationScaleMerge::process_rawrun and is bit-identical when the
// observation sits at its reflection mean.
// The signal part removed here is the one the observation's own sigma actually carries, so it is
// taken at max(0, I): a negative intensity has no Poisson signal to subtract, and subtracting it
// anyway ADDS to the background part and inflates the weight of exactly the down-fluctuated
// observations this correction exists to stop being mistreated.
double a_var = static_cast<double>(sigma_corr) * sigma_corr;
double MergeOnTheFly::CountingVariance(double sigma_corr, float image_scale_corr, float var_bkg, float partiality,
double I_mean) const {
// A weak observation's sigma^2 = a background/read part plus a Poisson signal part proportional to its
// OWN intensity; weighting the merge by 1/sigma^2 with that per-observation sigma biases the
// inverse-variance mean low at <1 photon (an up-fluctuated observation gets a larger sigma and is
// over-downweighted, so the weighted mean drifts below <I>). Rebuild the signal part at the reflection's
// EXPECTED intensity <I> instead - the background part plus corr*<I> - so the weight no longer
// correlates with the observation's own fluctuation. This mirrors the rotation combine in
// RotationScaleMerge::process_rawrun. A negative mean has no Poisson signal, hence max(0, <I>).
double v = sigma_corr * sigma_corr;
if (scaling_settings.GetExpectedVarianceMerge()) {
const double bkg_var = static_cast<double>(image_scale_corr) * image_scale_corr * var_bkg;
const double base = bkg_var + static_cast<double>(image_scale_corr) * std::max(0.0, I_for_b);
const double base = static_cast<double>(image_scale_corr) * image_scale_corr * var_bkg
+ static_cast<double>(image_scale_corr) * std::max(0.0, I_mean);
if (base > 0.0)
a_var = base;
v = base;
}
const double capture = capture_coeff * (1.0 - std::min(1.0f, partiality)) * std::max(0.0, I_for_b);
a_var += capture * capture;
const double capture = capture_coeff * (1.0 - std::min(1.0f, partiality)) * std::max(0.0, I_mean);
return v + capture * capture;
}
const double v = error_model_a * a_var
+ (error_model_b * I_for_b) * (error_model_b * I_for_b);
return (v > 0.0) ? static_cast<float>(std::sqrt(v)) : sigma_corr;
double MergeOnTheFly::PartialityTerm(float partiality) {
const double q = (1.0 - std::min(1.0f, partiality)) / partiality;
return q * q;
}
double MergeOnTheFly::CorrectedVariance(double I_corr, double sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const {
if (!error_model_active)
return sigma_corr * sigma_corr;
// The systematic terms use the reflection's mean (constant over its observations), falling back to
// this observation only if the mean is unknown: using the per-observation I_i instead would
// over-weight down-fluctuated points.
const auto it = error_model_mean_I.find(hkl_key);
const double I_mean = (it != error_model_mean_I.end()) ? it->second : I_corr;
const double v = error_model_a * CountingVariance(sigma_corr, image_scale_corr, var_bkg, partiality, I_mean)
+ (error_model_b * error_model_b + error_model_c2 * PartialityTerm(partiality)) * I_mean * I_mean;
return (v > 0.0) ? v : sigma_corr * sigma_corr;
}
MergeOnTheFly::Weighted MergeOnTheFly::Weigh(const Reflection &r, uint64_t hkl_key) const {
const double I_corr = static_cast<double>(r.I) * r.image_scale_corr;
const double sigma_corr = static_cast<double>(r.sigma) * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0)
return {};
const double w = 1.0 / CorrectedVariance(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality, hkl_key);
if (!std::isfinite(w) || !(w > 0.0))
return {};
return {I_corr, w};
}
void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outcomes) {
// Reset to identity up front: every early return below then leaves the model
// inactive (CorrectedSigma returns sigma unchanged) rather than keeping a stale
// a/b from a previous call alongside a freshly-cleared mean map.
// Median of the chi-square(1) distribution: a single observation's squared deviation from its
// reflection mean, divided by its variance, is chi-square(1)-distributed, so its median is this
// fraction of its mean. Used both to de-bias the median-based variance fit and to normalize the
// reported median reduced chi^2 so that honestly calibrated sigmas give 1.0 (not 0.4549).
constexpr double CHI2_1_MEDIAN = 0.454936;
// Reset to identity up front: every early return below then leaves the model inactive (CorrectedVariance
// returns sigma unchanged) rather than keeping a stale model from a previous call.
error_model_active = false;
error_model_a = 1.0;
error_model_b = 0.0;
error_model_c2 = 0.0;
error_model_chi2 = 0.0;
error_model_b_resolved = true;
error_model_mean_I.clear();
@@ -252,12 +311,11 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
reject_down = 0.0;
reject_count = 0;
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl,
// applying exactly the filters AddImage uses. ---
struct Obs { float I, sigma, p; };
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, applying the filters
// AddImage uses, down to the existence partiality.
struct Obs { float I, sigma, corr, var_bkg, p; };
struct KeyedObs { uint64_t key; Obs obs; };
std::unordered_map<uint64_t, std::vector<Obs>> groups;
std::vector<KeyedObs> all; // in image order
ReduceImagesInOrder<KeyedObs>(
outcomes.size(), nthreads,
[&](size_t i, std::vector<KeyedObs> &out) {
@@ -272,190 +330,191 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
continue;
if (exclude_ice_rings && r.on_ice_ring)
continue;
if (r.partiality < min_partiality)
if (r.partiality < existence_min_partiality)
continue;
const float I_corr = r.I * r.image_scale_corr;
const float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
continue;
out.push_back({generator(r).pack(), {I_corr, sigma_corr, r.partiality}});
out.push_back({generator(r).pack(), {I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality}});
}
},
[&](size_t, const std::vector<KeyedObs> &out) {
for (const auto &o: out)
groups[o.key].push_back(o.obs);
all.insert(all.end(), out.begin(), out.end());
});
// --- 2. One global pool of (sigma^2, <I>^2, bias-corrected squared deviation). For an
// observation in a group of n, the residual from the inverse-variance mean has
// E[(I_i - <I>)^2] = sigma_i^2 (1 - h_i), h_i = w_i / sum_w (its leverage). The
// (b*I)^2 term uses the reflection mean, so the mean (not I_i) is the abscissa. ---
struct Sample { double s2, I2, dev2; };
std::vector<Sample> samples;
std::vector<std::pair<double, double>> log_spread; // |ln(I / median)|, weight
for (const auto &[key, obs]: groups) {
if (obs.size() < 2)
continue;
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
sum_w += w;
sum_wI += w * o.I;
// The observations of each reflection side by side, in key order and within a reflection in image
// order - a sort on (key, place), a total order, so the same on any number of workers. Every sum
// below runs over them in that fixed order.
struct KeyPlace { uint64_t key; size_t place; };
std::vector<KeyPlace> order(all.size());
const size_t obs_threads = ThreadsForWork(all.size(), nthreads);
ParallelChunks(static_cast<int>(all.size()), obs_threads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
order[i] = {all[i].key, static_cast<size_t>(i)};
});
ParallelSort(order.begin(), order.end(), nthreads, [](const KeyPlace &x, const KeyPlace &y) {
return x.key < y.key || (x.key == y.key && x.place < y.place);
});
std::vector<Obs> flat(all.size());
ParallelChunks(static_cast<int>(all.size()), obs_threads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
flat[i] = all[order[i].place].obs;
});
all = {};
std::vector<uint64_t> keys;
std::vector<size_t> start; // reflection k is flat[start[k] .. start[k + 1])
for (size_t i = 0; i < order.size(); ++i)
if (i == 0 || order[i].key != order[i - 1].key) {
keys.push_back(order[i].key);
start.push_back(i);
}
if (!(sum_w > 0.0))
continue;
const double mean = sum_wI / sum_w;
error_model_mean_I[key] = static_cast<float>(mean);
// Robust centre for outlier rejection: the median intensity (resists the very
// outliers the inverse-variance mean is being protected from). Only when active, and only
// where there are three observations for a median to stand on.
if (reject_outliers && obs.size() >= 3) {
std::vector<float> iv;
iv.reserve(obs.size());
for (const auto &o: obs)
iv.push_back(o.I);
std::nth_element(iv.begin(), iv.begin() + iv.size() / 2, iv.end());
reject_median_I[key] = iv[iv.size() / 2];
start.push_back(order.size());
order = {};
const auto group = [&](int k) { return std::span<const Obs>(flat.data() + start[k], start[k + 1] - start[k]); };
const int n_keys = static_cast<int>(keys.size());
const size_t key_threads = ThreadsForWork(keys.size(), nthreads, 1024);
// --- 2. The starting mean of each reflection (counting-sigma weights) and, where there are three
// observations for it to stand on, the median - the robust centre of the outlier test.
std::vector<double> mean(n_keys);
std::vector<float> median(n_keys, NAN);
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k) {
const auto obs = group(k);
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
sum_w += w;
sum_wI += w * o.I;
}
mean[k] = sum_wI / sum_w;
if (reject_outliers && obs.size() >= 3) {
std::vector<float> iv;
iv.reserve(obs.size());
for (const auto &o: obs)
iv.push_back(o.I);
std::nth_element(iv.begin(), iv.begin() + iv.size() / 2, iv.end());
median[k] = iv[iv.size() / 2];
}
}
const double I2 = mean * mean;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
const double factor = 1.0 - w / sum_w;
if (factor < 0.05)
continue;
const double resid = static_cast<double>(o.I) - mean;
const double capture = capture_coeff * (1.0 - std::min(1.0f, o.p)) * std::max(0.0, mean);
const double s2 = static_cast<double>(o.sigma) * o.sigma + capture * capture;
samples.push_back({s2, I2, resid * resid / factor});
const auto mit = reject_median_I.find(key);
if (mit != reject_median_I.end() && o.I > 0.0f && mit->second > 0.0f && mean > 0.0)
log_spread.emplace_back(std::fabs(std::log(o.I / mit->second)), I2 / s2);
}
}
});
for (int k = 0; k < n_keys; ++k)
if (!std::isnan(median[k]))
reject_median_I[keys[k]] = median[k];
// The spread of ln(I / median) the band's multiplicative part is drawn from: its weighted median
// absolute value, weighted by (<I>/sigma)^2 so the observations whose ratio counting noise does not
// blur carry it - as the rotation merge measures it.
if (!log_spread.empty()) {
std::sort(log_spread.begin(), log_spread.end());
double total = 0.0;
for (const auto &[v, w]: log_spread) total += w;
double run = 0.0;
size_t j = 0;
for (; j + 1 < log_spread.size(); ++j) {
run += log_spread[j].second;
if (run >= 0.5 * total) break;
}
const double s = 1.4826 * log_spread[j].first;
reject_up = std::exp(reject_nsigma * s) - 1.0;
reject_down = 1.0 - std::exp(-reject_nsigma * s);
}
// --- 3. Fit global dev2 = a*sigma^2 + b^2*<I>^2. Bin by intensity (the per-observation
// dev2 is chi-square-1 noisy) and take medians; weight the bins by 1/dev2^2 so it
// is a *relative* fit - otherwise the strong bins (which fix b) swamp the weak
// bins (which fix a) and the weak sigmas stay over-confident. ---
constexpr int n_bins = 16;
if (samples.size() < static_cast<size_t>(8 * n_bins))
return; // too little multiplicity to fit -> leave identity
std::sort(samples.begin(), samples.end(),
[](const Sample &p, const Sample &q) { return p.I2 < q.I2; });
auto median = [](std::vector<double> &v) {
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
return v[v.size() / 2];
// --- 3. Fit the model (ErrorModel.h, FitStillsErrorModel) on the scatter about the reflection means,
// the counting variance rebuilt at the mean, and refit with the means the fitted model weights:
// the means and the model each depend on the other, and a single pass leaves the weights of
// the strong reflections at the counting-sigma means the fit started from. Measured on serial
// and rotation-as-serial sets, a few rounds raise the anomalous signal of the merge; they have
// settled after four, most often already after two or three: a round that leaves the model where
// the previous one put it (to 1e-3) would only reproduce the same means, and ends the iteration.
// Those rounds only carry the means to where the model settles, so they fit every stride-th
// observation (some four million of them); one last round fits them all and makes the final means.
// Each observation's sample goes to its own slot, so the pool is in key order whatever the number
// of workers; the slots left empty (single observations, the leverage cut) are dropped after.
constexpr int ROUNDS = 4;
const size_t stride = std::max<size_t>(1, flat.size() >> 22);
bool last_round = false;
std::vector<StillsErrorModelSample> samples;
StillsErrorModelFit fit;
double a = 1.0, b2 = 0.0, c2 = 0.0;
auto variance = [&](const Obs &o, double m) {
return a * CountingVariance(o.sigma, o.corr, o.var_bkg, o.p, m) + (b2 + c2 * PartialityTerm(o.p)) * m * m;
};
// Per-intensity-bin medians of (sigma^2, <I>^2, dev2).
std::vector<double> bs2, bI2, bd2;
bs2.reserve(n_bins); bI2.reserve(n_bins); bd2.reserve(n_bins);
const size_t per = samples.size() / n_bins;
for (int bin = 0; bin < n_bins; ++bin) {
const size_t lo = bin * per;
const size_t hi = (bin == n_bins - 1) ? samples.size() : lo + per;
std::vector<double> vs2, vI2, vd2;
vs2.reserve(hi - lo); vI2.reserve(hi - lo); vd2.reserve(hi - lo);
for (size_t i = lo; i < hi; ++i) {
vs2.push_back(samples[i].s2);
vI2.push_back(samples[i].I2);
vd2.push_back(samples[i].dev2);
}
bs2.push_back(median(vs2));
bI2.push_back(median(vI2));
// The per-observation dev2 is sigma^2 * chi-square(1)-distributed, whose MEDIAN is 0.4549 of
// its mean. Fitting the model to the robust median would therefore calibrate the variances to
// 0.4549x their true value (reduced chi^2 ~ 1/0.4549 = 2.2). Divide the median by that constant
// to recover an unbiased estimate of the mean (E[dev2] = sigma^2), keeping the robustness of
// the median while targeting reduced chi^2 = 1.
bd2.push_back(median(vd2) / CHI2_1_MEDIAN);
for (int round = 0; ; ++round) {
const size_t step = last_round ? 1 : stride;
samples.assign((flat.size() + step - 1) / step, StillsErrorModelSample{NAN, NAN, NAN, NAN});
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
std::vector<double> var;
for (int k = lo; k < hi; ++k) {
const auto obs = group(k);
if (obs.size() < 2)
continue;
const double m = mean[k];
// The residual from the weighted mean has E[(I_i - <I>)^2] = var_i (1 - h_i), h_i = w_i / sum_w
// its leverage, with the weights of the model that made the mean.
var.resize(obs.size());
double sum_w = 0.0;
for (size_t j = 0; j < obs.size(); ++j) {
var[j] = variance(obs[j], m);
sum_w += 1.0 / var[j];
}
for (size_t j = 0; j < obs.size(); ++j) {
if ((start[k] + j) % step != 0)
continue;
const auto &o = obs[j];
const double factor = 1.0 - 1.0 / (var[j] * sum_w);
if (factor < 0.05)
continue;
const double resid = static_cast<double>(o.I) - m;
samples[(start[k] + j) / step] = {CountingVariance(o.sigma, o.corr, o.var_bkg, o.p, m), m * m,
PartialityTerm(o.p) * m * m, resid * resid / factor};
}
}
});
samples.erase(std::remove_if(samples.begin(), samples.end(),
[](const StillsErrorModelSample &x) { return std::isnan(x.dev2); }),
samples.end());
const StillsErrorModelFit last = fit;
fit = FitStillsErrorModel(samples, nthreads, last); // each round starts from the last one's model
if (!fit.active)
return; // too little multiplicity to fit -> leave identity
a = fit.a; b2 = fit.b2; c2 = fit.c2;
const auto close = [](double x, double y) { return std::fabs(x - y) <= 1e-3 * std::fabs(y); };
const bool settled = round > 0 && close(a, last.a) && close(b2, last.b2) && close(c2, last.c2);
// The new means leave out what the merge's outlier test will leave out (the band of OutlierBand.h
// about the median, as Contributions draws it): the systematic terms of every variance scale with the
// mean, so a mean that took in the outliers - observations of the innermost reflections that carry a
// beam-stop flare the frame's scale does not, read at 10-30x the median on weak frames - widens the
// band by as much as it is off, keeps the outliers it should drop, and weights them like the rest.
const double band = reject_nsigma * std::sqrt(b2);
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k) {
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: group(k)) {
const double v = variance(o, mean[k]);
if (reject_outliers && !std::isnan(median[k])
&& OutsideOutlierBand(o.I, median[k], reject_nsigma, v, mean[k], std::sqrt(b2) * mean[k],
band, band))
continue;
const double w = 1.0 / v;
sum_w += w;
sum_wI += w * o.I;
}
mean[k] = sum_wI / sum_w;
}
});
if (last_round || ((settled || round + 1 == ROUNDS) && step == 1))
break;
last_round = settled || round + 1 == ROUNDS;
}
// Relative-weighted (1/dev2^2) least squares for (a, b^2). Floor the weight's dev2 at a
// small fraction of the typical bin dev2: an absolute floor (1e-30) does not stop a
// near-zero-scatter bin from acquiring a runaway weight and hijacking the fit, so the
// floor must scale with the data. The regression target keeps the unfloored dev2.
std::vector<double> bd2_sorted = bd2;
const double dev2_floor = std::max(1e-30, 1e-3 * median(bd2_sorted));
double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0;
for (int bin = 0; bin < n_bins; ++bin) {
const double s2 = bs2[bin], I2 = bI2[bin], d2 = bd2[bin];
const double d2w = std::max(d2, dev2_floor);
const double wgt = 1.0 / (d2w * d2w);
Ass += wgt * s2 * s2;
AsI += wgt * s2 * I2;
AII += wgt * I2 * I2;
Bs += wgt * s2 * d2;
BI += wgt * I2 * d2;
}
// Reject a near-collinear (ill-conditioned) system *relatively*: det lies in
// [0, Ass*AII] by Cauchy-Schwarz, so compare against that scale rather than 1e-30.
const double det = Ass * AII - AsI * AsI;
if (!(det > 1e-10 * Ass * AII))
return;
const double a_fit = (Bs * AII - BI * AsI) / det;
const double b2_fit = (Ass * BI - AsI * Bs) / det;
const double a = std::clamp(a_fit, 0.25, 100.0);
const double b2 = std::max(b2_fit, 0.0);
// Whether 1/b can be reported as ISa, decided as the rotation fit decides it (ErrorModelUpdate):
// b^2 two of its standard errors clear of zero, the error from the bins' scatter about the line,
// and a inside its bounds - beside a clamped a the b^2 above is not the solution of any fit, and
// it is a near-zero-scatter bin that puts a on its bound (see the weight floor above).
double chi = 0.0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double r = (bd2[bin] - a_fit * bs2[bin] - b2_fit * bI2[bin]) / bd2[bin];
chi += r * r;
}
error_model_b_resolved = a == a_fit && b2_fit >= 2.0 * std::sqrt(chi / (n_bins - 2) * Ass / det);
for (int k = 0; k < n_keys; ++k)
error_model_mean_I[keys[k]] = static_cast<float>(mean[k]);
error_model_a = a;
error_model_b = std::sqrt(b2);
error_model_c2 = c2;
error_model_b_resolved = fit.b_resolved;
error_model_active = true;
// A still's observations of one reflection spread by their partiality and per-crystal scale, not by a
// multiplicative factor about the median: many read near zero, so the spread of ln(I / median) is
// wide and the band drawn from it lets through even a powder grain or a neighbour's spot at hundreds
// of sigma. The error model's b is that spread measured as it is, so stills test the plain band of
// n sigma about the median.
if (stills) {
reject_up = reject_nsigma * error_model_b;
reject_down = reject_nsigma * error_model_b;
}
// multiplicative factor about the median: many read near zero, so a band drawn from the spread of
// ln(I / median) would let through even a powder grain or a neighbour's spot at hundreds of sigma.
// The band is the plain one of n sigma about the median.
reject_up = reject_nsigma * error_model_b;
reject_down = reject_nsigma * error_model_b;
// Achieved goodness of fit: the median of the per-observation dev2/(a*sigma^2 + (b*<I>)^2). That
// ratio is chi-square(1)-distributed (median 0.4549) when the sigmas are correct, so normalize by
// CHI2_1_MEDIAN to report a median reduced chi^2 that targets 1.0. The median (not mean) keeps it
// robust to the heavy outlier tail of serial data.
// Achieved goodness of fit: the median of dev2 / var over the samples, which is chi^2(1)-distributed
// (median 0.4549) when the sigmas are right; normalised so that honest sigmas report 1.0.
std::vector<double> chi2;
chi2.reserve(samples.size());
for (const auto &s: samples) {
const double v = a * s.s2 + b2 * s.I2;
if (v > 0.0)
chi2.push_back(s.dev2 / v);
for (const auto &x: samples)
chi2.push_back(x.dev2 / (a * x.s2 + b2 * x.I2 + c2 * x.P2));
if (!chi2.empty()) {
std::nth_element(chi2.begin(), chi2.begin() + chi2.size() / 2, chi2.end());
error_model_chi2 = chi2[chi2.size() / 2] / ERROR_MODEL_CHI2_1_MEDIAN;
}
error_model_chi2 = chi2.empty() ? 0.0 : median(chi2) / CHI2_1_MEDIAN;
}
bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) const {
@@ -490,7 +549,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
out.reserve(accumulator.size());
for (const auto &[key, accum_ptr]: by_key) {
const MergeAccum &accum = *accum_ptr;
if (accum.sum_w <= 0.0)
if (accum.sum_w <= 0.0 || !accum.exists)
continue;
MergedReflection mr{
@@ -505,7 +564,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
.on_ice_ring = accum.on_ice_ring
};
if (accum.n_half[0] + accum.n_half[1] > 0 && accum.sum_w_half[0] > 0.0 && accum.sum_w_half[1] > 0.0) {
if (accum.exists_half[0] && accum.exists_half[1]) {
for (int i = 0; i < 2; ++i) {
mr.I_half[i] = static_cast<float>(accum.sum_wI_half[i] / accum.sum_w_half[i]);
mr.sigma_half[i] = static_cast<float>(1.0 / std::sqrt(accum.sum_w_half[i]));
@@ -513,7 +572,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
}
for (int i = 0; i < 2; ++i)
if (accum.sum_w_split[i] > 0.0)
if (accum.exists_split[i])
mr.I_split[i] = static_cast<float>(accum.sum_wI_split[i] / accum.sum_w_split[i]);
if (!std::isfinite(accum.d) || accum.d <= 0.0f)
@@ -554,6 +613,9 @@ struct ShellAccum {
CorrelationCoefficient cc_ref;
CorrelationCoefficient cc_split;
CorrelationCoefficient cc_anom;
double anom_abs_diff = 0.0, anom_sigma = 0.0;
int anom_pairs = 0;
double split_abs_diff = 0.0;
double split_sum = 0.0;
int split_pairs = 0;
@@ -775,19 +837,47 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
}
}
// Per-reflection mean <I>, and a per-reflection accumulator for R_meas - it needs |I_i - <I>|,
// so the observations are visited again now that the means are known.
std::unordered_map<uint64_t, float> merged_I;
merged_I.reserve(merged.size());
// Anomalous statistics where the merge keeps the Friedel hands apart: an I(-) is stored at the negated
// index of its I(+). CC_anom between the frame-whole half-sets (I_split), so the two differences come
// from different crystals; SigAno against the merged sigmas. Acentric reflections only.
if (!scaling_settings.GetMergeFriedel()) {
std::map<std::tuple<int, int, int>, const MergedReflection *> by_index;
for (const auto &m: merged)
by_index[{m.h, m.k, m.l}] = &m;
const auto &ops = space_group.operations();
for (const auto &[hkl, plus]: by_index) {
const auto it = by_index.find({-plus->h, -plus->k, -plus->l});
if (it == by_index.end() || hkl <= it->first) // each pair once
continue;
const MergedReflection &minus = *it->second;
if (ops.is_reflection_centric({plus->h, plus->k, plus->l}))
continue;
const auto shell = shells.GetShell(plus->d);
if (!shell.has_value() || *shell < 0 || *shell >= n_shells)
continue;
for (auto *a: {&acc[*shell], &split_overall}) {
if (std::isfinite(plus->I) && std::isfinite(minus.I) && plus->sigma > 0.0f && minus.sigma > 0.0f) {
a->anom_abs_diff += std::fabs(plus->I - minus.I);
a->anom_sigma += std::hypot(plus->sigma, minus.sigma);
a->anom_pairs++;
}
if (std::isfinite(plus->I_split[0]) && std::isfinite(plus->I_split[1])
&& std::isfinite(minus.I_split[0]) && std::isfinite(minus.I_split[1]))
a->cc_anom.Add(plus->I_split[0] - minus.I_split[0], plus->I_split[1] - minus.I_split[1]);
}
}
}
// The observations are counted again, now only those of the reflections that are reported. No
// R_merge / R_meas / R_pim: they are rotation statistics, and on partials scaled to full equivalents
// they read hundreds of percent and mean nothing. R_split (below) is the serial one.
std::unordered_set<uint64_t> reported;
reported.reserve(merged.size());
for (const auto &m: merged)
if (std::isfinite(m.I))
merged_I[generator(m).pack()] = m.I;
reported.insert(generator(m).pack());
struct RmeasObs { double sum_abs_dev = 0.0; double sum_I = 0.0; int n = 0; int shell = -1; };
std::unordered_map<uint64_t, RmeasObs> rmeas_obs;
rmeas_obs.reserve(merged.size());
struct ShellObs { uint64_t key; float I_corr; float merged_I; int shell; bool merged; };
struct ShellObs { int shell; bool merged; };
ReduceImagesInOrder<ShellObs>(
integration_outcome.size(), nthreads,
[&](size_t i, std::vector<ShellObs> &out) {
@@ -800,62 +890,33 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
continue;
if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A))
continue;
if (r.partiality < min_partiality)
if (BelowMergeFloor(r.partiality))
continue;
const float I_corr = r.I * r.image_scale_corr;
const float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
const auto key = generator(r).pack();
const auto [I_corr, w] = Weigh(r, key);
if (w == 0.0)
continue;
const auto shell = shells.GetShell(r.d);
if (!shell.has_value())
continue;
const int s = *shell;
if (s >= 0 && s < n_shells) {
const auto key = generator(r).pack();
const auto mit = merged_I.find(key);
out.push_back({key, I_corr, mit != merged_I.end() ? mit->second : NAN, s,
mit != merged_I.end()});
}
if (s >= 0 && s < n_shells)
out.push_back({s, reported.contains(key)});
}
},
[&](size_t, const std::vector<ShellObs> &out) {
for (const auto &o: out) {
acc[o.shell].total_obs++;
if (o.merged) {
auto &ra = rmeas_obs[o.key];
ra.sum_abs_dev += std::abs(static_cast<double>(o.I_corr) - o.merged_I);
ra.sum_I += o.I_corr;
ra.n++;
ra.shell = o.shell;
}
}
for (const auto &o: out)
if (o.merged) // the observations of a reflection that is not reported are not counted
acc[o.shell].total_obs++;
});
// R_meas per shell: sum over reflections of sqrt(n/(n-1)) * sum_i|I_i - <I>|, over sum of I_i.
// R_merge and R_pim beside it, with the factor 1 and sqrt(1/(n-1)) - R_pim following Weiss (2001)
// J. Appl. Cryst. 34, 130-135.
std::vector<double> rmeas_num(n_shells, 0.0), rmeas_den(n_shells, 0.0);
std::vector<double> rmerge_num(n_shells, 0.0), rpim_num(n_shells, 0.0);
double rmeas_num_all = 0.0, rmeas_den_all = 0.0, rmerge_num_all = 0.0, rpim_num_all = 0.0;
for (const auto &[key, ra]: rmeas_obs) {
if (ra.n < 2 || ra.shell < 0 || ra.shell >= n_shells)
continue;
const double factor = std::sqrt(static_cast<double>(ra.n) / (ra.n - 1));
const double pim_factor = std::sqrt(1.0 / (ra.n - 1));
rmeas_num[ra.shell] += factor * ra.sum_abs_dev;
rmerge_num[ra.shell] += ra.sum_abs_dev;
rpim_num[ra.shell] += pim_factor * ra.sum_abs_dev;
rmeas_den[ra.shell] += ra.sum_I;
rmeas_num_all += factor * ra.sum_abs_dev;
rmerge_num_all += ra.sum_abs_dev;
rpim_num_all += pim_factor * ra.sum_abs_dev;
rmeas_den_all += ra.sum_I;
}
// R_split as White et al. (2012) J. Appl. Cryst. 45, 335-341 define it (CrystFEL's compare_hkl).
auto set_split = [](const ShellAccum &a, MergeStatisticsShell &ss) {
if (a.anom_pairs > 0) {
ss.abs_diff_over_sigma_anomalous = a.anom_abs_diff / a.anom_sigma;
ss.cc_anom = a.cc_anom.GetCC();
}
ss.n_split_pairs = a.split_pairs;
if (a.split_pairs == 0)
return;
@@ -865,6 +926,7 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
};
MergeStatistics out;
out.stills = true;
out.shells.resize(n_shells);
out.n_observations_rejected = reject_count;
@@ -883,10 +945,8 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
: 0.0;
ss.cc_half = sa.cc_half.GetCC();
ss.cc_half_internal = ss.cc_half; // stills weight nothing
ss.cc_ref = sa.cc_ref.GetCC();
ss.r_meas = rmeas_den[s] > 0.0 ? rmeas_num[s] / rmeas_den[s] : NAN;
ss.r_merge = rmeas_den[s] > 0.0 ? rmerge_num[s] / rmeas_den[s] : NAN;
ss.r_pim = rmeas_den[s] > 0.0 ? rpim_num[s] / rmeas_den[s] : NAN;
set_split(sa, ss);
}
@@ -913,10 +973,8 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
overall.unique_reflections = all_unique;
overall.mean_i_over_sigma = n_i_over_sigma > 0 ? sum_i_over_sigma / n_i_over_sigma : 0.0;
overall.cc_half = cc_half_overall.GetCC();
overall.cc_half_internal = overall.cc_half;
overall.cc_ref = cc_ref_overall.GetCC();
overall.r_meas = rmeas_den_all > 0.0 ? rmeas_num_all / rmeas_den_all : NAN;
overall.r_merge = rmeas_den_all > 0.0 ? rmerge_num_all / rmeas_den_all : NAN;
overall.r_pim = rmeas_den_all > 0.0 ? rpim_num_all / rmeas_den_all : NAN;
set_split(split_overall, overall);
return out;
@@ -980,7 +1038,7 @@ namespace {
}
}
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
std::string ShellTableRow(const MergeStatisticsShell &in, bool r_factors) {
// NaN, not 0.0, where the denominator is empty: Measured() below turns that into a dash, and a
// printed 0.0% reads as a measured total failure rather than as a quantity nobody could form.
double completeness = in.possible_unique_reflections > 0
@@ -988,33 +1046,44 @@ std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
double multiplicity = in.unique_reflections > 0
? static_cast<double>(in.total_observations) / in.unique_reflections : NAN;
output << fmt::format("{:8d} {:8d} {:8d} {:>8s} {:>7s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
std::string out = fmt::format("{:8d} {:8d} {:8d} {:>8s} {:>7s} {:>8s} ",
in.total_observations,
in.unique_reflections,
in.possible_unique_reflections,
Measured(completeness, "%"),
Measured(multiplicity),
Measured(in.mean_i_over_sigma),
Measured(in.mean_i_over_sigma));
if (r_factors)
out += fmt::format("{:>8s} {:>8s} {:>8s} ",
Measured(in.r_merge*100.0, "%"),
Measured(in.r_meas*100.0, "%"),
Measured(in.r_pim*100.0, "%"),
Measured(in.r_pim*100.0, "%"));
out += fmt::format("{:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
Measured(in.cc_half*100.0, "%"),
Measured(CCStar(in.cc_half)*100.0, "%"),
Measured(in.cc_ref*100.0, "%"),
Measured(in.cc_anom*100.0, "%"),
Measured(in.abs_diff_over_sigma_anomalous, "", 2));
return output;
return out;
}
std::string ShellTableHeader() {
return fmt::format(" {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>7s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
"d_min", "N_obs", "N_uniq", "N_possib", "Compl", "Mult", "<I/sig>", "R_merge", "R_meas", "R_pim",
"CC1/2", "CC*", "CCref", "CCanom", "SigAno");
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
return output << ShellTableRow(in, true);
}
std::string ShellTableRule() {
return fmt::format(" {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->7s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s}",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
std::string ShellTableHeader(bool r_factors) {
return fmt::format(" {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>7s} {:>8s} ",
"d_min", "N_obs", "N_uniq", "N_possib", "Compl", "Mult", "<I/sig>")
+ (r_factors ? fmt::format("{:>8s} {:>8s} {:>8s} ", "R_merge", "R_meas", "R_pim") : std::string())
+ fmt::format("{:>8s} {:>8s} {:>8s} {:>8s} {:>8s}", "CC1/2", "CC*", "CCref", "CCanom", "SigAno");
}
std::string ShellTableRule(bool r_factors) {
const std::string dash8(8, '-');
std::string rule = fmt::format(" {0} {0} {0} {0} {0} {1} {0}", dash8, std::string(7, '-'));
for (int i = 0; i < (r_factors ? 8 : 5); ++i)
rule += " " + dash8;
return rule;
}
std::string SerialShellTable(const MergeStatistics &in) {
@@ -1038,20 +1107,22 @@ std::string SerialShellTable(const MergeStatistics &in) {
}
std::ostream &operator<<(std::ostream &output, const MergeStatistics &in) {
// A stills merge has no R_merge / R_meas / R_pim columns: see MergeOnTheFly::MergeStats.
const bool r_factors = !in.stills;
output << std::endl;
output << ShellTableHeader() << std::endl;
output << ShellTableRule() << std::endl;
output << ShellTableHeader(r_factors) << std::endl;
output << ShellTableRule(r_factors) << std::endl;
for (const auto &sh: in.shells) {
if (sh.unique_reflections == 0)
continue;
output << fmt::format(" {:8.2f} ", sh.d_min);
output << sh;
output << ShellTableRow(sh, r_factors);
output << std::endl;
}
output << ShellTableRule() << std::endl;
output << ShellTableRule(r_factors) << std::endl;
output << fmt::format(" {:>8s} ", "Overall");
output << in.overall;
output << ShellTableRow(in.overall, r_factors);
output << std::endl;
output << SerialShellTable(in);
if (std::isfinite(in.wilson_b) && in.wilson_b > 0.0)
image_analysis/scale_merge/Merge.h
+98 -17
View File
@@ -29,7 +29,16 @@ struct MergeStatisticsShell {
double mean_i_over_sigma = 0.0;
// The reported CC1/2: the plain Pearson correlation of the two half-set means (each mean itself
// 1/sigma^2-weighted over its observations), every reflection counted alike - the CC1/2 of XDS,
// Aimless and DIALS, and the one the mmCIF item pdbx_CC_half defines.
double cc_half = 0.0f;
// Internal, not reported: the CC1/2 the processing DECIDES on. On rotation each reflection's pair
// enters with its MergedReflection::cc_weight, so a pair measured only on frames the crystal barely
// diffracted on does not count as much as a well-measured one; equal to cc_half where every weight
// is 1 (a sweep without a weak stretch), and on stills, which weight nothing. The automatic
// resolution cut is fitted to this same weighted form (ComputeCCHalfLogisticCutoff).
double cc_half_internal = 0.0f;
double cc_ref = NAN;
// Redundancy-independent merging R-factor (Diederichs & Karplus 1997), computed over the
@@ -48,14 +57,15 @@ struct MergeStatisticsShell {
// Anomalous signal-to-noise (XDS "SigAno" / mmCIF pdbx_absDiff_over_sigma_anomalous):
// <|I(+)-I(-)|> / <sigma(I(+)-I(-))> over acentric reflections measured in both hands. NaN when
// there is no anomalous split (e.g. Friedel-merged with no mates, or the stills path).
// there is no anomalous split (e.g. Friedel-merged with no mates).
double abs_diff_over_sigma_anomalous = NAN;
// CCanom (AIMLESS "CCanom", phenix "cc_anom"): the correlation of the anomalous difference
// measured twice - I(+)-I(-) from one half of the observations against the same difference from
// the other half, over acentric reflections whose BOTH mates split into two non-empty halves.
// Unlike SigAno it is not a ratio against the error model, so a mis-estimated sigma cannot move
// it. NaN when no reflection has an anomalous split (the stills path), which is not the same
// it. On stills the halves are the frame-whole ones of R_split, so the two differences come from
// different crystals. NaN when no reflection has an anomalous split, which is not the same
// statement as zero - zero means measured and absent.
double cc_anom = NAN;
@@ -181,6 +191,10 @@ struct MergeStatistics {
std::vector<MergeStatisticsShell> shells;
MergeStatisticsShell overall;
AnomalousMergeStatistics per_hand;
// Made by the stills merge (MergeOnTheFly). Its error model is reported as its terms, not as an
// ISa: on stills 1/b is the ceiling of one still observation under the stills model, which is not
// the instrument and crystal error CORRECT.LP's ISa measures.
bool stills = false;
// Dataset-wide isotropic Wilson B-factor estimate (A^2) from the log-linear fit of the shell-mean
// merged intensity against 1/d^2 (CalcGlobalWilsonB) - the analogue of XDS's "WILSON LINE ... B=".
@@ -233,10 +247,12 @@ inline double CCStar(double cc_half) {
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in);
std::ostream &operator<<(std::ostream &output, const MergeStatistics &in);
// The header and the rule of the table operator<< prints, for a table with the same columns rendered
// elsewhere (the report's reference-range table).
std::string ShellTableHeader();
std::string ShellTableRule();
// The header, the rule and one row of the table operator<< prints, for a table with the same columns
// rendered elsewhere (the report's reference-range table). Without r_factors the R_merge / R_meas / R_pim
// columns are left out, as on a stills merge.
std::string ShellTableHeader(bool r_factors = true);
std::string ShellTableRule(bool r_factors = true);
std::string ShellTableRow(const MergeStatisticsShell &in, bool r_factors = true);
// The serial table of R_split and the frame-split CC1/2, per shell and overall; empty where they were
// not computed (rotation).
std::string SerialShellTable(const MergeStatistics &in);
@@ -247,6 +263,9 @@ struct MergeAccum {
int32_t l = 0;
float d = NAN;
bool on_ice_ring = false;
bool exists = false; // an observation reached the existence partiality (MergeOnTheFly), so it is reported
bool exists_half[2] = {false, false}; // the same within each half-set ...
bool exists_split[2] = {false, false}; // ... and each frame-whole half-set
double sum_wI = 0.0;
double sum_w = 0.0;
double sum_wI_half[2] = {0.0, 0.0};
@@ -278,6 +297,29 @@ inline XdsErrorModel ToXdsErrorModel(double a, double b) {
return {a, b * b / a, 1.0 / b};
}
// The stills partiality term c (MergeOnTheFly::ErrorModelC) in the same convention:
// sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2).
inline double ToXdsPartialityTerm(double a, double c) {
return a > 0.0 ? c * c / a : 0.0;
}
// Why the stills merge took or left one observation: a bit for every test it fails, 0 = merged
// (MergeOnTheFly::Verdicts; a diagnostic, --export-observations).
enum ObservationFlag : uint32_t {
OBS_IMAGE_MASKED = 1, // the image is not merged: cell outside the run's population, or --min-image-cc
OBS_ABSENT = 2, // systematic absence of the merge's space group
OBS_NO_SCALE = 4, // no usable scale correction (unscaled image, collapsed scale, p * G = 0)
OBS_RESOLUTION = 8, // outside the merge's resolution limits, or not finite
OBS_ICE_EXCLUDED = 16, // on an ice ring, in a merge that leaves them out
OBS_LOW_PARTIALITY = 32, // partiality below --min-partiality
OBS_BAD_VALUE = 64, // corrected I or sigma not finite, or sigma <= 0
OBS_OUTLIER = 128, // outside the outlier band about the reflection's median
};
struct ObservationVerdict {
uint32_t flags = 0;
float sigma = NAN; // the sigma the merge weights it with (error model applied); NaN if not computable
};
class MergeOnTheFly {
mutable std::mutex merged_mutex;
@@ -292,7 +334,23 @@ class MergeOnTheFly {
// Apply image_cc_limit in Mask(). One flag for the whole engine, not a per-call argument, so the
// merge, the error model and MergeStats can never disagree about which images are in.
bool filter_by_image_cc = false;
// Which observations are merged. A merge without a fitted error model - the references the scale and
// the partiality are refined against - takes those at min_partiality and above. A merge with one takes
// every observation: it carries the partiality model's own error in each variance (ErrorModel.h, the
// stills model's c term), which weights a low-partiality observation by what it is worth. A reflection
// (each hand where the Friedel mates are kept apart) is then reported only if at least one of its
// observations reaches existence_min_partiality, and has a mean in a half-set only if one of its observations
// in that half does: one seen only in the tails of its rocking curve rests on the partiality model alone,
// and on a small set such reflections wreck the outer shells. The error
// model is fitted on the observations at that partiality and above. Measured on serial and
// rotation-as-serial sets: every observation with the term raises the anomalous map at the sulfur sites
// over a floor at 0.2, and the existence rule keeps the smallest sets where the floor had them.
static constexpr double EXISTENCE_MIN_PARTIALITY = 0.2;
double min_partiality = 0.02;
double existence_min_partiality = 0.02;
[[nodiscard]] bool BelowMergeFloor(float partiality) const {
return error_model_active ? partiality <= 0.0f : partiality < min_partiality;
}
// The share of a partial the model could not see, (1 - p), carried in its variance as
// (capture_coeff * (1 - p) * <I>)^2 - the rotation combine's capture term, per observation.
double capture_coeff = 0.0;
@@ -307,23 +365,35 @@ class MergeOnTheFly {
std::unordered_map<uint64_t, MergeAccum> accumulator; // exported in key order
// Global error model (XDS form): sigma_corr^2 = a*sigma^2 + (b*<I>)^2. a rescales the
// (under-estimated) counting variance; the (b*<I>)^2 term adds the intensity-
// proportional systematic error that counting statistics miss, so strong reflections
// are no longer over-weighted. ISa = 1/b is the asymptotic I/sigma. Refined from the
// scatter of symmetry equivalents (RefineErrorModel); identity until then.
// Global error model (XDS form plus a partiality term, ErrorModel.h):
// sigma_corr^2 = a*sigma^2 + (b*<I>)^2 + c^2 ((1 - p) / p)^2 <I>^2. a rescales the (under-estimated)
// counting variance; the (b*<I>)^2 term adds the intensity-proportional systematic error that counting
// statistics miss, so strong reflections are no longer over-weighted; the c term is the partiality
// model's error. ISa = 1/b is the asymptotic I/sigma. Refined from the scatter of symmetry equivalents
// (RefineErrorModel); identity until then.
bool error_model_active = false;
double error_model_a = 1.0;
double error_model_b = 0.0;
double error_model_c2 = 0.0; // the partiality term, c^2 ((1 - p) / p)^2 <I>^2 (ErrorModel.h)
double error_model_chi2 = 0.0; // achieved median reduced chi^2 (~1.0 = honestly calibrated sigmas)
bool error_model_b_resolved = true; // as ErrorModelFit::b_resolved; 1/b is reported as ISa only then
// The (b*I)^2 term uses the reflection's *mean* intensity (constant over its
// observations), so it inflates sigma without biasing the inverse-variance weights -
// using the per-observation I_i instead would over-weight down-fluctuated points.
std::unordered_map<uint64_t, float> error_model_mean_I;
[[nodiscard]] float CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const;
// An observation's counting variance on the merge's scale, its signal part rebuilt at the reflection
// mean I_mean (plus the capture term), and the partiality term's factor ((1 - p) / p)^2.
[[nodiscard]] double CountingVariance(double sigma_corr, float image_scale_corr, float var_bkg, float partiality,
double I_mean) const;
static double PartialityTerm(float partiality);
[[nodiscard]] double CorrectedVariance(double I_corr, double sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const;
// An observation's intensity on the merge's scale and its weight 1/variance, in double: a partial far out on
// its rocking curve (p down to 1e-36) has a 1/p no float holds. weight = 0 for an observation the merge
// cannot use - a value not finite, or a variance so large that the weight is zero: it carries no information.
struct Weighted { double I = 0.0, weight = 0.0; };
[[nodiscard]] Weighted Weigh(const Reflection &r, uint64_t hkl_key) const;
// Optional per-observation outlier rejection: drop observations whose corrected
// intensity lies more than reject_nsigma error-model sigmas from the reflection's
@@ -334,10 +404,9 @@ class MergeOnTheFly {
double reject_nsigma = 6.0;
std::unordered_map<uint64_t, float> reject_median_I;
// The asymmetric band of OutlierBand.h: exp(+-n s) - 1 about the reflection mean, s the spread of
// ln(I / median) measured on this merge's observations; on stills n b, the linear band.
// ln(I / median); the stills merge sets both to n b, the linear band.
double reject_up = 0.0, reject_down = 0.0;
size_t reject_count = 0;
bool stills = false;
bool Mask(const IntegrationOutcome &outcome) const;
@@ -345,9 +414,13 @@ class MergeOnTheFly {
struct Contribution {
uint64_t key;
int32_t h, k, l; // the reflection's own index, as the accumulator records it
float w, wI, d;
double w, wI;
float d;
bool on_ice_ring;
bool exists; // partiality at the existence floor or above
bool far_tail; // partiality below FAR_TAIL_PARTIALITY
};
size_t far_tail_count = 0; // merged observations with a partiality below FAR_TAIL_PARTIALITY
// The observations of one image that reach the merge; returns how many the outlier test dropped.
size_t Contributions(const IntegrationOutcome &outcome, std::vector<Contribution> &out) const;
void Accumulate(const std::vector<Contribution> &contributions, int64_t image_id);
@@ -366,11 +439,15 @@ public:
[[nodiscard]] bool ErrorModelActive() const { return error_model_active; }
[[nodiscard]] double ErrorModelA() const { return error_model_a; }
[[nodiscard]] double ErrorModelB() const { return error_model_b; }
[[nodiscard]] double ErrorModelC() const { return std::sqrt(error_model_c2); }
[[nodiscard]] double ErrorModelChi2() const { return error_model_chi2; }
[[nodiscard]] bool ErrorModelBResolved() const { return error_model_b_resolved; }
// Outlier rejection (driven by ScalingSettings::GetOutlierRejectNsigma) reports its count.
[[nodiscard]] size_t RejectedCount() const { return reject_count; }
// Merged observations with a partiality below FAR_TAIL_PARTIALITY, counted for the log.
static constexpr float FAR_TAIL_PARTIALITY = 1e-3f;
[[nodiscard]] size_t FarTailCount() const { return far_tail_count; }
// image_id is the image's stable identity (its index in the outcomes vector). It seeds the CC1/2
// half-set PREFERENCE, which the balancing rule in AddImage then overrides where a reflection's
@@ -384,6 +461,10 @@ public:
// per-observation work spread over the workers and the accumulation kept in image order.
void AddImages(const std::vector<IntegrationOutcome> &outcomes);
// Every test AddImages applies, each evaluated on its own, for every reflection of every outcome (in
// their order); call after AddImages. A diagnostic: the merge itself does not use it.
std::vector<std::vector<ObservationVerdict>> Verdicts(const std::vector<IntegrationOutcome> &outcomes) const;
// d_min_override, when set, is the effective high-resolution limit for the shell table (used for
// the automatic resolution cutoff computed by the caller); otherwise the manual
// ScalingSettings high-resolution limit stands. The number of shells is ScalingSettings::ReportShellCount.
image_analysis/scale_merge/ReferenceFreeReindex.cpp
+37 -20
View File
@@ -146,6 +146,19 @@ ReferenceFreeReindexResult ResolveReindexReferenceFree(std::vector<IntegrationOu
const auto corrected = [](const Reflection &r) {
return static_cast<double>(r.I) * (r.prescaling_corr * r.qe_corr * r.flight_corr) / r.partiality;
};
// The crystals are compared on the square root of the intensity, its sign kept. Intensities follow
// the Wilson distribution, whose long tail lets a few very strong reflections dominate every product
// of two crystals: then the strongest crystals settle among themselves, and a group of them can hold
// one another in the wrong indexing however much the weaker majority disagrees. Amplitudes have no
// such tail.
const auto amplitude = [&](const Reflection &r) {
const double i = corrected(r);
return std::copysign(std::sqrt(std::fabs(i)), i);
};
const auto amplitude_sigma = [&](const Reflection &r) {
const double sigma = r.sigma * (r.prescaling_corr * r.qe_corr * r.flight_corr) / r.partiality;
return sigma / (2.0 * std::sqrt(std::fabs(corrected(r)) + sigma));
};
// What each image has to offer: how many reflections, how fine, and how far its indices reach in
// any of the indexings (which sizes the index table below).
@@ -199,33 +212,38 @@ ReferenceFreeReindexResult ResolveReindexReferenceFree(std::vector<IntegrationOu
return out;
const size_t nnz = begin.back();
// The mean intensity in shells of equal reciprocal volume over the whole run, so that a crystal's
// intensities are compared after the fall-off with resolution, which every indexing shares, is out.
// The mean and spread of the amplitudes in shells of equal reciprocal volume over the whole run, so
// that a crystal's amplitudes are compared after the fall-off with resolution, which every indexing
// shares, is out. Past the resolution the run reaches, a shell holds only noise and its mean may be
// zero or negative, so a shell is put on its mean and spread rather than divided by its mean.
const auto bin_of = [&](const Reflection &r) {
const double s3 = 1.0 / (static_cast<double>(r.d) * r.d * r.d);
return std::min(RESOLUTION_BINS - 1, static_cast<int>(RESOLUTION_BINS * s3 / s3_top));
};
std::vector<std::array<double, 2 * RESOLUTION_BINS>> bin_part(N);
std::vector<std::array<double, 3 * RESOLUTION_BINS>> bin_part(N);
ParallelFor(N, nthreads, [&](int c) {
auto &b = bin_part[c];
b.fill(0.0);
for (const auto &r : outcomes[crystal_outcome[c]].reflections)
if (accept(r)) {
b[bin_of(r)] += corrected(r);
b[RESOLUTION_BINS + bin_of(r)] += 1.0;
const double a = amplitude(r);
b[bin_of(r)] += 1.0;
b[RESOLUTION_BINS + bin_of(r)] += a;
b[2 * RESOLUTION_BINS + bin_of(r)] += a * a;
}
});
std::array<double, RESOLUTION_BINS> bin_mean{};
std::array<double, RESOLUTION_BINS> bin_mean{}, bin_sd{};
{
std::array<double, 2 * RESOLUTION_BINS> total{};
std::array<double, 3 * RESOLUTION_BINS> total{};
for (const auto &b : bin_part) // in crystal order: the same sum at any thread count
for (int i = 0; i < 2 * RESOLUTION_BINS; ++i)
for (int i = 0; i < 3 * RESOLUTION_BINS; ++i)
total[i] += b[i];
for (int i = 0; i < RESOLUTION_BINS; ++i)
bin_mean[i] = total[RESOLUTION_BINS + i] > 0 ? total[i] / total[RESOLUTION_BINS + i] : 1.0;
if (std::any_of(bin_mean.begin(), bin_mean.end(), [](double m) { return !(m > 0.0); }))
for (auto &m : bin_mean)
m = 1.0; // a shell with a non-positive mean: compare raw intensities instead
for (int i = 0; i < RESOLUTION_BINS; ++i) {
const double n = total[i];
bin_mean[i] = n > 0 ? total[RESOLUTION_BINS + i] / n : 0.0;
const double var = n > 0 ? total[2 * RESOLUTION_BINS + i] / n - bin_mean[i] * bin_mean[i] : 0.0;
bin_sd[i] = var > 0 ? std::sqrt(var) : 1.0;
}
}
// Every index any indexing can produce, mapped to a dense number of its reflection in the
@@ -253,9 +271,9 @@ ReferenceFreeReindexResult ResolveReindexReferenceFree(std::vector<IntegrationOu
}
table_key = {};
// Each crystal's intensities, divided by the shell mean and standardised within the crystal (the
// unknown scale and offset of an image drop out), and where each lands in every indexing. Each is
// weighted by how much of it is signal: a standardised intensity has unit variance, of which its
// Each crystal's amplitudes, put on the shell's mean and spread and standardised within the crystal
// (the unknown scale and offset of an image drop out), and where each lands in every indexing. Each is
// weighted by how much of it is signal: a standardised amplitude has unit variance, of which its
// measurement error takes (sigma / sd)^2, so a weak or barely recorded reflection counts for little.
std::vector<float> z(nnz);
std::vector<std::vector<int32_t>> key(K, std::vector<int32_t>(nnz));
@@ -266,10 +284,9 @@ ReferenceFreeReindexResult ResolveReindexReferenceFree(std::vector<IntegrationOu
for (const auto &r : outcomes[crystal_outcome[c]].reflections) {
if (!accept(r))
continue;
const double v = corrected(r) / bin_mean[bin_of(r)];
const double v = (amplitude(r) - bin_mean[bin_of(r)]) / bin_sd[bin_of(r)];
z[j] = static_cast<float>(v);
err.push_back(r.sigma * (r.prescaling_corr * r.qe_corr * r.flight_corr) / r.partiality
/ bin_mean[bin_of(r)]);
err.push_back(amplitude_sigma(r) / bin_sd[bin_of(r)]);
sum += v;
sum2 += v * v;
for (int g = 0; g < K; ++g) {
@@ -290,7 +307,7 @@ ReferenceFreeReindexResult ResolveReindexReferenceFree(std::vector<IntegrationOu
// Following Brehm & Diederichs (2014) Acta Cryst. D70, 101-109: each crystal in turn takes the
// indexing in which it agrees best with all the others, until none changes. Agreement is the sum
// over all pairs of crystals of the products of their standardised intensities on common
// over all pairs of crystals of the products of their standardised amplitudes on common
// reflections, which is the dot product of a crystal with the sum of all the others - so a pass over
// the crystals costs one pass over the reflections, not one per pair of crystals. The sum of all
// pairs only ever grows, so it ends; a few starts are made and the one that ends highest is kept.
image_analysis/scale_merge/ResolutionCutoff.cpp
+4 -2
View File
@@ -63,7 +63,7 @@ std::vector<double> CCHalfFrameFactors(const std::vector<double> &frame_scale,
}
ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedReflection> &merged,
double cc_target, Logger &logger) {
double cc_target, Logger &logger, bool weighted) {
ResolutionCutoffResult result;
if (!(cc_target > 0.0 && cc_target < 1.0)) {
result.note = "invalid CC target";
@@ -130,7 +130,9 @@ ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedRefle
const int b = bin_of(m);
const uint32_t hash = static_cast<uint32_t>(m.h) * 73856093u ^ static_cast<uint32_t>(m.k) * 19349663u
^ static_cast<uint32_t>(m.l) * 83492791u;
group_cc[b][hash % bin_groups[b]].Add(m.I_half[0], m.I_half[1], m.cc_weight);
// Weighted by cc_weight: the cut is an internal decision and runs on the weighted CC1/2
// (MergeStatisticsShell::cc_half_internal), not on the plain one the report prints.
group_cc[b][hash % bin_groups[b]].Add(m.I_half[0], m.I_half[1], weighted ? m.cc_weight : 1.0f);
}
// Usable bins (enough counts), in ascending-s order, with their bin-centre s.
image_analysis/scale_merge/ResolutionCutoff.h
+5 -2
View File
@@ -44,12 +44,15 @@ double TypicalFrameScale(const std::vector<double> &frame_scale, const std::vect
// The per-frame factor of the CC1/2 weight (MergedReflection::cc_weight): max(1, (G_ref/G)^2), the
// variance an observation from a frame at scale G carries over the same observation at the typical
// scale G_ref (TypicalFrameScale).
// scale G_ref (TypicalFrameScale). Internal decisions only (the resolution cut, the batch delta-CC1/2
// ledger); the reported CC1/2 is unweighted.
std::vector<double> CCHalfFrameFactors(const std::vector<double> &frame_scale,
const std::vector<int64_t> &n_obs);
// weighted: each reflection's pair enters with its cc_weight, the internal form the cut is decided on;
// false fits the plain (reported) CC1/2 instead - the quoted FITTED_RESOLUTION; not applied as the cut.
ResolutionCutoffResult ComputeCCHalfLogisticCutoff(const std::vector<MergedReflection> &merged,
double cc_target, Logger &logger);
double cc_target, Logger &logger, bool weighted = true);
// Resolve the effective high-resolution limit and trim `merged` to it, in one place shared by the
// stills merge, the rotation merge and the offline --scale path. A manual limit (manual_limit) wins;
image_analysis/scale_merge/RotationScaleMerge.cpp
+508 -50
View File
@@ -365,6 +365,7 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
// Decay + absorption correction surfaces are one master toggle (on by default; both cross-validated,
// so a no-op when their systematic is absent).
refine_decay_b = s.GetCorrectionSurfaces();
segments = s.GetSweepSegments();
absorption_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
modulation_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
relative_b_deg = s.GetRelativeBDegrees(); // per-batch relative-B batch width (0 = off)
@@ -531,11 +532,28 @@ void RotationScaleMerge::IngestHost() {
if (frame_limit >= 0) n_frames = std::min(n_frames, frame_limit);
partials_released = false;
resident_ingest = false;
frame_segment.clear();
if (!segments.empty()) {
frame_segment.assign(n_frames, -1);
int32_t next = 0;
for (size_t k = 0; k < segments.size(); ++k) {
const SweepSegment &seg = segments[k];
if (seg.first_frame != next || seg.n_frames <= 0 || seg.first_frame + seg.n_frames > n_frames)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: the sweep segments do not cover the frames in order");
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
frame_segment[f] = static_cast<int32_t>(k);
next = seg.first_frame + seg.n_frames;
}
if (next != n_frames)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: the sweep segments do not cover the frames in order");
}
#ifdef JFJOCH_USE_CUDA
// Probed before the observations are built: whether the build lays down the full Obs array or
// only the ingest arrays depends on whether the device pipeline will be active.
gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
gpu_active_ = gpu_->Available() && !scale_on_cpu_;
gpu_active_ = gpu_->Available() && !scale_on_cpu_ && segments.empty();
resident_ingest = gpu_active_ && !observation_dump;
#endif
@@ -605,7 +623,12 @@ void RotationScaleMerge::IngestHost() {
}
});
MeasureIncidentFlux(mean_bkg);
// Fulls of several sweeps carry each sweep's own background: not one flux meter, so the per-frame
// scale alone takes the flux.
if (segments.empty())
MeasureIncidentFlux(mean_bkg);
else
frame_flux.assign(n_frames, 1.0);
// The occupied range of h, which is the buckets the sort works on.
int b_lo = H_ALL, b_hi = -1;
@@ -728,12 +751,19 @@ void RotationScaleMerge::IngestHost() {
}
});
}
BuildInRangeObservations(keys, static_cast<int>(total), SmoothedFrameGeometry());
// Pre-combined fulls have no rocking geometry left to smooth or to recompute a partiality from.
BuildInRangeObservations(keys, static_cast<int>(total),
segments.empty() ? SmoothedFrameGeometry() : SmoothedGeometry{});
rawrun_group.assign(rawrun_start.size(), -1);
logger->Info("RotationScaleMerge: ingested {} partial observations from {} frames ({} distinct hkl)",
n_partials_obs, n_frames, rawrun_start.size());
SmoothMosaicityAndPartiality();
if (segments.empty()) {
SmoothMosaicityAndPartiality();
} else {
ingest_zeta.reset();
ingest_d.reset();
}
ParallelChunks(static_cast<int>(partials.size()), ThreadsForWork(partials.size(), nthreads),
[&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
@@ -2174,6 +2204,7 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
has_fulls[f] = frame_obs_count[f] > 0 ? 1 : 0;
}
double best_step = INFINITY, lambda = 0.0;
std::vector<double> segment_lambda(segments.size(), 0.0);
bool lambda_kept = false;
std::vector<double> steps, g_best;
AndersonMixing anderson(SCALING_ANDERSON_DEPTH);
@@ -2188,9 +2219,16 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
if (fitted)
y[f] += g_fit[f] > 0.0 ? std::max(std::log(g_fit[f] / g[f]), -1.0) : -1.0;
}
if (!lambda_kept)
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
const std::vector<double> x = SmoothLogScale(y, J, lambda);
std::vector<double> x;
if (segments.empty()) {
if (!lambda_kept)
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
x = SmoothLogScale(y, J, lambda);
} else {
x = SmoothLogScaleBySegment(y, J, segment_lambda, !lambda_kept);
std::vector<double> l = segment_lambda;
lambda = median_of(l);
}
for (int f = 0; f < n_frames; ++f)
g_new[f] = std::exp(x[f]);
const double g_ref = TypicalFrameScale(g_new, n_obs);
@@ -2342,8 +2380,14 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
return o.group >= 0 && o.corr > 0.0f && std::isfinite(o.corr) && (o.d > 0.0f)
&& o.partiality >= min_partiality;
};
// Where an observation sits on the dose axis: its frame, or with several sweeps its place in the dose
// order the data put them in (MeasureSegmentDose).
auto dose = [&](const auto &o) {
return segments.empty() ? static_cast<double>(o.image_number)
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
};
auto decay_factor = [&](const auto &o, double slope) {
return std::exp(slope * (o.image_number - fcenter) * s2_of(o.d));
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
};
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
@@ -2358,6 +2402,8 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
BuildDecayTerms(n_groups, t);
decay_terms_kept = {};
decay_terms_kept_valid = false;
if (!segments.empty())
MeasureSegmentDose(t, n_groups);
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
std::vector<double> sw(n_groups), swI(n_groups);
@@ -2385,7 +2431,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
const double w = (Is / sc) * (Is / sc);
const double x = (o.image_number - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
@@ -2435,6 +2481,125 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
// sit at the same s^2 but different frames, so the correction spreads them) - net harmful. slope =
// 2 dB/dframe, so the total relative-B change across the run is |slope/2 * n_frames|.
constexpr double DECAY_MIN_DELTA_B = 2.0; // A^2, minimum total relative-B over the run to engage
if (!segments.empty()) {
// Two hypotheses of how damage accrued over several sweeps, decided on held-out frames: a B rising
// linearly along the dose order (frame_dose), or one B per sweep - a step at each sweep boundary,
// flat within a sweep. Both are regressions of y = ln(I_ref / I_obs) on s^2 = 1/(4 d^2), the ramp
// on x = (dose - centre) s^2, the steps on x = s^2 with a slope per sweep, at the weights the slope
// fit above uses and within each frame (below). Each is fitted on the even frames and its
// weighted squared residual taken on the odd ones, against their own reference, and the other way
// round; the hypothesis that predicts the held-out frames better is applied, where it beats no
// decay at all by the margin a correction has to clear on one sweep (and the physical floor).
// The held-out frames' residual is read at weights fixed by the observations, so a correction
// cannot win by reshaping sigma.
using Sums = std::array<double, 5>; // Sw, Sx, Sy, Sxx, Sxy
struct Line { double a = 0.0, b = 0.0; };
auto line_of = [](const Sums &S) {
Line l;
const double var = S[0] > 0.0 ? S[3] - S[1] * S[1] / S[0] : 0.0;
if (var > 0.0) l.b = (S[4] - S[1] * S[2] / S[0]) / var;
if (S[0] > 0.0) l.a = (S[2] - l.b * S[1]) / S[0];
return l;
};
auto reference = [&](int parity) {
std::fill(sw.begin(), sw.end(), 0.0);
std::fill(swI.begin(), swI.end(), 0.0);
for (const DecayTerm &o : t.term) {
if (parity >= 0 && (o.frame & 1) != parity) continue;
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
sw[o.group] += w;
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
}
};
// Every term of `parity` with its y and weight against the current reference.
auto for_terms = [&](int parity, const auto &fn) {
for (const DecayTerm &o : t.term) {
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group];
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
fn(o, std::log(Iref / Is), (Is / sc) * (Is / sc));
}
};
auto x_ramp = [&](const DecayTerm &o) { return (dose(o) - fcenter) * s2_of(o.d); };
// The terms of `parity` with y and both x taken about their frame's weighted mean: each frame has a
// scale of its own (the per-frame G), which took up its part of the B averaged over its
// reflections, so only how y varies with resolution WITHIN a frame is decay.
struct Point { int32_t seg; double w, y, xr, xs; };
auto points = [&](int parity) {
reference(parity);
std::vector<Point> pts;
std::vector<int32_t> frame;
std::vector<std::array<double, 4>> mean(n_frames, std::array<double, 4>{}); // Sw, Sy, Sxr, Sxs
for_terms(parity, [&](const DecayTerm &o, double y, double w) {
pts.push_back({frame_segment[o.frame], w, y, x_ramp(o), s2_of(o.d)});
frame.push_back(o.frame);
auto &m = mean[o.frame];
m[0] += w; m[1] += w * y; m[2] += w * pts.back().xr; m[3] += w * pts.back().xs;
});
for (size_t k = 0; k < pts.size(); ++k) {
const auto &m = mean[frame[k]];
pts[k].y -= m[1] / m[0]; pts[k].xr -= m[2] / m[0]; pts[k].xs -= m[3] / m[0];
}
return pts;
};
struct Fit { double ramp = 0.0; std::vector<double> step; };
auto fit = [&](int parity) {
Sums r{};
std::vector<Sums> st(segments.size(), Sums{});
for (const Point &q : points(parity)) {
r[0] += q.w; r[1] += q.w * q.xr; r[2] += q.w * q.y; r[3] += q.w * q.xr * q.xr; r[4] += q.w * q.xr * q.y;
Sums &a = st[q.seg];
a[0] += q.w; a[1] += q.w * q.xs; a[2] += q.w * q.y; a[3] += q.w * q.xs * q.xs; a[4] += q.w * q.xs * q.y;
}
Fit out{line_of(r).b, {}};
for (const Sums &a : st) out.step.push_back(line_of(a).b);
return out;
};
// Held-out weighted squared residuals of no decay, the ramp and the steps.
auto score = [&](int test, const Fit &m, std::array<double, 3> &ssr) {
for (const Point &q : points(test)) {
const double e1 = q.y - m.ramp * q.xr, e2 = q.y - m.step[q.seg] * q.xs;
ssr[0] += q.w * q.y * q.y; ssr[1] += q.w * e1 * e1; ssr[2] += q.w * e2 * e2;
}
};
std::array<double, 3> ssr{};
const Fit even = fit(0), odd = fit(1);
score(1, even, ssr);
score(0, odd, ssr);
const Fit all = fit(-1);
const double slope = all.ramp; // the slope decay_factor applies
std::vector<double> b(segments.size());
double b_mid = 0.0, b_lo = INFINITY, b_hi = -INFINITY;
for (size_t k = 0; k < segments.size(); ++k) {
b[k] = 0.5 * all.step[k];
b_mid += b[k] * segments[k].n_frames / static_cast<double>(n_frames);
b_lo = std::min(b_lo, b[k]);
b_hi = std::max(b_hi, b[k]);
}
const double delta_lin = std::fabs(0.5 * slope * n_frames), delta_step = b_hi - b_lo;
const bool lin_ok = delta_lin >= DECAY_MIN_DELTA_B && ssr[1] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
const bool step_ok = delta_step >= DECAY_MIN_DELTA_B && ssr[2] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
const bool use_step = step_ok && (!lin_ok || ssr[2] < ssr[1]);
const bool use_lin = lin_ok && !use_step;
segment_decay_model = use_step ? "per-sweep B" : use_lin ? "dose-linear" : "none";
const double rel = ssr[0] > 0.0 ? 100.0 / ssr[0] : 0.0;
logger->Info("Decay over {} sweeps, held-out residual against no decay: dose-linear (dB = {:.2f} A^2 over "
"the dose order) {:.1f}%, one B per sweep (spread {:.2f} A^2) {:.1f}% => {}", segments.size(),
delta_lin, ssr[1] * rel, delta_step, ssr[2] * rel, segment_decay_model);
if (use_step || use_lin)
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
Obs &o = fulls[i];
if (!usable(o)) continue;
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
: decay_factor(o, slope);
o.corr = static_cast<float>(o.corr * f);
}
});
return;
}
const double slope = fit_slope(-1);
const double total_delta_B = std::fabs(0.5 * slope * n_frames);
if (total_delta_B < DECAY_MIN_DELTA_B) {
@@ -2460,6 +2625,92 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
0.5 * slope * n_frames, 0.5 * slope);
}
std::vector<double> RotationScaleMerge::SegmentB(const DecayTerms &t, int n_groups, int parity) const {
// Per segment, the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against
// s^2 = 1/(4 d^2), which is 2 B, with an intercept of its own (the scale is the per-frame G's), over
// the frames of `parity` (< 0: all) against a reference made of the same frames. NaN where a segment
// has nothing to fit.
const size_t ns = segments.size();
std::vector<double> sw(n_groups, 0.0), swI(n_groups, 0.0);
for (const DecayTerm &o : t.term) {
if (parity >= 0 && (o.frame & 1) != parity) continue;
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
sw[o.group] += w;
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
}
std::vector<std::array<double, 5>> S(ns, std::array<double, 5>{}); // Sw, Sx, Sy, Sxx, Sxy
for (const DecayTerm &o : t.term) {
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group];
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
const double w = (Is / sc) * (Is / sc);
const double x = 1.0 / (4.0 * static_cast<double>(o.d) * o.d), y = std::log(Iref / Is);
auto &a = S[frame_segment[o.frame]];
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
}
std::vector<double> b(ns, NAN);
for (size_t k = 0; k < ns; ++k) {
const auto &a = S[k];
const double var = a[0] > 0.0 ? a[3] - a[1] * a[1] / a[0] : 0.0;
if (var > 0.0)
b[k] = 0.5 * (a[4] - a[1] * a[2] / a[0]) / var;
}
return b;
}
void RotationScaleMerge::MeasureSegmentDose(const DecayTerms &t, int n_groups) {
// A relative B per data set against the joint merge, as XSCALE fits it: Kabsch (2010) Acta Cryst.
// D66, 133-144; the dose read from it: Diederichs (2006) Acta Cryst. D62, 96-101.
// Radiation damage raises the relative B of a sweep collected later, so the B factors of the
// segments against their joint merge give the order the crystal took the dose in - read from the
// data, not from the order the sweeps were given in or the times their headers state. Per segment,
// the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against s^2 = 1/(4 d^2) is 2 B,
// with an intercept of its own (the scale is the per-frame G's). The least damaged segment is B = 0,
// and its frames start the dose axis.
const size_t ns = segments.size();
const std::vector<double> b = SegmentB(t, n_groups, -1);
// A segment with nothing to measure keeps its place in the input order, after the measured ones.
std::vector<int> order(ns);
std::iota(order.begin(), order.end(), 0);
std::stable_sort(order.begin(), order.end(), [&](int p, int q) {
if (std::isfinite(b[p]) != std::isfinite(b[q])) return std::isfinite(b[p]);
return std::isfinite(b[p]) && b[p] < b[q];
});
const double b0 = std::isfinite(b[order[0]]) ? b[order[0]] : 0.0;
frame_dose.assign(n_frames, 0.0);
double offset = 0.0;
std::string text;
for (size_t rank = 0; rank < ns; ++rank) {
const int k = order[rank];
const SweepSegment &seg = segments[k];
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
frame_dose[f] = offset + (f - seg.first_frame);
offset += seg.n_frames;
segment_results[k].relative_b = b[k] - b0;
segment_results[k].dose_rank = static_cast<int>(rank);
text += fmt::format("{}segment {} (B {:+.2f} A^2)", rank == 0 ? "" : ", ", k + 1, b[k] - b0);
}
logger->Info("Dose order of the {} sweep segments, from their relative B against the joint merge: {}",
ns, text);
}
std::vector<double> RotationScaleMerge::SmoothLogScaleBySegment(const std::vector<double> &y,
const std::vector<double> &J,
std::vector<double> &lambda, bool choose) const {
std::vector<double> x(y.size());
for (size_t k = 0; k < segments.size(); ++k) {
const auto lo = y.begin() + segments[k].first_frame, hi = lo + segments[k].n_frames;
const auto jlo = J.begin() + segments[k].first_frame;
const std::vector<double> ys(lo, hi), Js(jlo, jlo + segments[k].n_frames);
if (choose)
lambda[k] = ChooseLogScaleSmoothing(ys, Js, std::max(1, rocking_event_frames_at_start), nthreads);
const std::vector<double> xs = SmoothLogScale(ys, Js, lambda[k]);
std::copy(xs.begin(), xs.end(), x.begin() + segments[k].first_frame);
}
return x;
}
// A^2, the bound the Gauss-Seidel solve clamps each batch's relative-B to. It is a numerical guard against
// divergence, never an answer: a run of identical values in the solved curve means the solve reached the
// bound, not that the crystal did anything. The CORRECTION keeps the tight bound it was tuned with; the
@@ -3199,7 +3450,8 @@ void RotationScaleMerge::MeasureBatchDeltaCCHalf(int n_groups) {
const std::vector<double> frame_factor = CCHalfFrameFactors(frame_scale_fulls, n_obs_frame);
ParallelChunks(n_full, ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
obs_info[i] = 1.0 / frame_factor[obs_frame[i]]; // the information the observation carries, <= 1
// the information the observation carries, <= 1 (the cc_weight factor; internal, not reported)
obs_info[i] = 1.0 / frame_factor[obs_frame[i]];
});
for (int f = 0; f < n_frames; ++f) fstart[f + 1] += fstart[f];
// The usable observations once more, in frame order and with the three fields a measurement reads
@@ -3831,6 +4083,26 @@ void RotationScaleMerge::RefineAbsorptionSH(int n_iter, int n_groups) {
/*resolution_gauge=*/false, &basis);
}
void RotationScaleMerge::RefineAbsorptionJoint(int n_iter, int n_groups) {
// Pre-combined fulls of several sweeps: px/py hold the polar angles of the diffracted beam in the
// crystal, on the joint merge's axes (BuildJointMergeInput) - one direction frame for every sweep, so a
// chi or kappa series measures one surface from many directions.
const SphericalHarmonicBasis basis = MakeSphericalHarmonicBasis(32, 64, 6, 0.1);
std::vector<int32_t> cell(fulls.size(), -1);
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
const Obs &o = fulls[i];
if (!std::isfinite(o.px) || !std::isfinite(o.py))
continue;
cell[i] = SphericalHarmonicCell(basis, std::sin(o.px) * std::cos(o.py), std::sin(o.px) * std::sin(o.py),
std::cos(o.px));
}
});
ApplyCellSurface(cell, basis.NCell(), n_iter, n_groups,
"Absorption across the sweeps (crystal-frame spherical harmonics l<=6)",
/*resolution_gauge=*/false, &basis);
}
void RotationScaleMerge::RefineAbsorptionTime(int n_iter, int n_groups) {
// Time-dependent absorption: the same multiplicative surface as RefineAbsorption, but indexed by
// (rotation, detector position) rather than by the de-rotated crystal-frame direction alone. For a rigid
@@ -4716,12 +4988,15 @@ void RotationScaleMerge::Combine() {
size_t i = 0;
while (i < ev.size()) {
size_t kk = i + 1;
float last_frame = partials[ev[i]].r->image_number;
while (kk < ev.size()) {
const float frame = partials[ev[kk]].r->image_number;
if (frame - last_frame > max_frame_gap) break;
last_frame = frame;
++kk;
// A pre-combined full is a whole event already (SweepSegment).
if (segments.empty()) {
float last_frame = partials[ev[i]].r->image_number;
while (kk < ev.size()) {
const float frame = partials[ev[kk]].r->image_number;
if (frame - last_frame > max_frame_gap) break;
last_frame = frame;
++kk;
}
}
double pooled_bkg = 0.0;
@@ -4843,7 +5118,9 @@ void RotationScaleMerge::Combine() {
size_t n_used = 0;
std::vector<std::vector<Obs>> part; // the fulls of consecutive runs, in run order
if (observation_dump || nthreads <= 1) {
// No runs at all (a lattice that integrated nothing) takes the serial path too: the chunking below
// divides by the number of threads it would use, which is then 0.
if (observation_dump || nthreads <= 1 || n_run == 0) {
// Serial (diagnostic dump needs a single writer).
std::ostringstream dump;
if (observation_dump) {
@@ -5133,6 +5410,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
double chi2_a = 0.0, chi2_b2 = 0.0;
bool error_model_b_unmeasured = false; // b had no leverage; ISa is not reported
bool error_model_b_resolved = true; // b^2 stood clear of its own standard error
bool error_model_a_at_bound = false; // a stopped on its clamp
std::vector<Sample> &samples = em_samples;
// A full's counting variance at intensity I_for_b. Its own sigma carries its own Poisson fluctuation:
// a full that came out low has a smaller sigma, so 1/sigma^2 weights it up and the merged mean drifts
@@ -5152,6 +5430,24 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
// Fit (a, b) to a pool of samples (ErrorModel.h). A lambda because the pool changes once the cutoff
// below is known.
std::vector<ErrorModelBinned> &fit_scratch = em_fit_pool;
// Several sweeps merged as one (segments): each its own error model, fitted on its own observations'
// scatter about the joint means, as XSCALE gives every data set its own - one poor sweep must not
// inflate the sigmas of the others, nor a good one shrink a poor one's. A segment too thin for a fit of
// its own keeps the joint one. sample_segment: each sample's segment, beside `samples`.
std::vector<int32_t> sample_segment;
std::vector<double> segment_a(segments.size(), 1.0), segment_b(segments.size(), 0.0);
auto fit_segment_models = [&](const std::vector<Sample> &pool, const std::vector<int32_t> &pool_segment) {
std::vector<Sample> own;
for (size_t k = 0; k < segments.size(); ++k) {
own.clear();
for (size_t j = 0; j < pool.size(); ++j)
if (pool_segment[j] == static_cast<int32_t>(k)) own.push_back(pool[j]);
const ErrorModelFit fit = FitErrorModel(own, fit_scratch, nthreads);
segment_a[k] = fit.active ? fit.a : error_model_a;
segment_b[k] = fit.active ? std::sqrt(fit.b2) : error_model_b;
segment_results[k].isa = fit.active && fit.b_resolved ? ToXdsErrorModel(fit.a, std::sqrt(fit.b2)).isa : NAN;
}
};
// On the device where the merge is - the same fit to rounding (ErrorModelGPU.h).
auto fit_error_model = [&](const std::vector<Sample> &pool) {
ErrorModelFit fit;
@@ -5169,6 +5465,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
error_model_b = std::sqrt(fit.b2);
error_model_b_unmeasured = !fit.b_measured;
error_model_b_resolved = fit.b_resolved;
error_model_a_at_bound = fit.a_at_bound;
error_model_active = true;
chi2_pool = &pool; chi2_a = fit.a; chi2_b2 = fit.b2;
};
@@ -5234,7 +5531,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
const double factor = 1.0 - w / (on_hand ? swh[s] : sw[g]);
if (factor < 0.05) return false;
const double resid = static_cast<double>(o.I) * o.corr - mean;
out = {counting_variance(o, em_mean[g], s2), mean * mean, resid * resid / factor, o.d};
out = {counting_variance(o, em_mean[g], s2), mean * mean, resid * resid / factor, o.d, g};
return true;
};
const std::vector<int32_t> sampled = KeptIndices(n_full, nthreads, [&](int i) {
@@ -5246,6 +5543,11 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
[&](int lo, int hi) {
for (int j = lo; j < hi; ++j) sample_of(sampled[j], samples[j]);
});
if (!segments.empty()) {
sample_segment.resize(sampled.size());
for (size_t j = 0; j < sampled.size(); ++j)
sample_segment[j] = frame_segment[fulls[sampled[j]].frame];
}
}
// Per-group outlier-rejection median of I*corr (host both paths - a per-group median is awkward on
@@ -5417,15 +5719,22 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
}
sub("em fit");
fit_error_model(samples);
if (!segments.empty() && error_model_active)
fit_segment_models(samples, sample_segment);
sub("bijvoet excess");
}
// The full's sigma under the error model, with the variance evaluated at intensity I_for_b.
// The b of the error model an observation is under: its segment's where there are segments.
auto model_b = [&](const Obs &o) {
return segments.empty() ? error_model_b : segment_b[frame_segment[o.frame]];
};
auto model_sigma = [&](const Obs &o, double I_for_b, float sigma_corr) -> float {
if (!error_model_active) return sigma_corr;
const double a_var = counting_variance(o, I_for_b, static_cast<double>(sigma_corr) * sigma_corr);
const double v = error_model_a * a_var
+ (error_model_b * I_for_b) * (error_model_b * I_for_b);
const double a = segments.empty() ? error_model_a : segment_a[frame_segment[o.frame]];
const double b = model_b(o);
const double v = a * a_var + (b * I_for_b) * (b * I_for_b);
return v > 0.0 ? static_cast<float>(std::sqrt(v)) : sigma_corr;
};
auto corrected_sigma = [&](const Obs &o, float I_corr, float sigma_corr) -> float {
@@ -5577,7 +5886,9 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
// typical earns nothing (the factor is floored at 1), so on a sweep without a weak stretch
// every weight is 1 and the CC1/2 is the plain Pearson it was. G is the frame's scale taken
// before the correction surfaces (frame_scale_fulls), which are per-observation and
// resolution-dependent and are not a property of the frame. ----
// resolution-dependent and are not a property of the frame. Internal, not reported: the weight
// enters the CC1/2 the processing decides on (cc_half_internal, the resolution cut); the
// reported CC1/2 is the plain Pearson of the half-set means. ----
// Counts, so the blocks may add theirs up in any split.
std::vector<int64_t> n_obs_frame(frame_scale_fulls.size(), 0);
{
@@ -5744,7 +6055,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
const double I_b = std::isfinite(em_mean[o.group]) ? em_mean[o.group] : I_corr;
if (OutsideOutlierBand(I_corr, reject_median[o.group], reject_nsigma,
static_cast<double>(sigma_corr) * sigma_corr + v_add, I_b,
error_model_b * I_b, reject_up, reject_down)) {
model_b(o) * I_b, reject_up, reject_down)) {
++blk_rejected[blk];
rejected_obs[i] = 1;
continue;
@@ -5758,9 +6069,9 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
a.swIh[half] += wI; a.swh[half] += w; a.nh[half]++;
a.swh_typ[half] += w * frame_cc_factor[o.frame];
// Any contributing observation on an ice ring marks the merged reflection (as
// MergeOnTheFly does on the stills path): the intensity is still merged and written,
// the flag only lets a consumer that must not read ring signal as crystal signal -
// the CC1/2 resolution-cutoff fit - leave it out.
// MergeOnTheFly does on the stills path): the ice-ring bands decision below reads it,
// and so does the CC1/2 resolution-cutoff fit, which must not read ring signal as
// crystal signal.
if (o.unscaled & UNSCALED_ICE) a.on_ice = true;
if (!std::isfinite(a.d) && std::isfinite(o.d) && o.d > 0.0f) a.d = o.d;
}
@@ -5783,6 +6094,44 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
}
});
}
// ---- The ice-ring bands: merged as they are, down-weighted or left out, ring by ring
// (DecideIceRings). Applied to the accumulators, so the export, the statistics and the
// half-set CC all see the same reflections. Not on a search merge, which leaves the ice
// out already. ----
if (!for_search) {
std::vector<int32_t> ice_g;
std::vector<MergedReflection> ice_in;
for (int g = 0; g < n_groups; ++g) {
const auto &a = acc[g];
if (!(a.sw > 0.0) || !std::isfinite(a.d) || !(a.d > 0.0f)) continue;
MergedReflection m{};
m.I = static_cast<float>(a.swI / a.sw);
m.sigma = static_cast<float>(1.0 / std::sqrt(a.sw));
m.d = a.d;
m.on_ice_ring = a.on_ice;
ice_g.push_back(g);
ice_in.push_back(m);
}
const bool any_ice = std::any_of(ice_in.begin(), ice_in.end(),
[](const MergedReflection &m) { return m.on_ice_ring; });
if (any_ice) {
const IceRingDecision ice = DecideIceRings(ice_in);
for (size_t j = 0; j < ice_g.size(); ++j) {
const double f = ice.weight_factor[j];
if (f == 1.0) continue;
auto &a = acc[ice_g[j]];
a.swI *= f; a.sw *= f;
for (int i = 0; i < 2; ++i) { a.swIh[i] *= f; a.swh[i] *= f; a.swh_typ[i] *= f; }
if (f == 0.0) a.nh[0] = a.nh[1] = 0;
}
for (const auto &z : ice.zones)
logger->Info("Ice ring {:.3f} A {}: {} reflections against {} off the ring, bias {:+.1f} sigma, "
"variance x{:.2f} ({:+.1f} sigma) - {}", z.ring_d,
z.zone < 0 ? "low-q flank" : (z.zone > 0 ? "high-q flank" : "core"), z.n,
z.n_neighbours, z.bias_z, z.var_ratio, z.var_ratio_z, IceRingActionName(z.action));
result.ice_ring_verdicts = ice.zones;
}
}
// ---- Export merged reflections. ----
// Which groups are written, in group order, then each one's record on its own - on all threads.
// A group is written when it has weight and a resolution.
@@ -5829,6 +6178,17 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
// _process.h5 is written elsewhere from the partials, so no data is lost. A manual
// --scaling-high-resolution (d_min_limit) wins; the P1 search merge (for_search) is never cut, so
// the space-group search still sees the full range.
// Diagnostic only: the same fit on the plain CC1/2, before the cut trims the reflections. It is
// reported beside the applied cut and decides nothing.
if (!d_min_limit && !for_search && resolution_cutoff_method == ResolutionCutoffMethod::CCHalfLogistic) {
const auto plain = ComputeCCHalfLogisticCutoff(result.merged, resolution_cc_target, *logger,
/*weighted=*/false);
result.resolution_fit_plain_A = plain.d_fit;
result.resolution_cut_plain_A = plain.d_cut;
logger->Info("Resolution fit on the plain CC1/2 (diagnostic, not applied): crossing {}, cut {}",
plain.d_fit ? fmt::format("{:.2f} A", *plain.d_fit) : "none",
plain.d_cut ? fmt::format("{:.2f} A", *plain.d_cut) : "none");
}
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
result.merged, d_min_limit, resolution_cutoff_method, resolution_cc_target, for_search, *logger,
&result.resolution_fit_A);
@@ -5846,9 +6206,15 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
std::vector<Sample> &in_range = em_cut_pool;
in_range.clear();
in_range.reserve(samples.size());
for (const auto &s : samples)
if (s.d >= *effective_d_min) in_range.push_back(s);
std::vector<int32_t> in_range_segment;
for (size_t j = 0; j < samples.size(); ++j)
if (samples[j].d >= *effective_d_min) {
in_range.push_back(samples[j]);
if (!segments.empty()) in_range_segment.push_back(sample_segment[j]);
}
fit_error_model(in_range);
if (!segments.empty() && error_model_active)
fit_segment_models(in_range, in_range_segment);
run_merge();
std::erase_if(result.merged, [&](const MergedReflection &m) {
return std::isfinite(m.d) && m.d < *effective_d_min;
@@ -5860,17 +6226,43 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
// reflections are) - each with the sigma the merge weighted it by.
if (export_scaled_fulls && full_stats && !for_search) {
// Which fulls, in order, and then each one's record on its own - on all threads.
// The fulls of one sweep of several (uncut) also hand back the reflections the group extinguishes:
// a joint merge searches its own space group, and the screw axes are decided on exactly these.
const gemmi::GroupOps ops = Group().operations();
auto extinguished = [&](const Obs &o) {
return export_scaled_fulls_uncut && o.group < 0 && frame_cell_ok[o.frame] && o.corr > 0.0f
&& std::isfinite(o.corr) && std::isfinite(o.I) && o.d > 0.0f
&& ops.is_systematically_absent({{o.h, o.k, o.l}});
};
const std::vector<int32_t> kept = KeptIndices(n_full, nthreads, [&](int i) {
const int g = mf.group[i];
return g >= 0 && !rejected_obs[i] && std::isfinite(merged_I[g])
&& !(effective_d_min && acc[g].d < *effective_d_min);
if (g < 0)
return extinguished(fulls[i]);
return !rejected_obs[i] && std::isfinite(merged_I[g])
&& (export_scaled_fulls_uncut || !(effective_d_min && acc[g].d < *effective_d_min));
});
result.scaled_fulls.resize(kept.size());
const DiffractionGeometry export_geom = x.GetDiffractionGeometry();
ParallelChunks(static_cast<int>(kept.size()), ThreadsForWork(kept.size(), nthreads), [&](int lo, int hi) {
for (int j = lo; j < hi; ++j) {
const Obs &o = fulls[kept[j]];
const float I_corr = o.I * o.corr;
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, corrected_sigma(o, I_corr, o.sigma * o.corr)};
const float sigma = o.group >= 0 ? corrected_sigma(o, I_corr, o.sigma * o.corr)
: model_sigma(o, I_corr, o.sigma * o.corr);
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, sigma,
o.image_number, o.corr * o.corr * o.var_bkg, o.corr * o.var_per_I,
o.capture, o.d};
// The diffracted beam in the crystal, from the frame's own lattice (a single sweep only:
// a joint merge's fulls carry theirs from their own sweep, ScaledFull::s1_cell).
if (segments.empty() && std::isfinite(o.px) && std::isfinite(o.py)) {
const CrystalLattice &l = partials_out[o.frame].latt;
if (l.CalcVolume() > 0.0f) {
const Coord s1 = export_geom.LabCoord(o.px, o.py).Normalize();
result.scaled_fulls[j].s1_cell[0] = l.Vec0() * s1;
result.scaled_fulls[j].s1_cell[1] = l.Vec1() * s1;
result.scaled_fulls[j].s1_cell[2] = l.Vec2() * s1;
}
}
}
});
}
@@ -5990,6 +6382,23 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
result.error_model_a = em.a;
result.error_model_b = em.b;
}
// What the report prints of ISa: the interval of b^2 by a bootstrap over the reflections of the
// pool the reported fit was handed. The two-SE test above reads its error off how far the 16 bins
// miss the two-term line, which does not shrink with more data, so on a full sweep it can refuse a
// b that every resampling of the reflections reproduces. Report-only - nothing below reads it -
// and only where there is a fit to resample: b measured, a off its clamp, the final statistics.
sub("isa bootstrap");
if (full_stats && error_model_active && chi2_pool && !error_model_b_unmeasured && !error_model_a_at_bound) {
// 50 replicates of at most 100k samples each, one per thread: 0.2-0.5 s a merge on 8 threads.
constexpr int ISA_BOOTSTRAP_REPLICATES = 50;
constexpr size_t ISA_BOOTSTRAP_SAMPLES = 100000;
const auto ci = BootstrapErrorModelB2(*chi2_pool, chi2_b2, ISA_BOOTSTRAP_REPLICATES, ISA_BOOTSTRAP_SAMPLES,
nthreads);
if (ci && ci->high > 0.0) {
result.isa_low = 1.0 / std::sqrt(ci->high);
result.isa_high = ci->low > 0.0 ? 1.0 / std::sqrt(ci->low) : INFINITY;
}
}
// Report-only, and only the LAST successful fit's chi2 is ever printed - so its median is taken
// here, once, instead of in every fit: the pool is one sample per full and the median is a pass over
// it plus an nth_element, ten times a run.
@@ -6020,7 +6429,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
em.a, error_model_chi2);
else if (!error_model_b_resolved)
logger->Info("Error model (XDS convention): a={:.3f} b={:.3e} - b is not resolved from zero "
"(under two standard errors), so ISa is undetermined (1/b = {:.1f})",
"(under two standard errors of the bin fit), 1/b = {:.1f}",
em.a, em.b, em.isa);
else if (!full_stats)
// Neither the asymptote nor the chi2 median was measured on this merge, so neither is
@@ -6031,6 +6440,14 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
"(strong-reflection asymptote {:.1f}) chi2={:.2f}",
em.a, em.b, em.isa, result.isa_asymptotic, error_model_chi2);
}
if (std::isfinite(result.isa_low)) {
if (std::isfinite(result.isa_high))
logger->Info("ISa by bootstrap over the reflections: {:.1f}, 95% interval {:.1f}-{:.1f}",
result.isa, result.isa_low, result.isa_high);
else
logger->Info("ISa by bootstrap over the reflections: b reaches zero inside its 95% interval, "
"ISa >= {:.1f}", result.isa_low);
}
// R-free flags and French-Wilson amplitudes are fields of the WRITTEN reflection, so a merge that
// is not written needs neither. A search merge is never written either - the merge the run writes
@@ -6072,7 +6489,8 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
struct ShellAcc {
int unique = 0, total_obs = 0, possible = 0;
double sum_i_over_sigma = 0.0; int n_i_over_sigma = 0;
CorrelationCoefficient cc_half;
CorrelationCoefficient cc_half; // plain Pearson: the reported CC1/2
CorrelationCoefficient cc_half_internal; // cc_weight-weighted: internal decisions, not reported
};
std::vector<ShellAcc> sa(n_shells);
std::vector<int> possible(n_shells, 0);
@@ -6087,7 +6505,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
for (int s = 0; s < n_shells; ++s) sa[s].possible = possible[s];
sub("st merged walk");
CorrelationCoefficient cc_half_overall;
CorrelationCoefficient cc_half_overall, cc_half_internal_overall;
for (const auto &m : merged) {
const auto shell = shells.GetShell(m.d);
if (!shell || *shell < 0 || *shell >= n_shells) continue;
@@ -6096,8 +6514,10 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
s.unique++;
s.sum_i_over_sigma += m.I / m.sigma; s.n_i_over_sigma++;
if (std::isfinite(m.I_half[0]) && std::isfinite(m.I_half[1])) {
s.cc_half.Add(m.I_half[0], m.I_half[1], m.cc_weight);
cc_half_overall.Add(m.I_half[0], m.I_half[1], m.cc_weight);
s.cc_half.Add(m.I_half[0], m.I_half[1]);
cc_half_overall.Add(m.I_half[0], m.I_half[1]);
s.cc_half_internal.Add(m.I_half[0], m.I_half[1], m.cc_weight);
cc_half_internal_overall.Add(m.I_half[0], m.I_half[1], m.cc_weight);
}
}
}
@@ -6190,7 +6610,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
if (!shell || *shell < 0 || *shell >= n_shells) continue;
// Only what the merge kept counts towards multiplicity - see the GPU branch above.
// The R_meas accumulation below keeps its own, wider filter.
if (mf.group[i] == g) // usable_merge
if (mf.group[i] == g && acc[g].nh[0] + acc[g].nh[1] > 0) // usable_merge, not rejected
blk_obs[blk][*shell]++;
if (std::isfinite(merged_I[g])) {
auto &r = rmeas[g];
@@ -6500,6 +6920,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
ss.possible_unique_reflections = sa[s].possible;
ss.mean_i_over_sigma = sa[s].n_i_over_sigma > 0 ? sa[s].sum_i_over_sigma / sa[s].n_i_over_sigma : 0.0;
ss.cc_half = sa[s].cc_half.GetCC();
ss.cc_half_internal = sa[s].cc_half_internal.GetCC();
ss.cc_ref = NAN;
ss.r_meas = urmeas_den[s] > 0.0 ? urmeas_num[s] / urmeas_den[s] : NAN;
ss.r_meas_weighted = rmeas_den[s] > 0.0 ? rmeas_num[s] / rmeas_den[s] : NAN;
@@ -6518,6 +6939,7 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
}
overall.mean_i_over_sigma = n_ios > 0 ? sum_ios / n_ios : 0.0;
overall.cc_half = cc_half_overall.GetCC();
overall.cc_half_internal = cc_half_internal_overall.GetCC();
overall.cc_ref = NAN;
overall.r_meas = urmeas_den_all > 0.0 ? urmeas_num_all / urmeas_den_all : NAN;
overall.r_meas_weighted = rmeas_den_all > 0.0 ? rmeas_num_all / rmeas_den_all : NAN;
@@ -6572,9 +6994,14 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
if (effective_d_min && m.d < *effective_d_min) continue;
sd_min = std::min(sd_min, m.d); sd_max = std::max(sd_max, m.d);
}
// Nothing (or one resolution) merged: no rocking event of the lattice made a full reflection, which
// is what a lattice that does not describe the frames ends in.
if (!(sd_min < sd_max && sd_min > 0.0f))
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: resolution calculation failed");
fmt::format("nothing to merge ({} merged reflections): no rocking event of "
"the lattice was recorded whole enough to make a full reflection, so "
"the lattice probably does not describe these frames",
result.merged.size()));
// The completeness denominator is counted over the range the run DECLARES, not the range that
// survived: the shell grid shares its bounds (so nothing is enumerated outside a shell) and runs
// to the declared low-resolution limit, or over the whole sphere - 1/d^2 down to 0, spelled as an
@@ -6598,7 +7025,9 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
if (!wilson.rejected[j]) continue;
const Obs &o = fulls[wilson_full[j]];
if (effective_d_min && o.d < *effective_d_min) continue;
out.wilson_rejected.push_back({o.h, o.k, o.l, o.d, wilson.e2[j], o.image_number, o.px, o.py});
// A joint merge's fulls hold the beam direction in px/py (RefineAbsorptionJoint), not a detector position.
const float x_px = segments.empty() ? o.px : NAN, y_px = segments.empty() ? o.py : NAN;
out.wilson_rejected.push_back({o.h, o.k, o.l, o.d, wilson.e2[j], o.image_number, x_px, y_px});
}
out.radiation_damage_delta_b = rad_damage_delta_b;
out.radiation_damage_b_batch = rad_damage_b_batch;
@@ -6691,6 +7120,12 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st
const gemmi::SpaceGroup *group) {
const ScopedGpuPin pin(gpu_device); // see IngestHost
run_group = group;
// Pre-combined fulls: there are no partials to fit a scale on, and every event is one frame.
if (!segments.empty()) {
scale_partials_choice = false;
rocking_event_frames_at_start = 1;
rocking_events_at_start = n_partials_obs;
}
struct ClearGroup { const gemmi::SpaceGroup *&g; ~ClearGroup() { g = nullptr; } } clear_group{run_group};
// Whether a frame's scale can be fitted on its partials depends on whether the partials tell a change
// of scale from an error of the partiality model. The count of rocking events per frame is the prior
@@ -7041,7 +7476,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// kernels on it, and the restore re-uploads.
// --- 2a. On the de-novo search pass only, drop observations whose Lorentz geometry is poor. ---
if (for_search && search_min_zeta > 0.0) {
if (for_search && search_min_zeta > 0.0 && segments.empty()) { // a full has no rocking geometry
int64_t n_dropped = 0;
bool zeta_on_gpu = false;
#ifdef JFJOCH_USE_CUDA
@@ -7246,7 +7681,9 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
BuildFullsGroupCSR(n_groups);
// --- 4. Scale the fulls (XDS order, Unity model). ---
if (scale_fulls && !scaled_fulls_on_gpu) {
// On the fulls the host combined: the per-frame CSR this loop slices is built by Combine(). Fulls
// combined on the GPU were scaled there - unless there were none, and then there is nothing to scale.
if (scale_fulls && !combined_on_gpu) {
// On a working copy of the fulls, as the partials' loop (ScalingRef and ScalingFit in the header):
// full_index[q] is the full at position q, in the group CSR's order and then the fulls in no group.
const int nf = static_cast<int>(fulls.size());
@@ -7351,20 +7788,27 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// is a correction to the merged INTENSITIES, and a pass whose intensities are thrown away has
// nowhere to put one - the fitted gain lives on the fulls, which the next pass rebuilds. ---
const bool refine_surfaces = !for_search && full_stats;
// What runs along the frames of ONE sweep - the damage monitor, the sweep ledger, the per-batch B and
// the surfaces, each read in one goniometer and one detector setting - has no meaning across the
// segments of several, and is left out there. The decay is fitted along the dose order instead.
const bool along_frames = segments.empty();
const bool corrections =
refine_surfaces && (refine_decay_b || absorption_iter > 0 || modulation_iter > 0 || relative_b_deg > 0.0);
refine_surfaces && (refine_decay_b || absorption_iter > 0
|| (along_frames && (modulation_iter > 0 || relative_b_deg > 0.0)));
segment_results.assign(segments.size(), Result::Segment{});
segment_decay_model.clear();
// Radiation-damage monitor: measure the per-batch relative-B on the scaled fulls BEFORE any correction
// (report-only; captures the full damage signature, not a residual). Skipped on the P1 search pass.
std::unique_ptr<TimingMark> sub_mark;
auto sub = [&](const char *n) { sub_mark.reset(); sub_mark = std::make_unique<TimingMark>(*logger, std::string("rsm/s ") + n); };
sub("damage B");
if (refine_surfaces)
if (refine_surfaces && along_frames)
MeasureRadiationDamageB(n_groups);
// Sweep-quality diagnostic, on the per-frame scale the partial scaling just fitted (the flux is
// already out of it) and the per-frame CC computed above. It drops nothing itself; the ranges it
// segments are the ledger 4c below decides on.
sub("sweep quality");
if (refine_surfaces)
if (refine_surfaces && along_frames)
MeasureSweepQuality(partial_scaled, cc, cc_n);
// CC1/2 of these same fulls with the surfaces not yet folded in, for the caller's cross-pass quality
// guard: the pass this one is judged against fits no surfaces of its own, so it is the uncorrected
@@ -7379,7 +7823,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
if (measure_cc_before_corrections && corrections) {
cc_half_before_corrections = MergeAndStats(n_groups, for_search,
combined_on_gpu && scaled_fulls_on_gpu,
/*full_stats=*/false).statistics.overall.cc_half;
/*full_stats=*/false).statistics.overall.cc_half_internal;
logger->Info("CC1/2 before the correction surfaces: {:.4f} (the merge just above; the pass this "
"one is judged against fits no surfaces, so this is what the two are compared on)",
*cc_half_before_corrections);
@@ -7390,7 +7834,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
decay_terms_kept = {};
decay_terms_kept_valid = false;
sub("relative B");
if (refine_surfaces && relative_b_deg > 0.0)
if (refine_surfaces && along_frames && relative_b_deg > 0.0)
RefineRelativeB(n_groups); // per-batch relative-B on top of the single decay slope
// The three surfaces overlap, so the one fitted first gets the first claim and the others are tested
// on what it left - the order decides what is adopted. Detector first: every frame of the sweep
@@ -7406,17 +7850,20 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// the grid is then tested on what the smooth surface left, and where that surface is refused the
// grid sees exactly what it saw before.
sub("modulation");
if (refine_surfaces && modulation_iter > 0)
if (refine_surfaces && along_frames && modulation_iter > 0)
RefineModulation(modulation_iter, n_groups);
sub("absorption time");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorptionTime(absorption_iter, n_groups);
sub("absorption SH");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorptionSH(absorption_iter, n_groups);
sub("absorption gonio");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorption(absorption_iter, n_groups);
sub("absorption joint");
if (refine_surfaces && !along_frames && absorption_iter > 0)
RefineAbsorptionJoint(absorption_iter, n_groups);
sub("disposition");
// --- 4c. The disposition. What keeping each batch of the sweep costs the merged intensities
@@ -7455,7 +7902,18 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// With the surfaces off (or none of them adopted) the merge that just ran IS the uncorrected one, so
// its CC1/2 is the answer without a second merge. A caller that did not ask keeps the NaN.
if (measure_cc_before_corrections)
r.cc_half_before_corrections = cc_half_before_corrections.value_or(r.statistics.overall.cc_half);
r.cc_half_before_corrections = cc_half_before_corrections.value_or(r.statistics.overall.cc_half_internal);
// Each segment's scale: the median of its frames' scales, at the run's gauge.
for (size_t k = 0; k < segments.size(); ++k) {
std::vector<double> g;
for (int32_t f = segments[k].first_frame; f < segments[k].first_frame + segments[k].n_frames; ++f)
if (fulls_frame_count[f] > 0 && std::isfinite(g_full[f]) && g_full[f] > 0.0)
g.push_back(g_full[f]);
if (!g.empty())
segment_results[k].scale = median_of(g);
}
r.segments = segment_results;
r.segment_decay_model = segment_decay_model;
// The fulls and the merge's working arrays are rebuilt by every arm before they are read, so they go
// with it: kept, they sat beside the next arm's combine and through everything the caller did between
// merges - on a large cell in P1, the fulls and about 330 bytes per ASU group, gigabytes. Faulting them
image_analysis/scale_merge/RotationScaleMerge.h
+59 -3
View File
@@ -11,6 +11,7 @@
#include <memory>
#include <new>
#include <optional>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
@@ -25,6 +26,7 @@
#include "WilsonOutliers.h"
#include "Merge.h" // MergedReflection, MergeStatistics
#include "SphericalHarmonicSurface.h"
#include "IceRingVerdict.h"
#ifdef JFJOCH_USE_CUDA
#include <memory>
#include "RotationScaleMergeGPU.h"
@@ -97,6 +99,8 @@ public:
int block, size_t nthreads = 1);
struct Result {
std::vector<MergedReflection> merged;
// What the merge did with each ice-ring band (DecideIceRings); empty without flagged reflections.
std::vector<IceRingZoneVerdict> ice_ring_verdicts;
MergeStatistics statistics;
// Two tiers, and they are different quantities. `isa` is the whole-range 1/sqrt(a*b) - which
// in this parameterisation is 1/b - and is what XDS's ISa means, so it is the one exported.
@@ -108,19 +112,30 @@ public:
// fitted 1/b and every decision reads it as before; only what is PRINTED says "undetermined",
// since 1/b of a b that is zero within its error is the noise in b, not an I/sigma.
bool isa_resolved = true;
// The 95% interval of `isa` by a bootstrap over the reflections, which is what the report prints;
// isa_high is infinite where b's interval reaches zero (ISa >= isa_low), both NaN where there was
// no fit to resample. Report-only.
double isa_low = std::numeric_limits<double>::quiet_NaN();
double isa_high = std::numeric_limits<double>::quiet_NaN();
double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2)
double error_model_b = 0.0;
// The overall CC1/2 as it stood BEFORE the correction surfaces were folded in. That is what the
// The overall CC1/2 (internal weighted form, cc_half_internal; not reported) as it stood BEFORE the
// correction surfaces were folded in. That is what the
// two-pass quality guard compares one pass against the other with: a pass whose intensities are
// discarded fits no surfaces (see full_stats), so judging the pass that does fit them by its
// corrected CC1/2 would set two different measurements against each other. Equal to
// statistics.overall.cc_half whenever no surface was fitted, and NaN when the caller did not
// statistics.overall.cc_half_internal whenever no surface was fitted, and NaN when the caller did not
// ask for it (see measure_cc_before_corrections) - it costs a merge, so it is not measured on
// spec, and a caller that did not ask must not be handed a number that looks measured.
double cc_half_before_corrections = std::numeric_limits<double>::quiet_NaN();
// Where the automatic CC1/2 fit says the data reach, without the deliberate one-shell
// extension the reflections are written to. Empty when no automatic cut ran.
std::optional<double> resolution_fit_A;
// Diagnostic, inert: the same automatic fit made on the plain (reported) CC1/2 instead of the
// weighted one the cut is decided on - its crossing and the limit it would have written. Nothing
// is cut at them. Empty where the automatic cut did not run or that fit found no crossing.
std::optional<double> resolution_fit_plain_A;
std::optional<double> resolution_cut_plain_A;
// The merged reflections the run kept, binned over the reference range of
// ScalingSettings::ReportResolutionRange_A (--report-resolution) instead of the run's own,
// and the error model refitted on the observations of that table alone, for its ISa. Nothing
@@ -139,6 +154,17 @@ public:
// The fulls the merge kept, unmerged (see ScaledFull): what passed the outlier tests, inside the
// resolution cut. Filled only when SetExportScaledFulls asked for it, and never on a search merge.
std::vector<ScaledFull> scaled_fulls;
// Per sweep segment (SetSweepSegments), in segment order; empty without segments.
struct Segment {
double scale = NAN; // median per-frame scale of the segment's fulls (run gauge)
double relative_b = NAN; // A^2, against the joint merge, the least damaged segment at 0
int dose_rank = -1; // the segment's place in the dose order those B factors decide
double isa = NAN; // the ISa of the segment's own error model, NaN where b was not resolved
};
std::vector<Segment> segments;
// How the decay over the segments was modelled - "dose-linear" (along the dose order), "per-sweep B"
// (one B a segment) or "none" - as the held-out frames decided; empty without segments or decay.
std::string segment_decay_model;
};
// experiment: read live (its space group is changed by the caller between Run() calls).
@@ -288,6 +314,16 @@ public:
// Merge only the first n frames of the outcomes (by default all of them). Set before Ingest().
void SetFrameLimit(int n) { frame_limit = n; }
// The outcomes are several sweeps of pre-combined fulls laid end to end (SweepSegment; by default the
// experiment's ScalingSettings::GetSweepSegments). Set before Ingest(); the segments must cover the
// frames in order. Such an engine runs on the host: its fulls are a few per frame where a sweep's
// partials are millions, and the device pipeline is the partials'. Without segments nothing of this runs.
void SetSweepSegments(std::vector<SweepSegment> s) { segments = std::move(s); }
// Whether the fulls handed back (SetExportScaledFulls) are all the merge kept, past its resolution cut
// too: the fulls of one sweep of several, which the joint merge cuts again on its own.
void SetExportScaledFullsUncut(bool on) { export_scaled_fulls_uncut = on; }
// The GPU the engine runs on (device 0 by default). Set before Ingest(). The device only says where
// the merge runs: on cards of one model the result is the same on any of them.
void SetGpuDevice(int device) { gpu_device = device; }
@@ -378,7 +414,8 @@ private:
}
// Why an observation stays out of the scale fit (Obs::unscaled), with the bit values the device flags
// use: it sits on an ice ring, whose background is the ring's, or it is coarser than the scale fit's
// low-resolution limit (ScalingSettings::low_resolution_limit_A). Either is still merged and written.
// low-resolution limit (ScalingSettings::low_resolution_limit_A). A low-resolution one is still merged
// and written; what the merge does with one on an ice ring is decided ring by ring (DecideIceRings).
static constexpr uint8_t UNSCALED_ICE = 1;
static constexpr uint8_t UNSCALED_LOW_RES = 8;
uint8_t Unscaled(const Reflection &r) const {
@@ -724,8 +761,18 @@ private:
std::vector<int64_t> last_cc_n;
std::vector<uint8_t> last_frame_scaled;
int frame_limit = -1; // see SetFrameLimit
std::vector<SweepSegment> segments; // see SetSweepSegments
std::vector<int32_t> frame_segment; // each frame's segment (Ingest; empty without segments)
std::vector<Result::Segment> segment_results; // the last arm's, copied into its Result
// Each frame's place on the dose axis the decay is fitted along: the frames of the segments laid end
// to end in the order the segments' relative B factors put them (MeasureSegmentDose).
std::vector<double> frame_dose;
// SmoothLogScale / ChooseLogScaleSmoothing within each segment on its own; lambda per segment.
std::vector<double> SmoothLogScaleBySegment(const std::vector<double> &y, const std::vector<double> &J,
std::vector<double> &lambda, bool choose) const;
int gpu_device = 0; // see SetGpuDevice
bool export_scaled_fulls = false; // see SetExportScaledFulls
bool export_scaled_fulls_uncut = false; // see SetExportScaledFullsUncut
bool french_wilson = true; // see SetFrenchWilson
// --- helpers (each a flat pass; see the .cpp) ---
@@ -958,6 +1005,12 @@ private:
std::vector<int32_t> g_start;
};
void BuildDecayTerms(int n_groups, DecayTerms &t) const;
// Per segment: the relative B against the merged reference, and the dose order it puts the segments
// in (segment_results, frame_dose).
void MeasureSegmentDose(const DecayTerms &t, int n_groups);
// Each segment's B against a reference made of the same frames: those of `parity` (< 0: all).
std::vector<double> SegmentB(const DecayTerms &t, int n_groups, int parity) const;
std::string segment_decay_model; // the last decay's choice (Result::segment_decay_model)
// The terms the radiation-damage monitor built, handed on to RefineDecay: nothing between the two
// touches corr, so they are the terms RefineDecay would build. Empty outside that stretch of RunArm.
DecayTerms decay_terms_kept;
@@ -1018,6 +1071,9 @@ private:
// The crystal-frame absorption as real spherical harmonics of the diffracted-beam direction (l <= 6)
// instead of a grid; a candidate through the same cross-validated engine, fitted before the grid.
void RefineAbsorptionSH(int n_iter, int n_groups);
// Several sweeps merged as one (segments): the same spherical-harmonic surface over the diffracted
// beam's direction in the crystal, one surface shared by all the sweeps (px/py carry the direction).
void RefineAbsorptionJoint(int n_iter, int n_groups);
// Time-dependent absorption: the same cross-validated surface, indexed by (rotation, detector position)
// instead of by the crystal-frame direction alone. RefineAbsorption's parameterisation is the whole
// model only while the illuminated volume stays put; once the crystal drifts through the beam the exit
image_analysis/scale_merge/RotationScaleMergeGPU.cu
+1
View File
@@ -916,6 +916,7 @@ namespace {
out.I2 = mean * mean;
out.dev2 = resid * resid / factor;
out.d = p.d[i];
out.group = g;
return true;
}
image_analysis/scale_merge/ScaleOnTheFly.cpp
+5
View File
@@ -137,8 +137,13 @@ void ScaleOnTheFly::Scale(IntegrationOutcome &integration_outcome) const {
obs.push_back({coeff, static_cast<double>(r.I), sigma2, var_bkg});
}
// Too few reflections to put the image on the merge's scale: it is left out of the merge (a
// non-finite correction is what the merge skips on) rather than merged at the scale of 1 its
// prediction carries, which is no scale at all.
if (obs.size() < MIN_REFLECTIONS) {
clear_scale();
for (auto &r: integration_outcome.reflections)
r.image_scale_corr = NAN;
return;
}
Loaded 100 of 266 files, more files were not shown because too many files have changed in this diff. Show more