Merge stills-combined into rc181: stills partiality, error model and merging; force-still, observation export, outer-shadow mask, JUNGFRAU module geometry, model-inert stills

38 first-parent commits from the stills campaign (fork point bf0f95900, head e797ed720); details and
measured effects in the branch's commits. Conflicts resolved by keeping both sides: the rotation ISa
with its bootstrap interval (ISA_LOW / ISA_HIGH) stays on rotation, stills report ERROR_MODEL_A/B/C and
no ISa (ErrorModel.{h,cpp}, ResultReport.cpp, ResultReportTest.cpp, docs/RUGNUX_REPORT.md). The
rotation crash guard 4cb911d67 is the same patch as 9589e99b8 already here.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqrcrEZy6XBUCMZvyT
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-01 15:26:13 +01:00
commit a9e3dcb685
68 files changed
+2742 -918

No files matched your search

+37
View File
@@ -4,6 +4,7 @@
#include "DetectorModules.h"
#include "DetectorGeometry.h"
#include "Definitions.h"
#include "PixelMask.h"
#include <algorithm>
#include <cmath>
@@ -193,3 +194,39 @@ DetectorModules ModulesAsImaged(const DetectorGeometry &geom) {
}
return DetectorModules(out);
}
DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height) {
if (width <= 0 || height <= 0 || mask.size() != static_cast<size_t>(width * height))
return {};
const auto clear = [&](int64_t x, int64_t y) {
return x >= 0 && y >= 0 && x < width && y < height
&& (mask[y * width + x] & (1u << PixelMask::ModuleGapPixelBit)) == 0;
};
std::vector<uint8_t> taken(mask.size(), 0);
std::vector<DetectorModule> out;
for (int64_t y = 0; y < height; y++)
for (int64_t x = 0; x < width; x++) {
if (!clear(x, y) || taken[y * width + x])
continue;
// The first clear pixel of a region not yet taken is a module's corner: the module runs to the
// gap along the row and down the column, and has to be whole and bounded by gap all round.
int64_t w = 0, h = 0;
while (clear(x + w, y)) w++;
while (clear(x, y + h)) h++;
if (!((w == CONVERTED_MODULE_COLS && h == CONVERTED_MODULE_LINES)
|| (w == CONVERTED_MODULE_LINES && h == CONVERTED_MODULE_COLS)))
return {};
for (int64_t yy = y - 1; yy <= y + h; yy++)
for (int64_t xx = x - 1; xx <= x + w; xx++) {
const bool inside = xx >= x && xx < x + w && yy >= y && yy < y + h;
if (clear(xx, yy) != inside)
return {};
if (inside)
taken[yy * width + xx] = 1;
}
out.push_back(DetectorModule::AsImaged(x, y, w, h));
}
if (out.size() < 2)
return {};
return DetectorModules(out);
}
+6
View File
@@ -121,3 +121,9 @@ public:
// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES turned the way the geometry turns it, placed as imaged.
// Empty for a detector of one module.
DetectorModules ModulesAsImaged(const DetectorGeometry &geom);
// The modules of a JUNGFRAU image where only its pixel mask says where they are: every region of pixels
// clear of the module-gap bit (PixelMask::ModuleGapPixelBit) has to be one whole module,
// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES either way round, and is taken as placed as imaged.
// Empty where any region is not, or where there are fewer than two. The mask is row-major, width x height.
DetectorModules ModulesFromGapMask(const std::vector<uint32_t> &mask, int64_t width, int64_t height);
+4
View File
@@ -653,6 +653,10 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() {
void DiffractionExperiment::FillMessage(StartMessage &message) const {
message.images_per_file = GetImagesPerFile();
message.detector_modules = GetModules().Get();
// A JUNGFRAU image assembled here is made of modules this experiment places itself: they go into
// the file one by one, so that what reads it back knows them.
if (message.detector_modules.empty() && GetDetectorType() == DetectorType::JUNGFRAU && IsGeometryTransformed())
message.detector_modules = ModulesAsImaged(detector.GetGeometry()).Get();
message.beam_center_x = GetBeamX_pxl();
message.beam_center_y = GetBeamY_pxl();
const auto direct_beam = GetDiffractionGeometry().GetDirectBeam_pxl();
+3 -1
View File
@@ -473,8 +473,10 @@ struct EndMessage {
// setting the merge works in - is only chosen afterwards, so the two settings are not always the
// same one. Absent means they are, i.e. the identity.
std::optional<std::array<int32_t, 9>> reindex_matrix;
// Stills: the rocking curve the partialities were computed from (and the frames predicted through).
// Stills: the rocking curve the frames were predicted through (and the partialities computed from).
std::optional<StillsRunWidth> stills_run_width;
// Stills: the cell given on the command line, whose population of frames the merge keeps.
std::optional<UnitCell> stills_given_cell;
// Geometry refined by the offline analysis. When present these overwrite the master-file geometry
// at finalization (rugnux writes the refined values; the broker leaves them empty and keeps the
+1 -1
View File
@@ -24,7 +24,7 @@ struct Reflection {
float bkg;
float var_bkg; // non-signal (background) part of sigma^2, carried to the merge
float sigma;
float dist_ewald;
float dist_ewald; // stills: signed excitation error |S| - 1/lambda (1/A), positive outside the sphere
// The reciprocal Lorentz factor (rotation only - a still's Lorentz factor is one) times the
// reciprocal polarization factor, and nothing else. This is what LP means everywhere in the
// field, and it is what the CBOR key "rlp" and the HDF5 dataset "lp" store the reciprocal of.
+10
View File
@@ -78,6 +78,16 @@ high-throughput reduction and analysis of serial femtosecond X-ray diffraction d
J. Appl. Cryst. 47, 1118-1131
[doi:10.1107/S1600576714007626](https://doi.org/10.1107/S1600576714007626).
**Per-crystal post-refinement of serial stills** - refining, for each still, the partiality
parameters the spot positions cannot pin (a small reorientation, the crystal's own rocking width and the
radial scale of its reciprocal lattice, which a shot's photon energy or a crystal's cell moves) against
the merged intensities follows the post-refinement of PRIME and the mosaic-block model of Sauter et al.:
M. Uervirojnangkoorn, O. B. Zeldin, A. Y. Lyubimov, J. Hattne, A. S. Brewster, N. K. Sauter, A. T. Brunger
and W. I. Weis, "Enabling X-ray free electron laser crystallography for challenging biological systems
from a limited number of crystals" (2015), eLife 4, e05421; N. K. Sauter, J. Hattne, A. S. Brewster,
N. Echols, P. H. Zwart and P. D. Adams, "Improved crystal orientation and physical properties from
single-shot XFEL stills" (2014), Acta Cryst. D70, 3299-3309 (DOIs not verified here, so not given).
Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the
ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann,
A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP
+2 -1
View File
@@ -431,4 +431,5 @@ gauge holds their spot-weighted mean shift, turn and outward push, and with the
also the two tilt-like gradients across the detector, at zero, so the beam centre, distance, tilt and
crystal orientations carry what all modules share. A coarse search of each module's position comes
first, scored like the beam centre by the spots that land on a lattice. The placement is used from
there on for a foreign multi-panel detector; for a JUNGFRAU master it is measured and reported only.
there on for a foreign multi-panel detector and for modules given as a CrystFEL `.geom`; for a JUNGFRAU
master it is measured and reported only.
+3 -3
View File
@@ -341,7 +341,7 @@ The partiality applied is fixed by the data type and scaling stage, not chosen f
3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma(d^*)$ — the run's rocking width, measured once on the intensity-weighted excitation errors and shared by every crystal. It is measured per group of frames of similar prediction window (four per shell), each against its own window, the shell taking the median of the groups whose curve is narrower than their window, so the frames with the widest windows do not set it everywhere; and it is measured in eight shells of equal width in $d^*$, made non-decreasing (pool-adjacent-violators, weighted by each shell's summed signal) and interpolated linearly in $d^*$ as a table. The three-term form $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius, the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature — is still fitted, sets the prediction window (§8.2) and stands in for the table where fewer than three shells are measurable. The curve is centred on the run's **radial scale** of the reciprocal lattice — a wavelength or cell a little off, which the spot positions cannot see since they fix only $\lambda d^*$ — measured from where the curve's centre lies against $d^{*2}$ and applied to $\mathbf{q}$. Where the frames were predicted through the measured window (§8.2), the post-refinement uses that same curve rather than measuring it again; the curve is written to `_process.h5` (`/entry/MX/stillsRockingWidth*`, `stillsRadialScale`), so `--mode scale` merges with it too. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times; the refined partiality is adopted on every crystal it was refined on. The per-crystal fits of $G$ and of the tilt measure each residual against its expected variance at the model, background + counting + $(0.3\,I_\mathrm{model})^2$: a still's strong reflections scatter about the model by tens of per cent, and in counting sigmas alone every strong reflection would be an outlier and the scale would be set by the weak ones.
Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy.
Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. On stills this holds for the merges the scale and the partiality are refined against. The final merge, which has the fitted error model, takes every observation: the model's partiality term $c\,((1-p)/p)\,\langle I\rangle$ (§10.5) weights a low-partiality observation by what it is worth. A reflection - each Bijvoet hand under `-A` - is reported only if at least one of its observations reaches $p \ge 0.2$; one seen only far out on its rocking curve rests on the partiality model alone. The error model is fitted on the observations at $p \ge 0.2$.
### 10.3 Smoothing of per-frame scales
@@ -388,9 +388,9 @@ Per-shell and overall merging statistics are computed on corrected intensities,
itself count as missing instead of leaving the denominator along with the data,
- the anomalous signal-to-noise $\mathrm{SigAno}$ and the half-set anomalous correlation $\mathrm{CC}_\mathrm{anom}$ (below).
The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly.
The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path adds a partiality term, $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2 + c^2\left(\frac{1-p}{p}\right)^2\langle I\rangle^2$, and reports **no ISa**: there $1/b$ is the $I/\sigma$ ceiling of one fully recorded still under the stills model - the same frames processed as a rotation sweep read several times higher - which describes the processing model more than the instrument and the crystal. It reports $a$, $b$ and $c$ instead (`ERROR_MODEL_A/B/C`, `_reflns.jfjoch_error_model_a/b/c`); $1/b$ is still used inside, by the space-group search.
$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention.
$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$ (and on stills $c_\mathrm{XDS} = c^2/a$). Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention.
**Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ($\mathrm{CC}_\mathrm{anom}$) is reported beside it: $\Delta I$ is formed once per half-set and the two are correlated over the acentric pairs where **both** hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic $\sigma$ cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS's `CCanom` and `phenix.merging_statistics`' `cc_anom`; XDS's `Anomal Corr` is a **different quantity and is not comparable with it** — on the same observations it reads two to three times higher in the low shells. $\mathrm{CC}_\mathrm{anom}$ is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the `CCanom` column of the printed merge-statistics table and as `CC_ANOM=` in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints `-` and the results report writes no key, which is not the same statement as a value measured to be zero.
+7 -7
View File
@@ -30,8 +30,8 @@ the paper of the entry.
| `lcls242_s`, `lcls242_t` | [242](https://cxidb.org/id-242.html), [10.11577/3374214](https://doi.org/10.11577/3374214) | LCLS MFX | Rayonix MX340-HS; LCLS-I XTC | [9O4S](https://www.rcsb.org/structure/9O4S) (small cell), [9O4T](https://www.rcsb.org/structure/9O4T) (large cell) | P 1 21 1, 1.95 | Wolff et al., bioRxiv (2026) |
| `euxfel83` | [83](https://cxidb.org/id-83.html), [10.11577/1464102](https://doi.org/10.11577/1464102) | European XFEL SPB/SFX | AGIPD 1M, 16 modules of 512 x 128, 200 µm; Cheetah CXI | [6GTH](https://www.rcsb.org/structure/6GTH) | P 32 2 1, 1.69 | Wiedorn et al., Nat. Commun. (2018) |
| `euxfel152` | [152](https://cxidb.org/id-152.html), [10.11577/1660970](https://doi.org/10.11577/1660970) | European XFEL SPB/SFX | AGIPD 1M; Cheetah CXI | [6U57](https://www.rcsb.org/structure/6U57) | I 2 3, 2.80 | Echelmeier et al., Nat. Commun. (2020) |
| `alvra103`, `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) |
| `alvra213`, `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) |
| `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) |
| `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) |
Every set of the manifest is in the table and every set in the table is in the manifest, with the
same CXIDB entry, PDB entry, space group and d<sub>min</sub>; `tools/battery/doc_check.py` checks
@@ -63,8 +63,8 @@ broken download resumes with `curl -C -`.
| `lcls242_*` | `lcls_242/xtc/`, `lcls_242/xtc_more/`: `xtc/mfxl1013721-r0256` to `r0267` streams (the two crystal forms are in every run; each set keeps its own form) | ~90,000 | **subset**: runs 256-267 (44 of about 220 stream files; ~610 GB of ~3 TB) | ~700 GB |
| `euxfel83` | `euxfel_83/raw/`: `XFEL-r0243-c00.cxi`, `XFEL-r0278-c00.cxi`, `XFEL-r0280-c00.cxi` (Cheetah hits, from the entry's `cheetah-calib-mar-v1/hdf5/` tree on the NERSC archive); `euxfel_83/ref/agipd_mar18_v10.geom` from the entry | 2,790 | **subset**: 3 files; the deposited stream has 14,445 frames | 4.7 GB |
| `euxfel152` | `euxfel_152/raw/`: runs r0108, 0110, 0112, 0114, 0115, 0116, 0118, 0123, 0125 extracted from `XFEL2042-runs-cxi.tar` (range requests on the tar members); `euxfel_152/ref/kdo_embedded.geom`, the geometry embedded in the header of the entry's `KDO8PS.stream` | 3,801 | **subset**: 9 runs; the deposited stream has 37,006 frames | 5.7 GB |
| `alvra103` | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB |
| `alvra213` | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB |
| `alvra103_int16` (source) | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB |
| `alvra213_int16` (source) | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB |
**The masks of the SwissFEL geometry files.** rugnux takes the pixel masks and bad regions a CrystFEL
geometry file names. The SwissFEL files name their mask with an absolute path on the facility's disk,
@@ -78,9 +78,9 @@ that line differs.
**The `_int16` sets.** The SwissFEL files store converted JUNGFRAU data as uncompressed float32 keV,
68 MB (JF16M) or 17 MB (JF4M) a frame, which caps reading at about 74 or 295 frames/s at 5 GB/s. The
`_int16` sets are copies with the same layout in which `/data/data` holds whole photons (keV divided
by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller; the battery runs
them with the same arguments as the float32 originals (rugnux measures the gain from the frames), and
they merge to the same numbers. Make them with
by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller. The battery runs only the
copies: they merge to the same numbers as the float32 originals (rugnux measures the gain from the
frames), and the originals are kept only as the source of the conversion and as a test of the format. Make them with
`tools/battery/still/float32_to_int16.py <in.h5> <out.h5> <photon energy keV>` (h5py and hdf5plugin),
into `swissfel_103/conv/` (4.57 keV) and `swissfel_213/conv/` (12.037 keV, keeping the tarball's
directory tree), named `<input>_int16.h5`. One MbCO dark run (`run_000238`) holds no frames and has no copy.
+5 -2
View File
@@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data.
This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead
of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data`
VDS still resolves to the original images when they are available. `rugnux` can also process
VDS still resolves to the original images when they are available. Where the input is anything but a
single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no
`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process
an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set.
## 3. NXmx-standard content
@@ -397,7 +399,8 @@ variants.
| `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below |
| `stillsRockingWidthFormula` | 1/Å, rad, relative | stills: the run's rocking width σ² = γ₀² + (γₑ d\*)² + (σ_bw λ d\*²/2)², as `[γ₀, γₑ, σ_bw]`; offline (Rugnux) only |
| `stillsRadialScale` | | stills: the scale of the reciprocal lattice the rocking curve is centred on |
| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\*, which the partialities use (held beyond the first and last entry; the formula above is what the prediction window uses); `--mode scale` merges with formula and table instead of measuring another |
| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\* before the image pass, which the frames were predicted through (held beyond the first and last entry; the formula above is what the prediction window uses); the partiality post-refinement balances it against the merge, and `--mode scale` starts from this same curve instead of measuring another |
| `stillsGivenUnitCell` | Å, deg | stills: the cell given with `-C`, whose population of frames the merge kept (`[6]`); `--mode scale` leaves out the same frames |
**Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is
processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by
+17 -14
View File
@@ -46,10 +46,10 @@ against a foreign dataset injects that dataset's systematics — so `-z` never p
errors into the intensities. `--reference-column` picks the column to read where the automatic
choice (`F-model`, else `IMEAN`/`I`, else `FP`/`FOBS`/`F`) is not the right one.
For the indexing ambiguity an atomic model does as well: `--model` computes the intensities it
needs from the structure (a model that fits also sets the hand label and the setting, see
[Validating against a model](#validating-against-a-model-rugnux---model)). Where a reference dataset exists, prefer it;
where only a model does, it resolves the ambiguity just the same.
An atomic model does not take part in the processing: `--model` is placed and validated after the merge,
and a run with one merges exactly what a run without one does. A model that fits then sets the frame the
merge is written in - the hand label, the setting and the indexing (see
[Validating against a model](#validating-against-a-model-rugnux---model)) - by relabelling it, never by re-merging.
### The indexing ambiguity
@@ -102,16 +102,15 @@ How to resolve it:
| **Rotation**, a reference dataset exists | `-z reference.mtz`. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the *hkl* labels change; the cell does not |
| **Serial stills**, a reference dataset exists | `-z reference.mtz`. Resolved **per image**, at integration time, by correlating each crystal's intensities with the reference — so the merge never mixes hands in the first place. Where the crystals' own cells tell the indexings apart (pseudo-merohedry), a crystal its cell decides keeps that indexing and the reference decides the rest. The data-only resolution below runs as a check: `AMBIGUITY_DATA_DISAGREES=` |
| **Rotation**, only a model | `--model model.pdb`. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (`MODEL_FIT= ACCEPTED`) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (`MODEL_INDEXING_MARGIN_SIGMA`). It is the **data** that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model's indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand |
| **Serial stills**, only a model | `--model model.pdb`, with the cell and space group given (`-C` / `-S`). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be, the crystals their own cell decides again excepted. The data-only resolution below is then run only as a check: `AMBIGUITY_DATA_DISAGREES=` counts the crystals on which it would have chosen otherwise |
| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A model or reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing |
| **Serial stills**, only a model, space group given (`-S`) | The crystals are made consistent with each other from the data alone, as in the row below - the model plays no part in the processing. Validation then fits the merge to the model in each candidate indexing, as for rotation data, and relabels the written reflections into the model's indexing where it wins |
| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing |
| Neither | The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame; for stills whose space group the run determined, a merge of mixed indexings, which looks like a perfect twin and can pass for higher symmetry than the crystal has. Run again with `-S` |
Two things to know about where the choice lands:
- **`--mode scale` cannot repair a stills run after the fact.** The per-image test happens at
integration time, so a `_process.h5` whose images were integrated without a reference — an MTZ or a
model — has already lost the distinction, and no re-merge brings it back; the data-only resolution
runs in the full pipeline, not in `--mode scale`. On rotation data, where
- **`--mode scale` resolves it as the full run does.** On stills with no reference MTZ it makes the
crystals consistent from the data alone before merging; the per-image test against a reference
happens at integration time and is kept in the stored reflections. On rotation data, where
the ambiguity is one choice for the whole dataset, `--mode scale --model model.pdb` does resolve it.
- **The choice reaches the written reflections**, not only the R-factors and the maps: the merged
`.mtz` / `.cif` / `.hkl` (and `_unmerged.mtz`, where it is asked for) are written in the indexing
@@ -291,9 +290,13 @@ declines it and the run stops with a message saying so. `--model` works here as
that the model is placed by the rigid body only: a model it does not place, or a prediction, is not
searched for by molecular replacement, `--mr` is refused, and the placed model is not refined. On rotation data, where the
[indexing ambiguity](#the-indexing-ambiguity) is one choice for the whole dataset, it can settle the
enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. What no re-merge can
repair is a **stills** `_process.h5` integrated without a reference: there the ambiguity was
decided per image, at integration time, and the distinction is gone from the stored reflections.
enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. On
stills the re-merge is the full run's own merge: the same frames (a given cell's population, as
`/entry/MX/stillsGivenUnitCell` records it), the same rocking curve, the same data-only resolution of
the indexing ambiguity and the same scaling, so a
`_process.h5` merged again with the options it was written with reproduces the full run's reflections
(with a reference MTZ, the full run also lets the crystals' own cells overrule the reference's indexing
before merging, which `--mode scale` does not).
Where the full run re-seated the lattice — the space group it settled on is in a different setting
from the one each image was indexed in — the file records the change of basis as
@@ -312,7 +315,7 @@ describes them - the cell and space group; `_software` with Rugnux named twice,
beamline and its general class, the detector's make and model in `_diffrn_detector.type` and its
class in `_diffrn_detector.detector`, and the collection date from the NXmx start time); and the
merging statistics overall (`_reflns`) and per shell (`_reflns_shell`): completeness, multiplicity,
<I/σ(I)>, R_merge, R_meas, R_pim, CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries
<I/σ(I)>, R_merge, R_meas, R_pim (`?` on a stills merge, where they mean nothing), CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries
its e.s.d.s (`_cell.length_a_esd`, …). A
quantity the input does not state is left out rather than guessed. The unmerged observations are in
the `_unmerged.mtz` and the `.hkl` (both scaled on a rotation merge), not in the `.cif`.
+2 -2
View File
@@ -304,7 +304,7 @@ serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rota
| `-z, --reference <file>` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity), hands over the R-free set and reports CC<sub>ref</sub>. Not a scale anchor |
| `--free-flags <file>` | Take only the R-free test set of this MTZ or structure-factor mmCIF - e.g. the deposited reflections of the `--model` - for the validation, the refinement and the merged files; nothing else of the file is used, so unlike `-z` it fixes no space group, cell or frame. Taken on the model's axes where the merge matches the file's intensities (`R_FREE_FLAGS=`, see [Free set and model bias](RUGNUX_MODEL_AND_REFERENCE.md#free-set-and-model-bias)) |
| `--reference-column <label>` | Reference column to use, by MTZ label — an mmCIF under the labels `gemmi cif2mtz` gives it (default: auto — F-model, else IMEAN/I/…, else FP/FOBS/F) |
| `--model <file>` | Validate the merged intensities against this atomic model (PDB or mmCIF, gzipped or not; the format is taken from the file's content) — R-work / R-free and maps (see [Validating against a model](RUGNUX_ADVANCED.md#validating-against-a-model-rugnux---model)). An experimental model is placed as a rigid body, and by molecular replacement where the rigid body does not place it; a prediction (AlphaFold or ModelCIF) goes to molecular replacement straight away (`MODEL_PLACEMENT=`). It also settles the frame the reflections are written in: the enantiomorph, and the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity) where no `-z` did; another space group or the twin hypothesis' subgroup it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif`. For serial stills given `-C` / `-S`, the model's structure factors become the per-image reference. A placed model is then [refined](RUGNUX_MODEL_AND_REFERENCE.md#refinement) against the merge it was placed on (`<prefix>_refined_model.cif`, `<prefix>_refined_maps.mtz`, `<prefix>_blobs.json`, `REFINE_` keys) - seconds on a GPU, typically 2-15 minutes on the CPU - which decides nothing about the processing |
| `--model <file>` | Validate the merged intensities against this atomic model (PDB or mmCIF, gzipped or not; the format is taken from the file's content) — R-work / R-free and maps (see [Validating against a model](RUGNUX_ADVANCED.md#validating-against-a-model-rugnux---model)). An experimental model is placed as a rigid body, and by molecular replacement where the rigid body does not place it; a prediction (AlphaFold or ModelCIF) goes to molecular replacement straight away (`MODEL_PLACEMENT=`). It also settles the frame the reflections are written in: the enantiomorph, and the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity) where no `-z` did; another space group or the twin hypothesis' subgroup it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif`. It does not change the processing: the merge is the same with and without it. A placed model is then [refined](RUGNUX_MODEL_AND_REFERENCE.md#refinement) against the merge it was placed on (`<prefix>_refined_model.cif`, `<prefix>_refined_maps.mtz`, `<prefix>_blobs.json`, `REFINE_` keys) - seconds on a GPU, typically 2-15 minutes on the CPU - which decides nothing about the processing |
| `--mr <file>` | As `--model`, but placed by molecular replacement straight away, without trying the rigid body first: as many copies as the cell holds, in every space group the merge cannot tell apart and in the twin hypothesis' subgroup. A prediction is trimmed to the residues of pLDDT 70 and above, and where the AlphaFold DB's predicted-aligned-error JSON sits beside it under its own name it is cut into rigid domains; any other PDB or mmCIF is taken as an experimental structure, its polymer only (no waters, ligands, hydrogens or alternative conformers). It may relabel the enantiomorph of the answer, and a different space group it chooses is written beside the answer as `<prefix>_mr.mtz`/`.cif` with the placed model `<prefix>_mr_model.pdb`. `--mode mx` only; not with `--model` |
| `--no-refine` | Do not refine the placed model (for debugging): by default a model that `--model` or `--mr` places is refined |
| `--no-refine-solvent` | Refine the placed model with its solvent as given: no waters built or removed, no ion relabelled (the solvent audit - the `REFINE_SOLVENT_` keys and `<prefix>_solvent.json` - is still made). Also with `--mode refine` |
@@ -350,7 +350,7 @@ Defaults are taken from the input file; override them to reprocess with a correc
| `--saturation <num>` | Pixel value from which a pixel is overloaded, in the units the frames are read in. Replaces the file's limit and the one stills read off the frames |
| `--crystfel-geometry <file>` | Place the panels of an HDF5/CXI input by a CrystFEL geometry file instead of the file's own metadata: module geometry, camera length, mask arrays and bad regions (optional; [Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)). Photon energy, camera length and - where the geometry names no mask array - the masks otherwise come from the data files (Cheetah CXI, SACLA) or the command line; gain and saturation are measured. Tilted panels, `dimN` layouts, per-panel data and detector shifts are refused |
| `--module-geometry <file>` | Where the detector's modules are: an NXmx file with one NXdetector_module per module (what a run writes as `<prefix>_modules.h5`) or a CrystFEL geometry file whose panels are laid on the image. Overrides the input's own |
| `--refine-modules[=on\|off\|diagnose]` | Stills: place each module as a rigid block in the geometry bundle adjustment, where that predicts held-out spots better (default: on). Used for a foreign multi-panel detector, measured and reported only for a JUNGFRAU; the placement is written to `<prefix>_modules.h5`. `=diagnose` also runs a set of tests on the placement and writes them to the log |
| `--refine-modules[=on\|off\|diagnose]` | Stills: place each module as a rigid block in the geometry bundle adjustment, where that predicts held-out spots better (default: on). Used for a foreign multi-panel detector and for modules from a CrystFEL `.geom` (`--crystfel-geometry`, `--module-geometry`), measured and reported only for a JUNGFRAU; the placement is written to `<prefix>_modules.h5`. `=diagnose` also runs a set of tests on the placement and writes them to the log |
| `--mask-good [<file>:]<dataset>[@<bits>]` / `--mask-bad [<file>:]<dataset>[@<bits>]` | HDF5/CXI stills: a mask array in the frames' stored layout, in `<file>` or in each data file; usable only where non-zero (or with all `<bits>`) / unusable where non-zero (or with any `<bits>`). Repeatable ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)) |
| `--rot1 <num>` | PONI detector rotation 1 (rad). On stills, `--rot1` or `--rot2` holds the tilt (not refined); without them stills refine the tilt where held-out frames agree |
| `--rot2 <num>` | PONI detector rotation 2 (rad) |
+3 -2
View File
@@ -155,8 +155,9 @@ places each module with enough spots on its frames' lattices as a rigid block (s
`--refine-modules`, on by default) where that predicts held-out spots better; modules move only relative
to one another, the beam centre, distance and crystal orientations carrying what all of them share. The
placement is used from there on for the panels of a foreign multi-panel detector (the sensors of a
panel-table file, the panels of a `--crystfel-geometry` file); for a JUNGFRAU (an NXmx master) it is
measured and reported only. Either way it is written to `<prefix>_modules.h5`, which a later run reads
panel-table file, the panels of a `--crystfel-geometry` file or of a CrystFEL `.geom` given with
`--module-geometry`); for a JUNGFRAU (an NXmx master, with the modules the file gives) it is measured and
reported only. Either way it is written to `<prefix>_modules.h5`, which a later run reads
back with `--module-geometry`.
Spots are always found by `rugnux` itself, including for the two-pass rotation first pass — the
+2 -1
View File
@@ -24,9 +24,10 @@ ignores them, and one that does can find them without guessing.
| item | meaning |
|---|---|
| `_reflns.jfjoch_diffrn_ISa` | Asymptotic I/σ in **XDS's sense**: the whole-range `1/√(a·b)` of the error model, so it can be read directly against a `CORRECT.LP` |
| `_reflns.jfjoch_diffrn_ISa` | Asymptotic I/σ in **XDS's sense**: the whole-range `1/√(a·b)` of the error model, so it can be read directly against a `CORRECT.LP`. Rotation only: a stills merge writes the error-model terms instead |
| `_reflns.jfjoch_diffrn_ISa_asymptotic` | The **strong-reflection** tier — the counting-subtracted scatter of well-measured groups. XDS has no equivalent, and it can only ever be the more optimistic of the two. Rotation path only |
| `_reflns.jfjoch_error_model_a`, `_b` | The error model in XDS's convention, `σ² = a(σ₀² + b·I²)`, so the ISa above is re-derivable from the file rather than taken on trust |
| `_reflns.jfjoch_error_model_c` | Stills only: the partiality term, `σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²)` with p the observation's partiality |
| `_reflns.jfjoch_second_moment_I` | Twinning second moment ⟨I²⟩/⟨I⟩² — 2.00 untwinned, 1.50 for a perfect twin |
| `_reflns.jfjoch_L_test_mean_abs_L`, `_L_test_mean_L_squared` | Padilla–Yeates L-test. ⟨\|L\|⟩ is 0.500 untwinned / 0.375 for a perfect twin; ⟨L²⟩ is 0.333 / 0.200. Written only when the test found pairs |
| `_reflns.jfjoch_radiation_damage_relative_B` | Relative *B* from the first to the last rotation batch (Ų); positive is the usual direction, high-resolution intensity fading with dose |
+22 -5
View File
@@ -73,8 +73,13 @@ half-sets - as CrystFEL's partialator deals out crystals - and R_split = (1/sqrt
(1/2 sum (I_1 + I_2)) (White et al., 2012) and CC1/2 are taken over the reflections both halves
measured, so both compare directly with CrystFEL's `compare_hkl`. `CC_HALF=` deals each reflection's
observations out evenly instead, as XDS and cctbx do, and is the one the processing decides on; the two
agree overall and can differ in a single shell. On stills `R_MEAS=` measures the scatter of partial
observations, not the error of the merged intensity, and is not the number to quote.
agree overall and can differ in a single shell.
**`R_MERGE=`, `R_MEAS=`, `R_PIM=` and `R_MEAS_WEIGHTED=` are rotation statistics** and a stills report
has none of them, nor their `REFRES_` forms or their columns in the shell tables (the mmCIF writes `?`
for them). On stills they would measure the scatter of partial observations scaled to full
equivalents - hundreds of percent - not the error of the merged intensity. `R_SPLIT=`, `CC_HALF=`,
`CC_STAR=`, `CC_ANOM=`, `MULTIPLICITY=` and `I_OVER_SIGMA=` are reported as on rotation.
**`OBSERVATIONS_REJECTED=`** counts the observations the merge's outlier tests removed, and
**`OBSERVATIONS_REJECTED_WILSON=`** the part of them the Wilson test took (each listed in the developer
@@ -234,13 +239,25 @@ The keys most scripts read, by section; the rest of this page explains the ones
| `INCLUDE_RESOLUTION_RANGE` | `dmax dmin` of the written reflections (Å), as XDS's keyword |
| `FITTED_RESOLUTION` | Where the CC1/2 fall-off crosses the `--resolution-cc-target`; absent where the curve is not a fall-off (a `RESOLUTION_FIT` warning says why) |
| `UNIQUE_REFLECTIONS`, `TOTAL_OBSERVATIONS`, `COMPLETENESS`, `MULTIPLICITY` | Over the written range; completeness in percent |
| `I_OVER_SIGMA`, `R_MEAS`, `CC_HALF`, `WILSON_B` | The overall merging statistics; the shell table beside them is in the comment lines |
| `I_OVER_SIGMA`, `R_MEAS`, `CC_HALF`, `WILSON_B` | The overall merging statistics; the shell table beside them is in the comment lines. `R_MEAS` (and `R_MERGE`, `R_PIM`) rotation only, `R_SPLIT` stills only |
| `SIGANO`, `CC_ANOM`, `FRIEDELS_LAW` | The anomalous signal, and whether Friedel pairs were merged (`FALSE` with `-A`) |
| `ERROR_MODEL_A`, `ERROR_MODEL_B`, `ISA` | The error model in XDS's convention and ISa = 1/sqrt(a·b); `ISA= undetermined` where b is zero within its error, so 1/b would be noise |
| `ISA_LOW`, `ISA_HIGH` | Rotation data: the 95% interval of `ISA` from a bootstrap over the reflections (100 replicates, fixed draws). With `ISA= undetermined`, `ISA_LOW` alone is a lower bound: b reaches zero inside its interval, so only ISa ≥ `ISA_LOW` is measured. Both absent where there is no fit to resample (b out of reach of the data, or a on its bound) and on stills, where `ISA` follows the fit's own two-standard-error test |
| `ERROR_MODEL_A`, `ERROR_MODEL_B`, `ISA` | The error model in XDS's convention and ISa = 1/sqrt(a·b); `ISA= undetermined` where b is zero within its error, so 1/b would be noise. Rotation only for `ISA` |
| `ISA_LOW`, `ISA_HIGH` | Rotation data: the 95% interval of `ISA` from a bootstrap over the reflections (100 replicates, fixed draws). With `ISA= undetermined`, `ISA_LOW` alone is a lower bound: b reaches zero inside its interval, so only ISa ≥ `ISA_LOW` is measured. Both absent where there is no fit to resample (b out of reach of the data, or a on its bound) and on stills, which write no `ISA` |
| `ERROR_MODEL_C` | Stills only: the partiality term of the stills error model, σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²) in the same convention. A stills report writes no `ISA` ([below](#the-stills-error-model)), and `ERROR_MODEL_B= undetermined` where b is zero within its error |
| `R_WORK`, `R_FREE` | Section 5: the model as placed against these data ([below](#model-validation)); the refined model's are `REFINE_R_WORK`/`REFINE_R_FREE` |
| `R_FREE_FLAGS` | Section 5: the free set both are read on - `DRAWN` by this run, inherited from the `-z` `REFERENCE`, or from the `--free-flags` `FILE` |
### The stills error model
A stills merge reports its error model as its terms, `ERROR_MODEL_A`, `ERROR_MODEL_B` and
`ERROR_MODEL_C`, with σ² = a(σ₀² + b·I² + c·((1−p)/p)²·I²), p being the partiality of the observation:
b is the systematic error of a fully recorded still and c the error a partial one adds. It writes no
`ISA`. On stills 1/sqrt(a·b) is the I/σ ceiling of a single still observation under the stills model -
the same frames processed as a rotation sweep read several times higher - so it describes the
processing model more than the instrument and the crystal, and read on `CORRECT.LP`'s scale it would
call good serial data useless. The serial figures of merit are `R_SPLIT=`, `CC_HALF=` and `CC_STAR=`.
The mmCIF follows: `_reflns.jfjoch_error_model_c` instead of `_reflns.jfjoch_diffrn_ISa`.
## The summary, and what the run decided
The file opens with a **`SUMMARY`** section, above everything it summarises. It exists because the
+3 -3
View File
@@ -189,9 +189,9 @@ A crystal form with an [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambi
relatives — needs the space group on the run that integrates: every crystal is indexed in one of the
alternative indexings at random, and they are averaged together in the merge unless the crystals are put
into one first. With `-S` given, the run does that from the data alone, making the crystals consistent
with each other (`INDEXING_AMBIGUITY_RESOLVED= data`); a reference (`-z`), or `--model model.pdb`
together with `-C` and `-S`, puts each image into the reference's indexing instead, so that the merge
also comes out in it. Without `-S` and without either, the mixed merge can look more symmetric than the
with each other (`INDEXING_AMBIGUITY_RESOLVED= data`); a reference (`-z`) puts each image into the
reference's indexing instead, so that the merge also comes out in it (a `--model` relabels the finished
merge into its indexing and changes nothing else). Without `-S` and without a reference, the mixed merge can look more symmetric than the
crystal is, and the run warns.
`ffbidx` requires a known cell (`-C`) and is the indexer of choice for sparse serial stills; the
+1
View File
@@ -656,6 +656,7 @@ void IndexAndRefine::QuickPredictAndIntegrate(DataMessage &msg,
settings_prediction.window_ang = stills_window_->ang;
settings_prediction.window_bw = stills_window_->bw;
settings_prediction.window_q_scale = stills_window_->q_scale;
settings_prediction.window_radial = stills_window_->radial;
}
// Online is bounded by what the image-buffer slot can carry, offline by the prediction limit; both
+1
View File
@@ -97,6 +97,7 @@ struct StillsWindow {
float ang = 0.0f;
float bw = 0.0f;
float q_scale = 1.0f; // centred on the excitation error of the reciprocal lattice scaled by this
float radial = 0.0f; // where an off-sphere reflection is placed (BraggPredictionSettings::window_radial)
};
class IndexAndRefine {
+111 -2
View File
@@ -428,8 +428,9 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
// ISa in XDS's sense: the whole-range 1/sqrt(a*b) of the error model below, so this item can
// be read straight against a CORRECT.LP. The strong-reflection asymptote - a tier XDS does not
// have, and always the more optimistic of the two - is written separately rather than here.
out << "_reflns.jfjoch_diffrn_ISa " << CifStr(error_model.isa)
<< " # 1/sqrt(a*b), the XDS convention\n";
if (!statistics.stills)
out << "_reflns.jfjoch_diffrn_ISa " << CifStr(error_model.isa)
<< " # 1/sqrt(a*b), the XDS convention\n";
if (!error_model.isa_asymptotic.empty())
out << "_reflns.jfjoch_diffrn_ISa_asymptotic " << CifStr(error_model.isa_asymptotic)
<< " # strong-reflection asymptote (Diederichs); rotation path only\n";
@@ -438,6 +439,9 @@ void WriteMmcifReflections(const std::vector<MergedReflection> &reflections,
<< " # sigma^2 = a*(sigma0^2 + b*I^2), XDS convention\n";
if (!error_model.b.empty())
out << "_reflns.jfjoch_error_model_b " << CifStr(error_model.b) << "\n";
if (!error_model.c.empty())
out << "_reflns.jfjoch_error_model_c " << CifStr(error_model.c)
<< " # stills partiality term: + c*((1-p)/p)^2*I^2\n";
// Dataset-wide isotropic Wilson B-factor estimate (standard PDBx item), analogous to XDS's
// "WILSON LINE ... B=". Emitted only when the log-linear fit succeeded.
if (std::isfinite(statistics.wilson_b) && statistics.wilson_b > 0.0)
@@ -1360,3 +1364,108 @@ void WriteReflections(const std::vector<MergedReflection> &reflections,
WriteShelxIns(unitCell, nullptr, experiment, std::filesystem::path(filename).filename().string(),
filename + ".ins");
}
MtzTable ObservationMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const std::vector<std::vector<ObservationVerdict>> &verdicts,
std::optional<double> written_d_min,
const std::optional<StillsRunWidth> &run_width,
const UnitCell &unitCell,
const DiffractionExperiment &experiment) {
constexpr uint32_t OBS_BEYOND_DMIN = 4096, OBS_NO_VERDICT = 2048;
MtzTable mtz;
mtz.space_group_name = experiment.GetSpaceGroupOrP1().xhm();
mtz.unit_cell = unitCell;
mtz.title = "Rugnux stills observations (diagnostic)";
mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f},
{"obs", experiment.GetSampleName(), "obs", experiment.GetWavelength_A()}};
if (run_width) {
std::string t = "RUNWIDTH gamma0 " + std::to_string(run_width->gamma0) + " gamma_e " + std::to_string(run_width->gamma_e)
+ " bw " + std::to_string(run_width->bandwidth_sigma) + " radial_scale "
+ std::to_string(run_width->radial_scale);
mtz.history.push_back(t);
std::string x = "RUNWIDTH_DSTAR", y = "RUNWIDTH_S";
for (size_t i = 0; i < run_width->table_dstar.size(); ++i) {
x += " " + std::to_string(run_width->table_dstar[i]);
y += " " + std::to_string(run_width->table_s[i]);
}
mtz.history.push_back(x);
mtz.history.push_back(y);
}
if (written_d_min)
mtz.history.push_back("WRITTEN_DMIN " + std::to_string(*written_d_min));
const std::vector<std::string> labels = {
"H", "K", "L", "FRAME", "IMG", "XPRED", "YPRED", "XOBS", "YOBS", "PANEL", "D",
"EPS", "EPSR", "EPSABS", "QX", "QY", "QZ", "PART", "ROCKSIG", "ZETA", "DPHI",
"I", "SIGI", "BKG", "VARBKG", "LP", "QE", "FLIGHT", "SCALECORR", "ISCALED", "SIGSCALED", "SIGMERGE",
"G", "CCIMG", "MOSAIC", "ASX", "ASY", "ASZ", "BSX", "BSY", "BSZ", "CSX", "CSY", "CSZ",
"CELLA", "CELLB", "CELLC", "CELLAL", "CELLBE", "CELLGA", "FLAGS", "USED"};
for (const auto &l : labels)
mtz.columns.push_back({l, (l == "H" || l == "K" || l == "L") ? 'H' : (l == "FLAGS" || l == "USED" ? 'I' : 'R'), 0, {}});
// Predictions are on the detector plane, so the panel is the one whose placed footprint holds the
// point (-1 for a detector without panels, -2 in a gap or off every panel).
const auto panel_of = [&](float x, float y) {
const auto p = experiment.GetModules().PlaneToPanel(x, y);
return p ? static_cast<float>(p->panel) : -2.0f;
};
// The run's rocking width at d*, as StillsPartialityRefine evaluates it: its table, linear in d* and held
// beyond the end shells, else the three-term formula.
const auto rock_sigma = [&](double dstar, double qz) {
if (!run_width)
return NAN;
const auto &x = run_width->table_dstar;
const auto &y = run_width->table_s;
if (!x.empty()) {
if (dstar <= x.front()) return static_cast<float>(y.front());
if (dstar >= x.back()) return static_cast<float>(y.back());
const size_t i = std::upper_bound(x.begin(), x.end(), dstar) - x.begin();
return static_cast<float>(y[i - 1] + (dstar - x[i - 1]) / (x[i] - x[i - 1]) * (y[i] - y[i - 1]));
}
const double ang = run_width->gamma_e * dstar, bw = run_width->bandwidth_sigma * qz;
return static_cast<float>(std::sqrt(run_width->gamma0 * run_width->gamma0 + ang * ang + bw * bw));
};
const double radial = run_width ? run_width->radial_scale : 1.0;
std::vector<float> row(labels.size());
const auto push = [&]() {
for (size_t c = 0; c < row.size(); ++c)
mtz.columns[c].values.push_back(row[c]);
};
for (size_t i = 0; i < outcomes.size(); ++i) {
const auto &o = outcomes[i];
if (o.reflections.empty() || o.latt.CalcVolume() <= 1.0f) // an unindexed frame: no lattice
continue;
const bool have_verdicts = i < verdicts.size() && verdicts[i].size() == o.reflections.size();
const Coord A = o.latt.Astar(), B = o.latt.Bstar(), C = o.latt.Cstar();
const Coord S0 = o.geom.GetScatteringVector();
const double inv_lambda = 1.0 / o.geom.GetWavelength_A();
const UnitCell cell = o.latt.GetUnitCell();
for (size_t j = 0; j < o.reflections.size(); ++j) {
const Reflection &r = o.reflections[j];
const Coord q = A * static_cast<float>(r.h) + B * static_cast<float>(r.k) + C * static_cast<float>(r.l);
const auto eps_at = [&](double s) {
const double x = s * q.x + S0.x, y = s * q.y + S0.y, z = s * q.z + S0.z;
return static_cast<float>(std::sqrt(x * x + y * y + z * z) - inv_lambda);
};
const double dstar = std::sqrt(q.x * q.x + q.y * q.y + q.z * q.z);
uint32_t flags = have_verdicts ? verdicts[i][j].flags : OBS_NO_VERDICT;
if (written_d_min && r.d < *written_d_min)
flags |= OBS_BEYOND_DMIN;
row = {static_cast<float>(r.h), static_cast<float>(r.k), static_cast<float>(r.l), r.image_number,
static_cast<float>(i), r.predicted_x, r.predicted_y, r.observed_x, r.observed_y,
panel_of(r.predicted_x, r.predicted_y), r.d,
eps_at(1.0), eps_at(radial), r.dist_ewald, q.x, q.y, q.z, r.partiality,
rock_sigma(radial * dstar, std::fabs(radial * q.z)), r.zeta, r.delta_phi_deg,
r.I, r.sigma, r.bkg, r.var_bkg, r.prescaling_corr, r.qe_corr, r.flight_corr,
r.image_scale_corr, r.I * r.image_scale_corr, r.sigma * r.image_scale_corr,
have_verdicts ? verdicts[i][j].sigma : NAN,
o.image_scale_g.value_or(NAN), o.image_scale_cc.value_or(NAN), o.mosaicity_deg.value_or(NAN),
A.x, A.y, A.z, B.x, B.y, B.z, C.x, C.y, C.z,
cell.a, cell.b, cell.c, cell.alpha, cell.beta, cell.gamma,
static_cast<float>(flags), flags == 0 ? 1.0f : 0.0f};
push();
}
}
return mtz;
}
+18 -1
View File
@@ -11,6 +11,8 @@
#include "../common/DiffractionExperiment.h"
#include "../common/MtzTable.h"
#include "IntegrationOutcome.h"
#include "scale_merge/Merge.h"
#include "../common/StillsRunWidth.h"
struct MergeStatistics;
struct TwinningAnalysisResult;
@@ -19,12 +21,15 @@ struct CellEsd; // geom_refinement/CellEsd.h
// The error model as it is reported, already formatted. `isa` is the whole-range 1/sqrt(a*b), the
// same quantity XDS's ISa denotes, so a file written here is directly comparable with a CORRECT.LP;
// `isa_asymptotic` is the strong-reflection tier, which only the rotation path has. `a` and `b` are
// in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2). Empty strings are written as unknown.
// in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2), and `c` is the stills partiality term in the same
// convention, sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2). Empty strings are written as unknown.
// A stills merge writes no ISa (MergeStatistics::stills).
struct ErrorModelReport {
std::string isa;
std::string isa_asymptotic;
std::string a;
std::string b;
std::string c;
};
// What the reflection files record about where they came from, beyond the experiment itself and the
@@ -127,6 +132,18 @@ MtzTable UnmergedMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const std::vector<ScaledFull> *scaled_fulls = nullptr);
void SetUnmergedMtzMosaicity(MtzTable &mtz, const std::vector<IntegrationOutcome> &outcomes);
// Diagnostic (--export-observations, stills): every integrated observation of the merged run with what the
// merge made of it - one row per reflection of every outcome, the ones the merge left out included, with a
// FLAGS bit for each test it failed (ObservationFlag; BEYOND_DMIN 4096 = past the written resolution cut).
// verdicts: MergeOnTheFly::Verdicts of the written merge, in the outcomes' order. Columns are listed in the
// code and documented with the campaign's loader, not here; H K L are the indices as measured.
MtzTable ObservationMtzTable(const std::vector<IntegrationOutcome> &outcomes,
const std::vector<std::vector<ObservationVerdict>> &verdicts,
std::optional<double> written_d_min,
const std::optional<StillsRunWidth> &run_width,
const UnitCell &unitCell,
const DiffractionExperiment &experiment);
void WriteReflections(const std::vector<MergedReflection> &reflections,
const UnitCell &unitCell,
const DiffractionExperiment &experiment,
+12
View File
@@ -119,6 +119,18 @@ own radius**, not to a global threshold.
read as one enormous transmitting piece. The model is capped at 1, so it only ever removes dim
pixels and a sweep without such a piece still keeps its mask bit for bit.
12. **Outer shadow (`FindOuterShadow`, stills).** An aperture or cone in front of the detector can dim
everything beyond some radius. Beyond the largest ring the detector holds whole, a ring can then lie
wholly in the dimmed corners, its median is the dimmed level, and step 4 sees nothing. So there each
pixel's pooled background is compared with what the one `OUTER_STEP_PX` nearer the beam along its ray,
carried on by the ray's own fall-off, leads up to; a significant drop below `OUTER_STEP_RATIO` marks the
shadow's edge (pieces narrower than 2 `OUTER_OPENING_PX` or smaller than `OUTER_MIN_PIXELS` are dropped).
In each of `OUTER_RAYS` directions the shadow starts at the innermost edge pixel where the ray's own
background, pooled over the ray, shows the same step and stays below `OUTER_STEP_RATIO` of the
background before it out to the detector's edge, and it runs to that edge. `rugnux` runs it once the stills
beam centre is decided, on the pre-scan's projection, grows it by R3 - R1 (so an observation whose
background ring reaches the shadow is dropped with it, `DropShadowClipped`) and adds it to bit 9.
Three findings are baked in: the **mean** ratio (not the max) does the detecting — the max
projection is a great *picture* but a noisier detector; **size and counting statistics**, not a
position, are what distinguish hardware from a stray low patch; and **a lit pixel is never
+165
View File
@@ -1196,6 +1196,171 @@ std::vector<uint32_t> ShadowFinder::MaskOn(const Projection &p, const std::vecto
return mask;
}
namespace {
// FindOuterShadow on a W x H raster, with its known-masked pixels and the direct beam (cx, cy) on it.
std::vector<uint32_t> OuterShadowOn(const std::vector<float> &mean, const std::vector<uint32_t> &known_mask,
int W, int H, float cx, float cy, uint32_t frames, float gain,
int margin_px, size_t nthreads) {
const int n_pixels = W * H;
// The background pooled over the same box as in GetMask.
Plane<char> valid(n_pixels);
Plane<double> num(n_pixels);
Plane<int32_t> den(n_pixels);
ParallelChunks(n_pixels, nthreads, [&](int lo, int hi) {
for (int i = lo; i < hi; i++) {
valid[i] = known_mask[i] == 0 && std::isfinite(mean[i]);
num[i] = valid[i] ? mean[i] : 0.0;
den[i] = valid[i] ? 1 : 0;
}
});
const auto pooled_sum = box_sum(num, W, H, POOL_PX, nthreads);
const auto pooled_count = box_sum(den, W, H, POOL_PX, nthreads);
// Inside the largest ring the detector holds whole, GetMask's comparison along the ring works.
const float r_whole = std::min({cx, static_cast<float>(W - 1) - cx, cy, static_cast<float>(H - 1) - cy});
// The edge: pixels whose background stepped down from what the ray leads up to, by enough photons to be
// believed. What it leads up to is the background OUTER_STEP_PX nearer the beam, carried on by the fall-off
// over the OUTER_STEP_PX before that: a background can fall steeply with radius on its own (to 60 % over
// OUTER_STEP_PX at high angle on one panel detector), and a step is a drop against that trend. A background
// below zero is a detector's offset, not a shadow.
const auto pooled_at = [&](float x, float y) -> std::optional<double> {
const int xi = static_cast<int>(std::lround(x)), yi = static_cast<int>(std::lround(y));
if (xi < 0 || xi >= W || yi < 0 || yi >= H || pooled_count[yi * W + xi] == 0)
return std::nullopt;
return pooled_sum[yi * W + xi] / pooled_count[yi * W + xi];
};
Plane<char> edge(n_pixels);
ParallelChunks(H, nthreads, [&](int ylo, int yhi) {
for (int y = ylo; y < yhi; y++)
for (int x = 0; x < W; x++) {
const int i = y * W + x;
edge[i] = 0;
const float dx = static_cast<float>(x) - cx, dy = static_cast<float>(y) - cy;
const float r = std::hypot(dx, dy);
if (!valid[i] || r <= r_whole || r <= 2 * OUTER_STEP_PX)
continue;
const float s1 = 1.0f - OUTER_STEP_PX / r, s2 = 1.0f - 2 * OUTER_STEP_PX / r;
const auto inner = pooled_at(cx + dx * s1, cy + dy * s1);
const auto further_in = pooled_at(cx + dx * s2, cy + dy * s2);
if (!inner || !further_in || *inner <= 0.0 || *further_in <= 0.0)
continue;
const double here = pooled_sum[i] / pooled_count[i];
const double expected = *inner * std::min(1.0, *inner / *further_in);
const double photons = static_cast<double>(frames) * pooled_count[i] / gain;
edge[i] = here > 0.0 && here < OUTER_STEP_RATIO * expected
&& poisson_deficit_sigma(here * photons, expected * photons) > MIN_DEFICIT_SIGMA;
}
});
// Only pieces of the edge wide and large enough to have been cast by something.
const Plane<char> opened = dilate(erode(edge, W, H, OUTER_OPENING_PX, nthreads), W, H, OUTER_OPENING_PX, nthreads);
const auto pieces = label_components(opened, W, H, nthreads);
std::vector<int> piece_size(pieces.count, 0);
for (int i = 0; i < n_pixels; i++)
if (pieces.id[i] >= 0)
piece_size[pieces.id[i]]++;
// Along each ray, the shadow starts at the innermost edge pixel where the ray's own background - pooled over
// the ray, OUTER_STEP_PX at a time - steps down against its trend as well, and stays below OUTER_STEP_RATIO
// of the background before the step all the way out to the detector's edge; it runs to that edge. A single
// pixel's step can be the noise of a detector that is not Poisson; a step whose far side recovers further
// out is not an obstruction, but a dim panel edge or a patch of weak pixels.
const auto ray = [cx, cy](int x, int y) {
const double phi = std::atan2(static_cast<double>(y) - cy, static_cast<double>(x) - cx) + std::numbers::pi;
return std::min(OUTER_RAYS - 1, static_cast<int>(phi / (2.0 * std::numbers::pi) * OUTER_RAYS));
};
const auto radius = [cx, cy](int x, int y) {
return static_cast<int>(std::lround(std::hypot(static_cast<float>(x) - cx, static_cast<float>(y) - cy)));
};
const int n_r = std::max({radius(0, 0), radius(W - 1, 0), radius(0, H - 1), radius(W - 1, H - 1)}) + 1;
std::vector<double> sum(static_cast<size_t>(OUTER_RAYS) * n_r, 0.0);
std::vector<int> count(sum.size(), 0);
std::vector<char> at_edge(sum.size(), 0);
for (int i = 0; i < n_pixels; i++) {
const int x = i % W, y = i / W;
const size_t k = static_cast<size_t>(ray(x, y)) * n_r + radius(x, y);
if (valid[i]) {
sum[k] += pooled_sum[i] / pooled_count[i];
count[k]++;
}
if (pieces.id[i] >= 0 && piece_size[pieces.id[i]] >= OUTER_MIN_PIXELS)
at_edge[k] = 1;
}
std::vector<float> starts_at(OUTER_RAYS, std::numeric_limits<float>::infinity());
for (int k = 0; k < OUTER_RAYS; k++) {
const size_t o = static_cast<size_t>(k) * n_r;
std::vector<double> beyond_sum(n_r + 1, 0.0);
std::vector<int> beyond_count(n_r + 1, 0);
for (int r = n_r - 1; r >= 0; r--) {
beyond_sum[r] = beyond_sum[r + 1] + sum[o + r];
beyond_count[r] = beyond_count[r + 1] + count[o + r];
}
// The ray's mean over the radii [lo, hi).
const auto band = [&](int lo, int hi) -> std::optional<double> {
const int n = beyond_count[lo] - beyond_count[hi];
if (n == 0)
return std::nullopt;
return (beyond_sum[lo] - beyond_sum[hi]) / n;
};
for (int r = 2 * OUTER_STEP_PX; r + OUTER_STEP_PX <= n_r; r++) {
if (!at_edge[o + r])
continue;
const auto further_in = band(r - 2 * OUTER_STEP_PX, r - OUTER_STEP_PX);
const auto inner = band(r - OUTER_STEP_PX, r);
const auto outer = band(r, r + OUTER_STEP_PX);
const auto beyond = band(r, n_r);
if (!further_in || !inner || !outer || *further_in <= 0.0 || *inner <= 0.0)
continue;
const double expected = *inner * std::min(1.0, *inner / *further_in);
if (*outer < OUTER_STEP_RATIO * expected && *beyond < OUTER_STEP_RATIO * *inner) {
starts_at[k] = static_cast<float>(r);
break;
}
}
}
Plane<char> shadow(n_pixels);
ParallelChunks(H, nthreads, [&](int ylo, int yhi) {
for (int y = ylo; y < yhi; y++)
for (int x = 0; x < W; x++)
shadow[y * W + x] = static_cast<float>(radius(x, y)) >= starts_at[ray(x, y)];
});
shadow = dilate(shadow, W, H, margin_px, nthreads);
return {shadow.begin(), shadow.end()};
}
} // namespace
std::vector<uint32_t> FindOuterShadow(const DiffractionExperiment &experiment, const PixelMask &mask,
const std::vector<float> &mean, uint32_t frames, int margin_px,
size_t nthreads) {
if (nthreads == 0)
nthreads = AvailableCpus();
const int width = static_cast<int>(experiment.GetXPixelsNumConv());
const int height = static_cast<int>(experiment.GetYPixelsNumConv());
const float gain = experiment.GetBraggIntegrationSettings().GetPixelGain();
const auto [beam_x, beam_y] = experiment.GetDiffractionGeometry().GetDirectBeam_pxl();
// What the beam-stop shadow already masks is read too: a shadow it reached part of is found whole.
std::vector<uint32_t> known = mask.GetMask(experiment);
for (auto &v : known)
v &= ~(1u << PixelMask::BeamStopPixelBit);
const auto &modules = experiment.GetModules();
if (modules.empty())
return OuterShadowOn(mean, known, width, height, beam_x, beam_y, frames, gain, margin_px, nthreads);
// Panels: the rays run on the detector plane, as in GetMask.
const PlaneRaster raster = PlaneRasterOf(modules, width, height);
const auto shadow = OuterShadowOn(AssembleToPlane(modules, mean, width, height, raster, NAN),
AssembleToPlane(modules, known, width, height, raster,
uint32_t{1u << PixelMask::ModuleGapPixelBit}),
static_cast<int>(raster.width), static_cast<int>(raster.height),
beam_x - static_cast<float>(raster.x0), beam_y - static_cast<float>(raster.y0),
frames, gain, margin_px, nthreads);
return GatherFromPlane(modules, shadow, raster, width, height, uint32_t{0});
}
namespace shadow_finder {
int BlockedOutTo(const std::vector<float> &baseline, const std::vector<int> &ring_pixels) {
+13
View File
@@ -182,3 +182,16 @@ public:
[[nodiscard]] uint32_t GetFrameCount() const;
};
// The shadow of an obstruction at the outer edge of the beam path - an aperture or a cone in front of the
// detector that dims everything beyond some radius - which ShadowFinder cannot see: beyond the largest ring
// the detector holds whole, a ring can lie wholly in the corners, and where those are all dimmed the ring's
// median is the dimmed level, and nothing on the ring is below it. So there each pixel is compared along its
// own ray instead, with the background nearer the beam: a sharp, significant step down against the ray's own
// fall-off marks the edge of the shadow, and where the ray stays dim from there out to the detector's edge,
// everything beyond that edge on the ray is shadow. Returns 1 for shadow, 0 to keep, of the converted
// pixel count, grown by `margin_px` so that what lies that close to the shadow goes with it.
// `mean` is ShadowFinder::GetMeanProjection over `frames` frames; the rays start at the experiment's direct beam.
[[nodiscard]] std::vector<uint32_t> FindOuterShadow(const DiffractionExperiment &experiment, const PixelMask &mask,
const std::vector<float> &mean, uint32_t frames,
int margin_px, size_t nthreads = 0);
@@ -111,6 +111,18 @@ inline constexpr int MIN_RING_PIXELS = 32;
// its own and does not follow SHADOW_RATIO.
inline constexpr float BLOCKED_RING_RATIO = 0.35f;
// FindOuterShadow: a pixel is at the edge of an outer shadow when its background is below
// OUTER_STEP_RATIO of what the background OUTER_STEP_PX nearer the beam along its ray, carried on by its
// own fall-off, leads up to. Measured on Rayonix stills, such a shadow keeps 45-60 % of the background. The
// edge is a band OUTER_STEP_PX wide; a piece of it narrower than 2 OUTER_OPENING_PX or with fewer than
// OUTER_MIN_PIXELS pixels is not believed (a dim outermost pixel row, the noise). The rays are binned in
// OUTER_RAYS directions.
inline constexpr int OUTER_STEP_PX = 20;
inline constexpr float OUTER_STEP_RATIO = 0.75f;
inline constexpr int OUTER_OPENING_PX = 2;
inline constexpr int OUTER_MIN_PIXELS = 200;
inline constexpr int OUTER_RAYS = 1440;
// The rings that lie wholly inside the stop: walking outward, every judgeable ring (at least
// MIN_RING_PIXELS pixels) before the first whose baseline reaches BLOCKED_RING_RATIO of the typical
// background. -1 when there is none. See GetMask.
@@ -45,7 +45,8 @@ int BraggPrediction::TruncateToOutput(int count) {
reflections.begin() + count,
[](const Reflection &a, const Reflection &b) {
if (a.partiality != b.partiality) return a.partiality > b.partiality;
if (a.dist_ewald != b.dist_ewald) return a.dist_ewald < b.dist_ewald;
if (std::fabs(a.dist_ewald) != std::fabs(b.dist_ewald))
return std::fabs(a.dist_ewald) < std::fabs(b.dist_ewald);
if (a.h != b.h) return a.h < b.h;
if (a.k != b.k) return a.k < b.k;
return a.l < b.l;
@@ -142,7 +143,8 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
float S_y = recip_y + S0.y;
float S_z = recip_z + S0.z;
float S_len = sqrtf(S_x * S_x + S_y * S_y + S_z * S_z);
float dist_ewald_sphere = std::fabs(S_len - one_over_wavelength);
// Signed excitation error, |S| - 1/lambda: positive outside the sphere. The window tests its size.
float dist_ewald_sphere = S_len - one_over_wavelength;
// Energy bandwidth thickens the Ewald shell radially: at the
// diffraction condition |S|-1/λ shifts by recip_z·(Δλ/λ), i.e.
@@ -154,7 +156,7 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
const float bw_tol = kBandwidthCutoffSigmas * settings.bandwidth_sigma * std::fabs(recip_z);
radial_cutoff = std::sqrt(radial_cutoff * radial_cutoff + bw_tol * bw_tol);
}
float window_dist = dist_ewald_sphere;
float window_dist = std::fabs(dist_ewald_sphere);
if (settings.window_c0 > 0.0f || settings.window_ang > 0.0f || settings.window_bw > 0.0f) {
const float wb = settings.window_bw * std::fabs(recip_z);
radial_cutoff = std::sqrt(settings.window_c0 * settings.window_c0
@@ -165,6 +167,14 @@ int BraggPrediction::Calc(const DiffractionExperiment &experiment, const Crystal
}
if (window_dist <= radial_cutoff) {
if (settings.window_radial > 0.0f) {
const float az = std::fabs(recip_z);
const float f = dist_ewald_sphere * std::fmin(settings.window_radial * az / (radial_cutoff * radial_cutoff),
1.0f / az);
S_x += f * S0.x;
S_y += f * S0.y;
S_z += f * S0.z;
}
const float s0_p0 = S0.x * recip_x + S0.y * recip_y + S0.z * recip_z;
const float val = s0_sq * recip_sq - s0_p0 * s0_p0;
@@ -45,6 +45,12 @@ struct BraggPredictionSettings {
// The window is centred where the rocking curve is: on the excitation error of the reciprocal lattice
// scaled by this (StillsPartialityRefine::MeasureRadialScale). dist_ewald stays the lattice's own.
float window_q_scale = 1.0f;
// Stills, with the window: a reflection off the Ewald sphere is placed where the radial part of its
// smeared node diffracts (StillsPartialityRefine::MeasureRadialSpread) - its S moved along S0 by
// dk / k = window_radial |q_z| eps / W^2, W the window at the reflection, but never past the change that
// puts the node on the sphere (eps / |q_z|). window_radial = s^2 times the window's width in sigmas
// squared, s the run's relative radial spread. 0 leaves the positions as they were.
float window_radial = 0.0f;
};
class BraggPrediction {
@@ -106,7 +106,7 @@ namespace {
float Sy = recip_y + C.S0.y;
float Sz = recip_z + C.S0.z;
float S_len = sqrtf(Sx * Sx + Sy * Sy + Sz * Sz);
float dist_ewald = fabsf(S_len - C.one_over_wavelength);
float dist_ewald = S_len - C.one_over_wavelength; // signed, as on the CPU path
// Energy bandwidth thickens the Ewald shell radially: σ_bw = |recip_z|·(Δλ/λ)
// (= bλ/2d²). Broaden the acceptance window in quadrature (see CPU path).
float radial_cutoff = C.ewald_cutoff;
@@ -114,7 +114,7 @@ namespace {
const float bw_tol = kBandwidthCutoffSigmas * C.bandwidth_sigma * fabsf(recip_z);
radial_cutoff = sqrtf(radial_cutoff * radial_cutoff + bw_tol * bw_tol);
}
float window_dist = dist_ewald;
float window_dist = fabsf(dist_ewald);
if (C.window_c0 > 0.0f || C.window_ang > 0.0f || C.window_bw > 0.0f) {
const float wb = C.window_bw * fabsf(recip_z);
radial_cutoff = sqrtf(C.window_c0 * C.window_c0 + C.window_ang * C.window_ang * recip_sq + wb * wb);
@@ -123,6 +123,14 @@ namespace {
window_dist = fabsf(sqrtf(Scx * Scx + Scy * Scy + Scz * Scz) - C.one_over_wavelength);
}
if (window_dist > radial_cutoff) return false;
if (C.window_radial > 0.0f) { // see the CPU path
const float az = fabsf(recip_z);
const float f = dist_ewald * fminf(C.window_radial * az / (radial_cutoff * radial_cutoff), 1.0f / az);
Sx += f * C.S0.x;
Sy += f * C.S0.y;
Sz += f * C.S0.z;
S_len = sqrtf(Sx * Sx + Sy * Sy + Sz * Sz);
}
float Srx = C.rot[0] * Sx + C.rot[1] * Sy + C.rot[2] * Sz;
float Sry = C.rot[3] * Sx + C.rot[4] * Sy + C.rot[5] * Sz;
float Srz = C.rot[6] * Sx + C.rot[7] * Sy + C.rot[8] * Sz;
@@ -253,6 +261,7 @@ int BraggPredictionGPU::Calc(const DiffractionExperiment &experiment,
hK.window_ang = settings.window_ang;
hK.window_bw = settings.window_bw;
hK.window_q_scale = settings.window_q_scale;
hK.window_radial = settings.window_radial;
cudaMemcpyAsync(dK, &hK, sizeof(KernelConsts), cudaMemcpyHostToDevice, stream);
cudaMemsetAsync(d_count, 0, sizeof(int), stream);
@@ -18,6 +18,7 @@ struct KernelConsts {
float one_over_dmax_sq;
float ewald_cutoff;
float window_c0, window_ang, window_bw, window_q_scale; // the stills window curve; all zero: ewald_cutoff (see BraggPredictionSettings)
float window_radial; // where an off-sphere reflection is placed (see BraggPredictionSettings)
float bandwidth_sigma; // relative Δλ/λ (sigma); 0 = monochromatic
// Angle-dependent sensor efficiency, reduced to the two per-dataset numbers the kernel needs:
// the optical thickness at normal incidence and the absorbed fraction there. qe_a0 = 0 leaves
+15 -2
View File
@@ -83,6 +83,7 @@ HotPixelFinder::HotPixelFinder(const DiffractionExperiment &experiment, const Pi
key(std::make_unique_for_overwrite<int32_t[]>(width * height)),
n_lit(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
n_error(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
n_neg(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
sum_value(std::make_unique_for_overwrite<int64_t[]>(width * height)),
n_error_ring_ok(std::make_unique_for_overwrite<uint16_t[]>(width * height)),
error_level_sum(std::make_unique_for_overwrite<int64_t[]>(width * height)) {
@@ -91,7 +92,7 @@ HotPixelFinder::HotPixelFinder(const DiffractionExperiment &experiment, const Pi
const float ring_rad = RING_WIDTH_PX * geometry.GetPixelSize_mm() / geometry.GetDetectorDistance_mm();
ParallelChunks(static_cast<int>(height), nthreads, [&](int y0, int y1) {
for (size_t i = y0 * width; i < y1 * width; i++) {
n_lit[i] = n_error[i] = n_error_ring_ok[i] = 0;
n_lit[i] = n_error[i] = n_neg[i] = n_error_ring_ok[i] = 0;
sum_value[i] = error_level_sum[i] = 0;
key[i] = -1;
}
@@ -217,6 +218,8 @@ void HotPixelFinder::AddImage(const int32_t *image, std::vector<int32_t> &scratc
sum_value[i] += v;
if (static_cast<float>(v) > threshold[k])
n_lit[i]++;
if (static_cast<float>(v) < NegativeLimit(level[k], gain))
n_neg[i]++;
}
}
}
@@ -270,7 +273,9 @@ void HotPixelFinder::PrepareDevice() {
set_gpu(c);
return std::make_unique<HotPixelFinderGPU>(key.get(), width * height, key_begin, nrings, SECTORS,
HotPixelLevelRules{MIN_SECTOR_PIXELS, MIN_RING_PIXELS,
LIT_NSIGMA, LIT_OFFSET * gain, gain});
LIT_NSIGMA, LIT_OFFSET * gain, gain,
NEG_NSIGMA, NEG_OFFSET * gain,
NEG_FRACTION_INV, NEG_MIN_FRAMES});
}));
for (auto &b : built)
gpus.push_back(b.get());
@@ -368,6 +373,7 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
const size_t i = c.index[j];
sums_known[i] = 1;
n_lit[i] = c.n_lit[j];
n_neg[i] = c.n_neg[j];
n_error[i] = c.n_error[j];
n_error_ring_ok[i] = c.n_error_ring_ok[j];
sum_value[i] = c.sum_value[j];
@@ -412,6 +418,12 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
ret.mask[i] = 2;
continue;
}
// Unstable: far below zero on a twentieth of its frames, which no count of photons does.
if (sums_known[i] && n_valid(i) >= min_valid && n_neg[i] >= NEG_MIN_FRAMES
&& NEG_FRACTION_INV * n_neg[i] >= n_valid(i)) {
ret.mask[i] = 4;
continue;
}
// Dead: the mirror of hot. A pixel reading at most half of its own sector's level, by
// more than the bound a hot pixel's excess has to clear, is not recording the light that
// reaches it, and every reflection whose disk covers it is integrated short by that
@@ -440,6 +452,7 @@ HotPixelFinder::Result HotPixelFinder::GetMask(double oscillation_deg, double sp
ret.hot += v == 1;
ret.error += v == 2;
ret.dead += v == 3;
ret.unstable += v == 4;
}
return ret;
}
+23 -2
View File
@@ -42,11 +42,20 @@
// covers it is integrated short by the deficit. The detector's outermost row and column get no rule of
// their own - a dead edge row, or a hot pixel there, is caught like any other.
//
// An UNSTABLE pixel reads far below zero - ten times the Poisson noise of its ring's level and ten photons
// more under it - on at least a twentieth of its frames (and on two). No photon can make a count
// negative, and a reflection only adds counts, so such a pixel is broken whatever it reads in between:
// measured, a column of a JUNGFRAU 9M swinging between -2500 and +1000 counts from frame to frame,
// with a normal median. Its negative frames drag the background ring of every reflection near it far
// below zero, and a reflection whose disk then sums to nothing is integrated huge with a one-photon sigma.
//
// A CCD is read with an offset and is not Poisson, but neither test needs it to be: a dead CCD pixel
// reads its offset, far below a ring with any background on it, and a hot one ten times its ring.
// =============================================================================
#include <atomic>
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <memory>
#include <mutex>
@@ -82,14 +91,21 @@ public:
// strength at which the pixels behind the merged outliers were found, and well clear of a pixel
// that is merely a little over-responding.
static constexpr double STRONG_RATIO = 10.0;
// Unstable: below -(NEG_NSIGMA * noise of the level + NEG_OFFSET photons) on at least one frame in
// NEG_FRACTION_INV, and on NEG_MIN_FRAMES.
static constexpr float NEG_NSIGMA = 10.0f;
static constexpr float NEG_OFFSET = 10.0f;
static constexpr int NEG_FRACTION_INV = 20;
static constexpr int NEG_MIN_FRAMES = 2;
// The most frames a finder can be given: its per-pixel counters are 16-bit.
static constexpr int MAX_SAMPLED_FRAMES = UINT16_MAX;
struct Result {
std::vector<uint32_t> mask; // non-zero = masked (1 hot, 2 error value, 3 dead), converted geometry
std::vector<uint32_t> mask; // non-zero = masked (1 hot, 2 error value, 3 dead, 4 unstable), converted geometry
size_t hot = 0; // persistent above the ring
size_t error = 0; // holding the error value or the saturation limit on most frames
size_t dead = 0; // reading at most half of its sector's level
size_t unstable = 0; // far below zero on a twentieth of its frames
uint32_t frames = 0;
};
@@ -139,7 +155,7 @@ private:
std::atomic<size_t> next_band{0};
mutable std::mutex m;
uint32_t frames = 0;
std::unique_ptr<uint16_t[]> n_lit, n_error;
std::unique_ptr<uint16_t[]> n_lit, n_error, n_neg;
std::unique_ptr<int64_t[]> sum_value;
// A pixel's valid frames and the sum of its ring-sector level over them, kept per ring-sector - the
// frames its ring had a level on, and the sum of the level over those - less what the pixel's own
@@ -167,6 +183,11 @@ private:
const std::vector<float> &ring_spread, const std::vector<char> &ring_ok,
const std::vector<int32_t> &local_level,
std::vector<int32_t> &level, std::vector<float> &threshold);
// The count below which a pixel of level `level` is unstable on a frame - one rounding for the fma, as
// on the device (HotPixelsGPU.cu).
[[nodiscard]] static float NegativeLimit(int32_t level, float gain) {
return -std::fma(NEG_NSIGMA, std::sqrt(gain * std::max(static_cast<float>(level), gain)), NEG_OFFSET * gain);
}
[[nodiscard]] int n_valid(size_t i) const { return key_frames[key[i]] - n_error_ring_ok[i]; }
[[nodiscard]] int64_t sum_level(size_t i) const { return key_level_sum[key[i]] - error_level_sum[i]; }
};
+27 -13
View File
@@ -123,11 +123,11 @@ __global__ void ring_kernel(const int32_t *__restrict__ image, const uint32_t *_
// HotPixelFinder::AddImage's per-pixel loop, line for line.
__global__ void accumulate_kernel(const int32_t *__restrict__ image, const int32_t *__restrict__ key,
size_t npixels, int sectors, const int32_t *__restrict__ level,
size_t npixels, int sectors, HotPixelLevelRules rules, const int32_t *__restrict__ level,
const float *__restrict__ threshold, const char *__restrict__ ring_ok,
uint16_t *__restrict__ n_lit, uint16_t *__restrict__ n_error,
int64_t *__restrict__ sum_value, uint16_t *__restrict__ n_error_ring_ok,
int64_t *__restrict__ error_level_sum) {
int64_t *__restrict__ error_level_sum, uint16_t *__restrict__ n_neg) {
for (size_t i = blockIdx.x * static_cast<size_t>(blockDim.x) + threadIdx.x; i < npixels;
i += static_cast<size_t>(blockDim.x) * gridDim.x) {
const int32_t k = key[i], v = image[i];
@@ -144,6 +144,12 @@ __global__ void accumulate_kernel(const int32_t *__restrict__ image, const int32
sum_value[i] += v;
if (static_cast<float>(v) > threshold[k])
n_lit[i]++;
// HotPixelFinder::NegativeLimit, with its roundings
const float neg = -__fmaf_rn(rules.neg_nsigma,
__fsqrt_rn(__fmul_rn(rules.gain, fmaxf(static_cast<float>(level[k]), rules.gain))),
rules.neg_offset);
if (static_cast<float>(v) < neg)
n_neg[i]++;
}
}
@@ -202,12 +208,12 @@ __global__ void chance_kernel(size_t npixels, int sectors, const int32_t *__rest
}
// Writes the candidates' indices from `out` on when `out` is given, and counts them either way.
__global__ void candidate_kernel(size_t npixels, int sectors, uint32_t frames, int min_valid, bool spacing_ok,
__global__ void candidate_kernel(size_t npixels, int sectors, HotPixelLevelRules rules, uint32_t frames, int min_valid, bool spacing_ok,
const int32_t *__restrict__ key, const uint32_t *__restrict__ key_frames,
const uint16_t *__restrict__ n_lit, const uint16_t *__restrict__ n_error,
const uint16_t *__restrict__ n_error_ring_ok, const int64_t *__restrict__ sum_value,
const int64_t *__restrict__ key_local_level_sum, const int *__restrict__ k_chance,
uint32_t *__restrict__ count, uint32_t *__restrict__ out) {
const uint16_t *__restrict__ n_neg, uint32_t *__restrict__ count, uint32_t *__restrict__ out) {
const size_t i = blockIdx.x * static_cast<size_t>(blockDim.x) + threadIdx.x;
if (i >= npixels || key[i] < 0) return;
const int nv = valid_frames(key, key_frames, n_error_ring_ok, i);
@@ -217,7 +223,10 @@ __global__ void candidate_kernel(size_t npixels, int sectors, uint32_t frames, i
&& lit >= min(max(2, k_chance[key[i] / sectors]), nv);
// Can be dead (HotPixelFinder::GetMask): at most half of its sector's level, no error frames.
const bool dark = n_error[i] == 0 && nv >= min_valid && 2 * sum_value[i] <= key_local_level_sum[key[i]];
if (!error && !persistent && !dark) return;
// Can be unstable (HotPixelFinder::GetMask): far below zero on a twentieth of its valid frames.
const bool unstable = nv >= min_valid && n_neg[i] >= rules.neg_min_frames
&& rules.neg_fraction_inv * static_cast<int>(n_neg[i]) >= nv;
if (!error && !persistent && !dark && !unstable) return;
const uint32_t slot = atomicAdd(count, 1u);
if (out) out[slot] = static_cast<uint32_t>(i);
}
@@ -256,7 +265,7 @@ HotPixelFinderGPU::HotPixelFinderGPU(const int32_t *host_key, size_t npixels,
sectors(sectors), rules(rules),
key(npixels, ALLOC), pixels_by_key(std::max<size_t>(host_key_begin.back(), 1), ALLOC),
key_begin(host_key_begin.size(), ALLOC),
n_lit(npixels, ALLOC), n_error(npixels, ALLOC), n_error_ring_ok(npixels, ALLOC),
n_lit(npixels, ALLOC), n_error(npixels, ALLOC), n_error_ring_ok(npixels, ALLOC), n_neg(npixels, ALLOC),
sum_value(npixels, ALLOC), error_level_sum(npixels, ALLOC),
key_frames(std::max<size_t>(nkeys, 1), ALLOC), key_level_sum(std::max<size_t>(nkeys, 1), ALLOC),
key_local_level_sum(std::max<size_t>(nkeys, 1), ALLOC) {
@@ -269,6 +278,7 @@ HotPixelFinderGPU::HotPixelFinderGPU(const int32_t *host_key, size_t npixels,
cuda_err(cudaMemsetAsync(n_lit, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_error, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_error_ring_ok, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(n_neg, 0, npixels * sizeof(uint16_t), stream));
cuda_err(cudaMemsetAsync(sum_value, 0, npixels * sizeof(int64_t), stream));
cuda_err(cudaMemsetAsync(error_level_sum, 0, npixels * sizeof(int64_t), stream));
cuda_err(cudaMemsetAsync(key_frames, 0, std::max<size_t>(nkeys, 1) * sizeof(uint32_t), stream));
@@ -326,8 +336,8 @@ void HotPixelFinderGPU::Add(const int32_t *device_image, Frame &frame) {
std::lock_guard lock(accumulate_mutex);
cuda_err(cudaStreamWaitEvent(stream, last_accumulate, 0));
accumulate_kernel<<<static_cast<unsigned>((npixels + THREADS - 1) / THREADS), THREADS, 0, stream>>>(
device_image, key, npixels, sectors, frame.level, frame.threshold, frame.ring_ok,
n_lit, n_error, sum_value, n_error_ring_ok, error_level_sum);
device_image, key, npixels, sectors, rules, frame.level, frame.threshold, frame.ring_ok,
n_lit, n_error, sum_value, n_error_ring_ok, error_level_sum, n_neg);
cuda_err(cudaGetLastError());
cuda_err(cudaEventRecord(last_accumulate, stream));
}
@@ -348,6 +358,7 @@ void HotPixelFinderGPU::Fold(HotPixelFinderGPU &other) {
add(n_lit.get(), other.n_lit.get(), npixels);
add(n_error.get(), other.n_error.get(), npixels);
add(n_error_ring_ok.get(), other.n_error_ring_ok.get(), npixels);
add(n_neg.get(), other.n_neg.get(), npixels);
add(sum_value.get(), other.sum_value.get(), npixels);
add(error_level_sum.get(), other.error_level_sum.get(), npixels);
add(key_frames.get(), other.key_frames.get(), nkeys);
@@ -385,9 +396,9 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
cuda_err(cudaMemcpyAsync(d_k_chance, k_chance.data(), k_chance.size() * sizeof(int), cudaMemcpyHostToDevice,
stream));
cuda_err(cudaMemsetAsync(d_count, 0, sizeof(uint32_t), stream));
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, frames, min_valid, spacing_ok, key, key_frames,
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, rules, frames, min_valid, spacing_ok, key, key_frames,
n_lit, n_error, n_error_ring_ok, sum_value, key_local_level_sum,
d_k_chance, d_count, nullptr);
d_k_chance, n_neg, d_count, nullptr);
cuda_err(cudaGetLastError());
uint32_t n = 0;
cuda_err(cudaMemcpyAsync(&n, d_count, sizeof(uint32_t), cudaMemcpyDeviceToHost, stream));
@@ -407,17 +418,18 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
}
if (n > 0) {
CudaDevicePtr<uint32_t> index(n, ALLOC);
CudaDevicePtr<uint16_t> g_lit(n, ALLOC), g_error(n, ALLOC), g_error_ring_ok(n, ALLOC);
CudaDevicePtr<uint16_t> g_lit(n, ALLOC), g_error(n, ALLOC), g_error_ring_ok(n, ALLOC), g_neg(n, ALLOC);
CudaDevicePtr<int64_t> g_sum(n, ALLOC), g_error_level(n, ALLOC);
cuda_err(cudaMemsetAsync(d_count, 0, sizeof(uint32_t), stream));
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, frames, min_valid, spacing_ok, key,
candidate_kernel<<<blocks, THREADS, 0, stream>>>(npixels, sectors, rules, frames, min_valid, spacing_ok, key,
key_frames, n_lit, n_error, n_error_ring_ok, sum_value,
key_local_level_sum, d_k_chance, d_count, index);
key_local_level_sum, d_k_chance, n_neg, d_count, index);
cuda_err(cudaGetLastError());
const unsigned gb = (n + THREADS - 1) / THREADS;
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_lit.get(), g_lit.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_error.get(), g_error.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_error_ring_ok.get(), g_error_ring_ok.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, n_neg.get(), g_neg.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, sum_value.get(), g_sum.get());
gather_kernel<<<gb, THREADS, 0, stream>>>(n, index, error_level_sum.get(), g_error_level.get());
cuda_err(cudaGetLastError());
@@ -425,6 +437,7 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
c.n_lit.resize(n);
c.n_error.resize(n);
c.n_error_ring_ok.resize(n);
c.n_neg.resize(n);
c.sum_value.resize(n);
c.error_level_sum.resize(n);
cuda_err(cudaMemcpyAsync(c.index.data(), index, n * sizeof(uint32_t), cudaMemcpyDeviceToHost, stream));
@@ -432,6 +445,7 @@ HotPixelFinderGPU::Candidates HotPixelFinderGPU::GetCandidates(uint32_t frames,
cuda_err(cudaMemcpyAsync(c.n_error.data(), g_error, n * sizeof(uint16_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.n_error_ring_ok.data(), g_error_ring_ok, n * sizeof(uint16_t),
cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.n_neg.data(), g_neg, n * sizeof(uint16_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.sum_value.data(), g_sum, n * sizeof(int64_t), cudaMemcpyDeviceToHost, stream));
cuda_err(cudaMemcpyAsync(c.error_level_sum.data(), g_error_level, n * sizeof(int64_t),
cudaMemcpyDeviceToHost, stream));
+6 -2
View File
@@ -18,6 +18,10 @@ struct HotPixelLevelRules {
float lit_nsigma;
float lit_offset; // in the frames' units
float gain; // the frames' units per photon
float neg_nsigma; // HotPixelFinder::NegativeLimit
float neg_offset; // in the frames' units
int neg_fraction_inv;
int neg_min_frames;
};
// The device half of HotPixelFinder, for frames already preprocessed on the GPU. Everything a frame adds
@@ -41,7 +45,7 @@ class HotPixelFinderGPU {
CudaDevicePtr<uint32_t> key_begin; // where each key's pixels start in pixels_by_key
// The per-pixel and per-key sums, exactly those of HotPixelFinder.
CudaDevicePtr<uint16_t> n_lit, n_error, n_error_ring_ok;
CudaDevicePtr<uint16_t> n_lit, n_error, n_error_ring_ok, n_neg;
CudaDevicePtr<int64_t> sum_value, error_level_sum;
CudaDevicePtr<uint32_t> key_frames;
CudaDevicePtr<int64_t> key_level_sum, key_local_level_sum;
@@ -99,7 +103,7 @@ public:
// per-pixel sums; and every key's sums.
struct Candidates {
std::vector<uint32_t> index;
std::vector<uint16_t> n_lit, n_error, n_error_ring_ok;
std::vector<uint16_t> n_lit, n_error, n_error_ring_ok, n_neg;
std::vector<int64_t> sum_value, error_level_sum;
std::vector<uint32_t> key_frames;
std::vector<int64_t> key_level_sum, key_local_level_sum;
+122
View File
@@ -8,6 +8,8 @@
#include <functional>
#include <future>
#include <Eigen/Dense>
#include "../../common/ParallelFor.h"
#include "../../common/ThreadAffinity.h"
@@ -216,3 +218,123 @@ std::optional<ErrorModelB2Interval> BootstrapErrorModelB2(const std::vector<Erro
return ErrorModelB2Interval{q_low > 0.0 ? std::max(0.0, b2 + shrink * (q_low - b2)) : 0.0,
std::max(0.0, b2 + shrink * (q_high - b2))};
}
StillsErrorModelFit FitStillsErrorModel(const std::vector<StillsErrorModelSample> &pool, size_t nthreads,
const StillsErrorModelFit &start) {
constexpr int n_bins = ERROR_MODEL_BINS;
constexpr int n_sub = 4;
StillsErrorModelFit fit;
if (pool.size() < static_cast<size_t>(8 * n_bins * n_sub))
return fit;
// Bins of counting (I/sigma)^2, each split by the partiality term's share of the intensity term. The
// boundaries are put in place with nth_element on (key, position in the pool) - a total order, so which
// sample lands in which bin depends on the samples and the order the caller gave them in alone. The keys
// are computed once and carried in the comparison, never recomputed inside it.
struct Key { double v; size_t i; };
const auto by_key = [](const Key &p, const Key &q) { return p.v < q.v || (p.v == q.v && p.i < q.i); };
std::vector<Key> key(pool.size());
ParallelChunks(static_cast<int>(pool.size()), ThreadsForWork(pool.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
key[i] = {pool[i].I2 / pool[i].s2, static_cast<size_t>(i)};
});
const size_t per = pool.size() / n_bins;
// The split runs down the middle, and the two halves of a split are disjoint ranges, so they run side
// by side while the range is still worth a thread (as in FitErrorModel).
constexpr size_t SPLIT_MIN_PARALLEL = 1u << 17;
const std::function<void(size_t, size_t, int, int, int)> split = [&](size_t lo, size_t hi, int b0, int b1, int depth) {
if (b0 >= b1) return;
const int mid = (b0 + b1) / 2;
const size_t k = static_cast<size_t>(mid) * per;
std::nth_element(key.begin() + lo, key.begin() + k, key.begin() + hi, by_key);
if (depth > 0 && hi - lo > SPLIT_MIN_PARALLEL) {
auto left = std::async(std::launch::async, [&, lo, k, b0, mid, depth] {
RestoreThreadAffinity();
split(lo, k, b0, mid, depth - 1);
});
split(k, hi, mid + 1, b1, depth - 1);
left.get();
} else {
split(lo, k, b0, mid, 0);
split(k, hi, mid + 1, b1, 0);
}
};
split(0, key.size(), 1, n_bins, nthreads > 1 ? 3 : 0);
// b only where the strongest bin reaches past the counting term, as in FitErrorModel.
std::vector<double> top;
for (size_t j = (n_bins - 1) * per; j < key.size(); ++j)
top.push_back(key[j].v);
std::nth_element(top.begin(), top.begin() + top.size() / 2, top.end());
const bool b_measured = top[top.size() / 2] > ERROR_MODEL_B_LEVER_MIN;
// Each bin in four by the share, and the samples copied out in that order.
std::vector<std::pair<size_t, size_t>> groups(n_bins * n_sub);
std::vector<StillsErrorModelSample> x(pool.size());
ParallelFor(n_bins, nthreads, [&](int bin) {
const size_t lo = bin * per, hi = bin == n_bins - 1 ? key.size() : (bin + 1) * per;
for (size_t j = lo; j < hi; ++j) {
const auto &o = pool[key[j].i];
key[j].v = o.I2 > 0.0 ? o.P2 / o.I2 : 0.0;
}
const size_t sub = (hi - lo) / n_sub;
for (int q = 1; q < n_sub; ++q)
std::nth_element(key.begin() + lo + (q - 1) * sub, key.begin() + lo + q * sub, key.begin() + hi, by_key);
for (int q = 0; q < n_sub; ++q)
groups[bin * n_sub + q] = {lo + q * sub, q == n_sub - 1 ? hi : lo + (q + 1) * sub};
for (size_t j = lo; j < hi; ++j)
x[j] = pool[key[j].i];
});
key = {};
const int ng = static_cast<int>(groups.size());
std::vector<double> z2(pool.size());
Eigen::MatrixXd X(ng, 3);
Eigen::VectorXd y(ng);
double a = start.a, b2 = start.b2, c2 = start.c2;
for (int iter = 0; iter < 30; ++iter) {
// Per group: the three terms divided by the current variance, and the median normalised deviation
// expressed as the sum it stands for.
ParallelFor(ng, ThreadsForWork(pool.size(), nthreads), [&](int g) {
const auto [lo, hi] = groups[g];
double s = 0.0, I = 0.0, P = 0.0;
for (size_t i = lo; i < hi; ++i) {
const double v = a * x[i].s2 + b2 * x[i].I2 + c2 * x[i].P2;
s += x[i].s2 / v; I += x[i].I2 / v; P += x[i].P2 / v;
z2[i] = x[i].dev2 / v;
}
const auto mid = z2.begin() + lo + (hi - lo) / 2;
std::nth_element(z2.begin() + lo, mid, z2.begin() + hi);
X.row(g) << s, I, P;
y(g) = *mid / ERROR_MODEL_CHI2_1_MEDIAN * static_cast<double>(hi - lo);
});
// Every group weighs the same in relative terms; a negative term is dropped and the rest refitted.
const Eigen::VectorXd w = y.array().square().inverse();
std::vector<int> terms = b_measured ? std::vector<int>{0, 1, 2} : std::vector<int>{0, 2};
Eigen::VectorXd sol = Eigen::VectorXd::Zero(3);
while (true) {
Eigen::MatrixXd Xs(ng, terms.size());
for (size_t j = 0; j < terms.size(); ++j)
Xs.col(j) = X.col(terms[j]);
const Eigen::MatrixXd A = Xs.transpose() * w.asDiagonal() * Xs;
const Eigen::VectorXd s = A.ldlt().solve(Xs.transpose() * w.asDiagonal() * y);
sol.setZero();
for (size_t j = 0; j < terms.size(); ++j)
sol(terms[j]) = s(j);
const auto neg = std::find_if(terms.begin() + 1, terms.end(), [&](int t) { return sol(t) < 0.0; });
if (neg == terms.end())
break;
terms.erase(neg);
}
if (!std::isfinite(sol(0)) || !std::isfinite(sol(1)) || !std::isfinite(sol(2)))
break;
const double a_new = std::clamp(sol(0), 0.25, 100.0), b2_new = sol(1), c2_new = sol(2);
const bool settled = std::fabs(a_new - a) <= 1e-4 * a && std::fabs(b2_new - b2) <= 1e-4 * b2
&& std::fabs(c2_new - c2) <= 1e-4 * c2;
a = a_new; b2 = b2_new; c2 = c2_new;
fit.active = true;
fit.b_resolved = b_measured && b2 > 0.0 && a == sol(0);
if (settled)
break;
}
fit.a = a; fit.b2 = b2; fit.c2 = c2;
return fit;
}
+26
View File
@@ -77,3 +77,29 @@ struct ErrorModelB2Interval {
std::optional<ErrorModelB2Interval> BootstrapErrorModelB2(const std::vector<ErrorModelSample> &pool, double b2,
int replicates, size_t max_samples, size_t nthreads);
// The error model of the stills merge: the rotation model plus a partiality term,
// var = a * s2 + b^2 * <I>^2 + c^2 * ((1 - p) / p)^2 * <I>^2,
// p the partiality the observation is divided by. A still records one sample of its rocking curve, and the
// partiality model's error, carried through the division by p, grows like 1/p in the tail; without the term
// the observations below p ~ 0.5 cannot be merged at all, with it they add to the merge. Measured on four
// serial and rotation-as-serial sets, a term growing like 1/p fits the scatter better than the one a
// Gaussian rocking curve propagates (|eps| sigma_eps / sigma^2, which grows only like sqrt(-ln p)) and gives
// the stronger anomalous map; the curve's real tail is heavier than its Gaussian core.
// Fitted as FitErrorModel fits the rotation model - equal-count bins of counting I/sigma, the median
// normalised deviation of each, every bin relative, iterated - with each bin split in four by
// ((1 - p) / p)^2, so that c is told apart from b by observations of one strength and different partiality.
struct StillsErrorModelSample {
double s2, I2, P2, dev2; // P2 = ((1 - p) / p)^2 * I2
};
struct StillsErrorModelFit {
bool active = false;
double a = 1.0, b2 = 0.0, c2 = 0.0;
bool b_resolved = false; // as ErrorModelFit::b_resolved
};
// start: where the iteration begins - identity by default, or the fit of a previous round on a pool that
// has moved little since.
StillsErrorModelFit FitStillsErrorModel(const std::vector<StillsErrorModelSample> &pool, size_t nthreads = 1,
const StillsErrorModelFit &start = {});
+365 -296
View File
@@ -6,7 +6,11 @@
#include <algorithm>
#include <cmath>
#include <limits>
#include <map>
#include <span>
#include <tuple>
#include <unordered_map>
#include <unordered_set>
#include <spdlog/fmt/fmt.h>
@@ -20,6 +24,7 @@
#include "RfreeFlags.h"
#include "FrenchWilson.h"
#include "OutlierBand.h"
#include "ErrorModel.h"
namespace {
// Which half-set an observation prefers: a splitmix64 bit-mix of the image's stable index, so the
@@ -69,11 +74,11 @@ MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x)
? std::optional<double>(scaling_settings.GetMinCCForImage())
: std::nullopt),
min_partiality(scaling_settings.GetMinPartiality()),
existence_min_partiality(std::min(min_partiality, EXISTENCE_MIN_PARTIALITY)),
capture_coeff(scaling_settings.GetCaptureUncertaintyCoeff()),
generator(scaling_settings.GetMergeFriedel(), space_group),
reject_outliers(scaling_settings.GetOutlierRejectNsigma() > 0.0),
reject_nsigma(scaling_settings.GetOutlierRejectNsigma()),
stills(!x.IsRotationIndexing()) {
reject_nsigma(scaling_settings.GetOutlierRejectNsigma()) {
}
MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional<UnitCell> &cell) {
@@ -93,16 +98,14 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
continue;
if (exclude_ice_rings && r.on_ice_ring)
continue;
if (r.partiality < min_partiality)
if (BelowMergeFloor(r.partiality))
continue;
const float I_corr = r.I * r.image_scale_corr;
float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0)
continue;
auto hkl = generator(r);
auto hkl_key = hkl.pack();
sigma_corr = CorrectedSigma(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality, hkl_key);
const auto [I_corr, w] = Weigh(r, hkl_key);
if (w == 0.0)
continue;
// Robust outlier rejection: drop this observation if it sits outside the band about the
// reflection's median that the error model allows (OutlierBand.h). Needs the active error model
@@ -112,8 +115,7 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
const auto mit = reject_median_I.find(hkl_key);
if (mit != reject_median_I.end()) {
const double I_b = error_model_mean_I.at(hkl_key);
if (OutsideOutlierBand(I_corr, mit->second, reject_nsigma,
static_cast<double>(sigma_corr) * sigma_corr, I_b,
if (OutsideOutlierBand(I_corr, mit->second, reject_nsigma, 1.0 / w, I_b,
error_model_b * I_b, reject_up, reject_down)) {
++rejected;
continue;
@@ -121,7 +123,6 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
}
}
const float w = 1.0f / (sigma_corr * sigma_corr);
out.push_back(Contribution{
.key = hkl_key,
.h = hkl.plus ? hkl.h : -hkl.h,
@@ -130,12 +131,55 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
.w = w,
.wI = w * I_corr,
.d = r.d,
.on_ice_ring = r.on_ice_ring
.on_ice_ring = r.on_ice_ring,
.exists = r.partiality >= existence_min_partiality,
.far_tail = r.partiality < FAR_TAIL_PARTIALITY
});
}
return rejected;
}
// The tests of Contributions and Mask, each on its own rather than up to the first that fails.
std::vector<std::vector<ObservationVerdict>> MergeOnTheFly::Verdicts(const std::vector<IntegrationOutcome> &outcomes) const {
std::vector<std::vector<ObservationVerdict>> out(outcomes.size());
for (size_t i = 0; i < outcomes.size(); ++i) {
const uint32_t image_flags = Mask(outcomes[i]) ? OBS_IMAGE_MASKED : 0;
for (const auto &r: outcomes[i].reflections) {
ObservationVerdict v{.flags = image_flags};
if (generator.IsSystematicallyAbsent(r))
v.flags |= OBS_ABSENT;
const bool scaled = r.image_scale_corr > 0.0 && std::isfinite(r.image_scale_corr);
if (!scaled)
v.flags |= OBS_NO_SCALE;
if (!AcceptReflection(r, high_resolution_limit, std::nullopt))
v.flags |= OBS_RESOLUTION;
if (exclude_ice_rings && r.on_ice_ring)
v.flags |= OBS_ICE_EXCLUDED;
if (BelowMergeFloor(r.partiality))
v.flags |= OBS_LOW_PARTIALITY;
const uint64_t hkl_key = generator(r).pack();
const auto [I_corr, w] = scaled ? Weigh(r, hkl_key) : Weighted{};
if (w == 0.0) {
if (scaled)
v.flags |= OBS_BAD_VALUE;
out[i].push_back(v);
continue;
}
v.sigma = static_cast<float>(std::sqrt(1.0 / w));
if (reject_outliers && error_model_active) {
const auto mit = reject_median_I.find(hkl_key);
const auto bit = error_model_mean_I.find(hkl_key);
if (mit != reject_median_I.end() && bit != error_model_mean_I.end()
&& OutsideOutlierBand(I_corr, mit->second, reject_nsigma, 1.0 / w,
bit->second, error_model_b * bit->second, reject_up, reject_down))
v.flags |= OBS_OUTLIER;
}
out[i].push_back(v);
}
}
return out;
}
void MergeOnTheFly::Accumulate(const std::vector<Contribution> &contributions, int64_t image_id) {
const int prefer = HalfForImage(image_id);
@@ -158,6 +202,13 @@ void MergeOnTheFly::Accumulate(const std::vector<Contribution> &contributions, i
it->second.n_half[half]++;
it->second.sum_wI_split[prefer] += c.wI;
it->second.sum_w_split[prefer] += c.w;
if (c.exists) {
it->second.exists = true;
it->second.exists_half[half] = true;
it->second.exists_split[prefer] = true;
}
if (c.far_tail)
++far_tail_count;
if (c.on_ice_ring)
it->second.on_ice_ring = true;
@@ -193,57 +244,65 @@ void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
});
}
float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const {
if (!error_model_active)
return sigma_corr;
// Intensity for the (b*I)^2 term: the reflection's mean (constant over its
// observations), falling back to this observation only if the mean is unknown.
const auto it = error_model_mean_I.find(hkl_key);
const double I_for_b = (it != error_model_mean_I.end()) ? it->second : I_corr;
// Base variance for the a*sigma^2 term. A weak observation's sigma^2 = a background/read part plus a
// Poisson signal part proportional to its OWN intensity; weighting the merge by 1/sigma^2 with that
// per-observation sigma biases the inverse-variance mean low at <1 photon (an up-fluctuated observation
// gets a larger sigma and is over-downweighted, so the weighted mean drifts below <I>). Rebuild the
// signal part at the reflection's EXPECTED intensity <I> instead - decompose out the background part
// and re-add corr*<I> - so the weight no longer correlates with the observation's own fluctuation. This
// mirrors the rotation combine in RotationScaleMerge::process_rawrun and is bit-identical when the
// observation sits at its reflection mean.
// The signal part removed here is the one the observation's own sigma actually carries, so it is
// taken at max(0, I): a negative intensity has no Poisson signal to subtract, and subtracting it
// anyway ADDS to the background part and inflates the weight of exactly the down-fluctuated
// observations this correction exists to stop being mistreated.
double a_var = static_cast<double>(sigma_corr) * sigma_corr;
double MergeOnTheFly::CountingVariance(double sigma_corr, float image_scale_corr, float var_bkg, float partiality,
double I_mean) const {
// A weak observation's sigma^2 = a background/read part plus a Poisson signal part proportional to its
// OWN intensity; weighting the merge by 1/sigma^2 with that per-observation sigma biases the
// inverse-variance mean low at <1 photon (an up-fluctuated observation gets a larger sigma and is
// over-downweighted, so the weighted mean drifts below <I>). Rebuild the signal part at the reflection's
// EXPECTED intensity <I> instead - the background part plus corr*<I> - so the weight no longer
// correlates with the observation's own fluctuation. This mirrors the rotation combine in
// RotationScaleMerge::process_rawrun. A negative mean has no Poisson signal, hence max(0, <I>).
double v = sigma_corr * sigma_corr;
if (scaling_settings.GetExpectedVarianceMerge()) {
const double bkg_var = static_cast<double>(image_scale_corr) * image_scale_corr * var_bkg;
const double base = bkg_var + static_cast<double>(image_scale_corr) * std::max(0.0, I_for_b);
const double base = static_cast<double>(image_scale_corr) * image_scale_corr * var_bkg
+ static_cast<double>(image_scale_corr) * std::max(0.0, I_mean);
if (base > 0.0)
a_var = base;
v = base;
}
const double capture = capture_coeff * (1.0 - std::min(1.0f, partiality)) * std::max(0.0, I_for_b);
a_var += capture * capture;
const double capture = capture_coeff * (1.0 - std::min(1.0f, partiality)) * std::max(0.0, I_mean);
return v + capture * capture;
}
const double v = error_model_a * a_var
+ (error_model_b * I_for_b) * (error_model_b * I_for_b);
return (v > 0.0) ? static_cast<float>(std::sqrt(v)) : sigma_corr;
double MergeOnTheFly::PartialityTerm(float partiality) {
const double q = (1.0 - std::min(1.0f, partiality)) / partiality;
return q * q;
}
double MergeOnTheFly::CorrectedVariance(double I_corr, double sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const {
if (!error_model_active)
return sigma_corr * sigma_corr;
// The systematic terms use the reflection's mean (constant over its observations), falling back to
// this observation only if the mean is unknown: using the per-observation I_i instead would
// over-weight down-fluctuated points.
const auto it = error_model_mean_I.find(hkl_key);
const double I_mean = (it != error_model_mean_I.end()) ? it->second : I_corr;
const double v = error_model_a * CountingVariance(sigma_corr, image_scale_corr, var_bkg, partiality, I_mean)
+ (error_model_b * error_model_b + error_model_c2 * PartialityTerm(partiality)) * I_mean * I_mean;
return (v > 0.0) ? v : sigma_corr * sigma_corr;
}
MergeOnTheFly::Weighted MergeOnTheFly::Weigh(const Reflection &r, uint64_t hkl_key) const {
const double I_corr = static_cast<double>(r.I) * r.image_scale_corr;
const double sigma_corr = static_cast<double>(r.sigma) * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0)
return {};
const double w = 1.0 / CorrectedVariance(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality, hkl_key);
if (!std::isfinite(w) || !(w > 0.0))
return {};
return {I_corr, w};
}
void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outcomes) {
// Reset to identity up front: every early return below then leaves the model
// inactive (CorrectedSigma returns sigma unchanged) rather than keeping a stale
// a/b from a previous call alongside a freshly-cleared mean map.
// Median of the chi-square(1) distribution: a single observation's squared deviation from its
// reflection mean, divided by its variance, is chi-square(1)-distributed, so its median is this
// fraction of its mean. Used both to de-bias the median-based variance fit and to normalize the
// reported median reduced chi^2 so that honestly calibrated sigmas give 1.0 (not 0.4549).
constexpr double CHI2_1_MEDIAN = 0.454936;
// Reset to identity up front: every early return below then leaves the model inactive (CorrectedVariance
// returns sigma unchanged) rather than keeping a stale model from a previous call.
error_model_active = false;
error_model_a = 1.0;
error_model_b = 0.0;
error_model_c2 = 0.0;
error_model_chi2 = 0.0;
error_model_b_resolved = true;
error_model_mean_I.clear();
@@ -252,12 +311,11 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
reject_down = 0.0;
reject_count = 0;
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl,
// applying exactly the filters AddImage uses. ---
struct Obs { float I, sigma, p; };
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, applying the filters
// AddImage uses, down to the existence partiality.
struct Obs { float I, sigma, corr, var_bkg, p; };
struct KeyedObs { uint64_t key; Obs obs; };
std::unordered_map<uint64_t, std::vector<Obs>> groups;
std::vector<KeyedObs> all; // in image order
ReduceImagesInOrder<KeyedObs>(
outcomes.size(), nthreads,
[&](size_t i, std::vector<KeyedObs> &out) {
@@ -272,190 +330,191 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
continue;
if (exclude_ice_rings && r.on_ice_ring)
continue;
if (r.partiality < min_partiality)
if (r.partiality < existence_min_partiality)
continue;
const float I_corr = r.I * r.image_scale_corr;
const float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
continue;
out.push_back({generator(r).pack(), {I_corr, sigma_corr, r.partiality}});
out.push_back({generator(r).pack(), {I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality}});
}
},
[&](size_t, const std::vector<KeyedObs> &out) {
for (const auto &o: out)
groups[o.key].push_back(o.obs);
all.insert(all.end(), out.begin(), out.end());
});
// --- 2. One global pool of (sigma^2, <I>^2, bias-corrected squared deviation). For an
// observation in a group of n, the residual from the inverse-variance mean has
// E[(I_i - <I>)^2] = sigma_i^2 (1 - h_i), h_i = w_i / sum_w (its leverage). The
// (b*I)^2 term uses the reflection mean, so the mean (not I_i) is the abscissa. ---
struct Sample { double s2, I2, dev2; };
std::vector<Sample> samples;
std::vector<std::pair<double, double>> log_spread; // |ln(I / median)|, weight
for (const auto &[key, obs]: groups) {
if (obs.size() < 2)
continue;
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
sum_w += w;
sum_wI += w * o.I;
// The observations of each reflection side by side, in key order and within a reflection in image
// order - a sort on (key, place), a total order, so the same on any number of workers. Every sum
// below runs over them in that fixed order.
struct KeyPlace { uint64_t key; size_t place; };
std::vector<KeyPlace> order(all.size());
const size_t obs_threads = ThreadsForWork(all.size(), nthreads);
ParallelChunks(static_cast<int>(all.size()), obs_threads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
order[i] = {all[i].key, static_cast<size_t>(i)};
});
ParallelSort(order.begin(), order.end(), nthreads, [](const KeyPlace &x, const KeyPlace &y) {
return x.key < y.key || (x.key == y.key && x.place < y.place);
});
std::vector<Obs> flat(all.size());
ParallelChunks(static_cast<int>(all.size()), obs_threads, [&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
flat[i] = all[order[i].place].obs;
});
all = {};
std::vector<uint64_t> keys;
std::vector<size_t> start; // reflection k is flat[start[k] .. start[k + 1])
for (size_t i = 0; i < order.size(); ++i)
if (i == 0 || order[i].key != order[i - 1].key) {
keys.push_back(order[i].key);
start.push_back(i);
}
if (!(sum_w > 0.0))
continue;
const double mean = sum_wI / sum_w;
error_model_mean_I[key] = static_cast<float>(mean);
// Robust centre for outlier rejection: the median intensity (resists the very
// outliers the inverse-variance mean is being protected from). Only when active, and only
// where there are three observations for a median to stand on.
if (reject_outliers && obs.size() >= 3) {
std::vector<float> iv;
iv.reserve(obs.size());
for (const auto &o: obs)
iv.push_back(o.I);
std::nth_element(iv.begin(), iv.begin() + iv.size() / 2, iv.end());
reject_median_I[key] = iv[iv.size() / 2];
start.push_back(order.size());
order = {};
const auto group = [&](int k) { return std::span<const Obs>(flat.data() + start[k], start[k + 1] - start[k]); };
const int n_keys = static_cast<int>(keys.size());
const size_t key_threads = ThreadsForWork(keys.size(), nthreads, 1024);
// --- 2. The starting mean of each reflection (counting-sigma weights) and, where there are three
// observations for it to stand on, the median - the robust centre of the outlier test.
std::vector<double> mean(n_keys);
std::vector<float> median(n_keys, NAN);
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k) {
const auto obs = group(k);
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
sum_w += w;
sum_wI += w * o.I;
}
mean[k] = sum_wI / sum_w;
if (reject_outliers && obs.size() >= 3) {
std::vector<float> iv;
iv.reserve(obs.size());
for (const auto &o: obs)
iv.push_back(o.I);
std::nth_element(iv.begin(), iv.begin() + iv.size() / 2, iv.end());
median[k] = iv[iv.size() / 2];
}
}
const double I2 = mean * mean;
for (const auto &o: obs) {
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
const double factor = 1.0 - w / sum_w;
if (factor < 0.05)
continue;
const double resid = static_cast<double>(o.I) - mean;
const double capture = capture_coeff * (1.0 - std::min(1.0f, o.p)) * std::max(0.0, mean);
const double s2 = static_cast<double>(o.sigma) * o.sigma + capture * capture;
samples.push_back({s2, I2, resid * resid / factor});
const auto mit = reject_median_I.find(key);
if (mit != reject_median_I.end() && o.I > 0.0f && mit->second > 0.0f && mean > 0.0)
log_spread.emplace_back(std::fabs(std::log(o.I / mit->second)), I2 / s2);
}
}
});
for (int k = 0; k < n_keys; ++k)
if (!std::isnan(median[k]))
reject_median_I[keys[k]] = median[k];
// The spread of ln(I / median) the band's multiplicative part is drawn from: its weighted median
// absolute value, weighted by (<I>/sigma)^2 so the observations whose ratio counting noise does not
// blur carry it - as the rotation merge measures it.
if (!log_spread.empty()) {
std::sort(log_spread.begin(), log_spread.end());
double total = 0.0;
for (const auto &[v, w]: log_spread) total += w;
double run = 0.0;
size_t j = 0;
for (; j + 1 < log_spread.size(); ++j) {
run += log_spread[j].second;
if (run >= 0.5 * total) break;
}
const double s = 1.4826 * log_spread[j].first;
reject_up = std::exp(reject_nsigma * s) - 1.0;
reject_down = 1.0 - std::exp(-reject_nsigma * s);
}
// --- 3. Fit global dev2 = a*sigma^2 + b^2*<I>^2. Bin by intensity (the per-observation
// dev2 is chi-square-1 noisy) and take medians; weight the bins by 1/dev2^2 so it
// is a *relative* fit - otherwise the strong bins (which fix b) swamp the weak
// bins (which fix a) and the weak sigmas stay over-confident. ---
constexpr int n_bins = 16;
if (samples.size() < static_cast<size_t>(8 * n_bins))
return; // too little multiplicity to fit -> leave identity
std::sort(samples.begin(), samples.end(),
[](const Sample &p, const Sample &q) { return p.I2 < q.I2; });
auto median = [](std::vector<double> &v) {
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
return v[v.size() / 2];
// --- 3. Fit the model (ErrorModel.h, FitStillsErrorModel) on the scatter about the reflection means,
// the counting variance rebuilt at the mean, and refit with the means the fitted model weights:
// the means and the model each depend on the other, and a single pass leaves the weights of
// the strong reflections at the counting-sigma means the fit started from. Measured on serial
// and rotation-as-serial sets, a few rounds raise the anomalous signal of the merge; they have
// settled after four, most often already after two or three: a round that leaves the model where
// the previous one put it (to 1e-3) would only reproduce the same means, and ends the iteration.
// Those rounds only carry the means to where the model settles, so they fit every stride-th
// observation (some four million of them); one last round fits them all and makes the final means.
// Each observation's sample goes to its own slot, so the pool is in key order whatever the number
// of workers; the slots left empty (single observations, the leverage cut) are dropped after.
constexpr int ROUNDS = 4;
const size_t stride = std::max<size_t>(1, flat.size() >> 22);
bool last_round = false;
std::vector<StillsErrorModelSample> samples;
StillsErrorModelFit fit;
double a = 1.0, b2 = 0.0, c2 = 0.0;
auto variance = [&](const Obs &o, double m) {
return a * CountingVariance(o.sigma, o.corr, o.var_bkg, o.p, m) + (b2 + c2 * PartialityTerm(o.p)) * m * m;
};
// Per-intensity-bin medians of (sigma^2, <I>^2, dev2).
std::vector<double> bs2, bI2, bd2;
bs2.reserve(n_bins); bI2.reserve(n_bins); bd2.reserve(n_bins);
const size_t per = samples.size() / n_bins;
for (int bin = 0; bin < n_bins; ++bin) {
const size_t lo = bin * per;
const size_t hi = (bin == n_bins - 1) ? samples.size() : lo + per;
std::vector<double> vs2, vI2, vd2;
vs2.reserve(hi - lo); vI2.reserve(hi - lo); vd2.reserve(hi - lo);
for (size_t i = lo; i < hi; ++i) {
vs2.push_back(samples[i].s2);
vI2.push_back(samples[i].I2);
vd2.push_back(samples[i].dev2);
}
bs2.push_back(median(vs2));
bI2.push_back(median(vI2));
// The per-observation dev2 is sigma^2 * chi-square(1)-distributed, whose MEDIAN is 0.4549 of
// its mean. Fitting the model to the robust median would therefore calibrate the variances to
// 0.4549x their true value (reduced chi^2 ~ 1/0.4549 = 2.2). Divide the median by that constant
// to recover an unbiased estimate of the mean (E[dev2] = sigma^2), keeping the robustness of
// the median while targeting reduced chi^2 = 1.
bd2.push_back(median(vd2) / CHI2_1_MEDIAN);
for (int round = 0; ; ++round) {
const size_t step = last_round ? 1 : stride;
samples.assign((flat.size() + step - 1) / step, StillsErrorModelSample{NAN, NAN, NAN, NAN});
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
std::vector<double> var;
for (int k = lo; k < hi; ++k) {
const auto obs = group(k);
if (obs.size() < 2)
continue;
const double m = mean[k];
// The residual from the weighted mean has E[(I_i - <I>)^2] = var_i (1 - h_i), h_i = w_i / sum_w
// its leverage, with the weights of the model that made the mean.
var.resize(obs.size());
double sum_w = 0.0;
for (size_t j = 0; j < obs.size(); ++j) {
var[j] = variance(obs[j], m);
sum_w += 1.0 / var[j];
}
for (size_t j = 0; j < obs.size(); ++j) {
if ((start[k] + j) % step != 0)
continue;
const auto &o = obs[j];
const double factor = 1.0 - 1.0 / (var[j] * sum_w);
if (factor < 0.05)
continue;
const double resid = static_cast<double>(o.I) - m;
samples[(start[k] + j) / step] = {CountingVariance(o.sigma, o.corr, o.var_bkg, o.p, m), m * m,
PartialityTerm(o.p) * m * m, resid * resid / factor};
}
}
});
samples.erase(std::remove_if(samples.begin(), samples.end(),
[](const StillsErrorModelSample &x) { return std::isnan(x.dev2); }),
samples.end());
const StillsErrorModelFit last = fit;
fit = FitStillsErrorModel(samples, nthreads, last); // each round starts from the last one's model
if (!fit.active)
return; // too little multiplicity to fit -> leave identity
a = fit.a; b2 = fit.b2; c2 = fit.c2;
const auto close = [](double x, double y) { return std::fabs(x - y) <= 1e-3 * std::fabs(y); };
const bool settled = round > 0 && close(a, last.a) && close(b2, last.b2) && close(c2, last.c2);
// The new means leave out what the merge's outlier test will leave out (the band of OutlierBand.h
// about the median, as Contributions draws it): the systematic terms of every variance scale with the
// mean, so a mean that took in the outliers - observations of the innermost reflections that carry a
// beam-stop flare the frame's scale does not, read at 10-30x the median on weak frames - widens the
// band by as much as it is off, keeps the outliers it should drop, and weights them like the rest.
const double band = reject_nsigma * std::sqrt(b2);
ParallelChunks(n_keys, key_threads, [&](int lo, int hi) {
for (int k = lo; k < hi; ++k) {
double sum_w = 0.0, sum_wI = 0.0;
for (const auto &o: group(k)) {
const double v = variance(o, mean[k]);
if (reject_outliers && !std::isnan(median[k])
&& OutsideOutlierBand(o.I, median[k], reject_nsigma, v, mean[k], std::sqrt(b2) * mean[k],
band, band))
continue;
const double w = 1.0 / v;
sum_w += w;
sum_wI += w * o.I;
}
mean[k] = sum_wI / sum_w;
}
});
if (last_round || ((settled || round + 1 == ROUNDS) && step == 1))
break;
last_round = settled || round + 1 == ROUNDS;
}
// Relative-weighted (1/dev2^2) least squares for (a, b^2). Floor the weight's dev2 at a
// small fraction of the typical bin dev2: an absolute floor (1e-30) does not stop a
// near-zero-scatter bin from acquiring a runaway weight and hijacking the fit, so the
// floor must scale with the data. The regression target keeps the unfloored dev2.
std::vector<double> bd2_sorted = bd2;
const double dev2_floor = std::max(1e-30, 1e-3 * median(bd2_sorted));
double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0;
for (int bin = 0; bin < n_bins; ++bin) {
const double s2 = bs2[bin], I2 = bI2[bin], d2 = bd2[bin];
const double d2w = std::max(d2, dev2_floor);
const double wgt = 1.0 / (d2w * d2w);
Ass += wgt * s2 * s2;
AsI += wgt * s2 * I2;
AII += wgt * I2 * I2;
Bs += wgt * s2 * d2;
BI += wgt * I2 * d2;
}
// Reject a near-collinear (ill-conditioned) system *relatively*: det lies in
// [0, Ass*AII] by Cauchy-Schwarz, so compare against that scale rather than 1e-30.
const double det = Ass * AII - AsI * AsI;
if (!(det > 1e-10 * Ass * AII))
return;
const double a_fit = (Bs * AII - BI * AsI) / det;
const double b2_fit = (Ass * BI - AsI * Bs) / det;
const double a = std::clamp(a_fit, 0.25, 100.0);
const double b2 = std::max(b2_fit, 0.0);
// Whether 1/b can be reported as ISa, decided as the rotation fit decides it (ErrorModelUpdate):
// b^2 two of its standard errors clear of zero, the error from the bins' scatter about the line,
// and a inside its bounds - beside a clamped a the b^2 above is not the solution of any fit, and
// it is a near-zero-scatter bin that puts a on its bound (see the weight floor above).
double chi = 0.0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double r = (bd2[bin] - a_fit * bs2[bin] - b2_fit * bI2[bin]) / bd2[bin];
chi += r * r;
}
error_model_b_resolved = a == a_fit && b2_fit >= 2.0 * std::sqrt(chi / (n_bins - 2) * Ass / det);
for (int k = 0; k < n_keys; ++k)
error_model_mean_I[keys[k]] = static_cast<float>(mean[k]);
error_model_a = a;
error_model_b = std::sqrt(b2);
error_model_c2 = c2;
error_model_b_resolved = fit.b_resolved;
error_model_active = true;
// A still's observations of one reflection spread by their partiality and per-crystal scale, not by a
// multiplicative factor about the median: many read near zero, so the spread of ln(I / median) is
// wide and the band drawn from it lets through even a powder grain or a neighbour's spot at hundreds
// of sigma. The error model's b is that spread measured as it is, so stills test the plain band of
// n sigma about the median.
if (stills) {
reject_up = reject_nsigma * error_model_b;
reject_down = reject_nsigma * error_model_b;
}
// multiplicative factor about the median: many read near zero, so a band drawn from the spread of
// ln(I / median) would let through even a powder grain or a neighbour's spot at hundreds of sigma.
// The band is the plain one of n sigma about the median.
reject_up = reject_nsigma * error_model_b;
reject_down = reject_nsigma * error_model_b;
// Achieved goodness of fit: the median of the per-observation dev2/(a*sigma^2 + (b*<I>)^2). That
// ratio is chi-square(1)-distributed (median 0.4549) when the sigmas are correct, so normalize by
// CHI2_1_MEDIAN to report a median reduced chi^2 that targets 1.0. The median (not mean) keeps it
// robust to the heavy outlier tail of serial data.
// Achieved goodness of fit: the median of dev2 / var over the samples, which is chi^2(1)-distributed
// (median 0.4549) when the sigmas are right; normalised so that honest sigmas report 1.0.
std::vector<double> chi2;
chi2.reserve(samples.size());
for (const auto &s: samples) {
const double v = a * s.s2 + b2 * s.I2;
if (v > 0.0)
chi2.push_back(s.dev2 / v);
for (const auto &x: samples)
chi2.push_back(x.dev2 / (a * x.s2 + b2 * x.I2 + c2 * x.P2));
if (!chi2.empty()) {
std::nth_element(chi2.begin(), chi2.begin() + chi2.size() / 2, chi2.end());
error_model_chi2 = chi2[chi2.size() / 2] / ERROR_MODEL_CHI2_1_MEDIAN;
}
error_model_chi2 = chi2.empty() ? 0.0 : median(chi2) / CHI2_1_MEDIAN;
}
bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) const {
@@ -490,7 +549,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
out.reserve(accumulator.size());
for (const auto &[key, accum_ptr]: by_key) {
const MergeAccum &accum = *accum_ptr;
if (accum.sum_w <= 0.0)
if (accum.sum_w <= 0.0 || !accum.exists)
continue;
MergedReflection mr{
@@ -505,7 +564,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
.on_ice_ring = accum.on_ice_ring
};
if (accum.n_half[0] + accum.n_half[1] > 0 && accum.sum_w_half[0] > 0.0 && accum.sum_w_half[1] > 0.0) {
if (accum.exists_half[0] && accum.exists_half[1]) {
for (int i = 0; i < 2; ++i) {
mr.I_half[i] = static_cast<float>(accum.sum_wI_half[i] / accum.sum_w_half[i]);
mr.sigma_half[i] = static_cast<float>(1.0 / std::sqrt(accum.sum_w_half[i]));
@@ -513,7 +572,7 @@ std::vector<MergedReflection> MergeOnTheFly::ExportIntensities() {
}
for (int i = 0; i < 2; ++i)
if (accum.sum_w_split[i] > 0.0)
if (accum.exists_split[i])
mr.I_split[i] = static_cast<float>(accum.sum_wI_split[i] / accum.sum_w_split[i]);
if (!std::isfinite(accum.d) || accum.d <= 0.0f)
@@ -554,6 +613,9 @@ struct ShellAccum {
CorrelationCoefficient cc_ref;
CorrelationCoefficient cc_split;
CorrelationCoefficient cc_anom;
double anom_abs_diff = 0.0, anom_sigma = 0.0;
int anom_pairs = 0;
double split_abs_diff = 0.0;
double split_sum = 0.0;
int split_pairs = 0;
@@ -775,19 +837,47 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
}
}
// Per-reflection mean <I>, and a per-reflection accumulator for R_meas - it needs |I_i - <I>|,
// so the observations are visited again now that the means are known.
std::unordered_map<uint64_t, float> merged_I;
merged_I.reserve(merged.size());
// Anomalous statistics where the merge keeps the Friedel hands apart: an I(-) is stored at the negated
// index of its I(+). CC_anom between the frame-whole half-sets (I_split), so the two differences come
// from different crystals; SigAno against the merged sigmas. Acentric reflections only.
if (!scaling_settings.GetMergeFriedel()) {
std::map<std::tuple<int, int, int>, const MergedReflection *> by_index;
for (const auto &m: merged)
by_index[{m.h, m.k, m.l}] = &m;
const auto &ops = space_group.operations();
for (const auto &[hkl, plus]: by_index) {
const auto it = by_index.find({-plus->h, -plus->k, -plus->l});
if (it == by_index.end() || hkl <= it->first) // each pair once
continue;
const MergedReflection &minus = *it->second;
if (ops.is_reflection_centric({plus->h, plus->k, plus->l}))
continue;
const auto shell = shells.GetShell(plus->d);
if (!shell.has_value() || *shell < 0 || *shell >= n_shells)
continue;
for (auto *a: {&acc[*shell], &split_overall}) {
if (std::isfinite(plus->I) && std::isfinite(minus.I) && plus->sigma > 0.0f && minus.sigma > 0.0f) {
a->anom_abs_diff += std::fabs(plus->I - minus.I);
a->anom_sigma += std::hypot(plus->sigma, minus.sigma);
a->anom_pairs++;
}
if (std::isfinite(plus->I_split[0]) && std::isfinite(plus->I_split[1])
&& std::isfinite(minus.I_split[0]) && std::isfinite(minus.I_split[1]))
a->cc_anom.Add(plus->I_split[0] - minus.I_split[0], plus->I_split[1] - minus.I_split[1]);
}
}
}
// The observations are counted again, now only those of the reflections that are reported. No
// R_merge / R_meas / R_pim: they are rotation statistics, and on partials scaled to full equivalents
// they read hundreds of percent and mean nothing. R_split (below) is the serial one.
std::unordered_set<uint64_t> reported;
reported.reserve(merged.size());
for (const auto &m: merged)
if (std::isfinite(m.I))
merged_I[generator(m).pack()] = m.I;
reported.insert(generator(m).pack());
struct RmeasObs { double sum_abs_dev = 0.0; double sum_I = 0.0; int n = 0; int shell = -1; };
std::unordered_map<uint64_t, RmeasObs> rmeas_obs;
rmeas_obs.reserve(merged.size());
struct ShellObs { uint64_t key; float I_corr; float merged_I; int shell; bool merged; };
struct ShellObs { int shell; bool merged; };
ReduceImagesInOrder<ShellObs>(
integration_outcome.size(), nthreads,
[&](size_t i, std::vector<ShellObs> &out) {
@@ -800,62 +890,33 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
continue;
if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A))
continue;
if (r.partiality < min_partiality)
if (BelowMergeFloor(r.partiality))
continue;
const float I_corr = r.I * r.image_scale_corr;
const float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
const auto key = generator(r).pack();
const auto [I_corr, w] = Weigh(r, key);
if (w == 0.0)
continue;
const auto shell = shells.GetShell(r.d);
if (!shell.has_value())
continue;
const int s = *shell;
if (s >= 0 && s < n_shells) {
const auto key = generator(r).pack();
const auto mit = merged_I.find(key);
out.push_back({key, I_corr, mit != merged_I.end() ? mit->second : NAN, s,
mit != merged_I.end()});
}
if (s >= 0 && s < n_shells)
out.push_back({s, reported.contains(key)});
}
},
[&](size_t, const std::vector<ShellObs> &out) {
for (const auto &o: out) {
acc[o.shell].total_obs++;
if (o.merged) {
auto &ra = rmeas_obs[o.key];
ra.sum_abs_dev += std::abs(static_cast<double>(o.I_corr) - o.merged_I);
ra.sum_I += o.I_corr;
ra.n++;
ra.shell = o.shell;
}
}
for (const auto &o: out)
if (o.merged) // the observations of a reflection that is not reported are not counted
acc[o.shell].total_obs++;
});
// R_meas per shell: sum over reflections of sqrt(n/(n-1)) * sum_i|I_i - <I>|, over sum of I_i.
// R_merge and R_pim beside it, with the factor 1 and sqrt(1/(n-1)) - R_pim following Weiss (2001)
// J. Appl. Cryst. 34, 130-135.
std::vector<double> rmeas_num(n_shells, 0.0), rmeas_den(n_shells, 0.0);
std::vector<double> rmerge_num(n_shells, 0.0), rpim_num(n_shells, 0.0);
double rmeas_num_all = 0.0, rmeas_den_all = 0.0, rmerge_num_all = 0.0, rpim_num_all = 0.0;
for (const auto &[key, ra]: rmeas_obs) {
if (ra.n < 2 || ra.shell < 0 || ra.shell >= n_shells)
continue;
const double factor = std::sqrt(static_cast<double>(ra.n) / (ra.n - 1));
const double pim_factor = std::sqrt(1.0 / (ra.n - 1));
rmeas_num[ra.shell] += factor * ra.sum_abs_dev;
rmerge_num[ra.shell] += ra.sum_abs_dev;
rpim_num[ra.shell] += pim_factor * ra.sum_abs_dev;
rmeas_den[ra.shell] += ra.sum_I;
rmeas_num_all += factor * ra.sum_abs_dev;
rmerge_num_all += ra.sum_abs_dev;
rpim_num_all += pim_factor * ra.sum_abs_dev;
rmeas_den_all += ra.sum_I;
}
// R_split as White et al. (2012) J. Appl. Cryst. 45, 335-341 define it (CrystFEL's compare_hkl).
auto set_split = [](const ShellAccum &a, MergeStatisticsShell &ss) {
if (a.anom_pairs > 0) {
ss.abs_diff_over_sigma_anomalous = a.anom_abs_diff / a.anom_sigma;
ss.cc_anom = a.cc_anom.GetCC();
}
ss.n_split_pairs = a.split_pairs;
if (a.split_pairs == 0)
return;
@@ -865,6 +926,7 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
};
MergeStatistics out;
out.stills = true;
out.shells.resize(n_shells);
out.n_observations_rejected = reject_count;
@@ -884,9 +946,6 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
ss.cc_half = sa.cc_half.GetCC();
ss.cc_ref = sa.cc_ref.GetCC();
ss.r_meas = rmeas_den[s] > 0.0 ? rmeas_num[s] / rmeas_den[s] : NAN;
ss.r_merge = rmeas_den[s] > 0.0 ? rmerge_num[s] / rmeas_den[s] : NAN;
ss.r_pim = rmeas_den[s] > 0.0 ? rpim_num[s] / rmeas_den[s] : NAN;
set_split(sa, ss);
}
@@ -914,9 +973,6 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector<MergedReflection> &m
overall.mean_i_over_sigma = n_i_over_sigma > 0 ? sum_i_over_sigma / n_i_over_sigma : 0.0;
overall.cc_half = cc_half_overall.GetCC();
overall.cc_ref = cc_ref_overall.GetCC();
overall.r_meas = rmeas_den_all > 0.0 ? rmeas_num_all / rmeas_den_all : NAN;
overall.r_merge = rmeas_den_all > 0.0 ? rmerge_num_all / rmeas_den_all : NAN;
overall.r_pim = rmeas_den_all > 0.0 ? rpim_num_all / rmeas_den_all : NAN;
set_split(split_overall, overall);
return out;
@@ -980,7 +1036,7 @@ namespace {
}
}
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
std::string ShellTableRow(const MergeStatisticsShell &in, bool r_factors) {
// NaN, not 0.0, where the denominator is empty: Measured() below turns that into a dash, and a
// printed 0.0% reads as a measured total failure rather than as a quantity nobody could form.
double completeness = in.possible_unique_reflections > 0
@@ -988,33 +1044,44 @@ std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
double multiplicity = in.unique_reflections > 0
? static_cast<double>(in.total_observations) / in.unique_reflections : NAN;
output << fmt::format("{:8d} {:8d} {:8d} {:>8s} {:>7s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
std::string out = fmt::format("{:8d} {:8d} {:8d} {:>8s} {:>7s} {:>8s} ",
in.total_observations,
in.unique_reflections,
in.possible_unique_reflections,
Measured(completeness, "%"),
Measured(multiplicity),
Measured(in.mean_i_over_sigma),
Measured(in.mean_i_over_sigma));
if (r_factors)
out += fmt::format("{:>8s} {:>8s} {:>8s} ",
Measured(in.r_merge*100.0, "%"),
Measured(in.r_meas*100.0, "%"),
Measured(in.r_pim*100.0, "%"),
Measured(in.r_pim*100.0, "%"));
out += fmt::format("{:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
Measured(in.cc_half*100.0, "%"),
Measured(CCStar(in.cc_half)*100.0, "%"),
Measured(in.cc_ref*100.0, "%"),
Measured(in.cc_anom*100.0, "%"),
Measured(in.abs_diff_over_sigma_anomalous, "", 2));
return output;
return out;
}
std::string ShellTableHeader() {
return fmt::format(" {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>7s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>8s}",
"d_min", "N_obs", "N_uniq", "N_possib", "Compl", "Mult", "<I/sig>", "R_merge", "R_meas", "R_pim",
"CC1/2", "CC*", "CCref", "CCanom", "SigAno");
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in) {
return output << ShellTableRow(in, true);
}
std::string ShellTableRule() {
return fmt::format(" {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->7s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s} {:->8s}",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "");
std::string ShellTableHeader(bool r_factors) {
return fmt::format(" {:>8s} {:>8s} {:>8s} {:>8s} {:>8s} {:>7s} {:>8s} ",
"d_min", "N_obs", "N_uniq", "N_possib", "Compl", "Mult", "<I/sig>")
+ (r_factors ? fmt::format("{:>8s} {:>8s} {:>8s} ", "R_merge", "R_meas", "R_pim") : std::string())
+ fmt::format("{:>8s} {:>8s} {:>8s} {:>8s} {:>8s}", "CC1/2", "CC*", "CCref", "CCanom", "SigAno");
}
std::string ShellTableRule(bool r_factors) {
const std::string dash8(8, '-');
std::string rule = fmt::format(" {0} {0} {0} {0} {0} {1} {0}", dash8, std::string(7, '-'));
for (int i = 0; i < (r_factors ? 8 : 5); ++i)
rule += " " + dash8;
return rule;
}
std::string SerialShellTable(const MergeStatistics &in) {
@@ -1038,20 +1105,22 @@ std::string SerialShellTable(const MergeStatistics &in) {
}
std::ostream &operator<<(std::ostream &output, const MergeStatistics &in) {
// A stills merge has no R_merge / R_meas / R_pim columns: see MergeOnTheFly::MergeStats.
const bool r_factors = !in.stills;
output << std::endl;
output << ShellTableHeader() << std::endl;
output << ShellTableRule() << std::endl;
output << ShellTableHeader(r_factors) << std::endl;
output << ShellTableRule(r_factors) << std::endl;
for (const auto &sh: in.shells) {
if (sh.unique_reflections == 0)
continue;
output << fmt::format(" {:8.2f} ", sh.d_min);
output << sh;
output << ShellTableRow(sh, r_factors);
output << std::endl;
}
output << ShellTableRule() << std::endl;
output << ShellTableRule(r_factors) << std::endl;
output << fmt::format(" {:>8s} ", "Overall");
output << in.overall;
output << ShellTableRow(in.overall, r_factors);
output << std::endl;
output << SerialShellTable(in);
if (std::isfinite(in.wilson_b) && in.wilson_b > 0.0)
+89 -17
View File
@@ -48,14 +48,15 @@ struct MergeStatisticsShell {
// Anomalous signal-to-noise (XDS "SigAno" / mmCIF pdbx_absDiff_over_sigma_anomalous):
// <|I(+)-I(-)|> / <sigma(I(+)-I(-))> over acentric reflections measured in both hands. NaN when
// there is no anomalous split (e.g. Friedel-merged with no mates, or the stills path).
// there is no anomalous split (e.g. Friedel-merged with no mates).
double abs_diff_over_sigma_anomalous = NAN;
// CCanom (AIMLESS "CCanom", phenix "cc_anom"): the correlation of the anomalous difference
// measured twice - I(+)-I(-) from one half of the observations against the same difference from
// the other half, over acentric reflections whose BOTH mates split into two non-empty halves.
// Unlike SigAno it is not a ratio against the error model, so a mis-estimated sigma cannot move
// it. NaN when no reflection has an anomalous split (the stills path), which is not the same
// it. On stills the halves are the frame-whole ones of R_split, so the two differences come from
// different crystals. NaN when no reflection has an anomalous split, which is not the same
// statement as zero - zero means measured and absent.
double cc_anom = NAN;
@@ -181,6 +182,10 @@ struct MergeStatistics {
std::vector<MergeStatisticsShell> shells;
MergeStatisticsShell overall;
AnomalousMergeStatistics per_hand;
// Made by the stills merge (MergeOnTheFly). Its error model is reported as its terms, not as an
// ISa: on stills 1/b is the ceiling of one still observation under the stills model, which is not
// the instrument and crystal error CORRECT.LP's ISa measures.
bool stills = false;
// Dataset-wide isotropic Wilson B-factor estimate (A^2) from the log-linear fit of the shell-mean
// merged intensity against 1/d^2 (CalcGlobalWilsonB) - the analogue of XDS's "WILSON LINE ... B=".
@@ -233,10 +238,12 @@ inline double CCStar(double cc_half) {
std::ostream &operator<<(std::ostream &output, const MergeStatisticsShell &in);
std::ostream &operator<<(std::ostream &output, const MergeStatistics &in);
// The header and the rule of the table operator<< prints, for a table with the same columns rendered
// elsewhere (the report's reference-range table).
std::string ShellTableHeader();
std::string ShellTableRule();
// The header, the rule and one row of the table operator<< prints, for a table with the same columns
// rendered elsewhere (the report's reference-range table). Without r_factors the R_merge / R_meas / R_pim
// columns are left out, as on a stills merge.
std::string ShellTableHeader(bool r_factors = true);
std::string ShellTableRule(bool r_factors = true);
std::string ShellTableRow(const MergeStatisticsShell &in, bool r_factors = true);
// The serial table of R_split and the frame-split CC1/2, per shell and overall; empty where they were
// not computed (rotation).
std::string SerialShellTable(const MergeStatistics &in);
@@ -247,6 +254,9 @@ struct MergeAccum {
int32_t l = 0;
float d = NAN;
bool on_ice_ring = false;
bool exists = false; // an observation reached the existence partiality (MergeOnTheFly), so it is reported
bool exists_half[2] = {false, false}; // the same within each half-set ...
bool exists_split[2] = {false, false}; // ... and each frame-whole half-set
double sum_wI = 0.0;
double sum_w = 0.0;
double sum_wI_half[2] = {0.0, 0.0};
@@ -278,6 +288,29 @@ inline XdsErrorModel ToXdsErrorModel(double a, double b) {
return {a, b * b / a, 1.0 / b};
}
// The stills partiality term c (MergeOnTheFly::ErrorModelC) in the same convention:
// sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2).
inline double ToXdsPartialityTerm(double a, double c) {
return a > 0.0 ? c * c / a : 0.0;
}
// Why the stills merge took or left one observation: a bit for every test it fails, 0 = merged
// (MergeOnTheFly::Verdicts; a diagnostic, --export-observations).
enum ObservationFlag : uint32_t {
OBS_IMAGE_MASKED = 1, // the image is not merged: cell outside the run's population, or --min-image-cc
OBS_ABSENT = 2, // systematic absence of the merge's space group
OBS_NO_SCALE = 4, // no usable scale correction (unscaled image, collapsed scale, p * G = 0)
OBS_RESOLUTION = 8, // outside the merge's resolution limits, or not finite
OBS_ICE_EXCLUDED = 16, // on an ice ring, in a merge that leaves them out
OBS_LOW_PARTIALITY = 32, // partiality below --min-partiality
OBS_BAD_VALUE = 64, // corrected I or sigma not finite, or sigma <= 0
OBS_OUTLIER = 128, // outside the outlier band about the reflection's median
};
struct ObservationVerdict {
uint32_t flags = 0;
float sigma = NAN; // the sigma the merge weights it with (error model applied); NaN if not computable
};
class MergeOnTheFly {
mutable std::mutex merged_mutex;
@@ -292,7 +325,23 @@ class MergeOnTheFly {
// Apply image_cc_limit in Mask(). One flag for the whole engine, not a per-call argument, so the
// merge, the error model and MergeStats can never disagree about which images are in.
bool filter_by_image_cc = false;
// Which observations are merged. A merge without a fitted error model - the references the scale and
// the partiality are refined against - takes those at min_partiality and above. A merge with one takes
// every observation: it carries the partiality model's own error in each variance (ErrorModel.h, the
// stills model's c term), which weights a low-partiality observation by what it is worth. A reflection
// (each hand where the Friedel mates are kept apart) is then reported only if at least one of its
// observations reaches existence_min_partiality, and has a mean in a half-set only if one of its observations
// in that half does: one seen only in the tails of its rocking curve rests on the partiality model alone,
// and on a small set such reflections wreck the outer shells. The error
// model is fitted on the observations at that partiality and above. Measured on serial and
// rotation-as-serial sets: every observation with the term raises the anomalous map at the sulfur sites
// over a floor at 0.2, and the existence rule keeps the smallest sets where the floor had them.
static constexpr double EXISTENCE_MIN_PARTIALITY = 0.2;
double min_partiality = 0.02;
double existence_min_partiality = 0.02;
[[nodiscard]] bool BelowMergeFloor(float partiality) const {
return error_model_active ? partiality <= 0.0f : partiality < min_partiality;
}
// The share of a partial the model could not see, (1 - p), carried in its variance as
// (capture_coeff * (1 - p) * <I>)^2 - the rotation combine's capture term, per observation.
double capture_coeff = 0.0;
@@ -307,23 +356,35 @@ class MergeOnTheFly {
std::unordered_map<uint64_t, MergeAccum> accumulator; // exported in key order
// Global error model (XDS form): sigma_corr^2 = a*sigma^2 + (b*<I>)^2. a rescales the
// (under-estimated) counting variance; the (b*<I>)^2 term adds the intensity-
// proportional systematic error that counting statistics miss, so strong reflections
// are no longer over-weighted. ISa = 1/b is the asymptotic I/sigma. Refined from the
// scatter of symmetry equivalents (RefineErrorModel); identity until then.
// Global error model (XDS form plus a partiality term, ErrorModel.h):
// sigma_corr^2 = a*sigma^2 + (b*<I>)^2 + c^2 ((1 - p) / p)^2 <I>^2. a rescales the (under-estimated)
// counting variance; the (b*<I>)^2 term adds the intensity-proportional systematic error that counting
// statistics miss, so strong reflections are no longer over-weighted; the c term is the partiality
// model's error. ISa = 1/b is the asymptotic I/sigma. Refined from the scatter of symmetry equivalents
// (RefineErrorModel); identity until then.
bool error_model_active = false;
double error_model_a = 1.0;
double error_model_b = 0.0;
double error_model_c2 = 0.0; // the partiality term, c^2 ((1 - p) / p)^2 <I>^2 (ErrorModel.h)
double error_model_chi2 = 0.0; // achieved median reduced chi^2 (~1.0 = honestly calibrated sigmas)
bool error_model_b_resolved = true; // as ErrorModelFit::b_resolved; 1/b is reported as ISa only then
// The (b*I)^2 term uses the reflection's *mean* intensity (constant over its
// observations), so it inflates sigma without biasing the inverse-variance weights -
// using the per-observation I_i instead would over-weight down-fluctuated points.
std::unordered_map<uint64_t, float> error_model_mean_I;
[[nodiscard]] float CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const;
// An observation's counting variance on the merge's scale, its signal part rebuilt at the reflection
// mean I_mean (plus the capture term), and the partiality term's factor ((1 - p) / p)^2.
[[nodiscard]] double CountingVariance(double sigma_corr, float image_scale_corr, float var_bkg, float partiality,
double I_mean) const;
static double PartialityTerm(float partiality);
[[nodiscard]] double CorrectedVariance(double I_corr, double sigma_corr, float image_scale_corr,
float var_bkg, float partiality,
uint64_t hkl_key) const;
// An observation's intensity on the merge's scale and its weight 1/variance, in double: a partial far out on
// its rocking curve (p down to 1e-36) has a 1/p no float holds. weight = 0 for an observation the merge
// cannot use - a value not finite, or a variance so large that the weight is zero: it carries no information.
struct Weighted { double I = 0.0, weight = 0.0; };
[[nodiscard]] Weighted Weigh(const Reflection &r, uint64_t hkl_key) const;
// Optional per-observation outlier rejection: drop observations whose corrected
// intensity lies more than reject_nsigma error-model sigmas from the reflection's
@@ -334,10 +395,9 @@ class MergeOnTheFly {
double reject_nsigma = 6.0;
std::unordered_map<uint64_t, float> reject_median_I;
// The asymmetric band of OutlierBand.h: exp(+-n s) - 1 about the reflection mean, s the spread of
// ln(I / median) measured on this merge's observations; on stills n b, the linear band.
// ln(I / median); the stills merge sets both to n b, the linear band.
double reject_up = 0.0, reject_down = 0.0;
size_t reject_count = 0;
bool stills = false;
bool Mask(const IntegrationOutcome &outcome) const;
@@ -345,9 +405,13 @@ class MergeOnTheFly {
struct Contribution {
uint64_t key;
int32_t h, k, l; // the reflection's own index, as the accumulator records it
float w, wI, d;
double w, wI;
float d;
bool on_ice_ring;
bool exists; // partiality at the existence floor or above
bool far_tail; // partiality below FAR_TAIL_PARTIALITY
};
size_t far_tail_count = 0; // merged observations with a partiality below FAR_TAIL_PARTIALITY
// The observations of one image that reach the merge; returns how many the outlier test dropped.
size_t Contributions(const IntegrationOutcome &outcome, std::vector<Contribution> &out) const;
void Accumulate(const std::vector<Contribution> &contributions, int64_t image_id);
@@ -366,11 +430,15 @@ public:
[[nodiscard]] bool ErrorModelActive() const { return error_model_active; }
[[nodiscard]] double ErrorModelA() const { return error_model_a; }
[[nodiscard]] double ErrorModelB() const { return error_model_b; }
[[nodiscard]] double ErrorModelC() const { return std::sqrt(error_model_c2); }
[[nodiscard]] double ErrorModelChi2() const { return error_model_chi2; }
[[nodiscard]] bool ErrorModelBResolved() const { return error_model_b_resolved; }
// Outlier rejection (driven by ScalingSettings::GetOutlierRejectNsigma) reports its count.
[[nodiscard]] size_t RejectedCount() const { return reject_count; }
// Merged observations with a partiality below FAR_TAIL_PARTIALITY, counted for the log.
static constexpr float FAR_TAIL_PARTIALITY = 1e-3f;
[[nodiscard]] size_t FarTailCount() const { return far_tail_count; }
// image_id is the image's stable identity (its index in the outcomes vector). It seeds the CC1/2
// half-set PREFERENCE, which the balancing rule in AddImage then overrides where a reflection's
@@ -384,6 +452,10 @@ public:
// per-observation work spread over the workers and the accumulation kept in image order.
void AddImages(const std::vector<IntegrationOutcome> &outcomes);
// Every test AddImages applies, each evaluated on its own, for every reflection of every outcome (in
// their order); call after AddImages. A diagnostic: the merge itself does not use it.
std::vector<std::vector<ObservationVerdict>> Verdicts(const std::vector<IntegrationOutcome> &outcomes) const;
// d_min_override, when set, is the effective high-resolution limit for the shell table (used for
// the automatic resolution cutoff computed by the caller); otherwise the manual
// ScalingSettings high-resolution limit stands. The number of shells is ScalingSettings::ReportShellCount.
@@ -137,8 +137,13 @@ void ScaleOnTheFly::Scale(IntegrationOutcome &integration_outcome) const {
obs.push_back({coeff, static_cast<double>(r.I), sigma2, var_bkg});
}
// Too few reflections to put the image on the merge's scale: it is left out of the merge (a
// non-finite correction is what the merge skips on) rather than merged at the scale of 1 its
// prediction carries, which is no scale at all.
if (obs.size() < MIN_REFLECTIONS) {
clear_scale();
for (auto &r: integration_outcome.reflections)
r.image_scale_corr = NAN;
return;
}
@@ -8,6 +8,7 @@
#include <atomic>
#include <cmath>
#include <future>
#include <limits>
#include <thread>
#include <vector>
@@ -17,6 +18,7 @@
#include "Merge.h"
#include "../../common/Logger.h"
#include "../../common/ThreadAffinity.h"
#include "../../common/ParallelFor.h"
namespace {
constexpr size_t MIN_FIT_REFLECTIONS = 20;
@@ -25,6 +27,13 @@ namespace {
// Model error of the per-crystal scale fit, as a fraction of the predicted intensity - ScaleOnTheFly's.
constexpr double SCALE_MODEL_ERROR = 0.3;
// The width scales BalanceRunWidth searches: from 1/BALANCE_RANGE to BALANCE_RANGE times the table.
constexpr double BALANCE_RANGE = 3.0;
// The smallest priors on a crystal's radial scale and log width scale (RunPrior).
constexpr double MIN_PRIOR_ETA = 1e-5;
constexpr double MIN_PRIOR_LN_WIDTH = 0.01;
double SafeInv(double x, double fallback) {
if (!std::isfinite(x) || x == 0.0)
return fallback;
@@ -79,30 +88,6 @@ namespace {
}
};
// Analytic partiality for a reflection whose base reciprocal vector is q, tilted by (psi_x, psi_y).
// Mirrors BraggPrediction: dist_ewald = |S| - 1/lambda with S = q_rot + S0, and
// p = exp(-dist_ewald^2 / 2 sigma^2), sigma^2 = gamma0^2 + (gamma_e*d*)^2 + (bw*|q_z|)^2 - three
// independent broadenings added in quadrature: the reciprocal-lattice point's own radius (gamma0,
// ~1/domain size, resolution-INdependent), the mosaic/divergence spread (gamma_e*d*, proportional to
// d*) and the bandwidth smear along the beam. In practice the fit below returns gamma_e ~ 0 and the
// width is essentially gamma0 - see there.
double ComputeP(double qx, double qy, double qz,
double psi_x, double psi_y,
double s0x, double s0y, double s0z, double inv_lambda,
const Width &width) {
double aa[3] = {psi_x, psi_y, 0.0};
double q[3] = {qx, qy, qz};
double qr[3];
ceres::AngleAxisRotatePoint(aa, q, qr);
const double Sx = qr[0] + s0x, Sy = qr[1] + s0y, Sz = qr[2] + s0z;
const double de = std::sqrt(Sx * Sx + Sy * Sy + Sz * Sz) - inv_lambda;
const double dstar = std::sqrt(qr[0] * qr[0] + qr[1] * qr[1] + qr[2] * qr[2]);
const double sig2 = width.Sigma2(dstar, qr[2]);
if (!(sig2 > 0.0))
return 1.0;
return std::exp(-0.5 * de * de / sig2);
}
// The prediction window cuts the rocking curve. Reflections fill a window of half-width W evenly, so
// their plain moment of de is u = W^2 / 3, and the intensity-weighted moment m of a Gaussian of width
// s cut at +-W is u * g(a), a = W / s, with g(a) = 3/a^2 (1 - 2 a phi(a) / erf(a / sqrt 2)): 1 for a
@@ -170,79 +155,124 @@ namespace {
-s * ky, s * kx, c};
}
// ComputeP with the tilt given as its rotation matrix.
struct TiltModel {
// A crystal's model of its partialities: the orientation tilt (psi_x, psi_y) about the two axes
// perpendicular to the beam, a radial scale eta of its reciprocal lattice (q -> (1 + eta) q: a
// wavelength or a cell a little off for this crystal - on an XFEL the photon energy of its shot) and a
// scale exp(ln_width) of the run's rocking width (a crystal's own mosaicity or domain size).
// Per-crystal mosaicity and lattice scale in post-refinement, following Uervirojnangkoorn et al. (2015)
// eLife 4, e05421 and Sauter et al. (2014) Acta Cryst. D70, 3299-3309.
struct CrystalModel {
double psi_x = 0.0, psi_y = 0.0, eta = 0.0, ln_width = 0.0;
double width2 = 1.0; // exp(2 ln_width), the factor on the run's sigma^2
};
CrystalModel MakeCrystalModel(double psi_x, double psi_y, double eta, double ln_width) {
return {psi_x, psi_y, eta, ln_width, std::exp(2.0 * ln_width)};
}
// The partiality of a reflection whose base reciprocal vector is q under a crystal model. Mirrors
// BraggPrediction: dist_ewald = |S| - 1/lambda with S = q' + S0, q' = (1 + eta) R(psi) q, and
// p = exp(-dist_ewald^2 / 2 sigma^2), sigma = exp(ln_width) sigma_run(d*).
struct PartialityModel {
double s0x, s0y, s0z, inv_lambda;
Width width;
double P(const std::array<double, 9> &R, const FitObs &o) const {
const double qx = R[0] * o.qx + R[1] * o.qy + R[2] * o.qz;
const double qy = R[3] * o.qx + R[4] * o.qy + R[5] * o.qz;
const double qz = R[6] * o.qx + R[7] * o.qy + R[8] * o.qz;
double At(const std::array<double, 9> &R, const CrystalModel &m, double qx0, double qy0, double qz0) const {
const double f = 1.0 + m.eta;
const double qx = f * (R[0] * qx0 + R[1] * qy0 + R[2] * qz0);
const double qy = f * (R[3] * qx0 + R[4] * qy0 + R[5] * qz0);
const double qz = f * (R[6] * qx0 + R[7] * qy0 + R[8] * qz0);
const double de = VecLen(qx + s0x, qy + s0y, qz + s0z) - inv_lambda;
const double sig2 = width.Sigma2(VecLen(qx, qy, qz), qz);
const double sig2 = width.Sigma2(VecLen(qx, qy, qz), qz) * m.width2;
if (!(sig2 > 0.0))
return 1.0;
return std::exp(-0.5 * de * de / sig2);
}
double P(const std::array<double, 9> &R, const CrystalModel &m, const FitObs &o) const {
return At(R, m, o.qx, o.qy, o.qz);
}
};
// Refine the orientation tilt (psi_x, psi_y) holding the scale G fixed: minimise
// sum_i rho(r_i^2) / 2 + |psi / sigma_prior|^2 / 2, r_i = weight_i (G p_i(psi) Iref_i / corr_i - Iobs_i),
// with the Cauchy rho(s) = k^2 log(1 + s / k^2) - the intensity-space objective of ScaleOnTheFly, and
// the Gaussian prior a proper MAP term since r_i is already in sigma units. Levenberg-Marquardt on the
// normal equations reweighted by rho', the Jacobian by central differences, every trial tilt held inside
// the bound. Returns false when the cost cannot be evaluated, leaving psi as it came in.
bool RefineTilt(const std::vector<FitObs> &obs, const TiltModel &model, double G, double robust_k,
double inv_sigma_prior, double max_tilt, double psi[2]) {
// Refine the crystal model holding the scale G fixed: minimise
// sum_i rho(r_i^2) / 2 + sum_k (x_k / sigma_k)^2 / 2, r_i = weight_i (G p_i(x) Iref_i / corr_i - Iobs_i),
// over x = (psi_x, psi_y, eta, ln_width), with the Cauchy rho(s) = k^2 log(1 + s / k^2) - the intensity-
// space objective of ScaleOnTheFly - and Gaussian priors about the run (x = 0), proper MAP terms since
// r_i is already in sigma units. Levenberg-Marquardt on the normal equations reweighted by rho', the
// Jacobian by forward differences in the tilt and the radial scale (the model at x is the residual's
// already) and exact in the width, dp / d ln_width = -2 p ln p, every trial held inside the bounds. Returns false when the cost
// cannot be evaluated, leaving the model as it came in.
bool RefineCrystal(const std::vector<FitObs> &obs, const PartialityModel &model, double G, double robust_k,
const std::array<double, 4> &inv_sigma_prior, const std::array<double, 4> &bound,
CrystalModel &crystal, std::array<double, 4> &variance) {
constexpr int MAX_ITERATIONS = 25;
constexpr double STEP = 1e-7; // rad, for the central differences
constexpr std::array<double, 3> STEP = {1e-7, 1e-7, 1e-7}; // for the forward differences
const double k2 = robust_k * robust_k;
const double prior2 = inv_sigma_prior * inv_sigma_prior;
auto as_model = [](const Eigen::Vector4d &x) { return MakeCrystalModel(x(0), x(1), x(2), x(3)); };
auto cost = [&](const double p[2]) {
const auto R = TiltMatrix(p[0], p[1]);
double c = 0.5 * prior2 * (p[0] * p[0] + p[1] * p[1]);
auto cost = [&](const Eigen::Vector4d &x) {
const auto R = TiltMatrix(x(0), x(1));
const CrystalModel m = as_model(x);
double c = 0.0;
for (int k = 0; k < 4; ++k)
c += 0.5 * inv_sigma_prior[k] * inv_sigma_prior[k] * x(k) * x(k);
for (const auto &o: obs) {
const double r = o.weight * (G * model.P(R, o) * o.inv_corr * o.Iref - o.Iobs);
const double r = o.weight * (G * model.P(R, m, o) * o.inv_corr * o.Iref - o.Iobs);
c += 0.5 * k2 * std::log1p(r * r / k2);
}
return c;
};
double current = cost(psi);
Eigen::Vector4d x(crystal.psi_x, crystal.psi_y, crystal.eta, crystal.ln_width);
double current = cost(x);
if (!std::isfinite(current))
return false;
double lambda = 1e-4;
Eigen::Matrix4d data_A = Eigen::Matrix4d::Identity(); // the normal matrix of the data alone
for (int iter = 0; iter < MAX_ITERATIONS; ++iter) {
const auto R = TiltMatrix(psi[0], psi[1]);
const auto Rxp = TiltMatrix(psi[0] + STEP, psi[1]), Rxm = TiltMatrix(psi[0] - STEP, psi[1]);
const auto Ryp = TiltMatrix(psi[0], psi[1] + STEP), Rym = TiltMatrix(psi[0], psi[1] - STEP);
double A00 = prior2, A01 = 0.0, A11 = prior2;
double b0 = prior2 * psi[0], b1 = prior2 * psi[1];
Eigen::Matrix4d A = Eigen::Matrix4d::Zero();
Eigen::Vector4d b = Eigen::Vector4d::Zero();
for (int k = 0; k < 4; ++k) {
A(k, k) = inv_sigma_prior[k] * inv_sigma_prior[k];
b(k) = A(k, k) * x(k);
}
// The model at x and at x + one step in each of the tilts and the radial scale.
std::array<std::array<double, 9>, 4> R;
std::array<CrystalModel, 4> m;
for (int j = 0; j < 4; ++j) {
Eigen::Vector4d xj = x;
if (j > 0)
xj(j - 1) += STEP[j - 1];
R[j] = TiltMatrix(xj(0), xj(1));
m[j] = as_model(xj);
}
for (const auto &o: obs) {
const double a = o.weight * G * o.inv_corr * o.Iref;
const double r = a * model.P(R, o) - o.weight * o.Iobs;
const double jx = a * (model.P(Rxp, o) - model.P(Rxm, o)) / (2.0 * STEP);
const double jy = a * (model.P(Ryp, o) - model.P(Rym, o)) / (2.0 * STEP);
const double p0 = model.P(R[0], m[0], o);
const double r = a * p0 - o.weight * o.Iobs;
Eigen::Vector4d J;
for (int k = 0; k < 3; ++k)
J(k) = a * (model.P(R[k + 1], m[k + 1], o) - p0) / STEP[k];
J(3) = p0 > 0.0 ? -2.0 * a * p0 * std::log(p0) : 0.0;
const double w = 1.0 / (1.0 + r * r / k2); // rho'
A00 += w * jx * jx;
A01 += w * jx * jy;
A11 += w * jy * jy;
b0 += w * jx * r;
b1 += w * jy * r;
A += w * J * J.transpose();
b += w * r * J;
}
data_A = A;
for (int k = 0; k < 4; ++k)
data_A(k, k) -= inv_sigma_prior[k] * inv_sigma_prior[k];
bool accepted = false;
double trial[2] = {psi[0], psi[1]};
Eigen::Vector4d trial = x;
double trial_cost = current;
while (lambda < 1e10) {
const double M00 = A00 * (1.0 + lambda), M11 = A11 * (1.0 + lambda);
const double det = M00 * M11 - A01 * A01;
if (!(det > 0.0))
Eigen::Matrix4d M = A;
M.diagonal() *= 1.0 + lambda;
const Eigen::Vector4d dx = M.ldlt().solve(b);
if (!dx.allFinite())
break;
trial[0] = std::clamp(psi[0] - (M11 * b0 - A01 * b1) / det, -max_tilt, max_tilt);
trial[1] = std::clamp(psi[1] - (M00 * b1 - A01 * b0) / det, -max_tilt, max_tilt);
for (int k = 0; k < 4; ++k)
trial(k) = std::clamp(x(k) - dx(k), -bound[k], bound[k]);
trial_cost = cost(trial);
if (std::isfinite(trial_cost) && trial_cost < current) {
accepted = true;
@@ -253,14 +283,17 @@ namespace {
}
if (!accepted)
break;
const double step = std::hypot(trial[0] - psi[0], trial[1] - psi[1]);
const double decrease = current - trial_cost;
psi[0] = trial[0];
psi[1] = trial[1];
const double step = (trial - x).norm();
x = trial;
current = trial_cost;
if (decrease <= 1e-6 * current || step <= 1e-10)
break;
}
crystal = as_model(x);
const Eigen::Matrix4d cov = data_A.inverse();
for (int k = 0; k < 4; ++k)
variance[k] = cov(k, k);
return true;
}
}
@@ -273,12 +306,12 @@ StillsPartialityRefine::StillsPartialityRefine(const DiffractionExperiment &x)
min_partiality_(x.GetScalingSettings().GetMinPartiality()),
bandwidth_sigma_(x.GetBandwidthFWHM().value_or(0.0f) / 2.3548f) {}
double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
StillsPartialityRefine::CrystalFit StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
const std::map<HKLKey, double> &reference,
const std::optional<RunWidth> &run_width,
double radial_scale) const {
double radial_scale, const CrystalPrior &prior) const {
if (outcome.reflections.empty())
return 0.0;
return {};
const Coord Astar = outcome.latt.Astar();
const Coord Bstar = outcome.latt.Bstar();
@@ -337,7 +370,7 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
}
if (obs.size() < MIN_FIT_REFLECTIONS || n_de == 0)
return 0.0;
return {};
// Without the run's measured width (too few reflections in the run to measure it), the width comes
// from this crystal's own reflections. Solve the 2x2 normal equations for de^2 = A + B d*^2. A degenerate spread in d* (all reflections in
@@ -372,44 +405,47 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
.table_dstar = run_width ? &run_width->table_dstar : nullptr,
.table_s = run_width ? &run_width->table_s : nullptr};
double psi[2] = {0.0, 0.0};
CrystalModel crystal;
double G = 1.0;
const TiltModel model{.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda,
.width = width};
const bool refine_tilt = obs.size() >= settings_.min_reflections;
const int inner = refine_tilt ? settings_.inner_iterations : 1;
const PartialityModel model{.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda, .width = width};
const bool refine_crystal = obs.size() >= settings_.min_reflections;
const int inner = refine_crystal ? settings_.inner_iterations : 1;
const std::array<double, 4> inv_sigma_prior = {kRadToDeg / settings_.prior_sigma_deg,
kRadToDeg / settings_.prior_sigma_deg,
1.0 / prior.sigma_eta, 1.0 / prior.sigma_ln_width};
std::array<double, 4> variance{NAN, NAN, NAN, NAN};
const std::array<double, 4> bound = {settings_.max_tilt_rad, settings_.max_tilt_rad,
settings_.max_eta, settings_.max_ln_width};
for (int it = 0; it < inner; ++it) {
// (1) Solve G given the current partialities, and weight the tilt fit by the variances at that model.
// (1) Solve G given the current partialities, and weight the model fit by the variances at that model.
const auto R = TiltMatrix(crystal.psi_x, crystal.psi_y);
std::vector<double> coeff(obs.size());
for (size_t j = 0; j < obs.size(); ++j) {
const double p = ComputeP(obs[j].qx, obs[j].qy, obs[j].qz, psi[0], psi[1],
S0.x, S0.y, S0.z, inv_lambda, width);
coeff[j] = p * obs[j].inv_corr * obs[j].Iref;
}
for (size_t j = 0; j < obs.size(); ++j)
coeff[j] = model.P(R, crystal, obs[j]) * obs[j].inv_corr * obs[j].Iref;
G = SolveScaleIRLS(obs, coeff, settings_.robust_k);
if (!(G > 0.0) || !std::isfinite(G))
return 0.0;
return {};
for (size_t j = 0; j < obs.size(); ++j)
obs[j].weight = 1.0 / std::sqrt(ModelVariance(obs[j], G * coeff[j]));
if (!refine_tilt)
if (!refine_crystal)
break;
// (2) Refine the tilt holding G fixed. A cost that cannot be evaluated keeps the tilt this crystal
// came in with and stops refining it - G alone is still a usable model.
const double inv_sigma_prior = settings_.prior_sigma_deg > 0.0 ? kRadToDeg / settings_.prior_sigma_deg : 0.0;
if (!RefineTilt(obs, model, G, settings_.robust_k, inv_sigma_prior, settings_.max_tilt_rad, psi))
// (2) Refine the crystal model holding G fixed. A cost that cannot be evaluated keeps the model this
// crystal came in with and stops refining it - G alone is still a usable model.
if (!RefineCrystal(obs, model, G, settings_.robust_k, inv_sigma_prior, bound, crystal, variance))
break;
}
// Write the refined partiality + scale correction onto every reflection of the crystal (not only the
// fit subset), so the merge sees a consistent model. image_scale_corr = prescaling_corr * qe_corr * flight_corr /
// (partiality * G), the same composition ScaleOnTheFly writes.
const auto R = TiltMatrix(crystal.psi_x, crystal.psi_y);
for (auto &r: outcome.reflections) {
const Coord q = base_q(r);
const double p = ComputeP(q.x, q.y, q.z, psi[0], psi[1], S0.x, S0.y, S0.z, inv_lambda, width);
const double p = model.At(R, crystal, q.x, q.y, q.z);
r.partiality = static_cast<float>(p);
const float corr = r.prescaling_corr * r.qe_corr * r.flight_corr;
const double denom = p * G;
@@ -428,8 +464,8 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome,
outcome.image_scale_cc = cc;
outcome.image_scale_cc_n = cc_n;
const double tilt_deg = std::sqrt(psi[0] * psi[0] + psi[1] * psi[1]) * kRadToDeg;
return tilt_deg;
return {.refined = refine_crystal, .tilt_deg = std::hypot(crystal.psi_x, crystal.psi_y) * kRadToDeg,
.eta = crystal.eta, .ln_width = crystal.ln_width, .var_eta = variance[2], .var_ln_width = variance[3]};
}
double StillsPartialityRefine::MeasureRadialScale(const std::vector<IntegrationOutcome> &outcomes,
@@ -490,6 +526,67 @@ double StillsPartialityRefine::MeasureRadialScale(const std::vector<IntegrationO
return radial_scale * (1.0 - sxy / sxx);
}
double StillsPartialityRefine::MeasureRadialSpread(const std::vector<IntegrationOutcome> &outcomes,
const RunWidth &width) const {
// Strong spots only: their centroids are good to a fraction of a pixel.
constexpr double MIN_I_OVER_SIGMA = 10.0;
constexpr size_t MIN_SPOTS = 200;
constexpr float STEP = 1e-3f; // of the wavenumber, for the direction a spot moves in
// Per spot: m = how far (px, on the plane) the model moves it per unit s^2, and d = observed - predicted.
struct Spot { double mx, my, dx, dy; };
std::vector<Spot> spots;
for (const auto &o: outcomes) {
if (!o.image_scale_g || !(*o.image_scale_g > 0.0))
continue;
const Coord S0 = o.geom.GetScatteringVector();
for (const auto &r: o.reflections) {
if (r.on_ice_ring || !AcceptReflection(r, d_min_limit_, d_max_limit_) || !(r.d > 0.0f)
|| !(r.I > MIN_I_OVER_SIGMA * r.sigma) || !std::isfinite(r.observed_x) || !std::isfinite(r.observed_y)
|| !std::isfinite(r.dist_ewald))
continue;
const Coord q = o.latt.Astar() * static_cast<float>(r.h) + o.latt.Bstar() * static_cast<float>(r.k)
+ o.latt.Cstar() * static_cast<float>(r.l);
const double dstar = 1.0 / r.d, qz = std::fabs(q.z);
const double a = width.gamma_e * dstar, b = width.bandwidth_sigma * qz;
const double sigma2 = width.gamma0 * width.gamma0 + a * a + b * b;
if (!(sigma2 > 0.0))
continue;
const auto p0 = o.geom.RecipToDetector(q);
const auto p1 = o.geom.RecipToDetector(q + S0 * STEP);
const double f = qz * r.dist_ewald / sigma2 / STEP;
Spot s{f * (p1.first - p0.first), f * (p1.second - p0.second),
r.observed_x - r.predicted_x, r.observed_y - r.predicted_y};
if (std::isfinite(s.mx) && std::isfinite(s.my))
spots.push_back(s);
}
}
if (spots.size() < MIN_SPOTS)
return 0.0;
// Least squares, reweighted against the spots the model does not describe (overlaps, split spots):
// Cauchy weights at three times the typical residual.
double s2 = 0.0;
std::vector<double> r2(spots.size());
for (int iter = 0; iter < 5; ++iter) {
for (size_t i = 0; i < spots.size(); ++i) {
const double ex = spots[i].dx - s2 * spots[i].mx, ey = spots[i].dy - s2 * spots[i].my;
r2[i] = ex * ex + ey * ey;
}
std::vector<double> sorted = r2;
std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end());
const double scale2 = 9.0 * sorted[sorted.size() / 2] / 1.386; // median r^2 = 1.386 sigma^2 in 2D
double num = 0.0, den = 0.0;
for (size_t i = 0; i < spots.size(); ++i) {
const double w = 1.0 / (1.0 + r2[i] / scale2);
num += w * (spots[i].mx * spots[i].dx + spots[i].my * spots[i].dy);
den += w * (spots[i].mx * spots[i].mx + spots[i].my * spots[i].my);
}
if (!(den > 0.0))
return 0.0;
s2 = num / den;
}
return std::max(0.0, s2);
}
std::optional<StillsPartialityRefine::RunWidth> StillsPartialityRefine::MeasureRunWidth(
const std::vector<IntegrationOutcome> &outcomes, const std::optional<UnitCell> &reference_cell,
double radial_scale, const RunWidth *window_width) const {
@@ -718,6 +815,136 @@ std::optional<StillsPartialityRefine::RunWidth> StillsPartialityRefine::MeasureR
return width;
}
StillsPartialityRefine::RunWidth StillsPartialityRefine::BalanceRunWidth(
const std::vector<IntegrationOutcome> &outcomes, const std::map<HKLKey, double> &reference,
const std::optional<UnitCell> &reference_cell, const RunWidth &width, size_t nthreads) const {
// The width table is measured from moments of the excitation error through the prediction window, and
// the tails of the curve are where that is least sure: on rotation sweeps processed as stills, the
// reflections recorded at under a third of their peak came out 30% fainter than the model at low
// resolution and up to 50% brighter at high resolution, against those recorded near their peak. So
// each shell of the table is scaled by the factor s at which its reflections predicted below half
// their peak and those above it put the same scale on the merged intensities:
// sum I / sum p_s I_ref is the same for both, p_s the Gaussian of width s sigma.
if (width.table_dstar.empty())
return width;
const auto &indexing = experiment_.GetIndexingSettings();
struct Point { double u, x, ref; }; // eps / sigma, I / G (corrected), the reference intensity
struct ShellPoint { size_t shell; Point p; };
const Width w{.gamma0 = width.gamma0, .gamma_e = width.gamma_e, .bw = width.bandwidth_sigma,
.table_dstar = &width.table_dstar, .table_s = &width.table_s};
// Each frame's points on the workers, then dealt into the shells in frame order, so every sum below runs
// in the order of the serial loop.
std::vector<std::vector<ShellPoint>> per_frame(outcomes.size());
ParallelFor(static_cast<int>(outcomes.size()), nthreads, [&](int i) {
const auto &o = outcomes[i];
if (!o.image_scale_g || !(*o.image_scale_g > 0.0))
return;
if (reference_cell && !o.latt.GetUnitCell().is_close(*reference_cell, indexing.GetUnitCellDistTolerance(),
indexing.GetUnitCellAngleTolerance_deg()))
return;
const Coord S0 = o.geom.GetScatteringVector();
const double inv_lambda = 1.0 / o.geom.GetWavelength_A();
for (const auto &r: o.reflections) {
if (r.on_ice_ring || !AcceptReflection(r, d_min_limit_, d_max_limit_) || !std::isfinite(r.I))
continue;
const auto it = reference.find(hkl_key_generator_(r));
if (it == reference.end() || !(it->second > 0.0))
continue;
const Coord q = (o.latt.Astar() * static_cast<float>(r.h) + o.latt.Bstar() * static_cast<float>(r.k)
+ o.latt.Cstar() * static_cast<float>(r.l)) * static_cast<float>(width.radial_scale);
const double dstar = VecLen(q.x, q.y, q.z);
const double de = VecLen(q.x + S0.x, q.y + S0.y, q.z + S0.z) - inv_lambda;
const double x = r.I * r.prescaling_corr * r.qe_corr * r.flight_corr / *o.image_scale_g;
if (!std::isfinite(x))
continue;
// the table shell nearest in d*
size_t k = std::upper_bound(width.table_dstar.begin(), width.table_dstar.end(), dstar)
- width.table_dstar.begin();
if (k == width.table_dstar.size() || (k > 0 && dstar - width.table_dstar[k - 1] < width.table_dstar[k] - dstar))
k = k > 0 ? k - 1 : 0;
per_frame[i].push_back({k, {de / std::sqrt(w.Sigma2(dstar, q.z)), x, it->second}});
}
});
std::vector<std::vector<Point>> shells(width.table_dstar.size());
for (auto &f: per_frame) {
for (const auto &sp: f)
shells[sp.shell].push_back(sp.p);
f = {};
}
// log(sum I / sum p_s I_ref) over the reflections below half the table's peak, less the same over those
// above it; falls as s grows.
auto gap = [](const std::vector<Point> &pts, double s) {
const double half = std::sqrt(2.0 * std::log(2.0)); // |u| at half the peak
double x_lo = 0.0, m_lo = 0.0, x_hi = 0.0, m_hi = 0.0;
for (const auto &p: pts) {
const double ps = std::exp(-0.5 * p.u * p.u / (s * s));
(std::fabs(p.u) > half ? x_lo : x_hi) += p.x;
(std::fabs(p.u) > half ? m_lo : m_hi) += ps * p.ref;
}
if (!(x_lo > 0.0 && m_lo > 0.0 && x_hi > 0.0 && m_hi > 0.0))
return std::numeric_limits<double>::quiet_NaN();
return std::log(x_lo / m_lo) - std::log(x_hi / m_hi);
};
// Each shell on its own worker.
std::vector<double> factor(shells.size(), NAN);
ParallelFor(static_cast<int>(shells.size()), nthreads, [&](int k) {
double lo = 1.0 / BALANCE_RANGE, hi = BALANCE_RANGE;
const double g_lo = gap(shells[k], lo), g_hi = gap(shells[k], hi);
if (shells[k].size() < 400 || !(g_lo > 0.0) || !(g_hi < 0.0))
return;
for (int i = 0; i < 20; ++i) { // the factor to 2e-6
const double mid = std::sqrt(lo * hi);
(gap(shells[k], mid) > 0.0 ? lo : hi) = mid;
}
factor[k] = std::sqrt(lo * hi);
});
RunWidth balanced = width;
std::string table;
for (size_t k = 0; k < shells.size(); ++k) {
if (std::isnan(factor[k]))
continue;
balanced.table_s[k] *= factor[k];
table += fmt::format(" {:.2f} A x{:.2f}", 1.0 / width.table_dstar[k], factor[k]);
}
Logger("StillsPartialityRefine").Info("Stills rocking width balanced on the partials, against resolution:{}",
table.empty() ? " unchanged" : table);
return balanced;
}
StillsPartialityRefine::CrystalPrior StillsPartialityRefine::RunPrior(const std::vector<CrystalFit> &fits) const {
// The spread of a parameter over the crystals: the variance of the estimates less the median variance
// they carry from their own data, floored at a small value (a crystal is never held exactly).
auto spread = [&](auto value, auto variance, double floor) {
std::vector<double> x, v;
for (const auto &f: fits)
if (f.refined && std::isfinite(variance(f)) && variance(f) > 0.0) {
x.push_back(value(f));
v.push_back(variance(f));
}
if (x.size() < 10)
return floor;
double mean = 0.0;
for (double e: x)
mean += e;
mean /= static_cast<double>(x.size());
double var = 0.0;
for (double e: x)
var += (e - mean) * (e - mean);
var /= static_cast<double>(x.size() - 1);
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
return std::max(floor, std::sqrt(std::max(0.0, var - v[v.size() / 2])));
};
const CrystalPrior prior{
.sigma_eta = spread([](const CrystalFit &f) { return f.eta; },
[](const CrystalFit &f) { return f.var_eta; }, MIN_PRIOR_ETA),
.sigma_ln_width = spread([](const CrystalFit &f) { return f.ln_width; },
[](const CrystalFit &f) { return f.var_ln_width; }, MIN_PRIOR_LN_WIDTH)};
Logger("StillsPartialityRefine").Info("Stills crystals differ in radial scale by {:.1e} and in rocking width "
"by {:.0f}% (sd over the crystals, net of their own uncertainty)",
prior.sigma_eta, 100.0 * prior.sigma_ln_width);
return prior;
}
double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, size_t nthreads,
const std::optional<UnitCell> &reference_cell,
const std::optional<RunWidth> &given_width,
@@ -732,7 +959,6 @@ double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, si
nthreads = AvailableCpus();
nthreads = std::max<size_t>(1, nthreads);
double last_mean_tilt = 0.0;
// Where the frames were predicted through a curve measured before the image pass, the partialities use
// that same curve. Measured again here, through windows that follow it, the width reads narrower at low
// resolution (its constant term falls to about zero), and the merge it gave moved R-free against the
@@ -747,29 +973,29 @@ double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, si
run_width = MeasureRunWidth(outcomes, reference_cell, radial_scale);
}
const double radial_scale = run_width ? run_width->radial_scale : 1.0;
if (used_width)
*used_width = run_width;
// The priors on a crystal's radial scale and width are the run's own spread of them: the first cycle
// refines every crystal against loose priors, and the spread of those estimates less their own
// uncertainty (method of moments) is how much crystals really differ. On an XFEL each shot has its
// photon energy and the spread is large; on a monochromatic or a stable pink beam it comes out small,
// and the crystals are held to the run's curve.
CrystalPrior prior{.sigma_eta = settings_.max_eta, .sigma_ln_width = settings_.max_ln_width};
std::vector<CrystalFit> fits(outcomes.size());
for (int outer = 0; outer < settings_.outer_iterations; ++outer) {
// Reference full intensities from the current corrections.
const std::vector<MergedReflection> merged = MergeAll(experiment_, outcomes, nthreads, reference_cell);
std::map<HKLKey, double> reference;
for (const auto &m: merged)
reference[hkl_key_generator_(m)] = m.I;
if (outer == 0 && run_width)
run_width = BalanceRunWidth(outcomes, reference, reference_cell, *run_width, nthreads);
std::atomic<double> tilt_sum{0.0};
std::atomic<size_t> tilt_n{0};
std::atomic<size_t> next{0};
auto worker = [&]() {
size_t i = next.fetch_add(1);
while (i < outcomes.size()) {
const double t = merged_crystal(outcomes[i]) ? RefineOne(outcomes[i], reference, run_width, radial_scale) : 0.0;
if (t > 0.0) {
double prev = tilt_sum.load();
while (!tilt_sum.compare_exchange_weak(prev, prev + t)) {}
tilt_n.fetch_add(1);
}
fits[i] = merged_crystal(outcomes[i]) ? RefineOne(outcomes[i], reference, run_width, radial_scale, prior)
: CrystalFit{};
i = next.fetch_add(1);
}
};
@@ -786,8 +1012,20 @@ double StillsPartialityRefine::Run(std::vector<IntegrationOutcome> &outcomes, si
f.get();
}
last_mean_tilt = tilt_n > 0 ? tilt_sum.load() / static_cast<double>(tilt_n.load()) : 0.0;
if (outer == 0)
prior = RunPrior(fits);
}
if (used_width)
*used_width = run_width;
double tilt_sum = 0.0;
size_t tilt_n = 0;
for (const auto &f: fits)
if (f.refined) {
tilt_sum += f.tilt_deg;
++tilt_n;
}
const double last_mean_tilt = tilt_n > 0 ? tilt_sum / static_cast<double>(tilt_n) : 0.0;
return last_mean_tilt;
}
@@ -23,10 +23,18 @@
// reshapes the SPATIAL pattern of partialities in a way a single G cannot mimic. That breaks the
// degeneracy that killed the scalar-sigma fit.
//
// Two more parameters per crystal, each held by a prior about the run: a radial scale eta of its
// reciprocal lattice (q -> (1 + eta) q, which moves the excitation error by about eta lambda d*^2 / 2 -
// on an XFEL the photon energy of the crystal's own shot, which spot positions cannot pin) and a scale of
// the run's rocking width (the crystal's own mosaicity and size). Both change p differently across
// resolution, so neither is degenerate with G. Their priors are not constants: the first cycle refines
// against loose ones, and the run's spread of the estimates, net of their own uncertainty, becomes the
// prior of the next (RunPrior) - large on SASE XFEL data, small on a monochromatic or stable pink beam.
//
// Because the integrated intensity I_obs is fixed, refining the tilt only recomputes the partiality
// analytically from the stored per-image lattice/geometry (q = A*.h + B*.k + C*.l, then
// dist_ewald = | |q + S0| - 1/lambda |) - NO pixel re-integration. The loop is: merge -> per-crystal
// refine tilt (G profiled out by the same robust IRLS ScaleOnTheFly uses) -> recompute p and
// dist_ewald = |q + S0| - 1/lambda, signed) - NO pixel re-integration. The loop is: merge -> per-crystal
// refine tilt, eta and width (G profiled out by the same robust IRLS ScaleOnTheFly uses) -> recompute p and
// image_scale_corr -> re-merge, iterated a few times. Mutates each reflection's `partiality` and
// `image_scale_corr` in place; the existing MergeOnTheFly then consumes the improved corrections. The
// width of the Gaussian partiality is the run's, measured once from the intensity-weighted excitation
@@ -38,8 +46,10 @@ public:
int inner_iterations = 3; // (solve G) <-> (refine tilt) alternations per crystal
size_t min_reflections = 40; // skip tilt refinement below this (anti-overfit on sparse crystals)
double max_tilt_rad = 0.0175; // hard bound on |dpsi| (~1 deg); indexing already refined orientation
double prior_sigma_deg = 0.02; // soft prior pulling dpsi toward 0 (0 = off); tames weak-data overfit,
double prior_sigma_deg = 0.02; // soft prior pulling dpsi toward 0; tames weak-data overfit,
// inert on strong data (well-supported tilts overcome it)
double max_eta = 5e-3; // bound on a crystal's radial scale |eta|, and its prior in the first cycle
double max_ln_width = 1.1; // the same for the log of its width scale (a factor of 3)
double robust_k = 3.0; // Cauchy loss scale (sigma units)
double gamma_e = 0.0; // angular width sigma(d*) = gamma_e*d* (0 = measure from data)
};
@@ -75,6 +85,16 @@ public:
double MeasureRadialScale(const std::vector<IntegrationOutcome> &outcomes,
const std::optional<UnitCell> &reference_cell, double radial_scale) const;
// How far the run's spots move radially with their excitation error, as the relative spread s of the
// radial scale of the reciprocal lattice (a photon-energy spread, or a cell spread) that would move them
// so. A node off the Ewald sphere is recorded where the part of its smeared node that IS on the sphere
// diffracts: along the radial share of its rocking width, that is where the photon energy that puts the
// node on the sphere sends it. With the rocking width sigma (width's three-term formula) made of a radial
// part s |q_z| and the rest, the expected move is that of a change of the beam's wavenumber by
// dk / k = s^2 |q_z| eps / sigma^2 - fitted by least squares to where the strong spots were observed
// against where they were predicted. Returns s^2; 0 when the run has too few strong spots.
double MeasureRadialSpread(const std::vector<IntegrationOutcome> &outcomes, const RunWidth &width) const;
private:
const DiffractionExperiment experiment_;
const Settings settings_{};
@@ -84,8 +104,25 @@ private:
const double min_partiality_;
const float bandwidth_sigma_;
// Refine one crystal against the reference map; returns |dpsi| in degrees (0 if skipped).
double RefineOne(IntegrationOutcome &outcome,
const std::map<HKLKey, double> &reference,
const std::optional<RunWidth> &run_width, double radial_scale) const;
// The priors of a crystal's radial scale eta and log width scale (Gaussian, about 0).
struct CrystalPrior {
double sigma_eta, sigma_ln_width;
};
// One crystal's refined model: |dpsi| in degrees, eta, ln width and the variances of the last two from
// its own data (no prior). refined = false where only G (or nothing) was fitted.
struct CrystalFit {
bool refined = false;
double tilt_deg = 0.0, eta = 0.0, ln_width = 0.0, var_eta = NAN, var_ln_width = NAN;
};
// Refine one crystal against the reference map.
CrystalFit RefineOne(IntegrationOutcome &outcome, const std::map<HKLKey, double> &reference,
const std::optional<RunWidth> &run_width, double radial_scale,
const CrystalPrior &prior) const;
// The run's width table, each shell rescaled so that its weak and strong partials agree on the scale.
RunWidth BalanceRunWidth(const std::vector<IntegrationOutcome> &outcomes, const std::map<HKLKey, double> &reference,
const std::optional<UnitCell> &reference_cell, const RunWidth &width,
size_t nthreads) const;
// The run's spread of eta and ln width over the crystals, net of their own uncertainty.
CrystalPrior RunPrior(const std::vector<CrystalFit> &fits) const;
};
+113 -76
View File
@@ -1,6 +1,8 @@
// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstring>
#include <set>
@@ -329,18 +331,24 @@ bool ReadReflectionsFromGroup(HDF5Object &file,
auto bkg = file.ReadOptVector<float>(image_group_name + "/background_mean");
// Written since the merge stopped back-deriving it; older _process.h5 do not carry it.
auto var_bkg = file.ReadOptVector<float>(image_group_name + "/background_variance");
auto lp = file.ReadOptVector<float>(image_group_name + "/lp");
auto lp = file.ReadOptVector<double>(image_group_name + "/lp");
// The sensor-efficiency part of lp, written since the correction existed. Absent in every
// older file, and there the whole of lp is taken for Lorentz-polarization - which is what
// those files mean, since nothing had corrected for the sensor when they were written.
auto qe = file.ReadOptVector<float>(image_group_name + "/qe");
auto qe = file.ReadOptVector<double>(image_group_name + "/qe");
// The flight path, in the same reciprocal convention. Absent in every older file, and there
// it is taken as 1: nothing had corrected for the flight path when those files were written.
auto flight = file.ReadOptVector<float>(image_group_name + "/flight");
auto flight = file.ReadOptVector<double>(image_group_name + "/flight");
auto partiality = file.ReadOptVector<float>(image_group_name + "/partiality");
auto phi = file.ReadOptVector<float>(image_group_name + "/delta_phi");
auto zeta = file.ReadOptVector<float>(image_group_name + "/zeta");
auto image_scale_corr = file.ReadOptVector<float>(image_group_name + "/image_scale_corr");
// Written since --mode scale started from the state the merge of --mode mx starts from; absent in
// older files, where they read as zero and false.
auto dist_ewald = file.ReadOptVector<float>(image_group_name + "/excitation_error");
auto observed = file.ReadOptVector<uint8_t>(image_group_name + "/observed");
auto clipped = file.ReadOptVector<uint8_t>(image_group_name + "/clipped");
auto overloaded = file.ReadOptVector<uint8_t>(image_group_name + "/overloaded");
if (h.size() != l.size() || h.size() != k.size() || h.size() != d.size()
|| h.size() != predicted_x.size() || h.size() != predicted_y.size()
@@ -359,16 +367,16 @@ bool ReadReflectionsFromGroup(HDF5Object &file,
}
float lp_val = 0.0;
if (lp.size() > i && lp[i] != 0.0f)
lp_val = 1.0f / lp[i];
if (lp.size() > i && lp[i] != 0.0)
lp_val = static_cast<float>(1.0 / lp[i]);
float qe_val = 1.0f;
if (qe.size() > i && qe[i] != 0.0f)
qe_val = 1.0f / qe[i];
if (qe.size() > i && qe[i] != 0.0)
qe_val = static_cast<float>(1.0 / qe[i]);
float flight_val = 1.0f;
if (flight.size() > i && flight[i] != 0.0f)
flight_val = 1.0f / flight[i];
if (flight.size() > i && flight[i] != 0.0)
flight_val = static_cast<float>(1.0 / flight[i]);
float partiality_val = -1.0f;
if (partiality.size() > i && partiality[i] >= 0.0f)
@@ -416,12 +424,16 @@ bool ReadReflectionsFromGroup(HDF5Object &file,
.bkg = bkg.at(i),
.var_bkg = var_bkg_val,
.sigma = int_err.at(i),
.dist_ewald = dist_ewald.size() > i ? dist_ewald[i] : 0.0f,
.prescaling_corr = lp_val,
.qe_corr = qe_val,
.flight_corr = flight_val,
.partiality = partiality_val,
.zeta = zeta_val,
.image_scale_corr = image_scale_corr_val
.image_scale_corr = image_scale_corr_val,
.observed = observed.size() > i && observed[i],
.clipped = clipped.size() > i && clipped[i],
.overloaded = overloaded.size() > i && overloaded[i]
};
reflections.emplace_back(r);
}
@@ -536,6 +548,75 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
// Of a multichannel master; only its first channel is read
std::string first_channel;
// The per-image analysis and metadata a master holds itself (VDS, integrated, or a _process.h5 that
// holds no images at all).
auto read_master_per_image = [&] {
if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"))
dataset->efficiency = master_file->ReadVector<float>(
"/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image");
else
dataset->efficiency = std::vector<float>(number_of_images, 1.0);
if (master_file->Exists("/entry/roi"))
dataset->roi = master_file->FindLeafs("/entry/roi");
for (const auto &s: dataset->roi) {
dataset->roi_max.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/max"));
dataset->roi_sum.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum"));
dataset->roi_sum_sq.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum_sq"));
dataset->roi_npixel.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/npixel"));
dataset->roi_x.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/x"));
dataset->roi_y.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/y"));
}
if (master_file->Exists("/entry/MX")) {
if (master_file->Exists("/entry/MX/peakCountUnfiltered"))
dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/peakCountUnfiltered");
else
dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/nPeaks");
dataset->spot_count_low_res = master_file->ReadOptVector<float>("/entry/MX/peakCountLowRes");
dataset->spot_count_indexed = master_file->ReadOptVector<float>("/entry/MX/peakCountIndexed");
dataset->spot_count_ice_rings = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingRes");
dataset->spot_count_ice_control = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingControl");
dataset->indexing_result = master_file->ReadOptVector<float>("/entry/MX/imageIndexed");
dataset->bkg_estimate = master_file->ReadOptVector<float>("/entry/MX/bkgEstimate");
dataset->spindle_blind_fraction = master_file->ReadOptVector<float>("/entry/MX/spindleBlindFraction");
dataset->ice_ring_score = master_file->ReadOptVector<float>("/entry/MX/iceRingScore");
dataset->resolution_estimate = master_file->ReadOptVector<float>("/entry/MX/resolutionEstimate");
dataset->profile_radius = master_file->ReadOptVector<float>("/entry/MX/profileRadius");
// Master files write indexedLatticeCount; data files / the per-file MX
// plugin use indexingLatticeCount. Accept either for backward compatibility.
dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexedLatticeCount");
if (dataset->indexing_lattice_count.empty())
dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexingLatticeCount");
dataset->mosaicity_deg = master_file->ReadOptVector<float>("/entry/MX/mosaicity");
dataset->b_factor = master_file->ReadOptVector<float>("/entry/MX/bFactor");
dataset->image_scale_factor = master_file->ReadOptVector<float>("/entry/MX/imageScaleFactor");
dataset->image_scale_cc = master_file->ReadOptVector<float>("/entry/MX/imageScaleCC");
dataset->integrated_reflections = master_file->ReadOptVector<float>("/entry/MX/integratedReflections");
dataset->sweep_quality = master_file->ReadOptVector<uint8_t>("/entry/MX/sweepQuality");
if (master_file->Exists("/entry/MX/sweepQualityReasons")) {
const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons");
for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
dataset->sweep_quality_reasons.push_back(
master_file->ReadElement<std::string>("/entry/MX/sweepQualityReasons", i)
.value_or(""));
}
dataset->frame_disposition = master_file->ReadOptVector<uint8_t>("/entry/MX/frameDisposition");
if (master_file->Exists("/entry/MX/frameDispositionCodes")) {
const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes");
for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
dataset->frame_disposition_codes.push_back(
master_file->ReadElement<std::string>("/entry/MX/frameDispositionCodes", i)
.value_or(""));
}
}
if (master_file->Exists("/entry/image"))
dataset->max_value = master_file->ReadOptVector<int64_t>("/entry/image/max_value");
};
if (master_file->Exists("/entry/data/data")) {
HDF5DataSet data_dataset(*master_file, "/entry/data/data");
HDF5Dcpl dcpl(data_dataset);
@@ -552,70 +633,7 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
if (data_layout == HDF5DataSetLayout::VIRTUAL)
vds_data_mappings = ReadVDSImageMappings(*master_file, "/entry/data/data");
if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"))
dataset->efficiency = master_file->ReadVector<float>(
"/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image");
else
dataset->efficiency = std::vector<float>(number_of_images, 1.0);
if (master_file->Exists("/entry/roi"))
dataset->roi = master_file->FindLeafs("/entry/roi");
for (const auto &s: dataset->roi) {
dataset->roi_max.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/max"));
dataset->roi_sum.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum"));
dataset->roi_sum_sq.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/sum_sq"));
dataset->roi_npixel.emplace_back(master_file->ReadVector<int64_t>("/entry/roi/" + s + "/npixel"));
dataset->roi_x.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/x"));
dataset->roi_y.emplace_back(master_file->ReadVector<float>("/entry/roi/" + s + "/y"));
}
if (master_file->Exists("/entry/MX")) {
if (master_file->Exists("/entry/MX/peakCountUnfiltered"))
dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/peakCountUnfiltered");
else
dataset->spot_count = master_file->ReadOptVector<float>("/entry/MX/nPeaks");
dataset->spot_count_low_res = master_file->ReadOptVector<float>("/entry/MX/peakCountLowRes");
dataset->spot_count_indexed = master_file->ReadOptVector<float>("/entry/MX/peakCountIndexed");
dataset->spot_count_ice_rings = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingRes");
dataset->spot_count_ice_control = master_file->ReadOptVector<float>("/entry/MX/peakCountIceRingControl");
dataset->indexing_result = master_file->ReadOptVector<float>("/entry/MX/imageIndexed");
dataset->bkg_estimate = master_file->ReadOptVector<float>("/entry/MX/bkgEstimate");
dataset->spindle_blind_fraction = master_file->ReadOptVector<float>("/entry/MX/spindleBlindFraction");
dataset->ice_ring_score = master_file->ReadOptVector<float>("/entry/MX/iceRingScore");
dataset->resolution_estimate = master_file->ReadOptVector<float>("/entry/MX/resolutionEstimate");
dataset->profile_radius = master_file->ReadOptVector<float>("/entry/MX/profileRadius");
// Master files write indexedLatticeCount; data files / the per-file MX
// plugin use indexingLatticeCount. Accept either for backward compatibility.
dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexedLatticeCount");
if (dataset->indexing_lattice_count.empty())
dataset->indexing_lattice_count = master_file->ReadOptVector<float>("/entry/MX/indexingLatticeCount");
dataset->mosaicity_deg = master_file->ReadOptVector<float>("/entry/MX/mosaicity");
dataset->b_factor = master_file->ReadOptVector<float>("/entry/MX/bFactor");
dataset->image_scale_factor = master_file->ReadOptVector<float>("/entry/MX/imageScaleFactor");
dataset->image_scale_cc = master_file->ReadOptVector<float>("/entry/MX/imageScaleCC");
dataset->integrated_reflections = master_file->ReadOptVector<float>("/entry/MX/integratedReflections");
dataset->sweep_quality = master_file->ReadOptVector<uint8_t>("/entry/MX/sweepQuality");
if (master_file->Exists("/entry/MX/sweepQualityReasons")) {
const auto dim = master_file->GetDimension("/entry/MX/sweepQualityReasons");
for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
dataset->sweep_quality_reasons.push_back(
master_file->ReadElement<std::string>("/entry/MX/sweepQualityReasons", i)
.value_or(""));
}
dataset->frame_disposition = master_file->ReadOptVector<uint8_t>("/entry/MX/frameDisposition");
if (master_file->Exists("/entry/MX/frameDispositionCodes")) {
const auto dim = master_file->GetDimension("/entry/MX/frameDispositionCodes");
for (size_t i = 0; i < (dim.empty() ? 0 : dim[0]); i++)
dataset->frame_disposition_codes.push_back(
master_file->ReadElement<std::string>("/entry/MX/frameDispositionCodes", i)
.value_or(""));
}
}
if (master_file->Exists("/entry/image"))
dataset->max_value = master_file->ReadOptVector<int64_t>("/entry/image/max_value");
read_master_per_image();
format = FileWriterFormat::NXmxVDS;
} else if (master_file->Exists("/entry/data/data_000001")) {
@@ -792,6 +810,13 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector");
image_size_y = master_file->GetInt("/entry/instrument/detector/detectorSpecific/y_pixels_in_detector");
number_of_images = 0;
// A _process.h5 of an input it could not link to holds the per-image analysis without the
// images: as many images as it has per-image entries, none of them with pixels.
if (master_file->Exists("/entry/detector/number")) {
number_of_images = master_file->GetDimension("/entry/detector/number")[0];
images_per_file = number_of_images;
read_master_per_image();
}
}
if (master_file->Exists("/entry/MX")) {
@@ -971,6 +996,8 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
}
dataset->stills_run_width = w;
}
if (const auto c = master_file->ReadOptVector<float>("/entry/MX/stillsGivenUnitCell"); c.size() == 6)
dataset->stills_given_cell = UnitCell{.a = c[0], .b = c[1], .c = c[2], .alpha = c[3], .beta = c[4], .gamma = c[5]};
dataset->experiment.SampleName(master_file->GetString("/entry/sample/name"));
@@ -1148,6 +1175,14 @@ HDF5MetadataSource::OpenResult HDF5MetadataSource::Open(const std::string &filen
}
if (mask_tmp.empty())
mask_tmp = std::vector<uint32_t>(image_size_x * image_size_y);
// A JUNGFRAU file that does not state its modules one by one still marks the gaps between
// them in its mask: each region clear of the gap bit is one module, placed as imaged. Only for
// JUNGFRAU - a DECTRIS mask uses the same bit for its gaps, and its modules are not read here.
std::string detector_upper = detector_name;
std::transform(detector_upper.begin(), detector_upper.end(), detector_upper.begin(),
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
if (dataset->experiment.GetModules().empty() && detector_upper.find("JUNGFRAU") != std::string::npos)
dataset->experiment.Modules(ModulesFromGapMask(mask_tmp, image_size_x, image_size_y));
dataset->pixel_mask = std::make_shared<const PixelMask>(mask_tmp);
}
@@ -1666,8 +1701,10 @@ std::vector<IntegrationOutcome> HDF5MetadataSource::ReadReflections(size_t start
// ── per-image mosaicity ───────────────────────────────────────────────
if (meta_file->Exists("/entry/MX/mosaicity")) {
try {
outcome.mosaicity_deg =
meta_file->ReadElement<float>("/entry/MX/mosaicity", meta_image_id);
// NaN is how the file says the image had none.
const auto mos = meta_file->ReadElement<float>("/entry/MX/mosaicity", meta_image_id);
if (mos && std::isfinite(*mos))
outcome.mosaicity_deg = mos;
} catch (...) {
}
}
+4 -1
View File
@@ -15,6 +15,7 @@ void JFJochHDF5Reader::ReadFile(const std::string &filename) {
auto metadata = std::make_shared<HDF5MetadataSource>();
auto open_result = metadata->Open(filename, default_experiment);
has_pixels = open_result.image_layout.format != FileWriterFormat::NoFile;
image_source_.Configure(std::move(open_result.image_layout));
// Original file: per-image metadata is co-located with the pixels.
metadata->UseImageSourceForMetadata(&image_source_);
@@ -67,6 +68,8 @@ bool JFJochHDF5Reader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &
if (!active_metadata_)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Cannot load image if file not loaded");
if (!has_pixels)
return false;
auto loc = GetImageLocation(image_number);
chunk = image_source_.PrepareDirectRead(loc);
@@ -88,7 +91,7 @@ bool JFJochHDF5Reader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset
std::unique_lock ul(hdf5_mutex);
(void) update_dataset;
if (!dataset)
if (!dataset || !has_pixels)
return false;
if (!active_metadata_)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+3
View File
@@ -31,6 +31,9 @@ class JFJochHDF5Reader : public JFJochReader {
std::shared_ptr<HDF5MetadataSource> active_metadata_;
std::string active_snapshot_;
size_t number_of_images = 0;
// False for a file of per-image analysis without the images (a _process.h5 of an input it could
// not link to): its images have metadata, but no pixels to load.
bool has_pixels = false;
bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
DataMessage &message,
+3 -1
View File
@@ -45,8 +45,10 @@ struct JFJochReaderDataset {
// The e.s.d.s of the experiment's unit cell, a b c alpha beta gamma (/entry/sample/unit_cell_esd); empty
// where the file has none.
std::optional<UnitCell> unit_cell_esd;
// Stills: the rocking curve the run's partialities were computed from (/entry/MX/stillsRockingWidth*).
// Stills: the rocking curve the run's frames were predicted through (/entry/MX/stillsRockingWidth*).
std::optional<StillsRunWidth> stills_run_width;
// Stills: the cell the run was given, whose population of frames it merged (/entry/MX/stillsGivenUnitCell).
std::optional<UnitCell> stills_given_cell;
std::vector<float> az_int_bin_to_q;
std::vector<float> az_int_bin_to_phi;
+1 -1
View File
@@ -426,7 +426,7 @@ void JFJochReceiverRugnux::RugnuxThread() {
std::optional<double> d_min, isa;
if (result.has_merge_statistics)
SetIfFinite(d_min, result.merge_statistics.overall.d_min);
if (result.error_model_isa_resolved && result.error_model_isa > 0.0)
if (!result.merge_statistics.stills && result.error_model_isa_resolved && result.error_model_isa > 0.0)
isa = result.error_model_isa;
live_status.Headline(result.space_group ? result.space_group->xhm() : "", d_min, isa);
// The placed model and a copy of its map coefficients (the end message takes the originals), and the
+61 -36
View File
@@ -47,6 +47,9 @@ namespace {
// the XDS GEOMETRY KEYWORDS section; OVERLOADED_* / OVERLAPPED_* and the OVERLOADS / OVERLAPS flags.
// 17: INDEXING_AMBIGUITY_RESOLVED and the AMBIGUITY_* keys in section 2 - serial stills, the
// indexing ambiguity resolved from the data alone (or measured against the reference's answer).
// 18: stills report their error model as ERROR_MODEL_A / _B / _C (C the partiality term) and no
// ISA (nor ISa in the summary's Signal row); no R_MERGE / R_MEAS / R_PIM (nor REFRES_R_MEAS)
// and no R_merge / R_meas / R_pim columns in their shell tables.
// 8: every line that is not `KEY= value` and not blank starts with `#`, warnings included
// (`# WARNING:`); the header gains RUGNUX_DOWNLOAD and BUILD_CXX_FLAGS, and RUGNUX_GIT is
// the full build-time hash.
@@ -340,7 +343,8 @@ namespace {
// than as a row of zeros, so a reader cannot take it for a measured total failure.
std::string ReferenceShellTable(const MergeStatistics &rs, double own_d_min) {
std::ostringstream o;
o << "\n" << ShellTableHeader() << "\n" << ShellTableRule() << "\n";
const bool r_factors = !rs.stills;
o << "\n" << ShellTableHeader(r_factors) << "\n" << ShellTableRule(r_factors) << "\n";
for (const auto &sh : rs.shells) {
if (sh.unique_reflections == 0) {
if (sh.d_max <= own_d_min)
@@ -348,13 +352,13 @@ namespace {
sh.d_min, own_d_min, sh.possible_unique_reflections);
continue;
}
o << fmt::format(" {:8.2f} ", sh.d_min) << sh;
o << fmt::format(" {:8.2f} ", sh.d_min) << ShellTableRow(sh, r_factors);
if (sh.d_min < own_d_min)
o << fmt::format(" <- the run's own limit, {:.2f} A, is inside this shell", own_d_min);
o << "\n";
}
o << ShellTableRule() << "\n";
o << fmt::format(" {:>8s} ", "Overall") << rs.overall << "\n";
o << ShellTableRule(r_factors) << "\n";
o << fmt::format(" {:>8s} ", "Overall") << ShellTableRow(rs.overall, r_factors) << "\n";
o << SerialShellTable(rs);
return o.str();
}
@@ -1545,7 +1549,14 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
// The error model in XDS's convention, so the numbers are directly comparable with a
// CORRECT.LP. Kept in the default report because beamline monitoring scripts read them.
Add(s, KeyReal("ERROR_MODEL_A", result.error_model_a, "{:.4f}"));
Add(s, KeyReal("ERROR_MODEL_B", result.error_model_b, "{:.4e}"));
// On stills the terms are the whole report of the error model, so a b that is zero within
// its error says so here; on rotation ISA= says it.
if (ms.stills && !result.error_model_isa_resolved)
Add(s, KeyText("ERROR_MODEL_B", "undetermined"));
else
Add(s, KeyReal("ERROR_MODEL_B", result.error_model_b, "{:.4e}"));
if (ms.stills)
Add(s, KeyReal("ERROR_MODEL_C", result.error_model_c, "{:.4e}"));
// A b that is zero within its error gives a 1/b that is the noise in b, not an I/sigma.
Add(s, KeyText("SCALING_ITERATIONS",
fmt::format("{} partials, {} fulls{}", result.scaling_iterations_partials,
@@ -1560,21 +1571,26 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
? "the space-group search merge and the final merge"
: !result.scaling_converged ? "the final merge"
: "the space-group search merge"));
// A rotation merge's ISa is judged by its bootstrap interval (ISA_LOW / ISA_HIGH); where
// nothing was resampled (stills, b not measured, a on its clamp) by the bin fit's own test.
const bool isa_bootstrapped = std::isfinite(result.error_model_isa_low);
const bool isa_bounded = isa_bootstrapped && std::isfinite(result.error_model_isa_high);
const bool isa_reported = isa_bootstrapped ? isa_bounded : result.error_model_isa_resolved;
if (isa_reported)
Add(s, KeyReal("ISA", result.error_model_isa, "{:.2f}"));
else
Add(s, KeyText("ISA", "undetermined"));
if (isa_bootstrapped)
Add(s, KeyReal("ISA_LOW", result.error_model_isa_low, "{:.2f}"));
if (isa_bounded)
Add(s, KeyReal("ISA_HIGH", result.error_model_isa_high, "{:.2f}"));
if (isa_reported && result.error_model_isa_asymptotic > 0.0)
Add(s, KeyReal("ISA_ASYMPTOTIC", result.error_model_isa_asymptotic, "{:.2f}", true));
// No ISA on stills: there 1/b is the ceiling of one still observation under the stills
// model, a property of the processing more than of the instrument, and read on CORRECT.LP's
// scale it calls good serial data useless.
if (!ms.stills) {
// A rotation merge's ISa is judged by its bootstrap interval (ISA_LOW / ISA_HIGH); where
// nothing was resampled (stills, b not measured, a on its clamp) by the bin fit's own test.
const bool isa_bootstrapped = std::isfinite(result.error_model_isa_low);
const bool isa_bounded = isa_bootstrapped && std::isfinite(result.error_model_isa_high);
const bool isa_reported = isa_bootstrapped ? isa_bounded : result.error_model_isa_resolved;
if (isa_reported)
Add(s, KeyReal("ISA", result.error_model_isa, "{:.2f}"));
else
Add(s, KeyText("ISA", "undetermined"));
if (isa_bootstrapped)
Add(s, KeyReal("ISA_LOW", result.error_model_isa_low, "{:.2f}"));
if (isa_bounded)
Add(s, KeyReal("ISA_HIGH", result.error_model_isa_high, "{:.2f}"));
if (isa_reported && result.error_model_isa_asymptotic > 0.0)
Add(s, KeyReal("ISA_ASYMPTOTIC", result.error_model_isa_asymptotic, "{:.2f}", true));
}
// Only when TRUE: FALSE is the ordinary case and says nothing.
if (result.has_reference) {
Add(s, KeyBool("REFERENCE_DATA_USED", true));
@@ -1637,11 +1653,18 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
" both hands the quantity does not exist: the key is then absent here and the column is\n"
" a dash in the table below, which is not the same claim as a signal measured to be\n"
" zero.\n\n"
" ERROR_MODEL_A / ERROR_MODEL_B are in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2),\n"
" so ISA = 1/sqrt(a*b) means what CORRECT.LP's ISa means. ISA_LOW / ISA_HIGH are its 95%\n"
" interval from a bootstrap over the reflections; ISA= undetermined with ISA_LOW alone says\n"
" b is zero within that interval, so only ISa >= ISA_LOW is measured. ISA_ASYMPTOTIC, where\n"
" present, is the strong-reflection tier only.", true));
+ std::string(ms.stills
? " ERROR_MODEL_A / ERROR_MODEL_B / ERROR_MODEL_C are the stills error model in XDS's\n"
" convention, sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2), with p the partiality\n"
" of the observation: b is the systematic error of a fully recorded still, c the added\n"
" error of a partial one. No ISa is given: 1/sqrt(a*b) here is the I/sigma ceiling of\n"
" a single still observation under the stills model, not what CORRECT.LP's ISa\n"
" measures; R_SPLIT, CC_HALF and CC_STAR are the numbers to quote."
: " ERROR_MODEL_A / ERROR_MODEL_B are in XDS's convention, sigma^2 = a*(sigma0^2 + b*I^2),\n"
" so ISA = 1/sqrt(a*b) means what CORRECT.LP's ISa means. ISA_LOW / ISA_HIGH are its 95%\n"
" interval from a bootstrap over the reflections; ISA= undetermined with ISA_LOW alone says\n"
" b is zero within that interval, so only ISa >= ISA_LOW is measured. ISA_ASYMPTOTIC, where\n"
" present, is the strong-reflection tier only."), true));
{
std::ostringstream shells;
@@ -3370,17 +3393,19 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
sig += fmt::format(" R_meas {:.1f} %", 100.0 * o.r_meas);
if (std::isfinite(o.cc_half))
sig += fmt::format(" CC1/2 {:.3f}", o.cc_half);
const bool isa_bootstrapped = std::isfinite(result.error_model_isa_low);
const bool isa_bounded = isa_bootstrapped && std::isfinite(result.error_model_isa_high);
if (isa_bootstrapped && !isa_bounded)
sig += fmt::format(" ISa >= {:.1f}", result.error_model_isa_low);
else if (!(isa_bootstrapped || result.error_model_isa_resolved))
sig += " ISa undetermined";
else if (result.error_model_isa > 0.0) {
sig += fmt::format(" ISa {:.1f}", result.error_model_isa);
if (isa_bounded)
sig += fmt::format(" ({:.1f}-{:.1f})", result.error_model_isa_low,
result.error_model_isa_high);
if (!result.merge_statistics.stills) { // no ISa on stills (section 3)
const bool isa_bootstrapped = std::isfinite(result.error_model_isa_low);
const bool isa_bounded = isa_bootstrapped && std::isfinite(result.error_model_isa_high);
if (isa_bootstrapped && !isa_bounded)
sig += fmt::format(" ISa >= {:.1f}", result.error_model_isa_low);
else if (!(isa_bootstrapped || result.error_model_isa_resolved))
sig += " ISa undetermined";
else if (result.error_model_isa > 0.0) {
sig += fmt::format(" ISa {:.1f}", result.error_model_isa);
if (isa_bounded)
sig += fmt::format(" ({:.1f}-{:.1f})", result.error_model_isa_low,
result.error_model_isa_high);
}
}
row("Signal", sig);
if (std::isfinite(o.cc_anom))
+21 -27
View File
@@ -383,6 +383,12 @@ Rugnux::Rugnux(JFJochReader &reader, DiffractionExperiment experiment,
: fmt::format("{:.0f}% of the machine's {:.1f} GB", 100.0 * HOST_MEMORY_BUDGET_FRACTION,
static_cast<double>(machine) / 1e9));
// Modules where the image already has them (a JUNGFRAU file's own, or rebuilt from its mask) tell
// the stills module placement what may move; a rotation run has no such step and processes the
// image as one panel, as it always has.
if (experiment_.IsRotationIndexing() && !experiment_.GetModules().Moved())
experiment_.Modules({});
// Every decision on a rotation run - per-frame scaling, the correction surfaces, the outlier test,
// the error model, the CC1/2 half-sets and the resolution cut, the frame ledger, the space-group
// search - is taken with Friedel mates merged, -A or not. Splitting them halves the multiplicity
@@ -1003,6 +1009,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// tilt's pattern shift, so a tilt fitted before the centre is decided takes up the centre's error.
if (full)
DecideStillsTilt(refitted);
MaskOuterShadow();
// The geometry pre-pass integrates at the radius the run started with; the width the pre-scan just
// measured is applied to the canonical pass instead (RunAllPasses, between the two). See
@@ -1055,11 +1062,11 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
start_message.file_format = FileWriterFormat::NXmxIntegrated;
start_message.write_master_file = true;
start_message.write_images = false;
// The process file LINKS to the original images rather than writing its own, which only an HDF5
// input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs. So
// ask for the links only when the source is HDF5, and otherwise write the images into the file -
// the per-image analysis is the point of it either way, and without pictures nothing downstream
// could show the frame a reflection came from.
// The process file LINKS to the original images rather than writing its own, which only a single
// HDF5 input can offer: there is nothing for a virtual dataset to point at in a directory of CBFs,
// an XTC stream or several masters read as one. Such a process file holds no /entry/data/data at
// all - a copy of the images would make it as large as the data, and the per-image analysis is
// the point of it.
if (auto *hdf5 = dynamic_cast<JFJochHDF5Reader *>(&reader_)) {
// With the stride: image i of the process file is source image start_image + i * stride, so
// the linked pictures line up with the per-image analysis written beside them.
@@ -1083,8 +1090,6 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
? -(int64_t(1) << (stored.bit_depth - 1)) // INTx_MIN
: (int64_t(1) << stored.bit_depth) - 1; // UINTx_MAX
start_message.underload_value = stored.is_signed ? start_message.error_value.value() + 1 : 0;
} else {
start_message.write_images = true; // no file to link to; the process file carries them
}
// The full-detector mask is 4 B/px and the ROI map 2 B/px - tens of megabytes each - and nothing
@@ -1105,7 +1110,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
if (result.outputs && !sink)
scan_result.emplace(experiment_);
// Whether anything is in place to break the indexing ambiguity, for the warning further down: a
// reference MTZ, or (serial stills) the reference computed from a model just below.
// reference MTZ.
bool ambiguity_reference = !config_.reference_data.empty();
if (sink) {
fill_writer_maps();
@@ -1163,26 +1168,11 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// geometry pre-pass, which measures the spots against it (SpotFootprint.h).
if (!writer_queue && !(geometry_prepass && prescan_footprint_))
indexer->KeepReflectionsInMessage(false);
// The reference that breaks the per-image indexing ambiguity: a reference MTZ where there is
// one, otherwise intensities computed from the model, which carry the same information. Serial
// stills need it here or not at all - each crystal is indexed in its own hand, so by the time
// there is a merge for --model to fit, the hands have already been averaged together. Rotation
// data have one lattice and one hand, so their ambiguity is settled at the merge instead.
std::vector<MergedReflection> model_reference;
if (!config_.reference_data.empty()) {
// The reference that breaks the per-image indexing ambiguity, where there is a reference MTZ. Never
// intensities computed from --model: a model is placed and validated after the merge and does not
// change the processing, so a run with one merges exactly what a run without one does.
if (!config_.reference_data.empty())
indexer->ReferenceIntensities(config_.reference_data);
} else if (config_.model && !experiment_.IsRotationIndexing()
&& experiment_.GetUnitCell().has_value() && experiment_.GetGemmiSpaceGroup().has_value()) {
const auto d_min = experiment_.GetScalingSettings().GetHighResolutionLimit_A()
.value_or(experiment_.GetDetectorMaxResolution_A());
model_reference = ModelReferenceIntensities(
*config_.model, experiment_.GetUnitCell(),
&experiment_.GetSpaceGroupOrP1(), d_min, logger);
if (!model_reference.empty()) {
indexer->ReferenceIntensities(model_reference);
ambiguity_reference = true;
}
}
// Second pass of the rotation two-pass: predict with the smoothed mosaicity fitted in the pre-pass.
if (!geometry_prepass && !prepass_mosaicity_.empty())
indexer->SetPredictionMosaicityOverride(prepass_mosaicity_);
@@ -1346,6 +1336,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
SetUnmergedMtzMosaicity(mtz, indexer->GetIntegrationOutcome());
result.outputs->unmerged = std::move(mtz);
}
if (config_.export_observations && !observation_verdicts_.empty())
result.outputs->observations = ObservationMtzTable(indexer->GetIntegrationOutcome(), observation_verdicts_,
observation_d_min_, observation_run_width_,
*result.consensus_cell, experiment_);
if (config_.export_unmerged_partials) {
if (observer) observer->OnPhase("Unmerged reflections");
MtzTable mtz = UnmergedMtzTable(indexer->GetIntegrationOutcome(), *result.consensus_cell,
+56 -5
View File
@@ -172,8 +172,8 @@ struct ProcessConfig {
// With it, a run of tests on the placement (ModuleDiagnostics.h) goes to the log (--refine-modules=diagnose).
bool module_diagnostics = false;
// Whether the placement found is used from there on, or only measured and reported (and written to
// <prefix>_modules.h5). Used for a foreign multi-panel detector (SACLA, a geometry file's panels); a
// JUNGFRAU's metrology is measured only.
// <prefix>_modules.h5). Used for a foreign multi-panel detector (SACLA, a geometry file's panels, a
// CrystFEL .geom given with --module-geometry); a JUNGFRAU's metrology is measured only.
bool apply_modules = true;
// Beam-stop shadow detection (--detect-beam-stop). When set, a pre-pass projects this many
@@ -266,6 +266,14 @@ struct ProcessConfig {
// run can write both.
bool export_unmerged_partials = false;
// Diagnostic, stills: every integrated observation with what the merge made of it, as <prefix>_obs.mtz
// (ObservationMtzTable; --export-observations). Off by default; it does not change the processing.
bool export_observations = false;
// Stills: the prediction window's half-width, in run rocking widths (--stills-window-sigmas). Wider
// predicts, integrates and exports more of the rocking curve's tails.
double stills_window_sigmas = 2.0;
// If set, after merging, validate the merged intensities against this atomic model: compute
// R-free with an optimized bulk solvent and write 2Fo-Fc / Fo-Fc maps (--model). The caller parses
// it (ParseRugnuxModel, writer/RugnuxModelFile.h).
@@ -350,6 +358,34 @@ bool ValidationEvidencePrefers(const ValidationSpotEvidence &current, const Vali
std::pair<double, double> ValidationEvidenceDifference(const ValidationSpotEvidence &current,
const ValidationSpotEvidence &candidate);
// Stills given a cell: only the frames whose refined cell belongs to its population (GivenCellPopulation)
// are merged - a second crystal form, fitted to the same cell, is left out, its reflections cleared.
// Returns the number of frames left out.
size_t LeaveOutOtherCellForms(std::vector<IntegrationOutcome> &outcomes, const UnitCell &given_cell, Logger &logger);
// The stills scale and merge, the one both --mode mx and --mode scale make: each image scaled against the
// merge of all, the partialities post-refined against it (unless for_search or --simple-stills), the error
// model fitted, the observations merged, cut at the resolution fall-off and the statistics taken.
// run_width is the rocking curve the frames were predicted through (measured here where there is none);
// verdicts asks for each observation's merge verdict (--export-observations).
struct StillsMergeResult {
std::vector<MergedReflection> merged;
MergeStatistics statistics;
std::optional<MergeStatistics> reference_statistics; // --report-resolution
std::optional<double> d_min; // the cut applied to merged
std::optional<double> resolution_fit_A;
std::optional<StillsRunWidth> used_width; // the curve the post-refinement used
std::vector<std::vector<ObservationVerdict>> verdicts;
double isa = 0.0, a = 0.0, b = 0.0, c = 0.0; // error model, XDS convention (c: partiality term)
bool isa_resolved = true;
};
StillsMergeResult StillsScaleMerge(const DiffractionExperiment &experiment, std::vector<IntegrationOutcome> &outcomes,
const std::optional<UnitCell> &reference_cell,
const std::optional<StillsRunWidth> &run_width,
const std::vector<MergedReflection> &reference_data, bool for_search,
bool verdicts, int nthreads, Logger &logger,
const std::function<void(const std::string &)> &phase);
// The longest real-space axis whose lattice rows this detector can still separate. Neighbouring rows of
// an axis of length L lie 1/L apart in reciprocal space, which near the beam is D * lambda / L on the
// detector; they stay apart while that exceeds the recorded spot width. So L = D * lambda /
@@ -496,6 +532,7 @@ struct RugnuxOutputs {
std::optional<SolventBuildResult> solvent_build; // and the water build, in it
std::optional<MtzTable> unmerged; // <prefix>_unmerged.mtz
std::optional<MtzTable> unmerged_partials; // <prefix>_unmerged_partials.mtz
std::optional<MtzTable> observations; // <prefix>_obs.mtz (diagnostic)
std::string detector_jpeg; // <prefix>_detector.jpg; empty: none drawn
std::optional<PerImagePlot> plot; // <prefix>_plot.txt
// A run that indexed nothing, with no per-image analysis streamed (ProcessConfig::image_sink): the
@@ -594,7 +631,8 @@ struct ProcessResult {
// the error-model asymptotic I/sigma (the strong-reflection tier; the whole-range XDS-equivalent is 1/b); has_reference is true when a reference MTZ drove CCref.
bool has_merge_statistics = false;
MergeStatistics merge_statistics;
double error_model_isa = 0.0; // whole-range 1/sqrt(a*b), XDS's meaning
double error_model_isa = 0.0; // whole-range 1/sqrt(a*b), XDS's meaning; on stills 1/b of
// the stills model, used inside (space-group search), not reported
double error_model_isa_asymptotic = 0.0; // strong-reflection tier; rotation path only, 0 on stills
bool error_model_isa_resolved = true; // false: b is zero within two SE of the bin fit (the files' ISa "?")
// The report's ISa: its 95% interval by a bootstrap over the reflections (rotation only). high is
@@ -603,6 +641,9 @@ struct ProcessResult {
double error_model_isa_high = std::numeric_limits<double>::quiet_NaN();
double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2)
double error_model_b = 0.0;
// Stills only: the partiality term, in the same convention,
// sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2).
double error_model_c = 0.0;
bool has_reference = false;
// How the reference MTZ was matched: the operator the merged indices were put through to reach
// its frame (h,k,l where none), the intensity CC and matched fraction in that frame, and whether
@@ -1524,6 +1565,12 @@ class Rugnux {
// the defective pixels, and the recorded spot width that sets the integration radius
// (config_.adaptive_integration_radius).
void PreScan(int start_image, int images_to_process, int frame_count, RugnuxObserver *observer);
// The shadow of an obstruction at the edge of the beam path (FindOuterShadow), found on the projection
// the pre-scan built and added to the beam-stop shadow. Called once the beam centre is settled, so
// that the rays it is found along start at the beam; the projection is let go of once it is used.
void MaskOuterShadow();
std::shared_ptr<const std::vector<float>> outer_shadow_projection_;
uint32_t outer_shadow_frames_ = 0;
// The first part of a rotation sweep, [start_image, PrepassEnd): what the geometry pre-pass
// integrates, and the only frames the decisions taken before the canonical pass read - the
// pre-scan sample (mask, spot width, background beam centre) and the first pass's schemes and
@@ -1556,7 +1603,7 @@ class Rugnux {
struct Defects {
size_t pixels = 0;
std::vector<std::pair<uint32_t, uint32_t>> mask; // the mask's non-zero pixels
size_t hot = 0, error = 0, dead = 0;
size_t hot = 0, error = 0, dead = 0, unstable = 0;
uint32_t frames = 0;
};
struct Entry {
@@ -1668,7 +1715,11 @@ class Rugnux {
// Stills: the rocking width measured before the image pass, on a sample of its frames, which every
// frame is then predicted through (ImagePass) and the partialities are computed from (ScaleMerge).
std::optional<StillsPartialityRefine::RunWidth> stills_run_width_;
static constexpr double STILLS_WINDOW_SIGMAS = 2.0; // the window's half-width, in rocking widths
// --export-observations: the written stills merge's verdict on every observation, its resolution cut and
// the rocking curve its partialities were computed from.
std::vector<std::vector<ObservationVerdict>> observation_verdicts_;
std::optional<double> observation_d_min_;
std::optional<StillsRunWidth> observation_run_width_;
static constexpr int STILLS_WINDOW_MIN_FRAMES = 2000; // the sample: a tenth of the run, at least this
static constexpr int STILLS_WINDOW_FRACTION = 10;
// Stills: where the frames' pixel values pile up, against the file's saturation limit (see there).
+8 -4
View File
@@ -260,7 +260,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
// predicted five to eight widths at low resolution, mostly empty, and cut the curve at one or two at
// high resolution, so the partials there were never integrated. So the run's width is measured first,
// on a sample of the frames integrated the old way (MeasureRunWidth, as the merge measures it), and
// every frame is then predicted through STILLS_WINDOW_SIGMAS of it.
// every frame is then predicted through config_.stills_window_sigmas of it.
std::vector<int> window_frames;
const bool stills_window = full && per_image_analysis && !calibration && !geometry_prepass && indexer
&& !experiment_.IsRotationIndexing() && config_.spot_finding.indexing
@@ -449,16 +449,20 @@ void Rugnux::ImagePass(PipelineLocals &p) {
for (int i = 0; i < 3; ++i)
radial_scale = refine.MeasureRadialScale(outcomes, stills_user_cell_, radial_scale);
stills_run_width_ = refine.MeasureRunWidth(outcomes, stills_user_cell_, radial_scale);
const double radial_spread2 = stills_run_width_ ? refine.MeasureRadialSpread(outcomes, *stills_run_width_) : 0.0;
for (const int o: window_frames)
outcomes[o] = IntegrationOutcome{};
if (stills_run_width_) {
const auto k = static_cast<float>(STILLS_WINDOW_SIGMAS);
const auto k = static_cast<float>(config_.stills_window_sigmas);
indexer->SetStillsWindow(StillsWindow{.c0 = k * static_cast<float>(stills_run_width_->gamma0),
.ang = k * static_cast<float>(stills_run_width_->gamma_e),
.bw = k * static_cast<float>(stills_run_width_->bandwidth_sigma),
.q_scale = static_cast<float>(stills_run_width_->radial_scale)});
.q_scale = static_cast<float>(stills_run_width_->radial_scale),
.radial = k * k * static_cast<float>(radial_spread2)});
Note(fmt::format("prediction window {:.1f} x the rocking width measured on {} frames, centred at "
"radial scale {:+.2e}", STILLS_WINDOW_SIGMAS, window_frames.size(), radial_scale - 1.0));
"radial scale {:+.2e}", config_.stills_window_sigmas, window_frames.size(), radial_scale - 1.0));
Note(fmt::format("spots placed with their excitation error as by a radial spread of {:.1e}",
std::sqrt(radial_spread2)));
}
}
for (int i = 0; i < image_workers; ++i)
+4
View File
@@ -123,6 +123,10 @@ void WriteOutputs(const RugnuxOutputs &o, const std::string &prefix, const Outpu
WriteMtz(*o.unmerged_partials, prefix + "_unmerged_partials.mtz");
logger.Info("Unmerged partials written to {}_unmerged_partials.mtz", prefix);
}
if (o.observations) {
WriteMtz(*o.observations, prefix + "_obs.mtz");
logger.Info("Observations written to {}_obs.mtz", prefix);
}
if (!o.detector_jpeg.empty()) {
const std::string path = prefix + "_detector.jpg";
std::ofstream f(path, std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
+40 -4
View File
@@ -182,6 +182,39 @@ namespace {
}
}
void Rugnux::MaskOuterShadow() {
// Stills only. A rotation sweep's shadows are left as the beam-stop detection draws them: there the
// corners beyond a hardware shadow it already found would be added, and a margin about it.
if (!outer_shadow_projection_ || experiment_.IsRotationIndexing())
return;
Logger logger("Rugnux");
// An observation is left out where its signal disk reaches the shadow (DropShadowClipped), but what
// spoils one beside an outer shadow is its background ring, R3 across, straddling the shadow's edge:
// the ring is then read partly dimmed. So the shadow is grown by the difference.
const auto &bis = experiment_.GetBraggIntegrationSettings();
const int margin = static_cast<int>(std::ceil(bis.GetR3() - bis.GetR1()));
auto shadow = FindOuterShadow(experiment_, pixel_mask_, *outer_shadow_projection_, outer_shadow_frames_,
margin, config_.nthreads);
outer_shadow_projection_.reset();
const auto &mask = pixel_mask_.GetMask();
size_t added = 0;
for (size_t i = 0; i < shadow.size(); i++) {
added += shadow[i] != 0 && mask[i] == 0;
shadow[i] |= (mask[i] >> PixelMask::BeamStopPixelBit) & 1u;
}
const auto [beam_x, beam_y] = experiment_.GetDiffractionGeometry().GetDirectBeam_pxl();
if (added == 0) {
logger.Info("Outer shadow: none along the rays from the beam at ({:.2f},{:.2f})", beam_x, beam_y);
return;
}
pixel_mask_.LoadBeamStopMask(experiment_, shadow);
logger.Info("Outer shadow: {} pixels ({:.2f}% of the detector) beyond a step down in the background along the "
"rays from the beam at ({:.2f},{:.2f}), with {} px about it, are masked as shadow", added,
100.0 * static_cast<double>(added) / static_cast<double>(shadow.size()), beam_x, beam_y, margin);
Note(fmt::format("outer shadow masked: {:.2f}% of the detector",
100.0 * static_cast<double>(added) / static_cast<double>(shadow.size())));
}
// PRESCAN_MAX_WORKERS, fewer where their GPU engines would not fit the card.
size_t Rugnux::PrescanWorkers() const {
const int gpus = get_gpu_count();
@@ -1083,6 +1116,8 @@ void Rugnux::PreScan(int start_image, int images_to_process, int frame_count, Ru
"not as shadow", dead, shadow_frames);
const auto shadowed = std::count_if(shadow.begin(), shadow.end(), [](uint32_t v) { return v != 0; });
pixel_mask_.LoadBeamStopMask(experiment_, shadow);
outer_shadow_projection_ = projection;
outer_shadow_frames_ = static_cast<uint32_t>(shadow_frames);
opaque_shadow = shadow;
for (auto &v : opaque_shadow)
if (v == ShadowFinder::TRANSMITTING)
@@ -1266,10 +1301,11 @@ void Rugnux::MaskDefectivePixels(int start_image, const std::vector<int> &sample
for (const auto &[i, v] : defects->mask)
mask[i] = v;
pixel_mask_.LoadHotPixelMask(experiment_, mask);
const size_t masked = defects->hot + defects->error + defects->dead;
const size_t masked = defects->hot + defects->error + defects->dead + defects->unstable;
logger.Info("Defective pixels: {} masked from {} pre-scan frames - {} lit above their ring on more "
"frames than one reflection or chance explains, {} holding the detector's error value or saturation limit, {} dead",
masked, defects->frames, defects->hot, defects->error, defects->dead);
"frames than one reflection or chance explains, {} holding the detector's error value or saturation limit, {} dead, "
"{} unstable (far below zero on a twentieth of the frames)",
masked, defects->frames, defects->hot, defects->error, defects->dead, defects->unstable);
Note(fmt::format("{} defective pixel{} masked", masked, masked == 1 ? "" : "s"));
};
if (defects) {
@@ -1360,7 +1396,7 @@ void Rugnux::MaskDefectivePixels(int start_image, const std::vector<int> &sample
? oscillation * (sample.back() - sample.front()) / static_cast<double>(sample.size() - 1)
: 0.0;
const auto hot = finder.GetMask(oscillation, spacing, nthreads);
defects = PreScanMemo::Defects{hot.mask.size(), {}, hot.hot, hot.error, hot.dead, hot.frames};
defects = PreScanMemo::Defects{hot.mask.size(), {}, hot.hot, hot.error, hot.dead, hot.unstable, hot.frames};
for (size_t i = 0; i < hot.mask.size(); i++)
if (hot.mask[i] != 0)
defects->mask.emplace_back(static_cast<uint32_t>(i), hot.mask[i]);
+201 -121
View File
@@ -92,6 +92,9 @@
using namespace rugnux_internal;
namespace {
// Two groups of a given cell's frames whose mean volumes differ by less than this (relative) are one
// crystal form (see the given-cell population below).
constexpr double MIN_FORM_SEPARATION = 0.01;
// The space-group search's resolution cut on a P1 merge: where the best non-increasing fit of its
// thin-shell <I/sigma> drops under 1 (0: no cut). See the search below.
double SearchCut(const std::vector<MergedReflection> &merged) {
@@ -596,6 +599,175 @@ static void WarnIfMetricHostsMoreSymmetry(const std::optional<UnitCell> &cell,
metric_system, metric_rotations, sg.xhm(), group_rotations);
}
size_t LeaveOutOtherCellForms(std::vector<IntegrationOutcome> &outcomes, const UnitCell &given_cell, Logger &logger) {
const double given = CrystalLattice(given_cell).CalcVolume();
std::vector<double> rel;
for (const auto &o : outcomes)
if (!o.reflections.empty())
rel.push_back(o.latt.CalcVolume() / given - 1.0);
auto [low, high] = GivenCellPopulation(rel);
std::string all;
for (double r : rel)
all += fmt::format(" {:.4f}", r);
logger.Debug("Given cell: refined volumes relative to it:{}", all);
// A histogram valley alone does not make two crystal forms: one form whose cell drifts - a crystal
// swelling with dose, a cell refined a little differently in different orientations - falls into
// groups too (a drift through its range piles up at its two ends), and leaving one group out throws
// away part of the data of one structure. Measured: a rotation sweep processed as stills, its cell
// 0.25 % larger in volume in the second half than in the first, lost 841 of 1800 frames to the
// valley. Two forms of one protein that index on one cell differ by far more (9 % between the two
// forms of the battery's two-form sample), so groups closer than MIN_FORM_SEPARATION are one form.
// Their intensities would be the better judge, but a stills merge of groups taken from different
// orientations of one rotating crystal correlates only 0.8 with itself (0.66-0.88 measured), which
// is no clearer a line than the cells draw.
if (std::isfinite(low) || std::isfinite(high)) {
double sum_in = 0.0, sum_out = 0.0;
size_t n_in = 0, n_out = 0;
for (double r : rel)
(r >= low && r <= high) ? (sum_in += r, ++n_in) : (sum_out += r, ++n_out);
const double separation = std::fabs(sum_out / n_out - sum_in / n_in);
if (separation < MIN_FORM_SEPARATION) {
logger.Info("Given cell: the frames outside its volume population ({:+.2f}% to {:+.2f}%) are "
"{:.2f}% from it in volume, too close to be another crystal form - every frame is merged",
100 * low, 100 * high, 100 * separation);
low = -std::numeric_limits<double>::infinity();
high = std::numeric_limits<double>::infinity();
}
}
size_t below = 0, above = 0;
for (auto &o : outcomes) {
if (o.reflections.empty())
continue;
const double r = o.latt.CalcVolume() / given - 1.0;
if (r < low || r > high) {
o.reflections.clear();
(r < low ? below : above)++;
}
}
if (below + above > 0)
logger.Info("Given cell: {} of {} indexed frames refined to a cell outside its population "
"({:+.1f}% to {:+.1f}% in volume) and are left out of the merge - {} smaller, {} larger",
below + above, rel.size(), 100 * low, 100 * high, below, above);
return below + above;
}
StillsMergeResult StillsScaleMerge(const DiffractionExperiment &experiment, std::vector<IntegrationOutcome> &outcomes,
const std::optional<UnitCell> &reference_cell,
const std::optional<StillsRunWidth> &run_width,
const std::vector<MergedReflection> &reference_data, bool for_search,
bool verdicts, int nthreads, Logger &logger,
const std::function<void(const std::string &)> &phase) {
// Self-scale each image against the running merge with ScaleOnTheFly (fixed partiality), then merge
// directly. This runs even with an external reference: the reference is used only to break the
// per-image indexing ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor
// (scaling each image against a foreign dataset injects cross-dataset systematics and is a worse
// reference than the data's own merge). One pass: the per-image scale G is the exact one-pass
// solution, so iterating would only rebuild the reference from the freshly-scaled (noisy) data and
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
StillsMergeResult out;
// A search merge only looks: the frames leave it with the scales they came with, so the merge that is
// written starts from the frames as they were integrated - in --mode mx as in --mode scale, which reads
// them so from the _process.h5.
struct FrameScale { std::optional<float> cc, g; std::optional<int64_t> cc_n; std::vector<float> corr; };
std::vector<FrameScale> before;
if (for_search)
for (const auto &o : outcomes) {
FrameScale &f = before.emplace_back(FrameScale{o.image_scale_cc, o.image_scale_g, o.image_scale_cc_n, {}});
for (const auto &r : o.reflections)
f.corr.push_back(r.image_scale_corr);
}
{
phase("Scaling images");
TimingMark scale_mark(logger, "stills scaling images");
ScaleOnTheFly(experiment, MergeAll(experiment, outcomes, nthreads, reference_cell)).Scale(outcomes, nthreads);
}
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
// orientation tilt against the running merge and recompute each reflection's partiality + scale
// correction (no re-integration). Never for the P1 search pass.
if (!for_search && experiment.GetScalingSettings().GetStillsPartialityRefine()) {
phase("Partiality post-refine");
TimingMark refine_mark(logger, "stills partiality post-refine");
StillsPartialityRefine refiner(experiment);
const double mean_tilt = refiner.Run(outcomes, nthreads, reference_cell, run_width, &out.used_width);
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
// The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally
// when the image has no prior CC - which is exactly the state the guard above leaves. So
// the guard was protecting nothing: measured on a jet dataset, of 367 images it left
// unscaled only 8 were still unscaled in the output, and 53 reached the merge at or below
// a fiftieth of the run median, the worst at a 4525th. Whatever produced the final scale,
// it is the final scale that has to be credible.
ScaleOnTheFly::RejectCollapsedScales(outcomes);
}
phase("Merging");
TimingMark merge_mark(logger, "stills error model and merge");
MergeOnTheFly merge_engine(experiment);
merge_engine.Threads(nthreads);
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
// intensities and the error model, so the space-group search sees clean data; the final
// in-symmetry merge keeps them so completeness is not lost.
merge_engine.ExcludeIceRings(for_search);
if (reference_cell)
merge_engine.ReferenceCell(*reference_cell);
// Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells)
// most indexed crystals do not correlate with the true structure; --min-image-cc drops the
// crystals whose per-image CC to the reference is below the limit, so the merge keeps only
// the coherent (real) ones. Set before the error model so the model, the merge and the
// reported statistics are all fitted over the same set of images. The P1 search pass keeps
// every image: its job is to find the symmetry, not to produce final intensities.
merge_engine.FilterByImageCC(!for_search && experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
merge_engine.RefineErrorModel(outcomes);
if (merge_engine.ErrorModelActive()) {
// XDS convention, sigma^2 = a*(sigma0^2 + b*I^2 + c*((1-p)/p)^2*I^2). No ISa: on stills 1/b is
// the ceiling of one still observation, not what CORRECT.LP's ISa measures.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} c={:.3e} chi2={:.2f}",
em.a, em.b, ToXdsPartialityTerm(merge_engine.ErrorModelA(), merge_engine.ErrorModelC()),
merge_engine.ErrorModelChi2());
}
merge_engine.AddImages(outcomes);
if (!for_search)
logger.Info("Merge: {} merged observations have a partiality below {}", merge_engine.FarTailCount(),
MergeOnTheFly::FAR_TAIL_PARTIALITY);
if (verdicts)
out.verdicts = merge_engine.Verdicts(outcomes);
out.merged = merge_engine.ExportReflections();
merge_mark.Stop();
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
// range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A
// manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the
// space-group search still sees the full range.
const auto &cut_ss = experiment.GetScalingSettings();
out.d_min = ApplyResolutionCutoff(out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
cut_ss.GetResolutionCCTarget(), for_search, logger, &out.resolution_fit_A);
phase("Computing statistics");
TimingMark stats_mark(logger, "stills statistics");
out.statistics = merge_engine.MergeStats(out.merged, outcomes, reference_data, out.d_min);
// The same kept reflections binned over the reference range (--report-resolution); the
// stills error model is fitted over the whole range already, so no ISa of its own.
if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search)
out.reference_statistics = merge_engine.MergeStats(out.merged, outcomes, reference_data, std::nullopt, range);
// No asymptote tier, and the stills ISa already IS the whole-range 1/b.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
out.isa = em.isa;
out.isa_resolved = merge_engine.ErrorModelBResolved();
out.a = em.a;
out.b = em.b;
out.c = ToXdsPartialityTerm(merge_engine.ErrorModelA(), merge_engine.ErrorModelC());
for (size_t i = 0; i < before.size(); ++i) {
outcomes[i].image_scale_cc = before[i].cc;
outcomes[i].image_scale_g = before[i].g;
outcomes[i].image_scale_cc_n = before[i].cc_n;
for (size_t j = 0; j < before[i].corr.size(); ++j)
outcomes[i].reflections[j].image_scale_corr = before[i].corr[j];
}
return out;
}
bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
auto &logger = p.logger;
auto &result = p.result;
@@ -693,33 +865,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
logger.Info("Sweep ends: left out {} frame(s) at the start and {} at the end that recorded no "
"diffraction - a shutter or beam that was not on them", at_start, at_end);
}
// Stills given a cell: only the frames whose refined cell belongs to its population are merged
// (see GivenCellPopulation) - a second crystal form, fitted to the same cell, is left out.
// Stills given a cell: only the frames of its population are merged.
if (!experiment_.IsRotationIndexing() && stills_user_cell_) {
const double given = CrystalLattice(*stills_user_cell_).CalcVolume();
std::vector<double> rel;
for (const auto &o : outcomes)
if (!o.reflections.empty())
rel.push_back(o.latt.CalcVolume() / given - 1.0);
const auto [low, high] = GivenCellPopulation(rel);
std::string all;
for (double r : rel)
all += fmt::format(" {:.4f}", r);
logger.Debug("Given cell: refined volumes relative to it:{}", all);
size_t below = 0, above = 0;
for (auto &o : outcomes) {
if (o.reflections.empty())
continue;
const double r = o.latt.CalcVolume() / given - 1.0;
if (r < low || r > high) {
o.reflections.clear();
(r < low ? below : above)++;
}
}
if (below + above > 0) {
logger.Info("Given cell: {} of {} indexed frames refined to a cell outside its population "
"({:+.1f}% to {:+.1f}% in volume) and are left out of the merge - {} smaller, {} larger",
below + above, rel.size(), 100 * low, 100 * high, below, above);
end_msg.stills_given_cell = stills_user_cell_;
if (LeaveOutOtherCellForms(outcomes, *stills_user_cell_, logger) > 0) {
// The cell written is the mean over the frames merged, not over the other form's as well.
result.consensus_cell = indexer->GetConsensusUnitCell();
end_msg.unit_cell = result.consensus_cell;
@@ -1711,99 +1860,30 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
rsm->Run(for_search, /*full_stats=*/!geometry_prepass,
/*measure_cc_before_corrections=*/measure_cc));
}
// Stills (rotation goes through RotationScaleMerge above): self-scale each image against the
// running merge with ScaleOnTheFly (fixed partiality), then merge directly. This runs even
// with an external reference: the reference is used only to break the per-image indexing
// ambiguity and to report CCref / inherit the R-free set, NOT as a scale anchor (scaling each
// image against a foreign dataset injects cross-dataset systematics and is a worse reference
// than the data's own merge). One pass: the per-image scale G is the exact one-pass solution,
// so iterating would only rebuild the reference from the freshly-scaled (noisy) data and
// degrade weak stills (measured CC1/2 collapse at the default 3 iters).
{
phase("Scaling images (" + label + ")");
TimingMark scale_mark(logger, "stills scaling images " + label);
auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads,
result.consensus_cell);
indexer->ScaleAllImages(merge_result);
// Stills (rotation goes through RotationScaleMerge above).
const bool verdicts = config_.export_observations && !for_search && write_output;
auto sm = StillsScaleMerge(experiment_, indexer->GetIntegrationOutcome(), result.consensus_cell,
stills_run_width_, config_.reference_data, for_search, verdicts, config_.nthreads,
logger, [&](const std::string &name) { phase(name + " (" + label + ")"); });
if (verdicts) {
observation_verdicts_ = std::move(sm.verdicts);
observation_d_min_ = sm.d_min;
observation_run_width_ = sm.used_width;
}
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
// orientation tilt against the running merge and recompute each reflection's partiality + scale
// correction (no re-integration). Never for the P1 search pass.
if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) {
phase("Partiality post-refine (" + label + ")");
TimingMark refine_mark(logger, "stills partiality post-refine " + label);
StillsPartialityRefine refiner(experiment_);
const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads, result.consensus_cell,
stills_run_width_, &end_msg.stills_run_width);
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
// The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally
// when the image has no prior CC - which is exactly the state the guard above leaves. So
// the guard was protecting nothing: measured on a jet dataset, of 367 images it left
// unscaled only 8 were still unscaled in the output, and 53 reached the merge at or below
// a fiftieth of the run median, the worst at a 4525th. Whatever produced the final scale,
// it is the final scale that has to be credible.
ScaleOnTheFly::RejectCollapsedScales(indexer->GetIntegrationOutcome());
}
const std::vector<IntegrationOutcome> &merge_input = indexer->GetIntegrationOutcome();
phase("Merging");
TimingMark merge_mark(logger, "stills error model and merge " + label);
MergeOnTheFly merge_engine(experiment_);
merge_engine.Threads(config_.nthreads);
// For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged
// intensities and the error model, so the space-group search sees clean data; the final
// in-symmetry merge keeps them so completeness is not lost.
merge_engine.ExcludeIceRings(for_search);
if (result.consensus_cell.has_value())
merge_engine.ReferenceCell(*result.consensus_cell);
// Merge-consistency filter: on floods with many spurious lattices (e.g. XFEL large cells)
// most indexed crystals do not correlate with the true structure; --min-image-cc drops the
// crystals whose per-image CC to the reference is below the limit, so the merge keeps only
// the coherent (real) ones. Set before the error model so the model, the merge and the
// reported statistics are all fitted over the same set of images. The P1 search pass keeps
// every image: its job is to find the symmetry, not to produce final intensities.
merge_engine.FilterByImageCC(!for_search
&& experiment_.GetScalingSettings().GetMinCCForImage() > 0.0);
merge_engine.RefineErrorModel(merge_input);
if (merge_engine.ErrorModelActive())
{
// XDS convention, so the pair reads directly against a CORRECT.LP; the stills ISa is
// already the whole-range 1/b, there being no asymptote tier on this path.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
}
merge_engine.AddImages(merge_input);
// The curve the frames were predicted through, which --mode scale passes in again: the one the
// post-refinement used is balanced against this run's merge, and balancing it again would move it.
end_msg.stills_run_width = stills_run_width_;
if (sm.resolution_fit_A)
result.resolution_fit_A = sm.resolution_fit_A;
result.error_model_isa = sm.isa;
result.error_model_isa_resolved = sm.isa_resolved;
result.error_model_a = sm.a;
result.error_model_b = sm.b;
result.error_model_c = sm.c;
ScaleMergeResult out;
out.merged = merge_engine.ExportReflections();
merge_mark.Stop();
// Automatic high-resolution cutoff (post-merge): trim the written reflections + reported
// shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full
// range, and the _process.h5 is written from the per-image outcomes, so no data is lost. A
// manual --scaling-high-resolution wins; the P1 search merge (for_search) is never cut so the
// space-group search still sees the full range. (Rotation is cut inside RotationScaleMerge.)
const auto &cut_ss = experiment_.GetScalingSettings();
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
out.merged, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A);
phase("Computing statistics");
TimingMark stats_mark(logger, "stills statistics " + label);
out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min);
// The same kept reflections binned over the reference range (--report-resolution); the
// stills error model is fitted over the whole range already, so no ISa of its own.
if (const auto range = cut_ss.GetReportResolutionRange_A(); range && !for_search)
out.reference_statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data,
std::nullopt, range);
{ // stills: no asymptote tier, and its ISa already IS the whole-range 1/b
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
result.error_model_isa = em.isa;
result.error_model_isa_resolved = merge_engine.ErrorModelBResolved();
result.error_model_a = em.a;
result.error_model_b = em.b;
}
out.merged = std::move(sm.merged);
out.statistics = std::move(sm.statistics);
out.reference_statistics = std::move(sm.reference_statistics);
out.cc_half_before_corrections = out.statistics.overall.cc_half; // no surfaces on this path
logger.Info("Merge complete ({} unique reflections, {})", out.merged.size(), label);
Note(fmt::format("merge in {}: {} unique", label, out.merged.size()));
@@ -4449,7 +4529,6 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
// A model can resolve it too, but whether it does is only known once the model has
// been fitted (validation step below), so the warning stands here exactly as in a run
// without a model, and is withdrawn there if the model's indexing probe decided it.
// On serial stills with -C and -S the model is already in `ambiguity_reference`.
if (!ambiguity_reference && experiment_.IsRotationIndexing()) {
// One sweep has one orientation matrix, so every frame is in the same hand and the
// merge mixes nothing. The hand is arbitrary, which matters only against other
@@ -4464,8 +4543,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
"(reindex operator(s): {}). Serial-stills crystals are indexed in one hand at random, "
"and rugnux makes them consistent from the data only where the space group is given: "
"here it was determined by the run, so the merge mixes the hands and CC1/2 is degraded. "
"Giving the space group (-S) resolves it from the data; a reference MTZ (-z) or a model "
"(--model, which needs -C and -S here) resolves it against them.", laws);
"Giving the space group (-S) resolves it from the data; a reference MTZ (-z) resolves it "
"against that.", laws);
logger.Warning("{}", msg);
ambiguity_warning_text = " !! " + msg + "\n\n";
stats_text << ambiguity_warning_text;
@@ -5578,7 +5657,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
result.error_model_isa_asymptotic > 0 && result.error_model_isa_resolved
? fmt::format("{:.2f}", result.error_model_isa_asymptotic) : std::string(),
result.error_model_a > 0 ? fmt::format("{:.3f}", result.error_model_a) : std::string(),
result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string()};
result.error_model_b > 0 ? fmt::format("{:.4e}", result.error_model_b) : std::string(),
result.error_model_c > 0 ? fmt::format("{:.4e}", result.error_model_c) : std::string()};
MtzTable merged_mtz = MergedMtzTable(sm.merged, *result.consensus_cell, experiment_, config_.provenance);
// The unmerged MTZ reads the fulls until it is made.
if (unmerged_mtz.valid())
+83 -85
View File
@@ -65,6 +65,7 @@
#include "../image_analysis/scale_merge/Merge.h"
#include "../image_analysis/scale_merge/RfreeFlags.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h" // MatchReference
#include "../image_analysis/scale_merge/ReferenceFreeReindex.h"
#include "../image_analysis/scale_merge/ScaleOnTheFly.h"
#include "../image_analysis/scale_merge/StillsPartialityRefine.h"
#include "../image_analysis/scale_merge/SublatticeRelabel.h"
@@ -234,6 +235,8 @@ enum {
OPT_EXPORT_UNMERGED,
OPT_NO_EXPORT_UNMERGED,
OPT_EXPORT_UNMERGED_PARTIALS,
OPT_EXPORT_OBSERVATIONS,
OPT_STILLS_WINDOW_SIGMAS,
OPT_DUMP_OBSERVATIONS,
OPT_INTEGRATOR,
OPT_SIMPLE_STILLS,
@@ -317,6 +320,8 @@ static option long_options[] = {
{"export-unmerged", no_argument, nullptr, OPT_EXPORT_UNMERGED},
{"no-export-unmerged", no_argument, nullptr, OPT_NO_EXPORT_UNMERGED},
{"export-unmerged-partials", no_argument, nullptr, OPT_EXPORT_UNMERGED_PARTIALS},
{"export-observations", no_argument, nullptr, OPT_EXPORT_OBSERVATIONS},
{"stills-window-sigmas", required_argument, nullptr, OPT_STILLS_WINDOW_SIGMAS},
{"dump-observations", required_argument, nullptr, OPT_DUMP_OBSERVATIONS},
{"space-group", required_argument, nullptr, 'S'},
{"anomalous", no_argument, nullptr, 'A'},
@@ -911,6 +916,8 @@ struct RugnuxCli {
double refine_d_min = 0; // --refine-d-min
bool export_unmerged = true; // on by default; --no-export-unmerged turns it off
bool export_unmerged_partials = false; // --export-unmerged-partials: the same, one row per image
bool export_observations = false; // --export-observations: <prefix>_obs.mtz, stills diagnostic
std::optional<double> stills_window_sigmas_arg; // --stills-window-sigmas
std::string dump_observations; // diagnostic: dump unmerged -P rot3d fulls to this path
std::optional<double> min_partiality_arg; // --min-partiality; default per mode (RugnuxDefaultScalingSettings)
std::optional<double> min_captured_fraction_arg; // explicit --min-captured-fraction; default depends on rotation
@@ -1261,6 +1268,12 @@ int RugnuxCli::Parse(int argc, char **argv) {
case OPT_EXPORT_UNMERGED_PARTIALS:
export_unmerged_partials = true;
break;
case OPT_EXPORT_OBSERVATIONS:
export_observations = true;
break;
case OPT_STILLS_WINDOW_SIGMAS:
stills_window_sigmas_arg = parse_double_arg(optarg, "--stills-window-sigmas", logger);
break;
case OPT_DUMP_OBSERVATIONS:
dump_observations = optarg;
break;
@@ -2151,15 +2164,22 @@ int RugnuxInputRun::PrepareRugnuxInput() {
output_provenance.collection_date = dataset->arm_date;
{
// The header as the file states it, before anything is measured: what a question about a run
// is answered from first.
// is answered from first. The wavelength is the one used: --wavelength, where given, is not
// refined later, so the file's value would be printed for a run that never used it (--mode scale
// keeps the wavelength its reflections were integrated at).
const auto &x = dataset->experiment;
const auto gonio = x.GetGoniometer();
const auto [direct_x, direct_y] = x.GetDiffractionGeometry().GetDirectBeam_pxl();
logger.Info("Header: {} images, detector {} {} x {} px of {:.4f} mm, wavelength {:.5f} A, distance "
const bool wavelength_given = wavelength_A && mode != RugnuxMode::Scale;
const float used_wavelength_A = wavelength_given ? *wavelength_A : x.GetWavelength_A();
const std::string wavelength_source =
wavelength_given ? fmt::format(" (--wavelength; the file states {:.5f} A)", x.GetWavelength_A()) : "";
logger.Info("Header: {} images, detector {} {} x {} px of {:.4f} mm, wavelength {:.5f} A{}, distance "
"{:.3f} mm, beam {:.2f} {:.2f} px (direct beam {:.2f} {:.2f}), tilt {:.4f} {:.4f} deg, {}, "
"polarization {}",
reader.GetNumberOfImages(), x.GetDetectorDescription(), x.GetXPixelsNum(), x.GetYPixelsNum(),
x.GetPixelSize_mm(), x.GetWavelength_A(), x.GetDetectorDistance_mm(), x.GetBeamX_pxl(),
x.GetPixelSize_mm(), used_wavelength_A, wavelength_source, x.GetDetectorDistance_mm(),
x.GetBeamX_pxl(),
x.GetBeamY_pxl(), direct_x, direct_y, x.GetPoniRot1_rad() * 180.0 / PI,
x.GetPoniRot2_rad() * 180.0 / PI,
gonio ? fmt::format("rotation from {:.3f} deg by {:.4f} deg per image", gonio->GetStart_deg(),
@@ -2169,7 +2189,7 @@ int RugnuxInputRun::PrepareRugnuxInput() {
: std::string("not stated"));
console->OnNote(fmt::format("{} images, {} {} x {} px", reader.GetNumberOfImages(),
x.GetDetectorDescription(), x.GetXPixelsNum(), x.GetYPixelsNum()));
console->OnNote(fmt::format("{:.4f} A, {:.1f} mm, {}", x.GetWavelength_A(), x.GetDetectorDistance_mm(),
console->OnNote(fmt::format("{:.4f} A, {:.1f} mm, {}", used_wavelength_A, x.GetDetectorDistance_mm(),
gonio ? fmt::format("{:.3f} deg per image", gonio->GetIncrement_deg())
: std::string("no rotation axis")));
}
@@ -2295,6 +2315,11 @@ int RugnuxInputRun::PrepareRugnuxInput() {
// A rotation (goniometer) dataset uses RotationScaleMerge unless --force-still asks for stills scaling.
IndexingSettings indexing_settings;
indexing_settings.RotationIndexing(experiment.GetGoniometer().has_value() && !force_still);
const bool rot = experiment.GetGoniometer().has_value() && !force_still;
// Without the axis under --force-still, as in --mode mx: the unmerged export would otherwise read
// the frames as a sweep.
if (force_still)
experiment.Goniometer(std::nullopt);
// --detect-ice-rings, applied here as well as on the full path below: this block returns
// before that one runs, so without it the flag is silently ignored by --scale. Same
@@ -2308,7 +2333,6 @@ int RugnuxInputRun::PrepareRugnuxInput() {
// Start from the same defaults the full pipeline uses, so --scale reproduces the merge that wrote
// the _process.h5 rather than a weaker model of its own. Every CLI override below then applies on
// top, exactly as in the full-analysis path.
const bool rot = experiment.GetGoniometer().has_value() && !force_still;
ScalingSettings scaling_settings = RugnuxDefaultScalingSettings(rot);
if (d_min_scale_merge)
scaling_settings.HighResolutionLimit_A(d_min_scale_merge.value());
@@ -2417,18 +2441,29 @@ int RugnuxInputRun::PrepareRugnuxInput() {
}
}
// Stills given a cell: the frames of its population, as --mode mx merged them.
if (!experiment.IsRotationIndexing() && dataset->stills_given_cell)
LeaveOutOtherCellForms(reflections, *dataset->stills_given_cell, logger);
// Stills: a crystal indexed in a lattice containing the file's cell is put on that cell, as the full
// pipeline does - before the resolution of every reflection is recomputed from the cell below,
// which is right only for reflections indexed on it.
// pipeline does.
if (!experiment.IsRotationIndexing()) {
const auto relabel = RelabelOnSublattice(reflections, experiment.GetUnitCell().value(),
experiment.GetIndexingSettings());
logger.Info("Put {} crystal(s) indexed in a lattice containing the file's cell on that cell",
relabel.frames);
}
auto refl_stats = UpdateReflectionResolution(experiment.GetUnitCell().value(), reflections);
logger.Info("Read {} reflections from {} images", refl_stats.n_reflections, refl_stats.n_images);
experiment.ImagesPerTrigger(refl_stats.n_images);
// A rotation sweep's reflections are put on the one cell they are merged in; a still keeps the
// resolution of its own lattice, as --mode mx merges it.
if (experiment.IsRotationIndexing())
UpdateReflectionResolution(experiment.GetUnitCell().value(), reflections);
size_t n_reflections = 0, n_images = 0;
for (const auto &o : reflections)
if (!o.reflections.empty()) {
n_reflections += o.reflections.size();
++n_images;
}
logger.Info("Read {} reflections from {} images", n_reflections, n_images);
experiment.ImagesPerTrigger(n_images);
// Ice-ring handling, as the full pipeline does it (Rugnux.cpp): flag reflections on a
// hexagonal-ice powder ring so scaling skips them while the merge keeps them. The flag is not
@@ -2489,13 +2524,14 @@ int RugnuxInputRun::PrepareRugnuxInput() {
std::optional<double> resolution_fit_A; // CC1/2 crossing, without the one-shell extension
double error_model_a = 0.0;
double error_model_b = 0.0;
double error_model_c = 0.0; // stills only
std::optional<MergeStatistics> reference_statistics; // --report-resolution
double reference_isa = 0.0;
// Rotation (rot3d): the dedicated RotationScaleMerge does the whole self-scale -> 3D combine ->
// merge, including the default-on decay + absorption correction surfaces. It does not support
// external-reference scaling or wedge refinement.
// Everything else (stills, reference scaling) uses ScaleOnTheFly + MergeOnTheFly.
// Stills go through StillsScaleMerge, the merge of --mode mx.
const bool is_rotation = experiment.IsRotationIndexing();
if (is_rotation) {
if (!reference_data.empty()
@@ -2530,80 +2566,31 @@ int RugnuxInputRun::PrepareRugnuxInput() {
reference_statistics = std::move(r.reference_statistics);
reference_isa = r.reference_isa;
} else {
// Scaling self-references: the reference MTZ (if any) fixes the cell/space group, reports
// CCref and provides the R-free test set, but is NOT a scale anchor - scaling each image
// against a foreign dataset injects cross-dataset systematics and is a worse reference than
// the data's own merge. The per-image scale G is the exact one-pass solution, so one pass
// (iterating a self-rebuilt reference only re-fits the freshly-scaled noise on weak stills).
// The file holds each still as it was merged - with its post-refined partiality - and not the
// excitation errors its prediction made. Put every reflection back where the prediction left it,
// at p = 1 and with the excitation error of the stored lattice and geometry, so that the scaling
// and the post-refinement start from the state they start from in the full pipeline.
for (auto &o: reflections) {
const Coord S0 = o.geom.GetScatteringVector();
const double inv_lambda = 1.0 / o.geom.GetWavelength_A();
for (auto &r: o.reflections) {
const Coord S = o.latt.Astar() * static_cast<float>(r.h) + o.latt.Bstar() * static_cast<float>(r.k)
+ o.latt.Cstar() * static_cast<float>(r.l) + S0;
r.dist_ewald = static_cast<float>(std::fabs(std::sqrt(S.x * S.x + S.y * S.y + S.z * S.z) - inv_lambda));
r.partiality = 1.0f;
// A group whose lattice admits alternative indexings, with no reference to settle them: the
// crystals were integrated each in one of them at random, and are made consistent from the
// data alone, as --mode mx makes them before its merge.
if (const auto sg = experiment.GetGemmiSpaceGroup(); sg && reference_data.empty()) {
const auto ops = ReindexAmbiguityOperators(*experiment.GetUnitCell(), *sg);
if (!ops.empty()) {
const auto rf = ResolveReindexReferenceFree(reflections, experiment, *experiment.GetUnitCell(),
ops, nthreads, /*apply=*/true);
logger.Info("Indexing ambiguity resolved from the data: {} of {} crystals reindexed",
rf.reindexed, rf.crystals);
}
}
ScaleOnTheFly(experiment, MergeAll(experiment, reflections, nthreads, experiment.GetUnitCell())).Scale(reflections, nthreads);
// Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal
// orientation tilt against the merge and recompute each reflection's partiality + scale correction
// (no re-integration), then merge with the improved corrections.
if (experiment.GetScalingSettings().GetStillsPartialityRefine()) {
StillsPartialityRefine refiner(experiment);
// The curve the mx run computed its partialities from (and predicted the frames through),
// where the file carries it: measured again on frames predicted through it, it is another.
const double mean_tilt = refiner.Run(reflections, nthreads, experiment.GetUnitCell(),
dataset->stills_run_width);
logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt);
// As in the full pipeline: the refine fits its own per-image scale, with no floor.
ScaleOnTheFly::RejectCollapsedScales(reflections);
}
MergeOnTheFly merge_engine(experiment);
merge_engine.Threads(nthreads);
merge_engine.ReferenceCell(experiment.GetUnitCell());
// --min-image-cc has to hold for the merge itself, not only for the reported statistics.
merge_engine.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0);
// Fit the (a, b) error model from symmetry-mate scatter before merging, exactly as the full
// pipeline does (Rugnux.cpp). Without this the offline --scale merge would use the identity
// model and produce much worse stills intensities (no (b*I)^2 systematic term at all).
merge_engine.RefineErrorModel(reflections);
if (merge_engine.ErrorModelActive())
{
// XDS convention, so the pair reads directly against a CORRECT.LP; the stills ISa is
// already the whole-range 1/b, there being no asymptote tier on this path.
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}",
em.a, em.b, em.isa, merge_engine.ErrorModelChi2());
}
merge_engine.AddImages(reflections);
merged_reflections = merge_engine.ExportReflections();
// Automatic high-resolution cutoff (post-merge), matching the full-analysis path: a manual
// --scaling-high-resolution wins, otherwise trim the written reflections + reported shells
// to the CC1/2 fall-off. (Rotation is cut inside RotationScaleMerge above.)
const auto &cut_ss = experiment.GetScalingSettings();
// The offline --scale path re-scales a stored _process.h5 and is never a P1 search merge.
const std::optional<double> effective_d_min = ApplyResolutionCutoff(
merged_reflections, cut_ss.GetHighResolutionLimit_A(), cut_ss.GetResolutionCutoff(),
cut_ss.GetResolutionCCTarget(), /*for_search=*/false, logger, &resolution_fit_A);
merged_statistics = merge_engine.MergeStats(merged_reflections, reflections, reference_data,
effective_d_min);
if (const auto range = cut_ss.GetReportResolutionRange_A())
reference_statistics = merge_engine.MergeStats(merged_reflections, reflections, reference_data,
std::nullopt, range);
{ // stills: no asymptote tier, and its ISa already IS the whole-range 1/b
const auto em = ToXdsErrorModel(merge_engine.ErrorModelA(), merge_engine.ErrorModelB());
error_model_isa = em.isa;
error_model_isa_resolved = merge_engine.ErrorModelBResolved();
error_model_a = em.a;
error_model_b = em.b;
}
// The stills scale and merge of --mode mx, on the frames as they were written.
auto sm = StillsScaleMerge(experiment, reflections, experiment.GetUnitCell(), dataset->stills_run_width,
reference_data, /*for_search=*/false, /*verdicts=*/false, nthreads, logger,
[](const std::string &) {});
merged_reflections = std::move(sm.merged);
merged_statistics = std::move(sm.statistics);
reference_statistics = std::move(sm.reference_statistics);
resolution_fit_A = sm.resolution_fit_A;
error_model_isa = sm.isa;
error_model_isa_resolved = sm.isa_resolved;
error_model_a = sm.a;
error_model_b = sm.b;
error_model_c = sm.c;
}
logger.Info("Scale + merge completed in {:.2f} s ({} unique reflections)",
@@ -2792,7 +2779,8 @@ int RugnuxInputRun::PrepareRugnuxInput() {
error_model_isa_asymptotic > 0 && error_model_isa_resolved
? fmt::format("{:.2f}", error_model_isa_asymptotic) : std::string(),
error_model_a > 0 ? fmt::format("{:.3f}", error_model_a) : std::string(),
error_model_b > 0 ? fmt::format("{:.4e}", error_model_b) : std::string()};
error_model_b > 0 ? fmt::format("{:.4e}", error_model_b) : std::string(),
error_model_c > 0 ? fmt::format("{:.4e}", error_model_c) : std::string()};
WriteReflections(merged_reflections, *experiment.GetUnitCell(), experiment, merged_statistics,
em_report, twinning, output_provenance, output_prefix, static_cast<size_t>(nthreads),
&scaled_fulls);
@@ -2832,6 +2820,7 @@ int RugnuxInputRun::PrepareRugnuxInput() {
scale_result.error_model_isa_high = error_model_isa_high;
scale_result.error_model_a = error_model_a;
scale_result.error_model_b = error_model_b;
scale_result.error_model_c = error_model_c;
scale_result.resolution_fit_A = resolution_fit_A;
scale_result.reference_merge_statistics = reference_statistics;
scale_result.reference_isa = reference_isa;
@@ -3285,6 +3274,9 @@ int RugnuxInputRun::PrepareRugnuxInput() {
}
if (has_goniometer)
logger.Info("--force-still: treating the rotation dataset as independent stills");
// Without the axis, as on serial data: otherwise the prediction window, the mosaicity and the
// unmerged export still read the frames as a sweep (summing neighbouring stills into one event).
experiment.Goniometer(std::nullopt);
} else if (!rotation_indexing && has_goniometer) {
rotation_indexing = true;
two_pass_rotation = true;
@@ -3592,7 +3584,10 @@ int RugnuxInputRun::PrepareRugnuxInput() {
config.forced_rotation_lattice = forced_rotation_lattice;
config.refine_geometry = refine_geometry;
config.refine_modules = refine_modules;
config.apply_modules = input_is_panel_table || input_is_cxi || input_is_crystfel;
// The placement found replaces the given one only where the user gave a CrystFEL geometry; modules a
// file carries itself (a JUNGFRAU's, or rebuilt from its mask) are measured and reported only.
config.apply_modules = input_is_panel_table || input_is_cxi || input_is_crystfel
|| module_geometry_file.ends_with(".geom");
config.module_diagnostics = module_diagnostics;
config.run_scaling = run_scaling;
config.scaling_iter = scaling_iter;
@@ -3608,6 +3603,9 @@ int RugnuxInputRun::PrepareRugnuxInput() {
std::ofstream(path) << text;
};
config.export_unmerged = export_unmerged;
config.export_observations = export_observations;
if (stills_window_sigmas_arg)
config.stills_window_sigmas = *stills_window_sigmas_arg;
config.export_unmerged_partials = export_unmerged_partials;
config.model = model;
config.force_mr = !mr_model.empty();
+53 -10
View File
@@ -122,7 +122,7 @@ TEST_CASE("BraggPrediction_11keV", "[portable]") {
REQUIRE(std::abs(r.l) < settings.max_l );
REQUIRE(r.d >= settings.high_res_A);
REQUIRE(r.d == Catch::Approx(1/std::sqrt(recip * recip)).margin(0.01f));
REQUIRE(r.dist_ewald == Catch::Approx(std::abs(geom.DistFromEwaldSphere(recip))).epsilon(1e-4));
REQUIRE(r.dist_ewald == Catch::Approx(geom.DistFromEwaldSphere(recip)).epsilon(1e-4));
auto [x,y] = geom.RecipToDetector(recip);
REQUIRE(r.predicted_x == Catch::Approx(x).margin(0.01));
REQUIRE(r.predicted_y == Catch::Approx(y).margin(0.01));
@@ -158,7 +158,7 @@ TEST_CASE("BraggPrediction_15keV") {
REQUIRE(std::abs(r.l) < settings.max_l );
REQUIRE(r.d >= settings.high_res_A);
REQUIRE(r.d == Catch::Approx(1/std::sqrt(recip * recip)).margin(0.01f));
REQUIRE(r.dist_ewald == Catch::Approx(std::abs(geom.DistFromEwaldSphere(recip))).epsilon(1e-3));
REQUIRE(r.dist_ewald == Catch::Approx(geom.DistFromEwaldSphere(recip)).epsilon(1e-3));
auto [x,y] = geom.RecipToDetector(recip);
REQUIRE(r.predicted_x == Catch::Approx(x).margin(0.01));
REQUIRE(r.predicted_y == Catch::Approx(y).margin(0.01));
@@ -195,7 +195,7 @@ TEST_CASE("BraggPrediction_Rot1_Rot2", "[portable]") {
REQUIRE(std::abs(r.l) < settings.max_l );
REQUIRE(r.d >= settings.high_res_A);
REQUIRE(r.d == Catch::Approx(1/std::sqrt(recip * recip)).margin(0.01f));
REQUIRE(r.dist_ewald == Catch::Approx(std::abs(geom.DistFromEwaldSphere(recip))).epsilon(1e-4));
REQUIRE(r.dist_ewald == Catch::Approx(geom.DistFromEwaldSphere(recip)).epsilon(1e-4));
auto [x,y] = geom.RecipToDetector(recip);
REQUIRE(r.predicted_x == Catch::Approx(x).margin(0.01));
REQUIRE(r.predicted_y == Catch::Approx(y).margin(0.01));
@@ -419,15 +419,15 @@ TEST_CASE("BraggPrediction_WindowFollowsResolution", "[portable]") {
std::set<std::tuple<int, int, int>> expected, got;
for (int i = 0; i < n_wide; ++i) {
const auto &r = pred_wide.GetReflections()[i];
if (r.dist_ewald <= window(r))
if (std::fabs(r.dist_ewald) <= window(r))
expected.insert({r.h, r.k, r.l});
}
int beyond_cutoff = 0, low_res = 0;
for (int i = 0; i < n_curve; ++i) {
const auto &r = pred_curve.GetReflections()[i];
got.insert({r.h, r.k, r.l});
CHECK(r.dist_ewald <= window(r));
if (r.dist_ewald > curve.ewald_dist_cutoff) ++beyond_cutoff;
CHECK(std::fabs(r.dist_ewald) <= window(r));
if (std::fabs(r.dist_ewald) > curve.ewald_dist_cutoff) ++beyond_cutoff;
if (r.d > 6.0f) ++low_res;
}
CHECK(got == expected);
@@ -451,6 +451,49 @@ TEST_CASE("BraggPrediction_WindowFollowsResolution", "[portable]") {
CHECK(got_scaled != got);
}
TEST_CASE("BraggPrediction_WindowRadialPlacement", "[portable]") {
// With window_radial a reflection off the Ewald sphere is placed as by the wavenumber change
// dk / k = window_radial |q_z| eps / W^2, capped where it puts the node on the sphere: an outside node
// (eps > 0) moves toward the beam, an inside one away, by the amount the formula says.
DiffractionExperiment experiment(DetJF4M());
experiment.DetectorDistance_mm(100.0).BeamX_pxl(1500.0).BeamY_pxl(1000.0).IncidentEnergy_keV(12.0);
const CrystalLattice lattice(Coord{30, 10, 0}, Coord{-15, 45, 0}, Coord{0, 0, 150});
BraggPredictionSettings plain{.high_res_A = 1.8, .max_h = 40, .max_k = 40, .max_l = 90};
plain.window_c0 = 1e-4f;
plain.window_ang = 0.06f * static_cast<float>(PI) / 180.0f;
BraggPredictionSettings moved = plain;
moved.window_radial = 4.0f * 2e-3f * 2e-3f;
BraggPrediction pred_plain, pred_moved;
const int n = pred_plain.Calc(experiment, lattice, plain);
REQUIRE(n > 100);
REQUIRE(pred_moved.Calc(experiment, lattice, moved) == n);
const auto geom = experiment.GetDiffractionGeometry();
const Coord S0 = geom.GetScatteringVector();
int inward = 0, outward = 0;
for (int i = 0; i < n; ++i) {
const auto &a = pred_plain.GetReflections()[i], &b = pred_moved.GetReflections()[i];
REQUIRE(a.h == b.h);
REQUIRE(a.k == b.k);
REQUIRE(a.l == b.l);
CHECK(b.dist_ewald == a.dist_ewald);
const auto q = a.h * lattice.Astar() + a.k * lattice.Bstar() + a.l * lattice.Cstar();
const float W2 = plain.window_c0 * plain.window_c0 + plain.window_ang * plain.window_ang * (q * q);
const float az = std::fabs(q.z);
const float f = a.dist_ewald * std::fmin(moved.window_radial * az / W2, 1.0f / az);
const auto [ex, ey] = geom.RecipToDetector(q + S0 * f);
CHECK(b.predicted_x == Catch::Approx(ex).margin(1e-3));
CHECK(b.predicted_y == Catch::Approx(ey).margin(1e-3));
const float ra = std::hypot(a.predicted_x - 1500.0f, a.predicted_y - 1000.0f);
const float rb = std::hypot(b.predicted_x - 1500.0f, b.predicted_y - 1000.0f);
if (a.dist_ewald > 0 && rb < ra) ++inward;
if (a.dist_ewald < 0 && rb > ra) ++outward;
}
CHECK(inward > 10);
CHECK(outward > 10);
}
TEST_CASE("BraggPrediction_GrowsPastStartingCapacity", "[portable]") {
// A frame predicting more than the buffer starts with must come back whole - the same reflections a
// buffer large enough from the outset returns - and not as whichever the h/k/l walk reached first.
@@ -492,7 +535,7 @@ TEST_CASE("BraggPrediction_GrowsPastStartingCapacity", "[portable]") {
const auto &r = grown->GetReflections()[i];
kept.insert({r.h, r.k, r.l, r.delta_phi_deg});
worst_partiality = std::min(worst_partiality, r.partiality);
worst_dist_ewald = std::max(worst_dist_ewald, r.dist_ewald);
worst_dist_ewald = std::max(worst_dist_ewald, std::fabs(r.dist_ewald));
}
for (const auto &r : all) {
if (kept.contains({r.h, r.k, r.l, r.delta_phi_deg}))
@@ -500,7 +543,7 @@ TEST_CASE("BraggPrediction_GrowsPastStartingCapacity", "[portable]") {
if (rotation)
REQUIRE(r.partiality <= worst_partiality);
else
REQUIRE(r.dist_ewald >= worst_dist_ewald);
REQUIRE(std::fabs(r.dist_ewald) >= worst_dist_ewald);
}
}
}
@@ -628,7 +671,7 @@ TEST_CASE("BraggPredictionGPU") {
REQUIRE(std::abs(r.l) < settings.max_l );
REQUIRE(r.d >= settings.high_res_A);
REQUIRE(r.d == Catch::Approx(1/std::sqrt(recip * recip)).margin(0.01f));
REQUIRE(r.dist_ewald == Catch::Approx(std::abs(geom.DistFromEwaldSphere(recip))).epsilon(2e-2));
REQUIRE(r.dist_ewald == Catch::Approx(geom.DistFromEwaldSphere(recip)).epsilon(2e-2));
auto [x,y] = geom.RecipToDetector(recip);
REQUIRE(r.predicted_x == Catch::Approx(x).margin(0.05));
REQUIRE(r.predicted_y == Catch::Approx(y).margin(0.05));
@@ -667,7 +710,7 @@ TEST_CASE("BraggPredictionGPU_Rot1_Rot2") {
REQUIRE(std::abs(r.l) < settings.max_l );
REQUIRE(r.d >= settings.high_res_A);
REQUIRE(r.d == Catch::Approx(1/std::sqrt(recip * recip)).margin(0.01f));
REQUIRE(r.dist_ewald == Catch::Approx(std::abs(geom.DistFromEwaldSphere(recip))).epsilon(1e-3));
REQUIRE(r.dist_ewald == Catch::Approx(geom.DistFromEwaldSphere(recip)).epsilon(1e-3));
auto [x,y] = geom.RecipToDetector(recip);
REQUIRE(r.predicted_x == Catch::Approx(x).margin(0.05));
REQUIRE(r.predicted_y == Catch::Approx(y).margin(0.05));
+28
View File
@@ -9,6 +9,7 @@
#include "../common/DetectorModules.h"
#include "../common/DiffractionExperiment.h"
#include "../common/JFJochMath.h"
#include "../common/PixelMask.h"
#include "../reader/NXmxModules.h"
#include "../writer/NXmxModules.h"
@@ -110,3 +111,30 @@ TEST_CASE("DetectorModules_NXmxRoundTrip", "[DetectorModules]") {
CHECK(b.sy == Catch::Approx(a.sy).margin(1e-6));
}
}
// A JUNGFRAU file that marks only the gaps between its modules in the mask: the modules come back from
// the mask, one per gap-bounded region; a region that is not one whole module gives none at all.
TEST_CASE("DetectorModules_FromGapMask", "[DetectorModules]") {
const DetectorGeometryModular geom(4, 2, 8, 36, false);
const DetectorModules expected = ModulesAsImaged(geom);
const int64_t width = geom.GetWidth(true), height = geom.GetHeight(true);
std::vector<uint32_t> mask(width * height, 1u << PixelMask::ModuleGapPixelBit);
expected.ForEachPixel(width, height, [&](int64_t, size_t i, double, double) { mask[i] = 0; });
mask[5] |= 1u << PixelMask::ErrorPixelBit; // other bits do not matter
const DetectorModules modules = ModulesFromGapMask(mask, width, height);
REQUIRE(modules.size() == expected.size());
for (size_t i = 0; i < modules.size(); i++) {
CHECK(modules[i].x0 == expected[i].x0);
CHECK(modules[i].y0 == expected[i].y0);
CHECK(modules[i].width == expected[i].width);
CHECK(modules[i].height == expected[i].height);
}
// A gap pixel inside a module: no longer whole modules.
auto holed = mask;
holed[100 * width + 100] |= 1u << PixelMask::ModuleGapPixelBit;
CHECK(ModulesFromGapMask(holed, width, height).empty());
// No gaps at all: one region the size of the image, not a module.
CHECK(ModulesFromGapMask(std::vector<uint32_t>(width * height, 0), width, height).empty());
}
+8 -3
View File
@@ -31,7 +31,8 @@ namespace {
// but fewer than the one-reflection bound at that zeta. Only the first, third and fourth are masked - and
// a 3x3 patch reading high on every frame, which is stationary in the lab and so no reflection either.
// Three dark pixels besides: one dead on the powder ring, which is masked, one on the ring reading 70%
// of it, and one dead on the 5-count background, whose deficit is too small to make an outlier.
// of it, and one dead on the 5-count background, whose deficit is too small to make an outlier. And an
// unstable pixel: normal, but at -2000 on every fourth frame, which no photons do - masked as unstable.
TEST_CASE("HotPixelFinder_PersistentPixelNotBragg", "[HotPixelFinder]") {
DiffractionExperiment x(DetDECTRIS(W, H, "Test detector", ""));
x.IncidentEnergy_keV(WVL_1A_IN_KEV).DetectorDistance_mm(10.0f);
@@ -42,7 +43,7 @@ TEST_CASE("HotPixelFinder_PersistentPixelNotBragg", "[HotPixelFinder]") {
constexpr int HOT_X = 200, HOT_Y = 200, WARM_X = 190, WARM_Y = 60, ERR_X = 60, ERR_Y = 190;
constexpr int BRAGG_X = 40, BRAGG_Y = 115, STUCK_X = 100, STUCK_Y = 30;
constexpr int PATCH_X = 150, PATCH_Y = 225;
constexpr int PATCH_X = 150, PATCH_Y = 225, UNSTABLE_X = 30, UNSTABLE_Y = 150;
constexpr int DEAD_X = C + 82, DEAD_Y = C, DIM_X = C, DIM_Y = C + 82, DEAD_BKG_X = 20, DEAD_BKG_Y = 20;
std::mt19937 rng(1);
// The powder ring is a smooth radial profile, as a real one is: 45 counts over the background at
@@ -67,12 +68,14 @@ TEST_CASE("HotPixelFinder_PersistentPixelNotBragg", "[HotPixelFinder]") {
frame[I(DEAD_X, DEAD_Y)] = 0;
frame[I(DIM_X, DIM_Y)] = static_cast<int32_t>(0.7 * frame[I(DIM_X, DIM_Y)]);
frame[I(DEAD_BKG_X, DEAD_BKG_Y)] = 0;
if (f % 4 == 0)
frame[I(UNSTABLE_X, UNSTABLE_Y)] = -2000;
finder.AddImage(frame.data(), scratch);
}
const auto result = finder.GetMask(OSC_DEG, SPACING_DEG, 4);
CHECK(result.frames == NFRAMES);
CHECK(result.mask[I(HOT_X, HOT_Y)] == 1); // 1 = hot, 2 = error value, 3 = dead
CHECK(result.mask[I(HOT_X, HOT_Y)] == 1); // 1 = hot, 2 = error value, 3 = dead, 4 = unstable
CHECK(result.mask[I(ERR_X, ERR_Y)] == 2);
CHECK(result.mask[I(STUCK_X, STUCK_Y)] == 2);
CHECK(result.mask[I(WARM_X, WARM_Y)] == 0);
@@ -86,6 +89,8 @@ TEST_CASE("HotPixelFinder_PersistentPixelNotBragg", "[HotPixelFinder]") {
CHECK(result.hot == 10);
CHECK(result.error == 2);
CHECK(result.dead == 1);
CHECK(result.mask[I(UNSTABLE_X, UNSTABLE_Y)] == 4);
CHECK(result.unstable == 1);
}
#ifdef JFJOCH_USE_CUDA
+87 -22
View File
@@ -2527,18 +2527,22 @@ static Reflection MakeTestReflection(int i, int j) {
.I = 1000.0f + f,
.bkg = 10.0f + f,
.sigma = 2.0f + 0.5f * f,
.dist_ewald = -1.5e-3f + 1e-4f * f,
.prescaling_corr = 1.0f + 0.125f * f,
.qe_corr = 1.0f / (1.0f + 0.01f * f), // always in (0, 1], as the correction itself is
.flight_corr = 1.0f + 0.02f * f, // always >= 1, as the correction itself is
.partiality = 0.5f + 0.01f * f,
.zeta = 0.01f + 0.001f * f,
.image_scale_corr = 1.0f + 0.25f * f
.image_scale_corr = 1.0f + 0.25f * f,
.observed = j == 0,
.clipped = j == 1,
.overloaded = i == 3
};
}
// Every field of the round trip, against the reflection that was written. Deliberately NOT checked,
// because they are not part of it: dist_ewald and observed are prediction/integration scratch that
// is never written, and on_ice_ring is recomputed from the resolution by whoever scales (see the
// Every field of the round trip, against the reflection that was written, bit for bit: --mode scale merges
// what is read back, and reproduces the merge of the run that wrote it only if every field comes back as it
// was. Deliberately NOT checked: on_ice_ring, recomputed from the resolution by whoever scales (see the
// ice-ring handling in Rugnux and in the --scale path).
static void CheckReflectionRoundTrip(const Reflection &got, int i, int j) {
INFO("image " << i << " reflection " << j);
@@ -2546,22 +2550,26 @@ static void CheckReflectionRoundTrip(const Reflection &got, int i, int j) {
CHECK(got.h == want.h);
CHECK(got.k == want.k);
CHECK(got.l == want.l);
CHECK(got.image_number == Catch::Approx(want.image_number));
CHECK(got.delta_phi_deg == Catch::Approx(want.delta_phi_deg));
CHECK(got.predicted_x == Catch::Approx(want.predicted_x));
CHECK(got.predicted_y == Catch::Approx(want.predicted_y));
CHECK(got.observed_x == Catch::Approx(want.observed_x));
CHECK(got.observed_y == Catch::Approx(want.observed_y));
CHECK(got.d == Catch::Approx(want.d));
CHECK(got.I == Catch::Approx(want.I));
CHECK(got.bkg == Catch::Approx(want.bkg));
CHECK(got.sigma == Catch::Approx(want.sigma));
CHECK(got.prescaling_corr == Catch::Approx(want.prescaling_corr)); // stored as 1/prescaling_corr, inverted again on read
CHECK(got.qe_corr == Catch::Approx(want.qe_corr)); // likewise stored as its reciprocal
CHECK(got.flight_corr == Catch::Approx(want.flight_corr)); // likewise
CHECK(got.partiality == Catch::Approx(want.partiality));
CHECK(got.zeta == Catch::Approx(want.zeta));
CHECK(got.image_scale_corr == Catch::Approx(want.image_scale_corr));
CHECK(got.image_number == want.image_number);
CHECK(got.delta_phi_deg == want.delta_phi_deg);
CHECK(got.predicted_x == want.predicted_x);
CHECK(got.predicted_y == want.predicted_y);
CHECK(got.observed_x == want.observed_x);
CHECK(got.observed_y == want.observed_y);
CHECK(got.d == want.d);
CHECK(got.I == want.I);
CHECK(got.bkg == want.bkg);
CHECK(got.sigma == want.sigma);
CHECK(got.dist_ewald == want.dist_ewald);
CHECK(got.prescaling_corr == want.prescaling_corr); // stored as 1/prescaling_corr, inverted again on read
CHECK(got.qe_corr == want.qe_corr); // likewise stored as its reciprocal
CHECK(got.flight_corr == want.flight_corr); // likewise
CHECK(got.partiality == want.partiality);
CHECK(got.zeta == want.zeta);
CHECK(got.image_scale_corr == want.image_scale_corr);
CHECK(got.observed == want.observed);
CHECK(got.clipped == want.clipped);
CHECK(got.overloaded == want.overloaded);
}
TEST_CASE("JFJochReader_ReadReflections_VDS", "[HDF5][Full]") {
@@ -2806,8 +2814,11 @@ TEST_CASE("JFJochReader_StillsRunWidth", "[HDF5][Full]") {
REQUIRE_NOTHROW(writer.WriteHDF5(message));
EndMessage end_message;
end_message.max_image_number = 1;
if (with_width)
if (with_width) {
end_message.stills_run_width = w;
end_message.stills_given_cell = UnitCell{.a = 79.1f, .b = 79.1f, .c = 38.1f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
@@ -2823,7 +2834,10 @@ TEST_CASE("JFJochReader_StillsRunWidth", "[HDF5][Full]") {
CHECK(got->radial_scale == w.radial_scale);
CHECK(got->table_dstar == w.table_dstar);
CHECK(got->table_s == w.table_s);
}
REQUIRE(reader.GetDataset()->stills_given_cell.has_value());
CHECK(reader.GetDataset()->stills_given_cell->c == 38.1f);
} else
CHECK_FALSE(reader.GetDataset()->stills_given_cell.has_value());
}
remove("read_stills_run_width_master.h5");
remove("read_stills_run_width_data_000001.h5");
@@ -5168,3 +5182,54 @@ TEST_CASE("JFJochReader_PanelId", "[HDF5][PanelId]") {
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// A JUNGFRAU image assembled by the experiment is written with its modules one by one (module_0, ...),
// and the reader gives them back where the assembly put them; a DECTRIS detector is written without.
TEST_CASE("JFJochReader_JungfrauModules", "[HDF5][Full]") {
RegisterHDF5Filter();
for (const bool jungfrau: {true, false}) {
DiffractionExperiment x(jungfrau ? DetJF(4) : DetDECTRIS(2068, 1064, "Detector", {}));
x.FilePrefix("test_modules").ImagesPerTrigger(1).OverwriteExistingFiles(true);
x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150)
.IncidentEnergy_keV(WVL_1A_IN_KEV).PixelSigned(false).BitDepthImage(16);
if (jungfrau)
x.FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10));
const DetectorModules expected = jungfrau ? ModulesAsImaged(x.GetDetectorSetup().GetGeometry())
: DetectorModules();
CHECK(expected.size() == (jungfrau ? 4 : 0));
ScanResultGenerator generator(x);
std::vector<uint16_t> image(x.GetPixelsNum(), 0);
{
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = 0;
REQUIRE_NOTHROW(file_set.WriteHDF5(message));
REQUIRE_NOTHROW(generator.Add(message));
EndMessage end_message;
end_message.max_image_number = 1;
generator.FillEndMessage(end_message);
file_set.WriteHDF5(end_message);
file_set.Finalize();
}
{
JFJochHDF5Reader reader;
reader.ReadFile("test_modules_master.h5");
const DetectorModules &back = reader.GetDataset()->experiment.GetModules();
REQUIRE(back.size() == expected.size());
for (size_t i = 0; i < back.size(); i++) {
CHECK(back[i].x0 == expected[i].x0);
CHECK(back[i].y0 == expected[i].y0);
CHECK(back[i].width == expected[i].width);
CHECK(back[i].height == expected[i].height);
CHECK(back[i].ox == Catch::Approx(expected[i].ox).margin(1e-3));
CHECK(back[i].oy == Catch::Approx(expected[i].oy).margin(1e-3));
}
CHECK_FALSE(back.Moved());
}
remove("test_modules_master.h5");
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
+252 -1
View File
@@ -527,6 +527,54 @@ TEST_CASE("ErrorModel_SameOnAnyNumberOfThreads") {
CHECK(one.b2 == many.b2);
}
// The stills model: samples drawn with a known a, b and partiality term c - c^2 ((1 - p) / p)^2 <I>^2, p spread
// over [0.1, 1] - give those three back.
TEST_CASE("StillsErrorModel_RecoversThePartialityTerm") {
std::mt19937 rng(7);
std::uniform_real_distribution<double> u(0.0, 1.0);
std::normal_distribution<double> z(0.0, 1.0);
const double a = 1.2, b = 0.08, c = 0.15;
std::vector<StillsErrorModelSample> pool;
for (int i = 0; i < 200000; ++i) {
const double s2 = std::pow(10.0, 2.0 * u(rng));
const double snr = 300.0 * std::pow(10.0, -4.0 * u(rng));
const double I2 = snr * snr * s2;
const double p = 0.1 + 0.9 * u(rng);
const double P2 = std::pow((1.0 - p) / p, 2.0) * I2;
const double e = z(rng);
pool.push_back({s2, I2, P2, (a * s2 + b * b * I2 + c * c * P2) * e * e});
}
const auto fit = FitStillsErrorModel(pool);
REQUIRE(fit.active);
const auto many = FitStillsErrorModel(pool, 16); // the same numbers on any number of threads
CHECK(many.a == fit.a);
CHECK(many.b2 == fit.b2);
CHECK(many.c2 == fit.c2);
CHECK(fit.a == Catch::Approx(a).epsilon(0.05));
CHECK(std::sqrt(fit.b2) == Catch::Approx(b).epsilon(0.05));
CHECK(std::sqrt(fit.c2) == Catch::Approx(c).epsilon(0.05));
}
// Without a partiality error in the data the term stays at (near) zero.
TEST_CASE("StillsErrorModel_NoPartialityErrorNoTerm") {
std::mt19937 rng(8);
std::uniform_real_distribution<double> u(0.0, 1.0);
std::normal_distribution<double> z(0.0, 1.0);
std::vector<StillsErrorModelSample> pool;
for (int i = 0; i < 200000; ++i) {
const double s2 = std::pow(10.0, 2.0 * u(rng));
const double snr = 300.0 * std::pow(10.0, -4.0 * u(rng));
const double I2 = snr * snr * s2;
const double p = 0.1 + 0.9 * u(rng);
const double e = z(rng);
pool.push_back({s2, I2, std::pow((1.0 - p) / p, 2.0) * I2, (1.1 * s2 + 0.0025 * I2) * e * e});
}
const auto fit = FitStillsErrorModel(pool);
REQUIRE(fit.active);
CHECK(std::sqrt(fit.c2) < 0.01);
CHECK(std::sqrt(fit.b2) == Catch::Approx(0.05).epsilon(0.05));
}
// Data that never reach where b could be seen: a is fitted alone and b held at 0.
TEST_CASE("ErrorModel_BNotMeasuredOnWeakData") {
const auto pool = SyntheticErrorModelSamples(0.9, 0.05, 1.5, 0.0);
@@ -721,7 +769,147 @@ TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") {
const auto parallel_stats = parallel.MergeStats(parallel_merged, outcomes);
CHECK(serial_stats.overall.total_observations == parallel_stats.overall.total_observations);
CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half);
CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas);
CHECK(serial_stats.overall.r_split == parallel_stats.overall.r_split);
// R_merge / R_meas / R_pim are rotation statistics: the stills merge does not form them.
CHECK(std::isnan(serial_stats.overall.r_meas));
CHECK(std::isnan(serial_stats.overall.r_merge));
CHECK(std::isnan(serial_stats.overall.r_pim));
}
// With the error model fitted, the merge takes every observation, however small its partiality, but reports a
// reflection only if one of its observations reaches the existence partiality (0.2).
TEST_CASE("MergeOnTheFly_EveryObservationMergedReflectionReportedOnlyIfOneReachesTheExistencePartiality") {
DiffractionExperiment x;
x.SetSpaceGroup(TestSpaceGroup());
ScalingSettings s = x.GetScalingSettings();
x.ImportScalingSettings(s.MinPartiality(0.5)); // the stills default
std::mt19937 rng(7);
std::uniform_int_distribution<int> index(-6, 6);
std::uniform_real_distribution<float> unit(0.0f, 1.0f);
auto observation = [](int h, int k, int l, float I, float sigma, float p) {
Reflection r{};
r.h = h;
r.k = k;
r.l = l;
r.d = 20.0f / std::sqrt(static_cast<float>(h * h + k * k + l * l));
r.I = I;
r.sigma = sigma;
r.var_bkg = 0.5f * sigma * sigma;
r.prescaling_corr = 1.0f;
r.partiality = p;
r.image_scale_corr = 1.0f;
return r;
};
// Enough ordinary observations for the error model.
std::vector<IntegrationOutcome> outcomes(5000);
for (auto &o: outcomes)
for (int j = 0; j < 20; ++j) {
const int h = index(rng), k = index(rng), l = index(rng);
if (h != 0 || k != 0 || l != 0)
o.reflections.push_back(observation(h, k, l, 100.0f * unit(rng) - 5.0f, 1.0f + 5.0f * unit(rng),
0.5f + 0.5f * unit(rng)));
}
// (8,1,1): seen only in the tails. (9,1,1): one observation at p = 0.3 and three at p = 0.1.
// (9,2,1): the same p = 0.3 observation alone.
for (int i = 0; i < 4; ++i)
outcomes[i].reflections.push_back(observation(8, 1, 1, 50.0f, 2.0f, 0.1f));
outcomes[10].reflections.push_back(observation(9, 1, 1, 50.0f, 2.0f, 0.3f));
for (int i = 11; i < 14; ++i)
outcomes[i].reflections.push_back(observation(9, 1, 1, 50.0f, 2.0f, 0.1f));
outcomes[20].reflections.push_back(observation(9, 2, 1, 50.0f, 2.0f, 0.3f));
MergeOnTheFly merge(x);
merge.RefineErrorModel(outcomes);
REQUIRE(merge.ErrorModelActive());
merge.AddImages(outcomes);
const auto merged = merge.ExportReflections();
auto find = [&](int h, int k, int l) -> const MergedReflection * {
for (const auto &m: merged)
if (m.h == h && m.k == k && m.l == l)
return &m;
return nullptr;
};
CHECK(find(8, 1, 1) == nullptr);
const auto *with_tails = find(9, 1, 1);
const auto *alone = find(9, 2, 1);
REQUIRE(with_tails != nullptr);
REQUIRE(alone != nullptr);
CHECK(with_tails->sigma < alone->sigma); // the p = 0.1 observations are in its merge
// None of the tail observations is flagged as below the merge's floor.
const auto verdicts = merge.Verdicts(outcomes);
CHECK((verdicts[0].back().flags & OBS_LOW_PARTIALITY) == 0);
CHECK((verdicts[11].back().flags & OBS_LOW_PARTIALITY) == 0);
}
// The existence rule holds within each half-set too: a half that has only tail observations of a reflection has no
// mean of it. And observations far out on the rocking curve (p = 1e-30, a correction no float variance holds)
// leave every merged value and statistic finite.
TEST_CASE("MergeOnTheFly_HalfSetWithOnlyTailObservationsHasNoMean") {
DiffractionExperiment x;
x.SetSpaceGroup(TestSpaceGroup());
ScalingSettings s = x.GetScalingSettings();
x.ImportScalingSettings(s.MinPartiality(0.5));
std::mt19937 rng(7);
std::uniform_int_distribution<int> index(-6, 6);
std::uniform_real_distribution<float> unit(0.0f, 1.0f);
auto observation = [](int h, int k, int l, float I, float sigma, float p, float corr) {
Reflection r{};
r.h = h;
r.k = k;
r.l = l;
r.d = 20.0f / std::sqrt(static_cast<float>(h * h + k * k + l * l));
r.I = I;
r.sigma = sigma;
r.var_bkg = 0.5f * sigma * sigma;
r.prescaling_corr = 1.0f;
r.partiality = p;
r.image_scale_corr = corr;
return r;
};
std::vector<IntegrationOutcome> outcomes(5000);
for (auto &o: outcomes)
for (int j = 0; j < 20; ++j) {
const int h = index(rng), k = index(rng), l = index(rng);
if (h != 0 || k != 0 || l != 0)
o.reflections.push_back(observation(h, k, l, 100.0f * unit(rng) - 5.0f, 1.0f + 5.0f * unit(rng),
0.5f + 0.5f * unit(rng), 1.0f));
}
// Far-tail observations of the ordinary reflections, with the corrections such a partiality brings.
for (size_t i = 0; i < outcomes.size(); i += 5)
outcomes[i].reflections.push_back(observation(1, 2, 3, 1e4f, 30.0f, 1e-30f, 1e36f));
// (9,1,1): one observation at p = 0.3 on image 10, then tail observations on twenty more images - the
// other half-set, and the other frame-whole half-set, have only tail observations.
outcomes[10].reflections.push_back(observation(9, 1, 1, 50.0f, 2.0f, 0.3f, 1.0f));
for (int i = 11; i < 31; ++i)
outcomes[i].reflections.push_back(observation(9, 1, 1, 0.1f, 2.0f, 0.01f, 100.0f));
MergeOnTheFly merge(x);
merge.RefineErrorModel(outcomes);
REQUIRE(merge.ErrorModelActive());
merge.AddImages(outcomes);
CHECK(merge.FarTailCount() == 1000);
const auto merged = merge.ExportReflections();
const MergedReflection *m = nullptr;
for (const auto &r: merged) {
if (r.h == 9 && r.k == 1 && r.l == 1)
m = &r;
CHECK(std::isfinite(r.I));
CHECK(std::isfinite(r.sigma));
CHECK(r.sigma > 0.0f);
}
REQUIRE(m != nullptr);
CHECK(!std::isfinite(m->I_half[0]));
CHECK(!std::isfinite(m->I_half[1]));
CHECK(std::isfinite(m->I_split[0]) != std::isfinite(m->I_split[1]));
const auto stats = merge.MergeStats(merged, outcomes);
CHECK(std::isfinite(stats.overall.cc_half));
CHECK(std::isfinite(stats.overall.cc_half_split));
CHECK(std::isfinite(stats.overall.r_split));
}
// Stills partiality post-refinement on noiseless synthetic crystals: each crystal's true orientation is
@@ -1303,3 +1491,66 @@ TEST_CASE("ScalingSettings_MinPartialityRange") {
CHECK_NOTHROW(s.MinPartiality(0.5));
CHECK(s.GetMinPartiality() == 0.5);
}
// An image with too few reflections to fit its scale on is left out of the merge, not merged at the scale
// of 1 it was predicted with.
TEST_CASE("ScaleOnTheFly_TooFewReflectionsLeavesTheImageOut") {
DiffractionExperiment x;
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
std::vector<IntegrationOutcome> outcomes(3);
for (size_t i = 0; i < outcomes.size(); ++i) {
const int n = i == 2 ? 10 : 50;
for (int h = 1; h <= n; ++h) {
Reflection r{};
r.h = h; r.k = 1; r.l = 1;
r.d = 3.0f;
r.I = 100.0f * static_cast<float>(h) * (i == 1 ? 2.0f : 1.0f);
r.sigma = 1.0f;
r.var_bkg = 0.5f;
r.partiality = 1.0f;
r.prescaling_corr = 1.0f;
r.image_scale_corr = 1.0f;
outcomes[i].reflections.push_back(r);
}
}
ScaleOnTheFly(x, MergeAll(x, outcomes, 1, std::nullopt)).Scale(outcomes, 1);
CHECK(outcomes[0].image_scale_g.has_value());
CHECK(std::isfinite(outcomes[1].reflections[0].image_scale_corr));
CHECK(!outcomes[2].image_scale_g.has_value());
for (const auto &r: outcomes[2].reflections)
CHECK(std::isnan(r.image_scale_corr));
}
// Stills with the Friedel hands kept apart report CC_anom (between frame-whole half-sets) and SigAno: a
// consistent anomalous difference far above the noise correlates close to 1.
TEST_CASE("MergeOnTheFly_StillsAnomalousStatistics") {
DiffractionExperiment x;
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
ScalingSettings s = x.GetScalingSettings();
s.MergeFriedel(false);
x.ImportScalingSettings(s);
std::mt19937 rng(11);
std::normal_distribution<float> gauss(0.0f, 1.0f);
std::vector<float> dano(61);
for (auto &v: dano) v = 0.2f * gauss(rng);
std::vector<IntegrationOutcome> outcomes(40);
for (size_t i = 0; i < outcomes.size(); ++i)
for (int h = 1; h <= 60; ++h)
for (int sign: {1, -1}) {
Reflection r{};
r.h = sign * h; r.k = sign * 2; r.l = sign * 3;
r.d = 30.0f / std::sqrt(static_cast<float>(h * h + 13));
r.I = 1000.0f * (1.0f + 0.5f * sign * dano[h]) + 5.0f * gauss(rng);
r.sigma = 5.0f;
r.partiality = 1.0f;
r.prescaling_corr = 1.0f;
r.image_scale_corr = 1.0f;
outcomes[i].reflections.push_back(r);
}
MergeOnTheFly merge(x);
merge.AddImages(outcomes);
const auto merged = merge.ExportReflections();
const auto stats = merge.MergeStats(merged, outcomes);
CHECK(stats.overall.cc_anom > 0.9);
CHECK(stats.overall.abs_diff_over_sigma_anomalous > 5.0);
}
+26
View File
@@ -866,6 +866,32 @@ TEST_CASE("ResultReport_IsaBootstrapInterval", "[Diagnostics]") {
CHECK(lower.find("ISa >= 45.0") != std::string::npos);
}
// A stills merge reports its error model as its three terms and no ISa anywhere.
TEST_CASE("ResultReport_StillsErrorModelTerms", "[Diagnostics]") {
DiffractionExperiment x(DetJF(1));
ProcessResult result;
result.consensus_cell = UnitCell{.a = 79.0f, .b = 79.0f, .c = 38.0f,
.alpha = 90.0f, .beta = 90.0f, .gamma = 90.0f};
result.has_merge_statistics = true;
result.merge_statistics.stills = true;
result.merge_statistics.overall.d_max = 50.0f;
result.merge_statistics.overall.d_min = 1.50f;
result.merge_statistics.overall.cc_half = 0.99;
result.error_model_a = 1.1;
result.error_model_b = 0.1;
result.error_model_c = 0.02;
result.error_model_isa = 3.0;
const auto text = RenderResultReport("p", "in.h5", x, result);
CHECK(text.find("\nERROR_MODEL_B= 1.0000e-01\n") != std::string::npos);
CHECK(text.find("\nERROR_MODEL_C= 2.0000e-02\n") != std::string::npos);
CHECK(text.find("ISA=") == std::string::npos);
CHECK(text.find("ISa 3.0") == std::string::npos);
result.error_model_isa_resolved = false;
CHECK(RenderResultReport("p", "in.h5", x, result).find("\nERROR_MODEL_B= undetermined\n") != std::string::npos);
}
// The completeness of the range where the signal is, beside the completeness of the range that was
// written. Users compare the single overall figure against another program's and conclude the run
// lost data, when it is the deliberately generous cut diluting the denominator - so the report says
+45
View File
@@ -12,6 +12,7 @@
#include "../common/ScanResultGenerator.h"
#include "../writer/FileWriter.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../reader/JFJochMultiReader.h"
#include "../rugnux/Rugnux.h"
#include "../rugnux/RugnuxCommandLine.h"
#include "../rugnux/RugnuxOutputFiles.h"
@@ -94,6 +95,50 @@ TEST_CASE("Rugnux_AzInt", "[HDF5][Full]") {
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("Rugnux_ProcessFileWithoutImages", "[HDF5][Full]") {
// A reader with no single HDF5 file to link to (here several inputs read as one) gets a _process.h5
// with the per-image analysis and no /entry/data/data, which reads back with its images' metadata
// and no pixels.
WriteTestDataset("process_noimg_in", 4);
std::vector<std::unique_ptr<JFJochReader>> inputs;
auto input = std::make_unique<JFJochHDF5Reader>();
REQUIRE_NOTHROW(input->ReadFile("process_noimg_in_master.h5"));
inputs.push_back(std::move(input));
JFJochMultiReader reader(std::move(inputs));
auto dataset = reader.GetDataset();
REQUIRE(dataset);
ProcessConfig config;
config.mode = ProcessMode::AzimuthalIntegration;
config.nthreads = 2;
ProcessFileSink sink("process_noimg_out");
config.image_sink = &sink;
Rugnux process(reader, dataset->experiment, *dataset->pixel_mask, config);
ProcessResult result;
REQUIRE_NOTHROW(result = process.Run());
CHECK(result.images_processed == 4);
{
HDF5ReadOnlyFile file("process_noimg_out_process.h5");
CHECK_FALSE(file.Exists("/entry/data/data"));
}
{
JFJochHDF5Reader out;
REQUIRE_NOTHROW(out.ReadFile("process_noimg_out_process.h5"));
CHECK(out.GetNumberOfImages() == 4);
CHECK(out.LoadImage(0) == nullptr);
CHECK_NOTHROW(out.ReadReflections(0, 3));
}
reader.Close();
remove("process_noimg_in_master.h5");
remove("process_noimg_in_data_000001.h5");
remove("process_noimg_out_process.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("Rugnux_NoOutput", "[HDF5][Full]") {
WriteTestDataset("process_noout_in", 6);
+34
View File
@@ -790,3 +790,37 @@ TEST_CASE("ShadowAccumulatorGPU_FoldedCardsMatchOneCard", "[ShadowFinder]") {
CHECK(count_folded == count_one);
}
#endif
// Beyond the largest whole ring (radius C here) an aperture dims everything past radius 150 to half: every ring
// out there lies wholly in the dimmed corners, so only the step along the ray shows it. The background falls
// off smoothly with radius on its own, and that alone is not a shadow.
TEST_CASE("ShadowFinder_FindsAnOuterShadowAlongTheRays", "[ShadowFinder]") {
const DiffractionExperiment x = TestExperiment();
const PixelMask pixel_mask(x);
for (const bool aperture : {false, true}) {
std::vector<float> mean(static_cast<size_t>(W) * H);
for (int y = 0; y < H; y++)
for (int xx = 0; xx < W; xx++) {
const float r = std::hypot(static_cast<float>(xx - C), static_cast<float>(y - C));
mean[I(xx, y)] = 100.0f * (1.0f - r / 400.0f) * (aperture && r >= 150.0f ? 0.5f : 1.0f);
}
const auto shadow = FindOuterShadow(x, pixel_mask, mean, NFRAMES, 0);
size_t inside = 0, outside = 0, beyond = 0;
for (int y = 0; y < H; y++)
for (int xx = 0; xx < W; xx++) {
const float r = std::hypot(static_cast<float>(xx - C), static_cast<float>(y - C));
if (r >= 151.0f) {
beyond++;
outside += shadow[I(xx, y)] != 0;
} else if (r < 149.0f)
inside += shadow[I(xx, y)] != 0;
}
CHECK(inside == 0);
// A ray direction is a fraction of a pixel wide this close to the beam, and a few are left without an
// edge pixel of their own.
if (aperture)
CHECK(outside > 0.98 * beyond);
else
CHECK(outside == 0);
}
}
+2 -1
View File
@@ -46,7 +46,8 @@ def read_report(path):
else "REFRES_" if "REFRES_RANGE" in out else "")
if first:
out[prefix + "LOWRES_D"] = f[0]
out[prefix + "LOWRES_R_MEAS"] = percent(f[header.index("R_meas")])
if "R_meas" in header: # a stills table has no R_meas column
out[prefix + "LOWRES_R_MEAS"] = percent(f[header.index("R_meas")])
first = False
if not prefix: # every row of the own table; the last is the highest shell
out["HIGHRES_CC_HALF"] = percent(f[header.index("CC1/2")])
+8 -10
View File
@@ -17,10 +17,10 @@ worse". It runs rugnux over the SAME fixed image subset of the same datasets und
more) sets of arguments and tabulates, per dataset and per resolution shell:
indexing rate | integrated observations | merged reflections | completeness |
multiplicity | <I/sigma> | R_meas | CC1/2
multiplicity | <I/sigma> | CC1/2
and the arm-to-arm delta on each. Pooled numbers are printed too, but the per-shell table
is the one that decides: a change that improves the pooled R_meas by moving the resolution
is the one that decides: a change that improves the pooled CC1/2 by moving the resolution
cut-off has not improved anything, and pooled scoring has reversed the verdict on real
changes before.
@@ -180,7 +180,7 @@ def _i(v):
SHELL_COLS = ["d_res_high", "d_res_low", "number_measured_obs", "number_unique_obs",
"pdbx_redundancy", "percent_possible_obs", "meanI_over_sigI_obs",
"pdbx_Rrim_I_all", "pdbx_CC_half"]
"pdbx_CC_half"]
def parse_cif(path):
@@ -199,9 +199,8 @@ def parse_cif(path):
"mult": _f(val("_reflns.pdbx_redundancy")),
"compl": _f(val("_reflns.percent_possible_obs")),
"isig": _f(val("_reflns.pdbx_netI_over_sigmaI")),
"rmeas": _f(val("_reflns.pdbx_Rrim_I_all"), 100.0),
"cc": _f(val("_reflns.pdbx_CC_half"), 100.0),
"isa": _f(val("_reflns.jfjoch_diffrn_ISa")),
"em_b": _f(val("_reflns.jfjoch_error_model_b")), # the stills error model writes no ISa
"dmin": _f(val("_reflns.d_resolution_high")),
"dmax": _f(val("_reflns.d_resolution_low")),
}
@@ -214,7 +213,7 @@ def parse_cif(path):
"d_hi": d_hi, "d_lo": d_lo,
"obs": _i(row[2]), "uniq": _i(row[3]),
"mult": _f(row[4]), "compl": _f(row[5]), "isig": _f(row[6]),
"rmeas": _f(row[7], 100.0), "cc": _f(row[8], 100.0),
"cc": _f(row[7], 100.0),
})
shells.sort(key=lambda s: -s["d_hi"]) # low resolution first
r["shells"] = shells
@@ -533,9 +532,8 @@ SUMMARY_COLS = [
("Mult", "mult", 6, 2),
("Compl%", "compl", 7, 1),
("<I/sig>", "isig", 8, 2),
("Rmeas%", "rmeas", 7, 2),
("CC1/2%", "cc", 7, 2),
("ISa", "isa", 6, 2),
("EM b", "em_b", 7, 4),
]
@@ -556,8 +554,8 @@ def summary_row(label, r, secs):
SHELL_KEYS = [("obs", 9, 0), ("uniq", 7, 0), ("mult", 6, 2), ("compl", 7, 1),
("isig", 8, 2), ("rmeas", 8, 2), ("cc", 7, 2)]
SHELL_NAMES = ["Obs", "Refl", "Mult", "Compl%", "<I/sig>", "Rmeas%", "CC1/2%"]
("isig", 8, 2), ("cc", 7, 2)]
SHELL_NAMES = ["Obs", "Refl", "Mult", "Compl%", "<I/sig>", "CC1/2%"]
def print_shell_table(base_name, base, arm_name, arm, floor=None):
-2
View File
@@ -10,8 +10,6 @@
{"id": "lcls242_t", "cxidb": 242, "facility": "LCLS", "beamline": "MFX", "detector": "Rayonix MX340-HS (4x4 binned, 177 um)", "format": "LCLS-I XTC", "pdb": "9O4T", "sg": "P 1 21 1", "d_cut": 1.95, "r_free_deposited": 0.2348, "images_deposited": null, "inputs": ["lcls_242/xtc/*.xtc", "lcls_242/xtc_more/*.xtc"], "args": ["--wavelength", "1.294", "-C", "96.031,94.536,50.529,90,91.18,90", "-S", "P21"], "tiers": {}},
{"id": "euxfel83", "cxidb": 83, "facility": "European XFEL", "beamline": "SPB/SFX", "detector": "AGIPD 1M (200 um)", "format": "Cheetah CXI", "pdb": "6GTH", "sg": "P 32 2 1", "d_cut": 1.69, "r_free_deposited": 0.2096, "images_deposited": 14445, "inputs": ["euxfel_83/raw/*.cxi"], "args": ["--crystfel-geometry", "{data}/euxfel_83/ref/agipd_mar18_v10.geom", "--wavelength", "1.3332", "-C", "41.84,41.84,233.28,90,90,120", "-S", "P3221"], "tiers": {"smoke": {"images": 2000}}},
{"id": "euxfel152", "cxidb": 152, "facility": "European XFEL", "beamline": "SPB/SFX", "detector": "AGIPD 1M (200 um)", "format": "Cheetah CXI", "pdb": "6U57", "sg": "I 2 3", "d_cut": 2.8, "r_free_deposited": 0.2489, "images_deposited": 37006, "inputs": ["euxfel_152/raw/*.cxi"], "args": ["--crystfel-geometry", "{data}/euxfel_152/ref/kdo_embedded.geom", "--wavelength", "1.3332", "-C", "118,118,118,90,90,90", "-S", "I23"], "tiers": {"smoke": {"images": 2000}}},
{"id": "alvra103", "cxidb": 103, "facility": "SwissFEL", "beamline": "Alvra", "detector": "JUNGFRAU 16M (75 um)", "format": "SwissFEL HDF5, float32 keV", "pdb": "6S0L", "sg": "C 2 2 21", "d_cut": 2.65, "r_free_deposited": 0.2203, "images_deposited": 199136, "inputs": ["swissfel_103/raw/run*.JF06T32V01.h5"], "args": ["--crystfel-geometry", "{data}/swissfel_103/ref/jf-16M_mask.geom", "--wavelength", "2.7130", "-C", "40.34,180.66,143.05,90,90,90", "-S", "C2221"], "tiers": {}},
{"id": "alvra103_int16", "cxidb": 103, "facility": "SwissFEL", "beamline": "Alvra", "detector": "JUNGFRAU 16M (75 um)", "format": "int16 photons, bitshuffle/LZ4 copy", "pdb": "6S0L", "sg": "C 2 2 21", "d_cut": 2.65, "r_free_deposited": 0.2203, "images_deposited": 199136, "inputs": ["swissfel_103/conv/run*_int16.h5"], "args": ["--crystfel-geometry", "{data}/swissfel_103/ref/jf-16M_mask.geom", "--wavelength", "2.7130", "-C", "40.34,180.66,143.05,90,90,90", "-S", "C2221"], "tiers": {}},
{"id": "alvra213", "cxidb": 213, "facility": "SwissFEL", "beamline": "Alvra", "detector": "JUNGFRAU 4M (75 um)", "format": "SwissFEL HDF5, float32 keV", "pdb": "8BKH", "sg": "P 1 21 1", "d_cut": 1.35, "r_free_deposited": 0.2271, "images_deposited": null, "inputs": ["swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5"], "args": ["--crystfel-geometry", "{data}/swissfel_213/ref/mbco_march_mask.geom", "--wavelength", "1.0300", "-C", "64.1,28.8,35.8,90,106.9,90", "-S", "P21"], "tiers": {}},
{"id": "alvra213_int16", "cxidb": 213, "facility": "SwissFEL", "beamline": "Alvra", "detector": "JUNGFRAU 4M (75 um)", "format": "int16 photons, bitshuffle/LZ4 copy", "pdb": "8BKH", "sg": "P 1 21 1", "d_cut": 1.35, "r_free_deposited": 0.2271, "images_deposited": null, "inputs": ["swissfel_213/conv/march_dark_MbCO_II-March-06/data/dark/run_*_int16.h5"], "args": ["--crystfel-geometry", "{data}/swissfel_213/ref/mbco_march_mask.geom", "--wavelength", "1.0300", "-C", "64.1,28.8,35.8,90,106.9,90", "-S", "P21"], "tiers": {}}
]}
+15 -9
View File
@@ -34,10 +34,11 @@
namespace {
// Per-shell metric value (NaN -> dropped by the chart). Keep in sync with the combo entries.
double ShellMetric(const MergeStatisticsShell &s, int metric) {
// A stills merge has no R-meas; R-split, the serial R, takes its place.
double ShellMetric(const MergeStatisticsShell &s, int metric, bool stills) {
switch (metric) {
case 0: return s.cc_half * 100.0; // CC1/2 (%)
case 1: return s.r_meas * 100.0; // R-meas (%)
case 1: return (stills ? s.r_split : s.r_meas) * 100.0; // R-meas / R-split (%)
case 2: return s.mean_i_over_sigma; // <I/sigma>
case 3: return s.possible_unique_reflections > 0 // Completeness (%)
? 100.0 * s.unique_reflections / s.possible_unique_reflections : NAN;
@@ -149,9 +150,13 @@ JFJochMergeStatsWindow::JFJochMergeStatsWindow(const QString &title, const Merge
// by the CC1/2 logistic cutoff, so it varies per run - show it first.
if (o.d_min < o.d_max)
hero->addWidget(MakeCard(QString::asprintf("%.2f", o.d_min), "Resolution [Å]", this));
hero->addWidget(MakeCard(num(isa, 1), "ISa", this));
if (!std::isnan(isa)) // NaN: a stills merge, which reports no ISa
hero->addWidget(MakeCard(num(isa, 1), "ISa", this));
hero->addWidget(MakeCard(pct(o.cc_half * 100.0), "CC½", this));
hero->addWidget(MakeCard(pct(o.r_meas * 100.0), "R-meas", this));
if (stats_.stills)
hero->addWidget(MakeCard(pct(o.r_split * 100.0), "R-split", this));
else
hero->addWidget(MakeCard(pct(o.r_meas * 100.0), "R-meas", this));
hero->addWidget(MakeCard(pct(completeness), "Completeness", this));
hero->addWidget(MakeCard(num(multiplicity, 1), "Multiplicity", this));
// Wilson B describes how fast the intensities fall off with resolution; the radiation-damage
@@ -306,7 +311,7 @@ JFJochMergeStatsWindow::JFJochMergeStatsWindow(const QString &title, const Merge
row->addWidget(metricLabel_);
metric_ = new QComboBox(this);
metric_->addItem("CC½ (%)", 0);
metric_->addItem("R-meas (%)", 1);
metric_->addItem(stats_.stills ? "R-split (%)" : "R-meas (%)", 1);
metric_->addItem("⟨I/σ⟩", 2);
metric_->addItem("Completeness (%)", 3);
metric_->addItem("Multiplicity", 4);
@@ -324,7 +329,7 @@ JFJochMergeStatsWindow::JFJochMergeStatsWindow(const QString &title, const Merge
table_->verticalHeader()->setVisible(false);
stack_->addWidget(chart_); // page 0
stack_->addWidget(table_); // page 1
isa_page_ = buildIsaDiagnostic(merged_i_sigma, isa);
isa_page_ = std::isnan(isa) ? nullptr : buildIsaDiagnostic(merged_i_sigma, isa);
if (isa_page_)
stack_->addWidget(isa_page_); // page 2
else
@@ -436,7 +441,7 @@ void JFJochMergeStatsWindow::updatePlot() {
y.reserve(stats_.shells.size());
for (const auto &s: stats_.shells) {
x.push_back(s.mean_one_over_d2);
const double v = ShellMetric(s, metric);
const double v = ShellMetric(s, metric, stats_.stills);
y.push_back(static_cast<float>(v));
if (std::isfinite(v))
dmax = std::max(dmax, v);
@@ -455,7 +460,8 @@ void JFJochMergeStatsWindow::updatePlot() {
}
void JFJochMergeStatsWindow::buildTable() {
QStringList headers{"Res. [Å]", "N_obs", "Unique", "Compl. %", "Mult.", "⟨I/σ⟩", "R-meas %", "CC½ %"};
QStringList headers{"Res. [Å]", "N_obs", "Unique", "Compl. %", "Mult.", "⟨I/σ⟩",
stats_.stills ? "R-split %" : "R-meas %", "CC½ %"};
if (has_reference_)
headers << "CCref %";
table_->setColumnCount(headers.size());
@@ -479,7 +485,7 @@ void JFJochMergeStatsWindow::buildTable() {
set(std::isfinite(compl_) ? QString::number(compl_, 'f', 1) : QStringLiteral("—"));
set(std::isfinite(mult) ? QString::number(mult, 'f', 1) : QStringLiteral("—"));
set(QString::number(s.mean_i_over_sigma, 'f', 1));
set(QString::number(s.r_meas * 100.0, 'f', 1));
set(QString::number((stats_.stills ? s.r_split : s.r_meas) * 100.0, 'f', 1));
set(QString::number(s.cc_half * 100.0, 'f', 1));
if (has_reference_)
set(QString::number(s.cc_ref * 100.0, 'f', 1));
+3 -2
View File
@@ -22,7 +22,8 @@ class QTableWidget;
class JFJochSimpleChartView;
// Pop-up analysis window for a finished scaling/merging run: a row of eye-catching overall numbers
// (ISa, CC1/2, R-meas, completeness, multiplicity, the Wilson B and the radiation-damage Delta-B, and
// (ISa, CC1/2, R-meas - on stills no ISa and R-split for R-meas -, completeness, multiplicity, the Wilson B and
// the radiation-damage Delta-B, and
// CCref when a reference was used) and the report's pathology checklist above a per-resolution-shell plot whose metric is chosen from a
// combo, a per-shell table, and an ISa diagnostic scatter. Opened automatically when a merge job
// finishes and on demand from the processing-jobs dock. Modeless; deletes on close.
@@ -31,7 +32,7 @@ class JFJochMergeStatsWindow : public QWidget {
public:
// `report` is the rugnux result report text (<prefix>_report.txt), shown as its own view.
JFJochMergeStatsWindow(const QString &title, const MergeStatistics &stats,
double isa, bool has_reference,
double isa, bool has_reference, // isa NaN: none (stills)
const TwinningAnalysisResult &twinning,
const std::optional<gemmi::SpaceGroup> &space_group,
const std::optional<SearchSpaceGroupResult> &space_group_search,
@@ -9,6 +9,8 @@
#include "../../writer/RugnuxModelFile.h"
#include "../../common/ThreadAffinity.h"
#include <cmath>
#include <QApplication>
#include <QCheckBox>
#include <QClipboard>
@@ -797,7 +799,8 @@ void JFJochProcessingJobsWindow::onFinished(ProcessResult result, QString report
jobs_[row].report = report;
jobs_[row].checks = std::move(checks);
jobs_[row].merge_stats = result.merge_statistics;
jobs_[row].isa = result.error_model_isa;
// A stills merge reports its error model as terms, not as an ISa (the report says why): NaN, no card.
jobs_[row].isa = result.merge_statistics.stills ? NAN : result.error_model_isa;
jobs_[row].merge_has_reference = result.has_reference;
jobs_[row].twinning = result.twinning;
jobs_[row].space_group = result.space_group;
+18 -2
View File
@@ -15,9 +15,13 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_
return;
std::vector<int32_t> h, k, l;
std::vector<float> I, sigma, d, lp, qe, flight;
std::vector<float> I, sigma, d;
// The three corrections are stored as their reciprocals, in double so that the reciprocal of the
// value read back is the float the reflection held.
std::vector<double> lp, qe, flight;
std::vector<float> image, phi, pred_x, pred_y, obs_x, obs_y, bkg, var_bkg, partiality, zeta,
scale_factor;
scale_factor, dist_ewald;
std::vector<uint8_t> observed, clipped, overloaded;
h.reserve(msg.reflections.size());
k.reserve(msg.reflections.size());
@@ -39,6 +43,10 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_
phi.reserve(msg.reflections.size());
zeta.reserve(msg.reflections.size());
scale_factor.reserve(msg.reflections.size());
dist_ewald.reserve(msg.reflections.size());
observed.reserve(msg.reflections.size());
clipped.reserve(msg.reflections.size());
overloaded.reserve(msg.reflections.size());
for (const auto &refl : msg.reflections) {
image.emplace_back(refl.image_number);
@@ -61,6 +69,10 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_
phi.emplace_back(refl.delta_phi_deg);
zeta.emplace_back(refl.zeta);
scale_factor.emplace_back(refl.image_scale_corr);
dist_ewald.emplace_back(refl.dist_ewald);
observed.emplace_back(refl.observed);
clipped.emplace_back(refl.clipped);
overloaded.emplace_back(refl.overloaded);
}
std::string image_group_name = fmt::format("image_{:06d}", image_number);
@@ -86,6 +98,10 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_
image_group.SaveVector("partiality", partiality);
image_group.SaveVector("zeta", zeta);
image_group.SaveVector("image_scale_corr", scale_factor);
image_group.SaveVector("excitation_error", dist_ewald)->Units("Angstrom^-1");
image_group.SaveVector("observed", observed);
image_group.SaveVector("clipped", clipped);
image_group.SaveVector("overloaded", overloaded);
}
void HDF5DataFilePluginReflection::WriteFinal(HDF5File &data_file) {
+5
View File
@@ -1208,6 +1208,11 @@ void NXmx::Finalize(const EndMessage &end) {
SaveVector(*hdf5_file, "/entry/MX/stillsRockingWidthTable", w.table_s)->Units("Angstrom^-1");
}
}
if (end.stills_given_cell) {
const auto &c = *end.stills_given_cell;
SaveVector(*hdf5_file, "/entry/MX/stillsGivenUnitCell",
std::vector<float>{c.a, c.b, c.c, c.alpha, c.beta, c.gamma});
}
if (end.indexing_rate) {
SaveScalar(*hdf5_file, "/entry/MX/imageIndexedMean", end.indexing_rate.value());