diff --git a/common/DetectorModules.cpp b/common/DetectorModules.cpp index 7b313ed65..449b965f9 100644 --- a/common/DetectorModules.cpp +++ b/common/DetectorModules.cpp @@ -4,6 +4,7 @@ #include "DetectorModules.h" #include "DetectorGeometry.h" #include "Definitions.h" +#include "PixelMask.h" #include #include @@ -193,3 +194,39 @@ DetectorModules ModulesAsImaged(const DetectorGeometry &geom) { } return DetectorModules(out); } + +DetectorModules ModulesFromGapMask(const std::vector &mask, int64_t width, int64_t height) { + if (width <= 0 || height <= 0 || mask.size() != static_cast(width * height)) + return {}; + const auto clear = [&](int64_t x, int64_t y) { + return x >= 0 && y >= 0 && x < width && y < height + && (mask[y * width + x] & (1u << PixelMask::ModuleGapPixelBit)) == 0; + }; + std::vector taken(mask.size(), 0); + std::vector out; + for (int64_t y = 0; y < height; y++) + for (int64_t x = 0; x < width; x++) { + if (!clear(x, y) || taken[y * width + x]) + continue; + // The first clear pixel of a region not yet taken is a module's corner: the module runs to the + // gap along the row and down the column, and has to be whole and bounded by gap all round. + int64_t w = 0, h = 0; + while (clear(x + w, y)) w++; + while (clear(x, y + h)) h++; + if (!((w == CONVERTED_MODULE_COLS && h == CONVERTED_MODULE_LINES) + || (w == CONVERTED_MODULE_LINES && h == CONVERTED_MODULE_COLS))) + return {}; + for (int64_t yy = y - 1; yy <= y + h; yy++) + for (int64_t xx = x - 1; xx <= x + w; xx++) { + const bool inside = xx >= x && xx < x + w && yy >= y && yy < y + h; + if (clear(xx, yy) != inside) + return {}; + if (inside) + taken[yy * width + xx] = 1; + } + out.push_back(DetectorModule::AsImaged(x, y, w, h)); + } + if (out.size() < 2) + return {}; + return DetectorModules(out); +} diff --git a/common/DetectorModules.h b/common/DetectorModules.h index d38649d4f..4f979b952 100644 --- a/common/DetectorModules.h +++ b/common/DetectorModules.h @@ -121,3 +121,9 @@ public: // CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES turned the way the geometry turns it, placed as imaged. // Empty for a detector of one module. DetectorModules ModulesAsImaged(const DetectorGeometry &geom); + +// The modules of a JUNGFRAU image where only its pixel mask says where they are: every region of pixels +// clear of the module-gap bit (PixelMask::ModuleGapPixelBit) has to be one whole module, +// CONVERTED_MODULE_COLS x CONVERTED_MODULE_LINES either way round, and is taken as placed as imaged. +// Empty where any region is not, or where there are fewer than two. The mask is row-major, width x height. +DetectorModules ModulesFromGapMask(const std::vector &mask, int64_t width, int64_t height); diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 488a8f1fb..d8bb68ff4 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -653,6 +653,10 @@ SpotFindingSettings DiffractionExperiment::DefaultDataProcessingSettings() { void DiffractionExperiment::FillMessage(StartMessage &message) const { message.images_per_file = GetImagesPerFile(); message.detector_modules = GetModules().Get(); + // A JUNGFRAU image assembled here is made of modules this experiment places itself: they go into + // the file one by one, so that what reads it back knows them. + if (message.detector_modules.empty() && GetDetectorType() == DetectorType::JUNGFRAU && IsGeometryTransformed()) + message.detector_modules = ModulesAsImaged(detector.GetGeometry()).Get(); message.beam_center_x = GetBeamX_pxl(); message.beam_center_y = GetBeamY_pxl(); const auto direct_beam = GetDiffractionGeometry().GetDirectBeam_pxl(); diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 390707bd1..0aa2ee4fb 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -473,8 +473,10 @@ struct EndMessage { // setting the merge works in - is only chosen afterwards, so the two settings are not always the // same one. Absent means they are, i.e. the identity. std::optional> reindex_matrix; - // Stills: the rocking curve the partialities were computed from (and the frames predicted through). + // Stills: the rocking curve the frames were predicted through (and the partialities computed from). std::optional stills_run_width; + // Stills: the cell given on the command line, whose population of frames the merge keeps. + std::optional stills_given_cell; // Geometry refined by the offline analysis. When present these overwrite the master-file geometry // at finalization (rugnux writes the refined values; the broker leaves them empty and keeps the diff --git a/common/Reflection.h b/common/Reflection.h index 66eb37630..9c864b8f0 100644 --- a/common/Reflection.h +++ b/common/Reflection.h @@ -24,7 +24,7 @@ struct Reflection { float bkg; float var_bkg; // non-signal (background) part of sigma^2, carried to the merge float sigma; - float dist_ewald; + float dist_ewald; // stills: signed excitation error |S| - 1/lambda (1/A), positive outside the sphere // The reciprocal Lorentz factor (rotation only - a still's Lorentz factor is one) times the // reciprocal polarization factor, and nothing else. This is what LP means everywhere in the // field, and it is what the CBOR key "rlp" and the HDF5 dataset "lp" store the reciprocal of. diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index dccee57b3..9e726772e 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -78,6 +78,16 @@ high-throughput reduction and analysis of serial femtosecond X-ray diffraction d J. Appl. Cryst. 47, 1118-1131 [doi:10.1107/S1600576714007626](https://doi.org/10.1107/S1600576714007626). +**Per-crystal post-refinement of serial stills** - refining, for each still, the partiality +parameters the spot positions cannot pin (a small reorientation, the crystal's own rocking width and the +radial scale of its reciprocal lattice, which a shot's photon energy or a crystal's cell moves) against +the merged intensities follows the post-refinement of PRIME and the mosaic-block model of Sauter et al.: +M. Uervirojnangkoorn, O. B. Zeldin, A. Y. Lyubimov, J. Hattne, A. S. Brewster, N. K. Sauter, A. T. Brunger +and W. I. Weis, "Enabling X-ray free electron laser crystallography for challenging biological systems +from a limited number of crystals" (2015), eLife 4, e05421; N. K. Sauter, J. Hattne, A. S. Brewster, +N. Echols, P. H. Zwart and P. D. Adams, "Improved crystal orientation and physical properties from +single-shot XFEL stills" (2014), Acta Cryst. D70, 3299-3309 (DOIs not verified here, so not given). + Spot extraction groups strong pixels into spots with the sparse connected-component labelling of the ACTS traccc project: P. Gessinger, H. M. Gray, A. Krasznahorkay, C. Leggett, J. Niermann, A. Salzburger, S. N. Swatman and B. Yeo, "traccc: GPU track reconstruction library for HEP diff --git a/docs/CPU_DATA_ANALYSIS_INDEXING.md b/docs/CPU_DATA_ANALYSIS_INDEXING.md index e58c74043..e31e08ba5 100644 --- a/docs/CPU_DATA_ANALYSIS_INDEXING.md +++ b/docs/CPU_DATA_ANALYSIS_INDEXING.md @@ -431,4 +431,5 @@ gauge holds their spot-weighted mean shift, turn and outward push, and with the also the two tilt-like gradients across the detector, at zero, so the beam centre, distance, tilt and crystal orientations carry what all modules share. A coarse search of each module's position comes first, scored like the beam centre by the spots that land on a lattice. The placement is used from -there on for a foreign multi-panel detector; for a JUNGFRAU master it is measured and reported only. +there on for a foreign multi-panel detector and for modules given as a CrystFEL `.geom`; for a JUNGFRAU +master it is measured and reported only. diff --git a/docs/CPU_DATA_ANALYSIS_INTEGRATION.md b/docs/CPU_DATA_ANALYSIS_INTEGRATION.md index 90c85ebaa..7095ef913 100644 --- a/docs/CPU_DATA_ANALYSIS_INTEGRATION.md +++ b/docs/CPU_DATA_ANALYSIS_INTEGRATION.md @@ -341,7 +341,7 @@ The partiality applied is fixed by the data type and scaling stage, not chosen f 3. **Fixed**: use the per-reflection partiality carried from prediction. Still/serial images are predicted with $P=1$, so a single-pass stills scale is effectively unity/fixed — which is exactly what `--simple-stills` keeps. By default the stills path instead **post-refines a physical partiality**: a small per-crystal orientation tilt $(\delta\psi_x,\delta\psi_y)$ about the two axes perpendicular to the beam is refined against the running merge, and every reflection's partiality is then recomputed analytically from the refined lattice through its excitation error $\Delta_\mathrm{Ewald}=\big|\,|\mathbf{q}+\mathbf{S}_0|-1/\lambda\,\big|$ and a Gaussian width $\sigma(d^*)$ — the run's rocking width, measured once on the intensity-weighted excitation errors and shared by every crystal. It is measured per group of frames of similar prediction window (four per shell), each against its own window, the shell taking the median of the groups whose curve is narrower than their window, so the frames with the widest windows do not set it everywhere; and it is measured in eight shells of equal width in $d^*$, made non-decreasing (pool-adjacent-violators, weighted by each shell's summed signal) and interpolated linearly in $d^*$ as a table. The three-term form $\sigma^2=\gamma_0^2+(\gamma_e d^*)^2+(\mathrm{bw}\,|q_z|)^2$ — the reciprocal-lattice point's own radius, the mosaic/divergence spread, and the bandwidth smear along the beam, in quadrature — is still fitted, sets the prediction window (§8.2) and stands in for the table where fewer than three shells are measurable. The curve is centred on the run's **radial scale** of the reciprocal lattice — a wavelength or cell a little off, which the spot positions cannot see since they fix only $\lambda d^*$ — measured from where the curve's centre lies against $d^{*2}$ and applied to $\mathbf{q}$. Where the frames were predicted through the measured window (§8.2), the post-refinement uses that same curve rather than measuring it again; the curve is written to `_process.h5` (`/entry/MX/stillsRockingWidth*`, `stillsRadialScale`), so `--mode scale` merges with it too. A tilt moves reflections on opposite sides of the Ewald sphere in opposite directions, so it reshapes the *spatial* pattern of partialities — a degree of freedom the per-image scale $G$ does not have, and the reason the tilt is refined rather than a scalar partiality width, which would be degenerate with $G$. Nothing is re-integrated (the integrated intensities are fixed); the tilt is hard-bounded at about 1° and held by a soft prior, so it stays inert on sparse or weak crystals. The cycle is merge → per-crystal tilt refinement (with $G$ profiled out by the same robust Cauchy IRLS used for the per-frame scales, §10.3) → recompute $P$ → re-merge, repeated a few times; the refined partiality is adopted on every crystal it was refined on. The per-crystal fits of $G$ and of the tilt measure each residual against its expected variance at the model, background + counting + $(0.3\,I_\mathrm{model})^2$: a still's strong reflections scatter about the model by tens of per cent, and in counting sigmas alone every strong reflection would be an outlier and the scale would be set by the weak ones. -Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. +Reflections below a minimum partiality are left out of the merge (`--min-partiality`, default 0.02 on rotation and 0.5 on stills): the empirical rocking curve of a still has a floor far out on its tail, from mis-oriented crystals, and dividing such an observation by a small model $p$ makes it biased, not merely noisy. On stills this holds for the merges the scale and the partiality are refined against. The final merge, which has the fitted error model, takes every observation: the model's partiality term $c\,((1-p)/p)\,\langle I\rangle$ (§10.5) weights a low-partiality observation by what it is worth. A reflection - each Bijvoet hand under `-A` - is reported only if at least one of its observations reaches $p \ge 0.2$; one seen only far out on its rocking curve rests on the partiality model alone. The error model is fitted on the observations at $p \ge 0.2$. ### 10.3 Smoothing of per-frame scales @@ -388,9 +388,9 @@ Per-shell and overall merging statistics are computed on corrected intensities, itself count as missing instead of leaving the denominator along with the data, - the anomalous signal-to-noise $\mathrm{SigAno}$ and the half-set anomalous correlation $\mathrm{CC}_\mathrm{anom}$ (below). -The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path has no asymptotic estimate and reports $\mathrm{ISa}=1/b$ directly. +The error model is refined as $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2$, with $a$ set by the scatter of weak (counting-limited) reflections and $b$ the intensity-proportional systematic scatter of the strong ones. On the **rotation** path, **ISa** is the asymptotic ($I\to\infty$) signal-to-noise — by definition the reproducibility limit of the strongest reflections (Diederichs, *Acta Cryst.* **D66** (2010) 733) — and is read directly from the strong symmetry equivalents as the counting-subtracted fractional scatter of well-measured reflection groups (a robust median over strong groups; the $I/\sigma$ threshold is relaxed on weak or radiation-damaged data that has few strong reflections), rather than as $1/b$ of the whole-range fit, whose $b$ is raised slightly by an intermediate-intensity excess and so understates the limit. The asymptotic value is **report-only** — nothing downstream reads it, and the merged $\sigma$ is not floored at $b|I|$ (that floor was removed). The per-observation $\sigma_\mathrm{corr}$ (the merge weights) uses the whole-range $a,b$. The **stills** path adds a partiality term, $\sigma_\mathrm{corr}^2 = a\,\sigma^2 + (b\,\langle I\rangle)^2 + c^2\left(\frac{1-p}{p}\right)^2\langle I\rangle^2$, and reports **no ISa**: there $1/b$ is the $I/\sigma$ ceiling of one fully recorded still under the stills model - the same frames processed as a rotation sweep read several times higher - which describes the processing model more than the instrument and the crystal. It reports $a$, $b$ and $c$ instead (`ERROR_MODEL_A/B/C`, `_reflns.jfjoch_error_model_a/b/c`); $1/b$ is still used inside, by the space-group search. -$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$. Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. +$a$ and $b$ are **reported in XDS's convention**, which is $\sigma^2 = a(\sigma_0^2 + b I^2)$ with $\mathrm{ISa}=1/\sqrt{ab}$, so the printed pair can be read straight against a `CORRECT.LP`. The internal fit keeps the form above; only the report converts, as $b_\mathrm{XDS} = b^2/a$ (and on stills $c_\mathrm{XDS} = c^2/a$). Note that $a$ is the same in both conventions and that the two ISa expressions are the same number, $1/\sqrt{a\cdot b^2/a} = 1/b$ — so the rotation log prints **two** ISa, the whole-range $1/b$ (XDS's meaning) and the strong-reflection asymptote beside it, which can only ever be the more optimistic of the two. The mmCIF follows the same split: `_reflns.jfjoch_diffrn_ISa` is the whole-range value, directly comparable with a `CORRECT.LP`, and the asymptote is written separately as `_reflns.jfjoch_diffrn_ISa_asymptotic`, with `_reflns.jfjoch_error_model_a` and `_b` alongside so the number can be re-derived. Note that a file written before this change carries the *asymptote* under the plain `ISa` name. A third, unrelated $b$ appears in the space-group search (§13.1); it is fitted with the $\sigma^2$ coefficient held at 1 and its gate constants are calibrated in that convention. **Anomalous signal-to-noise (SigAno).** The strength of the anomalous signal is reported per shell and overall as $\mathrm{SigAno}=\langle|\Delta I|\rangle / \langle\sigma(\Delta I)\rangle$, where $\Delta I = I(+)-I(-)$ over acentric reflections measured in both Bijvoet hands and $\sigma(\Delta I)=\sqrt{\sigma_+^2+\sigma_-^2}$. It is computed from the **full-multiplicity** inverse-variance $I(+)/I(-)$ split (the same one written to the output), i.e. from all observations rather than a half-set. For pure noise $\mathrm{SigAno}$ approaches the half-normal value $\sqrt{2/\pi}\approx0.8$, and it rises above $1$ once a real anomalous difference is present. A half-set anomalous correlation ($\mathrm{CC}_\mathrm{anom}$) is reported beside it: $\Delta I$ is formed once per half-set and the two are correlated over the acentric pairs where **both** hands split into two non-empty halves, per shell and overall as one correlation rather than a mean of shells. Unlike SigAno it is not a ratio against the error model, so an optimistic $\sigma$ cannot inflate it. It has no floor either: subtracting the two Bijvoet hands cancels the large common intensity that keeps $\mathrm{CC}_{1/2}$ non-negative, so on data with little anomalous signal and about two observations per mate it goes strongly negative. That is a property of the statistic and is reported as measured. It agrees with AIMLESS's `CCanom` and `phenix.merging_statistics`' `cc_anom`; XDS's `Anomal Corr` is a **different quantity and is not comparable with it** — on the same observations it reads two to three times higher in the low shells. $\mathrm{CC}_\mathrm{anom}$ is a rotation-path statistic (the stills merge forms no half-set anomalous difference), and it is reported in the `CCanom` column of the printed merge-statistics table and as `CC_ANOM=` in the results report — not in the mmCIF. SigAno is emitted only when an anomalous split was made, using the standard PDBx items `_reflns.pdbx_absDiff_over_sigma_anomalous` (overall) and `_reflns_shell.pdbx_absDiff_over_sigma_anomalous` (per shell), and appears as the `SigAno` column of the same table. Where either could not be measured at all — a Friedel-merged run that split no Bijvoet pair, a shell too thin to split one in both hands — the table prints `-` and the results report writes no key, which is not the same statement as a value measured to be zero. diff --git a/docs/EXTERNAL_TEST_DATA_STILLS.md b/docs/EXTERNAL_TEST_DATA_STILLS.md index b68d40972..859a752a9 100644 --- a/docs/EXTERNAL_TEST_DATA_STILLS.md +++ b/docs/EXTERNAL_TEST_DATA_STILLS.md @@ -30,8 +30,8 @@ the paper of the entry. | `lcls242_s`, `lcls242_t` | [242](https://cxidb.org/id-242.html), [10.11577/3374214](https://doi.org/10.11577/3374214) | LCLS MFX | Rayonix MX340-HS; LCLS-I XTC | [9O4S](https://www.rcsb.org/structure/9O4S) (small cell), [9O4T](https://www.rcsb.org/structure/9O4T) (large cell) | P 1 21 1, 1.95 | Wolff et al., bioRxiv (2026) | | `euxfel83` | [83](https://cxidb.org/id-83.html), [10.11577/1464102](https://doi.org/10.11577/1464102) | European XFEL SPB/SFX | AGIPD 1M, 16 modules of 512 x 128, 200 µm; Cheetah CXI | [6GTH](https://www.rcsb.org/structure/6GTH) | P 32 2 1, 1.69 | Wiedorn et al., Nat. Commun. (2018) | | `euxfel152` | [152](https://cxidb.org/id-152.html), [10.11577/1660970](https://doi.org/10.11577/1660970) | European XFEL SPB/SFX | AGIPD 1M; Cheetah CXI | [6U57](https://www.rcsb.org/structure/6U57) | I 2 3, 2.80 | Echelmeier et al., Nat. Commun. (2020) | -| `alvra103`, `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) | -| `alvra213`, `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) | +| `alvra103_int16` | [103](https://cxidb.org/id-103.html), [10.11577/1650020](https://doi.org/10.11577/1650020) | SwissFEL Alvra | JUNGFRAU 16M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [6S0L](https://www.rcsb.org/structure/6S0L) | C 2 2 21, 2.65 | Nass et al., IUCrJ (2020) | +| `alvra213_int16` | [213](https://cxidb.org/id-213.html), [10.11577/2282689](https://doi.org/10.11577/2282689) | SwissFEL Alvra | JUNGFRAU 4M, 75 µm; SwissFEL HDF5, float32 keV (converted) | [8BKH](https://www.rcsb.org/structure/8BKH) (dark state) | P 1 21 1, 1.35 | Barends et al., Nature (2024) | Every set of the manifest is in the table and every set in the table is in the manifest, with the same CXIDB entry, PDB entry, space group and dmin; `tools/battery/doc_check.py` checks @@ -63,8 +63,8 @@ broken download resumes with `curl -C -`. | `lcls242_*` | `lcls_242/xtc/`, `lcls_242/xtc_more/`: `xtc/mfxl1013721-r0256` to `r0267` streams (the two crystal forms are in every run; each set keeps its own form) | ~90,000 | **subset**: runs 256-267 (44 of about 220 stream files; ~610 GB of ~3 TB) | ~700 GB | | `euxfel83` | `euxfel_83/raw/`: `XFEL-r0243-c00.cxi`, `XFEL-r0278-c00.cxi`, `XFEL-r0280-c00.cxi` (Cheetah hits, from the entry's `cheetah-calib-mar-v1/hdf5/` tree on the NERSC archive); `euxfel_83/ref/agipd_mar18_v10.geom` from the entry | 2,790 | **subset**: 3 files; the deposited stream has 14,445 frames | 4.7 GB | | `euxfel152` | `euxfel_152/raw/`: runs r0108, 0110, 0112, 0114, 0115, 0116, 0118, 0123, 0125 extracted from `XFEL2042-runs-cxi.tar` (range requests on the tar members); `euxfel_152/ref/kdo_embedded.geom`, the geometry embedded in the header of the entry's `KDO8PS.stream` | 3,801 | **subset**: 9 runs; the deposited stream has 37,006 frames | 5.7 GB | -| `alvra103` | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB | -| `alvra213` | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB | +| `alvra103_int16` (source) | `swissfel_103/raw/`: `Raw_Files/2018-08-07/a2a/run{10..24}.JF06T32V01.h5`; `swissfel_103/ref/jf-16M.geom` and the pixel mask `swissfel_103/ref/run47.JF06T32V01.mask.h5` from the entry; the set reads `swissfel_103/ref/jf-16M_mask.geom`, a copy with `mask_file` pointed at that mask (see below) | ~11,000 | **subset**: 15 runs of one day; the deposition indexed 199,136 of 2,251,787 images | ~750 GB | +| `alvra213_int16` (source) | `swissfel_213/raw/march_dark_MbCO_II-March-06/data/dark/run_*.JF06T08V01.h5`, from `march/march_dark_MbCO_II-March-06.tar.gz` (425 GB; download it to disk and extract, a streamed extraction does not survive a broken connection); `swissfel_213/ref/mbco_march.geom` from the JUNGFRAU geometry template of `2019-MARCH-SwissFEL-MbCO-supplement.tar.gz`; the pixel mask of run 243 (`data/pixel_mask/` in the same tarball) is used for all runs, through `swissfel_213/ref/mbco_march_mask.geom`, a copy with `mask_file` pointed at it (see below) | ~38,000 | all 24 dark runs of the March 2019 series | ~430 GB | **The masks of the SwissFEL geometry files.** rugnux takes the pixel masks and bad regions a CrystFEL geometry file names. The SwissFEL files name their mask with an absolute path on the facility's disk, @@ -78,9 +78,9 @@ that line differs. **The `_int16` sets.** The SwissFEL files store converted JUNGFRAU data as uncompressed float32 keV, 68 MB (JF16M) or 17 MB (JF4M) a frame, which caps reading at about 74 or 295 frames/s at 5 GB/s. The `_int16` sets are copies with the same layout in which `/data/data` holds whole photons (keV divided -by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller; the battery runs -them with the same arguments as the float32 originals (rugnux measures the gain from the frames), and -they merge to the same numbers. Make them with +by the photon energy, rounded) as int16 with bitshuffle/LZ4, about 25 times smaller. The battery runs only the +copies: they merge to the same numbers as the float32 originals (rugnux measures the gain from the +frames), and the originals are kept only as the source of the conversion and as a test of the format. Make them with `tools/battery/still/float32_to_int16.py ` (h5py and hdf5plugin), into `swissfel_103/conv/` (4.57 keV) and `swissfel_213/conv/` (12.037 keV, keeping the tarball's directory tree), named `_int16.h5`. One MbCO dark run (`run_000238`) holds no frames and has no copy. diff --git a/docs/HDF5.md b/docs/HDF5.md index e6908eab6..cca40f5f7 100644 --- a/docs/HDF5.md +++ b/docs/HDF5.md @@ -118,7 +118,9 @@ of the raw images — without duplicating terabytes of data. This is a particularly FAIR-friendly artefact: it can be shared or archived alongside (or instead of) the raw data to convey what is in a dataset and how it was processed, while the `/entry/data/data` -VDS still resolves to the original images when they are available. `rugnux` can also process +VDS still resolves to the original images when they are available. Where the input is anything but a +single HDF5 master it could link to (several masters read as one, CBF, XTC, CXI, ...), the file has no +`/entry/data/data` at all - only the per-image analysis, which `--mode scale` reads all the same. `rugnux` can also process an equally-spaced *subset* of images (start/end/stride), producing a down-sampled reference set. ## 3. NXmx-standard content @@ -397,7 +399,8 @@ variants. | `reindexMatrix` | | change of basis from the setting the per-image data are in to the setting of `/entry/sample/unit_cell` (`[9]`, `int32`, flattened 3×3, row major) — see below | | `stillsRockingWidthFormula` | 1/Å, rad, relative | stills: the run's rocking width σ² = γ₀² + (γₑ d\*)² + (σ_bw λ d\*²/2)², as `[γ₀, γₑ, σ_bw]`; offline (Rugnux) only | | `stillsRadialScale` | | stills: the scale of the reciprocal lattice the rocking curve is centred on | -| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\*, which the partialities use (held beyond the first and last entry; the formula above is what the prediction window uses); `--mode scale` merges with formula and table instead of measuring another | +| `stillsRockingWidthTableDstar`, `stillsRockingWidthTable` | 1/Å | stills: the rocking width σ measured against d\* before the image pass, which the frames were predicted through (held beyond the first and last entry; the formula above is what the prediction window uses); the partiality post-refinement balances it against the merge, and `--mode scale` starts from this same curve instead of measuring another | +| `stillsGivenUnitCell` | Å, deg | stills: the cell given with `-C`, whose population of frames the merge kept (`[6]`); `--mode scale` leaves out the same frames | **Reindex matrix.** The per-image `h`, `k`, `l` and `latticeIndexed` are written as each image is processed, in the setting that image was *indexed* in. The space group is only settled afterwards, by diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md index d387a1436..ee006c39a 100644 --- a/docs/RUGNUX_ADVANCED.md +++ b/docs/RUGNUX_ADVANCED.md @@ -46,10 +46,10 @@ against a foreign dataset injects that dataset's systematics — so `-z` never p errors into the intensities. `--reference-column` picks the column to read where the automatic choice (`F-model`, else `IMEAN`/`I`, else `FP`/`FOBS`/`F`) is not the right one. -For the indexing ambiguity an atomic model does as well: `--model` computes the intensities it -needs from the structure (a model that fits also sets the hand label and the setting, see -[Validating against a model](#validating-against-a-model-rugnux---model)). Where a reference dataset exists, prefer it; -where only a model does, it resolves the ambiguity just the same. +An atomic model does not take part in the processing: `--model` is placed and validated after the merge, +and a run with one merges exactly what a run without one does. A model that fits then sets the frame the +merge is written in - the hand label, the setting and the indexing (see +[Validating against a model](#validating-against-a-model-rugnux---model)) - by relabelling it, never by re-merging. ### The indexing ambiguity @@ -102,16 +102,15 @@ How to resolve it: | **Rotation**, a reference dataset exists | `-z reference.mtz`. Once the space group is settled, each candidate reindexing of the merged intensities is correlated with the reference and the best-correlating one is re-merged. Only the *hkl* labels change; the cell does not | | **Serial stills**, a reference dataset exists | `-z reference.mtz`. Resolved **per image**, at integration time, by correlating each crystal's intensities with the reference — so the merge never mixes hands in the first place. Where the crystals' own cells tell the indexings apart (pseudo-merohedry), a crystal its cell decides keeps that indexing and the reference decides the rest. The data-only resolution below runs as a check: `AMBIGUITY_DATA_DISAGREES=` | | **Rotation**, only a model | `--model model.pdb`. The merged data are fitted to the model in each candidate indexing and the lowest R-free wins — provided the model first beats its random-orientation null (`MODEL_FIT= ACCEPTED`) and its lead over the runner-up beats the lead a random placement of the same model takes; the written reflections are then reindexed into it, so the file, the R-factors and the maps agree. Where either bar is missed the data keep the indexing they were merged in, and the report says by how much (`MODEL_INDEXING_MARGIN_SIGMA`). It is the **data** that are relabelled, never the model: every dataset of one crystal form processed against the same model comes out in that model's indexing, so the reflection files and maps of a screening campaign compare directly. The relabelling is a proper rotation of the lattice, so the merging statistics are unchanged and I(+)/I(-) keep their hand | -| **Serial stills**, only a model | `--model model.pdb`, with the cell and space group given (`-C` / `-S`). Structure factors are computed from the model up front and used as the reference for the per-image test, exactly as a reference MTZ would be, the crystals their own cell decides again excepted. The data-only resolution below is then run only as a check: `AMBIGUITY_DATA_DISAGREES=` counts the crystals on which it would have chosen otherwise | -| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A model or reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing | +| **Serial stills**, only a model, space group given (`-S`) | The crystals are made consistent with each other from the data alone, as in the row below - the model plays no part in the processing. Validation then fits the merge to the model in each candidate indexing, as for rotation data, and relabels the written reflections into the model's indexing where it wins | +| **Serial stills**, neither, space group given (`-S`) | Nothing to do: the crystals are made consistent with each other from the data alone (`INDEXING_AMBIGUITY_RESOLVED= data`), in an indexing that is arbitrary against other data sets. A reference still settles the weakest crystals more reliably than the other crystals can, and puts the merge in its own indexing | | Neither | The run warns and merges in whichever indexing it found: for rotation data a usable dataset in an arbitrary frame; for stills whose space group the run determined, a merge of mixed indexings, which looks like a perfect twin and can pass for higher symmetry than the crystal has. Run again with `-S` | Two things to know about where the choice lands: -- **`--mode scale` cannot repair a stills run after the fact.** The per-image test happens at - integration time, so a `_process.h5` whose images were integrated without a reference — an MTZ or a - model — has already lost the distinction, and no re-merge brings it back; the data-only resolution - runs in the full pipeline, not in `--mode scale`. On rotation data, where +- **`--mode scale` resolves it as the full run does.** On stills with no reference MTZ it makes the + crystals consistent from the data alone before merging; the per-image test against a reference + happens at integration time and is kept in the stored reflections. On rotation data, where the ambiguity is one choice for the whole dataset, `--mode scale --model model.pdb` does resolve it. - **The choice reaches the written reflections**, not only the R-factors and the maps: the merged `.mtz` / `.cif` / `.hkl` (and `_unmerged.mtz`, where it is asked for) are written in the indexing @@ -291,9 +290,13 @@ declines it and the run stops with a message saying so. `--model` works here as that the model is placed by the rigid body only: a model it does not place, or a prediction, is not searched for by molecular replacement, `--mr` is refused, and the placed model is not refined. On rotation data, where the [indexing ambiguity](#the-indexing-ambiguity) is one choice for the whole dataset, it can settle the -enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. What no re-merge can -repair is a **stills** `_process.h5` integrated without a reference: there the ambiguity was -decided per image, at integration time, and the distinction is gone from the stored reflections. +enantiomorph label and the indexing — under the same fit test as always — and the re-merged reflections are written in what it settled. On +stills the re-merge is the full run's own merge: the same frames (a given cell's population, as +`/entry/MX/stillsGivenUnitCell` records it), the same rocking curve, the same data-only resolution of +the indexing ambiguity and the same scaling, so a +`_process.h5` merged again with the options it was written with reproduces the full run's reflections +(with a reference MTZ, the full run also lets the crystals' own cells overrule the reference's indexing +before merging, which `--mode scale` does not). Where the full run re-seated the lattice — the space group it settled on is in a different setting from the one each image was indexed in — the file records the change of basis as @@ -312,7 +315,7 @@ describes them - the cell and space group; `_software` with Rugnux named twice, beamline and its general class, the detector's make and model in `_diffrn_detector.type` and its class in `_diffrn_detector.detector`, and the collection date from the NXmx start time); and the merging statistics overall (`_reflns`) and per shell (`_reflns_shell`): completeness, multiplicity, -, R_merge, R_meas, R_pim, CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries +, R_merge, R_meas, R_pim (`?` on a stills merge, where they mean nothing), CC1/2 and CC\*, and SigAno where an anomalous split was made; on rotation data the cell carries its e.s.d.s (`_cell.length_a_esd`, …). A quantity the input does not state is left out rather than guessed. The unmerged observations are in the `_unmerged.mtz` and the `.hkl` (both scaled on a rotation merge), not in the `.cif`. diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md index 1e25bf517..c6658d137 100644 --- a/docs/RUGNUX_CLI.md +++ b/docs/RUGNUX_CLI.md @@ -304,7 +304,7 @@ serial stills; `fft` (GPU) / `fftw` (CPU) index **de novo** and suit strong rota | `-z, --reference ` | Reference data of the same crystal form — an MTZ or a PDB structure-factor mmCIF (`-sf.cif`), gzipped or not, recognised by content; `--reference-mtz` is the same option: fixes the space group, its setting and the cell, resolves the [indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity), hands over the R-free set and reports CCref. Not a scale anchor | | `--free-flags ` | Take only the R-free test set of this MTZ or structure-factor mmCIF - e.g. the deposited reflections of the `--model` - for the validation, the refinement and the merged files; nothing else of the file is used, so unlike `-z` it fixes no space group, cell or frame. Taken on the model's axes where the merge matches the file's intensities (`R_FREE_FLAGS=`, see [Free set and model bias](RUGNUX_MODEL_AND_REFERENCE.md#free-set-and-model-bias)) | | `--reference-column