Merge multisweep-p1d into rc181: Bruker readers (SFRM, imgCIF sweep axis, polarisation), several sweeps of one crystal merged jointly (--sweeps), per-sweep MTZ batches and the SWEEPS report section

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-29 02:13:59 +01:00
commit 03cdf2357b
42 files changed
+4069 -266

No files matched your search

+7
View File
@@ -76,6 +76,13 @@ struct ScaledFull {
float I = NAN;
float sigma = NAN;
float image_number = NAN; // the frame of the event's peak partial, which tells two events of one index apart
// The full's counting variance as the merge rebuilds it at an intensity I, before its error model:
// var_bkg + var_per_I * I + (capture * I)^2, all at the intensity's scale (what a later merge rebuilds
// the full from). NaN where not exported.
float var_bkg = NAN;
float var_per_I = NAN;
float capture = NAN;
float d = NAN; // resolution, A
};
struct MergedReflection {
+9
View File
@@ -315,3 +315,12 @@ ScalingSettings& ScalingSettings::ReportResolutionRange_A(std::optional<ReportRe
std::optional<ReportResolutionRange> ScalingSettings::GetReportResolutionRange_A() const {
return report_resolution_range;
}
ScalingSettings& ScalingSettings::SweepSegments(std::vector<SweepSegment> input) {
sweep_segments = std::move(input);
return *this;
}
const std::vector<SweepSegment> &ScalingSettings::GetSweepSegments() const {
return sweep_segments;
}
+17
View File
@@ -3,7 +3,9 @@
#pragma once
#include <cstdint>
#include <optional>
#include <vector>
#include "JFJochException.h"
// How the high-resolution cutoff for the written reflections and the reported shell table is chosen
@@ -17,8 +19,20 @@ struct ReportResolutionRange {
double d_max = 0.0;
};
// One sweep of several whose frames are laid end to end in one rotation merge (the joint merge of several
// sweeps): frames [first_frame, first_frame + n_frames). Every observation of a segment's frames is a full
// already combined in its own sweep (partiality 1, filed under the frame of its peak), so the merge
// combines nothing again, recomputes nothing from the rocking geometry, and lets nothing that runs along
// the frames - the per-frame scale's smoothness above all - reach across from one sweep into the next
// (RotationScaleMerge).
struct SweepSegment {
int32_t first_frame = 0;
int32_t n_frames = 0;
};
class ScalingSettings {
bool refine_wedge = false;
std::vector<SweepSegment> sweep_segments;
bool merge_friedel = true;
std::optional<double> high_resolution_limit_A;
@@ -150,6 +164,8 @@ public:
ScalingSettings& ResolutionCCTarget(double input);
ScalingSettings& ReportShellCount(int input);
ScalingSettings& ReportResolutionRange_A(std::optional<ReportResolutionRange> input);
// The sweeps a rotation merge's frames are made of; empty (the default): one sweep of partials.
ScalingSettings& SweepSegments(std::vector<SweepSegment> input);
[[nodiscard]] bool GetRefineWedge() const;
@@ -189,4 +205,5 @@ public:
[[nodiscard]] double GetResolutionCCTarget() const;
[[nodiscard]] int GetReportShellCount() const;
[[nodiscard]] std::optional<ReportResolutionRange> GetReportResolutionRange_A() const;
[[nodiscard]] const std::vector<SweepSegment> &GetSweepSegments() const;
};
+9
View File
@@ -384,6 +384,15 @@ J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in t
package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399
[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198).
**Several sweeps merged as one** - each sweep's partials combined into fulls on their own, and the
fulls of all the sweeps then scaled together, every sweep with its own frame scales and a relative B
against the joint merge, is how XSCALE merges several data sets (W. Kabsch, "Integration, scaling,
space-group assignment and post-refinement" (2010), above). Reading the radiation dose a sweep was taken
at from its relative B, which orders the sweeps by dose from the data rather than from the order they
were given in, follows K. Diederichs, "Some aspects of quantitative analysis and correction of radiation
damage" (2006), Acta Cryst. D62, 96-101
[doi:10.1107/S0907444905031537](https://doi.org/10.1107/S0907444905031537).
**Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature:
D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography:
Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75
+36 -15
View File
@@ -4,7 +4,7 @@ Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeli
ever sees its own detectors is not tested. The datasets below were collected by other people,
on detectors and in file formats we do not produce ourselves, and are used here to check that
`rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected
at other facilities, eleven on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
at other facilities, thirteen on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
else's detector and someone else's acquisition system. Their authors published all of these for
exactly this kind of reuse, and this page is where we credit them.
@@ -101,6 +101,8 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
| [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC |
| [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset |
| [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor |
| [6OQZ](https://www.rcsb.org/structure/6OQZ) | IRRMC [10.18430/m3.irrmc.5324](https://doi.org/10.18430/m3.irrmc.5324) | Home source, Bruker rotating anode | 1.60 | I 2 2 2 | 92.9 97.9 102.7 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of glucose isomerase from non-merohedrally twinned crystals |
| [6OR0](https://www.rcsb.org/structure/6OR0) | IRRMC [10.18430/m3.irrmc.5325](https://doi.org/10.18430/m3.irrmc.5325) | Home source, Bruker rotating anode | 1.55 | I 21 3 | 78.0 78.0 78.0 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of insulin from non-merohedrally twinned crystals |
| [6OWW](https://www.rcsb.org/structure/6OWW) | Zenodo [10.5281/zenodo.2943248](https://doi.org/10.5281/zenodo.2943248) | ALS 8.3.1 | 3.84 | P 1 21 1 | 85.8 86.0 214.3 90.0 89.9 90.0 | PILATUS3 6M | Crystal structure of a Human Cardiac Calsequestrin Filament Complexed with Ytterbium |
| [6P8J](https://www.rcsb.org/structure/6P8J) | SBGrid [10.15785/sbgrid/671](https://doi.org/10.15785/sbgrid/671) | APS 24-ID-E | 1.47 | P 1 21 1 | 46.5 152.2 88.3 90.0 90.1 90.0 | Dectris Eiger 16M | Structure of P. aeruginosa ATCC27853 CdnD D62N/D64N mutant bound to ATP |
| [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 |
@@ -244,7 +246,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
| [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 |
| [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) |
The table holds 201 PDB entries. Six more datasets, small-molecule / chemical-crystallography
The table holds 203 PDB entries. Eight more datasets, small-molecule / chemical-crystallography
sets kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero
detector 2θ, have no PDB entry and are listed [below](#datasets-with-no-pdb-entry).
@@ -313,7 +315,7 @@ the rest were deleted, so a run over the data directory sees a single collection
| 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å |
| 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** |
**Thirteen further archives hold more than one collection.** Their layout was read from the
**Seventeen further archives hold more than one collection.** Their layout was read from the
image files and repository listings; one sweep is kept for a run over the data directory unless
noted.
@@ -329,6 +331,10 @@ noted.
| lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run |
| 9E2T | one continuous sweep plus screening images | the 2700-frame sweep |
| 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) |
| 6OR0 | four 900 × 0.2° Bruker `.sfrm` runs of one twinned crystal at 10 cm: ω scans at 2θ = 0° (`01`) and 35° (`02`), φ scans on the fixed-χ (54.74°) platform at 2θ = 0° (`03`) and 35° (`04`) | the `01` run (2θ = 0°): the header distance of these SMART 6000 frames is ~4% short of what the data give, which the distance refinement corrects on a 2θ = 0° run and not yet on a swung one |
| 6OQZ | eleven `.sfrm` ω runs of 472-900 × 0.2° at 18 cm, 2θ = 10°, 25° and 45° (low, medium, high resolution) at different φ, plus two 5-frame runs at 2θ = 0° and the orientation-matrix runs `gdri_MATRIX_*` | the low-resolution `01` run (2θ = 10°), for the reason given for 6OR0; three crystals, the test set for multi-lattice integration |
| laspartic | seven Bruker-converted full-imgCIF kappa runs, φ (`01`, 1200 × 0.3°) and ω (`02`-`07`, 292-394 frames) at 2θ = -26.7° | the 1200-frame `01` φ run |
| onitroaniline | the same seven-run kappa layout (`01` 1200 × 0.3° φ, `02`-`07` ω), deposited twice: as Bruker `.sfrm` and as APEX's full-imgCIF conversion of the same frames | the `01` run, read from the `.sfrm` files |
| 7D98 | one 250 × 0.5° miniCBF sweep (frames 11-260), plus four HDF5 masters that repeat frames 61-260 in 60-frame pieces | the miniCBF sweep; the HDF5 files stay beside it and the row is pinned |
| 6H2P | two sweeps of one crystal, 1800 × 0.1° at 0.918 Å and 3600 × 0.1° at 1.893 Å (the Mn K edge) | **both**, as two battery rows, `6h2p_native` and `6h2p_1p89A`; the long-wavelength row has no deposited resolution to compare with |
| 8C3E | four 0.5° ω scans of one crystal at different (φ, χ) settings, 56, 150, 150 and 50 frames (`lcb2_8_newt_001` to `_004`) | the 150-frame `_003` scan |
@@ -349,7 +355,7 @@ noted.
## Datasets published as Raw Data Letters
Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a
Five of the datasets - 6R72, 8QQ7, 6RLR, laspartic and onitroaniline - were published as IUCrData Raw Data Letters, a
format whose purpose is to make raw images citable and re-processable in their own right. The
letters describe the collections and the difficulties in them, and are the reference for what the
data are:
@@ -364,6 +370,15 @@ data are:
IUCrData 7, x220852
[doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR.
- M. Lutz and L. Kroon-Batenburg, "Accurate intensity integration in the twinned γ-form of
*o*-nitroaniline" (2022), IUCrData 7
[doi:10.1107/S2414314622010598](https://doi.org/10.1107/S2414314622010598) - covers
onitroaniline.
- M. Lutz, "Twinned L-aspartic acid" (2025), IUCrData 10
[doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) - covers laspartic:
a twofold rotation about [100] with twin index 2 and obliquity 0.74°, whose coincidence lattice
is the doubled cell first reported for the compound (Bernal, 1931).
The authors of the second letter also published their own reciprocal-space reconstruction of the
6RLR data as a separate Zenodo record,
[10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763).
@@ -437,8 +452,8 @@ The PDB entry names the detector as a Rigaku HyPix-6000HE.
## Deposited models and structure factors
All 201 PDB entries in the table are released, and RCSB reports released structure factors
(`status_code_sf = REL`) for every one of them; the six small-molecule datasets have none. A merged
All 203 PDB entries in the table are released, and RCSB reports released structure factors
(`status_code_sf = REL`) for every one of them; the eight small-molecule datasets have none. A merged
result from this pipeline can therefore be checked against the deposited model or against the
deposited intensities.
@@ -609,8 +624,10 @@ reader that globs `*.img` will pick them up, so they are named here rather than
| `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition |
| `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition |
| `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition |
| `laspartic` | Zenodo record 10.5281/zenodo.15432050 | a small-molecule dataset, not a PDB deposition |
| `onitroaniline` | Zenodo record 10.5281/zenodo.7193538 | a small-molecule dataset, not a PDB deposition |
Five of the six small-molecule sets have a published structure to check a run against. These are
Six of the eight small-molecule sets have a published structure to check a run against. These are
reference values from the literature, not results obtained here.
| Dataset | Space group | Cell (A, deg) | T | Reference |
@@ -620,6 +637,10 @@ reference values from the literature, not results obtained here.
| `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) |
| `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) |
| `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) |
| `laspartic` | `P 1 21 1` (4) | 5.1237 6.9197 7.6006 / 90 100.442 90 | 150 K | twin fractions 0.537/0.463; COD 1577465 - M. Lutz, IUCrData **10** (2025) [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) |
`onitroaniline` has no cell here: the letter describing it could not be retrieved when the row was
added, so a run on it is not scored against a reference.
`cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**,
064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no
@@ -632,20 +653,20 @@ symmetries rather than to be easy to process. The counts below describe where it
like everything else on this page, they are metadata about the depositions and their files, not
measurements.
- **Repository**, over the 207 datasets on this page (the 201 rows of the table and the six
small-molecule sets): IRRMC 95, Zenodo 42, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
- **Repository**, over the 211 datasets on this page (the 203 rows of the table and the eight
small-molecule sets): IRRMC 97, Zenodo 44, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
ESRF 3, UQ eSpace 1.
- **Facility** - counted from the facility part of the Facility / beamline column, the beamline
ignored so that entries deposited with and without one count the same, over the 190 rows that
name one: APS 29, Diamond 21, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9,
SLS 8, SPring-8 8, SOLEIL 7, SSRF 7, PAL/PLS 5, CHESS 4, ALBA 3, Australian Synchrotron 3,
CLSI 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, Photon Factory 2, and one each from NSRRC,
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other eleven
rows were collected on laboratory sources: nine on rotating anodes, one on a liquid-metal jet
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other thirteen
rows were collected on laboratory sources: eleven on rotating anodes, one on a liquid-metal jet
and one on a sealed tube.
- **Crystal system, from the deposited space group of the 201 rows:** monoclinic 53,
orthorhombic 50, tetragonal 30, trigonal 22, hexagonal 20, cubic 13, triclinic 13.
- **Pink beam:** none of these datasets was collected with pink beam. All 201 rows are deposited
- **Crystal system, from the deposited space group of the 203 rows:** monoclinic 53,
orthorhombic 51, tetragonal 30, trigonal 22, hexagonal 20, cubic 14, triclinic 13.
- **Pink beam:** none of these datasets was collected with pink beam. All 203 rows are deposited
as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all but 5REO, which
leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` `M`); 9Q41 is the
one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The
@@ -654,7 +675,7 @@ measurements.
- **Long cell axes:** sixteen rows have a deposited cell axis longer than 320 Å - 4P8R,
4MTK, 7D98, 8V4O, 9ZMU, 9V16, 9Z72, 4NWV, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL.
The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
The marCCD, SMV, Bruker `.sfrm`, full-imgCIF and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with.
## Licences
+30 -6
View File
@@ -30,9 +30,10 @@ rugnux -o myrun dataset_master.h5 # NXmx / EIGER / Jungfraujoch mas
rugnux -o myrun sweep_00001.cbf # CBF, marCCD or SMV sweep: name one frame
rugnux -o myrun dataset_master.h5 model.cif # with a model (or --model model.cif)
rugnux -o myrun x_master.h5 http://host:5232/ # follow a broker's collection while it is written
rugnux -o myrun a_master.h5 b_master.h5 # several sweeps of one crystal, merged as one
```
Chosen when the file's goniometer axis turns. One sweep per run. A model is placed (rigid body, or
Chosen when the file's goniometer axis turns. Several sweeps can be given (below, Inputs). A model is placed (rigid body, or
molecular replacement), refined and validated after merging; it does not change the processing
decisions ([Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md)).
@@ -91,14 +92,36 @@ spot finding or integration ([Re-scaling and re-merging](RUGNUX_ADVANCED.md#re-s
## Inputs
Inputs are files, never a directory, read in the order given (a shell glob gives them sorted by
name). Several inputs are read together only as one stills dataset in `--mode mx`: all the files of
name). Several inputs are one stills dataset in `--mode mx`, or several rotation sweeps (below): all the files of
one dataset - `rugnux run*.h5` (SACLA), `*.cxi`, `*.xtc` (LCLS) - or several Jungfraujoch/NXmx
masters of stills, read as one run: their images one after another, merged together, with the
indexing rate reported per master as well. Such masters must share the detector, wavelength,
distance and beam centre. Several inputs must be one format; a rotation sweep (name one frame of a
CBF/marCCD/SMV sweep), any `--mode` other than `mx`, and following a broker take one. What each
format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read without a
geometry file where their own metadata places the detector
distance and beam centre. Several inputs must be one format; any `--mode` other than `mx`, and
following a broker, take one.
Several rotation sweeps are told apart from the files: one sweep per CBF/marCCD/SMV/SFRM file-name
template (the name with its frame number taken out - any frame of a sweep names the whole sweep, so
`rugnux a_*.cbf` is the one sweep `a`), one per rotation master. Only the inputs given are processed;
other sweeps beside them are named in the log. A mix of rotation and stills masters is refused, and
so are sweeps at different wavelengths, `-s`/`-e` and `--force-still` with several sweeps.
By default (`--sweeps joint`) the sweeps are one dataset, merged in two levels as XSCALE merges the
data sets XDS made of each sweep: each sweep is processed on its own first - indexed, integrated, its
partials combined and scaled, its space group determined - writing only its log `<prefix>_s<k>_log.txt`;
the sweeps are then put on the axes of the one with the most reflections (the change of basis, an
axis permutation or a twin law of the lattice, under which their merged intensities correlate best;
a sweep correlating below 0.5 on every axes is left out and named in the log), and their fulls are
scaled and merged together: a per-frame scale within each sweep, the sweeps' dose order read from
their relative B factors (not from the input order or the header times), a decay along that order or
one B per sweep, whichever predicts held-out frames better, then the space-group search, the
resolution cut and everything after it on the joint data, into `<prefix>.*`. The detector-dependent
corrections (absorption and detector surfaces) are each sweep's own. The unmerged MTZ holds the joint
merge's scaled observations in the batches of their own sweeps (sweep `s` of the merge: batches
`s * base + frame + 1`, `base` the power of ten past the longest sweep), under the batch headers that
sweep's own run wrote; the report names each sweep in section `1b. SWEEPS`. `--sweeps separate` processes each sweep alone instead, as if it were the only
input, into `<prefix>_s<k>_*` (k = 1, 2, ... in the order given).
What each format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read
without a geometry file where their own metadata places the detector
([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)).
An atomic model and a reference reflection file can be given among the inputs, in any order,
@@ -130,6 +153,7 @@ With the input already on disk, the URL is ignored.
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `<prefix>.*`; `separate` processes each alone into `<prefix>_s<k>_*` |
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
### Mode
+26 -2
View File
@@ -120,7 +120,8 @@ The header block above section 1 records **how the result was produced**: `RUGNU
Rates, per-image costs and progress remain on stdout only.
Sections, in order: the `SUMMARY` ([below](#the-summary-and-what-the-run-decided)), then
`1. DATA SET AND GEOMETRY`, `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
`1. DATA SET AND GEOMETRY`, on a joint merge of several sweeps `1b. SWEEPS`
([below](#several-sweeps-merged-as-one)), `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
was given (`--model`, `--mr`, or a model file among the inputs), `5. MODEL VALIDATION` - which also
holds the keys of [molecular replacement](#molecular-replacement) and of the
[refinement](#refinement-refine_) where they ran. A `--mode refine` report has one section,
@@ -252,7 +253,7 @@ and a run that produced a textbook data set read identically for their first thr
`GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`,
`LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_CELL_MISMATCH`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
`FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`, `SCALING_NOT_CONVERGED`, `HARMONIC_CONTAMINATION`,
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`. `NONE` when nothing fired. A code appears if and only if its
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`, `SWEEP_LEFT_OUT`. `NONE` when nothing fired. A code appears if and only if its
warning fired, so the flags and the `WARNING:` lines are two renderings of one list. Thresholds are
deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on
data that turn out fine — the closed
@@ -286,6 +287,29 @@ and a run that produced a textbook data set read identically for their first thr
Then the numbered sections: **1. DATA SET AND GEOMETRY**, **2. CRYSTAL**,
**3. MERGED DATA**, **4. DIAGNOSTICS**, and **5. MODEL VALIDATION** only where a model was given.
## Several sweeps merged as one
A run of several sweeps merged as one (`--sweeps joint`, the default with several rotation inputs, see
[Command-line options](RUGNUX_CLI.md)) describes the joint merge everywhere, and adds section
**1b. SWEEPS**; a run of one input has no such section and no such keys.
- **`SWEEP_COUNT=`** the sweeps given, **`SWEEP_REFERENCE=`** the one (1-based) the others were put on
the axes of - the one with the most reflections - and **`SWEEP_DECAY_MODEL=`** how the decay over the
sweeps was modelled: `dose-linear` (a B rising along the dose order), `per-sweep B` (one B a sweep) or
`none`, whichever predicted held-out frames better.
- Per sweep `k`: **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep
gave processed on its own - **`SWEEP_k_SPACE_GROUP_ALONE=`**, **`SWEEP_k_CELL=`**,
**`SWEEP_k_CC_HALF_ALONE=`**, **`SWEEP_k_COMPLETENESS_ALONE=`**, **`SWEEP_k_RESOLUTION_ALONE=`**; and,
for a sweep in the joint merge, **`SWEEP_k_REINDEX_OP=`** (onto the reference's axes) with
**`SWEEP_k_REINDEX_CC=`** (the merged-intensity correlation under it), **`SWEEP_k_BATCH_FIRST=`** /
**`SWEEP_k_BATCH_LAST=`** (its batches in the unmerged MTZ: sweep `s` of the merge is
`s * base + frame + 1`, `base` the power of ten past the longest sweep), **`SWEEP_k_SCALE=`** (the
median scale of its frames in the joint merge), **`SWEEP_k_RELATIVE_B=`** (its B against the joint
merge, the least damaged sweep at 0) and **`SWEEP_k_DOSE_RANK=`** (its place, from 1, in the dose
order those B factors decide - not the order the sweeps were given in, nor their header times).
- A sweep given but not merged - it gave no merge, or its intensities correlate with the reference
sweep's on no axes - is named by a `SWEEP_LEFT_OUT` warning.
## The reference-range table
`--report-resolution <dmin>[,<dmax>]` adds a second block of merging statistics to section 3, the
+325 -29
View File
@@ -365,6 +365,7 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
// Decay + absorption correction surfaces are one master toggle (on by default; both cross-validated,
// so a no-op when their systematic is absent).
refine_decay_b = s.GetCorrectionSurfaces();
segments = s.GetSweepSegments();
absorption_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
modulation_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
relative_b_deg = s.GetRelativeBDegrees(); // per-batch relative-B batch width (0 = off)
@@ -531,11 +532,28 @@ void RotationScaleMerge::IngestHost() {
if (frame_limit >= 0) n_frames = std::min(n_frames, frame_limit);
partials_released = false;
resident_ingest = false;
frame_segment.clear();
if (!segments.empty()) {
frame_segment.assign(n_frames, -1);
int32_t next = 0;
for (size_t k = 0; k < segments.size(); ++k) {
const SweepSegment &seg = segments[k];
if (seg.first_frame != next || seg.n_frames <= 0 || seg.first_frame + seg.n_frames > n_frames)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: the sweep segments do not cover the frames in order");
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
frame_segment[f] = static_cast<int32_t>(k);
next = seg.first_frame + seg.n_frames;
}
if (next != n_frames)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: the sweep segments do not cover the frames in order");
}
#ifdef JFJOCH_USE_CUDA
// Probed before the observations are built: whether the build lays down the full Obs array or
// only the ingest arrays depends on whether the device pipeline will be active.
gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
gpu_active_ = gpu_->Available() && !scale_on_cpu_;
gpu_active_ = gpu_->Available() && !scale_on_cpu_ && segments.empty();
resident_ingest = gpu_active_ && !observation_dump;
#endif
@@ -605,7 +623,12 @@ void RotationScaleMerge::IngestHost() {
}
});
MeasureIncidentFlux(mean_bkg);
// Fulls of several sweeps carry each sweep's own background: not one flux meter, so the per-frame
// scale alone takes the flux.
if (segments.empty())
MeasureIncidentFlux(mean_bkg);
else
frame_flux.assign(n_frames, 1.0);
// The occupied range of h, which is the buckets the sort works on.
int b_lo = H_ALL, b_hi = -1;
@@ -728,12 +751,19 @@ void RotationScaleMerge::IngestHost() {
}
});
}
BuildInRangeObservations(keys, static_cast<int>(total), SmoothedFrameGeometry());
// Pre-combined fulls have no rocking geometry left to smooth or to recompute a partiality from.
BuildInRangeObservations(keys, static_cast<int>(total),
segments.empty() ? SmoothedFrameGeometry() : SmoothedGeometry{});
rawrun_group.assign(rawrun_start.size(), -1);
logger->Info("RotationScaleMerge: ingested {} partial observations from {} frames ({} distinct hkl)",
n_partials_obs, n_frames, rawrun_start.size());
SmoothMosaicityAndPartiality();
if (segments.empty()) {
SmoothMosaicityAndPartiality();
} else {
ingest_zeta.reset();
ingest_d.reset();
}
ParallelChunks(static_cast<int>(partials.size()), ThreadsForWork(partials.size(), nthreads),
[&](int lo, int hi) {
for (int i = lo; i < hi; ++i)
@@ -2174,6 +2204,7 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
has_fulls[f] = frame_obs_count[f] > 0 ? 1 : 0;
}
double best_step = INFINITY, lambda = 0.0;
std::vector<double> segment_lambda(segments.size(), 0.0);
bool lambda_kept = false;
std::vector<double> steps, g_best;
AndersonMixing anderson(SCALING_ANDERSON_DEPTH);
@@ -2188,9 +2219,16 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
if (fitted)
y[f] += g_fit[f] > 0.0 ? std::max(std::log(g_fit[f] / g[f]), -1.0) : -1.0;
}
if (!lambda_kept)
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
const std::vector<double> x = SmoothLogScale(y, J, lambda);
std::vector<double> x;
if (segments.empty()) {
if (!lambda_kept)
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
x = SmoothLogScale(y, J, lambda);
} else {
x = SmoothLogScaleBySegment(y, J, segment_lambda, !lambda_kept);
std::vector<double> l = segment_lambda;
lambda = median_of(l);
}
for (int f = 0; f < n_frames; ++f)
g_new[f] = std::exp(x[f]);
const double g_ref = TypicalFrameScale(g_new, n_obs);
@@ -2342,8 +2380,14 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
return o.group >= 0 && o.corr > 0.0f && std::isfinite(o.corr) && (o.d > 0.0f)
&& o.partiality >= min_partiality;
};
// Where an observation sits on the dose axis: its frame, or with several sweeps its place in the dose
// order the data put them in (MeasureSegmentDose).
auto dose = [&](const auto &o) {
return segments.empty() ? static_cast<double>(o.image_number)
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
};
auto decay_factor = [&](const auto &o, double slope) {
return std::exp(slope * (o.image_number - fcenter) * s2_of(o.d));
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
};
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
@@ -2358,6 +2402,8 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
BuildDecayTerms(n_groups, t);
decay_terms_kept = {};
decay_terms_kept_valid = false;
if (!segments.empty())
MeasureSegmentDose(t, n_groups);
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
std::vector<double> sw(n_groups), swI(n_groups);
@@ -2385,7 +2431,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
const double w = (Is / sc) * (Is / sc);
const double x = (o.image_number - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
@@ -2435,6 +2481,125 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
// sit at the same s^2 but different frames, so the correction spreads them) - net harmful. slope =
// 2 dB/dframe, so the total relative-B change across the run is |slope/2 * n_frames|.
constexpr double DECAY_MIN_DELTA_B = 2.0; // A^2, minimum total relative-B over the run to engage
if (!segments.empty()) {
// Two hypotheses of how damage accrued over several sweeps, decided on held-out frames: a B rising
// linearly along the dose order (frame_dose), or one B per sweep - a step at each sweep boundary,
// flat within a sweep. Both are regressions of y = ln(I_ref / I_obs) on s^2 = 1/(4 d^2), the ramp
// on x = (dose - centre) s^2, the steps on x = s^2 with a slope per sweep, at the weights the slope
// fit above uses and within each frame (below). Each is fitted on the even frames and its
// weighted squared residual taken on the odd ones, against their own reference, and the other way
// round; the hypothesis that predicts the held-out frames better is applied, where it beats no
// decay at all by the margin a correction has to clear on one sweep (and the physical floor).
// The held-out frames' residual is read at weights fixed by the observations, so a correction
// cannot win by reshaping sigma.
using Sums = std::array<double, 5>; // Sw, Sx, Sy, Sxx, Sxy
struct Line { double a = 0.0, b = 0.0; };
auto line_of = [](const Sums &S) {
Line l;
const double var = S[0] > 0.0 ? S[3] - S[1] * S[1] / S[0] : 0.0;
if (var > 0.0) l.b = (S[4] - S[1] * S[2] / S[0]) / var;
if (S[0] > 0.0) l.a = (S[2] - l.b * S[1]) / S[0];
return l;
};
auto reference = [&](int parity) {
std::fill(sw.begin(), sw.end(), 0.0);
std::fill(swI.begin(), swI.end(), 0.0);
for (const DecayTerm &o : t.term) {
if (parity >= 0 && (o.frame & 1) != parity) continue;
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
sw[o.group] += w;
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
}
};
// Every term of `parity` with its y and weight against the current reference.
auto for_terms = [&](int parity, const auto &fn) {
for (const DecayTerm &o : t.term) {
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group];
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
fn(o, std::log(Iref / Is), (Is / sc) * (Is / sc));
}
};
auto x_ramp = [&](const DecayTerm &o) { return (dose(o) - fcenter) * s2_of(o.d); };
// The terms of `parity` with y and both x taken about their frame's weighted mean: each frame has a
// scale of its own (the per-frame G), which took up its part of the B averaged over its
// reflections, so only how y varies with resolution WITHIN a frame is decay.
struct Point { int32_t seg; double w, y, xr, xs; };
auto points = [&](int parity) {
reference(parity);
std::vector<Point> pts;
std::vector<int32_t> frame;
std::vector<std::array<double, 4>> mean(n_frames, std::array<double, 4>{}); // Sw, Sy, Sxr, Sxs
for_terms(parity, [&](const DecayTerm &o, double y, double w) {
pts.push_back({frame_segment[o.frame], w, y, x_ramp(o), s2_of(o.d)});
frame.push_back(o.frame);
auto &m = mean[o.frame];
m[0] += w; m[1] += w * y; m[2] += w * pts.back().xr; m[3] += w * pts.back().xs;
});
for (size_t k = 0; k < pts.size(); ++k) {
const auto &m = mean[frame[k]];
pts[k].y -= m[1] / m[0]; pts[k].xr -= m[2] / m[0]; pts[k].xs -= m[3] / m[0];
}
return pts;
};
struct Fit { double ramp = 0.0; std::vector<double> step; };
auto fit = [&](int parity) {
Sums r{};
std::vector<Sums> st(segments.size(), Sums{});
for (const Point &q : points(parity)) {
r[0] += q.w; r[1] += q.w * q.xr; r[2] += q.w * q.y; r[3] += q.w * q.xr * q.xr; r[4] += q.w * q.xr * q.y;
Sums &a = st[q.seg];
a[0] += q.w; a[1] += q.w * q.xs; a[2] += q.w * q.y; a[3] += q.w * q.xs * q.xs; a[4] += q.w * q.xs * q.y;
}
Fit out{line_of(r).b, {}};
for (const Sums &a : st) out.step.push_back(line_of(a).b);
return out;
};
// Held-out weighted squared residuals of no decay, the ramp and the steps.
auto score = [&](int test, const Fit &m, std::array<double, 3> &ssr) {
for (const Point &q : points(test)) {
const double e1 = q.y - m.ramp * q.xr, e2 = q.y - m.step[q.seg] * q.xs;
ssr[0] += q.w * q.y * q.y; ssr[1] += q.w * e1 * e1; ssr[2] += q.w * e2 * e2;
}
};
std::array<double, 3> ssr{};
const Fit even = fit(0), odd = fit(1);
score(1, even, ssr);
score(0, odd, ssr);
const Fit all = fit(-1);
const double slope = all.ramp; // the slope decay_factor applies
std::vector<double> b(segments.size());
double b_mid = 0.0, b_lo = INFINITY, b_hi = -INFINITY;
for (size_t k = 0; k < segments.size(); ++k) {
b[k] = 0.5 * all.step[k];
b_mid += b[k] * segments[k].n_frames / static_cast<double>(n_frames);
b_lo = std::min(b_lo, b[k]);
b_hi = std::max(b_hi, b[k]);
}
const double delta_lin = std::fabs(0.5 * slope * n_frames), delta_step = b_hi - b_lo;
const bool lin_ok = delta_lin >= DECAY_MIN_DELTA_B && ssr[1] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
const bool step_ok = delta_step >= DECAY_MIN_DELTA_B && ssr[2] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
const bool use_step = step_ok && (!lin_ok || ssr[2] < ssr[1]);
const bool use_lin = lin_ok && !use_step;
segment_decay_model = use_step ? "per-sweep B" : use_lin ? "dose-linear" : "none";
const double rel = ssr[0] > 0.0 ? 100.0 / ssr[0] : 0.0;
logger->Info("Decay over {} sweeps, held-out residual against no decay: dose-linear (dB = {:.2f} A^2 over "
"the dose order) {:.1f}%, one B per sweep (spread {:.2f} A^2) {:.1f}% => {}", segments.size(),
delta_lin, ssr[1] * rel, delta_step, ssr[2] * rel, segment_decay_model);
if (use_step || use_lin)
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) {
for (int i = lo; i < hi; ++i) {
Obs &o = fulls[i];
if (!usable(o)) continue;
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
: decay_factor(o, slope);
o.corr = static_cast<float>(o.corr * f);
}
});
return;
}
const double slope = fit_slope(-1);
const double total_delta_B = std::fabs(0.5 * slope * n_frames);
if (total_delta_B < DECAY_MIN_DELTA_B) {
@@ -2460,6 +2625,92 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
0.5 * slope * n_frames, 0.5 * slope);
}
std::vector<double> RotationScaleMerge::SegmentB(const DecayTerms &t, int n_groups, int parity) const {
// Per segment, the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against
// s^2 = 1/(4 d^2), which is 2 B, with an intercept of its own (the scale is the per-frame G's), over
// the frames of `parity` (< 0: all) against a reference made of the same frames. NaN where a segment
// has nothing to fit.
const size_t ns = segments.size();
std::vector<double> sw(n_groups, 0.0), swI(n_groups, 0.0);
for (const DecayTerm &o : t.term) {
if (parity >= 0 && (o.frame & 1) != parity) continue;
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
sw[o.group] += w;
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
}
std::vector<std::array<double, 5>> S(ns, std::array<double, 5>{}); // Sw, Sx, Sy, Sxx, Sxy
for (const DecayTerm &o : t.term) {
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
const double Iref = swI[o.group] / sw[o.group];
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
const double w = (Is / sc) * (Is / sc);
const double x = 1.0 / (4.0 * static_cast<double>(o.d) * o.d), y = std::log(Iref / Is);
auto &a = S[frame_segment[o.frame]];
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
}
std::vector<double> b(ns, NAN);
for (size_t k = 0; k < ns; ++k) {
const auto &a = S[k];
const double var = a[0] > 0.0 ? a[3] - a[1] * a[1] / a[0] : 0.0;
if (var > 0.0)
b[k] = 0.5 * (a[4] - a[1] * a[2] / a[0]) / var;
}
return b;
}
void RotationScaleMerge::MeasureSegmentDose(const DecayTerms &t, int n_groups) {
// A relative B per data set against the joint merge, as XSCALE fits it: Kabsch (2010) Acta Cryst.
// D66, 133-144; the dose read from it: Diederichs (2006) Acta Cryst. D62, 96-101.
// Radiation damage raises the relative B of a sweep collected later, so the B factors of the
// segments against their joint merge give the order the crystal took the dose in - read from the
// data, not from the order the sweeps were given in or the times their headers state. Per segment,
// the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against s^2 = 1/(4 d^2) is 2 B,
// with an intercept of its own (the scale is the per-frame G's). The least damaged segment is B = 0,
// and its frames start the dose axis.
const size_t ns = segments.size();
const std::vector<double> b = SegmentB(t, n_groups, -1);
// A segment with nothing to measure keeps its place in the input order, after the measured ones.
std::vector<int> order(ns);
std::iota(order.begin(), order.end(), 0);
std::stable_sort(order.begin(), order.end(), [&](int p, int q) {
if (std::isfinite(b[p]) != std::isfinite(b[q])) return std::isfinite(b[p]);
return std::isfinite(b[p]) && b[p] < b[q];
});
const double b0 = std::isfinite(b[order[0]]) ? b[order[0]] : 0.0;
frame_dose.assign(n_frames, 0.0);
double offset = 0.0;
std::string text;
for (size_t rank = 0; rank < ns; ++rank) {
const int k = order[rank];
const SweepSegment &seg = segments[k];
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
frame_dose[f] = offset + (f - seg.first_frame);
offset += seg.n_frames;
segment_results[k].relative_b = b[k] - b0;
segment_results[k].dose_rank = static_cast<int>(rank);
text += fmt::format("{}segment {} (B {:+.2f} A^2)", rank == 0 ? "" : ", ", k + 1, b[k] - b0);
}
logger->Info("Dose order of the {} sweep segments, from their relative B against the joint merge: {}",
ns, text);
}
std::vector<double> RotationScaleMerge::SmoothLogScaleBySegment(const std::vector<double> &y,
const std::vector<double> &J,
std::vector<double> &lambda, bool choose) const {
std::vector<double> x(y.size());
for (size_t k = 0; k < segments.size(); ++k) {
const auto lo = y.begin() + segments[k].first_frame, hi = lo + segments[k].n_frames;
const auto jlo = J.begin() + segments[k].first_frame;
const std::vector<double> ys(lo, hi), Js(jlo, jlo + segments[k].n_frames);
if (choose)
lambda[k] = ChooseLogScaleSmoothing(ys, Js, std::max(1, rocking_event_frames_at_start), nthreads);
const std::vector<double> xs = SmoothLogScale(ys, Js, lambda[k]);
std::copy(xs.begin(), xs.end(), x.begin() + segments[k].first_frame);
}
return x;
}
// A^2, the bound the Gauss-Seidel solve clamps each batch's relative-B to. It is a numerical guard against
// divergence, never an answer: a run of identical values in the solved curve means the solve reached the
// bound, not that the crystal did anything. The CORRECTION keeps the tight bound it was tuned with; the
@@ -4716,12 +4967,15 @@ void RotationScaleMerge::Combine() {
size_t i = 0;
while (i < ev.size()) {
size_t kk = i + 1;
float last_frame = partials[ev[i]].r->image_number;
while (kk < ev.size()) {
const float frame = partials[ev[kk]].r->image_number;
if (frame - last_frame > max_frame_gap) break;
last_frame = frame;
++kk;
// A pre-combined full is a whole event already (SweepSegment).
if (segments.empty()) {
float last_frame = partials[ev[i]].r->image_number;
while (kk < ev.size()) {
const float frame = partials[ev[kk]].r->image_number;
if (frame - last_frame > max_frame_gap) break;
last_frame = frame;
++kk;
}
}
double pooled_bkg = 0.0;
@@ -5862,18 +6116,31 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
// reflections are) - each with the sigma the merge weighted it by.
if (export_scaled_fulls && full_stats && !for_search) {
// Which fulls, in order, and then each one's record on its own - on all threads.
// The fulls of one sweep of several (uncut) also hand back the reflections the group extinguishes:
// a joint merge searches its own space group, and the screw axes are decided on exactly these.
const gemmi::GroupOps ops = Group().operations();
auto extinguished = [&](const Obs &o) {
return export_scaled_fulls_uncut && o.group < 0 && frame_cell_ok[o.frame] && o.corr > 0.0f
&& std::isfinite(o.corr) && std::isfinite(o.I) && o.d > 0.0f
&& ops.is_systematically_absent({{o.h, o.k, o.l}});
};
const std::vector<int32_t> kept = KeptIndices(n_full, nthreads, [&](int i) {
const int g = mf.group[i];
return g >= 0 && !rejected_obs[i] && std::isfinite(merged_I[g])
&& !(effective_d_min && acc[g].d < *effective_d_min);
if (g < 0)
return extinguished(fulls[i]);
return !rejected_obs[i] && std::isfinite(merged_I[g])
&& (export_scaled_fulls_uncut || !(effective_d_min && acc[g].d < *effective_d_min));
});
result.scaled_fulls.resize(kept.size());
ParallelChunks(static_cast<int>(kept.size()), ThreadsForWork(kept.size(), nthreads), [&](int lo, int hi) {
for (int j = lo; j < hi; ++j) {
const Obs &o = fulls[kept[j]];
const float I_corr = o.I * o.corr;
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, corrected_sigma(o, I_corr, o.sigma * o.corr),
o.image_number};
const float sigma = o.group >= 0 ? corrected_sigma(o, I_corr, o.sigma * o.corr)
: model_sigma(o, I_corr, o.sigma * o.corr);
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, sigma,
o.image_number, o.corr * o.corr * o.var_bkg, o.corr * o.var_per_I,
o.capture, o.d};
}
});
}
@@ -6575,9 +6842,14 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
if (effective_d_min && m.d < *effective_d_min) continue;
sd_min = std::min(sd_min, m.d); sd_max = std::max(sd_max, m.d);
}
// Nothing (or one resolution) merged: no rocking event of the lattice made a full reflection, which
// is what a lattice that does not describe the frames ends in.
if (!(sd_min < sd_max && sd_min > 0.0f))
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"RotationScaleMerge: resolution calculation failed");
fmt::format("nothing to merge ({} merged reflections): no rocking event of "
"the lattice was recorded whole enough to make a full reflection, so "
"the lattice probably does not describe these frames",
result.merged.size()));
// The completeness denominator is counted over the range the run DECLARES, not the range that
// survived: the shell grid shares its bounds (so nothing is enumerated outside a shell) and runs
// to the declared low-resolution limit, or over the whole sphere - 1/d^2 down to 0, spelled as an
@@ -6694,6 +6966,12 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st
const gemmi::SpaceGroup *group) {
const ScopedGpuPin pin(gpu_device); // see IngestHost
run_group = group;
// Pre-combined fulls: there are no partials to fit a scale on, and every event is one frame.
if (!segments.empty()) {
scale_partials_choice = false;
rocking_event_frames_at_start = 1;
rocking_events_at_start = n_partials_obs;
}
struct ClearGroup { const gemmi::SpaceGroup *&g; ~ClearGroup() { g = nullptr; } } clear_group{run_group};
// Whether a frame's scale can be fitted on its partials depends on whether the partials tell a change
// of scale from an error of the partiality model. The count of rocking events per frame is the prior
@@ -7044,7 +7322,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// kernels on it, and the restore re-uploads.
// --- 2a. On the de-novo search pass only, drop observations whose Lorentz geometry is poor. ---
if (for_search && search_min_zeta > 0.0) {
if (for_search && search_min_zeta > 0.0 && segments.empty()) { // a full has no rocking geometry
int64_t n_dropped = 0;
bool zeta_on_gpu = false;
#ifdef JFJOCH_USE_CUDA
@@ -7354,20 +7632,27 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// is a correction to the merged INTENSITIES, and a pass whose intensities are thrown away has
// nowhere to put one - the fitted gain lives on the fulls, which the next pass rebuilds. ---
const bool refine_surfaces = !for_search && full_stats;
// What runs along the frames of ONE sweep - the damage monitor, the sweep ledger, the per-batch B and
// the surfaces, each read in one goniometer and one detector setting - has no meaning across the
// segments of several, and is left out there. The decay is fitted along the dose order instead.
const bool along_frames = segments.empty();
const bool corrections =
refine_surfaces && (refine_decay_b || absorption_iter > 0 || modulation_iter > 0 || relative_b_deg > 0.0);
refine_surfaces && (refine_decay_b || (along_frames && (absorption_iter > 0 || modulation_iter > 0
|| relative_b_deg > 0.0)));
segment_results.assign(segments.size(), Result::Segment{});
segment_decay_model.clear();
// Radiation-damage monitor: measure the per-batch relative-B on the scaled fulls BEFORE any correction
// (report-only; captures the full damage signature, not a residual). Skipped on the P1 search pass.
std::unique_ptr<TimingMark> sub_mark;
auto sub = [&](const char *n) { sub_mark.reset(); sub_mark = std::make_unique<TimingMark>(*logger, std::string("rsm/s ") + n); };
sub("damage B");
if (refine_surfaces)
if (refine_surfaces && along_frames)
MeasureRadiationDamageB(n_groups);
// Sweep-quality diagnostic, on the per-frame scale the partial scaling just fitted (the flux is
// already out of it) and the per-frame CC computed above. It drops nothing itself; the ranges it
// segments are the ledger 4c below decides on.
sub("sweep quality");
if (refine_surfaces)
if (refine_surfaces && along_frames)
MeasureSweepQuality(partial_scaled, cc, cc_n);
// CC1/2 of these same fulls with the surfaces not yet folded in, for the caller's cross-pass quality
// guard: the pass this one is judged against fits no surfaces of its own, so it is the uncorrected
@@ -7393,7 +7678,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
decay_terms_kept = {};
decay_terms_kept_valid = false;
sub("relative B");
if (refine_surfaces && relative_b_deg > 0.0)
if (refine_surfaces && along_frames && relative_b_deg > 0.0)
RefineRelativeB(n_groups); // per-batch relative-B on top of the single decay slope
// The three surfaces overlap, so the one fitted first gets the first claim and the others are tested
// on what it left - the order decides what is adopted. Detector first: every frame of the sweep
@@ -7409,16 +7694,16 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// the grid is then tested on what the smooth surface left, and where that surface is refused the
// grid sees exactly what it saw before.
sub("modulation");
if (refine_surfaces && modulation_iter > 0)
if (refine_surfaces && along_frames && modulation_iter > 0)
RefineModulation(modulation_iter, n_groups);
sub("absorption time");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorptionTime(absorption_iter, n_groups);
sub("absorption SH");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorptionSH(absorption_iter, n_groups);
sub("absorption gonio");
if (refine_surfaces && absorption_iter > 0)
if (refine_surfaces && along_frames && absorption_iter > 0)
RefineAbsorption(absorption_iter, n_groups);
sub("disposition");
@@ -7459,6 +7744,17 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
// its CC1/2 is the answer without a second merge. A caller that did not ask keeps the NaN.
if (measure_cc_before_corrections)
r.cc_half_before_corrections = cc_half_before_corrections.value_or(r.statistics.overall.cc_half);
// Each segment's scale: the median of its frames' scales, at the run's gauge.
for (size_t k = 0; k < segments.size(); ++k) {
std::vector<double> g;
for (int32_t f = segments[k].first_frame; f < segments[k].first_frame + segments[k].n_frames; ++f)
if (fulls_frame_count[f] > 0 && std::isfinite(g_full[f]) && g_full[f] > 0.0)
g.push_back(g_full[f]);
if (!g.empty())
segment_results[k].scale = median_of(g);
}
r.segments = segment_results;
r.segment_decay_model = segment_decay_model;
// The fulls and the merge's working arrays are rebuilt by every arm before they are read, so they go
// with it: kept, they sat beside the next arm's combine and through everything the caller did between
// merges - on a large cell in P1, the fulls and about 330 bytes per ASU group, gigabytes. Faulting them
@@ -11,6 +11,7 @@
#include <memory>
#include <new>
#include <optional>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
@@ -139,6 +140,16 @@ public:
// The fulls the merge kept, unmerged (see ScaledFull): what passed the outlier tests, inside the
// resolution cut. Filled only when SetExportScaledFulls asked for it, and never on a search merge.
std::vector<ScaledFull> scaled_fulls;
// Per sweep segment (SetSweepSegments), in segment order; empty without segments.
struct Segment {
double scale = NAN; // median per-frame scale of the segment's fulls (run gauge)
double relative_b = NAN; // A^2, against the joint merge, the least damaged segment at 0
int dose_rank = -1; // the segment's place in the dose order those B factors decide
};
std::vector<Segment> segments;
// How the decay over the segments was modelled - "dose-linear" (along the dose order), "per-sweep B"
// (one B a segment) or "none" - as the held-out frames decided; empty without segments or decay.
std::string segment_decay_model;
};
// experiment: read live (its space group is changed by the caller between Run() calls).
@@ -288,6 +299,16 @@ public:
// Merge only the first n frames of the outcomes (by default all of them). Set before Ingest().
void SetFrameLimit(int n) { frame_limit = n; }
// The outcomes are several sweeps of pre-combined fulls laid end to end (SweepSegment; by default the
// experiment's ScalingSettings::GetSweepSegments). Set before Ingest(); the segments must cover the
// frames in order. Such an engine runs on the host: its fulls are a few per frame where a sweep's
// partials are millions, and the device pipeline is the partials'. Without segments nothing of this runs.
void SetSweepSegments(std::vector<SweepSegment> s) { segments = std::move(s); }
// Whether the fulls handed back (SetExportScaledFulls) are all the merge kept, past its resolution cut
// too: the fulls of one sweep of several, which the joint merge cuts again on its own.
void SetExportScaledFullsUncut(bool on) { export_scaled_fulls_uncut = on; }
// The GPU the engine runs on (device 0 by default). Set before Ingest(). The device only says where
// the merge runs: on cards of one model the result is the same on any of them.
void SetGpuDevice(int device) { gpu_device = device; }
@@ -724,8 +745,18 @@ private:
std::vector<int64_t> last_cc_n;
std::vector<uint8_t> last_frame_scaled;
int frame_limit = -1; // see SetFrameLimit
std::vector<SweepSegment> segments; // see SetSweepSegments
std::vector<int32_t> frame_segment; // each frame's segment (Ingest; empty without segments)
std::vector<Result::Segment> segment_results; // the last arm's, copied into its Result
// Each frame's place on the dose axis the decay is fitted along: the frames of the segments laid end
// to end in the order the segments' relative B factors put them (MeasureSegmentDose).
std::vector<double> frame_dose;
// SmoothLogScale / ChooseLogScaleSmoothing within each segment on its own; lambda per segment.
std::vector<double> SmoothLogScaleBySegment(const std::vector<double> &y, const std::vector<double> &J,
std::vector<double> &lambda, bool choose) const;
int gpu_device = 0; // see SetGpuDevice
bool export_scaled_fulls = false; // see SetExportScaledFulls
bool export_scaled_fulls_uncut = false; // see SetExportScaledFullsUncut
bool french_wilson = true; // see SetFrenchWilson
// --- helpers (each a flat pass; see the .cpp) ---
@@ -958,6 +989,12 @@ private:
std::vector<int32_t> g_start;
};
void BuildDecayTerms(int n_groups, DecayTerms &t) const;
// Per segment: the relative B against the merged reference, and the dose order it puts the segments
// in (segment_results, frame_dose).
void MeasureSegmentDose(const DecayTerms &t, int n_groups);
// Each segment's B against a reference made of the same frames: those of `parity` (< 0: all).
std::vector<double> SegmentB(const DecayTerms &t, int n_groups, int parity) const;
std::string segment_decay_model; // the last decay's choice (Result::segment_decay_model)
// The terms the radiation-damage monitor built, handed on to RefineDecay: nothing between the two
// touches corr, so they are the terms RefineDecay would build. Empty outside that stretch of RunArm.
DecayTerms decay_terms_kept;
+6
View File
@@ -6,6 +6,8 @@ ADD_LIBRARY(JFJochReader STATIC
JFJochMultiReader.h
SweepLayout.cpp
SweepLayout.h
ImgCIF.cpp
ImgCIF.h
MiniCBF.cpp
MiniCBF.h
JFJochCBFReader.cpp
@@ -29,6 +31,10 @@ ADD_LIBRARY(JFJochReader STATIC
JFJochXTCReader.h
NXmxModules.cpp
NXmxModules.h
SFRM.cpp
SFRM.h
JFJochSFRMReader.cpp
JFJochSFRMReader.h
HDF5ImageLocator.cpp
JFJochStartMessageReader.cpp
JFJochStartMessageReader.h
+415
View File
@@ -0,0 +1,415 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "ImgCIF.h"
#include <algorithm>
#include <cctype>
#include <cmath>
#include <set>
#include "../common/JFJochException.h"
#include "../common/DiffractionGeometry.h"
#include "../common/JFJochMath.h"
namespace imgcif {
namespace {
using Vec = std::array<double, 3>;
JFJochException Invalid(const std::string &what) {
return {JFJochExceptionCategory::InputParameterInvalid, "imgCIF header: " + what};
}
std::string Field(const std::map<std::string, std::string> &row, const std::string &name) {
const auto it = row.find(name);
return (it == row.end()) ? std::string() : it->second;
}
// A CIF number. "?" (unknown) and "." (not applicable) are how a table leaves a cell empty - an angle
// column on a translation row - and read as nothing.
std::optional<double> Number(const std::string &s) {
if (s.empty() || s == "?" || s == ".")
return {};
try {
size_t used = 0;
const double v = std::stod(s, &used);
// CIF allows a standard uncertainty in brackets after the value: "0.71073(2)"
if (used != s.size() && s[used] != '(')
throw Invalid("malformed number '" + s + "'");
return v;
} catch (const std::logic_error &) {
throw Invalid("malformed number '" + s + "'");
}
}
double NumberOr(const std::string &s, double fallback) {
return Number(s).value_or(fallback);
}
Vec Rotate(double angle_deg, Vec axis, const Vec &p) {
const double len = std::sqrt(axis[0] * axis[0] + axis[1] * axis[1] + axis[2] * axis[2]);
for (double &c: axis)
c /= len;
const double t = angle_deg * 3.14159265358979323846 / 180.0;
const double c = std::cos(t), s = std::sin(t);
const double dot = axis[0] * p[0] + axis[1] * p[1] + axis[2] * p[2];
const Vec cross = {axis[1] * p[2] - axis[2] * p[1],
axis[2] * p[0] - axis[0] * p[2],
axis[0] * p[1] - axis[1] * p[0]};
Vec out{};
for (int i = 0; i < 3; i++)
out[i] = p[i] * c + cross[i] * s + axis[i] * dot * (1.0 - c);
return out;
}
const Axis &Find(const Description &d, const std::string &id) {
const auto it = d.axes.find(id);
if (it == d.axes.end())
throw Invalid("axis '" + id + "' is used but not defined in the _axis table");
return it->second;
}
// Carry a point from the frame of axis `id` out to the laboratory: each axis on the way applies its own
// motion at its setting (or at the one given in `settings`), then moves by its offset from the axis it
// depends on. A child's vector is stated in its parent's frame, which is exactly what applying the
// parent after the child does to it.
Vec ToLab(const Description &d, std::string id, Vec p, const std::map<std::string, double> &settings) {
std::set<std::string> seen;
while (id != ".") {
if (!seen.insert(id).second)
throw Invalid("the axis chain through '" + id + "' loops back on itself");
const Axis &a = Find(d, id);
const auto s = settings.find(id);
const double value = (s != settings.end()) ? s->second : a.setting;
if (a.type == "rotation")
p = Rotate(value, a.vector, p);
else if (a.type == "translation")
for (int i = 0; i < 3; i++)
p[i] += value * a.vector[i];
else
throw Invalid("axis '" + id + "' is of type '" + a.type + "', which is not supported");
for (int i = 0; i < 3; i++)
p[i] += a.offset[i];
id = a.depends_on;
}
return p;
}
// Where stored pixel `stored` (0-based) sits along its axis. _array_structure_list.direction is not
// consulted: the dictionary says "decreasing" means the stored order runs from the last index to the
// first, but CBFlib - and every program built on it - places a pixel by the axis displacement alone,
// and the files written through CBFlib are only consistent read that way. Bruker's conversions state
// "decreasing" for their rows: read as CBFlib reads them a sweep of them indexes (and DIALS indexes
// it), read by the dictionary the image is mirrored and nothing does.
double Displacement(const ArrayIndex &index, double stored) {
return index.displacement + stored * index.increment;
}
} // namespace
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag) {
// The loop_ that introduces the tag, not the tag's own position: everything before it is another
// loop's data.
const size_t tag_at = text.find("\n" + tag);
if (tag_at == std::string::npos)
return {};
const size_t loop_at = text.rfind("loop_", tag_at);
if (loop_at == std::string::npos)
return {};
// Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would
// otherwise be counted as several columns and shift every row after it.
std::vector<std::string> tokens;
for (size_t i = loop_at + 5; i < text.size();) {
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
i++;
if (i >= text.size())
break;
size_t end;
if ((text[i] == '\'') || (text[i] == '"')) {
end = text.find(text[i], i + 1);
if (end == std::string::npos)
break;
tokens.push_back(text.substr(i + 1, end - i - 1));
end++;
} else {
end = i;
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
end++;
tokens.push_back(text.substr(i, end - i));
}
// A second loop_, or a tag belonging to another category, ends this one.
if ((tokens.back() == "loop_")
|| (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) {
tokens.pop_back();
break;
}
i = end;
}
std::vector<std::string> names;
size_t first_value = 0;
while ((first_value < tokens.size()) && tokens[first_value].starts_with("_"))
names.push_back(tokens[first_value++]);
// The loop found must be the tag's own: a tag stated outside any loop would otherwise be read as
// the rows of whatever loop came before it.
if (std::find(names.begin(), names.end(), tag) == names.end())
return {};
std::vector<std::map<std::string, std::string>> rows;
for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) {
std::map<std::string, std::string> row;
for (size_t j = 0; j < names.size(); j++)
row[names[j]] = tokens[i + j];
rows.push_back(std::move(row));
}
return rows;
}
std::optional<std::string> Item(const std::string &text, const std::string &tag) {
const std::string key = "\n" + tag;
for (size_t at = text.find(key); at != std::string::npos; at = text.find(key, at + 1)) {
size_t i = at + key.size();
// the whole tag, not a longer one that starts with it
if ((i < text.size()) && !std::isspace(static_cast<unsigned char>(text[i])))
continue;
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
i++;
if (i >= text.size())
return {};
if ((text[i] == '\'') || (text[i] == '"')) {
const size_t end = text.find(text[i], i + 1);
if (end == std::string::npos)
return {};
return text.substr(i + 1, end - i - 1);
}
size_t end = i;
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
end++;
const std::string value = text.substr(i, end - i);
if (value.starts_with("_") || value == "loop_")
return {}; // the tag heads a loop: not a single item
return value;
}
return {};
}
std::optional<Description> Parse(const std::string &text) {
const auto axis_rows = ParseLoop(text, "_axis.id");
auto frame_rows = ParseLoop(text, "_diffrn_scan_frame_axis.axis_id");
const auto scan_rows = ParseLoop(text, "_diffrn_scan_axis.axis_id");
if (axis_rows.empty() || (frame_rows.empty() && scan_rows.empty()))
return {};
Description d;
for (const auto &row: axis_rows) {
Axis a;
a.type = Field(row, "_axis.type");
a.equipment = Field(row, "_axis.equipment");
a.depends_on = Field(row, "_axis.depends_on");
if (a.depends_on.empty() || a.depends_on == "?")
a.depends_on = ".";
for (int i = 0; i < 3; i++) {
a.vector[i] = NumberOr(Field(row, "_axis.vector[" + std::to_string(i + 1) + "]"), 0.0);
a.offset[i] = NumberOr(Field(row, "_axis.offset[" + std::to_string(i + 1) + "]"), 0.0);
}
d.axes[Field(row, "_axis.id")] = a;
}
// Where each axis stands: the frame's own setting where the file states one, the scan's start
// otherwise; how far it moves over the frame, from the scan.
for (const auto &row: scan_rows) {
const auto it = d.axes.find(Field(row, "_diffrn_scan_axis.axis_id"));
if (it == d.axes.end())
continue;
const bool rotation = it->second.type == "rotation";
it->second.setting = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_start"
: "_diffrn_scan_axis.displacement_start"), 0.0);
it->second.increment = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_increment"
: "_diffrn_scan_axis.displacement_increment"), 0.0);
}
for (const auto &row: frame_rows) {
const auto it = d.axes.find(Field(row, "_diffrn_scan_frame_axis.axis_id"));
if (it == d.axes.end())
continue;
const bool rotation = it->second.type == "rotation";
if (const auto v = Number(Field(row, rotation ? "_diffrn_scan_frame_axis.angle"
: "_diffrn_scan_frame_axis.displacement")))
it->second.setting = *v;
}
// The two image indices, and the axis each runs along.
std::map<std::string, std::map<std::string, std::string>> set_rows;
for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
set_rows[Field(row, "_array_structure_list_axis.axis_set_id")] = row;
for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) {
ArrayIndex index;
const std::string set = Field(row, "_array_structure_list.axis_set_id");
const auto s = set_rows.find(set);
if (s == set_rows.end())
throw Invalid("image axis set '" + set + "' has no _array_structure_list_axis row");
index.axis_id = Field(s->second, "_array_structure_list_axis.axis_id");
index.displacement = NumberOr(Field(s->second, "_array_structure_list_axis.displacement"), 0.0);
index.increment = NumberOr(Field(s->second, "_array_structure_list_axis.displacement_increment"), 0.0);
index.dimension = static_cast<int64_t>(NumberOr(Field(row, "_array_structure_list.dimension"), 0.0));
const std::string which = Field(row, "_array_structure_list.index");
if (which == "1")
d.fast = index;
else if (which == "2")
d.slow = index;
}
if (d.fast.axis_id.empty() || d.slow.axis_id.empty())
throw Invalid("no _array_structure_list rows for the two image indices");
d.wavelength_A = NumberOr(Item(text, "_diffrn_radiation_wavelength.wavelength").value_or(""), 0.0);
if (d.wavelength_A == 0.0) {
const auto rows = ParseLoop(text, "_diffrn_radiation_wavelength.id");
if (!rows.empty())
d.wavelength_A = NumberOr(Field(rows.front(), "_diffrn_radiation_wavelength.wavelength"), 0.0);
}
d.overload = NumberOr(Item(text, "_array_intensities.overload").value_or(""), 0.0);
d.exposure_s = NumberOr(Item(text, "_diffrn_scan_frame.integration_time").value_or(""), 0.0);
d.detector = Item(text, "_diffrn_detector.type").value_or(Item(text, "_diffrn_detector.detector").value_or(""));
d.ccd = Item(text, "_diffrn_detector.detector").value_or("").find("CCD") != std::string::npos;
if (const auto ratio = Item(text, "_diffrn_radiation_wavelength.polarizn_source_ratio"))
d.polarization_ratio = NumberOr(*ratio, 0.0);
d.polarization_norm_deg = NumberOr(Item(text, "_diffrn_radiation_wavelength.polarizn_source_norm").value_or(""), 0.0);
return d;
}
std::optional<float> PolarizationFactor(const Description &d) {
if (!d.polarization_ratio)
return {};
return static_cast<float>(*d.polarization_ratio * std::cos(2.0 * d.polarization_norm_deg * PI / 180.0));
}
Geometry Evaluate(const Description &d) {
Geometry g;
// The image: the index axis that hangs off the other one is where the chain starts.
const Axis &fast = Find(d, d.fast.axis_id);
const Axis &slow = Find(d, d.slow.axis_id);
const bool slow_is_leaf = slow.depends_on == d.fast.axis_id;
if (!slow_is_leaf && fast.depends_on != d.slow.axis_id)
throw Invalid("the two image axes " + d.fast.axis_id + " and " + d.slow.axis_id
+ " do not depend on each other; such a detector is not supported");
const std::string leaf = slow_is_leaf ? d.slow.axis_id : d.fast.axis_id;
const auto pixel = [&](double col, double row) {
return ToLab(d, leaf, {0, 0, 0}, {{d.fast.axis_id, Displacement(d.fast, col)},
{d.slow.axis_id, Displacement(d.slow, row)}});
};
g.origin = pixel(0, 0);
const Vec p10 = pixel(1, 0), p01 = pixel(0, 1);
for (int i = 0; i < 3; i++) {
g.fast[i] = p10[i] - g.origin[i];
g.slow[i] = p01[i] - g.origin[i];
}
// The scan: exactly one axis may move, and it has to be a goniometer rotation. Its direction is
// its own vector carried by every axis it depends on, at their settings - on a kappa head a phi
// scan turns about phi as omega and kappa hold it. The axes it carries do not move it.
std::vector<std::string> moving;
for (const auto &[id, a]: d.axes)
if (a.increment != 0.0 && id != d.fast.axis_id && id != d.slow.axis_id)
moving.push_back(id);
if (moving.size() > 1) {
std::string names;
for (const auto &m: moving)
names += (names.empty() ? "" : ", ") + m;
throw Invalid("more than one axis moves during the frame (" + names + "); only a rotation "
"about one goniometer axis is supported");
}
if (moving.empty())
return g;
const std::string &id = moving.front();
const Axis &a = d.axes.at(id);
if (a.type != "rotation" || a.equipment != "goniometer")
throw Invalid("the axis that moves during the frame (" + id + ") is a " + a.equipment + " "
+ a.type + "; only a goniometer rotation is supported");
// Rotations only: an offset moves where the axis is, not which way it points.
Vec dir = a.vector;
std::set<std::string> seen;
for (std::string up = a.depends_on; up != "."; up = Find(d, up).depends_on) {
if (!seen.insert(up).second)
throw Invalid("the axis chain through '" + up + "' loops back on itself");
const Axis &parent = Find(d, up);
if (parent.type == "rotation")
dir = Rotate(parent.setting, parent.vector, dir);
}
const double len = std::sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2]);
if (!(len > 0.0))
throw Invalid("the scan axis " + id + " has no direction");
for (double &c: dir)
c /= len;
g.scan_axis = id;
g.scan_direction = dir;
g.scan_start_deg = a.setting;
g.scan_increment_deg = a.increment;
return g;
}
Coord ToInternal(const std::array<double, 3> &v) {
return {static_cast<float>(v[0]), static_cast<float>(-v[1]), static_cast<float>(-v[2])};
}
Coord GoniometerAxisOf(const std::array<double, 3> &v) {
return -ToInternal(v);
}
GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg,
const std::array<double, 3> &v) {
const double sign = increment_deg < 0.0 ? -1.0 : 1.0;
return GoniometerAxis(name, static_cast<float>(sign * start_deg), static_cast<float>(sign * increment_deg),
GoniometerAxisOf(v) * static_cast<float>(sign), {});
}
DetectorPlane PlaneOf(const Geometry &g, const std::string &path) {
const Coord origin = ToInternal(g.origin);
const Coord fast = ToInternal(g.fast);
const Coord slow = ToInternal(g.slow);
DetectorPlane p;
p.pixel_mm = fast.Length();
// One pixel size and a rectangular grid is what the geometry can describe.
if (std::fabs(slow.Length() - p.pixel_mm) > 1e-4f * p.pixel_mm
|| std::fabs(fast * slow) > 1e-4f * p.pixel_mm * p.pixel_mm)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
path + ": the imgCIF image axes are not square pixels on a rectangular "
"grid, which is not supported");
const Coord f = fast.Normalize(), s = slow.Normalize();
Coord n = f % s;
// fast x slow points away from the sample on an image stored the internal way round, and towards
// it on a mirrored one.
const bool mirrored = n * origin < 0.0f;
if (mirrored)
n = -n;
p.distance_mm = n * origin;
const Coord to_poni = n * p.distance_mm - origin;
p.beam_x_pxl = (to_poni * f) / p.pixel_mm;
p.beam_y_pxl = (to_poni * s) / p.pixel_mm;
// Of the four ways the image can be stored with this handedness, the one that leaves the least
// for the continuous rotation - the detector as it is mounted, with the arm and the tilts on top.
const RotMatrix m(f, s, n);
float best_trace = -10.0f;
for (int64_t q = 0; q < 4; q++) {
const DetectorOrientation o(mirrored, q);
const RotMatrix r = m * o.Matrix().transpose();
const float trace = r.Column(0).x + r.Column(1).y + r.Column(2).z;
if (trace > best_trace) {
best_trace = trace;
p.orientation = o;
PoniAnglesFromMatrix(r, p.rot1, p.rot2, p.rot3);
}
}
return p;
}
} // namespace imgcif
+132
View File
@@ -0,0 +1,132 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <array>
#include <cstdint>
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "../common/Coord.h"
#include "../common/DetectorOrientation.h"
#include "../common/GoniometerAxis.h"
// The parts of a full imgCIF header (the form CBFlib writes, and Bruker's APEX writes when it converts
// .sfrm frames) that say where the detector and the goniometer stand. Unlike a PILATUS miniCBF, which
// states its geometry in "# " comment lines, such a file states it only in CIF tables: an axis table
// (_axis), the setting of every axis at this frame (_diffrn_scan_frame_axis) and how it moves per
// frame (_diffrn_scan_axis), and which axes the image's two indices run along
// (_array_structure_list, _array_structure_list_axis).
//
// Every vector here is in the imgCIF laboratory frame (Z from the sample towards the source, Y
// opposite gravity, X completing a right-handed set) and every length in millimetres.
// Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458
namespace imgcif {
// One "loop_" of a CIF block, as rows keyed by tag. Tags and values are both read as
// whitespace-separated tokens rather than by line, because a template packs several tags onto one
// line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are laid out to
// match the tags, not the lines.
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag);
// A tag stated outside a loop ("_diffrn_radiation_wavelength.wavelength 0.71073"), or nothing.
std::optional<std::string> Item(const std::string &text, const std::string &tag);
struct Axis {
std::string type; // "rotation" or "translation"
std::string equipment; // "goniometer", "detector", ...
std::string depends_on; // "." for an axis that hangs off nothing
std::array<double, 3> vector{};
std::array<double, 3> offset{};
double setting = 0; // this frame's angle (deg) or displacement (mm)
double increment = 0; // its change over one frame
};
// One index of the stored image: the axis it runs along, and where pixel 1 and each next one sit on it.
struct ArrayIndex {
std::string axis_id;
int64_t dimension = 0;
double displacement = 0; // of index 1, mm
double increment = 0; // per index, mm
};
struct Description {
std::map<std::string, Axis> axes;
ArrayIndex fast; // _array_structure_list.index 1
ArrayIndex slow; // _array_structure_list.index 2
double wavelength_A = 0;
double overload = 0; // _array_intensities.overload; 0 where not stated
double exposure_s = 0;
std::string detector;
// _diffrn_detector.detector names a CCD: its pixel values are ADU, not counted photons.
bool ccd = false;
// _diffrn_radiation_wavelength.polarizn_source_ratio, (Ip - In) / (Ip + In), and
// polarizn_source_norm, the angle (deg) from the laboratory Y axis to the normal of the plane Ip
// is polarized in. Nothing where the header states no ratio.
std::optional<double> polarization_ratio;
double polarization_norm_deg = 0;
};
// The polarization as DiffractionExperiment::PolarizationFactor takes it - the degree, positive for a
// beam polarized in the horizontal (laboratory X-Z) plane - or nothing where the header states none.
// A plane at an angle psi from the horizontal enters as its projection, the degree times cos(2 psi),
// as the correction knows only those two planes.
std::optional<float> PolarizationFactor(const Description &d);
// The description, or nothing when the header is not a full imgCIF one - no axis table, or no
// statement of where the axes stand. Throws on a table that is there but cannot be read.
std::optional<Description> Parse(const std::string &text);
struct Geometry {
std::array<double, 3> origin{}; // centre of stored pixel (0, 0)
std::array<double, 3> fast{}; // one stored column further, minus origin
std::array<double, 3> slow{}; // one stored row further, minus origin
std::string scan_axis; // the goniometer axis that turns during the frame
std::array<double, 3> scan_direction{}; // its laboratory direction at this frame's settings
double scan_start_deg = 0;
double scan_increment_deg = 0;
};
// Where the image and the rotation axis are at this frame, through the whole axis chain. Throws, naming
// what it found, on anything this cannot describe as one rotation about one axis: two axes moving at
// once, a detector or translation axis moving, or a chain that does not close.
Geometry Evaluate(const Description &d);
// An imgCIF laboratory direction in the internal frame. imgCIF puts Z from the sample towards the
// source and Y opposite gravity, while the internal frame has z along the beam and y along increasing
// row, so the two differ by a half turn about x - a rotation and not a mirror, so an axis carried
// through it turns the same way by the same angle.
Coord ToInternal(const std::array<double, 3> &v);
// The goniometer axis as GoniometerAxis takes it, for an imgCIF rotation axis. GoniometerAxis turns
// the crystal by minus the angle about its axis (IndexAndRefine, rot_to_image), the sense an NXmx
// master's vector carries, so a right-handed imgCIF rotation about v is the internal -v.
Coord GoniometerAxisOf(const std::array<double, 3> &v);
// The sweep as GoniometerAxis takes it, turning the scanned axis v from start_deg by increment_deg per
// frame. A sweep that turns backwards (Bruker frames state a negative increment) is the same rotation
// about -v by the opposite angles, and is stated that way: the analysis takes the oscillation width of a
// frame to be positive, as every other format states it.
GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg,
const std::array<double, 3> &v);
// The detector plane in the terms the geometry is kept in: the point of normal incidence (PONI) in
// pixels, the distance along the normal, and the rotation that takes the detector frame to the
// laboratory, split into the discrete way the image is stored and the continuous rest. The table
// states the plane through three pixel centres, so this is only a change of description - a 2theta
// arm, a detector's small mounting tilts and a stored image turned or mirrored all come out of the same
// three points. Throws, naming path, on pixels that are not square or a grid that is not rectangular.
struct DetectorPlane {
float pixel_mm = 0;
float distance_mm = 0;
float beam_x_pxl = 0;
float beam_y_pxl = 0;
DetectorOrientation orientation;
float rot1 = 0, rot2 = 0, rot3 = 0; // PONI angles, rad
};
DetectorPlane PlaneOf(const Geometry &g, const std::string &path);
} // namespace imgcif
+80 -11
View File
@@ -5,6 +5,7 @@
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstring>
#include <filesystem>
#include <array>
@@ -16,6 +17,7 @@
#include "../common/Logger.h"
#include "../common/JFJochMath.h"
#include "SweepLayout.h"
#include "ImgCIF.h"
namespace {
@@ -178,6 +180,15 @@ Coord RotationAxis(const minicbf::Header &h) {
const Coord base = BaseAxis(h);
if (!minicbf::ScansPhi(h))
return base;
// A kappa head carries phi on an arm inclined to omega, and where the arm stands moves the phi
// axis by up to twice its inclination. These headers state the kappa angle but not the arm's
// inclination in any form this reader composes, so a phi scan at a non-zero kappa is refused
// rather than read about the wrong axis.
if (h.kappa_deg.has_value() && std::fabs(*h.kappa_deg) > 1e-3)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"a phi scan at kappa " + std::to_string(*h.kappa_deg)
+ " deg: the miniCBF header does not describe the kappa arm, so the "
"rotation axis cannot be composed; this is not supported");
const auto rad = [](double deg) { return static_cast<float>(deg * PI / 180.0); };
// Where the table states the inner axis outright, that IS the inclination, and the driven-circle
@@ -231,8 +242,24 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
dataset_ = std::make_shared<JFJochReaderDataset>();
dataset_->experiment = default_experiment;
// A full imgCIF header states its geometry through the axis chain, which is worked out once here
// and per frame below; a miniCBF states it in its "# " lines.
std::optional<imgcif::Geometry> geometry0;
std::optional<imgcif::DetectorPlane> plane0;
if (header0_.imgcif.has_value()) {
geometry0 = imgcif::Evaluate(*header0_.imgcif);
if (geometry0->scan_axis.empty())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
files_[0] + ": no goniometer axis turns during the frame; a full "
"imgCIF series of stills is not supported");
plane0 = imgcif::PlaneOf(*geometry0, files_[0]);
}
DetectorSetup detector = DetDECTRIS(header0_.nx, header0_.ny, header0_.detector, {});
detector.PixelSize_um(static_cast<int64_t>(std::lround(header0_.pixel_x_m * 1e6)));
if (plane0.has_value())
detector.PixelSize_um(plane0->pixel_mm * 1000.0f); // a CCD's pixel is not whole micrometres
else
detector.PixelSize_um(static_cast<int64_t>(std::lround(header0_.pixel_x_m * 1e6)));
detector.SensorThickness_um(static_cast<int64_t>(std::lround(header0_.thickness_m * 1e6)));
detector.SensorMaterial(header0_.material);
// Only when the header states one. An absent line reads as zero, and a limit of zero would mark
@@ -257,16 +284,25 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
// Images are handed out as signed 32-bit whatever the file stored, so that is the depth the rest
// of the code must see; the real overflow is the header's Count_cutoff, set above.
detector.BitDepthImage(32);
if (const auto orientation = ImageOrientation(header0_))
if (plane0.has_value())
detector.ImageOrientation(plane0->orientation);
else if (const auto orientation = ImageOrientation(header0_))
detector.ImageOrientation(orientation.value());
detector.MinFrameTime(std::chrono::microseconds(0));
detector.MinCountTime(std::chrono::microseconds(0));
detector.ReadOutTime(std::chrono::nanoseconds(0));
dataset_->experiment.Detector(detector);
dataset_->experiment.BeamX_pxl(static_cast<float>(header0_.beam_x_px));
dataset_->experiment.BeamY_pxl(static_cast<float>(header0_.beam_y_px));
dataset_->experiment.DetectorDistance_mm(static_cast<float>(header0_.distance_m * 1000.0));
if (plane0.has_value()) {
dataset_->experiment.BeamX_pxl(plane0->beam_x_pxl);
dataset_->experiment.BeamY_pxl(plane0->beam_y_pxl);
dataset_->experiment.DetectorDistance_mm(plane0->distance_mm);
dataset_->experiment.PoniRot1_rad(plane0->rot1).PoniRot2_rad(plane0->rot2).PoniRot3_rad(plane0->rot3);
} else {
dataset_->experiment.BeamX_pxl(static_cast<float>(header0_.beam_x_px));
dataset_->experiment.BeamY_pxl(static_cast<float>(header0_.beam_y_px));
dataset_->experiment.DetectorDistance_mm(static_cast<float>(header0_.distance_m * 1000.0));
}
// A detector swung out on a 2theta arm, which small-molecule collection uses routinely. The arm
// turns the detector about the sample, so it carries the square-on geometry with it: the header's
@@ -290,6 +326,10 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
// correction takes to contain the lab x axis; the degree it wants is 2f - 1.
if (header0_.polarization_fraction.has_value())
dataset_->experiment.PolarizationFactor(2.0f * static_cast<float>(*header0_.polarization_fraction) - 1.0f);
// A full imgCIF header states the degree itself, and the plane.
if (header0_.imgcif.has_value())
if (const auto p = imgcif::PolarizationFactor(*header0_.imgcif))
dataset_->experiment.PolarizationFactor(*p);
dataset_->experiment.FrameTime(
std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::duration<double>(header0_.period_s)),
@@ -300,21 +340,46 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
// Parsing one costs a few hundred microseconds - it is two dozen regular expressions - so on a
// sweep of several thousand frames this is seconds of startup before a single image is read, and
// the files are independent.
//
// A full imgCIF frame states every axis setting, so each one is checked to belong to the same
// sweep as the first: the same axis turning, about the same direction. A series in which the
// fixed axes moved between frames - kappa or omega reset for another run - is several sweeps,
// and taking it for one would put every frame after the move at the wrong orientation.
std::vector<sweep::Frame> frames(files_.size());
sweep::ForEachInOrder(files_.size(), [&](size_t i) {
const auto h = minicbf::ReadHeader(files_[i]);
frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m,
h.beam_x_px, h.beam_y_px, h.wavelength_A};
if (!geometry0.has_value()) {
frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m,
h.beam_x_px, h.beam_y_px, h.wavelength_A};
return;
}
if (!h.imgcif.has_value())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
files_[i] + " is not a full imgCIF frame like " + files_[0]);
const auto g = imgcif::Evaluate(*h.imgcif);
const Coord axis = ImgCIFToInternal(g.scan_direction);
if (g.scan_axis != geometry0->scan_axis
|| (axis - ImgCIFToInternal(geometry0->scan_direction)).Length() > 1e-4f)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
files_[i] + " turns " + (g.scan_axis.empty() ? "no axis" : g.scan_axis)
+ " about a different direction from " + files_[0]
+ ": the goniometer was reset between them, so the series is more "
"than one sweep. Name a frame of one sweep only.");
const auto plane = imgcif::PlaneOf(g, files_[i]);
frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3,
plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A};
});
// The sweep the headers describe, which is not always the files laid out end to end: a deposited
// series can be missing frames, and those are gaps in the rotation rather than images to close up.
const auto layout = sweep::Place(frames, "CBFReader");
files_ = layout.files;
dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name,
static_cast<float>(layout.start_deg),
static_cast<float>(layout.increment_deg),
RotationAxis(header0_), {}));
if (geometry0.has_value())
dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0->scan_axis, layout.start_deg, layout.increment_deg,
geometry0->scan_direction));
else
dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name, static_cast<float>(layout.start_deg),
static_cast<float>(layout.increment_deg), RotationAxis(header0_), {}));
dataset_->error_value = -1;
dataset_->experiment.ImagesPerTrigger(static_cast<int64_t>(files_.size()));
@@ -395,6 +460,10 @@ bool JFJochCBFReader::HasImage(int64_t image_number) const {
&& !files_[image_number].empty();
}
bool JFJochCBFReader::CountsInADU() const {
return header0_.imgcif.has_value() && header0_.imgcif->ccd;
}
bool JFJochCBFReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) {
if (!HasImage(image_number))
return false;
+4
View File
@@ -56,6 +56,10 @@ public:
// ReadRawImage hands such a slot back as "nothing to read".
[[nodiscard]] bool HasImage(int64_t image_number) const;
// Whether the pixel values are ADU rather than counted photons: a full imgCIF header that names a
// CCD, which is what Bruker's conversion of its frames states. A PILATUS miniCBF counts photons.
[[nodiscard]] bool CountsInADU() const;
bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override;
[[nodiscard]] std::vector<SpotToSave> ReadSpots(int64_t image) const override;
};
+231
View File
@@ -0,0 +1,231 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "JFJochSFRMReader.h"
#include <cmath>
#include "../common/JFJochException.h"
#include "../common/JFJochMath.h"
#include "../common/Logger.h"
#include "ImgCIF.h"
#include "SweepLayout.h"
namespace {
// The instrument a .sfrm header describes, as an imgCIF axis chain, so that one piece of code turns
// either description into the geometry. The chain is the one APEX writes when it converts frames to
// imgCIF (the pixels come out identical but for the baseline, so the image axes are the same), with
// one thing settled from data rather than taken from it: omega turns in the sense of the 2theta arm.
// An omega scan of APEX's conversion states the opposite increment to the frames it came from, and a
// SMART 6000 omega sweep indexes only that way round. Phi and chi are as the conversion composes them
// from a kappa setting. CENTER is placed as the conversion places it, each coordinate counted from the
// far edge of the stored image: on an APEX II phi sweep read both ways it indexes 97.5% of the frames
// against 90.3% and merges at ISa 17.5 against 2.1, where the image's own .cbf conversion gives 17.
//
// The goniometer is stated as Eulerian angles whatever the head is - a kappa instrument's frames carry
// the omega, chi and phi its kappa setting is equivalent to - so the chain is omega, carrying chi about
// the beam, carrying phi. On the 2theta arm the detector sits DISTANC (plus DETTYPE's face-to-grid
// offset) from the sample, tilted by the DETPAR pitch, roll and yaw. The stored rows run along the
// spindle axis and the stored columns along the direction the arm swings.
imgcif::Description Chain(const sfrm::Header &h, const std::string &path) {
if (h.center.size() < 2)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, path + " states no CENTER");
if (!(h.pixels_per_cm_512 > 0.0))
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
path + " states no pixel size (DETTYPE)");
if (h.detpar[0] != 0.0 || h.detpar[1] != 0.0 || h.detpar[2] != 0.0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
path + " states non-zero DETPAR offsets, whose meaning this reader does not know");
// DETTYPE gives pixels per centimetre on a 512-pixel frame; a binned or unbinned frame scales it.
const double pixel_mm = 10.0 / h.pixels_per_cm_512 * 512.0 / static_cast<double>(h.ncols);
const double distance_mm = (h.distance_cm + h.grid_offset_cm) * 10.0;
const double half_cols = static_cast<double>(h.ncols) / 2.0 * pixel_mm;
const double half_rows = static_cast<double>(h.nrows) / 2.0 * pixel_mm;
imgcif::Description d;
d.axes["OMEGA"] = {"rotation", "goniometer", ".", {-1, 0, 0}, {}, h.angles[1], 0};
d.axes["CHI"] = {"rotation", "goniometer", "OMEGA", {0, 0, 1}, {}, h.angles[3], 0};
d.axes["PHI"] = {"rotation", "goniometer", "CHI", {1, 0, 0}, {}, h.angles[2], 0};
d.axes["TWOTHETA"] = {"rotation", "detector", ".", {1, 0, 0}, {}, -h.angles[0], 0};
d.axes["DX"] = {"translation", "detector", "TWOTHETA", {0, 0, -1}, {}, distance_mm, 0};
d.axes["YAW"] = {"rotation", "detector", "DX", {-1, 0, 0}, {}, h.detpar[5], 0};
d.axes["PITCH"] = {"rotation", "detector", "YAW", {0, -1, 0}, {}, h.detpar[3], 0};
d.axes["ROLL"] = {"rotation", "detector", "PITCH", {0, 0, -1}, {}, h.detpar[4], 0};
d.axes["H"] = {"translation", "detector", "ROLL", {0, 1, 0}, {}, (h.center[0] * pixel_mm - half_cols), 0};
d.axes["V"] = {"translation", "detector", "H", {1, 0, 0}, {}, (h.center[1] * pixel_mm - half_rows), 0};
d.axes["ELEMENT_X"] = {"translation", "detector", "V", {0, 1, 0}, {-half_rows, -half_cols, 0}, 0, 0};
d.axes["ELEMENT_Y"] = {"translation", "detector", "ELEMENT_X", {1, 0, 0}, {}, 0, 0};
d.fast = {"ELEMENT_X", h.ncols, pixel_mm / 2.0, pixel_mm};
d.slow = {"ELEMENT_Y", h.nrows, pixel_mm / 2.0, pixel_mm};
static const char *AXIS_NAMES[] = {"", "TWOTHETA", "OMEGA", "PHI", "CHI"};
if (h.axis < 1 || h.axis > 4)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
path + " scans AXIS " + std::to_string(h.axis)
+ ", which is not a rotation this reader supports");
d.axes[AXIS_NAMES[h.axis]].increment = h.increment_deg;
d.wavelength_A = h.wavelength_A;
// The polarization APEX's conversion states for the same frame: 1 - cos^2 of the monochromator's
// scattering angle (FILTER2), in the horizontal plane. A tube with mirrors and no monochromator
// states 0, unpolarized - these are laboratory frames, and the synchrotron default does not apply.
const double cos_mono = std::cos(h.monochromator_2theta_deg * PI / 180.0);
d.polarization_ratio = 1.0 - cos_mono * cos_mono;
d.polarization_norm_deg = 0;
d.overload = h.saturation;
d.exposure_s = h.exposure_s;
d.detector = h.detector;
return d;
}
} // namespace
bool JFJochSFRMReader::CanRead(const std::string &path) {
return sfrm::CanRead(path);
}
void JFJochSFRMReader::ReadFiles(const std::string &path) {
files_ = sfrm::CollectSweep(path);
if (files_.empty())
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No SFRM images found for " + path);
header0_ = sfrm::ReadHeader(files_[0]);
if (!(header0_.wavelength_A > 0.0))
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
files_[0] + " states no wavelength in its SFRM header");
const auto geometry0 = imgcif::Evaluate(Chain(header0_, files_[0]));
const auto plane0 = imgcif::PlaneOf(geometry0, files_[0]);
dataset_ = std::make_shared<JFJochReaderDataset>();
dataset_->experiment = default_experiment;
DetectorSetup detector = DetDECTRIS(header0_.ncols, header0_.nrows, header0_.detector, {});
detector.PixelSize_um(plane0.pixel_mm * 1000.0f);
// A CCD has no sensor depth to correct for: the phosphor converts at the surface.
detector.SensorThickness_um(0);
// CCDPARM[4] is the count at which the CCD saturates; the stored container (with its overflow
// tables) holds far more, so it is the only limit there is.
if (header0_.saturation > 0.0)
detector.SaturationLimit(static_cast<int64_t>(header0_.saturation));
else
Logger("SFRMReader").Warning("{} states no saturation (CCDPARM), so no pixel will be called "
"saturated.", files_[0]);
detector.BitDepthImage(32);
detector.ImageOrientation(plane0.orientation);
detector.MinFrameTime(std::chrono::microseconds(0));
detector.MinCountTime(std::chrono::microseconds(0));
detector.ReadOutTime(std::chrono::nanoseconds(0));
dataset_->experiment.Detector(detector);
dataset_->experiment.BeamX_pxl(plane0.beam_x_pxl);
dataset_->experiment.BeamY_pxl(plane0.beam_y_pxl);
dataset_->experiment.DetectorDistance_mm(plane0.distance_mm);
dataset_->experiment.PoniRot1_rad(plane0.rot1).PoniRot2_rad(plane0.rot2).PoniRot3_rad(plane0.rot3);
dataset_->experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / static_cast<float>(header0_.wavelength_A));
dataset_->experiment.PolarizationFactor(imgcif::PolarizationFactor(Chain(header0_, files_[0])));
if (header0_.exposure_s > 0.0)
dataset_->experiment.FrameTime(
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<double>(header0_.exposure_s)),
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<double>(header0_.exposure_s)));
// Where every image sits on the spindle, from its own header - and the check that every frame is
// of the same sweep: the same axis turning about the same direction. A run whose fixed angles moved
// part-way is more than one sweep, and reading it as one would put every frame after the move at
// the wrong orientation.
std::vector<sweep::Frame> frames(files_.size());
sweep::ForEachInOrder(files_.size(), [&](size_t i) {
const auto h = sfrm::ReadHeader(files_[i]);
const auto g = imgcif::Evaluate(Chain(h, files_[i]));
if (g.scan_axis != geometry0.scan_axis
|| (imgcif::ToInternal(g.scan_direction) - imgcif::ToInternal(geometry0.scan_direction)).Length() > 1e-4f)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
files_[i] + " turns " + g.scan_axis + " about a different direction from "
+ files_[0] + ": the goniometer was reset between them, so the run is "
"more than one sweep");
const auto plane = imgcif::PlaneOf(g, files_[i]);
frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3,
plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A};
});
const auto layout = sweep::Place(frames, "SFRMReader");
files_ = layout.files;
dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0.scan_axis, layout.start_deg, layout.increment_deg,
geometry0.scan_direction));
dataset_->error_value = -1;
dataset_->experiment.ImagesPerTrigger(static_cast<int64_t>(files_.size()));
// Outside the phosphor's active area a frame reads below its baseline offset - the stored value
// is zero there in every frame - so frame 0's negative counts are the pixels no frame can be
// trusted at, exactly as a PILATUS marks its gaps.
std::vector<int32_t> first;
sfrm::Read(files_[0], first);
std::vector<uint32_t> mask(first.size(), 0);
for (size_t i = 0; i < first.size(); i++)
if (first[i] < 0)
mask[i] = 1;
dataset_->pixel_mask = std::make_shared<const PixelMask>(mask);
SetStartMessage(dataset_);
}
uint64_t JFJochSFRMReader::GetNumberOfImages() const {
return files_.size();
}
void JFJochSFRMReader::Close() {
files_.clear();
dataset_.reset();
}
template <class Buffer>
CompressedImage JFJochSFRMReader::DecodeInto(int64_t image_number, Buffer &buffer,
std::vector<uint8_t> &scratch) const {
if (!HasImage(image_number))
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"No image at this point of the sweep");
const size_t npixel = static_cast<size_t>(header0_.nrows) * static_cast<size_t>(header0_.ncols);
buffer.resize(npixel * sizeof(int32_t));
const auto h = sfrm::ReadInto(files_[image_number], reinterpret_cast<int32_t *>(buffer.data()), npixel,
scratch);
if (h.nrows != header0_.nrows || h.ncols != header0_.ncols)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"SFRM image size differs from the first image of the sweep");
return CompressedImage(buffer.data(), buffer.size(),
static_cast<size_t>(header0_.ncols), static_cast<size_t>(header0_.nrows),
CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION);
}
bool JFJochSFRMReader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
DataMessage &message,
std::vector<uint8_t> &buffer,
int64_t image_number,
bool update_dataset) {
(void) update_dataset;
if (!dataset || !HasImage(image_number))
return false;
std::vector<uint8_t> scratch;
message.image = DecodeInto(image_number, buffer, scratch);
message.number = image_number;
return true;
}
bool JFJochSFRMReader::HasImage(int64_t image_number) const {
return image_number >= 0 && static_cast<size_t>(image_number) < files_.size()
&& !files_[image_number].empty();
}
bool JFJochSFRMReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) {
if (!HasImage(image_number))
return false;
image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer);
return true;
}
std::vector<SpotToSave> JFJochSFRMReader::ReadSpots(int64_t) const {
return {}; // a raw frame stores no analysis results
}
+56
View File
@@ -0,0 +1,56 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <string>
#include <vector>
#include "JFJochReader.h"
#include "SFRM.h"
// Reads a rotation sweep straight from a directory of Bruker .sfrm frames - what SMART, PROTEUM
// and APEX write - with no conversion step.
//
// The same shape as the other one-file-per-image readers: the sweep's geometry comes from the first
// frame's header, the rotation angle of each image from its own, and images are decoded on demand.
// A Bruker collection directory holds one run per scan, each its own sweep; naming a frame reads its
// run. A raw frame carries no analysis results, so no spots and no reflections.
class JFJochSFRMReader : public JFJochReader {
std::vector<std::string> files_;
std::shared_ptr<JFJochReaderDataset> dataset_;
sfrm::Header header0_;
bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
DataMessage &message,
std::vector<uint8_t> &buffer,
int64_t image_number,
bool update_dataset) override;
// Decodes one image into the caller's byte buffer and returns the image that points at it. The
// file is read through scratch, which the caller keeps between frames.
template <class Buffer>
CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector<uint8_t> &scratch) const;
public:
~JFJochSFRMReader() override = default;
// True if the path names something this reader can open: a .sfrm frame, or a directory holding
// at least one. Cheap - it reads a few bytes.
static bool CanRead(const std::string &path);
// path is a directory of .sfrm frames, or one frame inside the run to take the whole run from.
void ReadFiles(const std::string &path);
[[nodiscard]] uint64_t GetNumberOfImages() const override;
void Close() override;
// Whether the sweep really has an image at this point. A deposited series can be missing frames,
// and they are kept as gaps so that every image keeps the spindle angle its own header states;
// ReadRawImage hands such a slot back as "nothing to read".
[[nodiscard]] bool HasImage(int64_t image_number) const;
bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override;
[[nodiscard]] std::vector<SpotToSave> ReadSpots(int64_t image) const override;
};
+18 -64
View File
@@ -7,6 +7,7 @@
#include <algorithm>
#include <cctype>
#include <cmath>
#include <cstring>
#include <fstream>
#include <limits>
@@ -91,67 +92,6 @@ std::optional<double> Angle(const std::string &text, const char *pattern) {
return v;
}
// One "loop_" of the imgCIF template block these headers carry, as rows keyed by tag. Tags and values
// are both read as whitespace-separated tokens rather than by line, because a template packs several
// tags onto one line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are
// laid out to match the tags, not the lines.
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag) {
// The loop_ that introduces the tag, not the tag's own position: everything before it is another
// loop's data.
const size_t tag_at = text.find("\n" + tag);
if (tag_at == std::string::npos)
return {};
const size_t loop_at = text.rfind("loop_", tag_at);
if (loop_at == std::string::npos)
return {};
// Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would
// otherwise be counted as several columns and shift every row after it.
std::vector<std::string> tokens;
for (size_t i = loop_at + 5; i < text.size();) {
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
i++;
if (i >= text.size())
break;
size_t end;
if ((text[i] == '\'') || (text[i] == '"')) {
end = text.find(text[i], i + 1);
if (end == std::string::npos)
break;
tokens.push_back(text.substr(i + 1, end - i - 1));
end++;
} else {
end = i;
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
end++;
tokens.push_back(text.substr(i, end - i));
}
// A second loop_, or a tag belonging to another category, ends this one.
if ((tokens.back() == "loop_")
|| (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) {
tokens.pop_back();
break;
}
i = end;
}
std::vector<std::string> names;
size_t first_value = 0;
while ((first_value < tokens.size()) && tokens[first_value].starts_with("_"))
names.push_back(tokens[first_value++]);
if (names.empty())
return {};
std::vector<std::map<std::string, std::string>> rows;
for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) {
std::map<std::string, std::string> row;
for (size_t j = 0; j < names.size(); j++)
row[names[j]] = tokens[i + j];
rows.push_back(std::move(row));
}
return rows;
}
std::string Field(const std::map<std::string, std::string> &row, const std::string &name) {
const auto it = row.find(name);
return (it == row.end()) ? std::string() : it->second;
@@ -166,7 +106,7 @@ std::string Field(const std::map<std::string, std::string> &row, const std::stri
// unswung detector frame - which is where the image orientation belongs, with the arm applied on top.
// Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458
void ParseAxisTable(const std::string &text, Header &h) {
const auto axes = ParseLoop(text, "_axis.id");
const auto axes = imgcif::ParseLoop(text, "_axis.id");
if (axes.empty())
return;
@@ -199,11 +139,11 @@ void ParseAxisTable(const std::string &text, Header &h) {
// Which axis the fast index runs along, and which the slow, through the two tables that say so.
std::map<std::string, std::string> axis_of_set;
for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
axis_of_set[Field(row, "_array_structure_list_axis.axis_set_id")]
= Field(row, "_array_structure_list_axis.axis_id");
for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) {
for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list.array_id")) {
const std::string set = Field(row, "_array_structure_list.axis_set_id");
const auto id = axis_of_set.contains(set) ? axis_of_set[set] : set;
const auto it = vector_of.find(id);
@@ -261,6 +201,7 @@ Header ParseHeader(const char *data, size_t size) {
// The whitespace after each name is what keeps "Chi_increment" out of "Chi".
h.chi_deg = Angle(t, R"(#\s*Chi\s+([\d.eE+-]+))");
h.omega_deg = Angle(t, R"(#\s*Omega\s+([\d.eE+-]+))");
h.kappa_deg = Angle(t, R"(#\s*Kappa\s+([\d.eE+-]+))");
h.chi_increment_deg = Angle(t, R"(#\s*Chi_increment\s+([\d.eE+-]+))").value_or(0.0);
h.phi_increment_deg = Angle(t, R"(#\s*Phi_increment\s+([\d.eE+-]+))").value_or(0.0);
h.omega_increment_deg = Angle(t, R"(#\s*Omega_increment\s+([\d.eE+-]+))").value_or(0.0);
@@ -292,6 +233,19 @@ Header ParseHeader(const char *data, size_t size) {
const auto conv = Match(t, R"RE(conversions\s*=\s*"([^"]+)")RE").value_or("");
h.byte_offset = conv.find("x-CBF_BYTE_OFFSET") != std::string::npos;
if (!Match(t, R"(#\s*(Pixel_size))").has_value()) { // Match returns the first group
h.imgcif = imgcif::Parse(t);
if (h.imgcif.has_value()) {
h.detector = h.imgcif->detector;
h.pixel_x_m = std::abs(h.imgcif->fast.increment) * 1e-3;
h.pixel_y_m = std::abs(h.imgcif->slow.increment) * 1e-3;
h.wavelength_A = h.imgcif->wavelength_A;
h.count_cutoff = static_cast<int64_t>(h.imgcif->overload);
h.exposure_s = h.imgcif->exposure_s;
h.thickness_m = 0;
}
}
if (h.period_s <= 0.0)
h.period_s = h.exposure_s;
+9
View File
@@ -10,6 +10,7 @@
#include <vector>
#include "../common/CompressedImage.h"
#include "ImgCIF.h"
// PILATUS miniCBF: an ASCII header, four separator bytes, then one byte-offset compressed image.
// No CIF parser and no libcbf are needed - every value is on a "# " comment line or a MIME line.
@@ -39,6 +40,7 @@ struct Header {
// does not really carry is absent here rather than 9999 degrees away from zero.
std::optional<double> chi_deg;
std::optional<double> omega_deg;
std::optional<double> kappa_deg;
double chi_increment_deg = 0;
double phi_increment_deg = 0;
double omega_increment_deg = 0;
@@ -70,6 +72,13 @@ struct Header {
// ScansPhi, and note that the one header carrying both a name and a table says "X" where its table
// says Y) but this token is not: where both are present they agree.
bool spindle_along_slow = false;
// A full imgCIF header - CBFlib's own form, which Bruker's APEX writes when it converts frames -
// states none of the "# " lines and says everything in its CIF tables instead. Where the header
// has no "# Pixel_size" line and those tables are there, this holds them, and the scalar fields
// above that the tables state (pixel size, wavelength, Count_cutoff, exposure, detector) are
// filled in from them. The geometry is the reader's to work out from the axis chain.
std::optional<imgcif::Description> imgcif;
};
// Whether the sweep turns PHI rather than the base (omega) axis. What moved is the axis with a
+338
View File
@@ -0,0 +1,338 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "SFRM.h"
#include <algorithm>
#include <cctype>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <optional>
#include <sstream>
#include "../common/JFJochException.h"
namespace sfrm {
namespace {
constexpr size_t BLOCK = 512;
constexpr size_t LINE = 80;
constexpr size_t KEY = 8; // "NROWS :" - seven characters of name and the colon
JFJochException Invalid(const std::string &what) {
return {JFJochExceptionCategory::InputParameterInvalid, what};
}
std::vector<uint8_t> ReadPrefix(const std::string &path, size_t bytes) {
std::ifstream f(path, std::ios::binary);
if (!f)
throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path);
std::vector<uint8_t> out(bytes);
f.read(reinterpret_cast<char *>(out.data()), static_cast<std::streamsize>(bytes));
out.resize(static_cast<size_t>(f.gcount()));
return out;
}
std::vector<std::string> Words(const std::map<std::string, std::string> &kv, const std::string &key) {
std::vector<std::string> out;
const auto it = kv.find(key);
if (it == kv.end())
return out;
std::istringstream in(it->second);
for (std::string w; in >> w;)
out.push_back(w);
return out;
}
double Num(const std::map<std::string, std::string> &kv, const std::string &key, size_t i,
double fallback = 0.0) {
const auto w = Words(kv, key);
if (i >= w.size())
return fallback;
try {
return std::stod(w[i]);
} catch (const std::logic_error &) {
throw Invalid("Malformed number in SFRM header " + key + ": '" + w[i] + "'");
}
}
int64_t Int(const std::map<std::string, std::string> &kv, const std::string &key, size_t i,
int64_t fallback = 0) {
return static_cast<int64_t>(Num(kv, key, i, static_cast<double>(fallback)));
}
// The 80-character lines of the header, as key -> value. A key that repeats (TITLE, CELL, MATRIX,
// DETPAR, ...) continues its value on the next line, so the pieces are joined.
std::map<std::string, std::string> Lines(const char *data, size_t size) {
std::map<std::string, std::string> kv;
for (size_t at = 0; at + LINE <= size; at += LINE) {
if (data[at + KEY - 1] != ':')
continue; // the padding after the last line
std::string key(data + at, KEY - 1);
key.erase(std::find_if(key.rbegin(), key.rend(), [](unsigned char c) { return !std::isspace(c); }).base(),
key.end());
const std::string value(data + at + KEY, LINE - KEY);
auto &v = kv[key];
v += (v.empty() ? "" : " ") + value;
}
return kv;
}
bool HasExtension(const std::filesystem::path &p) {
std::string ext = p.extension().string();
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
return ext == ".sfrm";
}
bool IsSFRM(const std::string &path) {
const auto head = ReadPrefix(path, KEY);
return head.size() == KEY && std::memcmp(head.data(), "FORMAT :", KEY) == 0;
}
// "ins_zwin_01_0042.sfrm" -> {"ins_zwin_01_", 4}: the run is part of the prefix, so naming a frame
// of run 01 reads run 01 and not the frames of every run in the directory.
struct Template {
std::string prefix;
size_t digits = 0;
bool Matches(const std::string &stem) const {
if (stem.size() != prefix.size() + digits || stem.compare(0, prefix.size(), prefix) != 0)
return false;
return std::all_of(stem.begin() + static_cast<std::ptrdiff_t>(prefix.size()), stem.end(),
[](unsigned char c) { return std::isdigit(c); });
}
};
std::optional<Template> TemplateOf(const std::filesystem::path &p) {
const std::string stem = p.stem().string();
size_t start = stem.size();
while (start > 0 && std::isdigit(static_cast<unsigned char>(stem[start - 1])))
start--;
if (start == stem.size())
return {};
return Template{stem.substr(0, start), stem.size() - start};
}
size_t Pad(size_t bytes, size_t to) {
return (bytes + to - 1) / to * to;
}
uint64_t ReadLE(const uint8_t *p, int64_t bytes) {
uint64_t v = 0;
for (int64_t i = bytes - 1; i >= 0; i--)
v = (v << 8) | p[i];
return v;
}
} // namespace
Header ParseHeader(const char *data, size_t size) {
const auto kv = Lines(data, size);
Header h;
h.format = Int(kv, "FORMAT", 0);
h.header_blocks = Int(kv, "HDRBLKS", 0);
h.nrows = Int(kv, "NROWS", 0);
h.ncols = Int(kv, "NCOLS", 0);
h.bytes_per_pixel = Int(kv, "NPIXELB", 0, 1);
h.bytes_per_underflow = Int(kv, "NPIXELB", 1, 1);
if (h.format == 100) {
for (size_t i = 0; i < 3; i++)
h.overflows[i] = Int(kv, "NOVERFL", i, 0);
} else {
h.overflows = {Int(kv, "NOVERFL", 0, 0), 0, 0};
}
h.baseline = Int(kv, "NEXP", 2, 0);
for (size_t i = 0; i < 4; i++)
h.angles[i] = Num(kv, "ANGLES", i);
h.axis = Int(kv, "AXIS", 0);
h.start_deg = Num(kv, "START", 0);
h.increment_deg = Num(kv, "INCREME", 0);
const auto distance = Words(kv, "DISTANC");
h.distance_cm = Num(kv, "DISTANC", distance.empty() ? 0 : distance.size() - 1);
for (size_t i = 0; i < Words(kv, "CENTER").size(); i++)
h.center.push_back(Num(kv, "CENTER", i));
const auto dettype = Words(kv, "DETTYPE");
h.detector = dettype.empty() ? "Bruker CCD" : dettype[0];
h.pixels_per_cm_512 = Num(kv, "DETTYPE", 1);
h.grid_offset_cm = Num(kv, "DETTYPE", 2);
h.wavelength_A = Num(kv, "WAVELEN", 0);
h.exposure_s = Num(kv, "CUMULAT", 0);
h.saturation = Num(kv, "CCDPARM", 4);
for (size_t i = 0; i < 6; i++)
h.detpar[i] = Num(kv, "DETPAR", i);
h.monochromator_2theta_deg = Num(kv, "FILTER2", 0);
h.raw = kv;
if (h.header_blocks <= 0 || h.nrows <= 0 || h.ncols <= 0)
throw Invalid("SFRM header states no HDRBLKS, NROWS or NCOLS");
if (h.bytes_per_pixel != 1 && h.bytes_per_pixel != 2 && h.bytes_per_pixel != 4)
throw Invalid("SFRM header states " + std::to_string(h.bytes_per_pixel) + " bytes per pixel");
if (Int(kv, "WORDORD", 0) != 0 || Int(kv, "LONGORD", 0) != 0)
throw Invalid("big-endian SFRM frames are not supported");
if (h.format != 86 && h.format != 100)
throw Invalid("SFRM FORMAT " + std::to_string(h.format) + " is not supported");
return h;
}
Header ReadHeader(const std::string &path) {
// Enough to reach HDRBLKS (line 3), then exactly the blocks it states.
auto buf = ReadPrefix(path, 5 * BLOCK);
const auto blocks = Int(Lines(reinterpret_cast<const char *>(buf.data()), buf.size()), "HDRBLKS", 0);
if (blocks <= 0)
throw Invalid(path + " states no HDRBLKS");
buf = ReadPrefix(path, static_cast<size_t>(blocks) * BLOCK);
return ParseHeader(reinterpret_cast<const char *>(buf.data()), buf.size());
}
Header Decode(const uint8_t *data, size_t size, int32_t *out, size_t capacity) {
const size_t header_bytes = 5 * BLOCK;
if (size < header_bytes)
throw Invalid("SFRM file shorter than its header");
const auto blocks = Int(Lines(reinterpret_cast<const char *>(data), header_bytes), "HDRBLKS", 0);
if (blocks <= 0 || static_cast<size_t>(blocks) * BLOCK > size)
throw Invalid("SFRM file shorter than its header");
const Header h = ParseHeader(reinterpret_cast<const char *>(data), static_cast<size_t>(blocks) * BLOCK);
const size_t npixel = static_cast<size_t>(h.nrows) * static_cast<size_t>(h.ncols);
if (npixel > capacity)
throw Invalid("SFRM image does not fit the buffer given for it");
size_t at = static_cast<size_t>(h.header_blocks) * BLOCK;
if (at + npixel * h.bytes_per_pixel > size)
throw Invalid("SFRM file ends before its image does");
for (size_t i = 0; i < npixel; i++)
out[i] = static_cast<int32_t>(ReadLE(data + at + i * h.bytes_per_pixel, h.bytes_per_pixel));
at += npixel * h.bytes_per_pixel;
if (h.format == 86) {
// One ASCII entry per overflowing pixel: its count in 9 characters, its offset in 7.
const int64_t n = h.overflows[0];
if (n > 0 && at + static_cast<size_t>(n) * 16 > size)
throw Invalid("SFRM file ends inside its overflow table");
for (int64_t k = 0; k < n; k++) {
const std::string entry(reinterpret_cast<const char *>(data + at + 16 * k), 16);
int64_t value, offset;
try {
value = std::stoll(entry.substr(0, 9));
offset = std::stoll(entry.substr(9, 7));
} catch (const std::logic_error &) {
throw Invalid("malformed SFRM overflow entry '" + entry + "'");
}
if (offset < 0 || static_cast<size_t>(offset) >= npixel)
throw Invalid("SFRM overflow entry points outside the image");
out[offset] = static_cast<int32_t>(value);
}
} else {
at = Pad(at, 16);
// The three tables, and where the next unread entry of each is.
const auto table = [&](int64_t n, int64_t bytes) {
const size_t begin = at;
if (n > 0) {
at = Pad(at + static_cast<size_t>(n) * bytes, 16);
if (begin + static_cast<size_t>(n) * bytes > size)
throw Invalid("SFRM file ends inside its overflow tables");
}
return begin;
};
const size_t under = table(h.overflows[0], h.bytes_per_underflow);
const size_t over16 = table(h.overflows[1], 2);
const size_t over32 = table(h.overflows[2], 4);
int64_t n_under = 0, n16 = 0, n32 = 0;
const auto next = [&](size_t begin, int64_t &used, int64_t total, int64_t bytes) {
if (used >= total)
throw Invalid("SFRM frame has more overflowing pixels than its tables hold");
return ReadLE(data + begin + static_cast<size_t>(used++) * bytes, bytes);
};
const uint64_t max_pixel = (h.bytes_per_pixel == 4) ? UINT32_MAX : (1ULL << (8 * h.bytes_per_pixel)) - 1;
for (size_t i = 0; i < npixel; i++) {
uint64_t v = static_cast<uint32_t>(out[i]);
if (v == 0 && h.overflows[0] > 0)
v = next(under, n_under, h.overflows[0], h.bytes_per_underflow);
else if (h.bytes_per_pixel < 4 && v == max_pixel) {
if (h.bytes_per_pixel == 1)
v = next(over16, n16, h.overflows[1], 2);
if (v == 0xFFFF) {
const auto w = static_cast<int32_t>(next(over32, n32, h.overflows[2], 4));
out[i] = static_cast<int32_t>(w - h.baseline);
continue;
}
}
out[i] = static_cast<int32_t>(static_cast<int64_t>(v) - h.baseline);
}
return h;
}
for (size_t i = 0; i < npixel; i++)
out[i] -= static_cast<int32_t>(h.baseline);
return h;
}
Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vector<uint8_t> &scratch) {
std::ifstream f(path, std::ios::binary);
if (!f)
throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path);
f.seekg(0, std::ios::end);
scratch.resize(static_cast<size_t>(f.tellg()));
f.seekg(0, std::ios::beg);
f.read(reinterpret_cast<char *>(scratch.data()), static_cast<std::streamsize>(scratch.size()));
try {
return Decode(scratch.data(), static_cast<size_t>(f.gcount()), out, capacity);
} catch (const JFJochException &e) {
throw Invalid(path + ": " + e.what());
}
}
Header Read(const std::string &path, std::vector<int32_t> &out) {
const Header h = ReadHeader(path);
out.resize(static_cast<size_t>(h.nrows) * static_cast<size_t>(h.ncols));
std::vector<uint8_t> scratch;
return ReadInto(path, out.data(), out.size(), scratch);
}
std::vector<std::string> CollectSweep(const std::string &path) {
const std::filesystem::path p(path);
std::error_code ec;
const bool is_dir = std::filesystem::is_directory(p, ec);
std::filesystem::path dir = is_dir ? p : p.parent_path();
if (dir.empty())
dir = ".";
// Naming a frame selects its run. Naming a directory selects the run with the most frames, which
// is the one a user pointing at a data directory means - a Bruker collection directory holds one
// run per scan, and every run is its own sweep.
std::optional<Template> want;
if (!is_dir)
want = TemplateOf(p);
std::map<std::pair<std::string, size_t>, std::vector<std::string>> runs;
for (const auto &e: std::filesystem::directory_iterator(dir, ec)) {
if (!e.is_regular_file() || !HasExtension(e.path()))
continue;
const auto t = TemplateOf(e.path());
if (!t.has_value() || (want.has_value() && !want->Matches(e.path().stem().string())))
continue;
runs[{t->prefix, t->digits}].push_back(e.path().string());
}
std::vector<std::string> out;
for (auto &[key, files]: runs)
if (files.size() > out.size())
out = std::move(files);
std::sort(out.begin(), out.end());
if (!out.empty() && !IsSFRM(out.front()))
return {};
return out;
}
bool CanRead(const std::string &path) {
std::error_code ec;
try {
if (std::filesystem::is_directory(path, ec))
return !CollectSweep(path).empty();
return HasExtension(std::filesystem::path(path)) && IsSFRM(path);
} catch (const JFJochException &) {
return false;
}
}
} // namespace sfrm
+85
View File
@@ -0,0 +1,85 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <array>
#include <cstdint>
#include <map>
#include <string>
#include <vector>
// Bruker .sfrm: the frame format of Bruker's SMART, PROTEUM and APEX software. An ASCII header of
// 80-character "KEY :value" lines, HDRBLKS blocks of 512 bytes long, then the pixels as unsigned
// little-endian integers of NPIXELB[0] bytes, row after row, then the tables that hold what did not
// fit in that container.
//
// FORMAT :100 VERSION:11 HDRBLKS:15
// NOVERFL:-1 5502 0 NPIXELB:1 1 NROWS :1024 NCOLS :1024
// ANGLES :2theta omega phi chi AXIS :2 START :0.0 INCREME:-0.2
// DISTANC:10.0 (cm) CENTER :506 511 ... WAVELEN:1.5419 1.5406 1.5444
//
// Two layouts of the tables exist. FORMAT 86 (VERSION < 11) has one ASCII overflow table after the
// image: NOVERFL entries of 16 characters, a 9-digit count and a 7-digit pixel offset, padded to a
// whole 512-byte block. FORMAT 100 has three binary tables, each padded to 16 bytes: NOVERFL[0]
// underflows of NPIXELB[1] bytes (absent where NOVERFL[0] is -1), NOVERFL[1] 16-bit overflows and
// NOVERFL[2] 32-bit ones. A pixel holding its container's maximum takes the next value of the next
// wider table, in pixel order; a pixel holding 0 takes the next underflow. The 32-bit table is
// signed - a pixel far enough below the baseline is stored there as a negative count. Every stored value carries
// the baseline offset NEXP[2], so the counts are the values less that offset.
//
// The layout as Bruker's frame format documents it, checked byte for byte against APEX's own
// conversion of the same frames to imgCIF: the pixels come out identical but for the baseline, which
// the conversion subtracts.
namespace sfrm {
struct Header {
int64_t format = 0; // 86 or 100
int64_t header_blocks = 0; // HDRBLKS
int64_t nrows = 0;
int64_t ncols = 0;
int64_t bytes_per_pixel = 1; // NPIXELB[0]
int64_t bytes_per_underflow = 1; // NPIXELB[1], FORMAT 100
std::array<int64_t, 3> overflows{-1, 0, 0}; // NOVERFL: FORMAT 86 uses [0] only
int64_t baseline = 0; // NEXP[2]
std::array<double, 4> angles{}; // ANGLES: 2theta, omega, phi, chi at the start of the frame (deg)
int64_t axis = 0; // AXIS: 1 2theta, 2 omega, 3 phi, 4 chi
double start_deg = 0; // START
double increment_deg = 0; // INCREME, signed
double distance_cm = 0; // DISTANC, its last value
std::vector<double> center; // CENTER, pixels
double pixels_per_cm_512 = 0; // DETTYPE[1]: pixels per cm on a 512-pixel frame
double grid_offset_cm = 0; // DETTYPE[2]: detector face to the calibration grid
double wavelength_A = 0; // WAVELEN[0]
double exposure_s = 0; // CUMULAT
double saturation = 0; // CCDPARM[4]; 0 where not stated
std::array<double, 6> detpar{}; // DETPAR: three offsets, then pitch, roll, yaw (deg)
double monochromator_2theta_deg = 0; // FILTER2[0]: the monochromator's scattering angle; 0 for none
std::string detector; // DETTYPE[0]
std::map<std::string, std::string> raw; // every key; repeated keys joined with a space
};
// True if the path names something this reader can open: a .sfrm frame, or a directory holding one.
bool CanRead(const std::string &path);
// The files of one sweep, in collection order. path is a directory, or one frame inside it.
std::vector<std::string> CollectSweep(const std::string &path);
// Parse a header from its bytes (at least HDRBLKS * 512 of them).
Header ParseHeader(const char *data, size_t size);
// Header only - reads just the header blocks.
Header ReadHeader(const std::string &path);
// Header + counts (stored value with the overflows resolved, less the baseline) into memory the
// caller already has. scratch carries the raw bytes between frames.
Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vector<uint8_t> &scratch);
// The same, decoding from the whole file already in memory. Throws if out holds fewer than
// nrows * ncols pixels or the tables run past the end.
Header Decode(const uint8_t *data, size_t size, int32_t *out, size_t capacity);
// Header + counts into a vector sized to fit.
Header Read(const std::string &path, std::vector<int32_t> &out);
} // namespace sfrm
+4 -1
View File
@@ -17,6 +17,8 @@ ADD_LIBRARY(JFJochRugnux STATIC
RugnuxFirstPassRescues.cpp
RugnuxFirstPassAccept.cpp
RugnuxScaleMerge.cpp
RugnuxJointSweeps.cpp
RugnuxJointSweeps.h
RugnuxDefaults.cpp
RugnuxDefaults.h
RugnuxLive.cpp
@@ -40,7 +42,8 @@ TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPrevie
# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux
# returns: the library itself writes none.
ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h RugnuxInputKind.cpp RugnuxInputKind.h)
ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h RugnuxInputKind.cpp RugnuxInputKind.h
RugnuxSweeps.cpp RugnuxSweeps.h)
TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter ZLIB::ZLIB)
# rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the
+1
View File
@@ -27,6 +27,7 @@ namespace PathologyCode {
constexpr const char *SYMMETRY_AMBIGUITY = "SYMMETRY_AMBIGUITY"; // the data do not pick a point group
constexpr const char *CENTERING_UNTESTED = "CENTERING_UNTESTED"; // centring from the metric only
constexpr const char *UNUSABLE_MERGE = "UNUSABLE_MERGE"; // the merged data carry no signal
constexpr const char *SWEEP_LEFT_OUT = "SWEEP_LEFT_OUT"; // a sweep given is not in the joint merge
constexpr const char *LOW_COMPLETENESS = "LOW_COMPLETENESS"; // too little of reciprocal space
constexpr const char *SWEEP_GAPS = "SWEEP_GAPS"; // ranges that delivered much less
constexpr const char *GONIO_SCALE = "GONIO_SCALE"; // the stage rotation is mis-scaled
+58
View File
@@ -941,6 +941,64 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
" the post-refined geometry. Every number in this report describes the pass named above,\n"
" whose files are {}.*; the header-geometry pass is not written.", output_prefix), true));
doc.sections.push_back(std::move(s));
}
// ---------------------------------------------- 1b. SWEEPS (a joint merge of several)
if (!result.joint_sweeps.empty()) {
ReportSection s;
s.title = "1b. SWEEPS";
Add(s, KeyInt("SWEEP_COUNT", static_cast<int64_t>(result.joint_sweeps.size())));
Add(s, KeyInt("SWEEP_REFERENCE", result.joint_reference_sweep + 1));
if (!result.segment_decay_model.empty())
Add(s, KeyText("SWEEP_DECAY_MODEL", result.segment_decay_model));
for (size_t k = 0; k < result.joint_sweeps.size(); ++k) {
const auto &w = result.joint_sweeps[k];
const std::string p = "SWEEP_" + std::to_string(k + 1) + "_";
const auto key = [&p](const char *name) { return p + name; };
std::vector<std::string> names; // the keys outlive the calls that take their c_str()
auto text = [&](const char *name, const std::string &v) {
names.push_back(key(name));
Add(s, KeyText(names.back().c_str(), v));
};
auto real = [&](const char *name, double v, const char *fmt_) {
names.push_back(key(name));
Add(s, KeyReal(names.back().c_str(), v, fmt_));
};
auto integer = [&](const char *name, int64_t v) {
names.push_back(key(name));
Add(s, KeyInt(names.back().c_str(), v));
};
text("INPUT", w.input);
integer("IMAGES", w.images);
text("INCLUDED", w.included ? "TRUE" : "FALSE");
if (!w.space_group_alone.empty())
text("SPACE_GROUP_ALONE", w.space_group_alone);
if (w.cell_alone)
text("CELL", CellString(*w.cell_alone));
if (std::isfinite(w.cc_half_alone))
real("CC_HALF_ALONE", w.cc_half_alone, "{:.4f}");
if (std::isfinite(w.completeness_alone))
real("COMPLETENESS_ALONE", w.completeness_alone, "{:.1f}");
if (std::isfinite(w.d_min_alone))
real("RESOLUTION_ALONE", w.d_min_alone, "{:.2f}");
if (w.included) {
text("REINDEX_OP", w.reindex_op);
real("REINDEX_CC", w.reindex_cc, "{:.4f}");
integer("BATCH_FIRST", w.batch_first);
integer("BATCH_LAST", w.batch_last);
if (w.segment >= 0 && w.segment < static_cast<int>(result.segment_results.size())) {
const auto &seg = result.segment_results[w.segment];
if (std::isfinite(seg.scale))
real("SCALE", seg.scale, "{:.4f}");
if (std::isfinite(seg.relative_b))
real("RELATIVE_B", seg.relative_b, "{:.2f}");
if (seg.dose_rank >= 0)
integer("DOSE_RANK", seg.dose_rank + 1);
}
}
}
doc.sections.push_back(std::move(s));
}
// ------------------------------------------------- XDS GEOMETRY KEYWORDS
+39 -5
View File
@@ -832,6 +832,21 @@ std::vector<bool> Rugnux::PrepassSubset(int prepass_images) const {
return keep;
}
ProcessResult Rugnux::RunJoint(std::vector<IntegrationOutcome> outcomes, const UnitCell &cell,
gemmi::CrystalSystem system, char centring, RugnuxObserver *observer) {
const auto run_start = std::chrono::steady_clock::now();
joint_outcomes_ = std::move(outcomes);
joint_cell_ = cell;
joint_system_ = system;
joint_centring_ = centring;
decide_late_ = true;
ProcessResult result = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
joint_outcomes_.reset();
result.pass_count = 1;
result.total_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - run_start).count();
return result;
}
ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
Logger logger("Rugnux");
ProcessResult result;
@@ -863,7 +878,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
if (config_.stride <= 0)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image stride must be positive");
const auto total_images_in_file = static_cast<int>(reader_.GetNumberOfImages());
const bool joint = joint_outcomes_.has_value();
const auto total_images_in_file = joint ? static_cast<int>(joint_outcomes_->size())
: static_cast<int>(reader_.GetNumberOfImages());
int end_image = config_.end_image;
if (end_image < 0 || end_image > total_images_in_file)
end_image = total_images_in_file;
@@ -952,7 +969,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// After any geometry refinement, so the shadow is found about the beam centre actually used,
// and before the azimuthal mapping and the output mask below, which both read pixel_mask_.
if (config_.detect_beam_stop.has_value() || config_.adaptive_integration_radius)
if (!joint && (config_.detect_beam_stop.has_value() || config_.adaptive_integration_radius))
PreScan(start_image, early_images,
std::min(config_.detect_beam_stop.value_or(BEAM_CENTER_PROJECTION_IMAGES),
HotPixelFinder::MAX_SAMPLED_FRAMES), observer);
@@ -1165,6 +1182,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
// (see the reflection mask in BraggIntegrationEngineCPU). A fixed group used to cost real
// intensity accuracy for that reason.
indexer->PredictAllCentringNodes(true);
if (joint) {
indexer->GetIntegrationOutcome() = std::move(*joint_outcomes_);
joint_outcomes_.reset();
}
}
const auto start_time = std::chrono::steady_clock::now();
@@ -1201,7 +1222,19 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
.scan_result = scan_result, .unmerged_mtz = unmerged_mtz};
// First pass of two-pass rotation indexing (full analysis only).
if (indexing && force_rotation_result_.has_value()) {
if (joint) {
// The sweeps were indexed, integrated and combined each on its own; their fulls are the frames.
result.rotation_lattice_found = true;
result.indexing_rate = 1.0f;
result.images_processed = static_cast<uint64_t>(images_to_process);
result.consensus_cell = joint_cell_;
result.consensus_centering = joint_centring_;
end_msg.rotation_lattice = CrystalLattice(*joint_cell_);
end_msg.rotation_lattice_type = LatticeMessage{.centering = joint_centring_, .niggli_class = 0,
.crystal_system = joint_system_};
result.rotation_lattice_type = end_msg.rotation_lattice_type;
end_msg.unit_cell = result.consensus_cell;
} else if (indexing && force_rotation_result_.has_value()) {
// Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
// search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
// integrates markedly worse.
@@ -1239,7 +1272,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
//
// Only an output pass probes: the answer the run returns is always one (RunAllPasses), and no other
// pass's probe is read - the pre-pass, the beam-centre arm and the probe passes leave theirs unread.
if (indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
if (!joint && indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
std::vector<int64_t> frames;
for (const int o : select_equally_spaced_image_ordinals(images_to_process, 60))
frames.push_back(o);
@@ -1250,7 +1283,8 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
std::optional<GPUImagePass> gpu_image_pass; // what a probe that does not fit beside it waits for
if (!gpu_work_beside_)
gpu_image_pass.emplace();
ImagePass(p);
if (!joint)
ImagePass(p);
gpu_image_pass.reset();
image_pass_mark.Stop();
+38
View File
@@ -115,6 +115,11 @@ struct ProcessConfig {
// scaling.
bool write_merged = true;
// One sweep of several merged as one (--sweeps joint): the run is made as on its own, but its written
// merge hands back all of its fulls, past the resolution cut too (MergedOutput::scaled_fulls), for the
// joint merge to scale and cut again (Rugnux::RunJoint).
bool sweep_member = false;
SpotFindingSettings spot_finding; // FullAnalysis spot finding (and Calibration by spots)
// Calibration: how the powder rings are measured, and the calibrant's rings as q = 2*pi/d (1/A).
@@ -942,6 +947,26 @@ struct ProcessResult {
// spindle, an indexing ambiguity). The same messages the log warns about, each carrying a closed
// PathologyCode for machinery beside the open sentence for a person.
std::vector<ReportWarning> warnings;
// A joint merge of several sweeps (Rugnux::RunJoint, --sweeps joint): what its merge measured per sweep
// segment (RotationScaleMerge::Result::segments, in segment order) and how it modelled the decay, and
// what the run knows of each sweep (filled by the caller, in input order). Empty on a run of one input.
struct JointSegment { double scale = NAN, relative_b = NAN; int dose_rank = -1; };
std::vector<JointSegment> segment_results;
std::string segment_decay_model;
struct JointSweep {
std::string input;
int64_t images = 0;
bool included = false;
int segment = -1; // its segment of the joint merge, where it is in it
std::string space_group_alone;
std::optional<UnitCell> cell_alone;
std::string reindex_op; // onto the reference sweep's axes
double reindex_cc = NAN;
int64_t batch_first = 0, batch_last = -1;
double cc_half_alone = NAN, completeness_alone = NAN, d_min_alone = NAN;
};
std::vector<JointSweep> joint_sweeps;
int joint_reference_sweep = -1;
// How the crystal sat on the spindle: the angle to the nearest symmetry axis and that axis's
// order. A property of the mounting rather than of the data, and the one number that says whether
@@ -1323,6 +1348,11 @@ class Rugnux {
// Whether the first passes decide their near ties on all the frames: offline, on a re-check, and on
// the run made again.
bool decide_late_ = false;
// RunJoint's frames and cell, for the one pass it makes (RunPipeline); empty otherwise.
std::optional<std::vector<IntegrationOutcome>> joint_outcomes_;
std::optional<UnitCell> joint_cell_;
gemmi::CrystalSystem joint_system_ = gemmi::CrystalSystem::Triclinic;
char joint_centring_ = 'P';
// Whether what the run otherwise decides on the pre-pass frames - the rotation-scale walk's refits,
// the demoted metric-symmetry arm, the integration radius (RefinedPasses) - is decided on all the
// frames: only on the run made again. Offline too these are decided on the pre-pass frames, and a near
@@ -1725,6 +1755,14 @@ public:
// Runs the configured workflow to completion or until Cancel(). Throws on setup failure.
ProcessResult Run(RugnuxObserver *observer = nullptr);
// The joint merge of several sweeps (--sweeps joint): `outcomes` are the frames of all of them laid
// end to end, each holding the fulls its own sweep combined (ScalingSettings::SweepSegments says which
// frames are whose), indexed on the axes of `cell`. Nothing is read from the images: the merge, the
// space-group search and everything after it run on these as on the outcomes of one sweep.
// `system` and `centring` are those of the lattice the cell is the conventional cell of.
ProcessResult RunJoint(std::vector<IntegrationOutcome> outcomes, const UnitCell &cell,
gemmi::CrystalSystem system, char centring, RugnuxObserver *observer = nullptr);
// The pinned host buffers the engines of a run kept for each other (CUDAMemHelpers.h), handed back to the
// driver. Not part of Run: a process that outlives the run (the broker, the viewer) calls it once the
// results are out, as it takes a tenth of a second the user would otherwise wait for; one that ends
+182
View File
@@ -0,0 +1,182 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "RugnuxJointSweeps.h"
#include <algorithm>
#include <cmath>
#include <map>
#include <numeric>
#include "../common/CrystalLattice.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
#include "../gemmi_gph/gemmi/mtz.hpp"
JointSweepRelation RelateJointSweeps(const std::vector<JointMemberSweep> &sweeps) {
JointSweepRelation out;
out.sweeps.resize(sweeps.size());
if (sweeps.empty())
return out;
out.reference = 0;
for (size_t k = 1; k < sweeps.size(); ++k)
if (sweeps[k].fulls.size() > sweeps[out.reference].fulls.size())
out.reference = static_cast<int>(k);
const JointMemberSweep &ref = sweeps[out.reference];
out.sweeps[out.reference] = {true, gemmi::Op::identity(), 1.0, 1.0, {}};
for (size_t k = 0; k < sweeps.size(); ++k) {
if (static_cast<int>(k) == out.reference)
continue;
JointSweepRelation::Sweep &s = out.sweeps[k];
if (sweeps[k].merged.empty()) {
s.why_left_out = "it merged nothing";
continue;
}
const ReferenceSettingChoice choice = ChooseReferenceSetting(sweeps[k].merged, sweeps[k].cell,
sweeps[k].space_group, ref.merged, ref.cell,
ref.space_group);
s.cob = choice.cob;
s.cc = choice.cc;
s.identity_cc = choice.identity_cc;
s.included = std::isfinite(choice.cc) && choice.cc >= MIN_REFERENCE_CC;
if (!s.included)
s.why_left_out = "its intensities correlate with the reference sweep's on no axes (best CC "
+ std::to_string(choice.cc) + ")";
}
return out;
}
JointMergeInput BuildJointMergeInput(const std::vector<JointMemberSweep> &sweeps,
const JointSweepRelation &relation) {
JointMergeInput out;
const CrystalLattice lattice(sweeps[relation.reference].cell);
int32_t first = 0;
for (size_t k = 0; k < sweeps.size(); ++k) {
const auto &sweep = sweeps[k];
const auto &rel = relation.sweeps[k];
if (!rel.included || sweep.n_frames <= 0)
continue;
out.segments.push_back({first, static_cast<int32_t>(sweep.n_frames)});
out.segment_sweep.push_back(static_cast<int>(k));
out.outcomes.resize(first + sweep.n_frames);
for (int64_t f = 0; f < sweep.n_frames; ++f)
out.outcomes[first + f].latt = lattice;
for (const ScaledFull &full : sweep.fulls) {
if (!std::isfinite(full.I) || !std::isfinite(full.var_bkg) || !(full.var_per_I > 0.0f)
|| !(full.d > 0.0f))
continue;
const auto frame = std::clamp<int64_t>(std::llround(full.image_number), 0, sweep.n_frames - 1);
const gemmi::Op::Miller hkl = rel.cob.apply_to_hkl({{full.h, full.k, full.l}});
// A full as the merge takes a pre-combined observation, so that the combine rebuilds the variance
// its own sweep's merge gave it: a count c of intensity I/c at correction c = var_per_I is a
// variance c^2 var_bkg' + c I at I, which is the sweep's var_bkg + var_per_I I with
// var_bkg' = var_bkg / c^2 (the capture term folded in at the full's own intensity).
const float c = full.var_per_I;
const float capture_var = std::isfinite(full.capture) ? full.capture * full.I * full.capture * full.I : 0.0f;
Reflection r{};
r.h = hkl[0]; r.k = hkl[1]; r.l = hkl[2];
r.image_number = static_cast<float>(first + frame);
r.d = full.d;
r.I = full.I / c;
r.var_bkg = (full.var_bkg + capture_var) / (c * c);
r.sigma = std::sqrt(std::max(r.var_bkg + std::max(0.0f, r.I), 1e-12f));
r.prescaling_corr = c;
r.partiality = 1.0f;
r.zeta = 1.0f;
r.image_scale_corr = c;
r.observed = true;
out.outcomes[first + frame].reflections.push_back(r);
}
first += static_cast<int32_t>(sweep.n_frames);
}
return out;
}
int64_t JointBatchBase(const JointMergeInput &input) {
int64_t longest = 0;
for (const auto &seg : input.segments)
longest = std::max<int64_t>(longest, seg.n_frames);
int64_t base = 1;
while (base <= longest)
base *= 10;
return base;
}
MtzTable JointUnmergedMtzTable(const std::vector<ScaledFull> &fulls, const UnitCell &cell,
const gemmi::SpaceGroup &space_group, const std::vector<JointMemberSweep> &sweeps,
const JointMergeInput &input, float wavelength_A,
const std::vector<std::string> &history) {
MtzTable mtz;
mtz.space_group_name = space_group.xhm();
mtz.unit_cell = cell;
mtz.title = "Rugnux unmerged observations, scaled, joint merge of several sweeps";
mtz.history = history;
mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f}, {"unmerged", "", "unmerged", wavelength_A}};
for (const auto &[label, type] : std::vector<std::pair<const char *, char>>{
{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'},
{"FRACTIONCALC", 'R'}, {"ROT", 'R'}})
mtz.columns.push_back({label, type, 0, {}});
const int64_t base = JointBatchBase(input);
// Per segment, a header for every frame: the sweep's own where its file has one, else its nearest one
// turned by the frames between them (a sweep's file holds the frames its observations span, and a
// batch with no header stops a reading program).
std::vector<std::map<int32_t, MtzTable::Batch>> own(input.segments.size());
for (size_t s = 0; s < input.segments.size(); ++s) {
const auto &batches = sweeps[input.segment_sweep[s]].batches;
if (batches.empty())
continue;
for (int32_t n = 1; n <= input.segments[s].n_frames; ++n) {
const auto nearest = std::min_element(batches.begin(), batches.end(), [n](const auto &a, const auto &b) {
return std::abs(a.number - n) < std::abs(b.number - n);
});
MtzTable::Batch copy = *nearest;
if (copy.number != n && copy.floats.size() > 47) {
const float shift = static_cast<float>(n - copy.number) * copy.floats[47];
copy.floats[36] += shift;
copy.floats[37] += shift;
}
own[s][n] = copy;
copy.number = static_cast<int32_t>((s + 1) * base + n);
mtz.batches.push_back(std::move(copy));
}
}
const gemmi::GroupOps ops = space_group.operations();
const auto centring_absent = [&](int h, int k, int l) {
for (const auto &t : ops.cen_ops)
if ((h * t[0] + k * t[1] + l * t[2]) % gemmi::Op::DEN != 0)
return true;
return false;
};
gemmi::UnmergedHklMover mover(&space_group);
std::vector<std::array<float, 9>> rows;
for (const ScaledFull &f : fulls) {
if (centring_absent(f.h, f.k, f.l))
continue;
const auto frame = static_cast<int32_t>(std::lround(f.image_number));
size_t s = 0;
while (s + 1 < input.segments.size() && frame >= input.segments[s + 1].first_frame)
++s;
const int32_t local = frame - input.segments[s].first_frame + 1; // the sweep's own batch
float rot = NAN;
if (const auto it = own[s].find(local); it != own[s].end() && it->second.floats.size() > 47)
rot = it->second.floats[36] + it->second.floats[47] / 2.0f;
std::array<int, 3> hkl{f.h, f.k, f.l};
const int isym = mover.move_to_asu(hkl);
rows.push_back({static_cast<float>(hkl[0]), static_cast<float>(hkl[1]), static_cast<float>(hkl[2]),
static_cast<float>(isym), static_cast<float>((s + 1) * base + local), f.I, f.sigma, 1.0f,
rot});
}
std::vector<size_t> order(rows.size());
std::iota(order.begin(), order.end(), 0);
std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) {
return std::lexicographical_compare(rows[a].begin(), rows[a].begin() + 5, rows[b].begin(), rows[b].begin() + 5);
});
for (size_t c = 0; c < mtz.columns.size(); ++c) {
mtz.columns[c].values.resize(rows.size());
for (size_t i = 0; i < order.size(); ++i)
mtz.columns[c].values[i] = rows[order[i]][c];
}
mtz.sort_order = {1, 2, 3, 4, 5};
return mtz;
}
+75
View File
@@ -0,0 +1,75 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <string>
#include <vector>
#include "../common/MtzTable.h"
#include "../common/Reflection.h"
#include "../common/ScalingSettings.h"
#include "../common/UnitCell.h"
#include "../gemmi_gph/gemmi/symmetry.hpp"
#include "../image_analysis/IntegrationOutcome.h"
// Several sweeps of one crystal merged as one (--sweeps joint), in two levels: each sweep is processed on
// its own - indexed, integrated, its partials combined into fulls and scaled - and the fulls of all of them
// are then scaled and merged together (RotationScaleMerge with SweepSegments), as XSCALE merges the data
// sets XDS made of each sweep. Between the two the sweeps are put on one set of axes: the indexing each
// was given is its own, and an ambiguous lattice or a different choice of axes indexes the same crystal
// differently from one sweep to the next.
// What the joint merge takes from one sweep's own run.
struct JointMemberSweep {
std::string label;
int64_t n_frames = 0;
UnitCell cell{}; // as its reflections are written
gemmi::SpaceGroup space_group; // the group its run determined
std::vector<MergedReflection> merged;
std::vector<ScaledFull> fulls; // all its fulls, unmerged, on the same axes
std::vector<MtzTable::Batch> batches; // the batch headers of its own unmerged MTZ
double cc_half = NAN, completeness = NAN, d_min = NAN; // its merge on its own
};
// How each sweep is put on the axes of the reference - the sweep with the most fulls: the change of basis
// (an axis permutation or a twin law of the lattice, ChooseReferenceSetting) under which its merged
// intensities correlate best with the reference's, and that correlation. A sweep whose best correlation is
// below MIN_REFERENCE_CC is not the same crystal form on any axes, and is left out.
struct JointSweepRelation {
struct Sweep {
bool included = false;
gemmi::Op cob = gemmi::Op::identity();
double cc = 0.0; // with the reference, under cob
double identity_cc = 0.0; // ... and on its own axes
std::string why_left_out;
};
int reference = -1;
std::vector<Sweep> sweeps;
};
JointSweepRelation RelateJointSweeps(const std::vector<JointMemberSweep> &sweeps);
// The joint merge's input: the frames of the included sweeps laid end to end, in the order given, each
// holding its sweep's fulls on the reference's axes as pre-combined observations, and the segments that
// say whose frames are whose.
struct JointMergeInput {
std::vector<IntegrationOutcome> outcomes;
std::vector<SweepSegment> segments;
std::vector<int> segment_sweep; // the sweep each segment is
};
JointMergeInput BuildJointMergeInput(const std::vector<JointMemberSweep> &sweeps,
const JointSweepRelation &relation);
// The batch numbers of the joint merge: segment s (from 0) is batches (s + 1) * base + frame + 1, base the
// power of ten past the longest sweep - 1001.. and 2001.. for sweeps of up to 999 frames - so a sweep's
// batches read as its own frames and never run into the next sweep's.
int64_t JointBatchBase(const JointMergeInput &input);
// The joint merge's unmerged MTZ: its scaled fulls (MergedOutput::scaled_fulls, on the merge's axes and
// frames), each in the batch of its sweep's frame, under the batch headers that sweep's own run wrote -
// its orientation, axis and angles - renumbered (JointBatchBase). Lattice-centring absences are left out,
// as in a single sweep's file.
MtzTable JointUnmergedMtzTable(const std::vector<ScaledFull> &fulls, const UnitCell &cell,
const gemmi::SpaceGroup &space_group, const std::vector<JointMemberSweep> &sweeps,
const JointMergeInput &input, float wavelength_A,
const std::vector<std::string> &history);
+5
View File
@@ -1671,6 +1671,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
result.scaling_iterations_partials = r.scaling_iterations_partials;
result.scaling_iterations_fulls = r.scaling_iterations_fulls;
result.scaling_converged = r.scaling_converged;
result.segment_results.clear();
for (const auto &seg : r.segments)
result.segment_results.push_back({seg.scale, seg.relative_b, seg.dose_rank});
result.segment_decay_model = r.segment_decay_model;
if (for_search)
result.search_scaling_converged = r.scaling_converged;
Note(fmt::format("merge in {}: {} unique", label, r.merged.size()));
@@ -1693,6 +1697,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
// Only a merge that may be written hands its fulls back: they are a copy as long as the
// fulls, and the search merges and the pre-pass never write a .hkl.
rsm->SetExportScaledFulls(!for_search && !geometry_prepass && write_output && config_.write_merged);
rsm->SetExportScaledFullsUncut(config_.sweep_member);
// Amplitudes are made once, on the merge that is written, where the anisotropy tensor
// their prior uses is known (see below) - not on every merge the tail makes on its way.
rsm->SetFrenchWilson(false);
+140
View File
@@ -0,0 +1,140 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "RugnuxSweeps.h"
#include <algorithm>
#include <cctype>
#include <cmath>
#include <filesystem>
#include <map>
#include <set>
#include <stdexcept>
#include <spdlog/fmt/fmt.h>
namespace {
// How far two sweeps may state the wavelength apart and still be one dataset: JFJochMultiReader's
// tolerance for the inputs of one stills run.
constexpr double WAVELENGTH_TOLERANCE = 1e-3; // relative
std::string Lower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
}
std::string RugnuxFrameTemplate(const std::string &path) {
const std::filesystem::path p(path);
std::string name = p.filename().string();
size_t end = name.size();
while (end > 0 && !std::isdigit(static_cast<unsigned char>(name[end - 1])))
end--;
if (end == 0)
return {};
size_t start = end;
while (start > 0 && std::isdigit(static_cast<unsigned char>(name[start - 1])))
start--;
name.replace(start, end - start, std::string(end - start, '#'));
return (p.parent_path() / name).string();
}
RugnuxSweepPlan PlanRugnuxSweeps(const std::vector<std::string> &inputs,
const std::function<RugnuxInputDescription(const std::string &)> &describe_file,
const std::function<RugnuxInputDescription(const std::string &)> &describe_sweep) {
RugnuxSweepPlan plan;
if (inputs.size() < 2)
return plan;
std::vector<RugnuxInputDescription> files;
for (const auto &f: inputs)
files.push_back(describe_file(f));
const bool any_foreign = std::any_of(files.begin(), files.end(), [](const auto &d) { return d.foreign; });
const size_t n_frames = std::count_if(files.begin(), files.end(),
[](const auto &d) { return !d.frame_template.empty(); });
// Inputs of the stills readers, or a mix of frames and masters: read as before, which takes the
// former and refuses the latter.
if (any_foreign || (n_frames != 0 && n_frames != inputs.size()))
return plan;
if (n_frames == inputs.size()) {
std::set<std::string> seen;
for (size_t i = 0; i < inputs.size(); i++)
if (seen.insert(files[i].frame_template).second)
plan.sweeps.push_back({inputs[i], files[i].frame_template, true});
if (plan.sweeps.size() == 1)
return plan;
} else {
// A first input of stills is a stills run, read together as before - and refused there if any
// other input turns. A rotation first input asks every other one.
if (!describe_sweep(inputs[0]).rotation)
return plan;
std::vector<std::string> rotation{inputs[0]}, stills;
for (size_t i = 1; i < inputs.size(); i++)
(describe_sweep(inputs[i]).rotation ? rotation : stills).push_back(inputs[i]);
if (!stills.empty())
throw std::runtime_error(fmt::format(
"{} is a rotation sweep and {} a stills collection: rotation sweeps and stills are not "
"processed together; give them in separate runs", rotation.front(), stills.front()));
for (const auto &f: inputs)
plan.sweeps.push_back({f, f, false});
}
std::vector<double> wavelength;
for (const auto &s: plan.sweeps)
wavelength.push_back(describe_sweep(s.input).wavelength_A);
for (size_t k = 1; k < plan.sweeps.size(); k++)
if (std::fabs(wavelength[k] - wavelength[0]) > WAVELENGTH_TOLERANCE * wavelength[0])
throw std::runtime_error(fmt::format(
"{} is at {:.5f} A and {} at {:.5f} A: sweeps at different wavelengths are not processed "
"together; give them in separate runs", plan.sweeps[0].label, wavelength[0],
plan.sweeps[k].label, wavelength[k]));
plan.wavelength_A = wavelength[0];
return plan;
}
std::vector<std::string> RugnuxSiblingSweeps(const RugnuxSweepPlan &plan, const std::vector<std::string> &extensions) {
namespace fs = std::filesystem;
std::set<std::string> given, dirs_frames, dirs_masters;
for (const auto &s: plan.sweeps) {
const fs::path p(s.input);
const std::string dir = p.parent_path().empty() ? "." : p.parent_path().string();
if (s.frames) {
given.insert(fs::path(s.label).filename().string());
dirs_frames.insert(dir);
} else {
given.insert(p.filename().string());
dirs_masters.insert(dir);
}
}
auto has_extension = [&](std::string name) {
name = Lower(name);
if (name.size() > 3 && name.ends_with(".gz"))
name.resize(name.size() - 3);
return std::any_of(extensions.begin(), extensions.end(), [&](const auto &e) { return name.ends_with(e); });
};
// template (or master) name -> files, in name order
std::map<std::string, size_t> found;
std::error_code ec;
for (const auto &dir: dirs_frames)
for (const auto &e: fs::directory_iterator(dir, ec)) {
const std::string name = e.path().filename().string();
if (!e.is_regular_file() || !has_extension(name))
continue;
const std::string t = fs::path(RugnuxFrameTemplate(e.path().string())).filename().string();
if (!t.empty() && !given.contains(t))
found[(fs::path(dir) / t).string()]++;
}
for (const auto &dir: dirs_masters)
for (const auto &e: fs::directory_iterator(dir, ec)) {
const std::string name = e.path().filename().string();
if (e.is_regular_file() && name.ends_with("_master.h5") && !given.contains(name))
found[(fs::path(dir) / name).string()] = 0;
}
std::vector<std::string> out;
for (const auto &[name, n]: found)
out.push_back(n > 0 ? fmt::format("{} ({} frame{})", name, n, n == 1 ? "" : "s") : name);
return out;
}
+56
View File
@@ -0,0 +1,56 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <functional>
#include <string>
#include <vector>
// The rotation sweeps a rugnux command line names, decided from the files in the order given:
// - frame files (CBF, marCCD, SMV, Bruker SFRM): one sweep per file-name template - the name with its last run of
// digits, the frame number, taken out. Any frame of a sweep names the whole sweep, so several frames
// of one template are one sweep;
// - masters: one sweep per rotation master. Several stills masters stay one stills run, as before, and
// a mix of rotation and stills masters is refused.
// Sweeps at different wavelengths are refused. Nothing is planned for a single input, or for inputs only
// the stills readers take (XFEL HDF5, CXI, XTC): those are read as they always were.
// What the planner needs to know about one input. `frame_template` is empty for anything but a frame
// file; `rotation` and `wavelength_A` are asked only of the input that stands for a sweep.
struct RugnuxInputDescription {
std::string frame_template;
bool foreign = false; // read only by the stills readers (XFEL HDF5, CXI, XTC)
bool rotation = false;
double wavelength_A = 0.0;
};
struct RugnuxSweep {
std::string input; // the file the sweep is read from: its first frame given, or its master
std::string label; // the frame template or the master, for messages
bool frames = false; // a sweep of frame files (label is its template)
};
struct RugnuxSweepPlan {
// Empty: the inputs are read together as before (one input, a stills run, or a mix the readers refuse).
std::vector<RugnuxSweep> sweeps;
double wavelength_A = 0.0; // of every sweep, where there are several
};
// `describe_file` says what an input is from its name and contents without reading the sweep (its
// frame template, or whether it is only for the stills readers); `describe_sweep` reads the input that
// stands for a sweep for its rotation and wavelength. Throws std::runtime_error, with a message for the
// user, on a mix of rotation sweeps and stills, or on sweeps at different wavelengths.
RugnuxSweepPlan PlanRugnuxSweeps(const std::vector<std::string> &inputs,
const std::function<RugnuxInputDescription(const std::string &)> &describe_file,
const std::function<RugnuxInputDescription(const std::string &)> &describe_sweep);
// The template of a frame file name, with its directory: the last run of digits in the name replaced by
// '#' characters, one per digit ("/d/x_1_0001.cbf" -> "/d/x_1_####.cbf"). Empty for a name with no digits.
std::string RugnuxFrameTemplate(const std::string &path);
// The sweeps beside the given ones that were not given: frame templates of `extensions` (lower case, with
// the dot; a trailing ".gz" is looked through) in the directories of the frame sweeps, and other
// "*_master.h5" files beside the given masters - one line each, for the log. Only the inputs given are
// processed; these are named so that a sweep left out by accident shows.
std::vector<std::string> RugnuxSiblingSweeps(const RugnuxSweepPlan &plan, const std::vector<std::string> &extensions);
+552 -130
View File
File diff suppressed because it is too large. Load diff
+501
View File
@@ -0,0 +1,501 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The two forms Bruker frames reach us in: a full imgCIF file (what APEX writes when it converts a
// frame, CBFlib's own form - geometry in CIF tables, no "# " lines) and the native .sfrm. Every
// fixture is synthetic and written byte for byte here.
#include <catch2/catch_all.hpp>
#include <cstring>
#include <fstream>
#include <iomanip>
#include <sstream>
#include "../common/JFJochMath.h"
#include "../reader/JFJochCBFReader.h"
#include "../reader/JFJochSFRMReader.h"
#include "../reader/MiniCBF.h"
#include "../reader/SFRM.h"
namespace {
// x-CBF_BYTE_OFFSET for small images: int8 deltas, escaping to int16.
std::string ByteOffset(const std::vector<int32_t> &pixels) {
std::string out;
int32_t prev = 0;
for (const int32_t v: pixels) {
const int32_t d = v - prev;
if (d > -128 && d < 128) {
out.push_back(static_cast<char>(static_cast<int8_t>(d)));
} else {
out.push_back(static_cast<char>(-128));
const auto d16 = static_cast<int16_t>(d);
out.append(reinterpret_cast<const char *>(&d16), 2);
}
prev = v;
}
return out;
}
struct ImgCIFFrame {
double omega = 30, kappa = 60, phi = 10;
double omega_inc = 0, kappa_inc = 0, phi_inc = 0.5;
double two_theta = 0;
double distance = 50;
std::string polarization; // extra _diffrn_radiation_wavelength lines, if any
};
// A full imgCIF frame of a kappa head scanning phi, with a detector on a 2theta arm. The image's
// fast index runs along laboratory +Y and its slow one along +X, both increasing, with the image
// centred on the arm's axis: pixel (cols-1)/2, (rows-1)/2 is where the unswung beam meets it.
void WriteImgCIF(const std::string &name, const ImgCIFFrame &f, int64_t nx, int64_t ny,
const std::vector<int32_t> &pixels) {
const double pixel = 0.12;
std::ostringstream h;
h << "###CBF: VERSION 1.7.11\n# CBF file written by CBFlib v0.9.6\n\ndata_image\n\n"
<< "_diffrn_scan_frame.integration_time 5\n"
<< "_diffrn_radiation_wavelength.id SOURCE\n_diffrn_radiation_wavelength.wavelength 0.71073\n"
<< f.polarization << "\n"
<< "loop_\n_axis.id\n_axis.depends_on\n_axis.equipment\n_axis.type\n"
<< "_axis.vector[1]\n_axis.vector[2]\n_axis.vector[3]\n_axis.offset[1]\n_axis.offset[2]\n_axis.offset[3]\n"
<< " OMEGA . goniometer rotation 1 0 0 0 0 0\n"
<< " KAPPA OMEGA goniometer rotation 0.642788 0 -0.766044 0 0 0\n"
<< " PHI KAPPA goniometer rotation 1 0 0 0 0 0\n"
<< " TWOTHETA . detector rotation 1 0 0 0 0 0\n"
<< " DX TWOTHETA detector translation 0 0 -1 0 0 0\n"
<< " ELEMENT_X DX detector translation 0 1 0 " << -ny * pixel / 2 << " " << -nx * pixel / 2 << " 0\n"
<< " ELEMENT_Y ELEMENT_X detector translation 1 0 0 0 0 0\n\n"
<< "loop_\n_diffrn_scan_frame_axis.axis_id\n_diffrn_scan_frame_axis.displacement\n"
<< "_diffrn_scan_frame_axis.angle\n"
<< " DX " << f.distance << " ?\n TWOTHETA ? " << f.two_theta << "\n OMEGA ? " << f.omega
<< "\n PHI ? " << f.phi << "\n KAPPA ? " << f.kappa << "\n\n"
<< "loop_\n_diffrn_scan_axis.axis_id\n_diffrn_scan_axis.displacement_start\n"
<< "_diffrn_scan_axis.displacement_increment\n_diffrn_scan_axis.displacement_range\n"
<< "_diffrn_scan_axis.angle_start\n_diffrn_scan_axis.angle_increment\n_diffrn_scan_axis.angle_range\n"
<< " DX " << f.distance << " 0 0 ? ? ?\n TWOTHETA ? ? ? " << f.two_theta << " 0 0\n"
<< " OMEGA ? ? ? " << f.omega << " " << f.omega_inc << " " << f.omega_inc << "\n"
<< " PHI ? ? ? " << f.phi << " " << f.phi_inc << " " << f.phi_inc << "\n"
<< " KAPPA ? ? ? " << f.kappa << " " << f.kappa_inc << " " << f.kappa_inc << "\n\n"
<< "_diffrn_measurement_axis.measurement_id GONIOMETER\n_diffrn_measurement_axis.axis_id PHI\n\n"
<< "_diffrn_detector.id DETECTOR\n_diffrn_detector.detector CCD\n_diffrn_detector.type APEX2\n\n"
<< "loop_\n_array_structure_list_axis.axis_set_id\n_array_structure_list_axis.axis_id\n"
<< "_array_structure_list_axis.displacement\n_array_structure_list_axis.displacement_increment\n"
<< " ELEMENT_X ELEMENT_X " << pixel / 2 << " " << pixel << "\n"
<< " ELEMENT_Y ELEMENT_Y " << pixel / 2 << " " << pixel << "\n\n"
<< "loop_\n_array_structure_list.array_id\n_array_structure_list.index\n_array_structure_list.dimension\n"
<< "_array_structure_list.precedence\n_array_structure_list.direction\n_array_structure_list.axis_set_id\n"
<< " IMAGE 1 " << nx << " 1 increasing ELEMENT_X\n IMAGE 2 " << ny << " 2 increasing ELEMENT_Y\n\n"
<< "_array_intensities.array_id IMAGE\n_array_intensities.gain 15.7\n_array_intensities.overload 600000\n\n"
<< "_array_data.array_id IMAGE\n_array_data.binary_id 1\n_array_data.data\n;\n"
<< "--CIF-BINARY-FORMAT-SECTION--\nContent-Type: application/octet-stream;\n"
<< " conversions=\"x-CBF_BYTE_OFFSET\"\nContent-Transfer-Encoding: BINARY\n"
<< "X-Binary-Element-Type: \"signed 32-bit integer\"\n"
<< "X-Binary-Number-of-Elements: " << nx * ny << "\n"
<< "X-Binary-Size-Fastest-Dimension: " << nx << "\n"
<< "X-Binary-Size-Second-Dimension: " << ny << "\n\n";
std::ofstream out(name, std::ios::binary);
const std::string text = h.str();
out.write(text.data(), static_cast<std::streamsize>(text.size()));
out.write(reinterpret_cast<const char *>(minicbf::BINARY_SEPARATOR), sizeof(minicbf::BINARY_SEPARATOR));
const std::string data = ByteOffset(pixels);
out.write(data.data(), static_cast<std::streamsize>(data.size()));
}
std::string FrameName(const std::string &prefix, int frame, const std::string &ext) {
std::ostringstream name;
name << prefix << "_" << std::setfill('0') << std::setw(4) << frame << ext;
return name.str();
}
Coord Internal(double x, double y, double z) {
return {static_cast<float>(x), static_cast<float>(-y), static_cast<float>(-z)};
}
} // namespace
TEST_CASE("JFJochCBFReader_FullImgCIF_KappaChain", "[HDF5][Full][portable]") {
const int64_t nx = 24, ny = 16;
std::vector<int32_t> pixels(nx * ny, 5);
pixels[3] = -32; // outside the active area
pixels[17] = 1000; // escapes to int16
SECTION("phi scan on a kappa head, square-on detector") {
for (int frame = 1; frame <= 3; frame++) {
ImgCIFFrame f;
f.phi = 10 + 0.5 * (frame - 1);
WriteImgCIF(FrameName("imgcif_kappa", frame, ".cbf"), f, nx, ny, pixels);
}
REQUIRE(JFJochCBFReader::CanRead("imgcif_kappa_0002.cbf"));
JFJochCBFReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("imgcif_kappa_0002.cbf"));
CHECK(reader.GetNumberOfImages() == 3);
CHECK(reader.CountsInADU());
const auto x = reader.GetDataset()->experiment;
// phi turns about its own axis as omega (30) and kappa (60) hold it; GoniometerAxis takes
// minus the right-handed direction
const Coord kappa_axis = Internal(0.642788, 0, -0.766044);
const Coord expected = -(RotMatrix(static_cast<float>(30 * PI / 180), Coord(1, 0, 0))
* (RotMatrix(static_cast<float>(60 * PI / 180), kappa_axis) * Coord(1, 0, 0)));
REQUIRE(x.GetGoniometer().has_value());
CHECK((x.GetGoniometer()->GetAxis() - expected).Length() < 1e-5f);
CHECK(x.GetGoniometer()->GetStart_deg() == Catch::Approx(10.0));
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
CHECK(x.GetGoniometer()->GetName() == "phi");
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(50.0));
CHECK(x.GetBeamX_pxl() == Catch::Approx((nx - 1) / 2.0));
CHECK(x.GetBeamY_pxl() == Catch::Approx((ny - 1) / 2.0));
CHECK(x.GetPixelSize_mm() == Catch::Approx(0.12));
CHECK(x.GetWavelength_A() == Catch::Approx(0.71073));
CHECK(x.GetSaturationLimit() == 600000);
CHECK_FALSE(x.GetPolarizationFactor().has_value()); // the header states none
// fast along laboratory +Y (internal -y), slow along +X
const auto geom = x.GetDiffractionGeometry();
CHECK((geom.GetFastAxis() - Coord(0, -1, 0)).Length() < 1e-5f);
CHECK((geom.GetSlowAxis() - Coord(1, 0, 0)).Length() < 1e-5f);
CHECK((geom.GetNormalAxis() - Coord(0, 0, 1)).Length() < 1e-5f);
// ReadRawImage hands the byte-offset stream on undecoded; LoadImage decodes it
const auto image = reader.LoadImage(1);
REQUIRE(image);
CHECK(image->Image()[0] == 5);
CHECK(image->Image()[17] == 1000);
CHECK(reader.GetDataset()->pixel_mask->GetMask()[3] != 0);
CHECK(reader.GetDataset()->pixel_mask->GetMask()[0] == 0);
reader.Close();
}
SECTION("a detector swung out on the 2theta arm") {
for (int frame = 1; frame <= 2; frame++) {
ImgCIFFrame f;
f.phi = 10 + 0.5 * (frame - 1);
f.two_theta = -20;
WriteImgCIF(FrameName("imgcif_swing", frame, ".cbf"), f, nx, ny, pixels);
}
JFJochCBFReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("imgcif_swing_0001.cbf"));
const auto x = reader.GetDataset()->experiment;
// The arm carries the square-on geometry with it: distance and foot of the normal stay.
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(50.0));
CHECK(x.GetBeamX_pxl() == Catch::Approx((nx - 1) / 2.0).margin(1e-3));
CHECK(x.GetBeamY_pxl() == Catch::Approx((ny - 1) / 2.0).margin(1e-3));
const Coord normal = RotMatrix(static_cast<float>(-20 * PI / 180), Coord(1, 0, 0)) * Coord(0, 0, 1);
CHECK((x.GetDiffractionGeometry().GetNormalAxis() - normal).Length() < 1e-5f);
reader.Close();
}
for (int frame = 1; frame <= 3; frame++) {
remove(FrameName("imgcif_kappa", frame, ".cbf").c_str());
remove(FrameName("imgcif_swing", frame, ".cbf").c_str());
}
}
// What the full imgCIF path must never do: read a series it cannot describe as one rotation about
// one axis as if it could.
TEST_CASE("JFJochCBFReader_FullImgCIF_Polarization", "[HDF5][Full][portable]") {
const int64_t nx = 24, ny = 16;
const std::vector<int32_t> pixels(nx * ny, 5);
auto read = [&](const std::string &prefix, const std::string &lines) {
for (int frame = 1; frame <= 2; frame++) {
ImgCIFFrame f;
f.phi = 10 + 0.5 * (frame - 1);
f.polarization = lines;
WriteImgCIF(FrameName(prefix, frame, ".cbf"), f, nx, ny, pixels);
}
JFJochCBFReader reader;
reader.ReadFiles(FrameName(prefix, 1, ".cbf"));
return reader.GetDataset()->experiment.GetPolarizationFactor();
};
// the degree as stated, in the horizontal plane when the plane's normal is the laboratory Y axis
const auto horizontal = read("imgcif_pol_h", "_diffrn_radiation_wavelength.polarizn_source_ratio 0.0444\n"
"_diffrn_radiation_wavelength.polarizn_source_norm 0\n");
REQUIRE(horizontal.has_value());
CHECK(*horizontal == Catch::Approx(0.0444));
// a normal turned by 90 degrees puts the plane vertical, which the correction takes as the opposite sign
const auto vertical = read("imgcif_pol_v", "_diffrn_radiation_wavelength.polarizn_source_ratio 0.5\n"
"_diffrn_radiation_wavelength.polarizn_source_norm 90\n");
REQUIRE(vertical.has_value());
CHECK(*vertical == Catch::Approx(-0.5));
}
TEST_CASE("JFJochCBFReader_FullImgCIF_refuses_what_it_cannot_describe", "[HDF5][Full][portable]") {
const int64_t nx = 8, ny = 8;
const std::vector<int32_t> pixels(nx * ny, 1);
SECTION("two axes moving at once") {
for (int frame = 1; frame <= 2; frame++) {
ImgCIFFrame f;
f.omega_inc = 0.5;
WriteImgCIF(FrameName("imgcif_two", frame, ".cbf"), f, nx, ny, pixels);
}
CHECK(JFJochCBFReader::CanRead("imgcif_two_0001.cbf"));
JFJochCBFReader reader;
CHECK_THROWS_AS(reader.ReadFiles("imgcif_two_0001.cbf"), JFJochException);
}
SECTION("kappa moved between frames of one name template") {
for (int frame = 1; frame <= 2; frame++) {
ImgCIFFrame f;
f.phi = 10 + 0.5 * (frame - 1);
f.kappa = (frame == 1) ? 60 : 0;
WriteImgCIF(FrameName("imgcif_reset", frame, ".cbf"), f, nx, ny, pixels);
}
JFJochCBFReader reader;
CHECK_THROWS_AS(reader.ReadFiles("imgcif_reset_0001.cbf"), JFJochException);
}
SECTION("nothing turns") {
for (int frame = 1; frame <= 2; frame++) {
ImgCIFFrame f;
f.phi_inc = 0;
WriteImgCIF(FrameName("imgcif_still", frame, ".cbf"), f, nx, ny, pixels);
}
JFJochCBFReader reader;
CHECK_THROWS_AS(reader.ReadFiles("imgcif_still_0001.cbf"), JFJochException);
}
for (int frame = 1; frame <= 2; frame++)
for (const char *p: {"imgcif_two", "imgcif_reset", "imgcif_still"})
remove(FrameName(p, frame, ".cbf").c_str());
}
TEST_CASE("ImgCIF_Item_and_Loop", "[HDF5][Full][portable]") {
const std::string text = "\n_a.x 1\nloop_\n_b.id\n_b.v\n p 1\n q 2\n\n_c.id 'quoted value'\n";
CHECK(imgcif::Item(text, "_a.x").value_or("") == "1");
CHECK(imgcif::Item(text, "_c.id").value_or("") == "quoted value");
CHECK_FALSE(imgcif::Item(text, "_b.id").has_value()); // heads a loop
const auto rows = imgcif::ParseLoop(text, "_b.id");
REQUIRE(rows.size() == 2);
CHECK(rows[1].at("_b.v") == "2");
// a tag outside any loop is not the rows of the loop before it
CHECK(imgcif::ParseLoop(text, "_c.id").empty());
}
namespace {
// One FORMAT 100 frame: 1-byte pixels, the 16- and 32-bit overflow tables, baseline 32.
struct SFRMFrame {
int64_t axis = 2;
double start = 0, increment = 0.5;
std::array<double, 4> angles{0, 0, 0, 0}; // 2theta omega phi chi
double distance_cm = 5.0;
double monochromator_2theta = 0; // FILTER2
std::vector<uint8_t> pixels;
std::vector<uint16_t> over16;
std::vector<int32_t> over32;
};
std::string Line(const std::string &key, const std::string &value) {
std::string k = key;
k.resize(7, ' ');
std::string v = value;
v.resize(72, ' ');
return k + ":" + v;
}
void Pad(std::string &s, size_t to) {
s.resize((s.size() + to - 1) / to * to, '\0');
}
void WriteSFRM(const std::string &name, const SFRMFrame &f, int64_t n) {
std::ostringstream angles;
for (double a: f.angles)
angles << a << " ";
std::string h;
h += Line("FORMAT", "100");
h += Line("VERSION", "11");
h += Line("HDRBLKS", "15");
h += Line("NOVERFL", "-1 " + std::to_string(f.over16.size()) + " " + std::to_string(f.over32.size()));
h += Line("NPIXELB", "1 1");
h += Line("NROWS", std::to_string(n));
h += Line("NCOLS", std::to_string(n));
h += Line("WORDORD", "0");
h += Line("LONGORD", "0");
h += Line("START", std::to_string(f.start));
h += Line("INCREME", std::to_string(f.increment));
h += Line("ANGLES", angles.str());
h += Line("AXIS", std::to_string(f.axis));
h += Line("DISTANC", std::to_string(f.distance_cm));
h += Line("CENTER", std::to_string(n / 2.0) + " " + std::to_string(n / 2.0));
h += Line("WAVELEN", "1.54184 1.54056 1.54439");
h += Line("CUMULAT", "10.0");
h += Line("DETTYPE", "CCD-TEST 64.0 0.1 0 0.254 0.05 1");
h += Line("NEXP", "2 0 32 0 1");
h += Line("CCDPARM", "14 10 170 0 900000");
h += Line("DETPAR", "0 0 0 0 0 0");
h += Line("FILTER2", std::to_string(f.monochromator_2theta) + " 0");
h.resize(15 * 512, ' ');
std::string body(f.pixels.begin(), f.pixels.end());
Pad(body, 16);
std::string t16(reinterpret_cast<const char *>(f.over16.data()), f.over16.size() * 2);
Pad(t16, 16);
std::string t32(reinterpret_cast<const char *>(f.over32.data()), f.over32.size() * 4);
Pad(t32, 16);
std::ofstream out(name, std::ios::binary);
for (const auto *s: {&h, &body, &t16, &t32})
out.write(s->data(), static_cast<std::streamsize>(s->size()));
}
} // namespace
TEST_CASE("SFRM_Decode_overflow_tables", "[HDF5][Full][portable]") {
const int64_t n = 8;
SFRMFrame f;
f.pixels.assign(n * n, 40); // 8 counts above the baseline of 32
f.pixels[0] = 0; // outside the active area: below the baseline
f.pixels[5] = 255; // -> 16-bit table: 1000
f.pixels[9] = 255; // -> 16-bit 65535 -> 32-bit 100000
f.pixels[20] = 255; // -> 16-bit 65535 -> 32-bit -14 (signed)
f.pixels[30] = 255; // -> 16-bit 300
f.over16 = {1000, 65535, 65535, 300};
f.over32 = {100000, -14};
WriteSFRM("sfrm_tables_0001.sfrm", f, n);
REQUIRE(sfrm::CanRead("sfrm_tables_0001.sfrm"));
std::vector<int32_t> v;
const auto h = sfrm::Read("sfrm_tables_0001.sfrm", v);
CHECK(h.format == 100);
CHECK(h.baseline == 32);
REQUIRE(v.size() == static_cast<size_t>(n * n));
CHECK(v[1] == 8);
CHECK(v[0] == -32);
CHECK(v[5] == 1000 - 32);
CHECK(v[9] == 100000 - 32);
CHECK(v[20] == -14 - 32);
CHECK(v[30] == 300 - 32);
remove("sfrm_tables_0001.sfrm");
SECTION("a table shorter than the pixels that point into it is an error") {
SFRMFrame g = f;
g.over16 = {1000};
g.over32 = {};
WriteSFRM("sfrm_short_0001.sfrm", g, n);
CHECK_THROWS_AS(sfrm::Read("sfrm_short_0001.sfrm", v), JFJochException);
remove("sfrm_short_0001.sfrm");
}
}
TEST_CASE("SFRM_Decode_format86_ascii_overflows", "[HDF5][Full][portable]") {
const int64_t n = 4;
std::string h;
h += Line("FORMAT", "86");
h += Line("VERSION", "9");
h += Line("HDRBLKS", "5");
h += Line("NOVERFL", "2");
h += Line("NPIXELB", "1");
h += Line("NROWS", std::to_string(n));
h += Line("NCOLS", std::to_string(n));
h.resize(5 * 512, ' ');
std::string body(n * n, static_cast<char>(7));
body[2] = static_cast<char>(255);
body[11] = static_cast<char>(255);
std::string table = " 70000 2 123456 11";
Pad(table, 512);
std::ofstream("sfrm_old_0001.sfrm", std::ios::binary) << h << body << table;
std::vector<int32_t> v;
const auto hdr = sfrm::Read("sfrm_old_0001.sfrm", v);
CHECK(hdr.format == 86);
CHECK(v[0] == 7);
CHECK(v[2] == 70000);
CHECK(v[11] == 123456);
remove("sfrm_old_0001.sfrm");
}
TEST_CASE("JFJochSFRMReader_Geometry", "[HDF5][Full][portable]") {
const int64_t n = 8;
SFRMFrame f;
f.pixels.assign(n * n, 40);
f.pixels[0] = 0;
SECTION("omega scan") {
for (int frame = 1; frame <= 3; frame++) {
f.start = 0.5 * (frame - 1);
f.angles = {0, f.start, 0, 54.74};
WriteSFRM(FrameName("sfrm_omega", frame, ".sfrm"), f, n);
}
REQUIRE(JFJochSFRMReader::CanRead("sfrm_omega_0001.sfrm"));
JFJochSFRMReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_omega_0001.sfrm"));
CHECK(reader.GetNumberOfImages() == 3);
const auto x = reader.GetDataset()->experiment;
REQUIRE(x.GetGoniometer().has_value());
// omega is the outermost axis, so a fixed chi does not tilt it
CHECK((x.GetGoniometer()->GetAxis() - Coord(1, 0, 0)).Length() < 1e-5f);
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
// DETTYPE: 64 pixels/cm on a 512 frame, so 1/64 cm * 512/8 per pixel; distance with the
// 0.1 cm face-to-grid offset
CHECK(x.GetPixelSize_mm() == Catch::Approx(10.0 / 64.0 * 512.0 / n));
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(51.0));
CHECK(x.GetBeamX_pxl() == Catch::Approx((n - 1) / 2.0).margin(1e-3));
CHECK(x.GetBeamY_pxl() == Catch::Approx((n - 1) / 2.0).margin(1e-3));
CHECK(x.GetSaturationLimit() == 900000);
CHECK(x.GetWavelength_A() == Catch::Approx(1.54184));
// laboratory frames with no monochromator: unpolarized, not the synchrotron default
REQUIRE(x.GetPolarizationFactor().has_value());
CHECK(*x.GetPolarizationFactor() == Catch::Approx(0.0).margin(1e-6));
CHECK(reader.GetDataset()->pixel_mask->GetMask()[0] != 0);
CHECK(reader.GetDataset()->pixel_mask->GetMask()[1] == 0);
JFJochReaderRawImage raw;
REQUIRE(reader.ReadRawImage(2, raw));
CHECK(reinterpret_cast<const int32_t *>(raw.image.GetCompressed())[1] == 8);
reader.Close();
}
SECTION("a sweep that turns backwards is stated forwards about the opposite axis") {
for (int frame = 1; frame <= 3; frame++) {
f.start = 10.0 - 0.5 * (frame - 1);
f.increment = -0.5;
f.angles = {0, f.start, 0, 54.74};
f.monochromator_2theta = 12.17;
WriteSFRM(FrameName("sfrm_back", frame, ".sfrm"), f, n);
}
JFJochSFRMReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_back_0001.sfrm"));
CHECK(reader.GetNumberOfImages() == 3);
const auto x = reader.GetDataset()->experiment;
REQUIRE(x.GetGoniometer().has_value());
CHECK((x.GetGoniometer()->GetAxis() - Coord(-1, 0, 0)).Length() < 1e-5f);
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
CHECK(x.GetGoniometer()->GetStart_deg() == Catch::Approx(-10.0));
// a graphite monochromator: the 1 - cos^2(2theta_m) APEX's conversion states, horizontal
REQUIRE(x.GetPolarizationFactor().has_value());
const double c = std::cos(12.17 * PI / 180.0);
CHECK(*x.GetPolarizationFactor() == Catch::Approx(1.0 - c * c));
reader.Close();
}
SECTION("phi scan on a fixed-chi head turns about the inclined phi axis") {
for (int frame = 1; frame <= 2; frame++) {
f.axis = 3;
f.start = 0.5 * (frame - 1);
f.angles = {0, 0, f.start, 54.74};
WriteSFRM(FrameName("sfrm_phi", frame, ".sfrm"), f, n);
}
JFJochSFRMReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_phi_0001.sfrm"));
const auto x = reader.GetDataset()->experiment;
// chi turns about the beam: imgCIF +Z, internal -z
const Coord expected = -(RotMatrix(static_cast<float>(54.74 * PI / 180), Coord(0, 0, -1)) * Coord(1, 0, 0));
CHECK((x.GetGoniometer()->GetAxis() - expected).Length() < 1e-5f);
reader.Close();
}
SECTION("chi moved between frames of one run is refused") {
for (int frame = 1; frame <= 2; frame++) {
f.axis = 3;
f.start = 0.5 * (frame - 1);
f.angles = {0, 0, f.start, frame == 1 ? 54.74 : 0.0};
WriteSFRM(FrameName("sfrm_reset", frame, ".sfrm"), f, n);
}
JFJochSFRMReader reader;
CHECK_THROWS_AS(reader.ReadFiles("sfrm_reset_0001.sfrm"), JFJochException);
}
for (int frame = 1; frame <= 3; frame++)
for (const char *p: {"sfrm_omega", "sfrm_phi", "sfrm_reset"})
remove(FrameName(p, frame, ".sfrm").c_str());
}
+5
View File
@@ -19,6 +19,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
MultiReaderTest.cpp
ModuleRefinementTest.cpp
SweepLayoutTest.cpp
BrukerReaderTest.cpp
CBORTest.cpp
ImageSpotFinderCPUTest.cpp
AdaptiveSpotFinderCPUTest.cpp
@@ -128,6 +129,7 @@ ADD_EXECUTABLE(jfjoch_test
RugnuxLiveStatusTest.cpp
RugnuxStructureTest.cpp
SweepLayoutTest.cpp
BrukerReaderTest.cpp
ShadowFinderTest.cpp
HotPixelFinderTest.cpp
BandwidthEstimateTest.cpp
@@ -139,6 +141,7 @@ ADD_EXECUTABLE(jfjoch_test
PostRefineTest.cpp
CellEsdTest.cpp
RotationScaleWalkTest.cpp
RotationScaleMergeSegmentsTest.cpp
AndersonMixingTest.cpp
RugnuxLargeTest.cpp
TestData.h
@@ -212,6 +215,8 @@ ADD_EXECUTABLE(jfjoch_test
WriteReflectionsTest.cpp
RugnuxResultFilesTest.cpp
RugnuxInputKindTest.cpp
RugnuxSweepsTest.cpp
RugnuxJointSweepsTest.cpp
UnitCellTest.cpp
CCTest.cpp
MultiLatticeSearchTest.cpp
+52
View File
@@ -4181,6 +4181,26 @@ TEST_CASE("JFJochCBFReader_Polarization", "[HDF5][Full][portable]") {
RemoveMiniCBFSweep("cbfnopol");
}
// A PILATUS header that also carries the imgCIF axis and scan tables (Diamond writes them) is still read
// as the PILATUS header it is: the "# Pixel_size" line decides, and the detector keeps its name.
TEST_CASE("JFJochCBFReader_PilatusHeaderWithScanTable", "[HDF5][Full][portable]") {
const int64_t nx = 24, ny = 16;
WriteMiniCBFSweep("cbfscan", "# Detector_2theta 0.0000 deg.\n# Oscillation_axis OMEGA\n"
+ AxisTable("GON_OMEGA rotation goniometer . 1 0 0 . . .\n"
"DET_Z translation detector . 0 0 -1 0 0 0\n"
"ELEMENT_X translation detector DET_Z 1 0 0 -1 1 0\n"
"ELEMENT_Y translation detector ELEMENT_X 0 -1 0 0 0 0\n", nx, ny)
+ "loop_\n_diffrn_scan_axis.axis_id\n_diffrn_scan_axis.angle_start\n"
"_diffrn_scan_axis.angle_range\n_diffrn_scan_axis.angle_increment\n"
"GON_OMEGA 0.0 0.1 0.1\n",
nx, ny);
JFJochCBFReader reader;
REQUIRE_NOTHROW(reader.ReadFiles("cbfscan_0001.cbf"));
CHECK(reader.GetDataset()->experiment.GetDetectorDescription().find("PILATUS3 6M") != std::string::npos);
reader.Close();
RemoveMiniCBFSweep("cbfscan");
}
TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full][portable]") {
const int64_t nx = 24, ny = 16;
@@ -4351,6 +4371,38 @@ TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Fu
}
}
// A phi scan on a kappa head turns about an axis that depends on where the kappa arm stands, and a
// miniCBF header states the kappa angle but not the arm. Reading it about the uninclined phi axis is
// a silent misread, so it is refused; an omega scan is not affected by kappa at all.
TEST_CASE("JFJochCBFReader_phi_scan_at_kappa_is_refused", "[HDF5][Full][portable]") {
const int64_t nx = 24, ny = 16;
const auto write = [&](const std::string &prefix, const std::string &scan_lines) {
for (int frame = 1; frame <= 2; frame++) {
std::ostringstream name;
name << prefix << "_" << std::setfill('0') << std::setw(4) << frame << ".cbf";
WriteRawMiniCBF(name.str(),
"# Pixel_size 172e-6 m x 172e-6 m\n# Wavelength 0.96864 A\n"
"# Detector_distance 0.33161 m\n# Count_cutoff 1000 counts\n"
"# Beam_xy (12.00, 8.00) pixels\n# Start_angle " + std::to_string(0.1 * frame)
+ " deg.\n# Angle_increment 0.1000 deg.\n# Kappa 30.0000 deg.\n" + scan_lines,
nx, ny);
}
};
write("cbfkappa_phi", "# Phi_increment 0.1000 deg.\n# Omega_increment 0.0000 deg.\n");
write("cbfkappa_omega", "# Phi_increment 0.0000 deg.\n# Omega_increment 0.1000 deg.\n");
{
JFJochCBFReader reader;
CHECK_THROWS_AS(reader.ReadFiles("cbfkappa_phi_0001.cbf"), JFJochException);
}
{
JFJochCBFReader reader;
CHECK_NOTHROW(reader.ReadFiles("cbfkappa_omega_0001.cbf"));
}
for (const char *f: {"cbfkappa_phi_0001.cbf", "cbfkappa_phi_0002.cbf",
"cbfkappa_omega_0001.cbf", "cbfkappa_omega_0002.cbf"})
remove(f);
}
TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full][portable]") {
// The header is read with a small first probe, and again with the large one when it does not end
// inside it; what stands past the first 16 kB must come out exactly as before.
+231
View File
@@ -0,0 +1,231 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <algorithm>
#include <cmath>
#include <cstring>
#include <map>
#include <random>
#include <vector>
#include "../common/CUDAWrapper.h"
#include "../common/Logger.h"
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
namespace {
// A synthetic crystal measured as pre-combined fulls on several sweeps: a cubic 20 A cell, Wilson
// intensities at B = 15 A^2 to 1.6 A, each sweep at its own scale and relative B, Poisson-like noise.
struct SweepSpec {
int n_frames;
double scale;
double relative_b; // at the sweep's first frame
double relative_b_end = NAN; // at its last; NaN: the same B throughout
};
struct Synthetic {
std::vector<IntegrationOutcome> frames;
std::map<std::array<int, 3>, double> I_true; // by the Friedel-merged index
};
constexpr double CELL = 20.0;
std::array<int, 3> FriedelKey(int h, int k, int l) {
const std::array<int, 3> a{h, k, l}, b{-h, -k, -l};
return std::max(a, b);
}
Synthetic MakeSweeps(const std::vector<SweepSpec> &sweeps, uint32_t seed) {
std::mt19937 rng(seed);
std::vector<std::array<int, 3>> hkl;
for (int h = -12; h <= 12; ++h)
for (int k = -12; k <= 12; ++k)
for (int l = -12; l <= 12; ++l) {
const double d = CELL / std::sqrt(static_cast<double>(h * h + k * k + l * l));
if ((h || k || l) && d >= 1.6) hkl.push_back({h, k, l});
}
Synthetic out;
std::exponential_distribution<double> wilson(1.0);
for (const auto &m: hkl) {
const auto key = FriedelKey(m[0], m[1], m[2]);
if (!out.I_true.contains(key)) {
const double s2 = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2]) / (4.0 * CELL * CELL);
out.I_true[key] = 2000.0 * std::exp(-2.0 * 15.0 * s2) * (0.05 + wilson(rng));
}
}
std::uniform_int_distribution<size_t> pick(0, hkl.size() - 1);
std::normal_distribution<double> noise(0.0, 1.0);
int frame = 0;
for (const auto &sweep: sweeps)
for (int f = 0; f < sweep.n_frames; ++f, ++frame) {
IntegrationOutcome o;
o.latt = CrystalLattice(UnitCell{static_cast<float>(CELL), static_cast<float>(CELL),
static_cast<float>(CELL), 90, 90, 90});
for (int j = 0; j < 400; ++j) {
const auto &m = hkl[pick(rng)];
const double s2 = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2]) / (4.0 * CELL * CELL);
const double b = std::isfinite(sweep.relative_b_end)
? sweep.relative_b + (sweep.relative_b_end - sweep.relative_b) * f / (sweep.n_frames - 1)
: sweep.relative_b;
const double expected = out.I_true.at(FriedelKey(m[0], m[1], m[2])) * sweep.scale
* std::exp(-2.0 * b * s2);
const double sigma = std::sqrt(expected + 20.0);
Reflection r{};
r.h = m[0]; r.k = m[1]; r.l = m[2];
r.image_number = static_cast<float>(frame);
r.d = static_cast<float>(1.0 / std::sqrt(4.0 * s2));
r.I = static_cast<float>(expected + sigma * noise(rng));
r.sigma = static_cast<float>(sigma);
// The combine rebuilds a full's variance as var_bkg + I: what is not the intensity's own
r.var_bkg = static_cast<float>(sigma * sigma - std::max(0.0, static_cast<double>(r.I)));
r.bkg = 20.0f;
r.prescaling_corr = 1.0f;
r.partiality = 1.0f;
r.zeta = 1.0f;
r.image_scale_corr = 1.0f;
r.predicted_x = 500.0f;
r.predicted_y = 500.0f;
o.reflections.push_back(r);
}
out.frames.push_back(std::move(o));
}
return out;
}
DiffractionExperiment SyntheticExperiment(bool correction_surfaces) {
DiffractionExperiment x;
x.IncidentEnergy_keV(12.4f);
x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, Coord(1, 0, 0), {}));
x.SetUnitCell(UnitCell{static_cast<float>(CELL), static_cast<float>(CELL), static_cast<float>(CELL), 90, 90, 90});
ScalingSettings s;
s.ScaleFulls(true).CorrectionSurfaces(correction_surfaces).ResolutionCutoff(ResolutionCutoffMethod::Off);
x.ImportScalingSettings(s);
return x;
}
std::vector<SweepSegment> Segments(const std::vector<SweepSpec> &sweeps) {
std::vector<SweepSegment> out;
int32_t first = 0;
for (const auto &s: sweeps) {
out.push_back({first, s.n_frames});
first += s.n_frames;
}
return out;
}
RotationScaleMerge::Result Merge(Synthetic &data, const DiffractionExperiment &x,
const std::vector<SweepSegment> &segments, bool export_fulls = false,
bool *on_gpu = nullptr) {
Logger logger("RotationScaleMergeSegmentsTest");
const UnitCell cell{static_cast<float>(CELL), static_cast<float>(CELL), static_cast<float>(CELL), 90, 90, 90};
RotationScaleMerge engine(x, data.frames, cell, 20, 4, logger);
if (!segments.empty())
engine.SetSweepSegments(segments);
engine.SetExportScaledFulls(export_fulls);
engine.Ingest();
if (on_gpu)
*on_gpu = engine.OnGPU();
return engine.Run(false, true, false);
}
// The median over a frame range of the scaled fulls' ratio to the true intensity (the strong ones).
double MedianRatio(const RotationScaleMerge::Result &r, const Synthetic &data, int f0, int f1) {
std::vector<double> ratio;
for (const auto &o: r.scaled_fulls)
if (o.image_number >= static_cast<float>(f0) && o.image_number < static_cast<float>(f1)) {
const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l));
if (t > 500.0)
ratio.push_back(o.I / t);
}
REQUIRE(ratio.size() > 20);
std::nth_element(ratio.begin(), ratio.begin() + ratio.size() / 2, ratio.end());
return ratio[ratio.size() / 2];
}
} // namespace
TEST_CASE("RotationScaleMergeSegments_PerSweepScaleRecovered", "[RotationScale]") {
const std::vector<SweepSpec> sweeps{{60, 1.0, 0.0}, {60, 0.5, 0.0}};
auto data = MakeSweeps(sweeps, 11);
const auto r = Merge(data, SyntheticExperiment(false), Segments(sweeps), true);
REQUIRE(r.segments.size() == 2);
CHECK(r.segments[1].scale / r.segments[0].scale == Catch::Approx(0.5).epsilon(0.03));
CHECK(r.statistics.overall.cc_half > 0.99);
// No coupling across the boundary: the frames on either side of it are put on the common scale as
// well as the middle of their sweeps are.
const double middle = MedianRatio(r, data, 20, 40);
CHECK(MedianRatio(r, data, 57, 60) / middle == Catch::Approx(1.0).epsilon(0.03));
CHECK(MedianRatio(r, data, 60, 63) / middle == Catch::Approx(1.0).epsilon(0.03));
CHECK(MedianRatio(r, data, 80, 100) / middle == Catch::Approx(1.0).epsilon(0.03));
}
TEST_CASE("RotationScaleMergeSegments_DoseOrderFromTheRelativeB", "[RotationScale]") {
// Given first, the sweep taken second: its B is 8 A^2 higher. The order comes from the data.
const std::vector<SweepSpec> sweeps{{60, 1.0, 8.0}, {60, 1.0, 0.0}, {60, 1.0, 4.0}};
auto data = MakeSweeps(sweeps, 12);
const auto r = Merge(data, SyntheticExperiment(true), Segments(sweeps));
REQUIRE(r.segments.size() == 3);
CHECK(r.segments[1].dose_rank == 0);
CHECK(r.segments[2].dose_rank == 1);
CHECK(r.segments[0].dose_rank == 2);
CHECK(r.segments[1].relative_b == Catch::Approx(0.0).margin(1e-9));
CHECK(r.segments[0].relative_b == Catch::Approx(8.0).margin(1.0));
CHECK(r.segments[2].relative_b == Catch::Approx(4.0).margin(1.0));
// Flat within each sweep: one B per sweep predicts the held-out frames better than a dose-linear ramp
CHECK(r.segment_decay_model == "per-sweep B");
}
TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationScale]") {
// One crystal damaged steadily: the sweep given first was taken second, its B going on from where the
// other's ended. Along the dose order the B rises in a straight line, which the ramp describes.
const std::vector<SweepSpec> sweeps{{60, 1.0, 6.0, 12.0}, {60, 1.0, 0.0, 6.0}};
auto data = MakeSweeps(sweeps, 15);
const auto r = Merge(data, SyntheticExperiment(true), Segments(sweeps));
REQUIRE(r.segments.size() == 2);
CHECK(r.segments[1].dose_rank == 0);
CHECK(r.segments[0].dose_rank == 1);
CHECK(r.segments[0].relative_b == Catch::Approx(6.0).margin(1.5));
CHECK(r.segment_decay_model == "dose-linear");
}
TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") {
const std::vector<SweepSpec> sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}};
auto data = MakeSweeps(sweeps, 13);
const auto x = SyntheticExperiment(false);
CHECK_THROWS_AS(Merge(data, x, {{0, 10}, {11, 9}}), JFJochException);
CHECK_THROWS_AS(Merge(data, x, {{0, 10}}), JFJochException);
}
TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[RotationScale]") {
// The same synthetic sweep merged without segments gives, to the bit, what the engine gave before
// segments existed: FNV-1a over the merged I and sigma, recorded on the engine of rc181 e56b19abb -
// on the CPU, and on the GPU for the card model it was recorded on (the device path matches the host
// path to the bit only where every reduction does; this one carries its own, so it is checked against
// its own record).
const std::vector<SweepSpec> sweeps{{40, 1.0, 0.0}, {40, 0.7, 3.0}};
auto data = MakeSweeps(sweeps, 14);
bool on_gpu = false;
const auto r = Merge(data, SyntheticExperiment(true), {}, false, &on_gpu);
uint64_t hash = 1469598103934665603ULL;
auto mix = [&hash](float v) {
uint32_t u;
std::memcpy(&u, &v, sizeof u);
for (int b = 0; b < 4; ++b) {
hash ^= (u >> (8 * b)) & 0xffu;
hash *= 1099511628211ULL;
}
};
for (const auto &m: r.merged) {
mix(m.I);
mix(m.sigma);
}
CHECK(r.segments.empty());
CHECK(r.merged.size() == 4106);
if (!on_gpu)
CHECK(hash == 3376995616949734726ULL);
else if (get_gpu_description().find("RTX 5080") != std::string::npos)
CHECK(hash == 6594155396455850264ULL);
}
+108
View File
@@ -0,0 +1,108 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <cmath>
#include <map>
#include <random>
#include "../rugnux/RugnuxJointSweeps.h"
namespace {
// A tetragonal P4 crystal: Laue class 4/m on a 4/mmm lattice, so (k, h, -l) indexes it a second way.
const gemmi::SpaceGroup &P4() { return *gemmi::find_spacegroup_by_name("P 4"); }
constexpr float A = 60.0f, C = 40.0f;
std::array<int, 3> P4Key(int h, int k, int l) {
// the 4/m ASU representative, enough to give equivalents one intensity
std::array<int, 3> best{h, k, l};
const std::array<std::array<int, 3>, 8> eq{{{h, k, l}, {-k, h, l}, {-h, -k, l}, {k, -h, l},
{-h, -k, -l}, {k, -h, -l}, {h, k, -l}, {-k, h, -l}}};
for (const auto &e: eq) best = std::max(best, e);
return best;
}
JointMemberSweep MakeSweep(const std::map<std::array<int, 3>, double> &truth, const gemmi::Op &indexed_as,
int n_frames, uint32_t seed) {
std::mt19937 rng(seed);
JointMemberSweep s;
s.n_frames = n_frames;
s.cell = UnitCell{A, A, C, 90, 90, 90};
s.space_group = P4();
for (const auto &[hkl, I]: truth) {
const auto m = indexed_as.apply_to_hkl({{hkl[0], hkl[1], hkl[2]}});
MergedReflection r;
r.h = m[0]; r.k = m[1]; r.l = m[2];
r.I = static_cast<float>(I);
r.sigma = 1.0f;
r.d = static_cast<float>(1.0 / std::sqrt((hkl[0] * hkl[0] + hkl[1] * hkl[1]) / (A * A) + hkl[2] * hkl[2] / (C * C)));
s.merged.push_back(r);
ScaledFull f;
f.h = m[0]; f.k = m[1]; f.l = m[2];
f.I = static_cast<float>(I);
f.sigma = 1.0f;
f.image_number = static_cast<float>(rng() % n_frames);
f.var_bkg = 4.0f;
f.var_per_I = 0.5f;
f.capture = 0.0f;
f.d = r.d;
s.fulls.push_back(f);
}
return s;
}
std::map<std::array<int, 3>, double> Truth(uint32_t seed) {
std::mt19937 rng(seed);
std::exponential_distribution<double> wilson(1.0);
std::map<std::array<int, 3>, double> t;
for (int h = -12; h <= 12; ++h)
for (int k = -12; k <= 12; ++k)
for (int l = -8; l <= 8; ++l) {
if (!(h || k || l)) continue;
const auto key = P4Key(h, k, l);
if (!t.contains(key)) t[key] = 1000.0 * wilson(rng);
}
return t;
}
} // namespace
TEST_CASE("RelateJointSweeps_AmbiguityRecoveredAndAStrangerLeftOut", "[Rugnux]") {
const auto truth = Truth(1);
const gemmi::Op twin = gemmi::parse_triplet("y,x,-z"); // h,k,l -> k,h,-l
std::vector<JointMemberSweep> sweeps;
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 2));
sweeps.push_back(MakeSweep(truth, twin, 80, 3));
sweeps.push_back(MakeSweep(Truth(99), gemmi::Op::identity(), 50, 4)); // another crystal
sweeps[0].fulls.push_back(sweeps[0].fulls.front()); // the reference: most fulls
const auto relation = RelateJointSweeps(sweeps);
REQUIRE(relation.reference == 0);
CHECK(relation.sweeps[0].included);
REQUIRE(relation.sweeps[1].included);
CHECK(relation.sweeps[1].cc > 0.99);
CHECK(relation.sweeps[1].identity_cc < 0.5);
// the operator found takes sweep 2's indices back onto the reference's (up to the Laue group)
const auto back = relation.sweeps[1].cob.apply_to_hkl(twin.apply_to_hkl({{3, 1, 2}}));
CHECK(P4Key(back[0], back[1], back[2]) == P4Key(3, 1, 2));
CHECK_FALSE(relation.sweeps[2].included);
const auto input = BuildJointMergeInput(sweeps, relation);
REQUIRE(input.segments.size() == 2);
CHECK(input.segments[0].first_frame == 0);
CHECK(input.segments[0].n_frames == 100);
CHECK(input.segments[1].first_frame == 100);
CHECK(input.segments[1].n_frames == 80);
CHECK(input.segment_sweep == std::vector<int>{0, 1});
REQUIRE(input.outcomes.size() == 180);
// a full as the combine takes it: count I/c at correction c = var_per_I, variance var_bkg / c^2
const Reflection *r = nullptr;
for (const auto &o: input.outcomes)
if (!o.reflections.empty()) { r = &o.reflections.front(); break; }
REQUIRE(r != nullptr);
CHECK(r->image_scale_corr == Catch::Approx(0.5));
CHECK(r->var_bkg == Catch::Approx(16.0));
CHECK(r->partiality == 1.0f);
}
+132
View File
@@ -0,0 +1,132 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <filesystem>
#include <fstream>
#include <map>
#include <stdexcept>
#include "../rugnux/RugnuxSweeps.h"
namespace {
// Inputs described by name alone: "<template>_<n>.cbf" frames, "<name>_rot.h5" / "<name>_still.h5"
// masters, "<name>.cxi" for the stills readers; the wavelength from a table.
struct FakeInputs {
std::map<std::string, double> wavelength;
int sweeps_described = 0;
RugnuxInputDescription File(const std::string &f) const {
RugnuxInputDescription d;
if (f.ends_with(".cbf"))
d.frame_template = RugnuxFrameTemplate(f);
d.foreign = f.ends_with(".cxi");
return d;
}
RugnuxInputDescription Sweep(const std::string &f) {
sweeps_described++;
RugnuxInputDescription d = File(f);
d.rotation = f.ends_with(".cbf") || f.ends_with("_rot.h5");
d.wavelength_A = wavelength.contains(f) ? wavelength.at(f) : 1.0;
return d;
}
RugnuxSweepPlan Plan(const std::vector<std::string> &inputs) {
return PlanRugnuxSweeps(inputs, [this](const std::string &f) { return File(f); },
[this](const std::string &f) { return Sweep(f); });
}
};
} // namespace
TEST_CASE("RugnuxFrameTemplate", "[Rugnux][portable]") {
CHECK(RugnuxFrameTemplate("/d/x_1_0001.cbf") == "/d/x_1_####.cbf");
CHECK(RugnuxFrameTemplate("x_00017.cbf.gz") == "x_#####.cbf.gz");
CHECK(RugnuxFrameTemplate("formal.001") == "formal.###");
CHECK(RugnuxFrameTemplate("noframe.cbf").empty());
}
TEST_CASE("PlanRugnuxSweeps_single_input_is_not_planned", "[Rugnux][portable]") {
FakeInputs in;
CHECK(in.Plan({"a_0001.cbf"}).sweeps.empty());
CHECK(in.Plan({"run_rot.h5"}).sweeps.empty());
CHECK(in.sweeps_described == 0); // nothing is opened for one input
}
TEST_CASE("PlanRugnuxSweeps_frames_by_template", "[Rugnux][portable]") {
FakeInputs in;
SECTION("frames of one template are its one sweep, named by the first given") {
const auto plan = in.Plan({"d/a_0003.cbf", "d/a_0001.cbf", "d/a_0002.cbf"});
REQUIRE(plan.sweeps.size() == 1);
CHECK(plan.sweeps[0].input == "d/a_0003.cbf");
CHECK(plan.sweeps[0].frames);
}
SECTION("two templates are two sweeps, in the order given") {
const auto plan = in.Plan({"d/b_0001.cbf", "d/b_0002.cbf", "d/a_0001.cbf"});
REQUIRE(plan.sweeps.size() == 2);
CHECK(plan.sweeps[0].input == "d/b_0001.cbf");
CHECK(plan.sweeps[0].label == std::filesystem::path("d/b_####.cbf").string());
CHECK(plan.sweeps[1].input == "d/a_0001.cbf");
CHECK(plan.wavelength_A == Catch::Approx(1.0));
}
SECTION("the same name in two directories is two sweeps") {
CHECK(in.Plan({"d1/a_0001.cbf", "d2/a_0001.cbf"}).sweeps.size() == 2);
}
SECTION("sweeps at different wavelengths are refused") {
in.wavelength["d/b_0001.cbf"] = 1.2;
CHECK_THROWS_AS(in.Plan({"d/a_0001.cbf", "d/b_0001.cbf"}), std::runtime_error);
}
}
TEST_CASE("PlanRugnuxSweeps_masters", "[Rugnux][portable]") {
FakeInputs in;
SECTION("stills masters stay one stills run") {
CHECK(in.Plan({"a_still.h5", "b_still.h5", "c_still.h5"}).sweeps.empty());
CHECK(in.sweeps_described == 1); // the first decides; the stills reader refuses a rotation one
}
SECTION("rotation masters are a sweep each") {
const auto plan = in.Plan({"a_rot.h5", "b_rot.h5"});
REQUIRE(plan.sweeps.size() == 2);
CHECK(plan.sweeps[1].input == "b_rot.h5");
CHECK_FALSE(plan.sweeps[0].frames);
}
SECTION("rotation and stills are refused") {
CHECK_THROWS_AS(in.Plan({"a_rot.h5", "b_still.h5"}), std::runtime_error);
}
SECTION("rotation masters at different wavelengths are refused") {
in.wavelength["b_rot.h5"] = 0.98;
CHECK_THROWS_AS(in.Plan({"a_rot.h5", "b_rot.h5"}), std::runtime_error);
}
SECTION("a wavelength within the tolerance is the same") {
in.wavelength["b_rot.h5"] = 1.0 + 1e-4;
CHECK(in.Plan({"a_rot.h5", "b_rot.h5"}).sweeps.size() == 2);
}
}
TEST_CASE("PlanRugnuxSweeps_leaves_the_stills_readers_and_mixtures_alone", "[Rugnux][portable]") {
FakeInputs in;
CHECK(in.Plan({"a.cxi", "b.cxi"}).sweeps.empty());
CHECK(in.Plan({"a_0001.cbf", "b_rot.h5"}).sweeps.empty()); // the readers refuse the mix, as before
CHECK(in.sweeps_described == 0);
}
TEST_CASE("RugnuxSiblingSweeps", "[Rugnux][portable]") {
namespace fs = std::filesystem;
const fs::path dir = fs::path("rugnux_sweeps_siblings");
fs::remove_all(dir);
fs::create_directories(dir);
for (const char *name: {"a_0001.cbf", "a_0002.cbf", "b_0001.cbf", "b_0002.cbf", "b_0003.cbf", "c_0001.cbf.gz",
"notes.txt", "x_master.h5", "y_master.h5"})
std::ofstream(dir / name) << "x";
RugnuxSweepPlan plan;
plan.sweeps.push_back({(dir / "a_0001.cbf").string(), RugnuxFrameTemplate((dir / "a_0001.cbf").string()), true});
plan.sweeps.push_back({(dir / "x_master.h5").string(), (dir / "x_master.h5").string(), false});
const auto siblings = RugnuxSiblingSweeps(plan, {".cbf"});
REQUIRE(siblings.size() == 3);
CHECK(siblings[0] == (dir / "b_####.cbf").string() + " (3 frames)");
CHECK(siblings[1] == (dir / "c_####.cbf.gz").string() + " (1 frame)");
CHECK(siblings[2] == (dir / "y_master.h5").string());
fs::remove_all(dir);
}
+4 -2
View File
@@ -41,8 +41,8 @@ def master_frames(path):
def detector_frame(path):
"""'marCCD' or 'SMV' if the file's header says so, else None. These formats have no reliable
extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next)."""
"""'marCCD', 'SMV' or 'sfrm' (Bruker) if the file's header says so, else None. These formats have
no reliable extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next)."""
try:
with open(path, "rb") as fh:
head = fh.read(64)
@@ -51,6 +51,8 @@ def detector_frame(path):
return "marCCD" if fh.read(3) == b"MMX" else None
if head[:1] == b"{" and b"HEADER_BYTES" in head:
return "SMV"
if head[:8] == b"FORMAT :":
return "sfrm"
except OSError:
pass
return None
+5 -1
View File
@@ -205,5 +205,9 @@
{"id": "4nwv", "input": "4nwv/sbgrid9/formal.001", "wavelength": 1.12706, "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [402.197, 369.862, 410.511, 90.0, 90.0, 90.0], "dmin": 3.25}, "tags": ["marCCD", "orthorhombic", "large-cell", "virus"], "tiers": {"ci": "virus capsid, 400 A body-centred cell, marCCD"}},
{"id": "7d98", "input": "7d98/xrda13/data/cbnrf006_01_00011.cbf", "wavelength": 1.9, "ref": {"sg": "P 61 2 2", "sgno": 178, "cell": [105.819, 105.819, 602.516, 90.0, 90.0, 120.0], "dmin": 3.6}, "tags": ["cbf", "hexagonal", "large-cell", "long-axis", "long-wavelength"], "pinned": true},
{"id": "9v16", "input": "9v16/xrd-00338/9V16/data01_master.h5", "wavelength": 0.9, "ref": {"sg": "P 41 21 2", "sgno": 92, "cell": [185.604, 185.604, 451.242, 90.0, 90.0, 90.0], "dmin": 3.45}, "tags": ["h5", "tetragonal", "large-cell", "long-axis"]},
{"id": "4p8r", "input": "4p8r/data/4p8r/series/gzn0-7.001", "wavelength": 0.97872, "ref": {"sg": "P 65 2 2", "sgno": 179, "cell": [86.7, 86.7, 701.37, 90.0, 90.0, 120.0], "dmin": 2.2}, "tags": ["marCCD", "hexagonal", "large-cell", "long-axis"]}
{"id": "4p8r", "input": "4p8r/data/4p8r/series/gzn0-7.001", "wavelength": 0.97872, "ref": {"sg": "P 65 2 2", "sgno": 179, "cell": [86.7, 86.7, 701.37, 90.0, 90.0, 120.0], "dmin": 2.2}, "tags": ["marCCD", "hexagonal", "large-cell", "long-axis"]},
{"id": "laspartic", "input": "laspartic/cbf/m0255a_01_0001.cbf", "wavelength": 0.71073, "pinned": true, "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [5.1237, 6.9197, 7.6006, 90.0, 100.442, 90.0], "dmin": null}, "cod": "1577465", "tags": ["cbf", "imgcif", "monoclinic", "small-molecule", "twin", "reticular-twin", "kappa", "bruker", "ccd", "home-source"]},
{"id": "onitroaniline", "input": "onitroaniline/sfrm/m0220c_01_0001.sfrm", "wavelength": 0.71073, "pinned": true, "tags": ["sfrm", "small-molecule", "twin", "pseudo-merohedral", "kappa", "bruker", "ccd", "home-source"]},
{"id": "6or0", "input": "6or0/data/ins_zwin_01_0001.sfrm", "wavelength": 1.54188, "pinned": true, "ref": {"sg": "I 21 3", "sgno": 199, "cell": [78.031, 78.031, 78.031, 90.0, 90.0, 90.0], "dmin": 1.55}, "tags": ["sfrm", "cubic", "twin", "non-merohedral", "bruker", "ccd", "home-source"]},
{"id": "6oqz", "input": "6oqz/data/gdri_01_0001.sfrm", "wavelength": 1.54188, "pinned": true, "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [92.931, 97.935, 102.706, 90.0, 90.0, 90.0], "dmin": 1.6}, "tags": ["sfrm", "orthorhombic", "twin", "non-merohedral", "multi-lattice", "bruker", "ccd", "home-source"]}
]}
+4
View File
@@ -110,6 +110,7 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se
cbf_reader.Experiment(experiment);
marccd_reader.Experiment(experiment);
smv_reader.Experiment(experiment);
sfrm_reader.Experiment(experiment);
autoload_timer = new QTimer(this);
autoload_timer->setInterval(autoload_interval);
@@ -326,6 +327,9 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
} else if (JFJochSMVReader::CanRead(path)) {
file_reader = &smv_reader;
smv_reader.ReadFiles(path);
} else if (JFJochSFRMReader::CanRead(path)) {
file_reader = &sfrm_reader;
sfrm_reader.ReadFiles(path);
} else {
cbf_mode = false;
file_reader = &hdf5_reader;
+2
View File
@@ -17,6 +17,7 @@
#include "../reader/JFJochSMVReader.h"
#include "../reader/JFJochXFELReader.h"
#include "../reader/JFJochXTCReader.h"
#include "../reader/JFJochSFRMReader.h"
#include "../common/Logger.h"
#include "JFJochHttpReader.h"
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
@@ -94,6 +95,7 @@ private:
JFJochXFELReader xfel_reader;
JFJochXTCReader xtc_reader;
QStringList pending_files_;
JFJochSFRMReader sfrm_reader;
JFJochReader *file_reader = &hdf5_reader;
bool cbf_mode = false; // "a raw sweep is open", whichever of the three formats it is
+5
View File
@@ -5,6 +5,7 @@
#include "../reader/JFJochCBFReader.h"
#include "../reader/JFJochMarCCDReader.h"
#include "../reader/JFJochSMVReader.h"
#include "../reader/JFJochSFRMReader.h"
#include "../reader/JFJochHDF5Reader.h"
#include "../rugnux/ResultReport.h"
#include "../rugnux/RugnuxOutputFiles.h"
@@ -60,6 +61,7 @@ void JFJochProcessController::run_(QString file_path, DiffractionExperiment expe
JFJochCBFReader cbf_reader;
JFJochMarCCDReader marccd_reader;
JFJochSMVReader smv_reader;
JFJochSFRMReader sfrm_reader;
JFJochReader *reader_ptr;
const std::string path = file_path.toStdString();
if (JFJochCBFReader::CanRead(path)) {
@@ -71,6 +73,9 @@ void JFJochProcessController::run_(QString file_path, DiffractionExperiment expe
} else if (JFJochSMVReader::CanRead(path)) {
smv_reader.ReadFiles(path);
reader_ptr = &smv_reader;
} else if (JFJochSFRMReader::CanRead(path)) {
sfrm_reader.ReadFiles(path);
reader_ptr = &sfrm_reader;
} else {
hdf5_reader.ReadFile(path);
reader_ptr = &hdf5_reader;