diff --git a/common/Reflection.h b/common/Reflection.h index 4b18846af..9f054599a 100644 --- a/common/Reflection.h +++ b/common/Reflection.h @@ -76,6 +76,13 @@ struct ScaledFull { float I = NAN; float sigma = NAN; float image_number = NAN; // the frame of the event's peak partial, which tells two events of one index apart + // The full's counting variance as the merge rebuilds it at an intensity I, before its error model: + // var_bkg + var_per_I * I + (capture * I)^2, all at the intensity's scale (what a later merge rebuilds + // the full from). NaN where not exported. + float var_bkg = NAN; + float var_per_I = NAN; + float capture = NAN; + float d = NAN; // resolution, A }; struct MergedReflection { diff --git a/common/ScalingSettings.cpp b/common/ScalingSettings.cpp index 8b70c6ac9..8fc2cd2c9 100644 --- a/common/ScalingSettings.cpp +++ b/common/ScalingSettings.cpp @@ -315,3 +315,12 @@ ScalingSettings& ScalingSettings::ReportResolutionRange_A(std::optional ScalingSettings::GetReportResolutionRange_A() const { return report_resolution_range; } + +ScalingSettings& ScalingSettings::SweepSegments(std::vector input) { + sweep_segments = std::move(input); + return *this; +} + +const std::vector &ScalingSettings::GetSweepSegments() const { + return sweep_segments; +} diff --git a/common/ScalingSettings.h b/common/ScalingSettings.h index 6e3d0b8e7..d1b3af5da 100644 --- a/common/ScalingSettings.h +++ b/common/ScalingSettings.h @@ -3,7 +3,9 @@ #pragma once +#include #include +#include #include "JFJochException.h" // How the high-resolution cutoff for the written reflections and the reported shell table is chosen @@ -17,8 +19,20 @@ struct ReportResolutionRange { double d_max = 0.0; }; +// One sweep of several whose frames are laid end to end in one rotation merge (the joint merge of several +// sweeps): frames [first_frame, first_frame + n_frames). Every observation of a segment's frames is a full +// already combined in its own sweep (partiality 1, filed under the frame of its peak), so the merge +// combines nothing again, recomputes nothing from the rocking geometry, and lets nothing that runs along +// the frames - the per-frame scale's smoothness above all - reach across from one sweep into the next +// (RotationScaleMerge). +struct SweepSegment { + int32_t first_frame = 0; + int32_t n_frames = 0; +}; + class ScalingSettings { bool refine_wedge = false; + std::vector sweep_segments; bool merge_friedel = true; std::optional high_resolution_limit_A; @@ -150,6 +164,8 @@ public: ScalingSettings& ResolutionCCTarget(double input); ScalingSettings& ReportShellCount(int input); ScalingSettings& ReportResolutionRange_A(std::optional input); + // The sweeps a rotation merge's frames are made of; empty (the default): one sweep of partials. + ScalingSettings& SweepSegments(std::vector input); [[nodiscard]] bool GetRefineWedge() const; @@ -189,4 +205,5 @@ public: [[nodiscard]] double GetResolutionCCTarget() const; [[nodiscard]] int GetReportShellCount() const; [[nodiscard]] std::optional GetReportResolutionRange_A() const; + [[nodiscard]] const std::vector &GetSweepSegments() const; }; diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index ede0116af..8f41d08a4 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -384,6 +384,15 @@ J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in t package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399 [doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198). +**Several sweeps merged as one** - each sweep's partials combined into fulls on their own, and the +fulls of all the sweeps then scaled together, every sweep with its own frame scales and a relative B +against the joint merge, is how XSCALE merges several data sets (W. Kabsch, "Integration, scaling, +space-group assignment and post-refinement" (2010), above). Reading the radiation dose a sweep was taken +at from its relative B, which orders the sweeps by dose from the data rather than from the order they +were given in, follows K. Diederichs, "Some aspects of quantitative analysis and correction of radiation +damage" (2006), Acta Cryst. D62, 96-101 +[doi:10.1107/S0907444905031537](https://doi.org/10.1107/S0907444905031537). + **Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature: D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography: Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75 diff --git a/docs/EXTERNAL_TEST_DATA.md b/docs/EXTERNAL_TEST_DATA.md index f374959ed..0f4033745 100644 --- a/docs/EXTERNAL_TEST_DATA.md +++ b/docs/EXTERNAL_TEST_DATA.md @@ -4,7 +4,7 @@ Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeli ever sees its own detectors is not tested. The datasets below were collected by other people, on detectors and in file formats we do not produce ourselves, and are used here to check that `rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected -at other facilities, eleven on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone +at other facilities, thirteen on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone else's detector and someone else's acquisition system. Their authors published all of these for exactly this kind of reuse, and this page is where we credit them. @@ -101,6 +101,8 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the | [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC | | [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset | | [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor | +| [6OQZ](https://www.rcsb.org/structure/6OQZ) | IRRMC [10.18430/m3.irrmc.5324](https://doi.org/10.18430/m3.irrmc.5324) | Home source, Bruker rotating anode | 1.60 | I 2 2 2 | 92.9 97.9 102.7 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of glucose isomerase from non-merohedrally twinned crystals | +| [6OR0](https://www.rcsb.org/structure/6OR0) | IRRMC [10.18430/m3.irrmc.5325](https://doi.org/10.18430/m3.irrmc.5325) | Home source, Bruker rotating anode | 1.55 | I 21 3 | 78.0 78.0 78.0 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of insulin from non-merohedrally twinned crystals | | [6OWW](https://www.rcsb.org/structure/6OWW) | Zenodo [10.5281/zenodo.2943248](https://doi.org/10.5281/zenodo.2943248) | ALS 8.3.1 | 3.84 | P 1 21 1 | 85.8 86.0 214.3 90.0 89.9 90.0 | PILATUS3 6M | Crystal structure of a Human Cardiac Calsequestrin Filament Complexed with Ytterbium | | [6P8J](https://www.rcsb.org/structure/6P8J) | SBGrid [10.15785/sbgrid/671](https://doi.org/10.15785/sbgrid/671) | APS 24-ID-E | 1.47 | P 1 21 1 | 46.5 152.2 88.3 90.0 90.1 90.0 | Dectris Eiger 16M | Structure of P. aeruginosa ATCC27853 CdnD D62N/D64N mutant bound to ATP | | [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 | @@ -244,7 +246,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the | [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 | | [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) | -The table holds 201 PDB entries. Six more datasets, small-molecule / chemical-crystallography +The table holds 203 PDB entries. Eight more datasets, small-molecule / chemical-crystallography sets kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero detector 2θ, have no PDB entry and are listed [below](#datasets-with-no-pdb-entry). @@ -313,7 +315,7 @@ the rest were deleted, so a run over the data directory sees a single collection | 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å | | 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** | -**Thirteen further archives hold more than one collection.** Their layout was read from the +**Seventeen further archives hold more than one collection.** Their layout was read from the image files and repository listings; one sweep is kept for a run over the data directory unless noted. @@ -329,6 +331,10 @@ noted. | lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run | | 9E2T | one continuous sweep plus screening images | the 2700-frame sweep | | 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) | +| 6OR0 | four 900 × 0.2° Bruker `.sfrm` runs of one twinned crystal at 10 cm: ω scans at 2θ = 0° (`01`) and 35° (`02`), φ scans on the fixed-χ (54.74°) platform at 2θ = 0° (`03`) and 35° (`04`) | the `01` run (2θ = 0°): the header distance of these SMART 6000 frames is ~4% short of what the data give, which the distance refinement corrects on a 2θ = 0° run and not yet on a swung one | +| 6OQZ | eleven `.sfrm` ω runs of 472-900 × 0.2° at 18 cm, 2θ = 10°, 25° and 45° (low, medium, high resolution) at different φ, plus two 5-frame runs at 2θ = 0° and the orientation-matrix runs `gdri_MATRIX_*` | the low-resolution `01` run (2θ = 10°), for the reason given for 6OR0; three crystals, the test set for multi-lattice integration | +| laspartic | seven Bruker-converted full-imgCIF kappa runs, φ (`01`, 1200 × 0.3°) and ω (`02`-`07`, 292-394 frames) at 2θ = -26.7° | the 1200-frame `01` φ run | +| onitroaniline | the same seven-run kappa layout (`01` 1200 × 0.3° φ, `02`-`07` ω), deposited twice: as Bruker `.sfrm` and as APEX's full-imgCIF conversion of the same frames | the `01` run, read from the `.sfrm` files | | 7D98 | one 250 × 0.5° miniCBF sweep (frames 11-260), plus four HDF5 masters that repeat frames 61-260 in 60-frame pieces | the miniCBF sweep; the HDF5 files stay beside it and the row is pinned | | 6H2P | two sweeps of one crystal, 1800 × 0.1° at 0.918 Å and 3600 × 0.1° at 1.893 Å (the Mn K edge) | **both**, as two battery rows, `6h2p_native` and `6h2p_1p89A`; the long-wavelength row has no deposited resolution to compare with | | 8C3E | four 0.5° ω scans of one crystal at different (φ, χ) settings, 56, 150, 150 and 50 frames (`lcb2_8_newt_001` to `_004`) | the 150-frame `_003` scan | @@ -349,7 +355,7 @@ noted. ## Datasets published as Raw Data Letters -Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a +Five of the datasets - 6R72, 8QQ7, 6RLR, laspartic and onitroaniline - were published as IUCrData Raw Data Letters, a format whose purpose is to make raw images citable and re-processable in their own right. The letters describe the collections and the difficulties in them, and are the reference for what the data are: @@ -364,6 +370,15 @@ data are: IUCrData 7, x220852 [doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR. +- M. Lutz and L. Kroon-Batenburg, "Accurate intensity integration in the twinned γ-form of + *o*-nitroaniline" (2022), IUCrData 7 + [doi:10.1107/S2414314622010598](https://doi.org/10.1107/S2414314622010598) - covers + onitroaniline. +- M. Lutz, "Twinned L-aspartic acid" (2025), IUCrData 10 + [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) - covers laspartic: + a twofold rotation about [100] with twin index 2 and obliquity 0.74°, whose coincidence lattice + is the doubled cell first reported for the compound (Bernal, 1931). + The authors of the second letter also published their own reciprocal-space reconstruction of the 6RLR data as a separate Zenodo record, [10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763). @@ -437,8 +452,8 @@ The PDB entry names the detector as a Rigaku HyPix-6000HE. ## Deposited models and structure factors -All 201 PDB entries in the table are released, and RCSB reports released structure factors -(`status_code_sf = REL`) for every one of them; the six small-molecule datasets have none. A merged +All 203 PDB entries in the table are released, and RCSB reports released structure factors +(`status_code_sf = REL`) for every one of them; the eight small-molecule datasets have none. A merged result from this pipeline can therefore be checked against the deposited model or against the deposited intensities. @@ -609,8 +624,10 @@ reader that globs `*.img` will pick them up, so they are named here rather than | `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition | | `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition | | `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition | +| `laspartic` | Zenodo record 10.5281/zenodo.15432050 | a small-molecule dataset, not a PDB deposition | +| `onitroaniline` | Zenodo record 10.5281/zenodo.7193538 | a small-molecule dataset, not a PDB deposition | -Five of the six small-molecule sets have a published structure to check a run against. These are +Six of the eight small-molecule sets have a published structure to check a run against. These are reference values from the literature, not results obtained here. | Dataset | Space group | Cell (A, deg) | T | Reference | @@ -620,6 +637,10 @@ reference values from the literature, not results obtained here. | `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) | | `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) | | `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) | +| `laspartic` | `P 1 21 1` (4) | 5.1237 6.9197 7.6006 / 90 100.442 90 | 150 K | twin fractions 0.537/0.463; COD 1577465 - M. Lutz, IUCrData **10** (2025) [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) | + +`onitroaniline` has no cell here: the letter describing it could not be retrieved when the row was +added, so a run on it is not scored against a reference. `cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**, 064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no @@ -632,20 +653,20 @@ symmetries rather than to be easy to process. The counts below describe where it like everything else on this page, they are metadata about the depositions and their files, not measurements. -- **Repository**, over the 207 datasets on this page (the 201 rows of the table and the six - small-molecule sets): IRRMC 95, Zenodo 42, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5, +- **Repository**, over the 211 datasets on this page (the 203 rows of the table and the eight + small-molecule sets): IRRMC 97, Zenodo 44, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5, ESRF 3, UQ eSpace 1. - **Facility** - counted from the facility part of the Facility / beamline column, the beamline ignored so that entries deposited with and without one count the same, over the 190 rows that name one: APS 29, Diamond 21, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9, SLS 8, SPring-8 8, SOLEIL 7, SSRF 7, PAL/PLS 5, CHESS 4, ALBA 3, Australian Synchrotron 3, CLSI 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, Photon Factory 2, and one each from NSRRC, - RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other eleven - rows were collected on laboratory sources: nine on rotating anodes, one on a liquid-metal jet + RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other thirteen + rows were collected on laboratory sources: eleven on rotating anodes, one on a liquid-metal jet and one on a sealed tube. -- **Crystal system, from the deposited space group of the 201 rows:** monoclinic 53, - orthorhombic 50, tetragonal 30, trigonal 22, hexagonal 20, cubic 13, triclinic 13. -- **Pink beam:** none of these datasets was collected with pink beam. All 201 rows are deposited +- **Crystal system, from the deposited space group of the 203 rows:** monoclinic 53, + orthorhombic 51, tetragonal 30, trigonal 22, hexagonal 20, cubic 14, triclinic 13. +- **Pink beam:** none of these datasets was collected with pink beam. All 203 rows are deposited as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all but 5REO, which leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` `M`); 9Q41 is the one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The @@ -654,7 +675,7 @@ measurements. - **Long cell axes:** sixteen rows have a deposited cell axis longer than 320 Å - 4P8R, 4MTK, 7D98, 8V4O, 9ZMU, 9V16, 9Z72, 4NWV, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL. -The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats +The marCCD, SMV, Bruker `.sfrm`, full-imgCIF and gzip-compressed miniCBF datasets are the reason rugnux reads those formats natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with. ## Licences diff --git a/docs/RUGNUX_CLI.md b/docs/RUGNUX_CLI.md index 5d5834b33..45674058f 100644 --- a/docs/RUGNUX_CLI.md +++ b/docs/RUGNUX_CLI.md @@ -30,9 +30,10 @@ rugnux -o myrun dataset_master.h5 # NXmx / EIGER / Jungfraujoch mas rugnux -o myrun sweep_00001.cbf # CBF, marCCD or SMV sweep: name one frame rugnux -o myrun dataset_master.h5 model.cif # with a model (or --model model.cif) rugnux -o myrun x_master.h5 http://host:5232/ # follow a broker's collection while it is written +rugnux -o myrun a_master.h5 b_master.h5 # several sweeps of one crystal, merged as one ``` -Chosen when the file's goniometer axis turns. One sweep per run. A model is placed (rigid body, or +Chosen when the file's goniometer axis turns. Several sweeps can be given (below, Inputs). A model is placed (rigid body, or molecular replacement), refined and validated after merging; it does not change the processing decisions ([Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md)). @@ -91,14 +92,36 @@ spot finding or integration ([Re-scaling and re-merging](RUGNUX_ADVANCED.md#re-s ## Inputs Inputs are files, never a directory, read in the order given (a shell glob gives them sorted by -name). Several inputs are read together only as one stills dataset in `--mode mx`: all the files of +name). Several inputs are one stills dataset in `--mode mx`, or several rotation sweeps (below): all the files of one dataset - `rugnux run*.h5` (SACLA), `*.cxi`, `*.xtc` (LCLS) - or several Jungfraujoch/NXmx masters of stills, read as one run: their images one after another, merged together, with the indexing rate reported per master as well. Such masters must share the detector, wavelength, -distance and beam centre. Several inputs must be one format; a rotation sweep (name one frame of a -CBF/marCCD/SMV sweep), any `--mode` other than `mx`, and following a broker take one. What each -format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read without a -geometry file where their own metadata places the detector +distance and beam centre. Several inputs must be one format; any `--mode` other than `mx`, and +following a broker, take one. + +Several rotation sweeps are told apart from the files: one sweep per CBF/marCCD/SMV/SFRM file-name +template (the name with its frame number taken out - any frame of a sweep names the whole sweep, so +`rugnux a_*.cbf` is the one sweep `a`), one per rotation master. Only the inputs given are processed; +other sweeps beside them are named in the log. A mix of rotation and stills masters is refused, and +so are sweeps at different wavelengths, `-s`/`-e` and `--force-still` with several sweeps. +By default (`--sweeps joint`) the sweeps are one dataset, merged in two levels as XSCALE merges the +data sets XDS made of each sweep: each sweep is processed on its own first - indexed, integrated, its +partials combined and scaled, its space group determined - writing only its log `_s_log.txt`; +the sweeps are then put on the axes of the one with the most reflections (the change of basis, an +axis permutation or a twin law of the lattice, under which their merged intensities correlate best; +a sweep correlating below 0.5 on every axes is left out and named in the log), and their fulls are +scaled and merged together: a per-frame scale within each sweep, the sweeps' dose order read from +their relative B factors (not from the input order or the header times), a decay along that order or +one B per sweep, whichever predicts held-out frames better, then the space-group search, the +resolution cut and everything after it on the joint data, into `.*`. The detector-dependent +corrections (absorption and detector surfaces) are each sweep's own. The unmerged MTZ holds the joint +merge's scaled observations in the batches of their own sweeps (sweep `s` of the merge: batches +`s * base + frame + 1`, `base` the power of ten past the longest sweep), under the batch headers that +sweep's own run wrote; the report names each sweep in section `1b. SWEEPS`. `--sweeps separate` processes each sweep alone instead, as if it were the only +input, into `_s_*` (k = 1, 2, ... in the order given). + +What each format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read +without a geometry file where their own metadata places the detector ([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)). An atomic model and a reference reflection file can be given among the inputs, in any order, @@ -130,6 +153,7 @@ With the input already on disk, the URL is ignored. | `-v, --verbose` | Show the whole log on the terminal in place of the steps; `_log.txt` is written either way | | `--simulate-live ` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only | | `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse | +| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `.*`; `separate` processes each alone into `_s_*` | | `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) | ### Mode diff --git a/docs/RUGNUX_REPORT.md b/docs/RUGNUX_REPORT.md index 79ddbfb6b..5dd08bb6d 100644 --- a/docs/RUGNUX_REPORT.md +++ b/docs/RUGNUX_REPORT.md @@ -120,7 +120,8 @@ The header block above section 1 records **how the result was produced**: `RUGNU Rates, per-image costs and progress remain on stdout only. Sections, in order: the `SUMMARY` ([below](#the-summary-and-what-the-run-decided)), then -`1. DATA SET AND GEOMETRY`, `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model +`1. DATA SET AND GEOMETRY`, on a joint merge of several sweeps `1b. SWEEPS` +([below](#several-sweeps-merged-as-one)), `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model was given (`--model`, `--mr`, or a model file among the inputs), `5. MODEL VALIDATION` - which also holds the keys of [molecular replacement](#molecular-replacement) and of the [refinement](#refinement-refine_) where they ran. A `--mode refine` report has one section, @@ -252,7 +253,7 @@ and a run that produced a textbook data set read identically for their first thr `GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`, `LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_CELL_MISMATCH`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`, `FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`, `SCALING_NOT_CONVERGED`, `HARMONIC_CONTAMINATION`, - `SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`. `NONE` when nothing fired. A code appears if and only if its + `SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`, `SWEEP_LEFT_OUT`. `NONE` when nothing fired. A code appears if and only if its warning fired, so the flags and the `WARNING:` lines are two renderings of one list. Thresholds are deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on data that turn out fine — the closed @@ -286,6 +287,29 @@ and a run that produced a textbook data set read identically for their first thr Then the numbered sections: **1. DATA SET AND GEOMETRY**, **2. CRYSTAL**, **3. MERGED DATA**, **4. DIAGNOSTICS**, and **5. MODEL VALIDATION** only where a model was given. +## Several sweeps merged as one + +A run of several sweeps merged as one (`--sweeps joint`, the default with several rotation inputs, see +[Command-line options](RUGNUX_CLI.md)) describes the joint merge everywhere, and adds section +**1b. SWEEPS**; a run of one input has no such section and no such keys. + +- **`SWEEP_COUNT=`** the sweeps given, **`SWEEP_REFERENCE=`** the one (1-based) the others were put on + the axes of - the one with the most reflections - and **`SWEEP_DECAY_MODEL=`** how the decay over the + sweeps was modelled: `dose-linear` (a B rising along the dose order), `per-sweep B` (one B a sweep) or + `none`, whichever predicted held-out frames better. +- Per sweep `k`: **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep + gave processed on its own - **`SWEEP_k_SPACE_GROUP_ALONE=`**, **`SWEEP_k_CELL=`**, + **`SWEEP_k_CC_HALF_ALONE=`**, **`SWEEP_k_COMPLETENESS_ALONE=`**, **`SWEEP_k_RESOLUTION_ALONE=`**; and, + for a sweep in the joint merge, **`SWEEP_k_REINDEX_OP=`** (onto the reference's axes) with + **`SWEEP_k_REINDEX_CC=`** (the merged-intensity correlation under it), **`SWEEP_k_BATCH_FIRST=`** / + **`SWEEP_k_BATCH_LAST=`** (its batches in the unmerged MTZ: sweep `s` of the merge is + `s * base + frame + 1`, `base` the power of ten past the longest sweep), **`SWEEP_k_SCALE=`** (the + median scale of its frames in the joint merge), **`SWEEP_k_RELATIVE_B=`** (its B against the joint + merge, the least damaged sweep at 0) and **`SWEEP_k_DOSE_RANK=`** (its place, from 1, in the dose + order those B factors decide - not the order the sweeps were given in, nor their header times). +- A sweep given but not merged - it gave no merge, or its intensities correlate with the reference + sweep's on no axes - is named by a `SWEEP_LEFT_OUT` warning. + ## The reference-range table `--report-resolution [,]` adds a second block of merging statistics to section 3, the diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index fbdbc000e..802b9d7b9 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -365,6 +365,7 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment, // Decay + absorption correction surfaces are one master toggle (on by default; both cross-validated, // so a no-op when their systematic is absent). refine_decay_b = s.GetCorrectionSurfaces(); + segments = s.GetSweepSegments(); absorption_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0; modulation_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0; relative_b_deg = s.GetRelativeBDegrees(); // per-batch relative-B batch width (0 = off) @@ -531,11 +532,28 @@ void RotationScaleMerge::IngestHost() { if (frame_limit >= 0) n_frames = std::min(n_frames, frame_limit); partials_released = false; resident_ingest = false; + frame_segment.clear(); + if (!segments.empty()) { + frame_segment.assign(n_frames, -1); + int32_t next = 0; + for (size_t k = 0; k < segments.size(); ++k) { + const SweepSegment &seg = segments[k]; + if (seg.first_frame != next || seg.n_frames <= 0 || seg.first_frame + seg.n_frames > n_frames) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "RotationScaleMerge: the sweep segments do not cover the frames in order"); + for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f) + frame_segment[f] = static_cast(k); + next = seg.first_frame + seg.n_frames; + } + if (next != n_frames) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "RotationScaleMerge: the sweep segments do not cover the frames in order"); + } #ifdef JFJOCH_USE_CUDA // Probed before the observations are built: whether the build lays down the full Obs array or // only the ingest arrays depends on whether the device pipeline will be active. gpu_ = std::make_unique(gpu_device); - gpu_active_ = gpu_->Available() && !scale_on_cpu_; + gpu_active_ = gpu_->Available() && !scale_on_cpu_ && segments.empty(); resident_ingest = gpu_active_ && !observation_dump; #endif @@ -605,7 +623,12 @@ void RotationScaleMerge::IngestHost() { } }); - MeasureIncidentFlux(mean_bkg); + // Fulls of several sweeps carry each sweep's own background: not one flux meter, so the per-frame + // scale alone takes the flux. + if (segments.empty()) + MeasureIncidentFlux(mean_bkg); + else + frame_flux.assign(n_frames, 1.0); // The occupied range of h, which is the buckets the sort works on. int b_lo = H_ALL, b_hi = -1; @@ -728,12 +751,19 @@ void RotationScaleMerge::IngestHost() { } }); } - BuildInRangeObservations(keys, static_cast(total), SmoothedFrameGeometry()); + // Pre-combined fulls have no rocking geometry left to smooth or to recompute a partiality from. + BuildInRangeObservations(keys, static_cast(total), + segments.empty() ? SmoothedFrameGeometry() : SmoothedGeometry{}); rawrun_group.assign(rawrun_start.size(), -1); logger->Info("RotationScaleMerge: ingested {} partial observations from {} frames ({} distinct hkl)", n_partials_obs, n_frames, rawrun_start.size()); - SmoothMosaicityAndPartiality(); + if (segments.empty()) { + SmoothMosaicityAndPartiality(); + } else { + ingest_zeta.reset(); + ingest_d.reset(); + } ParallelChunks(static_cast(partials.size()), ThreadsForWork(partials.size(), nthreads), [&](int lo, int hi) { for (int i = lo; i < hi; ++i) @@ -2174,6 +2204,7 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( has_fulls[f] = frame_obs_count[f] > 0 ? 1 : 0; } double best_step = INFINITY, lambda = 0.0; + std::vector segment_lambda(segments.size(), 0.0); bool lambda_kept = false; std::vector steps, g_best; AndersonMixing anderson(SCALING_ANDERSON_DEPTH); @@ -2188,9 +2219,16 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( if (fitted) y[f] += g_fit[f] > 0.0 ? std::max(std::log(g_fit[f] / g[f]), -1.0) : -1.0; } - if (!lambda_kept) - lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads); - const std::vector x = SmoothLogScale(y, J, lambda); + std::vector x; + if (segments.empty()) { + if (!lambda_kept) + lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads); + x = SmoothLogScale(y, J, lambda); + } else { + x = SmoothLogScaleBySegment(y, J, segment_lambda, !lambda_kept); + std::vector l = segment_lambda; + lambda = median_of(l); + } for (int f = 0; f < n_frames; ++f) g_new[f] = std::exp(x[f]); const double g_ref = TypicalFrameScale(g_new, n_obs); @@ -2342,8 +2380,14 @@ void RotationScaleMerge::RefineDecay(int n_groups) { return o.group >= 0 && o.corr > 0.0f && std::isfinite(o.corr) && (o.d > 0.0f) && o.partiality >= min_partiality; }; + // Where an observation sits on the dose axis: its frame, or with several sweeps its place in the dose + // order the data put them in (MeasureSegmentDose). + auto dose = [&](const auto &o) { + return segments.empty() ? static_cast(o.image_number) + : frame_dose[o.frame] + (o.image_number - static_cast(o.frame)); + }; auto decay_factor = [&](const auto &o, double slope) { - return std::exp(slope * (o.image_number - fcenter) * s2_of(o.d)); + return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d)); }; // The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the @@ -2358,6 +2402,8 @@ void RotationScaleMerge::RefineDecay(int n_groups) { BuildDecayTerms(n_groups, t); decay_terms_kept = {}; decay_terms_kept_valid = false; + if (!segments.empty()) + MeasureSegmentDose(t, n_groups); const int nt = static_cast(ThreadsForWork(t.term.size(), nthreads)); std::vector sw(n_groups), swI(n_groups); @@ -2385,7 +2431,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) { const double Is = static_cast(o.I) * o.corr, sc = static_cast(o.sigma) * o.corr; if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return; const double w = (Is / sc) * (Is / sc); - const double x = (o.image_number - fcenter) * s2_of(o.d), y = std::log(Iref / Is); + const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is); a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y; }, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; }); const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4]; @@ -2435,6 +2481,125 @@ void RotationScaleMerge::RefineDecay(int n_groups) { // sit at the same s^2 but different frames, so the correction spreads them) - net harmful. slope = // 2 dB/dframe, so the total relative-B change across the run is |slope/2 * n_frames|. constexpr double DECAY_MIN_DELTA_B = 2.0; // A^2, minimum total relative-B over the run to engage + if (!segments.empty()) { + // Two hypotheses of how damage accrued over several sweeps, decided on held-out frames: a B rising + // linearly along the dose order (frame_dose), or one B per sweep - a step at each sweep boundary, + // flat within a sweep. Both are regressions of y = ln(I_ref / I_obs) on s^2 = 1/(4 d^2), the ramp + // on x = (dose - centre) s^2, the steps on x = s^2 with a slope per sweep, at the weights the slope + // fit above uses and within each frame (below). Each is fitted on the even frames and its + // weighted squared residual taken on the odd ones, against their own reference, and the other way + // round; the hypothesis that predicts the held-out frames better is applied, where it beats no + // decay at all by the margin a correction has to clear on one sweep (and the physical floor). + // The held-out frames' residual is read at weights fixed by the observations, so a correction + // cannot win by reshaping sigma. + using Sums = std::array; // Sw, Sx, Sy, Sxx, Sxy + struct Line { double a = 0.0, b = 0.0; }; + auto line_of = [](const Sums &S) { + Line l; + const double var = S[0] > 0.0 ? S[3] - S[1] * S[1] / S[0] : 0.0; + if (var > 0.0) l.b = (S[4] - S[1] * S[2] / S[0]) / var; + if (S[0] > 0.0) l.a = (S[2] - l.b * S[1]) / S[0]; + return l; + }; + auto reference = [&](int parity) { + std::fill(sw.begin(), sw.end(), 0.0); + std::fill(swI.begin(), swI.end(), 0.0); + for (const DecayTerm &o : t.term) { + if (parity >= 0 && (o.frame & 1) != parity) continue; + const double sc = static_cast(o.sigma) * o.corr, w = 1.0 / (sc * sc); + sw[o.group] += w; + swI[o.group] += w * static_cast(o.I) * o.corr; + } + }; + // Every term of `parity` with its y and weight against the current reference. + auto for_terms = [&](int parity, const auto &fn) { + for (const DecayTerm &o : t.term) { + if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue; + const double Iref = swI[o.group] / sw[o.group]; + const double Is = static_cast(o.I) * o.corr, sc = static_cast(o.sigma) * o.corr; + if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue; + fn(o, std::log(Iref / Is), (Is / sc) * (Is / sc)); + } + }; + auto x_ramp = [&](const DecayTerm &o) { return (dose(o) - fcenter) * s2_of(o.d); }; + // The terms of `parity` with y and both x taken about their frame's weighted mean: each frame has a + // scale of its own (the per-frame G), which took up its part of the B averaged over its + // reflections, so only how y varies with resolution WITHIN a frame is decay. + struct Point { int32_t seg; double w, y, xr, xs; }; + auto points = [&](int parity) { + reference(parity); + std::vector pts; + std::vector frame; + std::vector> mean(n_frames, std::array{}); // Sw, Sy, Sxr, Sxs + for_terms(parity, [&](const DecayTerm &o, double y, double w) { + pts.push_back({frame_segment[o.frame], w, y, x_ramp(o), s2_of(o.d)}); + frame.push_back(o.frame); + auto &m = mean[o.frame]; + m[0] += w; m[1] += w * y; m[2] += w * pts.back().xr; m[3] += w * pts.back().xs; + }); + for (size_t k = 0; k < pts.size(); ++k) { + const auto &m = mean[frame[k]]; + pts[k].y -= m[1] / m[0]; pts[k].xr -= m[2] / m[0]; pts[k].xs -= m[3] / m[0]; + } + return pts; + }; + struct Fit { double ramp = 0.0; std::vector step; }; + auto fit = [&](int parity) { + Sums r{}; + std::vector st(segments.size(), Sums{}); + for (const Point &q : points(parity)) { + r[0] += q.w; r[1] += q.w * q.xr; r[2] += q.w * q.y; r[3] += q.w * q.xr * q.xr; r[4] += q.w * q.xr * q.y; + Sums &a = st[q.seg]; + a[0] += q.w; a[1] += q.w * q.xs; a[2] += q.w * q.y; a[3] += q.w * q.xs * q.xs; a[4] += q.w * q.xs * q.y; + } + Fit out{line_of(r).b, {}}; + for (const Sums &a : st) out.step.push_back(line_of(a).b); + return out; + }; + // Held-out weighted squared residuals of no decay, the ramp and the steps. + auto score = [&](int test, const Fit &m, std::array &ssr) { + for (const Point &q : points(test)) { + const double e1 = q.y - m.ramp * q.xr, e2 = q.y - m.step[q.seg] * q.xs; + ssr[0] += q.w * q.y * q.y; ssr[1] += q.w * e1 * e1; ssr[2] += q.w * e2 * e2; + } + }; + std::array ssr{}; + const Fit even = fit(0), odd = fit(1); + score(1, even, ssr); + score(0, odd, ssr); + const Fit all = fit(-1); + + const double slope = all.ramp; // the slope decay_factor applies + std::vector b(segments.size()); + double b_mid = 0.0, b_lo = INFINITY, b_hi = -INFINITY; + for (size_t k = 0; k < segments.size(); ++k) { + b[k] = 0.5 * all.step[k]; + b_mid += b[k] * segments[k].n_frames / static_cast(n_frames); + b_lo = std::min(b_lo, b[k]); + b_hi = std::max(b_hi, b[k]); + } + const double delta_lin = std::fabs(0.5 * slope * n_frames), delta_step = b_hi - b_lo; + const bool lin_ok = delta_lin >= DECAY_MIN_DELTA_B && ssr[1] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0]; + const bool step_ok = delta_step >= DECAY_MIN_DELTA_B && ssr[2] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0]; + const bool use_step = step_ok && (!lin_ok || ssr[2] < ssr[1]); + const bool use_lin = lin_ok && !use_step; + segment_decay_model = use_step ? "per-sweep B" : use_lin ? "dose-linear" : "none"; + const double rel = ssr[0] > 0.0 ? 100.0 / ssr[0] : 0.0; + logger->Info("Decay over {} sweeps, held-out residual against no decay: dose-linear (dB = {:.2f} A^2 over " + "the dose order) {:.1f}%, one B per sweep (spread {:.2f} A^2) {:.1f}% => {}", segments.size(), + delta_lin, ssr[1] * rel, delta_step, ssr[2] * rel, segment_decay_model); + if (use_step || use_lin) + ParallelChunks(static_cast(fulls.size()), ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) { + for (int i = lo; i < hi; ++i) { + Obs &o = fulls[i]; + if (!usable(o)) continue; + const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d)) + : decay_factor(o, slope); + o.corr = static_cast(o.corr * f); + } + }); + return; + } const double slope = fit_slope(-1); const double total_delta_B = std::fabs(0.5 * slope * n_frames); if (total_delta_B < DECAY_MIN_DELTA_B) { @@ -2460,6 +2625,92 @@ void RotationScaleMerge::RefineDecay(int n_groups) { 0.5 * slope * n_frames, 0.5 * slope); } +std::vector RotationScaleMerge::SegmentB(const DecayTerms &t, int n_groups, int parity) const { + // Per segment, the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against + // s^2 = 1/(4 d^2), which is 2 B, with an intercept of its own (the scale is the per-frame G's), over + // the frames of `parity` (< 0: all) against a reference made of the same frames. NaN where a segment + // has nothing to fit. + const size_t ns = segments.size(); + std::vector sw(n_groups, 0.0), swI(n_groups, 0.0); + for (const DecayTerm &o : t.term) { + if (parity >= 0 && (o.frame & 1) != parity) continue; + const double sc = static_cast(o.sigma) * o.corr, w = 1.0 / (sc * sc); + sw[o.group] += w; + swI[o.group] += w * static_cast(o.I) * o.corr; + } + std::vector> S(ns, std::array{}); // Sw, Sx, Sy, Sxx, Sxy + for (const DecayTerm &o : t.term) { + if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue; + const double Iref = swI[o.group] / sw[o.group]; + const double Is = static_cast(o.I) * o.corr, sc = static_cast(o.sigma) * o.corr; + if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue; + const double w = (Is / sc) * (Is / sc); + const double x = 1.0 / (4.0 * static_cast(o.d) * o.d), y = std::log(Iref / Is); + auto &a = S[frame_segment[o.frame]]; + a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y; + } + std::vector b(ns, NAN); + for (size_t k = 0; k < ns; ++k) { + const auto &a = S[k]; + const double var = a[0] > 0.0 ? a[3] - a[1] * a[1] / a[0] : 0.0; + if (var > 0.0) + b[k] = 0.5 * (a[4] - a[1] * a[2] / a[0]) / var; + } + return b; +} + +void RotationScaleMerge::MeasureSegmentDose(const DecayTerms &t, int n_groups) { + // A relative B per data set against the joint merge, as XSCALE fits it: Kabsch (2010) Acta Cryst. + // D66, 133-144; the dose read from it: Diederichs (2006) Acta Cryst. D62, 96-101. + // Radiation damage raises the relative B of a sweep collected later, so the B factors of the + // segments against their joint merge give the order the crystal took the dose in - read from the + // data, not from the order the sweeps were given in or the times their headers state. Per segment, + // the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against s^2 = 1/(4 d^2) is 2 B, + // with an intercept of its own (the scale is the per-frame G's). The least damaged segment is B = 0, + // and its frames start the dose axis. + const size_t ns = segments.size(); + const std::vector b = SegmentB(t, n_groups, -1); + // A segment with nothing to measure keeps its place in the input order, after the measured ones. + std::vector order(ns); + std::iota(order.begin(), order.end(), 0); + std::stable_sort(order.begin(), order.end(), [&](int p, int q) { + if (std::isfinite(b[p]) != std::isfinite(b[q])) return std::isfinite(b[p]); + return std::isfinite(b[p]) && b[p] < b[q]; + }); + const double b0 = std::isfinite(b[order[0]]) ? b[order[0]] : 0.0; + frame_dose.assign(n_frames, 0.0); + double offset = 0.0; + std::string text; + for (size_t rank = 0; rank < ns; ++rank) { + const int k = order[rank]; + const SweepSegment &seg = segments[k]; + for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f) + frame_dose[f] = offset + (f - seg.first_frame); + offset += seg.n_frames; + segment_results[k].relative_b = b[k] - b0; + segment_results[k].dose_rank = static_cast(rank); + text += fmt::format("{}segment {} (B {:+.2f} A^2)", rank == 0 ? "" : ", ", k + 1, b[k] - b0); + } + logger->Info("Dose order of the {} sweep segments, from their relative B against the joint merge: {}", + ns, text); +} + +std::vector RotationScaleMerge::SmoothLogScaleBySegment(const std::vector &y, + const std::vector &J, + std::vector &lambda, bool choose) const { + std::vector x(y.size()); + for (size_t k = 0; k < segments.size(); ++k) { + const auto lo = y.begin() + segments[k].first_frame, hi = lo + segments[k].n_frames; + const auto jlo = J.begin() + segments[k].first_frame; + const std::vector ys(lo, hi), Js(jlo, jlo + segments[k].n_frames); + if (choose) + lambda[k] = ChooseLogScaleSmoothing(ys, Js, std::max(1, rocking_event_frames_at_start), nthreads); + const std::vector xs = SmoothLogScale(ys, Js, lambda[k]); + std::copy(xs.begin(), xs.end(), x.begin() + segments[k].first_frame); + } + return x; +} + // A^2, the bound the Gauss-Seidel solve clamps each batch's relative-B to. It is a numerical guard against // divergence, never an answer: a run of identical values in the solved curve means the solve reached the // bound, not that the crystal did anything. The CORRECTION keeps the tight bound it was tuned with; the @@ -4716,12 +4967,15 @@ void RotationScaleMerge::Combine() { size_t i = 0; while (i < ev.size()) { size_t kk = i + 1; - float last_frame = partials[ev[i]].r->image_number; - while (kk < ev.size()) { - const float frame = partials[ev[kk]].r->image_number; - if (frame - last_frame > max_frame_gap) break; - last_frame = frame; - ++kk; + // A pre-combined full is a whole event already (SweepSegment). + if (segments.empty()) { + float last_frame = partials[ev[i]].r->image_number; + while (kk < ev.size()) { + const float frame = partials[ev[kk]].r->image_number; + if (frame - last_frame > max_frame_gap) break; + last_frame = frame; + ++kk; + } } double pooled_bkg = 0.0; @@ -5862,18 +6116,31 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool // reflections are) - each with the sigma the merge weighted it by. if (export_scaled_fulls && full_stats && !for_search) { // Which fulls, in order, and then each one's record on its own - on all threads. + // The fulls of one sweep of several (uncut) also hand back the reflections the group extinguishes: + // a joint merge searches its own space group, and the screw axes are decided on exactly these. + const gemmi::GroupOps ops = Group().operations(); + auto extinguished = [&](const Obs &o) { + return export_scaled_fulls_uncut && o.group < 0 && frame_cell_ok[o.frame] && o.corr > 0.0f + && std::isfinite(o.corr) && std::isfinite(o.I) && o.d > 0.0f + && ops.is_systematically_absent({{o.h, o.k, o.l}}); + }; const std::vector kept = KeptIndices(n_full, nthreads, [&](int i) { const int g = mf.group[i]; - return g >= 0 && !rejected_obs[i] && std::isfinite(merged_I[g]) - && !(effective_d_min && acc[g].d < *effective_d_min); + if (g < 0) + return extinguished(fulls[i]); + return !rejected_obs[i] && std::isfinite(merged_I[g]) + && (export_scaled_fulls_uncut || !(effective_d_min && acc[g].d < *effective_d_min)); }); result.scaled_fulls.resize(kept.size()); ParallelChunks(static_cast(kept.size()), ThreadsForWork(kept.size(), nthreads), [&](int lo, int hi) { for (int j = lo; j < hi; ++j) { const Obs &o = fulls[kept[j]]; const float I_corr = o.I * o.corr; - result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, corrected_sigma(o, I_corr, o.sigma * o.corr), - o.image_number}; + const float sigma = o.group >= 0 ? corrected_sigma(o, I_corr, o.sigma * o.corr) + : model_sigma(o, I_corr, o.sigma * o.corr); + result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, sigma, + o.image_number, o.corr * o.corr * o.var_bkg, o.corr * o.var_per_I, + o.capture, o.d}; } }); } @@ -6575,9 +6842,14 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool if (effective_d_min && m.d < *effective_d_min) continue; sd_min = std::min(sd_min, m.d); sd_max = std::max(sd_max, m.d); } + // Nothing (or one resolution) merged: no rocking event of the lattice made a full reflection, which + // is what a lattice that does not describe the frames ends in. if (!(sd_min < sd_max && sd_min > 0.0f)) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "RotationScaleMerge: resolution calculation failed"); + fmt::format("nothing to merge ({} merged reflections): no rocking event of " + "the lattice was recorded whole enough to make a full reflection, so " + "the lattice probably does not describe these frames", + result.merged.size())); // The completeness denominator is counted over the range the run DECLARES, not the range that // survived: the shell grid shares its bounds (so nothing is enumerated outside a shell) and runs // to the declared low-resolution limit, or over the whole sphere - 1/d^2 down to 0, spelled as an @@ -6694,6 +6966,12 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st const gemmi::SpaceGroup *group) { const ScopedGpuPin pin(gpu_device); // see IngestHost run_group = group; + // Pre-combined fulls: there are no partials to fit a scale on, and every event is one frame. + if (!segments.empty()) { + scale_partials_choice = false; + rocking_event_frames_at_start = 1; + rocking_events_at_start = n_partials_obs; + } struct ClearGroup { const gemmi::SpaceGroup *&g; ~ClearGroup() { g = nullptr; } } clear_group{run_group}; // Whether a frame's scale can be fitted on its partials depends on whether the partials tell a change // of scale from an error of the partiality model. The count of rocking events per frame is the prior @@ -7044,7 +7322,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool // kernels on it, and the restore re-uploads. // --- 2a. On the de-novo search pass only, drop observations whose Lorentz geometry is poor. --- - if (for_search && search_min_zeta > 0.0) { + if (for_search && search_min_zeta > 0.0 && segments.empty()) { // a full has no rocking geometry int64_t n_dropped = 0; bool zeta_on_gpu = false; #ifdef JFJOCH_USE_CUDA @@ -7354,20 +7632,27 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool // is a correction to the merged INTENSITIES, and a pass whose intensities are thrown away has // nowhere to put one - the fitted gain lives on the fulls, which the next pass rebuilds. --- const bool refine_surfaces = !for_search && full_stats; + // What runs along the frames of ONE sweep - the damage monitor, the sweep ledger, the per-batch B and + // the surfaces, each read in one goniometer and one detector setting - has no meaning across the + // segments of several, and is left out there. The decay is fitted along the dose order instead. + const bool along_frames = segments.empty(); const bool corrections = - refine_surfaces && (refine_decay_b || absorption_iter > 0 || modulation_iter > 0 || relative_b_deg > 0.0); + refine_surfaces && (refine_decay_b || (along_frames && (absorption_iter > 0 || modulation_iter > 0 + || relative_b_deg > 0.0))); + segment_results.assign(segments.size(), Result::Segment{}); + segment_decay_model.clear(); // Radiation-damage monitor: measure the per-batch relative-B on the scaled fulls BEFORE any correction // (report-only; captures the full damage signature, not a residual). Skipped on the P1 search pass. std::unique_ptr sub_mark; auto sub = [&](const char *n) { sub_mark.reset(); sub_mark = std::make_unique(*logger, std::string("rsm/s ") + n); }; sub("damage B"); - if (refine_surfaces) + if (refine_surfaces && along_frames) MeasureRadiationDamageB(n_groups); // Sweep-quality diagnostic, on the per-frame scale the partial scaling just fitted (the flux is // already out of it) and the per-frame CC computed above. It drops nothing itself; the ranges it // segments are the ledger 4c below decides on. sub("sweep quality"); - if (refine_surfaces) + if (refine_surfaces && along_frames) MeasureSweepQuality(partial_scaled, cc, cc_n); // CC1/2 of these same fulls with the surfaces not yet folded in, for the caller's cross-pass quality // guard: the pass this one is judged against fits no surfaces of its own, so it is the uncorrected @@ -7393,7 +7678,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool decay_terms_kept = {}; decay_terms_kept_valid = false; sub("relative B"); - if (refine_surfaces && relative_b_deg > 0.0) + if (refine_surfaces && along_frames && relative_b_deg > 0.0) RefineRelativeB(n_groups); // per-batch relative-B on top of the single decay slope // The three surfaces overlap, so the one fitted first gets the first claim and the others are tested // on what it left - the order decides what is adopted. Detector first: every frame of the sweep @@ -7409,16 +7694,16 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool // the grid is then tested on what the smooth surface left, and where that surface is refused the // grid sees exactly what it saw before. sub("modulation"); - if (refine_surfaces && modulation_iter > 0) + if (refine_surfaces && along_frames && modulation_iter > 0) RefineModulation(modulation_iter, n_groups); sub("absorption time"); - if (refine_surfaces && absorption_iter > 0) + if (refine_surfaces && along_frames && absorption_iter > 0) RefineAbsorptionTime(absorption_iter, n_groups); sub("absorption SH"); - if (refine_surfaces && absorption_iter > 0) + if (refine_surfaces && along_frames && absorption_iter > 0) RefineAbsorptionSH(absorption_iter, n_groups); sub("absorption gonio"); - if (refine_surfaces && absorption_iter > 0) + if (refine_surfaces && along_frames && absorption_iter > 0) RefineAbsorption(absorption_iter, n_groups); sub("disposition"); @@ -7459,6 +7744,17 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool // its CC1/2 is the answer without a second merge. A caller that did not ask keeps the NaN. if (measure_cc_before_corrections) r.cc_half_before_corrections = cc_half_before_corrections.value_or(r.statistics.overall.cc_half); + // Each segment's scale: the median of its frames' scales, at the run's gauge. + for (size_t k = 0; k < segments.size(); ++k) { + std::vector g; + for (int32_t f = segments[k].first_frame; f < segments[k].first_frame + segments[k].n_frames; ++f) + if (fulls_frame_count[f] > 0 && std::isfinite(g_full[f]) && g_full[f] > 0.0) + g.push_back(g_full[f]); + if (!g.empty()) + segment_results[k].scale = median_of(g); + } + r.segments = segment_results; + r.segment_decay_model = segment_decay_model; // The fulls and the merge's working arrays are rebuilt by every arm before they are read, so they go // with it: kept, they sat beside the next arm's combine and through everything the caller did between // merges - on a large cell in P1, the fulls and about 330 bytes per ASU group, gigabytes. Faulting them diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index 6291bdf1e..d1ebd935a 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -139,6 +140,16 @@ public: // The fulls the merge kept, unmerged (see ScaledFull): what passed the outlier tests, inside the // resolution cut. Filled only when SetExportScaledFulls asked for it, and never on a search merge. std::vector scaled_fulls; + // Per sweep segment (SetSweepSegments), in segment order; empty without segments. + struct Segment { + double scale = NAN; // median per-frame scale of the segment's fulls (run gauge) + double relative_b = NAN; // A^2, against the joint merge, the least damaged segment at 0 + int dose_rank = -1; // the segment's place in the dose order those B factors decide + }; + std::vector segments; + // How the decay over the segments was modelled - "dose-linear" (along the dose order), "per-sweep B" + // (one B a segment) or "none" - as the held-out frames decided; empty without segments or decay. + std::string segment_decay_model; }; // experiment: read live (its space group is changed by the caller between Run() calls). @@ -288,6 +299,16 @@ public: // Merge only the first n frames of the outcomes (by default all of them). Set before Ingest(). void SetFrameLimit(int n) { frame_limit = n; } + // The outcomes are several sweeps of pre-combined fulls laid end to end (SweepSegment; by default the + // experiment's ScalingSettings::GetSweepSegments). Set before Ingest(); the segments must cover the + // frames in order. Such an engine runs on the host: its fulls are a few per frame where a sweep's + // partials are millions, and the device pipeline is the partials'. Without segments nothing of this runs. + void SetSweepSegments(std::vector s) { segments = std::move(s); } + + // Whether the fulls handed back (SetExportScaledFulls) are all the merge kept, past its resolution cut + // too: the fulls of one sweep of several, which the joint merge cuts again on its own. + void SetExportScaledFullsUncut(bool on) { export_scaled_fulls_uncut = on; } + // The GPU the engine runs on (device 0 by default). Set before Ingest(). The device only says where // the merge runs: on cards of one model the result is the same on any of them. void SetGpuDevice(int device) { gpu_device = device; } @@ -724,8 +745,18 @@ private: std::vector last_cc_n; std::vector last_frame_scaled; int frame_limit = -1; // see SetFrameLimit + std::vector segments; // see SetSweepSegments + std::vector frame_segment; // each frame's segment (Ingest; empty without segments) + std::vector segment_results; // the last arm's, copied into its Result + // Each frame's place on the dose axis the decay is fitted along: the frames of the segments laid end + // to end in the order the segments' relative B factors put them (MeasureSegmentDose). + std::vector frame_dose; + // SmoothLogScale / ChooseLogScaleSmoothing within each segment on its own; lambda per segment. + std::vector SmoothLogScaleBySegment(const std::vector &y, const std::vector &J, + std::vector &lambda, bool choose) const; int gpu_device = 0; // see SetGpuDevice bool export_scaled_fulls = false; // see SetExportScaledFulls + bool export_scaled_fulls_uncut = false; // see SetExportScaledFullsUncut bool french_wilson = true; // see SetFrenchWilson // --- helpers (each a flat pass; see the .cpp) --- @@ -958,6 +989,12 @@ private: std::vector g_start; }; void BuildDecayTerms(int n_groups, DecayTerms &t) const; + // Per segment: the relative B against the merged reference, and the dose order it puts the segments + // in (segment_results, frame_dose). + void MeasureSegmentDose(const DecayTerms &t, int n_groups); + // Each segment's B against a reference made of the same frames: those of `parity` (< 0: all). + std::vector SegmentB(const DecayTerms &t, int n_groups, int parity) const; + std::string segment_decay_model; // the last decay's choice (Result::segment_decay_model) // The terms the radiation-damage monitor built, handed on to RefineDecay: nothing between the two // touches corr, so they are the terms RefineDecay would build. Empty outside that stretch of RunArm. DecayTerms decay_terms_kept; diff --git a/reader/CMakeLists.txt b/reader/CMakeLists.txt index 20111de43..a8f507003 100644 --- a/reader/CMakeLists.txt +++ b/reader/CMakeLists.txt @@ -6,6 +6,8 @@ ADD_LIBRARY(JFJochReader STATIC JFJochMultiReader.h SweepLayout.cpp SweepLayout.h + ImgCIF.cpp + ImgCIF.h MiniCBF.cpp MiniCBF.h JFJochCBFReader.cpp @@ -29,6 +31,10 @@ ADD_LIBRARY(JFJochReader STATIC JFJochXTCReader.h NXmxModules.cpp NXmxModules.h + SFRM.cpp + SFRM.h + JFJochSFRMReader.cpp + JFJochSFRMReader.h HDF5ImageLocator.cpp JFJochStartMessageReader.cpp JFJochStartMessageReader.h diff --git a/reader/ImgCIF.cpp b/reader/ImgCIF.cpp new file mode 100644 index 000000000..9d1800b36 --- /dev/null +++ b/reader/ImgCIF.cpp @@ -0,0 +1,415 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "ImgCIF.h" + +#include +#include +#include +#include + +#include "../common/JFJochException.h" +#include "../common/DiffractionGeometry.h" +#include "../common/JFJochMath.h" + +namespace imgcif { + +namespace { + +using Vec = std::array; + +JFJochException Invalid(const std::string &what) { + return {JFJochExceptionCategory::InputParameterInvalid, "imgCIF header: " + what}; +} + +std::string Field(const std::map &row, const std::string &name) { + const auto it = row.find(name); + return (it == row.end()) ? std::string() : it->second; +} + +// A CIF number. "?" (unknown) and "." (not applicable) are how a table leaves a cell empty - an angle +// column on a translation row - and read as nothing. +std::optional Number(const std::string &s) { + if (s.empty() || s == "?" || s == ".") + return {}; + try { + size_t used = 0; + const double v = std::stod(s, &used); + // CIF allows a standard uncertainty in brackets after the value: "0.71073(2)" + if (used != s.size() && s[used] != '(') + throw Invalid("malformed number '" + s + "'"); + return v; + } catch (const std::logic_error &) { + throw Invalid("malformed number '" + s + "'"); + } +} + +double NumberOr(const std::string &s, double fallback) { + return Number(s).value_or(fallback); +} + +Vec Rotate(double angle_deg, Vec axis, const Vec &p) { + const double len = std::sqrt(axis[0] * axis[0] + axis[1] * axis[1] + axis[2] * axis[2]); + for (double &c: axis) + c /= len; + const double t = angle_deg * 3.14159265358979323846 / 180.0; + const double c = std::cos(t), s = std::sin(t); + const double dot = axis[0] * p[0] + axis[1] * p[1] + axis[2] * p[2]; + const Vec cross = {axis[1] * p[2] - axis[2] * p[1], + axis[2] * p[0] - axis[0] * p[2], + axis[0] * p[1] - axis[1] * p[0]}; + Vec out{}; + for (int i = 0; i < 3; i++) + out[i] = p[i] * c + cross[i] * s + axis[i] * dot * (1.0 - c); + return out; +} + +const Axis &Find(const Description &d, const std::string &id) { + const auto it = d.axes.find(id); + if (it == d.axes.end()) + throw Invalid("axis '" + id + "' is used but not defined in the _axis table"); + return it->second; +} + +// Carry a point from the frame of axis `id` out to the laboratory: each axis on the way applies its own +// motion at its setting (or at the one given in `settings`), then moves by its offset from the axis it +// depends on. A child's vector is stated in its parent's frame, which is exactly what applying the +// parent after the child does to it. +Vec ToLab(const Description &d, std::string id, Vec p, const std::map &settings) { + std::set seen; + while (id != ".") { + if (!seen.insert(id).second) + throw Invalid("the axis chain through '" + id + "' loops back on itself"); + const Axis &a = Find(d, id); + const auto s = settings.find(id); + const double value = (s != settings.end()) ? s->second : a.setting; + if (a.type == "rotation") + p = Rotate(value, a.vector, p); + else if (a.type == "translation") + for (int i = 0; i < 3; i++) + p[i] += value * a.vector[i]; + else + throw Invalid("axis '" + id + "' is of type '" + a.type + "', which is not supported"); + for (int i = 0; i < 3; i++) + p[i] += a.offset[i]; + id = a.depends_on; + } + return p; +} + +// Where stored pixel `stored` (0-based) sits along its axis. _array_structure_list.direction is not +// consulted: the dictionary says "decreasing" means the stored order runs from the last index to the +// first, but CBFlib - and every program built on it - places a pixel by the axis displacement alone, +// and the files written through CBFlib are only consistent read that way. Bruker's conversions state +// "decreasing" for their rows: read as CBFlib reads them a sweep of them indexes (and DIALS indexes +// it), read by the dictionary the image is mirrored and nothing does. +double Displacement(const ArrayIndex &index, double stored) { + return index.displacement + stored * index.increment; +} + +} // namespace + +std::vector> ParseLoop(const std::string &text, const std::string &tag) { + // The loop_ that introduces the tag, not the tag's own position: everything before it is another + // loop's data. + const size_t tag_at = text.find("\n" + tag); + if (tag_at == std::string::npos) + return {}; + const size_t loop_at = text.rfind("loop_", tag_at); + if (loop_at == std::string::npos) + return {}; + + // Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would + // otherwise be counted as several columns and shift every row after it. + std::vector tokens; + for (size_t i = loop_at + 5; i < text.size();) { + while ((i < text.size()) && std::isspace(static_cast(text[i]))) + i++; + if (i >= text.size()) + break; + size_t end; + if ((text[i] == '\'') || (text[i] == '"')) { + end = text.find(text[i], i + 1); + if (end == std::string::npos) + break; + tokens.push_back(text.substr(i + 1, end - i - 1)); + end++; + } else { + end = i; + while ((end < text.size()) && !std::isspace(static_cast(text[end]))) + end++; + tokens.push_back(text.substr(i, end - i)); + } + // A second loop_, or a tag belonging to another category, ends this one. + if ((tokens.back() == "loop_") + || (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) { + tokens.pop_back(); + break; + } + i = end; + } + + std::vector names; + size_t first_value = 0; + while ((first_value < tokens.size()) && tokens[first_value].starts_with("_")) + names.push_back(tokens[first_value++]); + // The loop found must be the tag's own: a tag stated outside any loop would otherwise be read as + // the rows of whatever loop came before it. + if (std::find(names.begin(), names.end(), tag) == names.end()) + return {}; + + std::vector> rows; + for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) { + std::map row; + for (size_t j = 0; j < names.size(); j++) + row[names[j]] = tokens[i + j]; + rows.push_back(std::move(row)); + } + return rows; +} + +std::optional Item(const std::string &text, const std::string &tag) { + const std::string key = "\n" + tag; + for (size_t at = text.find(key); at != std::string::npos; at = text.find(key, at + 1)) { + size_t i = at + key.size(); + // the whole tag, not a longer one that starts with it + if ((i < text.size()) && !std::isspace(static_cast(text[i]))) + continue; + while ((i < text.size()) && std::isspace(static_cast(text[i]))) + i++; + if (i >= text.size()) + return {}; + if ((text[i] == '\'') || (text[i] == '"')) { + const size_t end = text.find(text[i], i + 1); + if (end == std::string::npos) + return {}; + return text.substr(i + 1, end - i - 1); + } + size_t end = i; + while ((end < text.size()) && !std::isspace(static_cast(text[end]))) + end++; + const std::string value = text.substr(i, end - i); + if (value.starts_with("_") || value == "loop_") + return {}; // the tag heads a loop: not a single item + return value; + } + return {}; +} + +std::optional Parse(const std::string &text) { + const auto axis_rows = ParseLoop(text, "_axis.id"); + auto frame_rows = ParseLoop(text, "_diffrn_scan_frame_axis.axis_id"); + const auto scan_rows = ParseLoop(text, "_diffrn_scan_axis.axis_id"); + if (axis_rows.empty() || (frame_rows.empty() && scan_rows.empty())) + return {}; + + Description d; + for (const auto &row: axis_rows) { + Axis a; + a.type = Field(row, "_axis.type"); + a.equipment = Field(row, "_axis.equipment"); + a.depends_on = Field(row, "_axis.depends_on"); + if (a.depends_on.empty() || a.depends_on == "?") + a.depends_on = "."; + for (int i = 0; i < 3; i++) { + a.vector[i] = NumberOr(Field(row, "_axis.vector[" + std::to_string(i + 1) + "]"), 0.0); + a.offset[i] = NumberOr(Field(row, "_axis.offset[" + std::to_string(i + 1) + "]"), 0.0); + } + d.axes[Field(row, "_axis.id")] = a; + } + + // Where each axis stands: the frame's own setting where the file states one, the scan's start + // otherwise; how far it moves over the frame, from the scan. + for (const auto &row: scan_rows) { + const auto it = d.axes.find(Field(row, "_diffrn_scan_axis.axis_id")); + if (it == d.axes.end()) + continue; + const bool rotation = it->second.type == "rotation"; + it->second.setting = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_start" + : "_diffrn_scan_axis.displacement_start"), 0.0); + it->second.increment = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_increment" + : "_diffrn_scan_axis.displacement_increment"), 0.0); + } + for (const auto &row: frame_rows) { + const auto it = d.axes.find(Field(row, "_diffrn_scan_frame_axis.axis_id")); + if (it == d.axes.end()) + continue; + const bool rotation = it->second.type == "rotation"; + if (const auto v = Number(Field(row, rotation ? "_diffrn_scan_frame_axis.angle" + : "_diffrn_scan_frame_axis.displacement"))) + it->second.setting = *v; + } + + // The two image indices, and the axis each runs along. + std::map> set_rows; + for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id")) + set_rows[Field(row, "_array_structure_list_axis.axis_set_id")] = row; + for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) { + ArrayIndex index; + const std::string set = Field(row, "_array_structure_list.axis_set_id"); + const auto s = set_rows.find(set); + if (s == set_rows.end()) + throw Invalid("image axis set '" + set + "' has no _array_structure_list_axis row"); + index.axis_id = Field(s->second, "_array_structure_list_axis.axis_id"); + index.displacement = NumberOr(Field(s->second, "_array_structure_list_axis.displacement"), 0.0); + index.increment = NumberOr(Field(s->second, "_array_structure_list_axis.displacement_increment"), 0.0); + index.dimension = static_cast(NumberOr(Field(row, "_array_structure_list.dimension"), 0.0)); + const std::string which = Field(row, "_array_structure_list.index"); + if (which == "1") + d.fast = index; + else if (which == "2") + d.slow = index; + } + if (d.fast.axis_id.empty() || d.slow.axis_id.empty()) + throw Invalid("no _array_structure_list rows for the two image indices"); + + d.wavelength_A = NumberOr(Item(text, "_diffrn_radiation_wavelength.wavelength").value_or(""), 0.0); + if (d.wavelength_A == 0.0) { + const auto rows = ParseLoop(text, "_diffrn_radiation_wavelength.id"); + if (!rows.empty()) + d.wavelength_A = NumberOr(Field(rows.front(), "_diffrn_radiation_wavelength.wavelength"), 0.0); + } + d.overload = NumberOr(Item(text, "_array_intensities.overload").value_or(""), 0.0); + d.exposure_s = NumberOr(Item(text, "_diffrn_scan_frame.integration_time").value_or(""), 0.0); + d.detector = Item(text, "_diffrn_detector.type").value_or(Item(text, "_diffrn_detector.detector").value_or("")); + d.ccd = Item(text, "_diffrn_detector.detector").value_or("").find("CCD") != std::string::npos; + if (const auto ratio = Item(text, "_diffrn_radiation_wavelength.polarizn_source_ratio")) + d.polarization_ratio = NumberOr(*ratio, 0.0); + d.polarization_norm_deg = NumberOr(Item(text, "_diffrn_radiation_wavelength.polarizn_source_norm").value_or(""), 0.0); + return d; +} + +std::optional PolarizationFactor(const Description &d) { + if (!d.polarization_ratio) + return {}; + return static_cast(*d.polarization_ratio * std::cos(2.0 * d.polarization_norm_deg * PI / 180.0)); +} + +Geometry Evaluate(const Description &d) { + Geometry g; + + // The image: the index axis that hangs off the other one is where the chain starts. + const Axis &fast = Find(d, d.fast.axis_id); + const Axis &slow = Find(d, d.slow.axis_id); + const bool slow_is_leaf = slow.depends_on == d.fast.axis_id; + if (!slow_is_leaf && fast.depends_on != d.slow.axis_id) + throw Invalid("the two image axes " + d.fast.axis_id + " and " + d.slow.axis_id + + " do not depend on each other; such a detector is not supported"); + const std::string leaf = slow_is_leaf ? d.slow.axis_id : d.fast.axis_id; + const auto pixel = [&](double col, double row) { + return ToLab(d, leaf, {0, 0, 0}, {{d.fast.axis_id, Displacement(d.fast, col)}, + {d.slow.axis_id, Displacement(d.slow, row)}}); + }; + g.origin = pixel(0, 0); + const Vec p10 = pixel(1, 0), p01 = pixel(0, 1); + for (int i = 0; i < 3; i++) { + g.fast[i] = p10[i] - g.origin[i]; + g.slow[i] = p01[i] - g.origin[i]; + } + + // The scan: exactly one axis may move, and it has to be a goniometer rotation. Its direction is + // its own vector carried by every axis it depends on, at their settings - on a kappa head a phi + // scan turns about phi as omega and kappa hold it. The axes it carries do not move it. + std::vector moving; + for (const auto &[id, a]: d.axes) + if (a.increment != 0.0 && id != d.fast.axis_id && id != d.slow.axis_id) + moving.push_back(id); + if (moving.size() > 1) { + std::string names; + for (const auto &m: moving) + names += (names.empty() ? "" : ", ") + m; + throw Invalid("more than one axis moves during the frame (" + names + "); only a rotation " + "about one goniometer axis is supported"); + } + if (moving.empty()) + return g; + + const std::string &id = moving.front(); + const Axis &a = d.axes.at(id); + if (a.type != "rotation" || a.equipment != "goniometer") + throw Invalid("the axis that moves during the frame (" + id + ") is a " + a.equipment + " " + + a.type + "; only a goniometer rotation is supported"); + + // Rotations only: an offset moves where the axis is, not which way it points. + Vec dir = a.vector; + std::set seen; + for (std::string up = a.depends_on; up != "."; up = Find(d, up).depends_on) { + if (!seen.insert(up).second) + throw Invalid("the axis chain through '" + up + "' loops back on itself"); + const Axis &parent = Find(d, up); + if (parent.type == "rotation") + dir = Rotate(parent.setting, parent.vector, dir); + } + const double len = std::sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2]); + if (!(len > 0.0)) + throw Invalid("the scan axis " + id + " has no direction"); + for (double &c: dir) + c /= len; + + g.scan_axis = id; + g.scan_direction = dir; + g.scan_start_deg = a.setting; + g.scan_increment_deg = a.increment; + return g; +} + +Coord ToInternal(const std::array &v) { + return {static_cast(v[0]), static_cast(-v[1]), static_cast(-v[2])}; +} + +Coord GoniometerAxisOf(const std::array &v) { + return -ToInternal(v); +} + +GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg, + const std::array &v) { + const double sign = increment_deg < 0.0 ? -1.0 : 1.0; + return GoniometerAxis(name, static_cast(sign * start_deg), static_cast(sign * increment_deg), + GoniometerAxisOf(v) * static_cast(sign), {}); +} + +DetectorPlane PlaneOf(const Geometry &g, const std::string &path) { + const Coord origin = ToInternal(g.origin); + const Coord fast = ToInternal(g.fast); + const Coord slow = ToInternal(g.slow); + + DetectorPlane p; + p.pixel_mm = fast.Length(); + // One pixel size and a rectangular grid is what the geometry can describe. + if (std::fabs(slow.Length() - p.pixel_mm) > 1e-4f * p.pixel_mm + || std::fabs(fast * slow) > 1e-4f * p.pixel_mm * p.pixel_mm) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + path + ": the imgCIF image axes are not square pixels on a rectangular " + "grid, which is not supported"); + + const Coord f = fast.Normalize(), s = slow.Normalize(); + Coord n = f % s; + // fast x slow points away from the sample on an image stored the internal way round, and towards + // it on a mirrored one. + const bool mirrored = n * origin < 0.0f; + if (mirrored) + n = -n; + + p.distance_mm = n * origin; + const Coord to_poni = n * p.distance_mm - origin; + p.beam_x_pxl = (to_poni * f) / p.pixel_mm; + p.beam_y_pxl = (to_poni * s) / p.pixel_mm; + + // Of the four ways the image can be stored with this handedness, the one that leaves the least + // for the continuous rotation - the detector as it is mounted, with the arm and the tilts on top. + const RotMatrix m(f, s, n); + float best_trace = -10.0f; + for (int64_t q = 0; q < 4; q++) { + const DetectorOrientation o(mirrored, q); + const RotMatrix r = m * o.Matrix().transpose(); + const float trace = r.Column(0).x + r.Column(1).y + r.Column(2).z; + if (trace > best_trace) { + best_trace = trace; + p.orientation = o; + PoniAnglesFromMatrix(r, p.rot1, p.rot2, p.rot3); + } + } + return p; +} + +} // namespace imgcif diff --git a/reader/ImgCIF.h b/reader/ImgCIF.h new file mode 100644 index 000000000..fedce33c8 --- /dev/null +++ b/reader/ImgCIF.h @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "../common/Coord.h" +#include "../common/DetectorOrientation.h" +#include "../common/GoniometerAxis.h" + +// The parts of a full imgCIF header (the form CBFlib writes, and Bruker's APEX writes when it converts +// .sfrm frames) that say where the detector and the goniometer stand. Unlike a PILATUS miniCBF, which +// states its geometry in "# " comment lines, such a file states it only in CIF tables: an axis table +// (_axis), the setting of every axis at this frame (_diffrn_scan_frame_axis) and how it moves per +// frame (_diffrn_scan_axis), and which axes the image's two indices run along +// (_array_structure_list, _array_structure_list_axis). +// +// Every vector here is in the imgCIF laboratory frame (Z from the sample towards the source, Y +// opposite gravity, X completing a right-handed set) and every length in millimetres. +// Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458 +namespace imgcif { + +// One "loop_" of a CIF block, as rows keyed by tag. Tags and values are both read as +// whitespace-separated tokens rather than by line, because a template packs several tags onto one +// line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are laid out to +// match the tags, not the lines. +std::vector> ParseLoop(const std::string &text, const std::string &tag); + +// A tag stated outside a loop ("_diffrn_radiation_wavelength.wavelength 0.71073"), or nothing. +std::optional Item(const std::string &text, const std::string &tag); + +struct Axis { + std::string type; // "rotation" or "translation" + std::string equipment; // "goniometer", "detector", ... + std::string depends_on; // "." for an axis that hangs off nothing + std::array vector{}; + std::array offset{}; + double setting = 0; // this frame's angle (deg) or displacement (mm) + double increment = 0; // its change over one frame +}; + +// One index of the stored image: the axis it runs along, and where pixel 1 and each next one sit on it. +struct ArrayIndex { + std::string axis_id; + int64_t dimension = 0; + double displacement = 0; // of index 1, mm + double increment = 0; // per index, mm +}; + +struct Description { + std::map axes; + ArrayIndex fast; // _array_structure_list.index 1 + ArrayIndex slow; // _array_structure_list.index 2 + double wavelength_A = 0; + double overload = 0; // _array_intensities.overload; 0 where not stated + double exposure_s = 0; + std::string detector; + // _diffrn_detector.detector names a CCD: its pixel values are ADU, not counted photons. + bool ccd = false; + // _diffrn_radiation_wavelength.polarizn_source_ratio, (Ip - In) / (Ip + In), and + // polarizn_source_norm, the angle (deg) from the laboratory Y axis to the normal of the plane Ip + // is polarized in. Nothing where the header states no ratio. + std::optional polarization_ratio; + double polarization_norm_deg = 0; +}; + +// The polarization as DiffractionExperiment::PolarizationFactor takes it - the degree, positive for a +// beam polarized in the horizontal (laboratory X-Z) plane - or nothing where the header states none. +// A plane at an angle psi from the horizontal enters as its projection, the degree times cos(2 psi), +// as the correction knows only those two planes. +std::optional PolarizationFactor(const Description &d); + +// The description, or nothing when the header is not a full imgCIF one - no axis table, or no +// statement of where the axes stand. Throws on a table that is there but cannot be read. +std::optional Parse(const std::string &text); + +struct Geometry { + std::array origin{}; // centre of stored pixel (0, 0) + std::array fast{}; // one stored column further, minus origin + std::array slow{}; // one stored row further, minus origin + std::string scan_axis; // the goniometer axis that turns during the frame + std::array scan_direction{}; // its laboratory direction at this frame's settings + double scan_start_deg = 0; + double scan_increment_deg = 0; +}; + +// Where the image and the rotation axis are at this frame, through the whole axis chain. Throws, naming +// what it found, on anything this cannot describe as one rotation about one axis: two axes moving at +// once, a detector or translation axis moving, or a chain that does not close. +Geometry Evaluate(const Description &d); + +// An imgCIF laboratory direction in the internal frame. imgCIF puts Z from the sample towards the +// source and Y opposite gravity, while the internal frame has z along the beam and y along increasing +// row, so the two differ by a half turn about x - a rotation and not a mirror, so an axis carried +// through it turns the same way by the same angle. +Coord ToInternal(const std::array &v); + +// The goniometer axis as GoniometerAxis takes it, for an imgCIF rotation axis. GoniometerAxis turns +// the crystal by minus the angle about its axis (IndexAndRefine, rot_to_image), the sense an NXmx +// master's vector carries, so a right-handed imgCIF rotation about v is the internal -v. +Coord GoniometerAxisOf(const std::array &v); + +// The sweep as GoniometerAxis takes it, turning the scanned axis v from start_deg by increment_deg per +// frame. A sweep that turns backwards (Bruker frames state a negative increment) is the same rotation +// about -v by the opposite angles, and is stated that way: the analysis takes the oscillation width of a +// frame to be positive, as every other format states it. +GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg, + const std::array &v); + +// The detector plane in the terms the geometry is kept in: the point of normal incidence (PONI) in +// pixels, the distance along the normal, and the rotation that takes the detector frame to the +// laboratory, split into the discrete way the image is stored and the continuous rest. The table +// states the plane through three pixel centres, so this is only a change of description - a 2theta +// arm, a detector's small mounting tilts and a stored image turned or mirrored all come out of the same +// three points. Throws, naming path, on pixels that are not square or a grid that is not rectangular. +struct DetectorPlane { + float pixel_mm = 0; + float distance_mm = 0; + float beam_x_pxl = 0; + float beam_y_pxl = 0; + DetectorOrientation orientation; + float rot1 = 0, rot2 = 0, rot3 = 0; // PONI angles, rad +}; +DetectorPlane PlaneOf(const Geometry &g, const std::string &path); + +} // namespace imgcif diff --git a/reader/JFJochCBFReader.cpp b/reader/JFJochCBFReader.cpp index f3d52e8b6..09c3d5be1 100644 --- a/reader/JFJochCBFReader.cpp +++ b/reader/JFJochCBFReader.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include "../common/Logger.h" #include "../common/JFJochMath.h" #include "SweepLayout.h" +#include "ImgCIF.h" namespace { @@ -178,6 +180,15 @@ Coord RotationAxis(const minicbf::Header &h) { const Coord base = BaseAxis(h); if (!minicbf::ScansPhi(h)) return base; + // A kappa head carries phi on an arm inclined to omega, and where the arm stands moves the phi + // axis by up to twice its inclination. These headers state the kappa angle but not the arm's + // inclination in any form this reader composes, so a phi scan at a non-zero kappa is refused + // rather than read about the wrong axis. + if (h.kappa_deg.has_value() && std::fabs(*h.kappa_deg) > 1e-3) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "a phi scan at kappa " + std::to_string(*h.kappa_deg) + + " deg: the miniCBF header does not describe the kappa arm, so the " + "rotation axis cannot be composed; this is not supported"); const auto rad = [](double deg) { return static_cast(deg * PI / 180.0); }; // Where the table states the inner axis outright, that IS the inclination, and the driven-circle @@ -231,8 +242,24 @@ void JFJochCBFReader::ReadFiles(const std::string &path) { dataset_ = std::make_shared(); dataset_->experiment = default_experiment; + // A full imgCIF header states its geometry through the axis chain, which is worked out once here + // and per frame below; a miniCBF states it in its "# " lines. + std::optional geometry0; + std::optional plane0; + if (header0_.imgcif.has_value()) { + geometry0 = imgcif::Evaluate(*header0_.imgcif); + if (geometry0->scan_axis.empty()) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + files_[0] + ": no goniometer axis turns during the frame; a full " + "imgCIF series of stills is not supported"); + plane0 = imgcif::PlaneOf(*geometry0, files_[0]); + } + DetectorSetup detector = DetDECTRIS(header0_.nx, header0_.ny, header0_.detector, {}); - detector.PixelSize_um(static_cast(std::lround(header0_.pixel_x_m * 1e6))); + if (plane0.has_value()) + detector.PixelSize_um(plane0->pixel_mm * 1000.0f); // a CCD's pixel is not whole micrometres + else + detector.PixelSize_um(static_cast(std::lround(header0_.pixel_x_m * 1e6))); detector.SensorThickness_um(static_cast(std::lround(header0_.thickness_m * 1e6))); detector.SensorMaterial(header0_.material); // Only when the header states one. An absent line reads as zero, and a limit of zero would mark @@ -257,16 +284,25 @@ void JFJochCBFReader::ReadFiles(const std::string &path) { // Images are handed out as signed 32-bit whatever the file stored, so that is the depth the rest // of the code must see; the real overflow is the header's Count_cutoff, set above. detector.BitDepthImage(32); - if (const auto orientation = ImageOrientation(header0_)) + if (plane0.has_value()) + detector.ImageOrientation(plane0->orientation); + else if (const auto orientation = ImageOrientation(header0_)) detector.ImageOrientation(orientation.value()); detector.MinFrameTime(std::chrono::microseconds(0)); detector.MinCountTime(std::chrono::microseconds(0)); detector.ReadOutTime(std::chrono::nanoseconds(0)); dataset_->experiment.Detector(detector); - dataset_->experiment.BeamX_pxl(static_cast(header0_.beam_x_px)); - dataset_->experiment.BeamY_pxl(static_cast(header0_.beam_y_px)); - dataset_->experiment.DetectorDistance_mm(static_cast(header0_.distance_m * 1000.0)); + if (plane0.has_value()) { + dataset_->experiment.BeamX_pxl(plane0->beam_x_pxl); + dataset_->experiment.BeamY_pxl(plane0->beam_y_pxl); + dataset_->experiment.DetectorDistance_mm(plane0->distance_mm); + dataset_->experiment.PoniRot1_rad(plane0->rot1).PoniRot2_rad(plane0->rot2).PoniRot3_rad(plane0->rot3); + } else { + dataset_->experiment.BeamX_pxl(static_cast(header0_.beam_x_px)); + dataset_->experiment.BeamY_pxl(static_cast(header0_.beam_y_px)); + dataset_->experiment.DetectorDistance_mm(static_cast(header0_.distance_m * 1000.0)); + } // A detector swung out on a 2theta arm, which small-molecule collection uses routinely. The arm // turns the detector about the sample, so it carries the square-on geometry with it: the header's @@ -290,6 +326,10 @@ void JFJochCBFReader::ReadFiles(const std::string &path) { // correction takes to contain the lab x axis; the degree it wants is 2f - 1. if (header0_.polarization_fraction.has_value()) dataset_->experiment.PolarizationFactor(2.0f * static_cast(*header0_.polarization_fraction) - 1.0f); + // A full imgCIF header states the degree itself, and the plane. + if (header0_.imgcif.has_value()) + if (const auto p = imgcif::PolarizationFactor(*header0_.imgcif)) + dataset_->experiment.PolarizationFactor(*p); dataset_->experiment.FrameTime( std::chrono::duration_cast( std::chrono::duration(header0_.period_s)), @@ -300,21 +340,46 @@ void JFJochCBFReader::ReadFiles(const std::string &path) { // Parsing one costs a few hundred microseconds - it is two dozen regular expressions - so on a // sweep of several thousand frames this is seconds of startup before a single image is read, and // the files are independent. + // + // A full imgCIF frame states every axis setting, so each one is checked to belong to the same + // sweep as the first: the same axis turning, about the same direction. A series in which the + // fixed axes moved between frames - kappa or omega reset for another run - is several sweeps, + // and taking it for one would put every frame after the move at the wrong orientation. std::vector frames(files_.size()); sweep::ForEachInOrder(files_.size(), [&](size_t i) { const auto h = minicbf::ReadHeader(files_[i]); - frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m, - h.beam_x_px, h.beam_y_px, h.wavelength_A}; + if (!geometry0.has_value()) { + frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m, + h.beam_x_px, h.beam_y_px, h.wavelength_A}; + return; + } + if (!h.imgcif.has_value()) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + files_[i] + " is not a full imgCIF frame like " + files_[0]); + const auto g = imgcif::Evaluate(*h.imgcif); + const Coord axis = ImgCIFToInternal(g.scan_direction); + if (g.scan_axis != geometry0->scan_axis + || (axis - ImgCIFToInternal(geometry0->scan_direction)).Length() > 1e-4f) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + files_[i] + " turns " + (g.scan_axis.empty() ? "no axis" : g.scan_axis) + + " about a different direction from " + files_[0] + + ": the goniometer was reset between them, so the series is more " + "than one sweep. Name a frame of one sweep only."); + const auto plane = imgcif::PlaneOf(g, files_[i]); + frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3, + plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A}; }); // The sweep the headers describe, which is not always the files laid out end to end: a deposited // series can be missing frames, and those are gaps in the rotation rather than images to close up. const auto layout = sweep::Place(frames, "CBFReader"); files_ = layout.files; - dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name, - static_cast(layout.start_deg), - static_cast(layout.increment_deg), - RotationAxis(header0_), {})); + if (geometry0.has_value()) + dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0->scan_axis, layout.start_deg, layout.increment_deg, + geometry0->scan_direction)); + else + dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name, static_cast(layout.start_deg), + static_cast(layout.increment_deg), RotationAxis(header0_), {})); dataset_->error_value = -1; dataset_->experiment.ImagesPerTrigger(static_cast(files_.size())); @@ -395,6 +460,10 @@ bool JFJochCBFReader::HasImage(int64_t image_number) const { && !files_[image_number].empty(); } +bool JFJochCBFReader::CountsInADU() const { + return header0_.imgcif.has_value() && header0_.imgcif->ccd; +} + bool JFJochCBFReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) { if (!HasImage(image_number)) return false; diff --git a/reader/JFJochCBFReader.h b/reader/JFJochCBFReader.h index 1ee73ab89..ff20b7cc9 100644 --- a/reader/JFJochCBFReader.h +++ b/reader/JFJochCBFReader.h @@ -56,6 +56,10 @@ public: // ReadRawImage hands such a slot back as "nothing to read". [[nodiscard]] bool HasImage(int64_t image_number) const; + // Whether the pixel values are ADU rather than counted photons: a full imgCIF header that names a + // CCD, which is what Bruker's conversion of its frames states. A PILATUS miniCBF counts photons. + [[nodiscard]] bool CountsInADU() const; + bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override; [[nodiscard]] std::vector ReadSpots(int64_t image) const override; }; diff --git a/reader/JFJochSFRMReader.cpp b/reader/JFJochSFRMReader.cpp new file mode 100644 index 000000000..c1dbd0cae --- /dev/null +++ b/reader/JFJochSFRMReader.cpp @@ -0,0 +1,231 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochSFRMReader.h" + +#include + +#include "../common/JFJochException.h" +#include "../common/JFJochMath.h" +#include "../common/Logger.h" +#include "ImgCIF.h" +#include "SweepLayout.h" + +namespace { + + +// The instrument a .sfrm header describes, as an imgCIF axis chain, so that one piece of code turns +// either description into the geometry. The chain is the one APEX writes when it converts frames to +// imgCIF (the pixels come out identical but for the baseline, so the image axes are the same), with +// one thing settled from data rather than taken from it: omega turns in the sense of the 2theta arm. +// An omega scan of APEX's conversion states the opposite increment to the frames it came from, and a +// SMART 6000 omega sweep indexes only that way round. Phi and chi are as the conversion composes them +// from a kappa setting. CENTER is placed as the conversion places it, each coordinate counted from the +// far edge of the stored image: on an APEX II phi sweep read both ways it indexes 97.5% of the frames +// against 90.3% and merges at ISa 17.5 against 2.1, where the image's own .cbf conversion gives 17. +// +// The goniometer is stated as Eulerian angles whatever the head is - a kappa instrument's frames carry +// the omega, chi and phi its kappa setting is equivalent to - so the chain is omega, carrying chi about +// the beam, carrying phi. On the 2theta arm the detector sits DISTANC (plus DETTYPE's face-to-grid +// offset) from the sample, tilted by the DETPAR pitch, roll and yaw. The stored rows run along the +// spindle axis and the stored columns along the direction the arm swings. +imgcif::Description Chain(const sfrm::Header &h, const std::string &path) { + if (h.center.size() < 2) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, path + " states no CENTER"); + if (!(h.pixels_per_cm_512 > 0.0)) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + path + " states no pixel size (DETTYPE)"); + if (h.detpar[0] != 0.0 || h.detpar[1] != 0.0 || h.detpar[2] != 0.0) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + path + " states non-zero DETPAR offsets, whose meaning this reader does not know"); + // DETTYPE gives pixels per centimetre on a 512-pixel frame; a binned or unbinned frame scales it. + const double pixel_mm = 10.0 / h.pixels_per_cm_512 * 512.0 / static_cast(h.ncols); + const double distance_mm = (h.distance_cm + h.grid_offset_cm) * 10.0; + const double half_cols = static_cast(h.ncols) / 2.0 * pixel_mm; + const double half_rows = static_cast(h.nrows) / 2.0 * pixel_mm; + + imgcif::Description d; + d.axes["OMEGA"] = {"rotation", "goniometer", ".", {-1, 0, 0}, {}, h.angles[1], 0}; + d.axes["CHI"] = {"rotation", "goniometer", "OMEGA", {0, 0, 1}, {}, h.angles[3], 0}; + d.axes["PHI"] = {"rotation", "goniometer", "CHI", {1, 0, 0}, {}, h.angles[2], 0}; + d.axes["TWOTHETA"] = {"rotation", "detector", ".", {1, 0, 0}, {}, -h.angles[0], 0}; + d.axes["DX"] = {"translation", "detector", "TWOTHETA", {0, 0, -1}, {}, distance_mm, 0}; + d.axes["YAW"] = {"rotation", "detector", "DX", {-1, 0, 0}, {}, h.detpar[5], 0}; + d.axes["PITCH"] = {"rotation", "detector", "YAW", {0, -1, 0}, {}, h.detpar[3], 0}; + d.axes["ROLL"] = {"rotation", "detector", "PITCH", {0, 0, -1}, {}, h.detpar[4], 0}; + d.axes["H"] = {"translation", "detector", "ROLL", {0, 1, 0}, {}, (h.center[0] * pixel_mm - half_cols), 0}; + d.axes["V"] = {"translation", "detector", "H", {1, 0, 0}, {}, (h.center[1] * pixel_mm - half_rows), 0}; + d.axes["ELEMENT_X"] = {"translation", "detector", "V", {0, 1, 0}, {-half_rows, -half_cols, 0}, 0, 0}; + d.axes["ELEMENT_Y"] = {"translation", "detector", "ELEMENT_X", {1, 0, 0}, {}, 0, 0}; + d.fast = {"ELEMENT_X", h.ncols, pixel_mm / 2.0, pixel_mm}; + d.slow = {"ELEMENT_Y", h.nrows, pixel_mm / 2.0, pixel_mm}; + + static const char *AXIS_NAMES[] = {"", "TWOTHETA", "OMEGA", "PHI", "CHI"}; + if (h.axis < 1 || h.axis > 4) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + path + " scans AXIS " + std::to_string(h.axis) + + ", which is not a rotation this reader supports"); + d.axes[AXIS_NAMES[h.axis]].increment = h.increment_deg; + d.wavelength_A = h.wavelength_A; + // The polarization APEX's conversion states for the same frame: 1 - cos^2 of the monochromator's + // scattering angle (FILTER2), in the horizontal plane. A tube with mirrors and no monochromator + // states 0, unpolarized - these are laboratory frames, and the synchrotron default does not apply. + const double cos_mono = std::cos(h.monochromator_2theta_deg * PI / 180.0); + d.polarization_ratio = 1.0 - cos_mono * cos_mono; + d.polarization_norm_deg = 0; + d.overload = h.saturation; + d.exposure_s = h.exposure_s; + d.detector = h.detector; + return d; +} + +} // namespace + +bool JFJochSFRMReader::CanRead(const std::string &path) { + return sfrm::CanRead(path); +} + +void JFJochSFRMReader::ReadFiles(const std::string &path) { + files_ = sfrm::CollectSweep(path); + if (files_.empty()) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "No SFRM images found for " + path); + + header0_ = sfrm::ReadHeader(files_[0]); + if (!(header0_.wavelength_A > 0.0)) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + files_[0] + " states no wavelength in its SFRM header"); + const auto geometry0 = imgcif::Evaluate(Chain(header0_, files_[0])); + const auto plane0 = imgcif::PlaneOf(geometry0, files_[0]); + + dataset_ = std::make_shared(); + dataset_->experiment = default_experiment; + + DetectorSetup detector = DetDECTRIS(header0_.ncols, header0_.nrows, header0_.detector, {}); + detector.PixelSize_um(plane0.pixel_mm * 1000.0f); + // A CCD has no sensor depth to correct for: the phosphor converts at the surface. + detector.SensorThickness_um(0); + // CCDPARM[4] is the count at which the CCD saturates; the stored container (with its overflow + // tables) holds far more, so it is the only limit there is. + if (header0_.saturation > 0.0) + detector.SaturationLimit(static_cast(header0_.saturation)); + else + Logger("SFRMReader").Warning("{} states no saturation (CCDPARM), so no pixel will be called " + "saturated.", files_[0]); + detector.BitDepthImage(32); + detector.ImageOrientation(plane0.orientation); + detector.MinFrameTime(std::chrono::microseconds(0)); + detector.MinCountTime(std::chrono::microseconds(0)); + detector.ReadOutTime(std::chrono::nanoseconds(0)); + dataset_->experiment.Detector(detector); + + dataset_->experiment.BeamX_pxl(plane0.beam_x_pxl); + dataset_->experiment.BeamY_pxl(plane0.beam_y_pxl); + dataset_->experiment.DetectorDistance_mm(plane0.distance_mm); + dataset_->experiment.PoniRot1_rad(plane0.rot1).PoniRot2_rad(plane0.rot2).PoniRot3_rad(plane0.rot3); + dataset_->experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / static_cast(header0_.wavelength_A)); + dataset_->experiment.PolarizationFactor(imgcif::PolarizationFactor(Chain(header0_, files_[0]))); + if (header0_.exposure_s > 0.0) + dataset_->experiment.FrameTime( + std::chrono::duration_cast(std::chrono::duration(header0_.exposure_s)), + std::chrono::duration_cast(std::chrono::duration(header0_.exposure_s))); + + // Where every image sits on the spindle, from its own header - and the check that every frame is + // of the same sweep: the same axis turning about the same direction. A run whose fixed angles moved + // part-way is more than one sweep, and reading it as one would put every frame after the move at + // the wrong orientation. + std::vector frames(files_.size()); + sweep::ForEachInOrder(files_.size(), [&](size_t i) { + const auto h = sfrm::ReadHeader(files_[i]); + const auto g = imgcif::Evaluate(Chain(h, files_[i])); + if (g.scan_axis != geometry0.scan_axis + || (imgcif::ToInternal(g.scan_direction) - imgcif::ToInternal(geometry0.scan_direction)).Length() > 1e-4f) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + files_[i] + " turns " + g.scan_axis + " about a different direction from " + + files_[0] + ": the goniometer was reset between them, so the run is " + "more than one sweep"); + const auto plane = imgcif::PlaneOf(g, files_[i]); + frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3, + plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A}; + }); + + const auto layout = sweep::Place(frames, "SFRMReader"); + files_ = layout.files; + dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0.scan_axis, layout.start_deg, layout.increment_deg, + geometry0.scan_direction)); + + dataset_->error_value = -1; + dataset_->experiment.ImagesPerTrigger(static_cast(files_.size())); + + // Outside the phosphor's active area a frame reads below its baseline offset - the stored value + // is zero there in every frame - so frame 0's negative counts are the pixels no frame can be + // trusted at, exactly as a PILATUS marks its gaps. + std::vector first; + sfrm::Read(files_[0], first); + std::vector mask(first.size(), 0); + for (size_t i = 0; i < first.size(); i++) + if (first[i] < 0) + mask[i] = 1; + dataset_->pixel_mask = std::make_shared(mask); + + SetStartMessage(dataset_); +} + +uint64_t JFJochSFRMReader::GetNumberOfImages() const { + return files_.size(); +} + +void JFJochSFRMReader::Close() { + files_.clear(); + dataset_.reset(); +} + +template +CompressedImage JFJochSFRMReader::DecodeInto(int64_t image_number, Buffer &buffer, + std::vector &scratch) const { + if (!HasImage(image_number)) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "No image at this point of the sweep"); + + const size_t npixel = static_cast(header0_.nrows) * static_cast(header0_.ncols); + buffer.resize(npixel * sizeof(int32_t)); + const auto h = sfrm::ReadInto(files_[image_number], reinterpret_cast(buffer.data()), npixel, + scratch); + if (h.nrows != header0_.nrows || h.ncols != header0_.ncols) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "SFRM image size differs from the first image of the sweep"); + + return CompressedImage(buffer.data(), buffer.size(), + static_cast(header0_.ncols), static_cast(header0_.nrows), + CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION); +} + +bool JFJochSFRMReader::LoadImage_i(std::shared_ptr &dataset, + DataMessage &message, + std::vector &buffer, + int64_t image_number, + bool update_dataset) { + (void) update_dataset; + if (!dataset || !HasImage(image_number)) + return false; + std::vector scratch; + message.image = DecodeInto(image_number, buffer, scratch); + message.number = image_number; + return true; +} + +bool JFJochSFRMReader::HasImage(int64_t image_number) const { + return image_number >= 0 && static_cast(image_number) < files_.size() + && !files_[image_number].empty(); +} + +bool JFJochSFRMReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) { + if (!HasImage(image_number)) + return false; + image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer); + return true; +} + +std::vector JFJochSFRMReader::ReadSpots(int64_t) const { + return {}; // a raw frame stores no analysis results +} diff --git a/reader/JFJochSFRMReader.h b/reader/JFJochSFRMReader.h new file mode 100644 index 000000000..0eb040373 --- /dev/null +++ b/reader/JFJochSFRMReader.h @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +#include "JFJochReader.h" +#include "SFRM.h" + +// Reads a rotation sweep straight from a directory of Bruker .sfrm frames - what SMART, PROTEUM +// and APEX write - with no conversion step. +// +// The same shape as the other one-file-per-image readers: the sweep's geometry comes from the first +// frame's header, the rotation angle of each image from its own, and images are decoded on demand. +// A Bruker collection directory holds one run per scan, each its own sweep; naming a frame reads its +// run. A raw frame carries no analysis results, so no spots and no reflections. +class JFJochSFRMReader : public JFJochReader { + std::vector files_; + std::shared_ptr dataset_; + sfrm::Header header0_; + + bool LoadImage_i(std::shared_ptr &dataset, + DataMessage &message, + std::vector &buffer, + int64_t image_number, + bool update_dataset) override; + + // Decodes one image into the caller's byte buffer and returns the image that points at it. The + // file is read through scratch, which the caller keeps between frames. + template + CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector &scratch) const; + +public: + ~JFJochSFRMReader() override = default; + + // True if the path names something this reader can open: a .sfrm frame, or a directory holding + // at least one. Cheap - it reads a few bytes. + static bool CanRead(const std::string &path); + + // path is a directory of .sfrm frames, or one frame inside the run to take the whole run from. + void ReadFiles(const std::string &path); + + [[nodiscard]] uint64_t GetNumberOfImages() const override; + void Close() override; + + + // Whether the sweep really has an image at this point. A deposited series can be missing frames, + // and they are kept as gaps so that every image keeps the spindle angle its own header states; + // ReadRawImage hands such a slot back as "nothing to read". + [[nodiscard]] bool HasImage(int64_t image_number) const; + + bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override; + [[nodiscard]] std::vector ReadSpots(int64_t image) const override; +}; diff --git a/reader/MiniCBF.cpp b/reader/MiniCBF.cpp index 955976f6d..84204cbec 100644 --- a/reader/MiniCBF.cpp +++ b/reader/MiniCBF.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -91,67 +92,6 @@ std::optional Angle(const std::string &text, const char *pattern) { return v; } -// One "loop_" of the imgCIF template block these headers carry, as rows keyed by tag. Tags and values -// are both read as whitespace-separated tokens rather than by line, because a template packs several -// tags onto one line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are -// laid out to match the tags, not the lines. -std::vector> ParseLoop(const std::string &text, const std::string &tag) { - // The loop_ that introduces the tag, not the tag's own position: everything before it is another - // loop's data. - const size_t tag_at = text.find("\n" + tag); - if (tag_at == std::string::npos) - return {}; - const size_t loop_at = text.rfind("loop_", tag_at); - if (loop_at == std::string::npos) - return {}; - - // Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would - // otherwise be counted as several columns and shift every row after it. - std::vector tokens; - for (size_t i = loop_at + 5; i < text.size();) { - while ((i < text.size()) && std::isspace(static_cast(text[i]))) - i++; - if (i >= text.size()) - break; - size_t end; - if ((text[i] == '\'') || (text[i] == '"')) { - end = text.find(text[i], i + 1); - if (end == std::string::npos) - break; - tokens.push_back(text.substr(i + 1, end - i - 1)); - end++; - } else { - end = i; - while ((end < text.size()) && !std::isspace(static_cast(text[end]))) - end++; - tokens.push_back(text.substr(i, end - i)); - } - // A second loop_, or a tag belonging to another category, ends this one. - if ((tokens.back() == "loop_") - || (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) { - tokens.pop_back(); - break; - } - i = end; - } - - std::vector names; - size_t first_value = 0; - while ((first_value < tokens.size()) && tokens[first_value].starts_with("_")) - names.push_back(tokens[first_value++]); - if (names.empty()) - return {}; - - std::vector> rows; - for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) { - std::map row; - for (size_t j = 0; j < names.size(); j++) - row[names[j]] = tokens[i + j]; - rows.push_back(std::move(row)); - } - return rows; -} - std::string Field(const std::map &row, const std::string &name) { const auto it = row.find(name); return (it == row.end()) ? std::string() : it->second; @@ -166,7 +106,7 @@ std::string Field(const std::map &row, const std::stri // unswung detector frame - which is where the image orientation belongs, with the arm applied on top. // Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458 void ParseAxisTable(const std::string &text, Header &h) { - const auto axes = ParseLoop(text, "_axis.id"); + const auto axes = imgcif::ParseLoop(text, "_axis.id"); if (axes.empty()) return; @@ -199,11 +139,11 @@ void ParseAxisTable(const std::string &text, Header &h) { // Which axis the fast index runs along, and which the slow, through the two tables that say so. std::map axis_of_set; - for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id")) + for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list_axis.axis_set_id")) axis_of_set[Field(row, "_array_structure_list_axis.axis_set_id")] = Field(row, "_array_structure_list_axis.axis_id"); - for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) { + for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list.array_id")) { const std::string set = Field(row, "_array_structure_list.axis_set_id"); const auto id = axis_of_set.contains(set) ? axis_of_set[set] : set; const auto it = vector_of.find(id); @@ -261,6 +201,7 @@ Header ParseHeader(const char *data, size_t size) { // The whitespace after each name is what keeps "Chi_increment" out of "Chi". h.chi_deg = Angle(t, R"(#\s*Chi\s+([\d.eE+-]+))"); h.omega_deg = Angle(t, R"(#\s*Omega\s+([\d.eE+-]+))"); + h.kappa_deg = Angle(t, R"(#\s*Kappa\s+([\d.eE+-]+))"); h.chi_increment_deg = Angle(t, R"(#\s*Chi_increment\s+([\d.eE+-]+))").value_or(0.0); h.phi_increment_deg = Angle(t, R"(#\s*Phi_increment\s+([\d.eE+-]+))").value_or(0.0); h.omega_increment_deg = Angle(t, R"(#\s*Omega_increment\s+([\d.eE+-]+))").value_or(0.0); @@ -292,6 +233,19 @@ Header ParseHeader(const char *data, size_t size) { const auto conv = Match(t, R"RE(conversions\s*=\s*"([^"]+)")RE").value_or(""); h.byte_offset = conv.find("x-CBF_BYTE_OFFSET") != std::string::npos; + if (!Match(t, R"(#\s*(Pixel_size))").has_value()) { // Match returns the first group + h.imgcif = imgcif::Parse(t); + if (h.imgcif.has_value()) { + h.detector = h.imgcif->detector; + h.pixel_x_m = std::abs(h.imgcif->fast.increment) * 1e-3; + h.pixel_y_m = std::abs(h.imgcif->slow.increment) * 1e-3; + h.wavelength_A = h.imgcif->wavelength_A; + h.count_cutoff = static_cast(h.imgcif->overload); + h.exposure_s = h.imgcif->exposure_s; + h.thickness_m = 0; + } + } + if (h.period_s <= 0.0) h.period_s = h.exposure_s; diff --git a/reader/MiniCBF.h b/reader/MiniCBF.h index 3b82eaafb..48d9c0bde 100644 --- a/reader/MiniCBF.h +++ b/reader/MiniCBF.h @@ -10,6 +10,7 @@ #include #include "../common/CompressedImage.h" +#include "ImgCIF.h" // PILATUS miniCBF: an ASCII header, four separator bytes, then one byte-offset compressed image. // No CIF parser and no libcbf are needed - every value is on a "# " comment line or a MIME line. @@ -39,6 +40,7 @@ struct Header { // does not really carry is absent here rather than 9999 degrees away from zero. std::optional chi_deg; std::optional omega_deg; + std::optional kappa_deg; double chi_increment_deg = 0; double phi_increment_deg = 0; double omega_increment_deg = 0; @@ -70,6 +72,13 @@ struct Header { // ScansPhi, and note that the one header carrying both a name and a table says "X" where its table // says Y) but this token is not: where both are present they agree. bool spindle_along_slow = false; + + // A full imgCIF header - CBFlib's own form, which Bruker's APEX writes when it converts frames - + // states none of the "# " lines and says everything in its CIF tables instead. Where the header + // has no "# Pixel_size" line and those tables are there, this holds them, and the scalar fields + // above that the tables state (pixel size, wavelength, Count_cutoff, exposure, detector) are + // filled in from them. The geometry is the reader's to work out from the axis chain. + std::optional imgcif; }; // Whether the sweep turns PHI rather than the base (omega) axis. What moved is the axis with a diff --git a/reader/SFRM.cpp b/reader/SFRM.cpp new file mode 100644 index 000000000..f8da7bf4d --- /dev/null +++ b/reader/SFRM.cpp @@ -0,0 +1,338 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "SFRM.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../common/JFJochException.h" + +namespace sfrm { + +namespace { + +constexpr size_t BLOCK = 512; +constexpr size_t LINE = 80; +constexpr size_t KEY = 8; // "NROWS :" - seven characters of name and the colon + +JFJochException Invalid(const std::string &what) { + return {JFJochExceptionCategory::InputParameterInvalid, what}; +} + +std::vector ReadPrefix(const std::string &path, size_t bytes) { + std::ifstream f(path, std::ios::binary); + if (!f) + throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path); + std::vector out(bytes); + f.read(reinterpret_cast(out.data()), static_cast(bytes)); + out.resize(static_cast(f.gcount())); + return out; +} + +std::vector Words(const std::map &kv, const std::string &key) { + std::vector out; + const auto it = kv.find(key); + if (it == kv.end()) + return out; + std::istringstream in(it->second); + for (std::string w; in >> w;) + out.push_back(w); + return out; +} + +double Num(const std::map &kv, const std::string &key, size_t i, + double fallback = 0.0) { + const auto w = Words(kv, key); + if (i >= w.size()) + return fallback; + try { + return std::stod(w[i]); + } catch (const std::logic_error &) { + throw Invalid("Malformed number in SFRM header " + key + ": '" + w[i] + "'"); + } +} + +int64_t Int(const std::map &kv, const std::string &key, size_t i, + int64_t fallback = 0) { + return static_cast(Num(kv, key, i, static_cast(fallback))); +} + +// The 80-character lines of the header, as key -> value. A key that repeats (TITLE, CELL, MATRIX, +// DETPAR, ...) continues its value on the next line, so the pieces are joined. +std::map Lines(const char *data, size_t size) { + std::map kv; + for (size_t at = 0; at + LINE <= size; at += LINE) { + if (data[at + KEY - 1] != ':') + continue; // the padding after the last line + std::string key(data + at, KEY - 1); + key.erase(std::find_if(key.rbegin(), key.rend(), [](unsigned char c) { return !std::isspace(c); }).base(), + key.end()); + const std::string value(data + at + KEY, LINE - KEY); + auto &v = kv[key]; + v += (v.empty() ? "" : " ") + value; + } + return kv; +} + +bool HasExtension(const std::filesystem::path &p) { + std::string ext = p.extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); }); + return ext == ".sfrm"; +} + +bool IsSFRM(const std::string &path) { + const auto head = ReadPrefix(path, KEY); + return head.size() == KEY && std::memcmp(head.data(), "FORMAT :", KEY) == 0; +} + +// "ins_zwin_01_0042.sfrm" -> {"ins_zwin_01_", 4}: the run is part of the prefix, so naming a frame +// of run 01 reads run 01 and not the frames of every run in the directory. +struct Template { + std::string prefix; + size_t digits = 0; + + bool Matches(const std::string &stem) const { + if (stem.size() != prefix.size() + digits || stem.compare(0, prefix.size(), prefix) != 0) + return false; + return std::all_of(stem.begin() + static_cast(prefix.size()), stem.end(), + [](unsigned char c) { return std::isdigit(c); }); + } +}; + +std::optional