Merge multisweep-p1d into rc181: Bruker readers (SFRM, imgCIF sweep axis, polarisation), several sweeps of one crystal merged jointly (--sweeps), per-sweep MTZ batches and the SWEEPS report section
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQqN9qRD57vnShzTC37
This commit is contained in:
commit
03cdf2357b
42 files changed
+4069
-266
No files matched your search
@@ -76,6 +76,13 @@ struct ScaledFull {
|
||||
float I = NAN;
|
||||
float sigma = NAN;
|
||||
float image_number = NAN; // the frame of the event's peak partial, which tells two events of one index apart
|
||||
// The full's counting variance as the merge rebuilds it at an intensity I, before its error model:
|
||||
// var_bkg + var_per_I * I + (capture * I)^2, all at the intensity's scale (what a later merge rebuilds
|
||||
// the full from). NaN where not exported.
|
||||
float var_bkg = NAN;
|
||||
float var_per_I = NAN;
|
||||
float capture = NAN;
|
||||
float d = NAN; // resolution, A
|
||||
};
|
||||
|
||||
struct MergedReflection {
|
||||
|
||||
@@ -315,3 +315,12 @@ ScalingSettings& ScalingSettings::ReportResolutionRange_A(std::optional<ReportRe
|
||||
std::optional<ReportResolutionRange> ScalingSettings::GetReportResolutionRange_A() const {
|
||||
return report_resolution_range;
|
||||
}
|
||||
|
||||
ScalingSettings& ScalingSettings::SweepSegments(std::vector<SweepSegment> input) {
|
||||
sweep_segments = std::move(input);
|
||||
return *this;
|
||||
}
|
||||
|
||||
const std::vector<SweepSegment> &ScalingSettings::GetSweepSegments() const {
|
||||
return sweep_segments;
|
||||
}
|
||||
@@ -3,7 +3,9 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
#include "JFJochException.h"
|
||||
|
||||
// How the high-resolution cutoff for the written reflections and the reported shell table is chosen
|
||||
@@ -17,8 +19,20 @@ struct ReportResolutionRange {
|
||||
double d_max = 0.0;
|
||||
};
|
||||
|
||||
// One sweep of several whose frames are laid end to end in one rotation merge (the joint merge of several
|
||||
// sweeps): frames [first_frame, first_frame + n_frames). Every observation of a segment's frames is a full
|
||||
// already combined in its own sweep (partiality 1, filed under the frame of its peak), so the merge
|
||||
// combines nothing again, recomputes nothing from the rocking geometry, and lets nothing that runs along
|
||||
// the frames - the per-frame scale's smoothness above all - reach across from one sweep into the next
|
||||
// (RotationScaleMerge).
|
||||
struct SweepSegment {
|
||||
int32_t first_frame = 0;
|
||||
int32_t n_frames = 0;
|
||||
};
|
||||
|
||||
class ScalingSettings {
|
||||
bool refine_wedge = false;
|
||||
std::vector<SweepSegment> sweep_segments;
|
||||
|
||||
bool merge_friedel = true;
|
||||
std::optional<double> high_resolution_limit_A;
|
||||
@@ -150,6 +164,8 @@ public:
|
||||
ScalingSettings& ResolutionCCTarget(double input);
|
||||
ScalingSettings& ReportShellCount(int input);
|
||||
ScalingSettings& ReportResolutionRange_A(std::optional<ReportResolutionRange> input);
|
||||
// The sweeps a rotation merge's frames are made of; empty (the default): one sweep of partials.
|
||||
ScalingSettings& SweepSegments(std::vector<SweepSegment> input);
|
||||
|
||||
[[nodiscard]] bool GetRefineWedge() const;
|
||||
|
||||
@@ -189,4 +205,5 @@ public:
|
||||
[[nodiscard]] double GetResolutionCCTarget() const;
|
||||
[[nodiscard]] int GetReportShellCount() const;
|
||||
[[nodiscard]] std::optional<ReportResolutionRange> GetReportResolutionRange_A() const;
|
||||
[[nodiscard]] const std::vector<SweepSegment> &GetSweepSegments() const;
|
||||
};
|
||||
@@ -384,6 +384,15 @@ J. Beilsten-Edmands, G. Winter, R. Gildea et al., "Scaling diffraction data in t
|
||||
package: algorithms and new approaches for multi-crystal scaling" (2020), Acta Cryst. D76, 385-399
|
||||
[doi:10.1107/S2059798320003198](https://doi.org/10.1107/S2059798320003198).
|
||||
|
||||
**Several sweeps merged as one** - each sweep's partials combined into fulls on their own, and the
|
||||
fulls of all the sweeps then scaled together, every sweep with its own frame scales and a relative B
|
||||
against the joint merge, is how XSCALE merges several data sets (W. Kabsch, "Integration, scaling,
|
||||
space-group assignment and post-refinement" (2010), above). Reading the radiation dose a sweep was taken
|
||||
at from its relative B, which orders the sweeps by dose from the data rather than from the order they
|
||||
were given in, follows K. Diederichs, "Some aspects of quantitative analysis and correction of radiation
|
||||
damage" (2006), Acta Cryst. D62, 96-101
|
||||
[doi:10.1107/S0907444905031537](https://doi.org/10.1107/S0907444905031537).
|
||||
|
||||
**Uncertainty conventions** follow the IUCr Commission on Crystallographic Nomenclature:
|
||||
D. Schwarzenbach, S. C. Abrahams, H. D. Flack et al., "Statistical descriptors in crystallography:
|
||||
Report of the IUCr Subcommittee on Statistical Descriptors" (1989), Acta Cryst. A45, 63-75
|
||||
|
||||
+36
-15
@@ -4,7 +4,7 @@ Jungfraujoch is developed at the Swiss Light Source, but a data-reduction pipeli
|
||||
ever sees its own detectors is not tested. The datasets below were collected by other people,
|
||||
on detectors and in file formats we do not produce ourselves, and are used here to check that
|
||||
`rugnux` reads foreign files correctly and reduces them to sensible results. Most were collected
|
||||
at other facilities, eleven on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
|
||||
at other facilities, thirteen on laboratory X-ray sources; a few come from SLS beamlines, where the data are still written by someone
|
||||
else's detector and someone else's acquisition system. Their authors published all of these for
|
||||
exactly this kind of reuse, and this page is where we credit them.
|
||||
|
||||
@@ -101,6 +101,8 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
|
||||
| [6NEN](https://www.rcsb.org/structure/6NEN) | UQ eSpace [10.14264/uql.2018.843](https://doi.org/10.14264/uql.2018.843) | Australian Synchrotron MX2 | 2.15 | P 3 1 2 | 105.5 105.5 35.1 90.0 90.0 120.0 | SMV, S/N 928 | Catalytic domain of Proteus mirabilis ScsC |
|
||||
| [6O2H](https://www.rcsb.org/structure/6O2H) | SBGrid [10.15785/sbgrid/747](https://doi.org/10.15785/sbgrid/747) | CHESS F1 | 1.21 | P 1 | 27.4 32.1 34.5 88.7 108.5 111.9 | PILATUS3 6M | Hen lysozyme in triclinic space group at ambient temperature - diffuse scattering dataset |
|
||||
| [6OEL](https://www.rcsb.org/structure/6OEL) | SBGrid [10.15785/sbgrid/652](https://doi.org/10.15785/sbgrid/652) | ALS 8.2.1 | 3.10 | F 41 3 2 | 328.1 328.1 328.1 90.0 90.0 90.0 | SMV, S/N 905 | Engineered Fab bound to IL-4 receptor |
|
||||
| [6OQZ](https://www.rcsb.org/structure/6OQZ) | IRRMC [10.18430/m3.irrmc.5324](https://doi.org/10.18430/m3.irrmc.5324) | Home source, Bruker rotating anode | 1.60 | I 2 2 2 | 92.9 97.9 102.7 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of glucose isomerase from non-merohedrally twinned crystals |
|
||||
| [6OR0](https://www.rcsb.org/structure/6OR0) | IRRMC [10.18430/m3.irrmc.5325](https://doi.org/10.18430/m3.irrmc.5325) | Home source, Bruker rotating anode | 1.55 | I 21 3 | 78.0 78.0 78.0 90.0 90.0 90.0 | Bruker SMART 6000 | Crystal structure of insulin from non-merohedrally twinned crystals |
|
||||
| [6OWW](https://www.rcsb.org/structure/6OWW) | Zenodo [10.5281/zenodo.2943248](https://doi.org/10.5281/zenodo.2943248) | ALS 8.3.1 | 3.84 | P 1 21 1 | 85.8 86.0 214.3 90.0 89.9 90.0 | PILATUS3 6M | Crystal structure of a Human Cardiac Calsequestrin Filament Complexed with Ytterbium |
|
||||
| [6P8J](https://www.rcsb.org/structure/6P8J) | SBGrid [10.15785/sbgrid/671](https://doi.org/10.15785/sbgrid/671) | APS 24-ID-E | 1.47 | P 1 21 1 | 46.5 152.2 88.3 90.0 90.1 90.0 | Dectris Eiger 16M | Structure of P. aeruginosa ATCC27853 CdnD D62N/D64N mutant bound to ATP |
|
||||
| [6P8P](https://www.rcsb.org/structure/6P8P) | SBGrid [10.15785/sbgrid/673](https://doi.org/10.15785/sbgrid/673) | APS 24-ID-C | 1.64 | P 4 | 97.5 97.5 60.1 90.0 90.0 90.0 | PILATUS 6M-F | Structure of P. aeruginosa ATCC27853 HORMA1 |
|
||||
@@ -244,7 +246,7 @@ This page lists the rotation datasets. The serial (XFEL stills) datasets of the
|
||||
| [9ZM0](https://www.rcsb.org/structure/9ZM0) | IRRMC [10.18430/M39ZM0](https://doi.org/10.18430/M39ZM0) | NSLS-II 17-ID-1 | 2.10 | P 1 21 1 | 50.4 30.1 91.2 90.0 97.1 90.0 | Dectris EIGER1 Si 9M | Crystal structure of monomeric Atg23 |
|
||||
| [9ZMU](https://www.rcsb.org/structure/9ZMU) | IRRMC [10.18430/M39ZMU](https://doi.org/10.18430/M39ZMU) | NSLS-II 19-ID | 1.98 | P 65 2 2 | 47.8 47.8 492.6 90.0 90.0 120.0 | Dectris EIGER2 Si 9M | Crystal structure of an Iole protein from Brucella melitensis (hexagonal P form) |
|
||||
|
||||
The table holds 201 PDB entries. Six more datasets, small-molecule / chemical-crystallography
|
||||
The table holds 203 PDB entries. Eight more datasets, small-molecule / chemical-crystallography
|
||||
sets kept because they exercise short wavelengths, CdTe sensors, fine slicing and non-zero
|
||||
detector 2θ, have no PDB entry and are listed [below](#datasets-with-no-pdb-entry).
|
||||
|
||||
@@ -313,7 +315,7 @@ the rest were deleted, so a run over the data directory sees a single collection
|
||||
| 9CRW | a dose pair on one crystal 37 min apart - 0.025 s at 289 mm, 0.010 s at 276 mm | the 0.025 s sweep, whose 2.5 Å target matches the deposited 2.49 Å |
|
||||
| 7RIS | two crystals at two wavelengths - 1.53494 Å (Ho derivative) and 1.03329 Å (the deposited native) | **both** |
|
||||
|
||||
**Thirteen further archives hold more than one collection.** Their layout was read from the
|
||||
**Seventeen further archives hold more than one collection.** Their layout was read from the
|
||||
image files and repository listings; one sweep is kept for a run over the data directory unless
|
||||
noted.
|
||||
|
||||
@@ -329,6 +331,10 @@ noted.
|
||||
| lalanine | four runs of the RODIN L-alanine deposition at 2θ = 20° | the 900-frame `pgw240050_01` run |
|
||||
| 9E2T | one continuous sweep plus screening images | the 2700-frame sweep |
|
||||
| 8OWM | three MXRDR zips covering one 1800-frame sweep, plus a processed-data zip | the three sweep zips (proc zip skipped) |
|
||||
| 6OR0 | four 900 × 0.2° Bruker `.sfrm` runs of one twinned crystal at 10 cm: ω scans at 2θ = 0° (`01`) and 35° (`02`), φ scans on the fixed-χ (54.74°) platform at 2θ = 0° (`03`) and 35° (`04`) | the `01` run (2θ = 0°): the header distance of these SMART 6000 frames is ~4% short of what the data give, which the distance refinement corrects on a 2θ = 0° run and not yet on a swung one |
|
||||
| 6OQZ | eleven `.sfrm` ω runs of 472-900 × 0.2° at 18 cm, 2θ = 10°, 25° and 45° (low, medium, high resolution) at different φ, plus two 5-frame runs at 2θ = 0° and the orientation-matrix runs `gdri_MATRIX_*` | the low-resolution `01` run (2θ = 10°), for the reason given for 6OR0; three crystals, the test set for multi-lattice integration |
|
||||
| laspartic | seven Bruker-converted full-imgCIF kappa runs, φ (`01`, 1200 × 0.3°) and ω (`02`-`07`, 292-394 frames) at 2θ = -26.7° | the 1200-frame `01` φ run |
|
||||
| onitroaniline | the same seven-run kappa layout (`01` 1200 × 0.3° φ, `02`-`07` ω), deposited twice: as Bruker `.sfrm` and as APEX's full-imgCIF conversion of the same frames | the `01` run, read from the `.sfrm` files |
|
||||
| 7D98 | one 250 × 0.5° miniCBF sweep (frames 11-260), plus four HDF5 masters that repeat frames 61-260 in 60-frame pieces | the miniCBF sweep; the HDF5 files stay beside it and the row is pinned |
|
||||
| 6H2P | two sweeps of one crystal, 1800 × 0.1° at 0.918 Å and 3600 × 0.1° at 1.893 Å (the Mn K edge) | **both**, as two battery rows, `6h2p_native` and `6h2p_1p89A`; the long-wavelength row has no deposited resolution to compare with |
|
||||
| 8C3E | four 0.5° ω scans of one crystal at different (φ, χ) settings, 56, 150, 150 and 50 frames (`lcb2_8_newt_001` to `_004`) | the 150-frame `_003` scan |
|
||||
@@ -349,7 +355,7 @@ noted.
|
||||
|
||||
## Datasets published as Raw Data Letters
|
||||
|
||||
Three of the datasets - 6R72, 8QQ7 and 6RLR - were published as IUCrData Raw Data Letters, a
|
||||
Five of the datasets - 6R72, 8QQ7, 6RLR, laspartic and onitroaniline - were published as IUCrData Raw Data Letters, a
|
||||
format whose purpose is to make raw images citable and re-processable in their own right. The
|
||||
letters describe the collections and the difficulties in them, and are the reference for what the
|
||||
data are:
|
||||
@@ -364,6 +370,15 @@ data are:
|
||||
IUCrData 7, x220852
|
||||
[doi:10.1107/S2414314622008525](https://doi.org/10.1107/S2414314622008525) - covers 6RLR.
|
||||
|
||||
- M. Lutz and L. Kroon-Batenburg, "Accurate intensity integration in the twinned γ-form of
|
||||
*o*-nitroaniline" (2022), IUCrData 7
|
||||
[doi:10.1107/S2414314622010598](https://doi.org/10.1107/S2414314622010598) - covers
|
||||
onitroaniline.
|
||||
- M. Lutz, "Twinned L-aspartic acid" (2025), IUCrData 10
|
||||
[doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) - covers laspartic:
|
||||
a twofold rotation about [100] with twin index 2 and obliquity 0.74°, whose coincidence lattice
|
||||
is the doubled cell first reported for the compound (Bernal, 1931).
|
||||
|
||||
The authors of the second letter also published their own reciprocal-space reconstruction of the
|
||||
6RLR data as a separate Zenodo record,
|
||||
[10.5281/zenodo.6961763](https://doi.org/10.5281/zenodo.6961763).
|
||||
@@ -437,8 +452,8 @@ The PDB entry names the detector as a Rigaku HyPix-6000HE.
|
||||
|
||||
## Deposited models and structure factors
|
||||
|
||||
All 201 PDB entries in the table are released, and RCSB reports released structure factors
|
||||
(`status_code_sf = REL`) for every one of them; the six small-molecule datasets have none. A merged
|
||||
All 203 PDB entries in the table are released, and RCSB reports released structure factors
|
||||
(`status_code_sf = REL`) for every one of them; the eight small-molecule datasets have none. A merged
|
||||
result from this pipeline can therefore be checked against the deposited model or against the
|
||||
deposited intensities.
|
||||
|
||||
@@ -609,8 +624,10 @@ reader that globs `*.img` will pick them up, so they are named here rather than
|
||||
| `nidppe` | Zenodo record 10.5281/zenodo.20041091 | a small-molecule dataset, not a PDB deposition |
|
||||
| `cytidine` | Zenodo record 10.5281/zenodo.33555 | a small-molecule dataset, not a PDB deposition |
|
||||
| `lalanine` | Zenodo record 10.5281/zenodo.11946282 | a small-molecule dataset, not a PDB deposition |
|
||||
| `laspartic` | Zenodo record 10.5281/zenodo.15432050 | a small-molecule dataset, not a PDB deposition |
|
||||
| `onitroaniline` | Zenodo record 10.5281/zenodo.7193538 | a small-molecule dataset, not a PDB deposition |
|
||||
|
||||
Five of the six small-molecule sets have a published structure to check a run against. These are
|
||||
Six of the eight small-molecule sets have a published structure to check a run against. These are
|
||||
reference values from the literature, not results obtained here.
|
||||
|
||||
| Dataset | Space group | Cell (A, deg) | T | Reference |
|
||||
@@ -620,6 +637,10 @@ reference values from the literature, not results obtained here.
|
||||
| `nidppe` | `P 1 21/c 1` (14) | 11.2779 13.3386 15.8739 / 90 98.7953 90 | 150 K | COD 2012031 - Acta Cryst. C**57** (2001) 690-693 [doi:10.1107/S0108270101003961](https://doi.org/10.1107/S0108270101003961) |
|
||||
| `cytidine` | `P 21 21 21` (19) | 13.98 14.788 5.119 / 90 90 90 | 296 K | β-cytidine; COD 2001311 - D. L. Ward, Acta Cryst. C**49** (1993) 1789-1792 [doi:10.1107/S0108270193003464](https://doi.org/10.1107/S0108270193003464) |
|
||||
| `lalanine` | `P 21 21 21` (19) | 5.791 5.944 12.269 / 90 90 90 | 100 K | COD 2311261 - S. Parsons, H. D. Flack, T. Wagner, Acta Cryst. B**69** (2013) 249-259 [doi:10.1107/S2052519213010014](https://doi.org/10.1107/S2052519213010014) |
|
||||
| `laspartic` | `P 1 21 1` (4) | 5.1237 6.9197 7.6006 / 90 100.442 90 | 150 K | twin fractions 0.537/0.463; COD 1577465 - M. Lutz, IUCrData **10** (2025) [doi:10.1107/S241431462500879X](https://doi.org/10.1107/S241431462500879X) |
|
||||
|
||||
`onitroaniline` has no cell here: the letter describing it could not be retrieved when the row was
|
||||
added, so a run on it is not scored against a reference.
|
||||
|
||||
`cuhf2` has no confirmed cell. Its space group is published as `P 4/n m m` (Phys. Rev. B **81**,
|
||||
064422 (2010) [doi:10.1103/PhysRevB.81.064422](https://doi.org/10.1103/PhysRevB.81.064422)) but no
|
||||
@@ -632,20 +653,20 @@ symmetries rather than to be easy to process. The counts below describe where it
|
||||
like everything else on this page, they are metadata about the depositions and their files, not
|
||||
measurements.
|
||||
|
||||
- **Repository**, over the 207 datasets on this page (the 201 rows of the table and the six
|
||||
small-molecule sets): IRRMC 95, Zenodo 42, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
|
||||
- **Repository**, over the 211 datasets on this page (the 203 rows of the table and the eight
|
||||
small-molecule sets): IRRMC 97, Zenodo 44, SBGrid 40, MXRDR 16, Keele University 5, XRDa 5,
|
||||
ESRF 3, UQ eSpace 1.
|
||||
- **Facility** - counted from the facility part of the Facility / beamline column, the beamline
|
||||
ignored so that entries deposited with and without one count the same, over the 190 rows that
|
||||
name one: APS 29, Diamond 21, ESRF 19, NSLS-II 14, PETRA III 13, BESSY 12, SSRL 11, ALS 9,
|
||||
SLS 8, SPring-8 8, SOLEIL 7, SSRF 7, PAL/PLS 5, CHESS 4, ALBA 3, Australian Synchrotron 3,
|
||||
CLSI 3, ELETTRA 2, LNLS 2, MAX IV 2, NSLS 2, Photon Factory 2, and one each from NSRRC,
|
||||
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other eleven
|
||||
rows were collected on laboratory sources: nine on rotating anodes, one on a liquid-metal jet
|
||||
RRCAT Indus-2, SRS Daresbury and EMBL/DESY Hamburg (DORIS) - 26 facilities. The other thirteen
|
||||
rows were collected on laboratory sources: eleven on rotating anodes, one on a liquid-metal jet
|
||||
and one on a sealed tube.
|
||||
- **Crystal system, from the deposited space group of the 201 rows:** monoclinic 53,
|
||||
orthorhombic 50, tetragonal 30, trigonal 22, hexagonal 20, cubic 13, triclinic 13.
|
||||
- **Pink beam:** none of these datasets was collected with pink beam. All 201 rows are deposited
|
||||
- **Crystal system, from the deposited space group of the 203 rows:** monoclinic 53,
|
||||
orthorhombic 51, tetragonal 30, trigonal 22, hexagonal 20, cubic 14, triclinic 13.
|
||||
- **Pink beam:** none of these datasets was collected with pink beam. All 203 rows are deposited
|
||||
as `SINGLE WAVELENGTH` (`_diffrn_radiation.pdbx_diffrn_protocol`), and all but 5REO, which
|
||||
leaves the field blank, as monochromatic (`pdbx_monochromatic_or_laue_m_l` `M`); 9Q41 is the
|
||||
one row recorded with a multilayer rather than a crystal monochromator (CHESS Rh/B4C). The
|
||||
@@ -654,7 +675,7 @@ measurements.
|
||||
- **Long cell axes:** sixteen rows have a deposited cell axis longer than 320 Å - 4P8R,
|
||||
4MTK, 7D98, 8V4O, 9ZMU, 9V16, 9Z72, 4NWV, 9YL4, 5NW5, 6QAJ, 7QIJ, 8T7R, 9H0Q, 6G1F and 6OEL.
|
||||
|
||||
The marCCD, SMV and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
|
||||
The marCCD, SMV, Bruker `.sfrm`, full-imgCIF and gzip-compressed miniCBF datasets are the reason rugnux reads those formats
|
||||
natively, and accepts the `.img` and numeric-suffix (`.001`) file names they arrive with.
|
||||
|
||||
## Licences
|
||||
|
||||
+30
-6
@@ -30,9 +30,10 @@ rugnux -o myrun dataset_master.h5 # NXmx / EIGER / Jungfraujoch mas
|
||||
rugnux -o myrun sweep_00001.cbf # CBF, marCCD or SMV sweep: name one frame
|
||||
rugnux -o myrun dataset_master.h5 model.cif # with a model (or --model model.cif)
|
||||
rugnux -o myrun x_master.h5 http://host:5232/ # follow a broker's collection while it is written
|
||||
rugnux -o myrun a_master.h5 b_master.h5 # several sweeps of one crystal, merged as one
|
||||
```
|
||||
|
||||
Chosen when the file's goniometer axis turns. One sweep per run. A model is placed (rigid body, or
|
||||
Chosen when the file's goniometer axis turns. Several sweeps can be given (below, Inputs). A model is placed (rigid body, or
|
||||
molecular replacement), refined and validated after merging; it does not change the processing
|
||||
decisions ([Models and reference data](RUGNUX_MODEL_AND_REFERENCE.md)).
|
||||
|
||||
@@ -91,14 +92,36 @@ spot finding or integration ([Re-scaling and re-merging](RUGNUX_ADVANCED.md#re-s
|
||||
## Inputs
|
||||
|
||||
Inputs are files, never a directory, read in the order given (a shell glob gives them sorted by
|
||||
name). Several inputs are read together only as one stills dataset in `--mode mx`: all the files of
|
||||
name). Several inputs are one stills dataset in `--mode mx`, or several rotation sweeps (below): all the files of
|
||||
one dataset - `rugnux run*.h5` (SACLA), `*.cxi`, `*.xtc` (LCLS) - or several Jungfraujoch/NXmx
|
||||
masters of stills, read as one run: their images one after another, merged together, with the
|
||||
indexing rate reported per master as well. Such masters must share the detector, wavelength,
|
||||
distance and beam centre. Several inputs must be one format; a rotation sweep (name one frame of a
|
||||
CBF/marCCD/SMV sweep), any `--mode` other than `mx`, and following a broker take one. What each
|
||||
format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read without a
|
||||
geometry file where their own metadata places the detector
|
||||
distance and beam centre. Several inputs must be one format; any `--mode` other than `mx`, and
|
||||
following a broker, take one.
|
||||
|
||||
Several rotation sweeps are told apart from the files: one sweep per CBF/marCCD/SMV/SFRM file-name
|
||||
template (the name with its frame number taken out - any frame of a sweep names the whole sweep, so
|
||||
`rugnux a_*.cbf` is the one sweep `a`), one per rotation master. Only the inputs given are processed;
|
||||
other sweeps beside them are named in the log. A mix of rotation and stills masters is refused, and
|
||||
so are sweeps at different wavelengths, `-s`/`-e` and `--force-still` with several sweeps.
|
||||
By default (`--sweeps joint`) the sweeps are one dataset, merged in two levels as XSCALE merges the
|
||||
data sets XDS made of each sweep: each sweep is processed on its own first - indexed, integrated, its
|
||||
partials combined and scaled, its space group determined - writing only its log `<prefix>_s<k>_log.txt`;
|
||||
the sweeps are then put on the axes of the one with the most reflections (the change of basis, an
|
||||
axis permutation or a twin law of the lattice, under which their merged intensities correlate best;
|
||||
a sweep correlating below 0.5 on every axes is left out and named in the log), and their fulls are
|
||||
scaled and merged together: a per-frame scale within each sweep, the sweeps' dose order read from
|
||||
their relative B factors (not from the input order or the header times), a decay along that order or
|
||||
one B per sweep, whichever predicts held-out frames better, then the space-group search, the
|
||||
resolution cut and everything after it on the joint data, into `<prefix>.*`. The detector-dependent
|
||||
corrections (absorption and detector surfaces) are each sweep's own. The unmerged MTZ holds the joint
|
||||
merge's scaled observations in the batches of their own sweeps (sweep `s` of the merge: batches
|
||||
`s * base + frame + 1`, `base` the power of ten past the longest sweep), under the batch headers that
|
||||
sweep's own run wrote; the report names each sweep in section `1b. SWEEPS`. `--sweeps separate` processes each sweep alone instead, as if it were the only
|
||||
input, into `<prefix>_s<k>_*` (k = 1, 2, ... in the order given).
|
||||
|
||||
What each format needs is in [What Rugnux reads](RUGNUX_FORMATS.md); serial XFEL stills are read
|
||||
without a geometry file where their own metadata places the detector
|
||||
([Serial XFEL inputs](RUGNUX_FORMATS.md#serial-xfel-inputs)).
|
||||
|
||||
An atomic model and a reference reflection file can be given among the inputs, in any order,
|
||||
@@ -130,6 +153,7 @@ With the input already on disk, the URL is ignored.
|
||||
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
|
||||
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
|
||||
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
|
||||
| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `<prefix>.*`; `separate` processes each alone into `<prefix>_s<k>_*` |
|
||||
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
|
||||
|
||||
### Mode
|
||||
|
||||
+26
-2
@@ -120,7 +120,8 @@ The header block above section 1 records **how the result was produced**: `RUGNU
|
||||
Rates, per-image costs and progress remain on stdout only.
|
||||
|
||||
Sections, in order: the `SUMMARY` ([below](#the-summary-and-what-the-run-decided)), then
|
||||
`1. DATA SET AND GEOMETRY`, `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
|
||||
`1. DATA SET AND GEOMETRY`, on a joint merge of several sweeps `1b. SWEEPS`
|
||||
([below](#several-sweeps-merged-as-one)), `2. CRYSTAL`, `3. MERGED DATA`, `4. DIAGNOSTICS` and, only where a model
|
||||
was given (`--model`, `--mr`, or a model file among the inputs), `5. MODEL VALIDATION` - which also
|
||||
holds the keys of [molecular replacement](#molecular-replacement) and of the
|
||||
[refinement](#refinement-refine_) where they ran. A `--mode refine` report has one section,
|
||||
@@ -252,7 +253,7 @@ and a run that produced a textbook data set read identically for their first thr
|
||||
`GONIO_SCALE`, `SPINDLE_CAP`, `ANISOTROPY`, `TWINNING`, `PSEUDO_TRANSLATION`,
|
||||
`LATTICE_TRANSLATION`, `MODEL_HAND`, `MODEL_NOT_VALIDATED`, `MODEL_CELL_MISMATCH`, `MODEL_FREE_SET`, `MODEL_SPACE_GROUP`, `REFERENCE_MISMATCH`, `CANCELLED`, `RESOLUTION_FIT`,
|
||||
`FLIGHT_PATH`, `GEOMETRY_NOT_CONVERGED`, `SCALING_NOT_CONVERGED`, `HARMONIC_CONTAMINATION`,
|
||||
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`. `NONE` when nothing fired. A code appears if and only if its
|
||||
`SUPERCELL_POSSIBLE`, `MULTIPLE_LATTICES`, `LATTICE_EXPLAINS_FEW_SPOTS`, `ICE_RINGS`, `POWDER_RINGS`, `BEAM_CENTRE_OFF`, `OVERLOADS`, `OVERLAPS`, `SWEEP_LEFT_OUT`. `NONE` when nothing fired. A code appears if and only if its
|
||||
warning fired, so the flags and the `WARNING:` lines are two renderings of one list. Thresholds are
|
||||
deliberately low: a warning is a prompt to check something, not a verdict, and some of them fire on
|
||||
data that turn out fine — the closed
|
||||
@@ -286,6 +287,29 @@ and a run that produced a textbook data set read identically for their first thr
|
||||
Then the numbered sections: **1. DATA SET AND GEOMETRY**, **2. CRYSTAL**,
|
||||
**3. MERGED DATA**, **4. DIAGNOSTICS**, and **5. MODEL VALIDATION** only where a model was given.
|
||||
|
||||
## Several sweeps merged as one
|
||||
|
||||
A run of several sweeps merged as one (`--sweeps joint`, the default with several rotation inputs, see
|
||||
[Command-line options](RUGNUX_CLI.md)) describes the joint merge everywhere, and adds section
|
||||
**1b. SWEEPS**; a run of one input has no such section and no such keys.
|
||||
|
||||
- **`SWEEP_COUNT=`** the sweeps given, **`SWEEP_REFERENCE=`** the one (1-based) the others were put on
|
||||
the axes of - the one with the most reflections - and **`SWEEP_DECAY_MODEL=`** how the decay over the
|
||||
sweeps was modelled: `dose-linear` (a B rising along the dose order), `per-sweep B` (one B a sweep) or
|
||||
`none`, whichever predicted held-out frames better.
|
||||
- Per sweep `k`: **`SWEEP_k_INPUT=`**, **`SWEEP_k_IMAGES=`**, **`SWEEP_k_INCLUDED=`**; what the sweep
|
||||
gave processed on its own - **`SWEEP_k_SPACE_GROUP_ALONE=`**, **`SWEEP_k_CELL=`**,
|
||||
**`SWEEP_k_CC_HALF_ALONE=`**, **`SWEEP_k_COMPLETENESS_ALONE=`**, **`SWEEP_k_RESOLUTION_ALONE=`**; and,
|
||||
for a sweep in the joint merge, **`SWEEP_k_REINDEX_OP=`** (onto the reference's axes) with
|
||||
**`SWEEP_k_REINDEX_CC=`** (the merged-intensity correlation under it), **`SWEEP_k_BATCH_FIRST=`** /
|
||||
**`SWEEP_k_BATCH_LAST=`** (its batches in the unmerged MTZ: sweep `s` of the merge is
|
||||
`s * base + frame + 1`, `base` the power of ten past the longest sweep), **`SWEEP_k_SCALE=`** (the
|
||||
median scale of its frames in the joint merge), **`SWEEP_k_RELATIVE_B=`** (its B against the joint
|
||||
merge, the least damaged sweep at 0) and **`SWEEP_k_DOSE_RANK=`** (its place, from 1, in the dose
|
||||
order those B factors decide - not the order the sweeps were given in, nor their header times).
|
||||
- A sweep given but not merged - it gave no merge, or its intensities correlate with the reference
|
||||
sweep's on no axes - is named by a `SWEEP_LEFT_OUT` warning.
|
||||
|
||||
## The reference-range table
|
||||
|
||||
`--report-resolution <dmin>[,<dmax>]` adds a second block of merging statistics to section 3, the
|
||||
|
||||
@@ -365,6 +365,7 @@ RotationScaleMerge::RotationScaleMerge(const DiffractionExperiment &experiment,
|
||||
// Decay + absorption correction surfaces are one master toggle (on by default; both cross-validated,
|
||||
// so a no-op when their systematic is absent).
|
||||
refine_decay_b = s.GetCorrectionSurfaces();
|
||||
segments = s.GetSweepSegments();
|
||||
absorption_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
|
||||
modulation_iter = s.GetCorrectionSurfaces() ? s.GetAbsorptionIter() : 0;
|
||||
relative_b_deg = s.GetRelativeBDegrees(); // per-batch relative-B batch width (0 = off)
|
||||
@@ -531,11 +532,28 @@ void RotationScaleMerge::IngestHost() {
|
||||
if (frame_limit >= 0) n_frames = std::min(n_frames, frame_limit);
|
||||
partials_released = false;
|
||||
resident_ingest = false;
|
||||
frame_segment.clear();
|
||||
if (!segments.empty()) {
|
||||
frame_segment.assign(n_frames, -1);
|
||||
int32_t next = 0;
|
||||
for (size_t k = 0; k < segments.size(); ++k) {
|
||||
const SweepSegment &seg = segments[k];
|
||||
if (seg.first_frame != next || seg.n_frames <= 0 || seg.first_frame + seg.n_frames > n_frames)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"RotationScaleMerge: the sweep segments do not cover the frames in order");
|
||||
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
|
||||
frame_segment[f] = static_cast<int32_t>(k);
|
||||
next = seg.first_frame + seg.n_frames;
|
||||
}
|
||||
if (next != n_frames)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"RotationScaleMerge: the sweep segments do not cover the frames in order");
|
||||
}
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
// Probed before the observations are built: whether the build lays down the full Obs array or
|
||||
// only the ingest arrays depends on whether the device pipeline will be active.
|
||||
gpu_ = std::make_unique<RotationScaleMergeGPU>(gpu_device);
|
||||
gpu_active_ = gpu_->Available() && !scale_on_cpu_;
|
||||
gpu_active_ = gpu_->Available() && !scale_on_cpu_ && segments.empty();
|
||||
resident_ingest = gpu_active_ && !observation_dump;
|
||||
#endif
|
||||
|
||||
@@ -605,7 +623,12 @@ void RotationScaleMerge::IngestHost() {
|
||||
}
|
||||
});
|
||||
|
||||
MeasureIncidentFlux(mean_bkg);
|
||||
// Fulls of several sweeps carry each sweep's own background: not one flux meter, so the per-frame
|
||||
// scale alone takes the flux.
|
||||
if (segments.empty())
|
||||
MeasureIncidentFlux(mean_bkg);
|
||||
else
|
||||
frame_flux.assign(n_frames, 1.0);
|
||||
|
||||
// The occupied range of h, which is the buckets the sort works on.
|
||||
int b_lo = H_ALL, b_hi = -1;
|
||||
@@ -728,12 +751,19 @@ void RotationScaleMerge::IngestHost() {
|
||||
}
|
||||
});
|
||||
}
|
||||
BuildInRangeObservations(keys, static_cast<int>(total), SmoothedFrameGeometry());
|
||||
// Pre-combined fulls have no rocking geometry left to smooth or to recompute a partiality from.
|
||||
BuildInRangeObservations(keys, static_cast<int>(total),
|
||||
segments.empty() ? SmoothedFrameGeometry() : SmoothedGeometry{});
|
||||
rawrun_group.assign(rawrun_start.size(), -1);
|
||||
logger->Info("RotationScaleMerge: ingested {} partial observations from {} frames ({} distinct hkl)",
|
||||
n_partials_obs, n_frames, rawrun_start.size());
|
||||
|
||||
SmoothMosaicityAndPartiality();
|
||||
if (segments.empty()) {
|
||||
SmoothMosaicityAndPartiality();
|
||||
} else {
|
||||
ingest_zeta.reset();
|
||||
ingest_d.reset();
|
||||
}
|
||||
ParallelChunks(static_cast<int>(partials.size()), ThreadsForWork(partials.size(), nthreads),
|
||||
[&](int lo, int hi) {
|
||||
for (int i = lo; i < hi; ++i)
|
||||
@@ -2174,6 +2204,7 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
|
||||
has_fulls[f] = frame_obs_count[f] > 0 ? 1 : 0;
|
||||
}
|
||||
double best_step = INFINITY, lambda = 0.0;
|
||||
std::vector<double> segment_lambda(segments.size(), 0.0);
|
||||
bool lambda_kept = false;
|
||||
std::vector<double> steps, g_best;
|
||||
AndersonMixing anderson(SCALING_ANDERSON_DEPTH);
|
||||
@@ -2188,9 +2219,16 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop(
|
||||
if (fitted)
|
||||
y[f] += g_fit[f] > 0.0 ? std::max(std::log(g_fit[f] / g[f]), -1.0) : -1.0;
|
||||
}
|
||||
if (!lambda_kept)
|
||||
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
|
||||
const std::vector<double> x = SmoothLogScale(y, J, lambda);
|
||||
std::vector<double> x;
|
||||
if (segments.empty()) {
|
||||
if (!lambda_kept)
|
||||
lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads);
|
||||
x = SmoothLogScale(y, J, lambda);
|
||||
} else {
|
||||
x = SmoothLogScaleBySegment(y, J, segment_lambda, !lambda_kept);
|
||||
std::vector<double> l = segment_lambda;
|
||||
lambda = median_of(l);
|
||||
}
|
||||
for (int f = 0; f < n_frames; ++f)
|
||||
g_new[f] = std::exp(x[f]);
|
||||
const double g_ref = TypicalFrameScale(g_new, n_obs);
|
||||
@@ -2342,8 +2380,14 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
return o.group >= 0 && o.corr > 0.0f && std::isfinite(o.corr) && (o.d > 0.0f)
|
||||
&& o.partiality >= min_partiality;
|
||||
};
|
||||
// Where an observation sits on the dose axis: its frame, or with several sweeps its place in the dose
|
||||
// order the data put them in (MeasureSegmentDose).
|
||||
auto dose = [&](const auto &o) {
|
||||
return segments.empty() ? static_cast<double>(o.image_number)
|
||||
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
|
||||
};
|
||||
auto decay_factor = [&](const auto &o, double slope) {
|
||||
return std::exp(slope * (o.image_number - fcenter) * s2_of(o.d));
|
||||
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
|
||||
};
|
||||
|
||||
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
|
||||
@@ -2358,6 +2402,8 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
BuildDecayTerms(n_groups, t);
|
||||
decay_terms_kept = {};
|
||||
decay_terms_kept_valid = false;
|
||||
if (!segments.empty())
|
||||
MeasureSegmentDose(t, n_groups);
|
||||
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
|
||||
std::vector<double> sw(n_groups), swI(n_groups);
|
||||
|
||||
@@ -2385,7 +2431,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double x = (o.image_number - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
|
||||
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
|
||||
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
|
||||
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
|
||||
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
|
||||
@@ -2435,6 +2481,125 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
// sit at the same s^2 but different frames, so the correction spreads them) - net harmful. slope =
|
||||
// 2 dB/dframe, so the total relative-B change across the run is |slope/2 * n_frames|.
|
||||
constexpr double DECAY_MIN_DELTA_B = 2.0; // A^2, minimum total relative-B over the run to engage
|
||||
if (!segments.empty()) {
|
||||
// Two hypotheses of how damage accrued over several sweeps, decided on held-out frames: a B rising
|
||||
// linearly along the dose order (frame_dose), or one B per sweep - a step at each sweep boundary,
|
||||
// flat within a sweep. Both are regressions of y = ln(I_ref / I_obs) on s^2 = 1/(4 d^2), the ramp
|
||||
// on x = (dose - centre) s^2, the steps on x = s^2 with a slope per sweep, at the weights the slope
|
||||
// fit above uses and within each frame (below). Each is fitted on the even frames and its
|
||||
// weighted squared residual taken on the odd ones, against their own reference, and the other way
|
||||
// round; the hypothesis that predicts the held-out frames better is applied, where it beats no
|
||||
// decay at all by the margin a correction has to clear on one sweep (and the physical floor).
|
||||
// The held-out frames' residual is read at weights fixed by the observations, so a correction
|
||||
// cannot win by reshaping sigma.
|
||||
using Sums = std::array<double, 5>; // Sw, Sx, Sy, Sxx, Sxy
|
||||
struct Line { double a = 0.0, b = 0.0; };
|
||||
auto line_of = [](const Sums &S) {
|
||||
Line l;
|
||||
const double var = S[0] > 0.0 ? S[3] - S[1] * S[1] / S[0] : 0.0;
|
||||
if (var > 0.0) l.b = (S[4] - S[1] * S[2] / S[0]) / var;
|
||||
if (S[0] > 0.0) l.a = (S[2] - l.b * S[1]) / S[0];
|
||||
return l;
|
||||
};
|
||||
auto reference = [&](int parity) {
|
||||
std::fill(sw.begin(), sw.end(), 0.0);
|
||||
std::fill(swI.begin(), swI.end(), 0.0);
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if (parity >= 0 && (o.frame & 1) != parity) continue;
|
||||
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
|
||||
sw[o.group] += w;
|
||||
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
|
||||
}
|
||||
};
|
||||
// Every term of `parity` with its y and weight against the current reference.
|
||||
auto for_terms = [&](int parity, const auto &fn) {
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
|
||||
const double Iref = swI[o.group] / sw[o.group];
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
|
||||
fn(o, std::log(Iref / Is), (Is / sc) * (Is / sc));
|
||||
}
|
||||
};
|
||||
auto x_ramp = [&](const DecayTerm &o) { return (dose(o) - fcenter) * s2_of(o.d); };
|
||||
// The terms of `parity` with y and both x taken about their frame's weighted mean: each frame has a
|
||||
// scale of its own (the per-frame G), which took up its part of the B averaged over its
|
||||
// reflections, so only how y varies with resolution WITHIN a frame is decay.
|
||||
struct Point { int32_t seg; double w, y, xr, xs; };
|
||||
auto points = [&](int parity) {
|
||||
reference(parity);
|
||||
std::vector<Point> pts;
|
||||
std::vector<int32_t> frame;
|
||||
std::vector<std::array<double, 4>> mean(n_frames, std::array<double, 4>{}); // Sw, Sy, Sxr, Sxs
|
||||
for_terms(parity, [&](const DecayTerm &o, double y, double w) {
|
||||
pts.push_back({frame_segment[o.frame], w, y, x_ramp(o), s2_of(o.d)});
|
||||
frame.push_back(o.frame);
|
||||
auto &m = mean[o.frame];
|
||||
m[0] += w; m[1] += w * y; m[2] += w * pts.back().xr; m[3] += w * pts.back().xs;
|
||||
});
|
||||
for (size_t k = 0; k < pts.size(); ++k) {
|
||||
const auto &m = mean[frame[k]];
|
||||
pts[k].y -= m[1] / m[0]; pts[k].xr -= m[2] / m[0]; pts[k].xs -= m[3] / m[0];
|
||||
}
|
||||
return pts;
|
||||
};
|
||||
struct Fit { double ramp = 0.0; std::vector<double> step; };
|
||||
auto fit = [&](int parity) {
|
||||
Sums r{};
|
||||
std::vector<Sums> st(segments.size(), Sums{});
|
||||
for (const Point &q : points(parity)) {
|
||||
r[0] += q.w; r[1] += q.w * q.xr; r[2] += q.w * q.y; r[3] += q.w * q.xr * q.xr; r[4] += q.w * q.xr * q.y;
|
||||
Sums &a = st[q.seg];
|
||||
a[0] += q.w; a[1] += q.w * q.xs; a[2] += q.w * q.y; a[3] += q.w * q.xs * q.xs; a[4] += q.w * q.xs * q.y;
|
||||
}
|
||||
Fit out{line_of(r).b, {}};
|
||||
for (const Sums &a : st) out.step.push_back(line_of(a).b);
|
||||
return out;
|
||||
};
|
||||
// Held-out weighted squared residuals of no decay, the ramp and the steps.
|
||||
auto score = [&](int test, const Fit &m, std::array<double, 3> &ssr) {
|
||||
for (const Point &q : points(test)) {
|
||||
const double e1 = q.y - m.ramp * q.xr, e2 = q.y - m.step[q.seg] * q.xs;
|
||||
ssr[0] += q.w * q.y * q.y; ssr[1] += q.w * e1 * e1; ssr[2] += q.w * e2 * e2;
|
||||
}
|
||||
};
|
||||
std::array<double, 3> ssr{};
|
||||
const Fit even = fit(0), odd = fit(1);
|
||||
score(1, even, ssr);
|
||||
score(0, odd, ssr);
|
||||
const Fit all = fit(-1);
|
||||
|
||||
const double slope = all.ramp; // the slope decay_factor applies
|
||||
std::vector<double> b(segments.size());
|
||||
double b_mid = 0.0, b_lo = INFINITY, b_hi = -INFINITY;
|
||||
for (size_t k = 0; k < segments.size(); ++k) {
|
||||
b[k] = 0.5 * all.step[k];
|
||||
b_mid += b[k] * segments[k].n_frames / static_cast<double>(n_frames);
|
||||
b_lo = std::min(b_lo, b[k]);
|
||||
b_hi = std::max(b_hi, b[k]);
|
||||
}
|
||||
const double delta_lin = std::fabs(0.5 * slope * n_frames), delta_step = b_hi - b_lo;
|
||||
const bool lin_ok = delta_lin >= DECAY_MIN_DELTA_B && ssr[1] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
|
||||
const bool step_ok = delta_step >= DECAY_MIN_DELTA_B && ssr[2] < (1.0 - CV_MIN_RELATIVE_GAIN) * ssr[0];
|
||||
const bool use_step = step_ok && (!lin_ok || ssr[2] < ssr[1]);
|
||||
const bool use_lin = lin_ok && !use_step;
|
||||
segment_decay_model = use_step ? "per-sweep B" : use_lin ? "dose-linear" : "none";
|
||||
const double rel = ssr[0] > 0.0 ? 100.0 / ssr[0] : 0.0;
|
||||
logger->Info("Decay over {} sweeps, held-out residual against no decay: dose-linear (dB = {:.2f} A^2 over "
|
||||
"the dose order) {:.1f}%, one B per sweep (spread {:.2f} A^2) {:.1f}% => {}", segments.size(),
|
||||
delta_lin, ssr[1] * rel, delta_step, ssr[2] * rel, segment_decay_model);
|
||||
if (use_step || use_lin)
|
||||
ParallelChunks(static_cast<int>(fulls.size()), ThreadsForWork(fulls.size(), nthreads), [&](int lo, int hi) {
|
||||
for (int i = lo; i < hi; ++i) {
|
||||
Obs &o = fulls[i];
|
||||
if (!usable(o)) continue;
|
||||
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
|
||||
: decay_factor(o, slope);
|
||||
o.corr = static_cast<float>(o.corr * f);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
const double slope = fit_slope(-1);
|
||||
const double total_delta_B = std::fabs(0.5 * slope * n_frames);
|
||||
if (total_delta_B < DECAY_MIN_DELTA_B) {
|
||||
@@ -2460,6 +2625,92 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
0.5 * slope * n_frames, 0.5 * slope);
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::SegmentB(const DecayTerms &t, int n_groups, int parity) const {
|
||||
// Per segment, the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against
|
||||
// s^2 = 1/(4 d^2), which is 2 B, with an intercept of its own (the scale is the per-frame G's), over
|
||||
// the frames of `parity` (< 0: all) against a reference made of the same frames. NaN where a segment
|
||||
// has nothing to fit.
|
||||
const size_t ns = segments.size();
|
||||
std::vector<double> sw(n_groups, 0.0), swI(n_groups, 0.0);
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if (parity >= 0 && (o.frame & 1) != parity) continue;
|
||||
const double sc = static_cast<double>(o.sigma) * o.corr, w = 1.0 / (sc * sc);
|
||||
sw[o.group] += w;
|
||||
swI[o.group] += w * static_cast<double>(o.I) * o.corr;
|
||||
}
|
||||
std::vector<std::array<double, 5>> S(ns, std::array<double, 5>{}); // Sw, Sx, Sy, Sxx, Sxy
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if ((parity >= 0 && (o.frame & 1) != parity) || sw[o.group] <= 0.0) continue;
|
||||
const double Iref = swI[o.group] / sw[o.group];
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) continue;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double x = 1.0 / (4.0 * static_cast<double>(o.d) * o.d), y = std::log(Iref / Is);
|
||||
auto &a = S[frame_segment[o.frame]];
|
||||
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
|
||||
}
|
||||
std::vector<double> b(ns, NAN);
|
||||
for (size_t k = 0; k < ns; ++k) {
|
||||
const auto &a = S[k];
|
||||
const double var = a[0] > 0.0 ? a[3] - a[1] * a[1] / a[0] : 0.0;
|
||||
if (var > 0.0)
|
||||
b[k] = 0.5 * (a[4] - a[1] * a[2] / a[0]) / var;
|
||||
}
|
||||
return b;
|
||||
}
|
||||
|
||||
void RotationScaleMerge::MeasureSegmentDose(const DecayTerms &t, int n_groups) {
|
||||
// A relative B per data set against the joint merge, as XSCALE fits it: Kabsch (2010) Acta Cryst.
|
||||
// D66, 133-144; the dose read from it: Diederichs (2006) Acta Cryst. D62, 96-101.
|
||||
// Radiation damage raises the relative B of a sweep collected later, so the B factors of the
|
||||
// segments against their joint merge give the order the crystal took the dose in - read from the
|
||||
// data, not from the order the sweeps were given in or the times their headers state. Per segment,
|
||||
// the inverse-variance-of-log weighted slope of ln(I_ref / I_obs) against s^2 = 1/(4 d^2) is 2 B,
|
||||
// with an intercept of its own (the scale is the per-frame G's). The least damaged segment is B = 0,
|
||||
// and its frames start the dose axis.
|
||||
const size_t ns = segments.size();
|
||||
const std::vector<double> b = SegmentB(t, n_groups, -1);
|
||||
// A segment with nothing to measure keeps its place in the input order, after the measured ones.
|
||||
std::vector<int> order(ns);
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
std::stable_sort(order.begin(), order.end(), [&](int p, int q) {
|
||||
if (std::isfinite(b[p]) != std::isfinite(b[q])) return std::isfinite(b[p]);
|
||||
return std::isfinite(b[p]) && b[p] < b[q];
|
||||
});
|
||||
const double b0 = std::isfinite(b[order[0]]) ? b[order[0]] : 0.0;
|
||||
frame_dose.assign(n_frames, 0.0);
|
||||
double offset = 0.0;
|
||||
std::string text;
|
||||
for (size_t rank = 0; rank < ns; ++rank) {
|
||||
const int k = order[rank];
|
||||
const SweepSegment &seg = segments[k];
|
||||
for (int32_t f = seg.first_frame; f < seg.first_frame + seg.n_frames; ++f)
|
||||
frame_dose[f] = offset + (f - seg.first_frame);
|
||||
offset += seg.n_frames;
|
||||
segment_results[k].relative_b = b[k] - b0;
|
||||
segment_results[k].dose_rank = static_cast<int>(rank);
|
||||
text += fmt::format("{}segment {} (B {:+.2f} A^2)", rank == 0 ? "" : ", ", k + 1, b[k] - b0);
|
||||
}
|
||||
logger->Info("Dose order of the {} sweep segments, from their relative B against the joint merge: {}",
|
||||
ns, text);
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::SmoothLogScaleBySegment(const std::vector<double> &y,
|
||||
const std::vector<double> &J,
|
||||
std::vector<double> &lambda, bool choose) const {
|
||||
std::vector<double> x(y.size());
|
||||
for (size_t k = 0; k < segments.size(); ++k) {
|
||||
const auto lo = y.begin() + segments[k].first_frame, hi = lo + segments[k].n_frames;
|
||||
const auto jlo = J.begin() + segments[k].first_frame;
|
||||
const std::vector<double> ys(lo, hi), Js(jlo, jlo + segments[k].n_frames);
|
||||
if (choose)
|
||||
lambda[k] = ChooseLogScaleSmoothing(ys, Js, std::max(1, rocking_event_frames_at_start), nthreads);
|
||||
const std::vector<double> xs = SmoothLogScale(ys, Js, lambda[k]);
|
||||
std::copy(xs.begin(), xs.end(), x.begin() + segments[k].first_frame);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
// A^2, the bound the Gauss-Seidel solve clamps each batch's relative-B to. It is a numerical guard against
|
||||
// divergence, never an answer: a run of identical values in the solved curve means the solve reached the
|
||||
// bound, not that the crystal did anything. The CORRECTION keeps the tight bound it was tuned with; the
|
||||
@@ -4716,12 +4967,15 @@ void RotationScaleMerge::Combine() {
|
||||
size_t i = 0;
|
||||
while (i < ev.size()) {
|
||||
size_t kk = i + 1;
|
||||
float last_frame = partials[ev[i]].r->image_number;
|
||||
while (kk < ev.size()) {
|
||||
const float frame = partials[ev[kk]].r->image_number;
|
||||
if (frame - last_frame > max_frame_gap) break;
|
||||
last_frame = frame;
|
||||
++kk;
|
||||
// A pre-combined full is a whole event already (SweepSegment).
|
||||
if (segments.empty()) {
|
||||
float last_frame = partials[ev[i]].r->image_number;
|
||||
while (kk < ev.size()) {
|
||||
const float frame = partials[ev[kk]].r->image_number;
|
||||
if (frame - last_frame > max_frame_gap) break;
|
||||
last_frame = frame;
|
||||
++kk;
|
||||
}
|
||||
}
|
||||
|
||||
double pooled_bkg = 0.0;
|
||||
@@ -5862,18 +6116,31 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
// reflections are) - each with the sigma the merge weighted it by.
|
||||
if (export_scaled_fulls && full_stats && !for_search) {
|
||||
// Which fulls, in order, and then each one's record on its own - on all threads.
|
||||
// The fulls of one sweep of several (uncut) also hand back the reflections the group extinguishes:
|
||||
// a joint merge searches its own space group, and the screw axes are decided on exactly these.
|
||||
const gemmi::GroupOps ops = Group().operations();
|
||||
auto extinguished = [&](const Obs &o) {
|
||||
return export_scaled_fulls_uncut && o.group < 0 && frame_cell_ok[o.frame] && o.corr > 0.0f
|
||||
&& std::isfinite(o.corr) && std::isfinite(o.I) && o.d > 0.0f
|
||||
&& ops.is_systematically_absent({{o.h, o.k, o.l}});
|
||||
};
|
||||
const std::vector<int32_t> kept = KeptIndices(n_full, nthreads, [&](int i) {
|
||||
const int g = mf.group[i];
|
||||
return g >= 0 && !rejected_obs[i] && std::isfinite(merged_I[g])
|
||||
&& !(effective_d_min && acc[g].d < *effective_d_min);
|
||||
if (g < 0)
|
||||
return extinguished(fulls[i]);
|
||||
return !rejected_obs[i] && std::isfinite(merged_I[g])
|
||||
&& (export_scaled_fulls_uncut || !(effective_d_min && acc[g].d < *effective_d_min));
|
||||
});
|
||||
result.scaled_fulls.resize(kept.size());
|
||||
ParallelChunks(static_cast<int>(kept.size()), ThreadsForWork(kept.size(), nthreads), [&](int lo, int hi) {
|
||||
for (int j = lo; j < hi; ++j) {
|
||||
const Obs &o = fulls[kept[j]];
|
||||
const float I_corr = o.I * o.corr;
|
||||
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, corrected_sigma(o, I_corr, o.sigma * o.corr),
|
||||
o.image_number};
|
||||
const float sigma = o.group >= 0 ? corrected_sigma(o, I_corr, o.sigma * o.corr)
|
||||
: model_sigma(o, I_corr, o.sigma * o.corr);
|
||||
result.scaled_fulls[j] = {o.h, o.k, o.l, I_corr, sigma,
|
||||
o.image_number, o.corr * o.corr * o.var_bkg, o.corr * o.var_per_I,
|
||||
o.capture, o.d};
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -6575,9 +6842,14 @@ RotationScaleMerge::Result RotationScaleMerge::MergeAndStats(int n_groups, bool
|
||||
if (effective_d_min && m.d < *effective_d_min) continue;
|
||||
sd_min = std::min(sd_min, m.d); sd_max = std::max(sd_max, m.d);
|
||||
}
|
||||
// Nothing (or one resolution) merged: no rocking event of the lattice made a full reflection, which
|
||||
// is what a lattice that does not describe the frames ends in.
|
||||
if (!(sd_min < sd_max && sd_min > 0.0f))
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"RotationScaleMerge: resolution calculation failed");
|
||||
fmt::format("nothing to merge ({} merged reflections): no rocking event of "
|
||||
"the lattice was recorded whole enough to make a full reflection, so "
|
||||
"the lattice probably does not describe these frames",
|
||||
result.merged.size()));
|
||||
// The completeness denominator is counted over the range the run DECLARES, not the range that
|
||||
// survived: the shell grid shares its bounds (so nothing is enumerated outside a shell) and runs
|
||||
// to the declared low-resolution limit, or over the whole sphere - 1/d^2 down to 0, spelled as an
|
||||
@@ -6694,6 +6966,12 @@ RotationScaleMerge::Result RotationScaleMerge::Run(bool for_search, bool full_st
|
||||
const gemmi::SpaceGroup *group) {
|
||||
const ScopedGpuPin pin(gpu_device); // see IngestHost
|
||||
run_group = group;
|
||||
// Pre-combined fulls: there are no partials to fit a scale on, and every event is one frame.
|
||||
if (!segments.empty()) {
|
||||
scale_partials_choice = false;
|
||||
rocking_event_frames_at_start = 1;
|
||||
rocking_events_at_start = n_partials_obs;
|
||||
}
|
||||
struct ClearGroup { const gemmi::SpaceGroup *&g; ~ClearGroup() { g = nullptr; } } clear_group{run_group};
|
||||
// Whether a frame's scale can be fitted on its partials depends on whether the partials tell a change
|
||||
// of scale from an error of the partiality model. The count of rocking events per frame is the prior
|
||||
@@ -7044,7 +7322,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
|
||||
// kernels on it, and the restore re-uploads.
|
||||
|
||||
// --- 2a. On the de-novo search pass only, drop observations whose Lorentz geometry is poor. ---
|
||||
if (for_search && search_min_zeta > 0.0) {
|
||||
if (for_search && search_min_zeta > 0.0 && segments.empty()) { // a full has no rocking geometry
|
||||
int64_t n_dropped = 0;
|
||||
bool zeta_on_gpu = false;
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
@@ -7354,20 +7632,27 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
|
||||
// is a correction to the merged INTENSITIES, and a pass whose intensities are thrown away has
|
||||
// nowhere to put one - the fitted gain lives on the fulls, which the next pass rebuilds. ---
|
||||
const bool refine_surfaces = !for_search && full_stats;
|
||||
// What runs along the frames of ONE sweep - the damage monitor, the sweep ledger, the per-batch B and
|
||||
// the surfaces, each read in one goniometer and one detector setting - has no meaning across the
|
||||
// segments of several, and is left out there. The decay is fitted along the dose order instead.
|
||||
const bool along_frames = segments.empty();
|
||||
const bool corrections =
|
||||
refine_surfaces && (refine_decay_b || absorption_iter > 0 || modulation_iter > 0 || relative_b_deg > 0.0);
|
||||
refine_surfaces && (refine_decay_b || (along_frames && (absorption_iter > 0 || modulation_iter > 0
|
||||
|| relative_b_deg > 0.0)));
|
||||
segment_results.assign(segments.size(), Result::Segment{});
|
||||
segment_decay_model.clear();
|
||||
// Radiation-damage monitor: measure the per-batch relative-B on the scaled fulls BEFORE any correction
|
||||
// (report-only; captures the full damage signature, not a residual). Skipped on the P1 search pass.
|
||||
std::unique_ptr<TimingMark> sub_mark;
|
||||
auto sub = [&](const char *n) { sub_mark.reset(); sub_mark = std::make_unique<TimingMark>(*logger, std::string("rsm/s ") + n); };
|
||||
sub("damage B");
|
||||
if (refine_surfaces)
|
||||
if (refine_surfaces && along_frames)
|
||||
MeasureRadiationDamageB(n_groups);
|
||||
// Sweep-quality diagnostic, on the per-frame scale the partial scaling just fitted (the flux is
|
||||
// already out of it) and the per-frame CC computed above. It drops nothing itself; the ranges it
|
||||
// segments are the ledger 4c below decides on.
|
||||
sub("sweep quality");
|
||||
if (refine_surfaces)
|
||||
if (refine_surfaces && along_frames)
|
||||
MeasureSweepQuality(partial_scaled, cc, cc_n);
|
||||
// CC1/2 of these same fulls with the surfaces not yet folded in, for the caller's cross-pass quality
|
||||
// guard: the pass this one is judged against fits no surfaces of its own, so it is the uncorrected
|
||||
@@ -7393,7 +7678,7 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
|
||||
decay_terms_kept = {};
|
||||
decay_terms_kept_valid = false;
|
||||
sub("relative B");
|
||||
if (refine_surfaces && relative_b_deg > 0.0)
|
||||
if (refine_surfaces && along_frames && relative_b_deg > 0.0)
|
||||
RefineRelativeB(n_groups); // per-batch relative-B on top of the single decay slope
|
||||
// The three surfaces overlap, so the one fitted first gets the first claim and the others are tested
|
||||
// on what it left - the order decides what is adopted. Detector first: every frame of the sweep
|
||||
@@ -7409,16 +7694,16 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
|
||||
// the grid is then tested on what the smooth surface left, and where that surface is refused the
|
||||
// grid sees exactly what it saw before.
|
||||
sub("modulation");
|
||||
if (refine_surfaces && modulation_iter > 0)
|
||||
if (refine_surfaces && along_frames && modulation_iter > 0)
|
||||
RefineModulation(modulation_iter, n_groups);
|
||||
sub("absorption time");
|
||||
if (refine_surfaces && absorption_iter > 0)
|
||||
if (refine_surfaces && along_frames && absorption_iter > 0)
|
||||
RefineAbsorptionTime(absorption_iter, n_groups);
|
||||
sub("absorption SH");
|
||||
if (refine_surfaces && absorption_iter > 0)
|
||||
if (refine_surfaces && along_frames && absorption_iter > 0)
|
||||
RefineAbsorptionSH(absorption_iter, n_groups);
|
||||
sub("absorption gonio");
|
||||
if (refine_surfaces && absorption_iter > 0)
|
||||
if (refine_surfaces && along_frames && absorption_iter > 0)
|
||||
RefineAbsorption(absorption_iter, n_groups);
|
||||
sub("disposition");
|
||||
|
||||
@@ -7459,6 +7744,17 @@ RotationScaleMerge::Result RotationScaleMerge::RunArm(bool scale_partials, bool
|
||||
// its CC1/2 is the answer without a second merge. A caller that did not ask keeps the NaN.
|
||||
if (measure_cc_before_corrections)
|
||||
r.cc_half_before_corrections = cc_half_before_corrections.value_or(r.statistics.overall.cc_half);
|
||||
// Each segment's scale: the median of its frames' scales, at the run's gauge.
|
||||
for (size_t k = 0; k < segments.size(); ++k) {
|
||||
std::vector<double> g;
|
||||
for (int32_t f = segments[k].first_frame; f < segments[k].first_frame + segments[k].n_frames; ++f)
|
||||
if (fulls_frame_count[f] > 0 && std::isfinite(g_full[f]) && g_full[f] > 0.0)
|
||||
g.push_back(g_full[f]);
|
||||
if (!g.empty())
|
||||
segment_results[k].scale = median_of(g);
|
||||
}
|
||||
r.segments = segment_results;
|
||||
r.segment_decay_model = segment_decay_model;
|
||||
// The fulls and the merge's working arrays are rebuilt by every arm before they are read, so they go
|
||||
// with it: kept, they sat beside the next arm's combine and through everything the caller did between
|
||||
// merges - on a large cell in P1, the fulls and about 330 bytes per ASU group, gigabytes. Faulting them
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <memory>
|
||||
#include <new>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
@@ -139,6 +140,16 @@ public:
|
||||
// The fulls the merge kept, unmerged (see ScaledFull): what passed the outlier tests, inside the
|
||||
// resolution cut. Filled only when SetExportScaledFulls asked for it, and never on a search merge.
|
||||
std::vector<ScaledFull> scaled_fulls;
|
||||
// Per sweep segment (SetSweepSegments), in segment order; empty without segments.
|
||||
struct Segment {
|
||||
double scale = NAN; // median per-frame scale of the segment's fulls (run gauge)
|
||||
double relative_b = NAN; // A^2, against the joint merge, the least damaged segment at 0
|
||||
int dose_rank = -1; // the segment's place in the dose order those B factors decide
|
||||
};
|
||||
std::vector<Segment> segments;
|
||||
// How the decay over the segments was modelled - "dose-linear" (along the dose order), "per-sweep B"
|
||||
// (one B a segment) or "none" - as the held-out frames decided; empty without segments or decay.
|
||||
std::string segment_decay_model;
|
||||
};
|
||||
|
||||
// experiment: read live (its space group is changed by the caller between Run() calls).
|
||||
@@ -288,6 +299,16 @@ public:
|
||||
// Merge only the first n frames of the outcomes (by default all of them). Set before Ingest().
|
||||
void SetFrameLimit(int n) { frame_limit = n; }
|
||||
|
||||
// The outcomes are several sweeps of pre-combined fulls laid end to end (SweepSegment; by default the
|
||||
// experiment's ScalingSettings::GetSweepSegments). Set before Ingest(); the segments must cover the
|
||||
// frames in order. Such an engine runs on the host: its fulls are a few per frame where a sweep's
|
||||
// partials are millions, and the device pipeline is the partials'. Without segments nothing of this runs.
|
||||
void SetSweepSegments(std::vector<SweepSegment> s) { segments = std::move(s); }
|
||||
|
||||
// Whether the fulls handed back (SetExportScaledFulls) are all the merge kept, past its resolution cut
|
||||
// too: the fulls of one sweep of several, which the joint merge cuts again on its own.
|
||||
void SetExportScaledFullsUncut(bool on) { export_scaled_fulls_uncut = on; }
|
||||
|
||||
// The GPU the engine runs on (device 0 by default). Set before Ingest(). The device only says where
|
||||
// the merge runs: on cards of one model the result is the same on any of them.
|
||||
void SetGpuDevice(int device) { gpu_device = device; }
|
||||
@@ -724,8 +745,18 @@ private:
|
||||
std::vector<int64_t> last_cc_n;
|
||||
std::vector<uint8_t> last_frame_scaled;
|
||||
int frame_limit = -1; // see SetFrameLimit
|
||||
std::vector<SweepSegment> segments; // see SetSweepSegments
|
||||
std::vector<int32_t> frame_segment; // each frame's segment (Ingest; empty without segments)
|
||||
std::vector<Result::Segment> segment_results; // the last arm's, copied into its Result
|
||||
// Each frame's place on the dose axis the decay is fitted along: the frames of the segments laid end
|
||||
// to end in the order the segments' relative B factors put them (MeasureSegmentDose).
|
||||
std::vector<double> frame_dose;
|
||||
// SmoothLogScale / ChooseLogScaleSmoothing within each segment on its own; lambda per segment.
|
||||
std::vector<double> SmoothLogScaleBySegment(const std::vector<double> &y, const std::vector<double> &J,
|
||||
std::vector<double> &lambda, bool choose) const;
|
||||
int gpu_device = 0; // see SetGpuDevice
|
||||
bool export_scaled_fulls = false; // see SetExportScaledFulls
|
||||
bool export_scaled_fulls_uncut = false; // see SetExportScaledFullsUncut
|
||||
bool french_wilson = true; // see SetFrenchWilson
|
||||
|
||||
// --- helpers (each a flat pass; see the .cpp) ---
|
||||
@@ -958,6 +989,12 @@ private:
|
||||
std::vector<int32_t> g_start;
|
||||
};
|
||||
void BuildDecayTerms(int n_groups, DecayTerms &t) const;
|
||||
// Per segment: the relative B against the merged reference, and the dose order it puts the segments
|
||||
// in (segment_results, frame_dose).
|
||||
void MeasureSegmentDose(const DecayTerms &t, int n_groups);
|
||||
// Each segment's B against a reference made of the same frames: those of `parity` (< 0: all).
|
||||
std::vector<double> SegmentB(const DecayTerms &t, int n_groups, int parity) const;
|
||||
std::string segment_decay_model; // the last decay's choice (Result::segment_decay_model)
|
||||
// The terms the radiation-damage monitor built, handed on to RefineDecay: nothing between the two
|
||||
// touches corr, so they are the terms RefineDecay would build. Empty outside that stretch of RunArm.
|
||||
DecayTerms decay_terms_kept;
|
||||
|
||||
@@ -6,6 +6,8 @@ ADD_LIBRARY(JFJochReader STATIC
|
||||
JFJochMultiReader.h
|
||||
SweepLayout.cpp
|
||||
SweepLayout.h
|
||||
ImgCIF.cpp
|
||||
ImgCIF.h
|
||||
MiniCBF.cpp
|
||||
MiniCBF.h
|
||||
JFJochCBFReader.cpp
|
||||
@@ -29,6 +31,10 @@ ADD_LIBRARY(JFJochReader STATIC
|
||||
JFJochXTCReader.h
|
||||
NXmxModules.cpp
|
||||
NXmxModules.h
|
||||
SFRM.cpp
|
||||
SFRM.h
|
||||
JFJochSFRMReader.cpp
|
||||
JFJochSFRMReader.h
|
||||
HDF5ImageLocator.cpp
|
||||
JFJochStartMessageReader.cpp
|
||||
JFJochStartMessageReader.h
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "ImgCIF.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <set>
|
||||
|
||||
#include "../common/JFJochException.h"
|
||||
#include "../common/DiffractionGeometry.h"
|
||||
#include "../common/JFJochMath.h"
|
||||
|
||||
namespace imgcif {
|
||||
|
||||
namespace {
|
||||
|
||||
using Vec = std::array<double, 3>;
|
||||
|
||||
JFJochException Invalid(const std::string &what) {
|
||||
return {JFJochExceptionCategory::InputParameterInvalid, "imgCIF header: " + what};
|
||||
}
|
||||
|
||||
std::string Field(const std::map<std::string, std::string> &row, const std::string &name) {
|
||||
const auto it = row.find(name);
|
||||
return (it == row.end()) ? std::string() : it->second;
|
||||
}
|
||||
|
||||
// A CIF number. "?" (unknown) and "." (not applicable) are how a table leaves a cell empty - an angle
|
||||
// column on a translation row - and read as nothing.
|
||||
std::optional<double> Number(const std::string &s) {
|
||||
if (s.empty() || s == "?" || s == ".")
|
||||
return {};
|
||||
try {
|
||||
size_t used = 0;
|
||||
const double v = std::stod(s, &used);
|
||||
// CIF allows a standard uncertainty in brackets after the value: "0.71073(2)"
|
||||
if (used != s.size() && s[used] != '(')
|
||||
throw Invalid("malformed number '" + s + "'");
|
||||
return v;
|
||||
} catch (const std::logic_error &) {
|
||||
throw Invalid("malformed number '" + s + "'");
|
||||
}
|
||||
}
|
||||
|
||||
double NumberOr(const std::string &s, double fallback) {
|
||||
return Number(s).value_or(fallback);
|
||||
}
|
||||
|
||||
Vec Rotate(double angle_deg, Vec axis, const Vec &p) {
|
||||
const double len = std::sqrt(axis[0] * axis[0] + axis[1] * axis[1] + axis[2] * axis[2]);
|
||||
for (double &c: axis)
|
||||
c /= len;
|
||||
const double t = angle_deg * 3.14159265358979323846 / 180.0;
|
||||
const double c = std::cos(t), s = std::sin(t);
|
||||
const double dot = axis[0] * p[0] + axis[1] * p[1] + axis[2] * p[2];
|
||||
const Vec cross = {axis[1] * p[2] - axis[2] * p[1],
|
||||
axis[2] * p[0] - axis[0] * p[2],
|
||||
axis[0] * p[1] - axis[1] * p[0]};
|
||||
Vec out{};
|
||||
for (int i = 0; i < 3; i++)
|
||||
out[i] = p[i] * c + cross[i] * s + axis[i] * dot * (1.0 - c);
|
||||
return out;
|
||||
}
|
||||
|
||||
const Axis &Find(const Description &d, const std::string &id) {
|
||||
const auto it = d.axes.find(id);
|
||||
if (it == d.axes.end())
|
||||
throw Invalid("axis '" + id + "' is used but not defined in the _axis table");
|
||||
return it->second;
|
||||
}
|
||||
|
||||
// Carry a point from the frame of axis `id` out to the laboratory: each axis on the way applies its own
|
||||
// motion at its setting (or at the one given in `settings`), then moves by its offset from the axis it
|
||||
// depends on. A child's vector is stated in its parent's frame, which is exactly what applying the
|
||||
// parent after the child does to it.
|
||||
Vec ToLab(const Description &d, std::string id, Vec p, const std::map<std::string, double> &settings) {
|
||||
std::set<std::string> seen;
|
||||
while (id != ".") {
|
||||
if (!seen.insert(id).second)
|
||||
throw Invalid("the axis chain through '" + id + "' loops back on itself");
|
||||
const Axis &a = Find(d, id);
|
||||
const auto s = settings.find(id);
|
||||
const double value = (s != settings.end()) ? s->second : a.setting;
|
||||
if (a.type == "rotation")
|
||||
p = Rotate(value, a.vector, p);
|
||||
else if (a.type == "translation")
|
||||
for (int i = 0; i < 3; i++)
|
||||
p[i] += value * a.vector[i];
|
||||
else
|
||||
throw Invalid("axis '" + id + "' is of type '" + a.type + "', which is not supported");
|
||||
for (int i = 0; i < 3; i++)
|
||||
p[i] += a.offset[i];
|
||||
id = a.depends_on;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
// Where stored pixel `stored` (0-based) sits along its axis. _array_structure_list.direction is not
|
||||
// consulted: the dictionary says "decreasing" means the stored order runs from the last index to the
|
||||
// first, but CBFlib - and every program built on it - places a pixel by the axis displacement alone,
|
||||
// and the files written through CBFlib are only consistent read that way. Bruker's conversions state
|
||||
// "decreasing" for their rows: read as CBFlib reads them a sweep of them indexes (and DIALS indexes
|
||||
// it), read by the dictionary the image is mirrored and nothing does.
|
||||
double Displacement(const ArrayIndex &index, double stored) {
|
||||
return index.displacement + stored * index.increment;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag) {
|
||||
// The loop_ that introduces the tag, not the tag's own position: everything before it is another
|
||||
// loop's data.
|
||||
const size_t tag_at = text.find("\n" + tag);
|
||||
if (tag_at == std::string::npos)
|
||||
return {};
|
||||
const size_t loop_at = text.rfind("loop_", tag_at);
|
||||
if (loop_at == std::string::npos)
|
||||
return {};
|
||||
|
||||
// Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would
|
||||
// otherwise be counted as several columns and shift every row after it.
|
||||
std::vector<std::string> tokens;
|
||||
for (size_t i = loop_at + 5; i < text.size();) {
|
||||
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
|
||||
i++;
|
||||
if (i >= text.size())
|
||||
break;
|
||||
size_t end;
|
||||
if ((text[i] == '\'') || (text[i] == '"')) {
|
||||
end = text.find(text[i], i + 1);
|
||||
if (end == std::string::npos)
|
||||
break;
|
||||
tokens.push_back(text.substr(i + 1, end - i - 1));
|
||||
end++;
|
||||
} else {
|
||||
end = i;
|
||||
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
|
||||
end++;
|
||||
tokens.push_back(text.substr(i, end - i));
|
||||
}
|
||||
// A second loop_, or a tag belonging to another category, ends this one.
|
||||
if ((tokens.back() == "loop_")
|
||||
|| (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) {
|
||||
tokens.pop_back();
|
||||
break;
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
|
||||
std::vector<std::string> names;
|
||||
size_t first_value = 0;
|
||||
while ((first_value < tokens.size()) && tokens[first_value].starts_with("_"))
|
||||
names.push_back(tokens[first_value++]);
|
||||
// The loop found must be the tag's own: a tag stated outside any loop would otherwise be read as
|
||||
// the rows of whatever loop came before it.
|
||||
if (std::find(names.begin(), names.end(), tag) == names.end())
|
||||
return {};
|
||||
|
||||
std::vector<std::map<std::string, std::string>> rows;
|
||||
for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) {
|
||||
std::map<std::string, std::string> row;
|
||||
for (size_t j = 0; j < names.size(); j++)
|
||||
row[names[j]] = tokens[i + j];
|
||||
rows.push_back(std::move(row));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
std::optional<std::string> Item(const std::string &text, const std::string &tag) {
|
||||
const std::string key = "\n" + tag;
|
||||
for (size_t at = text.find(key); at != std::string::npos; at = text.find(key, at + 1)) {
|
||||
size_t i = at + key.size();
|
||||
// the whole tag, not a longer one that starts with it
|
||||
if ((i < text.size()) && !std::isspace(static_cast<unsigned char>(text[i])))
|
||||
continue;
|
||||
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
|
||||
i++;
|
||||
if (i >= text.size())
|
||||
return {};
|
||||
if ((text[i] == '\'') || (text[i] == '"')) {
|
||||
const size_t end = text.find(text[i], i + 1);
|
||||
if (end == std::string::npos)
|
||||
return {};
|
||||
return text.substr(i + 1, end - i - 1);
|
||||
}
|
||||
size_t end = i;
|
||||
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
|
||||
end++;
|
||||
const std::string value = text.substr(i, end - i);
|
||||
if (value.starts_with("_") || value == "loop_")
|
||||
return {}; // the tag heads a loop: not a single item
|
||||
return value;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<Description> Parse(const std::string &text) {
|
||||
const auto axis_rows = ParseLoop(text, "_axis.id");
|
||||
auto frame_rows = ParseLoop(text, "_diffrn_scan_frame_axis.axis_id");
|
||||
const auto scan_rows = ParseLoop(text, "_diffrn_scan_axis.axis_id");
|
||||
if (axis_rows.empty() || (frame_rows.empty() && scan_rows.empty()))
|
||||
return {};
|
||||
|
||||
Description d;
|
||||
for (const auto &row: axis_rows) {
|
||||
Axis a;
|
||||
a.type = Field(row, "_axis.type");
|
||||
a.equipment = Field(row, "_axis.equipment");
|
||||
a.depends_on = Field(row, "_axis.depends_on");
|
||||
if (a.depends_on.empty() || a.depends_on == "?")
|
||||
a.depends_on = ".";
|
||||
for (int i = 0; i < 3; i++) {
|
||||
a.vector[i] = NumberOr(Field(row, "_axis.vector[" + std::to_string(i + 1) + "]"), 0.0);
|
||||
a.offset[i] = NumberOr(Field(row, "_axis.offset[" + std::to_string(i + 1) + "]"), 0.0);
|
||||
}
|
||||
d.axes[Field(row, "_axis.id")] = a;
|
||||
}
|
||||
|
||||
// Where each axis stands: the frame's own setting where the file states one, the scan's start
|
||||
// otherwise; how far it moves over the frame, from the scan.
|
||||
for (const auto &row: scan_rows) {
|
||||
const auto it = d.axes.find(Field(row, "_diffrn_scan_axis.axis_id"));
|
||||
if (it == d.axes.end())
|
||||
continue;
|
||||
const bool rotation = it->second.type == "rotation";
|
||||
it->second.setting = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_start"
|
||||
: "_diffrn_scan_axis.displacement_start"), 0.0);
|
||||
it->second.increment = NumberOr(Field(row, rotation ? "_diffrn_scan_axis.angle_increment"
|
||||
: "_diffrn_scan_axis.displacement_increment"), 0.0);
|
||||
}
|
||||
for (const auto &row: frame_rows) {
|
||||
const auto it = d.axes.find(Field(row, "_diffrn_scan_frame_axis.axis_id"));
|
||||
if (it == d.axes.end())
|
||||
continue;
|
||||
const bool rotation = it->second.type == "rotation";
|
||||
if (const auto v = Number(Field(row, rotation ? "_diffrn_scan_frame_axis.angle"
|
||||
: "_diffrn_scan_frame_axis.displacement")))
|
||||
it->second.setting = *v;
|
||||
}
|
||||
|
||||
// The two image indices, and the axis each runs along.
|
||||
std::map<std::string, std::map<std::string, std::string>> set_rows;
|
||||
for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
|
||||
set_rows[Field(row, "_array_structure_list_axis.axis_set_id")] = row;
|
||||
for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) {
|
||||
ArrayIndex index;
|
||||
const std::string set = Field(row, "_array_structure_list.axis_set_id");
|
||||
const auto s = set_rows.find(set);
|
||||
if (s == set_rows.end())
|
||||
throw Invalid("image axis set '" + set + "' has no _array_structure_list_axis row");
|
||||
index.axis_id = Field(s->second, "_array_structure_list_axis.axis_id");
|
||||
index.displacement = NumberOr(Field(s->second, "_array_structure_list_axis.displacement"), 0.0);
|
||||
index.increment = NumberOr(Field(s->second, "_array_structure_list_axis.displacement_increment"), 0.0);
|
||||
index.dimension = static_cast<int64_t>(NumberOr(Field(row, "_array_structure_list.dimension"), 0.0));
|
||||
const std::string which = Field(row, "_array_structure_list.index");
|
||||
if (which == "1")
|
||||
d.fast = index;
|
||||
else if (which == "2")
|
||||
d.slow = index;
|
||||
}
|
||||
if (d.fast.axis_id.empty() || d.slow.axis_id.empty())
|
||||
throw Invalid("no _array_structure_list rows for the two image indices");
|
||||
|
||||
d.wavelength_A = NumberOr(Item(text, "_diffrn_radiation_wavelength.wavelength").value_or(""), 0.0);
|
||||
if (d.wavelength_A == 0.0) {
|
||||
const auto rows = ParseLoop(text, "_diffrn_radiation_wavelength.id");
|
||||
if (!rows.empty())
|
||||
d.wavelength_A = NumberOr(Field(rows.front(), "_diffrn_radiation_wavelength.wavelength"), 0.0);
|
||||
}
|
||||
d.overload = NumberOr(Item(text, "_array_intensities.overload").value_or(""), 0.0);
|
||||
d.exposure_s = NumberOr(Item(text, "_diffrn_scan_frame.integration_time").value_or(""), 0.0);
|
||||
d.detector = Item(text, "_diffrn_detector.type").value_or(Item(text, "_diffrn_detector.detector").value_or(""));
|
||||
d.ccd = Item(text, "_diffrn_detector.detector").value_or("").find("CCD") != std::string::npos;
|
||||
if (const auto ratio = Item(text, "_diffrn_radiation_wavelength.polarizn_source_ratio"))
|
||||
d.polarization_ratio = NumberOr(*ratio, 0.0);
|
||||
d.polarization_norm_deg = NumberOr(Item(text, "_diffrn_radiation_wavelength.polarizn_source_norm").value_or(""), 0.0);
|
||||
return d;
|
||||
}
|
||||
|
||||
std::optional<float> PolarizationFactor(const Description &d) {
|
||||
if (!d.polarization_ratio)
|
||||
return {};
|
||||
return static_cast<float>(*d.polarization_ratio * std::cos(2.0 * d.polarization_norm_deg * PI / 180.0));
|
||||
}
|
||||
|
||||
Geometry Evaluate(const Description &d) {
|
||||
Geometry g;
|
||||
|
||||
// The image: the index axis that hangs off the other one is where the chain starts.
|
||||
const Axis &fast = Find(d, d.fast.axis_id);
|
||||
const Axis &slow = Find(d, d.slow.axis_id);
|
||||
const bool slow_is_leaf = slow.depends_on == d.fast.axis_id;
|
||||
if (!slow_is_leaf && fast.depends_on != d.slow.axis_id)
|
||||
throw Invalid("the two image axes " + d.fast.axis_id + " and " + d.slow.axis_id
|
||||
+ " do not depend on each other; such a detector is not supported");
|
||||
const std::string leaf = slow_is_leaf ? d.slow.axis_id : d.fast.axis_id;
|
||||
const auto pixel = [&](double col, double row) {
|
||||
return ToLab(d, leaf, {0, 0, 0}, {{d.fast.axis_id, Displacement(d.fast, col)},
|
||||
{d.slow.axis_id, Displacement(d.slow, row)}});
|
||||
};
|
||||
g.origin = pixel(0, 0);
|
||||
const Vec p10 = pixel(1, 0), p01 = pixel(0, 1);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
g.fast[i] = p10[i] - g.origin[i];
|
||||
g.slow[i] = p01[i] - g.origin[i];
|
||||
}
|
||||
|
||||
// The scan: exactly one axis may move, and it has to be a goniometer rotation. Its direction is
|
||||
// its own vector carried by every axis it depends on, at their settings - on a kappa head a phi
|
||||
// scan turns about phi as omega and kappa hold it. The axes it carries do not move it.
|
||||
std::vector<std::string> moving;
|
||||
for (const auto &[id, a]: d.axes)
|
||||
if (a.increment != 0.0 && id != d.fast.axis_id && id != d.slow.axis_id)
|
||||
moving.push_back(id);
|
||||
if (moving.size() > 1) {
|
||||
std::string names;
|
||||
for (const auto &m: moving)
|
||||
names += (names.empty() ? "" : ", ") + m;
|
||||
throw Invalid("more than one axis moves during the frame (" + names + "); only a rotation "
|
||||
"about one goniometer axis is supported");
|
||||
}
|
||||
if (moving.empty())
|
||||
return g;
|
||||
|
||||
const std::string &id = moving.front();
|
||||
const Axis &a = d.axes.at(id);
|
||||
if (a.type != "rotation" || a.equipment != "goniometer")
|
||||
throw Invalid("the axis that moves during the frame (" + id + ") is a " + a.equipment + " "
|
||||
+ a.type + "; only a goniometer rotation is supported");
|
||||
|
||||
// Rotations only: an offset moves where the axis is, not which way it points.
|
||||
Vec dir = a.vector;
|
||||
std::set<std::string> seen;
|
||||
for (std::string up = a.depends_on; up != "."; up = Find(d, up).depends_on) {
|
||||
if (!seen.insert(up).second)
|
||||
throw Invalid("the axis chain through '" + up + "' loops back on itself");
|
||||
const Axis &parent = Find(d, up);
|
||||
if (parent.type == "rotation")
|
||||
dir = Rotate(parent.setting, parent.vector, dir);
|
||||
}
|
||||
const double len = std::sqrt(dir[0] * dir[0] + dir[1] * dir[1] + dir[2] * dir[2]);
|
||||
if (!(len > 0.0))
|
||||
throw Invalid("the scan axis " + id + " has no direction");
|
||||
for (double &c: dir)
|
||||
c /= len;
|
||||
|
||||
g.scan_axis = id;
|
||||
g.scan_direction = dir;
|
||||
g.scan_start_deg = a.setting;
|
||||
g.scan_increment_deg = a.increment;
|
||||
return g;
|
||||
}
|
||||
|
||||
Coord ToInternal(const std::array<double, 3> &v) {
|
||||
return {static_cast<float>(v[0]), static_cast<float>(-v[1]), static_cast<float>(-v[2])};
|
||||
}
|
||||
|
||||
Coord GoniometerAxisOf(const std::array<double, 3> &v) {
|
||||
return -ToInternal(v);
|
||||
}
|
||||
|
||||
GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg,
|
||||
const std::array<double, 3> &v) {
|
||||
const double sign = increment_deg < 0.0 ? -1.0 : 1.0;
|
||||
return GoniometerAxis(name, static_cast<float>(sign * start_deg), static_cast<float>(sign * increment_deg),
|
||||
GoniometerAxisOf(v) * static_cast<float>(sign), {});
|
||||
}
|
||||
|
||||
DetectorPlane PlaneOf(const Geometry &g, const std::string &path) {
|
||||
const Coord origin = ToInternal(g.origin);
|
||||
const Coord fast = ToInternal(g.fast);
|
||||
const Coord slow = ToInternal(g.slow);
|
||||
|
||||
DetectorPlane p;
|
||||
p.pixel_mm = fast.Length();
|
||||
// One pixel size and a rectangular grid is what the geometry can describe.
|
||||
if (std::fabs(slow.Length() - p.pixel_mm) > 1e-4f * p.pixel_mm
|
||||
|| std::fabs(fast * slow) > 1e-4f * p.pixel_mm * p.pixel_mm)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
path + ": the imgCIF image axes are not square pixels on a rectangular "
|
||||
"grid, which is not supported");
|
||||
|
||||
const Coord f = fast.Normalize(), s = slow.Normalize();
|
||||
Coord n = f % s;
|
||||
// fast x slow points away from the sample on an image stored the internal way round, and towards
|
||||
// it on a mirrored one.
|
||||
const bool mirrored = n * origin < 0.0f;
|
||||
if (mirrored)
|
||||
n = -n;
|
||||
|
||||
p.distance_mm = n * origin;
|
||||
const Coord to_poni = n * p.distance_mm - origin;
|
||||
p.beam_x_pxl = (to_poni * f) / p.pixel_mm;
|
||||
p.beam_y_pxl = (to_poni * s) / p.pixel_mm;
|
||||
|
||||
// Of the four ways the image can be stored with this handedness, the one that leaves the least
|
||||
// for the continuous rotation - the detector as it is mounted, with the arm and the tilts on top.
|
||||
const RotMatrix m(f, s, n);
|
||||
float best_trace = -10.0f;
|
||||
for (int64_t q = 0; q < 4; q++) {
|
||||
const DetectorOrientation o(mirrored, q);
|
||||
const RotMatrix r = m * o.Matrix().transpose();
|
||||
const float trace = r.Column(0).x + r.Column(1).y + r.Column(2).z;
|
||||
if (trace > best_trace) {
|
||||
best_trace = trace;
|
||||
p.orientation = o;
|
||||
PoniAnglesFromMatrix(r, p.rot1, p.rot2, p.rot3);
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
} // namespace imgcif
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../common/Coord.h"
|
||||
#include "../common/DetectorOrientation.h"
|
||||
#include "../common/GoniometerAxis.h"
|
||||
|
||||
// The parts of a full imgCIF header (the form CBFlib writes, and Bruker's APEX writes when it converts
|
||||
// .sfrm frames) that say where the detector and the goniometer stand. Unlike a PILATUS miniCBF, which
|
||||
// states its geometry in "# " comment lines, such a file states it only in CIF tables: an axis table
|
||||
// (_axis), the setting of every axis at this frame (_diffrn_scan_frame_axis) and how it moves per
|
||||
// frame (_diffrn_scan_axis), and which axes the image's two indices run along
|
||||
// (_array_structure_list, _array_structure_list_axis).
|
||||
//
|
||||
// Every vector here is in the imgCIF laboratory frame (Z from the sample towards the source, Y
|
||||
// opposite gravity, X completing a right-handed set) and every length in millimetres.
|
||||
// Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458
|
||||
namespace imgcif {
|
||||
|
||||
// One "loop_" of a CIF block, as rows keyed by tag. Tags and values are both read as
|
||||
// whitespace-separated tokens rather than by line, because a template packs several tags onto one
|
||||
// line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are laid out to
|
||||
// match the tags, not the lines.
|
||||
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag);
|
||||
|
||||
// A tag stated outside a loop ("_diffrn_radiation_wavelength.wavelength 0.71073"), or nothing.
|
||||
std::optional<std::string> Item(const std::string &text, const std::string &tag);
|
||||
|
||||
struct Axis {
|
||||
std::string type; // "rotation" or "translation"
|
||||
std::string equipment; // "goniometer", "detector", ...
|
||||
std::string depends_on; // "." for an axis that hangs off nothing
|
||||
std::array<double, 3> vector{};
|
||||
std::array<double, 3> offset{};
|
||||
double setting = 0; // this frame's angle (deg) or displacement (mm)
|
||||
double increment = 0; // its change over one frame
|
||||
};
|
||||
|
||||
// One index of the stored image: the axis it runs along, and where pixel 1 and each next one sit on it.
|
||||
struct ArrayIndex {
|
||||
std::string axis_id;
|
||||
int64_t dimension = 0;
|
||||
double displacement = 0; // of index 1, mm
|
||||
double increment = 0; // per index, mm
|
||||
};
|
||||
|
||||
struct Description {
|
||||
std::map<std::string, Axis> axes;
|
||||
ArrayIndex fast; // _array_structure_list.index 1
|
||||
ArrayIndex slow; // _array_structure_list.index 2
|
||||
double wavelength_A = 0;
|
||||
double overload = 0; // _array_intensities.overload; 0 where not stated
|
||||
double exposure_s = 0;
|
||||
std::string detector;
|
||||
// _diffrn_detector.detector names a CCD: its pixel values are ADU, not counted photons.
|
||||
bool ccd = false;
|
||||
// _diffrn_radiation_wavelength.polarizn_source_ratio, (Ip - In) / (Ip + In), and
|
||||
// polarizn_source_norm, the angle (deg) from the laboratory Y axis to the normal of the plane Ip
|
||||
// is polarized in. Nothing where the header states no ratio.
|
||||
std::optional<double> polarization_ratio;
|
||||
double polarization_norm_deg = 0;
|
||||
};
|
||||
|
||||
// The polarization as DiffractionExperiment::PolarizationFactor takes it - the degree, positive for a
|
||||
// beam polarized in the horizontal (laboratory X-Z) plane - or nothing where the header states none.
|
||||
// A plane at an angle psi from the horizontal enters as its projection, the degree times cos(2 psi),
|
||||
// as the correction knows only those two planes.
|
||||
std::optional<float> PolarizationFactor(const Description &d);
|
||||
|
||||
// The description, or nothing when the header is not a full imgCIF one - no axis table, or no
|
||||
// statement of where the axes stand. Throws on a table that is there but cannot be read.
|
||||
std::optional<Description> Parse(const std::string &text);
|
||||
|
||||
struct Geometry {
|
||||
std::array<double, 3> origin{}; // centre of stored pixel (0, 0)
|
||||
std::array<double, 3> fast{}; // one stored column further, minus origin
|
||||
std::array<double, 3> slow{}; // one stored row further, minus origin
|
||||
std::string scan_axis; // the goniometer axis that turns during the frame
|
||||
std::array<double, 3> scan_direction{}; // its laboratory direction at this frame's settings
|
||||
double scan_start_deg = 0;
|
||||
double scan_increment_deg = 0;
|
||||
};
|
||||
|
||||
// Where the image and the rotation axis are at this frame, through the whole axis chain. Throws, naming
|
||||
// what it found, on anything this cannot describe as one rotation about one axis: two axes moving at
|
||||
// once, a detector or translation axis moving, or a chain that does not close.
|
||||
Geometry Evaluate(const Description &d);
|
||||
|
||||
// An imgCIF laboratory direction in the internal frame. imgCIF puts Z from the sample towards the
|
||||
// source and Y opposite gravity, while the internal frame has z along the beam and y along increasing
|
||||
// row, so the two differ by a half turn about x - a rotation and not a mirror, so an axis carried
|
||||
// through it turns the same way by the same angle.
|
||||
Coord ToInternal(const std::array<double, 3> &v);
|
||||
|
||||
// The goniometer axis as GoniometerAxis takes it, for an imgCIF rotation axis. GoniometerAxis turns
|
||||
// the crystal by minus the angle about its axis (IndexAndRefine, rot_to_image), the sense an NXmx
|
||||
// master's vector carries, so a right-handed imgCIF rotation about v is the internal -v.
|
||||
Coord GoniometerAxisOf(const std::array<double, 3> &v);
|
||||
|
||||
// The sweep as GoniometerAxis takes it, turning the scanned axis v from start_deg by increment_deg per
|
||||
// frame. A sweep that turns backwards (Bruker frames state a negative increment) is the same rotation
|
||||
// about -v by the opposite angles, and is stated that way: the analysis takes the oscillation width of a
|
||||
// frame to be positive, as every other format states it.
|
||||
GoniometerAxis SweepAxis(const std::string &name, double start_deg, double increment_deg,
|
||||
const std::array<double, 3> &v);
|
||||
|
||||
// The detector plane in the terms the geometry is kept in: the point of normal incidence (PONI) in
|
||||
// pixels, the distance along the normal, and the rotation that takes the detector frame to the
|
||||
// laboratory, split into the discrete way the image is stored and the continuous rest. The table
|
||||
// states the plane through three pixel centres, so this is only a change of description - a 2theta
|
||||
// arm, a detector's small mounting tilts and a stored image turned or mirrored all come out of the same
|
||||
// three points. Throws, naming path, on pixels that are not square or a grid that is not rectangular.
|
||||
struct DetectorPlane {
|
||||
float pixel_mm = 0;
|
||||
float distance_mm = 0;
|
||||
float beam_x_pxl = 0;
|
||||
float beam_y_pxl = 0;
|
||||
DetectorOrientation orientation;
|
||||
float rot1 = 0, rot2 = 0, rot3 = 0; // PONI angles, rad
|
||||
};
|
||||
DetectorPlane PlaneOf(const Geometry &g, const std::string &path);
|
||||
|
||||
} // namespace imgcif
|
||||
+80
-11
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <array>
|
||||
@@ -16,6 +17,7 @@
|
||||
#include "../common/Logger.h"
|
||||
#include "../common/JFJochMath.h"
|
||||
#include "SweepLayout.h"
|
||||
#include "ImgCIF.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -178,6 +180,15 @@ Coord RotationAxis(const minicbf::Header &h) {
|
||||
const Coord base = BaseAxis(h);
|
||||
if (!minicbf::ScansPhi(h))
|
||||
return base;
|
||||
// A kappa head carries phi on an arm inclined to omega, and where the arm stands moves the phi
|
||||
// axis by up to twice its inclination. These headers state the kappa angle but not the arm's
|
||||
// inclination in any form this reader composes, so a phi scan at a non-zero kappa is refused
|
||||
// rather than read about the wrong axis.
|
||||
if (h.kappa_deg.has_value() && std::fabs(*h.kappa_deg) > 1e-3)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"a phi scan at kappa " + std::to_string(*h.kappa_deg)
|
||||
+ " deg: the miniCBF header does not describe the kappa arm, so the "
|
||||
"rotation axis cannot be composed; this is not supported");
|
||||
|
||||
const auto rad = [](double deg) { return static_cast<float>(deg * PI / 180.0); };
|
||||
// Where the table states the inner axis outright, that IS the inclination, and the driven-circle
|
||||
@@ -231,8 +242,24 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
|
||||
dataset_ = std::make_shared<JFJochReaderDataset>();
|
||||
dataset_->experiment = default_experiment;
|
||||
|
||||
// A full imgCIF header states its geometry through the axis chain, which is worked out once here
|
||||
// and per frame below; a miniCBF states it in its "# " lines.
|
||||
std::optional<imgcif::Geometry> geometry0;
|
||||
std::optional<imgcif::DetectorPlane> plane0;
|
||||
if (header0_.imgcif.has_value()) {
|
||||
geometry0 = imgcif::Evaluate(*header0_.imgcif);
|
||||
if (geometry0->scan_axis.empty())
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
files_[0] + ": no goniometer axis turns during the frame; a full "
|
||||
"imgCIF series of stills is not supported");
|
||||
plane0 = imgcif::PlaneOf(*geometry0, files_[0]);
|
||||
}
|
||||
|
||||
DetectorSetup detector = DetDECTRIS(header0_.nx, header0_.ny, header0_.detector, {});
|
||||
detector.PixelSize_um(static_cast<int64_t>(std::lround(header0_.pixel_x_m * 1e6)));
|
||||
if (plane0.has_value())
|
||||
detector.PixelSize_um(plane0->pixel_mm * 1000.0f); // a CCD's pixel is not whole micrometres
|
||||
else
|
||||
detector.PixelSize_um(static_cast<int64_t>(std::lround(header0_.pixel_x_m * 1e6)));
|
||||
detector.SensorThickness_um(static_cast<int64_t>(std::lround(header0_.thickness_m * 1e6)));
|
||||
detector.SensorMaterial(header0_.material);
|
||||
// Only when the header states one. An absent line reads as zero, and a limit of zero would mark
|
||||
@@ -257,16 +284,25 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
|
||||
// Images are handed out as signed 32-bit whatever the file stored, so that is the depth the rest
|
||||
// of the code must see; the real overflow is the header's Count_cutoff, set above.
|
||||
detector.BitDepthImage(32);
|
||||
if (const auto orientation = ImageOrientation(header0_))
|
||||
if (plane0.has_value())
|
||||
detector.ImageOrientation(plane0->orientation);
|
||||
else if (const auto orientation = ImageOrientation(header0_))
|
||||
detector.ImageOrientation(orientation.value());
|
||||
detector.MinFrameTime(std::chrono::microseconds(0));
|
||||
detector.MinCountTime(std::chrono::microseconds(0));
|
||||
detector.ReadOutTime(std::chrono::nanoseconds(0));
|
||||
dataset_->experiment.Detector(detector);
|
||||
|
||||
dataset_->experiment.BeamX_pxl(static_cast<float>(header0_.beam_x_px));
|
||||
dataset_->experiment.BeamY_pxl(static_cast<float>(header0_.beam_y_px));
|
||||
dataset_->experiment.DetectorDistance_mm(static_cast<float>(header0_.distance_m * 1000.0));
|
||||
if (plane0.has_value()) {
|
||||
dataset_->experiment.BeamX_pxl(plane0->beam_x_pxl);
|
||||
dataset_->experiment.BeamY_pxl(plane0->beam_y_pxl);
|
||||
dataset_->experiment.DetectorDistance_mm(plane0->distance_mm);
|
||||
dataset_->experiment.PoniRot1_rad(plane0->rot1).PoniRot2_rad(plane0->rot2).PoniRot3_rad(plane0->rot3);
|
||||
} else {
|
||||
dataset_->experiment.BeamX_pxl(static_cast<float>(header0_.beam_x_px));
|
||||
dataset_->experiment.BeamY_pxl(static_cast<float>(header0_.beam_y_px));
|
||||
dataset_->experiment.DetectorDistance_mm(static_cast<float>(header0_.distance_m * 1000.0));
|
||||
}
|
||||
|
||||
// A detector swung out on a 2theta arm, which small-molecule collection uses routinely. The arm
|
||||
// turns the detector about the sample, so it carries the square-on geometry with it: the header's
|
||||
@@ -290,6 +326,10 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
|
||||
// correction takes to contain the lab x axis; the degree it wants is 2f - 1.
|
||||
if (header0_.polarization_fraction.has_value())
|
||||
dataset_->experiment.PolarizationFactor(2.0f * static_cast<float>(*header0_.polarization_fraction) - 1.0f);
|
||||
// A full imgCIF header states the degree itself, and the plane.
|
||||
if (header0_.imgcif.has_value())
|
||||
if (const auto p = imgcif::PolarizationFactor(*header0_.imgcif))
|
||||
dataset_->experiment.PolarizationFactor(*p);
|
||||
dataset_->experiment.FrameTime(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::duration<double>(header0_.period_s)),
|
||||
@@ -300,21 +340,46 @@ void JFJochCBFReader::ReadFiles(const std::string &path) {
|
||||
// Parsing one costs a few hundred microseconds - it is two dozen regular expressions - so on a
|
||||
// sweep of several thousand frames this is seconds of startup before a single image is read, and
|
||||
// the files are independent.
|
||||
//
|
||||
// A full imgCIF frame states every axis setting, so each one is checked to belong to the same
|
||||
// sweep as the first: the same axis turning, about the same direction. A series in which the
|
||||
// fixed axes moved between frames - kappa or omega reset for another run - is several sweeps,
|
||||
// and taking it for one would put every frame after the move at the wrong orientation.
|
||||
std::vector<sweep::Frame> frames(files_.size());
|
||||
sweep::ForEachInOrder(files_.size(), [&](size_t i) {
|
||||
const auto h = minicbf::ReadHeader(files_[i]);
|
||||
frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m,
|
||||
h.beam_x_px, h.beam_y_px, h.wavelength_A};
|
||||
if (!geometry0.has_value()) {
|
||||
frames[i] = {files_[i], h.start_angle_deg, h.angle_increment_deg, h.distance_m,
|
||||
h.beam_x_px, h.beam_y_px, h.wavelength_A};
|
||||
return;
|
||||
}
|
||||
if (!h.imgcif.has_value())
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
files_[i] + " is not a full imgCIF frame like " + files_[0]);
|
||||
const auto g = imgcif::Evaluate(*h.imgcif);
|
||||
const Coord axis = ImgCIFToInternal(g.scan_direction);
|
||||
if (g.scan_axis != geometry0->scan_axis
|
||||
|| (axis - ImgCIFToInternal(geometry0->scan_direction)).Length() > 1e-4f)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
files_[i] + " turns " + (g.scan_axis.empty() ? "no axis" : g.scan_axis)
|
||||
+ " about a different direction from " + files_[0]
|
||||
+ ": the goniometer was reset between them, so the series is more "
|
||||
"than one sweep. Name a frame of one sweep only.");
|
||||
const auto plane = imgcif::PlaneOf(g, files_[i]);
|
||||
frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3,
|
||||
plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A};
|
||||
});
|
||||
|
||||
// The sweep the headers describe, which is not always the files laid out end to end: a deposited
|
||||
// series can be missing frames, and those are gaps in the rotation rather than images to close up.
|
||||
const auto layout = sweep::Place(frames, "CBFReader");
|
||||
files_ = layout.files;
|
||||
dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name,
|
||||
static_cast<float>(layout.start_deg),
|
||||
static_cast<float>(layout.increment_deg),
|
||||
RotationAxis(header0_), {}));
|
||||
if (geometry0.has_value())
|
||||
dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0->scan_axis, layout.start_deg, layout.increment_deg,
|
||||
geometry0->scan_direction));
|
||||
else
|
||||
dataset_->experiment.Goniometer(GoniometerAxis(header0_.axis_name, static_cast<float>(layout.start_deg),
|
||||
static_cast<float>(layout.increment_deg), RotationAxis(header0_), {}));
|
||||
|
||||
dataset_->error_value = -1;
|
||||
dataset_->experiment.ImagesPerTrigger(static_cast<int64_t>(files_.size()));
|
||||
@@ -395,6 +460,10 @@ bool JFJochCBFReader::HasImage(int64_t image_number) const {
|
||||
&& !files_[image_number].empty();
|
||||
}
|
||||
|
||||
bool JFJochCBFReader::CountsInADU() const {
|
||||
return header0_.imgcif.has_value() && header0_.imgcif->ccd;
|
||||
}
|
||||
|
||||
bool JFJochCBFReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) {
|
||||
if (!HasImage(image_number))
|
||||
return false;
|
||||
|
||||
@@ -56,6 +56,10 @@ public:
|
||||
// ReadRawImage hands such a slot back as "nothing to read".
|
||||
[[nodiscard]] bool HasImage(int64_t image_number) const;
|
||||
|
||||
// Whether the pixel values are ADU rather than counted photons: a full imgCIF header that names a
|
||||
// CCD, which is what Bruker's conversion of its frames states. A PILATUS miniCBF counts photons.
|
||||
[[nodiscard]] bool CountsInADU() const;
|
||||
|
||||
bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override;
|
||||
[[nodiscard]] std::vector<SpotToSave> ReadSpots(int64_t image) const override;
|
||||
};
|
||||
@@ -0,0 +1,231 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "JFJochSFRMReader.h"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
#include "../common/JFJochException.h"
|
||||
#include "../common/JFJochMath.h"
|
||||
#include "../common/Logger.h"
|
||||
#include "ImgCIF.h"
|
||||
#include "SweepLayout.h"
|
||||
|
||||
namespace {
|
||||
|
||||
|
||||
// The instrument a .sfrm header describes, as an imgCIF axis chain, so that one piece of code turns
|
||||
// either description into the geometry. The chain is the one APEX writes when it converts frames to
|
||||
// imgCIF (the pixels come out identical but for the baseline, so the image axes are the same), with
|
||||
// one thing settled from data rather than taken from it: omega turns in the sense of the 2theta arm.
|
||||
// An omega scan of APEX's conversion states the opposite increment to the frames it came from, and a
|
||||
// SMART 6000 omega sweep indexes only that way round. Phi and chi are as the conversion composes them
|
||||
// from a kappa setting. CENTER is placed as the conversion places it, each coordinate counted from the
|
||||
// far edge of the stored image: on an APEX II phi sweep read both ways it indexes 97.5% of the frames
|
||||
// against 90.3% and merges at ISa 17.5 against 2.1, where the image's own .cbf conversion gives 17.
|
||||
//
|
||||
// The goniometer is stated as Eulerian angles whatever the head is - a kappa instrument's frames carry
|
||||
// the omega, chi and phi its kappa setting is equivalent to - so the chain is omega, carrying chi about
|
||||
// the beam, carrying phi. On the 2theta arm the detector sits DISTANC (plus DETTYPE's face-to-grid
|
||||
// offset) from the sample, tilted by the DETPAR pitch, roll and yaw. The stored rows run along the
|
||||
// spindle axis and the stored columns along the direction the arm swings.
|
||||
imgcif::Description Chain(const sfrm::Header &h, const std::string &path) {
|
||||
if (h.center.size() < 2)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, path + " states no CENTER");
|
||||
if (!(h.pixels_per_cm_512 > 0.0))
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
path + " states no pixel size (DETTYPE)");
|
||||
if (h.detpar[0] != 0.0 || h.detpar[1] != 0.0 || h.detpar[2] != 0.0)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
path + " states non-zero DETPAR offsets, whose meaning this reader does not know");
|
||||
// DETTYPE gives pixels per centimetre on a 512-pixel frame; a binned or unbinned frame scales it.
|
||||
const double pixel_mm = 10.0 / h.pixels_per_cm_512 * 512.0 / static_cast<double>(h.ncols);
|
||||
const double distance_mm = (h.distance_cm + h.grid_offset_cm) * 10.0;
|
||||
const double half_cols = static_cast<double>(h.ncols) / 2.0 * pixel_mm;
|
||||
const double half_rows = static_cast<double>(h.nrows) / 2.0 * pixel_mm;
|
||||
|
||||
imgcif::Description d;
|
||||
d.axes["OMEGA"] = {"rotation", "goniometer", ".", {-1, 0, 0}, {}, h.angles[1], 0};
|
||||
d.axes["CHI"] = {"rotation", "goniometer", "OMEGA", {0, 0, 1}, {}, h.angles[3], 0};
|
||||
d.axes["PHI"] = {"rotation", "goniometer", "CHI", {1, 0, 0}, {}, h.angles[2], 0};
|
||||
d.axes["TWOTHETA"] = {"rotation", "detector", ".", {1, 0, 0}, {}, -h.angles[0], 0};
|
||||
d.axes["DX"] = {"translation", "detector", "TWOTHETA", {0, 0, -1}, {}, distance_mm, 0};
|
||||
d.axes["YAW"] = {"rotation", "detector", "DX", {-1, 0, 0}, {}, h.detpar[5], 0};
|
||||
d.axes["PITCH"] = {"rotation", "detector", "YAW", {0, -1, 0}, {}, h.detpar[3], 0};
|
||||
d.axes["ROLL"] = {"rotation", "detector", "PITCH", {0, 0, -1}, {}, h.detpar[4], 0};
|
||||
d.axes["H"] = {"translation", "detector", "ROLL", {0, 1, 0}, {}, (h.center[0] * pixel_mm - half_cols), 0};
|
||||
d.axes["V"] = {"translation", "detector", "H", {1, 0, 0}, {}, (h.center[1] * pixel_mm - half_rows), 0};
|
||||
d.axes["ELEMENT_X"] = {"translation", "detector", "V", {0, 1, 0}, {-half_rows, -half_cols, 0}, 0, 0};
|
||||
d.axes["ELEMENT_Y"] = {"translation", "detector", "ELEMENT_X", {1, 0, 0}, {}, 0, 0};
|
||||
d.fast = {"ELEMENT_X", h.ncols, pixel_mm / 2.0, pixel_mm};
|
||||
d.slow = {"ELEMENT_Y", h.nrows, pixel_mm / 2.0, pixel_mm};
|
||||
|
||||
static const char *AXIS_NAMES[] = {"", "TWOTHETA", "OMEGA", "PHI", "CHI"};
|
||||
if (h.axis < 1 || h.axis > 4)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
path + " scans AXIS " + std::to_string(h.axis)
|
||||
+ ", which is not a rotation this reader supports");
|
||||
d.axes[AXIS_NAMES[h.axis]].increment = h.increment_deg;
|
||||
d.wavelength_A = h.wavelength_A;
|
||||
// The polarization APEX's conversion states for the same frame: 1 - cos^2 of the monochromator's
|
||||
// scattering angle (FILTER2), in the horizontal plane. A tube with mirrors and no monochromator
|
||||
// states 0, unpolarized - these are laboratory frames, and the synchrotron default does not apply.
|
||||
const double cos_mono = std::cos(h.monochromator_2theta_deg * PI / 180.0);
|
||||
d.polarization_ratio = 1.0 - cos_mono * cos_mono;
|
||||
d.polarization_norm_deg = 0;
|
||||
d.overload = h.saturation;
|
||||
d.exposure_s = h.exposure_s;
|
||||
d.detector = h.detector;
|
||||
return d;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool JFJochSFRMReader::CanRead(const std::string &path) {
|
||||
return sfrm::CanRead(path);
|
||||
}
|
||||
|
||||
void JFJochSFRMReader::ReadFiles(const std::string &path) {
|
||||
files_ = sfrm::CollectSweep(path);
|
||||
if (files_.empty())
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"No SFRM images found for " + path);
|
||||
|
||||
header0_ = sfrm::ReadHeader(files_[0]);
|
||||
if (!(header0_.wavelength_A > 0.0))
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
files_[0] + " states no wavelength in its SFRM header");
|
||||
const auto geometry0 = imgcif::Evaluate(Chain(header0_, files_[0]));
|
||||
const auto plane0 = imgcif::PlaneOf(geometry0, files_[0]);
|
||||
|
||||
dataset_ = std::make_shared<JFJochReaderDataset>();
|
||||
dataset_->experiment = default_experiment;
|
||||
|
||||
DetectorSetup detector = DetDECTRIS(header0_.ncols, header0_.nrows, header0_.detector, {});
|
||||
detector.PixelSize_um(plane0.pixel_mm * 1000.0f);
|
||||
// A CCD has no sensor depth to correct for: the phosphor converts at the surface.
|
||||
detector.SensorThickness_um(0);
|
||||
// CCDPARM[4] is the count at which the CCD saturates; the stored container (with its overflow
|
||||
// tables) holds far more, so it is the only limit there is.
|
||||
if (header0_.saturation > 0.0)
|
||||
detector.SaturationLimit(static_cast<int64_t>(header0_.saturation));
|
||||
else
|
||||
Logger("SFRMReader").Warning("{} states no saturation (CCDPARM), so no pixel will be called "
|
||||
"saturated.", files_[0]);
|
||||
detector.BitDepthImage(32);
|
||||
detector.ImageOrientation(plane0.orientation);
|
||||
detector.MinFrameTime(std::chrono::microseconds(0));
|
||||
detector.MinCountTime(std::chrono::microseconds(0));
|
||||
detector.ReadOutTime(std::chrono::nanoseconds(0));
|
||||
dataset_->experiment.Detector(detector);
|
||||
|
||||
dataset_->experiment.BeamX_pxl(plane0.beam_x_pxl);
|
||||
dataset_->experiment.BeamY_pxl(plane0.beam_y_pxl);
|
||||
dataset_->experiment.DetectorDistance_mm(plane0.distance_mm);
|
||||
dataset_->experiment.PoniRot1_rad(plane0.rot1).PoniRot2_rad(plane0.rot2).PoniRot3_rad(plane0.rot3);
|
||||
dataset_->experiment.IncidentEnergy_keV(WVL_1A_IN_KEV / static_cast<float>(header0_.wavelength_A));
|
||||
dataset_->experiment.PolarizationFactor(imgcif::PolarizationFactor(Chain(header0_, files_[0])));
|
||||
if (header0_.exposure_s > 0.0)
|
||||
dataset_->experiment.FrameTime(
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<double>(header0_.exposure_s)),
|
||||
std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::duration<double>(header0_.exposure_s)));
|
||||
|
||||
// Where every image sits on the spindle, from its own header - and the check that every frame is
|
||||
// of the same sweep: the same axis turning about the same direction. A run whose fixed angles moved
|
||||
// part-way is more than one sweep, and reading it as one would put every frame after the move at
|
||||
// the wrong orientation.
|
||||
std::vector<sweep::Frame> frames(files_.size());
|
||||
sweep::ForEachInOrder(files_.size(), [&](size_t i) {
|
||||
const auto h = sfrm::ReadHeader(files_[i]);
|
||||
const auto g = imgcif::Evaluate(Chain(h, files_[i]));
|
||||
if (g.scan_axis != geometry0.scan_axis
|
||||
|| (imgcif::ToInternal(g.scan_direction) - imgcif::ToInternal(geometry0.scan_direction)).Length() > 1e-4f)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
files_[i] + " turns " + g.scan_axis + " about a different direction from "
|
||||
+ files_[0] + ": the goniometer was reset between them, so the run is "
|
||||
"more than one sweep");
|
||||
const auto plane = imgcif::PlaneOf(g, files_[i]);
|
||||
frames[i] = {files_[i], g.scan_start_deg, g.scan_increment_deg, plane.distance_mm * 1e-3,
|
||||
plane.beam_x_pxl, plane.beam_y_pxl, h.wavelength_A};
|
||||
});
|
||||
|
||||
const auto layout = sweep::Place(frames, "SFRMReader");
|
||||
files_ = layout.files;
|
||||
dataset_->experiment.Goniometer(imgcif::SweepAxis(geometry0.scan_axis, layout.start_deg, layout.increment_deg,
|
||||
geometry0.scan_direction));
|
||||
|
||||
dataset_->error_value = -1;
|
||||
dataset_->experiment.ImagesPerTrigger(static_cast<int64_t>(files_.size()));
|
||||
|
||||
// Outside the phosphor's active area a frame reads below its baseline offset - the stored value
|
||||
// is zero there in every frame - so frame 0's negative counts are the pixels no frame can be
|
||||
// trusted at, exactly as a PILATUS marks its gaps.
|
||||
std::vector<int32_t> first;
|
||||
sfrm::Read(files_[0], first);
|
||||
std::vector<uint32_t> mask(first.size(), 0);
|
||||
for (size_t i = 0; i < first.size(); i++)
|
||||
if (first[i] < 0)
|
||||
mask[i] = 1;
|
||||
dataset_->pixel_mask = std::make_shared<const PixelMask>(mask);
|
||||
|
||||
SetStartMessage(dataset_);
|
||||
}
|
||||
|
||||
uint64_t JFJochSFRMReader::GetNumberOfImages() const {
|
||||
return files_.size();
|
||||
}
|
||||
|
||||
void JFJochSFRMReader::Close() {
|
||||
files_.clear();
|
||||
dataset_.reset();
|
||||
}
|
||||
|
||||
template <class Buffer>
|
||||
CompressedImage JFJochSFRMReader::DecodeInto(int64_t image_number, Buffer &buffer,
|
||||
std::vector<uint8_t> &scratch) const {
|
||||
if (!HasImage(image_number))
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"No image at this point of the sweep");
|
||||
|
||||
const size_t npixel = static_cast<size_t>(header0_.nrows) * static_cast<size_t>(header0_.ncols);
|
||||
buffer.resize(npixel * sizeof(int32_t));
|
||||
const auto h = sfrm::ReadInto(files_[image_number], reinterpret_cast<int32_t *>(buffer.data()), npixel,
|
||||
scratch);
|
||||
if (h.nrows != header0_.nrows || h.ncols != header0_.ncols)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
|
||||
"SFRM image size differs from the first image of the sweep");
|
||||
|
||||
return CompressedImage(buffer.data(), buffer.size(),
|
||||
static_cast<size_t>(header0_.ncols), static_cast<size_t>(header0_.nrows),
|
||||
CompressedImageMode::Int32, CompressionAlgorithm::NO_COMPRESSION);
|
||||
}
|
||||
|
||||
bool JFJochSFRMReader::LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
|
||||
DataMessage &message,
|
||||
std::vector<uint8_t> &buffer,
|
||||
int64_t image_number,
|
||||
bool update_dataset) {
|
||||
(void) update_dataset;
|
||||
if (!dataset || !HasImage(image_number))
|
||||
return false;
|
||||
std::vector<uint8_t> scratch;
|
||||
message.image = DecodeInto(image_number, buffer, scratch);
|
||||
message.number = image_number;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool JFJochSFRMReader::HasImage(int64_t image_number) const {
|
||||
return image_number >= 0 && static_cast<size_t>(image_number) < files_.size()
|
||||
&& !files_[image_number].empty();
|
||||
}
|
||||
|
||||
bool JFJochSFRMReader::ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) {
|
||||
if (!HasImage(image_number))
|
||||
return false;
|
||||
image.image = DecodeInto(image_number, image.image_buffer, image.read_buffer);
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<SpotToSave> JFJochSFRMReader::ReadSpots(int64_t) const {
|
||||
return {}; // a raw frame stores no analysis results
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "JFJochReader.h"
|
||||
#include "SFRM.h"
|
||||
|
||||
// Reads a rotation sweep straight from a directory of Bruker .sfrm frames - what SMART, PROTEUM
|
||||
// and APEX write - with no conversion step.
|
||||
//
|
||||
// The same shape as the other one-file-per-image readers: the sweep's geometry comes from the first
|
||||
// frame's header, the rotation angle of each image from its own, and images are decoded on demand.
|
||||
// A Bruker collection directory holds one run per scan, each its own sweep; naming a frame reads its
|
||||
// run. A raw frame carries no analysis results, so no spots and no reflections.
|
||||
class JFJochSFRMReader : public JFJochReader {
|
||||
std::vector<std::string> files_;
|
||||
std::shared_ptr<JFJochReaderDataset> dataset_;
|
||||
sfrm::Header header0_;
|
||||
|
||||
bool LoadImage_i(std::shared_ptr<JFJochReaderDataset> &dataset,
|
||||
DataMessage &message,
|
||||
std::vector<uint8_t> &buffer,
|
||||
int64_t image_number,
|
||||
bool update_dataset) override;
|
||||
|
||||
// Decodes one image into the caller's byte buffer and returns the image that points at it. The
|
||||
// file is read through scratch, which the caller keeps between frames.
|
||||
template <class Buffer>
|
||||
CompressedImage DecodeInto(int64_t image_number, Buffer &buffer, std::vector<uint8_t> &scratch) const;
|
||||
|
||||
public:
|
||||
~JFJochSFRMReader() override = default;
|
||||
|
||||
// True if the path names something this reader can open: a .sfrm frame, or a directory holding
|
||||
// at least one. Cheap - it reads a few bytes.
|
||||
static bool CanRead(const std::string &path);
|
||||
|
||||
// path is a directory of .sfrm frames, or one frame inside the run to take the whole run from.
|
||||
void ReadFiles(const std::string &path);
|
||||
|
||||
[[nodiscard]] uint64_t GetNumberOfImages() const override;
|
||||
void Close() override;
|
||||
|
||||
|
||||
// Whether the sweep really has an image at this point. A deposited series can be missing frames,
|
||||
// and they are kept as gaps so that every image keeps the spindle angle its own header states;
|
||||
// ReadRawImage hands such a slot back as "nothing to read".
|
||||
[[nodiscard]] bool HasImage(int64_t image_number) const;
|
||||
|
||||
bool ReadRawImage(int64_t image_number, JFJochReaderRawImage &image) override;
|
||||
[[nodiscard]] std::vector<SpotToSave> ReadSpots(int64_t image) const override;
|
||||
};
|
||||
+18
-64
@@ -7,6 +7,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <limits>
|
||||
@@ -91,67 +92,6 @@ std::optional<double> Angle(const std::string &text, const char *pattern) {
|
||||
return v;
|
||||
}
|
||||
|
||||
// One "loop_" of the imgCIF template block these headers carry, as rows keyed by tag. Tags and values
|
||||
// are both read as whitespace-separated tokens rather than by line, because a template packs several
|
||||
// tags onto one line ("_axis.vector[1] _axis.vector[2] _axis.vector[3]") and the rows that follow are
|
||||
// laid out to match the tags, not the lines.
|
||||
std::vector<std::map<std::string, std::string>> ParseLoop(const std::string &text, const std::string &tag) {
|
||||
// The loop_ that introduces the tag, not the tag's own position: everything before it is another
|
||||
// loop's data.
|
||||
const size_t tag_at = text.find("\n" + tag);
|
||||
if (tag_at == std::string::npos)
|
||||
return {};
|
||||
const size_t loop_at = text.rfind("loop_", tag_at);
|
||||
if (loop_at == std::string::npos)
|
||||
return {};
|
||||
|
||||
// Bare tokens, and the quoted ones a CIF value may be - a quoted value holding spaces would
|
||||
// otherwise be counted as several columns and shift every row after it.
|
||||
std::vector<std::string> tokens;
|
||||
for (size_t i = loop_at + 5; i < text.size();) {
|
||||
while ((i < text.size()) && std::isspace(static_cast<unsigned char>(text[i])))
|
||||
i++;
|
||||
if (i >= text.size())
|
||||
break;
|
||||
size_t end;
|
||||
if ((text[i] == '\'') || (text[i] == '"')) {
|
||||
end = text.find(text[i], i + 1);
|
||||
if (end == std::string::npos)
|
||||
break;
|
||||
tokens.push_back(text.substr(i + 1, end - i - 1));
|
||||
end++;
|
||||
} else {
|
||||
end = i;
|
||||
while ((end < text.size()) && !std::isspace(static_cast<unsigned char>(text[end])))
|
||||
end++;
|
||||
tokens.push_back(text.substr(i, end - i));
|
||||
}
|
||||
// A second loop_, or a tag belonging to another category, ends this one.
|
||||
if ((tokens.back() == "loop_")
|
||||
|| (tokens.back().starts_with("_") && !tokens.back().starts_with(tag.substr(0, tag.find('.') + 1)))) {
|
||||
tokens.pop_back();
|
||||
break;
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
|
||||
std::vector<std::string> names;
|
||||
size_t first_value = 0;
|
||||
while ((first_value < tokens.size()) && tokens[first_value].starts_with("_"))
|
||||
names.push_back(tokens[first_value++]);
|
||||
if (names.empty())
|
||||
return {};
|
||||
|
||||
std::vector<std::map<std::string, std::string>> rows;
|
||||
for (size_t i = first_value; i + names.size() <= tokens.size(); i += names.size()) {
|
||||
std::map<std::string, std::string> row;
|
||||
for (size_t j = 0; j < names.size(); j++)
|
||||
row[names[j]] = tokens[i + j];
|
||||
rows.push_back(std::move(row));
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
std::string Field(const std::map<std::string, std::string> &row, const std::string &name) {
|
||||
const auto it = row.find(name);
|
||||
return (it == row.end()) ? std::string() : it->second;
|
||||
@@ -166,7 +106,7 @@ std::string Field(const std::map<std::string, std::string> &row, const std::stri
|
||||
// unswung detector frame - which is where the image orientation belongs, with the arm applied on top.
|
||||
// Following Hammersley, Bernstein & Westbrook (2006) Int. Tables Cryst. G, 444-458
|
||||
void ParseAxisTable(const std::string &text, Header &h) {
|
||||
const auto axes = ParseLoop(text, "_axis.id");
|
||||
const auto axes = imgcif::ParseLoop(text, "_axis.id");
|
||||
if (axes.empty())
|
||||
return;
|
||||
|
||||
@@ -199,11 +139,11 @@ void ParseAxisTable(const std::string &text, Header &h) {
|
||||
|
||||
// Which axis the fast index runs along, and which the slow, through the two tables that say so.
|
||||
std::map<std::string, std::string> axis_of_set;
|
||||
for (const auto &row: ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
|
||||
for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list_axis.axis_set_id"))
|
||||
axis_of_set[Field(row, "_array_structure_list_axis.axis_set_id")]
|
||||
= Field(row, "_array_structure_list_axis.axis_id");
|
||||
|
||||
for (const auto &row: ParseLoop(text, "_array_structure_list.array_id")) {
|
||||
for (const auto &row: imgcif::ParseLoop(text, "_array_structure_list.array_id")) {
|
||||
const std::string set = Field(row, "_array_structure_list.axis_set_id");
|
||||
const auto id = axis_of_set.contains(set) ? axis_of_set[set] : set;
|
||||
const auto it = vector_of.find(id);
|
||||
@@ -261,6 +201,7 @@ Header ParseHeader(const char *data, size_t size) {
|
||||
// The whitespace after each name is what keeps "Chi_increment" out of "Chi".
|
||||
h.chi_deg = Angle(t, R"(#\s*Chi\s+([\d.eE+-]+))");
|
||||
h.omega_deg = Angle(t, R"(#\s*Omega\s+([\d.eE+-]+))");
|
||||
h.kappa_deg = Angle(t, R"(#\s*Kappa\s+([\d.eE+-]+))");
|
||||
h.chi_increment_deg = Angle(t, R"(#\s*Chi_increment\s+([\d.eE+-]+))").value_or(0.0);
|
||||
h.phi_increment_deg = Angle(t, R"(#\s*Phi_increment\s+([\d.eE+-]+))").value_or(0.0);
|
||||
h.omega_increment_deg = Angle(t, R"(#\s*Omega_increment\s+([\d.eE+-]+))").value_or(0.0);
|
||||
@@ -292,6 +233,19 @@ Header ParseHeader(const char *data, size_t size) {
|
||||
const auto conv = Match(t, R"RE(conversions\s*=\s*"([^"]+)")RE").value_or("");
|
||||
h.byte_offset = conv.find("x-CBF_BYTE_OFFSET") != std::string::npos;
|
||||
|
||||
if (!Match(t, R"(#\s*(Pixel_size))").has_value()) { // Match returns the first group
|
||||
h.imgcif = imgcif::Parse(t);
|
||||
if (h.imgcif.has_value()) {
|
||||
h.detector = h.imgcif->detector;
|
||||
h.pixel_x_m = std::abs(h.imgcif->fast.increment) * 1e-3;
|
||||
h.pixel_y_m = std::abs(h.imgcif->slow.increment) * 1e-3;
|
||||
h.wavelength_A = h.imgcif->wavelength_A;
|
||||
h.count_cutoff = static_cast<int64_t>(h.imgcif->overload);
|
||||
h.exposure_s = h.imgcif->exposure_s;
|
||||
h.thickness_m = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (h.period_s <= 0.0)
|
||||
h.period_s = h.exposure_s;
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "../common/CompressedImage.h"
|
||||
#include "ImgCIF.h"
|
||||
|
||||
// PILATUS miniCBF: an ASCII header, four separator bytes, then one byte-offset compressed image.
|
||||
// No CIF parser and no libcbf are needed - every value is on a "# " comment line or a MIME line.
|
||||
@@ -39,6 +40,7 @@ struct Header {
|
||||
// does not really carry is absent here rather than 9999 degrees away from zero.
|
||||
std::optional<double> chi_deg;
|
||||
std::optional<double> omega_deg;
|
||||
std::optional<double> kappa_deg;
|
||||
double chi_increment_deg = 0;
|
||||
double phi_increment_deg = 0;
|
||||
double omega_increment_deg = 0;
|
||||
@@ -70,6 +72,13 @@ struct Header {
|
||||
// ScansPhi, and note that the one header carrying both a name and a table says "X" where its table
|
||||
// says Y) but this token is not: where both are present they agree.
|
||||
bool spindle_along_slow = false;
|
||||
|
||||
// A full imgCIF header - CBFlib's own form, which Bruker's APEX writes when it converts frames -
|
||||
// states none of the "# " lines and says everything in its CIF tables instead. Where the header
|
||||
// has no "# Pixel_size" line and those tables are there, this holds them, and the scalar fields
|
||||
// above that the tables state (pixel size, wavelength, Count_cutoff, exposure, detector) are
|
||||
// filled in from them. The geometry is the reader's to work out from the axis chain.
|
||||
std::optional<imgcif::Description> imgcif;
|
||||
};
|
||||
|
||||
// Whether the sweep turns PHI rather than the base (omega) axis. What moved is the axis with a
|
||||
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "SFRM.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
|
||||
#include "../common/JFJochException.h"
|
||||
|
||||
namespace sfrm {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr size_t BLOCK = 512;
|
||||
constexpr size_t LINE = 80;
|
||||
constexpr size_t KEY = 8; // "NROWS :" - seven characters of name and the colon
|
||||
|
||||
JFJochException Invalid(const std::string &what) {
|
||||
return {JFJochExceptionCategory::InputParameterInvalid, what};
|
||||
}
|
||||
|
||||
std::vector<uint8_t> ReadPrefix(const std::string &path, size_t bytes) {
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f)
|
||||
throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path);
|
||||
std::vector<uint8_t> out(bytes);
|
||||
f.read(reinterpret_cast<char *>(out.data()), static_cast<std::streamsize>(bytes));
|
||||
out.resize(static_cast<size_t>(f.gcount()));
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::string> Words(const std::map<std::string, std::string> &kv, const std::string &key) {
|
||||
std::vector<std::string> out;
|
||||
const auto it = kv.find(key);
|
||||
if (it == kv.end())
|
||||
return out;
|
||||
std::istringstream in(it->second);
|
||||
for (std::string w; in >> w;)
|
||||
out.push_back(w);
|
||||
return out;
|
||||
}
|
||||
|
||||
double Num(const std::map<std::string, std::string> &kv, const std::string &key, size_t i,
|
||||
double fallback = 0.0) {
|
||||
const auto w = Words(kv, key);
|
||||
if (i >= w.size())
|
||||
return fallback;
|
||||
try {
|
||||
return std::stod(w[i]);
|
||||
} catch (const std::logic_error &) {
|
||||
throw Invalid("Malformed number in SFRM header " + key + ": '" + w[i] + "'");
|
||||
}
|
||||
}
|
||||
|
||||
int64_t Int(const std::map<std::string, std::string> &kv, const std::string &key, size_t i,
|
||||
int64_t fallback = 0) {
|
||||
return static_cast<int64_t>(Num(kv, key, i, static_cast<double>(fallback)));
|
||||
}
|
||||
|
||||
// The 80-character lines of the header, as key -> value. A key that repeats (TITLE, CELL, MATRIX,
|
||||
// DETPAR, ...) continues its value on the next line, so the pieces are joined.
|
||||
std::map<std::string, std::string> Lines(const char *data, size_t size) {
|
||||
std::map<std::string, std::string> kv;
|
||||
for (size_t at = 0; at + LINE <= size; at += LINE) {
|
||||
if (data[at + KEY - 1] != ':')
|
||||
continue; // the padding after the last line
|
||||
std::string key(data + at, KEY - 1);
|
||||
key.erase(std::find_if(key.rbegin(), key.rend(), [](unsigned char c) { return !std::isspace(c); }).base(),
|
||||
key.end());
|
||||
const std::string value(data + at + KEY, LINE - KEY);
|
||||
auto &v = kv[key];
|
||||
v += (v.empty() ? "" : " ") + value;
|
||||
}
|
||||
return kv;
|
||||
}
|
||||
|
||||
bool HasExtension(const std::filesystem::path &p) {
|
||||
std::string ext = p.extension().string();
|
||||
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
|
||||
return ext == ".sfrm";
|
||||
}
|
||||
|
||||
bool IsSFRM(const std::string &path) {
|
||||
const auto head = ReadPrefix(path, KEY);
|
||||
return head.size() == KEY && std::memcmp(head.data(), "FORMAT :", KEY) == 0;
|
||||
}
|
||||
|
||||
// "ins_zwin_01_0042.sfrm" -> {"ins_zwin_01_", 4}: the run is part of the prefix, so naming a frame
|
||||
// of run 01 reads run 01 and not the frames of every run in the directory.
|
||||
struct Template {
|
||||
std::string prefix;
|
||||
size_t digits = 0;
|
||||
|
||||
bool Matches(const std::string &stem) const {
|
||||
if (stem.size() != prefix.size() + digits || stem.compare(0, prefix.size(), prefix) != 0)
|
||||
return false;
|
||||
return std::all_of(stem.begin() + static_cast<std::ptrdiff_t>(prefix.size()), stem.end(),
|
||||
[](unsigned char c) { return std::isdigit(c); });
|
||||
}
|
||||
};
|
||||
|
||||
std::optional<Template> TemplateOf(const std::filesystem::path &p) {
|
||||
const std::string stem = p.stem().string();
|
||||
size_t start = stem.size();
|
||||
while (start > 0 && std::isdigit(static_cast<unsigned char>(stem[start - 1])))
|
||||
start--;
|
||||
if (start == stem.size())
|
||||
return {};
|
||||
return Template{stem.substr(0, start), stem.size() - start};
|
||||
}
|
||||
|
||||
size_t Pad(size_t bytes, size_t to) {
|
||||
return (bytes + to - 1) / to * to;
|
||||
}
|
||||
|
||||
uint64_t ReadLE(const uint8_t *p, int64_t bytes) {
|
||||
uint64_t v = 0;
|
||||
for (int64_t i = bytes - 1; i >= 0; i--)
|
||||
v = (v << 8) | p[i];
|
||||
return v;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Header ParseHeader(const char *data, size_t size) {
|
||||
const auto kv = Lines(data, size);
|
||||
Header h;
|
||||
h.format = Int(kv, "FORMAT", 0);
|
||||
h.header_blocks = Int(kv, "HDRBLKS", 0);
|
||||
h.nrows = Int(kv, "NROWS", 0);
|
||||
h.ncols = Int(kv, "NCOLS", 0);
|
||||
h.bytes_per_pixel = Int(kv, "NPIXELB", 0, 1);
|
||||
h.bytes_per_underflow = Int(kv, "NPIXELB", 1, 1);
|
||||
if (h.format == 100) {
|
||||
for (size_t i = 0; i < 3; i++)
|
||||
h.overflows[i] = Int(kv, "NOVERFL", i, 0);
|
||||
} else {
|
||||
h.overflows = {Int(kv, "NOVERFL", 0, 0), 0, 0};
|
||||
}
|
||||
h.baseline = Int(kv, "NEXP", 2, 0);
|
||||
for (size_t i = 0; i < 4; i++)
|
||||
h.angles[i] = Num(kv, "ANGLES", i);
|
||||
h.axis = Int(kv, "AXIS", 0);
|
||||
h.start_deg = Num(kv, "START", 0);
|
||||
h.increment_deg = Num(kv, "INCREME", 0);
|
||||
const auto distance = Words(kv, "DISTANC");
|
||||
h.distance_cm = Num(kv, "DISTANC", distance.empty() ? 0 : distance.size() - 1);
|
||||
for (size_t i = 0; i < Words(kv, "CENTER").size(); i++)
|
||||
h.center.push_back(Num(kv, "CENTER", i));
|
||||
const auto dettype = Words(kv, "DETTYPE");
|
||||
h.detector = dettype.empty() ? "Bruker CCD" : dettype[0];
|
||||
h.pixels_per_cm_512 = Num(kv, "DETTYPE", 1);
|
||||
h.grid_offset_cm = Num(kv, "DETTYPE", 2);
|
||||
h.wavelength_A = Num(kv, "WAVELEN", 0);
|
||||
h.exposure_s = Num(kv, "CUMULAT", 0);
|
||||
h.saturation = Num(kv, "CCDPARM", 4);
|
||||
for (size_t i = 0; i < 6; i++)
|
||||
h.detpar[i] = Num(kv, "DETPAR", i);
|
||||
h.monochromator_2theta_deg = Num(kv, "FILTER2", 0);
|
||||
h.raw = kv;
|
||||
|
||||
if (h.header_blocks <= 0 || h.nrows <= 0 || h.ncols <= 0)
|
||||
throw Invalid("SFRM header states no HDRBLKS, NROWS or NCOLS");
|
||||
if (h.bytes_per_pixel != 1 && h.bytes_per_pixel != 2 && h.bytes_per_pixel != 4)
|
||||
throw Invalid("SFRM header states " + std::to_string(h.bytes_per_pixel) + " bytes per pixel");
|
||||
if (Int(kv, "WORDORD", 0) != 0 || Int(kv, "LONGORD", 0) != 0)
|
||||
throw Invalid("big-endian SFRM frames are not supported");
|
||||
if (h.format != 86 && h.format != 100)
|
||||
throw Invalid("SFRM FORMAT " + std::to_string(h.format) + " is not supported");
|
||||
return h;
|
||||
}
|
||||
|
||||
Header ReadHeader(const std::string &path) {
|
||||
// Enough to reach HDRBLKS (line 3), then exactly the blocks it states.
|
||||
auto buf = ReadPrefix(path, 5 * BLOCK);
|
||||
const auto blocks = Int(Lines(reinterpret_cast<const char *>(buf.data()), buf.size()), "HDRBLKS", 0);
|
||||
if (blocks <= 0)
|
||||
throw Invalid(path + " states no HDRBLKS");
|
||||
buf = ReadPrefix(path, static_cast<size_t>(blocks) * BLOCK);
|
||||
return ParseHeader(reinterpret_cast<const char *>(buf.data()), buf.size());
|
||||
}
|
||||
|
||||
Header Decode(const uint8_t *data, size_t size, int32_t *out, size_t capacity) {
|
||||
const size_t header_bytes = 5 * BLOCK;
|
||||
if (size < header_bytes)
|
||||
throw Invalid("SFRM file shorter than its header");
|
||||
const auto blocks = Int(Lines(reinterpret_cast<const char *>(data), header_bytes), "HDRBLKS", 0);
|
||||
if (blocks <= 0 || static_cast<size_t>(blocks) * BLOCK > size)
|
||||
throw Invalid("SFRM file shorter than its header");
|
||||
const Header h = ParseHeader(reinterpret_cast<const char *>(data), static_cast<size_t>(blocks) * BLOCK);
|
||||
|
||||
const size_t npixel = static_cast<size_t>(h.nrows) * static_cast<size_t>(h.ncols);
|
||||
if (npixel > capacity)
|
||||
throw Invalid("SFRM image does not fit the buffer given for it");
|
||||
size_t at = static_cast<size_t>(h.header_blocks) * BLOCK;
|
||||
if (at + npixel * h.bytes_per_pixel > size)
|
||||
throw Invalid("SFRM file ends before its image does");
|
||||
for (size_t i = 0; i < npixel; i++)
|
||||
out[i] = static_cast<int32_t>(ReadLE(data + at + i * h.bytes_per_pixel, h.bytes_per_pixel));
|
||||
at += npixel * h.bytes_per_pixel;
|
||||
|
||||
if (h.format == 86) {
|
||||
// One ASCII entry per overflowing pixel: its count in 9 characters, its offset in 7.
|
||||
const int64_t n = h.overflows[0];
|
||||
if (n > 0 && at + static_cast<size_t>(n) * 16 > size)
|
||||
throw Invalid("SFRM file ends inside its overflow table");
|
||||
for (int64_t k = 0; k < n; k++) {
|
||||
const std::string entry(reinterpret_cast<const char *>(data + at + 16 * k), 16);
|
||||
int64_t value, offset;
|
||||
try {
|
||||
value = std::stoll(entry.substr(0, 9));
|
||||
offset = std::stoll(entry.substr(9, 7));
|
||||
} catch (const std::logic_error &) {
|
||||
throw Invalid("malformed SFRM overflow entry '" + entry + "'");
|
||||
}
|
||||
if (offset < 0 || static_cast<size_t>(offset) >= npixel)
|
||||
throw Invalid("SFRM overflow entry points outside the image");
|
||||
out[offset] = static_cast<int32_t>(value);
|
||||
}
|
||||
} else {
|
||||
at = Pad(at, 16);
|
||||
// The three tables, and where the next unread entry of each is.
|
||||
const auto table = [&](int64_t n, int64_t bytes) {
|
||||
const size_t begin = at;
|
||||
if (n > 0) {
|
||||
at = Pad(at + static_cast<size_t>(n) * bytes, 16);
|
||||
if (begin + static_cast<size_t>(n) * bytes > size)
|
||||
throw Invalid("SFRM file ends inside its overflow tables");
|
||||
}
|
||||
return begin;
|
||||
};
|
||||
const size_t under = table(h.overflows[0], h.bytes_per_underflow);
|
||||
const size_t over16 = table(h.overflows[1], 2);
|
||||
const size_t over32 = table(h.overflows[2], 4);
|
||||
int64_t n_under = 0, n16 = 0, n32 = 0;
|
||||
const auto next = [&](size_t begin, int64_t &used, int64_t total, int64_t bytes) {
|
||||
if (used >= total)
|
||||
throw Invalid("SFRM frame has more overflowing pixels than its tables hold");
|
||||
return ReadLE(data + begin + static_cast<size_t>(used++) * bytes, bytes);
|
||||
};
|
||||
const uint64_t max_pixel = (h.bytes_per_pixel == 4) ? UINT32_MAX : (1ULL << (8 * h.bytes_per_pixel)) - 1;
|
||||
for (size_t i = 0; i < npixel; i++) {
|
||||
uint64_t v = static_cast<uint32_t>(out[i]);
|
||||
if (v == 0 && h.overflows[0] > 0)
|
||||
v = next(under, n_under, h.overflows[0], h.bytes_per_underflow);
|
||||
else if (h.bytes_per_pixel < 4 && v == max_pixel) {
|
||||
if (h.bytes_per_pixel == 1)
|
||||
v = next(over16, n16, h.overflows[1], 2);
|
||||
if (v == 0xFFFF) {
|
||||
const auto w = static_cast<int32_t>(next(over32, n32, h.overflows[2], 4));
|
||||
out[i] = static_cast<int32_t>(w - h.baseline);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out[i] = static_cast<int32_t>(static_cast<int64_t>(v) - h.baseline);
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < npixel; i++)
|
||||
out[i] -= static_cast<int32_t>(h.baseline);
|
||||
return h;
|
||||
}
|
||||
|
||||
Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vector<uint8_t> &scratch) {
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f)
|
||||
throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path);
|
||||
f.seekg(0, std::ios::end);
|
||||
scratch.resize(static_cast<size_t>(f.tellg()));
|
||||
f.seekg(0, std::ios::beg);
|
||||
f.read(reinterpret_cast<char *>(scratch.data()), static_cast<std::streamsize>(scratch.size()));
|
||||
try {
|
||||
return Decode(scratch.data(), static_cast<size_t>(f.gcount()), out, capacity);
|
||||
} catch (const JFJochException &e) {
|
||||
throw Invalid(path + ": " + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
Header Read(const std::string &path, std::vector<int32_t> &out) {
|
||||
const Header h = ReadHeader(path);
|
||||
out.resize(static_cast<size_t>(h.nrows) * static_cast<size_t>(h.ncols));
|
||||
std::vector<uint8_t> scratch;
|
||||
return ReadInto(path, out.data(), out.size(), scratch);
|
||||
}
|
||||
|
||||
std::vector<std::string> CollectSweep(const std::string &path) {
|
||||
const std::filesystem::path p(path);
|
||||
std::error_code ec;
|
||||
const bool is_dir = std::filesystem::is_directory(p, ec);
|
||||
std::filesystem::path dir = is_dir ? p : p.parent_path();
|
||||
if (dir.empty())
|
||||
dir = ".";
|
||||
|
||||
// Naming a frame selects its run. Naming a directory selects the run with the most frames, which
|
||||
// is the one a user pointing at a data directory means - a Bruker collection directory holds one
|
||||
// run per scan, and every run is its own sweep.
|
||||
std::optional<Template> want;
|
||||
if (!is_dir)
|
||||
want = TemplateOf(p);
|
||||
|
||||
std::map<std::pair<std::string, size_t>, std::vector<std::string>> runs;
|
||||
for (const auto &e: std::filesystem::directory_iterator(dir, ec)) {
|
||||
if (!e.is_regular_file() || !HasExtension(e.path()))
|
||||
continue;
|
||||
const auto t = TemplateOf(e.path());
|
||||
if (!t.has_value() || (want.has_value() && !want->Matches(e.path().stem().string())))
|
||||
continue;
|
||||
runs[{t->prefix, t->digits}].push_back(e.path().string());
|
||||
}
|
||||
std::vector<std::string> out;
|
||||
for (auto &[key, files]: runs)
|
||||
if (files.size() > out.size())
|
||||
out = std::move(files);
|
||||
std::sort(out.begin(), out.end());
|
||||
if (!out.empty() && !IsSFRM(out.front()))
|
||||
return {};
|
||||
return out;
|
||||
}
|
||||
|
||||
bool CanRead(const std::string &path) {
|
||||
std::error_code ec;
|
||||
try {
|
||||
if (std::filesystem::is_directory(path, ec))
|
||||
return !CollectSweep(path).empty();
|
||||
return HasExtension(std::filesystem::path(path)) && IsSFRM(path);
|
||||
} catch (const JFJochException &) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace sfrm
|
||||
@@ -0,0 +1,85 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Bruker .sfrm: the frame format of Bruker's SMART, PROTEUM and APEX software. An ASCII header of
|
||||
// 80-character "KEY :value" lines, HDRBLKS blocks of 512 bytes long, then the pixels as unsigned
|
||||
// little-endian integers of NPIXELB[0] bytes, row after row, then the tables that hold what did not
|
||||
// fit in that container.
|
||||
//
|
||||
// FORMAT :100 VERSION:11 HDRBLKS:15
|
||||
// NOVERFL:-1 5502 0 NPIXELB:1 1 NROWS :1024 NCOLS :1024
|
||||
// ANGLES :2theta omega phi chi AXIS :2 START :0.0 INCREME:-0.2
|
||||
// DISTANC:10.0 (cm) CENTER :506 511 ... WAVELEN:1.5419 1.5406 1.5444
|
||||
//
|
||||
// Two layouts of the tables exist. FORMAT 86 (VERSION < 11) has one ASCII overflow table after the
|
||||
// image: NOVERFL entries of 16 characters, a 9-digit count and a 7-digit pixel offset, padded to a
|
||||
// whole 512-byte block. FORMAT 100 has three binary tables, each padded to 16 bytes: NOVERFL[0]
|
||||
// underflows of NPIXELB[1] bytes (absent where NOVERFL[0] is -1), NOVERFL[1] 16-bit overflows and
|
||||
// NOVERFL[2] 32-bit ones. A pixel holding its container's maximum takes the next value of the next
|
||||
// wider table, in pixel order; a pixel holding 0 takes the next underflow. The 32-bit table is
|
||||
// signed - a pixel far enough below the baseline is stored there as a negative count. Every stored value carries
|
||||
// the baseline offset NEXP[2], so the counts are the values less that offset.
|
||||
//
|
||||
// The layout as Bruker's frame format documents it, checked byte for byte against APEX's own
|
||||
// conversion of the same frames to imgCIF: the pixels come out identical but for the baseline, which
|
||||
// the conversion subtracts.
|
||||
namespace sfrm {
|
||||
|
||||
struct Header {
|
||||
int64_t format = 0; // 86 or 100
|
||||
int64_t header_blocks = 0; // HDRBLKS
|
||||
int64_t nrows = 0;
|
||||
int64_t ncols = 0;
|
||||
int64_t bytes_per_pixel = 1; // NPIXELB[0]
|
||||
int64_t bytes_per_underflow = 1; // NPIXELB[1], FORMAT 100
|
||||
std::array<int64_t, 3> overflows{-1, 0, 0}; // NOVERFL: FORMAT 86 uses [0] only
|
||||
int64_t baseline = 0; // NEXP[2]
|
||||
std::array<double, 4> angles{}; // ANGLES: 2theta, omega, phi, chi at the start of the frame (deg)
|
||||
int64_t axis = 0; // AXIS: 1 2theta, 2 omega, 3 phi, 4 chi
|
||||
double start_deg = 0; // START
|
||||
double increment_deg = 0; // INCREME, signed
|
||||
double distance_cm = 0; // DISTANC, its last value
|
||||
std::vector<double> center; // CENTER, pixels
|
||||
double pixels_per_cm_512 = 0; // DETTYPE[1]: pixels per cm on a 512-pixel frame
|
||||
double grid_offset_cm = 0; // DETTYPE[2]: detector face to the calibration grid
|
||||
double wavelength_A = 0; // WAVELEN[0]
|
||||
double exposure_s = 0; // CUMULAT
|
||||
double saturation = 0; // CCDPARM[4]; 0 where not stated
|
||||
std::array<double, 6> detpar{}; // DETPAR: three offsets, then pitch, roll, yaw (deg)
|
||||
double monochromator_2theta_deg = 0; // FILTER2[0]: the monochromator's scattering angle; 0 for none
|
||||
std::string detector; // DETTYPE[0]
|
||||
std::map<std::string, std::string> raw; // every key; repeated keys joined with a space
|
||||
};
|
||||
|
||||
// True if the path names something this reader can open: a .sfrm frame, or a directory holding one.
|
||||
bool CanRead(const std::string &path);
|
||||
|
||||
// The files of one sweep, in collection order. path is a directory, or one frame inside it.
|
||||
std::vector<std::string> CollectSweep(const std::string &path);
|
||||
|
||||
// Parse a header from its bytes (at least HDRBLKS * 512 of them).
|
||||
Header ParseHeader(const char *data, size_t size);
|
||||
|
||||
// Header only - reads just the header blocks.
|
||||
Header ReadHeader(const std::string &path);
|
||||
|
||||
// Header + counts (stored value with the overflows resolved, less the baseline) into memory the
|
||||
// caller already has. scratch carries the raw bytes between frames.
|
||||
Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vector<uint8_t> &scratch);
|
||||
|
||||
// The same, decoding from the whole file already in memory. Throws if out holds fewer than
|
||||
// nrows * ncols pixels or the tables run past the end.
|
||||
Header Decode(const uint8_t *data, size_t size, int32_t *out, size_t capacity);
|
||||
|
||||
// Header + counts into a vector sized to fit.
|
||||
Header Read(const std::string &path, std::vector<int32_t> &out);
|
||||
|
||||
} // namespace sfrm
|
||||
@@ -17,6 +17,8 @@ ADD_LIBRARY(JFJochRugnux STATIC
|
||||
RugnuxFirstPassRescues.cpp
|
||||
RugnuxFirstPassAccept.cpp
|
||||
RugnuxScaleMerge.cpp
|
||||
RugnuxJointSweeps.cpp
|
||||
RugnuxJointSweeps.h
|
||||
RugnuxDefaults.cpp
|
||||
RugnuxDefaults.h
|
||||
RugnuxLive.cpp
|
||||
@@ -40,7 +42,8 @@ TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPrevie
|
||||
|
||||
# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux
|
||||
# returns: the library itself writes none.
|
||||
ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h RugnuxInputKind.cpp RugnuxInputKind.h)
|
||||
ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h RugnuxInputKind.cpp RugnuxInputKind.h
|
||||
RugnuxSweeps.cpp RugnuxSweeps.h)
|
||||
TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter ZLIB::ZLIB)
|
||||
|
||||
# rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the
|
||||
|
||||
@@ -27,6 +27,7 @@ namespace PathologyCode {
|
||||
constexpr const char *SYMMETRY_AMBIGUITY = "SYMMETRY_AMBIGUITY"; // the data do not pick a point group
|
||||
constexpr const char *CENTERING_UNTESTED = "CENTERING_UNTESTED"; // centring from the metric only
|
||||
constexpr const char *UNUSABLE_MERGE = "UNUSABLE_MERGE"; // the merged data carry no signal
|
||||
constexpr const char *SWEEP_LEFT_OUT = "SWEEP_LEFT_OUT"; // a sweep given is not in the joint merge
|
||||
constexpr const char *LOW_COMPLETENESS = "LOW_COMPLETENESS"; // too little of reciprocal space
|
||||
constexpr const char *SWEEP_GAPS = "SWEEP_GAPS"; // ranges that delivered much less
|
||||
constexpr const char *GONIO_SCALE = "GONIO_SCALE"; // the stage rotation is mis-scaled
|
||||
|
||||
@@ -941,6 +941,64 @@ ReportDocument BuildReportDocument(const std::string &output_prefix,
|
||||
" the post-refined geometry. Every number in this report describes the pass named above,\n"
|
||||
" whose files are {}.*; the header-geometry pass is not written.", output_prefix), true));
|
||||
doc.sections.push_back(std::move(s));
|
||||
|
||||
}
|
||||
|
||||
// ---------------------------------------------- 1b. SWEEPS (a joint merge of several)
|
||||
if (!result.joint_sweeps.empty()) {
|
||||
ReportSection s;
|
||||
s.title = "1b. SWEEPS";
|
||||
Add(s, KeyInt("SWEEP_COUNT", static_cast<int64_t>(result.joint_sweeps.size())));
|
||||
Add(s, KeyInt("SWEEP_REFERENCE", result.joint_reference_sweep + 1));
|
||||
if (!result.segment_decay_model.empty())
|
||||
Add(s, KeyText("SWEEP_DECAY_MODEL", result.segment_decay_model));
|
||||
for (size_t k = 0; k < result.joint_sweeps.size(); ++k) {
|
||||
const auto &w = result.joint_sweeps[k];
|
||||
const std::string p = "SWEEP_" + std::to_string(k + 1) + "_";
|
||||
const auto key = [&p](const char *name) { return p + name; };
|
||||
std::vector<std::string> names; // the keys outlive the calls that take their c_str()
|
||||
auto text = [&](const char *name, const std::string &v) {
|
||||
names.push_back(key(name));
|
||||
Add(s, KeyText(names.back().c_str(), v));
|
||||
};
|
||||
auto real = [&](const char *name, double v, const char *fmt_) {
|
||||
names.push_back(key(name));
|
||||
Add(s, KeyReal(names.back().c_str(), v, fmt_));
|
||||
};
|
||||
auto integer = [&](const char *name, int64_t v) {
|
||||
names.push_back(key(name));
|
||||
Add(s, KeyInt(names.back().c_str(), v));
|
||||
};
|
||||
text("INPUT", w.input);
|
||||
integer("IMAGES", w.images);
|
||||
text("INCLUDED", w.included ? "TRUE" : "FALSE");
|
||||
if (!w.space_group_alone.empty())
|
||||
text("SPACE_GROUP_ALONE", w.space_group_alone);
|
||||
if (w.cell_alone)
|
||||
text("CELL", CellString(*w.cell_alone));
|
||||
if (std::isfinite(w.cc_half_alone))
|
||||
real("CC_HALF_ALONE", w.cc_half_alone, "{:.4f}");
|
||||
if (std::isfinite(w.completeness_alone))
|
||||
real("COMPLETENESS_ALONE", w.completeness_alone, "{:.1f}");
|
||||
if (std::isfinite(w.d_min_alone))
|
||||
real("RESOLUTION_ALONE", w.d_min_alone, "{:.2f}");
|
||||
if (w.included) {
|
||||
text("REINDEX_OP", w.reindex_op);
|
||||
real("REINDEX_CC", w.reindex_cc, "{:.4f}");
|
||||
integer("BATCH_FIRST", w.batch_first);
|
||||
integer("BATCH_LAST", w.batch_last);
|
||||
if (w.segment >= 0 && w.segment < static_cast<int>(result.segment_results.size())) {
|
||||
const auto &seg = result.segment_results[w.segment];
|
||||
if (std::isfinite(seg.scale))
|
||||
real("SCALE", seg.scale, "{:.4f}");
|
||||
if (std::isfinite(seg.relative_b))
|
||||
real("RELATIVE_B", seg.relative_b, "{:.2f}");
|
||||
if (seg.dose_rank >= 0)
|
||||
integer("DOSE_RANK", seg.dose_rank + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
doc.sections.push_back(std::move(s));
|
||||
}
|
||||
|
||||
// ------------------------------------------------- XDS GEOMETRY KEYWORDS
|
||||
|
||||
+39
-5
@@ -832,6 +832,21 @@ std::vector<bool> Rugnux::PrepassSubset(int prepass_images) const {
|
||||
return keep;
|
||||
}
|
||||
|
||||
ProcessResult Rugnux::RunJoint(std::vector<IntegrationOutcome> outcomes, const UnitCell &cell,
|
||||
gemmi::CrystalSystem system, char centring, RugnuxObserver *observer) {
|
||||
const auto run_start = std::chrono::steady_clock::now();
|
||||
joint_outcomes_ = std::move(outcomes);
|
||||
joint_cell_ = cell;
|
||||
joint_system_ = system;
|
||||
joint_centring_ = centring;
|
||||
decide_late_ = true;
|
||||
ProcessResult result = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false);
|
||||
joint_outcomes_.reset();
|
||||
result.pass_count = 1;
|
||||
result.total_time_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - run_start).count();
|
||||
return result;
|
||||
}
|
||||
|
||||
ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
|
||||
Logger logger("Rugnux");
|
||||
ProcessResult result;
|
||||
@@ -863,7 +878,9 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
if (config_.stride <= 0)
|
||||
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Image stride must be positive");
|
||||
|
||||
const auto total_images_in_file = static_cast<int>(reader_.GetNumberOfImages());
|
||||
const bool joint = joint_outcomes_.has_value();
|
||||
const auto total_images_in_file = joint ? static_cast<int>(joint_outcomes_->size())
|
||||
: static_cast<int>(reader_.GetNumberOfImages());
|
||||
int end_image = config_.end_image;
|
||||
if (end_image < 0 || end_image > total_images_in_file)
|
||||
end_image = total_images_in_file;
|
||||
@@ -952,7 +969,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
|
||||
// After any geometry refinement, so the shadow is found about the beam centre actually used,
|
||||
// and before the azimuthal mapping and the output mask below, which both read pixel_mask_.
|
||||
if (config_.detect_beam_stop.has_value() || config_.adaptive_integration_radius)
|
||||
if (!joint && (config_.detect_beam_stop.has_value() || config_.adaptive_integration_radius))
|
||||
PreScan(start_image, early_images,
|
||||
std::min(config_.detect_beam_stop.value_or(BEAM_CENTER_PROJECTION_IMAGES),
|
||||
HotPixelFinder::MAX_SAMPLED_FRAMES), observer);
|
||||
@@ -1165,6 +1182,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
// (see the reflection mask in BraggIntegrationEngineCPU). A fixed group used to cost real
|
||||
// intensity accuracy for that reason.
|
||||
indexer->PredictAllCentringNodes(true);
|
||||
if (joint) {
|
||||
indexer->GetIntegrationOutcome() = std::move(*joint_outcomes_);
|
||||
joint_outcomes_.reset();
|
||||
}
|
||||
}
|
||||
|
||||
const auto start_time = std::chrono::steady_clock::now();
|
||||
@@ -1201,7 +1222,19 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
.scan_result = scan_result, .unmerged_mtz = unmerged_mtz};
|
||||
|
||||
// First pass of two-pass rotation indexing (full analysis only).
|
||||
if (indexing && force_rotation_result_.has_value()) {
|
||||
if (joint) {
|
||||
// The sweeps were indexed, integrated and combined each on its own; their fulls are the frames.
|
||||
result.rotation_lattice_found = true;
|
||||
result.indexing_rate = 1.0f;
|
||||
result.images_processed = static_cast<uint64_t>(images_to_process);
|
||||
result.consensus_cell = joint_cell_;
|
||||
result.consensus_centering = joint_centring_;
|
||||
end_msg.rotation_lattice = CrystalLattice(*joint_cell_);
|
||||
end_msg.rotation_lattice_type = LatticeMessage{.centering = joint_centring_, .niggli_class = 0,
|
||||
.crystal_system = joint_system_};
|
||||
result.rotation_lattice_type = end_msg.rotation_lattice_type;
|
||||
end_msg.unit_cell = result.consensus_cell;
|
||||
} else if (indexing && force_rotation_result_.has_value()) {
|
||||
// Supercell-collapse fallback: force pass-1's WHOLE indexing result (lattice + refined orientation /
|
||||
// search metadata / axis), not just its lattice - a lattice-only force loses that metadata and
|
||||
// integrates markedly worse.
|
||||
@@ -1239,7 +1272,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
//
|
||||
// Only an output pass probes: the answer the run returns is always one (RunAllPasses), and no other
|
||||
// pass's probe is read - the pre-pass, the beam-centre arm and the probe passes leave theirs unread.
|
||||
if (indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
|
||||
if (!joint && indexing && indexer && experiment_.IsRotationIndexing() && write_output && !geometry_prepass) {
|
||||
std::vector<int64_t> frames;
|
||||
for (const int o : select_equally_spaced_image_ordinals(images_to_process, 60))
|
||||
frames.push_back(o);
|
||||
@@ -1250,7 +1283,8 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
|
||||
std::optional<GPUImagePass> gpu_image_pass; // what a probe that does not fit beside it waits for
|
||||
if (!gpu_work_beside_)
|
||||
gpu_image_pass.emplace();
|
||||
ImagePass(p);
|
||||
if (!joint)
|
||||
ImagePass(p);
|
||||
gpu_image_pass.reset();
|
||||
image_pass_mark.Stop();
|
||||
|
||||
|
||||
@@ -115,6 +115,11 @@ struct ProcessConfig {
|
||||
// scaling.
|
||||
bool write_merged = true;
|
||||
|
||||
// One sweep of several merged as one (--sweeps joint): the run is made as on its own, but its written
|
||||
// merge hands back all of its fulls, past the resolution cut too (MergedOutput::scaled_fulls), for the
|
||||
// joint merge to scale and cut again (Rugnux::RunJoint).
|
||||
bool sweep_member = false;
|
||||
|
||||
SpotFindingSettings spot_finding; // FullAnalysis spot finding (and Calibration by spots)
|
||||
|
||||
// Calibration: how the powder rings are measured, and the calibrant's rings as q = 2*pi/d (1/A).
|
||||
@@ -942,6 +947,26 @@ struct ProcessResult {
|
||||
// spindle, an indexing ambiguity). The same messages the log warns about, each carrying a closed
|
||||
// PathologyCode for machinery beside the open sentence for a person.
|
||||
std::vector<ReportWarning> warnings;
|
||||
// A joint merge of several sweeps (Rugnux::RunJoint, --sweeps joint): what its merge measured per sweep
|
||||
// segment (RotationScaleMerge::Result::segments, in segment order) and how it modelled the decay, and
|
||||
// what the run knows of each sweep (filled by the caller, in input order). Empty on a run of one input.
|
||||
struct JointSegment { double scale = NAN, relative_b = NAN; int dose_rank = -1; };
|
||||
std::vector<JointSegment> segment_results;
|
||||
std::string segment_decay_model;
|
||||
struct JointSweep {
|
||||
std::string input;
|
||||
int64_t images = 0;
|
||||
bool included = false;
|
||||
int segment = -1; // its segment of the joint merge, where it is in it
|
||||
std::string space_group_alone;
|
||||
std::optional<UnitCell> cell_alone;
|
||||
std::string reindex_op; // onto the reference sweep's axes
|
||||
double reindex_cc = NAN;
|
||||
int64_t batch_first = 0, batch_last = -1;
|
||||
double cc_half_alone = NAN, completeness_alone = NAN, d_min_alone = NAN;
|
||||
};
|
||||
std::vector<JointSweep> joint_sweeps;
|
||||
int joint_reference_sweep = -1;
|
||||
|
||||
// How the crystal sat on the spindle: the angle to the nearest symmetry axis and that axis's
|
||||
// order. A property of the mounting rather than of the data, and the one number that says whether
|
||||
@@ -1323,6 +1348,11 @@ class Rugnux {
|
||||
// Whether the first passes decide their near ties on all the frames: offline, on a re-check, and on
|
||||
// the run made again.
|
||||
bool decide_late_ = false;
|
||||
// RunJoint's frames and cell, for the one pass it makes (RunPipeline); empty otherwise.
|
||||
std::optional<std::vector<IntegrationOutcome>> joint_outcomes_;
|
||||
std::optional<UnitCell> joint_cell_;
|
||||
gemmi::CrystalSystem joint_system_ = gemmi::CrystalSystem::Triclinic;
|
||||
char joint_centring_ = 'P';
|
||||
// Whether what the run otherwise decides on the pre-pass frames - the rotation-scale walk's refits,
|
||||
// the demoted metric-symmetry arm, the integration radius (RefinedPasses) - is decided on all the
|
||||
// frames: only on the run made again. Offline too these are decided on the pre-pass frames, and a near
|
||||
@@ -1725,6 +1755,14 @@ public:
|
||||
// Runs the configured workflow to completion or until Cancel(). Throws on setup failure.
|
||||
ProcessResult Run(RugnuxObserver *observer = nullptr);
|
||||
|
||||
// The joint merge of several sweeps (--sweeps joint): `outcomes` are the frames of all of them laid
|
||||
// end to end, each holding the fulls its own sweep combined (ScalingSettings::SweepSegments says which
|
||||
// frames are whose), indexed on the axes of `cell`. Nothing is read from the images: the merge, the
|
||||
// space-group search and everything after it run on these as on the outcomes of one sweep.
|
||||
// `system` and `centring` are those of the lattice the cell is the conventional cell of.
|
||||
ProcessResult RunJoint(std::vector<IntegrationOutcome> outcomes, const UnitCell &cell,
|
||||
gemmi::CrystalSystem system, char centring, RugnuxObserver *observer = nullptr);
|
||||
|
||||
// The pinned host buffers the engines of a run kept for each other (CUDAMemHelpers.h), handed back to the
|
||||
// driver. Not part of Run: a process that outlives the run (the broker, the viewer) calls it once the
|
||||
// results are out, as it takes a tenth of a second the user would otherwise wait for; one that ends
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "RugnuxJointSweeps.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <map>
|
||||
#include <numeric>
|
||||
|
||||
#include "../common/CrystalLattice.h"
|
||||
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
|
||||
#include "../gemmi_gph/gemmi/mtz.hpp"
|
||||
|
||||
JointSweepRelation RelateJointSweeps(const std::vector<JointMemberSweep> &sweeps) {
|
||||
JointSweepRelation out;
|
||||
out.sweeps.resize(sweeps.size());
|
||||
if (sweeps.empty())
|
||||
return out;
|
||||
out.reference = 0;
|
||||
for (size_t k = 1; k < sweeps.size(); ++k)
|
||||
if (sweeps[k].fulls.size() > sweeps[out.reference].fulls.size())
|
||||
out.reference = static_cast<int>(k);
|
||||
const JointMemberSweep &ref = sweeps[out.reference];
|
||||
out.sweeps[out.reference] = {true, gemmi::Op::identity(), 1.0, 1.0, {}};
|
||||
for (size_t k = 0; k < sweeps.size(); ++k) {
|
||||
if (static_cast<int>(k) == out.reference)
|
||||
continue;
|
||||
JointSweepRelation::Sweep &s = out.sweeps[k];
|
||||
if (sweeps[k].merged.empty()) {
|
||||
s.why_left_out = "it merged nothing";
|
||||
continue;
|
||||
}
|
||||
const ReferenceSettingChoice choice = ChooseReferenceSetting(sweeps[k].merged, sweeps[k].cell,
|
||||
sweeps[k].space_group, ref.merged, ref.cell,
|
||||
ref.space_group);
|
||||
s.cob = choice.cob;
|
||||
s.cc = choice.cc;
|
||||
s.identity_cc = choice.identity_cc;
|
||||
s.included = std::isfinite(choice.cc) && choice.cc >= MIN_REFERENCE_CC;
|
||||
if (!s.included)
|
||||
s.why_left_out = "its intensities correlate with the reference sweep's on no axes (best CC "
|
||||
+ std::to_string(choice.cc) + ")";
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
JointMergeInput BuildJointMergeInput(const std::vector<JointMemberSweep> &sweeps,
|
||||
const JointSweepRelation &relation) {
|
||||
JointMergeInput out;
|
||||
const CrystalLattice lattice(sweeps[relation.reference].cell);
|
||||
int32_t first = 0;
|
||||
for (size_t k = 0; k < sweeps.size(); ++k) {
|
||||
const auto &sweep = sweeps[k];
|
||||
const auto &rel = relation.sweeps[k];
|
||||
if (!rel.included || sweep.n_frames <= 0)
|
||||
continue;
|
||||
out.segments.push_back({first, static_cast<int32_t>(sweep.n_frames)});
|
||||
out.segment_sweep.push_back(static_cast<int>(k));
|
||||
out.outcomes.resize(first + sweep.n_frames);
|
||||
for (int64_t f = 0; f < sweep.n_frames; ++f)
|
||||
out.outcomes[first + f].latt = lattice;
|
||||
for (const ScaledFull &full : sweep.fulls) {
|
||||
if (!std::isfinite(full.I) || !std::isfinite(full.var_bkg) || !(full.var_per_I > 0.0f)
|
||||
|| !(full.d > 0.0f))
|
||||
continue;
|
||||
const auto frame = std::clamp<int64_t>(std::llround(full.image_number), 0, sweep.n_frames - 1);
|
||||
const gemmi::Op::Miller hkl = rel.cob.apply_to_hkl({{full.h, full.k, full.l}});
|
||||
// A full as the merge takes a pre-combined observation, so that the combine rebuilds the variance
|
||||
// its own sweep's merge gave it: a count c of intensity I/c at correction c = var_per_I is a
|
||||
// variance c^2 var_bkg' + c I at I, which is the sweep's var_bkg + var_per_I I with
|
||||
// var_bkg' = var_bkg / c^2 (the capture term folded in at the full's own intensity).
|
||||
const float c = full.var_per_I;
|
||||
const float capture_var = std::isfinite(full.capture) ? full.capture * full.I * full.capture * full.I : 0.0f;
|
||||
Reflection r{};
|
||||
r.h = hkl[0]; r.k = hkl[1]; r.l = hkl[2];
|
||||
r.image_number = static_cast<float>(first + frame);
|
||||
r.d = full.d;
|
||||
r.I = full.I / c;
|
||||
r.var_bkg = (full.var_bkg + capture_var) / (c * c);
|
||||
r.sigma = std::sqrt(std::max(r.var_bkg + std::max(0.0f, r.I), 1e-12f));
|
||||
r.prescaling_corr = c;
|
||||
r.partiality = 1.0f;
|
||||
r.zeta = 1.0f;
|
||||
r.image_scale_corr = c;
|
||||
r.observed = true;
|
||||
out.outcomes[first + frame].reflections.push_back(r);
|
||||
}
|
||||
first += static_cast<int32_t>(sweep.n_frames);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
int64_t JointBatchBase(const JointMergeInput &input) {
|
||||
int64_t longest = 0;
|
||||
for (const auto &seg : input.segments)
|
||||
longest = std::max<int64_t>(longest, seg.n_frames);
|
||||
int64_t base = 1;
|
||||
while (base <= longest)
|
||||
base *= 10;
|
||||
return base;
|
||||
}
|
||||
|
||||
MtzTable JointUnmergedMtzTable(const std::vector<ScaledFull> &fulls, const UnitCell &cell,
|
||||
const gemmi::SpaceGroup &space_group, const std::vector<JointMemberSweep> &sweeps,
|
||||
const JointMergeInput &input, float wavelength_A,
|
||||
const std::vector<std::string> &history) {
|
||||
MtzTable mtz;
|
||||
mtz.space_group_name = space_group.xhm();
|
||||
mtz.unit_cell = cell;
|
||||
mtz.title = "Rugnux unmerged observations, scaled, joint merge of several sweeps";
|
||||
mtz.history = history;
|
||||
mtz.datasets = {{"HKL_base", "HKL_base", "HKL_base", 0.0f}, {"unmerged", "", "unmerged", wavelength_A}};
|
||||
for (const auto &[label, type] : std::vector<std::pair<const char *, char>>{
|
||||
{"H", 'H'}, {"K", 'H'}, {"L", 'H'}, {"M/ISYM", 'Y'}, {"BATCH", 'B'}, {"I", 'J'}, {"SIGI", 'Q'},
|
||||
{"FRACTIONCALC", 'R'}, {"ROT", 'R'}})
|
||||
mtz.columns.push_back({label, type, 0, {}});
|
||||
|
||||
const int64_t base = JointBatchBase(input);
|
||||
// Per segment, a header for every frame: the sweep's own where its file has one, else its nearest one
|
||||
// turned by the frames between them (a sweep's file holds the frames its observations span, and a
|
||||
// batch with no header stops a reading program).
|
||||
std::vector<std::map<int32_t, MtzTable::Batch>> own(input.segments.size());
|
||||
for (size_t s = 0; s < input.segments.size(); ++s) {
|
||||
const auto &batches = sweeps[input.segment_sweep[s]].batches;
|
||||
if (batches.empty())
|
||||
continue;
|
||||
for (int32_t n = 1; n <= input.segments[s].n_frames; ++n) {
|
||||
const auto nearest = std::min_element(batches.begin(), batches.end(), [n](const auto &a, const auto &b) {
|
||||
return std::abs(a.number - n) < std::abs(b.number - n);
|
||||
});
|
||||
MtzTable::Batch copy = *nearest;
|
||||
if (copy.number != n && copy.floats.size() > 47) {
|
||||
const float shift = static_cast<float>(n - copy.number) * copy.floats[47];
|
||||
copy.floats[36] += shift;
|
||||
copy.floats[37] += shift;
|
||||
}
|
||||
own[s][n] = copy;
|
||||
copy.number = static_cast<int32_t>((s + 1) * base + n);
|
||||
mtz.batches.push_back(std::move(copy));
|
||||
}
|
||||
}
|
||||
|
||||
const gemmi::GroupOps ops = space_group.operations();
|
||||
const auto centring_absent = [&](int h, int k, int l) {
|
||||
for (const auto &t : ops.cen_ops)
|
||||
if ((h * t[0] + k * t[1] + l * t[2]) % gemmi::Op::DEN != 0)
|
||||
return true;
|
||||
return false;
|
||||
};
|
||||
gemmi::UnmergedHklMover mover(&space_group);
|
||||
std::vector<std::array<float, 9>> rows;
|
||||
for (const ScaledFull &f : fulls) {
|
||||
if (centring_absent(f.h, f.k, f.l))
|
||||
continue;
|
||||
const auto frame = static_cast<int32_t>(std::lround(f.image_number));
|
||||
size_t s = 0;
|
||||
while (s + 1 < input.segments.size() && frame >= input.segments[s + 1].first_frame)
|
||||
++s;
|
||||
const int32_t local = frame - input.segments[s].first_frame + 1; // the sweep's own batch
|
||||
float rot = NAN;
|
||||
if (const auto it = own[s].find(local); it != own[s].end() && it->second.floats.size() > 47)
|
||||
rot = it->second.floats[36] + it->second.floats[47] / 2.0f;
|
||||
std::array<int, 3> hkl{f.h, f.k, f.l};
|
||||
const int isym = mover.move_to_asu(hkl);
|
||||
rows.push_back({static_cast<float>(hkl[0]), static_cast<float>(hkl[1]), static_cast<float>(hkl[2]),
|
||||
static_cast<float>(isym), static_cast<float>((s + 1) * base + local), f.I, f.sigma, 1.0f,
|
||||
rot});
|
||||
}
|
||||
std::vector<size_t> order(rows.size());
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) {
|
||||
return std::lexicographical_compare(rows[a].begin(), rows[a].begin() + 5, rows[b].begin(), rows[b].begin() + 5);
|
||||
});
|
||||
for (size_t c = 0; c < mtz.columns.size(); ++c) {
|
||||
mtz.columns[c].values.resize(rows.size());
|
||||
for (size_t i = 0; i < order.size(); ++i)
|
||||
mtz.columns[c].values[i] = rows[order[i]][c];
|
||||
}
|
||||
mtz.sort_order = {1, 2, 3, 4, 5};
|
||||
return mtz;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "../common/MtzTable.h"
|
||||
#include "../common/Reflection.h"
|
||||
#include "../common/ScalingSettings.h"
|
||||
#include "../common/UnitCell.h"
|
||||
#include "../gemmi_gph/gemmi/symmetry.hpp"
|
||||
#include "../image_analysis/IntegrationOutcome.h"
|
||||
|
||||
// Several sweeps of one crystal merged as one (--sweeps joint), in two levels: each sweep is processed on
|
||||
// its own - indexed, integrated, its partials combined into fulls and scaled - and the fulls of all of them
|
||||
// are then scaled and merged together (RotationScaleMerge with SweepSegments), as XSCALE merges the data
|
||||
// sets XDS made of each sweep. Between the two the sweeps are put on one set of axes: the indexing each
|
||||
// was given is its own, and an ambiguous lattice or a different choice of axes indexes the same crystal
|
||||
// differently from one sweep to the next.
|
||||
|
||||
// What the joint merge takes from one sweep's own run.
|
||||
struct JointMemberSweep {
|
||||
std::string label;
|
||||
int64_t n_frames = 0;
|
||||
UnitCell cell{}; // as its reflections are written
|
||||
gemmi::SpaceGroup space_group; // the group its run determined
|
||||
std::vector<MergedReflection> merged;
|
||||
std::vector<ScaledFull> fulls; // all its fulls, unmerged, on the same axes
|
||||
std::vector<MtzTable::Batch> batches; // the batch headers of its own unmerged MTZ
|
||||
double cc_half = NAN, completeness = NAN, d_min = NAN; // its merge on its own
|
||||
};
|
||||
|
||||
// How each sweep is put on the axes of the reference - the sweep with the most fulls: the change of basis
|
||||
// (an axis permutation or a twin law of the lattice, ChooseReferenceSetting) under which its merged
|
||||
// intensities correlate best with the reference's, and that correlation. A sweep whose best correlation is
|
||||
// below MIN_REFERENCE_CC is not the same crystal form on any axes, and is left out.
|
||||
struct JointSweepRelation {
|
||||
struct Sweep {
|
||||
bool included = false;
|
||||
gemmi::Op cob = gemmi::Op::identity();
|
||||
double cc = 0.0; // with the reference, under cob
|
||||
double identity_cc = 0.0; // ... and on its own axes
|
||||
std::string why_left_out;
|
||||
};
|
||||
int reference = -1;
|
||||
std::vector<Sweep> sweeps;
|
||||
};
|
||||
JointSweepRelation RelateJointSweeps(const std::vector<JointMemberSweep> &sweeps);
|
||||
|
||||
// The joint merge's input: the frames of the included sweeps laid end to end, in the order given, each
|
||||
// holding its sweep's fulls on the reference's axes as pre-combined observations, and the segments that
|
||||
// say whose frames are whose.
|
||||
struct JointMergeInput {
|
||||
std::vector<IntegrationOutcome> outcomes;
|
||||
std::vector<SweepSegment> segments;
|
||||
std::vector<int> segment_sweep; // the sweep each segment is
|
||||
};
|
||||
JointMergeInput BuildJointMergeInput(const std::vector<JointMemberSweep> &sweeps,
|
||||
const JointSweepRelation &relation);
|
||||
|
||||
// The batch numbers of the joint merge: segment s (from 0) is batches (s + 1) * base + frame + 1, base the
|
||||
// power of ten past the longest sweep - 1001.. and 2001.. for sweeps of up to 999 frames - so a sweep's
|
||||
// batches read as its own frames and never run into the next sweep's.
|
||||
int64_t JointBatchBase(const JointMergeInput &input);
|
||||
|
||||
// The joint merge's unmerged MTZ: its scaled fulls (MergedOutput::scaled_fulls, on the merge's axes and
|
||||
// frames), each in the batch of its sweep's frame, under the batch headers that sweep's own run wrote -
|
||||
// its orientation, axis and angles - renumbered (JointBatchBase). Lattice-centring absences are left out,
|
||||
// as in a single sweep's file.
|
||||
MtzTable JointUnmergedMtzTable(const std::vector<ScaledFull> &fulls, const UnitCell &cell,
|
||||
const gemmi::SpaceGroup &space_group, const std::vector<JointMemberSweep> &sweeps,
|
||||
const JointMergeInput &input, float wavelength_A,
|
||||
const std::vector<std::string> &history);
|
||||
@@ -1671,6 +1671,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
result.scaling_iterations_partials = r.scaling_iterations_partials;
|
||||
result.scaling_iterations_fulls = r.scaling_iterations_fulls;
|
||||
result.scaling_converged = r.scaling_converged;
|
||||
result.segment_results.clear();
|
||||
for (const auto &seg : r.segments)
|
||||
result.segment_results.push_back({seg.scale, seg.relative_b, seg.dose_rank});
|
||||
result.segment_decay_model = r.segment_decay_model;
|
||||
if (for_search)
|
||||
result.search_scaling_converged = r.scaling_converged;
|
||||
Note(fmt::format("merge in {}: {} unique", label, r.merged.size()));
|
||||
@@ -1693,6 +1697,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
// Only a merge that may be written hands its fulls back: they are a copy as long as the
|
||||
// fulls, and the search merges and the pre-pass never write a .hkl.
|
||||
rsm->SetExportScaledFulls(!for_search && !geometry_prepass && write_output && config_.write_merged);
|
||||
rsm->SetExportScaledFullsUncut(config_.sweep_member);
|
||||
// Amplitudes are made once, on the merge that is written, where the anisotropy tensor
|
||||
// their prior uses is known (see below) - not on every merge the tail makes on its way.
|
||||
rsm->SetFrenchWilson(false);
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "RugnuxSweeps.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cmath>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <spdlog/fmt/fmt.h>
|
||||
|
||||
namespace {
|
||||
// How far two sweeps may state the wavelength apart and still be one dataset: JFJochMultiReader's
|
||||
// tolerance for the inputs of one stills run.
|
||||
constexpr double WAVELENGTH_TOLERANCE = 1e-3; // relative
|
||||
|
||||
std::string Lower(std::string s) {
|
||||
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
std::string RugnuxFrameTemplate(const std::string &path) {
|
||||
const std::filesystem::path p(path);
|
||||
std::string name = p.filename().string();
|
||||
size_t end = name.size();
|
||||
while (end > 0 && !std::isdigit(static_cast<unsigned char>(name[end - 1])))
|
||||
end--;
|
||||
if (end == 0)
|
||||
return {};
|
||||
size_t start = end;
|
||||
while (start > 0 && std::isdigit(static_cast<unsigned char>(name[start - 1])))
|
||||
start--;
|
||||
name.replace(start, end - start, std::string(end - start, '#'));
|
||||
return (p.parent_path() / name).string();
|
||||
}
|
||||
|
||||
RugnuxSweepPlan PlanRugnuxSweeps(const std::vector<std::string> &inputs,
|
||||
const std::function<RugnuxInputDescription(const std::string &)> &describe_file,
|
||||
const std::function<RugnuxInputDescription(const std::string &)> &describe_sweep) {
|
||||
RugnuxSweepPlan plan;
|
||||
if (inputs.size() < 2)
|
||||
return plan;
|
||||
|
||||
std::vector<RugnuxInputDescription> files;
|
||||
for (const auto &f: inputs)
|
||||
files.push_back(describe_file(f));
|
||||
const bool any_foreign = std::any_of(files.begin(), files.end(), [](const auto &d) { return d.foreign; });
|
||||
const size_t n_frames = std::count_if(files.begin(), files.end(),
|
||||
[](const auto &d) { return !d.frame_template.empty(); });
|
||||
// Inputs of the stills readers, or a mix of frames and masters: read as before, which takes the
|
||||
// former and refuses the latter.
|
||||
if (any_foreign || (n_frames != 0 && n_frames != inputs.size()))
|
||||
return plan;
|
||||
|
||||
if (n_frames == inputs.size()) {
|
||||
std::set<std::string> seen;
|
||||
for (size_t i = 0; i < inputs.size(); i++)
|
||||
if (seen.insert(files[i].frame_template).second)
|
||||
plan.sweeps.push_back({inputs[i], files[i].frame_template, true});
|
||||
if (plan.sweeps.size() == 1)
|
||||
return plan;
|
||||
} else {
|
||||
// A first input of stills is a stills run, read together as before - and refused there if any
|
||||
// other input turns. A rotation first input asks every other one.
|
||||
if (!describe_sweep(inputs[0]).rotation)
|
||||
return plan;
|
||||
std::vector<std::string> rotation{inputs[0]}, stills;
|
||||
for (size_t i = 1; i < inputs.size(); i++)
|
||||
(describe_sweep(inputs[i]).rotation ? rotation : stills).push_back(inputs[i]);
|
||||
if (!stills.empty())
|
||||
throw std::runtime_error(fmt::format(
|
||||
"{} is a rotation sweep and {} a stills collection: rotation sweeps and stills are not "
|
||||
"processed together; give them in separate runs", rotation.front(), stills.front()));
|
||||
for (const auto &f: inputs)
|
||||
plan.sweeps.push_back({f, f, false});
|
||||
}
|
||||
|
||||
std::vector<double> wavelength;
|
||||
for (const auto &s: plan.sweeps)
|
||||
wavelength.push_back(describe_sweep(s.input).wavelength_A);
|
||||
for (size_t k = 1; k < plan.sweeps.size(); k++)
|
||||
if (std::fabs(wavelength[k] - wavelength[0]) > WAVELENGTH_TOLERANCE * wavelength[0])
|
||||
throw std::runtime_error(fmt::format(
|
||||
"{} is at {:.5f} A and {} at {:.5f} A: sweeps at different wavelengths are not processed "
|
||||
"together; give them in separate runs", plan.sweeps[0].label, wavelength[0],
|
||||
plan.sweeps[k].label, wavelength[k]));
|
||||
plan.wavelength_A = wavelength[0];
|
||||
return plan;
|
||||
}
|
||||
|
||||
std::vector<std::string> RugnuxSiblingSweeps(const RugnuxSweepPlan &plan, const std::vector<std::string> &extensions) {
|
||||
namespace fs = std::filesystem;
|
||||
std::set<std::string> given, dirs_frames, dirs_masters;
|
||||
for (const auto &s: plan.sweeps) {
|
||||
const fs::path p(s.input);
|
||||
const std::string dir = p.parent_path().empty() ? "." : p.parent_path().string();
|
||||
if (s.frames) {
|
||||
given.insert(fs::path(s.label).filename().string());
|
||||
dirs_frames.insert(dir);
|
||||
} else {
|
||||
given.insert(p.filename().string());
|
||||
dirs_masters.insert(dir);
|
||||
}
|
||||
}
|
||||
auto has_extension = [&](std::string name) {
|
||||
name = Lower(name);
|
||||
if (name.size() > 3 && name.ends_with(".gz"))
|
||||
name.resize(name.size() - 3);
|
||||
return std::any_of(extensions.begin(), extensions.end(), [&](const auto &e) { return name.ends_with(e); });
|
||||
};
|
||||
|
||||
// template (or master) name -> files, in name order
|
||||
std::map<std::string, size_t> found;
|
||||
std::error_code ec;
|
||||
for (const auto &dir: dirs_frames)
|
||||
for (const auto &e: fs::directory_iterator(dir, ec)) {
|
||||
const std::string name = e.path().filename().string();
|
||||
if (!e.is_regular_file() || !has_extension(name))
|
||||
continue;
|
||||
const std::string t = fs::path(RugnuxFrameTemplate(e.path().string())).filename().string();
|
||||
if (!t.empty() && !given.contains(t))
|
||||
found[(fs::path(dir) / t).string()]++;
|
||||
}
|
||||
for (const auto &dir: dirs_masters)
|
||||
for (const auto &e: fs::directory_iterator(dir, ec)) {
|
||||
const std::string name = e.path().filename().string();
|
||||
if (e.is_regular_file() && name.ends_with("_master.h5") && !given.contains(name))
|
||||
found[(fs::path(dir) / name).string()] = 0;
|
||||
}
|
||||
|
||||
std::vector<std::string> out;
|
||||
for (const auto &[name, n]: found)
|
||||
out.push_back(n > 0 ? fmt::format("{} ({} frame{})", name, n, n == 1 ? "" : "s") : name);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// The rotation sweeps a rugnux command line names, decided from the files in the order given:
|
||||
// - frame files (CBF, marCCD, SMV, Bruker SFRM): one sweep per file-name template - the name with its last run of
|
||||
// digits, the frame number, taken out. Any frame of a sweep names the whole sweep, so several frames
|
||||
// of one template are one sweep;
|
||||
// - masters: one sweep per rotation master. Several stills masters stay one stills run, as before, and
|
||||
// a mix of rotation and stills masters is refused.
|
||||
// Sweeps at different wavelengths are refused. Nothing is planned for a single input, or for inputs only
|
||||
// the stills readers take (XFEL HDF5, CXI, XTC): those are read as they always were.
|
||||
|
||||
// What the planner needs to know about one input. `frame_template` is empty for anything but a frame
|
||||
// file; `rotation` and `wavelength_A` are asked only of the input that stands for a sweep.
|
||||
struct RugnuxInputDescription {
|
||||
std::string frame_template;
|
||||
bool foreign = false; // read only by the stills readers (XFEL HDF5, CXI, XTC)
|
||||
bool rotation = false;
|
||||
double wavelength_A = 0.0;
|
||||
};
|
||||
|
||||
struct RugnuxSweep {
|
||||
std::string input; // the file the sweep is read from: its first frame given, or its master
|
||||
std::string label; // the frame template or the master, for messages
|
||||
bool frames = false; // a sweep of frame files (label is its template)
|
||||
};
|
||||
|
||||
struct RugnuxSweepPlan {
|
||||
// Empty: the inputs are read together as before (one input, a stills run, or a mix the readers refuse).
|
||||
std::vector<RugnuxSweep> sweeps;
|
||||
double wavelength_A = 0.0; // of every sweep, where there are several
|
||||
};
|
||||
|
||||
// `describe_file` says what an input is from its name and contents without reading the sweep (its
|
||||
// frame template, or whether it is only for the stills readers); `describe_sweep` reads the input that
|
||||
// stands for a sweep for its rotation and wavelength. Throws std::runtime_error, with a message for the
|
||||
// user, on a mix of rotation sweeps and stills, or on sweeps at different wavelengths.
|
||||
RugnuxSweepPlan PlanRugnuxSweeps(const std::vector<std::string> &inputs,
|
||||
const std::function<RugnuxInputDescription(const std::string &)> &describe_file,
|
||||
const std::function<RugnuxInputDescription(const std::string &)> &describe_sweep);
|
||||
|
||||
// The template of a frame file name, with its directory: the last run of digits in the name replaced by
|
||||
// '#' characters, one per digit ("/d/x_1_0001.cbf" -> "/d/x_1_####.cbf"). Empty for a name with no digits.
|
||||
std::string RugnuxFrameTemplate(const std::string &path);
|
||||
|
||||
// The sweeps beside the given ones that were not given: frame templates of `extensions` (lower case, with
|
||||
// the dot; a trailing ".gz" is looked through) in the directories of the frame sweeps, and other
|
||||
// "*_master.h5" files beside the given masters - one line each, for the log. Only the inputs given are
|
||||
// processed; these are named so that a sweep left out by accident shows.
|
||||
std::vector<std::string> RugnuxSiblingSweeps(const RugnuxSweepPlan &plan, const std::vector<std::string> &extensions);
|
||||
+552
-130
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,501 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
// The two forms Bruker frames reach us in: a full imgCIF file (what APEX writes when it converts a
|
||||
// frame, CBFlib's own form - geometry in CIF tables, no "# " lines) and the native .sfrm. Every
|
||||
// fixture is synthetic and written byte for byte here.
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
#include "../common/JFJochMath.h"
|
||||
#include "../reader/JFJochCBFReader.h"
|
||||
#include "../reader/JFJochSFRMReader.h"
|
||||
#include "../reader/MiniCBF.h"
|
||||
#include "../reader/SFRM.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// x-CBF_BYTE_OFFSET for small images: int8 deltas, escaping to int16.
|
||||
std::string ByteOffset(const std::vector<int32_t> &pixels) {
|
||||
std::string out;
|
||||
int32_t prev = 0;
|
||||
for (const int32_t v: pixels) {
|
||||
const int32_t d = v - prev;
|
||||
if (d > -128 && d < 128) {
|
||||
out.push_back(static_cast<char>(static_cast<int8_t>(d)));
|
||||
} else {
|
||||
out.push_back(static_cast<char>(-128));
|
||||
const auto d16 = static_cast<int16_t>(d);
|
||||
out.append(reinterpret_cast<const char *>(&d16), 2);
|
||||
}
|
||||
prev = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct ImgCIFFrame {
|
||||
double omega = 30, kappa = 60, phi = 10;
|
||||
double omega_inc = 0, kappa_inc = 0, phi_inc = 0.5;
|
||||
double two_theta = 0;
|
||||
double distance = 50;
|
||||
std::string polarization; // extra _diffrn_radiation_wavelength lines, if any
|
||||
};
|
||||
|
||||
// A full imgCIF frame of a kappa head scanning phi, with a detector on a 2theta arm. The image's
|
||||
// fast index runs along laboratory +Y and its slow one along +X, both increasing, with the image
|
||||
// centred on the arm's axis: pixel (cols-1)/2, (rows-1)/2 is where the unswung beam meets it.
|
||||
void WriteImgCIF(const std::string &name, const ImgCIFFrame &f, int64_t nx, int64_t ny,
|
||||
const std::vector<int32_t> &pixels) {
|
||||
const double pixel = 0.12;
|
||||
std::ostringstream h;
|
||||
h << "###CBF: VERSION 1.7.11\n# CBF file written by CBFlib v0.9.6\n\ndata_image\n\n"
|
||||
<< "_diffrn_scan_frame.integration_time 5\n"
|
||||
<< "_diffrn_radiation_wavelength.id SOURCE\n_diffrn_radiation_wavelength.wavelength 0.71073\n"
|
||||
<< f.polarization << "\n"
|
||||
<< "loop_\n_axis.id\n_axis.depends_on\n_axis.equipment\n_axis.type\n"
|
||||
<< "_axis.vector[1]\n_axis.vector[2]\n_axis.vector[3]\n_axis.offset[1]\n_axis.offset[2]\n_axis.offset[3]\n"
|
||||
<< " OMEGA . goniometer rotation 1 0 0 0 0 0\n"
|
||||
<< " KAPPA OMEGA goniometer rotation 0.642788 0 -0.766044 0 0 0\n"
|
||||
<< " PHI KAPPA goniometer rotation 1 0 0 0 0 0\n"
|
||||
<< " TWOTHETA . detector rotation 1 0 0 0 0 0\n"
|
||||
<< " DX TWOTHETA detector translation 0 0 -1 0 0 0\n"
|
||||
<< " ELEMENT_X DX detector translation 0 1 0 " << -ny * pixel / 2 << " " << -nx * pixel / 2 << " 0\n"
|
||||
<< " ELEMENT_Y ELEMENT_X detector translation 1 0 0 0 0 0\n\n"
|
||||
<< "loop_\n_diffrn_scan_frame_axis.axis_id\n_diffrn_scan_frame_axis.displacement\n"
|
||||
<< "_diffrn_scan_frame_axis.angle\n"
|
||||
<< " DX " << f.distance << " ?\n TWOTHETA ? " << f.two_theta << "\n OMEGA ? " << f.omega
|
||||
<< "\n PHI ? " << f.phi << "\n KAPPA ? " << f.kappa << "\n\n"
|
||||
<< "loop_\n_diffrn_scan_axis.axis_id\n_diffrn_scan_axis.displacement_start\n"
|
||||
<< "_diffrn_scan_axis.displacement_increment\n_diffrn_scan_axis.displacement_range\n"
|
||||
<< "_diffrn_scan_axis.angle_start\n_diffrn_scan_axis.angle_increment\n_diffrn_scan_axis.angle_range\n"
|
||||
<< " DX " << f.distance << " 0 0 ? ? ?\n TWOTHETA ? ? ? " << f.two_theta << " 0 0\n"
|
||||
<< " OMEGA ? ? ? " << f.omega << " " << f.omega_inc << " " << f.omega_inc << "\n"
|
||||
<< " PHI ? ? ? " << f.phi << " " << f.phi_inc << " " << f.phi_inc << "\n"
|
||||
<< " KAPPA ? ? ? " << f.kappa << " " << f.kappa_inc << " " << f.kappa_inc << "\n\n"
|
||||
<< "_diffrn_measurement_axis.measurement_id GONIOMETER\n_diffrn_measurement_axis.axis_id PHI\n\n"
|
||||
<< "_diffrn_detector.id DETECTOR\n_diffrn_detector.detector CCD\n_diffrn_detector.type APEX2\n\n"
|
||||
<< "loop_\n_array_structure_list_axis.axis_set_id\n_array_structure_list_axis.axis_id\n"
|
||||
<< "_array_structure_list_axis.displacement\n_array_structure_list_axis.displacement_increment\n"
|
||||
<< " ELEMENT_X ELEMENT_X " << pixel / 2 << " " << pixel << "\n"
|
||||
<< " ELEMENT_Y ELEMENT_Y " << pixel / 2 << " " << pixel << "\n\n"
|
||||
<< "loop_\n_array_structure_list.array_id\n_array_structure_list.index\n_array_structure_list.dimension\n"
|
||||
<< "_array_structure_list.precedence\n_array_structure_list.direction\n_array_structure_list.axis_set_id\n"
|
||||
<< " IMAGE 1 " << nx << " 1 increasing ELEMENT_X\n IMAGE 2 " << ny << " 2 increasing ELEMENT_Y\n\n"
|
||||
<< "_array_intensities.array_id IMAGE\n_array_intensities.gain 15.7\n_array_intensities.overload 600000\n\n"
|
||||
<< "_array_data.array_id IMAGE\n_array_data.binary_id 1\n_array_data.data\n;\n"
|
||||
<< "--CIF-BINARY-FORMAT-SECTION--\nContent-Type: application/octet-stream;\n"
|
||||
<< " conversions=\"x-CBF_BYTE_OFFSET\"\nContent-Transfer-Encoding: BINARY\n"
|
||||
<< "X-Binary-Element-Type: \"signed 32-bit integer\"\n"
|
||||
<< "X-Binary-Number-of-Elements: " << nx * ny << "\n"
|
||||
<< "X-Binary-Size-Fastest-Dimension: " << nx << "\n"
|
||||
<< "X-Binary-Size-Second-Dimension: " << ny << "\n\n";
|
||||
std::ofstream out(name, std::ios::binary);
|
||||
const std::string text = h.str();
|
||||
out.write(text.data(), static_cast<std::streamsize>(text.size()));
|
||||
out.write(reinterpret_cast<const char *>(minicbf::BINARY_SEPARATOR), sizeof(minicbf::BINARY_SEPARATOR));
|
||||
const std::string data = ByteOffset(pixels);
|
||||
out.write(data.data(), static_cast<std::streamsize>(data.size()));
|
||||
}
|
||||
|
||||
std::string FrameName(const std::string &prefix, int frame, const std::string &ext) {
|
||||
std::ostringstream name;
|
||||
name << prefix << "_" << std::setfill('0') << std::setw(4) << frame << ext;
|
||||
return name.str();
|
||||
}
|
||||
|
||||
Coord Internal(double x, double y, double z) {
|
||||
return {static_cast<float>(x), static_cast<float>(-y), static_cast<float>(-z)};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("JFJochCBFReader_FullImgCIF_KappaChain", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 24, ny = 16;
|
||||
std::vector<int32_t> pixels(nx * ny, 5);
|
||||
pixels[3] = -32; // outside the active area
|
||||
pixels[17] = 1000; // escapes to int16
|
||||
|
||||
SECTION("phi scan on a kappa head, square-on detector") {
|
||||
for (int frame = 1; frame <= 3; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.phi = 10 + 0.5 * (frame - 1);
|
||||
WriteImgCIF(FrameName("imgcif_kappa", frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
REQUIRE(JFJochCBFReader::CanRead("imgcif_kappa_0002.cbf"));
|
||||
JFJochCBFReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("imgcif_kappa_0002.cbf"));
|
||||
CHECK(reader.GetNumberOfImages() == 3);
|
||||
CHECK(reader.CountsInADU());
|
||||
const auto x = reader.GetDataset()->experiment;
|
||||
|
||||
// phi turns about its own axis as omega (30) and kappa (60) hold it; GoniometerAxis takes
|
||||
// minus the right-handed direction
|
||||
const Coord kappa_axis = Internal(0.642788, 0, -0.766044);
|
||||
const Coord expected = -(RotMatrix(static_cast<float>(30 * PI / 180), Coord(1, 0, 0))
|
||||
* (RotMatrix(static_cast<float>(60 * PI / 180), kappa_axis) * Coord(1, 0, 0)));
|
||||
REQUIRE(x.GetGoniometer().has_value());
|
||||
CHECK((x.GetGoniometer()->GetAxis() - expected).Length() < 1e-5f);
|
||||
CHECK(x.GetGoniometer()->GetStart_deg() == Catch::Approx(10.0));
|
||||
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
|
||||
CHECK(x.GetGoniometer()->GetName() == "phi");
|
||||
|
||||
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(50.0));
|
||||
CHECK(x.GetBeamX_pxl() == Catch::Approx((nx - 1) / 2.0));
|
||||
CHECK(x.GetBeamY_pxl() == Catch::Approx((ny - 1) / 2.0));
|
||||
CHECK(x.GetPixelSize_mm() == Catch::Approx(0.12));
|
||||
CHECK(x.GetWavelength_A() == Catch::Approx(0.71073));
|
||||
CHECK(x.GetSaturationLimit() == 600000);
|
||||
CHECK_FALSE(x.GetPolarizationFactor().has_value()); // the header states none
|
||||
// fast along laboratory +Y (internal -y), slow along +X
|
||||
const auto geom = x.GetDiffractionGeometry();
|
||||
CHECK((geom.GetFastAxis() - Coord(0, -1, 0)).Length() < 1e-5f);
|
||||
CHECK((geom.GetSlowAxis() - Coord(1, 0, 0)).Length() < 1e-5f);
|
||||
CHECK((geom.GetNormalAxis() - Coord(0, 0, 1)).Length() < 1e-5f);
|
||||
|
||||
// ReadRawImage hands the byte-offset stream on undecoded; LoadImage decodes it
|
||||
const auto image = reader.LoadImage(1);
|
||||
REQUIRE(image);
|
||||
CHECK(image->Image()[0] == 5);
|
||||
CHECK(image->Image()[17] == 1000);
|
||||
CHECK(reader.GetDataset()->pixel_mask->GetMask()[3] != 0);
|
||||
CHECK(reader.GetDataset()->pixel_mask->GetMask()[0] == 0);
|
||||
reader.Close();
|
||||
}
|
||||
|
||||
SECTION("a detector swung out on the 2theta arm") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.phi = 10 + 0.5 * (frame - 1);
|
||||
f.two_theta = -20;
|
||||
WriteImgCIF(FrameName("imgcif_swing", frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
JFJochCBFReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("imgcif_swing_0001.cbf"));
|
||||
const auto x = reader.GetDataset()->experiment;
|
||||
// The arm carries the square-on geometry with it: distance and foot of the normal stay.
|
||||
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(50.0));
|
||||
CHECK(x.GetBeamX_pxl() == Catch::Approx((nx - 1) / 2.0).margin(1e-3));
|
||||
CHECK(x.GetBeamY_pxl() == Catch::Approx((ny - 1) / 2.0).margin(1e-3));
|
||||
const Coord normal = RotMatrix(static_cast<float>(-20 * PI / 180), Coord(1, 0, 0)) * Coord(0, 0, 1);
|
||||
CHECK((x.GetDiffractionGeometry().GetNormalAxis() - normal).Length() < 1e-5f);
|
||||
reader.Close();
|
||||
}
|
||||
|
||||
for (int frame = 1; frame <= 3; frame++) {
|
||||
remove(FrameName("imgcif_kappa", frame, ".cbf").c_str());
|
||||
remove(FrameName("imgcif_swing", frame, ".cbf").c_str());
|
||||
}
|
||||
}
|
||||
|
||||
// What the full imgCIF path must never do: read a series it cannot describe as one rotation about
|
||||
// one axis as if it could.
|
||||
TEST_CASE("JFJochCBFReader_FullImgCIF_Polarization", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 24, ny = 16;
|
||||
const std::vector<int32_t> pixels(nx * ny, 5);
|
||||
auto read = [&](const std::string &prefix, const std::string &lines) {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.phi = 10 + 0.5 * (frame - 1);
|
||||
f.polarization = lines;
|
||||
WriteImgCIF(FrameName(prefix, frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
JFJochCBFReader reader;
|
||||
reader.ReadFiles(FrameName(prefix, 1, ".cbf"));
|
||||
return reader.GetDataset()->experiment.GetPolarizationFactor();
|
||||
};
|
||||
// the degree as stated, in the horizontal plane when the plane's normal is the laboratory Y axis
|
||||
const auto horizontal = read("imgcif_pol_h", "_diffrn_radiation_wavelength.polarizn_source_ratio 0.0444\n"
|
||||
"_diffrn_radiation_wavelength.polarizn_source_norm 0\n");
|
||||
REQUIRE(horizontal.has_value());
|
||||
CHECK(*horizontal == Catch::Approx(0.0444));
|
||||
// a normal turned by 90 degrees puts the plane vertical, which the correction takes as the opposite sign
|
||||
const auto vertical = read("imgcif_pol_v", "_diffrn_radiation_wavelength.polarizn_source_ratio 0.5\n"
|
||||
"_diffrn_radiation_wavelength.polarizn_source_norm 90\n");
|
||||
REQUIRE(vertical.has_value());
|
||||
CHECK(*vertical == Catch::Approx(-0.5));
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochCBFReader_FullImgCIF_refuses_what_it_cannot_describe", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 8, ny = 8;
|
||||
const std::vector<int32_t> pixels(nx * ny, 1);
|
||||
|
||||
SECTION("two axes moving at once") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.omega_inc = 0.5;
|
||||
WriteImgCIF(FrameName("imgcif_two", frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
CHECK(JFJochCBFReader::CanRead("imgcif_two_0001.cbf"));
|
||||
JFJochCBFReader reader;
|
||||
CHECK_THROWS_AS(reader.ReadFiles("imgcif_two_0001.cbf"), JFJochException);
|
||||
}
|
||||
|
||||
SECTION("kappa moved between frames of one name template") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.phi = 10 + 0.5 * (frame - 1);
|
||||
f.kappa = (frame == 1) ? 60 : 0;
|
||||
WriteImgCIF(FrameName("imgcif_reset", frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
JFJochCBFReader reader;
|
||||
CHECK_THROWS_AS(reader.ReadFiles("imgcif_reset_0001.cbf"), JFJochException);
|
||||
}
|
||||
|
||||
SECTION("nothing turns") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
ImgCIFFrame f;
|
||||
f.phi_inc = 0;
|
||||
WriteImgCIF(FrameName("imgcif_still", frame, ".cbf"), f, nx, ny, pixels);
|
||||
}
|
||||
JFJochCBFReader reader;
|
||||
CHECK_THROWS_AS(reader.ReadFiles("imgcif_still_0001.cbf"), JFJochException);
|
||||
}
|
||||
|
||||
for (int frame = 1; frame <= 2; frame++)
|
||||
for (const char *p: {"imgcif_two", "imgcif_reset", "imgcif_still"})
|
||||
remove(FrameName(p, frame, ".cbf").c_str());
|
||||
}
|
||||
|
||||
TEST_CASE("ImgCIF_Item_and_Loop", "[HDF5][Full][portable]") {
|
||||
const std::string text = "\n_a.x 1\nloop_\n_b.id\n_b.v\n p 1\n q 2\n\n_c.id 'quoted value'\n";
|
||||
CHECK(imgcif::Item(text, "_a.x").value_or("") == "1");
|
||||
CHECK(imgcif::Item(text, "_c.id").value_or("") == "quoted value");
|
||||
CHECK_FALSE(imgcif::Item(text, "_b.id").has_value()); // heads a loop
|
||||
const auto rows = imgcif::ParseLoop(text, "_b.id");
|
||||
REQUIRE(rows.size() == 2);
|
||||
CHECK(rows[1].at("_b.v") == "2");
|
||||
// a tag outside any loop is not the rows of the loop before it
|
||||
CHECK(imgcif::ParseLoop(text, "_c.id").empty());
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
// One FORMAT 100 frame: 1-byte pixels, the 16- and 32-bit overflow tables, baseline 32.
|
||||
struct SFRMFrame {
|
||||
int64_t axis = 2;
|
||||
double start = 0, increment = 0.5;
|
||||
std::array<double, 4> angles{0, 0, 0, 0}; // 2theta omega phi chi
|
||||
double distance_cm = 5.0;
|
||||
double monochromator_2theta = 0; // FILTER2
|
||||
std::vector<uint8_t> pixels;
|
||||
std::vector<uint16_t> over16;
|
||||
std::vector<int32_t> over32;
|
||||
};
|
||||
|
||||
std::string Line(const std::string &key, const std::string &value) {
|
||||
std::string k = key;
|
||||
k.resize(7, ' ');
|
||||
std::string v = value;
|
||||
v.resize(72, ' ');
|
||||
return k + ":" + v;
|
||||
}
|
||||
|
||||
void Pad(std::string &s, size_t to) {
|
||||
s.resize((s.size() + to - 1) / to * to, '\0');
|
||||
}
|
||||
|
||||
void WriteSFRM(const std::string &name, const SFRMFrame &f, int64_t n) {
|
||||
std::ostringstream angles;
|
||||
for (double a: f.angles)
|
||||
angles << a << " ";
|
||||
std::string h;
|
||||
h += Line("FORMAT", "100");
|
||||
h += Line("VERSION", "11");
|
||||
h += Line("HDRBLKS", "15");
|
||||
h += Line("NOVERFL", "-1 " + std::to_string(f.over16.size()) + " " + std::to_string(f.over32.size()));
|
||||
h += Line("NPIXELB", "1 1");
|
||||
h += Line("NROWS", std::to_string(n));
|
||||
h += Line("NCOLS", std::to_string(n));
|
||||
h += Line("WORDORD", "0");
|
||||
h += Line("LONGORD", "0");
|
||||
h += Line("START", std::to_string(f.start));
|
||||
h += Line("INCREME", std::to_string(f.increment));
|
||||
h += Line("ANGLES", angles.str());
|
||||
h += Line("AXIS", std::to_string(f.axis));
|
||||
h += Line("DISTANC", std::to_string(f.distance_cm));
|
||||
h += Line("CENTER", std::to_string(n / 2.0) + " " + std::to_string(n / 2.0));
|
||||
h += Line("WAVELEN", "1.54184 1.54056 1.54439");
|
||||
h += Line("CUMULAT", "10.0");
|
||||
h += Line("DETTYPE", "CCD-TEST 64.0 0.1 0 0.254 0.05 1");
|
||||
h += Line("NEXP", "2 0 32 0 1");
|
||||
h += Line("CCDPARM", "14 10 170 0 900000");
|
||||
h += Line("DETPAR", "0 0 0 0 0 0");
|
||||
h += Line("FILTER2", std::to_string(f.monochromator_2theta) + " 0");
|
||||
h.resize(15 * 512, ' ');
|
||||
std::string body(f.pixels.begin(), f.pixels.end());
|
||||
Pad(body, 16);
|
||||
std::string t16(reinterpret_cast<const char *>(f.over16.data()), f.over16.size() * 2);
|
||||
Pad(t16, 16);
|
||||
std::string t32(reinterpret_cast<const char *>(f.over32.data()), f.over32.size() * 4);
|
||||
Pad(t32, 16);
|
||||
std::ofstream out(name, std::ios::binary);
|
||||
for (const auto *s: {&h, &body, &t16, &t32})
|
||||
out.write(s->data(), static_cast<std::streamsize>(s->size()));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("SFRM_Decode_overflow_tables", "[HDF5][Full][portable]") {
|
||||
const int64_t n = 8;
|
||||
SFRMFrame f;
|
||||
f.pixels.assign(n * n, 40); // 8 counts above the baseline of 32
|
||||
f.pixels[0] = 0; // outside the active area: below the baseline
|
||||
f.pixels[5] = 255; // -> 16-bit table: 1000
|
||||
f.pixels[9] = 255; // -> 16-bit 65535 -> 32-bit 100000
|
||||
f.pixels[20] = 255; // -> 16-bit 65535 -> 32-bit -14 (signed)
|
||||
f.pixels[30] = 255; // -> 16-bit 300
|
||||
f.over16 = {1000, 65535, 65535, 300};
|
||||
f.over32 = {100000, -14};
|
||||
WriteSFRM("sfrm_tables_0001.sfrm", f, n);
|
||||
|
||||
REQUIRE(sfrm::CanRead("sfrm_tables_0001.sfrm"));
|
||||
std::vector<int32_t> v;
|
||||
const auto h = sfrm::Read("sfrm_tables_0001.sfrm", v);
|
||||
CHECK(h.format == 100);
|
||||
CHECK(h.baseline == 32);
|
||||
REQUIRE(v.size() == static_cast<size_t>(n * n));
|
||||
CHECK(v[1] == 8);
|
||||
CHECK(v[0] == -32);
|
||||
CHECK(v[5] == 1000 - 32);
|
||||
CHECK(v[9] == 100000 - 32);
|
||||
CHECK(v[20] == -14 - 32);
|
||||
CHECK(v[30] == 300 - 32);
|
||||
remove("sfrm_tables_0001.sfrm");
|
||||
|
||||
SECTION("a table shorter than the pixels that point into it is an error") {
|
||||
SFRMFrame g = f;
|
||||
g.over16 = {1000};
|
||||
g.over32 = {};
|
||||
WriteSFRM("sfrm_short_0001.sfrm", g, n);
|
||||
CHECK_THROWS_AS(sfrm::Read("sfrm_short_0001.sfrm", v), JFJochException);
|
||||
remove("sfrm_short_0001.sfrm");
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("SFRM_Decode_format86_ascii_overflows", "[HDF5][Full][portable]") {
|
||||
const int64_t n = 4;
|
||||
std::string h;
|
||||
h += Line("FORMAT", "86");
|
||||
h += Line("VERSION", "9");
|
||||
h += Line("HDRBLKS", "5");
|
||||
h += Line("NOVERFL", "2");
|
||||
h += Line("NPIXELB", "1");
|
||||
h += Line("NROWS", std::to_string(n));
|
||||
h += Line("NCOLS", std::to_string(n));
|
||||
h.resize(5 * 512, ' ');
|
||||
std::string body(n * n, static_cast<char>(7));
|
||||
body[2] = static_cast<char>(255);
|
||||
body[11] = static_cast<char>(255);
|
||||
std::string table = " 70000 2 123456 11";
|
||||
Pad(table, 512);
|
||||
std::ofstream("sfrm_old_0001.sfrm", std::ios::binary) << h << body << table;
|
||||
|
||||
std::vector<int32_t> v;
|
||||
const auto hdr = sfrm::Read("sfrm_old_0001.sfrm", v);
|
||||
CHECK(hdr.format == 86);
|
||||
CHECK(v[0] == 7);
|
||||
CHECK(v[2] == 70000);
|
||||
CHECK(v[11] == 123456);
|
||||
remove("sfrm_old_0001.sfrm");
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochSFRMReader_Geometry", "[HDF5][Full][portable]") {
|
||||
const int64_t n = 8;
|
||||
SFRMFrame f;
|
||||
f.pixels.assign(n * n, 40);
|
||||
f.pixels[0] = 0;
|
||||
|
||||
SECTION("omega scan") {
|
||||
for (int frame = 1; frame <= 3; frame++) {
|
||||
f.start = 0.5 * (frame - 1);
|
||||
f.angles = {0, f.start, 0, 54.74};
|
||||
WriteSFRM(FrameName("sfrm_omega", frame, ".sfrm"), f, n);
|
||||
}
|
||||
REQUIRE(JFJochSFRMReader::CanRead("sfrm_omega_0001.sfrm"));
|
||||
JFJochSFRMReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_omega_0001.sfrm"));
|
||||
CHECK(reader.GetNumberOfImages() == 3);
|
||||
const auto x = reader.GetDataset()->experiment;
|
||||
REQUIRE(x.GetGoniometer().has_value());
|
||||
// omega is the outermost axis, so a fixed chi does not tilt it
|
||||
CHECK((x.GetGoniometer()->GetAxis() - Coord(1, 0, 0)).Length() < 1e-5f);
|
||||
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
|
||||
// DETTYPE: 64 pixels/cm on a 512 frame, so 1/64 cm * 512/8 per pixel; distance with the
|
||||
// 0.1 cm face-to-grid offset
|
||||
CHECK(x.GetPixelSize_mm() == Catch::Approx(10.0 / 64.0 * 512.0 / n));
|
||||
CHECK(x.GetDetectorDistance_mm() == Catch::Approx(51.0));
|
||||
CHECK(x.GetBeamX_pxl() == Catch::Approx((n - 1) / 2.0).margin(1e-3));
|
||||
CHECK(x.GetBeamY_pxl() == Catch::Approx((n - 1) / 2.0).margin(1e-3));
|
||||
CHECK(x.GetSaturationLimit() == 900000);
|
||||
CHECK(x.GetWavelength_A() == Catch::Approx(1.54184));
|
||||
// laboratory frames with no monochromator: unpolarized, not the synchrotron default
|
||||
REQUIRE(x.GetPolarizationFactor().has_value());
|
||||
CHECK(*x.GetPolarizationFactor() == Catch::Approx(0.0).margin(1e-6));
|
||||
CHECK(reader.GetDataset()->pixel_mask->GetMask()[0] != 0);
|
||||
CHECK(reader.GetDataset()->pixel_mask->GetMask()[1] == 0);
|
||||
|
||||
JFJochReaderRawImage raw;
|
||||
REQUIRE(reader.ReadRawImage(2, raw));
|
||||
CHECK(reinterpret_cast<const int32_t *>(raw.image.GetCompressed())[1] == 8);
|
||||
reader.Close();
|
||||
}
|
||||
|
||||
SECTION("a sweep that turns backwards is stated forwards about the opposite axis") {
|
||||
for (int frame = 1; frame <= 3; frame++) {
|
||||
f.start = 10.0 - 0.5 * (frame - 1);
|
||||
f.increment = -0.5;
|
||||
f.angles = {0, f.start, 0, 54.74};
|
||||
f.monochromator_2theta = 12.17;
|
||||
WriteSFRM(FrameName("sfrm_back", frame, ".sfrm"), f, n);
|
||||
}
|
||||
JFJochSFRMReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_back_0001.sfrm"));
|
||||
CHECK(reader.GetNumberOfImages() == 3);
|
||||
const auto x = reader.GetDataset()->experiment;
|
||||
REQUIRE(x.GetGoniometer().has_value());
|
||||
CHECK((x.GetGoniometer()->GetAxis() - Coord(-1, 0, 0)).Length() < 1e-5f);
|
||||
CHECK(x.GetGoniometer()->GetIncrement_deg() == Catch::Approx(0.5));
|
||||
CHECK(x.GetGoniometer()->GetStart_deg() == Catch::Approx(-10.0));
|
||||
// a graphite monochromator: the 1 - cos^2(2theta_m) APEX's conversion states, horizontal
|
||||
REQUIRE(x.GetPolarizationFactor().has_value());
|
||||
const double c = std::cos(12.17 * PI / 180.0);
|
||||
CHECK(*x.GetPolarizationFactor() == Catch::Approx(1.0 - c * c));
|
||||
reader.Close();
|
||||
}
|
||||
|
||||
SECTION("phi scan on a fixed-chi head turns about the inclined phi axis") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
f.axis = 3;
|
||||
f.start = 0.5 * (frame - 1);
|
||||
f.angles = {0, 0, f.start, 54.74};
|
||||
WriteSFRM(FrameName("sfrm_phi", frame, ".sfrm"), f, n);
|
||||
}
|
||||
JFJochSFRMReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("sfrm_phi_0001.sfrm"));
|
||||
const auto x = reader.GetDataset()->experiment;
|
||||
// chi turns about the beam: imgCIF +Z, internal -z
|
||||
const Coord expected = -(RotMatrix(static_cast<float>(54.74 * PI / 180), Coord(0, 0, -1)) * Coord(1, 0, 0));
|
||||
CHECK((x.GetGoniometer()->GetAxis() - expected).Length() < 1e-5f);
|
||||
reader.Close();
|
||||
}
|
||||
|
||||
SECTION("chi moved between frames of one run is refused") {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
f.axis = 3;
|
||||
f.start = 0.5 * (frame - 1);
|
||||
f.angles = {0, 0, f.start, frame == 1 ? 54.74 : 0.0};
|
||||
WriteSFRM(FrameName("sfrm_reset", frame, ".sfrm"), f, n);
|
||||
}
|
||||
JFJochSFRMReader reader;
|
||||
CHECK_THROWS_AS(reader.ReadFiles("sfrm_reset_0001.sfrm"), JFJochException);
|
||||
}
|
||||
|
||||
for (int frame = 1; frame <= 3; frame++)
|
||||
for (const char *p: {"sfrm_omega", "sfrm_phi", "sfrm_reset"})
|
||||
remove(FrameName(p, frame, ".sfrm").c_str());
|
||||
}
|
||||
@@ -19,6 +19,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
|
||||
MultiReaderTest.cpp
|
||||
ModuleRefinementTest.cpp
|
||||
SweepLayoutTest.cpp
|
||||
BrukerReaderTest.cpp
|
||||
CBORTest.cpp
|
||||
ImageSpotFinderCPUTest.cpp
|
||||
AdaptiveSpotFinderCPUTest.cpp
|
||||
@@ -128,6 +129,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
RugnuxLiveStatusTest.cpp
|
||||
RugnuxStructureTest.cpp
|
||||
SweepLayoutTest.cpp
|
||||
BrukerReaderTest.cpp
|
||||
ShadowFinderTest.cpp
|
||||
HotPixelFinderTest.cpp
|
||||
BandwidthEstimateTest.cpp
|
||||
@@ -139,6 +141,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
PostRefineTest.cpp
|
||||
CellEsdTest.cpp
|
||||
RotationScaleWalkTest.cpp
|
||||
RotationScaleMergeSegmentsTest.cpp
|
||||
AndersonMixingTest.cpp
|
||||
RugnuxLargeTest.cpp
|
||||
TestData.h
|
||||
@@ -212,6 +215,8 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
WriteReflectionsTest.cpp
|
||||
RugnuxResultFilesTest.cpp
|
||||
RugnuxInputKindTest.cpp
|
||||
RugnuxSweepsTest.cpp
|
||||
RugnuxJointSweepsTest.cpp
|
||||
UnitCellTest.cpp
|
||||
CCTest.cpp
|
||||
MultiLatticeSearchTest.cpp
|
||||
|
||||
@@ -4181,6 +4181,26 @@ TEST_CASE("JFJochCBFReader_Polarization", "[HDF5][Full][portable]") {
|
||||
RemoveMiniCBFSweep("cbfnopol");
|
||||
}
|
||||
|
||||
// A PILATUS header that also carries the imgCIF axis and scan tables (Diamond writes them) is still read
|
||||
// as the PILATUS header it is: the "# Pixel_size" line decides, and the detector keeps its name.
|
||||
TEST_CASE("JFJochCBFReader_PilatusHeaderWithScanTable", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 24, ny = 16;
|
||||
WriteMiniCBFSweep("cbfscan", "# Detector_2theta 0.0000 deg.\n# Oscillation_axis OMEGA\n"
|
||||
+ AxisTable("GON_OMEGA rotation goniometer . 1 0 0 . . .\n"
|
||||
"DET_Z translation detector . 0 0 -1 0 0 0\n"
|
||||
"ELEMENT_X translation detector DET_Z 1 0 0 -1 1 0\n"
|
||||
"ELEMENT_Y translation detector ELEMENT_X 0 -1 0 0 0 0\n", nx, ny)
|
||||
+ "loop_\n_diffrn_scan_axis.axis_id\n_diffrn_scan_axis.angle_start\n"
|
||||
"_diffrn_scan_axis.angle_range\n_diffrn_scan_axis.angle_increment\n"
|
||||
"GON_OMEGA 0.0 0.1 0.1\n",
|
||||
nx, ny);
|
||||
JFJochCBFReader reader;
|
||||
REQUIRE_NOTHROW(reader.ReadFiles("cbfscan_0001.cbf"));
|
||||
CHECK(reader.GetDataset()->experiment.GetDetectorDescription().find("PILATUS3 6M") != std::string::npos);
|
||||
reader.Close();
|
||||
RemoveMiniCBFSweep("cbfscan");
|
||||
}
|
||||
|
||||
TEST_CASE("JFJochCBFReader_AxisTableStatesTheMounting", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 24, ny = 16;
|
||||
|
||||
@@ -4351,6 +4371,38 @@ TEST_CASE("JFJochCBFReader_incomplete_header_is_refused_not_misread", "[HDF5][Fu
|
||||
}
|
||||
}
|
||||
|
||||
// A phi scan on a kappa head turns about an axis that depends on where the kappa arm stands, and a
|
||||
// miniCBF header states the kappa angle but not the arm. Reading it about the uninclined phi axis is
|
||||
// a silent misread, so it is refused; an omega scan is not affected by kappa at all.
|
||||
TEST_CASE("JFJochCBFReader_phi_scan_at_kappa_is_refused", "[HDF5][Full][portable]") {
|
||||
const int64_t nx = 24, ny = 16;
|
||||
const auto write = [&](const std::string &prefix, const std::string &scan_lines) {
|
||||
for (int frame = 1; frame <= 2; frame++) {
|
||||
std::ostringstream name;
|
||||
name << prefix << "_" << std::setfill('0') << std::setw(4) << frame << ".cbf";
|
||||
WriteRawMiniCBF(name.str(),
|
||||
"# Pixel_size 172e-6 m x 172e-6 m\n# Wavelength 0.96864 A\n"
|
||||
"# Detector_distance 0.33161 m\n# Count_cutoff 1000 counts\n"
|
||||
"# Beam_xy (12.00, 8.00) pixels\n# Start_angle " + std::to_string(0.1 * frame)
|
||||
+ " deg.\n# Angle_increment 0.1000 deg.\n# Kappa 30.0000 deg.\n" + scan_lines,
|
||||
nx, ny);
|
||||
}
|
||||
};
|
||||
write("cbfkappa_phi", "# Phi_increment 0.1000 deg.\n# Omega_increment 0.0000 deg.\n");
|
||||
write("cbfkappa_omega", "# Phi_increment 0.0000 deg.\n# Omega_increment 0.1000 deg.\n");
|
||||
{
|
||||
JFJochCBFReader reader;
|
||||
CHECK_THROWS_AS(reader.ReadFiles("cbfkappa_phi_0001.cbf"), JFJochException);
|
||||
}
|
||||
{
|
||||
JFJochCBFReader reader;
|
||||
CHECK_NOTHROW(reader.ReadFiles("cbfkappa_omega_0001.cbf"));
|
||||
}
|
||||
for (const char *f: {"cbfkappa_phi_0001.cbf", "cbfkappa_phi_0002.cbf",
|
||||
"cbfkappa_omega_0001.cbf", "cbfkappa_omega_0002.cbf"})
|
||||
remove(f);
|
||||
}
|
||||
|
||||
TEST_CASE("MiniCBF_header_longer_than_the_first_probe", "[HDF5][Full][portable]") {
|
||||
// The header is read with a small first probe, and again with the large one when it does not end
|
||||
// inside it; what stands past the first 16 kB must come out exactly as before.
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include "../common/CUDAWrapper.h"
|
||||
#include "../common/Logger.h"
|
||||
#include "../image_analysis/scale_merge/RotationScaleMerge.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// A synthetic crystal measured as pre-combined fulls on several sweeps: a cubic 20 A cell, Wilson
|
||||
// intensities at B = 15 A^2 to 1.6 A, each sweep at its own scale and relative B, Poisson-like noise.
|
||||
struct SweepSpec {
|
||||
int n_frames;
|
||||
double scale;
|
||||
double relative_b; // at the sweep's first frame
|
||||
double relative_b_end = NAN; // at its last; NaN: the same B throughout
|
||||
};
|
||||
|
||||
struct Synthetic {
|
||||
std::vector<IntegrationOutcome> frames;
|
||||
std::map<std::array<int, 3>, double> I_true; // by the Friedel-merged index
|
||||
};
|
||||
|
||||
constexpr double CELL = 20.0;
|
||||
|
||||
std::array<int, 3> FriedelKey(int h, int k, int l) {
|
||||
const std::array<int, 3> a{h, k, l}, b{-h, -k, -l};
|
||||
return std::max(a, b);
|
||||
}
|
||||
|
||||
Synthetic MakeSweeps(const std::vector<SweepSpec> &sweeps, uint32_t seed) {
|
||||
std::mt19937 rng(seed);
|
||||
std::vector<std::array<int, 3>> hkl;
|
||||
for (int h = -12; h <= 12; ++h)
|
||||
for (int k = -12; k <= 12; ++k)
|
||||
for (int l = -12; l <= 12; ++l) {
|
||||
const double d = CELL / std::sqrt(static_cast<double>(h * h + k * k + l * l));
|
||||
if ((h || k || l) && d >= 1.6) hkl.push_back({h, k, l});
|
||||
}
|
||||
Synthetic out;
|
||||
std::exponential_distribution<double> wilson(1.0);
|
||||
for (const auto &m: hkl) {
|
||||
const auto key = FriedelKey(m[0], m[1], m[2]);
|
||||
if (!out.I_true.contains(key)) {
|
||||
const double s2 = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2]) / (4.0 * CELL * CELL);
|
||||
out.I_true[key] = 2000.0 * std::exp(-2.0 * 15.0 * s2) * (0.05 + wilson(rng));
|
||||
}
|
||||
}
|
||||
std::uniform_int_distribution<size_t> pick(0, hkl.size() - 1);
|
||||
std::normal_distribution<double> noise(0.0, 1.0);
|
||||
int frame = 0;
|
||||
for (const auto &sweep: sweeps)
|
||||
for (int f = 0; f < sweep.n_frames; ++f, ++frame) {
|
||||
IntegrationOutcome o;
|
||||
o.latt = CrystalLattice(UnitCell{static_cast<float>(CELL), static_cast<float>(CELL),
|
||||
static_cast<float>(CELL), 90, 90, 90});
|
||||
for (int j = 0; j < 400; ++j) {
|
||||
const auto &m = hkl[pick(rng)];
|
||||
const double s2 = (m[0] * m[0] + m[1] * m[1] + m[2] * m[2]) / (4.0 * CELL * CELL);
|
||||
const double b = std::isfinite(sweep.relative_b_end)
|
||||
? sweep.relative_b + (sweep.relative_b_end - sweep.relative_b) * f / (sweep.n_frames - 1)
|
||||
: sweep.relative_b;
|
||||
const double expected = out.I_true.at(FriedelKey(m[0], m[1], m[2])) * sweep.scale
|
||||
* std::exp(-2.0 * b * s2);
|
||||
const double sigma = std::sqrt(expected + 20.0);
|
||||
Reflection r{};
|
||||
r.h = m[0]; r.k = m[1]; r.l = m[2];
|
||||
r.image_number = static_cast<float>(frame);
|
||||
r.d = static_cast<float>(1.0 / std::sqrt(4.0 * s2));
|
||||
r.I = static_cast<float>(expected + sigma * noise(rng));
|
||||
r.sigma = static_cast<float>(sigma);
|
||||
// The combine rebuilds a full's variance as var_bkg + I: what is not the intensity's own
|
||||
r.var_bkg = static_cast<float>(sigma * sigma - std::max(0.0, static_cast<double>(r.I)));
|
||||
r.bkg = 20.0f;
|
||||
r.prescaling_corr = 1.0f;
|
||||
r.partiality = 1.0f;
|
||||
r.zeta = 1.0f;
|
||||
r.image_scale_corr = 1.0f;
|
||||
r.predicted_x = 500.0f;
|
||||
r.predicted_y = 500.0f;
|
||||
o.reflections.push_back(r);
|
||||
}
|
||||
out.frames.push_back(std::move(o));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
DiffractionExperiment SyntheticExperiment(bool correction_surfaces) {
|
||||
DiffractionExperiment x;
|
||||
x.IncidentEnergy_keV(12.4f);
|
||||
x.Goniometer(GoniometerAxis("omega", 0.0f, 0.1f, Coord(1, 0, 0), {}));
|
||||
x.SetUnitCell(UnitCell{static_cast<float>(CELL), static_cast<float>(CELL), static_cast<float>(CELL), 90, 90, 90});
|
||||
ScalingSettings s;
|
||||
s.ScaleFulls(true).CorrectionSurfaces(correction_surfaces).ResolutionCutoff(ResolutionCutoffMethod::Off);
|
||||
x.ImportScalingSettings(s);
|
||||
return x;
|
||||
}
|
||||
|
||||
std::vector<SweepSegment> Segments(const std::vector<SweepSpec> &sweeps) {
|
||||
std::vector<SweepSegment> out;
|
||||
int32_t first = 0;
|
||||
for (const auto &s: sweeps) {
|
||||
out.push_back({first, s.n_frames});
|
||||
first += s.n_frames;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
RotationScaleMerge::Result Merge(Synthetic &data, const DiffractionExperiment &x,
|
||||
const std::vector<SweepSegment> &segments, bool export_fulls = false,
|
||||
bool *on_gpu = nullptr) {
|
||||
Logger logger("RotationScaleMergeSegmentsTest");
|
||||
const UnitCell cell{static_cast<float>(CELL), static_cast<float>(CELL), static_cast<float>(CELL), 90, 90, 90};
|
||||
RotationScaleMerge engine(x, data.frames, cell, 20, 4, logger);
|
||||
if (!segments.empty())
|
||||
engine.SetSweepSegments(segments);
|
||||
engine.SetExportScaledFulls(export_fulls);
|
||||
engine.Ingest();
|
||||
if (on_gpu)
|
||||
*on_gpu = engine.OnGPU();
|
||||
return engine.Run(false, true, false);
|
||||
}
|
||||
|
||||
// The median over a frame range of the scaled fulls' ratio to the true intensity (the strong ones).
|
||||
double MedianRatio(const RotationScaleMerge::Result &r, const Synthetic &data, int f0, int f1) {
|
||||
std::vector<double> ratio;
|
||||
for (const auto &o: r.scaled_fulls)
|
||||
if (o.image_number >= static_cast<float>(f0) && o.image_number < static_cast<float>(f1)) {
|
||||
const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l));
|
||||
if (t > 500.0)
|
||||
ratio.push_back(o.I / t);
|
||||
}
|
||||
REQUIRE(ratio.size() > 20);
|
||||
std::nth_element(ratio.begin(), ratio.begin() + ratio.size() / 2, ratio.end());
|
||||
return ratio[ratio.size() / 2];
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_PerSweepScaleRecovered", "[RotationScale]") {
|
||||
const std::vector<SweepSpec> sweeps{{60, 1.0, 0.0}, {60, 0.5, 0.0}};
|
||||
auto data = MakeSweeps(sweeps, 11);
|
||||
const auto r = Merge(data, SyntheticExperiment(false), Segments(sweeps), true);
|
||||
REQUIRE(r.segments.size() == 2);
|
||||
CHECK(r.segments[1].scale / r.segments[0].scale == Catch::Approx(0.5).epsilon(0.03));
|
||||
CHECK(r.statistics.overall.cc_half > 0.99);
|
||||
|
||||
// No coupling across the boundary: the frames on either side of it are put on the common scale as
|
||||
// well as the middle of their sweeps are.
|
||||
const double middle = MedianRatio(r, data, 20, 40);
|
||||
CHECK(MedianRatio(r, data, 57, 60) / middle == Catch::Approx(1.0).epsilon(0.03));
|
||||
CHECK(MedianRatio(r, data, 60, 63) / middle == Catch::Approx(1.0).epsilon(0.03));
|
||||
CHECK(MedianRatio(r, data, 80, 100) / middle == Catch::Approx(1.0).epsilon(0.03));
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_DoseOrderFromTheRelativeB", "[RotationScale]") {
|
||||
// Given first, the sweep taken second: its B is 8 A^2 higher. The order comes from the data.
|
||||
const std::vector<SweepSpec> sweeps{{60, 1.0, 8.0}, {60, 1.0, 0.0}, {60, 1.0, 4.0}};
|
||||
auto data = MakeSweeps(sweeps, 12);
|
||||
const auto r = Merge(data, SyntheticExperiment(true), Segments(sweeps));
|
||||
REQUIRE(r.segments.size() == 3);
|
||||
CHECK(r.segments[1].dose_rank == 0);
|
||||
CHECK(r.segments[2].dose_rank == 1);
|
||||
CHECK(r.segments[0].dose_rank == 2);
|
||||
CHECK(r.segments[1].relative_b == Catch::Approx(0.0).margin(1e-9));
|
||||
CHECK(r.segments[0].relative_b == Catch::Approx(8.0).margin(1.0));
|
||||
CHECK(r.segments[2].relative_b == Catch::Approx(4.0).margin(1.0));
|
||||
// Flat within each sweep: one B per sweep predicts the held-out frames better than a dose-linear ramp
|
||||
CHECK(r.segment_decay_model == "per-sweep B");
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationScale]") {
|
||||
// One crystal damaged steadily: the sweep given first was taken second, its B going on from where the
|
||||
// other's ended. Along the dose order the B rises in a straight line, which the ramp describes.
|
||||
const std::vector<SweepSpec> sweeps{{60, 1.0, 6.0, 12.0}, {60, 1.0, 0.0, 6.0}};
|
||||
auto data = MakeSweeps(sweeps, 15);
|
||||
const auto r = Merge(data, SyntheticExperiment(true), Segments(sweeps));
|
||||
REQUIRE(r.segments.size() == 2);
|
||||
CHECK(r.segments[1].dose_rank == 0);
|
||||
CHECK(r.segments[0].dose_rank == 1);
|
||||
CHECK(r.segments[0].relative_b == Catch::Approx(6.0).margin(1.5));
|
||||
CHECK(r.segment_decay_model == "dose-linear");
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") {
|
||||
const std::vector<SweepSpec> sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}};
|
||||
auto data = MakeSweeps(sweeps, 13);
|
||||
const auto x = SyntheticExperiment(false);
|
||||
CHECK_THROWS_AS(Merge(data, x, {{0, 10}, {11, 9}}), JFJochException);
|
||||
CHECK_THROWS_AS(Merge(data, x, {{0, 10}}), JFJochException);
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[RotationScale]") {
|
||||
// The same synthetic sweep merged without segments gives, to the bit, what the engine gave before
|
||||
// segments existed: FNV-1a over the merged I and sigma, recorded on the engine of rc181 e56b19abb -
|
||||
// on the CPU, and on the GPU for the card model it was recorded on (the device path matches the host
|
||||
// path to the bit only where every reduction does; this one carries its own, so it is checked against
|
||||
// its own record).
|
||||
const std::vector<SweepSpec> sweeps{{40, 1.0, 0.0}, {40, 0.7, 3.0}};
|
||||
auto data = MakeSweeps(sweeps, 14);
|
||||
bool on_gpu = false;
|
||||
const auto r = Merge(data, SyntheticExperiment(true), {}, false, &on_gpu);
|
||||
uint64_t hash = 1469598103934665603ULL;
|
||||
auto mix = [&hash](float v) {
|
||||
uint32_t u;
|
||||
std::memcpy(&u, &v, sizeof u);
|
||||
for (int b = 0; b < 4; ++b) {
|
||||
hash ^= (u >> (8 * b)) & 0xffu;
|
||||
hash *= 1099511628211ULL;
|
||||
}
|
||||
};
|
||||
for (const auto &m: r.merged) {
|
||||
mix(m.I);
|
||||
mix(m.sigma);
|
||||
}
|
||||
CHECK(r.segments.empty());
|
||||
CHECK(r.merged.size() == 4106);
|
||||
if (!on_gpu)
|
||||
CHECK(hash == 3376995616949734726ULL);
|
||||
else if (get_gpu_description().find("RTX 5080") != std::string::npos)
|
||||
CHECK(hash == 6594155396455850264ULL);
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <map>
|
||||
#include <random>
|
||||
|
||||
#include "../rugnux/RugnuxJointSweeps.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// A tetragonal P4 crystal: Laue class 4/m on a 4/mmm lattice, so (k, h, -l) indexes it a second way.
|
||||
const gemmi::SpaceGroup &P4() { return *gemmi::find_spacegroup_by_name("P 4"); }
|
||||
constexpr float A = 60.0f, C = 40.0f;
|
||||
|
||||
std::array<int, 3> P4Key(int h, int k, int l) {
|
||||
// the 4/m ASU representative, enough to give equivalents one intensity
|
||||
std::array<int, 3> best{h, k, l};
|
||||
const std::array<std::array<int, 3>, 8> eq{{{h, k, l}, {-k, h, l}, {-h, -k, l}, {k, -h, l},
|
||||
{-h, -k, -l}, {k, -h, -l}, {h, k, -l}, {-k, h, -l}}};
|
||||
for (const auto &e: eq) best = std::max(best, e);
|
||||
return best;
|
||||
}
|
||||
|
||||
JointMemberSweep MakeSweep(const std::map<std::array<int, 3>, double> &truth, const gemmi::Op &indexed_as,
|
||||
int n_frames, uint32_t seed) {
|
||||
std::mt19937 rng(seed);
|
||||
JointMemberSweep s;
|
||||
s.n_frames = n_frames;
|
||||
s.cell = UnitCell{A, A, C, 90, 90, 90};
|
||||
s.space_group = P4();
|
||||
for (const auto &[hkl, I]: truth) {
|
||||
const auto m = indexed_as.apply_to_hkl({{hkl[0], hkl[1], hkl[2]}});
|
||||
MergedReflection r;
|
||||
r.h = m[0]; r.k = m[1]; r.l = m[2];
|
||||
r.I = static_cast<float>(I);
|
||||
r.sigma = 1.0f;
|
||||
r.d = static_cast<float>(1.0 / std::sqrt((hkl[0] * hkl[0] + hkl[1] * hkl[1]) / (A * A) + hkl[2] * hkl[2] / (C * C)));
|
||||
s.merged.push_back(r);
|
||||
ScaledFull f;
|
||||
f.h = m[0]; f.k = m[1]; f.l = m[2];
|
||||
f.I = static_cast<float>(I);
|
||||
f.sigma = 1.0f;
|
||||
f.image_number = static_cast<float>(rng() % n_frames);
|
||||
f.var_bkg = 4.0f;
|
||||
f.var_per_I = 0.5f;
|
||||
f.capture = 0.0f;
|
||||
f.d = r.d;
|
||||
s.fulls.push_back(f);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
std::map<std::array<int, 3>, double> Truth(uint32_t seed) {
|
||||
std::mt19937 rng(seed);
|
||||
std::exponential_distribution<double> wilson(1.0);
|
||||
std::map<std::array<int, 3>, double> t;
|
||||
for (int h = -12; h <= 12; ++h)
|
||||
for (int k = -12; k <= 12; ++k)
|
||||
for (int l = -8; l <= 8; ++l) {
|
||||
if (!(h || k || l)) continue;
|
||||
const auto key = P4Key(h, k, l);
|
||||
if (!t.contains(key)) t[key] = 1000.0 * wilson(rng);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("RelateJointSweeps_AmbiguityRecoveredAndAStrangerLeftOut", "[Rugnux]") {
|
||||
const auto truth = Truth(1);
|
||||
const gemmi::Op twin = gemmi::parse_triplet("y,x,-z"); // h,k,l -> k,h,-l
|
||||
std::vector<JointMemberSweep> sweeps;
|
||||
sweeps.push_back(MakeSweep(truth, gemmi::Op::identity(), 100, 2));
|
||||
sweeps.push_back(MakeSweep(truth, twin, 80, 3));
|
||||
sweeps.push_back(MakeSweep(Truth(99), gemmi::Op::identity(), 50, 4)); // another crystal
|
||||
sweeps[0].fulls.push_back(sweeps[0].fulls.front()); // the reference: most fulls
|
||||
|
||||
const auto relation = RelateJointSweeps(sweeps);
|
||||
REQUIRE(relation.reference == 0);
|
||||
CHECK(relation.sweeps[0].included);
|
||||
REQUIRE(relation.sweeps[1].included);
|
||||
CHECK(relation.sweeps[1].cc > 0.99);
|
||||
CHECK(relation.sweeps[1].identity_cc < 0.5);
|
||||
// the operator found takes sweep 2's indices back onto the reference's (up to the Laue group)
|
||||
const auto back = relation.sweeps[1].cob.apply_to_hkl(twin.apply_to_hkl({{3, 1, 2}}));
|
||||
CHECK(P4Key(back[0], back[1], back[2]) == P4Key(3, 1, 2));
|
||||
CHECK_FALSE(relation.sweeps[2].included);
|
||||
|
||||
const auto input = BuildJointMergeInput(sweeps, relation);
|
||||
REQUIRE(input.segments.size() == 2);
|
||||
CHECK(input.segments[0].first_frame == 0);
|
||||
CHECK(input.segments[0].n_frames == 100);
|
||||
CHECK(input.segments[1].first_frame == 100);
|
||||
CHECK(input.segments[1].n_frames == 80);
|
||||
CHECK(input.segment_sweep == std::vector<int>{0, 1});
|
||||
REQUIRE(input.outcomes.size() == 180);
|
||||
// a full as the combine takes it: count I/c at correction c = var_per_I, variance var_bkg / c^2
|
||||
const Reflection *r = nullptr;
|
||||
for (const auto &o: input.outcomes)
|
||||
if (!o.reflections.empty()) { r = &o.reflections.front(); break; }
|
||||
REQUIRE(r != nullptr);
|
||||
CHECK(r->image_scale_corr == Catch::Approx(0.5));
|
||||
CHECK(r->var_bkg == Catch::Approx(16.0));
|
||||
CHECK(r->partiality == 1.0f);
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "../rugnux/RugnuxSweeps.h"
|
||||
|
||||
namespace {
|
||||
|
||||
// Inputs described by name alone: "<template>_<n>.cbf" frames, "<name>_rot.h5" / "<name>_still.h5"
|
||||
// masters, "<name>.cxi" for the stills readers; the wavelength from a table.
|
||||
struct FakeInputs {
|
||||
std::map<std::string, double> wavelength;
|
||||
int sweeps_described = 0;
|
||||
|
||||
RugnuxInputDescription File(const std::string &f) const {
|
||||
RugnuxInputDescription d;
|
||||
if (f.ends_with(".cbf"))
|
||||
d.frame_template = RugnuxFrameTemplate(f);
|
||||
d.foreign = f.ends_with(".cxi");
|
||||
return d;
|
||||
}
|
||||
RugnuxInputDescription Sweep(const std::string &f) {
|
||||
sweeps_described++;
|
||||
RugnuxInputDescription d = File(f);
|
||||
d.rotation = f.ends_with(".cbf") || f.ends_with("_rot.h5");
|
||||
d.wavelength_A = wavelength.contains(f) ? wavelength.at(f) : 1.0;
|
||||
return d;
|
||||
}
|
||||
RugnuxSweepPlan Plan(const std::vector<std::string> &inputs) {
|
||||
return PlanRugnuxSweeps(inputs, [this](const std::string &f) { return File(f); },
|
||||
[this](const std::string &f) { return Sweep(f); });
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("RugnuxFrameTemplate", "[Rugnux][portable]") {
|
||||
CHECK(RugnuxFrameTemplate("/d/x_1_0001.cbf") == "/d/x_1_####.cbf");
|
||||
CHECK(RugnuxFrameTemplate("x_00017.cbf.gz") == "x_#####.cbf.gz");
|
||||
CHECK(RugnuxFrameTemplate("formal.001") == "formal.###");
|
||||
CHECK(RugnuxFrameTemplate("noframe.cbf").empty());
|
||||
}
|
||||
|
||||
TEST_CASE("PlanRugnuxSweeps_single_input_is_not_planned", "[Rugnux][portable]") {
|
||||
FakeInputs in;
|
||||
CHECK(in.Plan({"a_0001.cbf"}).sweeps.empty());
|
||||
CHECK(in.Plan({"run_rot.h5"}).sweeps.empty());
|
||||
CHECK(in.sweeps_described == 0); // nothing is opened for one input
|
||||
}
|
||||
|
||||
TEST_CASE("PlanRugnuxSweeps_frames_by_template", "[Rugnux][portable]") {
|
||||
FakeInputs in;
|
||||
SECTION("frames of one template are its one sweep, named by the first given") {
|
||||
const auto plan = in.Plan({"d/a_0003.cbf", "d/a_0001.cbf", "d/a_0002.cbf"});
|
||||
REQUIRE(plan.sweeps.size() == 1);
|
||||
CHECK(plan.sweeps[0].input == "d/a_0003.cbf");
|
||||
CHECK(plan.sweeps[0].frames);
|
||||
}
|
||||
SECTION("two templates are two sweeps, in the order given") {
|
||||
const auto plan = in.Plan({"d/b_0001.cbf", "d/b_0002.cbf", "d/a_0001.cbf"});
|
||||
REQUIRE(plan.sweeps.size() == 2);
|
||||
CHECK(plan.sweeps[0].input == "d/b_0001.cbf");
|
||||
CHECK(plan.sweeps[0].label == std::filesystem::path("d/b_####.cbf").string());
|
||||
CHECK(plan.sweeps[1].input == "d/a_0001.cbf");
|
||||
CHECK(plan.wavelength_A == Catch::Approx(1.0));
|
||||
}
|
||||
SECTION("the same name in two directories is two sweeps") {
|
||||
CHECK(in.Plan({"d1/a_0001.cbf", "d2/a_0001.cbf"}).sweeps.size() == 2);
|
||||
}
|
||||
SECTION("sweeps at different wavelengths are refused") {
|
||||
in.wavelength["d/b_0001.cbf"] = 1.2;
|
||||
CHECK_THROWS_AS(in.Plan({"d/a_0001.cbf", "d/b_0001.cbf"}), std::runtime_error);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("PlanRugnuxSweeps_masters", "[Rugnux][portable]") {
|
||||
FakeInputs in;
|
||||
SECTION("stills masters stay one stills run") {
|
||||
CHECK(in.Plan({"a_still.h5", "b_still.h5", "c_still.h5"}).sweeps.empty());
|
||||
CHECK(in.sweeps_described == 1); // the first decides; the stills reader refuses a rotation one
|
||||
}
|
||||
SECTION("rotation masters are a sweep each") {
|
||||
const auto plan = in.Plan({"a_rot.h5", "b_rot.h5"});
|
||||
REQUIRE(plan.sweeps.size() == 2);
|
||||
CHECK(plan.sweeps[1].input == "b_rot.h5");
|
||||
CHECK_FALSE(plan.sweeps[0].frames);
|
||||
}
|
||||
SECTION("rotation and stills are refused") {
|
||||
CHECK_THROWS_AS(in.Plan({"a_rot.h5", "b_still.h5"}), std::runtime_error);
|
||||
}
|
||||
SECTION("rotation masters at different wavelengths are refused") {
|
||||
in.wavelength["b_rot.h5"] = 0.98;
|
||||
CHECK_THROWS_AS(in.Plan({"a_rot.h5", "b_rot.h5"}), std::runtime_error);
|
||||
}
|
||||
SECTION("a wavelength within the tolerance is the same") {
|
||||
in.wavelength["b_rot.h5"] = 1.0 + 1e-4;
|
||||
CHECK(in.Plan({"a_rot.h5", "b_rot.h5"}).sweeps.size() == 2);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("PlanRugnuxSweeps_leaves_the_stills_readers_and_mixtures_alone", "[Rugnux][portable]") {
|
||||
FakeInputs in;
|
||||
CHECK(in.Plan({"a.cxi", "b.cxi"}).sweeps.empty());
|
||||
CHECK(in.Plan({"a_0001.cbf", "b_rot.h5"}).sweeps.empty()); // the readers refuse the mix, as before
|
||||
CHECK(in.sweeps_described == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("RugnuxSiblingSweeps", "[Rugnux][portable]") {
|
||||
namespace fs = std::filesystem;
|
||||
const fs::path dir = fs::path("rugnux_sweeps_siblings");
|
||||
fs::remove_all(dir);
|
||||
fs::create_directories(dir);
|
||||
for (const char *name: {"a_0001.cbf", "a_0002.cbf", "b_0001.cbf", "b_0002.cbf", "b_0003.cbf", "c_0001.cbf.gz",
|
||||
"notes.txt", "x_master.h5", "y_master.h5"})
|
||||
std::ofstream(dir / name) << "x";
|
||||
|
||||
RugnuxSweepPlan plan;
|
||||
plan.sweeps.push_back({(dir / "a_0001.cbf").string(), RugnuxFrameTemplate((dir / "a_0001.cbf").string()), true});
|
||||
plan.sweeps.push_back({(dir / "x_master.h5").string(), (dir / "x_master.h5").string(), false});
|
||||
const auto siblings = RugnuxSiblingSweeps(plan, {".cbf"});
|
||||
REQUIRE(siblings.size() == 3);
|
||||
CHECK(siblings[0] == (dir / "b_####.cbf").string() + " (3 frames)");
|
||||
CHECK(siblings[1] == (dir / "c_####.cbf.gz").string() + " (1 frame)");
|
||||
CHECK(siblings[2] == (dir / "y_master.h5").string());
|
||||
fs::remove_all(dir);
|
||||
}
|
||||
@@ -41,8 +41,8 @@ def master_frames(path):
|
||||
|
||||
|
||||
def detector_frame(path):
|
||||
"""'marCCD' or 'SMV' if the file's header says so, else None. These formats have no reliable
|
||||
extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next)."""
|
||||
"""'marCCD', 'SMV' or 'sfrm' (Bruker) if the file's header says so, else None. These formats have
|
||||
no reliable extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next)."""
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
head = fh.read(64)
|
||||
@@ -51,6 +51,8 @@ def detector_frame(path):
|
||||
return "marCCD" if fh.read(3) == b"MMX" else None
|
||||
if head[:1] == b"{" and b"HEADER_BYTES" in head:
|
||||
return "SMV"
|
||||
if head[:8] == b"FORMAT :":
|
||||
return "sfrm"
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
@@ -205,5 +205,9 @@
|
||||
{"id": "4nwv", "input": "4nwv/sbgrid9/formal.001", "wavelength": 1.12706, "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [402.197, 369.862, 410.511, 90.0, 90.0, 90.0], "dmin": 3.25}, "tags": ["marCCD", "orthorhombic", "large-cell", "virus"], "tiers": {"ci": "virus capsid, 400 A body-centred cell, marCCD"}},
|
||||
{"id": "7d98", "input": "7d98/xrda13/data/cbnrf006_01_00011.cbf", "wavelength": 1.9, "ref": {"sg": "P 61 2 2", "sgno": 178, "cell": [105.819, 105.819, 602.516, 90.0, 90.0, 120.0], "dmin": 3.6}, "tags": ["cbf", "hexagonal", "large-cell", "long-axis", "long-wavelength"], "pinned": true},
|
||||
{"id": "9v16", "input": "9v16/xrd-00338/9V16/data01_master.h5", "wavelength": 0.9, "ref": {"sg": "P 41 21 2", "sgno": 92, "cell": [185.604, 185.604, 451.242, 90.0, 90.0, 90.0], "dmin": 3.45}, "tags": ["h5", "tetragonal", "large-cell", "long-axis"]},
|
||||
{"id": "4p8r", "input": "4p8r/data/4p8r/series/gzn0-7.001", "wavelength": 0.97872, "ref": {"sg": "P 65 2 2", "sgno": 179, "cell": [86.7, 86.7, 701.37, 90.0, 90.0, 120.0], "dmin": 2.2}, "tags": ["marCCD", "hexagonal", "large-cell", "long-axis"]}
|
||||
{"id": "4p8r", "input": "4p8r/data/4p8r/series/gzn0-7.001", "wavelength": 0.97872, "ref": {"sg": "P 65 2 2", "sgno": 179, "cell": [86.7, 86.7, 701.37, 90.0, 90.0, 120.0], "dmin": 2.2}, "tags": ["marCCD", "hexagonal", "large-cell", "long-axis"]},
|
||||
{"id": "laspartic", "input": "laspartic/cbf/m0255a_01_0001.cbf", "wavelength": 0.71073, "pinned": true, "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [5.1237, 6.9197, 7.6006, 90.0, 100.442, 90.0], "dmin": null}, "cod": "1577465", "tags": ["cbf", "imgcif", "monoclinic", "small-molecule", "twin", "reticular-twin", "kappa", "bruker", "ccd", "home-source"]},
|
||||
{"id": "onitroaniline", "input": "onitroaniline/sfrm/m0220c_01_0001.sfrm", "wavelength": 0.71073, "pinned": true, "tags": ["sfrm", "small-molecule", "twin", "pseudo-merohedral", "kappa", "bruker", "ccd", "home-source"]},
|
||||
{"id": "6or0", "input": "6or0/data/ins_zwin_01_0001.sfrm", "wavelength": 1.54188, "pinned": true, "ref": {"sg": "I 21 3", "sgno": 199, "cell": [78.031, 78.031, 78.031, 90.0, 90.0, 90.0], "dmin": 1.55}, "tags": ["sfrm", "cubic", "twin", "non-merohedral", "bruker", "ccd", "home-source"]},
|
||||
{"id": "6oqz", "input": "6oqz/data/gdri_01_0001.sfrm", "wavelength": 1.54188, "pinned": true, "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [92.931, 97.935, 102.706, 90.0, 90.0, 90.0], "dmin": 1.6}, "tags": ["sfrm", "orthorhombic", "twin", "non-merohedral", "multi-lattice", "bruker", "ccd", "home-source"]}
|
||||
]}
|
||||
@@ -110,6 +110,7 @@ JFJochImageReadingWorker::JFJochImageReadingWorker(const SpotFindingSettings &se
|
||||
cbf_reader.Experiment(experiment);
|
||||
marccd_reader.Experiment(experiment);
|
||||
smv_reader.Experiment(experiment);
|
||||
sfrm_reader.Experiment(experiment);
|
||||
|
||||
autoload_timer = new QTimer(this);
|
||||
autoload_timer->setInterval(autoload_interval);
|
||||
@@ -326,6 +327,9 @@ void JFJochImageReadingWorker::LoadFile_i(const QString &filename, qint64 image_
|
||||
} else if (JFJochSMVReader::CanRead(path)) {
|
||||
file_reader = &smv_reader;
|
||||
smv_reader.ReadFiles(path);
|
||||
} else if (JFJochSFRMReader::CanRead(path)) {
|
||||
file_reader = &sfrm_reader;
|
||||
sfrm_reader.ReadFiles(path);
|
||||
} else {
|
||||
cbf_mode = false;
|
||||
file_reader = &hdf5_reader;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "../reader/JFJochSMVReader.h"
|
||||
#include "../reader/JFJochXFELReader.h"
|
||||
#include "../reader/JFJochXTCReader.h"
|
||||
#include "../reader/JFJochSFRMReader.h"
|
||||
#include "../common/Logger.h"
|
||||
#include "JFJochHttpReader.h"
|
||||
#include "../image_analysis/MXAnalysisWithoutFPGA.h"
|
||||
@@ -94,6 +95,7 @@ private:
|
||||
JFJochXFELReader xfel_reader;
|
||||
JFJochXTCReader xtc_reader;
|
||||
QStringList pending_files_;
|
||||
JFJochSFRMReader sfrm_reader;
|
||||
JFJochReader *file_reader = &hdf5_reader;
|
||||
bool cbf_mode = false; // "a raw sweep is open", whichever of the three formats it is
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "../reader/JFJochCBFReader.h"
|
||||
#include "../reader/JFJochMarCCDReader.h"
|
||||
#include "../reader/JFJochSMVReader.h"
|
||||
#include "../reader/JFJochSFRMReader.h"
|
||||
#include "../reader/JFJochHDF5Reader.h"
|
||||
#include "../rugnux/ResultReport.h"
|
||||
#include "../rugnux/RugnuxOutputFiles.h"
|
||||
@@ -60,6 +61,7 @@ void JFJochProcessController::run_(QString file_path, DiffractionExperiment expe
|
||||
JFJochCBFReader cbf_reader;
|
||||
JFJochMarCCDReader marccd_reader;
|
||||
JFJochSMVReader smv_reader;
|
||||
JFJochSFRMReader sfrm_reader;
|
||||
JFJochReader *reader_ptr;
|
||||
const std::string path = file_path.toStdString();
|
||||
if (JFJochCBFReader::CanRead(path)) {
|
||||
@@ -71,6 +73,9 @@ void JFJochProcessController::run_(QString file_path, DiffractionExperiment expe
|
||||
} else if (JFJochSMVReader::CanRead(path)) {
|
||||
smv_reader.ReadFiles(path);
|
||||
reader_ptr = &smv_reader;
|
||||
} else if (JFJochSFRMReader::CanRead(path)) {
|
||||
sfrm_reader.ReadFiles(path);
|
||||
reader_ptr = &sfrm_reader;
|
||||
} else {
|
||||
hdf5_reader.ReadFile(path);
|
||||
reader_ptr = &hdf5_reader;
|
||||
|
||||
Reference in new issue
Block a user