602 lines
41 KiB
C++
602 lines
41 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#pragma once
|
|
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <functional>
|
|
#include <limits>
|
|
#include <optional>
|
|
#include <vector>
|
|
|
|
#include "../../common/DiffractionExperiment.h"
|
|
#include "../../common/Logger.h"
|
|
#include "../../common/Reflection.h"
|
|
#include "../../common/UnitCell.h"
|
|
#include "../IntegrationOutcome.h"
|
|
|
|
#include "Merge.h" // MergedReflection, MergeStatistics
|
|
#ifdef JFJOCH_USE_CUDA
|
|
#include <memory>
|
|
#include "RotationScaleMergeGPU.h"
|
|
#endif
|
|
|
|
// Dedicated, allocate-once scale+combine+merge for rotation data (the -P rot3d path): recompute the
|
|
// per-frame partiality from the (smoothed) mosaicity, robustly fit a per-image scale G, 3D-combine each
|
|
// rocking event's partials into fulls, refit a per-frame scale on the fulls (XDS order), and merge with
|
|
// a global error model.
|
|
//
|
|
// The per-frame partial observations are ingested ONCE into flat vectors; the hkl->ASU grouping is
|
|
// computed once per space group (by a sort, not a map) and reused across all scaling iterations; every
|
|
// hot step is a flat loop over those vectors, so the whole pipeline maps onto CUDA kernels (segmented
|
|
// reduction + per-frame solve) and runs GPU-resident when a GPU is present, with the CPU loops as the
|
|
// bit-parity fallback. CC1/2 and the per-image CC are computed once at the end, not every iteration.
|
|
//
|
|
// Used only for the self-scaling rotation case with per-image G (Rotation partiality, a fixed/forced
|
|
// mosaicity is honoured by the recompute). Post-scale-fulls correction stages (on by default via
|
|
// ScalingSettings::CorrectionSurfaces): a global Debye-Waller decay and a goniometer-frame absorption
|
|
// surface, both fitted on the host and pushed back to the resident (GPU) fulls before the merge.
|
|
// External-reference scaling, the stills B-factor and wedge refinement are unsupported (caller rejects).
|
|
// Stills use the per-image ScaleOnTheFly (fixed partiality) instead.
|
|
class RotationScaleMerge {
|
|
public:
|
|
struct Result {
|
|
std::vector<MergedReflection> merged;
|
|
MergeStatistics statistics;
|
|
// Two tiers, and they are different quantities. `isa` is the whole-range 1/sqrt(a*b) - which
|
|
// in this parameterisation is 1/b - and is what XDS's ISa means, so it is the one exported.
|
|
// `isa_asymptotic` is the strong-reflection tier, which XDS has no equivalent of and which can
|
|
// only ever be the more optimistic of the two. Both 0 if the model stayed at identity.
|
|
double isa = 0.0;
|
|
double isa_asymptotic = 0.0;
|
|
// b^2 stood at least two standard errors clear of zero. When it did not, `isa` is still the
|
|
// fitted 1/b and every decision reads it as before; only what is PRINTED says "undetermined",
|
|
// since 1/b of a b that is zero within its error is the noise in b, not an I/sigma.
|
|
bool isa_resolved = true;
|
|
double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2)
|
|
double error_model_b = 0.0;
|
|
// The overall CC1/2 as it stood BEFORE the correction surfaces were folded in. That is what the
|
|
// two-pass quality guard compares one pass against the other with: a pass whose intensities are
|
|
// discarded fits no surfaces (see full_stats), so judging the pass that does fit them by its
|
|
// corrected CC1/2 would set two different measurements against each other. Equal to
|
|
// statistics.overall.cc_half whenever no surface was fitted, and NaN when the caller did not
|
|
// ask for it (see measure_cc_before_corrections) - it costs a merge, so it is not measured on
|
|
// spec, and a caller that did not ask must not be handed a number that looks measured.
|
|
double cc_half_before_corrections = std::numeric_limits<double>::quiet_NaN();
|
|
// Where the automatic CC1/2 fit says the data reach, without the deliberate one-shell
|
|
// extension the reflections are written to. Empty when no automatic cut ran.
|
|
std::optional<double> resolution_fit_A;
|
|
// The merged reflections the run kept, binned over the reference range of
|
|
// ScalingSettings::ReportResolutionRange_A (--report-resolution) instead of the run's own,
|
|
// and the error model refitted on the observations of that table alone, for its ISa. Nothing
|
|
// past the run's own cut is in it: those shells are empty, and the overall numbers are over
|
|
// what the run kept. Absent when no range was given, on a search merge, and on a merge whose
|
|
// numbers are not written.
|
|
std::optional<MergeStatistics> reference_statistics;
|
|
double reference_isa = 0.0;
|
|
// The per-frame scaling loops (partials, then the fulls): the iterations each used and
|
|
// whether it settled (rms |log(G_new/G_old)| below its tolerance) before the cap. A merge
|
|
// whose scales were still moving when the cap stopped them is reported as such, and every
|
|
// gate reading it is reading an unsettled state.
|
|
int scaling_iterations_partials = 0;
|
|
int scaling_iterations_fulls = 0;
|
|
bool scaling_converged = true;
|
|
};
|
|
|
|
// experiment: read live (its space group is changed by the caller between Run() calls).
|
|
// partial_outcomes: the per-frame partials; the final per-frame scale (G, CC, mosaicity) is written
|
|
// back onto them so the offline per-image scaling table is still exported.
|
|
// reference_cell: the consensus cell (for the completeness count and the cell-consistency mask).
|
|
RotationScaleMerge(const DiffractionExperiment &experiment,
|
|
std::vector<IntegrationOutcome> &partial_outcomes,
|
|
std::optional<UnitCell> reference_cell,
|
|
int scaling_iterations,
|
|
size_t nthreads,
|
|
Logger &logger,
|
|
std::string observation_dump_path = {});
|
|
|
|
// Copy the per-frame partials into the flat buffers. Call once before the first Run().
|
|
void Ingest();
|
|
|
|
// Scale (per-frame G) -> smooth G -> 3D combine -> scale fulls -> merge -> error model -> statistics
|
|
// for the space group currently set on the experiment, reusing the ingested buffers.
|
|
// for_search: the de-novo P1 pass whose merged intensities feed the space-group search - ice-ring
|
|
// reflections are dropped from the merge and the error model (kept otherwise, for completeness).
|
|
// full_stats: these merged intensities are an OUTPUT. False on the rotation two-pass geometry
|
|
// pre-pass, whose merge exists only to choose the space group and post-refine the geometry and
|
|
// whose reflections are never written: the correction surfaces, the report-only diagnostics, the
|
|
// R_meas re-walk, the anomalous split, the R-free flags and the French-Wilson amplitudes are then
|
|
// all skipped, because computing them fills in fields nothing reads. What the pre-pass IS read for
|
|
// - the merged intensities themselves, the error model, and the completeness / CC1/2 the second
|
|
// pass is judged against - is computed either way.
|
|
// measure_cc_before_corrections: also merge ONCE MORE, just before the correction surfaces, and
|
|
// report that merge's overall CC1/2 in the result. Only a caller comparing two passes of the same
|
|
// data needs it (the pre-pass fits no surfaces, so only the uncorrected number is the same
|
|
// measurement on both sides); it is a whole extra merge, so the offline re-scale path, which
|
|
// compares nothing, asks for it to be left out.
|
|
Result Run(bool for_search, bool full_stats, bool measure_cc_before_corrections);
|
|
|
|
// Override the high-resolution cut for the next Run() - used to gate the de-novo P1 search pass at
|
|
// <I/sigma> >= 1 without cutting the final in-symmetry merge. Reset to the manual limit afterwards.
|
|
void SetDMinLimit(std::optional<double> d_min_A) { d_min_limit = d_min_A; }
|
|
|
|
// Toggle the search-pass Lorentz filter (see search_min_zeta) between Run() calls, so the caller can
|
|
// produce both a filtered and an unfiltered search merge from the same ingested partials.
|
|
void SetSearchMinZeta(double zeta) { search_min_zeta = zeta; }
|
|
[[nodiscard]] double GetSearchMinZeta() const { return search_min_zeta; }
|
|
|
|
private:
|
|
// One integrated observation - a per-frame partial during scaling/combine, or a combined full during
|
|
// scale-fulls/merge. Flat (not nested per image); a POD so the arrays translate straight to CUDA.
|
|
struct Obs {
|
|
int32_t h, k, l;
|
|
float I, sigma, d, prescaling_corr, partiality, zeta, delta_phi, bkg, var_bkg;
|
|
// Fulls only, written by the combine: the full's variance as a function of intensity,
|
|
// var(I) = var_bkg + var_per_I * I. The merge rebuilds it at the reflection's mean.
|
|
float var_per_I = 0.0f;
|
|
float px = NAN, py = NAN; // predicted detector position (for the absorption surface; CPU path only)
|
|
float image_number; // fractional frame position (for 3D-combine contiguity)
|
|
int32_t frame; // index of the outcome whose per-frame scale G applies to this obs
|
|
uint8_t on_ice;
|
|
float corr; // image_scale_corr (working; updated by scaling)
|
|
int32_t group; // dense ASU-group id for the current space group; <0 = never mergeable
|
|
};
|
|
|
|
// One leverage-corrected error-model sample per usable full: its raw variance, its group's mean
|
|
// intensity, its squared deviation from that mean - and the resolution it sits at, because the fit is
|
|
// re-run over the samples that survive the automatic resolution cutoff. See MergeAndStats.
|
|
struct Sample { double s2, I2, dev2; float d; };
|
|
|
|
// The narrow per-observation record the ingest sort orders: the raw hkl the runs are cut on, the
|
|
// frame position that breaks a tie inside one, the observation's own index (which makes the order
|
|
// total - see the .cpp), and the resolution the range test reads. Twenty bytes against the Obs's
|
|
// eighty, and it is all the ingest needs before it knows which observations survive. Sixteen bits
|
|
// are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the longest axis at
|
|
// atomic resolution - and on a long axis this array sets the sort's memory high-water mark.
|
|
struct SortKey {
|
|
int16_t h, k, l;
|
|
float image_number;
|
|
int32_t idx;
|
|
float d;
|
|
};
|
|
|
|
const DiffractionExperiment &x;
|
|
std::vector<IntegrationOutcome> &partials_out; // written back at the end of scaling
|
|
std::optional<UnitCell> reference_cell;
|
|
size_t nthreads;
|
|
Logger &logger;
|
|
std::string observation_dump_path;
|
|
|
|
// Fixed settings snapshot (read once in the ctor).
|
|
int n_frames = 0;
|
|
double min_partiality = 0.02;
|
|
std::optional<double> d_min_limit;
|
|
std::optional<double> d_max_limit;
|
|
bool merge_friedel = true;
|
|
double capture_uncertainty_coeff = 0.0;
|
|
double min_captured_fraction = 0.0;
|
|
// RockingEventFrameGap of this run's oscillation width, taken once so that the two places that
|
|
// cut events on the CPU and the GPU kernel that cuts them on the device all compare the same
|
|
// float (RotationScaleMergeGPU.cu documents that bit-parity contract).
|
|
float max_frame_gap = 2.0f;
|
|
|
|
// Drop a frame's observations entirely when the frame disagrees with the merged reference below this
|
|
// correlation (--min-image-cc). A mis-centred or off-crystal frame still produces spots, still
|
|
// indexes and still integrates - it just measures something that is not the crystal's diffraction,
|
|
// and nothing downstream removes it. 0 = off.
|
|
double min_cc_for_image = 0.0;
|
|
|
|
// Exclude observations with |zeta| below this from the DE-NOVO SEARCH merge only (the final merge
|
|
// keeps everything). zeta is the sine of the angle between a reflection's rocking path and the
|
|
// spindle: near 0 it crosses the Ewald sphere almost tangentially, spends many frames in
|
|
// diffracting position and is measured badly. The symmetry search compares how equal an operator's
|
|
// paired intensities are, so it is answered by whichever reflections are worst measured - and when
|
|
// the spindle lies in a lattice plane, an operator permuting the two in-plane axes samples a
|
|
// different mixture of qualities than one that only flips signs, which is not a fair comparison.
|
|
// Unlike a bound on I/sigma this is pure geometry, identical in meaning on every dataset. 0 = off.
|
|
double search_min_zeta = 0.0;
|
|
double reject_nsigma = 0.0;
|
|
bool reject_outliers = false;
|
|
double rfree_fraction = 0.0;
|
|
// The cap on the per-frame scaling loops. The loops run to a tolerance (see RunScalingLoop);
|
|
// this is how far they may go before giving up and saying so.
|
|
int scaling_iter = 100;
|
|
bool scale_fulls = true;
|
|
bool refine_decay_b = false; // per-time-block Debye-Waller decay correction (radiation damage)
|
|
int absorption_iter = 0; // >0: fit a goniometer-frame absorption surface over this many iterations
|
|
int modulation_iter = 0; // >0: fit a detector-plane modulation (flat-field) surface, this many iterations
|
|
double relative_b_deg = 0.0; // >0: fit a per-batch relative-B (batch width in deg); 0 = off
|
|
double mosaicity_deg = 0.1;
|
|
// Automatic high-resolution cutoff for the written reflections + reported shells (post-merge; the
|
|
// scaling, combine and error model always run over the full range). Manual d_min_limit wins.
|
|
ResolutionCutoffMethod resolution_cutoff_method = ResolutionCutoffMethod::Off;
|
|
double resolution_cc_target = 0.30;
|
|
int report_shell_count = 9;
|
|
std::optional<ReportResolutionRange> report_range; // the second, reference-range table
|
|
|
|
// Flat buffers, allocated once by Ingest() and reused across Run() calls. The full Obs array is
|
|
// built only when the device pipeline is NOT resident (no GPU, or an observation dump was asked
|
|
// for); on the resident path (resident_ingest) the device holds the observations and the host
|
|
// keeps only the ingest arrays below, until the end of Ingest.
|
|
std::vector<Obs> partials; // all per-frame partials, grouped by frame
|
|
bool resident_ingest = false;
|
|
// What the ingest-time geometry smoothing (SmoothMosaicityAndPartiality) needs beyond the source
|
|
// reflections, one array per field in `partials` order: the two fields it rewrites, and the two the
|
|
// rocking-event walks read in raw-hkl order, where a source reflection is out of reach. Every other
|
|
// field it reads - hkl, frame, d, zeta - is read from the source reflection itself (ForEachKeptIn),
|
|
// so on the resident path no per-observation record is built at all: on a fine-sliced long axis
|
|
// that record was gigabytes, held beside the source reflections it was copied from. Filled by
|
|
// BuildInRangeObservations and handed back at the end of Ingest.
|
|
std::vector<float> ingest_delta_phi, ingest_partiality, ingest_image_number;
|
|
// The rocking-event usability test (RockingEventFrames) on the ingest-time values - which is what
|
|
// that walk reads on the resident path, where the host corr is never refreshed (see
|
|
// partials_released). Resident path only.
|
|
std::vector<uint8_t> ingest_rock_ok;
|
|
// Which source reflections the resolution range kept (indexed like the sort keys: frame by frame,
|
|
// from ingest_src_start), so that the i-th kept one of a frame can be found again.
|
|
std::vector<uint8_t> ingest_keep;
|
|
std::vector<int32_t> ingest_src_start;
|
|
// Call fn(observation index, source reflection) for each reflection of frame o that the ingest
|
|
// kept, in order.
|
|
template <class Fn> void ForEachKeptIn(int o, Fn fn) const {
|
|
int32_t src = ingest_src_start[o];
|
|
int32_t at = frame_start[o];
|
|
for (const auto &r : partials_out[o].reflections)
|
|
if (ingest_keep[src++]) fn(at++, r);
|
|
}
|
|
// How many observations Ingest built (== partials.size() until any release below), so the
|
|
// sizes survive when the array itself does not.
|
|
int n_partials_obs = 0;
|
|
// With the GPU resident and no observation dump, no host stage reads the 80-byte Obs records
|
|
// after the device upload: scaling, combine, scale-fulls and merge run on the device,
|
|
// ComputeAsuGroups stamps the flat group_ids array, every pass restarts corr from
|
|
// corr_ingested, and the CPU combine only runs for the dump. The one late host reader,
|
|
// RockingEventFrames, sees only ingest-time values on that path (the host corr is refreshed
|
|
// only in the dump fallback), so Ingest takes its answer and hands the array - a second
|
|
// full-size copy of the partials, gigabytes on a fine-sliced long axis - straight back.
|
|
bool partials_released = false;
|
|
int rocking_event_frames_at_ingest = 0;
|
|
std::vector<int32_t> frame_start, frame_count; // CSR ranges of `partials` per frame
|
|
std::vector<uint8_t> frame_cell_ok; // per-frame cell-consistency mask (1 = kept)
|
|
std::vector<uint8_t> finite_ok; // per-obs AcceptReflection finiteness (immutable; 1 = kept)
|
|
std::vector<double> g_partial; // per-frame partial scale G (the RESIDUAL after the flux)
|
|
std::vector<double> frame_flux; // per-frame incident flux, run median = 1 (see the .cpp)
|
|
// corr as Ingest built it, kept for the whole life of the object and put back at the start of every
|
|
// Run. A pass that starts anywhere else is not the pass that would have run on its own, and the
|
|
// whole of the difference is the ITERATION COUNT: the scaling loop's fixed point is where it left
|
|
// off, so a pass handed the previous pass's fitted corr does not redo its scaling_iter iterations,
|
|
// it CONTINUES them. A de-novo run calls Run four times, and the fourth was therefore scaled at
|
|
// twelve iterations where the caller asked for three.
|
|
//
|
|
// The surplus is harmless where multiplicity is high and harmful where it is not, and every
|
|
// space-group search pass sits in the second case. Merged in a determined point group the
|
|
// per-frame fit converges by the second iteration and the collapsed-scale guard's drop count is
|
|
// stationary from there - over the rotation battery it moves 377 -> 397 frames between three
|
|
// iterations and twelve, and 30 of 39 crystals drop nothing at any count. Merged in P1, where a
|
|
// reflection has a couple of observations rather than a couple of dozen, it does not converge in
|
|
// twelve: each iteration rebuilds the reference from the intensities the last one scaled, the
|
|
// healthy frames drift down with the weak ones, and the guard drops more of the sweep every time.
|
|
// Same guard, same battery, two regimes: summed over the search passes it drops 417 frames when a
|
|
// search pass runs one iteration and 1101 when it runs three, against the 377 -> 397 above. On one
|
|
// rotation crystal that repeatedly left the beam a single search pass drops 93 frames of 1800 at
|
|
// three iterations, 124 at six and 264 at nine, and the space group determined from the
|
|
// nine-iteration merge is wrong. A later pass inherits that state rather than creating it, so this
|
|
// is a defect of the P1 search arm, and it reaches the user through the point group that arm picks.
|
|
std::vector<float> corr_ingested;
|
|
|
|
// Raw-hkl ordering, built ONCE by Ingest and reused: `perm` lists partial indices sorted by
|
|
// (raw h,k,l, image_number); each distinct raw hkl is a contiguous run [rawrun_start, +count) of it.
|
|
// The expensive sort happens once here, so per-pass combine (event split) and ASU grouping are linear.
|
|
std::vector<int32_t> perm;
|
|
std::vector<int32_t> rawrun_start, rawrun_count;
|
|
std::vector<int32_t> rawrun_h, rawrun_k, rawrun_l;
|
|
std::vector<float> rawrun_d; // representative resolution per raw hkl
|
|
std::vector<int32_t> rawrun_group; // dense ASU-group id per raw hkl (<0 = absent/out of range)
|
|
|
|
std::vector<Obs> fulls; // combined fulls (rebuilt each Run), sorted by frame
|
|
std::vector<int32_t> fulls_frame_start, fulls_frame_count; // CSR ranges of `fulls` per frame
|
|
std::vector<double> g_full; // per-frame scale on the fulls
|
|
std::vector<double> frame_scale_fulls; // per-frame total scale of the fulls before the surfaces (cc_weight)
|
|
|
|
// The five fields the host-side walks over the fulls actually read, pulled out of the 80-byte record
|
|
// once per merge (see MergeAndStats). `group` carries the usability decision: -1 means the full is
|
|
// not in this merge, which is what a negative group already meant. Members rather than locals for the
|
|
// same reason as FullsStaging below - the merge runs several times per run and this is tens of
|
|
// megabytes each time.
|
|
struct MergeFields {
|
|
std::vector<int32_t> group;
|
|
std::vector<float> I, sigma, corr, d;
|
|
void Resize(int n) { group.resize(n); I.resize(n); sigma.resize(n); corr.resize(n); d.resize(n); }
|
|
};
|
|
MergeFields merge_fields;
|
|
|
|
// One host array per field for the fulls download: the device hands back an array per field and the
|
|
// host gathers them into `fulls`. Members rather than locals in Run() because the whole
|
|
// scale->combine->merge chain runs several times per run and these are a few hundred megabytes
|
|
// between them, so as locals every chain allocates, faults in and zeroes the lot again.
|
|
struct FullsStaging {
|
|
std::vector<int32_t> h, k, l, frame, group;
|
|
std::vector<float> I, sigma, d, image_number, corr, px, py, var_bkg, var_per_I;
|
|
std::vector<uint8_t> on_ice;
|
|
void Resize(int n) {
|
|
h.resize(n); k.resize(n); l.resize(n); frame.resize(n); group.resize(n);
|
|
I.resize(n); sigma.resize(n); d.resize(n); image_number.resize(n); corr.resize(n);
|
|
px.resize(n); py.resize(n); var_bkg.resize(n); var_per_I.resize(n);
|
|
on_ice.resize(n);
|
|
}
|
|
};
|
|
FullsStaging fulls_staging;
|
|
|
|
// The merge accumulators (see MergeAndStats' run_merge): one entry per ASU group, plus the arrays
|
|
// the device kernel fills that the host unpacks into them. Members for the same reason as
|
|
// FullsStaging - a merge runs several times per run and this is a hundred megabytes between them.
|
|
// swh_typ: each half-set's precision had every observation been at the run's typical frame scale
|
|
// (see MergeAndStats, cc_weight).
|
|
struct Accum { double swI = 0, sw = 0, swIh[2] = {0, 0}, swh[2] = {0, 0}, swh_typ[2] = {0, 0};
|
|
size_t nh[2] = {0, 0}; float d = NAN; bool on_ice = false; };
|
|
std::vector<Accum> merge_acc;
|
|
struct MergeAccumStaging {
|
|
std::vector<double> swI, sw, swIh0, swIh1, swh0, swh1, swh_typ0, swh_typ1, d;
|
|
std::vector<int32_t> nh0, nh1, rej;
|
|
std::vector<uint8_t> on_ice;
|
|
void Resize(int n) {
|
|
swI.resize(n); sw.resize(n); swIh0.resize(n); swIh1.resize(n); swh0.resize(n); swh1.resize(n);
|
|
swh_typ0.resize(n); swh_typ1.resize(n);
|
|
d.resize(n); nh0.resize(n); nh1.resize(n); rej.resize(n); on_ice.resize(n);
|
|
}
|
|
};
|
|
MergeAccumStaging merge_accum;
|
|
|
|
// Per-group scatter for the strong-reflection ISa asymptote (see MergeAndStats). A member for the
|
|
// same reason: 32 bytes a group, once per merge.
|
|
struct GroupScatter { double sum = 0, sum_sq = 0, sum_var = 0; int n = 0; };
|
|
std::vector<GroupScatter> asymptote_scatter;
|
|
|
|
// The error model's working pools (see MergeAndStats): the samples themselves, the scratch copy each
|
|
// fit partitions, the misfit-free subset the refit uses, the subset inside the resolution cutoff, and
|
|
// the per-sample chi2 the reported number is the median of. Members for the same reason as FullsStaging - one is 32 bytes per full and
|
|
// there are two dozen fits per run, so as locals this is gigabytes of pages faulted in and handed
|
|
// straight back. Every one of them is cleared and refilled before it is read.
|
|
std::vector<Sample> em_samples, em_fit_pool, em_refit_pool, em_cut_pool;
|
|
std::vector<double> em_chi2;
|
|
|
|
// Set by FitPerFrameG: which frames were fitted this call (so corr/G is updated only there).
|
|
std::vector<uint8_t> frame_scaled_scratch;
|
|
|
|
// Per-frame mosaicity smoothed in frame order (deterministic); used to recompute partiality and
|
|
// written back for the per-image scaling table. Empty if there is no per-frame mosaicity.
|
|
std::vector<float> mos_smooth;
|
|
|
|
// Radiation-damage monitor (measured by MeasureRadiationDamageB on the scaled fulls before any decay
|
|
// correction; report-only, copied into the result statistics by MergeAndStats). NaN / empty until set.
|
|
double rad_damage_delta_b = std::numeric_limits<double>::quiet_NaN(); // relative-B first->last (A^2)
|
|
std::vector<float> rad_damage_b_batch; // per-batch relative-B curve (A^2)
|
|
double rad_damage_batch_deg = 0.0; // rotation width per batch (deg)
|
|
|
|
// Sweep-quality diagnostic (MeasureSweepQuality, then the delta-CC1/2 and the disposition;
|
|
// copied into the result statistics by MergeAndStats). Empty and not measured until it runs.
|
|
SweepQuality sweep_quality;
|
|
|
|
// The two per-frame channels MeasureSweepQuality segmented, kept so the ledger can recompute a
|
|
// range's numbers after the ranges have been split at the rejection boundaries. Relative to the
|
|
// run median; 0 in both means the frame contributed nothing.
|
|
std::vector<double> sweep_scale, sweep_cc;
|
|
|
|
// Frames whose observations delta-CC1/2 convicted (1 = out of the merge). Sized n_frames and all
|
|
// zero until MeasureBatchDeltaCCHalf runs; Run() is what takes the observations out.
|
|
std::vector<uint8_t> frame_rejected;
|
|
|
|
// Frames whose observations reach this pass's merge at all (1 = in), before delta-CC1/2 has its say.
|
|
// Built by Run(), where every per-frame drop is decided. A full cannot answer this: the combine
|
|
// attributes a rocking event to its PEAK frame, so on fine slicing most frames own no full at all
|
|
// while their measurements sit inside one.
|
|
std::vector<uint8_t> frame_in_merge;
|
|
|
|
// Working per-group arrays (sized to the current group count; reused).
|
|
std::vector<int32_t> group_h, group_k, group_l;
|
|
|
|
#ifdef JFJOCH_USE_CUDA
|
|
// GPU engine: the whole hot path (scaling, combine, scale-fulls, per-frame CC, smooth-G, merge +
|
|
// error model) runs on the device, resident, when a GPU is present. Null / inactive otherwise, with
|
|
// the CPU loops as the bit-parity fallback. Built in Ingest.
|
|
std::unique_ptr<RotationScaleMergeGPU> gpu_;
|
|
bool gpu_active_ = false;
|
|
#endif
|
|
|
|
// --- helpers (each a flat pass; see the .cpp) ---
|
|
// Turn the per-frame mean background under the reflections (accumulated by the ingest fill loop) into
|
|
// the per-frame incident flux, which the finiteness pass then folds into prescaling_corr so that
|
|
// corr = prescaling_corr / (partiality * G) divides it out and G fits only the residual. See the .cpp for why a
|
|
// background is a usable flux meter and what it costs when it is not.
|
|
void MeasureIncidentFlux(const std::vector<double> &mean_bkg);
|
|
|
|
// Build the flat `partials` array (and the per-frame CSR, the finiteness mask and `perm`) from the
|
|
// source reflections, skipping the observations whose resolution can never be in range:
|
|
// --scaling-high/low-resolution are the coarsest limits any Run() uses (the space-group search only
|
|
// ever RAISES d_min), and an out-of-range raw hkl gets group -1 in every pass, which keeps it out of
|
|
// the scaling reference, the per-frame fit, the combine, the merge and the error model alike. On a
|
|
// crystal that integrates to the detector corner and merges well short of it that is most of the
|
|
// array, and an eighty-byte record built for it is eighty bytes written and then thrown away. Whole
|
|
// raw-hkl RUNS are skipped, on the same per-hkl resolution ComputeAsuGroups tests, so what survives -
|
|
// and the order of every sum formed over it - is exactly what it would have been had the whole array
|
|
// been built and then filtered. Everything is built when no manual limit was given.
|
|
// `keys` is CONSUMED: it is freed as soon as the permutation has been remapped, before the
|
|
// observation array is built, so the two never coexist - together they would be another third of
|
|
// the payload on top of it.
|
|
void BuildInRangeObservations(std::vector<SortKey> &keys);
|
|
|
|
// Compute the dense ASU-group id for the current space group by grouping the (pre-sorted) raw-hkl
|
|
// runs by their ASU key - one gemmi ASU reduction per distinct raw hkl, not per observation. Fills
|
|
// rawrun_group, the group_h/k/l representative tables, and partials[].group; returns the group count.
|
|
int ComputeAsuGroups(const HKLKeyGenerator &key_generator);
|
|
|
|
// The observation's inverse-variance weight in the scaling reference, and its scaled intensity;
|
|
// 0 when the observation is not in the reference at all.
|
|
static double ReferenceWeight(const Obs &o, double min_partiality, float &I_corr);
|
|
// Inverse-variance per-group mean of I*corr over `obs` (the merge reference).
|
|
void ReduceGroupMeans(const std::vector<Obs> &obs, int n_groups, std::vector<double> &out_mean) const;
|
|
|
|
// Robust per-frame G fit (IRLS, Cauchy k=3), unity=false uses the rotation partiality, unity=true the
|
|
// scale-fulls (partiality already folded in). Reads out_mean[group] as the reference intensity.
|
|
void FitPerFrameG(std::vector<Obs> &obs, const std::vector<int32_t> &fstart,
|
|
const std::vector<int32_t> &fcount, const std::vector<double> &group_mean_in,
|
|
bool unity, std::vector<double> &g);
|
|
|
|
// The alternating per-frame scale fit run to convergence: `iterate` fits every frame once against
|
|
// the current reference (leaving G in `g` and the frames it fitted in frame_scaled_scratch),
|
|
// `rescale` multiplies the corr of the flagged frames' observations by a per-frame ratio. Pins the
|
|
// gauge after every iteration and stops when the scales settle or the cap is reached. On return
|
|
// frame_scaled_scratch flags every frame fitted in any iteration.
|
|
struct ScalingLoopOutcome { int iterations = 0; bool converged = false; double step = 0.0; };
|
|
ScalingLoopOutcome RunScalingLoop(const char *what, std::vector<double> &g,
|
|
const std::vector<int32_t> &frame_obs_count,
|
|
const std::function<void()> &iterate,
|
|
const std::function<void(const std::vector<uint8_t> &,
|
|
const std::vector<double> &)> &rescale);
|
|
|
|
// corr = prescaling_corr / (partiality * G[frame]); leaves corr unchanged for frames that could not be fit.
|
|
void UpdateCorr(std::vector<Obs> &obs, const std::vector<double> &g,
|
|
const std::vector<uint8_t> &frame_scaled) const;
|
|
|
|
void SmoothG(std::vector<Obs> &obs, std::vector<double> &g, int window) const;
|
|
|
|
// The windowed geometric mean of G over frames (the shared first half of SmoothG); the GPU path
|
|
// applies the resulting ratio to the resident corr in a kernel instead of the host obs loop.
|
|
void ComputeSmoothGWindow(const std::vector<double> &g, int window,
|
|
std::vector<double> &g_smooth) const;
|
|
|
|
// Drop the observations of any frame whose fitted per-frame scale collapsed far below the run
|
|
// median, reporting the per-frame corr factor the caller has to apply. See the .cpp for why nothing
|
|
// downstream can catch a collapsed scale on its own.
|
|
bool DropCollapsedScales(const std::vector<uint8_t> &fitted_mask, std::vector<double> &g,
|
|
std::vector<uint8_t> &apply, std::vector<double> &ratio) const;
|
|
|
|
// Smooth per-frame mosaicity in frame order and recompute each partial's partiality from it, so the
|
|
// per-frame partials of one rocking event tile the curve consistently (they sum toward 1) before the
|
|
// 3D combine. Deterministic (frame order); replaces the old arrival-order mosaicity moving average
|
|
// that prediction applied. SG-independent, so done once in Ingest.
|
|
void SmoothGeometry();
|
|
void SmoothMosaicityAndPartiality();
|
|
|
|
void Combine(); // partials -> fulls (CPU)
|
|
|
|
// Is this full in the merge? group >= 0 already encodes "not absent and passes AcceptReflection";
|
|
// the rest is the frame's cell-consistency mask, a usable scale and a usable sigma. The P1 search
|
|
// pass adds its own ice test on top of this (see MergeAndStats).
|
|
[[nodiscard]] bool UsableFull(const Obs &o) const;
|
|
|
|
// Drop the fulls of any frame whose scale collapsed toward zero. The fulls are scaled with the Unity
|
|
// model, so their corr IS 1/G and a collapsed G multiplies every intensity on that frame without
|
|
// bound. Covers the CPU and GPU scaling paths alike; `from_staging` says the fulls' frame and corr
|
|
// are still in fulls_staging, which is where the scan reads them from when they are. Returns true if
|
|
// anything was dropped (the caller then has to push the corrected corr back to the device).
|
|
bool DropCollapsedFullScales(bool from_staging);
|
|
|
|
// Post-scale-fulls correction surfaces, each an alternating multiplicative fit of the host fulls' corr
|
|
// against the merged reference (cheap host loops; the corrected corr is re-uploaded to the resident
|
|
// fulls afterwards). Each is cross-validated (fit even frames, keep only if held-out odd equivalents
|
|
// improve) so it is a no-op when its systematic is absent. RefineDecay fits a global Debye-Waller B
|
|
// (resolution x time - radiation damage the resolution-flat per-frame G cannot capture; also gated on a
|
|
// physical total-dB floor). RefineAbsorption fits a smooth factor over the diffracted-beam direction in
|
|
// the goniometer frame (path-length / absorption; negligible at hard X-rays, matters at low energy).
|
|
void RefineDecay(int n_groups);
|
|
// The observation fields the decay / relative-B passes read, pulled out of `fulls` once. Those
|
|
// passes run a dozen walks over the eighty-byte records for twenty bytes of each, and their
|
|
// per-group references had to stay serial because they scatter into a per-group array. `term` keeps
|
|
// fulls order, so a sum formed over it is formed from the same terms in the same order the serial
|
|
// walk used; `gterm` holds the same terms in ASU-group order, by a stable counting sort over `term`,
|
|
// so a group's terms are still added in fulls order - which lets a thread own whole groups and gives
|
|
// the same per-group sums. `g_start` is that array's CSR.
|
|
struct DecayTerm { float I, sigma, corr, d, image_number; int32_t frame, group; };
|
|
struct DecayTerms {
|
|
std::vector<DecayTerm> term, gterm;
|
|
std::vector<int32_t> g_start;
|
|
};
|
|
void BuildDecayTerms(int n_groups, DecayTerms &t) const;
|
|
// Solve a smooth per-batch relative-B from the per-batch normal equations for b (num_c, den_c):
|
|
// data-fidelity + a second-difference (curvature) penalty, by Gauss-Seidel, each batch clamped to
|
|
// +-b_max. Returns the un-anchored curve; the caller sets the gauge and the clamp it can live with.
|
|
// Shared by the correction and the radiation-damage monitor.
|
|
std::vector<double> SolveCurvatureSmoothedB(const std::vector<double> &num,
|
|
const std::vector<double> &den, double b_max) const;
|
|
// Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset
|
|
// {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a
|
|
// subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction.
|
|
std::vector<double> FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
|
|
int frames_per_batch, int gparity) const;
|
|
// Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before
|
|
// any decay correction and store the first->last relative-B change + the per-batch curve on this object
|
|
// (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF).
|
|
void MeasureRadiationDamageB(int n_groups);
|
|
// Sweep-quality diagnostic: find the contiguous stretches of the sweep over which the crystal
|
|
// delivered much less than the rest of the run, and say what each one looks like. Reads the
|
|
// per-frame scale (with the incident flux already divided out) and the per-frame CC to merge.
|
|
// It only DETECTS and NAMES; what is done about a stretch is MeasureBatchDeltaCCHalf's decision.
|
|
void MeasureSweepQuality(const std::vector<uint8_t> &partial_scaled, const std::vector<double> &cc,
|
|
const std::vector<int64_t> &cc_n);
|
|
// Fill in every ledger range's numbers from sweep_scale / sweep_cc. assign_reasons=false keeps the
|
|
// reasons already on the ranges, which is what a re-fill after the ranges were split wants.
|
|
void FillSweepRanges(bool assign_reasons);
|
|
// The frames one rocking event spans, as the combine cuts them: the median over the run's events.
|
|
// Walked on the partials, because the combine also runs on the GPU and a full keeps only the frame
|
|
// of its peak partial.
|
|
[[nodiscard]] int RockingEventFrames() const;
|
|
template <class UsableFn, class ImgFn>
|
|
[[nodiscard]] int RockingEventFramesOver(UsableFn usable, ImgFn img) const;
|
|
// Per-batch delta-CC1/2 on the corrected fulls: measure what keeping each batch of the sweep costs
|
|
// the merged intensities, convict the batches that cost significantly, slide the conviction's edges
|
|
// onto the frames that carry it, and turn the result into the disposition ledger. Fills
|
|
// frame_rejected, which Run() then takes out of the merge. See the .cpp for the statistic and the
|
|
// discipline.
|
|
void MeasureBatchDeltaCCHalf(int n_groups);
|
|
// Split the ledger ranges at the boundaries of frame_rejected and add a range for every rejected
|
|
// stretch no ledger range covers, so that each range is wholly rejected or wholly kept.
|
|
void SplitSweepRanges();
|
|
// Give every frame and every ledger range its disposition, and count the sweep.
|
|
void CountSweepDisposition();
|
|
// Per-batch relative-B, applied after RefineDecay: the single decay slope removes the average
|
|
// radiation-damage falloff, but the relative scattering power drifts NON-monotonically across a run
|
|
// (absorption path, crystal slippage, dose bursts). Refine one relative Debye-Waller B per batch
|
|
// (FitRelativeBCurve), anchored to zero mean (the constant part is a global Wilson-B, degenerate with
|
|
// overall scale). Guarded by a physical peak-to-peak floor and cross-validated by ASU-GROUP parity (a
|
|
// per-batch parameter cannot be scored on a held-out FRAME the batch owns; splitting the equivalents
|
|
// tests whether a batch's B generalises to reflections it was not fit on). Opt-in (--relative-b).
|
|
void RefineRelativeB(int n_groups);
|
|
void RefineAbsorption(int n_iter, int n_groups);
|
|
// Time-dependent absorption: the same cross-validated surface, indexed by (rotation, detector position)
|
|
// instead of by the crystal-frame direction alone. RefineAbsorption's parameterisation is the whole
|
|
// model only while the illuminated volume stays put; once the crystal drifts through the beam the exit
|
|
// path becomes a function of the spindle angle too, and nothing time-independent reaches it.
|
|
void RefineAbsorptionTime(int n_iter, int n_groups);
|
|
// Detector-plane modulation (flat-field): the same cross-validated surface fit as absorption, but the
|
|
// cell is the predicted detector position (px, py) instead of the goniometer-frame direction. Corrects
|
|
// detector-response / geometric systematics that vary with where a reflection lands; because it lives
|
|
// in the detector frame (not tied to the rotation) the same correction concept applies to stills.
|
|
void RefineModulation(int n_iter, int n_groups);
|
|
// Shared engine for the correction surfaces: given a per-full cell assignment (cell[i] in [0,ncell), or
|
|
// <0 to skip), fit a Tikhonov-regularised multiplicative factor per cell against the merged reference,
|
|
// cross-validate on even/odd frames, and fold it into corr only if the held-out equivalents improve.
|
|
void ApplyCellSurface(const std::vector<int32_t> &cell, int ncell, int n_iter, int n_groups,
|
|
const char *name);
|
|
|
|
// Sort `fulls` by peak frame and (re)build fulls_frame_start/count (the per-frame CSR the scale-fulls
|
|
// step slices). Shared by the CPU Combine tail and the GPU combine path.
|
|
void SortFullsByFrame();
|
|
|
|
// Per-frame CC vs the partial merge reference (CPU; the GPU equivalent is gpu_->ComputePartialCC).
|
|
void ComputePerFrameCC(const std::vector<double> &partial_group_mean,
|
|
std::vector<double> &cc, std::vector<int64_t> &cc_n) const;
|
|
|
|
// Write G/CC/mosaicity back onto the partials (once, at the end of partial scaling) from the given
|
|
// per-frame cc/cc_n, so the offline per-image scaling table is still exported.
|
|
void FinalizePerFrameScale(const std::vector<double> &cc, const std::vector<int64_t> &cc_n,
|
|
const std::vector<uint8_t> &frame_scaled);
|
|
|
|
// Error model + merge + statistics over the fulls (the last stage). n_groups is the fulls group count.
|
|
// fulls_resident: the (scaled) fulls + their group CSR are still on the GPU, so the em-stats / samples
|
|
// / merge-accumulate / R_meas reductions run there (only per-group + samples come back).
|
|
// full_stats: see Run().
|
|
Result MergeAndStats(int n_groups, bool for_search, bool fulls_resident, bool full_stats);
|
|
};
|