// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #pragma once #include #include #include #include #include #include #include #include #include #include #include #include "../../common/DiffractionExperiment.h" #include "../../common/Logger.h" #include "../../common/Reflection.h" #include "../../common/UnitCell.h" #include "../IntegrationOutcome.h" #include "ErrorModel.h" #include "WilsonOutliers.h" #include "Merge.h" // MergedReflection, MergeStatistics #include "SphericalHarmonicSurface.h" #ifdef JFJOCH_USE_CUDA #include #include "RotationScaleMergeGPU.h" #endif // A vector allocator that leaves each new element default-initialised - for a type without initialisers // of its own, not written at all - instead of zeroing it. An array sized on one thread and then filled // on all of them is otherwise first written page by page by that one thread: on a long sweep, gigabytes // of page faults taken serially before the parallel fill starts. Everything else is std::allocator's. template struct DefaultInitAllocator : std::allocator { template struct rebind { using other = DefaultInitAllocator; }; DefaultInitAllocator() noexcept = default; template DefaultInitAllocator(const DefaultInitAllocator &) noexcept {} template void construct(U *p) noexcept(std::is_nothrow_default_constructible_v) { ::new (static_cast(p)) U; } template void construct(U *p, Args &&... args) { ::new (static_cast(p)) U(std::forward(args)...); } }; template using FillVector = std::vector>; // Dedicated, allocate-once scale+combine+merge for rotation data (the -P rot3d path): recompute the // per-frame partiality from the (smoothed) mosaicity, robustly fit a per-image scale G, 3D-combine each // rocking event's partials into fulls, refit a per-frame scale on the fulls (XDS order), and merge with // a global error model. // // The per-frame partial observations are ingested ONCE into flat vectors; the hkl->ASU grouping is // computed once per space group (by a sort, not a map) and reused across all scaling iterations; every // hot step is a flat loop over those vectors, so the whole pipeline maps onto CUDA kernels (segmented // reduction + per-frame solve) and runs GPU-resident when a GPU is present, with the CPU loops as the // bit-parity fallback. CC1/2 and the per-image CC are computed once at the end, not every iteration. // // Used only for the self-scaling rotation case with per-image G (Rotation partiality, a fixed/forced // mosaicity is honoured by the recompute). Post-scale-fulls correction stages (on by default via // ScalingSettings::CorrectionSurfaces): a global Debye-Waller decay and a goniometer-frame absorption // surface, both fitted on the host and pushed back to the resident (GPU) fulls before the merge. // External-reference scaling, the stills B-factor and wedge refinement are unsupported (caller rejects). // Stills use the per-image ScaleOnTheFly (fixed partiality) instead. // // Before any of it: TrimBlankSweepEnds clears the reflections of the frames at either end of the sweep // that the beam never reached, and returns how many it cleared at the start and at the end. std::pair TrimBlankSweepEnds(std::vector &outcomes, size_t nthreads); class RotationScaleMerge { public: // The log scale x minimising sum_f J_f (x_f - y_f)^2 + lambda * sum_f (x_f - 2 x_{f+1} + x_{f+2})^2: // each frame's own estimate y_f at its information J_f, held to a curve that bends only as far as // the data ask. A frame with J_f = 0 takes the curve through its neighbours. Needs at least two // frames with J_f > 0; returns y unchanged otherwise. static std::vector SmoothLogScale(const std::vector &y, const std::vector &J, double lambda); // The lambda of SmoothLogScale that best predicts frames left out of the fit, over a fixed grid // relative to the typical J. The frames are left out in blocks of `block` (five interleaved folds of // them): a full sums the frames of its rocking curve, so neighbours closer than that share their // measurement and would predict each other whatever the scale does. // The (lambda, fold) solves run on nthreads workers; the choice does not depend on how many. static double ChooseLogScaleSmoothing(const std::vector &y, const std::vector &J, int block, size_t nthreads = 1); struct Result { std::vector merged; MergeStatistics statistics; // Two tiers, and they are different quantities. `isa` is the whole-range 1/sqrt(a*b) - which // in this parameterisation is 1/b - and is what XDS's ISa means, so it is the one exported. // `isa_asymptotic` is the strong-reflection tier, which XDS has no equivalent of and which can // only ever be the more optimistic of the two. Both 0 if the model stayed at identity. double isa = 0.0; double isa_asymptotic = 0.0; // b^2 stood at least two standard errors clear of zero. When it did not, `isa` is still the // fitted 1/b and every decision reads it as before; only what is PRINTED says "undetermined", // since 1/b of a b that is zero within its error is the noise in b, not an I/sigma. bool isa_resolved = true; double error_model_a = 0.0; // XDS convention: sigma^2 = a*(sigma0^2 + b*I^2) double error_model_b = 0.0; // The overall CC1/2 as it stood BEFORE the correction surfaces were folded in. That is what the // two-pass quality guard compares one pass against the other with: a pass whose intensities are // discarded fits no surfaces (see full_stats), so judging the pass that does fit them by its // corrected CC1/2 would set two different measurements against each other. Equal to // statistics.overall.cc_half whenever no surface was fitted, and NaN when the caller did not // ask for it (see measure_cc_before_corrections) - it costs a merge, so it is not measured on // spec, and a caller that did not ask must not be handed a number that looks measured. double cc_half_before_corrections = std::numeric_limits::quiet_NaN(); // Where the automatic CC1/2 fit says the data reach, without the deliberate one-shell // extension the reflections are written to. Empty when no automatic cut ran. std::optional resolution_fit_A; // The merged reflections the run kept, binned over the reference range of // ScalingSettings::ReportResolutionRange_A (--report-resolution) instead of the run's own, // and the error model refitted on the observations of that table alone, for its ISa. Nothing // past the run's own cut is in it: those shells are empty, and the overall numbers are over // what the run kept. Absent when no range was given, on a search merge, and on a merge whose // numbers are not written. std::optional reference_statistics; double reference_isa = 0.0; // The per-frame scaling loops (partials, then the fulls): the iterations each used and // whether it settled (rms |log(G_new/G_old)| below its tolerance) before the cap. A merge // whose scales were still moving when the cap stopped them is reported as such, and every // gate reading it is reading an unsettled state. int scaling_iterations_partials = 0; int scaling_iterations_fulls = 0; bool scaling_converged = true; // The fulls the merge kept, unmerged (see ScaledFull): what passed the outlier tests, inside the // resolution cut. Filled only when SetExportScaledFulls asked for it, and never on a search merge. std::vector scaled_fulls; }; // experiment: read live (its space group is changed by the caller between Run() calls). // partial_outcomes: the per-frame partials; the final per-frame scale (G, CC, mosaicity) is written // back onto them so the offline per-image scaling table is still exported. Where the engine scales on // the host (ReadsReflections), it reads the reflections themselves from the ingest on, so they // must stay where they are, as they are, for as long as it merges: not freed, moved or changed, // but for their h, k and l, which it does not read after the ingest. // reference_cell: the consensus cell (for the completeness count and the cell-consistency mask). RotationScaleMerge(const DiffractionExperiment &experiment, std::vector &partial_outcomes, std::optional reference_cell, int scaling_iterations, size_t nthreads, Logger &logger, std::function observation_dump = {}); // Copy the per-frame partials into the flat buffers. Call once before the first Run(). void Ingest(); // Ingest in its two halves, which is what Ingest() calls: the host half reads the outcomes and // builds every host array, the device half allocates the GPU's arrays and fills them. A caller can // run the host half beside other work and the device half once the GPU has room. With the GPU // resident (no observation dump) the host half only reads the per-frame cells: the reflections go // to the card, and are sorted and cut there, in the device half. void IngestHost(); // With `observations_from` - the engine this one was copied from (CloneIngest), its own // IngestDevice done - the observations are copied from that engine's card arrays rather than // staged from the reflections again. void IngestDevice(const RotationScaleMerge *observations_from = nullptr); // Scale (per-frame G) -> smooth G -> 3D combine -> scale fulls -> merge -> error model -> statistics // for the space group currently set on the experiment, reusing the ingested buffers. // for_search: the de-novo P1 pass whose merged intensities feed the space-group search - ice-ring // reflections are dropped from the merge and the error model (kept otherwise, for completeness). // full_stats: these merged intensities are an OUTPUT. False on the rotation two-pass geometry // pre-pass, whose merge exists only to choose the space group and post-refine the geometry and // whose reflections are never written: the correction surfaces, the report-only diagnostics, the // R_meas re-walk, the anomalous split, the R-free flags and the French-Wilson amplitudes are then // all skipped, because computing them fills in fields nothing reads. What the pre-pass IS read for // - the merged intensities themselves, the error model, and the completeness / CC1/2 the second // pass is judged against - is computed either way. // measure_cc_before_corrections: also merge ONCE MORE, just before the correction surfaces, and // report that merge's overall CC1/2 in the result. Only a caller comparing two passes of the same // data needs it (the pre-pass fits no surfaces, so only the uncorrected number is the same // measurement on both sides); it is a whole extra merge, so the offline re-scale path, which // compares nothing, asks for it to be left out. // group: the space group to merge in, for this Run only; null takes the experiment's. Given, the merge // reads the experiment for nothing that changes during a run, so a caller may merge in its own group // (the finalist ledger's P1 merge) while the experiment's group is another. Result Run(bool for_search, bool full_stats, bool measure_cc_before_corrections, const gemmi::SpaceGroup *group = nullptr); // Whether Run() writes the per-frame G / CC / mosaicity back onto the outcomes (on by default). Off // for a merge that is not the run's answer - the quality guard's merge of the pre-pass frames, the // finalist ledger's P1 merge - so the per-image table and the unmerged MTZ describe the merge that // was written. void SetWriteBackPerFrameScale(bool on) { write_back_per_frame_scale = on; } // Write back what the last Run() would have written with the write-back on: for a merge that ran // beside others with it off, at the point where it would have run in series. void WriteBackPerFrameScale() { FinalizePerFrameScale(last_cc, last_cc_n, last_frame_scaled); } // Hands an array (or a struct of them) back to the allocator. `v = {}` would not: on a std::vector it // is the initializer-list assignment, which empties the vector and keeps its memory. template static void Free(T &v) { v = T(); } // Hand back the merge's working arrays - MergeAndStats' accumulators, R_meas sums, ISa scatter, // error-model pools and per-full fields. Each is resized and refilled before it is read, so the // next merge starts from them as an engine's first merge does, and gives the same answer. On a // large cell in P1 they are gigabytes, kept between merges only to save faulting them in again; // this is for a gap between merges where something else sits on the run's memory peak. void ReleaseMergeScratch() { Free(merge_fields); Free(merge_acc); Free(merge_accum); Free(merge_merged_I); Free(merge_rejected); Free(merge_half_set); Free(merge_obs_key); Free(merge_wobs); Free(merge_rmeas); Free(merge_rmeas_staging); Free(asymptote_scatter); Free(em_samples); Free(em_cut_pool); Free(em_fit_pool); Free(em_chi2); } // For an engine that will not merge again: everything on the host but the per-frame state // WriteBackPerFrameScale reads (partials_out, g_partial, frame_flux, mos_smooth, last_cc, // last_cc_n, last_frame_scaled). Its card memory goes with the engine. void ReleaseAllButPerFrameScale() { ReleaseMergeScratch(); Free(partials); Free(fulls); Free(fulls_staging); Free(fulls_group_start); Free(fulls_group_perm); Free(perm); Free(rawrun_start); Free(rawrun_count); Free(rawrun_h); Free(rawrun_k); Free(rawrun_l); Free(rawrun_d); Free(rawrun_group); Free(finite_ok); Free(corr_ingested); obs_position.reset(); } // A second engine on this one's ingest, on `device` and logging to `log`, for a merge to run beside // this engine's: the host half of the ingest is copied, and the copy's IngestDevice(this) is still // to be called. Call after this engine's IngestDevice(). No merge depends on the merges run // before it (see RunArm), so a merge on the copy gives what the same merge gives on this engine. // The copy does not write its per-frame scales back (WriteBackPerFrameScale does). [[nodiscard]] std::unique_ptr CloneIngest(int device, Logger &log) const; // The card memory an engine may hold during a merge, the device helpers its merge calls included: // what a caller goes by when it decides whether a second engine fits beside one already on the // card. Call after IngestHost(). It counts what the ingest knows - the observations, the rocking // events (the combine makes at most one full of each) and the distinct hkl (at most one ASU group // each) - at what each costs at the engine's peak (the correction-surface terms, or the merge with // its outlier test), plus a fifth. Measured on 14 engines of 23-123 M observations, every engine's // peak was at most 88% of this. Where the ingest on the card peaks higher, its peak. // Call after IngestDevice(). [[nodiscard]] size_t DeviceBytes() const; // The device scaling's estimated peak (see the .cpp). Call after the ingest. [[nodiscard]] size_t GPUScalingPeakBytes() const; // The host memory an engine holds, for a caller deciding what may run beside it within the host budget // (HostMemory.h); it decides nothing in the engine. HostIngestBytes: what the ingest holds from then on, // between merges - call after IngestDevice(). HostMergeBytes: what one merge holds on top of it at its // peak, from the counts the ingest knows (see the .cpp). HostBytesBound: the two together, before an // ingest of `n_reflections`, at most. [[nodiscard]] size_t HostIngestBytes() const; [[nodiscard]] size_t HostMergeBytes() const; [[nodiscard]] static size_t HostBytesBound(size_t n_reflections, bool on_gpu); // Above this fraction of the card's memory the scaling runs on the CPU. static constexpr double CPU_SCALING_ABOVE_CARD_FRACTION = 0.8; // Merge only the first n frames of the outcomes (by default all of them). Set before Ingest(). void SetFrameLimit(int n) { frame_limit = n; } // The GPU the engine runs on (device 0 by default). Set before Ingest(). The device only says where // the merge runs: on cards of one model the result is the same on any of them. void SetGpuDevice(int device) { gpu_device = device; } [[nodiscard]] int GetGpuDevice() const { return gpu_device; } // Whether this engine scales on the GPU (decided by the ingest). [[nodiscard]] bool OnGPU() const; // Whether the engine still reads the source reflections in later Runs (see the constructor): where it // scales on the host, or keeps its partials for the observation dump. [[nodiscard]] bool ReadsReflections() const { return !partials.empty(); } // Whether Run() also hands back the merge's fulls unmerged (Result::scaled_fulls). Off by default: // it is a copy as long as the fulls, wanted only by a caller that writes them. void SetExportScaledFulls(bool on) { export_scaled_fulls = on; } // Whether a full-statistics Run() gives the merge French-Wilson amplitudes (on by default). Rugnux // turns it off and makes the amplitudes once, on the merge it writes, with the anisotropic prior. void SetFrenchWilson(bool on) { french_wilson = on; } // Where the engine logs from now on (by default the logger it was made with) - a merge run beside // other work logs into a held buffer, so the run's log keeps one order. void SetLogger(Logger &to) { logger = &to; } // Override the high-resolution cut for the next Run() - used to gate the de-novo P1 search pass at // >= 1 without cutting the final in-symmetry merge. Reset to the manual limit afterwards. void SetDMinLimit(std::optional d_min_A) { d_min_limit = d_min_A; } // Toggle the search-pass Lorentz filter (see search_min_zeta) between Run() calls, so the caller can // produce both a filtered and an unfiltered search merge from the same ingested partials. void SetSearchMinZeta(double zeta) { search_min_zeta = zeta; } [[nodiscard]] double GetSearchMinZeta() const { return search_min_zeta; } private: // A unique_ptr that a copy of the engine starts without (see CloneIngest). template struct NotCopied : std::unique_ptr { using std::unique_ptr::unique_ptr; using std::unique_ptr::operator=; NotCopied() = default; NotCopied(NotCopied &&) noexcept = default; NotCopied &operator=(NotCopied &&) noexcept = default; NotCopied(const NotCopied &) {} }; RotationScaleMerge(const RotationScaleMerge &) = default; // CloneIngest only static constexpr uint8_t OBS_CLIPPED = 1; static constexpr uint8_t OBS_OVERLOADED = 2; // One combined full during scale-fulls/merge (a per-frame partial is a PartialObs, below). Flat (not // nested per image); a POD so the arrays translate straight to CUDA. No member initialisers - here or in // PartialObs - so that `fulls` and `partials` (FillVector) are first written by the parallel passes // that fill them: whoever makes one sets every field. struct Obs { int32_t h, k, l; float I, sigma, d, prescaling_corr, partiality, zeta, delta_phi, bkg, var_bkg; // Fulls only, written by the combine: the full's variance as a function of intensity, // var(I) = var_bkg + var_per_I * I. The merge rebuilds it at the reflection's mean. 0 on a partial. float var_per_I; // Fulls only: the capture uncertainty, a relative sigma - capture_uncertainty_coeff times the // part of the rocking curve the event did not catch - so the merge can add (capture * )^2 to // the variance it rebuilds at the reflection's mean, as the combine added it to the full's own. // 0 on a partial. float capture; float px, py; // predicted detector position (for the absorption surface; CPU path only) float image_number; // fractional frame position (for 3D-combine contiguity) int32_t frame; // index of the outcome whose per-frame scale G applies to this obs uint8_t on_ice; uint8_t clipped; // OBS_CLIPPED | OBS_OVERLOADED (Reflection::clipped / ::overloaded); a full // is clipped if any of its partials is, and an overloaded event is no full float corr; // image_scale_corr (working; updated by scaling) int32_t group; // dense ASU-group id for the current space group; <0 = never mergeable }; // One per-frame partial as the host holds it - without a GPU, or for the observation dump: where its // reflection is, and the four values the ingest works out for it or the scaling changes. Every other // field is the reflection's own, read from it: the run keeps the reflections for as long as the engine // merges, and nothing changes in them after the images are integrated but the Miller indices of a // reindexing, which only the ingest reads (from the reflections, into the raw-hkl runs). Twenty-four // bytes where a copy of the fields was eighty-four - on a long axis, a hundred million partials. struct PartialObs { const Reflection *r; int32_t frame; // index of the outcome whose per-frame scale G applies to this obs float prescaling_corr; // the reflection's whole deterministic correction over the frame's flux float partiality; // recomputed from the frame-order-smoothed mosaicity (Ingest) float corr; // image_scale_corr (working; updated by scaling) }; // A reflection's Obs::clipped. static uint8_t ClippedFlags(const Reflection &r) { return (r.clipped ? OBS_CLIPPED : 0) | (r.overloaded ? OBS_OVERLOADED : 0); } // One leverage-corrected error-model sample per usable full: its counting variance at the group's // mean, that mean squared, its squared deviation from the mean - and the resolution it sits at, // because the fit is re-run over the samples that survive the automatic resolution cutoff. See // MergeAndStats and ErrorModel.h. using Sample = ErrorModelSample; // The narrow per-observation record the ingest sort orders: the raw hkl the runs are cut on, the // frame position that breaks a tie inside one, the observation's own index (which makes the order // total - see the .cpp), and the resolution the range test reads. Twenty bytes against the Obs's // eighty, and it is all the ingest needs before it knows which observations survive. Sixteen bits // are plenty for a Miller index - |h| <= a / d_min, in the hundreds even on the longest axis at // atomic resolution - and on a long axis this array sets the sort's memory high-water mark. struct SortKey { int16_t h, k, l; float image_number; int32_t idx; float d; }; // The per-frame crystal orientation smoothed in frame order (SmoothedFrameGeometry): each frame's // reciprocal basis, and the goniometer frame delta_phi is solved in. Empty `ok` = not smoothed. struct SmoothedGeometry { int half = 0; // the half-window cross-validation chose std::vector> recip; // per frame std::vector ok; // per frame: recip holds a smoothed basis Coord S0, m1, m2, m3; float m2_S0 = 0.0f, m3_S0 = 0.0f; }; SmoothedGeometry SmoothedFrameGeometry() const; static float SmoothedDeltaPhi(const SmoothedGeometry &sg, int o, int32_t h, int32_t k, int32_t l, float delta_phi_deg, int64_t &n); const DiffractionExperiment &x; std::vector &partials_out; // written back at the end of scaling std::optional reference_cell; size_t nthreads; Logger *logger; // Handed the text of the diagnostic observation dump each time the combine runs (CPU only). std::function observation_dump; // The group the current Run merges in (see Run); null outside a Run that was given one. const gemmi::SpaceGroup *run_group = nullptr; const gemmi::SpaceGroup &Group() const { return run_group ? *run_group : x.GetSpaceGroupOrP1(); } // Fixed settings snapshot (read once in the ctor). int n_frames = 0; double min_partiality = 0.02; std::optional d_min_limit; std::optional d_max_limit; bool merge_friedel = true; double capture_uncertainty_coeff = 0.0; double min_captured_fraction = 0.0; // RockingEventFrameGap of this run's oscillation width, taken once so that the two places that // cut events on the CPU and the GPU kernel that cuts them on the device all compare the same // float (RotationScaleMergeGPU.cu documents that bit-parity contract). float max_frame_gap = 2.0f; // Drop a frame's observations entirely when the frame disagrees with the merged reference below this // correlation (--min-image-cc). A mis-centred or off-crystal frame still produces spots, still // indexes and still integrates - it just measures something that is not the crystal's diffraction, // and nothing downstream removes it. 0 = off. double min_cc_for_image = 0.0; // Exclude observations with |zeta| below this from the DE-NOVO SEARCH merge only (the final merge // keeps everything). zeta is the sine of the angle between a reflection's rocking path and the // spindle: near 0 it crosses the Ewald sphere almost tangentially, spends many frames in // diffracting position and is measured badly. The symmetry search compares how equal an operator's // paired intensities are, so it is answered by whichever reflections are worst measured - and when // the spindle lies in a lattice plane, an operator permuting the two in-plane axes samples a // different mixture of qualities than one that only flips signs, which is not a fair comparison. // Unlike a bound on I/sigma this is pure geometry, identical in meaning on every dataset. 0 = off. double search_min_zeta = 0.0; double reject_nsigma = 0.0; bool reject_outliers = false; double rfree_fraction = 0.0; // The cap on the per-frame scaling loops. The loops run to a tolerance (see RunScalingLoop); // this is how far they may go before giving up and saying so. int scaling_iter = 100; bool scale_fulls = true; bool refine_decay_b = false; // per-time-block Debye-Waller decay correction (radiation damage) int absorption_iter = 0; // >0: fit a goniometer-frame absorption surface over this many iterations int modulation_iter = 0; // >0: fit a detector-plane modulation (flat-field) surface, this many iterations double relative_b_deg = 0.0; // >0: fit a per-batch relative-B (batch width in deg); 0 = off double mosaicity_deg = 0.1; // Automatic high-resolution cutoff for the written reflections + reported shells (post-merge; the // scaling, combine and error model always run over the full range). Manual d_min_limit wins. ResolutionCutoffMethod resolution_cutoff_method = ResolutionCutoffMethod::Off; double resolution_cc_target = 0.30; int report_shell_count = 9; std::optional report_range; // the second, reference-range table // Flat buffers, allocated once by Ingest() and reused across Run() calls. The full Obs array is // built only when the device pipeline is NOT resident (no GPU, or an observation dump was asked // for); on the resident path (resident_ingest) the device holds the observations and the host // keeps only the ingest arrays below, until the end of Ingest. FillVector partials; // all per-frame partials, grouped by frame bool resident_ingest = false; // What the ingest-time geometry smoothing (SmoothMosaicityAndPartiality) needs, one array per field // in `partials` order: the two fields it rewrites, the two the rocking-event walks read in raw-hkl // order, and what the partiality recompute reads besides them (zeta, and d under an energy // bandwidth - handed back as soon as it is done). So on the resident path no per-observation // record is built at all - on a fine-sliced long axis that record was gigabytes, held beside the // source reflections it was copied from - and the smoothing does not sweep the source reflections // again either. Filled by // BuildInRangeObservations and handed back at the end of Ingest (the resident path keeps them on the // card, IngestOnDevice). Plain arrays, not sized vectors, for // the reason the staging buffers are (see Ingest): the build writes every element, and a sized vector // would first zero all of them on one thread. NotCopied ingest_delta_phi, ingest_partiality, ingest_image_number, ingest_zeta, ingest_d; // How many observations Ingest built (== partials.size() until any release below), so the // sizes survive when the array itself does not. int n_partials_obs = 0; // With the GPU resident and no observation dump, no host stage reads the 80-byte Obs records // after the device upload: scaling, combine, scale-fulls and merge run on the device, // ComputeAsuGroups stamps the flat group_ids array, every pass restarts corr from // corr_ingested, and the CPU combine only runs for the dump. The one late host reader, // RockingEventFrames, sees only ingest-time values on that path (the host corr is refreshed // only in the dump fallback), so Ingest takes its answer and hands the array - a second // full-size copy of the partials, gigabytes on a fine-sliced long axis - straight back. bool partials_released = false; int rocking_event_frames_at_ingest = 0; int64_t rocking_events_at_ingest = 0; // Each partial's predicted detector position, for the fulls the device path combines (GetFullsPeak): // only the host reads it, so the card does not hold it. Shared by the engines CloneIngest makes. struct ObsPosition { std::vector px, py; }; std::shared_ptr obs_position; // The reflections and raw-hkl runs the resident ingest read before the resolution cut (DeviceBytes). size_t ingest_reflections = 0, ingest_runs = 0; std::vector frame_start, frame_count; // CSR ranges of `partials` per frame std::vector frame_cell_ok; // per-frame cell-consistency mask (1 = kept) std::vector finite_ok; // per-obs AcceptReflection finiteness (immutable; 1 = kept) std::vector g_partial; // per-frame partial scale G (the RESIDUAL after the flux); 1 at the start of every merge std::vector frame_flux; // per-frame incident flux, run median = 1 (see the .cpp) // corr as Ingest built it, kept for the whole life of the object and put back at the start of every // Run. A pass that starts anywhere else is not the pass that would have run on its own, and the // whole of the difference is the ITERATION COUNT: the scaling loop's fixed point is where it left // off, so a pass handed the previous pass's fitted corr does not redo its scaling_iter iterations, // it CONTINUES them. A de-novo run calls Run four times, and the fourth was therefore scaled at // twelve iterations where the caller asked for three. // // The surplus is harmless where multiplicity is high and harmful where it is not, and every // space-group search pass sits in the second case. Merged in a determined point group the // per-frame fit converges by the second iteration and the collapsed-scale guard's drop count is // stationary from there - over the rotation battery it moves 377 -> 397 frames between three // iterations and twelve, and 30 of 39 crystals drop nothing at any count. Merged in P1, where a // reflection has a couple of observations rather than a couple of dozen, it does not converge in // twelve: each iteration rebuilds the reference from the intensities the last one scaled, the // healthy frames drift down with the weak ones, and the guard drops more of the sweep every time. // Same guard, same battery, two regimes: summed over the search passes it drops 417 frames when a // search pass runs one iteration and 1101 when it runs three, against the 377 -> 397 above. On one // rotation crystal that repeatedly left the beam a single search pass drops 93 frames of 1800 at // three iterations, 124 at six and 264 at nine, and the space group determined from the // nine-iteration merge is wrong. A later pass inherits that state rather than creating it, so this // is a defect of the P1 search arm, and it reaches the user through the point group that arm picks. std::vector corr_ingested; // RockingEventFrames at the start of a Run, where corr has just been restored to corr_ingested - so // the same for every Run on the same ingest; -1 until the first Run takes it. int rocking_event_frames_at_start = -1; int64_t rocking_events_at_start = 0; // the events that walk counted // Whether the per-frame scale is fitted on the partials, decided by the first Run on this ingest // (PartialScalingWins) and kept for the rest; empty until then. std::optional scale_partials_choice; // Raw-hkl ordering, built ONCE by Ingest and reused: `perm` lists partial indices sorted by // (raw h,k,l, image_number); each distinct raw hkl is a contiguous run [rawrun_start, +count) of it. // The expensive sort happens once here, so per-pass combine (event split) and ASU grouping are linear. std::vector perm; std::vector rawrun_start, rawrun_count; std::vector rawrun_h, rawrun_k, rawrun_l; std::vector rawrun_d; // representative resolution per raw hkl std::vector rawrun_group; // dense ASU-group id per raw hkl (<0 = absent/out of range) FillVector fulls; // combined fulls (rebuilt each Run), sorted by frame // The fulls by ASU group, each group's in index order: group g is fulls_group_perm[fulls_group_start[g]] // up to fulls_group_start[g + 1]. A pass over every group's fulls can then give each thread whole // groups and still add a group up in the order a serial walk over the fulls would. Rebuilt with the // fulls (BuildFullsGroupCSR). std::vector fulls_group_start, fulls_group_perm; void BuildFullsGroupCSR(int n_groups); std::vector fulls_frame_start, fulls_frame_count; // CSR ranges of `fulls` per frame std::vector g_full; // per-frame scale on the fulls std::vector frame_scale_fulls; // per-frame total scale of the fulls before the surfaces (cc_weight) // The five fields the host-side walks over the fulls actually read, pulled out of the 80-byte record // once per merge (see MergeAndStats). `group` carries the usability decision: -1 means the full is // not in this merge, which is what a negative group already meant. Members rather than locals for the // same reason as FullsStaging below - the merge runs several times per run and this is tens of // megabytes each time. struct MergeFields { FillVector group; FillVector I, sigma, corr, d; void Resize(int n) { group.resize(n); I.resize(n); sigma.resize(n); corr.resize(n); d.resize(n); } }; MergeFields merge_fields; // Rocking events the last combine dropped because a pixel of one of their partials was saturated. int64_t fulls_dropped_overloaded = 0; // One host array per field for the fulls download: the device hands back an array per field and the // host gathers them into `fulls`. Members rather than locals in Run() because the whole // scale->combine->merge chain runs several times per run and these are a few hundred megabytes // between them, so as locals every chain allocates, faults in and zeroes the lot again. struct FullsStaging { FillVector run, peak, frame, group; FillVector I, sigma, d, image_number, corr, var_bkg, var_per_I, capture; FillVector on_ice, clipped; void Resize(int n) { run.resize(n); peak.resize(n); frame.resize(n); group.resize(n); I.resize(n); sigma.resize(n); d.resize(n); image_number.resize(n); corr.resize(n); var_bkg.resize(n); var_per_I.resize(n); capture.resize(n); on_ice.resize(n); clipped.resize(n); } }; FullsStaging fulls_staging; // The merge accumulators (see MergeAndStats' run_merge): one entry per ASU group, plus the arrays // the device kernel fills that the host unpacks into them. Members for the same reason as // FullsStaging - a merge runs several times per run and this is a hundred megabytes between them. // swh_typ: each half-set's precision had every observation been at the run's typical frame scale // (see MergeAndStats, cc_weight). // No initialisers of its own, so that the device path - whose unpack writes every field of every // group - sizes merge_acc without zeroing it first on one thread; the host path assigns EMPTY. struct Accum { double swI, sw, swIh[2], swh[2], swh_typ[2]; size_t nh[2]; float d; bool on_ice; }; static constexpr Accum ACCUM_EMPTY{0, 0, {0, 0}, {0, 0}, {0, 0}, {0, 0}, NAN, false}; FillVector merge_acc; struct MergeAccumStaging { FillVector swI, sw, swIh0, swIh1, swh0, swh1, swh_typ0, swh_typ1, d; FillVector nh0, nh1, rej; FillVector on_ice; void Resize(int n) { swI.resize(n); sw.resize(n); swIh0.resize(n); swIh1.resize(n); swh0.resize(n); swh1.resize(n); swh_typ0.resize(n); swh_typ1.resize(n); d.resize(n); nh0.resize(n); nh1.resize(n); rej.resize(n); on_ice.resize(n); } }; MergeAccumStaging merge_accum; // MergeAndStats' other per-full and per-group arrays, members for the same reason: five merges a // run, each over up to tens of millions of fulls and millions of groups, and as locals each one // allocated, faulted in and zeroed them again on one thread. As members they are only resized, and // reset on all threads where a pass needs them reset. std::vector merge_merged_I; std::vector merge_rejected, merge_half_set; std::vector merge_obs_key; std::vector merge_wobs; // The R_meas accumulators per group (see MergeAndStats' statistics), and the device's per-group sums. // No initialisers, as Accum: the merge resets them on all threads to RMEAS_EMPTY. struct RmeasObs { double sum_abs_dev, sum_I, w_abs_dev, w_I, sum_v, sum_v2; int n, shell; }; static constexpr RmeasObs RMEAS_EMPTY{0, 0, 0, 0, 0, 0, 0, -1}; FillVector merge_rmeas; struct RmeasStaging { FillVector absdev, sumI, wabsdev, wsumI, sumv, sumv2; FillVector n, nusable; void Resize(int g) { absdev.resize(g); sumI.resize(g); wabsdev.resize(g); wsumI.resize(g); sumv.resize(g); sumv2.resize(g); n.resize(g); nusable.resize(g); } }; RmeasStaging merge_rmeas_staging; // Per-group scatter for the strong-reflection ISa asymptote (see MergeAndStats). A member for the // same reason: 32 bytes a group, once per merge. struct GroupScatter { double sum, sum_sq, sum_var; int n; }; // no initialisers, as Accum FillVector asymptote_scatter; // The error model's working pools (see MergeAndStats): the samples themselves, the scratch copy each // fit partitions, the subset inside the resolution cutoff, and // the per-sample chi2 the reported number is the median of. Members for the same reason as FullsStaging - one is 32 bytes per full and // there are two dozen fits per run, so as locals this is gigabytes of pages faulted in and handed // straight back. Every one of them is cleared and refilled before it is read. std::vector em_samples, em_cut_pool; std::vector em_fit_pool; std::vector em_chi2; // Set by FitScalingFrames: which frames were fitted this call (so corr/G is updated only there), and // the information each fit had on G, sum w^2 c^2 (0 where the frame was not fitted). std::vector frame_scaled_scratch; std::vector frame_info_scratch; // Per-frame mosaicity smoothed in frame order (deterministic); used to recompute partiality and // written back for the per-image scaling table. Empty if there is no per-frame mosaicity. std::vector mos_smooth; // Radiation-damage monitor (measured by MeasureRadiationDamageB on the scaled fulls before any decay // correction; report-only, copied into the result statistics by MergeAndStats). NaN / empty until set. double rad_damage_delta_b = std::numeric_limits::quiet_NaN(); // relative-B first->last (A^2) std::vector rad_damage_b_batch; // per-batch relative-B curve (A^2) double rad_damage_batch_deg = 0.0; // rotation width per batch (deg) // Sweep-quality diagnostic (MeasureSweepQuality, then the delta-CC1/2 and the disposition; // copied into the result statistics by MergeAndStats). Empty and not measured until it runs. SweepQuality sweep_quality; // The two per-frame channels MeasureSweepQuality segmented, kept so the ledger can recompute a // range's numbers after the ranges have been split at the rejection boundaries. Relative to the // run median; 0 in both means the frame contributed nothing. std::vector sweep_scale, sweep_cc; // The first and last frame with anything integrated on them. A sweep whose ends recorded no // diffraction reaches this engine with those frames already empty (the caller trims them), so the // sweep ledger looks for degraded stretches and edges between these two and not beyond. int data_first = 0, data_last = -1; // Frames whose observations delta-CC1/2 convicted (1 = out of the merge). Sized n_frames and all // zero until MeasureBatchDeltaCCHalf runs; Run() is what takes the observations out. std::vector frame_rejected; // Frames whose observations reach this pass's merge at all (1 = in), before delta-CC1/2 has its say. // Built by Run(), where every per-frame drop is decided. A full cannot answer this: the combine // attributes a rocking event to its PEAK frame, so on fine slicing most frames own no full at all // while their measurements sit inside one. std::vector frame_in_merge; // Working per-group arrays (sized to the current group count; reused). std::vector group_h, group_k, group_l; #ifdef JFJOCH_USE_CUDA // GPU engine: the whole hot path (scaling, combine, scale-fulls, per-frame CC, smooth-G, merge + // error model) runs on the device, resident, when a GPU is present. Null / inactive otherwise, with // the CPU loops as the bit-parity fallback. Built in Ingest. NotCopied gpu_; bool gpu_active_ = false; // The scaling does not fit the card (IngestDevice): this engine scales on the CPU. bool scale_on_cpu_ = false; // The fulls of this pass live on the device (the GPU combine ran), so the correction surfaces can // build their terms there. Set by RunArm. bool fulls_on_device_ = false; // The fulls' current corr, gathered for the device term build of each surface (a member: one per // surface, a few tens of megabytes each). std::vector surface_corr; #endif bool write_back_per_frame_scale = true; // see SetWriteBackPerFrameScale // What the last Run wrote back, or would have (WriteBackPerFrameScale). std::vector last_cc; std::vector last_cc_n; std::vector last_frame_scaled; int frame_limit = -1; // see SetFrameLimit int gpu_device = 0; // see SetGpuDevice bool export_scaled_fulls = false; // see SetExportScaledFulls bool french_wilson = true; // see SetFrenchWilson // --- helpers (each a flat pass; see the .cpp) --- // Turn the per-frame mean background under the reflections (accumulated by the ingest fill loop) into // the per-frame incident flux, which the finiteness pass then folds into prescaling_corr so that // corr = prescaling_corr / (partiality * G) divides it out and G fits only the residual. See the .cpp for why a // background is a usable flux meter and what it costs when it is not. void MeasureIncidentFlux(const std::vector &mean_bkg); // Build the flat `partials` array (and the per-frame CSR, the finiteness mask and `perm`) from the // source reflections, skipping the observations whose resolution can never be in range: // --scaling-high/low-resolution are the coarsest limits any Run() uses (the space-group search only // ever RAISES d_min), and an out-of-range raw hkl gets group -1 in every pass, which keeps it out of // the scaling reference, the per-frame fit, the combine, the merge and the error model alike. On a // crystal that integrates to the detector corner and merges well short of it that is most of the // array, and an eighty-byte record built for it is eighty bytes written and then thrown away. Whole // raw-hkl RUNS are skipped, on the same per-hkl resolution ComputeAsuGroups tests, so what survives - // and the order of every sum formed over it - is exactly what it would have been had the whole array // been built and then filtered. Everything is built when no manual limit was given. // `keys` is CONSUMED: it is freed as soon as the permutation has been remapped, before the // observation array is built, so the two never coexist - together they would be another third of // the payload on top of it. // Every kept partial's delta_phi is re-solved against `sg` as it is built (SmoothedDeltaPhi). void BuildInRangeObservations(std::unique_ptr &keys, int n_obs, const SmoothedGeometry &sg); #ifdef JFJOCH_USE_CUDA // The resident path's ingest (see IngestDevice): the observations and the raw-hkl runs built on the // card from the reflections, exactly as BuildInRangeObservations builds them on the host. void IngestOnDevice(); // Where the scaling's estimated peak is over CPU_SCALING_ABOVE_CARD_FRACTION of the card: says so, // gives the device up (scale_on_cpu_) and returns true. Call once the ingest has counted. bool ScaleOnCPUWhereTooLarge(); // The same for the ingest's sort of n_reflections, decided before anything is on the card. bool ScaleOnCPUWhereSortTooLarge(size_t n_reflections); // What the rule above says for `bytes` of the card, `what` naming the work in the warning. bool ScaleOnCPUWhereOverCard(size_t bytes, const std::string &what); #endif // Compute the dense ASU-group id for the current space group by grouping the (pre-sorted) raw-hkl // runs by their ASU key - one gemmi ASU reduction per distinct raw hkl, not per observation. Fills // rawrun_group, the group_h/k/l representative tables, and partials[].group; returns the group count. int ComputeAsuGroups(const HKLKeyGenerator &key_generator); // Without a GPU: the partials by group, in observation order (a CSR - start and count per group), // which ComputeAsuGroups builds for the GPU reduction anyway. ReducePartialGroupMeans sums each // group over it, on all threads and in the order the serial loop would. std::vector partial_group_start, partial_group_count; NotCopied partial_group_perm; // Without a GPU: each partial's ASU group (-1 = in none), in observation order - where the CPU // passes read a partial's group from. It is not kept on the partial: written into the records at // random, once per grouping, it cost more than the grouping itself. NotCopied partial_group; // The CPU scaling loops' working copy of the partials, or of the fulls. A loop passes over them // several times per iteration and reads a handful of fields each time, so it works on these rather // than on the partials and their reflections. ScalingRef holds what the group means and the corr update read, in the CSR's // group order with the partials in no group after them, and corr is kept beside it in the same // order - so neither pass gathers. ScalingFit holds, frame by frame, the observations // FitScalingFrames fits: the fit's filters, which do not change over the loop, applied once, // and the factors of its sums that do not depend on the reference worked out once. The same // values, so the same arithmetic in the same order. struct ScalingRef { int32_t frame; float I, sigma, partiality, prescaling_corr; }; // The fit's factors that do not depend on the reference - the coefficient of a mean of 1, // partiality / prescaling_corr (1 for a full), and the weight 1/sigma^2 - are worked out from these // floats where they are used: twenty bytes an observation read every iteration instead of thirty-two, // and the same doubles. struct ScalingFit { int32_t group; float I, sigma, partiality, prescaling_corr; }; // The observation's inverse-variance weight in the scaling reference, and its scaled intensity; // 0 when the observation is not in the reference at all. static double ReferenceWeight(int32_t group, float corr, float partiality, float I, float sigma, double min_partiality, float &I_corr) { if (group < 0) return 0.0; if (!(corr > 0.0f) || !std::isfinite(corr)) return 0.0; if (partiality < min_partiality) return 0.0; const float Ic = I * corr; const float sigma_corr = sigma * corr; if (!std::isfinite(Ic) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) return 0.0; I_corr = Ic; return 1.0 / (static_cast(sigma_corr) * sigma_corr); } // Inverse-variance per-group mean of I*corr over the partials (the scaling reference; no cell mask - // only the final merge applies one), using the CSR above, each group summed in index order. void ReducePartialGroupMeans(int n_groups, std::vector &out_mean) const; // The partials' loop's corr update of one iteration - UpdateCorr from the G it fitted, then the // rescale RunScalingLoop asked for - per frame: `fitted` and `g` as the iteration left them, `apply` // and `ratio` the rescale's. struct CorrUpdate { std::vector fitted, apply; std::vector g, ratio; void Apply(const ScalingRef &o, float &corr) const { if (fitted[o.frame]) { // UpdateCorr const double denom = static_cast(o.partiality) * g[o.frame]; if (std::isfinite(o.prescaling_corr) && std::isfinite(denom) && denom > 0.0) corr = static_cast(o.prescaling_corr / denom); else corr = NAN; } if (apply[o.frame] && std::isfinite(corr)) corr = static_cast(corr * ratio[o.frame]); } }; // The same again over a scaling loop's copy, refs and corr both in group order: group g's at // [group_start[g], group_start[g] + group_count[g]). With `update`, each corr is first brought up to // date by it, in the same walk (see the partials' loop in RunArm). void ReduceScalingGroupMeans(int n_groups, const int32_t *group_start, const int32_t *group_count, const FillVector &refs, FillVector &corr, std::vector &out_mean, const CorrUpdate *update = nullptr) const; // Per-frame G fit against the reference group_mean_in[group], over a scaling loop's ScalingFit, frame // f's at [fit_start[f], fit_start[f + 1]). A frame with fewer than min_obs of them keeps its G: a // partial frame needs MIN_REFLECTIONS of its own (as ScaleOnTheFly), while a full is a whole rocking // event filed under one frame, so on a sparse sweep a frame holds a handful - its fit goes in at // the information it has (min_obs 1), and the smoothing in RunFullsScalingLoop decides how far its // neighbours carry it. void FitScalingFrames(const FillVector &fit, const std::vector &fit_start, size_t min_obs, const std::vector &group_mean_in, std::vector &g); // The alternating per-frame scale fit run to convergence: `iterate` fits every frame once against // the current reference (leaving G in `g` and the frames it fitted in frame_scaled_scratch), // `rescale` multiplies the corr of the flagged frames' observations by a per-frame ratio. Smooths G // over `window` frames and pins the gauge after every iteration; stops when the scales settle, stop // settling, or the cap is reached. On return frame_scaled_scratch flags every frame fitted in any // iteration. struct ScalingLoopOutcome { int iterations = 0; bool converged = false; bool stalled = false; double step = 0.0; double best_step = 0.0; }; void LogScalingOutcome(const char *what, const ScalingLoopOutcome &out) const; // Shrink a free per-frame scale back toward the restrained one, frame by frame, by how much of // the difference between them is real (see ShrinkToRestrained in the .cpp). void ShrinkToRestrained(const char *what, const std::vector &g_restrained, const std::vector &fitted, const std::vector &frame_obs_count, std::vector &g) const; // The CPU partial scaling loop of an earlier Run, for a later Run that would run it on the same // inputs - which the P1 merges of a pass do. The loop restarts from corr_ingested, fits each frame // from its observations and the group means alone, and writes G and corr only on the frames it // fits; so over the same ASU grouping and settings it ends where it ended before. Kept: which frames // it fitted, their G, and every partial's corr. struct PartialLoopKey { std::string space_group; bool merge_friedel; std::optional d_min, d_max; double min_partiality; int window, iterations; bool operator==(const PartialLoopKey &) const = default; }; struct PartialLoopMemo { PartialLoopKey key; ScalingLoopOutcome outcome; std::vector fitted; std::vector g; FillVector corr; }; // A pass alternates its groupings (P1 full range, P1 at the search limit, the adopted group), so a // few are kept; the oldest go first. std::vector partial_loop_memos; static constexpr size_t MAX_PARTIAL_LOOP_MEMOS = 3; ScalingLoopOutcome RunScalingLoop(const char *what, std::vector &g, const std::vector &frame_obs_count, int window, const std::function &iterate, const std::function &, const std::vector &)> &rescale); // The per-frame scale of the fulls, run to convergence: `fit` fits every frame once against the // current reference without touching corr (leaving the free G in `g_fit`, the fitted frames in // frame_scaled_scratch and their information in frame_info_scratch); the scale is then the // penalised smooth of those fits (SmoothLogScale, lambda by ChooseLogScaleSmoothing), gauge-pinned, // and `rescale` multiplies the corr of each frame's fulls by g_old / g_new. `g` is the scale corr // stands at (corr = 1/g) on entry and on return. ScalingLoopOutcome RunFullsScalingLoop(std::vector &g, std::vector &g_fit, const std::vector &frame_obs_count, const std::function &fit, const std::function &, const std::vector &)> &rescale); // The windowed geometric mean of G over frames; the scaling loop applies the resulting ratio to // the observations' corr (on the GPU, to the resident corr in a kernel). void ComputeSmoothGWindow(const std::vector &g, int window, double g_floor, std::vector &g_smooth) const; // Drop the observations of any frame whose fitted per-frame scale collapsed far below the run's // typical frame, reporting the per-frame corr factor the caller has to apply. See the .cpp for why nothing // downstream can catch a collapsed scale on its own. bool DropCollapsedScales(const std::vector &fitted_mask, std::vector &g, std::vector &apply, std::vector &ratio) const; // Smooth per-frame mosaicity in frame order and recompute each partial's partiality from it, so the // per-frame partials of one rocking event tile the curve consistently (they sum toward 1) before the // 3D combine. Deterministic (frame order); replaces the old arrival-order mosaicity moving average // that prediction applied. SG-independent, so done once in Ingest. void SmoothMosaicityAndPartiality(); void SmoothMosaicity(); // its mos_smooth half void Combine(); // partials -> fulls (CPU) // Is this full in the merge? group >= 0 already encodes "not absent and passes AcceptReflection"; // the rest is the frame's cell-consistency mask, a usable scale and a usable sigma. The P1 search // pass adds its own ice test on top of this (see MergeAndStats). [[nodiscard]] bool UsableFull(const Obs &o) const; // Drop the fulls of any frame whose scale collapsed toward zero. The fulls are scaled with the Unity // model, so their corr IS 1/G and a collapsed G multiplies every intensity on that frame without // bound. Covers the CPU and GPU scaling paths alike; `from_staging` says the fulls' frame and corr // are still in fulls_staging, which is where the scan reads them from when they are. Returns true if // anything was dropped (the caller then has to push the corrected corr back to the device). bool DropCollapsedFullScales(bool from_staging); // Post-scale-fulls correction surfaces, each an alternating multiplicative fit of the host fulls' corr // against the merged reference (cheap host loops; the corrected corr is re-uploaded to the resident // fulls afterwards). Each is cross-validated (fit even frames, keep only if held-out odd equivalents // improve) so it is a no-op when its systematic is absent. RefineDecay fits a global Debye-Waller B // (resolution x time - radiation damage the resolution-flat per-frame G cannot capture; also gated on a // physical total-dB floor). RefineAbsorption fits a smooth factor over the diffracted-beam direction in // the goniometer frame (path-length / absorption; negligible at hard X-rays, matters at low energy). void RefineDecay(int n_groups); // The observation fields the decay / relative-B passes read, pulled out of `fulls` once. Those // passes run a dozen walks over the eighty-byte records for twenty bytes of each, and their // per-group references had to stay serial because they scatter into a per-group array. `term` keeps // fulls order, so a sum formed over it is formed from the same terms in the same order the serial // walk used; `gterm` holds the same terms in ASU-group order, by a stable counting sort over `term`, // so a group's terms are still added in fulls order - which lets a thread own whole groups and gives // the same per-group sums. `g_start` is that array's CSR. struct DecayTerm { float I, sigma, corr, d, image_number; int32_t frame, group; }; struct DecayTerms { std::vector term, gterm; std::vector g_start; }; void BuildDecayTerms(int n_groups, DecayTerms &t) const; // The terms the radiation-damage monitor built, handed on to RefineDecay: nothing between the two // touches corr, so they are the terms RefineDecay would build. Empty outside that stretch of RunArm. DecayTerms decay_terms_kept; bool decay_terms_kept_valid = false; // Solve a smooth per-batch relative-B from the per-batch normal equations for b (num_c, den_c): // data-fidelity + a second-difference (curvature) penalty, by Gauss-Seidel, each batch clamped to // +-b_max. Returns the un-anchored curve; the caller sets the gauge and the clamp it can live with. // Shared by the correction and the radiation-damage monitor. std::vector SolveCurvatureSmoothedB(const std::vector &num, const std::vector &den, double b_max) const; // Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset // {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a // subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction. std::vector FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch, int frames_per_batch, int gparity) const; // Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before // any decay correction and store the first->last relative-B change + the per-batch curve on this object // (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF). void MeasureRadiationDamageB(int n_groups); // Sweep-quality diagnostic: find the contiguous stretches of the sweep over which the crystal // delivered much less than the rest of the run, and say what each one looks like. Reads the // per-frame scale (with the incident flux already divided out) and the per-frame CC to merge. // It only DETECTS and NAMES; what is done about a stretch is MeasureBatchDeltaCCHalf's decision. void MeasureSweepQuality(const std::vector &partial_scaled, const std::vector &cc, const std::vector &cc_n); // Fill in every ledger range's numbers from sweep_scale / sweep_cc. assign_reasons=false keeps the // reasons already on the ranges, which is what a re-fill after the ranges were split wants. void FillSweepRanges(bool assign_reasons); // The frames one rocking event spans, as the combine cuts them: the median over the run's events. // Walked on the partials, because the combine also runs on the GPU and a full keeps only the frame // of its peak partial. // n_events (if given) receives how many rocking events the walk counted. [[nodiscard]] int RockingEventFrames(int64_t *n_events = nullptr) const; template [[nodiscard]] int RockingEventFramesOver(UsableFn usable, ImgFn img, int64_t *n_events) const; // The median of a histogram of event spans (bins 0 .. n_frames + 1), and its total in n_events. [[nodiscard]] int MedianEventSpan(const std::vector &hist, int64_t *n_events) const; // Per-batch delta-CC1/2 on the corrected fulls: measure what keeping each batch of the sweep costs // the merged intensities, convict the batches that cost significantly, slide the conviction's edges // onto the frames that carry it, and turn the result into the disposition ledger. Fills // frame_rejected, which Run() then takes out of the merge. See the .cpp for the statistic and the // discipline. void MeasureBatchDeltaCCHalf(int n_groups); // Split the ledger ranges at the boundaries of frame_rejected and add a range for every rejected // stretch no ledger range covers, so that each range is wholly rejected or wholly kept. void SplitSweepRanges(); // Give every frame and every ledger range its disposition, and count the sweep. void CountSweepDisposition(); // Per-batch relative-B, applied after RefineDecay: the single decay slope removes the average // radiation-damage falloff, but the relative scattering power drifts NON-monotonically across a run // (absorption path, crystal slippage, dose bursts). Refine one relative Debye-Waller B per batch // (FitRelativeBCurve), anchored to zero mean (the constant part is a global Wilson-B, degenerate with // overall scale). Guarded by a physical peak-to-peak floor and cross-validated by ASU-GROUP parity (a // per-batch parameter cannot be scored on a held-out FRAME the batch owns; splitting the equivalents // tests whether a batch's B generalises to reflections it was not fit on). Opt-in (--relative-b). void RefineRelativeB(int n_groups); void RefineAbsorption(int n_iter, int n_groups); // The crystal-frame absorption as real spherical harmonics of the diffracted-beam direction (l <= 6) // instead of a grid; a candidate through the same cross-validated engine, fitted before the grid. void RefineAbsorptionSH(int n_iter, int n_groups); // Time-dependent absorption: the same cross-validated surface, indexed by (rotation, detector position) // instead of by the crystal-frame direction alone. RefineAbsorption's parameterisation is the whole // model only while the illuminated volume stays put; once the crystal drifts through the beam the exit // path becomes a function of the spindle angle too, and nothing time-independent reaches it. void RefineAbsorptionTime(int n_iter, int n_groups); // Detector-plane modulation (flat-field): the same cross-validated surface fit as absorption, but the // cell is the predicted detector position (px, py) instead of the goniometer-frame direction. Corrects // detector-response / geometric systematics that vary with where a reflection lands; because it lives // in the detector frame (not tied to the rotation) the same correction concept applies to stills. void RefineModulation(int n_iter, int n_groups); // Shared engine for the correction surfaces: given a per-full cell assignment (cell[i] in [0,ncell), or // <0 to skip), fit a multiplicative factor per cell against the merged reference under a prior pull to 1, // cross-validate on even/odd frames, and fold it into corr only if the held-out half-set CC improves. // resolution_gauge: the cells are patches of the detector, each at one resolution, so the component of // the surface that is a function of resolution is a null direction and is projected out per shell. // sh_basis: the cells are fine direction bins of that basis, and the fit holds log A to its span. void ApplyCellSurface(const std::vector &cell, int ncell, int n_iter, int n_groups, const char *name, bool resolution_gauge, const SphericalHarmonicBasis *sh_basis = nullptr); // Fill `fulls` with the parts' fulls sorted by peak frame and (re)build fulls_frame_start/count (the // per-frame CSR the scale-fulls step slices). The CPU Combine's tail. void SortFullsByFrame(const std::vector> &part); // Per-frame CC vs the partial merge reference (CPU; the GPU equivalent is gpu_->ComputePartialCC). void ComputePerFrameCC(const std::vector &partial_group_mean, std::vector &cc, std::vector &cc_n) const; // Write G/CC/mosaicity back onto the partials (once, at the end of partial scaling) from the given // per-frame cc/cc_n, so the offline per-image scaling table is still exported. void FinalizePerFrameScale(const std::vector &cc, const std::vector &cc_n, const std::vector &frame_scaled); // Error model + merge + statistics over the fulls (the last stage). n_groups is the fulls group count. // fulls_resident: the (scaled) fulls + their group CSR are still on the GPU, so the em-stats / samples // / merge-accumulate / R_meas reductions run there (only per-group + samples come back). // full_stats: see Run(). Result MergeAndStats(int n_groups, bool for_search, bool fulls_resident, bool full_stats); // One whole pass of Run() with the per-frame scale fitted on the partials (scale_partials) or taken // from the fulls alone. Result RunArm(bool scale_partials, bool for_search, bool full_stats, bool measure_cc_before_corrections); };