From 34d5bb563138055c31c3cd9ceb482f4871eb3ab5 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 10:09:23 +0200 Subject: [PATCH 01/11] stills merge 2.3-2.5x faster, output unchanged: parallel observation passes, hash accumulator, no amplitudes on scaling references The stills scale/merge after the image loop was almost entirely serial: - every scaling-reference merge (MergeAll: the per-image scaling reference and the two partiality post-refinement references) ran French-Wilson single-threaded on its export, about 3 s each, and assigned free-R flags - neither is read by a reference, which uses I only. MergeAll now exports intensities alone (MergeOnTheFly::ExportIntensities). - the merge accumulated into a std::map; it is now an unordered_map exported in key order, which is the std::map order. - AddImage, the error-model collection and the R_meas pass of MergeStats did the per-observation work (systematic-absence test, ASU key, filters, error-model sigma, outlier test, merged-I lookup) in one serial loop. They now reduce a block of images in parallel and consume it in image order (ReduceImagesInOrder), so every floating-point sum, half-set choice and hash-map insertion is made in the serial order. Measured on a 25k-frame XFEL stills run (box loaded, load 9-14): merge P1 19.0 -> 7.1 s, final-group merge 27.7 -> 13.2 s; on a second run 23.4 -> 7.7 s and 29.7 -> 13.6 s. mtz/unmerged mtz data, hkl, cif and report identical on both runs, mx and --mode scale. New test MergeOnTheFly_AddImagesIsTheSerialMerge checks the parallel merge against the serial loop across a block boundary. Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/scale_merge/Merge.cpp | 253 ++++++++++++------ image_analysis/scale_merge/Merge.h | 28 +- .../scale_merge/StillsPartialityRefine.cpp | 2 +- rugnux/RugnuxScaleMerge.cpp | 11 +- rugnux/rugnux_cli.cpp | 6 +- tests/MergeScaleTest.cpp | 65 +++++ 6 files changed, 269 insertions(+), 96 deletions(-) diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 54ec8d9eb..2c3bf37f1 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -15,6 +15,7 @@ #include "../../common/CorrelationCoefficient.h" #include "../../common/ResolutionShells.h" #include "../../common/Definitions.h" +#include "../../common/ParallelFor.h" #include "HKLKey.h" #include "RfreeFlags.h" #include "FrenchWilson.h" @@ -32,6 +33,27 @@ namespace { z = z ^ (z >> 31); return static_cast(z & 1ULL); } + + // Calls reduce(i, out) for the images in parallel, a block of them at a time, and then consume(i, out) + // for each image of the block in image order - the order that every floating-point sum, every + // half-set choice and every hash-map insertion of the merge depends on. The result is therefore the + // serial loop's, bit for bit, at any thread count. + template + void ReduceImagesInOrder(size_t n_images, size_t nthreads, Reduce reduce, Consume consume) { + constexpr size_t BLOCK_IMAGES = 4096; + std::vector> out(std::min(BLOCK_IMAGES, n_images)); + for (size_t first = 0; first < n_images; first += BLOCK_IMAGES) { + const int n = static_cast(std::min(BLOCK_IMAGES, n_images - first)); + ParallelChunks(n, nthreads, [&](int lo, int hi) { + for (int j = lo; j < hi; ++j) { + out[j].clear(); + reduce(first + j, out[j]); + } + }); + for (int j = 0; j < n; ++j) + consume(first + j, out[j]); + } + } } MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x) @@ -57,14 +79,8 @@ MergeOnTheFly &MergeOnTheFly::ReferenceCell(const std::optional &cell) return *this; } -void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id) { - std::unique_lock ul(merged_mutex); - - if (Mask(outcome)) - return; - - const int prefer = HalfForImage(image_id); - +size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vector &out) const { + size_t rejected = 0; for (const auto &r: outcome.reflections) { if (generator.IsSystematicallyAbsent(r)) continue; @@ -94,24 +110,36 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id const auto mit = reject_median_I.find(hkl_key); if (mit != reject_median_I.end() && std::fabs(I_corr - mit->second) > reject_nsigma * sigma_corr) { - ++reject_count; + ++rejected; continue; } } - auto it = accumulator.find(hkl_key); - if (it == accumulator.end()) - it = accumulator.emplace(hkl_key, MergeAccum{ - .h = hkl.plus ? hkl.h : -hkl.h, - .k = hkl.plus ? hkl.k : -hkl.k, - .l = hkl.plus ? hkl.l : -hkl.l, - }).first; - const float w = 1.0f / (sigma_corr * sigma_corr); - const float wI = w * I_corr; + out.push_back(Contribution{ + .key = hkl_key, + .h = hkl.plus ? hkl.h : -hkl.h, + .k = hkl.plus ? hkl.k : -hkl.k, + .l = hkl.plus ? hkl.l : -hkl.l, + .w = w, + .wI = w * I_corr, + .d = r.d, + .on_ice_ring = r.on_ice_ring + }); + } + return rejected; +} - it->second.sum_wI += wI; - it->second.sum_w += w; +void MergeOnTheFly::Accumulate(const std::vector &contributions, int64_t image_id) { + const int prefer = HalfForImage(image_id); + + for (const auto &c: contributions) { + auto it = accumulator.find(c.key); + if (it == accumulator.end()) + it = accumulator.emplace(c.key, MergeAccum{.h = c.h, .k = c.k, .l = c.l}).first; + + it->second.sum_wI += c.wI; + it->second.sum_w += c.w; // CC1/2 correlates the two half-set means, so a reflection whose observations all land in one // half has no second mean and drops out of the statistic entirely. Taking the image's own @@ -119,18 +147,44 @@ void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id // reflection floor(n/2) against ceil(n/2), which is the split cctbx and XDS use. const int half = it->second.n_half[prefer] > it->second.n_half[1 - prefer] ? 1 - prefer : prefer; - it->second.sum_wI_half[half] += wI; - it->second.sum_w_half[half] += w; + it->second.sum_wI_half[half] += c.wI; + it->second.sum_w_half[half] += c.w; it->second.n_half[half]++; - if (r.on_ice_ring) + if (c.on_ice_ring) it->second.on_ice_ring = true; - if (!std::isfinite(it->second.d) && std::isfinite(r.d) && r.d > 0.0f) - it->second.d = r.d; + if (!std::isfinite(it->second.d) && std::isfinite(c.d) && c.d > 0.0f) + it->second.d = c.d; } } +void MergeOnTheFly::AddImage(const IntegrationOutcome &outcome, int64_t image_id) { + std::unique_lock ul(merged_mutex); + + if (Mask(outcome)) + return; + std::vector contributions; + reject_count += Contributions(outcome, contributions); + Accumulate(contributions, image_id); +} + +void MergeOnTheFly::AddImages(const std::vector &outcomes) { + std::unique_lock ul(merged_mutex); + + std::vector rejected(outcomes.size(), 0); + ReduceImagesInOrder( + outcomes.size(), nthreads, + [&](size_t i, std::vector &out) { + if (!Mask(outcomes[i])) + rejected[i] = Contributions(outcomes[i], out); + }, + [&](size_t i, const std::vector &out) { + reject_count += rejected[i]; + Accumulate(out, static_cast(i)); + }); +} + float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr, float var_bkg, uint64_t hkl_key) const { @@ -188,29 +242,36 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc // --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl, // applying exactly the filters AddImage uses. --- struct Obs { float I, sigma; }; + struct KeyedObs { uint64_t key; Obs obs; }; std::unordered_map> groups; - for (const auto &outcome: outcomes) { - if (Mask(outcome)) - continue; - for (const auto &r: outcome.reflections) { - if (generator.IsSystematicallyAbsent(r)) - continue; - if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) - continue; - if (!AcceptReflection(r, high_resolution_limit, low_resolution_limit)) - continue; - if (exclude_ice_rings && r.on_ice_ring) - continue; - if (r.partiality < min_partiality) - continue; - const float I_corr = r.I * r.image_scale_corr; - const float sigma_corr = r.sigma * r.image_scale_corr; - if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) - continue; - groups[generator(r).pack()].push_back({I_corr, sigma_corr}); - } - } + ReduceImagesInOrder( + outcomes.size(), nthreads, + [&](size_t i, std::vector &out) { + if (Mask(outcomes[i])) + return; + for (const auto &r: outcomes[i].reflections) { + if (generator.IsSystematicallyAbsent(r)) + continue; + if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) + continue; + if (!AcceptReflection(r, high_resolution_limit, low_resolution_limit)) + continue; + if (exclude_ice_rings && r.on_ice_ring) + continue; + if (r.partiality < min_partiality) + continue; + const float I_corr = r.I * r.image_scale_corr; + const float sigma_corr = r.sigma * r.image_scale_corr; + if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) + continue; + out.push_back({generator(r).pack(), {I_corr, sigma_corr}}); + } + }, + [&](size_t, const std::vector &out) { + for (const auto &o: out) + groups[o.key].push_back(o.obs); + }); // --- 2. One global pool of (sigma^2, ^2, bias-corrected squared deviation). For an // observation in a group of n, the residual from the inverse-variance mean has @@ -337,7 +398,7 @@ void MergeOnTheFly::RefineErrorModel(const std::vector &outc error_model_chi2 = chi2.empty() ? 0.0 : median(chi2) / CHI2_1_MEDIAN; } -bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) { +bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) const { if (reference_cell) { auto cell = outcome.latt.GetUnitCell(); if (!cell.is_close(*reference_cell, @@ -355,12 +416,20 @@ bool MergeOnTheFly::Mask(const IntegrationOutcome &outcome) { return false; } -std::vector MergeOnTheFly::ExportReflections() { +std::vector MergeOnTheFly::ExportIntensities() { std::unique_lock ul(merged_mutex); std::vector out; + std::vector> by_key; + by_key.reserve(accumulator.size()); + for (const auto &[key, accum]: accumulator) + by_key.emplace_back(key, &accum); + std::sort(by_key.begin(), by_key.end(), + [](const auto &a, const auto &b) { return a.first < b.first; }); + out.reserve(accumulator.size()); - for (const auto &accum: accumulator | std::views::values) { + for (const auto &[key, accum_ptr]: by_key) { + const MergeAccum &accum = *accum_ptr; if (accum.sum_w <= 0.0) continue; @@ -388,18 +457,22 @@ std::vector MergeOnTheFly::ExportReflections() { out.emplace_back(mr); } + return out; +} +std::vector MergeOnTheFly::ExportReflections() { + auto out = ExportIntensities(); AssignRfreeFlags(out, space_group, scaling_settings.GetRfreeFraction(), 500, reference_cell); ApplyFrenchWilson(out, space_group); return out; } std::vector MergeAll(const DiffractionExperiment &x, - const std::vector &integration_outcome) { + const std::vector &integration_outcome, + size_t nthreads) { MergeOnTheFly merge(x); - for (size_t i = 0; i < integration_outcome.size(); ++i) - merge.AddImage(integration_outcome[i], static_cast(i)); - return merge.ExportReflections(); + merge.Threads(nthreads).AddImages(integration_outcome); + return merge.ExportIntensities(); } struct ShellAccum { @@ -630,43 +703,51 @@ MergeStatistics MergeOnTheFly::MergeStats(const std::vector &m std::unordered_map rmeas_obs; rmeas_obs.reserve(merged.size()); - for (int i = 0; i < integration_outcome.size(); ++i) { - if (Mask(integration_outcome[i])) - continue; + struct ShellObs { uint64_t key; float I_corr; float merged_I; int shell; bool merged; }; + ReduceImagesInOrder( + integration_outcome.size(), nthreads, + [&](size_t i, std::vector &out) { + if (Mask(integration_outcome[i])) + return; + for (const auto &r: integration_outcome[i].reflections) { + if (generator.IsSystematicallyAbsent(r)) + continue; + if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) + continue; + if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A)) + continue; + if (r.partiality < min_partiality) + continue; - for (const auto &r: integration_outcome[i].reflections) { - if (generator.IsSystematicallyAbsent(r)) - continue; - if (r.image_scale_corr <= 0.0 || !std::isfinite(r.image_scale_corr)) - continue; - if (!AcceptReflection(r, d_min_limit_A, d_max_limit_A)) - continue; - if (r.partiality < min_partiality) - continue; + const float I_corr = r.I * r.image_scale_corr; + const float sigma_corr = r.sigma * r.image_scale_corr; + if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) + continue; - const float I_corr = r.I * r.image_scale_corr; - const float sigma_corr = r.sigma * r.image_scale_corr; - if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f) - continue; - - const auto shell = shells.GetShell(r.d); - if (!shell.has_value()) - continue; - const int s = *shell; - if (s >= 0 && s < n_shells) { - acc[s].total_obs++; - const auto key = generator(r).pack(); - const auto mit = merged_I.find(key); - if (mit != merged_I.end()) { - auto &ra = rmeas_obs[key]; - ra.sum_abs_dev += std::abs(static_cast(I_corr) - mit->second); - ra.sum_I += I_corr; - ra.n++; - ra.shell = s; + const auto shell = shells.GetShell(r.d); + if (!shell.has_value()) + continue; + const int s = *shell; + if (s >= 0 && s < n_shells) { + const auto key = generator(r).pack(); + const auto mit = merged_I.find(key); + out.push_back({key, I_corr, mit != merged_I.end() ? mit->second : NAN, s, + mit != merged_I.end()}); } } - } - } + }, + [&](size_t, const std::vector &out) { + for (const auto &o: out) { + acc[o.shell].total_obs++; + if (o.merged) { + auto &ra = rmeas_obs[o.key]; + ra.sum_abs_dev += std::abs(static_cast(o.I_corr) - o.merged_I); + ra.sum_I += o.I_corr; + ra.n++; + ra.shell = o.shell; + } + } + }); // R_meas per shell: sum over reflections of sqrt(n/(n-1)) * sum_i|I_i - |, over sum of I_i. // R_merge and R_pim beside it, with the factor 1 and sqrt(1/(n-1)) - R_pim following Weiss (2001) diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index b8f2b797e..7cca243a3 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -283,10 +283,11 @@ class MergeOnTheFly { // merged intensities feed the space-group search and the error model - those model fits must not // see the ice-contaminated intensities. The final in-symmetry merge keeps them (for completeness). bool exclude_ice_rings = false; + size_t nthreads = 1; HKLKeyGenerator generator; - std::map accumulator; + std::unordered_map accumulator; // exported in key order // Global error model (XDS form): sigma_corr^2 = a*sigma^2 + (b*)^2. a rescales the // (under-estimated) counting variance; the (b*)^2 term adds the intensity- @@ -315,12 +316,26 @@ class MergeOnTheFly { std::unordered_map reject_median_I; size_t reject_count = 0; - bool Mask(const IntegrationOutcome &outcome); + bool Mask(const IntegrationOutcome &outcome) const; + + // One accepted observation, reduced to what the accumulator adds for it. + struct Contribution { + uint64_t key; + int32_t h, k, l; // the reflection's own index, as the accumulator records it + float w, wI, d; + bool on_ice_ring; + }; + // The observations of one image that reach the merge; returns how many the outlier test dropped. + size_t Contributions(const IntegrationOutcome &outcome, std::vector &out) const; + void Accumulate(const std::vector &contributions, int64_t image_id); public: MergeOnTheFly(const DiffractionExperiment &x); MergeOnTheFly& ReferenceCell(const std::optional &cell); MergeOnTheFly& ExcludeIceRings(bool input) { exclude_ice_rings = input; return *this; } MergeOnTheFly& FilterByImageCC(bool input) { filter_by_image_cc = input; return *this; } + // Workers for the passes over every observation (AddImages, RefineErrorModel, MergeStats); the + // result does not depend on the number. + MergeOnTheFly& Threads(size_t input) { nthreads = std::max(1, input); return *this; } // Fit the global error model from the spread of symmetry-equivalent observations. // Call once before merging; AddImage then applies it. @@ -341,6 +356,9 @@ public: // because it holds every observation at once; a streaming accumulator cannot, and there is no // second, device-side walk of the stills observations for this to disagree with. void AddImage(const IntegrationOutcome& outcome, int64_t image_id); + // AddImage for every outcome, the image id being its index: the same merge, bit for bit, with the + // per-observation work spread over the workers and the accumulation kept in image order. + void AddImages(const std::vector &outcomes); // d_min_override, when set, is the effective high-resolution limit for the shell table (used for // the automatic resolution cutoff computed by the caller); otherwise the manual @@ -355,10 +373,14 @@ public: std::optional declared_range = std::nullopt); std::vector ExportReflections(); + // The merged intensities alone - no free-R flags, no French-Wilson amplitudes - which is all a + // scaling reference reads. + std::vector ExportIntensities(); }; std::vector MergeAll(const DiffractionExperiment &x, - const std::vector &reflections); + const std::vector &reflections, + size_t nthreads); // Pearson CC between one image's corrected intensities (I * image_scale_corr) and a reference set of // full intensities, over the reflections that would enter the merge (non-ice, within the resolution diff --git a/image_analysis/scale_merge/StillsPartialityRefine.cpp b/image_analysis/scale_merge/StillsPartialityRefine.cpp index 44910f6cb..12265a510 100644 --- a/image_analysis/scale_merge/StillsPartialityRefine.cpp +++ b/image_analysis/scale_merge/StillsPartialityRefine.cpp @@ -358,7 +358,7 @@ double StillsPartialityRefine::Run(std::vector &outcomes, si for (int outer = 0; outer < settings_.outer_iterations; ++outer) { // Reference full intensities from the current corrections. - const std::vector merged = MergeAll(experiment_, outcomes); + const std::vector merged = MergeAll(experiment_, outcomes, nthreads); std::map reference; for (const auto &m: merged) reference[hkl_key_generator_(m)] = m.I; diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index a2b5f44a4..e42731bfa 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -1250,7 +1250,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // degrade weak stills (measured CC1/2 collapse at the default 3 iters). { phase("Scaling images (" + label + ")"); - auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome()); + TimingMark scale_mark(logger, "stills scaling images " + label); + auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads); indexer->ScaleAllImages(merge_result); } // Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal @@ -1258,6 +1259,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // correction (no re-integration). Never for the P1 search pass. if (!for_search && experiment_.GetScalingSettings().GetStillsPartialityRefine()) { phase("Partiality post-refine (" + label + ")"); + TimingMark refine_mark(logger, "stills partiality post-refine " + label); StillsPartialityRefine refiner(experiment_); const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads); logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt); @@ -1272,7 +1274,9 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { const std::vector &merge_input = indexer->GetIntegrationOutcome(); phase("Merging"); + TimingMark merge_mark(logger, "stills error model and merge " + label); MergeOnTheFly merge_engine(experiment_); + merge_engine.Threads(config_.nthreads); // For the de-novo P1 pass (for_search) drop the ice-ring reflections from the merged // intensities and the error model, so the space-group search sees clean data; the final // in-symmetry merge keeps them so completeness is not lost. @@ -1296,11 +1300,11 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}", em.a, em.b, em.isa, merge_engine.ErrorModelChi2()); } - for (size_t i = 0; i < merge_input.size(); ++i) - merge_engine.AddImage(merge_input[i], static_cast(i)); + merge_engine.AddImages(merge_input); ScaleMergeResult out; out.merged = merge_engine.ExportReflections(); + merge_mark.Stop(); // Automatic high-resolution cutoff (post-merge): trim the written reflections + reported // shells to the CC1/2 fall-off. The merge, scaling and error model above ran over the full @@ -1313,6 +1317,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { cut_ss.GetResolutionCCTarget(), for_search, logger, &result.resolution_fit_A); phase("Computing statistics"); + TimingMark stats_mark(logger, "stills statistics " + label); out.statistics = merge_engine.MergeStats(out.merged, merge_input, config_.reference_data, effective_d_min); // The same kept reflections binned over the reference range (--report-resolution); the // stills error model is fitted over the whole range already, so no ISa of its own. diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 714bd73d1..5527130e9 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -2103,7 +2103,7 @@ static int RunRugnux(int argc, char **argv) { // against a foreign dataset injects cross-dataset systematics and is a worse reference than // the data's own merge. The per-image scale G is the exact one-pass solution, so one pass // (iterating a self-rebuilt reference only re-fits the freshly-scaled noise on weak stills). - ScaleOnTheFly(experiment, MergeAll(experiment, reflections)).Scale(reflections, nthreads); + ScaleOnTheFly(experiment, MergeAll(experiment, reflections, nthreads)).Scale(reflections, nthreads); // Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal // orientation tilt against the merge and recompute each reflection's partiality + scale correction // (no re-integration), then merge with the improved corrections. @@ -2113,6 +2113,7 @@ static int RunRugnux(int argc, char **argv) { logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt); } MergeOnTheFly merge_engine(experiment); + merge_engine.Threads(nthreads); merge_engine.ReferenceCell(experiment.GetUnitCell()); // --min-image-cc has to hold for the merge itself, not only for the reported statistics. merge_engine.FilterByImageCC(experiment.GetScalingSettings().GetMinCCForImage() > 0.0); @@ -2128,8 +2129,7 @@ static int RunRugnux(int argc, char **argv) { logger.Info("Error model (XDS convention): a={:.3f} b={:.3e} ISa={:.1f} chi2={:.2f}", em.a, em.b, em.isa, merge_engine.ErrorModelChi2()); } - for (size_t i = 0; i < reflections.size(); ++i) - merge_engine.AddImage(reflections[i], static_cast(i)); + merge_engine.AddImages(reflections); merged_reflections = merge_engine.ExportReflections(); // Automatic high-resolution cutoff (post-merge), matching the full-analysis path: a manual diff --git a/tests/MergeScaleTest.cpp b/tests/MergeScaleTest.cpp index aab3d5e9d..ed548a294 100644 --- a/tests/MergeScaleTest.cpp +++ b/tests/MergeScaleTest.cpp @@ -3,6 +3,7 @@ #include +#include #include #include "../image_analysis/bragg_integration/CalcISigma.h" #include "../image_analysis/scale_merge/ErrorModel.h" @@ -585,3 +586,67 @@ TEST_CASE("Global Wilson B over a widened window, with no upper cap", "[wilson]" CHECK(1.0 / (3.9 * 3.9) - 1.0 / (low_res.d_low_A * low_res.d_low_A) == Catch::Approx(1.0 / 9.0 - 1.0 / 16.0).epsilon(0.01)); } + +// The parallel AddImages is the serial AddImage loop, bit for bit: the per-observation work is spread +// over the workers, but every sum and every half-set choice is still made in image order. More images +// than one of its blocks, so the block boundary is crossed. +TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") { + DiffractionExperiment x; + x.SetSpaceGroup(TestSpaceGroup()); + + std::mt19937 rng(7); + std::uniform_int_distribution index(-6, 6); + std::uniform_real_distribution unit(0.0f, 1.0f); + std::vector outcomes(5000); + for (auto &o: outcomes) { + for (int j = 0; j < 20; ++j) { + Reflection r{}; + r.h = index(rng); + r.k = index(rng); + r.l = index(rng); + if (r.h == 0 && r.k == 0 && r.l == 0) + continue; + r.d = 20.0f / std::sqrt(static_cast(r.h * r.h + r.k * r.k + r.l * r.l)); + r.I = 100.0f * unit(rng) - 5.0f; + r.sigma = 1.0f + 5.0f * unit(rng); + r.var_bkg = r.sigma * r.sigma * unit(rng); + r.prescaling_corr = 1.0f; + r.partiality = 0.5f + 0.5f * unit(rng); + r.image_scale_corr = 0.5f + unit(rng); + r.on_ice_ring = unit(rng) < 0.05f; + o.reflections.push_back(r); + } + } + + MergeOnTheFly serial(x); + serial.RefineErrorModel(outcomes); + for (size_t i = 0; i < outcomes.size(); ++i) + serial.AddImage(outcomes[i], static_cast(i)); + const auto serial_merged = serial.ExportReflections(); + + MergeOnTheFly parallel(x); + parallel.Threads(8); + parallel.RefineErrorModel(outcomes); + parallel.AddImages(outcomes); + const auto parallel_merged = parallel.ExportReflections(); + + REQUIRE(serial.ErrorModelActive()); + CHECK(serial.ErrorModelA() == parallel.ErrorModelA()); + CHECK(serial.ErrorModelB() == parallel.ErrorModelB()); + REQUIRE(serial_merged.size() == parallel_merged.size()); + for (size_t i = 0; i < serial_merged.size(); ++i) { + CHECK(serial_merged[i].h == parallel_merged[i].h); + CHECK(serial_merged[i].k == parallel_merged[i].k); + CHECK(serial_merged[i].l == parallel_merged[i].l); + CHECK(serial_merged[i].I == parallel_merged[i].I); + CHECK(serial_merged[i].sigma == parallel_merged[i].sigma); + for (int half = 0; half < 2; ++half) // NaN where a half is empty, on both sides + CHECK(std::memcmp(&serial_merged[i].I_half[half], ¶llel_merged[i].I_half[half], sizeof(float)) == 0); + } + + const auto serial_stats = serial.MergeStats(serial_merged, outcomes); + const auto parallel_stats = parallel.MergeStats(parallel_merged, outcomes); + CHECK(serial_stats.overall.total_observations == parallel_stats.overall.total_observations); + CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half); + CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas); +} From bfd715b64ddf6653f1bc5afe4a3717e7792b280a Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 09:27:21 +0200 Subject: [PATCH 02/11] rugnux --no-gpu: run on the CPU only, without touching CUDA, on a machine with a GPU hide_gpus() in CUDAWrapper makes get_gpu_count() answer 0 without calling cudaGetDeviceCount, so every GPU decision in the process (indexer Auto -> FFTW, preprocessing/spot finding/decode, pre-scan, scaling engine, model validation, MR) takes the path it takes on a machine without a GPU, and no CUDA context is created. rugnux reads --no-gpu in main, ahead of the parser, because set_gpu_blocking_sync() already asks CUDA for the devices. The banner says 'CPU only (--no-gpu)'. The 'run it on the CPU' advice in the out-of-GPU-memory messages and in the install docs now names --no-gpu instead of CUDA_VISIBLE_DEVICES=. Co-Authored-By: Claude Opus 5.5 (1M context) --- common/CUDAWrapper.cpp | 2 ++ common/CUDAWrapper.cu | 23 ++++++++++++++---- common/CUDAWrapper.h | 6 +++++ docs/RUGNUX_ADVANCED.md | 1 + docs/RUGNUX_INSTALL.md | 4 ++-- image_analysis/indexing/CUDAMemHelpers.h | 2 +- .../scale_merge/RotationScaleMergeGPU.cu | 6 ++--- rugnux/rugnux_cli.cpp | 24 +++++++++++++++---- 8 files changed, 53 insertions(+), 15 deletions(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 5d3f45ecc..88aeb1800 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -80,6 +80,8 @@ int32_t get_gpu_count() { return 0; } +void hide_gpus() {} + std::vector get_gpu_names() { return {}; } diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index f12c984a6..4df5a9daa 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -18,7 +18,17 @@ inline void cuda_err(cudaError_t val) { throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); } +namespace { + std::atomic gpus_hidden{false}; +} + +void hide_gpus() { + gpus_hidden = true; +} + int32_t get_gpu_count() { + if (gpus_hidden) + return 0; int device_count; cudaError_t val = cudaGetDeviceCount(&device_count); switch (val) { @@ -137,11 +147,13 @@ void set_gpu_blocking_sync() { // cudaSetDevice creates the device's context, serially, and on four cards that was most of a // second before the run had started. The function is taken from the driver at run time, so the // program does not link against it and still starts on a machine without one. + // Without a device nothing is called at all: even cudaGetLastError() would load the driver. + if (get_gpu_count() == 0) + return; void *fn = nullptr; cudaDriverEntryPointQueryResult found{}; - if (get_gpu_count() == 0 - || cudaGetDriverEntryPointByVersion("cuDevicePrimaryCtxSetFlags", &fn, 12000, cudaEnableDefault, - &found) != cudaSuccess + if (cudaGetDriverEntryPointByVersion("cuDevicePrimaryCtxSetFlags", &fn, 12000, cudaEnableDefault, &found) + != cudaSuccess || found != cudaDriverEntryPointSuccess) { cudaGetLastError(); return; @@ -175,10 +187,13 @@ void release_gpu_contexts() { } void cuda_clear_error() { - cudaGetLastError(); + if (!gpus_hidden) // hidden, nothing has used CUDA, and this call would still load the driver + cudaGetLastError(); } void cuda_throw_if_context_lost() { + if (gpus_hidden) + return; // cudaFree(nullptr) frees nothing, but a lost context fails it. The last error cannot say the same: // once cudaGetLastError() has returned a sticky error, it and cudaPeekAtLastError() report success. const cudaError_t err = cudaFree(nullptr); diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index a61937f5a..0b895f1fb 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -11,6 +11,12 @@ int32_t get_gpu_count(); +// From here on get_gpu_count() is 0 without asking CUDA, so nothing in the process creates a context or +// touches a device and every GPU path takes its CPU path, as on a machine without a GPU: what +// CUDA_VISIBLE_DEVICES= does, asked for by the program (rugnux --no-gpu). Call it before anything uses +// CUDA; there is no way back. +void hide_gpus(); + // Names of the visible GPUs, in device order and one entry per device, so repeated cards repeat. // Empty without CUDA and on a machine with no device, which is also what get_gpu_count() == 0 says. std::vector get_gpu_names(); diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md index 4bcf8edd9..5510b61c8 100644 --- a/docs/RUGNUX_ADVANCED.md +++ b/docs/RUGNUX_ADVANCED.md @@ -330,6 +330,7 @@ General: | `-e, --end-image ` | Last image to process (default: all) | | `-t, --stride ` | Process every *n*-th image (default: 1) | | `-v, --verbose` | Show the whole log on the terminal in place of the steps; `_log.txt` is written either way | +| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched and every GPU step takes its CPU path (`-X Auto` indexes with FFTW; `-X FFT` and `-X FFBIDX` are refused) | Mode — `--mode ` (default `mx`): diff --git a/docs/RUGNUX_INSTALL.md b/docs/RUGNUX_INSTALL.md index 650b2153c..eb5b0a61b 100644 --- a/docs/RUGNUX_INSTALL.md +++ b/docs/RUGNUX_INSTALL.md @@ -124,7 +124,7 @@ fits and rigid-body placement of the validation and the rotation and translation molecular replacement. The CPU implementations are the path where there is no GPU, and the two agree to rounding rather than bit for bit (parts of the scaling and the model fits round differently on the card), so a GPU and a CPU run can differ in the last digits of the merged data. The choice is automatic; -a CUDA build runs on the CPU when the GPU is hidden from it (`CUDA_VISIBLE_DEVICES= rugnux ...`). +a CUDA build runs on the CPU when told to (`rugnux --no-gpu ...`). The thread count (`-N`) governs the CPU side of all of it. The released CUDA builds need only an NVIDIA **driver** on the host, no CUDA toolkit: 525.60.13 or @@ -189,7 +189,7 @@ a big cell. One sweep of a crystal with an axis of about 640 Å predicted 236 mi peaked at **17.5 GB of host memory on a GPU build and 72.5 GB on a CPU-only build**. A CPU-only build keeps in host RAM what a GPU build keeps on the card, so it is the one that needs the large machine. When such a set is too large for GPU scaling on the card, the run stops and says to run it on the -CPU (`CUDA_VISIBLE_DEVICES= rugnux ...`, or a CPU-only build) — which then wants that much host memory. +CPU (`rugnux --no-gpu ...`, or a CPU-only build) — which then wants that much host memory. ### When host memory runs out diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 0da8b4719..0aa741556 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -32,7 +32,7 @@ inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_ msg += buf; } return msg + ". Either another program is using this GPU, or this data set needs more GPU memory than " - "the card has - free the GPU and run again, or run on the CPU (CUDA_VISIBLE_DEVICES= rugnux " + "the card has - free the GPU and run again, or run on the CPU (rugnux --no-gpu " "..., or a CPU-only build)"; } diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.cu b/image_analysis/scale_merge/RotationScaleMergeGPU.cu index a4d52a8d7..2582589b5 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.cu +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.cu @@ -1127,7 +1127,7 @@ namespace { "Scaling %lld partial observations needs more GPU memory than this card has: %.1f GB for " "their per-observation arrays alone (%zu bytes each), before the merge's other buffers, " "on a %.1f GB card. This data set is too large for GPU scaling on this card - run it on " - "the CPU (CUDA_VISIBLE_DEVICES= rugnux ..., or a CPU-only build) on a machine with plenty of " + "the CPU (rugnux --no-gpu ..., or a CPU-only build) on a machine with plenty of " "host memory: the CPU path keeps all of it in RAM, which for a set this large can be " "tens of GB", static_cast(n_obs), double(n_obs) * OBS_BYTES / 1e9, OBS_BYTES, @@ -1146,7 +1146,7 @@ namespace { "Scaling %lld partial observations ran out of GPU memory: a buffer of %.1f MB did not fit, " "with %.1f GB of the %.1f GB card free (the per-observation arrays take %.1f GB). Either " "another program is using this GPU, or this data set is too large for GPU scaling on this " - "card - run it on the CPU (CUDA_VISIBLE_DEVICES= rugnux ..., or a CPU-only build) on a " + "card - run it on the CPU (rugnux --no-gpu ..., or a CPU-only build) on a " "machine with plenty of host memory", static_cast(n_obs), double(bytes) / 1e6, double(free_bytes) / 1e9, double(total_bytes) / 1e9, double(n_obs) * OBS_BYTES / 1e9); @@ -1162,7 +1162,7 @@ namespace { snprintf(msg, sizeof(msg), "GPU busy: scaling %lld partial observations waited %lld minutes for GPU work running " "beside it in this process to give back its memory, and it did not. The data set fits " - "this card; run it again, or on the CPU (CUDA_VISIBLE_DEVICES= rugnux ...)", + "this card; run it again, or on the CPU (rugnux --no-gpu ...)", static_cast(n_obs), static_cast(std::chrono::duration_cast(GPU_BUSY_TIMEOUT).count())); throw JFJochException(JFJochExceptionCategory::GPUCUDAError, msg); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 714bd73d1..4853c5119 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -118,6 +118,8 @@ void print_usage() { "(debug level); an empty prefix writes nothing at all" << std::endl; std::cout << " -N, --threads Number of threads (default: every CPU the run may use - its affinity and cgroup quota - of which the " "per-image loop uses at most 16 per GPU; an explicit value is used as given)" << std::endl; + std::cout << " --no-gpu Run on the CPU only, as on a machine without a GPU: CUDA is not " + "touched and every GPU step takes its CPU path (-X Auto indexes with FFTW)" << std::endl; std::cout << " -s, --start-image Start image number (default: 0)" << std::endl; std::cout << " -e, --end-image End image number (default: all)" << std::endl; std::cout << " -t, --stride Image stride (default: 1)" << std::endl; @@ -318,6 +320,7 @@ enum { OPT_FINALIST_LEDGER, OPT_SIMULATE_LIVE, OPT_DEVELOPER, + OPT_NO_GPU, OPT_FORCE_STILL, OPT_AZIM_MIN_Q, OPT_AZIM_MAX_Q, @@ -375,6 +378,7 @@ static option long_options[] = { {"finalist-ledger", no_argument, nullptr, OPT_FINALIST_LEDGER}, {"simulate-live", required_argument, nullptr, OPT_SIMULATE_LIVE}, {"developer", no_argument, nullptr, OPT_DEVELOPER}, + {"no-gpu", no_argument, nullptr, OPT_NO_GPU}, {"smooth-g", optional_argument, nullptr, OPT_SMOOTH_G}, {"relative-b", optional_argument, nullptr, OPT_RELATIVE_B}, {"no-scaling-corrections", no_argument, nullptr, OPT_NO_SCALING_CORRECTIONS}, @@ -852,7 +856,7 @@ extern "C" void fftwf_assertion_failed(const char *s, int line, const char *file // cannot reject on its own (a resolution limit of zero, a distance of zero, a polarisation above one, // an unreadable file), so main wraps this and reports rather than letting the exception terminate the // process with no diagnostic and exit code 134. -static int RunRugnux(int argc, char **argv) { +static int RunRugnux(int argc, char **argv, bool no_gpu) { // The whole invocation, reported at the end and written into the results report. Started before // anything else so it covers reading the file and setting up as well as the processing. const auto invocation_start = std::chrono::steady_clock::now(); @@ -898,8 +902,8 @@ static int RunRugnux(int argc, char **argv) { : fmt::format(" (git {} {})", jfjoch_git_sha1().substr(0, 6), jfjoch_git_date()), build); logger.Info("Build flags: {}", jfjoch_build_cxx_flags().empty() ? "none" : jfjoch_build_cxx_flags()); logger.Info("Command line: {}", provenance.command_line); - logger.Info("GPU: {}", provenance.gpu_count > 0 - ? fmt::format("{} ({} visible)", provenance.gpu_description, provenance.gpu_count) + logger.Info("GPU: {}", no_gpu ? std::string("not used (--no-gpu) - running on the CPU") + : provenance.gpu_count > 0 ? fmt::format("{} ({} visible)", provenance.gpu_description, provenance.gpu_count) : std::string("none visible - running on the CPU")); RegisterHDF5Filter(); @@ -1361,6 +1365,8 @@ static int RunRugnux(int argc, char **argv) { case OPT_DEVELOPER: provenance.developer = true; break; + case OPT_NO_GPU: // acted on in main, before anything asks CUDA for the devices + break; case OPT_SMOOTH_G: smooth_g_deg_arg = optarg ? parse_double_arg(optarg, "--smooth-g", logger) : SMOOTH_G_DEFAULT_DEG; break; @@ -1639,7 +1645,8 @@ static int RunRugnux(int argc, char **argv) { // Said before the run rather than after it: the GPUs are what makes rugnux fast, and a machine // that turns out to have none - a driver mismatch, a CUDA_VISIBLE_DEVICES left over from another // job - is worth knowing about while there is still time to stop and fix it. - console->Field("Compute", provenance.gpu_count > 0 + console->Field("Compute", no_gpu ? fmt::format("CPU only (--no-gpu), {} threads", nthreads) + : provenance.gpu_count > 0 ? fmt::format("{}{}, {} threads", provenance.gpu_count > 1 ? fmt::format("{} x ", provenance.gpu_count) : "", provenance.gpu_description, nthreads) : fmt::format("no GPU visible - {} CPU threads, which is much slower", nthreads)); @@ -3297,9 +3304,16 @@ int main(int argc, char **argv) { // Before anything creates a CUDA context, which fixes the flags for the device. A thread // waiting on the GPU then sleeps instead of spinning on a core - measured on a 16M rotation // run, a fifth of all CPU time went to that spinning, with no change in wall time either way. + // --no-gpu is looked for here, ahead of the parser, because that already asks CUDA for the devices. + bool no_gpu = false; + for (int i = 1; i < argc; i++) + if (std::strcmp(argv[i], "--no-gpu") == 0) + no_gpu = true; + if (no_gpu) + hide_gpus(); set_gpu_blocking_sync(); enable_gpu_numa_binding(); - const int ret = RunRugnux(argc, argv); + const int ret = RunRugnux(argc, argv, no_gpu); // Everything that used a card has been destroyed with the run. release_gpu_contexts(); return ret; From 593590b3574b3b4b95efd5d4fad43e63ea1074c2 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 09:28:21 +0200 Subject: [PATCH 03/11] rugnux -X FFT is the algorithm: cuFFT on a GPU, FFTW without one; -X cuFFT names the GPU implementation In rugnux people choose the algorithm, not where it runs. -X fft (and the old -F) now resolves at parse time to the GPU FFT indexer where a GPU is visible and --no-gpu is not given, and to FFTW otherwise. -X cufft is the GPU implementation alone, as -X fft was. -X cufft and -X ffbidx without a GPU now stop at once with a clear message instead of failing later in the indexer pool. Auto is unchanged and not redundant: on stills with a known cell and a GPU it takes FFBIDX. Only rugnux's CLI spelling changes. The internal IndexingAlgorithmEnum, the broker and its OpenAPI enum keep FFT = the GPU implementation; the CLI maps onto it, and the command line the viewer writes for a run (RugnuxCommandLine) gives the enum's FFT back as -X cufft, which re-parses to the same thing. A run without -X, and -X fft on a machine with a GPU, process exactly as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 7 +++++-- docs/RUGNUX_ADVANCED.md | 4 ++-- docs/RUGNUX_INSTALL.md | 4 ++-- rugnux/RugnuxCommandLine.cpp | 2 +- rugnux/rugnux_cli.cpp | 20 ++++++++++++++++---- tests/RugnuxTest.cpp | 2 +- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 218801403..3f430a2e5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -367,9 +367,12 @@ generated API model and internal types. (`beam_stop/ShadowFinder` is built and runs by default in `rugnux` — it is no longer the unbuilt prototype an older version of this file described). Least-squares refinement uses **Ceres** (fetched in `image_analysis/CMakeLists.txt`, built with miniglog, no MKL, no Ceres-CUDA, CXX_THREADS). -The indexer is chosen with `-X` (`FFBIDX|FFT|FFTW|Auto|None`, default `Auto`, which resolves to a +The indexer is chosen with `-X` (`Auto|FFT|cuFFT|FFTW|FFBIDX|None`, default `Auto`, which resolves to a GPU indexer when one is present and `fftw` otherwise): `ffbidx` wants a known cell (`-C`) and suits -sparse serial stills; `fft`/`fftw` index de novo and suit strong rotation data. +sparse serial stills; `fft` indexes de novo and suits strong rotation data - it is the algorithm, run +with cuFFT on a GPU and with FFTW without one (or under `--no-gpu`), while `cufft`/`fftw` name one +implementation each. Only rugnux's CLI spells it this way: internally (and in the broker's OpenAPI) +`FFT` is the GPU implementation, which the CLI's `cufft` maps onto. **FPGA** (`fpga/`): `hls/` is the Vitis HLS source (image-analysis kernels), `hls_simulation/` runs that same HLS on CPU for hardware-free testing, `hdl/` is the Verilog RTL, `host_library/` is diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md index 5510b61c8..4b8e13ef8 100644 --- a/docs/RUGNUX_ADVANCED.md +++ b/docs/RUGNUX_ADVANCED.md @@ -330,7 +330,7 @@ General: | `-e, --end-image ` | Last image to process (default: all) | | `-t, --stride ` | Process every *n*-th image (default: 1) | | `-v, --verbose` | Show the whole log on the terminal in place of the steps; `_log.txt` is written either way | -| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched and every GPU step takes its CPU path (`-X Auto` indexes with FFTW; `-X FFT` and `-X FFBIDX` are refused) | +| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched and every GPU step takes its CPU path (`-X Auto` and `-X FFT` index with FFTW; `-X cuFFT` and `-X FFBIDX` are refused) | Mode — `--mode ` (default `mx`): @@ -408,7 +408,7 @@ For very long axes the direction grid's angular resolution binds well below 500 | Option | Description | | --- | --- | | `--force-still` | Treat a rotation (goniometer) dataset as independent stills instead of rotation | -| `-X, --indexing-algorithm ` | `FFBIDX` \| `FFT` \| `FFTW` \| `Auto` \| `None` | +| `-X, --indexing-algorithm ` | `Auto` (default) \| `FFT` \| `cuFFT` \| `FFTW` \| `FFBIDX` \| `None`, case-insensitive. `FFT` is the de-novo FFT search, on the GPU (cuFFT) where there is one and `--no-gpu` is not given, otherwise on the CPU (FFTW); `cuFFT` and `FFTW` name one implementation each. `cuFFT` and `FFBIDX` (known cell, `-C`) need a GPU and are refused without one. `Auto` takes `FFBIDX` on stills with a known cell and a GPU, and `FFT` otherwise; rotation data always use the FFT search | | `-C, --unit-cell ` | Reference unit cell `"a,b,c,alpha,beta,gamma"` (required by `ffbidx`). On rotation data it also widens the FFT search to cover the cell given, at both ends: the longest axis looked for is raised to reach it, and `--fft-min-unit-cell` is lowered to admit it | | `--fft-min-unit-cell ` | Shortest cell axis the FFT search accepts, Å (default: 10). A candidate with a shorter axis is discarded — but a de-novo rotation run also runs a **second first-pass hypothesis with the floor lowered to 5 Å by default**, adopted when the standing cell turns out to be an integer supercell of what it finds, so a small-molecule cell is indexed without this flag. `-C` lowers the floor on its own to cover the cell given (and the second hypothesis then stays out of the way). On stills, where neither applies, a crystal below the floor cannot be indexed unless this is lowered | | `--min-indexed-spots ` | Spots a frame must have on the lattice before it counts as indexed (default: 6, minimum 4). It sets the reported indexing rate and the count the rotation first pass ranks candidate lattices by; whether the run has a lattice at all is decided on the pooled spots instead ([§4.1](CPU_DATA_ANALYSIS_INDEXING.md#41-indexed-spot-decision-inlier-test)), and integration is not gated by it | diff --git a/docs/RUGNUX_INSTALL.md b/docs/RUGNUX_INSTALL.md index eb5b0a61b..5f9b42422 100644 --- a/docs/RUGNUX_INSTALL.md +++ b/docs/RUGNUX_INSTALL.md @@ -113,8 +113,8 @@ build has no CUDA there and needs nothing installed beyond Xcode and CMake. ## Hardware As with the rest of Jungfraujoch, **serious performance requires an NVIDIA GPU**. The CUDA build -provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`fft`); without CUDA -only the CPU `fftw` indexer is available. With a GPU present most of the per-image pipeline runs on +provides the GPU fast-feedback indexer (`ffbidx`) and the GPU FFT indexer (`cufft`); without CUDA +only the CPU `fftw` indexer is available, and `-X fft` runs on it. With a GPU present most of the per-image pipeline runs on the device — bitshuffle+LZ4 decompression, miniCBF byte-offset decoding and marCCD widening, image preprocessing, azimuthal integration, spot finding, prediction and Bragg integration — as do the pre-scan (beam-stop mask, background beam-centre fit, spot widths, defective pixels; one accumulator diff --git a/rugnux/RugnuxCommandLine.cpp b/rugnux/RugnuxCommandLine.cpp index 83d61b1bf..eb7c08495 100644 --- a/rugnux/RugnuxCommandLine.cpp +++ b/rugnux/RugnuxCommandLine.cpp @@ -25,7 +25,7 @@ namespace { const char *indexing_alg_flag(IndexingAlgorithmEnum a) { switch (a) { case IndexingAlgorithmEnum::FFBIDX: return "ffbidx"; - case IndexingAlgorithmEnum::FFT: return "fft"; + case IndexingAlgorithmEnum::FFT: return "cufft"; // the GPU implementation; rugnux's -X fft is the algorithm case IndexingAlgorithmEnum::FFTW: return "fftw"; case IndexingAlgorithmEnum::None: return "none"; case IndexingAlgorithmEnum::Auto: diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 4853c5119..da20f59e6 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -119,7 +119,7 @@ void print_usage() { std::cout << " -N, --threads Number of threads (default: every CPU the run may use - its affinity and cgroup quota - of which the " "per-image loop uses at most 16 per GPU; an explicit value is used as given)" << std::endl; std::cout << " --no-gpu Run on the CPU only, as on a machine without a GPU: CUDA is not " - "touched and every GPU step takes its CPU path (-X Auto indexes with FFTW)" << std::endl; + "touched and every GPU step takes its CPU path (-X Auto and FFT index with FFTW)" << std::endl; std::cout << " -s, --start-image Start image number (default: 0)" << std::endl; std::cout << " -e, --end-image End image number (default: all)" << std::endl; std::cout << " -t, --stride Image stride (default: 1)" << std::endl; @@ -169,7 +169,9 @@ void print_usage() { std::cout << " --rotation-scale Goniometer rotation scale: the stage turned k times the angle stored in the file (the commanded one). Applied to both passes; overrides the fitted correction" << std::endl; std::cout << " --rotation-no-postrefine Disable the (default-on) two-pass rotation post-refine (post-refine detector distance/beam + cell/axis, then re-integrate; only the refined pass is written, as the canonical _* output)" << std::endl; std::cout << " --prepass-fraction Integrate the first fraction f of the sweep, but at least 120 deg of it, in the two-pass rotation geometry pre-pass (0.05-1, default 0.5; 1 = the whole sweep); pass 2 reads every frame. The pre-scan and the first-pass indexing sample the same part of the sweep" << std::endl; - std::cout << " -X, --indexing-algorithm Indexing algorithm (FFBIDX|FFT|FFTW|Auto|None)" << std::endl; + std::cout << " -X, --indexing-algorithm Indexing algorithm (Auto|FFT|cuFFT|FFTW|FFBIDX|None, default Auto). FFT is " + "the de-novo FFT search, on the GPU (cuFFT) where there is one and --no-gpu is not given, otherwise on the " + "CPU (FFTW); cuFFT and FFTW name one implementation each. cuFFT and FFBIDX (known cell) need a GPU" << std::endl; std::cout << " -S, --space-group Space group number (96) or symbol (P43212) - for indexing and scaling" << std::endl; std::cout << " -C, --unit-cell Fix reference unit cell: \"a,b,c,alpha,beta,gamma\"" << std::endl; std::cout << " --fft-min-unit-cell Shortest cell axis the FFT search accepts, in A (default: 10). A candidate with a shorter axis is discarded, but a de-novo rotation run also tries a second first-pass hypothesis with the floor at 5 A, so a small-molecule cell is indexed without this flag; -C lowers it on its own to cover the cell given" << std::endl; @@ -1133,7 +1135,9 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { if (alg == "ffbidx") indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; - else if (alg == "fft") + else if (alg == "fft") // the algorithm: on the GPU where there is one, else with FFTW + indexing_algorithm = get_gpu_count() > 0 ? IndexingAlgorithmEnum::FFT : IndexingAlgorithmEnum::FFTW; + else if (alg == "cufft") indexing_algorithm = IndexingAlgorithmEnum::FFT; else if (alg == "fftw") indexing_algorithm = IndexingAlgorithmEnum::FFTW; @@ -1207,7 +1211,7 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { dump_observations = optarg; break; case 'F': - indexing_algorithm = IndexingAlgorithmEnum::FFT; + indexing_algorithm = get_gpu_count() > 0 ? IndexingAlgorithmEnum::FFT : IndexingAlgorithmEnum::FFTW; break; case 'A': anomalous_mode = true; @@ -1594,6 +1598,14 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { exit(EXIT_FAILURE); } + if ((indexing_algorithm == IndexingAlgorithmEnum::FFT || indexing_algorithm == IndexingAlgorithmEnum::FFBIDX) + && get_gpu_count() == 0) { + logger.Error("-X {} runs on the GPU only, and {}; -X FFT or Auto index on the CPU", + indexing_algorithm == IndexingAlgorithmEnum::FFT ? "cuFFT" : "FFBIDX", + no_gpu ? "--no-gpu is given" : "no GPU is visible"); + exit(EXIT_FAILURE); + } + input_file = argv[optind]; const std::string broker_url = (optind == argc - 2) ? argv[optind + 1] : ""; diff --git a/tests/RugnuxTest.cpp b/tests/RugnuxTest.cpp index bb8367cb7..70c8621f7 100644 --- a/tests/RugnuxTest.cpp +++ b/tests/RugnuxTest.cpp @@ -263,7 +263,7 @@ TEST_CASE("RugnuxCommandLine_Full", "[process]") { CHECK(cmd.find("-N 8") != std::string::npos); CHECK(cmd.find("-e 500") != std::string::npos); CHECK(cmd.find("-o run1") != std::string::npos); - CHECK(cmd.find("-X fft") != std::string::npos); + CHECK(cmd.find("-X cufft") != std::string::npos); CHECK(cmd.find("-S 96") != std::string::npos); // -R takes an optional argument, so its value must be attached (-R30); a separate "-R 30" token // would not re-parse (getopt would leave 30 as a positional and drop the count). From 39b488e323e3e10b620cdc0921af825f555d4b1a Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 10:50:56 +0200 Subject: [PATCH 04/11] Probe passes and the main image pass take turns on a card under 12 GB On a 16 GB card a probe pass (GPUWorkBeside) runs its image work beside the main line's image pass (GPUImagePass). Measured on a 16 Mpx detector with nsys, the two together held 8.6 GB of device buffers (two sets of 24 per-worker 67 MB image buffers plus five 0.6 GB FFT indexers), so on an 8 GB card the refined-geometry pass ran out of memory ("Allocating GPU memory (53.0 MB) failed"). Below 12 GB of card memory each of the two now waits, before it starts, for the other kind to end. Only when each runs depends on the card, never what it computes; neither waits once running, so nothing waits in a circle. On a 16 GB card nothing changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- common/CUDAWrapper.cpp | 62 +++++++++++++++++++++++++----------------- common/CUDAWrapper.h | 6 ++-- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 5d3f45ecc..2cb53c297 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -28,50 +28,62 @@ std::string get_gpu_description() { } namespace { - // How many of one kind of GPU work are running, for a thread to wait until none is. - struct GPUWorkCount { - std::mutex m; - std::condition_variable none; - int n = 0; - void Add() { - std::lock_guard lock(m); - ++n; + // How many probe passes (GPUWorkBeside) and main-line image passes (GPUImagePass) are running, for a + // thread to wait until none of one kind is. + std::mutex gpu_work_m; + std::condition_variable gpu_work_changed; + int gpu_work_beside_n = 0, gpu_image_passes_n = 0; + + // A card too small for a probe pass beside the main line's image pass: measured on a 16 Mpx detector, + // the two together held 8.6 GB. On such a card each of the two waits, before it starts, for the other + // kind to end - so they take turns, and only when each runs depends on the card, never what it + // computes. Neither waits once it is running, so nothing waits in a circle. + constexpr size_t TAKE_TURNS_BELOW_BYTES = 12'000'000'000ull; + + bool take_turns() { + static const bool small = get_gpu_count() > 0 && get_gpu_total_memory(0) < TAKE_TURNS_BELOW_BYTES; + return small; + } + + void remove(int &n) { + { + std::lock_guard lock(gpu_work_m); + --n; } - void Remove() { - { - std::lock_guard lock(m); - --n; - } - none.notify_all(); - } - }; - GPUWorkCount gpu_work_beside, gpu_image_passes; + gpu_work_changed.notify_all(); + } } GPUWorkBeside::GPUWorkBeside() { - gpu_work_beside.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); + ++gpu_work_beside_n; } GPUWorkBeside::~GPUWorkBeside() { - gpu_work_beside.Remove(); + remove(gpu_work_beside_n); } bool wait_for_gpu_work_beside(std::chrono::seconds timeout) { - std::unique_lock lock(gpu_work_beside.m); - return gpu_work_beside.none.wait_for(lock, timeout, [] { return gpu_work_beside.n == 0; }); + std::unique_lock lock(gpu_work_m); + return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; }); } GPUImagePass::GPUImagePass() { - gpu_image_passes.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_work_beside_n == 0; }); + ++gpu_image_passes_n; } GPUImagePass::~GPUImagePass() { - gpu_image_passes.Remove(); + remove(gpu_image_passes_n); } void wait_for_gpu_image_passes() { - std::unique_lock lock(gpu_image_passes.m); - gpu_image_passes.none.wait(lock, [] { return gpu_image_passes.n == 0; }); + std::unique_lock lock(gpu_work_m); + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); } #ifndef JFJOCH_USE_CUDA diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index a61937f5a..429443b59 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -89,7 +89,8 @@ void cuda_throw_if_context_lost(); // GPU work that runs beside the main line of a process and gives all of its device memory back when it // ends: a probe pass run on a copy of the run while the run itself scales and merges. Hold one for as -// long as that work runs. Build-independent - it only counts. +// long as that work runs. On a card too small for both, it waits first for the main line's image passes +// to end (GPUImagePass). Build-independent - it only counts. class GPUWorkBeside { public: GPUWorkBeside(); @@ -103,7 +104,8 @@ bool wait_for_gpu_work_beside(std::chrono::seconds timeout); // The image pass of a run's main line - not of a probe holding a GPUWorkBeside: the stretch that holds // the most of the card, and that never waits for a probe, so a probe that does not fit beside it can -// wait for it to end. Hold one for as long as the image pass runs. Build-independent - it only counts. +// wait for it to end. Hold one for as long as the image pass runs. On a card too small for both, it +// waits first for the work beside to end. Build-independent - it only counts. class GPUImagePass { public: GPUImagePass(); From 4936f9b408c042371131651ed4eaefedbf8820db Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 11:32:51 +0200 Subject: [PATCH 05/11] RESULT-CHANGING (owner decision): stills scaling reference and tilt refinement only over the crystals the merge keeps The final stills merge drops every crystal whose cell is not close to the consensus cell (MergeOnTheFly::Mask), but the scaling reference (MergeAll) and the partiality post-refinement were made over all of them. On two 25k-frame XFEL runs of a tetragonal crystal those were 26-30% of the indexed frames and 60% of the integrated reflections: weak frames (median 8 indexed spots) indexed in a 2x/3x/4x supercell or an unrelated lattice, whose supercell-only reflections are pure noise and whose indices do not name the true reflections. They went into the reference every merged crystal was scaled against, and through the tilt refinement. MergeAll and StillsPartialityRefine::Run now take the reference cell and use only the crystals it accepts - the same test, with the same tolerances, as the merge. Measured (both runs, mx path): CC1/2 0.9599 -> 0.9598 and 0.9645 -> 0.9644, I/sigma +0.05 and +0.02, R_pim unchanged, space group and resolution unchanged; --mode scale against a deposited tetragonal lysozyme model CC 0.9452 -> 0.9477 and 0.9461 -> 0.9473. Scale+merge 2.1x faster on --mode scale (14.6 -> 6.9 s, loaded box). Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/scale_merge/Merge.cpp | 5 +++-- image_analysis/scale_merge/Merge.h | 5 ++++- .../scale_merge/StillsPartialityRefine.cpp | 13 ++++++++++--- image_analysis/scale_merge/StillsPartialityRefine.h | 5 ++++- rugnux/RugnuxScaleMerge.cpp | 5 +++-- rugnux/rugnux_cli.cpp | 4 ++-- 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/image_analysis/scale_merge/Merge.cpp b/image_analysis/scale_merge/Merge.cpp index 2c3bf37f1..4f72271c7 100644 --- a/image_analysis/scale_merge/Merge.cpp +++ b/image_analysis/scale_merge/Merge.cpp @@ -469,9 +469,10 @@ std::vector MergeOnTheFly::ExportReflections() { std::vector MergeAll(const DiffractionExperiment &x, const std::vector &integration_outcome, - size_t nthreads) { + size_t nthreads, + const std::optional &reference_cell) { MergeOnTheFly merge(x); - merge.Threads(nthreads).AddImages(integration_outcome); + merge.Threads(nthreads).ReferenceCell(reference_cell).AddImages(integration_outcome); return merge.ExportIntensities(); } diff --git a/image_analysis/scale_merge/Merge.h b/image_analysis/scale_merge/Merge.h index 7cca243a3..cfb5cd615 100644 --- a/image_analysis/scale_merge/Merge.h +++ b/image_analysis/scale_merge/Merge.h @@ -378,9 +378,12 @@ public: std::vector ExportIntensities(); }; +// The merged intensities of the images whose cell is close to reference_cell (all of them without one): +// the reference a per-image scale is fitted against. std::vector MergeAll(const DiffractionExperiment &x, const std::vector &reflections, - size_t nthreads); + size_t nthreads, + const std::optional &reference_cell); // Pearson CC between one image's corrected intensities (I * image_scale_corr) and a reference set of // full intensities, over the reflections that would enter the merge (non-ice, within the resolution diff --git a/image_analysis/scale_merge/StillsPartialityRefine.cpp b/image_analysis/scale_merge/StillsPartialityRefine.cpp index 12265a510..60ee3f7dd 100644 --- a/image_analysis/scale_merge/StillsPartialityRefine.cpp +++ b/image_analysis/scale_merge/StillsPartialityRefine.cpp @@ -349,7 +349,14 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome, return tilt_deg; } -double StillsPartialityRefine::Run(std::vector &outcomes, size_t nthreads) const { +double StillsPartialityRefine::Run(std::vector &outcomes, size_t nthreads, + const std::optional &reference_cell) const { + const auto &indexing = experiment_.GetIndexingSettings(); + auto merged_crystal = [&](const IntegrationOutcome &o) { + return !reference_cell || o.latt.GetUnitCell().is_close(*reference_cell, + indexing.GetUnitCellDistTolerance(), + indexing.GetUnitCellAngleTolerance_deg()); + }; if (nthreads == 0) nthreads = AvailableCpus(); nthreads = std::max(1, nthreads); @@ -358,7 +365,7 @@ double StillsPartialityRefine::Run(std::vector &outcomes, si for (int outer = 0; outer < settings_.outer_iterations; ++outer) { // Reference full intensities from the current corrections. - const std::vector merged = MergeAll(experiment_, outcomes, nthreads); + const std::vector merged = MergeAll(experiment_, outcomes, nthreads, reference_cell); std::map reference; for (const auto &m: merged) reference[hkl_key_generator_(m)] = m.I; @@ -370,7 +377,7 @@ double StillsPartialityRefine::Run(std::vector &outcomes, si auto worker = [&]() { size_t i = next.fetch_add(1); while (i < outcomes.size()) { - const double t = RefineOne(outcomes[i], reference); + const double t = merged_crystal(outcomes[i]) ? RefineOne(outcomes[i], reference) : 0.0; if (t > 0.0) { double prev = tilt_sum.load(); while (!tilt_sum.compare_exchange_weak(prev, prev + t)) {} diff --git a/image_analysis/scale_merge/StillsPartialityRefine.h b/image_analysis/scale_merge/StillsPartialityRefine.h index fbce8c1dd..f74568b65 100644 --- a/image_analysis/scale_merge/StillsPartialityRefine.h +++ b/image_analysis/scale_merge/StillsPartialityRefine.h @@ -43,7 +43,10 @@ public: explicit StillsPartialityRefine(const DiffractionExperiment &x); // Refine all crystals in place. Returns the mean |dpsi| applied (degrees), for diagnostics. - double Run(std::vector &outcomes, size_t nthreads = 0) const; + // Only the crystals whose cell is close to reference_cell (all without one) are refined and make the + // reference: the others are not merged. + double Run(std::vector &outcomes, size_t nthreads, + const std::optional &reference_cell) const; private: const DiffractionExperiment experiment_; diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index e42731bfa..f390ebc3f 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -1251,7 +1251,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { { phase("Scaling images (" + label + ")"); TimingMark scale_mark(logger, "stills scaling images " + label); - auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads); + auto merge_result = MergeAll(experiment_, indexer->GetIntegrationOutcome(), config_.nthreads, + result.consensus_cell); indexer->ScaleAllImages(merge_result); } // Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal @@ -1261,7 +1262,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { phase("Partiality post-refine (" + label + ")"); TimingMark refine_mark(logger, "stills partiality post-refine " + label); StillsPartialityRefine refiner(experiment_); - const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads); + const double mean_tilt = refiner.Run(indexer->GetIntegrationOutcome(), config_.nthreads, result.consensus_cell); logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt); // The refine fits its OWN per-image scale, with no floor, and adopts it unconditionally // when the image has no prior CC - which is exactly the state the guard above leaves. So diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 5527130e9..6b29580ca 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -2103,13 +2103,13 @@ static int RunRugnux(int argc, char **argv) { // against a foreign dataset injects cross-dataset systematics and is a worse reference than // the data's own merge. The per-image scale G is the exact one-pass solution, so one pass // (iterating a self-rebuilt reference only re-fits the freshly-scaled noise on weak stills). - ScaleOnTheFly(experiment, MergeAll(experiment, reflections, nthreads)).Scale(reflections, nthreads); + ScaleOnTheFly(experiment, MergeAll(experiment, reflections, nthreads, experiment.GetUnitCell())).Scale(reflections, nthreads); // Physical partiality post-refinement (default on; --simple-stills disables): refine a per-crystal // orientation tilt against the merge and recompute each reflection's partiality + scale correction // (no re-integration), then merge with the improved corrections. if (experiment.GetScalingSettings().GetStillsPartialityRefine()) { StillsPartialityRefine refiner(experiment); - const double mean_tilt = refiner.Run(reflections, nthreads); + const double mean_tilt = refiner.Run(reflections, nthreads, experiment.GetUnitCell()); logger.Info("Stills partiality post-refine: mean |dpsi| = {:.3f} deg", mean_tilt); } MergeOnTheFly merge_engine(experiment); From 3c4d730f898a32001d19fdf5a4233db531a146d9 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 11:59:09 +0200 Subject: [PATCH 06/11] RESULT-CHANGING (owner decision): stills tilt post-refinement solved directly instead of by Ceres Each crystal's (psi_x, psi_y) tilt was refined by a Ceres problem rebuilt for every inner iteration: one auto-differentiated residual block and one Cauchy loss per reflection. With the scaling reference cleaned up it was 70% of the stills scale/merge. The same objective - Cauchy on the sigma-weighted intensity residuals plus the Gaussian prior on the tilt, the tilt bounded - is now minimised by a small Levenberg-Marquardt on the rho'-reweighted 2x2 normal equations, the tilt applied to all reflections through one rotation matrix and the Jacobian taken by central differences. It reaches the same answer to optimizer precision: on the new synthetic test (StillsPartialityRefine_RecoversTheTilt) partiality error 0.0559 -> 0.03019 with Ceres and 0.03016 with this, mean tilt 0.02065 vs 0.02067 deg; on two 25k-frame XFEL runs (--mode scale) CC1/2, I/sigma, R_pim and the CC against a deposited lysozyme model agree to 0.0005, mean |dpsi| 0.023 deg on both. Scale+merge 6.9 -> 3.0 s (--mode scale, loaded box). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scale_merge/StillsPartialityRefine.cpp | 176 +++++++++++------- tests/MergeScaleTest.cpp | 96 ++++++++++ 2 files changed, 205 insertions(+), 67 deletions(-) diff --git a/image_analysis/scale_merge/StillsPartialityRefine.cpp b/image_analysis/scale_merge/StillsPartialityRefine.cpp index 60ee3f7dd..21785a263 100644 --- a/image_analysis/scale_merge/StillsPartialityRefine.cpp +++ b/image_analysis/scale_merge/StillsPartialityRefine.cpp @@ -3,13 +3,14 @@ #include "StillsPartialityRefine.h" +#include +#include #include #include #include #include #include -#include #include #include "Merge.h" @@ -101,44 +102,114 @@ namespace { return G; } - // Ceres residual: refine the orientation tilt (psi_x, psi_y) holding the scale G fixed. The tilt - // rotates the base reciprocal vector q; partiality follows analytically. Residual is the intensity - // mismatch weighted by 1/sigma, exactly matching ScaleOnTheFly's intensity-space objective. - struct PsiResidual { - double qx, qy, qz; + // The rotation by the angle-axis vector (psi_x, psi_y, 0) as a matrix (Rodrigues), so that one tilt + // is applied to all the reflections of a crystal by a matrix product each. + std::array TiltMatrix(double psi_x, double psi_y) { + const double theta = std::sqrt(psi_x * psi_x + psi_y * psi_y); + if (theta < 1e-15) + return {1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0}; + const double kx = psi_x / theta, ky = psi_y / theta; + const double c = std::cos(theta), s = std::sin(theta), t = 1.0 - c; + return {c + t * kx * kx, t * kx * ky, s * ky, + t * kx * ky, c + t * ky * ky, -s * kx, + -s * ky, s * kx, c}; + } + + // ComputeP with the tilt given as its rotation matrix. + struct TiltModel { double s0x, s0y, s0z, inv_lambda; double gamma0, gamma_e, bw; - double G, inv_corr, Iref, Iobs, weight; - template - bool operator()(const T *const psi, T *residual) const { - T q[3] = {T(qx), T(qy), T(qz)}; - T aa[3] = {psi[0], psi[1], T(0.0)}; - T qr[3]; - ceres::AngleAxisRotatePoint(aa, q, qr); - const T Sx = qr[0] + T(s0x), Sy = qr[1] + T(s0y), Sz = qr[2] + T(s0z); - const T de = ceres::sqrt(Sx * Sx + Sy * Sy + Sz * Sz) - T(inv_lambda); - const T dstar = ceres::sqrt(qr[0] * qr[0] + qr[1] * qr[1] + qr[2] * qr[2]); - const T sig_ang = T(gamma_e) * dstar; - const T sbw = T(bw) * ceres::abs(qr[2]); - const T sig2 = T(gamma0) * T(gamma0) + sig_ang * sig_ang + sbw * sbw; - const T p = ceres::exp(T(-0.5) * de * de / sig2); - residual[0] = T(weight) * (T(G) * p * T(inv_corr) * T(Iref) - T(Iobs)); - return true; + double P(const std::array &R, const FitObs &o) const { + const double qx = R[0] * o.qx + R[1] * o.qy + R[2] * o.qz; + const double qy = R[3] * o.qx + R[4] * o.qy + R[5] * o.qz; + const double qz = R[6] * o.qx + R[7] * o.qy + R[8] * o.qz; + const double de = VecLen(qx + s0x, qy + s0y, qz + s0z) - inv_lambda; + const double sig_ang = gamma_e * VecLen(qx, qy, qz); + const double sbw = bw * std::fabs(qz); + const double sig2 = gamma0 * gamma0 + sig_ang * sig_ang + sbw * sbw; + if (!(sig2 > 0.0)) + return 1.0; + return std::exp(-0.5 * de * de / sig2); } }; - // Gaussian prior N(0, sigma_prior^2) on the tilt. The data residuals above are (model-obs)/sigma, a - // proper chi^2, so the MAP prior residual is simply dpsi/sigma_prior - no scale calibration needed. - struct PsiPrior { - double inv_sigma; - template - bool operator()(const T *const psi, T *residual) const { - residual[0] = T(inv_sigma) * psi[0]; - residual[1] = T(inv_sigma) * psi[1]; - return true; + // Refine the orientation tilt (psi_x, psi_y) holding the scale G fixed: minimise + // sum_i rho(r_i^2) / 2 + |psi / sigma_prior|^2 / 2, r_i = weight_i (G p_i(psi) Iref_i / corr_i - Iobs_i), + // with the Cauchy rho(s) = k^2 log(1 + s / k^2) - the intensity-space objective of ScaleOnTheFly, and + // the Gaussian prior a proper MAP term since r_i is already in sigma units. Levenberg-Marquardt on the + // normal equations reweighted by rho', the Jacobian by central differences, every trial tilt held inside + // the bound. Returns false when the cost cannot be evaluated, leaving psi as it came in. + bool RefineTilt(const std::vector &obs, const TiltModel &model, double G, double robust_k, + double inv_sigma_prior, double max_tilt, double psi[2]) { + constexpr int MAX_ITERATIONS = 25; + constexpr double STEP = 1e-7; // rad, for the central differences + const double k2 = robust_k * robust_k; + const double prior2 = inv_sigma_prior * inv_sigma_prior; + + auto cost = [&](const double p[2]) { + const auto R = TiltMatrix(p[0], p[1]); + double c = 0.5 * prior2 * (p[0] * p[0] + p[1] * p[1]); + for (const auto &o: obs) { + const double r = o.weight * (G * model.P(R, o) * o.inv_corr * o.Iref - o.Iobs); + c += 0.5 * k2 * std::log1p(r * r / k2); + } + return c; + }; + + double current = cost(psi); + if (!std::isfinite(current)) + return false; + double lambda = 1e-4; + for (int iter = 0; iter < MAX_ITERATIONS; ++iter) { + const auto R = TiltMatrix(psi[0], psi[1]); + const auto Rxp = TiltMatrix(psi[0] + STEP, psi[1]), Rxm = TiltMatrix(psi[0] - STEP, psi[1]); + const auto Ryp = TiltMatrix(psi[0], psi[1] + STEP), Rym = TiltMatrix(psi[0], psi[1] - STEP); + double A00 = prior2, A01 = 0.0, A11 = prior2; + double b0 = prior2 * psi[0], b1 = prior2 * psi[1]; + for (const auto &o: obs) { + const double a = o.weight * G * o.inv_corr * o.Iref; + const double r = a * model.P(R, o) - o.weight * o.Iobs; + const double jx = a * (model.P(Rxp, o) - model.P(Rxm, o)) / (2.0 * STEP); + const double jy = a * (model.P(Ryp, o) - model.P(Rym, o)) / (2.0 * STEP); + const double w = 1.0 / (1.0 + r * r / k2); // rho' + A00 += w * jx * jx; + A01 += w * jx * jy; + A11 += w * jy * jy; + b0 += w * jx * r; + b1 += w * jy * r; + } + + bool accepted = false; + double trial[2] = {psi[0], psi[1]}; + double trial_cost = current; + while (lambda < 1e10) { + const double M00 = A00 * (1.0 + lambda), M11 = A11 * (1.0 + lambda); + const double det = M00 * M11 - A01 * A01; + if (!(det > 0.0)) + break; + trial[0] = std::clamp(psi[0] - (M11 * b0 - A01 * b1) / det, -max_tilt, max_tilt); + trial[1] = std::clamp(psi[1] - (M00 * b1 - A01 * b0) / det, -max_tilt, max_tilt); + trial_cost = cost(trial); + if (std::isfinite(trial_cost) && trial_cost < current) { + accepted = true; + lambda = std::max(lambda / 10.0, 1e-12); + break; + } + lambda *= 10.0; + } + if (!accepted) + break; + const double step = std::hypot(trial[0] - psi[0], trial[1] - psi[1]); + const double decrease = current - trial_cost; + psi[0] = trial[0]; + psi[1] = trial[1]; + current = trial_cost; + if (decrease <= 1e-6 * current || step <= 1e-10) + break; } - }; + return true; + } } StillsPartialityRefine::StillsPartialityRefine(const DiffractionExperiment &x) @@ -238,6 +309,8 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome, double psi[2] = {0.0, 0.0}; double G = 1.0; + const TiltModel model{.s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda, + .gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw}; const bool refine_tilt = obs.size() >= settings_.min_reflections; const int inner = refine_tilt ? settings_.inner_iterations : 1; @@ -258,42 +331,11 @@ double StillsPartialityRefine::RefineOne(IntegrationOutcome &outcome, if (!refine_tilt) break; - // (2) Refine the tilt holding G fixed. - ceres::Problem problem; - for (const auto &o: obs) { - auto *cost = new ceres::AutoDiffCostFunction(new PsiResidual{ - .qx = o.qx, .qy = o.qy, .qz = o.qz, - .s0x = S0.x, .s0y = S0.y, .s0z = S0.z, .inv_lambda = inv_lambda, - .gamma0 = gamma0, .gamma_e = gamma_e, .bw = bw, - .G = G, .inv_corr = o.inv_corr, .Iref = o.Iref, .Iobs = o.Iobs, .weight = o.weight}); - problem.AddResidualBlock(cost, new ceres::CauchyLoss(settings_.robust_k), psi); - } - if (settings_.prior_sigma_deg > 0.0) { - const double inv_sigma = kRadToDeg / settings_.prior_sigma_deg; // 1 / sigma_prior (rad) - problem.AddResidualBlock(new ceres::AutoDiffCostFunction( - new PsiPrior{inv_sigma}), nullptr, psi); - } - problem.SetParameterLowerBound(psi, 0, -settings_.max_tilt_rad); - problem.SetParameterUpperBound(psi, 0, settings_.max_tilt_rad); - problem.SetParameterLowerBound(psi, 1, -settings_.max_tilt_rad); - problem.SetParameterUpperBound(psi, 1, settings_.max_tilt_rad); - - ceres::Solver::Options options; - options.linear_solver_type = ceres::DENSE_QR; - options.minimizer_progress_to_stdout = false; - options.num_threads = 1; - options.max_num_iterations = 25; - ceres::Solver::Summary summary; - const double psi_before[2] = {psi[0], psi[1]}; - ceres::Solve(options, &problem, &summary); - // A solve that failed leaves whatever the minimiser last wrote in psi, and that tilt would go - // straight onto every reflection's partiality below. Keep the tilt this crystal came in with - // and stop refining it instead - G alone is still a usable model. - if (!summary.IsSolutionUsable()) { - psi[0] = psi_before[0]; - psi[1] = psi_before[1]; + // (2) Refine the tilt holding G fixed. A cost that cannot be evaluated keeps the tilt this crystal + // came in with and stops refining it - G alone is still a usable model. + const double inv_sigma_prior = settings_.prior_sigma_deg > 0.0 ? kRadToDeg / settings_.prior_sigma_deg : 0.0; + if (!RefineTilt(obs, model, G, settings_.robust_k, inv_sigma_prior, settings_.max_tilt_rad, psi)) break; - } } // Keep what the crystal came in with. This refinement is on by default, so a crystal the tilt model diff --git a/tests/MergeScaleTest.cpp b/tests/MergeScaleTest.cpp index ed548a294..d862fd213 100644 --- a/tests/MergeScaleTest.cpp +++ b/tests/MergeScaleTest.cpp @@ -10,6 +10,8 @@ #include "../image_analysis/scale_merge/HKLKey.h" #include "../image_analysis/scale_merge/Merge.h" #include "../image_analysis/scale_merge/ResolutionCutoff.h" +#include "../image_analysis/scale_merge/ScaleOnTheFly.h" +#include "../image_analysis/scale_merge/StillsPartialityRefine.h" #include "gemmi/reciproc.hpp" TEST_CASE("HKLKey_NoSG_noMergeFriedel") { @@ -650,3 +652,97 @@ TEST_CASE("MergeOnTheFly_AddImagesIsTheSerialMerge") { CHECK(serial_stats.overall.cc_half == parallel_stats.overall.cc_half); CHECK(serial_stats.overall.r_meas == parallel_stats.overall.r_meas); } + +// Stills partiality post-refinement on noiseless synthetic crystals: each crystal's true orientation is +// tilted from the stored one by up to 0.05 deg, its intensities are the true partials of a fixed set of +// full intensities, and its stored partialities those of the stored orientation. The refinement has to +// move the partialities towards the true ones. (The Ceres solve this replaced gave the same numbers: error +// 0.0559 -> 0.0302, mean tilt 0.0206 deg.) +TEST_CASE("StillsPartialityRefine_RecoversTheTilt") { + constexpr float CELL = 50.0f, WAVELENGTH = 1.0f, D_MIN = 2.0f; + constexpr double GAMMA0 = 0.001, ACCEPT = 1.7 * GAMMA0; // RLP width; |dist_ewald| accepted at prediction + DiffractionExperiment x; + x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1")); + DiffractionGeometry geom; + geom.Wavelength_A(WAVELENGTH); + const Coord S0 = geom.GetScatteringVector(); + + std::mt19937 rng(11); + std::normal_distribution gauss(0.0, 1.0); + std::uniform_real_distribution unit(0.0, 1.0); + auto full_I = [](int h, int k, int l) { // a fixed, Friedel-symmetric pseudo-random full intensity + const uint64_t key = HKLKeyGenerator(true, gemmi::get_spacegroup_by_name("P 1"))(h, k, l).pack(); + return 10.0 + 990.0 * static_cast((key * 0x9e3779b97f4a7c15ULL) >> 40) / static_cast(1ULL << 24); + }; + auto rotate = [](const double R[9], const Coord &v) { + return Coord(static_cast(R[0] * v.x + R[1] * v.y + R[2] * v.z), + static_cast(R[3] * v.x + R[4] * v.y + R[5] * v.z), + static_cast(R[6] * v.x + R[7] * v.y + R[8] * v.z)); + }; + auto rotation = [](double ax, double ay, double az, double angle, double R[9]) { // Rodrigues + const double n = std::sqrt(ax * ax + ay * ay + az * az); + ax /= n; ay /= n; az /= n; + const double c = std::cos(angle), s = std::sin(angle), t = 1.0 - c; + const double M[9] = {c + t * ax * ax, t * ax * ay - s * az, t * ax * az + s * ay, + t * ax * ay + s * az, c + t * ay * ay, t * ay * az - s * ax, + t * ax * az - s * ay, t * ay * az + s * ax, c + t * az * az}; + std::copy(M, M + 9, R); + }; + auto partiality = [&](const Coord &q) { + const double de = (q + S0).Length() - 1.0 / WAVELENGTH; + return std::exp(-0.5 * de * de / (GAMMA0 * GAMMA0)); + }; + + std::vector outcomes(150); + std::vector> true_p(outcomes.size()); + const int hmax = static_cast(CELL / D_MIN); + for (size_t c = 0; c < outcomes.size(); ++c) { + double R[9], T[9]; + rotation(gauss(rng), gauss(rng), gauss(rng), 2.0 * M_PI * unit(rng), R); + const double tilt = (0.02 + 0.03 * unit(rng)) * M_PI / 180.0, phi = 2.0 * M_PI * unit(rng); + rotation(std::cos(phi), std::sin(phi), 0.0, tilt, T); // about an axis normal to the beam + auto &o = outcomes[c]; + o.geom = geom; + o.latt = CrystalLattice(rotate(R, Coord(CELL, 0, 0)), rotate(R, Coord(0, CELL, 0)), + rotate(R, Coord(0, 0, CELL))); + const double G = 0.5 + unit(rng); + for (int h = -hmax; h <= hmax; ++h) + for (int k = -hmax; k <= hmax; ++k) + for (int l = -hmax; l <= hmax; ++l) { + const Coord q = o.latt.Astar() * h + o.latt.Bstar() * k + o.latt.Cstar() * l; + const float dstar = q.Length(); + if (dstar == 0.0f || dstar > 1.0f / D_MIN) + continue; + if (std::fabs((q + S0).Length() - 1.0 / WAVELENGTH) > ACCEPT) + continue; + const double p_true = partiality(rotate(T, q)); + Reflection r{}; + r.h = h; r.k = k; r.l = l; + r.d = 1.0f / dstar; + r.I = static_cast(G * p_true * full_I(h, k, l)); + r.sigma = static_cast(1.0 + 0.01 * r.I); + r.prescaling_corr = 1.0f; + r.partiality = static_cast(partiality(q)); + r.image_scale_corr = 1.0f / r.partiality; + o.reflections.push_back(r); + true_p[c].push_back(p_true); + } + REQUIRE(o.reflections.size() >= 40); + } + + auto mean_abs_error = [&]() { + double sum = 0.0; + size_t n = 0; + for (size_t c = 0; c < outcomes.size(); ++c) + for (size_t i = 0; i < outcomes[c].reflections.size(); ++i, ++n) + sum += std::fabs(outcomes[c].reflections[i].partiality - true_p[c][i]); + return sum / static_cast(n); + }; + const double before = mean_abs_error(); + ScaleOnTheFly(x, MergeAll(x, outcomes, 4, std::nullopt)).Scale(outcomes, 4); + const double mean_tilt = StillsPartialityRefine(x).Run(outcomes, 4, std::nullopt); + const double after = mean_abs_error(); + CHECK(mean_tilt > 0.015); + CHECK(mean_tilt < 0.06); + CHECK(after < 0.6 * before); // measured 0.54: the prior holds the larger tilts back +} From beba8f03018ee90b630a466648077dc040d34913 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 12:01:57 +0200 Subject: [PATCH 07/11] Rotation scaling runs on the CPU where its GPU footprint is over 80% of the card The device scaling's peak, measured with nsys on four large sets (33-123 M observations, 4-16 M distinct hkl), is 81 bytes an observation plus 291 bytes a distinct hkl to within 5%. Once the ingest has counted both, an engine whose estimate is over 80% of the card's total memory scales on the CPU instead, with one warning line saying why; a resident ingest is made again on the host, which the CPU path needs. Decided once, before anything is scaled, on the card's total memory: the answer depends on the card, not on the moment. On a simulated 8 GB card (total clamped to 8.2e9, 7.75 GB free) a 33 M-observation, 15 M-hkl set whose final merge alone needs ~7.5 GB on the GPU ran out of memory there; it now completes, and its merged MTZ matches the 16 GB GPU run to float rounding (max relative difference 9e-7 on any intensity column, same reflections, cell, space group and resolution cut). The largest estimate on the battery is 12.45 GB, under 80% of a 16 GB card, so nothing changes there. The all-observations arm is copied from the engine's card arrays only when that engine is on the GPU. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scale_merge/RotationScaleMerge.cpp | 38 ++++++++++++++++++- .../scale_merge/RotationScaleMerge.h | 8 ++++ rugnux/RugnuxScaleMerge.cpp | 2 +- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index b969ff62f..6840ec09e 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -395,7 +395,7 @@ void RotationScaleMerge::IngestHost() { // Probed before the observations are built: whether the build lays down the full Obs array or // only the ingest arrays depends on whether the device pipeline will be active. gpu_ = std::make_unique(gpu_device); - gpu_active_ = gpu_->Available(); + gpu_active_ = gpu_->Available() && !scale_on_cpu_; resident_ingest = gpu_active_ && observation_dump_path.empty(); #endif @@ -690,6 +690,22 @@ void RotationScaleMerge::IngestOnDevice() { } #endif +bool RotationScaleMerge::OnGPU() const { +#ifdef JFJOCH_USE_CUDA + return gpu_active_; +#else + return false; +#endif +} + +size_t RotationScaleMerge::GPUScalingPeakBytes() const { + // The device scaling's peak, fitted to what nsys measured on four large data sets (33-123 M + // observations, 4-16 M distinct hkl): within 5% on each. Unlike DeviceBytes, an estimate and not + // an upper bound - it decides where the GPU clearly cannot take the scaling. + constexpr size_t PEAK_BYTES_PER_OBS = 81, PEAK_BYTES_PER_HKL = 291; + return PEAK_BYTES_PER_OBS * static_cast(n_partials_obs) + PEAK_BYTES_PER_HKL * rawrun_start.size(); +} + size_t RotationScaleMerge::DeviceBytes() const { // What each costs at the peak, measured: an observation's arrays and its group CSR; a full's arrays // with the surface terms, or with the outlier test's and the error model's samples; an hkl's raw run @@ -734,6 +750,26 @@ void RotationScaleMerge::IngestDevice(const RotationScaleMerge *observations_fro gpu_->CopyObservationsFrom(*observations_from->gpu_); else if (resident_ingest) IngestOnDevice(); + // The scaling's footprint on the card is known once the ingest has counted the observations and the + // distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run + // scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the + // card but not on the moment. The CPU path builds its observations on the host, so a resident + // ingest is made again there. + if (gpu_active_ && !observations_from) { + const size_t card = get_gpu_total_memory(gpu_device); + if (GPUScalingPeakBytes() > CPU_SCALING_ABOVE_CARD_FRACTION * card) { + logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about " + "{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead", + n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9, + 100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9); + scale_on_cpu_ = true; + gpu_active_ = false; + if (resident_ingest) { + gpu_.reset(); + IngestHost(); + } + } + } if (gpu_active_ && !resident_ingest && !observations_from) { const int n = static_cast(partials.size()); gpu_->SetPartialsLayout(n, n_frames, frame_start.data(), frame_count.data()); diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index 1c3a8e2f7..34baebcd9 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -177,6 +177,10 @@ public: // engine's peak was at most 81% of this. Where the ingest on the card peaks higher, its peak. // Call after IngestDevice(). [[nodiscard]] size_t DeviceBytes() const; + // The device scaling's estimated peak (see the .cpp). Call after the ingest. + [[nodiscard]] size_t GPUScalingPeakBytes() const; + // Above this fraction of the card's memory the scaling runs on the CPU. + static constexpr double CPU_SCALING_ABOVE_CARD_FRACTION = 0.8; // Merge only the first n frames of the outcomes (by default all of them). Set before Ingest(). void SetFrameLimit(int n) { frame_limit = n; } @@ -185,6 +189,8 @@ public: // the merge runs: on cards of one model the result is the same on any of them. void SetGpuDevice(int device) { gpu_device = device; } [[nodiscard]] int GetGpuDevice() const { return gpu_device; } + // Whether this engine scales on the GPU (decided by the ingest). + [[nodiscard]] bool OnGPU() const; // Whether Run() also hands back the merge's fulls unmerged (Result::scaled_fulls). Off by default: // it is a copy as long as the fulls, wanted only by a caller that writes them. @@ -551,6 +557,8 @@ private: // the CPU loops as the bit-parity fallback. Built in Ingest. NotCopied gpu_; bool gpu_active_ = false; + // The scaling does not fit the card (IngestDevice): this engine scales on the CPU. + bool scale_on_cpu_ = false; // The fulls of this pass live on the device (the GPU combine ran), so the correction surfaces can // build their terms there. Set by RunArm. bool fulls_on_device_ = false; diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index a2b5f44a4..8e0e66b7f 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -1161,7 +1161,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { // is the data sets up to about 50 M observations at high multiplicity; the large ones merge // their arms one after the other. if (!geometry_prepass && !experiment_.GetGemmiSpaceGroup().has_value() && rsm->GetSearchMinZeta() > 0.0 - && cards > 0) { + && cards > 0 && rsm->OnGPU()) { const int32_t device = std::min(cards - 1, 2); constexpr size_t CARD_BYTES_RESERVED = size_t(3) << 30; if (cards >= 3 From c1b745fdfc7e27467eef181d38ecae543a339275 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 12:01:57 +0200 Subject: [PATCH 08/11] Image-loop workers per card bounded by the card's memory A worker of the full-analysis image loop holds about 24 bytes a detector pixel on its card (sixteen held 6.3 GB on a 16.8 Mpx detector, measured with nsys). The workers now keep to half of the card's total memory: on a 16 GB card that is still the sixteen of before for detectors up to ~21 Mpx; on an 8 GB card with a 16.8 Mpx detector it is ten, where sixteen ran out of memory with ~7 GB free. How many workers there are changes only the loop's speed. An explicit -N is obeyed as before. Co-Authored-By: Claude Opus 5.5 (1M context) --- rugnux/RugnuxImagePass.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index 90060a378..3ba388961 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -368,9 +368,19 @@ void Rugnux::ImagePass(PipelineLocals &p) { // which is what a previous attempt at this got wrong. // Only the full-analysis worker drives a card; the azimuthal one preprocesses and integrates on // the CPU and wants every thread it can have, so the cap must not reach it. + // A worker also holds about 24 bytes a detector pixel of its card - sixteen held 6.3 GB on a 16.8 Mpx + // detector - and the workers keep to half of the card, so a small card gets fewer of them. How many + // there are changes only how fast the loop runs, never what it computes. const int gpus = get_gpu_count(); + int workers_per_card = 16; + if (gpus > 0) { + constexpr int64_t WORKER_BYTES_PER_PIXEL = 24; + const auto fit = static_cast(get_gpu_total_memory(0) / 2) + / (WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum()); + workers_per_card = static_cast(std::clamp(fit, 1, 16)); + } const int image_workers = (per_image_analysis && config_.nthreads_auto && gpus > 0) - ? std::min(config_.nthreads, std::max(16, 16 * gpus)) + ? std::min(config_.nthreads, workers_per_card * gpus) : config_.nthreads; if (image_workers < config_.nthreads) logger.Info("Per-image loop: {} of {} threads ({} GPUs) - past what a card can take queue " From 4b92ec25a65b98a0b73992768c9c1eda9c7a16d1 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 12:13:17 +0200 Subject: [PATCH 09/11] On a card under 12 GB probe passes also take turns with each other With the main image pass already waited for, a simulated 8 GB card with 7.0 GB free (the display's share taken) still ran out of memory: the rotation-scale walk's index-only probe and the speculative geometry probe ran beside each other during the merge, and the walk's cuFFT plan failed. On a small card a probe pass now also waits, before it starts, for every other probe pass to end. Scheduling only; nothing changes on a 16 GB card. Co-Authored-By: Claude Opus 5.5 (1M context) --- common/CUDAWrapper.cpp | 11 ++++++----- common/CUDAWrapper.h | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 2cb53c297..862d90df2 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -34,10 +34,11 @@ namespace { std::condition_variable gpu_work_changed; int gpu_work_beside_n = 0, gpu_image_passes_n = 0; - // A card too small for a probe pass beside the main line's image pass: measured on a 16 Mpx detector, - // the two together held 8.6 GB. On such a card each of the two waits, before it starts, for the other - // kind to end - so they take turns, and only when each runs depends on the card, never what it - // computes. Neither waits once it is running, so nothing waits in a circle. + // A card too small for a probe pass beside the main line's image pass, or beside another probe pass: + // measured on a 16 Mpx detector, two of them together held 8.6 GB. On such a card a probe pass waits, + // before it starts, for every other one and for the image passes to end, and an image pass for the + // probe passes - so they take turns, and only when each runs depends on the card, never what it + // computes. None waits once it is running, so nothing waits in a circle. constexpr size_t TAKE_TURNS_BELOW_BYTES = 12'000'000'000ull; bool take_turns() { @@ -57,7 +58,7 @@ namespace { GPUWorkBeside::GPUWorkBeside() { std::unique_lock lock(gpu_work_m); if (take_turns()) - gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0 && gpu_work_beside_n == 0; }); ++gpu_work_beside_n; } diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index 429443b59..5f17fe419 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -89,8 +89,8 @@ void cuda_throw_if_context_lost(); // GPU work that runs beside the main line of a process and gives all of its device memory back when it // ends: a probe pass run on a copy of the run while the run itself scales and merges. Hold one for as -// long as that work runs. On a card too small for both, it waits first for the main line's image passes -// to end (GPUImagePass). Build-independent - it only counts. +// long as that work runs. On a small card it waits first for the main line's image passes (GPUImagePass) +// and for other work beside to end. Build-independent - it only counts. class GPUWorkBeside { public: GPUWorkBeside(); From bdb5aaa44b997fe75c0194fe0fafd343a6b4eee1 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 13:14:25 +0200 Subject: [PATCH 10/11] Indexing algorithm FFT without a GPU resolves to None: live indexing is off instead of the collection cancelled The broker cancelled the whole data collection on the first indexed image when its algorithm was FFT and no GPU was visible (the indexer pool refuses FFT there, and the receiver cancels on the exception). FFTW is too slow for live stills, so it is not switched to; with None the pool reports the frame as not indexed and the collection runs on. rugnux resolves -X FFT itself (cuFFT or FFTW) and never reaches this case. Co-Authored-By: Claude Opus 5.5 (1M context) --- common/DiffractionExperiment.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index 4e1bf3d9b..d01da5c51 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -1710,7 +1710,8 @@ IndexingAlgorithmEnum DiffractionExperiment::GetIndexingAlgorithm() const { return IndexingAlgorithmEnum::FFT; return IndexingAlgorithmEnum::FFBIDX; case IndexingAlgorithmEnum::FFT: - return IndexingAlgorithmEnum::FFT; + // No silent switch to FFTW, too slow for live stills: a missing GPU turns indexing off + return get_gpu_count() == 0 ? IndexingAlgorithmEnum::None : IndexingAlgorithmEnum::FFT; case IndexingAlgorithmEnum::FFTW: return IndexingAlgorithmEnum::FFTW; default: From 86f17740f6b5a980dee326d14f035b11631973da Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 13:15:22 +0200 Subject: [PATCH 11/11] ACKNOWLEDGEMENT: Markstein reference without the working note Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/ACKNOWLEDGEMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 33feb7705..be74d7ca6 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -521,7 +521,7 @@ Parallel Connected-Component Labelling on GPUs" (2018), IEEE Trans. Parallel Dis The FFT indexer's GPU peak search divides by a fixed window width with a reciprocal multiplication corrected once by the exact remainder, which returns the correctly rounded quotient, following P. W. Markstein, "Computation of elementary functions on the IBM RISC System/6000 processor" -(1990), IBM J. Res. Dev. 34, 111-119 (DOI not verified here). +(1990), IBM J. Res. Dev. 34, 111-119. This software uses the Viridis, Magma and Inferno colormaps from Matplotlib under its BSD-compatible license. J. D. Hunter, "Matplotlib: A 2D graphics environment" (2007), Comput. Sci.