From 9c8347031dda1bebf6390f8046c3042fbc41d438 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 22:20:17 +0100 Subject: [PATCH 1/4] FFTIndexer: DeAlias asks its rows on the refinement's threads Each shortlisted row's unfold test (the transform at its own length and at its folded partner's, summed over up to 4096 spots in double) is independent of the others, so the rows are spread over refine_threads with ParallelFor and the answers applied afterwards in row order, logging included. With one refine thread (stills) it is the serial loop as before. Results identical by construction. Rotation first pass, -N 28, 5950X + RTX 5080, machine shared with other jobs (median of 2, s): Step 3 Indexing / Step 6 Indexing at the refined geometry thau 1000 images: 2.75 / 1.23 -> 2.46 / 1.06 cytc: 1.81 -> 1.65 lyso: 1.96 / 0.90 -> 1.73 / 0.85 _report.txt (minus run-specific lines) and .hkl identical on all three. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- image_analysis/indexing/FFTIndexer.cpp | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/image_analysis/indexing/FFTIndexer.cpp b/image_analysis/indexing/FFTIndexer.cpp index d4a1ea9d8..ab666f957 100644 --- a/image_analysis/indexing/FFTIndexer.cpp +++ b/image_analysis/indexing/FFTIndexer.cpp @@ -9,6 +9,7 @@ #include #include "PostIndexingRefinement.h" #include "../../common/Logger.h" +#include "../../common/ParallelFor.h" // Following Steller, Bolotovsky & Rossmann (1997) J. Appl. Cryst. 30, 1036-1040, after Bricogne (1986) EEC PSD Software Workshop, LURE FFTIndexer::FFTIndexer(const IndexingSettings &settings) @@ -399,30 +400,36 @@ void FFTIndexer::DeAlias(const std::vector &coord, size_t nspots, std::ve // error, and the spots' phases stay coherent only over a fraction of it at high resolution. Search // both lengths over +-1 bin in quarter-bin steps. const double bin_A = 2.0 * static_cast(max_length_A) / static_cast(histogram_size); - std::vector proj; - proj.reserve(nspots / stride + 1); - const auto amplitude = [&](double centre_A) { return RowAmplitude(proj, centre_A, bin_A).first; }; - - for (auto &row : rows) { - const double len = row.Length(); + // Each row is asked on its own, so the rows are spread over the refinement's threads; the answers + // are applied afterwards in row order. + std::vector a_len(rows.size(), 0.0), a_partner(rows.size(), 0.0); + ParallelFor(static_cast(rows.size()), refine_threads, [&](int r) { + const double len = rows[r].Length(); if (!(len > 0.0)) - continue; + return; const double partner = 2.0 * static_cast(max_length_A) - len; const double ratio = partner / len; if (std::abs(ratio - std::round(ratio)) < 0.1) - continue; - const Coord dir = row.Normalize(); - proj.clear(); + return; + const Coord dir = rows[r].Normalize(); + std::vector proj; + proj.reserve(nspots / stride + 1); for (size_t i = 0; i < nspots; i += stride) proj.push_back(dir * coord[i]); - const double a_len = amplitude(len), a_partner = amplitude(partner); - if (a_partner > a_len) { - Logger("FFTIndexer").Debug("FFT row of {:.1f} A unfolded to {:.1f} A: the spots carry {:.0f} at " - "the longer length against {:.0f} at the folded one", - len, partner, a_partner, a_len); - row = dir * static_cast(partner); - } + a_len[r] = RowAmplitude(proj, len, bin_A).first; + a_partner[r] = RowAmplitude(proj, partner, bin_A).first; + }); + + for (size_t r = 0; r < rows.size(); r++) { + if (!(a_partner[r] > a_len[r])) + continue; + const double len = rows[r].Length(); + const double partner = 2.0 * static_cast(max_length_A) - len; + Logger("FFTIndexer").Debug("FFT row of {:.1f} A unfolded to {:.1f} A: the spots carry {:.0f} at " + "the longer length against {:.0f} at the folded one", + len, partner, a_partner[r], a_len[r]); + rows[r] = rows[r].Normalize() * static_cast(partner); } } From 388474ba4aa99f1d2542e6d47ed080e05d127f48 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 22:20:17 +0100 Subject: [PATCH 2/4] Refine: test the candidate cells on the refinement's threads After the candidate cells are refined, each one is tested on its own - the reference match, StepDownToReference where it does not match, the angle and volume tests, and ComputeIndexedMask over the accumulated spots. That loop ran on one thread per scheme; on a re-index with a reference cell (the long-axis hypothesis) StepDownToReference and the mask took most of the call. The candidates now go over refine_threads with ParallelFor, each into its own slot, and are collected in candidate order before the ranking, so the ranking sees the same list. Results identical by construction. Rotation first pass, -N 28, 5950X + RTX 5080, machine shared with other jobs (median of 2, s): Step 3 / Step 6, after the DeAlias commit thau 1000 images: 2.46 / 1.06 -> 2.10 / 1.00 cytc: 1.65 -> 1.62 lyso: 1.73 / 0.85 -> 1.76 / 0.84 (noise) _report.txt (minus run-specific lines) and .hkl identical on all three. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- .../indexing/PostIndexingRefinement.cpp | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/image_analysis/indexing/PostIndexingRefinement.cpp b/image_analysis/indexing/PostIndexingRefinement.cpp index 59a51d709..20dd2823e 100644 --- a/image_analysis/indexing/PostIndexingRefinement.cpp +++ b/image_analysis/indexing/PostIndexingRefinement.cpp @@ -3,6 +3,7 @@ #include "../../common/JFJochMath.h" #include "PostIndexingRefinement.h" +#include "../../common/ParallelFor.h" #include #include @@ -335,7 +336,11 @@ std::vector Refine(const std::vector &in_spots, reference_cells.push_back(reference.NiggliReduce().GetUnitCell()); } - for (int i = 0; i < scores.size(); i++) { + // Each candidate is tested on its own - the step-down to the reference and the indexed mask are the + // costly parts - so they are spread over the same threads as the refinement above, each into its own + // slot, and kept in candidate order. + std::vector> tested(scores.size()); + ParallelFor(static_cast(scores.size()), p.refine_threads, [&](int i) { Eigen::Matrix3f cell_rows = oCell.block(3u * i, 0u, 3u, 3u); Eigen::Matrix3f cell_cols = cell_rows.transpose(); Eigen::Vector3f row_norms = cell_rows.rowwise().norm(); @@ -343,18 +348,18 @@ std::vector Refine(const std::vector &in_spots, if (!reference_cells.empty()) { if (!MatchesReference(cell_rows, reference_cells, p.dist_tolerance_vs_reference)) { if (!p.step_down_to_reference) - continue; + return; const auto sub = StepDownToReference(cell_rows, reference_cells, p.dist_tolerance_vs_reference, spots.topRows(nspots), p.indexing_tolerance); if (!sub) - continue; + return; cell_rows = *sub; cell_cols = cell_rows.transpose(); row_norms = cell_rows.rowwise().norm(); } } else { if (row_norms.minCoeff() < p.min_length_A || row_norms.maxCoeff() > p.max_length_A) - continue; + return; } // Filter for wrong angles. Compared as COSINES, not angles: acos is strictly decreasing on @@ -369,28 +374,31 @@ std::vector Refine(const std::vector &in_spots, if (cos_alpha > cos_min_angle || cos_alpha < cos_max_angle || cos_beta > cos_min_angle || cos_beta < cos_max_angle || cos_gamma > cos_min_angle || cos_gamma < cos_max_angle) - continue; + return; // Three coplanar rows pass both tests above - 120 deg apart in one plane is inside any angle // window - and the refinement can walk a candidate there. Such a cell has no reciprocal basis, // so it is not a lattice: the same volume test as CrystalLattice::VolumeFraction(). if (std::abs(cell_rows.determinant()) < MIN_BASIS_VOLUME_FRACTION * row_norms.prod()) - continue; + return; int64_t indexed_spot_count = 0; auto indexed_mask = ComputeIndexedMask(spots.topRows(nspots), cell_cols, p.indexing_tolerance, indexed_spot_count); if (indexed_spot_count < p.viable_cell_min_spots) - continue; + return; - candidates.emplace_back(RefinedCandidate{ + tested[i] = RefinedCandidate{ .cell = cell_rows, .score = scores(i), .volume = std::abs(cell_rows.determinant()), .indexed_spot_count = indexed_spot_count, .indexed_mask = std::move(indexed_mask) - }); - } + }; + }); + for (auto &t : tested) + if (t) + candidates.push_back(std::move(*t)); std::sort(candidates.begin(), candidates.end(), [](const RefinedCandidate &a, const RefinedCandidate &b) { From 556e61b3c1647ab4cdb9922ec232d78647fca54c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 22:20:30 +0100 Subject: [PATCH 3/4] First pass: start the long-axis searches ahead of the hypothesis The long-axis hypothesis (the coarse 1200 A search, then a re-index of each cell it offers) was 1.10 s of thau's Step 3 and 0.36 s of its Step 6, almost all of it CPU work after a few ms of FFT, while the rest of the machine sat idle. Its searches need only the fed spots and the experiment, so they now start ahead of it, on a copy of the experiment and a pool of their own (FirstPassRun::long_axis_spec): the coarse search beside the file centre's schemes (IndexSchemes), the re-index of each cell it offers beside the beam-centre check. The hypothesis then asks the same questions, and RotationIndexer's memo, keyed on every input an indexer reads, answers them. Where the run moved in between (the axis-sign rescue, another centre), the key differs and the hypothesis indexes as before; a search that failed leaves no memo and the hypothesis meets the failure itself. The parameter code the two share is factored out (LongAxisBound, LongAxisCoarseSettings, LongAxisReindexSettings, LongAxisOffers). Results identical by construction. Rotation first pass, -N 28, 5950X + RTX 5080, machine shared with other jobs (median of 2 before, 4 after, s): Step 3 / Step 6, after the two indexer commits thau 1000 images: 2.10 / 1.00 -> 1.65 / 0.84 (long-axis line 1.16 -> 0.02 s) cytc: 1.62 -> 1.50 lyso: 1.76 / 0.84 -> 1.40 / 0.65 a ~1000 A-axis crystal (private): wall 84 -> 70 s _report.txt (minus run-specific lines) and .hkl identical on all four. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- docs/CPU_DATA_ANALYSIS_INDEXING.md | 2 +- rugnux/RugnuxFirstPass.cpp | 6 +- rugnux/RugnuxFirstPass.h | 33 +++++++ rugnux/RugnuxFirstPassRescues.cpp | 133 +++++++++++++++++++++++------ 4 files changed, 144 insertions(+), 30 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS_INDEXING.md b/docs/CPU_DATA_ANALYSIS_INDEXING.md index 1689ab3af..4e482dabc 100644 --- a/docs/CPU_DATA_ANALYSIS_INDEXING.md +++ b/docs/CPU_DATA_ANALYSIS_INDEXING.md @@ -121,7 +121,7 @@ For each direction, the FFT magnitude spectrum is computed; peaks correspond to The running-mean background window keeps a **constant width** and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel's running sum is still valid. -The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames. +The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames. Its searches are started ahead of the question - the coarse one beside the file centre's schemes, the re-index of what it offers beside the beam-centre check - and the question then reads their answers from the indexer's memo: this changes when they run, not what they find. Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 5605c45d0..7816abe10 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -484,6 +484,7 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { run.PrefetchCertainFrames(); FirstPass best = run.IndexSchemes(); best = run.AxisSignRescue(std::move(best)); + run.StartLongAxisReindex(best); bool long_axis_asked = false; best = run.BeamCentreCheck(std::move(best), long_axis_asked); // The long-axis hypothesis, at the centre the run now stands on. @@ -1516,7 +1517,10 @@ void Rugnux::FirstPassRun::StartCentreSpeculation() { } auto Rugnux::FirstPassRun::IndexSchemes() -> FirstPass { - return PickBest(*indexer_pool, *indexer, [&] { StartCentreSpeculation(); }); + return PickBest(*indexer_pool, *indexer, [&] { + StartLongAxisCoarse(); + StartCentreSpeculation(); + }); } // Twinning by reticular merohedry: two domains of one lattice whose nodes coincide on a sub-lattice diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index d219b0ae4..2e4ddfa2d 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -69,6 +69,10 @@ struct Rugnux::FirstPassRun { // The beam-centre check; sets long_axis_asked where it has asked the long-axis hypothesis itself. FirstPass BeamCentreCheck(FirstPass best, bool &long_axis_asked); FirstPass LongAxisHypothesis(FirstPass standing); + // Its searches, started ahead of it (see long_axis_spec): the coarse one beside the file centre's + // schemes, the re-index of what that offers beside the beam-centre check. + void StartLongAxisCoarse(); + void StartLongAxisReindex(const FirstPass &standing); FirstPass AddEndWedge(FirstPass bp, IndexerThreadPool &pool, IndexAndRefine &idx); void ReleaseSpotEngines(); // Throws where the pass ends with no lattice at all. @@ -143,6 +147,16 @@ struct Rugnux::FirstPassRun { const std::function &alongside = {}); void ApplyState(const FirstPassState &st); void StartCentreSpeculation(); + // How long an axis the detector separates the rows of, for the spots the run has. + struct LongAxisReach { + float width_px = 0.0f, resolvable_A = 0.0f, bound_A = 0.0f; + }; + LongAxisReach LongAxisBound() const; + // The settings of the coarse search, and of the re-index of the cell `x` holds. + IndexingSettings LongAxisCoarseSettings(float bound_A) const; + IndexingSettings LongAxisReindexSettings(const DiffractionExperiment &x) const; + // The coarse cells other than the standing lattice, each once. + static std::vector LongAxisOffers(const SchemeIndexers &ris, const FirstPass &standing); // --- Near ties --- // @@ -330,4 +344,23 @@ struct Rugnux::FirstPassRun { std::future done; // last, so it is waited for before the rest goes }; std::optional centre_spec; + + // The long-axis hypothesis's FFT searches need only the fed spots and the experiment, and leave + // most of the machine idle (the card does a few percent of the work). So they are started ahead of + // it, on a copy of the experiment and a pool of their own: the coarse search beside the file + // centre's schemes, and the re-index of each cell it offers beside the beam-centre check. The + // hypothesis then asks the same questions and RotationIndexer's memo, keyed on everything an + // indexer reads, answers them; where the run has moved in between (another centre or axis sign) + // the keys differ and it indexes as before. Nothing here is read but the memo. + struct LongAxisSpeculation { + std::unique_ptr experiment; + std::unique_ptr own_pool; // none where the run's pool is wide enough + SchemeIndexers ris; + std::future done; // last, so it is waited for before the rest goes + }; + std::vector long_axis_spec; + // The experiment the coarse search was started on. + std::vector long_axis_spec_key; + void StartLongAxisSearch(std::unique_ptr x, const IndexingSettings &settings, bool own_pool); + void JoinLongAxisSpeculation(); }; diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index 4dd8d9ea0..5d599e2fe 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -143,6 +143,99 @@ auto Rugnux::FirstPassRun::AxisSignRescue(FirstPass best) -> FirstPass { return best; } +// The parts of the long-axis hypothesis below, shared with the searches started ahead of it (long_axis_spec). +auto Rugnux::FirstPassRun::LongAxisBound() const -> LongAxisReach { + LongAxisReach r; + r.width_px = rx.spot_width_r80_px_.value_or(rx.experiment_.GetBraggIntegrationSettings().GetR1()); + r.resolvable_A = ResolvableAxisLength_A(rx.experiment_.GetDetectorDistance_mm(), rx.experiment_.GetWavelength_A(), + rx.experiment_.GetPixelSize_mm(), r.width_px); + r.bound_A = std::min(r.resolvable_A, IndexingSettings::fft_max_unit_cell_limit_A); + return r; +} + +IndexingSettings Rugnux::FirstPassRun::LongAxisCoarseSettings(float bound_A) const { + auto settings = rx.experiment_.GetIndexingSettings(); + settings.FFT_HighResolution_A(3.5f); + settings.FFT_MaxUnitCell_A(bound_A); + settings.IndexingThreads(rx.FirstPassIndexerThreads()); + settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); + return settings; +} + +IndexingSettings Rugnux::FirstPassRun::LongAxisReindexSettings(const DiffractionExperiment &x) const { + auto settings = x.GetIndexingSettings(); + settings.IndexingThreads(rx.FirstPassIndexerThreads()); + settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); + RaiseFFTBoundForKnownCell(settings, x); + return settings; +} + +std::vector Rugnux::FirstPassRun::LongAxisOffers(const SchemeIndexers &ris, const FirstPass &standing) { + std::vector offered; + for (auto &ri : ris) { + auto r = ri->GetLattice(); + if (!r.has_value() || (standing.result && SameLattice(*r, *standing.result)) + || std::ranges::any_of(offered, [&](const auto &o) { return SameLattice(*r, o); })) + continue; + offered.push_back(std::move(*r)); + } + std::vector cells; + for (const auto &o : offered) + cells.push_back(o.lattice.GetUnitCell()); + return cells; +} + +// One search of long_axis_spec, on its own copy of the experiment, in a pool of its own or in the run's. +void Rugnux::FirstPassRun::StartLongAxisSearch(std::unique_ptr x, + const IndexingSettings &settings, bool own_pool) { + auto &s = long_axis_spec.emplace_back(); + s.experiment = std::move(x); + if (own_pool) + s.own_pool = std::make_unique(settings, IndexerConstruction::OnFirstUse, + rx.gpu_work_beside_); + s.ris = FeedSchemes(own_pool ? *s.own_pool : *indexer_pool, s.experiment.get()); + std::vector ris; + for (const auto &ri : s.ris) + ris.push_back(ri.get()); + s.done = std::async(std::launch::async, [ris] { RunIndexing(ris); }); +} + +void Rugnux::FirstPassRun::StartLongAxisCoarse() { + if (rx.cancelled_ || rx.experiment_.GetUnitCell().has_value()) + return; + const LongAxisReach reach = LongAxisBound(); + if (!(reach.bound_A > indexing_bound_A)) + return; + long_axis_spec_key = rx.ExperimentKey(); + StartLongAxisSearch(std::make_unique(rx.experiment_), + LongAxisCoarseSettings(reach.bound_A), true); +} + +void Rugnux::FirstPassRun::StartLongAxisReindex(const FirstPass &standing) { + if (long_axis_spec.empty()) + return; + // A search that failed leaves no memo; the hypothesis then meets the failure itself. + long_axis_spec.front().done.wait(); + const bool same_run = long_axis_spec_key == rx.ExperimentKey(); + const std::vector offered = same_run ? LongAxisOffers(long_axis_spec.front().ris, standing) + : std::vector{}; + long_axis_spec.clear(); + for (const auto &cell : offered) { + if (rx.cancelled_) + break; + auto x = std::make_unique(rx.experiment_); + x->SetUnitCell(cell); + const IndexingSettings settings = LongAxisReindexSettings(*x); + StartLongAxisSearch(std::move(x), settings, settings.GetFFT_MaxUnitCell_A() > indexing_bound_A); + } +} + +void Rugnux::FirstPassRun::JoinLongAxisSpeculation() { + for (auto &s : long_axis_spec) + s.done.wait(); + long_axis_spec.clear(); +} + // Long-axis hypothesis. The standard search reaches fft_max_unit_cell_A (500 A), and DeAlias unfolds // rows up to twice that, so a longer axis is not a candidate at all: it comes back as a sub-multiple, // or as a folded row that unfolds to the wrong length, and the cell built on it puts only a minority @@ -168,14 +261,12 @@ auto Rugnux::FirstPassRun::AxisSignRescue(FirstPass best) -> FirstPass { // // Not when a cell was given: the standard pass was already sized for it (RaiseFFTBoundForKnownCell). auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { + JoinLongAxisSpeculation(); if (rx.cancelled_ || rx.experiment_.GetUnitCell().has_value()) return standing; const auto start = std::chrono::steady_clock::now(); - const float width_px = rx.spot_width_r80_px_.value_or(rx.experiment_.GetBraggIntegrationSettings().GetR1()); - const float resolvable_A = ResolvableAxisLength_A(rx.experiment_.GetDetectorDistance_mm(), - rx.experiment_.GetWavelength_A(), - rx.experiment_.GetPixelSize_mm(), width_px); - const float bound_A = std::min(resolvable_A, IndexingSettings::fft_max_unit_cell_limit_A); + const LongAxisReach reach = LongAxisBound(); + const float width_px = reach.width_px, resolvable_A = reach.resolvable_A, bound_A = reach.bound_A; if (!(bound_A > indexing_bound_A)) { logger.Info("Long-axis hypothesis: not asked - at {:.0f} mm, {:.4f} A and {:.0f} um pixels the rows of an " "axis are separated up to {:.0f} A for {:.2f} px spots, within the {:.0f} A the standard " @@ -183,30 +274,19 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { 1000.0f * rx.experiment_.GetPixelSize_mm(), resolvable_A, width_px, indexing_bound_A); return standing; } - - auto coarse_settings = rx.experiment_.GetIndexingSettings(); - coarse_settings.FFT_HighResolution_A(3.5f); - coarse_settings.FFT_MaxUnitCell_A(bound_A); - coarse_settings.IndexingThreads(rx.FirstPassIndexerThreads()); - coarse_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); - std::vector offered; + std::vector offered; { - IndexerThreadPool coarse_pool(coarse_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); + IndexerThreadPool coarse_pool(LongAxisCoarseSettings(bound_A), IndexerConstruction::OnFirstUse, + rx.gpu_work_beside_); const SchemeIndexers cris = FeedSchemes(coarse_pool); RunIndexing(cris); - for (auto &ri : cris) { - auto r = ri->GetLattice(); - if (!r.has_value() || (standing.result && SameLattice(*r, *standing.result)) - || std::ranges::any_of(offered, [&](const auto &o) { return SameLattice(*r, o); })) - continue; - offered.push_back(std::move(*r)); - } + offered = LongAxisOffers(cris, standing); } - const auto cell_text = [](const RotationIndexerResult &r) { - const auto uc = r.lattice.GetUnitCell(); + const auto uc_text = [](const UnitCell &uc) { return fmt::format("{:.1f} {:.1f} {:.1f} {:.1f} {:.1f} {:.1f}", uc.a, uc.b, uc.c, uc.alpha, uc.beta, uc.gamma); }; + const auto cell_text = [&](const RotationIndexerResult &r) { return uc_text(r.lattice.GetUnitCell()); }; const auto share = [](const PooledEvidence &e) { return fmt::format("{}/{} pooled spots ({} by chance)", e.on_lattice, e.spots, e.by_chance); }; @@ -222,11 +302,8 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { // The re-index must be able to REPRESENT the cell the coarse pass offered; the standard pool was // sized at the default bound. CountIndexed forces the lattice and never touches the pool, so the // run's IndexAndRefine is reused and only the FFT needs the wider one. - rx.experiment_.SetUnitCell(coarse.lattice.GetUnitCell()); - auto wide_settings = rx.experiment_.GetIndexingSettings(); - wide_settings.IndexingThreads(rx.FirstPassIndexerThreads()); - wide_settings.RefineThreads(std::clamp(rx.config_.nthreads / 2, 1, MAX_REFINE_THREADS)); - RaiseFFTBoundForKnownCell(wide_settings, rx.experiment_); + rx.experiment_.SetUnitCell(coarse); + const auto wide_settings = LongAxisReindexSettings(rx.experiment_); FirstPass c; if (wide_settings.GetFFT_MaxUnitCell_A() > indexing_bound_A) { IndexerThreadPool wide_pool(wide_settings, IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); @@ -235,7 +312,7 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { c = PickBest(*indexer_pool, *indexer); } rx.experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo - counts += fmt::format("; coarse {}", cell_text(coarse)); + counts += fmt::format("; coarse {}", uc_text(coarse)); if (!c.result.has_value()) { counts += " re-indexes to nothing"; continue; From f51cc35c86da5455f34ac97d00f848a50d83dd4b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 22:39:52 +0100 Subject: [PATCH 4/4] First pass: ask the long-axis hypothesis once per run An axis long enough for the hypothesis is rare, and whether a crystal has one does not change between the passes of a run, yet every first pass - the refined-geometry pass, the probes, the distance-check passes - ran the coarse search again (thau: 0.36 s in Step 6 and 0.06-0.70 s in each later pass, offering nothing in any of them). The first pass that asks now leaves its answer on the run (Rugnux::long_axis_answer_, reset per RunAllPasses): a later pass keeps its standing cell where that pass kept one, and re-indexes straight at the cell it adopted otherwise, without the coarse search. Only the long-axis path changes; within a pass both centres of the beam-centre check are still asked. Rotation first pass, -N 28, 5950X + RTX 5080, machine shared with other jobs (median of 2, s): Step 3 / Step 6 / wall, after the speculation commit thau 1000 images: 1.65 / 0.84 / 15.0 -> 1.58 / 0.67 / 13.9 cytc: 1.50 / / 20.7 -> 1.35 / / 19.5 lyso: 1.40 / 0.65 / 12.7 -> 1.30 / 0.48 / 11.5 _report.txt (minus run-specific lines) and .hkl identical to rc184-port on these three and on a 13-set battery subset (lyso_x06da_ref, thau_x10sa_16keV, thau_x10sa_injection, insu_I_x06da_ref, cytc_x10sa, 6toc, 4bwl, 5vml, kdp_x10sa_20keV, and the long-axis sets 8v4o, 7d98, 4mtk, 4p8r): battery compare 0 changed, every report and .hkl equal. On a crystal with a ~1000 A axis (private) the run ends on the same cell, space group and resolution; its distance-check passes now re-index at the carried cell instead of searching, which moves its merging statistics in the fourth digit. Wall 70 -> 60 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- docs/CPU_DATA_ANALYSIS_INDEXING.md | 2 +- rugnux/Rugnux.h | 7 +++++ rugnux/RugnuxFirstPass.cpp | 3 +++ rugnux/RugnuxFirstPass.h | 5 ++++ rugnux/RugnuxFirstPassRescues.cpp | 43 ++++++++++++++++++++++-------- rugnux/RugnuxPasses.cpp | 1 + 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS_INDEXING.md b/docs/CPU_DATA_ANALYSIS_INDEXING.md index 4e482dabc..c7a48a0c2 100644 --- a/docs/CPU_DATA_ANALYSIS_INDEXING.md +++ b/docs/CPU_DATA_ANALYSIS_INDEXING.md @@ -121,7 +121,7 @@ For each direction, the FFT magnitude spectrum is computed; peaks correspond to The running-mean background window keeps a **constant width** and is slid inward at the ends of the spectrum rather than truncated there, so a peak within half a window of either end — which is where the longest cells sit — is judged against as much background as any other. Both window bounds stay monotonically non-decreasing in the bin index, so the GPU kernel's running sum is still valid. -The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames. Its searches are started ahead of the question - the coarse one beside the file centre's schemes, the re-index of what it offers beside the beam-centre check - and the question then reads their answers from the indexer's memo: this changes when they run, not what they find. +The longest basis vector the transform can return is `fft_max_unit_cell_A`, since the histogram is sized from it and its last usable bin *is* that length; the shortest is `fft_min_unit_cell_A` (`rugnux --fft-min-unit-cell`, default 10 Å), below which a candidate is discarded. The defaults are unchanged (500 Å and 10 Å), but the accepted range for the maximum now reaches 1200 Å, and a reference cell given with `-C` moves **both** bounds on its own — up to reach a long axis, down to admit a small-molecule cell — since a cell the search cannot represent cannot be found by it. A de-novo rotation run asks the long end a second time as a **long-axis hypothesis**: the detector separates the rows of an axis of length $L$ while their spacing near the beam, $D\lambda/L$, exceeds the measured spot width, so wherever $D\lambda/(p\,r_{80})$ ($p$ the pixel size, $r_{80}$ the pre-scan's spot width) exceeds the standard bound, a coarse search (3.5 Å histogram) to that length, at most 1200 Å, offers its lattices. Each is re-indexed at full resolution with its cell as the reference, and the one putting the largest share of the pooled validation spots on itself beyond chance replaces the standing lattice where that share is significantly larger (the pooled-spot comparison the first pass uses for every competing lattice) — except an exact superlattice of the standing cell, for which the pooled count is biased and which is left to the supercell probe of the merge. It is asked of every such run, whatever the standing pass indexed: a wrong axis along a fine row still indexes most frames. It is asked **once per run**, by the first pass that reaches it: a later pass (at the refined geometry, or a probe) carries that answer, keeping its standing cell where the first one kept its own and otherwise re-indexing straight at the long-axis cell the first one adopted, without the coarse search. The searches themselves are started ahead of the question - the coarse one beside the file centre's schemes, the re-index of what it offers beside the beam-centre check - and the question then reads their answers from the indexer's memo: this changes when they run, not what they find. Candidate vectors are $\mathbf{v}_d = L_d\,\mathbf{u}_d$. diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index b062da167..ff0c84c5d 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -1552,6 +1552,13 @@ class Rugnux { // keeps the smaller), Niggli-reduced primitive: the cells an indexing scheme found as lattices in their own // right, which DoubledCellHypothesis reads the added class of. std::vector harmonic_offers_; + // The long-axis hypothesis's answer, from the first pass of this run that asked it: the cell it + // adopted, or none where it kept the standing cell. A later pass carries it instead of asking again + // (FirstPassRun::LongAxisHypothesis). + struct LongAxisAnswer { + std::optional adopted; + }; + std::optional long_axis_answer_; // The first pass's spot lists, kept across passes under everything a frame's list depends on // (SpotFindingKey): a probe and the canonical pass after it, or a pass repeating an earlier // pass's rescue, find the spots of the same frames under the same settings again. Shared with the diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 7816abe10..f63d8eb7a 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -490,6 +490,9 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { // The long-axis hypothesis, at the centre the run now stands on. if (!long_axis_asked) best = run.LongAxisHypothesis(std::move(best)); + // What later passes of the run carry (Rugnux::long_axis_answer_). + if (run.long_axis_asked_here || run.long_axis_adopted) + long_axis_answer_ = LongAxisAnswer{run.long_axis_adopted}; // The end wedge, at the geometry the rescues settled on. best = run.AddEndWedge(std::move(best), *p.indexer_pool, *p.indexer); run.ReleaseSpotEngines(); // no more frames to read; give the cards their memory back diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index 2e4ddfa2d..5dcf13326 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -363,4 +363,9 @@ struct Rugnux::FirstPassRun { std::vector long_axis_spec_key; void StartLongAxisSearch(std::unique_ptr x, const IndexingSettings &settings, bool own_pool); void JoinLongAxisSpeculation(); + + // Whether this pass asked the long-axis question itself, and the cell it adopted (see + // Rugnux::long_axis_answer_). + bool long_axis_asked_here = false; + std::optional long_axis_adopted; }; diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index 5d599e2fe..6e89db80d 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -201,7 +201,7 @@ void Rugnux::FirstPassRun::StartLongAxisSearch(std::unique_ptr indexing_bound_A)) @@ -274,8 +274,20 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { 1000.0f * rx.experiment_.GetPixelSize_mm(), resolvable_A, width_px, indexing_bound_A); return standing; } + // Asked once per run. An axis this long is rare, and whether the crystal has one does not change + // between the passes of a run, so a later pass carries the first answer: it keeps its standing cell + // where that pass kept one, and re-indexes at the cell it adopted otherwise - the coarse search, + // which is what finds the axis, is not run again. std::vector offered; - { + if (rx.long_axis_answer_) { + if (!rx.long_axis_answer_->adopted) { + logger.Info("Long-axis hypothesis: not asked again - an earlier pass of this run asked it and kept " + "the standing cell"); + return standing; + } + offered.push_back(*rx.long_axis_answer_->adopted); + } else { + long_axis_asked_here = true; IndexerThreadPool coarse_pool(LongAxisCoarseSettings(bound_A), IndexerConstruction::OnFirstUse, rx.gpu_work_beside_); const SchemeIndexers cris = FeedSchemes(coarse_pool); @@ -312,7 +324,7 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { c = PickBest(*indexer_pool, *indexer); } rx.experiment_.SetUnitCell(std::nullopt); // leave the space-group / cell determination de-novo - counts += fmt::format("; coarse {}", uc_text(coarse)); + counts += fmt::format("; {} {}", rx.long_axis_answer_ ? "carried" : "coarse", uc_text(coarse)); if (!c.result.has_value()) { counts += " re-indexes to nothing"; continue; @@ -351,15 +363,24 @@ auto Rugnux::FirstPassRun::LongAxisHypothesis(FirstPass standing) -> FirstPass { choice.reset(); } const double seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); - logger.Info("Long-axis hypothesis: rows separated up to {:.0f} A ({:.2f} px spots), so the coarse search reached " - "{:.0f} A and offered {} lattice(s) other than the standing one{}; the standing cell puts {} on " - "itself - {}; {:.2f} s", - resolvable_A, width_px, bound_A, offered.size(), counts, - standing.result ? share(at_standing) : std::string("no lattice"), - choice ? "adopting " + cell_text(*constrained[*choice].result) : std::string("keeping the standing cell"), - seconds); - if (choice) + if (rx.long_axis_answer_) + logger.Info("Long-axis hypothesis: an earlier pass of this run adopted a long-axis cell, re-indexed here{}; " + "the standing cell puts {} on itself - {}; {:.2f} s", counts, + standing.result ? share(at_standing) : std::string("no lattice"), + choice ? "adopting " + cell_text(*constrained[*choice].result) : std::string("keeping the standing cell"), + seconds); + else + logger.Info("Long-axis hypothesis: rows separated up to {:.0f} A ({:.2f} px spots), so the coarse search reached " + "{:.0f} A and offered {} lattice(s) other than the standing one{}; the standing cell puts {} on " + "itself - {}; {:.2f} s", + resolvable_A, width_px, bound_A, offered.size(), counts, + standing.result ? share(at_standing) : std::string("no lattice"), + choice ? "adopting " + cell_text(*constrained[*choice].result) : std::string("keeping the standing cell"), + seconds); + if (choice) { + long_axis_adopted = constrained[*choice].result->lattice.GetUnitCell(); return std::move(constrained[*choice]); + } return standing; } diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 383a2b5b2..3737a397b 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -365,6 +365,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // only while this holds one; a second Run() on the same object must not inherit the last one's. quality_guard_pass1_.reset(); harmonic_offers_.clear(); + long_axis_answer_.reset(); // Rotation two-pass geometry post-refinement: the first pass integrates at the header geometry and // post-refines the detector geometry (distance + beam from the observed spot positions, cell scale + axis // from phi_obs); the second pass re-indexes and re-integrates with the refined geometry. Only the second