From 1eda55ab38461aa5ce7f4d735670f11531dc26f0 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 10 Oct 2026 23:49:54 +0200 Subject: [PATCH 1/2] rugnux: the rotation-scale walk's and the geometry walk's indexing probes run on copies of the run The rotation-scale walk ran its probe passes on the run itself and put back a list of members afterwards (experiment, geometry, pre-pass mosaicity, scale). The list missed what an integrating refit probe leaves - bkg_starved_fraction_, last_rotation_result_ - and the spot-finding settings a first-pass ladder rung adopts were carried on purpose, with a check that threw the refined passes away when the walk had changed them. Each probe now runs on a copy of the run as the walk found it, and only its first-pass memos (an exact cache) come back; the run ends the walk as it began it, the restore lines and the spot-finding check go, and the refined passes are overturned only where a scale is adopted. The geometry walk's index_at_geometry probe runs on a copy the same way, so a probe the speculative one already answered (memo hit) and one run there (miss) leave the run identical. Measured before the change with a temporary state dump at the start of the refined passes, walk run against a run that skips the walk and applies the walk's k directly (8r5r, 9gdj, 9qw8, 2ygz adopt k != 1; 9t6s, lalanine walk first and keep k = 1): p.hkl identical on all six; the run state differed only in last_rotation_result_, bkg_starved_fraction_ and the memo count - leaks the canonical pass overwrote before reading, so latent. After (GPU, gpulock, shared machine): p.hkl and MTZ reflection data identical to the base binary on 26 ci-tier sets with a merge (+ both no-crystal controls, same verdict), on the 11 open sets whose walk adopts k != 1 or runs first, on 7 sets whose geometry walk takes a sub-step round, and on the one private set that adopts a scale. Catch2 [RotationScale] and Rugnux_FirstPassMemoMatchesRecomputation (verify_first_pass_memo through the walk) pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- rugnux/Rugnux.cpp | 2 +- rugnux/Rugnux.h | 9 ++-- rugnux/RugnuxPasses.cpp | 105 ++++++++++++++++------------------------ 3 files changed, 49 insertions(+), 67 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 9fdd59ca9..f67aff928 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -413,7 +413,7 @@ void Rugnux::AddFirstPassMemo(const FirstPassMemo &memo) { } void Rugnux::StartSpeculativeGeometryProbe(std::shared_ptr run, const std::array &geometry) { - // What the walk's index_at_geometry sets up on the run before it indexes at this geometry. + // What the walk's index_at_geometry sets up on its copy of the run before it indexes at this geometry. run->prepass_detector_geometry_ = prepass_detector_geometry_; run->speculative_probe_.reset(); run->speculate_geometry_probe_ = false; diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 26f61c77c..0bb4d5da6 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -1072,7 +1072,8 @@ class Rugnux { ProcessResult RunAllPasses(RugnuxObserver *observer); // The rotation-scale walk of the rotation two-pass and the refined passes after it run side by side - // (see RunAllPasses): the walk on the run, the refined passes on a copy of it, at the stored angles. + // (see RunAllPasses): the refined passes on a copy of the run, at the stored angles, and the walk + // beside them, its probes on copies of their own. // What the two share. struct RotationScaleCheck { std::mutex m; @@ -1081,7 +1082,7 @@ class Rugnux { bool output_ended = false; // the refined passes have returned bool called_off = false; // the walk has called them off bool done = false; // the walk has ended - bool overturns = false; // the refined passes did not start where the walk leaves the run + bool overturns = false; // the walk adopted a scale, so the refined passes run again int passes = 0; // the probe passes the walk ran std::string text; // its trail, for the pass decision void Set(bool &flag) { @@ -1121,8 +1122,8 @@ class Rugnux { const std::string &base_prefix); std::optional RefinedPasses(RugnuxObserver *observer, ProcessResult pass1, const RefinedPassStart &start); - // The rotation-scale walk, run on the run while the refined passes run on a copy of it; `check` is - // what the two share, and output_cancel calls the copy off. + // The rotation-scale walk, its probes on copies of the run, while the refined passes run on a copy + // of their own; `check` is what the two share, and output_cancel calls that copy off. RotationScaleWalk WalkRotationScaleBeside(const ProcessResult &pass1, RotationScaleCheck &check, CancelFlag output_cancel); diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 4c10582ec..a1f73d2ec 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -565,15 +565,13 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // geometry the second pass will, so every hypothesis, the stored angles included, is scored // there, and at that geometry only. Its passes // only measure - an indexing probe stops once the lattice is scored, a refit stops once the - // post-refinement has measured - and they leave nothing behind: the experiment, the geometry - // the second pass is to run at and pass 1's mosaicity (a width in degrees fitted against the - // stored angles) are put back when the walk ends. + // post-refinement has measured - and each runs on a copy of the run, so they leave nothing + // behind: the run ends the walk as it began it, and the adopted scale is all it takes from it. // // The walk runs BESIDE the output pass, not before it. It adopts a scale other than the stored // one only rarely, so the refined passes start at once at the stored angles, on a copy of the - // run as the walk finds it, while the walk runs on the run itself. Where the walk adopts a - // scale, or leaves the run changed behind it, the copy's passes are thrown away and run again on - // the run itself - which the walk has then left exactly where the sequential walk would have. + // run as the walk finds it. Where the walk adopts a scale, the copy's passes are thrown away and + // run again on the run itself, at that scale. // The walk starts once the output pass has indexed: its probe at the stored angles is that same // first pass, which it then finds done, and its probe at the fit finds no frame's spots again // (SpotFindingKey). The output pass waits for the walk only where it would act on the answer - @@ -632,9 +630,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { "integrates at the corrected angles; the fault is in the hardware and should " "be fixed there", walk.scale, 100.0 * (walk.scale - 1.0), walk.scale > 1.0f ? "further" : "less far"); - } else if (check->overturns) - logger.Info("Two-pass: the rotation-scale walk changed the spot finding the refined passes " - "start from, so they run again after it"); + } } return DoubledCellHypothesis(observer, *RefinedPasses(observer, std::move(pass1), start), base_prefix); } @@ -643,7 +639,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // The rotation-scale walk of RunAllPasses, run beside the refined passes on a copy of the run (`check` // is what the two share): it starts once their first pass has indexed, calls them off (output_cancel) -// before a refit, and says at its end whether they started where it leaves the run. +// before a refit, and says at its end whether it overturns them. RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, RotationScaleCheck &check, CancelFlag output_cancel) { Logger logger("Rugnux"); @@ -653,66 +649,54 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro }; try { check.Wait(check.output_indexed); - const DiffractionExperiment before_walk = experiment_; - const auto geometry_before_walk = prepass_detector_geometry_; - const auto mosaicity_before_walk = prepass_mosaicity_; int probe_passes = 0; - // A probe pass at scale k, run on r - this run, or a copy of it (see below). - const auto probe_on = [&](Rugnux &r, float k, bool index_only) { - r.experiment_ = before_walk; - r.experiment_.Goniometer(ScaleRotation(*before_walk.GetGoniometer(), k)); - r.prepass_rotation_scale_ = k; - r.prepass_mosaicity_.clear(); - r.indexing_probe_only_ = index_only; - r.postrefine_probe_ = r.postrefine_probe_only_ = !index_only; + // Every probe runs on a copy of the run as the walk found it, so a probe leaves nothing on the + // run - not the spot-finding settings a first-pass ladder rung adopts, not the reflection + // footprint or rotation result an integrating refit leaves - and the run ends the walk exactly + // as it began it, whatever scale is adopted. Only its first-pass memos are kept: they are a + // record of what the first pass finds on given inputs, which a later probe on the same inputs + // then reads instead of indexing again. + const auto probe_on = [this, &logger](std::shared_ptr r, float k, bool index_only) { + r->experiment_.Goniometer(ScaleRotation(*experiment_.GetGoniometer(), k)); + r->prepass_rotation_scale_ = k; + r->prepass_mosaicity_.clear(); + r->indexing_probe_only_ = index_only; + r->postrefine_probe_ = r->postrefine_probe_only_ = !index_only; ProcessResult res; try { - res = r.RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); + res = r->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); } catch (const std::exception &e) { if (IsFatalResourceError(e)) throw; // Angles under which nothing indexes have scored nothing, which is what an // empty result reads as. logger.Info("Rotation scale {:.5f}: the probe pass did not complete ({})", k, e.what()); } - r.indexing_probe_only_ = r.postrefine_probe_ = r.postrefine_probe_only_ = false; return res; }; const auto probe = [&](float k, bool index_only) { ++probe_passes; - return probe_on(*this, k, index_only); + const auto r = std::make_shared(*this); + const ProcessResult res = probe_on(r, k, index_only); + for (const auto &m : r->first_pass_memo_) + AddFirstPassMemo(m); + return res; }; // The walk always indexes at the stored angles first and at the pass-1 fit next (see - // WalkRotationScale). Both are known now and each is its own pass, so the one at the fit - // runs at the same time, on a copy of the run as it stands before either. What a probe - // leaves behind for the next pass - the spot-finding settings a first-pass ladder rung - // adopts; the experiment is put back by the probe itself - is checked when the fit's turn - // comes: if the probe at the stored angles changed it, the copy started from a state the - // sequential walk would not have, and the probe is run again here. The same check, at the - // end, says whether the refined passes beside the walk started where it leaves the run. + // WalkRotationScale). Both are known now, so the one at the fit runs at the same time. const float first_fit = static_cast(pass1.post_refine->rotation_scale); std::shared_ptr fit_copy; std::future at_fit; - const SpotFindingSettings walk_start_spot_finding = config_.spot_finding; if (first_fit != 1.0f && !cancelled_) { fit_copy = std::make_shared(*this); - at_fit = std::async(std::launch::async, [&probe_on, fit_copy, first_fit] { - return probe_on(*fit_copy, first_fit, true); - }); + at_fit = std::async(std::launch::async, probe_on, fit_copy, first_fit, true); } - const auto same_spot_finding = [&] { - return config_.spot_finding.high_resolution_limit == walk_start_spot_finding.high_resolution_limit - && config_.spot_finding.measured_ring_q_recipA == walk_start_spot_finding.measured_ring_q_recipA; - }; const auto index_at = [&](float k) { if (at_fit.valid() && k == first_fit) { - ProcessResult r = at_fit.get(); - if (same_spot_finding()) { - ++probe_passes; - config_.spot_finding = fit_copy->config_.spot_finding; - for (const auto &m : fit_copy->first_pass_memo_) - AddFirstPassMemo(m); - return r.validation_evidence; - } + ++probe_passes; + const ProcessResult res = at_fit.get(); + for (const auto &m : fit_copy->first_pass_memo_) + AddFirstPassMemo(m); + return res.validation_evidence; } return probe(k, true).validation_evidence; }; @@ -732,10 +716,6 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro constexpr int MAX_ROTATION_SCALE_ROUNDS = 8; const RotationScaleWalk walk = WalkRotationScale(pass1.post_refine->rotation_scale, index_at, refit_at, MAX_ROTATION_SCALE_ROUNDS); - experiment_ = before_walk; - prepass_detector_geometry_ = geometry_before_walk; - prepass_mosaicity_ = mosaicity_before_walk; - prepass_rotation_scale_.reset(); std::string text; if (!walk.trail.empty()) { text = fmt::format( @@ -747,7 +727,7 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro } { std::lock_guard lock(check.m); - check.overturns = check.called_off || walk.scale != 1.0f || !same_spot_finding(); + check.overturns = check.called_off || walk.scale != 1.0f; check.passes = probe_passes; check.text = text; check.done = true; @@ -1090,25 +1070,26 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro / static_cast(std::max(1, e.spots)); }; // An indexing probe at a geometry: the validation evidence of the lattice indexed there, and - // nothing else - the pass stops once the lattice is scored, and the experiment is put back. + // nothing else - the pass stops once the lattice is scored. It runs on a copy of the run, as the + // rotation-scale walk's probes do, so it leaves nothing on the run but its first-pass memo: a + // probe the speculative one has already answered and one run here leave the run the same. const auto index_at_geometry = [&](const std::array &g) { - const DiffractionExperiment before_probe = experiment_; - const bool probe_was = postrefine_probe_; - set_geometry(g); CollectSpeculativeProbe(); - indexing_probe_only_ = true; - postrefine_probe_ = false; + Rugnux probe(*this); + probe.experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2]) + .PoniRot1_rad(g[3]).PoniRot2_rad(g[4]); + probe.indexing_probe_only_ = true; + probe.postrefine_probe_ = false; ProcessResult r; try { - r = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + r = probe.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); } catch (const std::exception &e) { if (IsFatalResourceError(e)) throw; logger.Info("Two-pass: the indexing probe at distance {:.3f} mm did not complete ({})", g[2], e.what()); } - indexing_probe_only_ = false; - postrefine_probe_ = probe_was; - experiment_ = before_probe; + for (const auto &m : probe.first_pass_memo_) + AddFirstPassMemo(m); ++arm_passes; return r.validation_evidence; }; From de69e6d0d77f6db9377d254beced9825a4e55891 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 10 Oct 2026 23:56:15 +0200 Subject: [PATCH 2/2] rugnux: the metric-symmetry check's demoted arm runs on a copy; the canonical pass no longer starts from its spot budget The demoted arm is a probe pass, run on the run itself before the canonical pass (the promoted arm). Its first pass measures a spot budget and writes it onto the experiment, and nothing put it back, so the canonical pass started from the probe's budget instead of the full spot list - measured, 446 spots per image instead of 1000 on one in-house crystal (validation spots 22055 instead of 43582) and 58 instead of 1000 on 6yqf. The same held where the arm lost: the demoted canonical pass started from the budget the stopped promoted pass measured. The demoted arm now runs on a copy of the run and gives back only the geometry its post-refinement asks for (which the promoted arm reads, as before) and its first-pass memo; where it wins, the demoted canonical pass starts from the experiment the promoted one started from. The indexing-settings and flag juggling around the probe goes. GPU, gpulock, shared machine, on the 22 sets whose baseline runs the check: p.hkl and MTZ reflection data identical to the base binary on 20; lcystine_x10sa_20keV fails with a cuFFT out-of-memory on the shared card with both binaries (four attempts). Changed: 6yqf only (its canonical first pass now reads the full spot list): merge 3.23 -> 3.15 A, R_meas 0.654 -> 0.568, ISa 4.97 -> 5.36; rugnux model R_free 0.459 -> 0.504 at its own d_min, phenix.model_vs_data R_free at the common 3.23 A 0.443 -> 0.435 (0.443 -> 0.447 at full resolution). The check's decision is unchanged on all 22. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- rugnux/RugnuxPasses.cpp | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index a1f73d2ec..212851a93 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -912,21 +912,22 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // The demoted arm runs first, as a probe: it is compared on the post-refinement's held-out // residual, so it has to measure one, and a canonical pass only does where the pre-pass // moved the geometry. The canonical pass below is the promoted arm (lattice_arm_check_). - const bool probe_was = postrefine_probe_; - postrefine_probe_ = true; - postrefine_probe_only_ = true; + // The probe runs on a copy of the run, so it leaves nothing on the run - not the spot + // budget its first pass measures, which the canonical pass would otherwise start from - + // except the geometry its post-refinement asks for, which the promoted arm reads (see + // ScaleMergeAndSymmetry), and its first-pass memo. + Rugnux demoted_run(*this); + demoted_run.postrefine_probe_ = true; + demoted_run.postrefine_probe_only_ = true; IndexingSettings demoted_settings = experiment_.GetIndexingSettings(); demoted_settings.LatticeClassDemoted(length_arm); - experiment_.ImportIndexingSettings(demoted_settings); - lattice_angles_demoted_ = angle_arm; - demoted_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + demoted_run.experiment_.ImportIndexingSettings(demoted_settings); + demoted_run.lattice_angles_demoted_ = angle_arm; + demoted_arm = demoted_run.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); ++arm_passes; - postrefine_probe_only_ = false; - postrefine_probe_ = probe_was; - IndexingSettings promoted_settings = experiment_.GetIndexingSettings(); - promoted_settings.LatticeClassDemoted(false); - experiment_.ImportIndexingSettings(promoted_settings); - lattice_angles_demoted_ = false; + prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; + for (const auto &m : demoted_run.first_pass_memo_) + AddFirstPassMemo(m); lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; } else { logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, " @@ -942,6 +943,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro base_prefix); speculate_geometry_probe_ = true; // a walk that follows starts with an indexing probe rerun_if_starved_ = true; + const DiffractionExperiment before_pass2 = experiment_; auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); if (lattice_arm_check_) { // The metric-symmetry check above, decided inside the pass just run (the promoted arm). @@ -970,8 +972,10 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro length_arm ? "axes" : "an angle", length_arm ? 100.0 * split : split * 180.0 / PI, length_arm ? "% apart" : "deg from 90", demoted_class, realised_demoted, realised_promoted); - // The promoted arm stopped before its merge; the demoted class is the canonical pass. + // The promoted arm stopped before its merge; the demoted class is the canonical pass, and + // it starts where the promoted one did, not from the spot budget that one measured. ++arm_passes; + experiment_ = before_pass2; IndexingSettings demoted_settings = experiment_.GetIndexingSettings(); demoted_settings.LatticeClassDemoted(length_arm); experiment_.ImportIndexingSettings(demoted_settings);