Rugnux: reuse the canonical pass's first-pass evidence for a probe on the same inputs

The geometry walk after the canonical pass starts with an indexing probe at
the geometry in hand (probe_at_best), which is the canonical pass's own first
pass run over again - a full first-pass indexing whose answer is already
known (identical validation-spot counts in the logs).

The first pass is deterministic in its inputs, so each canonical or probe
first pass now leaves its validation evidence behind keyed by what it read
(geometry, goniometer, spot budget, cell, indexing and spot-finding settings,
and the run state it consults), and an indexing-only probe with the same key
takes it. Nothing is stored for the geometry pre-pass, for a pass that
changed its own inputs on the way (a rescue adopted), or for one that ran the
beam-centre ladder, which a probe never runs. RUGNUX_VERIFY_FIRST_PASS_MEMO
runs the probe anyway and fails the run on any difference.

md5-identical output; 37.2 -> 34.0 s on a 16M rotation set with a geometry
walk (one probe fewer).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C

rugnux/Rugnux.cpp

@@ -2966,6 +2966,43 @@ namespace {
     }
 }
 
+std::vector<double> Rugnux::FirstPassInputKey() const {
+    std::vector<double> k{experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(),
+                          experiment_.GetDetectorDistance_mm(), experiment_.GetPoniRot1_rad(),
+                          experiment_.GetPoniRot2_rad(), experiment_.GetWavelength_A(),
+                          static_cast<double>(experiment_.GetMaxSpotCount())};
+    if (const auto g = experiment_.GetGoniometer()) {
+        const Coord a = g->GetAxis();
+        const auto helical = g->GetHelicalStep().value_or(Coord(-1, -1, -1));
+        k.insert(k.end(), {1.0, a.x, a.y, a.z, g->GetStart_deg(), g->GetIncrement_deg(), g->GetWedge_deg(),
+                           g->GetScreeningWedge().value_or(-1.0f), helical.x, helical.y, helical.z});
+    } else
+        k.push_back(0.0);
+    if (const auto uc = experiment_.GetUnitCell())
+        k.insert(k.end(), {1.0, uc->a, uc->b, uc->c, uc->alpha, uc->beta, uc->gamma});
+    else
+        k.push_back(0.0);
+    const auto &x = experiment_.GetIndexingSettings();
+    k.insert(k.end(), {x.GetTolerance(), x.GetFFT_MinUnitCell_A(), x.GetFFT_MaxUnitCell_A(),
+                       static_cast<double>(x.GetFFT_NumVectors()), x.GetFFT_HighResolution_A(),
+                       x.GetFFT_MinAngle_deg(), x.GetFFT_MaxAngle_deg(), x.GetUnitCellDistTolerance(),
+                       x.GetUnitCellAngleTolerance_deg(), static_cast<double>(x.GetIndexIceRings()),
+                       x.GetRotationIndexingMinAngularRange_deg(), x.GetRotationIndexingAngularStride_deg(),
+                       static_cast<double>(x.GetMaxExtraLattices()), static_cast<double>(x.GetLatticeClassDemoted()),
+                       static_cast<double>(x.GetViableCellMinSpots()), static_cast<double>(x.GetAlgorithm()),
+                       static_cast<double>(x.GetGeomRefinementAlgorithm())});
+    const auto &f = config_.spot_finding;
+    k.insert(k.end(), {f.signal_to_noise_threshold, static_cast<double>(f.photon_count_threshold),
+                       static_cast<double>(f.min_pix_per_spot.value_or(-1)), static_cast<double>(f.max_pix_per_spot),
+                       f.high_resolution_limit.value_or(-1.0f)});
+    // Run state the first pass reads besides the experiment.
+    k.insert(k.end(), {static_cast<double>(lattice_angles_demoted_), prepass_rotation_scale_.value_or(-1.0f),
+                       static_cast<double>(force_rotation_result_.has_value()),
+                       static_cast<double>(background_center_.has_value()),
+                       static_cast<double>(beam_center_alternative_state_.has_value())});
+    return k;
+}
+
 ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, bool geometry_prepass) {
     Logger logger("Rugnux");
     ProcessResult result;
@@ -3230,6 +3267,22 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
         indexer->ForceRotationIndexerLattice(*config_.forced_rotation_lattice);
         logger.Info("Rotation indexer lattice forced externally - skipping first pass");
     } else if (full && config_.rotation_indexing && config_.two_pass_rotation) {
+        // An indexing probe over inputs a first pass has already been run on (first_pass_memo_).
+        const std::vector<double> first_pass_key = FirstPassInputKey();
+        const bool verify_memo = std::getenv("RUGNUX_VERIFY_FIRST_PASS_MEMO") != nullptr;
+        std::optional<ValidationSpotEvidence> memo_hit;
+        if (indexing_probe_only_ && first_pass_memo_ && first_pass_memo_->key == first_pass_key) {
+            memo_hit = first_pass_memo_->evidence;
+            if (!verify_memo) {
+                logger.Info("Indexing probe: the first pass has already run on these inputs - {}/{} "
+                            "validation spots on the lattice, {} at a wrong spindle angle",
+                            memo_hit->on_lattice, memo_hit->spots, memo_hit->by_chance);
+                result.validation_evidence = *memo_hit;
+                return result;
+            }
+        }
+        bool beam_center_ladder_ran = false;
+
         if (observer)
             observer->OnPhase("Rotation indexing (first pass)");
 
@@ -4520,6 +4573,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
         if (!cancelled_ && !beam_center_searched_ && config_.beam_center_search_pxl > 0
             && best.score < 0.5 * static_cast<double>(validation.size())) {
             beam_center_searched_ = true;
+            beam_center_ladder_ran = true;
             const float beam_x = experiment_.GetBeamX_pxl(), beam_y = experiment_.GetBeamY_pxl();
             const int reach = config_.beam_center_search_pxl;
             // The background centre is measured off the whole projection rather than off a lattice, so
@@ -4963,6 +5017,18 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
             const double pooled_chance = static_cast<double>(evidence.by_chance)
                                          / static_cast<double>(std::max<int64_t>(1, evidence.spots));
             result.validation_evidence = evidence;
+            if (memo_hit && (memo_hit->spots != evidence.spots || memo_hit->on_lattice != evidence.on_lattice
+                             || memo_hit->by_chance != evidence.by_chance))
+                throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                                      fmt::format("First-pass memo mismatch: stored {}/{}/{}, recomputed {}/{}/{}",
+                                                  memo_hit->on_lattice, memo_hit->spots, memo_hit->by_chance,
+                                                  evidence.on_lattice, evidence.spots, evidence.by_chance));
+            // Kept only where the pass ended on the inputs it started from, and did not take the
+            // beam-centre ladder, which a probe (beam_center_searched_) never would.
+            if (!geometry_prepass && !beam_center_ladder_ran && FirstPassInputKey() == first_pass_key)
+                first_pass_memo_ = FirstPassMemo{first_pass_key, evidence};
+            else
+                first_pass_memo_.reset();
             // A pass run only to score a lattice has its score, whatever it is: a lattice that does not
             // beat chance is a result for the comparison that asked, not a reason to stop the run.
             if (indexing_probe_only_) {

rugnux/Rugnux.h

@@ -706,6 +706,17 @@ class Rugnux {
     // lattice on the validation frames (ProcessResult::validation_evidence), before a single image is
     // integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score.
     bool indexing_probe_only_ = false;
+    // The validation evidence of the last canonical-pass or probe first pass, keyed by the inputs that
+    // pass read (FirstPassInputKey). The first pass is deterministic in them, so an indexing probe
+    // whose key matches takes the stored evidence instead of indexing again - the geometry walk's
+    // probe at the geometry in hand is the canonical pass's own first pass run over. With
+    // RUGNUX_VERIFY_FIRST_PASS_MEMO set the probe runs anyway and the run fails if the two differ.
+    struct FirstPassMemo {
+        std::vector<double> key;
+        ValidationSpotEvidence evidence;
+    };
+    std::optional<FirstPassMemo> first_pass_memo_;
+    [[nodiscard]] std::vector<double> FirstPassInputKey() const;
     // The file's detector distance, from before the first pass of the rotation two-pass: a pass whose
     // distance is still this one asks the post-refinement to test "the header distance is right"
     // (PostRefineSettings::distance_at_header); a pass that has walked off it does not.