From af88d62201667bba1c360dda74fc032b5a08053c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 11 Oct 2026 06:46:59 +0200 Subject: [PATCH 1/2] rugnux metric-symmetry check: no demoted arm where it indexes the same lattice The demoted arm of the metric-symmetry check integrated and post-refined the whole sweep even where, released from the equality, its first pass ended on exactly the lattice the promoted class indexes. It then integrates the same reflections the canonical pass does, realises the same held-out residual, and the promoted class stands on the tie - 4nwv spent 14 s to report "I-centred orthorhombic does not beat I-centred orthorhombic". The promoted class's first pass now runs first as an indexing probe (memo cleared, so it always records its outcome), and the demoted arm stops right after its own first pass when that ends on the same first-pass outcome string (lattice vectors to the last bit, path, class, geometry, spot-finding state). No comparison is made and the canonical pass runs as before. The decision is unchanged by construction; the geometry the arm used to hand on is the promoted pass's own fit in that case either way. Output-identical: p.hkl md5 and MTZ arrays equal on all 34 metric-check and scale-walk sets of the battery (31 open/in-house + 3 private) against 56bad5fdb. The skip fired on 5 sets; metric-check step time 4nwv 14.0 -> 2.9 s, 7kcn 6.9 -> 1.6 s, in-house lysozyme sets -1.3..-1.7 s; the promoted probe costs ~0.1-0.8 s on the sets where the lattices differ. Battery metric-check step total 129 -> 107 s. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- rugnux/Rugnux.h | 5 +++++ rugnux/RugnuxFirstPass.cpp | 5 +++++ rugnux/RugnuxPasses.cpp | 37 +++++++++++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 24b901afc..f63008b89 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -938,6 +938,11 @@ class Rugnux { // lattice on the validation frames (ProcessResult::validation_evidence), before a single image is // integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score. bool indexing_probe_only_ = false; + // Where a pass stops right after its first pass: when that pass ends exactly on this outcome + // (first_pass_outcome_). The demoted arm of the metric-symmetry check (see RefinedPasses) is + // given the promoted class's own, because a demoted arm that indexes the same lattice the same + // way integrates the same reflections and can only tie. + std::string stop_at_first_pass_outcome_; // --- Near ties: decided early, re-checked late (see FirstPassRun::DecideLate) --- // diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 7f8281b00..e1bd20d49 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -444,6 +444,11 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { 100.0 * verdict.pooled, 100.0 * verdict.pooled_chance); return true; } + if (!stop_at_first_pass_outcome_.empty() && first_pass_outcome_ == stop_at_first_pass_outcome_) { + logger.Info("First pass: scheme '{}' ends exactly where the promoted class's first pass does, " + "so this pass stops before integrating", best.name); + return true; + } run.RejectChanceLattice(best, verdict); run.AdoptLattice(best, verdict); if (!p.geometry_prepass && !postrefine_probe_only_) diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 212851a93..77eb1b006 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -916,6 +916,26 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // budget its first pass measures, which the canonical pass would otherwise start from - // except the geometry its post-refinement asks for, which the promoted arm reads (see // ScaleMergeAndSymmetry), and its first-pass memo. + // + // Before it, the promoted class's own first pass, as an indexing probe: the demoted arm + // stops after its first pass where that ends exactly where this one does. Released from the + // equality, the indexing can still find the same lattice - on the same spots, by the same + // path, to the last bit - and then it integrates the same reflections the canonical pass + // will and realises the same residual, which cannot beat it. Measured: an I-centred + // orthorhombic arm integrated a whole sweep to report "I-ortho does not beat I-ortho". + Rugnux promoted_probe(*this); + promoted_probe.indexing_probe_only_ = true; + promoted_probe.postrefine_probe_ = false; + promoted_probe.first_pass_memo_.clear(); // a memo answers without the outcome + try { + promoted_probe.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + } catch (const std::exception &e) { + if (IsFatalResourceError(e)) throw; + promoted_probe.first_pass_outcome_.clear(); + } + ++arm_passes; + for (const auto &m : promoted_probe.first_pass_memo_) + AddFirstPassMemo(m); Rugnux demoted_run(*this); demoted_run.postrefine_probe_ = true; demoted_run.postrefine_probe_only_ = true; @@ -923,12 +943,25 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro demoted_settings.LatticeClassDemoted(length_arm); demoted_run.experiment_.ImportIndexingSettings(demoted_settings); demoted_run.lattice_angles_demoted_ = angle_arm; + demoted_run.stop_at_first_pass_outcome_ = promoted_probe.first_pass_outcome_; demoted_arm = demoted_run.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); ++arm_passes; - prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; for (const auto &m : demoted_run.first_pass_memo_) AddFirstPassMemo(m); - lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; + const bool same_first_pass = !promoted_probe.first_pass_outcome_.empty() + && demoted_run.first_pass_outcome_ == promoted_probe.first_pass_outcome_; + if (same_first_pass) { + logger.Info("Metric symmetry: released from the equality the first pass ends exactly where " + "the promoted class's does, so the two arms are one and the higher symmetry " + "stands with nothing to compare"); + lattice_arm = fmt::format( + "the metric symmetry stands: released from the {} the first pass indexes the same " + "lattice, so there was nothing to compare", length_arm ? "a = b equality" : "90 deg angle"); + note("the metric symmetry stands: the demoted arm indexes the same lattice"); + } else { + prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; + lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; + } } else { logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, " "{:.1f} px at the far corner, inside the integration disc of {:.1f} px, so " From bcb059591b6c298be64604d8cfe579287a68dbaa Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 11 Oct 2026 07:12:05 +0200 Subject: [PATCH 2/2] rugnux hypothesis arms read their residual on the pre-pass frames The demoted arm of the metric-symmetry check and the rotation-scale walk's refits integrated and post-refined every frame of the sweep, only to read a held-out residual or a scale fitted on top. They now integrate only the frames the geometry pre-pass read (prepass_frames_only_, PrepassEnd), as the pre-pass asked its own fit: the run decides early on the evidence it has. Metric-symmetry check: the canonical pass, the promoted arm, fits its comparison post-refinement on the same frames (GatherPostRefineObservations takes a frame limit). That fit only measures and leaves nothing on the run; the canonical pass then measures its own fit over every frame where it measures one anyway, as any canonical pass does. A decision whose margin from the HeldOutResidualFell line lies within NEAR_TIE_SIGMA of its noise is a near tie in the part-B sense: the canonical pass also fits over every frame while its reflections are in hand, and after the run the demoted arm runs again on every frame (deciding late) and is judged against that fit. Where the check ends elsewhere the run is made again deciding late, which asks both arms on all the frames as before. No new threshold. Rotation-scale walk: a refit proposes the next scale; the validation frames of the whole sweep still decide whether it is taken, so it needs no re-check. Battery, 31 open/in-house metric-check and scale-walk sets + 3 private, against 56bad5fdb: - p.hkl and MTZ arrays identical on 32 of 34. Every metric-symmetry decision ends as before. 6yqf decided a near tie the other way on 1200 of 2000 frames, the re-check on all frames overturned it and the run made again gives base's output (24 -> 55 s). 6cdl and 9vyb were near ties the re-check confirmed. - 9gdj: the scale walk takes one more refit and adopts 0.97020 instead of 0.97045; R_meas 11.41 -> 11.36 %, R_free 0.1709 -> 0.1708, d_min, SG same. - One private scale-walk set: the pre-pass refit pointed up where the all-frame one pointed down; the whole-sweep validation frames prefer the higher scale, and the merge improves on every statistic. - ci tier: 0 sets changed beyond noise (only 9gdj's data differ). - rugnux wall over the 31 sets 1059 -> 993 s (with 6yqf's re-run), metric check step 129 -> 85 s; ci tier 640 -> 626 s. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3 --- image_analysis/geom_refinement/PostRefine.cpp | 4 +- image_analysis/geom_refinement/PostRefine.h | 6 +- rugnux/Rugnux.cpp | 21 ++++++- rugnux/Rugnux.h | 19 +++++- rugnux/RugnuxImagePass.cpp | 2 +- rugnux/RugnuxPasses.cpp | 26 +++++++- rugnux/RugnuxScaleMerge.cpp | 59 +++++++++++++------ tests/PostRefineTest.cpp | 17 ++++++ 8 files changed, 125 insertions(+), 29 deletions(-) diff --git a/image_analysis/geom_refinement/PostRefine.cpp b/image_analysis/geom_refinement/PostRefine.cpp index 299736af7..43a0921fb 100644 --- a/image_analysis/geom_refinement/PostRefine.cpp +++ b/image_analysis/geom_refinement/PostRefine.cpp @@ -151,7 +151,7 @@ struct RotationScaleResidual { PostRefineObservations GatherPostRefineObservations(std::vector &outcomes, size_t nthreads, bool release_reflections, - Logger &logger) { + Logger &logger, size_t max_images) { PostRefineObservations out; try { nthreads = std::max(1, nthreads); @@ -161,7 +161,7 @@ PostRefineObservations GatherPostRefineObservations(std::vector(outcomes.size()); + const int n_out = static_cast(std::min(outcomes.size(), max_images)); std::vector counts(n_out, 0); std::vector h_lo_of(n_out), h_hi_of(n_out); ParallelChunks(n_out, nthreads, [&](int lo, int hi) { diff --git a/image_analysis/geom_refinement/PostRefine.h b/image_analysis/geom_refinement/PostRefine.h index 6ed478587..7d4c40d20 100644 --- a/image_analysis/geom_refinement/PostRefine.h +++ b/image_analysis/geom_refinement/PostRefine.h @@ -128,10 +128,12 @@ struct PostRefineObservations { // image's reflection vector is freed the moment it has been read - only for a caller that will // never touch them again (the rotation geometry pre-pass, whose reflections are never written). // An allocation failure returns an empty gather (n_pts = 0), which the fit refuses - the same -// silent refusal the fit gives every other failure. +// silent refusal the fit gives every other failure. max_images: only the first that many outcomes +// are read - a comparison against a pass that integrated only the start of the sweep. PostRefineObservations GatherPostRefineObservations(std::vector &outcomes, size_t nthreads, bool release_reflections, - Logger &logger); + Logger &logger, + size_t max_images = SIZE_MAX); // nominal_geom / reference_latt: the current detector geometry and the phi=0 reference lattice (orientation // + cell) from rotation indexing. obs: the gathered per-image integrated reflections (observed_x/y, I, diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 105b8138f..4d5e408e2 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -461,6 +461,25 @@ bool Rugnux::NearTieOverturned() { logger.Info("Near ties re-checked on the rest of the sweep: {} first pass(es), {} ending elsewhere", near_tie_rechecks_->pending.size(), elsewhere); near_tie_rechecks_->pending.clear(); + // The metric-symmetry checks decided early, where no first pass has already sent the run again. + for (auto &[demoted, promoted, lost] : near_tie_rechecks_->arms) { + if (elsewhere > 0) + break; + ProcessResult r; + try { + r = demoted->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); + } catch (const std::exception &e) { + if (IsFatalResourceError(e)) throw; + } + const bool lost_late = r.post_refine && HeldOutResidualFell(promoted, *r.post_refine); + logger.Info("Near-tie re-check: on all the frames the demoted class realises a held-out residual of " + "{:.3e} against the promoted class's {:.3e} - the metric-symmetry check {}", + r.post_refine ? r.post_refine->held_out_before : NAN, promoted.held_out_before, + lost_late == lost ? "ends where it did" : "ends elsewhere"); + if (lost_late != lost) + elsewhere++; + } + near_tie_rechecks_->arms.clear(); return elsewhere > 0; } @@ -900,7 +919,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b const auto start_time = std::chrono::steady_clock::now(); - const int loop_end = geometry_prepass ? prepass_end : end_image; + const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image; // Declared here rather than where they are first filled, so that the phases below can share them // through PipelineLocals: the bytes the image loop read, the end message (also written to the file), diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index f63008b89..a476554ac 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -915,6 +915,11 @@ class Rugnux { // Everything below that point - the merges, the space-group search, the correction surfaces, the // reports - is work no comparison looks at, and on a long sweep it is four fifths of the pass. bool postrefine_probe_only_ = false; + // Whether such a probe integrates only the frames the geometry pre-pass read (PrepassEnd). The + // hypothesis arms that read nothing but a probe's post-refinement - the demoted class of the + // metric-symmetry check, the rotation-scale walk's refits - ask it there, as the pre-pass asked + // its own fit: the frames the run decides on early. + bool prepass_frames_only_ = false; // Whether RunAllPasses re-runs this pass at the fixed integration radius when its background // ring is starved (bkg_starved_fraction_ over the bound). Such a pass learns it at integration, // and what is read from it afterwards - the post-refinement and the validation evidence the @@ -929,9 +934,12 @@ class Rugnux { // it loses (lost), so the class that wins is integrated once. struct LatticeArmCheck { std::optional demoted; // the demoted arm's post-refinement - std::optional promoted; // the canonical pass's own + std::optional promoted; // the canonical pass's own, on the same frames + bool prepass_frames = false; // the demoted arm integrated the pre-pass frames only bool asked = false; bool lost = false; + bool near_tie = false; // decided early on a margin within the noise ... + std::optional promoted_all; // ... so the canonical pass's fit over all frames }; std::optional lattice_arm_check_; // Whether a pass exists only to index: it stops as soon as its first-pass indexing has scored the @@ -971,9 +979,18 @@ class Rugnux { bool geometry_prepass; std::string outcome; // what its first pass ended on }; + // A metric-symmetry check decided on the pre-pass frames within its noise (see RefinedPasses): + // the demoted arm, to run again on every frame and be judged against the promoted arm's fit + // over every frame. + struct PendingArm { + std::shared_ptr demoted; + PostRefineResult promoted; + bool lost; // what the check decided early + }; std::mutex m; std::atomic carried = false; // a first pass decided a near tie early std::vector pending; + std::vector arms; }; std::shared_ptr near_tie_rechecks_ = std::make_shared(); // Leaves the first pass the pass just run made to be re-checked; `run` is the copy of this run taken diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index caa246819..1498143b2 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -373,7 +373,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // smoothing in the merge extrapolates over: measured, the fulls' log-scale curve, carried straight // across 900 empty frames, fell further every round until the scale there was zero, and the // cross-validation of its smoothness then took every scale in the sweep to NaN. - if (geometry_prepass) + if (geometry_prepass || prepass_frames_only_) indexer->GetIntegrationOutcome().resize(prepass_images); // The footprint of the reflection rather than of the spot: the pre-scan's widths plus where the diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 77eb1b006..93bcbb6bb 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -662,6 +662,10 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro r->prepass_mosaicity_.clear(); r->indexing_probe_only_ = index_only; r->postrefine_probe_ = r->postrefine_probe_only_ = !index_only; + // A refit reads only the scale its post-refinement fits on top of k - a step for the walk + // to take, which the validation frames of the whole sweep then judge - so it is fitted on + // the frames pass 1 fitted the first one on. + r->prepass_frames_only_ = !index_only && !decide_late_; ProcessResult res; try { res = r->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); @@ -881,6 +885,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro const bool angle_arm = !length_arm && pass1.lattice_angle_departure_deg.has_value(); double split = 0.0; std::optional demoted_arm; // run where the data resolve the equality + std::shared_ptr demoted_recheck; // the demoted arm again on all the frames, for a near tie if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) { split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0; const double dx = std::max(experiment_.GetBeamX_pxl(), @@ -914,8 +919,8 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // moved the geometry. The canonical pass below is the promoted arm (lattice_arm_check_). // The probe runs on a copy of the run, so it leaves nothing on the run - not the spot // budget its first pass measures, which the canonical pass would otherwise start from - - // except the geometry its post-refinement asks for, which the promoted arm reads (see - // ScaleMergeAndSymmetry), and its first-pass memo. + // except the geometry its post-refinement asks for, which stands where the canonical pass + // measures no fit of its own (see ScaleMergeAndSymmetry), and its first-pass memo. // // Before it, the promoted class's own first pass, as an indexing probe: the demoted arm // stops after its first pass where that ends exactly where this one does. Released from the @@ -944,6 +949,16 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro demoted_run.experiment_.ImportIndexingSettings(demoted_settings); demoted_run.lattice_angles_demoted_ = angle_arm; demoted_run.stop_at_first_pass_outcome_ = promoted_probe.first_pass_outcome_; + // The arm is asked on the frames the pre-pass read, as the pre-pass asked its own fit, and + // the canonical pass compares its fit on the same frames. A run deciding late asks it on + // all of them; a near tie keeps this copy to do that once the run is over. + demoted_recheck = std::make_shared(demoted_run); + demoted_recheck->decide_late_ = true; + demoted_recheck->near_tie_recheck_ = true; + demoted_recheck->first_pass_memo_.clear(); + demoted_recheck->speculative_probe_.reset(); + demoted_recheck->rotation_scale_check_.reset(); + demoted_run.prepass_frames_only_ = !decide_late_; demoted_arm = demoted_run.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); ++arm_passes; for (const auto &m : demoted_run.first_pass_memo_) @@ -960,7 +975,8 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro note("the metric symmetry stands: the demoted arm indexes the same lattice"); } else { prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; - lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; + lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine, + .prepass_frames = demoted_run.prepass_frames_only_}; } } else { logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, " @@ -982,6 +998,10 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // The metric-symmetry check above, decided inside the pass just run (the promoted arm). const LatticeArmCheck arm = *lattice_arm_check_; lattice_arm_check_.reset(); + if (arm.near_tie && arm.promoted_all) { + std::lock_guard lock(near_tie_rechecks_->m); + near_tie_rechecks_->arms.push_back({demoted_recheck, *arm.promoted_all, arm.lost}); + } const double realised_promoted = arm.promoted ? arm.promoted->held_out_before : NAN; const double realised_demoted = arm.demoted ? arm.demoted->held_out_before : NAN; const auto class_text = [](const ProcessResult &r) { diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 3e3a63028..091232951 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -12,6 +12,7 @@ #include "../image_analysis/hot_pixels/HotPixels.h" #include "../image_analysis/SensorAbsorption.h" #include "DiagnosticOutput.h" +#include "RugnuxFirstPass.h" #include #include @@ -729,7 +730,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { PostRefineSettings prs; PostRefineObservations obs; }; - const auto prepare_post_refine = [&]() -> std::optional { + const auto prepare_post_refine = [&](size_t max_images = SIZE_MAX) -> std::optional { const auto &outcomes = indexer->GetIntegrationOutcome(); if (geometry_prepass) { prepass_mosaicity_.assign(outcomes.size(), NAN); @@ -790,7 +791,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { ? std::move(*prepass_postrefine_obs) : GatherPostRefineObservations(indexer->GetIntegrationOutcome(), config_.nthreads, - /*release_reflections=*/false, logger); + /*release_reflections=*/false, logger, + max_images); prepass_postrefine_obs.reset(); return PostRefineJob{*rot, prs, std::move(pr_obs)}; } @@ -906,22 +908,44 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { rsm->IngestHost(); }); // The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the - // comparison reads the held-out residual of its post-refinement, so the fit runs here, before the - // merge, and the pass stops if the demoted arm realised the lower residual. Otherwise it goes on - // exactly as the canonical pass it is, leaving behind what the check, run as two probes before - // it, left: the post-refinement only where the pass measures one anyway (postrefine_probe_), and - // the geometry it asks for as the later of the two arms' fits that asked for one. + // comparison reads the held-out residual of a post-refinement on the frames the demoted arm + // integrated, so that fit runs here, before the merge, and the pass stops if the demoted arm + // realised the lower residual. The fit only measures and leaves nothing on the run; otherwise the + // pass goes on exactly as the canonical pass it is, measuring its own post-refinement over all + // the frames where it measures one anyway (postrefine_probe_). const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass && is_rotation && !postrefine_probe_only_; if (lattice_arm_asked) { LatticeArmCheck &arm = *lattice_arm_check_; arm.asked = true; - const auto geometry_before = prepass_detector_geometry_; - post_refine_geometry(); - arm.promoted = result.post_refine; - arm.lost = !cancelled_ && arm.promoted && arm.demoted && HeldOutResidualFell(*arm.promoted, *arm.demoted); - if ((arm.lost || !postrefine_probe_) && arm.demoted && arm.demoted->ok && arm.demoted->detector_refined) - prepass_detector_geometry_ = geometry_before; + const auto arm_fit = [&](size_t max_images) -> std::optional { + TimingMark mark(logger, "post-refinement (metric-symmetry arm)"); + auto job = prepare_post_refine(max_images); + if (!job) + return std::nullopt; + return PostRefineRotationGeometry(std::move(job->obs), *job->rot.axis, job->rot.geom, + job->rot.lattice, job->prs, logger); + }; + arm.promoted = arm_fit(arm.prepass_frames ? static_cast(prepass_images) : SIZE_MAX); + if (!cancelled_ && arm.promoted && arm.demoted) { + arm.lost = HeldOutResidualFell(*arm.promoted, *arm.demoted); + // Decided on the start of the sweep, a near tie is re-checked on all of it once the run is + // over (RefinedPasses), against this pass's own fit over every frame, taken while its + // reflections are still here. The margin is the residual's distance from the line + // HeldOutResidualFell draws, against the noise that draws it. + const double noise = std::hypot(arm.promoted->held_out_before_se, arm.demoted->held_out_before_se); + const double margin = arm.promoted->held_out_before - arm.demoted->held_out_before - noise; + arm.near_tie = arm.prepass_frames && std::isfinite(noise) + && std::abs(margin) <= FirstPassRun::NEAR_TIE_SIGMA * noise; + logger.Info("Metric-symmetry check: on the first {} images the held-out residual is {:.3e} +- {:.1e} " + "as the promoted class against {:.3e} +- {:.1e} as the demoted one{}", + arm.prepass_frames ? prepass_images : images_to_process, + arm.promoted->held_out_before, arm.promoted->held_out_before_se, + arm.demoted->held_out_before, arm.demoted->held_out_before_se, + arm.near_tie ? " - a near tie, re-checked on all the frames after the run" : ""); + if (arm.near_tie) + arm.promoted_all = arm_fit(SIZE_MAX); + } if (arm.lost) { if (census_future.valid()) census_future.wait(); @@ -932,13 +956,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { result.images_processed, result.processing_time_s); if (!writer) return true; - } else if (!postrefine_probe_) - result.post_refine.reset(); - else if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine - && result.post_refine->detector_refined && !result.post_refine->large_move) - StartSpeculativeGeometryProbe(std::make_shared(*this), *prepass_detector_geometry_); + } } - if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ && !lattice_arm_asked) { + if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ + && !(lattice_arm_asked && lattice_arm_check_->lost)) { // Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined // it), so only the gather runs here: it reads the outcomes, which the chain re-indexes in // place once the space group is settled. The fit and the speculative probe it may start diff --git a/tests/PostRefineTest.cpp b/tests/PostRefineTest.cpp index e0479e9b4..a363e735e 100644 --- a/tests/PostRefineTest.cpp +++ b/tests/PostRefineTest.cpp @@ -125,3 +125,20 @@ TEST_CASE("PostRefine_RotationScale", "[PostRefine]") { CHECK(FittedRotationScale(0.97) == Catch::Approx(0.97).margin(0.001)); CHECK(FittedRotationScale(1.0) == Catch::Approx(1.0).margin(0.001)); } + +TEST_CASE("PostRefine_GatherReadsOnlyTheFirstImages", "[PostRefine]") { + std::vector outcomes(4); + for (int i = 0; i < 4; ++i) + for (int j = 0; j <= i; ++j) { + Reflection r{}; + r.h = j; r.k = 1; r.l = 2; + r.image_number = static_cast(i); + r.I = 100.0f; + r.sigma = 10.0f; + outcomes[i].reflections.push_back(r); + } + Logger logger("PostRefineTest"); + // Images 0 and 1 hold 1 + 2 reflections; all four hold 1 + 2 + 3 + 4. + CHECK(GatherPostRefineObservations(outcomes, 2, false, logger, 2).n_pts == 3); + CHECK(GatherPostRefineObservations(outcomes, 2, false, logger).n_pts == 10); +}