diff --git a/image_analysis/geom_refinement/PostRefine.cpp b/image_analysis/geom_refinement/PostRefine.cpp index 299736af7..43a0921fb 100644 --- a/image_analysis/geom_refinement/PostRefine.cpp +++ b/image_analysis/geom_refinement/PostRefine.cpp @@ -151,7 +151,7 @@ struct RotationScaleResidual { PostRefineObservations GatherPostRefineObservations(std::vector &outcomes, size_t nthreads, bool release_reflections, - Logger &logger) { + Logger &logger, size_t max_images) { PostRefineObservations out; try { nthreads = std::max(1, nthreads); @@ -161,7 +161,7 @@ PostRefineObservations GatherPostRefineObservations(std::vector(outcomes.size()); + const int n_out = static_cast(std::min(outcomes.size(), max_images)); std::vector counts(n_out, 0); std::vector h_lo_of(n_out), h_hi_of(n_out); ParallelChunks(n_out, nthreads, [&](int lo, int hi) { diff --git a/image_analysis/geom_refinement/PostRefine.h b/image_analysis/geom_refinement/PostRefine.h index 6ed478587..7d4c40d20 100644 --- a/image_analysis/geom_refinement/PostRefine.h +++ b/image_analysis/geom_refinement/PostRefine.h @@ -128,10 +128,12 @@ struct PostRefineObservations { // image's reflection vector is freed the moment it has been read - only for a caller that will // never touch them again (the rotation geometry pre-pass, whose reflections are never written). // An allocation failure returns an empty gather (n_pts = 0), which the fit refuses - the same -// silent refusal the fit gives every other failure. +// silent refusal the fit gives every other failure. max_images: only the first that many outcomes +// are read - a comparison against a pass that integrated only the start of the sweep. PostRefineObservations GatherPostRefineObservations(std::vector &outcomes, size_t nthreads, bool release_reflections, - Logger &logger); + Logger &logger, + size_t max_images = SIZE_MAX); // nominal_geom / reference_latt: the current detector geometry and the phi=0 reference lattice (orientation // + cell) from rotation indexing. obs: the gathered per-image integrated reflections (observed_x/y, I, diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 105b8138f..4d5e408e2 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -461,6 +461,25 @@ bool Rugnux::NearTieOverturned() { logger.Info("Near ties re-checked on the rest of the sweep: {} first pass(es), {} ending elsewhere", near_tie_rechecks_->pending.size(), elsewhere); near_tie_rechecks_->pending.clear(); + // The metric-symmetry checks decided early, where no first pass has already sent the run again. + for (auto &[demoted, promoted, lost] : near_tie_rechecks_->arms) { + if (elsewhere > 0) + break; + ProcessResult r; + try { + r = demoted->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); + } catch (const std::exception &e) { + if (IsFatalResourceError(e)) throw; + } + const bool lost_late = r.post_refine && HeldOutResidualFell(promoted, *r.post_refine); + logger.Info("Near-tie re-check: on all the frames the demoted class realises a held-out residual of " + "{:.3e} against the promoted class's {:.3e} - the metric-symmetry check {}", + r.post_refine ? r.post_refine->held_out_before : NAN, promoted.held_out_before, + lost_late == lost ? "ends where it did" : "ends elsewhere"); + if (lost_late != lost) + elsewhere++; + } + near_tie_rechecks_->arms.clear(); return elsewhere > 0; } @@ -900,7 +919,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b const auto start_time = std::chrono::steady_clock::now(); - const int loop_end = geometry_prepass ? prepass_end : end_image; + const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image; // Declared here rather than where they are first filled, so that the phases below can share them // through PipelineLocals: the bytes the image loop read, the end message (also written to the file), diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 24b901afc..a476554ac 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -915,6 +915,11 @@ class Rugnux { // Everything below that point - the merges, the space-group search, the correction surfaces, the // reports - is work no comparison looks at, and on a long sweep it is four fifths of the pass. bool postrefine_probe_only_ = false; + // Whether such a probe integrates only the frames the geometry pre-pass read (PrepassEnd). The + // hypothesis arms that read nothing but a probe's post-refinement - the demoted class of the + // metric-symmetry check, the rotation-scale walk's refits - ask it there, as the pre-pass asked + // its own fit: the frames the run decides on early. + bool prepass_frames_only_ = false; // Whether RunAllPasses re-runs this pass at the fixed integration radius when its background // ring is starved (bkg_starved_fraction_ over the bound). Such a pass learns it at integration, // and what is read from it afterwards - the post-refinement and the validation evidence the @@ -929,15 +934,23 @@ class Rugnux { // it loses (lost), so the class that wins is integrated once. struct LatticeArmCheck { std::optional demoted; // the demoted arm's post-refinement - std::optional promoted; // the canonical pass's own + std::optional promoted; // the canonical pass's own, on the same frames + bool prepass_frames = false; // the demoted arm integrated the pre-pass frames only bool asked = false; bool lost = false; + bool near_tie = false; // decided early on a margin within the noise ... + std::optional promoted_all; // ... so the canonical pass's fit over all frames }; std::optional lattice_arm_check_; // Whether a pass exists only to index: it stops as soon as its first-pass indexing has scored the // lattice on the validation frames (ProcessResult::validation_evidence), before a single image is // integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score. bool indexing_probe_only_ = false; + // Where a pass stops right after its first pass: when that pass ends exactly on this outcome + // (first_pass_outcome_). The demoted arm of the metric-symmetry check (see RefinedPasses) is + // given the promoted class's own, because a demoted arm that indexes the same lattice the same + // way integrates the same reflections and can only tie. + std::string stop_at_first_pass_outcome_; // --- Near ties: decided early, re-checked late (see FirstPassRun::DecideLate) --- // @@ -966,9 +979,18 @@ class Rugnux { bool geometry_prepass; std::string outcome; // what its first pass ended on }; + // A metric-symmetry check decided on the pre-pass frames within its noise (see RefinedPasses): + // the demoted arm, to run again on every frame and be judged against the promoted arm's fit + // over every frame. + struct PendingArm { + std::shared_ptr demoted; + PostRefineResult promoted; + bool lost; // what the check decided early + }; std::mutex m; std::atomic carried = false; // a first pass decided a near tie early std::vector pending; + std::vector arms; }; std::shared_ptr near_tie_rechecks_ = std::make_shared(); // Leaves the first pass the pass just run made to be re-checked; `run` is the copy of this run taken diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 7f8281b00..e1bd20d49 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -444,6 +444,11 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { 100.0 * verdict.pooled, 100.0 * verdict.pooled_chance); return true; } + if (!stop_at_first_pass_outcome_.empty() && first_pass_outcome_ == stop_at_first_pass_outcome_) { + logger.Info("First pass: scheme '{}' ends exactly where the promoted class's first pass does, " + "so this pass stops before integrating", best.name); + return true; + } run.RejectChanceLattice(best, verdict); run.AdoptLattice(best, verdict); if (!p.geometry_prepass && !postrefine_probe_only_) diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index caa246819..1498143b2 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -373,7 +373,7 @@ void Rugnux::ImagePass(PipelineLocals &p) { // smoothing in the merge extrapolates over: measured, the fulls' log-scale curve, carried straight // across 900 empty frames, fell further every round until the scale there was zero, and the // cross-validation of its smoothness then took every scale in the sweep to NaN. - if (geometry_prepass) + if (geometry_prepass || prepass_frames_only_) indexer->GetIntegrationOutcome().resize(prepass_images); // The footprint of the reflection rather than of the spot: the pre-scan's widths plus where the diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index 212851a93..93bcbb6bb 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -662,6 +662,10 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro r->prepass_mosaicity_.clear(); r->indexing_probe_only_ = index_only; r->postrefine_probe_ = r->postrefine_probe_only_ = !index_only; + // A refit reads only the scale its post-refinement fits on top of k - a step for the walk + // to take, which the validation frames of the whole sweep then judge - so it is fitted on + // the frames pass 1 fitted the first one on. + r->prepass_frames_only_ = !index_only && !decide_late_; ProcessResult res; try { res = r->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false); @@ -881,6 +885,7 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro const bool angle_arm = !length_arm && pass1.lattice_angle_departure_deg.has_value(); double split = 0.0; std::optional demoted_arm; // run where the data resolve the equality + std::shared_ptr demoted_recheck; // the demoted arm again on all the frames, for a near tie if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) { split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0; const double dx = std::max(experiment_.GetBeamX_pxl(), @@ -914,8 +919,28 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // moved the geometry. The canonical pass below is the promoted arm (lattice_arm_check_). // The probe runs on a copy of the run, so it leaves nothing on the run - not the spot // budget its first pass measures, which the canonical pass would otherwise start from - - // except the geometry its post-refinement asks for, which the promoted arm reads (see - // ScaleMergeAndSymmetry), and its first-pass memo. + // except the geometry its post-refinement asks for, which stands where the canonical pass + // measures no fit of its own (see ScaleMergeAndSymmetry), and its first-pass memo. + // + // Before it, the promoted class's own first pass, as an indexing probe: the demoted arm + // stops after its first pass where that ends exactly where this one does. Released from the + // equality, the indexing can still find the same lattice - on the same spots, by the same + // path, to the last bit - and then it integrates the same reflections the canonical pass + // will and realises the same residual, which cannot beat it. Measured: an I-centred + // orthorhombic arm integrated a whole sweep to report "I-ortho does not beat I-ortho". + Rugnux promoted_probe(*this); + promoted_probe.indexing_probe_only_ = true; + promoted_probe.postrefine_probe_ = false; + promoted_probe.first_pass_memo_.clear(); // a memo answers without the outcome + try { + promoted_probe.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + } catch (const std::exception &e) { + if (IsFatalResourceError(e)) throw; + promoted_probe.first_pass_outcome_.clear(); + } + ++arm_passes; + for (const auto &m : promoted_probe.first_pass_memo_) + AddFirstPassMemo(m); Rugnux demoted_run(*this); demoted_run.postrefine_probe_ = true; demoted_run.postrefine_probe_only_ = true; @@ -923,12 +948,36 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro demoted_settings.LatticeClassDemoted(length_arm); demoted_run.experiment_.ImportIndexingSettings(demoted_settings); demoted_run.lattice_angles_demoted_ = angle_arm; + demoted_run.stop_at_first_pass_outcome_ = promoted_probe.first_pass_outcome_; + // The arm is asked on the frames the pre-pass read, as the pre-pass asked its own fit, and + // the canonical pass compares its fit on the same frames. A run deciding late asks it on + // all of them; a near tie keeps this copy to do that once the run is over. + demoted_recheck = std::make_shared(demoted_run); + demoted_recheck->decide_late_ = true; + demoted_recheck->near_tie_recheck_ = true; + demoted_recheck->first_pass_memo_.clear(); + demoted_recheck->speculative_probe_.reset(); + demoted_recheck->rotation_scale_check_.reset(); + demoted_run.prepass_frames_only_ = !decide_late_; demoted_arm = demoted_run.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); ++arm_passes; - prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; for (const auto &m : demoted_run.first_pass_memo_) AddFirstPassMemo(m); - lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; + const bool same_first_pass = !promoted_probe.first_pass_outcome_.empty() + && demoted_run.first_pass_outcome_ == promoted_probe.first_pass_outcome_; + if (same_first_pass) { + logger.Info("Metric symmetry: released from the equality the first pass ends exactly where " + "the promoted class's does, so the two arms are one and the higher symmetry " + "stands with nothing to compare"); + lattice_arm = fmt::format( + "the metric symmetry stands: released from the {} the first pass indexes the same " + "lattice, so there was nothing to compare", length_arm ? "a = b equality" : "90 deg angle"); + note("the metric symmetry stands: the demoted arm indexes the same lattice"); + } else { + prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_; + lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine, + .prepass_frames = demoted_run.prepass_frames_only_}; + } } else { logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, " "{:.1f} px at the far corner, inside the integration disc of {:.1f} px, so " @@ -949,6 +998,10 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // The metric-symmetry check above, decided inside the pass just run (the promoted arm). const LatticeArmCheck arm = *lattice_arm_check_; lattice_arm_check_.reset(); + if (arm.near_tie && arm.promoted_all) { + std::lock_guard lock(near_tie_rechecks_->m); + near_tie_rechecks_->arms.push_back({demoted_recheck, *arm.promoted_all, arm.lost}); + } const double realised_promoted = arm.promoted ? arm.promoted->held_out_before : NAN; const double realised_demoted = arm.demoted ? arm.demoted->held_out_before : NAN; const auto class_text = [](const ProcessResult &r) { diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 3e3a63028..091232951 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -12,6 +12,7 @@ #include "../image_analysis/hot_pixels/HotPixels.h" #include "../image_analysis/SensorAbsorption.h" #include "DiagnosticOutput.h" +#include "RugnuxFirstPass.h" #include #include @@ -729,7 +730,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { PostRefineSettings prs; PostRefineObservations obs; }; - const auto prepare_post_refine = [&]() -> std::optional { + const auto prepare_post_refine = [&](size_t max_images = SIZE_MAX) -> std::optional { const auto &outcomes = indexer->GetIntegrationOutcome(); if (geometry_prepass) { prepass_mosaicity_.assign(outcomes.size(), NAN); @@ -790,7 +791,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { ? std::move(*prepass_postrefine_obs) : GatherPostRefineObservations(indexer->GetIntegrationOutcome(), config_.nthreads, - /*release_reflections=*/false, logger); + /*release_reflections=*/false, logger, + max_images); prepass_postrefine_obs.reset(); return PostRefineJob{*rot, prs, std::move(pr_obs)}; } @@ -906,22 +908,44 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { rsm->IngestHost(); }); // The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the - // comparison reads the held-out residual of its post-refinement, so the fit runs here, before the - // merge, and the pass stops if the demoted arm realised the lower residual. Otherwise it goes on - // exactly as the canonical pass it is, leaving behind what the check, run as two probes before - // it, left: the post-refinement only where the pass measures one anyway (postrefine_probe_), and - // the geometry it asks for as the later of the two arms' fits that asked for one. + // comparison reads the held-out residual of a post-refinement on the frames the demoted arm + // integrated, so that fit runs here, before the merge, and the pass stops if the demoted arm + // realised the lower residual. The fit only measures and leaves nothing on the run; otherwise the + // pass goes on exactly as the canonical pass it is, measuring its own post-refinement over all + // the frames where it measures one anyway (postrefine_probe_). const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass && is_rotation && !postrefine_probe_only_; if (lattice_arm_asked) { LatticeArmCheck &arm = *lattice_arm_check_; arm.asked = true; - const auto geometry_before = prepass_detector_geometry_; - post_refine_geometry(); - arm.promoted = result.post_refine; - arm.lost = !cancelled_ && arm.promoted && arm.demoted && HeldOutResidualFell(*arm.promoted, *arm.demoted); - if ((arm.lost || !postrefine_probe_) && arm.demoted && arm.demoted->ok && arm.demoted->detector_refined) - prepass_detector_geometry_ = geometry_before; + const auto arm_fit = [&](size_t max_images) -> std::optional { + TimingMark mark(logger, "post-refinement (metric-symmetry arm)"); + auto job = prepare_post_refine(max_images); + if (!job) + return std::nullopt; + return PostRefineRotationGeometry(std::move(job->obs), *job->rot.axis, job->rot.geom, + job->rot.lattice, job->prs, logger); + }; + arm.promoted = arm_fit(arm.prepass_frames ? static_cast(prepass_images) : SIZE_MAX); + if (!cancelled_ && arm.promoted && arm.demoted) { + arm.lost = HeldOutResidualFell(*arm.promoted, *arm.demoted); + // Decided on the start of the sweep, a near tie is re-checked on all of it once the run is + // over (RefinedPasses), against this pass's own fit over every frame, taken while its + // reflections are still here. The margin is the residual's distance from the line + // HeldOutResidualFell draws, against the noise that draws it. + const double noise = std::hypot(arm.promoted->held_out_before_se, arm.demoted->held_out_before_se); + const double margin = arm.promoted->held_out_before - arm.demoted->held_out_before - noise; + arm.near_tie = arm.prepass_frames && std::isfinite(noise) + && std::abs(margin) <= FirstPassRun::NEAR_TIE_SIGMA * noise; + logger.Info("Metric-symmetry check: on the first {} images the held-out residual is {:.3e} +- {:.1e} " + "as the promoted class against {:.3e} +- {:.1e} as the demoted one{}", + arm.prepass_frames ? prepass_images : images_to_process, + arm.promoted->held_out_before, arm.promoted->held_out_before_se, + arm.demoted->held_out_before, arm.demoted->held_out_before_se, + arm.near_tie ? " - a near tie, re-checked on all the frames after the run" : ""); + if (arm.near_tie) + arm.promoted_all = arm_fit(SIZE_MAX); + } if (arm.lost) { if (census_future.valid()) census_future.wait(); @@ -932,13 +956,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { result.images_processed, result.processing_time_s); if (!writer) return true; - } else if (!postrefine_probe_) - result.post_refine.reset(); - else if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine - && result.post_refine->detector_refined && !result.post_refine->large_move) - StartSpeculativeGeometryProbe(std::make_shared(*this), *prepass_detector_geometry_); + } } - if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ && !lattice_arm_asked) { + if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ + && !(lattice_arm_asked && lattice_arm_check_->lost)) { // Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined // it), so only the gather runs here: it reads the outcomes, which the chain re-indexes in // place once the space group is settled. The fit and the speculative probe it may start diff --git a/tests/PostRefineTest.cpp b/tests/PostRefineTest.cpp index e0479e9b4..a363e735e 100644 --- a/tests/PostRefineTest.cpp +++ b/tests/PostRefineTest.cpp @@ -125,3 +125,20 @@ TEST_CASE("PostRefine_RotationScale", "[PostRefine]") { CHECK(FittedRotationScale(0.97) == Catch::Approx(0.97).margin(0.001)); CHECK(FittedRotationScale(1.0) == Catch::Approx(1.0).margin(0.001)); } + +TEST_CASE("PostRefine_GatherReadsOnlyTheFirstImages", "[PostRefine]") { + std::vector outcomes(4); + for (int i = 0; i < 4; ++i) + for (int j = 0; j <= i; ++j) { + Reflection r{}; + r.h = j; r.k = 1; r.l = 2; + r.image_number = static_cast(i); + r.I = 100.0f; + r.sigma = 10.0f; + outcomes[i].reflections.push_back(r); + } + Logger logger("PostRefineTest"); + // Images 0 and 1 hold 1 + 2 reflections; all four hold 1 + 2 + 3 + 4. + CHECK(GatherPostRefineObservations(outcomes, 2, false, logger, 2).n_pts == 3); + CHECK(GatherPostRefineObservations(outcomes, 2, false, logger).n_pts == 10); +}