diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index d9523860e..105b8138f 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -388,19 +388,82 @@ ProcessResult Rugnux::Run(RugnuxObserver *observer) { // first pass being invisible in it. Time the whole thing here, where every pass is inside. const auto run_start = std::chrono::steady_clock::now(); DiffractionExperiment header_experiment = experiment_; - ProcessResult result = RunAllPasses(observer); + // The run as it starts, to be made again where a near tie decided early ends elsewhere on the rest + // of the sweep (decide_late_). + const auto again = std::make_shared(*this); + again->near_tie_rechecks_ = std::make_shared(); + again->decide_late_ = true; + Rugnux *run = this; + ProcessResult result; + bool lost = false; + try { + result = RunAllPasses(observer); + } catch (const NoLatticeFound &) { + // A run that found no lattice after deciding a near tie early is made again deciding late. + if (!near_tie_rechecks_->carried) + throw; + lost = true; + } + if (lost || (!result.cancelled && NearTieOverturned())) { + JoinBeamCenterCapture(); + Logger logger("Rugnux"); + logger.Info("A near tie decided early ends elsewhere on the rest of the sweep - the run is made again, " + "deciding near ties on all the frames"); + result = again->RunAllPasses(observer); + run = again.get(); + } result.total_time_s = std::chrono::duration(std::chrono::steady_clock::now() - run_start).count(); // For the report's beam-centre warning: what the background measured, and what this crystal asks // of the centre - at the file's geometry, as the pre-scan's check asks it, but on the cell the run // determined rather than the long axis that check has to assume. - JoinBeamCenterCapture(); - result.background_beam_centre = measured_beam_center_; + run->JoinBeamCenterCapture(); + result.background_beam_centre = run->measured_beam_center_; if (result.consensus_cell) header_experiment.SetUnitCell(*result.consensus_cell); result.beam_centre_need_pxl = BeamCenterNeed_pxl(header_experiment); return result; } +void Rugnux::StartNearTieRecheck(std::shared_ptr run, bool geometry_prepass) { + // The pass again, up to its first pass, as the probe it would be: the same inputs, no memo to + // answer from, nothing started beside it. + run->decide_late_ = true; + run->near_tie_recheck_ = true; + run->indexing_probe_only_ = true; + run->first_pass_memo_.clear(); + run->first_pass_outcome_.clear(); + run->speculative_probe_.reset(); + run->speculate_geometry_probe_ = false; + run->rotation_scale_check_.reset(); + std::lock_guard lock(near_tie_rechecks_->m); + near_tie_rechecks_->pending.push_back({std::move(run), geometry_prepass, first_pass_outcome_}); +} + +bool Rugnux::NearTieOverturned() { + // After the run: by then every frame is in, live too, and nothing else holds the card. + Logger logger("Rugnux"); + TimingMark mark(logger, "near-tie re-checks"); + std::lock_guard lock(near_tie_rechecks_->m); + int elsewhere = 0; + for (auto &[run, geometry_prepass, outcome] : near_tie_rechecks_->pending) { + try { + run->RunPipeline(nullptr, /*write_output=*/false, geometry_prepass); + } catch (const NoLatticeFound &) { + run->first_pass_outcome_.clear(); // the pass found one; on all the frames it finds none + } + if (run->first_pass_outcome_ != outcome) { + logger.Info("Near-tie re-check: the first pass ends elsewhere on all the frames - {} against {}", + run->first_pass_outcome_, outcome); + elsewhere++; + } + } + if (!near_tie_rechecks_->pending.empty()) + logger.Info("Near ties re-checked on the rest of the sweep: {} first pass(es), {} ending elsewhere", + near_tie_rechecks_->pending.size(), elsewhere); + near_tie_rechecks_->pending.clear(); + return elsewhere > 0; +} + void Rugnux::AddFirstPassMemo(const FirstPassMemo &memo) { for (const auto &m : first_pass_memo_) if (m.key == memo.key) @@ -558,6 +621,13 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // ASKED for. A pass that determined one leaves it on experiment_, and the pass after it - the // re-runs in Run(), or a second Run() on the same object - would otherwise inherit that answer and // skip the search that is the whole point of running again. + // The run as this pass finds it, for the re-check of a near tie its first pass may decide early. + std::shared_ptr recheck; + if (!decide_late_ && config_.mode == ProcessMode::FullAnalysis && config_.rotation_indexing + && config_.two_pass_rotation && !force_rotation_result_.has_value() + && !config_.forced_rotation_lattice.has_value() && experiment_.IsRotationIndexing()) + recheck = std::make_shared(*this); + experiment_.SetSpaceGroup(user_fixed_sg_); const auto dataset = reader_.GetDataset(); @@ -866,7 +936,10 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b logger.Info("Rotation indexer lattice forced externally - skipping first pass"); } else if (full && config_.rotation_indexing && config_.two_pass_rotation) { TimingMark mark(logger, "first-pass indexing"); - if (FirstPassRotationIndexing(p)) + const bool probe_done = FirstPassRotationIndexing(p); + if (recheck && near_tie_carried_) + StartNearTieRecheck(std::move(recheck), geometry_prepass); + if (probe_done) return result; } // The rotation-scale walk beside this pass starts once its frames are indexed (see RunAllPasses). diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 0bb4d5da6..24b901afc 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -938,6 +938,44 @@ class Rugnux { // lattice on the validation frames (ProcessResult::validation_evidence), before a single image is // integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score. bool indexing_probe_only_ = false; + + // --- Near ties: decided early, re-checked late (see FirstPassRun::DecideLate) --- + // + // A first pass acts on its validation frames at once. Where it decided a near tie that the rest + // of the sweep could re-check, the copy of the run taken as its pass began is kept, and once the + // run is over - every frame is in by then, live too - that pass is run again on it as a probe, + // deciding late. Only where a re-check's first pass ends elsewhere than the pass's did is the + // run made again, every first pass deciding late. Offline the answer is then the same as deciding + // late throughout; live, no near tie holds the run up while the collection goes on. + // + // Whether the first passes decide their near ties on all the frames: on a re-check, and on the run + // made again. + bool decide_late_ = false; + // Whether this copy of the run is a re-check (for the log). + bool near_tie_recheck_ = false; + // Set by the first pass now running where it decided a near tie the later frames can re-check. + bool near_tie_carried_ = false; + // What the last first pass ended on - the lattice, the path to it, the centre and the spot-finding + // settings it stands on, the centres it carries forward - for its re-check to compare with. + std::string first_pass_outcome_; + // The re-checks of the whole run, shared with the copies the run makes of itself: a probe on a copy + // leaves its re-check here too. + struct NearTieRechecks { + struct Pending { + std::shared_ptr run; // the run as the pass found it + bool geometry_prepass; + std::string outcome; // what its first pass ended on + }; + std::mutex m; + std::atomic carried = false; // a first pass decided a near tie early + std::vector pending; + }; + std::shared_ptr near_tie_rechecks_ = std::make_shared(); + // Leaves the first pass the pass just run made to be re-checked; `run` is the copy of this run taken + // as the pass began. + void StartNearTieRecheck(std::shared_ptr run, bool geometry_prepass); + // Runs the re-checks. Whether one ended elsewhere than its pass. + bool NearTieOverturned(); // The validation evidence of the last canonical-pass or probe first pass, keyed by the inputs that // pass read (FirstPassInputKey). The first pass is deterministic in them, so an indexing probe // whose key matches takes the stored evidence instead of indexing again - the geometry walk's diff --git a/rugnux/RugnuxFirstPass.cpp b/rugnux/RugnuxFirstPass.cpp index 62dbfdafa..7f8281b00 100644 --- a/rugnux/RugnuxFirstPass.cpp +++ b/rugnux/RugnuxFirstPass.cpp @@ -369,6 +369,7 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { p.observer->OnPhase("Rotation indexing (first pass)"); Step(!p.geometry_prepass && prepass_detector_geometry_ ? "Indexing at the refined geometry" : "Indexing"); + near_tie_carried_ = false; FirstPassRun run(*this, p); using FirstPass = FirstPassRun::FirstPass; const int n_validation = static_cast(run.validation.size()); @@ -415,6 +416,25 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) { } const FirstPassRun::Verdict verdict = run.MeasureVerdict(best, first_pass_key, memo_hit); + // What the pass ended on, for the re-check of a near tie it decided early to compare with. The + // numbers are printed exactly (shortest round trip), so equal means equal. + { + const auto vec = [](const Coord &c) { return fmt::format("{} {} {}", c.x, c.y, c.z); }; + const auto centre = [](const std::optional> &c) { + return c ? fmt::format("{} {}", (*c)[0], (*c)[1]) : std::string("-"); + }; + const auto state = [](const FirstPassState &st) { + return fmt::format("{} {} {}", st.rings, st.seed, st.d_min.value_or(0.0f)); + }; + const CrystalLattice &l = best.result->lattice; + first_pass_outcome_ = fmt::format( + "'{}' {} {} | {} | {} | {} {} {} | {} {} {} | {} | {} | {} | {}", best.name, best.score, + best.larger_settled, vec(l.Vec0()), vec(l.Vec1()), vec(l.Vec2()), best.result->search_result.centering, + static_cast(best.result->search_result.system), experiment_.GetBeamX_pxl(), + experiment_.GetBeamY_pxl(), experiment_.GetDetectorDistance_mm(), state(run.committed), + centre(beam_center_alternative_), centre(beam_center_same_lattice_alternative_), + beam_center_alternative_state_ ? state(*beam_center_alternative_state_) : std::string("-")); + } // A pass run only to score a lattice has its score, whatever it is: a lattice that does not // beat chance is a result for the comparison that asked, not a reason to stop the run. if (indexing_probe_only_) { @@ -874,7 +894,7 @@ auto Rugnux::FirstPassRun::ScoreSchemes(const SchemeIndexers &ris, IndexAndRefin bool works = alt_score > majority; if (NearTieGate("metric-symmetry demotion", fmt::format("scheme '{}' unconstrained", name), alt_score, majority) - && !late_validation.empty()) { + && DecideLate()) { const int late = CountIndexed(idx, alt, late_validation); const bool early = works; works = LateOverGate(alt_score, late, majority); @@ -918,7 +938,7 @@ auto Rugnux::FirstPassRun::ScoreSchemes(const SchemeIndexers &ris, IndexAndRefin if (bp.result.has_value() && !SameLattice(result, *bp.result) && NearTieFrames("scheme choice", fmt::format("'{}'", name), score, fmt::format("'{}'", bp.name), bp.score) - && !late_validation.empty()) { + && DecideLate()) { const int late = CountIndexed(idx, result, late_validation); const int bp_late = CountIndexed(idx, *bp.result, late_validation); count += late; @@ -994,7 +1014,7 @@ auto Rugnux::FirstPassRun::ScoreSchemes(const SchemeIndexers &ris, IndexAndRefin const bool tie_sub = NearTieCounts("axis-harmonic arbiter (own sub-lattice)", "larger cell", static_cast(c.larger_p), "sub-lattice", static_cast(c.sub)); - if ((tie_smaller || tie_sub) && !late_validation.empty()) { + if ((tie_smaller || tie_sub) && DecideLate()) { const bool early = c.larger > c.smaller && c.larger_p > c.sub; PrefetchSpots(late_validation); c = count_spots(AllValidationFrames()); @@ -1283,7 +1303,7 @@ auto Rugnux::FirstPassRun::FirstPassLadder(int score_before, int min_gain, int m std::sqrt(var(rung_score[top]) + var(std::max(0, score_before))), "binomial on validation frames"); if (min_score > 0 && NearTieGate("ladder adoption (bar)", "best rung", rung_score[top], min_score) - && !late_validation.empty()) { + && DecideLate()) { ApplyState({rungs[top].rings, rungs[top].seed, rungs[top].d_min}); const int late = CountIndexed(*indexer, *rung_pass[top].result, late_validation); // At least min_score of the validation frames is more than min_score - 1 of them. @@ -1665,7 +1685,8 @@ auto Rugnux::FirstPassRun::TwinCompositeHypothesis(FirstPass best) -> FirstPass // Which first pass a near-tie line comes from: the geometry pre-pass, the canonical pass, or a probe // that only scores a lattice. std::string Rugnux::FirstPassRun::PassTag() const { - return geometry_prepass ? "pass 1" : rx.indexing_probe_only_ ? "probe" : "pass 2"; + const char *pass = geometry_prepass ? "pass 1" : rx.indexing_probe_only_ ? "probe" : "pass 2"; + return rx.near_tie_recheck_ ? std::string(pass) + " re-check" : pass; } bool Rugnux::FirstPassRun::NearTie(const std::string &decision, const std::string &a, const std::string &b, @@ -1723,6 +1744,16 @@ bool Rugnux::FirstPassRun::NearTiePooled(const std::string &decision, const std: -difference, sigma, "binomial on pooled validation spots"); } +bool Rugnux::FirstPassRun::DecideLate() const { + if (late_validation.empty()) + return false; + if (rx.decide_late_) + return true; + rx.near_tie_carried_ = true; + rx.near_tie_rechecks_->carried = true; + return false; +} + std::vector Rugnux::FirstPassRun::AllValidationFrames() const { std::vector all = validation; all.insert(all.end(), late_validation.begin(), late_validation.end()); diff --git a/rugnux/RugnuxFirstPass.h b/rugnux/RugnuxFirstPass.h index 8b03c32db..4d3ddfb35 100644 --- a/rugnux/RugnuxFirstPass.h +++ b/rugnux/RugnuxFirstPass.h @@ -172,14 +172,16 @@ struct Rugnux::FirstPassRun { bool NearTiePooled(const std::string &decision, const std::string &a, const PooledEvidence &ea, const std::string &b, const PooledEvidence &eb) const; - // A near tie is not decided on the first part of the sweep. Both candidates are carried until - // the frames of the rest of it are in, and the comparison is then made again, unchanged, on the - // validation frames and late_validation together - a gate set at a share of the validation - // frames at the same share of all of them. Live, that is where the decision waits for the - // collection; offline the frames are already there. A decision that is not a near tie never - // reads them, so it is taken exactly as before; nor does one on a sweep the first part already - // covers, where there is nothing later to read (late_validation, below). + // A near tie is decided early and checked late. The pass acts on the validation frames at once; + // the comparison made again, unchanged, on the validation frames and late_validation together - + // a gate set at a share of the validation frames at the same share of all of them - is the + // re-check, which Run() makes once the rest of the sweep is in (Rugnux::decide_late_). A + // decision that is not a near tie never reads later frames; nor does one on a sweep the first + // part already covers, where there is nothing later to read (late_validation, below). // + // Whether this near tie is decided on all the frames now: on the re-check, and on the run Run() + // makes again where a re-check ended elsewhere. Otherwise it notes that the pass carried one. + bool DecideLate() const; // The validation frames followed by late_validation. std::vector AllValidationFrames() const; // A frame count on the validation frames, plus `late` on late_validation, over the gate set at diff --git a/rugnux/RugnuxFirstPassRescues.cpp b/rugnux/RugnuxFirstPassRescues.cpp index d24af7369..765007839 100644 --- a/rugnux/RugnuxFirstPassRescues.cpp +++ b/rugnux/RugnuxFirstPassRescues.cpp @@ -225,7 +225,7 @@ auto Rugnux::FirstPassRun::LongAxisRescue(const FirstPass &standing) -> std::opt bool more = constrained.score > std::max(standing.score, 0); if (standing.result.has_value() && !SameLattice(*constrained.result, *standing.result) && NearTieFrames("long-axis rescue", "re-index", constrained.score, "standing", standing.score) - && !late_validation.empty()) { + && DecideLate()) { const int late = CountIndexed(*indexer, *constrained.result, late_validation); const int standing_late = CountIndexed(*indexer, *standing.result, late_validation); const bool early = more; @@ -515,7 +515,7 @@ auto Rugnux::FirstPassRun::CentreByPooledSpots(FirstPass best, FirstPass alt, co long_axis_asked = true; if (alt.result.has_value() && best.result.has_value() && NearTiePooled("beam centre (pooled spots)", "file centre", at_header, "measured centre", at_measured) - && !late_validation.empty()) { + && DecideLate()) { // Each centre on its own spots, as above. const bool early = ValidationEvidenceBeatsChance(at_measured) && ValidationEvidencePrefers(at_header, at_measured); @@ -973,7 +973,7 @@ auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass { // Carried only where the gate is what decides: a sub-lattice that indexes fewer frames than // the standing cell is not nominated whatever the gate says. if (sub_best && NearTieGate("sub-lattice majority gate", "best sub-lattice", sub_best->score, majority) - && sub_best->score >= best.score && !late_validation.empty()) { + && sub_best->score >= best.score && DecideLate()) { const int late = CountIndexed(*indexer, *sub_best->result, late_validation); const int standing_late = CountIndexed(*indexer, *best.result, late_validation); const bool early = nominated; @@ -988,7 +988,7 @@ auto Rugnux::FirstPassRun::SubLatticeHypothesis(FirstPass best) -> FirstPass { PooledEvidence at_standing = MeasurePooled(*indexer, *best.result); PooledEvidence at_sub = MeasurePooled(*indexer, *sub_best->result); if (NearTiePooled("sub-lattice hypothesis", "standing cell", at_standing, "sub-lattice", at_sub) - && !late_validation.empty()) { + && DecideLate()) { const bool early = ValidationEvidencePrefers(at_standing, at_sub); const std::vector all = AllValidationFrames(); at_standing = MeasurePooled(*indexer, *best.result, all); @@ -1077,7 +1077,7 @@ auto Rugnux::FirstPassRun::Pass1Fallback(FirstPass best) -> FirstPass { bool pass1_more = pass1_score > best.score; if (pass1_score >= 0 && !same_lattice && NearTieFrames("pass-1 fallback", "re-index", best.score, "pass-1 lattice", pass1_score) - && !late_validation.empty()) { + && DecideLate()) { const int late = CountIndexed(*indexer, *best.result, late_validation); const int pass1_late = CountIndexed(*indexer, pass1_lattice, late_validation); const bool early = pass1_more;