diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 1147a40f1..3a1404ad1 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -804,6 +804,17 @@ class Rugnux { // The demoted arm of an ANGLE promotion (see RunAllPasses): the first pass takes the indexer's // free (triclinic) refinement in place of a monoclinic or orthorhombic class. bool lattice_angles_demoted_ = false; + // The metric-symmetry check of the refined passes (see RefinedPasses): the demoted arm has run as + // a probe, and the canonical pass after it is the promoted arm. That pass fits its post-refinement + // before it merges, compares the held-out residual against the demoted arm's, and stops there if + // it loses (lost), so the class that wins is integrated once. + struct LatticeArmCheck { + std::optional demoted; // the demoted arm's post-refinement + std::optional promoted; // the canonical pass's own + bool asked = false; + bool lost = false; + }; + std::optional lattice_arm_check_; // Whether a pass exists only to index: it stops as soon as its first-pass indexing has scored the // lattice on the validation frames (ProcessResult::validation_evidence), before a single image is // integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score. diff --git a/rugnux/RugnuxPasses.cpp b/rugnux/RugnuxPasses.cpp index ad420551e..caf84a643 100644 --- a/rugnux/RugnuxPasses.cpp +++ b/rugnux/RugnuxPasses.cpp @@ -341,6 +341,7 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { force_rotation_result_.reset(); bragg_adaptive_.reset(); beam_center_same_lattice_alternative_.reset(); + lattice_arm_check_.reset(); logger.Info("Rotation two-pass geometry post-refinement: first pass (header geometry) -> {}_01_*", base_prefix); @@ -805,10 +806,13 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // hypotheses are run - the promoted class, and the class the same walk carries when it is // granted no length equality at all - and whichever REALISES the lower held-out residual on a // re-integrated, re-indexed pass is kept, the - // promoted class on a tie, because it is the one that asserts more. Both arms are probe-only - // passes (postrefine_probe_only_): the residual is all the comparison reads and it is in hand - // before the scaling engine is built, so neither arm pays for a merge, a space-group search or - // a set of correction surfaces, and only the class that wins is ever run as a canonical pass. + // promoted class on a tie, because it is the one that asserts more. The residual is all the + // comparison reads and it is in hand before the scaling engine is built, so the demoted arm is a + // probe-only pass (postrefine_probe_only_), and the promoted arm is the canonical pass itself: it + // fits its post-refinement before it merges, compares there (lattice_arm_check_), and stops only + // if it loses - and only then is the demoted class run as the canonical pass. So no arm pays for + // a merge, a space-group search or a set of correction surfaces, and the class that stands is + // integrated once. std::string lattice_arm; // the arm decision, for the pass decision text // // An ANGLE promotion is the same question: the walk holds a monoclinic or orthorhombic angle at @@ -820,9 +824,10 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro // validation frame as primitive orthorhombic, so the per-frame guard never fired. const bool length_arm = pass1.lattice_length_split.has_value(); const bool angle_arm = !length_arm && pass1.lattice_angle_departure_deg.has_value(); + double split = 0.0; + std::optional demoted_arm; // run where the data resolve the equality if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) { - const double split = length_arm ? *pass1.lattice_length_split - : *pass1.lattice_angle_departure_deg * PI / 180.0; + split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0; const double dx = std::max(experiment_.GetBeamX_pxl(), static_cast(experiment_.GetXPixelsNum()) - experiment_.GetBeamX_pxl()); const double dy = std::max(experiment_.GetBeamY_pxl(), @@ -849,77 +854,25 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro note(fmt::format("{} re-integrated with and without its {} equality", gemmi::crystal_system_str(pass1.rotation_lattice_type->crystal_system), length_arm ? "a = b" : "90 deg angle")); - // The arms are compared on the post-refinement's held-out residual, so they have to - // measure one: a canonical pass only does where the pre-pass moved the geometry. + // The demoted arm runs first, as a probe: it is compared on the post-refinement's held-out + // residual, so it has to measure one, and a canonical pass only does where the pre-pass + // moved the geometry. The canonical pass below is the promoted arm (lattice_arm_check_). const bool probe_was = postrefine_probe_; postrefine_probe_ = true; postrefine_probe_only_ = true; - const auto promoted_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); - ++arm_passes; IndexingSettings demoted_settings = experiment_.GetIndexingSettings(); demoted_settings.LatticeClassDemoted(length_arm); experiment_.ImportIndexingSettings(demoted_settings); lattice_angles_demoted_ = angle_arm; - const auto demoted_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + demoted_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); ++arm_passes; postrefine_probe_only_ = false; postrefine_probe_ = probe_was; - const double realised_promoted = promoted_arm.post_refine ? promoted_arm.post_refine->held_out_before : NAN; - const double realised_demoted = demoted_arm.post_refine ? demoted_arm.post_refine->held_out_before : NAN; - const auto class_text = [](const ProcessResult &r) { - return r.rotation_lattice_type - ? fmt::format("{}-centred {}", r.rotation_lattice_type->centering, - gemmi::crystal_system_str(r.rotation_lattice_type->crystal_system)) - : std::string("no lattice"); - }; - if (!cancelled_ && promoted_arm.post_refine && demoted_arm.post_refine - && HeldOutResidualFell(*promoted_arm.post_refine, *demoted_arm.post_refine)) { - logger.Warning("Metric symmetry: re-integrated as {} the held-out residual is {:.3e} " - "against {:.3e} as {} - the equality is not this crystal's, and the " - "run goes on without it", class_text(demoted_arm), realised_demoted, - realised_promoted, class_text(promoted_arm)); - note(fmt::format("the equality dropped: {} fits better", class_text(demoted_arm))); - lattice_arm = fmt::format( - "the metric symmetry was promoted to {} on {} the free refinement leaves " - "{:.2f} {}; re-integrated as {} the held-out residual is {:.3e} against " - "{:.3e}, so the equality was dropped", class_text(promoted_arm), - length_arm ? "axes" : "an angle", length_arm ? 100.0 * split : split * 180.0 / PI, - length_arm ? "% apart" : "deg from 90", - class_text(demoted_arm), realised_demoted, realised_promoted); - } else if (!cancelled_) { - note(fmt::format("the metric symmetry stands: {}", class_text(promoted_arm))); - IndexingSettings promoted_settings = experiment_.GetIndexingSettings(); - promoted_settings.LatticeClassDemoted(false); - experiment_.ImportIndexingSettings(promoted_settings); - lattice_angles_demoted_ = false; - // An arm that indexed nothing, or post-refined nothing, measures no residual. The - // metric symmetry stands either way - that is the arm whose absence of evidence is - // not evidence against it - but the run must not report a comparison it could not - // make: written as a number, a missing residual came out as "realised a held-out - // residual of nan against 1.234e-03, so the metric symmetry stands". - const bool compared = std::isfinite(realised_promoted) && std::isfinite(realised_demoted); - if (compared) { - logger.Info("Metric symmetry: re-integrated as {} the held-out residual is {:.3e}, " - "which {} does not beat ({:.3e}) - the higher symmetry stands", - class_text(promoted_arm), realised_promoted, class_text(demoted_arm), - realised_demoted); - lattice_arm = fmt::format( - "{} was run as well and realised a held-out residual of {:.3e} against {:.3e} " - "as {}, so the metric symmetry stands", class_text(demoted_arm), - realised_demoted, realised_promoted, class_text(promoted_arm)); - } else { - const char *which = !std::isfinite(realised_demoted) && !std::isfinite(realised_promoted) - ? "neither arm measured a held-out residual" - : !std::isfinite(realised_demoted) - ? "the demoted arm measured no held-out residual" - : "the promoted arm measured no held-out residual"; - logger.Info("Metric symmetry: {}, so the two could not be compared - the higher " - "symmetry stands", which); - lattice_arm = fmt::format( - "{} was run as well, but {}, so the two could not be compared and the " - "metric symmetry stands untested", class_text(demoted_arm), which); - } - } + IndexingSettings promoted_settings = experiment_.GetIndexingSettings(); + promoted_settings.LatticeClassDemoted(false); + experiment_.ImportIndexingSettings(promoted_settings); + lattice_angles_demoted_ = false; + lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine}; } else { logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, " "{:.1f} px at the far corner, inside the integration disc of {:.1f} px, so " @@ -935,6 +888,70 @@ std::optional Rugnux::RefinedPasses(RugnuxObserver *observer, Pro speculate_geometry_probe_ = true; // a walk that follows starts with an indexing probe rerun_if_starved_ = true; auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); + if (lattice_arm_check_) { + // The metric-symmetry check above, decided inside the pass just run (the promoted arm). + const LatticeArmCheck arm = *lattice_arm_check_; + lattice_arm_check_.reset(); + const double realised_promoted = arm.promoted ? arm.promoted->held_out_before : NAN; + const double realised_demoted = arm.demoted ? arm.demoted->held_out_before : NAN; + const auto class_text = [](const ProcessResult &r) { + return r.rotation_lattice_type + ? fmt::format("{}-centred {}", r.rotation_lattice_type->centering, + gemmi::crystal_system_str(r.rotation_lattice_type->crystal_system)) + : std::string("no lattice"); + }; + const std::string promoted_class = class_text(pass2); + const std::string demoted_class = class_text(*demoted_arm); + if (arm.lost) { + logger.Warning("Metric symmetry: re-integrated as {} the held-out residual is {:.3e} " + "against {:.3e} as {} - the equality is not this crystal's, and the " + "run goes on without it", demoted_class, realised_demoted, + realised_promoted, promoted_class); + note(fmt::format("the equality dropped: {} fits better", demoted_class)); + lattice_arm = fmt::format( + "the metric symmetry was promoted to {} on {} the free refinement leaves " + "{:.2f} {}; re-integrated as {} the held-out residual is {:.3e} against " + "{:.3e}, so the equality was dropped", promoted_class, + length_arm ? "axes" : "an angle", length_arm ? 100.0 * split : split * 180.0 / PI, + length_arm ? "% apart" : "deg from 90", + demoted_class, realised_demoted, realised_promoted); + // The promoted arm stopped before its merge; the demoted class is the canonical pass. + ++arm_passes; + IndexingSettings demoted_settings = experiment_.GetIndexingSettings(); + demoted_settings.LatticeClassDemoted(length_arm); + experiment_.ImportIndexingSettings(demoted_settings); + lattice_angles_demoted_ = angle_arm; + pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); + } else if (!cancelled_) { + note(fmt::format("the metric symmetry stands: {}", promoted_class)); + // An arm that indexed nothing, or post-refined nothing, measures no residual. The + // metric symmetry stands either way - that is the arm whose absence of evidence is + // not evidence against it - but the run must not report a comparison it could not + // make: written as a number, a missing residual came out as "realised a held-out + // residual of nan against 1.234e-03, so the metric symmetry stands". + const bool compared = std::isfinite(realised_promoted) && std::isfinite(realised_demoted); + if (compared) { + logger.Info("Metric symmetry: re-integrated as {} the held-out residual is {:.3e}, " + "which {} does not beat ({:.3e}) - the higher symmetry stands", + promoted_class, realised_promoted, demoted_class, realised_demoted); + lattice_arm = fmt::format( + "{} was run as well and realised a held-out residual of {:.3e} against {:.3e} " + "as {}, so the metric symmetry stands", demoted_class, + realised_demoted, realised_promoted, promoted_class); + } else { + const char *which = !std::isfinite(realised_demoted) && !std::isfinite(realised_promoted) + ? "neither arm measured a held-out residual" + : !std::isfinite(realised_demoted) + ? "the demoted arm measured no held-out residual" + : "the promoted arm measured no held-out residual"; + logger.Info("Metric symmetry: {}, so the two could not be compared - the higher " + "symmetry stands", which); + lattice_arm = fmt::format( + "{} was run as well, but {}, so the two could not be compared and the " + "metric symmetry stands untested", demoted_class, which); + } + } + } rerun_if_starved_ = false; speculate_geometry_probe_ = false; // Run beside the rotation-scale walk, the passes stand only where the walk leaves them standing. diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index 65d5b7d04..be7bef8ca 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -801,7 +801,40 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { config_.nthreads, engine_log, config_.observation_dump_path); rsm->IngestHost(); }); - if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_) { + // The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the + // comparison reads the held-out residual of its post-refinement, so the fit runs here, before the + // merge, and the pass stops if the demoted arm realised the lower residual. Otherwise it goes on + // exactly as the canonical pass it is, leaving behind what the check, run as two probes before + // it, left: the post-refinement only where the pass measures one anyway (postrefine_probe_), and + // the geometry it asks for as the later of the two arms' fits that asked for one. + const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass + && is_rotation && !postrefine_probe_only_; + if (lattice_arm_asked) { + LatticeArmCheck &arm = *lattice_arm_check_; + arm.asked = true; + const auto geometry_before = prepass_detector_geometry_; + post_refine_geometry(); + arm.promoted = result.post_refine; + arm.lost = !cancelled_ && arm.promoted && arm.demoted && HeldOutResidualFell(*arm.promoted, *arm.demoted); + if ((arm.lost || !postrefine_probe_) && arm.demoted && arm.demoted->ok && arm.demoted->detector_refined) + prepass_detector_geometry_ = geometry_before; + if (arm.lost) { + if (census_future.valid()) + census_future.wait(); + result.processing_time_s = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + logger.Info("Metric-symmetry check: {} images integrated and post-refined in {:.2f} s - the " + "demoted class realises the lower held-out residual, so this pass does not merge", + result.images_processed, result.processing_time_s); + if (!writer) + return true; + } else if (!postrefine_probe_) + result.post_refine.reset(); + else if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine + && result.post_refine->detector_refined && !result.post_refine->large_move) + StartSpeculativeGeometryProbe(std::make_shared(*this), *prepass_detector_geometry_); + } + if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ && !lattice_arm_asked) { // Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined // it), so only the gather runs here: it reads the outcomes, which the chain re-indexes in // place once the space group is settled. The fit and the speculative probe it may start @@ -2749,6 +2782,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { && ((bragg_adaptive_ && bkg_starved_fraction_ && *bkg_starved_fraction_ > spot_width::BKG_STARVED_MAX_FRACTION) || !RefinedPassIsWorse(*quality_guard_pass1_, result).empty())) + || (lattice_arm_check_ && lattice_arm_check_->lost) // only with a _process.h5 open, see above || (rotation_scale_check_ && rotation_scale_check_->Overturns()); if (superseded) { logger.Info("This pass is going to be re-run, so it makes no report and writes no files.");