From 1149c541fca7aa3db46ea7029798c6d4e7815e0c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 20 Sep 2026 08:47:24 +0200 Subject: [PATCH] rugnux: the distance arm is decided on probes, not on written passes Where the pre-pass cannot tell whether its distance move paid, the two hypotheses were judged by running each one as a full canonical pass - scaling, space-group search, correction surfaces, every merge, the model step and the output files - although the comparison reads exactly one number from each: the held-out residual the post-refinement measures right after integration, before the scaling engine is even built. Where the header arm won, a third full pass then reproduced the second's numbers digit for digit to write the files. Both arms are now measured before the canonical pass by passes that stop as soon as that post-refinement has measured, and only the arm that wins is run as a canonical pass. So a run that takes the arm pays two image loops plus one canonical pass instead of two or three canonical passes. Result-neutral by construction - the residuals compared, the comparison, the geometry adopted and the pass that writes are unchanged. Verified byte-identical p.mtz, p.hkl, p.cif and p_P1.mtz, and identical reports bar the command line and the clock, on eight rotation datasets covering both arms (seven where the header wins, one where the post-refined distance does). Measured on the datasets whose logs show the arm: it takes an open-arm corpus pass from 261 to 218 minutes, the in-house arm from 25 to 22 and the private arm from 24 to 19, with the worst single set going 594 -> 316 s. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT --- rugnux/Rugnux.cpp | 187 +++++++++++++++++++++++++--------------------- rugnux/Rugnux.h | 6 ++ 2 files changed, 109 insertions(+), 84 deletions(-) diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 1e353fcb7..789718e9b 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -2060,12 +2060,100 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // Let every refined pass from here on ask the quality guard's question of itself, so one that // is going to be thrown away can skip the report and the files it would have superseded. quality_guard_pass1_ = pass1; - logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical) -> {}_*", - base_prefix); config_.output_prefix = base_prefix; // the refined pass is the canonical result (no _02 suffix) // The canonical pass measures the post-refinement again where the pre-pass moved the geometry // (it applies nothing), so that the walk below has something to decide on. postrefine_probe_ = prepass_detector_geometry_.has_value(); + + const auto current_geometry = [&] { + return std::array{experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(), + experiment_.GetDetectorDistance_mm(), + experiment_.GetPoniRot1_rad(), experiment_.GetPoniRot2_rad()}; + }; + const auto set_geometry = [&](const std::array &g) { + experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2]) + .PoniRot1_rad(g[3]).PoniRot2_rad(g[4]); + }; + + // Where the pre-pass could not tell whether its distance move paid (held_arm_beam_px: the + // free distance lowered the fit's own held-out residual by less than that residual's noise), + // the other hypothesis - the header distance, with the beam the same fit found there - is + // measured too, and the two are judged on what they REALISE: the held-out residual a pass + // re-integrated and re-indexed at that geometry measures there, the comparison the walk below + // makes between its rounds. Where the free arm wins the run goes on at the post-refined + // distance; where the header's wins, or the two cannot be told apart, the header's stands - + // it is the hypothesis that asserts nothing, so a tie is its - and the distance walk below is + // not entered: the data have just said they cannot place the distance. Measured on a sweep at + // 820 mm reaching 2theta ~ 9 deg, where the fit's own gain never realises: the free arm's + // re-integrated residual sat within 0.6 % of the header's at a noise of 8 %, and the cell it + // carried was 1 % too large. On a sweep at 110 mm reaching 57 deg, a 0.14 % distance move the + // pre-pass could not decide realised 3.7 % against a noise of 1.5 %, and the cell it carried + // matched the reference to 0.04 %. + // + // Both arms are measured BEFORE the canonical pass, by passes that stop as soon as the + // post-refinement has measured (postrefine_probe_only_). The residual is all the comparison + // reads, and it is in hand before the scaling engine is built, so neither arm pays for a + // merge, a space-group search or a set of correction surfaces - and only the arm that wins is + // ever run as a canonical pass. Judging the arms on written passes instead cost the loser's + // whole pass and, where the header won, a third pass that reproduced the loser's numbers + // digit for digit. + std::string distance_arm; // the arm decision, for the pass decision text + bool distance_arm_kept_header = false; + int arm_passes = 0; + if (!cancelled_ && prepass_detector_geometry_ && pass1.post_refine + && pass1.post_refine->held_arm_beam_px) { + const std::array free_geometry = current_geometry(); + std::array held_geometry = free_geometry; + held_geometry[0] = static_cast((*pass1.post_refine->held_arm_beam_px)[0]); + held_geometry[1] = static_cast((*pass1.post_refine->held_arm_beam_px)[1]); + held_geometry[2] = header_geometry[2]; + logger.Info("Two-pass: the pre-pass could not tell whether moving the distance {:.3f} -> {:.3f} mm " + "paid, so the header distance is re-integrated as well and the two are " + "judged on the held-out residual each realises", header_geometry[2], + free_geometry[2]); + postrefine_probe_only_ = true; + const auto free_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + ++arm_passes; + set_geometry(held_geometry); + const auto held_arm = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); + ++arm_passes; + postrefine_probe_only_ = false; + const double realised_free = free_arm.post_refine ? free_arm.post_refine->held_out_before : NAN; + const double realised_held = held_arm.post_refine ? held_arm.post_refine->held_out_before : NAN; + const double se_free = free_arm.post_refine ? free_arm.post_refine->held_out_before_se : NAN; + const double se_held = held_arm.post_refine ? held_arm.post_refine->held_out_before_se : NAN; + if (!cancelled_ && free_arm.post_refine && held_arm.post_refine + && HeldOutResidualFell(*held_arm.post_refine, *free_arm.post_refine)) { + logger.Info("Two-pass: re-integrated at the post-refined distance {:.3f} mm the held-out " + "residual is {:.3e} against {:.3e} at the header's {:.3f} mm (standard errors " + "{:.1e} and {:.1e}) - the move is realised and stands", free_geometry[2], + realised_free, realised_held, held_geometry[2], se_free, se_held); + set_geometry(free_geometry); + distance_arm = fmt::format( + "the header distance {:.3f} mm was run as well and realised a held-out residual of " + "{:.3e} against {:.3e} at the post-refined {:.3f} mm, so the post-refined distance " + "stands", held_geometry[2], realised_held, realised_free, free_geometry[2]); + } else if (!cancelled_) { + logger.Info("Two-pass: re-integrated at the header distance {:.3f} mm the held-out residual " + "is {:.3e}, which the post-refined {:.3f} mm does not beat ({:.3e}; standard " + "errors {:.1e} and {:.1e}) - these data do not determine the distance, and the " + "header's stands", held_geometry[2], realised_held, free_geometry[2], + realised_free, se_held, se_free); + distance_arm_kept_header = true; + pass1.post_refine->distance_held = true; + pass1.post_refine->beam_x_after_px = held_geometry[0]; + pass1.post_refine->beam_y_after_px = held_geometry[1]; + pass1.post_refine->distance_after_mm = held_geometry[2]; + distance_arm = fmt::format( + "the header distance {:.3f} mm kept: re-integrated there the held-out residual is " + "{:.3e}, which the post-refined {:.3f} mm does not beat ({:.3e}), so these data do " + "not determine the distance", held_geometry[2], realised_held, free_geometry[2], + realised_free); + } + } + + logger.Info("Rotation two-pass geometry post-refinement: second pass (refined geometry, canonical) -> {}_*", + base_prefix); auto pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); // Walk the geometry to the fit's fixed point. The post-refinement is fitted to reflections @@ -2099,87 +2187,6 @@ ProcessResult Rugnux::RunAllPasses(RugnuxObserver *observer) { // last - and says it did not converge; the quality guard below judges it against the header // pass as it judges every refined pass. constexpr int MAX_GEOMETRY_ROUNDS = 8; - const auto current_geometry = [&] { - return std::array{experiment_.GetBeamX_pxl(), experiment_.GetBeamY_pxl(), - experiment_.GetDetectorDistance_mm(), - experiment_.GetPoniRot1_rad(), experiment_.GetPoniRot2_rad()}; - }; - const auto set_geometry = [&](const std::array &g) { - experiment_.BeamX_pxl(g[0]).BeamY_pxl(g[1]).DetectorDistance_mm(g[2]) - .PoniRot1_rad(g[3]).PoniRot2_rad(g[4]); - }; - - // Where the pre-pass could not tell whether its distance move paid (held_arm_beam_px: the - // free distance lowered the fit's own held-out residual by less than that residual's noise), - // the other hypothesis - the header distance, with the beam the same fit found there - is run - // as a canonical pass of its own, and the two are judged on what they REALISE: the held-out - // residual each re-integrated pass measures at its own geometry, the comparison the walk - // below makes between its rounds. The free arm ran first and wrote the canonical files, so - // where it wins nothing is re-run; where the header's wins, or the two cannot be told apart, - // the header's is re-run to write them - it is the hypothesis that asserts nothing, so a tie - // is its, and the distance walk below is not entered: the data have just said they cannot - // place the distance. Measured on a sweep at 820 mm reaching 2theta ~ 9 deg, where the fit's - // own gain never realises: the free arm's re-integrated residual sat within 0.6 % of the - // header's at a noise of 8 %, and the cell it carried was 1 % too large. On a sweep at 110 mm - // reaching 57 deg, a 0.14 % distance move the pre-pass could not decide realised 3.7 % - // against a noise of 1.5 %, and the cell it carried matched the reference to 0.04 %. - std::string distance_arm; // the arm decision, for the pass decision text - bool distance_arm_kept_header = false; - int arm_passes = 0; - if (!cancelled_ && prepass_detector_geometry_ && pass1.post_refine - && pass1.post_refine->held_arm_beam_px && pass2.post_refine) { - const std::array free_geometry = current_geometry(); - std::array held_geometry = free_geometry; - held_geometry[0] = static_cast((*pass1.post_refine->held_arm_beam_px)[0]); - held_geometry[1] = static_cast((*pass1.post_refine->held_arm_beam_px)[1]); - held_geometry[2] = header_geometry[2]; - logger.Info("Two-pass: the pre-pass could not tell whether moving the distance {:.3f} -> {:.3f} mm " - "paid, so the canonical pass runs at the header distance as well and the two are " - "judged on the held-out residual each realises", header_geometry[2], - free_geometry[2]); - // The pass at the header distance measures its own post-refinement and integration, and - // neither may replace the free pass's where the free pass is the one kept. - const auto free_wish = prepass_detector_geometry_; - const auto free_starved = bkg_starved_fraction_; - set_geometry(held_geometry); - auto held = RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false); - ++arm_passes; - const double realised_free = pass2.post_refine->held_out_before; - const double realised_held = held.post_refine ? held.post_refine->held_out_before : NAN; - const double se_free = pass2.post_refine->held_out_before_se; - const double se_held = held.post_refine ? held.post_refine->held_out_before_se : NAN; - if (!cancelled_ && held.post_refine && HeldOutResidualFell(*held.post_refine, *pass2.post_refine)) { - logger.Info("Two-pass: re-integrated at the post-refined distance {:.3f} mm the held-out " - "residual is {:.3e} against {:.3e} at the header's {:.3f} mm (standard errors " - "{:.1e} and {:.1e}) - the move is realised and stands", free_geometry[2], - realised_free, realised_held, held_geometry[2], se_free, se_held); - set_geometry(free_geometry); - prepass_detector_geometry_ = free_wish; - bkg_starved_fraction_ = free_starved; - distance_arm = fmt::format( - "the header distance {:.3f} mm was run as well and realised a held-out residual of " - "{:.3e} against {:.3e} at the post-refined {:.3f} mm, so the post-refined distance " - "stands", held_geometry[2], realised_held, realised_free, free_geometry[2]); - } else if (!cancelled_) { - logger.Info("Two-pass: re-integrated at the header distance {:.3f} mm the held-out residual " - "is {:.3e}, which the post-refined {:.3f} mm does not beat ({:.3e}; standard " - "errors {:.1e} and {:.1e}) - these data do not determine the distance, and the " - "header's stands", held_geometry[2], realised_held, free_geometry[2], - realised_free, se_held, se_free); - pass2 = RunPipeline(observer, /*write_output=*/true, /*geometry_prepass=*/false); - ++arm_passes; - distance_arm_kept_header = true; - pass1.post_refine->distance_held = true; - pass1.post_refine->beam_x_after_px = held_geometry[0]; - pass1.post_refine->beam_y_after_px = held_geometry[1]; - pass1.post_refine->distance_after_mm = held_geometry[2]; - distance_arm = fmt::format( - "the header distance {:.3f} mm kept: re-integrated there the held-out residual is " - "{:.3e}, which the post-refined {:.3f} mm does not beat ({:.3e}), so these data do " - "not determine the distance", held_geometry[2], realised_held, free_geometry[2], - realised_free); - } - } int geometry_rounds = 0; // passes run at a geometry the walk moved to int walk_passes = 0; // every pass the walk ran, the return to its best round included int best_round = 0; @@ -5267,8 +5274,20 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b // before the engine is built rather than beside it - the gathered observations and their // rocking events are gigabytes on a long axis, and beside the engine's own arrays they set // the pass's memory high-water mark. - if (postrefine_probe_ && !geometry_prepass && is_rotation) + if (postrefine_probe_ && !geometry_prepass && is_rotation) { post_refine_geometry(); + // A pass the distance arm runs to compare two geometries has now produced the only thing + // the comparison reads - the held-out residual of the fit just measured. Stop here rather + // than scale, search, fit surfaces and merge for a result nothing looks at. + if (postrefine_probe_only_) { + result.processing_time_s = std::chrono::duration( + std::chrono::steady_clock::now() - start_time).count(); + logger.Info("Distance-arm probe: {} images integrated and post-refined in {:.2f} s " + "(the arm reads the held-out residual only, so this pass does not merge)", + result.images_processed, result.processing_time_s); + return result; + } + } if (is_rotation) { if (experiment_.GetRefineRotationWedgeInScaling() || rot_ss.GetRotationWedgeForScaling().has_value()) diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 32441c661..6f8fe4f7c 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -555,6 +555,12 @@ class Rugnux { // whether it has finished walking is a question only reflections integrated after the move can // answer; RunAllPasses turns this on for the passes that answer it. bool postrefine_probe_ = false; + // Whether such a pass exists ONLY for that measurement. The distance arm reads the held-out + // residual the probe returns and nothing else, so a pass run for it stops as soon as the probe + // has measured - which is before the scaling engine is built. Everything below that point (the + // merges, the space-group search, the correction surfaces, the reports) is work no comparison + // looks at, and on a long sweep it is four fifths of the pass. + bool postrefine_probe_only_ = false; // Pre-scan: read a spread sample of frames and take three things off them - the shadow of the beam // stop and its holder, added to the pixel mask (config_.detect_beam_stop), the beam centre