mx/Jungfraujoch
3
2
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 81 Wiki Activity

Merge branch 'b1-prepass-arms' into rc175: metric-symmetry check skips a demoted arm that indexes the same lattice; hypothesis arms read their residual on the pre-pass frames

Browse Source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EizBKhTqrAYago9KJG3dA3
This commit is contained in:
leonarski_f
2026-10-11 07:13:02 +02:00
co-authored by Claude Opus 5.5
parent 56bad5fdba bcb059591b
commit 46be3f6474
9 changed files with 169 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image_analysis/geom_refinement/PostRefine.cpp
+2 -2
View File
@@ -151,7 +151,7 @@ struct RotationScaleResidual {
PostRefineObservations GatherPostRefineObservations(std::vector<IntegrationOutcome> &outcomes,
size_t nthreads, bool release_reflections,
Logger &logger) {
Logger &logger, size_t max_images) {
PostRefineObservations out;
try {
nthreads = std::max<size_t>(1, nthreads);
@@ -161,7 +161,7 @@ PostRefineObservations GatherPostRefineObservations(std::vector<IntegrationOutco
// copying - and the counts are cheap to take. The h range comes out of the same sweep: the
// sort below buckets by h and needs to know how many buckets that is, and this pass already
// reads every reflection.
const int n_out = static_cast<int>(outcomes.size());
const int n_out = static_cast<int>(std::min(outcomes.size(), max_images));
std::vector<size_t> counts(n_out, 0);
std::vector<int> h_lo_of(n_out), h_hi_of(n_out);
ParallelChunks(n_out, nthreads, [&](int lo, int hi) {
image_analysis/geom_refinement/PostRefine.h
+4 -2
View File
@@ -128,10 +128,12 @@ struct PostRefineObservations {
// image's reflection vector is freed the moment it has been read - only for a caller that will
// never touch them again (the rotation geometry pre-pass, whose reflections are never written).
// An allocation failure returns an empty gather (n_pts = 0), which the fit refuses - the same
// silent refusal the fit gives every other failure.
// silent refusal the fit gives every other failure. max_images: only the first that many outcomes
// are read - a comparison against a pass that integrated only the start of the sweep.
PostRefineObservations GatherPostRefineObservations(std::vector<IntegrationOutcome> &outcomes,
size_t nthreads, bool release_reflections,
Logger &logger);
Logger &logger,
size_t max_images = SIZE_MAX);
// nominal_geom / reference_latt: the current detector geometry and the phi=0 reference lattice (orientation
// + cell) from rotation indexing. obs: the gathered per-image integrated reflections (observed_x/y, I,
rugnux/Rugnux.cpp
+20 -1
View File
@@ -461,6 +461,25 @@ bool Rugnux::NearTieOverturned() {
logger.Info("Near ties re-checked on the rest of the sweep: {} first pass(es), {} ending elsewhere",
near_tie_rechecks_->pending.size(), elsewhere);
near_tie_rechecks_->pending.clear();
// The metric-symmetry checks decided early, where no first pass has already sent the run again.
for (auto &[demoted, promoted, lost] : near_tie_rechecks_->arms) {
if (elsewhere > 0)
break;
ProcessResult r;
try {
r = demoted->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
}
const bool lost_late = r.post_refine && HeldOutResidualFell(promoted, *r.post_refine);
logger.Info("Near-tie re-check: on all the frames the demoted class realises a held-out residual of "
"{:.3e} against the promoted class's {:.3e} - the metric-symmetry check {}",
r.post_refine ? r.post_refine->held_out_before : NAN, promoted.held_out_before,
lost_late == lost ? "ends where it did" : "ends elsewhere");
if (lost_late != lost)
elsewhere++;
}
near_tie_rechecks_->arms.clear();
return elsewhere > 0;
}
@@ -900,7 +919,7 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b
const auto start_time = std::chrono::steady_clock::now();
const int loop_end = geometry_prepass ? prepass_end : end_image;
const int loop_end = geometry_prepass || prepass_frames_only_ ? prepass_end : end_image;
// Declared here rather than where they are first filled, so that the phases below can share them
// through PipelineLocals: the bytes the image loop read, the end message (also written to the file),
rugnux/Rugnux.h
+23 -1
View File
@@ -915,6 +915,11 @@ class Rugnux {
// Everything below that point - the merges, the space-group search, the correction surfaces, the
// reports - is work no comparison looks at, and on a long sweep it is four fifths of the pass.
bool postrefine_probe_only_ = false;
// Whether such a probe integrates only the frames the geometry pre-pass read (PrepassEnd). The
// hypothesis arms that read nothing but a probe's post-refinement - the demoted class of the
// metric-symmetry check, the rotation-scale walk's refits - ask it there, as the pre-pass asked
// its own fit: the frames the run decides on early.
bool prepass_frames_only_ = false;
// Whether RunAllPasses re-runs this pass at the fixed integration radius when its background
// ring is starved (bkg_starved_fraction_ over the bound). Such a pass learns it at integration,
// and what is read from it afterwards - the post-refinement and the validation evidence the
@@ -929,15 +934,23 @@ class Rugnux {
// it loses (lost), so the class that wins is integrated once.
struct LatticeArmCheck {
std::optional<PostRefineResult> demoted; // the demoted arm's post-refinement
std::optional<PostRefineResult> promoted; // the canonical pass's own
std::optional<PostRefineResult> promoted; // the canonical pass's own, on the same frames
bool prepass_frames = false; // the demoted arm integrated the pre-pass frames only
bool asked = false;
bool lost = false;
bool near_tie = false; // decided early on a margin within the noise ...
std::optional<PostRefineResult> promoted_all; // ... so the canonical pass's fit over all frames
};
std::optional<LatticeArmCheck> lattice_arm_check_;
// Whether a pass exists only to index: it stops as soon as its first-pass indexing has scored the
// lattice on the validation frames (ProcessResult::validation_evidence), before a single image is
// integrated. The rotation-scale walk (see RunAllPasses) compares angle models on that score.
bool indexing_probe_only_ = false;
// Where a pass stops right after its first pass: when that pass ends exactly on this outcome
// (first_pass_outcome_). The demoted arm of the metric-symmetry check (see RefinedPasses) is
// given the promoted class's own, because a demoted arm that indexes the same lattice the same
// way integrates the same reflections and can only tie.
std::string stop_at_first_pass_outcome_;
// --- Near ties: decided early, re-checked late (see FirstPassRun::DecideLate) ---
//
@@ -966,9 +979,18 @@ class Rugnux {
bool geometry_prepass;
std::string outcome; // what its first pass ended on
};
// A metric-symmetry check decided on the pre-pass frames within its noise (see RefinedPasses):
// the demoted arm, to run again on every frame and be judged against the promoted arm's fit
// over every frame.
struct PendingArm {
std::shared_ptr<Rugnux> demoted;
PostRefineResult promoted;
bool lost; // what the check decided early
};
std::mutex m;
std::atomic<bool> carried = false; // a first pass decided a near tie early
std::vector<Pending> pending;
std::vector<PendingArm> arms;
};
std::shared_ptr<NearTieRechecks> near_tie_rechecks_ = std::make_shared<NearTieRechecks>();
// Leaves the first pass the pass just run made to be re-checked; `run` is the copy of this run taken
rugnux/RugnuxFirstPass.cpp
+5
View File
@@ -444,6 +444,11 @@ bool Rugnux::FirstPassRotationIndexing(PipelineLocals &p) {
100.0 * verdict.pooled, 100.0 * verdict.pooled_chance);
return true;
}
if (!stop_at_first_pass_outcome_.empty() && first_pass_outcome_ == stop_at_first_pass_outcome_) {
logger.Info("First pass: scheme '{}' ends exactly where the promoted class's first pass does, "
"so this pass stops before integrating", best.name);
return true;
}
run.RejectChanceLattice(best, verdict);
run.AdoptLattice(best, verdict);
if (!p.geometry_prepass && !postrefine_probe_only_)
rugnux/RugnuxImagePass.cpp
+1 -1
View File
@@ -373,7 +373,7 @@ void Rugnux::ImagePass(PipelineLocals &p) {
// smoothing in the merge extrapolates over: measured, the fulls' log-scale curve, carried straight
// across 900 empty frames, fell further every round until the scale there was zero, and the
// cross-validation of its smoothness then took every scale in the sweep to NaN.
if (geometry_prepass)
if (geometry_prepass || prepass_frames_only_)
indexer->GetIntegrationOutcome().resize(prepass_images);
// The footprint of the reflection rather than of the spot: the pre-scan's widths plus where the
rugnux/RugnuxPasses.cpp
+57 -4
View File
@@ -662,6 +662,10 @@ RotationScaleWalk Rugnux::WalkRotationScaleBeside(const ProcessResult &pass1, Ro
r->prepass_mosaicity_.clear();
r->indexing_probe_only_ = index_only;
r->postrefine_probe_ = r->postrefine_probe_only_ = !index_only;
// A refit reads only the scale its post-refinement fits on top of k - a step for the walk
// to take, which the validation frames of the whole sweep then judge - so it is fitted on
// the frames pass 1 fitted the first one on.
r->prepass_frames_only_ = !index_only && !decide_late_;
ProcessResult res;
try {
res = r->RunPipeline(nullptr, /*write_output=*/false, /*geometry_prepass=*/false);
@@ -881,6 +885,7 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
const bool angle_arm = !length_arm && pass1.lattice_angle_departure_deg.has_value();
double split = 0.0;
std::optional<ProcessResult> demoted_arm; // run where the data resolve the equality
std::shared_ptr<Rugnux> demoted_recheck; // the demoted arm again on all the frames, for a near tie
if (!cancelled_ && (length_arm || angle_arm) && pass1.rotation_lattice_type) {
split = length_arm ? *pass1.lattice_length_split : *pass1.lattice_angle_departure_deg * PI / 180.0;
const double dx = std::max(experiment_.GetBeamX_pxl(),
@@ -914,8 +919,28 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
// moved the geometry. The canonical pass below is the promoted arm (lattice_arm_check_).
// The probe runs on a copy of the run, so it leaves nothing on the run - not the spot
// budget its first pass measures, which the canonical pass would otherwise start from -
// except the geometry its post-refinement asks for, which the promoted arm reads (see
// ScaleMergeAndSymmetry), and its first-pass memo.
// except the geometry its post-refinement asks for, which stands where the canonical pass
// measures no fit of its own (see ScaleMergeAndSymmetry), and its first-pass memo.
//
// Before it, the promoted class's own first pass, as an indexing probe: the demoted arm
// stops after its first pass where that ends exactly where this one does. Released from the
// equality, the indexing can still find the same lattice - on the same spots, by the same
// path, to the last bit - and then it integrates the same reflections the canonical pass
// will and realises the same residual, which cannot beat it. Measured: an I-centred
// orthorhombic arm integrated a whole sweep to report "I-ortho does not beat I-ortho".
Rugnux promoted_probe(*this);
promoted_probe.indexing_probe_only_ = true;
promoted_probe.postrefine_probe_ = false;
promoted_probe.first_pass_memo_.clear(); // a memo answers without the outcome
try {
promoted_probe.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false);
} catch (const std::exception &e) {
if (IsFatalResourceError(e)) throw;
promoted_probe.first_pass_outcome_.clear();
}
++arm_passes;
for (const auto &m : promoted_probe.first_pass_memo_)
AddFirstPassMemo(m);
Rugnux demoted_run(*this);
demoted_run.postrefine_probe_ = true;
demoted_run.postrefine_probe_only_ = true;
@@ -923,12 +948,36 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
demoted_settings.LatticeClassDemoted(length_arm);
demoted_run.experiment_.ImportIndexingSettings(demoted_settings);
demoted_run.lattice_angles_demoted_ = angle_arm;
demoted_run.stop_at_first_pass_outcome_ = promoted_probe.first_pass_outcome_;
// The arm is asked on the frames the pre-pass read, as the pre-pass asked its own fit, and
// the canonical pass compares its fit on the same frames. A run deciding late asks it on
// all of them; a near tie keeps this copy to do that once the run is over.
demoted_recheck = std::make_shared<Rugnux>(demoted_run);
demoted_recheck->decide_late_ = true;
demoted_recheck->near_tie_recheck_ = true;
demoted_recheck->first_pass_memo_.clear();
demoted_recheck->speculative_probe_.reset();
demoted_recheck->rotation_scale_check_.reset();
demoted_run.prepass_frames_only_ = !decide_late_;
demoted_arm = demoted_run.RunPipeline(observer, /*write_output=*/false, /*geometry_prepass=*/false);
++arm_passes;
prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_;
for (const auto &m : demoted_run.first_pass_memo_)
AddFirstPassMemo(m);
lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine};
const bool same_first_pass = !promoted_probe.first_pass_outcome_.empty()
&& demoted_run.first_pass_outcome_ == promoted_probe.first_pass_outcome_;
if (same_first_pass) {
logger.Info("Metric symmetry: released from the equality the first pass ends exactly where "
"the promoted class's does, so the two arms are one and the higher symmetry "
"stands with nothing to compare");
lattice_arm = fmt::format(
"the metric symmetry stands: released from the {} the first pass indexes the same "
"lattice, so there was nothing to compare", length_arm ? "a = b equality" : "90 deg angle");
note("the metric symmetry stands: the demoted arm indexes the same lattice");
} else {
prepass_detector_geometry_ = demoted_run.prepass_detector_geometry_;
lattice_arm_check_ = LatticeArmCheck{.demoted = demoted_arm->post_refine,
.prepass_frames = demoted_run.prepass_frames_only_};
}
} else {
logger.Info("Metric symmetry: {} - the free refinement departs from it by {:.2f} {}, "
"{:.1f} px at the far corner, inside the integration disc of {:.1f} px, so "
@@ -949,6 +998,10 @@ std::optional<ProcessResult> Rugnux::RefinedPasses(RugnuxObserver *observer, Pro
// The metric-symmetry check above, decided inside the pass just run (the promoted arm).
const LatticeArmCheck arm = *lattice_arm_check_;
lattice_arm_check_.reset();
if (arm.near_tie && arm.promoted_all) {
std::lock_guard lock(near_tie_rechecks_->m);
near_tie_rechecks_->arms.push_back({demoted_recheck, *arm.promoted_all, arm.lost});
}
const double realised_promoted = arm.promoted ? arm.promoted->held_out_before : NAN;
const double realised_demoted = arm.demoted ? arm.demoted->held_out_before : NAN;
const auto class_text = [](const ProcessResult &r) {
rugnux/RugnuxScaleMerge.cpp
+40 -19
View File
@@ -12,6 +12,7 @@
#include "../image_analysis/hot_pixels/HotPixels.h"
#include "../image_analysis/SensorAbsorption.h"
#include "DiagnosticOutput.h"
#include "RugnuxFirstPass.h"
#include <algorithm>
#include <atomic>
@@ -729,7 +730,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
PostRefineSettings prs;
PostRefineObservations obs;
};
const auto prepare_post_refine = [&]() -> std::optional<PostRefineJob> {
const auto prepare_post_refine = [&](size_t max_images = SIZE_MAX) -> std::optional<PostRefineJob> {
const auto &outcomes = indexer->GetIntegrationOutcome();
if (geometry_prepass) {
prepass_mosaicity_.assign(outcomes.size(), NAN);
@@ -790,7 +791,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
? std::move(*prepass_postrefine_obs)
: GatherPostRefineObservations(indexer->GetIntegrationOutcome(),
config_.nthreads,
/*release_reflections=*/false, logger);
/*release_reflections=*/false, logger,
max_images);
prepass_postrefine_obs.reset();
return PostRefineJob{*rot, prs, std::move(pr_obs)};
}
@@ -906,22 +908,44 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
rsm->IngestHost();
});
// The canonical pass that is the promoted arm of the metric-symmetry check (RefinedPasses): the
// comparison reads the held-out residual of its post-refinement, so the fit runs here, before the
// merge, and the pass stops if the demoted arm realised the lower residual. Otherwise it goes on
// exactly as the canonical pass it is, leaving behind what the check, run as two probes before
// it, left: the post-refinement only where the pass measures one anyway (postrefine_probe_), and
// the geometry it asks for as the later of the two arms' fits that asked for one.
// comparison reads the held-out residual of a post-refinement on the frames the demoted arm
// integrated, so that fit runs here, before the merge, and the pass stops if the demoted arm
// realised the lower residual. The fit only measures and leaves nothing on the run; otherwise the
// pass goes on exactly as the canonical pass it is, measuring its own post-refinement over all
// the frames where it measures one anyway (postrefine_probe_).
const bool lattice_arm_asked = lattice_arm_check_ && !lattice_arm_check_->asked && !geometry_prepass
&& is_rotation && !postrefine_probe_only_;
if (lattice_arm_asked) {
LatticeArmCheck &arm = *lattice_arm_check_;
arm.asked = true;
const auto geometry_before = prepass_detector_geometry_;
post_refine_geometry();
arm.promoted = result.post_refine;
arm.lost = !cancelled_ && arm.promoted && arm.demoted && HeldOutResidualFell(*arm.promoted, *arm.demoted);
if ((arm.lost || !postrefine_probe_) && arm.demoted && arm.demoted->ok && arm.demoted->detector_refined)
prepass_detector_geometry_ = geometry_before;
const auto arm_fit = [&](size_t max_images) -> std::optional<PostRefineResult> {
TimingMark mark(logger, "post-refinement (metric-symmetry arm)");
auto job = prepare_post_refine(max_images);
if (!job)
return std::nullopt;
return PostRefineRotationGeometry(std::move(job->obs), *job->rot.axis, job->rot.geom,
job->rot.lattice, job->prs, logger);
};
arm.promoted = arm_fit(arm.prepass_frames ? static_cast<size_t>(prepass_images) : SIZE_MAX);
if (!cancelled_ && arm.promoted && arm.demoted) {
arm.lost = HeldOutResidualFell(*arm.promoted, *arm.demoted);
// Decided on the start of the sweep, a near tie is re-checked on all of it once the run is
// over (RefinedPasses), against this pass's own fit over every frame, taken while its
// reflections are still here. The margin is the residual's distance from the line
// HeldOutResidualFell draws, against the noise that draws it.
const double noise = std::hypot(arm.promoted->held_out_before_se, arm.demoted->held_out_before_se);
const double margin = arm.promoted->held_out_before - arm.demoted->held_out_before - noise;
arm.near_tie = arm.prepass_frames && std::isfinite(noise)
&& std::abs(margin) <= FirstPassRun::NEAR_TIE_SIGMA * noise;
logger.Info("Metric-symmetry check: on the first {} images the held-out residual is {:.3e} +- {:.1e} "
"as the promoted class against {:.3e} +- {:.1e} as the demoted one{}",
arm.prepass_frames ? prepass_images : images_to_process,
arm.promoted->held_out_before, arm.promoted->held_out_before_se,
arm.demoted->held_out_before, arm.demoted->held_out_before_se,
arm.near_tie ? " - a near tie, re-checked on all the frames after the run" : "");
if (arm.near_tie)
arm.promoted_all = arm_fit(SIZE_MAX);
}
if (arm.lost) {
if (census_future.valid())
census_future.wait();
@@ -932,13 +956,10 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
result.images_processed, result.processing_time_s);
if (!writer)
return true;
} else if (!postrefine_probe_)
result.post_refine.reset();
else if (speculate_geometry_probe_ && !cancelled_ && prepass_detector_geometry_ && result.post_refine
&& result.post_refine->detector_refined && !result.post_refine->large_move)
StartSpeculativeGeometryProbe(std::make_shared<Rugnux>(*this), *prepass_detector_geometry_);
}
}
if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_ && !lattice_arm_asked) {
if (postrefine_probe_ && !geometry_prepass && is_rotation && !postrefine_probe_only_
&& !(lattice_arm_asked && lattice_arm_check_->lost)) {
// Nothing in the merge chain reads the fit (RunAllPasses does, after RunPipeline has joined
// it), so only the gather runs here: it reads the outcomes, which the chain re-indexes in
// place once the space group is settled. The fit and the speculative probe it may start
tests/PostRefineTest.cpp
+17
View File
@@ -125,3 +125,20 @@ TEST_CASE("PostRefine_RotationScale", "[PostRefine]") {
CHECK(FittedRotationScale(0.97) == Catch::Approx(0.97).margin(0.001));
CHECK(FittedRotationScale(1.0) == Catch::Approx(1.0).margin(0.001));
}
TEST_CASE("PostRefine_GatherReadsOnlyTheFirstImages", "[PostRefine]") {
std::vector<IntegrationOutcome> outcomes(4);
for (int i = 0; i < 4; ++i)
for (int j = 0; j <= i; ++j) {
Reflection r{};
r.h = j; r.k = 1; r.l = 2;
r.image_number = static_cast<float>(i);
r.I = 100.0f;
r.sigma = 10.0f;
outcomes[i].reflections.push_back(r);
}
Logger logger("PostRefineTest");
// Images 0 and 1 hold 1 + 2 reflections; all four hold 1 + 2 + 3 + 4.
CHECK(GatherPostRefineObservations(outcomes, 2, false, logger, 2).n_pts == 3);
CHECK(GatherPostRefineObservations(outcomes, 2, false, logger).n_pts == 10);
}