From 1359d4ce86c93a5a939b65b28d588edb2f13c48e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 23:40:18 +0100 Subject: [PATCH] Rotation scaling: keep the fulls smoothness once its choice turns back The fulls loop re-chooses the smoothness (lambda) of the per-frame scale curve by cross-validation every round until its step falls below the acceleration onset. The held-out error is often bimodal - a curve following a few frames and one straight over tens of them, minima a few percent apart - and which minimum is lower depends on the scales the round started from, i.e. on the last choice. On the flip-flopping loops the choice walked down a decade a round and then jumped back up four or five decades; every jump moved the scales by rms |dlogG| 0.2-0.8, so the step never reached the onset and the loop was stopped unsettled (thaumatin 3.8 keV: 1.5e4 <-> 1.5e9 every ~4 rounds). Now the first choice that turns back the way the previous ones went is kept, and the loop is one map from there. A loop whose choice only moves one way is unchanged. CPU battery (--no-gpu --no-refine), 33 open/in-house sets incl. the guard list: unsettled fulls loops 65 -> 54, final merge unsettled 9 -> 6; thaumatin 3.8 keV ISa 27.8 -> 31.5 (XDS 30.8), 4.6 keV 38.1 -> 38.6, 6jgj 13.5 -> 13.7, all three now settled; every other set's ISa, R_meas, low-res R_meas and REFMAC R-free unchanged. Private arm (5): unsettled loops 9 -> 5, both unsettled final merges settle, values within 0.4 ISa. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../scale_merge/RotationScaleMerge.cpp | 47 ++++++++++++++----- .../scale_merge/RotationScaleMerge.h | 12 ++++- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 7818ea042..e21fde542 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -2204,6 +2204,15 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( // on 149 of the 245 fulls loops that ended unsettled over one battery the choice was still jumping // 2x or more at the end, one sweep's cycling between two choices 300x apart for as long as the loop // ran. The acceleration, too, models one map. + // + // Nor is it chosen afresh once it has turned back (WalkSmoothing). The held-out error over the grid + // often has two minima a few percent apart - a curve following a few frames, and one straight over + // tens of them - and which of the two is lower depends on the scales the round started from, which + // the last choice made. On 28 sets of one battery a loop's choice walked down a decade a round and + // then jumped back up by four or five, every jump moving the scales by up to a factor two (rms + // |dlogG| 0.2-0.8), so the step never came within SCALING_ANDERSON_ONSET and the loop was stopped + // unsettled. A choice that only moves one way is left as it was; the first one that turns back is + // kept, and the loop is one map from there on. ScalingLoopOutcome out; std::vector y(n_frames), J(n_frames), g_new(n_frames), ratio(n_frames); std::vector n_obs(n_frames); @@ -2213,8 +2222,8 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( has_fulls[f] = frame_obs_count[f] > 0 ? 1 : 0; } double best_step = INFINITY, lambda = 0.0; - std::vector segment_lambda(segments.size(), 0.0); - bool lambda_kept = false; + SmoothingWalk walk; + std::vector segment_walks(segments.size()); std::vector steps, g_best; AndersonMixing anderson(SCALING_ANDERSON_DEPTH); std::vector log_g(n_frames), log_g_new(n_frames), weight(n_frames); @@ -2230,12 +2239,13 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( } std::vector x; if (segments.empty()) { - if (!lambda_kept) - lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads); + WalkSmoothing(walk, y, J); + lambda = walk.lambda; x = SmoothLogScale(y, J, lambda); } else { - x = SmoothLogScaleBySegment(y, J, segment_lambda, !lambda_kept); - std::vector l = segment_lambda; + x = SmoothLogScaleBySegment(y, J, segment_walks); + std::vector l; + for (const auto &w : segment_walks) l.push_back(w.lambda); lambda = median_of(l); } for (int f = 0; f < n_frames; ++f) @@ -2265,7 +2275,10 @@ RotationScaleMerge::ScalingLoopOutcome RotationScaleMerge::RunFullsScalingLoop( logger->Debug("Per-frame scaling (fulls): iteration {}, rms |dlogG| {:.2e}, lambda {:.3g}, gauge {:.4f}, " "{} restarts{}", it + 1, out.step, lambda, g_ref, anderson.Restarts(), anderson.Stopped() ? ", plain" : ""); - if (out.step < SCALING_ANDERSON_ONSET) lambda_kept = true; + if (out.step < SCALING_ANDERSON_ONSET) { + walk.kept = true; + for (auto &w : segment_walks) w.kept = true; + } // Stopped and ended on the best round as RunScalingLoop's loop is. steps.push_back(out.step); if (out.step < best_step) { @@ -2724,17 +2737,29 @@ void RotationScaleMerge::MeasureSegmentDose(const DecayTerms &t, int n_groups) { ns, text); } +void RotationScaleMerge::WalkSmoothing(SmoothingWalk &walk, const std::vector &y, + const std::vector &J) const { + if (walk.kept) return; + const double lambda = ChooseLogScaleSmoothing(y, J, std::max(1, rocking_event_frames_at_start), nthreads); + // A grid step is half a decade; the grid itself drifts a little with the typical J between rounds. + const int direction = walk.lambda <= 0.0 ? 0 : lambda > 1.5 * walk.lambda ? 1 : lambda < walk.lambda / 1.5 ? -1 : 0; + if (direction != 0 && direction == -walk.direction) + walk.kept = true; + if (direction != 0) + walk.direction = direction; + walk.lambda = lambda; +} + std::vector RotationScaleMerge::SmoothLogScaleBySegment(const std::vector &y, const std::vector &J, - std::vector &lambda, bool choose) const { + std::vector &walks) const { std::vector x(y.size()); for (size_t k = 0; k < segments.size(); ++k) { const auto lo = y.begin() + segments[k].first_frame, hi = lo + segments[k].n_frames; const auto jlo = J.begin() + segments[k].first_frame; const std::vector ys(lo, hi), Js(jlo, jlo + segments[k].n_frames); - if (choose) - lambda[k] = ChooseLogScaleSmoothing(ys, Js, std::max(1, rocking_event_frames_at_start), nthreads); - const std::vector xs = SmoothLogScale(ys, Js, lambda[k]); + WalkSmoothing(walks[k], ys, Js); + const std::vector xs = SmoothLogScale(ys, Js, walks[k].lambda); std::copy(xs.begin(), xs.end(), x.begin() + segments[k].first_frame); } return x; diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index fa849d68c..9b9f671e4 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -786,9 +786,17 @@ private: // Each frame's place on the dose axis the decay is fitted along: the frames of the segments laid end // to end in the order the segments' relative B factors put them (MeasureSegmentDose). std::vector frame_dose; - // SmoothLogScale / ChooseLogScaleSmoothing within each segment on its own; lambda per segment. + // The smoothness of one fulls scale curve over the rounds of its loop (RunFullsScalingLoop): chosen + // afresh each round, and kept from the round the choice turns back the way it came. + struct SmoothingWalk { + double lambda = 0.0; + int direction = 0; + bool kept = false; + }; + void WalkSmoothing(SmoothingWalk &walk, const std::vector &y, const std::vector &J) const; + // SmoothLogScale within each segment on its own, with a SmoothingWalk per segment. std::vector SmoothLogScaleBySegment(const std::vector &y, const std::vector &J, - std::vector &lambda, bool choose) const; + std::vector &walks) const; int gpu_device = 0; // see SetGpuDevice bool export_scaled_fulls = false; // see SetExportScaledFulls bool export_scaled_fulls_uncut = false; // see SetExportScaledFullsUncut