diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 6f87043b7..7818ea042 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -2367,10 +2367,26 @@ void RotationScaleMerge::BuildDecayTerms(int n_groups, DecayTerms &t) const { for (const DecayTerm &d : t.term) t.gterm[fill[d.group]++] = d; } +std::vector RotationScaleMerge::FrameMeanS2(const DecayTerms &t) const { + std::vector sw(n_frames, 0.0), sws2(n_frames, 0.0); + for (const DecayTerm &o : t.term) { + if (!(o.I > 0.0f)) continue; + const double w = (static_cast(o.I) / o.sigma) * (static_cast(o.I) / o.sigma); + sw[o.frame] += w; + sws2[o.frame] += w / (4.0 * static_cast(o.d) * o.d); + } + std::vector m(n_frames, 0.0); + for (int f = 0; f < n_frames; ++f) + if (sw[f] > 0.0) m[f] = sws2[f] / sw[f]; + return m; +} + void RotationScaleMerge::RefineDecay(int n_groups) { // Radiation damage weakens later frames more at higher resolution - a resolution x time (Debye-Waller) // systematic the resolution-flat per-frame G leaves in. Model it as a single global relative-B rate: - // ln(I_ref / I_obs) = slope * (frame - frame_center) * s^2, s^2 = 1/(4 d^2), slope = 2 dB/dframe, + // ln(I_ref / I_obs) = slope * (frame - frame_center) * (s^2 - _frame), s^2 = 1/(4 d^2), + // slope = 2 dB/dframe, with s^2 about the frame's own mean because the per-frame G already holds the + // frame's average scale - applied about s^2 = 0 the correction would move that average a second time, // fitted (inverse-variance-of-log weighted) against the merged reference. // // The fit is dominated by strong LOW-res reflections and EXTRAPOLATED to high s^2, so a slope fitted to @@ -2395,8 +2411,11 @@ void RotationScaleMerge::RefineDecay(int n_groups) { return segments.empty() ? static_cast(o.image_number) : frame_dose[o.frame] + (o.image_number - static_cast(o.frame)); }; + // s^2 about the frame's mean (FrameMeanS2): the per-frame G has the frame's average already. + std::vector frame_s2; + auto s2c = [&](const auto &o) { return s2_of(o.d) - frame_s2[o.frame]; }; auto decay_factor = [&](const auto &o, double slope) { - return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d)); + return std::exp(slope * (dose(o) - fcenter) * s2c(o)); }; // The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the @@ -2413,6 +2432,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) { decay_terms_kept_valid = false; if (!segments.empty()) MeasureSegmentDose(t, n_groups); + frame_s2 = FrameMeanS2(t); const int nt = static_cast(ThreadsForWork(t.term.size(), nthreads)); std::vector sw(n_groups), swI(n_groups); @@ -2440,7 +2460,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) { const double Is = static_cast(o.I) * o.corr, sc = static_cast(o.sigma) * o.corr; if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return; const double w = (Is / sc) * (Is / sc); - const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is); + const double x = (dose(o) - fcenter) * s2c(o), y = std::log(Iref / Is); a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y; }, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; }); const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4]; @@ -2602,7 +2622,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) { for (int i = lo; i < hi; ++i) { Obs &o = fulls[i]; if (!usable(o)) continue; - const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d)) + const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2c(o)) : decay_factor(o, slope); o.corr = static_cast(o.corr * f); } @@ -2767,12 +2787,14 @@ std::vector RotationScaleMerge::SolveCurvatureSmoothedB(const std::vecto return b; } -std::vector RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch, - int frames_per_batch, int gparity) const { +std::vector RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, const std::vector &frame_s2, + int n_groups, int n_batch, int frames_per_batch, + int gparity) const { // Fit one relative-B per batch over the ASU-group subset {group&1 == gparity} (gparity < 0 = all fulls), // against an inverse-variance reference built from that same subset (no leakage). Per batch the model is - // y = ln(I_ref / I_obs) = b * s^2 through the origin (the resolution-flat intercept is the per-frame G, - // already applied), so the data term is b_c = sum(w s^2 y) / sum(w s^2 s^2); SolveCurvatureSmoothedB then + // y = ln(I_ref / I_obs) = b * (s^2 - _frame): the per-frame G already holds each frame's average, + // so b only tilts a frame about its own mean s^2 (FrameMeanS2), and the data term is + // b_c = sum(w s^2 y) / sum(w s^2 s^2) with s^2 taken about that mean; SolveCurvatureSmoothedB then // smooths it. Finally subtract the leverage-weighted mean: a constant B is a global Wilson-B degenerate // with overall scale, and cannot change equivalent agreement (within one ASU group every equivalent sits // at the same s^2, so only b VARYING between batches moves the merge). Drives the per-batch correction. @@ -2802,7 +2824,7 @@ std::vector RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, i const double Is = static_cast(o.I) * o.corr, sc = static_cast(o.sigma) * o.corr; if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return; const double w = (Is / sc) * (Is / sc); - const double s2 = s2_of(o.d), y = std::log(Iref / Is); + const double s2 = s2_of(o.d) - frame_s2[o.frame], y = std::log(Iref / Is); a[0][batch_of(o)] += w * s2 * y; a[1][batch_of(o)] += w * s2 * s2; }, AddArrays<2>); const std::vector &num = nd[0], &den = nd[1]; @@ -4058,13 +4080,14 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) { && o.partiality >= min_partiality; }; auto batch_of = [&](const auto &o) { return std::min(n_batch - 1, o.frame / frames_per_batch); }; - auto b_factor = [&](const auto &o, const std::vector &b) { - return std::exp(b[batch_of(o)] * s2_of(o.d)); - }; // The fields the three curve fits and the three held-out scores below read, pulled out of the fat // records once, with the group-ordered copy their per-group references run over (BuildDecayTerms). DecayTerms t; BuildDecayTerms(n_groups, t); + const std::vector frame_s2 = FrameMeanS2(t); + auto b_factor = [&](const auto &o, const std::vector &b) { + return std::exp(b[batch_of(o)] * (s2_of(o.d) - frame_s2[o.frame])); + }; const int nt = static_cast(ThreadsForWork(t.term.size(), nthreads)); std::vector sw(n_groups), swI(n_groups); // Held-out disagreement over the group subset {group&1 == gparity} with `b` applied: the same @@ -4100,7 +4123,7 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) { // equivalents (same s^2, different batch) into extra scatter - net harmful, exactly like RefineDecay's // sub-floor slope. (RELATIVE_B_MIN_SPREAD ~ DECAY_MIN_DELTA_B.) constexpr double RELATIVE_B_MIN_SPREAD = 2.0; // A^2, minimum peak-to-peak relative-B to engage - const std::vector b_all = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, -1); + const std::vector b_all = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, -1); const double bmin = *std::min_element(b_all.begin(), b_all.end()); const double bmax = *std::max_element(b_all.begin(), b_all.end()); if (bmax - bmin < RELATIVE_B_MIN_SPREAD) { @@ -4112,8 +4135,8 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) { // per-batch structure generalises across the equivalent split; over-fit per-batch noise does not. const std::vector zero(n_batch, 0.0); const double base = subset_disagreement(0, zero) + subset_disagreement(1, zero); - const std::vector b_even = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 0); - const std::vector b_odd = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 1); + const std::vector b_even = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 0); + const std::vector b_odd = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 1); const double gain = base - (subset_disagreement(1, b_even) + subset_disagreement(0, b_odd)); if (!(gain > CV_MIN_RELATIVE_GAIN * base)) { logger->Info("Relative-B: not cross-validated ({} batches, peak-to-peak {:.2f} A^2, held-out gain {:.1f}%, skipped)", diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index 0671bf5e4..fa849d68c 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -1043,8 +1043,12 @@ private: // Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset // {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a // subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction. - std::vector FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch, - int frames_per_batch, int gparity) const; + std::vector FitRelativeBCurve(const DecayTerms &t, const std::vector &frame_s2, int n_groups, + int n_batch, int frames_per_batch, int gparity) const; + // Per frame, the mean s^2 = 1/(4 d^2) of its terms at the weights the per-frame G was fitted at, + // (I/sigma)^2. A relative B is applied about it: the frame's G already holds the frame's average + // scale, so exp(B s^2) alone would move that average a second time. + std::vector FrameMeanS2(const DecayTerms &t) const; // Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before // any decay correction and store the first->last relative-B change + the per-batch curve on this object // (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF). diff --git a/tests/RotationScaleMergeSegmentsTest.cpp b/tests/RotationScaleMergeSegmentsTest.cpp index 1a1957398..a24845b38 100644 --- a/tests/RotationScaleMergeSegmentsTest.cpp +++ b/tests/RotationScaleMergeSegmentsTest.cpp @@ -192,6 +192,27 @@ TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationSc CHECK(r.segment_decay_model == "dose-linear"); } +TEST_CASE("RotationScaleMerge_DecayRampWithinOneSweep", "[RotationScale]") { + // One sweep whose B rises steadily from 0 to 10 A^2: the per-frame G takes each frame's average, the + // decay the tilt about it, so the high-resolution fulls come out on one scale from the first frame to + // the last. + auto data = MakeSweeps({{200, 1.0, 0.0, 10.0}}, 23); + const auto r = Merge(data, SyntheticExperiment(true), {}, true); + auto ratio = [&](int f0, int f1) { + std::vector v; + for (const auto &o: r.scaled_fulls) { + const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l)); + if (o.image_number >= static_cast(f0) && o.image_number < static_cast(f1) + && o.d < 2.2f && t > 200.0) + v.push_back(o.I / t); + } + REQUIRE(v.size() > 20); + std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end()); + return v[v.size() / 2]; + }; + CHECK(ratio(170, 200) / ratio(0, 30) == Catch::Approx(1.0).epsilon(0.08)); +} + TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") { const std::vector sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}}; auto data = MakeSweeps(sweeps, 13); @@ -228,7 +249,7 @@ TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[Rotation if (!on_gpu) CHECK(hash == 15900277682318835699ULL); else if (get_gpu_description().find("RTX 5080") != std::string::npos) - CHECK(hash == 6640579017825061097ULL); + CHECK(hash == 1680029721518244960ULL); } TEST_CASE("RotationScaleMerge_ShadowDoesNotBendTheResolutionShape", "[RotationScale]") { @@ -325,6 +346,13 @@ TEST_CASE("RotationScaleMerge_SecondCrystalStretchRemovedWhole", "[RotationScale r.image_number += 600.0f; data.frames.push_back(std::move(o)); } + // A 3% error proportional to the intensity, as every real measurement has: without it the error + // model's b is not resolved, and the merge is taken from the fulls alone, which this test is not about. + std::mt19937 rng(24); + std::normal_distribution noise(0.0, 1.0); + for (auto &o: data.frames) + for (auto &r: o.reflections) + r.I *= static_cast(1.0 + 0.03 * noise(rng)); const auto r = Merge(data, SyntheticExperiment(true), {}); const auto &sq = r.statistics.sweep_quality; const auto &removed = sq.removed_stretches;