Decay and per-batch relative B tilt each frame about its own mean s^2
The per-frame scale G already holds each frame's average scale, so a radiation-damage B only tilts a frame about the frame's (I/sigma)^2-weighted mean s^2. The single-sweep decay and the per-batch relative B were fitted and applied about s^2 = 0 instead: the fit was attenuated (it had to explain frame offsets G had already removed) and the applied factor moved every frame's average a second time, which the even/odd-frame cross-validation then scored as a loss. Measured: a synthetic single sweep whose B rises 0 -> 10 A^2 now fits 8.95 A^2 (before 2.74), and its high-resolution fulls come out on one scale from first to last frame (new test). The two-sweep synthetic with a 3 A^2 step now fits 3.43 (before 0.97) and applies it (GPU hash re-pinned; the CPU pin is not reproducible on this machine at the base either). On the rotation battery the decay was never applied: the six damaged in-house sets read held-out gains of -17%, -146%, -8%, -3%, -3% - the signature of the double-applied average. With the fix they read -1.1, -0.2, -6.2, -3.1 and a negligible dB; still skipped, and no merged result moves beyond noise (guard: 10-set battery + those 5 sets against the rc186 baseline, only time differs). On a 360 deg trigonal sweep with relative B +11 A^2 the decay fit goes 2.91 -> 6.45 A^2, held-out gain -3.8% -> +0.4% (still under the 2% gate). The second-crystal stretch test passed only because the spurious decay (-19.9 A^2 on data with no B) gave the partials arm a resolved error model; on error-free synthetic data the run takes the fulls-alone merge, which the test is not about, so the test data get a 3% proportional error. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
This commit is contained in:
1 parent
aecd48b9d3
commit
77737e2582
3 files changed
+73
-18
No files matched your search
@@ -2367,10 +2367,26 @@ void RotationScaleMerge::BuildDecayTerms(int n_groups, DecayTerms &t) const {
|
||||
for (const DecayTerm &d : t.term) t.gterm[fill[d.group]++] = d;
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::FrameMeanS2(const DecayTerms &t) const {
|
||||
std::vector<double> sw(n_frames, 0.0), sws2(n_frames, 0.0);
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if (!(o.I > 0.0f)) continue;
|
||||
const double w = (static_cast<double>(o.I) / o.sigma) * (static_cast<double>(o.I) / o.sigma);
|
||||
sw[o.frame] += w;
|
||||
sws2[o.frame] += w / (4.0 * static_cast<double>(o.d) * o.d);
|
||||
}
|
||||
std::vector<double> m(n_frames, 0.0);
|
||||
for (int f = 0; f < n_frames; ++f)
|
||||
if (sw[f] > 0.0) m[f] = sws2[f] / sw[f];
|
||||
return m;
|
||||
}
|
||||
|
||||
void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
// Radiation damage weakens later frames more at higher resolution - a resolution x time (Debye-Waller)
|
||||
// systematic the resolution-flat per-frame G leaves in. Model it as a single global relative-B rate:
|
||||
// ln(I_ref / I_obs) = slope * (frame - frame_center) * s^2, s^2 = 1/(4 d^2), slope = 2 dB/dframe,
|
||||
// ln(I_ref / I_obs) = slope * (frame - frame_center) * (s^2 - <s^2>_frame), s^2 = 1/(4 d^2),
|
||||
// slope = 2 dB/dframe, with s^2 about the frame's own mean because the per-frame G already holds the
|
||||
// frame's average scale - applied about s^2 = 0 the correction would move that average a second time,
|
||||
// fitted (inverse-variance-of-log weighted) against the merged reference.
|
||||
//
|
||||
// The fit is dominated by strong LOW-res reflections and EXTRAPOLATED to high s^2, so a slope fitted to
|
||||
@@ -2395,8 +2411,11 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
return segments.empty() ? static_cast<double>(o.image_number)
|
||||
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
|
||||
};
|
||||
// s^2 about the frame's mean (FrameMeanS2): the per-frame G has the frame's average already.
|
||||
std::vector<double> frame_s2;
|
||||
auto s2c = [&](const auto &o) { return s2_of(o.d) - frame_s2[o.frame]; };
|
||||
auto decay_factor = [&](const auto &o, double slope) {
|
||||
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
|
||||
return std::exp(slope * (dose(o) - fcenter) * s2c(o));
|
||||
};
|
||||
|
||||
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
|
||||
@@ -2413,6 +2432,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
decay_terms_kept_valid = false;
|
||||
if (!segments.empty())
|
||||
MeasureSegmentDose(t, n_groups);
|
||||
frame_s2 = FrameMeanS2(t);
|
||||
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
|
||||
std::vector<double> sw(n_groups), swI(n_groups);
|
||||
|
||||
@@ -2440,7 +2460,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
|
||||
const double x = (dose(o) - fcenter) * s2c(o), y = std::log(Iref / Is);
|
||||
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
|
||||
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
|
||||
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
|
||||
@@ -2602,7 +2622,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
for (int i = lo; i < hi; ++i) {
|
||||
Obs &o = fulls[i];
|
||||
if (!usable(o)) continue;
|
||||
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
|
||||
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2c(o))
|
||||
: decay_factor(o, slope);
|
||||
o.corr = static_cast<float>(o.corr * f);
|
||||
}
|
||||
@@ -2767,12 +2787,14 @@ std::vector<double> RotationScaleMerge::SolveCurvatureSmoothedB(const std::vecto
|
||||
return b;
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
|
||||
int frames_per_batch, int gparity) const {
|
||||
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2,
|
||||
int n_groups, int n_batch, int frames_per_batch,
|
||||
int gparity) const {
|
||||
// Fit one relative-B per batch over the ASU-group subset {group&1 == gparity} (gparity < 0 = all fulls),
|
||||
// against an inverse-variance reference built from that same subset (no leakage). Per batch the model is
|
||||
// y = ln(I_ref / I_obs) = b * s^2 through the origin (the resolution-flat intercept is the per-frame G,
|
||||
// already applied), so the data term is b_c = sum(w s^2 y) / sum(w s^2 s^2); SolveCurvatureSmoothedB then
|
||||
// y = ln(I_ref / I_obs) = b * (s^2 - <s^2>_frame): the per-frame G already holds each frame's average,
|
||||
// so b only tilts a frame about its own mean s^2 (FrameMeanS2), and the data term is
|
||||
// b_c = sum(w s^2 y) / sum(w s^2 s^2) with s^2 taken about that mean; SolveCurvatureSmoothedB then
|
||||
// smooths it. Finally subtract the leverage-weighted mean: a constant B is a global Wilson-B degenerate
|
||||
// with overall scale, and cannot change equivalent agreement (within one ASU group every equivalent sits
|
||||
// at the same s^2, so only b VARYING between batches moves the merge). Drives the per-batch correction.
|
||||
@@ -2802,7 +2824,7 @@ std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, i
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double s2 = s2_of(o.d), y = std::log(Iref / Is);
|
||||
const double s2 = s2_of(o.d) - frame_s2[o.frame], y = std::log(Iref / Is);
|
||||
a[0][batch_of(o)] += w * s2 * y; a[1][batch_of(o)] += w * s2 * s2;
|
||||
}, AddArrays<2>);
|
||||
const std::vector<double> &num = nd[0], &den = nd[1];
|
||||
@@ -4058,13 +4080,14 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
&& o.partiality >= min_partiality;
|
||||
};
|
||||
auto batch_of = [&](const auto &o) { return std::min(n_batch - 1, o.frame / frames_per_batch); };
|
||||
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
|
||||
return std::exp(b[batch_of(o)] * s2_of(o.d));
|
||||
};
|
||||
// The fields the three curve fits and the three held-out scores below read, pulled out of the fat
|
||||
// records once, with the group-ordered copy their per-group references run over (BuildDecayTerms).
|
||||
DecayTerms t;
|
||||
BuildDecayTerms(n_groups, t);
|
||||
const std::vector<double> frame_s2 = FrameMeanS2(t);
|
||||
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
|
||||
return std::exp(b[batch_of(o)] * (s2_of(o.d) - frame_s2[o.frame]));
|
||||
};
|
||||
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
|
||||
std::vector<double> sw(n_groups), swI(n_groups);
|
||||
// Held-out disagreement over the group subset {group&1 == gparity} with `b` applied: the same
|
||||
@@ -4100,7 +4123,7 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
// equivalents (same s^2, different batch) into extra scatter - net harmful, exactly like RefineDecay's
|
||||
// sub-floor slope. (RELATIVE_B_MIN_SPREAD ~ DECAY_MIN_DELTA_B.)
|
||||
constexpr double RELATIVE_B_MIN_SPREAD = 2.0; // A^2, minimum peak-to-peak relative-B to engage
|
||||
const std::vector<double> b_all = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, -1);
|
||||
const std::vector<double> b_all = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, -1);
|
||||
const double bmin = *std::min_element(b_all.begin(), b_all.end());
|
||||
const double bmax = *std::max_element(b_all.begin(), b_all.end());
|
||||
if (bmax - bmin < RELATIVE_B_MIN_SPREAD) {
|
||||
@@ -4112,8 +4135,8 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
// per-batch structure generalises across the equivalent split; over-fit per-batch noise does not.
|
||||
const std::vector<double> zero(n_batch, 0.0);
|
||||
const double base = subset_disagreement(0, zero) + subset_disagreement(1, zero);
|
||||
const std::vector<double> b_even = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 0);
|
||||
const std::vector<double> b_odd = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 1);
|
||||
const std::vector<double> b_even = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 0);
|
||||
const std::vector<double> b_odd = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 1);
|
||||
const double gain = base - (subset_disagreement(1, b_even) + subset_disagreement(0, b_odd));
|
||||
if (!(gain > CV_MIN_RELATIVE_GAIN * base)) {
|
||||
logger->Info("Relative-B: not cross-validated ({} batches, peak-to-peak {:.2f} A^2, held-out gain {:.1f}%, skipped)",
|
||||
|
||||
@@ -1043,8 +1043,12 @@ private:
|
||||
// Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset
|
||||
// {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a
|
||||
// subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction.
|
||||
std::vector<double> FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
|
||||
int frames_per_batch, int gparity) const;
|
||||
std::vector<double> FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2, int n_groups,
|
||||
int n_batch, int frames_per_batch, int gparity) const;
|
||||
// Per frame, the mean s^2 = 1/(4 d^2) of its terms at the weights the per-frame G was fitted at,
|
||||
// (I/sigma)^2. A relative B is applied about it: the frame's G already holds the frame's average
|
||||
// scale, so exp(B s^2) alone would move that average a second time.
|
||||
std::vector<double> FrameMeanS2(const DecayTerms &t) const;
|
||||
// Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before
|
||||
// any decay correction and store the first->last relative-B change + the per-batch curve on this object
|
||||
// (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF).
|
||||
|
||||
@@ -192,6 +192,27 @@ TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationSc
|
||||
CHECK(r.segment_decay_model == "dose-linear");
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMerge_DecayRampWithinOneSweep", "[RotationScale]") {
|
||||
// One sweep whose B rises steadily from 0 to 10 A^2: the per-frame G takes each frame's average, the
|
||||
// decay the tilt about it, so the high-resolution fulls come out on one scale from the first frame to
|
||||
// the last.
|
||||
auto data = MakeSweeps({{200, 1.0, 0.0, 10.0}}, 23);
|
||||
const auto r = Merge(data, SyntheticExperiment(true), {}, true);
|
||||
auto ratio = [&](int f0, int f1) {
|
||||
std::vector<double> v;
|
||||
for (const auto &o: r.scaled_fulls) {
|
||||
const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l));
|
||||
if (o.image_number >= static_cast<float>(f0) && o.image_number < static_cast<float>(f1)
|
||||
&& o.d < 2.2f && t > 200.0)
|
||||
v.push_back(o.I / t);
|
||||
}
|
||||
REQUIRE(v.size() > 20);
|
||||
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
|
||||
return v[v.size() / 2];
|
||||
};
|
||||
CHECK(ratio(170, 200) / ratio(0, 30) == Catch::Approx(1.0).epsilon(0.08));
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") {
|
||||
const std::vector<SweepSpec> sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}};
|
||||
auto data = MakeSweeps(sweeps, 13);
|
||||
@@ -228,7 +249,7 @@ TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[Rotation
|
||||
if (!on_gpu)
|
||||
CHECK(hash == 15900277682318835699ULL);
|
||||
else if (get_gpu_description().find("RTX 5080") != std::string::npos)
|
||||
CHECK(hash == 6640579017825061097ULL);
|
||||
CHECK(hash == 1680029721518244960ULL);
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMerge_ShadowDoesNotBendTheResolutionShape", "[RotationScale]") {
|
||||
@@ -325,6 +346,13 @@ TEST_CASE("RotationScaleMerge_SecondCrystalStretchRemovedWhole", "[RotationScale
|
||||
r.image_number += 600.0f;
|
||||
data.frames.push_back(std::move(o));
|
||||
}
|
||||
// A 3% error proportional to the intensity, as every real measurement has: without it the error
|
||||
// model's b is not resolved, and the merge is taken from the fulls alone, which this test is not about.
|
||||
std::mt19937 rng(24);
|
||||
std::normal_distribution<double> noise(0.0, 1.0);
|
||||
for (auto &o: data.frames)
|
||||
for (auto &r: o.reflections)
|
||||
r.I *= static_cast<float>(1.0 + 0.03 * noise(rng));
|
||||
const auto r = Merge(data, SyntheticExperiment(true), {});
|
||||
const auto &sq = r.statistics.sweep_quality;
|
||||
const auto &removed = sq.removed_stretches;
|
||||
|
||||
Reference in new issue
Block a user