Merge rc187-cytc-proc: decay/relative-B about each frame's mean s^2, live integrates at the adaptive radius as offline

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-14 07:28:29 +01:00
commit 7b8196e770
7 files changed
+80 -25

No files matched your search

+1 -1
View File
@@ -162,7 +162,7 @@ With the input already on disk, the URL is ignored.
| `-t, --stride <num>` | Process every *n*-th image (default: 1) |
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in (the integration radius from the spot width stays where the pre-pass frames put it), and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `<prefix>.*`; `separate` processes each alone into `<prefix>_s<k>_*` |
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
@@ -1470,8 +1470,8 @@ AnisotropyResult AnalyzeAnisotropy(const std::vector<MergedReflection> &merged,
"reachable by a lab-fixed systematic and reduces the power of the test");
if (std::isfinite(run.radiation_damage_relative_b) && std::fabs(run.radiation_damage_relative_b) > 10.0)
result.cautions.emplace_back(
"the run carries substantial radiation damage; the scale model removes its average "
"monotone part, but a non-monotone dose ramp needs --relative-b to be taken out too");
"the run carries substantial radiation damage; the scale model takes out a decay only "
"where it cross-validates, so part of the damage can remain in the fall-off");
// R2's Laue-class caution, fired on its measured signature rather than on the Laue class alone.
// Too high a symmetry assignment pushes real anisotropy into the directions the systematic error is
// measured in, so sigma_sys is inflated by the signal itself and the gate collapses (measured: a
@@ -2367,10 +2367,26 @@ void RotationScaleMerge::BuildDecayTerms(int n_groups, DecayTerms &t) const {
for (const DecayTerm &d : t.term) t.gterm[fill[d.group]++] = d;
}
std::vector<double> RotationScaleMerge::FrameMeanS2(const DecayTerms &t) const {
std::vector<double> sw(n_frames, 0.0), sws2(n_frames, 0.0);
for (const DecayTerm &o : t.term) {
if (!(o.I > 0.0f)) continue;
const double w = (static_cast<double>(o.I) / o.sigma) * (static_cast<double>(o.I) / o.sigma);
sw[o.frame] += w;
sws2[o.frame] += w / (4.0 * static_cast<double>(o.d) * o.d);
}
std::vector<double> m(n_frames, 0.0);
for (int f = 0; f < n_frames; ++f)
if (sw[f] > 0.0) m[f] = sws2[f] / sw[f];
return m;
}
void RotationScaleMerge::RefineDecay(int n_groups) {
// Radiation damage weakens later frames more at higher resolution - a resolution x time (Debye-Waller)
// systematic the resolution-flat per-frame G leaves in. Model it as a single global relative-B rate:
// ln(I_ref / I_obs) = slope * (frame - frame_center) * s^2, s^2 = 1/(4 d^2), slope = 2 dB/dframe,
// ln(I_ref / I_obs) = slope * (frame - frame_center) * (s^2 - <s^2>_frame), s^2 = 1/(4 d^2),
// slope = 2 dB/dframe, with s^2 about the frame's own mean because the per-frame G already holds the
// frame's average scale - applied about s^2 = 0 the correction would move that average a second time,
// fitted (inverse-variance-of-log weighted) against the merged reference.
//
// The fit is dominated by strong LOW-res reflections and EXTRAPOLATED to high s^2, so a slope fitted to
@@ -2395,8 +2411,11 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
return segments.empty() ? static_cast<double>(o.image_number)
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
};
// s^2 about the frame's mean (FrameMeanS2): the per-frame G has the frame's average already.
std::vector<double> frame_s2;
auto s2c = [&](const auto &o) { return s2_of(o.d) - frame_s2[o.frame]; };
auto decay_factor = [&](const auto &o, double slope) {
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
return std::exp(slope * (dose(o) - fcenter) * s2c(o));
};
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
@@ -2413,6 +2432,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
decay_terms_kept_valid = false;
if (!segments.empty())
MeasureSegmentDose(t, n_groups);
frame_s2 = FrameMeanS2(t);
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
std::vector<double> sw(n_groups), swI(n_groups);
@@ -2440,7 +2460,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
const double w = (Is / sc) * (Is / sc);
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
const double x = (dose(o) - fcenter) * s2c(o), y = std::log(Iref / Is);
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
@@ -2602,7 +2622,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
for (int i = lo; i < hi; ++i) {
Obs &o = fulls[i];
if (!usable(o)) continue;
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2c(o))
: decay_factor(o, slope);
o.corr = static_cast<float>(o.corr * f);
}
@@ -2767,12 +2787,14 @@ std::vector<double> RotationScaleMerge::SolveCurvatureSmoothedB(const std::vecto
return b;
}
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
int frames_per_batch, int gparity) const {
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2,
int n_groups, int n_batch, int frames_per_batch,
int gparity) const {
// Fit one relative-B per batch over the ASU-group subset {group&1 == gparity} (gparity < 0 = all fulls),
// against an inverse-variance reference built from that same subset (no leakage). Per batch the model is
// y = ln(I_ref / I_obs) = b * s^2 through the origin (the resolution-flat intercept is the per-frame G,
// already applied), so the data term is b_c = sum(w s^2 y) / sum(w s^2 s^2); SolveCurvatureSmoothedB then
// y = ln(I_ref / I_obs) = b * (s^2 - <s^2>_frame): the per-frame G already holds each frame's average,
// so b only tilts a frame about its own mean s^2 (FrameMeanS2), and the data term is
// b_c = sum(w s^2 y) / sum(w s^2 s^2) with s^2 taken about that mean; SolveCurvatureSmoothedB then
// smooths it. Finally subtract the leverage-weighted mean: a constant B is a global Wilson-B degenerate
// with overall scale, and cannot change equivalent agreement (within one ASU group every equivalent sits
// at the same s^2, so only b VARYING between batches moves the merge). Drives the per-batch correction.
@@ -2802,7 +2824,7 @@ std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, i
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
const double w = (Is / sc) * (Is / sc);
const double s2 = s2_of(o.d), y = std::log(Iref / Is);
const double s2 = s2_of(o.d) - frame_s2[o.frame], y = std::log(Iref / Is);
a[0][batch_of(o)] += w * s2 * y; a[1][batch_of(o)] += w * s2 * s2;
}, AddArrays<2>);
const std::vector<double> &num = nd[0], &den = nd[1];
@@ -4058,13 +4080,14 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
&& o.partiality >= min_partiality;
};
auto batch_of = [&](const auto &o) { return std::min(n_batch - 1, o.frame / frames_per_batch); };
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
return std::exp(b[batch_of(o)] * s2_of(o.d));
};
// The fields the three curve fits and the three held-out scores below read, pulled out of the fat
// records once, with the group-ordered copy their per-group references run over (BuildDecayTerms).
DecayTerms t;
BuildDecayTerms(n_groups, t);
const std::vector<double> frame_s2 = FrameMeanS2(t);
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
return std::exp(b[batch_of(o)] * (s2_of(o.d) - frame_s2[o.frame]));
};
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
std::vector<double> sw(n_groups), swI(n_groups);
// Held-out disagreement over the group subset {group&1 == gparity} with `b` applied: the same
@@ -4100,7 +4123,7 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
// equivalents (same s^2, different batch) into extra scatter - net harmful, exactly like RefineDecay's
// sub-floor slope. (RELATIVE_B_MIN_SPREAD ~ DECAY_MIN_DELTA_B.)
constexpr double RELATIVE_B_MIN_SPREAD = 2.0; // A^2, minimum peak-to-peak relative-B to engage
const std::vector<double> b_all = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, -1);
const std::vector<double> b_all = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, -1);
const double bmin = *std::min_element(b_all.begin(), b_all.end());
const double bmax = *std::max_element(b_all.begin(), b_all.end());
if (bmax - bmin < RELATIVE_B_MIN_SPREAD) {
@@ -4112,8 +4135,8 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
// per-batch structure generalises across the equivalent split; over-fit per-batch noise does not.
const std::vector<double> zero(n_batch, 0.0);
const double base = subset_disagreement(0, zero) + subset_disagreement(1, zero);
const std::vector<double> b_even = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 0);
const std::vector<double> b_odd = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 1);
const std::vector<double> b_even = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 0);
const std::vector<double> b_odd = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 1);
const double gain = base - (subset_disagreement(1, b_even) + subset_disagreement(0, b_odd));
if (!(gain > CV_MIN_RELATIVE_GAIN * base)) {
logger->Info("Relative-B: not cross-validated ({} batches, peak-to-peak {:.2f} A^2, held-out gain {:.1f}%, skipped)",
@@ -1043,8 +1043,12 @@ private:
// Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset
// {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a
// subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction.
std::vector<double> FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
int frames_per_batch, int gparity) const;
std::vector<double> FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2, int n_groups,
int n_batch, int frames_per_batch, int gparity) const;
// Per frame, the mean s^2 = 1/(4 d^2) of its terms at the weights the per-frame G was fitted at,
// (I/sigma)^2. A relative B is applied about it: the frame's G already holds the frame's average
// scale, so exp(B s^2) alone would move that average a second time.
std::vector<double> FrameMeanS2(const DecayTerms &t) const;
// Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before
// any decay correction and store the first->last relative-B change + the per-batch curve on this object
// (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF).
+4 -2
View File
@@ -59,8 +59,10 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr
c.refine = true;
c.live = true;
c.fast_live = true;
// Decided late, on the whole sweep, as it re-checks its radius there - which fast_live does not.
c.adaptive_integration_radius = false;
// The signal radius from the spot width of the pre-scan frames, as in `rugnux`; where the pre-pass
// counts the widened radius crowding the background rings the canonical pass keeps the default, and
// a near tie stays where it was decided (fast_live re-checks nothing on the whole sweep).
c.adaptive_integration_radius = true;
return s;
}
-2
View File
@@ -3723,8 +3723,6 @@ int RugnuxInputRun::PrepareRugnuxInput() {
config.finalist_ledger = finalist_ledger_flag;
config.live = broker_feed || simulate_live;
config.fast_live = fast_live && config.live;
if (config.fast_live)
config.adaptive_integration_radius = false; // as MakeRugnuxLiveSetup
if (simulate_live)
logger.Info("Simulated live collection: image i readable (i+1) x {:.1f} ms after the run starts",
simulated_frame_time.count() / 1000.0);
+29 -1
View File
@@ -192,6 +192,27 @@ TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationSc
CHECK(r.segment_decay_model == "dose-linear");
}
TEST_CASE("RotationScaleMerge_DecayRampWithinOneSweep", "[RotationScale]") {
// One sweep whose B rises steadily from 0 to 10 A^2: the per-frame G takes each frame's average, the
// decay the tilt about it, so the high-resolution fulls come out on one scale from the first frame to
// the last.
auto data = MakeSweeps({{200, 1.0, 0.0, 10.0}}, 23);
const auto r = Merge(data, SyntheticExperiment(true), {}, true);
auto ratio = [&](int f0, int f1) {
std::vector<double> v;
for (const auto &o: r.scaled_fulls) {
const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l));
if (o.image_number >= static_cast<float>(f0) && o.image_number < static_cast<float>(f1)
&& o.d < 2.2f && t > 200.0)
v.push_back(o.I / t);
}
REQUIRE(v.size() > 20);
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
return v[v.size() / 2];
};
CHECK(ratio(170, 200) / ratio(0, 30) == Catch::Approx(1.0).epsilon(0.08));
}
TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") {
const std::vector<SweepSpec> sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}};
auto data = MakeSweeps(sweeps, 13);
@@ -228,7 +249,7 @@ TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[Rotation
if (!on_gpu)
CHECK(hash == 15900277682318835699ULL);
else if (get_gpu_description().find("RTX 5080") != std::string::npos)
CHECK(hash == 6640579017825061097ULL);
CHECK(hash == 1680029721518244960ULL);
}
TEST_CASE("RotationScaleMerge_ShadowDoesNotBendTheResolutionShape", "[RotationScale]") {
@@ -325,6 +346,13 @@ TEST_CASE("RotationScaleMerge_SecondCrystalStretchRemovedWhole", "[RotationScale
r.image_number += 600.0f;
data.frames.push_back(std::move(o));
}
// A 3% error proportional to the intensity, as every real measurement has: without it the error
// model's b is not resolved, and the merge is taken from the fulls alone, which this test is not about.
std::mt19937 rng(24);
std::normal_distribution<double> noise(0.0, 1.0);
for (auto &o: data.frames)
for (auto &r: o.reflections)
r.I *= static_cast<float>(1.0 + 0.03 * noise(rng));
const auto r = Merge(data, SyntheticExperiment(true), {});
const auto &sq = r.statistics.sweep_quality;
const auto &removed = sq.removed_stretches;