Merge rc187-cytc-proc: decay/relative-B about each frame's mean s^2, live integrates at the adaptive radius as offline
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
commit
7b8196e770
7 files changed
+80
-25
No files matched your search
+1
-1
@@ -162,7 +162,7 @@ With the input already on disk, the URL is ignored.
|
||||
| `-t, --stride <num>` | Process every *n*-th image (default: 1) |
|
||||
| `-v, --verbose` | Show the whole log on the terminal in place of the steps; `<prefix>_log.txt` is written either way |
|
||||
| `--simulate-live <ms>` | Read a stored input as the collection it was while being written: image *i* becomes readable (*i*+1)·`ms` ms after the start (0: all at once). The run takes the live path a broker URL takes, with the same result; for testing the live workflow without a broker. `--mode mx` only |
|
||||
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in, the integration radius is fixed, and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
|
||||
| `--fast-live` | With `--simulate-live` or a broker URL: run as the broker's rugnux live mode does — every decision stands where it was first taken, nothing is re-checked or run again once the sweep is in (the integration radius from the spot width stays where the pre-pass frames put it), and SIGINT cuts the run to what it has established. A difficult crystal can come out worse |
|
||||
| `--sweeps joint\|separate` | Several rotation sweeps (see Inputs): `joint` (the default) merges them as one dataset into `<prefix>.*`; `separate` processes each alone into `<prefix>_s<k>_*` |
|
||||
| `--no-gpu` | Run on the CPU only, as on a machine without a GPU, even where one is present: CUDA is not touched (nor, on macOS, the Apple GPU that Bragg integration, the model density and the molecular-replacement rotation function otherwise run on) and every GPU step takes its CPU path (`-X FFT` indexes with FFTW and `-X FFBIDX` with its CPU implementation; `-X cuFFT` is refused) |
|
||||
|
||||
|
||||
@@ -1470,8 +1470,8 @@ AnisotropyResult AnalyzeAnisotropy(const std::vector<MergedReflection> &merged,
|
||||
"reachable by a lab-fixed systematic and reduces the power of the test");
|
||||
if (std::isfinite(run.radiation_damage_relative_b) && std::fabs(run.radiation_damage_relative_b) > 10.0)
|
||||
result.cautions.emplace_back(
|
||||
"the run carries substantial radiation damage; the scale model removes its average "
|
||||
"monotone part, but a non-monotone dose ramp needs --relative-b to be taken out too");
|
||||
"the run carries substantial radiation damage; the scale model takes out a decay only "
|
||||
"where it cross-validates, so part of the damage can remain in the fall-off");
|
||||
// R2's Laue-class caution, fired on its measured signature rather than on the Laue class alone.
|
||||
// Too high a symmetry assignment pushes real anisotropy into the directions the systematic error is
|
||||
// measured in, so sigma_sys is inflated by the signal itself and the gate collapses (measured: a
|
||||
|
||||
@@ -2367,10 +2367,26 @@ void RotationScaleMerge::BuildDecayTerms(int n_groups, DecayTerms &t) const {
|
||||
for (const DecayTerm &d : t.term) t.gterm[fill[d.group]++] = d;
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::FrameMeanS2(const DecayTerms &t) const {
|
||||
std::vector<double> sw(n_frames, 0.0), sws2(n_frames, 0.0);
|
||||
for (const DecayTerm &o : t.term) {
|
||||
if (!(o.I > 0.0f)) continue;
|
||||
const double w = (static_cast<double>(o.I) / o.sigma) * (static_cast<double>(o.I) / o.sigma);
|
||||
sw[o.frame] += w;
|
||||
sws2[o.frame] += w / (4.0 * static_cast<double>(o.d) * o.d);
|
||||
}
|
||||
std::vector<double> m(n_frames, 0.0);
|
||||
for (int f = 0; f < n_frames; ++f)
|
||||
if (sw[f] > 0.0) m[f] = sws2[f] / sw[f];
|
||||
return m;
|
||||
}
|
||||
|
||||
void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
// Radiation damage weakens later frames more at higher resolution - a resolution x time (Debye-Waller)
|
||||
// systematic the resolution-flat per-frame G leaves in. Model it as a single global relative-B rate:
|
||||
// ln(I_ref / I_obs) = slope * (frame - frame_center) * s^2, s^2 = 1/(4 d^2), slope = 2 dB/dframe,
|
||||
// ln(I_ref / I_obs) = slope * (frame - frame_center) * (s^2 - <s^2>_frame), s^2 = 1/(4 d^2),
|
||||
// slope = 2 dB/dframe, with s^2 about the frame's own mean because the per-frame G already holds the
|
||||
// frame's average scale - applied about s^2 = 0 the correction would move that average a second time,
|
||||
// fitted (inverse-variance-of-log weighted) against the merged reference.
|
||||
//
|
||||
// The fit is dominated by strong LOW-res reflections and EXTRAPOLATED to high s^2, so a slope fitted to
|
||||
@@ -2395,8 +2411,11 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
return segments.empty() ? static_cast<double>(o.image_number)
|
||||
: frame_dose[o.frame] + (o.image_number - static_cast<float>(o.frame));
|
||||
};
|
||||
// s^2 about the frame's mean (FrameMeanS2): the per-frame G has the frame's average already.
|
||||
std::vector<double> frame_s2;
|
||||
auto s2c = [&](const auto &o) { return s2_of(o.d) - frame_s2[o.frame]; };
|
||||
auto decay_factor = [&](const auto &o, double slope) {
|
||||
return std::exp(slope * (dose(o) - fcenter) * s2_of(o.d));
|
||||
return std::exp(slope * (dose(o) - fcenter) * s2c(o));
|
||||
};
|
||||
|
||||
// The fields the seven passes below read (three slope fits, four held-out scores), pulled out of the
|
||||
@@ -2413,6 +2432,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
decay_terms_kept_valid = false;
|
||||
if (!segments.empty())
|
||||
MeasureSegmentDose(t, n_groups);
|
||||
frame_s2 = FrameMeanS2(t);
|
||||
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
|
||||
std::vector<double> sw(n_groups), swI(n_groups);
|
||||
|
||||
@@ -2440,7 +2460,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double x = (dose(o) - fcenter) * s2_of(o.d), y = std::log(Iref / Is);
|
||||
const double x = (dose(o) - fcenter) * s2c(o), y = std::log(Iref / Is);
|
||||
a[0] += w; a[1] += w * x; a[2] += w * y; a[3] += w * x * x; a[4] += w * x * y;
|
||||
}, [](Lin &a, const Lin &b) { for (int j = 0; j < 5; ++j) a[j] += b[j]; });
|
||||
const double Sw = S[0], Sx = S[1], Sy = S[2], Sxx = S[3], Sxy = S[4];
|
||||
@@ -2602,7 +2622,7 @@ void RotationScaleMerge::RefineDecay(int n_groups) {
|
||||
for (int i = lo; i < hi; ++i) {
|
||||
Obs &o = fulls[i];
|
||||
if (!usable(o)) continue;
|
||||
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2_of(o.d))
|
||||
const double f = use_step ? std::exp(2.0 * (b[frame_segment[o.frame]] - b_mid) * s2c(o))
|
||||
: decay_factor(o, slope);
|
||||
o.corr = static_cast<float>(o.corr * f);
|
||||
}
|
||||
@@ -2767,12 +2787,14 @@ std::vector<double> RotationScaleMerge::SolveCurvatureSmoothedB(const std::vecto
|
||||
return b;
|
||||
}
|
||||
|
||||
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
|
||||
int frames_per_batch, int gparity) const {
|
||||
std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2,
|
||||
int n_groups, int n_batch, int frames_per_batch,
|
||||
int gparity) const {
|
||||
// Fit one relative-B per batch over the ASU-group subset {group&1 == gparity} (gparity < 0 = all fulls),
|
||||
// against an inverse-variance reference built from that same subset (no leakage). Per batch the model is
|
||||
// y = ln(I_ref / I_obs) = b * s^2 through the origin (the resolution-flat intercept is the per-frame G,
|
||||
// already applied), so the data term is b_c = sum(w s^2 y) / sum(w s^2 s^2); SolveCurvatureSmoothedB then
|
||||
// y = ln(I_ref / I_obs) = b * (s^2 - <s^2>_frame): the per-frame G already holds each frame's average,
|
||||
// so b only tilts a frame about its own mean s^2 (FrameMeanS2), and the data term is
|
||||
// b_c = sum(w s^2 y) / sum(w s^2 s^2) with s^2 taken about that mean; SolveCurvatureSmoothedB then
|
||||
// smooths it. Finally subtract the leverage-weighted mean: a constant B is a global Wilson-B degenerate
|
||||
// with overall scale, and cannot change equivalent agreement (within one ASU group every equivalent sits
|
||||
// at the same s^2, so only b VARYING between batches moves the merge). Drives the per-batch correction.
|
||||
@@ -2802,7 +2824,7 @@ std::vector<double> RotationScaleMerge::FitRelativeBCurve(const DecayTerms &t, i
|
||||
const double Is = static_cast<double>(o.I) * o.corr, sc = static_cast<double>(o.sigma) * o.corr;
|
||||
if (!std::isfinite(Iref) || Iref <= 0.0 || !(Is > 0.0)) return;
|
||||
const double w = (Is / sc) * (Is / sc);
|
||||
const double s2 = s2_of(o.d), y = std::log(Iref / Is);
|
||||
const double s2 = s2_of(o.d) - frame_s2[o.frame], y = std::log(Iref / Is);
|
||||
a[0][batch_of(o)] += w * s2 * y; a[1][batch_of(o)] += w * s2 * s2;
|
||||
}, AddArrays<2>);
|
||||
const std::vector<double> &num = nd[0], &den = nd[1];
|
||||
@@ -4058,13 +4080,14 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
&& o.partiality >= min_partiality;
|
||||
};
|
||||
auto batch_of = [&](const auto &o) { return std::min(n_batch - 1, o.frame / frames_per_batch); };
|
||||
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
|
||||
return std::exp(b[batch_of(o)] * s2_of(o.d));
|
||||
};
|
||||
// The fields the three curve fits and the three held-out scores below read, pulled out of the fat
|
||||
// records once, with the group-ordered copy their per-group references run over (BuildDecayTerms).
|
||||
DecayTerms t;
|
||||
BuildDecayTerms(n_groups, t);
|
||||
const std::vector<double> frame_s2 = FrameMeanS2(t);
|
||||
auto b_factor = [&](const auto &o, const std::vector<double> &b) {
|
||||
return std::exp(b[batch_of(o)] * (s2_of(o.d) - frame_s2[o.frame]));
|
||||
};
|
||||
const int nt = static_cast<int>(ThreadsForWork(t.term.size(), nthreads));
|
||||
std::vector<double> sw(n_groups), swI(n_groups);
|
||||
// Held-out disagreement over the group subset {group&1 == gparity} with `b` applied: the same
|
||||
@@ -4100,7 +4123,7 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
// equivalents (same s^2, different batch) into extra scatter - net harmful, exactly like RefineDecay's
|
||||
// sub-floor slope. (RELATIVE_B_MIN_SPREAD ~ DECAY_MIN_DELTA_B.)
|
||||
constexpr double RELATIVE_B_MIN_SPREAD = 2.0; // A^2, minimum peak-to-peak relative-B to engage
|
||||
const std::vector<double> b_all = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, -1);
|
||||
const std::vector<double> b_all = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, -1);
|
||||
const double bmin = *std::min_element(b_all.begin(), b_all.end());
|
||||
const double bmax = *std::max_element(b_all.begin(), b_all.end());
|
||||
if (bmax - bmin < RELATIVE_B_MIN_SPREAD) {
|
||||
@@ -4112,8 +4135,8 @@ void RotationScaleMerge::RefineRelativeB(int n_groups) {
|
||||
// per-batch structure generalises across the equivalent split; over-fit per-batch noise does not.
|
||||
const std::vector<double> zero(n_batch, 0.0);
|
||||
const double base = subset_disagreement(0, zero) + subset_disagreement(1, zero);
|
||||
const std::vector<double> b_even = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 0);
|
||||
const std::vector<double> b_odd = FitRelativeBCurve(t, n_groups, n_batch, frames_per_batch, 1);
|
||||
const std::vector<double> b_even = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 0);
|
||||
const std::vector<double> b_odd = FitRelativeBCurve(t, frame_s2, n_groups, n_batch, frames_per_batch, 1);
|
||||
const double gain = base - (subset_disagreement(1, b_even) + subset_disagreement(0, b_odd));
|
||||
if (!(gain > CV_MIN_RELATIVE_GAIN * base)) {
|
||||
logger->Info("Relative-B: not cross-validated ({} batches, peak-to-peak {:.2f} A^2, held-out gain {:.1f}%, skipped)",
|
||||
|
||||
@@ -1043,8 +1043,12 @@ private:
|
||||
// Fit a smoothed per-batch relative-B curve (A^2 per batch) on the fulls over the ASU-group subset
|
||||
// {group&1==gparity} (gparity<0 = all): the weighted s^2 slope of ln(Iref/Iobs) per batch against a
|
||||
// subset-global reference, smoothed and zero-mean-anchored. Drives the per-batch correction.
|
||||
std::vector<double> FitRelativeBCurve(const DecayTerms &t, int n_groups, int n_batch,
|
||||
int frames_per_batch, int gparity) const;
|
||||
std::vector<double> FitRelativeBCurve(const DecayTerms &t, const std::vector<double> &frame_s2, int n_groups,
|
||||
int n_batch, int frames_per_batch, int gparity) const;
|
||||
// Per frame, the mean s^2 = 1/(4 d^2) of its terms at the weights the per-frame G was fitted at,
|
||||
// (I/sigma)^2. A relative B is applied about it: the frame's G already holds the frame's average
|
||||
// scale, so exp(B s^2) alone would move that average a second time.
|
||||
std::vector<double> FrameMeanS2(const DecayTerms &t) const;
|
||||
// Radiation-damage MONITOR (report-only): measure the per-batch relative-B on the scaled fulls before
|
||||
// any decay correction and store the first->last relative-B change + the per-batch curve on this object
|
||||
// (copied into the result statistics by MergeAndStats, then printed / logged / written to the mmCIF).
|
||||
|
||||
@@ -59,8 +59,10 @@ RugnuxLiveSetup MakeRugnuxLiveSetup(const JFJochReaderDataset &dataset, int nthr
|
||||
c.refine = true;
|
||||
c.live = true;
|
||||
c.fast_live = true;
|
||||
// Decided late, on the whole sweep, as it re-checks its radius there - which fast_live does not.
|
||||
c.adaptive_integration_radius = false;
|
||||
// The signal radius from the spot width of the pre-scan frames, as in `rugnux`; where the pre-pass
|
||||
// counts the widened radius crowding the background rings the canonical pass keeps the default, and
|
||||
// a near tie stays where it was decided (fast_live re-checks nothing on the whole sweep).
|
||||
c.adaptive_integration_radius = true;
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
@@ -3723,8 +3723,6 @@ int RugnuxInputRun::PrepareRugnuxInput() {
|
||||
config.finalist_ledger = finalist_ledger_flag;
|
||||
config.live = broker_feed || simulate_live;
|
||||
config.fast_live = fast_live && config.live;
|
||||
if (config.fast_live)
|
||||
config.adaptive_integration_radius = false; // as MakeRugnuxLiveSetup
|
||||
if (simulate_live)
|
||||
logger.Info("Simulated live collection: image i readable (i+1) x {:.1f} ms after the run starts",
|
||||
simulated_frame_time.count() / 1000.0);
|
||||
|
||||
@@ -192,6 +192,27 @@ TEST_CASE("RotationScaleMergeSegments_DoseLinearDecayAcrossSweeps", "[RotationSc
|
||||
CHECK(r.segment_decay_model == "dose-linear");
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMerge_DecayRampWithinOneSweep", "[RotationScale]") {
|
||||
// One sweep whose B rises steadily from 0 to 10 A^2: the per-frame G takes each frame's average, the
|
||||
// decay the tilt about it, so the high-resolution fulls come out on one scale from the first frame to
|
||||
// the last.
|
||||
auto data = MakeSweeps({{200, 1.0, 0.0, 10.0}}, 23);
|
||||
const auto r = Merge(data, SyntheticExperiment(true), {}, true);
|
||||
auto ratio = [&](int f0, int f1) {
|
||||
std::vector<double> v;
|
||||
for (const auto &o: r.scaled_fulls) {
|
||||
const double t = data.I_true.at(FriedelKey(o.h, o.k, o.l));
|
||||
if (o.image_number >= static_cast<float>(f0) && o.image_number < static_cast<float>(f1)
|
||||
&& o.d < 2.2f && t > 200.0)
|
||||
v.push_back(o.I / t);
|
||||
}
|
||||
REQUIRE(v.size() > 20);
|
||||
std::nth_element(v.begin(), v.begin() + v.size() / 2, v.end());
|
||||
return v[v.size() / 2];
|
||||
};
|
||||
CHECK(ratio(170, 200) / ratio(0, 30) == Catch::Approx(1.0).epsilon(0.08));
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMergeSegments_SegmentsMustCoverTheFrames", "[RotationScale]") {
|
||||
const std::vector<SweepSpec> sweeps{{10, 1.0, 0.0}, {10, 1.0, 0.0}};
|
||||
auto data = MakeSweeps(sweeps, 13);
|
||||
@@ -228,7 +249,7 @@ TEST_CASE("RotationScaleMergeSegments_NoSegmentsIsTheEngineAsBefore", "[Rotation
|
||||
if (!on_gpu)
|
||||
CHECK(hash == 15900277682318835699ULL);
|
||||
else if (get_gpu_description().find("RTX 5080") != std::string::npos)
|
||||
CHECK(hash == 6640579017825061097ULL);
|
||||
CHECK(hash == 1680029721518244960ULL);
|
||||
}
|
||||
|
||||
TEST_CASE("RotationScaleMerge_ShadowDoesNotBendTheResolutionShape", "[RotationScale]") {
|
||||
@@ -325,6 +346,13 @@ TEST_CASE("RotationScaleMerge_SecondCrystalStretchRemovedWhole", "[RotationScale
|
||||
r.image_number += 600.0f;
|
||||
data.frames.push_back(std::move(o));
|
||||
}
|
||||
// A 3% error proportional to the intensity, as every real measurement has: without it the error
|
||||
// model's b is not resolved, and the merge is taken from the fulls alone, which this test is not about.
|
||||
std::mt19937 rng(24);
|
||||
std::normal_distribution<double> noise(0.0, 1.0);
|
||||
for (auto &o: data.frames)
|
||||
for (auto &r: o.reflections)
|
||||
r.I *= static_cast<float>(1.0 + 0.03 * noise(rng));
|
||||
const auto r = Merge(data, SyntheticExperiment(true), {});
|
||||
const auto &sq = r.statistics.sweep_quality;
|
||||
const auto &removed = sq.removed_stretches;
|
||||
|
||||
Reference in new issue
Block a user