MergeOnTheFly: capture term and multiplicative outlier band for the stills merge (both opt-in)

--capture-uncertainty now reaches the stills merge: each partial carries (c (1 - p) <I>)^2 in its
variance, the rotation combine's capture term per observation, in the merge weights and in the
error-model samples alike. --reject-outliers on stills now uses the rotation merge's band
(OutlierBand.h): exp(+-n s) about the reflection mean for the systematic part, s the weighted median
|ln(I / median)| of the merge's own observations, instead of a symmetric +-n sigma band that reaches
below zero when b is large (stills b 1.4-1.7). A median needs three observations.

Neither is switched on for stills: measured with the run-measured partiality width before its
window-truncation correction (dimple R-free, SwissFEL 4k / SACLA 243 / MicroMAX mono / MicroMAX pink):
no options 0.2567 / 0.2497 / 0.3279 / 0.2477, --capture-uncertainty 1 0.2510 / 0.2470 / 0.3261 /
0.2527, --reject-outliers 6 0.2619 / 0.2487 / 0.3317 / 0.2630 - mixed, and the photon-counting pink
set loses with both. Raising --min-partiality to 0.1 or 0.2 lost 0.008-0.019 on SwissFEL and mono.
Defaults unchanged, so stills output is unchanged; rotation does not use MergeOnTheFly.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-21 19:55:08 +02:00
1 parent 3e0285c425
commit 419db6ba0e
4 files changed
+131 -18

No files matched your search

+50 -15
View File
@@ -19,6 +19,7 @@
#include "HKLKey.h"
#include "RfreeFlags.h"
#include "FrenchWilson.h"
#include "OutlierBand.h"
namespace {
// Which half-set an observation prefers: a splitmix64 bit-mix of the image's stable index, so the
@@ -69,6 +70,7 @@ MergeOnTheFly::MergeOnTheFly(const DiffractionExperiment &x)
? std::optional<double>(scaling_settings.GetMinCCForImage())
: std::nullopt),
min_partiality(scaling_settings.GetMinPartiality()),
capture_coeff(scaling_settings.GetCaptureUncertaintyCoeff()),
generator(scaling_settings.GetMergeFriedel(), space_group),
reject_outliers(scaling_settings.GetOutlierRejectNsigma() > 0.0),
reject_nsigma(scaling_settings.GetOutlierRejectNsigma()) {
@@ -100,18 +102,22 @@ size_t MergeOnTheFly::Contributions(const IntegrationOutcome &outcome, std::vect
continue;
auto hkl = generator(r);
auto hkl_key = hkl.pack();
sigma_corr = CorrectedSigma(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, hkl_key);
sigma_corr = CorrectedSigma(I_corr, sigma_corr, r.image_scale_corr, r.var_bkg, r.partiality, hkl_key);
// Robust outlier rejection: drop this observation if it sits more than
// reject_nsigma error-model sigmas from the reflection's median. Needs the active
// error model so sigma_corr reflects the real scatter (else the threshold is the
// bare counting sigma and would cull good partials).
// Robust outlier rejection: drop this observation if it sits outside the band about the
// reflection's median that the error model allows (OutlierBand.h). Needs the active error model
// so sigma_corr reflects the real scatter (else the threshold is the bare counting sigma and
// would cull good partials).
if (reject_outliers && error_model_active) {
const auto mit = reject_median_I.find(hkl_key);
if (mit != reject_median_I.end() &&
std::fabs(I_corr - mit->second) > reject_nsigma * sigma_corr) {
++rejected;
continue;
if (mit != reject_median_I.end()) {
const double I_b = error_model_mean_I.at(hkl_key);
if (OutsideOutlierBand(I_corr, mit->second, reject_nsigma,
static_cast<double>(sigma_corr) * sigma_corr, I_b,
error_model_b * I_b, reject_up, reject_down)) {
++rejected;
continue;
}
}
}
@@ -188,7 +194,7 @@ void MergeOnTheFly::AddImages(const std::vector<IntegrationOutcome> &outcomes) {
}
float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg,
float var_bkg, float partiality,
uint64_t hkl_key) const {
if (!error_model_active)
return sigma_corr;
@@ -217,6 +223,8 @@ float MergeOnTheFly::CorrectedSigma(float I_corr, float sigma_corr, float image_
if (base > 0.0)
a_var = base;
}
const double capture = capture_coeff * (1.0 - std::min(1.0f, partiality)) * std::max(0.0, I_for_b);
a_var += capture * capture;
const double v = error_model_a * a_var
+ (error_model_b * I_for_b) * (error_model_b * I_for_b);
@@ -239,11 +247,13 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
error_model_chi2 = 0.0;
error_model_mean_I.clear();
reject_median_I.clear();
reject_up = 0.0;
reject_down = 0.0;
reject_count = 0;
// --- 1. Collect accepted, scaled observations grouped by symmetry-equivalent hkl,
// applying exactly the filters AddImage uses. ---
struct Obs { float I, sigma; };
struct Obs { float I, sigma, p; };
struct KeyedObs { uint64_t key; Obs obs; };
std::unordered_map<uint64_t, std::vector<Obs>> groups;
@@ -267,7 +277,7 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
const float sigma_corr = r.sigma * r.image_scale_corr;
if (!std::isfinite(I_corr) || !std::isfinite(sigma_corr) || sigma_corr <= 0.0f)
continue;
out.push_back({generator(r).pack(), {I_corr, sigma_corr}});
out.push_back({generator(r).pack(), {I_corr, sigma_corr, r.partiality}});
}
},
[&](size_t, const std::vector<KeyedObs> &out) {
@@ -281,6 +291,7 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
// (b*I)^2 term uses the reflection mean, so the mean (not I_i) is the abscissa. ---
struct Sample { double s2, I2, dev2; };
std::vector<Sample> samples;
std::vector<std::pair<double, double>> log_spread; // |ln(I / median)|, weight
for (const auto &[key, obs]: groups) {
if (obs.size() < 2)
@@ -296,8 +307,9 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
const double mean = sum_wI / sum_w;
error_model_mean_I[key] = static_cast<float>(mean);
// Robust centre for outlier rejection: the median intensity (resists the very
// outliers the inverse-variance mean is being protected from). Only when active.
if (reject_outliers) {
// outliers the inverse-variance mean is being protected from). Only when active, and only
// where there are three observations for a median to stand on.
if (reject_outliers && obs.size() >= 3) {
std::vector<float> iv;
iv.reserve(obs.size());
for (const auto &o: obs)
@@ -312,10 +324,33 @@ void MergeOnTheFly::RefineErrorModel(const std::vector<IntegrationOutcome> &outc
if (factor < 0.05)
continue;
const double resid = static_cast<double>(o.I) - mean;
samples.push_back({static_cast<double>(o.sigma) * o.sigma, I2, resid * resid / factor});
const double capture = capture_coeff * (1.0 - std::min(1.0f, o.p)) * std::max(0.0, mean);
const double s2 = static_cast<double>(o.sigma) * o.sigma + capture * capture;
samples.push_back({s2, I2, resid * resid / factor});
const auto mit = reject_median_I.find(key);
if (mit != reject_median_I.end() && o.I > 0.0f && mit->second > 0.0f && mean > 0.0)
log_spread.emplace_back(std::fabs(std::log(o.I / mit->second)), I2 / s2);
}
}
// The spread of ln(I / median) the band's multiplicative part is drawn from: its weighted median
// absolute value, weighted by (<I>/sigma)^2 so the observations whose ratio counting noise does not
// blur carry it - as the rotation merge measures it.
if (!log_spread.empty()) {
std::sort(log_spread.begin(), log_spread.end());
double total = 0.0;
for (const auto &[v, w]: log_spread) total += w;
double run = 0.0;
size_t j = 0;
for (; j + 1 < log_spread.size(); ++j) {
run += log_spread[j].second;
if (run >= 0.5 * total) break;
}
const double s = 1.4826 * log_spread[j].first;
reject_up = std::exp(reject_nsigma * s) - 1.0;
reject_down = 1.0 - std::exp(-reject_nsigma * s);
}
// --- 3. Fit global dev2 = a*sigma^2 + b^2*<I>^2. Bin by intensity (the per-observation
// dev2 is chi-square-1 noisy) and take medians; weight the bins by 1/dev2^2 so it
// is a *relative* fit - otherwise the strong bins (which fix b) swamp the weak
+7 -1
View File
@@ -294,6 +294,9 @@ class MergeOnTheFly {
// merge, the error model and MergeStats can never disagree about which images are in.
bool filter_by_image_cc = false;
double min_partiality = 0.02;
// The share of a partial the model could not see, (1 - p), carried in its variance as
// (capture_coeff * (1 - p) * <I>)^2 - the rotation combine's capture term, per observation.
double capture_coeff = 0.0;
// When set, ice-ring-flagged reflections are left out of this merge. Used for the P1 pass whose
// merged intensities feed the space-group search and the error model - those model fits must not
@@ -319,7 +322,7 @@ class MergeOnTheFly {
// using the per-observation I_i instead would over-weight down-fluctuated points.
std::unordered_map<uint64_t, float> error_model_mean_I;
[[nodiscard]] float CorrectedSigma(float I_corr, float sigma_corr, float image_scale_corr,
float var_bkg,
float var_bkg, float partiality,
uint64_t hkl_key) const;
// Optional per-observation outlier rejection: drop observations whose corrected
@@ -330,6 +333,9 @@ class MergeOnTheFly {
bool reject_outliers = false;
double reject_nsigma = 6.0;
std::unordered_map<uint64_t, float> reject_median_I;
// The asymmetric band of OutlierBand.h: exp(+-n s) - 1 about the reflection mean, s the spread of
// ln(I / median) measured on this merge's observations.
double reject_up = 0.0, reject_down = 0.0;
size_t reject_count = 0;
bool Mask(const IntegrationOutcome &outcome) const;
+2 -2
View File
@@ -229,11 +229,11 @@ void print_usage() {
std::cout << " --ice-min-score <num> Ice-presence gate: measured ice score (1 = no ice) a run must reach before ANY ice handling is applied - the flagging and the exclusion from scaling (default: 1.5). The eleven hexagonal bands cover 16-26% of the unique reflections whether or not the crystal has ice, so handling ice on a clean crystal is a pure loss. 0 = no gate (always handle ice)" << std::endl;
std::cout << " --ice-min-spot-ratio <num> Second ice-presence channel: found spots on the hexagonal rings over the same q width of ice-free flanks beside them (1 = spots spread evenly). Ice in large crystallites diffracts as discrete spots and leaves the radial profile flat, so --ice-min-score alone is blind to it. Default 2.0; 0 disables this channel" << std::endl;
std::cout << " --min-partiality <num> Minimum partiality to accept reflection (default: 0.02)" << std::endl;
std::cout << " --capture-uncertainty <num> rot3d: systematic sigma ~num*(1-captured_fraction)*I on under-captured fulls (default: 1.0 for rot3d, 0 otherwise)" << std::endl;
std::cout << " --capture-uncertainty <num> Systematic sigma num*(1-captured fraction)*<I>: on rot3d fulls, the share of the rocking curve not captured; on stills, the share of each partial its partiality does not see (default: 1.0 for rot3d, 0 otherwise)" << std::endl;
std::cout << " --min-captured-fraction <num> rot3d: drop a combined full whose rocking curve was captured below this fraction (edge-of-sweep truncated fulls) (default: 0.7 for rotation, 0 otherwise; 0 = off)" << std::endl;
std::cout << " --mosaicity <num> Diagnostic: fix the scaling mosaicity (deg) instead of the per-image seed" << std::endl;
std::cout << " --prediction-mosaicity <num> Diagnostic: fix the rocking width the PREDICTION window opens to (deg), leaving the partiality on the per-image sigma_M. Prediction and partiality are one number by default, so a sigma_M that moves takes the integrated reflection population with it" << std::endl;
std::cout << " --reject-outliers <num> Per-observation merge outlier rejection, N sigma from the per-reflection median (default: 6 for rot3d, XDS/DIALS-style; 0 = off)" << std::endl;
std::cout << " --reject-outliers <num> Per-observation merge outlier rejection, N sigma from the per-reflection median, the systematic part of the band multiplicative (default: 6 for rot3d, XDS/DIALS-style; 0 = off)" << std::endl;
std::cout << " --search-min-zeta <num> De-novo space-group search only: also search a merge of just the observations whose Lorentz geometry |zeta| reaches this, and report both answers (default: 0.85 for rotation, 0 = single search). Reflections crossing the Ewald sphere near-tangentially are measured worst and can make a real symmetry operator look like a twin law; where the two searches disagree, the merge of ALL the observations decides" << std::endl;
std::cout << " --min-image-cc <num> Per-image CC limit in percent (default: no limit)" << std::endl;
std::cout << " --scaling-iterations <num> Cap on the per-frame scaling iterations; the loop stops earlier once the scales settle (default: 100)" << std::endl;
+72
View File
@@ -956,3 +956,75 @@ TEST_CASE("StillsPartialityRefine_MeasureRunWidth") {
CHECK(width->gamma_e == Catch::Approx(GAMMA_E).epsilon(0.1));
CHECK(width->bandwidth_sigma < 1e-3);
}
// Outlier rejection on the stills merge: one observation at ten times its reflection's intensity among
// nineteen consistent ones is dropped, and nothing else is.
TEST_CASE("MergeOnTheFly_RejectsAnOutlierInTheBand") {
DiffractionExperiment x;
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
ScalingSettings s = x.GetScalingSettings();
s.OutlierRejectNsigma(6.0);
x.ImportScalingSettings(s);
std::mt19937 rng(3);
std::normal_distribution<float> gauss(0.0f, 1.0f);
std::vector<IntegrationOutcome> outcomes(20);
for (size_t i = 0; i < outcomes.size(); ++i) {
for (int h = 1; h <= 50; ++h) {
Reflection r{};
r.h = h; r.k = 2; r.l = 3;
r.d = 2.0f;
const float I_true = 100.0f * static_cast<float>(h);
r.I = I_true * (1.0f + 0.1f * gauss(rng)) + 3.0f * gauss(rng);
if (i == 7 && h == 25)
r.I = 10.0f * I_true;
r.sigma = 3.0f;
r.prescaling_corr = 1.0f;
r.partiality = 1.0f;
r.image_scale_corr = 1.0f;
outcomes[i].reflections.push_back(r);
}
}
MergeOnTheFly merge(x);
merge.RefineErrorModel(outcomes);
REQUIRE(merge.ErrorModelActive());
merge.AddImages(outcomes);
CHECK(merge.RejectedCount() == 1);
}
// The capture term widens the sigma of a partial by (c (1 - p) <I>): a partial merges at a lower weight
// than a full of the same intensity, and not at all differently with c = 0.
TEST_CASE("MergeOnTheFly_CaptureTermDownweightsPartials") {
auto merged_I = [](double c) {
DiffractionExperiment x;
x.SetSpaceGroup(gemmi::get_spacegroup_by_name("P 1"));
ScalingSettings s = x.GetScalingSettings();
s.CaptureUncertaintyCoeff(c);
x.ImportScalingSettings(s);
std::mt19937 rng(9);
std::normal_distribution<float> gauss(0.0f, 1.0f);
std::vector<IntegrationOutcome> outcomes(40);
for (size_t i = 0; i < outcomes.size(); ++i)
for (int h = 1; h <= 40; ++h) {
Reflection r{};
r.h = h; r.k = 1; r.l = 1;
r.d = 2.0f;
// Every other image records the reflection as a 30 % partial that reads 20 % high.
const bool partial = i % 2 == 1;
r.partiality = partial ? 0.3f : 1.0f;
r.image_scale_corr = 1.0f / r.partiality;
r.I = r.partiality * 100.0f * static_cast<float>(h) * (partial ? 1.2f : 1.0f) * (1.0f + 0.05f * gauss(rng));
r.sigma = 2.0f;
r.prescaling_corr = 1.0f;
outcomes[i].reflections.push_back(r);
}
MergeOnTheFly merge(x);
merge.RefineErrorModel(outcomes);
merge.AddImages(outcomes);
const auto m = merge.ExportReflections();
return m.at(10).I / (100.0f * static_cast<float>(m.at(10).h));
};
const float without = merged_I(0.0), with = merged_I(1.0);
CHECK(with < without);
CHECK(with < 1.08f);
}