mx/Jungfraujoch
3
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 81 Wiki Activity

RotationScaleMerge: fit the error model on the device in the GPU build (tier F)

Browse Source 
FitErrorModelGPU puts the samples in the host's rank order with four stable
radix passes, so the equal-count bins hold the same samples; computes every
normalised deviation with the host's rounding, so every bin median is the
host's to the bit; and hands the per-bin terms to the host's own update step,
now ErrorModelUpdate (a pure extraction from FitErrorModel). The one thing
that differs is the order each bin's two sums are added in: a fixed tree on the
device, the order nth_element happened to leave the bin in on the host. a and
b^2 can therefore differ from the host fit in their last bits - measured on two
synthetic pools: relative difference 2e-16 and 3e-13 in a, 5e-16 in b^2. The
device fit is deterministic (same pool, same bits).

p.mtz md5 nevertheless unchanged on myob, cytc, 8a1a and 8qaw (GPU build): the
difference does not survive the float rounding of the merged sigmas there. A
knife-edge decision elsewhere could still see it, which is why this is kept as
its own commit.

The host fit took 0.4-0.6 s per fit on the large merges, about half of it the
initial equal-count split, with only 16-way parallelism in the iterations.
Device peak about 72 bytes per sample (the rank sort; each stage's buffers are
freed when it is done); the free memory is checked first and a device without
it stops with a message naming the CPU build.

Kept last on the branch so that it can be dropped on its own.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
This commit is contained in:
leonarski_f
2026-10-08 08:52:04 +02:00
co-authored by Claude Opus 5.5
parent 6e17ae36c8
commit 87f0bdc870
7 changed files with 304 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image_analysis/scale_merge/ErrorModel.cpp
+48 -45
View File
@@ -12,7 +12,7 @@
ErrorModelFit FitErrorModel(const std::vector<ErrorModelSample> &pool, std::vector<ErrorModelBinned> &smp,
size_t nthreads) {
constexpr int n_bins = 16;
constexpr int n_bins = ERROR_MODEL_BINS;
ErrorModelFit fit;
if (pool.size() < static_cast<size_t>(8 * n_bins))
return fit;
@@ -63,16 +63,12 @@ ErrorModelFit FitErrorModel(const std::vector<ErrorModelSample> &pool, std::vect
// hands b the bins' own noise instead (measured: b = 5.6, ISa 0.5, on a crystal XDS gives 6.1). Then
// fit a alone and leave b at zero. Over the rotation battery the two crystals this fires on sit at
// (I/sigma)^2 0.22 and 0.84 while the next is 31.7, so the threshold is not delicate.
constexpr double B_LEVER_MIN = 4.0;
{
const auto top_mid = smp.begin() + (bin_lo(n_bins - 1) + bin_hi(n_bins - 1)) / 2;
std::nth_element(smp.begin() + bin_lo(n_bins - 1), top_mid, smp.end(), by_snr);
fit.b_measured = top_mid->snr2 > B_LEVER_MIN;
fit.b_measured = top_mid->snr2 > ERROR_MODEL_B_LEVER_MIN;
}
// The median of a chi^2 with one degree of freedom: a normalised squared deviation that is right
// on average has half of its values below this.
constexpr double CHI2_1_MEDIAN = 0.454936;
std::vector<double> bs2(n_bins), bI2(n_bins), bd2(n_bins);
double a = 1.0, b2 = 0.0;
for (int iter = 0; iter < 30; ++iter) {
@@ -93,48 +89,55 @@ ErrorModelFit FitErrorModel(const std::vector<ErrorModelSample> &pool, std::vect
const auto mid = smp.begin() + bin_lo(bin) + n / 2;
std::nth_element(smp.begin() + bin_lo(bin), mid, smp.begin() + bin_hi(bin),
[](const ErrorModelBinned &p, const ErrorModelBinned &q) { return p.z2 < q.z2; });
bs2[bin] = s; bI2[bin] = I; bd2[bin] = mid->z2 / CHI2_1_MEDIAN * static_cast<double>(n);
bs2[bin] = s; bI2[bin] = I; bd2[bin] = mid->z2 / ERROR_MODEL_CHI2_1_MEDIAN * static_cast<double>(n);
}
});
// Every bin weighs the same in RELATIVE terms, so the weak bins still constrain a.
double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double w = 1.0 / (bd2[bin] * bd2[bin]);
Ass += w * bs2[bin] * bs2[bin]; AsI += w * bs2[bin] * bI2[bin]; AII += w * bI2[bin] * bI2[bin];
Bs += w * bs2[bin] * bd2[bin]; BI += w * bI2[bin] * bd2[bin];
}
const double det = Ass * AII - AsI * AsI;
double a_new, b2_new;
if (!fit.b_measured) {
if (!(Ass > 0.0)) return fit;
a_new = std::clamp(Bs / Ass, 0.25, 100.0);
b2_new = 0.0;
} else {
if (!(det > 1e-10 * Ass * AII)) return fit;
const double a_fit = (Bs * AII - BI * AsI) / det;
const double b2_fit = (Ass * BI - AsI * Bs) / det;
a_new = std::clamp(a_fit, 0.25, 100.0);
b2_new = std::max(b2_fit, 0.0);
// Leverage is not enough: b^2 can have it and still come out at zero within its own error,
// and 1/b then reads the noise in b as an I/sigma. The standard error of b^2 comes from the
// bins' own scatter about the fitted line; this only decides what is printed.
double chi = 0.0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double r = (bd2[bin] - a_fit * bs2[bin] - b2_fit * bI2[bin]) / bd2[bin];
chi += r * r;
}
fit.b_resolved = b2_fit >= 2.0 * std::sqrt(chi / (n_bins - 2) * Ass / det);
}
const bool settled = std::fabs(a_new - a) <= 1e-4 * a && std::fabs(b2_new - b2) <= 1e-4 * b2;
a = a_new;
b2 = b2_new;
fit.active = true;
fit.a = a;
fit.b2 = b2;
if (settled) break;
if (!ErrorModelUpdate(bs2, bI2, bd2, a, b2, fit))
break;
}
return fit;
}
bool ErrorModelUpdate(const std::vector<double> &bs2, const std::vector<double> &bI2, const std::vector<double> &bd2,
double &a, double &b2, ErrorModelFit &fit) {
const int n_bins = static_cast<int>(bs2.size());
// Every bin weighs the same in RELATIVE terms, so the weak bins still constrain a.
double Ass = 0, AsI = 0, AII = 0, Bs = 0, BI = 0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double w = 1.0 / (bd2[bin] * bd2[bin]);
Ass += w * bs2[bin] * bs2[bin]; AsI += w * bs2[bin] * bI2[bin]; AII += w * bI2[bin] * bI2[bin];
Bs += w * bs2[bin] * bd2[bin]; BI += w * bI2[bin] * bd2[bin];
}
const double det = Ass * AII - AsI * AsI;
double a_new, b2_new;
if (!fit.b_measured) {
if (!(Ass > 0.0)) return false;
a_new = std::clamp(Bs / Ass, 0.25, 100.0);
b2_new = 0.0;
} else {
if (!(det > 1e-10 * Ass * AII)) return false;
const double a_fit = (Bs * AII - BI * AsI) / det;
const double b2_fit = (Ass * BI - AsI * Bs) / det;
a_new = std::clamp(a_fit, 0.25, 100.0);
b2_new = std::max(b2_fit, 0.0);
// Leverage is not enough: b^2 can have it and still come out at zero within its own error,
// and 1/b then reads the noise in b as an I/sigma. The standard error of b^2 comes from the
// bins' own scatter about the fitted line; this only decides what is printed.
double chi = 0.0;
for (int bin = 0; bin < n_bins; ++bin) {
if (!(bd2[bin] > 0.0)) continue;
const double r = (bd2[bin] - a_fit * bs2[bin] - b2_fit * bI2[bin]) / bd2[bin];
chi += r * r;
}
fit.b_resolved = b2_fit >= 2.0 * std::sqrt(chi / (n_bins - 2) * Ass / det);
}
const bool settled = std::fabs(a_new - a) <= 1e-4 * a && std::fabs(b2_new - b2) <= 1e-4 * b2;
a = a_new;
b2 = b2_new;
fit.active = true;
fit.a = a;
fit.b2 = b2;
return !settled;
}