leonarski_fandClaude Opus 5.5
70859bcb60
Exact (tier E): the same observations in the same order, the same verdicts. - The judged observations are compacted in fixed blocks (count, prefix, fill) instead of a serial push_back. - Both sorts (by d, by reflection) sort (key, index) records, so a comparison reads the two elements it compares instead of two observations elsewhere in memory; same total order. - Each reflection is decided on its own observations, so the reflections go side by side in claimed blocks; the rejection count is an integer sum. Measured (prototype, GPU): Wilson test 8tyy 3.4 -> 3.0 s, 8a1a 2.8 -> 2.6 s per run (the sort-on-records part is not in the measured binary yet); md5 identical. Clean branch rebuilt from scratch (GPU and CPU) and re-verified: p.mtz md5 identical to production on myob/cytc/thau/8a1a/8tyy/8qaw/9gdj (GPU), cytc -N 4, myob/cytc and myob -N 4 (CPU). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi
238 lines
12 KiB
C++
238 lines
12 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "WilsonOutliers.h"
|
|
|
|
#include "../../common/ParallelFor.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <numeric>
|
|
|
|
WilsonOutlierResult WilsonOutliers(const std::vector<WilsonObservation> &obs, double alpha, size_t nthreads) {
|
|
WilsonOutlierResult out;
|
|
out.rejected.assign(obs.size(), 0);
|
|
out.e2.assign(obs.size(), NAN);
|
|
|
|
// The observations that can be judged, in index order: fixed blocks count theirs, a prefix places
|
|
// them, and every block writes its own stretch.
|
|
const auto judged = [&](size_t i) {
|
|
const auto &o = obs[i];
|
|
return o.unit >= 0 && o.d > 0.0f && o.d < WILSON_OUTLIER_D_MAX && std::isfinite(o.I)
|
|
&& o.sigma > 0.0f && std::isfinite(o.sigma);
|
|
};
|
|
const int n_obs = static_cast<int>(obs.size());
|
|
const int nb = ReductionBlocks(n_obs, 16384);
|
|
const auto block_begin = [&](int b) { return static_cast<size_t>(static_cast<int64_t>(n_obs) * b / nb); };
|
|
std::vector<size_t> block_at(nb + 1, 0);
|
|
ParallelFor(nb, nthreads, [&](int b) {
|
|
size_t c = 0;
|
|
for (size_t i = block_begin(b); i < block_begin(b + 1); ++i) c += judged(i) ? 1 : 0;
|
|
block_at[b + 1] = c;
|
|
});
|
|
for (int b = 0; b < nb; ++b) block_at[b + 1] += block_at[b];
|
|
std::vector<size_t> idx(nb > 0 ? block_at[nb] : 0);
|
|
ParallelFor(nb, nthreads, [&](int b) {
|
|
size_t k = block_at[b];
|
|
for (size_t i = block_begin(b); i < block_begin(b + 1); ++i)
|
|
if (judged(i)) idx[k++] = i;
|
|
});
|
|
if (idx.empty())
|
|
return out;
|
|
|
|
// alpha / N per observation, half of it to each tail. Wilson's acentric law is P(E^2 > t) = exp(-t)
|
|
// and the centric one P(E^2 > t) = erfc(sqrt(t/2)) <= exp(-t/2), so t and 2t; the Gaussian tail is
|
|
// P(> z sigma) <= exp(-z^2/2) / 2. Both bounds are the conservative side of the exact quantile.
|
|
// Following Wilson (1949) Acta Cryst. 2, 318-321; the significance guard as in Aimless, Evans (2006)
|
|
// Acta Cryst. D62, 72-82.
|
|
const double p = alpha / (2.0 * static_cast<double>(idx.size()));
|
|
const double t = std::log(1.0 / p);
|
|
const double z = std::sqrt(2.0 * std::log(1.0 / (2.0 * p)));
|
|
|
|
// <I/epsilon> in shells of equal observation count, low to high resolution. Two thousand observations
|
|
// put the shell mean at 2% (an exponential's sd equals its mean); narrow shells keep the fall-off
|
|
// across one from reading as spread.
|
|
constexpr size_t OBS_PER_SHELL = 2000;
|
|
// Sorted with the key beside the index, so a comparison reads the two elements it compares rather
|
|
// than two observations somewhere else in memory; the same order.
|
|
{
|
|
struct ByD { float d; size_t i; };
|
|
std::vector<ByD> by_d(idx.size());
|
|
ParallelChunks(static_cast<int>(idx.size()), nthreads, [&](int lo, int hi) {
|
|
for (int j = lo; j < hi; ++j) by_d[j] = {obs[idx[j]].d, idx[j]};
|
|
});
|
|
ParallelSort(by_d.begin(), by_d.end(), nthreads, [](const ByD &a, const ByD &b) {
|
|
return a.d != b.d ? a.d > b.d : a.i < b.i;
|
|
});
|
|
ParallelChunks(static_cast<int>(idx.size()), nthreads, [&](int lo, int hi) {
|
|
for (int j = lo; j < hi; ++j) idx[j] = by_d[j].i;
|
|
});
|
|
}
|
|
const size_t n_shells = std::max<size_t>(1, idx.size() / OBS_PER_SHELL);
|
|
// The observations in that order, so the walks below read them one after another instead of each
|
|
// through idx, and where each shell starts: shell s is the j with j * n_shells / N == s, which begins
|
|
// at the first j with j * n_shells >= s * N. Each shell is then one task, and every sum below is still
|
|
// taken over its shell in the order of j, so the split changes no rounding.
|
|
std::vector<WilsonObservation> so(idx.size());
|
|
ParallelChunks(static_cast<int>(idx.size()), nthreads, [&](int lo, int hi) {
|
|
for (int j = lo; j < hi; ++j) so[j] = obs[idx[j]];
|
|
});
|
|
std::vector<size_t> shell_start(n_shells + 1);
|
|
for (size_t sh = 0; sh <= n_shells; ++sh)
|
|
shell_start[sh] = (sh * idx.size() + n_shells - 1) / n_shells;
|
|
const int n_shell_tasks = static_cast<int>(n_shells);
|
|
const auto lower_e2 = [&](const WilsonObservation &o, double mu) {
|
|
return (o.I - z * o.sigma) / (o.epsilon * mu);
|
|
};
|
|
const auto wilson_bound = [&](const WilsonObservation &o) { return o.centric ? 2.0 * t : t; };
|
|
// A shell is judged only where its mean is itself established at the same significance: with no
|
|
// measured <I>, nothing in the shell can be called improbable against it.
|
|
std::vector<double> mu(n_shells, 0.0);
|
|
std::vector<uint8_t> shell_valid(n_shells, 0);
|
|
ParallelFor(n_shell_tasks, nthreads, [&](int sh) {
|
|
double sum = 0.0;
|
|
size_t cnt = 0;
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j) {
|
|
sum += so[j].I / so[j].epsilon;
|
|
++cnt;
|
|
}
|
|
// Once more without the observations the plain Wilson bound already calls improbable: a single
|
|
// artefact thousands of times <I> would otherwise raise the mean of its own shell several-fold
|
|
// and hide the smaller artefacts sharing it.
|
|
double sum_kept = 0.0, sum2_kept = 0.0;
|
|
size_t cnt_kept = 0;
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j) {
|
|
const auto &o = so[j];
|
|
if (sum > 0.0 && lower_e2(o, sum / cnt) > wilson_bound(o))
|
|
continue;
|
|
const double x = o.I / o.epsilon;
|
|
sum_kept += x; sum2_kept += x * x; ++cnt_kept;
|
|
}
|
|
if (cnt_kept < 2) return;
|
|
const double n = static_cast<double>(cnt_kept);
|
|
mu[sh] = sum_kept / n;
|
|
const double var = std::max(0.0, (sum2_kept - n * mu[sh] * mu[sh]) / (n - 1.0));
|
|
shell_valid[sh] = mu[sh] > 0.0 && mu[sh] > z * std::sqrt(var / n);
|
|
});
|
|
|
|
// The tail's scale, peaks over threshold: above any threshold an exponential's excess is the same
|
|
// exponential, so the median excess of the acentric observations over a common threshold u is
|
|
// k ln 2. u is where Wilson leaves a hundred acentric observations above it. Only shells whose
|
|
// typical observation is significant at u take part: where sigma is comparable to u <I>, what
|
|
// exceeds u is the noise tail, and it would read as a heavy intensity tail.
|
|
size_t n_acentric = 0;
|
|
for (size_t sh = 0; sh < n_shells; ++sh)
|
|
if (shell_valid[sh])
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j)
|
|
if (!so[j].centric) ++n_acentric;
|
|
const double u = std::max(0.0, std::log(static_cast<double>(n_acentric) / 100.0));
|
|
std::vector<uint8_t> shell_measures_tail(n_shells, 0);
|
|
ParallelFor(n_shell_tasks, nthreads, [&](int sh) {
|
|
if (!shell_valid[sh]) return;
|
|
std::vector<double> v; // sigma / (epsilon <I/epsilon>)
|
|
v.reserve(shell_start[sh + 1] - shell_start[sh]);
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j)
|
|
v.push_back(so[j].sigma / (so[j].epsilon * mu[sh]));
|
|
if (v.empty()) return;
|
|
const size_t mid = v.size() / 2;
|
|
std::nth_element(v.begin(), v.begin() + mid, v.end());
|
|
shell_measures_tail[sh] = z * v[mid] <= u;
|
|
});
|
|
std::vector<double> excess;
|
|
for (size_t sh = 0; sh < n_shells; ++sh) {
|
|
if (!shell_measures_tail[sh]) continue;
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j) {
|
|
const auto &o = so[j];
|
|
if (o.centric) continue;
|
|
const double e2 = o.I / (o.epsilon * mu[sh]);
|
|
if (e2 > u) excess.push_back(e2 - u);
|
|
}
|
|
}
|
|
if (!excess.empty()) {
|
|
const size_t mid = excess.size() / 2;
|
|
std::nth_element(excess.begin(), excess.begin() + mid, excess.end());
|
|
out.tail_scale = std::max(1.0, excess[mid] / std::log(2.0));
|
|
}
|
|
out.bound = out.tail_scale * t;
|
|
|
|
// Which observations are improbable, and the observations of each reflection (in valid shells).
|
|
// Each observation is written once, at its own index, so the shells can do it side by side; the
|
|
// reflections are then formed by a sort on a total order, whatever order tested was filled in.
|
|
std::vector<uint8_t> improbable(obs.size(), 0);
|
|
ParallelFor(n_shell_tasks, nthreads, [&](int sh) {
|
|
if (!shell_valid[sh]) return;
|
|
for (size_t j = shell_start[sh]; j < shell_start[sh + 1]; ++j) {
|
|
const auto &o = so[j];
|
|
out.e2[idx[j]] = static_cast<float>(o.I / (o.epsilon * mu[sh]));
|
|
improbable[idx[j]] = lower_e2(o, mu[sh]) > out.tail_scale * wilson_bound(o);
|
|
}
|
|
});
|
|
std::vector<size_t> tested;
|
|
for (size_t sh = 0; sh < n_shells; ++sh)
|
|
if (shell_valid[sh])
|
|
tested.insert(tested.end(), idx.begin() + shell_start[sh], idx.begin() + shell_start[sh + 1]);
|
|
out.n_tested = tested.size();
|
|
{
|
|
struct ByUnit { int32_t unit; size_t i; };
|
|
std::vector<ByUnit> by_unit(tested.size());
|
|
ParallelChunks(static_cast<int>(tested.size()), nthreads, [&](int lo, int hi) {
|
|
for (int j = lo; j < hi; ++j) by_unit[j] = {obs[tested[j]].unit, tested[j]};
|
|
});
|
|
ParallelSort(by_unit.begin(), by_unit.end(), nthreads, [](const ByUnit &a, const ByUnit &b) {
|
|
return a.unit != b.unit ? a.unit < b.unit : a.i < b.i;
|
|
});
|
|
ParallelChunks(static_cast<int>(tested.size()), nthreads, [&](int lo, int hi) {
|
|
for (int j = lo; j < hi; ++j) tested[j] = by_unit[j].i;
|
|
});
|
|
}
|
|
|
|
// An improbable observation measured once is rejected. One with company is rejected when most of
|
|
// its reflection's other observations are probable, unclipped witnesses and it disagrees with their
|
|
// mean beyond the errors - for a pair, the other member. Where most are large, the reflection is
|
|
// (Aimless's "keep if most observations are large").
|
|
// Each reflection is decided on its own observations alone, so the reflections go side by side:
|
|
// where each one's stretch of `tested` starts, then the stretches in blocks.
|
|
const int n_tested = static_cast<int>(tested.size());
|
|
std::vector<uint8_t> starts_unit(n_tested);
|
|
ParallelChunks(n_tested, ThreadsForWork(tested.size(), nthreads), [&](int lo, int hi) {
|
|
for (int q = lo; q < hi; ++q)
|
|
starts_unit[q] = q == 0 || obs[tested[q]].unit != obs[tested[q - 1]].unit;
|
|
});
|
|
std::vector<int32_t> unit_start;
|
|
for (int q = 0; q < n_tested; ++q)
|
|
if (starts_unit[q]) unit_start.push_back(q);
|
|
const int n_units = static_cast<int>(unit_start.size());
|
|
unit_start.push_back(n_tested);
|
|
std::vector<size_t> rejected_in_block(ReductionBlocks(n_units, 1024), 0);
|
|
ParallelBlocks(n_units, nthreads, [&](int blk, int u0, int u1) {
|
|
for (int u = u0; u < u1; ++u) {
|
|
const size_t lo = unit_start[u], hi = unit_start[u + 1];
|
|
for (size_t q = lo; q < hi; ++q) {
|
|
const size_t i = tested[q];
|
|
if (!improbable[i]) continue;
|
|
size_t n_probable = 0;
|
|
double sw = 0.0, swI = 0.0;
|
|
for (size_t r = lo; r < hi; ++r) {
|
|
const auto &o = obs[tested[r]];
|
|
if (r == q || improbable[tested[r]] || o.clipped) continue;
|
|
const double w = 1.0 / (static_cast<double>(o.sigma) * o.sigma);
|
|
sw += w; swI += w * o.I;
|
|
++n_probable;
|
|
}
|
|
const size_t n_others = hi - lo - 1;
|
|
bool reject = n_others == 0;
|
|
if (2 * n_probable > n_others) {
|
|
const auto &o = obs[i];
|
|
reject = std::fabs(o.I - swI / sw) > z * std::sqrt(static_cast<double>(o.sigma) * o.sigma + 1.0 / sw);
|
|
}
|
|
if (reject) {
|
|
out.rejected[i] = 1;
|
|
++rejected_in_block[blk];
|
|
}
|
|
}
|
|
}
|
|
}, 1024);
|
|
for (const size_t r : rejected_in_block) out.n_rejected += r;
|
|
return out;
|
|
}
|