CPU spot finder and rotation prediction: same results, less work per pixel

AdaptiveSpotFinderCPU::AccumulateRingsBlock reads each pixel once for both the
ring histogram and the fused azimuthal profile (was two loops), and no longer
keeps the per-ring integer sums: they are taken from the histogram, as the two
sigma-clip passes already were (ClipRings(INFINITY)). Integer sums, so the same
totals; the profile's float sums keep their pixel order.

FlagRow is branch-free and works a 32-pixel word at a time, so it vectorises;
pixels outside every ring meet a +inf threshold in an extra ring_thr entry.

BraggPredictionRot::Calc takes A*h, A*h + B*k, C*l and 4*S0*S0 out of the inner
loops; p0 is the same ((A*h) + (B*k)) + (C*l) as before.

Measured on cytc (CPU build, both binaries run concurrently on a loaded box):
AccumulateRingsBlock -35%, FlagRow -47%, Calc + Coord ops -30% cycles; whole run
-5% cycles. p.mtz md5 unchanged on myob/cytc/thau, CPU and GPU builds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01K5K8jvPPbmCrbqnWkddTuB
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-03 10:06:05 +02:00
1 parent 2c97e654ed
commit 56f0c5dd0a
3 files changed
+77 -84

No files matched your search

@@ -45,6 +45,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
const Coord m3 = (m1 % m2).Normalize();
const float m2_S0 = m2 * S0;
const float four_S0_sq = 4 * S0 * S0;
const float m3_S0 = m3 * S0;
int i = 0;
@@ -99,17 +100,24 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
cos_phi_limit = std::cos(phi_limit);
}
// p0 = A* h + B* k + C* l, evaluated as ((A* h) + (B* k)) + (C* l) exactly as before, with the
// terms that do not change in the inner loops taken out of them.
std::vector<Coord> Cstar_l(2 * settings.max_l + 1);
for (int l = -settings.max_l; l <= settings.max_l; l++)
Cstar_l[l + settings.max_l] = Cstar * l;
for (int h = -settings.max_h; h <= settings.max_h; h++) {
// Precompute A* h contribution
const Coord Astar_h = Astar * h;
for (int k = -settings.max_k; k <= settings.max_k; k++) {
// Accumulate B* k contribution
const Coord AB = Astar_h + Bstar * k;
for (int l = -settings.max_l; l <= settings.max_l; l++) {
if (systematic_absence(h, k, l, settings.centering))
continue;
Coord p0 = Astar * h + Bstar * k + Cstar * l;
const Coord &Cl = Cstar_l[l + settings.max_l];
const Coord p0(AB.x + Cl.x, AB.y + Cl.y, AB.z + Cl.z);
float p0_sq = p0 * p0;
if (p0_sq <= 0.0f || p0_sq > one_over_dmax_sq)
@@ -129,7 +137,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
};
// No solution for Laue equations
if ((rho_sq < p_m3 * p_m3) || (p0_sq > 4 * S0 * S0))
if ((rho_sq < p_m3 * p_m3) || (p0_sq > four_S0_sq))
continue;
// Effective rocking width for this reflection: mosaicity broadened by the bandwidth
@@ -17,7 +17,7 @@ AdaptiveSpotFinderCPU::AdaptiveSpotFinderCPU(const AzimuthalIntegrationMapping &
ring_cnt.assign(nbins, 0);
ring_mean.assign(nbins, 0.0f);
ring_sigma.assign(nbins, 0.0f);
ring_thr.assign(nbins, 0.0f);
ring_thr.assign(nbins + 1, INFINITY); // the last entry is for pixels outside every ring
ring_bkg.assign(nbins, NAN);
ring_bits.assign(OutputSize(), 0);
ring_hist.assign(nbins * HIST_VALUES, 0);
@@ -54,84 +54,54 @@ void AdaptiveSpotFinderCPU::BeginRings() {
rings_from_blocks = true;
}
// The plain pass over pixels [first, first + n).
// The plain pass over pixels [first, first + n): the histogram of each ring's values, from which
// PlainRings() takes the integer sums, and the fused profile when asked for. One loop reads each pixel
// once for both.
void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n) {
const auto &pixel_to_bin = mapping.GetPixelToBin();
const size_t nbins = ring_sum.size();
const float *corrections = mapping.Corrections().data();
// Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and
// written back when the ring changes: the same additions in the same order, without a store and a
// reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own
// over the block, so that neither loop runs out of registers for its sums.
if (fuse_azint) {
size_t cur = nbins; // the ring held in the locals below; nbins = none
float az_sum = 0.0f, az_sum2 = 0.0f;
uint32_t az_cnt = 0;
for (size_t pxl = first; pxl < first + n; ++pxl) {
const int32_t v = image[pxl];
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
const uint16_t b = pixel_to_bin[pxl];
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
if (b != cur) {
if (cur != nbins) {
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
cur = b;
az_sum = azint_sum[b];
az_sum2 = azint_sum2[b];
az_cnt = azint_count[b];
}
const float val = static_cast<float>(v) * corrections[pxl];
const float val_sq = val * val;
az_sum += val;
az_sum2 += val_sq;
++az_cnt;
}
if (cur != nbins) {
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
}
// Values outside the histogram are listed by a second loop, run only when the block has any: a
// call in this loop would leave the sums in memory again.
uint32_t *hist = ring_hist.data();
// Consecutive pixels mostly share a ring, so the ring's profile sums are held in locals while they
// do and written back when the ring changes: the same additions in the same order, without a store
// and a reload of the same address on every pixel. Values outside the histogram are listed by a
// second loop, run only when the block has any.
bool overflow = false;
size_t cur = nbins;
int64_t sum = 0, cnt = 0;
uint64_t sum2 = 0;
size_t cur = nbins; // the ring held in the locals below; nbins = none
float az_sum = 0.0f, az_sum2 = 0.0f;
uint32_t az_cnt = 0;
for (size_t pxl = first; pxl < first + n; ++pxl) {
const int32_t v = image[pxl];
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
const uint16_t b = pixel_to_bin[pxl];
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
if (b != cur) {
if (cur != nbins) {
ring_sum[cur] = sum;
ring_sum2[cur] = sum2;
ring_cnt[cur] = cnt;
}
cur = b;
sum = ring_sum[b];
sum2 = ring_sum2[b];
cnt = ring_cnt[b];
}
sum += v;
sum2 += static_cast<uint64_t>(static_cast<int64_t>(v) * v);
cnt += 1;
if (v >= 0 && v < HIST_VALUES)
if (static_cast<uint32_t>(v) < HIST_VALUES)
hist[b * HIST_VALUES + v] += 1;
else
overflow = true;
if (!fuse_azint) continue;
if (b != cur) {
if (cur != nbins) {
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
cur = b;
az_sum = azint_sum[b];
az_sum2 = azint_sum2[b];
az_cnt = azint_count[b];
}
const float val = static_cast<float>(v) * corrections[pxl];
const float val_sq = val * val;
az_sum += val;
az_sum2 += val_sq;
++az_cnt;
}
if (cur != nbins) {
ring_sum[cur] = sum;
ring_sum2[cur] = sum2;
ring_cnt[cur] = cnt;
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
if (overflow)
@@ -146,7 +116,8 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
}
// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the
// pixels holding it would, and its pixels are added as a count.
// pixels holding it would, and its pixels are added as a count. Integer sums, so with nothing clipped
// (clip_k = INFINITY) they are the plain sums over the pixels themselves.
void AdaptiveSpotFinderCPU::ClipRings(float clip_k) {
const size_t nbins = ring_sum.size();
@@ -155,6 +126,7 @@ void AdaptiveSpotFinderCPU::ClipRings(float clip_k) {
std::fill(ring_cnt.begin(), ring_cnt.end(), 0);
const auto keep = [&](uint16_t b, int32_t v) {
if (std::isinf(clip_k)) return true;
const float lo = ring_mean[b] - clip_k * ring_sigma[b];
const float hi = ring_mean[b] + clip_k * ring_sigma[b];
return !(v < lo || v > hi); // exclude peaks / outliers
@@ -197,6 +169,7 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image,
AccumulateRingsBlock(image, 0, static_cast<size_t>(width) * height);
}
rings_from_blocks = false;
ClipRings(INFINITY);
UpdateRingStatistics();
ClipRings(3.0f);
UpdateRingStatistics();
@@ -259,19 +232,31 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image,
void AdaptiveSpotFinderCPU::FlagRow(const ImagePreprocessorBuffer &image, int32_t row) {
const auto &pixel_to_bin = mapping.GetPixelToBin();
const size_t nbins = ring_thr.size();
const auto nbins = static_cast<uint32_t>(ring_thr.size() - 1); // ring_thr[nbins] is +inf
const float *thr = ring_thr.data();
const int32_t *img = image.data();
const uint16_t *bin = pixel_to_bin.data();
const size_t first = static_cast<size_t>(row) * width;
const size_t end = first + width;
for (size_t pxl = first; pxl < first + width; ++pxl) {
const int32_t v = image[pxl];
const uint16_t b = pixel_to_bin[pxl];
bool strong = false;
if (v == INT32_MAX)
strong = true;
else if (v != INT32_MIN && b < nbins && v >= ring_thr[b])
strong = true;
if (strong)
ring_bits[pxl / 32] |= 1U << (pxl % 32);
// Saturated is strong, bad is not, and a pixel outside every ring (bin >= nbins) meets the +inf
// threshold. Written without branches and a word of 32 pixels at a time, so that the loop vectorises.
const auto strong = [&](size_t pxl) -> uint32_t {
const int32_t v = img[pxl];
const uint32_t b = std::min<uint32_t>(bin[pxl], nbins);
return (v == INT32_MAX) | ((v != INT32_MIN) & (v >= thr[b]));
};
size_t pxl = first;
while (pxl < end && pxl % 32 != 0) {
ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32);
++pxl;
}
for (; pxl + 32 <= end; pxl += 32) {
uint32_t word = 0;
for (uint32_t j = 0; j < 32; ++j)
word |= strong(pxl + j) << j;
ring_bits[pxl / 32] |= word;
}
for (; pxl < end; ++pxl)
ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32);
}
@@ -57,7 +57,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
std::vector<int64_t> ring_cnt;
std::vector<float> ring_mean;
std::vector<float> ring_sigma;
std::vector<float> ring_thr;
std::vector<float> ring_thr; // nbins + 1: the last is +inf, the threshold of a pixel outside every ring
// ring_mean of the last Detect(), NaN where the ring holds too few pixels to be its own background.
// Kept separately because ring_mean carries the previous frame's value for an empty ring.
std::vector<float> ring_bkg;
@@ -65,8 +65,8 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
// local-box mask that ImageSpotFinderCPU::Detect leaves in output_buffer.
std::vector<uint32_t> ring_bits;
// The plain pass's valid pixels as a per-ring histogram of their values (HIST_VALUES bins per
// ring) plus a list of the values outside it, so the two sigma-clip passes sum over distinct
// values instead of over the image again. Integer sums, so the same totals.
// ring) plus a list of the values outside it. The plain sums and the two sigma-clip passes are all
// taken from it, over distinct values instead of over the image. Integer sums, so the same totals.
static constexpr int32_t HIST_VALUES = 1024;
std::vector<uint32_t> ring_hist;
std::vector<std::pair<uint16_t, int32_t>> ring_overflow; // (ring, value)
@@ -84,7 +84,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
// Zero the sums of the plain ring pass (and of the fused profile).
void ResetRings();
// One sigma-clip pass over the plain pass's values.
// One sigma-clip pass over the plain pass's values; clip_k = INFINITY gives the plain sums.
void ClipRings(float clip_k);
// ring_mean / ring_sigma from the current sums.
void UpdateRingStatistics();