CPU pixel loops: keep the ring sums and the vertical slide out of memory for real

Two fixes found in the post-merge profile, where the previous commit's loops did not do what they
were written to do:

- AccumulateRingsBlock: the ring_overflow.emplace_back call inside the loop left no callee-saved
  registers for the locals, so GCC kept sum/sum2/count on the stack and the store-reload chain was
  still there (29% of the function's samples on one stack add). The azint sums get a loop of their own
  over the block, and the out-of-histogram values are listed by a second loop run only when the block
  has any, in the same pixel order; now the sums live in registers. Same additions in the same order.
- DetectPass: inlined into DetectPass's main loop the vertical slide was not vectorised (the
  vectorised copies GCC made were for the other call sites). It is now a free function,
  SlideVertical, which GCC vectorises on its own.

Measured (perf, 75 s of myob, CPU-only, relative to the unchanged FlagRow/AnalyzeBlock):
AccumulateRingsBlock -4..-11%, DetectPass + SlideVertical -7..-14%. Byte-identical p.hkl, p.mtz,
p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob (GPU); ImageSpotFinderCPU*,
AdaptiveSpotFinder, SpotFinding, AzimuthalIntegration and portable tests pass.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-09-28 03:57:10 +02:00
1 parent a277b8c458
commit 8e6ecedd5c
2 files changed
+83 -44

No files matched your search

@@ -60,48 +60,65 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
const size_t nbins = ring_sum.size();
const float *corrections = mapping.Corrections().data();
// Consecutive pixels mostly share a ring, so the ring's sums are held in locals while they do and
// Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and
// written back when the ring changes: the same additions in the same order, without a store and a
// reload of the same address on every pixel.
uint32_t *hist = ring_hist.data();
size_t cur = nbins; // the ring held in the locals below; nbins = none
int64_t sum = 0, cnt = 0;
uint64_t sum2 = 0;
float az_sum = 0.0f, az_sum2 = 0.0f;
uint32_t az_cnt = 0;
const auto store = [&] {
if (cur == nbins)
return;
ring_sum[cur] = sum;
ring_sum2[cur] = sum2;
ring_cnt[cur] = cnt;
if (fuse_azint) {
// reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own
// over the block, so that neither loop runs out of registers for its sums.
if (fuse_azint) {
size_t cur = nbins; // the ring held in the locals below; nbins = none
float az_sum = 0.0f, az_sum2 = 0.0f;
uint32_t az_cnt = 0;
for (size_t pxl = first; pxl < first + n; ++pxl) {
const int32_t v = image[pxl];
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
const uint16_t b = pixel_to_bin[pxl];
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
if (b != cur) {
if (cur != nbins) {
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
cur = b;
az_sum = azint_sum[b];
az_sum2 = azint_sum2[b];
az_cnt = azint_count[b];
}
const float val = static_cast<float>(v) * corrections[pxl];
const float val_sq = val * val;
az_sum += val;
az_sum2 += val_sq;
++az_cnt;
}
if (cur != nbins) {
azint_sum[cur] = az_sum;
azint_sum2[cur] = az_sum2;
azint_count[cur] = az_cnt;
}
};
}
// Values outside the histogram are listed by a second loop, run only when the block has any: a
// call in this loop would leave the sums in memory again.
uint32_t *hist = ring_hist.data();
bool overflow = false;
size_t cur = nbins;
int64_t sum = 0, cnt = 0;
uint64_t sum2 = 0;
for (size_t pxl = first; pxl < first + n; ++pxl) {
const int32_t v = image[pxl];
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
const uint16_t b = pixel_to_bin[pxl];
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
if (b != cur) {
store();
if (cur != nbins) {
ring_sum[cur] = sum;
ring_sum2[cur] = sum2;
ring_cnt[cur] = cnt;
}
cur = b;
sum = ring_sum[b];
sum2 = ring_sum2[b];
cnt = ring_cnt[b];
az_sum = azint_sum[b];
az_sum2 = azint_sum2[b];
az_cnt = azint_count[b];
}
if (fuse_azint) {
const float val = static_cast<float>(v) * corrections[pxl];
const float val_sq = val * val;
az_sum += val;
az_sum2 += val_sq;
++az_cnt;
}
sum += v;
sum2 += static_cast<uint64_t>(static_cast<int64_t>(v) * v);
@@ -109,9 +126,23 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
if (v >= 0 && v < HIST_VALUES)
hist[b * HIST_VALUES + v] += 1;
else
ring_overflow.emplace_back(b, v);
overflow = true;
}
store();
if (cur != nbins) {
ring_sum[cur] = sum;
ring_sum2[cur] = sum2;
ring_cnt[cur] = cnt;
}
if (overflow)
for (size_t pxl = first; pxl < first + n; ++pxl) {
const int32_t v = image[pxl];
if (v == INT32_MIN || v == INT32_MAX) continue;
const uint16_t b = pixel_to_bin[pxl];
if (b >= nbins) continue;
if (v < 0 || v >= HIST_VALUES)
ring_overflow.emplace_back(b, v);
}
}
// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the
@@ -45,6 +45,24 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid,
// pixel is above SNR threshold
}
// Adds row `in` to the vertical sums and takes row `out` out of them; nullptr is no row. A bad or
// saturated pixel adds 0. Written without branches, and as a function of its own so that the loop
// vectorises whatever it is inlined into; integer sums, so the same totals as pixel by pixel.
void SlideVertical(const int32_t *in, const int32_t *out, int32_t width,
int64_t *sum, int64_t *sum2, uint16_t *valid) {
for (int32_t col = 0; col < width; col++) {
const int64_t a = in ? in[col] : INT32_MIN;
const int64_t r = out ? out[col] : INT32_MIN;
const bool a_ok = a != INT32_MAX && a != INT32_MIN;
const bool r_ok = r != INT32_MAX && r != INT32_MIN;
const int64_t a_v = a_ok ? a : 0;
const int64_t r_v = r_ok ? r : 0;
sum[col] += a_v - r_v;
sum2[col] += a_v * a_v - r_v * r_v;
valid[col] += static_cast<uint16_t>(a_ok - r_ok);
}
}
} // namespace
void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings,
@@ -164,27 +182,17 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image,
std::vector<uint16_t> valid_vert(width, 0);
// Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is
// no row. A bad or saturated pixel adds 0. Written without branches, so the loop vectorises.
// A pixel strong in the previous pass is not counted either; it is added here with the rest
// of its row and taken out again below - there are few, and the sums are integers, so the
// totals are the same as never adding it.
// no row. A pixel strong in the previous pass is not counted either; SlideVertical adds it with
// the rest of its row and it is taken out again below - there are few, and the sums are
// integers, so the totals are the same as never adding it.
const int32_t *img = image.data();
const int32_t w = width;
int64_t *sv = sum_vert.data();
int64_t *sv2 = sum2_vert.data();
uint16_t *vv = valid_vert.data();
auto slide_vert = [&](int line_in, int line_out) {
for (int32_t col = 0; col < w; col++) {
const int64_t in = line_in >= 0 ? img[line_in * w + col] : INT32_MIN;
const int64_t out = line_out >= 0 ? img[line_out * w + col] : INT32_MIN;
const bool in_ok = in != INT32_MAX && in != INT32_MIN;
const bool out_ok = out != INT32_MAX && out != INT32_MIN;
const int64_t in_v = in_ok ? in : 0;
const int64_t out_v = out_ok ? out : 0;
sv[col] += in_v - out_v;
sv2[col] += in_v * in_v - out_v * out_v;
vv[col] += static_cast<uint16_t>(in_ok - out_ok);
}
SlideVertical(line_in >= 0 ? img + static_cast<size_t>(line_in) * w : nullptr,
line_out >= 0 ? img + static_cast<size_t>(line_out) * w : nullptr, w, sv, sv2, vv);
if (!prev_strong)
return;
// sign = +1 takes out what line_in added, -1 puts back what line_out took out.