From 8e6ecedd5c5762ce7cd6f30767555d65810bd629 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 28 Sep 2026 03:57:10 +0200 Subject: [PATCH] CPU pixel loops: keep the ring sums and the vertical slide out of memory for real Two fixes found in the post-merge profile, where the previous commit's loops did not do what they were written to do: - AccumulateRingsBlock: the ring_overflow.emplace_back call inside the loop left no callee-saved registers for the locals, so GCC kept sum/sum2/count on the stack and the store-reload chain was still there (29% of the function's samples on one stack add). The azint sums get a loop of their own over the block, and the out-of-histogram values are listed by a second loop run only when the block has any, in the same pixel order; now the sums live in registers. Same additions in the same order. - DetectPass: inlined into DetectPass's main loop the vertical slide was not vectorised (the vectorised copies GCC made were for the other call sites). It is now a free function, SlideVertical, which GCC vectorises on its own. Measured (perf, 75 s of myob, CPU-only, relative to the unchanged FlagRow/AnalyzeBlock): AccumulateRingsBlock -4..-11%, DetectPass + SlideVertical -7..-14%. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob (GPU); ImageSpotFinderCPU*, AdaptiveSpotFinder, SpotFinding, AzimuthalIntegration and portable tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../spot_finding/AdaptiveSpotFinderCPU.cpp | 89 +++++++++++++------ .../spot_finding/ImageSpotFinderCPU.cpp | 38 ++++---- 2 files changed, 83 insertions(+), 44 deletions(-) diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 4a8107b22..62474e7e3 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -60,48 +60,65 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer & const size_t nbins = ring_sum.size(); const float *corrections = mapping.Corrections().data(); - // Consecutive pixels mostly share a ring, so the ring's sums are held in locals while they do and + // Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and // written back when the ring changes: the same additions in the same order, without a store and a - // reload of the same address on every pixel. - uint32_t *hist = ring_hist.data(); - size_t cur = nbins; // the ring held in the locals below; nbins = none - int64_t sum = 0, cnt = 0; - uint64_t sum2 = 0; - float az_sum = 0.0f, az_sum2 = 0.0f; - uint32_t az_cnt = 0; - const auto store = [&] { - if (cur == nbins) - return; - ring_sum[cur] = sum; - ring_sum2[cur] = sum2; - ring_cnt[cur] = cnt; - if (fuse_azint) { + // reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own + // over the block, so that neither loop runs out of registers for its sums. + if (fuse_azint) { + size_t cur = nbins; // the ring held in the locals below; nbins = none + float az_sum = 0.0f, az_sum2 = 0.0f; + uint32_t az_cnt = 0; + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; // masked / out of range (UINT16_MAX) + if (b != cur) { + if (cur != nbins) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + cur = b; + az_sum = azint_sum[b]; + az_sum2 = azint_sum2[b]; + az_cnt = azint_count[b]; + } + const float val = static_cast(v) * corrections[pxl]; + const float val_sq = val * val; + az_sum += val; + az_sum2 += val_sq; + ++az_cnt; + } + if (cur != nbins) { azint_sum[cur] = az_sum; azint_sum2[cur] = az_sum2; azint_count[cur] = az_cnt; } - }; + } + + // Values outside the histogram are listed by a second loop, run only when the block has any: a + // call in this loop would leave the sums in memory again. + uint32_t *hist = ring_hist.data(); + bool overflow = false; + size_t cur = nbins; + int64_t sum = 0, cnt = 0; + uint64_t sum2 = 0; for (size_t pxl = first; pxl < first + n; ++pxl) { const int32_t v = image[pxl]; if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated const uint16_t b = pixel_to_bin[pxl]; if (b >= nbins) continue; // masked / out of range (UINT16_MAX) if (b != cur) { - store(); + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } cur = b; sum = ring_sum[b]; sum2 = ring_sum2[b]; cnt = ring_cnt[b]; - az_sum = azint_sum[b]; - az_sum2 = azint_sum2[b]; - az_cnt = azint_count[b]; - } - if (fuse_azint) { - const float val = static_cast(v) * corrections[pxl]; - const float val_sq = val * val; - az_sum += val; - az_sum2 += val_sq; - ++az_cnt; } sum += v; sum2 += static_cast(static_cast(v) * v); @@ -109,9 +126,23 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer & if (v >= 0 && v < HIST_VALUES) hist[b * HIST_VALUES + v] += 1; else - ring_overflow.emplace_back(b, v); + overflow = true; } - store(); + if (cur != nbins) { + ring_sum[cur] = sum; + ring_sum2[cur] = sum2; + ring_cnt[cur] = cnt; + } + + if (overflow) + for (size_t pxl = first; pxl < first + n; ++pxl) { + const int32_t v = image[pxl]; + if (v == INT32_MIN || v == INT32_MAX) continue; + const uint16_t b = pixel_to_bin[pxl]; + if (b >= nbins) continue; + if (v < 0 || v >= HIST_VALUES) + ring_overflow.emplace_back(b, v); + } } // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the diff --git a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp index ea7549f2f..1cb6773f9 100644 --- a/image_analysis/spot_finding/ImageSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/ImageSpotFinderCPU.cpp @@ -45,6 +45,24 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid, // pixel is above SNR threshold } +// Adds row `in` to the vertical sums and takes row `out` out of them; nullptr is no row. A bad or +// saturated pixel adds 0. Written without branches, and as a function of its own so that the loop +// vectorises whatever it is inlined into; integer sums, so the same totals as pixel by pixel. +void SlideVertical(const int32_t *in, const int32_t *out, int32_t width, + int64_t *sum, int64_t *sum2, uint16_t *valid) { + for (int32_t col = 0; col < width; col++) { + const int64_t a = in ? in[col] : INT32_MIN; + const int64_t r = out ? out[col] : INT32_MIN; + const bool a_ok = a != INT32_MAX && a != INT32_MIN; + const bool r_ok = r != INT32_MAX && r != INT32_MIN; + const int64_t a_v = a_ok ? a : 0; + const int64_t r_v = r_ok ? r : 0; + sum[col] += a_v - r_v; + sum2[col] += a_v * a_v - r_v * r_v; + valid[col] += static_cast(a_ok - r_ok); + } +} + } // namespace void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings, @@ -164,27 +182,17 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image, std::vector valid_vert(width, 0); // Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is - // no row. A bad or saturated pixel adds 0. Written without branches, so the loop vectorises. - // A pixel strong in the previous pass is not counted either; it is added here with the rest - // of its row and taken out again below - there are few, and the sums are integers, so the - // totals are the same as never adding it. + // no row. A pixel strong in the previous pass is not counted either; SlideVertical adds it with + // the rest of its row and it is taken out again below - there are few, and the sums are + // integers, so the totals are the same as never adding it. const int32_t *img = image.data(); const int32_t w = width; int64_t *sv = sum_vert.data(); int64_t *sv2 = sum2_vert.data(); uint16_t *vv = valid_vert.data(); auto slide_vert = [&](int line_in, int line_out) { - for (int32_t col = 0; col < w; col++) { - const int64_t in = line_in >= 0 ? img[line_in * w + col] : INT32_MIN; - const int64_t out = line_out >= 0 ? img[line_out * w + col] : INT32_MIN; - const bool in_ok = in != INT32_MAX && in != INT32_MIN; - const bool out_ok = out != INT32_MAX && out != INT32_MIN; - const int64_t in_v = in_ok ? in : 0; - const int64_t out_v = out_ok ? out : 0; - sv[col] += in_v - out_v; - sv2[col] += in_v * in_v - out_v * out_v; - vv[col] += static_cast(in_ok - out_ok); - } + SlideVertical(line_in >= 0 ? img + static_cast(line_in) * w : nullptr, + line_out >= 0 ? img + static_cast(line_out) * w : nullptr, w, sv, sv2, vv); if (!prev_strong) return; // sign = +1 takes out what line_in added, -1 puts back what line_out took out.