From 56f0c5dd0acca9aed6793b4606771772fcfd2f85 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 3 Oct 2026 10:06:05 +0200 Subject: [PATCH] CPU spot finder and rotation prediction: same results, less work per pixel AdaptiveSpotFinderCPU::AccumulateRingsBlock reads each pixel once for both the ring histogram and the fused azimuthal profile (was two loops), and no longer keeps the per-ring integer sums: they are taken from the histogram, as the two sigma-clip passes already were (ClipRings(INFINITY)). Integer sums, so the same totals; the profile's float sums keep their pixel order. FlagRow is branch-free and works a 32-pixel word at a time, so it vectorises; pixels outside every ring meet a +inf threshold in an extra ring_thr entry. BraggPredictionRot::Calc takes A*h, A*h + B*k, C*l and 4*S0*S0 out of the inner loops; p0 is the same ((A*h) + (B*k)) + (C*l) as before. Measured on cytc (CPU build, both binaries run concurrently on a loaded box): AccumulateRingsBlock -35%, FlagRow -47%, Calc + Coord ops -30% cycles; whole run -5% cycles. p.mtz md5 unchanged on myob/cytc/thau, CPU and GPU builds. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01K5K8jvPPbmCrbqnWkddTuB --- .../bragg_prediction/BraggPredictionRot.cpp | 16 +- .../spot_finding/AdaptiveSpotFinderCPU.cpp | 137 ++++++++---------- .../spot_finding/AdaptiveSpotFinderCPU.h | 8 +- 3 files changed, 77 insertions(+), 84 deletions(-) diff --git a/image_analysis/bragg_prediction/BraggPredictionRot.cpp b/image_analysis/bragg_prediction/BraggPredictionRot.cpp index 33b022a6c..e9deb1428 100644 --- a/image_analysis/bragg_prediction/BraggPredictionRot.cpp +++ b/image_analysis/bragg_prediction/BraggPredictionRot.cpp @@ -45,6 +45,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys const Coord m3 = (m1 % m2).Normalize(); const float m2_S0 = m2 * S0; + const float four_S0_sq = 4 * S0 * S0; const float m3_S0 = m3 * S0; int i = 0; @@ -99,17 +100,24 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys cos_phi_limit = std::cos(phi_limit); } + // p0 = A* h + B* k + C* l, evaluated as ((A* h) + (B* k)) + (C* l) exactly as before, with the + // terms that do not change in the inner loops taken out of them. + std::vector Cstar_l(2 * settings.max_l + 1); + for (int l = -settings.max_l; l <= settings.max_l; l++) + Cstar_l[l + settings.max_l] = Cstar * l; + for (int h = -settings.max_h; h <= settings.max_h; h++) { - // Precompute A* h contribution + const Coord Astar_h = Astar * h; for (int k = -settings.max_k; k <= settings.max_k; k++) { - // Accumulate B* k contribution + const Coord AB = Astar_h + Bstar * k; for (int l = -settings.max_l; l <= settings.max_l; l++) { if (systematic_absence(h, k, l, settings.centering)) continue; - Coord p0 = Astar * h + Bstar * k + Cstar * l; + const Coord &Cl = Cstar_l[l + settings.max_l]; + const Coord p0(AB.x + Cl.x, AB.y + Cl.y, AB.z + Cl.z); float p0_sq = p0 * p0; if (p0_sq <= 0.0f || p0_sq > one_over_dmax_sq) @@ -129,7 +137,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys }; // No solution for Laue equations - if ((rho_sq < p_m3 * p_m3) || (p0_sq > 4 * S0 * S0)) + if ((rho_sq < p_m3 * p_m3) || (p0_sq > four_S0_sq)) continue; // Effective rocking width for this reflection: mosaicity broadened by the bandwidth diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp index 62474e7e3..995df923b 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.cpp @@ -17,7 +17,7 @@ AdaptiveSpotFinderCPU::AdaptiveSpotFinderCPU(const AzimuthalIntegrationMapping & ring_cnt.assign(nbins, 0); ring_mean.assign(nbins, 0.0f); ring_sigma.assign(nbins, 0.0f); - ring_thr.assign(nbins, 0.0f); + ring_thr.assign(nbins + 1, INFINITY); // the last entry is for pixels outside every ring ring_bkg.assign(nbins, NAN); ring_bits.assign(OutputSize(), 0); ring_hist.assign(nbins * HIST_VALUES, 0); @@ -54,84 +54,54 @@ void AdaptiveSpotFinderCPU::BeginRings() { rings_from_blocks = true; } -// The plain pass over pixels [first, first + n). +// The plain pass over pixels [first, first + n): the histogram of each ring's values, from which +// PlainRings() takes the integer sums, and the fused profile when asked for. One loop reads each pixel +// once for both. void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n) { const auto &pixel_to_bin = mapping.GetPixelToBin(); const size_t nbins = ring_sum.size(); const float *corrections = mapping.Corrections().data(); - - // Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and - // written back when the ring changes: the same additions in the same order, without a store and a - // reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own - // over the block, so that neither loop runs out of registers for its sums. - if (fuse_azint) { - size_t cur = nbins; // the ring held in the locals below; nbins = none - float az_sum = 0.0f, az_sum2 = 0.0f; - uint32_t az_cnt = 0; - for (size_t pxl = first; pxl < first + n; ++pxl) { - const int32_t v = image[pxl]; - if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated - const uint16_t b = pixel_to_bin[pxl]; - if (b >= nbins) continue; // masked / out of range (UINT16_MAX) - if (b != cur) { - if (cur != nbins) { - azint_sum[cur] = az_sum; - azint_sum2[cur] = az_sum2; - azint_count[cur] = az_cnt; - } - cur = b; - az_sum = azint_sum[b]; - az_sum2 = azint_sum2[b]; - az_cnt = azint_count[b]; - } - const float val = static_cast(v) * corrections[pxl]; - const float val_sq = val * val; - az_sum += val; - az_sum2 += val_sq; - ++az_cnt; - } - if (cur != nbins) { - azint_sum[cur] = az_sum; - azint_sum2[cur] = az_sum2; - azint_count[cur] = az_cnt; - } - } - - // Values outside the histogram are listed by a second loop, run only when the block has any: a - // call in this loop would leave the sums in memory again. uint32_t *hist = ring_hist.data(); + + // Consecutive pixels mostly share a ring, so the ring's profile sums are held in locals while they + // do and written back when the ring changes: the same additions in the same order, without a store + // and a reload of the same address on every pixel. Values outside the histogram are listed by a + // second loop, run only when the block has any. bool overflow = false; - size_t cur = nbins; - int64_t sum = 0, cnt = 0; - uint64_t sum2 = 0; + size_t cur = nbins; // the ring held in the locals below; nbins = none + float az_sum = 0.0f, az_sum2 = 0.0f; + uint32_t az_cnt = 0; for (size_t pxl = first; pxl < first + n; ++pxl) { const int32_t v = image[pxl]; if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated const uint16_t b = pixel_to_bin[pxl]; if (b >= nbins) continue; // masked / out of range (UINT16_MAX) - if (b != cur) { - if (cur != nbins) { - ring_sum[cur] = sum; - ring_sum2[cur] = sum2; - ring_cnt[cur] = cnt; - } - cur = b; - sum = ring_sum[b]; - sum2 = ring_sum2[b]; - cnt = ring_cnt[b]; - } - sum += v; - sum2 += static_cast(static_cast(v) * v); - cnt += 1; - if (v >= 0 && v < HIST_VALUES) + if (static_cast(v) < HIST_VALUES) hist[b * HIST_VALUES + v] += 1; else overflow = true; + if (!fuse_azint) continue; + if (b != cur) { + if (cur != nbins) { + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; + } + cur = b; + az_sum = azint_sum[b]; + az_sum2 = azint_sum2[b]; + az_cnt = azint_count[b]; + } + const float val = static_cast(v) * corrections[pxl]; + const float val_sq = val * val; + az_sum += val; + az_sum2 += val_sq; + ++az_cnt; } if (cur != nbins) { - ring_sum[cur] = sum; - ring_sum2[cur] = sum2; - ring_cnt[cur] = cnt; + azint_sum[cur] = az_sum; + azint_sum2[cur] = az_sum2; + azint_count[cur] = az_cnt; } if (overflow) @@ -146,7 +116,8 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer & } // A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the -// pixels holding it would, and its pixels are added as a count. +// pixels holding it would, and its pixels are added as a count. Integer sums, so with nothing clipped +// (clip_k = INFINITY) they are the plain sums over the pixels themselves. void AdaptiveSpotFinderCPU::ClipRings(float clip_k) { const size_t nbins = ring_sum.size(); @@ -155,6 +126,7 @@ void AdaptiveSpotFinderCPU::ClipRings(float clip_k) { std::fill(ring_cnt.begin(), ring_cnt.end(), 0); const auto keep = [&](uint16_t b, int32_t v) { + if (std::isinf(clip_k)) return true; const float lo = ring_mean[b] - clip_k * ring_sigma[b]; const float hi = ring_mean[b] + clip_k * ring_sigma[b]; return !(v < lo || v > hi); // exclude peaks / outliers @@ -197,6 +169,7 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, AccumulateRingsBlock(image, 0, static_cast(width) * height); } rings_from_blocks = false; + ClipRings(INFINITY); UpdateRingStatistics(); ClipRings(3.0f); UpdateRingStatistics(); @@ -259,19 +232,31 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image, void AdaptiveSpotFinderCPU::FlagRow(const ImagePreprocessorBuffer &image, int32_t row) { const auto &pixel_to_bin = mapping.GetPixelToBin(); - const size_t nbins = ring_thr.size(); + const auto nbins = static_cast(ring_thr.size() - 1); // ring_thr[nbins] is +inf + const float *thr = ring_thr.data(); + const int32_t *img = image.data(); + const uint16_t *bin = pixel_to_bin.data(); const size_t first = static_cast(row) * width; + const size_t end = first + width; - for (size_t pxl = first; pxl < first + width; ++pxl) { - const int32_t v = image[pxl]; - const uint16_t b = pixel_to_bin[pxl]; - bool strong = false; - if (v == INT32_MAX) - strong = true; - else if (v != INT32_MIN && b < nbins && v >= ring_thr[b]) - strong = true; - - if (strong) - ring_bits[pxl / 32] |= 1U << (pxl % 32); + // Saturated is strong, bad is not, and a pixel outside every ring (bin >= nbins) meets the +inf + // threshold. Written without branches and a word of 32 pixels at a time, so that the loop vectorises. + const auto strong = [&](size_t pxl) -> uint32_t { + const int32_t v = img[pxl]; + const uint32_t b = std::min(bin[pxl], nbins); + return (v == INT32_MAX) | ((v != INT32_MIN) & (v >= thr[b])); + }; + size_t pxl = first; + while (pxl < end && pxl % 32 != 0) { + ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32); + ++pxl; } + for (; pxl + 32 <= end; pxl += 32) { + uint32_t word = 0; + for (uint32_t j = 0; j < 32; ++j) + word |= strong(pxl + j) << j; + ring_bits[pxl / 32] |= word; + } + for (; pxl < end; ++pxl) + ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32); } diff --git a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h index d0a82aa5f..184ef2379 100644 --- a/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h +++ b/image_analysis/spot_finding/AdaptiveSpotFinderCPU.h @@ -57,7 +57,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { std::vector ring_cnt; std::vector ring_mean; std::vector ring_sigma; - std::vector ring_thr; + std::vector ring_thr; // nbins + 1: the last is +inf, the threshold of a pixel outside every ring // ring_mean of the last Detect(), NaN where the ring holds too few pixels to be its own background. // Kept separately because ring_mean carries the previous frame's value for an empty ring. std::vector ring_bkg; @@ -65,8 +65,8 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { // local-box mask that ImageSpotFinderCPU::Detect leaves in output_buffer. std::vector ring_bits; // The plain pass's valid pixels as a per-ring histogram of their values (HIST_VALUES bins per - // ring) plus a list of the values outside it, so the two sigma-clip passes sum over distinct - // values instead of over the image again. Integer sums, so the same totals. + // ring) plus a list of the values outside it. The plain sums and the two sigma-clip passes are all + // taken from it, over distinct values instead of over the image. Integer sums, so the same totals. static constexpr int32_t HIST_VALUES = 1024; std::vector ring_hist; std::vector> ring_overflow; // (ring, value) @@ -84,7 +84,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU { // Zero the sums of the plain ring pass (and of the fused profile). void ResetRings(); - // One sigma-clip pass over the plain pass's values. + // One sigma-clip pass over the plain pass's values; clip_k = INFINITY gives the plain sums. void ClipRings(float clip_k); // ring_mean / ring_sigma from the current sums. void UpdateRingStatistics();