CPU pixel loops: keep the ring sums and the vertical slide out of memory for real
Two fixes found in the post-merge profile, where the previous commit's loops did not do what they were written to do: - AccumulateRingsBlock: the ring_overflow.emplace_back call inside the loop left no callee-saved registers for the locals, so GCC kept sum/sum2/count on the stack and the store-reload chain was still there (29% of the function's samples on one stack add). The azint sums get a loop of their own over the block, and the out-of-histogram values are listed by a second loop run only when the block has any, in the same pixel order; now the sums live in registers. Same additions in the same order. - DetectPass: inlined into DetectPass's main loop the vertical slide was not vectorised (the vectorised copies GCC made were for the other call sites). It is now a free function, SlideVertical, which GCC vectorises on its own. Measured (perf, 75 s of myob, CPU-only, relative to the unchanged FlagRow/AnalyzeBlock): AccumulateRingsBlock -4..-11%, DetectPass + SlideVertical -7..-14%. Byte-identical p.hkl, p.mtz, p_P1.mtz, p_unmerged.mtz on myob, cytc, lyso, sparse (CPU) and myob (GPU); ImageSpotFinderCPU*, AdaptiveSpotFinder, SpotFinding, AzimuthalIntegration and portable tests pass. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
This commit is contained in:
1 parent
a277b8c458
commit
8e6ecedd5c
2 files changed
+83
-44
No files matched your search
@@ -60,48 +60,65 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
|
||||
const size_t nbins = ring_sum.size();
|
||||
const float *corrections = mapping.Corrections().data();
|
||||
|
||||
// Consecutive pixels mostly share a ring, so the ring's sums are held in locals while they do and
|
||||
// Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and
|
||||
// written back when the ring changes: the same additions in the same order, without a store and a
|
||||
// reload of the same address on every pixel.
|
||||
uint32_t *hist = ring_hist.data();
|
||||
size_t cur = nbins; // the ring held in the locals below; nbins = none
|
||||
int64_t sum = 0, cnt = 0;
|
||||
uint64_t sum2 = 0;
|
||||
float az_sum = 0.0f, az_sum2 = 0.0f;
|
||||
uint32_t az_cnt = 0;
|
||||
const auto store = [&] {
|
||||
if (cur == nbins)
|
||||
return;
|
||||
ring_sum[cur] = sum;
|
||||
ring_sum2[cur] = sum2;
|
||||
ring_cnt[cur] = cnt;
|
||||
if (fuse_azint) {
|
||||
// reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own
|
||||
// over the block, so that neither loop runs out of registers for its sums.
|
||||
if (fuse_azint) {
|
||||
size_t cur = nbins; // the ring held in the locals below; nbins = none
|
||||
float az_sum = 0.0f, az_sum2 = 0.0f;
|
||||
uint32_t az_cnt = 0;
|
||||
for (size_t pxl = first; pxl < first + n; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
|
||||
if (b != cur) {
|
||||
if (cur != nbins) {
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
cur = b;
|
||||
az_sum = azint_sum[b];
|
||||
az_sum2 = azint_sum2[b];
|
||||
az_cnt = azint_count[b];
|
||||
}
|
||||
const float val = static_cast<float>(v) * corrections[pxl];
|
||||
const float val_sq = val * val;
|
||||
az_sum += val;
|
||||
az_sum2 += val_sq;
|
||||
++az_cnt;
|
||||
}
|
||||
if (cur != nbins) {
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Values outside the histogram are listed by a second loop, run only when the block has any: a
|
||||
// call in this loop would leave the sums in memory again.
|
||||
uint32_t *hist = ring_hist.data();
|
||||
bool overflow = false;
|
||||
size_t cur = nbins;
|
||||
int64_t sum = 0, cnt = 0;
|
||||
uint64_t sum2 = 0;
|
||||
for (size_t pxl = first; pxl < first + n; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
|
||||
if (b != cur) {
|
||||
store();
|
||||
if (cur != nbins) {
|
||||
ring_sum[cur] = sum;
|
||||
ring_sum2[cur] = sum2;
|
||||
ring_cnt[cur] = cnt;
|
||||
}
|
||||
cur = b;
|
||||
sum = ring_sum[b];
|
||||
sum2 = ring_sum2[b];
|
||||
cnt = ring_cnt[b];
|
||||
az_sum = azint_sum[b];
|
||||
az_sum2 = azint_sum2[b];
|
||||
az_cnt = azint_count[b];
|
||||
}
|
||||
if (fuse_azint) {
|
||||
const float val = static_cast<float>(v) * corrections[pxl];
|
||||
const float val_sq = val * val;
|
||||
az_sum += val;
|
||||
az_sum2 += val_sq;
|
||||
++az_cnt;
|
||||
}
|
||||
sum += v;
|
||||
sum2 += static_cast<uint64_t>(static_cast<int64_t>(v) * v);
|
||||
@@ -109,9 +126,23 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
|
||||
if (v >= 0 && v < HIST_VALUES)
|
||||
hist[b * HIST_VALUES + v] += 1;
|
||||
else
|
||||
ring_overflow.emplace_back(b, v);
|
||||
overflow = true;
|
||||
}
|
||||
store();
|
||||
if (cur != nbins) {
|
||||
ring_sum[cur] = sum;
|
||||
ring_sum2[cur] = sum2;
|
||||
ring_cnt[cur] = cnt;
|
||||
}
|
||||
|
||||
if (overflow)
|
||||
for (size_t pxl = first; pxl < first + n; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
if (v == INT32_MIN || v == INT32_MAX) continue;
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
if (b >= nbins) continue;
|
||||
if (v < 0 || v >= HIST_VALUES)
|
||||
ring_overflow.emplace_back(b, v);
|
||||
}
|
||||
}
|
||||
|
||||
// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the
|
||||
|
||||
@@ -45,6 +45,24 @@ bool StrongInWindow(int64_t pxl_val, int64_t sum, int64_t sum2, int64_t valid,
|
||||
// pixel is above SNR threshold
|
||||
}
|
||||
|
||||
// Adds row `in` to the vertical sums and takes row `out` out of them; nullptr is no row. A bad or
|
||||
// saturated pixel adds 0. Written without branches, and as a function of its own so that the loop
|
||||
// vectorises whatever it is inlined into; integer sums, so the same totals as pixel by pixel.
|
||||
void SlideVertical(const int32_t *in, const int32_t *out, int32_t width,
|
||||
int64_t *sum, int64_t *sum2, uint16_t *valid) {
|
||||
for (int32_t col = 0; col < width; col++) {
|
||||
const int64_t a = in ? in[col] : INT32_MIN;
|
||||
const int64_t r = out ? out[col] : INT32_MIN;
|
||||
const bool a_ok = a != INT32_MAX && a != INT32_MIN;
|
||||
const bool r_ok = r != INT32_MAX && r != INT32_MIN;
|
||||
const int64_t a_v = a_ok ? a : 0;
|
||||
const int64_t r_v = r_ok ? r : 0;
|
||||
sum[col] += a_v - r_v;
|
||||
sum2[col] += a_v * a_v - r_v * r_v;
|
||||
valid[col] += static_cast<uint16_t>(a_ok - r_ok);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void ImageSpotFinderCPU::DetectAt(const ImagePreprocessorBuffer &image, const SpotFindingSettings &settings,
|
||||
@@ -164,27 +182,17 @@ void ImageSpotFinderCPU::DetectPass(const ImagePreprocessorBuffer &image,
|
||||
std::vector<uint16_t> valid_vert(width, 0);
|
||||
|
||||
// Add the row entering the window (line_in) and take out the one leaving it (line_out); -1 is
|
||||
// no row. A bad or saturated pixel adds 0. Written without branches, so the loop vectorises.
|
||||
// A pixel strong in the previous pass is not counted either; it is added here with the rest
|
||||
// of its row and taken out again below - there are few, and the sums are integers, so the
|
||||
// totals are the same as never adding it.
|
||||
// no row. A pixel strong in the previous pass is not counted either; SlideVertical adds it with
|
||||
// the rest of its row and it is taken out again below - there are few, and the sums are
|
||||
// integers, so the totals are the same as never adding it.
|
||||
const int32_t *img = image.data();
|
||||
const int32_t w = width;
|
||||
int64_t *sv = sum_vert.data();
|
||||
int64_t *sv2 = sum2_vert.data();
|
||||
uint16_t *vv = valid_vert.data();
|
||||
auto slide_vert = [&](int line_in, int line_out) {
|
||||
for (int32_t col = 0; col < w; col++) {
|
||||
const int64_t in = line_in >= 0 ? img[line_in * w + col] : INT32_MIN;
|
||||
const int64_t out = line_out >= 0 ? img[line_out * w + col] : INT32_MIN;
|
||||
const bool in_ok = in != INT32_MAX && in != INT32_MIN;
|
||||
const bool out_ok = out != INT32_MAX && out != INT32_MIN;
|
||||
const int64_t in_v = in_ok ? in : 0;
|
||||
const int64_t out_v = out_ok ? out : 0;
|
||||
sv[col] += in_v - out_v;
|
||||
sv2[col] += in_v * in_v - out_v * out_v;
|
||||
vv[col] += static_cast<uint16_t>(in_ok - out_ok);
|
||||
}
|
||||
SlideVertical(line_in >= 0 ? img + static_cast<size_t>(line_in) * w : nullptr,
|
||||
line_out >= 0 ? img + static_cast<size_t>(line_out) * w : nullptr, w, sv, sv2, vv);
|
||||
if (!prev_strong)
|
||||
return;
|
||||
// sign = +1 takes out what line_in added, -1 puts back what line_out took out.
|
||||
|
||||
Reference in new issue
Block a user