CPU spot finder and rotation prediction: same results, less work per pixel
AdaptiveSpotFinderCPU::AccumulateRingsBlock reads each pixel once for both the ring histogram and the fused azimuthal profile (was two loops), and no longer keeps the per-ring integer sums: they are taken from the histogram, as the two sigma-clip passes already were (ClipRings(INFINITY)). Integer sums, so the same totals; the profile's float sums keep their pixel order. FlagRow is branch-free and works a 32-pixel word at a time, so it vectorises; pixels outside every ring meet a +inf threshold in an extra ring_thr entry. BraggPredictionRot::Calc takes A*h, A*h + B*k, C*l and 4*S0*S0 out of the inner loops; p0 is the same ((A*h) + (B*k)) + (C*l) as before. Measured on cytc (CPU build, both binaries run concurrently on a loaded box): AccumulateRingsBlock -35%, FlagRow -47%, Calc + Coord ops -30% cycles; whole run -5% cycles. p.mtz md5 unchanged on myob/cytc/thau, CPU and GPU builds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K5K8jvPPbmCrbqnWkddTuB
This commit is contained in:
1 parent
2c97e654ed
commit
56f0c5dd0a
3 files changed
+77
-84
No files matched your search
@@ -45,6 +45,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
|
||||
const Coord m3 = (m1 % m2).Normalize();
|
||||
|
||||
const float m2_S0 = m2 * S0;
|
||||
const float four_S0_sq = 4 * S0 * S0;
|
||||
const float m3_S0 = m3 * S0;
|
||||
|
||||
int i = 0;
|
||||
@@ -99,17 +100,24 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
|
||||
cos_phi_limit = std::cos(phi_limit);
|
||||
}
|
||||
|
||||
// p0 = A* h + B* k + C* l, evaluated as ((A* h) + (B* k)) + (C* l) exactly as before, with the
|
||||
// terms that do not change in the inner loops taken out of them.
|
||||
std::vector<Coord> Cstar_l(2 * settings.max_l + 1);
|
||||
for (int l = -settings.max_l; l <= settings.max_l; l++)
|
||||
Cstar_l[l + settings.max_l] = Cstar * l;
|
||||
|
||||
for (int h = -settings.max_h; h <= settings.max_h; h++) {
|
||||
// Precompute A* h contribution
|
||||
const Coord Astar_h = Astar * h;
|
||||
|
||||
for (int k = -settings.max_k; k <= settings.max_k; k++) {
|
||||
// Accumulate B* k contribution
|
||||
const Coord AB = Astar_h + Bstar * k;
|
||||
|
||||
for (int l = -settings.max_l; l <= settings.max_l; l++) {
|
||||
if (systematic_absence(h, k, l, settings.centering))
|
||||
continue;
|
||||
|
||||
Coord p0 = Astar * h + Bstar * k + Cstar * l;
|
||||
const Coord &Cl = Cstar_l[l + settings.max_l];
|
||||
const Coord p0(AB.x + Cl.x, AB.y + Cl.y, AB.z + Cl.z);
|
||||
|
||||
float p0_sq = p0 * p0;
|
||||
if (p0_sq <= 0.0f || p0_sq > one_over_dmax_sq)
|
||||
@@ -129,7 +137,7 @@ int BraggPredictionRot::Calc(const DiffractionExperiment &experiment, const Crys
|
||||
};
|
||||
|
||||
// No solution for Laue equations
|
||||
if ((rho_sq < p_m3 * p_m3) || (p0_sq > 4 * S0 * S0))
|
||||
if ((rho_sq < p_m3 * p_m3) || (p0_sq > four_S0_sq))
|
||||
continue;
|
||||
|
||||
// Effective rocking width for this reflection: mosaicity broadened by the bandwidth
|
||||
|
||||
@@ -17,7 +17,7 @@ AdaptiveSpotFinderCPU::AdaptiveSpotFinderCPU(const AzimuthalIntegrationMapping &
|
||||
ring_cnt.assign(nbins, 0);
|
||||
ring_mean.assign(nbins, 0.0f);
|
||||
ring_sigma.assign(nbins, 0.0f);
|
||||
ring_thr.assign(nbins, 0.0f);
|
||||
ring_thr.assign(nbins + 1, INFINITY); // the last entry is for pixels outside every ring
|
||||
ring_bkg.assign(nbins, NAN);
|
||||
ring_bits.assign(OutputSize(), 0);
|
||||
ring_hist.assign(nbins * HIST_VALUES, 0);
|
||||
@@ -54,84 +54,54 @@ void AdaptiveSpotFinderCPU::BeginRings() {
|
||||
rings_from_blocks = true;
|
||||
}
|
||||
|
||||
// The plain pass over pixels [first, first + n).
|
||||
// The plain pass over pixels [first, first + n): the histogram of each ring's values, from which
|
||||
// PlainRings() takes the integer sums, and the fused profile when asked for. One loop reads each pixel
|
||||
// once for both.
|
||||
void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &image, size_t first, size_t n) {
|
||||
const auto &pixel_to_bin = mapping.GetPixelToBin();
|
||||
const size_t nbins = ring_sum.size();
|
||||
const float *corrections = mapping.Corrections().data();
|
||||
|
||||
// Consecutive pixels mostly share a ring, so a ring's sums are held in locals while they do and
|
||||
// written back when the ring changes: the same additions in the same order, without a store and a
|
||||
// reload of the same address on every pixel. The azimuthal-integration sums get a loop of their own
|
||||
// over the block, so that neither loop runs out of registers for its sums.
|
||||
if (fuse_azint) {
|
||||
size_t cur = nbins; // the ring held in the locals below; nbins = none
|
||||
float az_sum = 0.0f, az_sum2 = 0.0f;
|
||||
uint32_t az_cnt = 0;
|
||||
for (size_t pxl = first; pxl < first + n; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
|
||||
if (b != cur) {
|
||||
if (cur != nbins) {
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
cur = b;
|
||||
az_sum = azint_sum[b];
|
||||
az_sum2 = azint_sum2[b];
|
||||
az_cnt = azint_count[b];
|
||||
}
|
||||
const float val = static_cast<float>(v) * corrections[pxl];
|
||||
const float val_sq = val * val;
|
||||
az_sum += val;
|
||||
az_sum2 += val_sq;
|
||||
++az_cnt;
|
||||
}
|
||||
if (cur != nbins) {
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
}
|
||||
|
||||
// Values outside the histogram are listed by a second loop, run only when the block has any: a
|
||||
// call in this loop would leave the sums in memory again.
|
||||
uint32_t *hist = ring_hist.data();
|
||||
|
||||
// Consecutive pixels mostly share a ring, so the ring's profile sums are held in locals while they
|
||||
// do and written back when the ring changes: the same additions in the same order, without a store
|
||||
// and a reload of the same address on every pixel. Values outside the histogram are listed by a
|
||||
// second loop, run only when the block has any.
|
||||
bool overflow = false;
|
||||
size_t cur = nbins;
|
||||
int64_t sum = 0, cnt = 0;
|
||||
uint64_t sum2 = 0;
|
||||
size_t cur = nbins; // the ring held in the locals below; nbins = none
|
||||
float az_sum = 0.0f, az_sum2 = 0.0f;
|
||||
uint32_t az_cnt = 0;
|
||||
for (size_t pxl = first; pxl < first + n; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
if (v == INT32_MIN || v == INT32_MAX) continue; // bad / saturated
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
if (b >= nbins) continue; // masked / out of range (UINT16_MAX)
|
||||
if (b != cur) {
|
||||
if (cur != nbins) {
|
||||
ring_sum[cur] = sum;
|
||||
ring_sum2[cur] = sum2;
|
||||
ring_cnt[cur] = cnt;
|
||||
}
|
||||
cur = b;
|
||||
sum = ring_sum[b];
|
||||
sum2 = ring_sum2[b];
|
||||
cnt = ring_cnt[b];
|
||||
}
|
||||
sum += v;
|
||||
sum2 += static_cast<uint64_t>(static_cast<int64_t>(v) * v);
|
||||
cnt += 1;
|
||||
if (v >= 0 && v < HIST_VALUES)
|
||||
if (static_cast<uint32_t>(v) < HIST_VALUES)
|
||||
hist[b * HIST_VALUES + v] += 1;
|
||||
else
|
||||
overflow = true;
|
||||
if (!fuse_azint) continue;
|
||||
if (b != cur) {
|
||||
if (cur != nbins) {
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
cur = b;
|
||||
az_sum = azint_sum[b];
|
||||
az_sum2 = azint_sum2[b];
|
||||
az_cnt = azint_count[b];
|
||||
}
|
||||
const float val = static_cast<float>(v) * corrections[pxl];
|
||||
const float val_sq = val * val;
|
||||
az_sum += val;
|
||||
az_sum2 += val_sq;
|
||||
++az_cnt;
|
||||
}
|
||||
if (cur != nbins) {
|
||||
ring_sum[cur] = sum;
|
||||
ring_sum2[cur] = sum2;
|
||||
ring_cnt[cur] = cnt;
|
||||
azint_sum[cur] = az_sum;
|
||||
azint_sum2[cur] = az_sum2;
|
||||
azint_count[cur] = az_cnt;
|
||||
}
|
||||
|
||||
if (overflow)
|
||||
@@ -146,7 +116,8 @@ void AdaptiveSpotFinderCPU::AccumulateRingsBlock(const ImagePreprocessorBuffer &
|
||||
}
|
||||
|
||||
// A sigma-clip pass over the plain pass's values: each distinct value of a ring meets the same test the
|
||||
// pixels holding it would, and its pixels are added as a count.
|
||||
// pixels holding it would, and its pixels are added as a count. Integer sums, so with nothing clipped
|
||||
// (clip_k = INFINITY) they are the plain sums over the pixels themselves.
|
||||
void AdaptiveSpotFinderCPU::ClipRings(float clip_k) {
|
||||
const size_t nbins = ring_sum.size();
|
||||
|
||||
@@ -155,6 +126,7 @@ void AdaptiveSpotFinderCPU::ClipRings(float clip_k) {
|
||||
std::fill(ring_cnt.begin(), ring_cnt.end(), 0);
|
||||
|
||||
const auto keep = [&](uint16_t b, int32_t v) {
|
||||
if (std::isinf(clip_k)) return true;
|
||||
const float lo = ring_mean[b] - clip_k * ring_sigma[b];
|
||||
const float hi = ring_mean[b] + clip_k * ring_sigma[b];
|
||||
return !(v < lo || v > hi); // exclude peaks / outliers
|
||||
@@ -197,6 +169,7 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image,
|
||||
AccumulateRingsBlock(image, 0, static_cast<size_t>(width) * height);
|
||||
}
|
||||
rings_from_blocks = false;
|
||||
ClipRings(INFINITY);
|
||||
UpdateRingStatistics();
|
||||
ClipRings(3.0f);
|
||||
UpdateRingStatistics();
|
||||
@@ -259,19 +232,31 @@ void AdaptiveSpotFinderCPU::Detect(const ImagePreprocessorBuffer &image,
|
||||
|
||||
void AdaptiveSpotFinderCPU::FlagRow(const ImagePreprocessorBuffer &image, int32_t row) {
|
||||
const auto &pixel_to_bin = mapping.GetPixelToBin();
|
||||
const size_t nbins = ring_thr.size();
|
||||
const auto nbins = static_cast<uint32_t>(ring_thr.size() - 1); // ring_thr[nbins] is +inf
|
||||
const float *thr = ring_thr.data();
|
||||
const int32_t *img = image.data();
|
||||
const uint16_t *bin = pixel_to_bin.data();
|
||||
const size_t first = static_cast<size_t>(row) * width;
|
||||
const size_t end = first + width;
|
||||
|
||||
for (size_t pxl = first; pxl < first + width; ++pxl) {
|
||||
const int32_t v = image[pxl];
|
||||
const uint16_t b = pixel_to_bin[pxl];
|
||||
bool strong = false;
|
||||
if (v == INT32_MAX)
|
||||
strong = true;
|
||||
else if (v != INT32_MIN && b < nbins && v >= ring_thr[b])
|
||||
strong = true;
|
||||
|
||||
if (strong)
|
||||
ring_bits[pxl / 32] |= 1U << (pxl % 32);
|
||||
// Saturated is strong, bad is not, and a pixel outside every ring (bin >= nbins) meets the +inf
|
||||
// threshold. Written without branches and a word of 32 pixels at a time, so that the loop vectorises.
|
||||
const auto strong = [&](size_t pxl) -> uint32_t {
|
||||
const int32_t v = img[pxl];
|
||||
const uint32_t b = std::min<uint32_t>(bin[pxl], nbins);
|
||||
return (v == INT32_MAX) | ((v != INT32_MIN) & (v >= thr[b]));
|
||||
};
|
||||
size_t pxl = first;
|
||||
while (pxl < end && pxl % 32 != 0) {
|
||||
ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32);
|
||||
++pxl;
|
||||
}
|
||||
for (; pxl + 32 <= end; pxl += 32) {
|
||||
uint32_t word = 0;
|
||||
for (uint32_t j = 0; j < 32; ++j)
|
||||
word |= strong(pxl + j) << j;
|
||||
ring_bits[pxl / 32] |= word;
|
||||
}
|
||||
for (; pxl < end; ++pxl)
|
||||
ring_bits[pxl / 32] |= strong(pxl) << (pxl % 32);
|
||||
}
|
||||
@@ -57,7 +57,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
|
||||
std::vector<int64_t> ring_cnt;
|
||||
std::vector<float> ring_mean;
|
||||
std::vector<float> ring_sigma;
|
||||
std::vector<float> ring_thr;
|
||||
std::vector<float> ring_thr; // nbins + 1: the last is +inf, the threshold of a pixel outside every ring
|
||||
// ring_mean of the last Detect(), NaN where the ring holds too few pixels to be its own background.
|
||||
// Kept separately because ring_mean carries the previous frame's value for an empty ring.
|
||||
std::vector<float> ring_bkg;
|
||||
@@ -65,8 +65,8 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
|
||||
// local-box mask that ImageSpotFinderCPU::Detect leaves in output_buffer.
|
||||
std::vector<uint32_t> ring_bits;
|
||||
// The plain pass's valid pixels as a per-ring histogram of their values (HIST_VALUES bins per
|
||||
// ring) plus a list of the values outside it, so the two sigma-clip passes sum over distinct
|
||||
// values instead of over the image again. Integer sums, so the same totals.
|
||||
// ring) plus a list of the values outside it. The plain sums and the two sigma-clip passes are all
|
||||
// taken from it, over distinct values instead of over the image. Integer sums, so the same totals.
|
||||
static constexpr int32_t HIST_VALUES = 1024;
|
||||
std::vector<uint32_t> ring_hist;
|
||||
std::vector<std::pair<uint16_t, int32_t>> ring_overflow; // (ring, value)
|
||||
@@ -84,7 +84,7 @@ class AdaptiveSpotFinderCPU : public ImageSpotFinderCPU {
|
||||
|
||||
// Zero the sums of the plain ring pass (and of the fused profile).
|
||||
void ResetRings();
|
||||
// One sigma-clip pass over the plain pass's values.
|
||||
// One sigma-clip pass over the plain pass's values; clip_k = INFINITY gives the plain sums.
|
||||
void ClipRings(float clip_k);
|
||||
// ring_mean / ring_sigma from the current sums.
|
||||
void UpdateRingStatistics();
|
||||
|
||||
Reference in new issue
Block a user