Merge branch 'rc173' into cpu-bragg-locality

rc173 stores refl_mask and owner as 16x16 tiles (cpu-loops-tail), so
the rows of those two can no longer be prefetched as flat rows; the
prefetch now covers the image only, and the box helper uses a local
struct in place of the removed Rect.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-09-28 04:06:15 +02:00
commit 9d5e72ba09
39 files changed
+1094 -349

No files matched your search

@@ -63,7 +63,9 @@ struct ImageSampler {
} // namespace
BraggIntegrationEngineCPU::BraggIntegrationEngineCPU(const DiffractionExperiment &experiment)
: BraggIntegrationEngine(experiment) {}
: BraggIntegrationEngine(experiment),
refl_mask(static_cast<int>(xpixel), static_cast<int>(ypixel), 0),
owner(static_cast<int>(xpixel), static_cast<int>(ypixel), BRAGG_OWNER_NONE) {}
template <class Sampler>
std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
@@ -90,17 +92,8 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
// signal is excluded from this reflection's r2..r3 background ring. The region is the INNER
// stencil ellipse, taken in each neighbour's OWN frame - an elongated neighbour whose streak
// is still marked as a disk would leak its tails into this reflection's ring. ---
// Back to empty what the previous call wrote (see the header).
for (const Rect &b : refl_mask_dirty)
for (int y = b.y0; y <= b.y1; ++y)
std::fill(refl_mask.begin() + y * W + b.x0, refl_mask.begin() + y * W + b.x1 + 1, 0);
refl_mask_dirty.clear();
for (const Rect &b : owner_dirty)
for (int y = b.y0; y <= b.y1; ++y)
std::fill(owner.begin() + y * W + b.x0, owner.begin() + y * W + b.x1 + 1, BRAGG_OWNER_NONE);
owner_dirty.clear();
refl_mask.resize(npixel, 0);
refl_mask.Clear();
owner.Clear();
for (size_t i = 0; i < npredicted; ++i) {
const auto &r = predicted[i];
const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
@@ -108,12 +101,10 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_in + 1.0f)));
const int y0 = std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_in - 1.0f)));
const int y1 = std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_in + 1.0f)));
if (x0 <= x1 && y0 <= y1)
refl_mask_dirty.push_back({x0, x1, y0, y1});
for (int y = y0; y <= y1; ++y)
for (int x = x0; x <= x1; ++x) {
const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
if (d.inner < r2_sq) refl_mask[y * W + x] = 1;
if (d.inner < r2_sq) refl_mask.At(x, y) = 1;
}
}
@@ -125,7 +116,6 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
const bool exclude = overlap == OverlapMode::Exclude && mode != IntegratorMode::BoxSum;
const bool use_owner = overlap != OverlapMode::Off;
if (use_owner) {
owner.resize(npixel, BRAGG_OWNER_NONE);
const float claim_sq = claim * claim;
for (size_t i = 0; i < npredicted; ++i) {
const auto &r = predicted[i];
@@ -133,21 +123,19 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + claim)));
const int y0 = std::max(0, static_cast<int>(std::floor(r.predicted_y - claim)));
const int y1 = std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + claim)));
if (x0 <= x1 && y0 <= y1)
owner_dirty.push_back({x0, x1, y0, y1});
for (int y = y0; y <= y1; ++y)
for (int x = x0; x <= x1; ++x) {
const float dx = x - r.predicted_x, dy = y - r.predicted_y;
const float d2 = dx * dx + dy * dy;
if (d2 >= claim_sq) continue;
uint32_t &o = owner[y * W + x];
uint32_t &o = owner.At(x, y);
o = std::min(o, BraggOwnerKey(std::sqrt(d2), inv_claim, static_cast<int>(i)));
}
}
}
auto clean = [&](int x, int y, size_t i) {
return !use_owner
|| BraggOwnedBy(owner[static_cast<size_t>(y) * W + x], static_cast<int>(i));
|| BraggOwnedBy(owner.Get(x, y), static_cast<int>(i));
};
// --- Pass A: box-sum every reflection (rough I, background, centroid, strong flag). ---
@@ -182,35 +170,31 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
// The box pass A reads: the OUTER ellipse, tightly - taking the largest semi-axis in both
// directions instead would read up to 60% more pixels for nothing.
struct Box { int x0, x1, y0, y1; };
auto box_of = [&](const Reflection &r, const BraggStencil &st) {
return Rect{std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f))),
return Box{std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f))),
std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_out + 1.0f))),
std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_out - 1.0f))),
std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_out + 1.0f)))};
};
for (size_t i = 0; i < npredicted; ++i) {
// The next reflection's rows - image and reflection mask over its box, owner map over its
// claim - asked for while this one is worked on: the reflections are spread over frames far
// larger than the caches, so each box would otherwise wait on memory row by row. (Written
// out here: GCC drops a call to a function that does nothing but prefetch.)
// The next reflection's image rows, asked for while this one is worked on: the reflections
// are spread over a frame far larger than the caches, so each box would otherwise wait on
// memory row by row. (Written out here: GCC drops a call to a function that does nothing but
// prefetch.)
if (i + 1 < npredicted) {
const Reflection &n = predicted[i + 1];
const Rect b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil));
const int ox = static_cast<int>(std::lround(n.predicted_x)), oy = static_cast<int>(std::lround(n.predicted_y));
for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y) {
const size_t row = static_cast<size_t>(y) * W;
prefetch_row(img.address(row + b.x0), img.address(row + b.x1));
prefetch_row(&refl_mask[row + b.x0], &refl_mask[row + b.x1]);
if (use_owner && std::abs(y - oy) <= R)
prefetch_row(&owner[row + std::max(b.x0, ox - R)], &owner[row + std::min(b.x1, ox + R)]);
}
const Box b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil));
for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y)
prefetch_row(img.address(static_cast<size_t>(y) * W + b.x0),
img.address(static_cast<size_t>(y) * W + b.x1));
}
const auto &r = predicted[i];
Rough out;
// This reflection's stencil: the r1 signal disk, and the r2..r3 ring elongated radially by
// the analytic smear.
const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
const Rect box = box_of(r, st);
const Box box = box_of(r, st);
const int x0 = box.x0, x1 = box.x1, y0 = box.y0, y1 = box.y1;
// Both from the stencil's own radius, so nothing downstream is derived from a second one.
@@ -253,7 +237,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
y_sum += y;
++n_inner_valid;
} else if (d.inner >= r2_sq && d.outer < r3_sq) {
if (refl_mask[y * W + x]) { ++n_bkg_neighbour; continue; }
if (refl_mask.Get(x, y)) { ++n_bkg_neighbour; continue; }
if (!valid(px)) continue;
bkg_sum += static_cast<double>(px);
if (bkg_trim_frac > 0.0) bkg_vals.push_back(px);
@@ -307,7 +291,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
for (int x = x0; x <= x1; ++x) {
const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
if (!(d.inner >= r2_sq && d.outer < r3_sq)) continue;
if (refl_mask[y * W + x]) continue;
if (refl_mask.Get(x, y)) continue;
const int32_t px = img[y * W + x];
if (!valid(px)) continue;
if (static_cast<double>(px) <= thr) {
@@ -549,14 +533,12 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
// needed) radially elongated, on a grid grown to hold the streak.
const double fx = predicted[i].predicted_x - rh.cx, fy = predicted[i].predicted_y - rh.cy;
Rf = elong ? std::min(3 * R, static_cast<int>(std::ceil(r2 + 2.0 * std::sqrt(s2r)))) : R;
// The grid's image and owner rows, asked for now (as in pass A): building the profile
// below takes long enough for them to arrive before the fit reads them.
// The grid's image rows, asked for now (as in pass A): building the profile below takes
// long enough for them to arrive before the fit reads them.
const int gx0 = std::max(0, rh.cx - Rf), gx1 = std::min(W - 1, rh.cx + Rf);
for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y) {
const size_t row = static_cast<size_t>(y) * W;
prefetch_row(img.address(row + gx0), img.address(row + gx1));
if (use_owner) prefetch_row(&owner[row + gx0], &owner[row + gx1]);
}
for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y)
prefetch_row(img.address(static_cast<size_t>(y) * W + gx0),
img.address(static_cast<size_t>(y) * W + gx1));
const int Gf = 2 * Rf + 1;
Pbuf.assign(static_cast<size_t>(Gf) * Gf, 0.0);
double gs = 0.0;