Merge branch 'rc173' into cpu-bragg-locality
rc173 stores refl_mask and owner as 16x16 tiles (cpu-loops-tail), so the rows of those two can no longer be prefetched as flat rows; the prefetch now covers the image only, and the box helper uses a local struct in place of the removed Rect. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
This commit is contained in:
commit
9d5e72ba09
39 files changed
+1094
-349
No files matched your search
@@ -63,7 +63,9 @@ struct ImageSampler {
|
||||
} // namespace
|
||||
|
||||
BraggIntegrationEngineCPU::BraggIntegrationEngineCPU(const DiffractionExperiment &experiment)
|
||||
: BraggIntegrationEngine(experiment) {}
|
||||
: BraggIntegrationEngine(experiment),
|
||||
refl_mask(static_cast<int>(xpixel), static_cast<int>(ypixel), 0),
|
||||
owner(static_cast<int>(xpixel), static_cast<int>(ypixel), BRAGG_OWNER_NONE) {}
|
||||
|
||||
template <class Sampler>
|
||||
std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
@@ -90,17 +92,8 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
// signal is excluded from this reflection's r2..r3 background ring. The region is the INNER
|
||||
// stencil ellipse, taken in each neighbour's OWN frame - an elongated neighbour whose streak
|
||||
// is still marked as a disk would leak its tails into this reflection's ring. ---
|
||||
// Back to empty what the previous call wrote (see the header).
|
||||
for (const Rect &b : refl_mask_dirty)
|
||||
for (int y = b.y0; y <= b.y1; ++y)
|
||||
std::fill(refl_mask.begin() + y * W + b.x0, refl_mask.begin() + y * W + b.x1 + 1, 0);
|
||||
refl_mask_dirty.clear();
|
||||
for (const Rect &b : owner_dirty)
|
||||
for (int y = b.y0; y <= b.y1; ++y)
|
||||
std::fill(owner.begin() + y * W + b.x0, owner.begin() + y * W + b.x1 + 1, BRAGG_OWNER_NONE);
|
||||
owner_dirty.clear();
|
||||
|
||||
refl_mask.resize(npixel, 0);
|
||||
refl_mask.Clear();
|
||||
owner.Clear();
|
||||
for (size_t i = 0; i < npredicted; ++i) {
|
||||
const auto &r = predicted[i];
|
||||
const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
|
||||
@@ -108,12 +101,10 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_in + 1.0f)));
|
||||
const int y0 = std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_in - 1.0f)));
|
||||
const int y1 = std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_in + 1.0f)));
|
||||
if (x0 <= x1 && y0 <= y1)
|
||||
refl_mask_dirty.push_back({x0, x1, y0, y1});
|
||||
for (int y = y0; y <= y1; ++y)
|
||||
for (int x = x0; x <= x1; ++x) {
|
||||
const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
|
||||
if (d.inner < r2_sq) refl_mask[y * W + x] = 1;
|
||||
if (d.inner < r2_sq) refl_mask.At(x, y) = 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +116,6 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
const bool exclude = overlap == OverlapMode::Exclude && mode != IntegratorMode::BoxSum;
|
||||
const bool use_owner = overlap != OverlapMode::Off;
|
||||
if (use_owner) {
|
||||
owner.resize(npixel, BRAGG_OWNER_NONE);
|
||||
const float claim_sq = claim * claim;
|
||||
for (size_t i = 0; i < npredicted; ++i) {
|
||||
const auto &r = predicted[i];
|
||||
@@ -133,21 +123,19 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
const int x1 = std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + claim)));
|
||||
const int y0 = std::max(0, static_cast<int>(std::floor(r.predicted_y - claim)));
|
||||
const int y1 = std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + claim)));
|
||||
if (x0 <= x1 && y0 <= y1)
|
||||
owner_dirty.push_back({x0, x1, y0, y1});
|
||||
for (int y = y0; y <= y1; ++y)
|
||||
for (int x = x0; x <= x1; ++x) {
|
||||
const float dx = x - r.predicted_x, dy = y - r.predicted_y;
|
||||
const float d2 = dx * dx + dy * dy;
|
||||
if (d2 >= claim_sq) continue;
|
||||
uint32_t &o = owner[y * W + x];
|
||||
uint32_t &o = owner.At(x, y);
|
||||
o = std::min(o, BraggOwnerKey(std::sqrt(d2), inv_claim, static_cast<int>(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
auto clean = [&](int x, int y, size_t i) {
|
||||
return !use_owner
|
||||
|| BraggOwnedBy(owner[static_cast<size_t>(y) * W + x], static_cast<int>(i));
|
||||
|| BraggOwnedBy(owner.Get(x, y), static_cast<int>(i));
|
||||
};
|
||||
|
||||
// --- Pass A: box-sum every reflection (rough I, background, centroid, strong flag). ---
|
||||
@@ -182,35 +170,31 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
|
||||
// The box pass A reads: the OUTER ellipse, tightly - taking the largest semi-axis in both
|
||||
// directions instead would read up to 60% more pixels for nothing.
|
||||
struct Box { int x0, x1, y0, y1; };
|
||||
auto box_of = [&](const Reflection &r, const BraggStencil &st) {
|
||||
return Rect{std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f))),
|
||||
return Box{std::max(0, static_cast<int>(std::floor(r.predicted_x - st.ex_out - 1.0f))),
|
||||
std::min(W - 1, static_cast<int>(std::ceil(r.predicted_x + st.ex_out + 1.0f))),
|
||||
std::max(0, static_cast<int>(std::floor(r.predicted_y - st.ey_out - 1.0f))),
|
||||
std::min(H - 1, static_cast<int>(std::ceil(r.predicted_y + st.ey_out + 1.0f)))};
|
||||
};
|
||||
for (size_t i = 0; i < npredicted; ++i) {
|
||||
// The next reflection's rows - image and reflection mask over its box, owner map over its
|
||||
// claim - asked for while this one is worked on: the reflections are spread over frames far
|
||||
// larger than the caches, so each box would otherwise wait on memory row by row. (Written
|
||||
// out here: GCC drops a call to a function that does nothing but prefetch.)
|
||||
// The next reflection's image rows, asked for while this one is worked on: the reflections
|
||||
// are spread over a frame far larger than the caches, so each box would otherwise wait on
|
||||
// memory row by row. (Written out here: GCC drops a call to a function that does nothing but
|
||||
// prefetch.)
|
||||
if (i + 1 < npredicted) {
|
||||
const Reflection &n = predicted[i + 1];
|
||||
const Rect b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil));
|
||||
const int ox = static_cast<int>(std::lround(n.predicted_x)), oy = static_cast<int>(std::lround(n.predicted_y));
|
||||
for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y) {
|
||||
const size_t row = static_cast<size_t>(y) * W;
|
||||
prefetch_row(img.address(row + b.x0), img.address(row + b.x1));
|
||||
prefetch_row(&refl_mask[row + b.x0], &refl_mask[row + b.x1]);
|
||||
if (use_owner && std::abs(y - oy) <= R)
|
||||
prefetch_row(&owner[row + std::max(b.x0, ox - R)], &owner[row + std::min(b.x1, ox + R)]);
|
||||
}
|
||||
const Box b = box_of(n, MakeBraggStencil(n.predicted_x, n.predicted_y, stencil));
|
||||
for (int y = b.y0; y <= b.y1 && b.x0 <= b.x1; ++y)
|
||||
prefetch_row(img.address(static_cast<size_t>(y) * W + b.x0),
|
||||
img.address(static_cast<size_t>(y) * W + b.x1));
|
||||
}
|
||||
const auto &r = predicted[i];
|
||||
Rough out;
|
||||
// This reflection's stencil: the r1 signal disk, and the r2..r3 ring elongated radially by
|
||||
// the analytic smear.
|
||||
const BraggStencil st = MakeBraggStencil(r.predicted_x, r.predicted_y, stencil);
|
||||
const Rect box = box_of(r, st);
|
||||
const Box box = box_of(r, st);
|
||||
const int x0 = box.x0, x1 = box.x1, y0 = box.y0, y1 = box.y1;
|
||||
|
||||
// Both from the stencil's own radius, so nothing downstream is derived from a second one.
|
||||
@@ -253,7 +237,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
y_sum += y;
|
||||
++n_inner_valid;
|
||||
} else if (d.inner >= r2_sq && d.outer < r3_sq) {
|
||||
if (refl_mask[y * W + x]) { ++n_bkg_neighbour; continue; }
|
||||
if (refl_mask.Get(x, y)) { ++n_bkg_neighbour; continue; }
|
||||
if (!valid(px)) continue;
|
||||
bkg_sum += static_cast<double>(px);
|
||||
if (bkg_trim_frac > 0.0) bkg_vals.push_back(px);
|
||||
@@ -307,7 +291,7 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
for (int x = x0; x <= x1; ++x) {
|
||||
const auto d = BraggStencilDistances(st, x - r.predicted_x, y - r.predicted_y);
|
||||
if (!(d.inner >= r2_sq && d.outer < r3_sq)) continue;
|
||||
if (refl_mask[y * W + x]) continue;
|
||||
if (refl_mask.Get(x, y)) continue;
|
||||
const int32_t px = img[y * W + x];
|
||||
if (!valid(px)) continue;
|
||||
if (static_cast<double>(px) <= thr) {
|
||||
@@ -549,14 +533,12 @@ std::vector<Reflection> BraggIntegrationEngineCPU::RunImpl(const Sampler &img,
|
||||
// needed) radially elongated, on a grid grown to hold the streak.
|
||||
const double fx = predicted[i].predicted_x - rh.cx, fy = predicted[i].predicted_y - rh.cy;
|
||||
Rf = elong ? std::min(3 * R, static_cast<int>(std::ceil(r2 + 2.0 * std::sqrt(s2r)))) : R;
|
||||
// The grid's image and owner rows, asked for now (as in pass A): building the profile
|
||||
// below takes long enough for them to arrive before the fit reads them.
|
||||
// The grid's image rows, asked for now (as in pass A): building the profile below takes
|
||||
// long enough for them to arrive before the fit reads them.
|
||||
const int gx0 = std::max(0, rh.cx - Rf), gx1 = std::min(W - 1, rh.cx + Rf);
|
||||
for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y) {
|
||||
const size_t row = static_cast<size_t>(y) * W;
|
||||
prefetch_row(img.address(row + gx0), img.address(row + gx1));
|
||||
if (use_owner) prefetch_row(&owner[row + gx0], &owner[row + gx1]);
|
||||
}
|
||||
for (int y = std::max(0, rh.cy - Rf); y <= std::min(H - 1, rh.cy + Rf); ++y)
|
||||
prefetch_row(img.address(static_cast<size_t>(y) * W + gx0),
|
||||
img.address(static_cast<size_t>(y) * W + gx1));
|
||||
const int Gf = 2 * Rf + 1;
|
||||
Pbuf.assign(static_cast<size_t>(Gf) * Gf, 0.0);
|
||||
double gs = 0.0;
|
||||
|
||||
Reference in new issue
Block a user