From f8eb9ede5cbe69bf05930e175e5ae8747a986cff Mon Sep 17 00:00:00 2001 From: jungfrau Date: Mon, 16 Nov 2026 17:14:54 +0100 Subject: [PATCH] Model density: walk each row of an atom's box as one run of points The points of a row within an atom's radius are one run (r2 falls and then rises along the row), so RowRun() finds its ends from the roots of r2 = max_r2 and moves them onto the points the exact single-precision test takes. The row's points are then filled without a test in the loop, and AtomDensity adds them to the grid a run of consecutive grid points at a time instead of by a gathered index per point. The fill and the adds now vectorise with -march=x86-64-v3. The points, their arithmetic and the order of the adds are unchanged, so the grid is the same bit for bit (ModelDensityGPU_BitIdenticalToCPU holds); refinement results do not move. The same run applied to GatherMapPoints() was slower: its rows are about ten points long, and finding the run cost more than the well-predicted branch it removed. Vectorising the gradient reductions by summing in lanes was faster (-16% in AtomGradientsFromMap) but changes the summation order, which moved the optimiser's path and a final R-free by 0.003; both were left out. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL --- .../structure_refinement/ModelGrid.cpp | 128 +++++++++++------- 1 file changed, 77 insertions(+), 51 deletions(-) diff --git a/image_analysis/structure_refinement/ModelGrid.cpp b/image_analysis/structure_refinement/ModelGrid.cpp index 65468c364..5ae6d6248 100644 --- a/image_analysis/structure_refinement/ModelGrid.cpp +++ b/image_analysis/structure_refinement/ModelGrid.cpp @@ -178,24 +178,53 @@ DensityGeometry ModelDensityGeometry(const gemmi::Grid &grid) { static_cast(m.a22), static_cast(m.a23), static_cast(m.a33)}; } -// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their index -// in the grid, squared distance and delta (the atom's centre minus the point, orthogonal). +// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their squared +// distance and delta (the atom's centre minus the point, orthogonal), and where they are in the grid - runs of +// consecutive points along u, a row's run split where it wraps. density and x are AtomDensity's scratch. struct DensityPoints { + struct Run { + size_t start; // the grid index of the run's first point + int n; + }; int n = 0; - std::vector idx; - std::vector r2, dx, dy, dz; + std::vector runs; + std::vector r2, dx, dy, dz, density, x; void Clear(size_t capacity) { n = 0; - if (idx.size() < capacity) { - idx.resize(capacity); + runs.clear(); + if (r2.size() < capacity) { r2.resize(capacity); dx.resize(capacity); dy.resize(capacity); dz.resize(capacity); + density.resize(capacity); + x.resize(capacity); } } }; +// The points of one row of a density box, u_lo to u_hi, that the walk takes: those where r2 = yz + x x, +// x = m00 (nctr0 - u) + x_yz, is not above max_r2. r2 falls and then rises along the row, so they are one run, first to +// last (none where first > last). Its ends are put at the roots of r2 = max_r2, then moved onto the points the test +// itself takes, so the run is the walk's, point for point, and the loops over it need no test. +void RowRun(float m00, float nctr0, float x_yz, float yz, float max_r2, int u_lo, int u_hi, int &first, int &last) { + const auto inside = [&](int u) { + const float x = m00 * (nctr0 - static_cast(u)) + x_yz; + return !(yz + x * x > max_r2); + }; + const double centre = nctr0 + double(x_yz) / m00, half = std::sqrt(std::max(0.0, double(max_r2) - yz)) / m00; + first = std::max(u_lo, static_cast(std::ceil(centre - half))); + last = std::min(u_hi, static_cast(std::floor(centre + half))); + while (first <= last && !inside(first)) + first++; + while (first > u_lo && inside(first - 1)) + first--; + while (last < u_hi && inside(last + 1)) + last++; + while (last >= first && !inside(last)) + last--; +} + // UseAtomBoxesByPlane() for the density's boxes: the same planes, atoms and walk. A point at (u, v, w), the // box's own (unwrapped) indices, is f = centre - (u, v, w) from the atom, and in A // x = m00 fx + (m01 fy + m02 fz), y = m11 fy + m12 fz, z = m22 fz, r2 = (y y + z z) + x x, @@ -220,7 +249,7 @@ void UseDensityBoxesByPlane(gemmi::Grid &grid, const std::vector(2 * b.d[0] + 1) * (2 * b.d[1] + 1) * (1 + 2 * b.d[2] / nw)); int n = 0; for (int w = w_lo + gemmi::modulo(plane - w_lo, nw); w <= w_hi; w += nw) { @@ -233,22 +262,21 @@ void UseDensityBoxesByPlane(gemmi::Grid &grid, const std::vector b.max_r2) continue; // so is every point of the row: x x only adds const float x_yz = m.m01 * fy + m.m02 * fz; + int first, last; + RowRun(m.m00, b.nctr[0], x_yz, yz, b.max_r2, u_lo, u_hi, first, last); + for (int k = 0; k < last - first + 1; ++k) { + const float x = m.m00 * (b.nctr[0] - static_cast(first + k)) + x_yz; + points.r2[n + k] = yz + x * x; + points.dx[n + k] = x; + points.dy[n + k] = y; + points.dz[n + k] = z; + } const size_t row = grid.index_q(0, v_, plane); - for (int u = u_lo, u_ = u_0;;) { - const float x = m.m00 * (b.nctr[0] - static_cast(u)) + x_yz; - const float r2 = yz + x * x; - if (!(r2 > b.max_r2)) { - points.idx[n] = row + u_; - points.r2[n] = r2; - points.dx[n] = x; - points.dy[n] = y; - points.dz[n] = z; - ++n; - } - if (u >= u_hi) - break; - ++u; - u_ = (u_ + 1 == nu ? 0 : u_ + 1); + for (int u = first; u <= last;) { + const int u_ = gemmi::modulo(u, nu), len = std::min(last - u + 1, nu - u_); + points.runs.push_back({row + u_, len}); + u += len; + n += len; } } } @@ -310,37 +338,35 @@ struct AtomDensity { // gemmi's ExpSum / ExpAnisoSum::calculate() for the points of a box, a Gaussian at a time: every // point still sums its Gaussians in gemmi's order, starting from 0, and the loops over the points // vectorise where gemmi's loop over one point's few Gaussians does not. The points are then added - // to the grid in the order given. - void operator()(float *data, const DensityPoints &p) const { - constexpr int BLOCK = 256; + // to the grid in the order given, a run of consecutive grid points at a time. + void operator()(float *data, DensityPoints &p) const { const int n = p.n; - for (int k0 = 0; k0 < n; k0 += BLOCK) { - const int m = std::min(BLOCK, n - k0); - float density[BLOCK], x[BLOCK]; - for (int k = 0; k < m; ++k) - density[k] = 0; - if (!is_aniso) { - float r2[BLOCK]; - for (int k = 0; k < m; ++k) - r2[k] = p.r2[k0 + k]; - for (size_t i = 0; i < std::size(iso.a); ++i) { - for (int k = 0; k < m; ++k) - x[k] = std::max(iso.b[i] * r2[k], -88.f); - for (int k = 0; k < m; ++k) - density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]); - } - } else { - for (size_t i = 0; i < std::size(aniso.a); ++i) { - for (int k = 0; k < m; ++k) { - const gemmi::Vec3f r(p.dx[k0 + k], p.dy[k0 + k], p.dz[k0 + k]); - x[k] = std::max(aniso.b[i].r_u_r(r), -88.f); - } - for (int k = 0; k < m; ++k) - density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]); - } + float *density = p.density.data(), *x = p.x.data(); + for (int k = 0; k < n; ++k) + density[k] = 0; + if (!is_aniso) { + const float *r2 = p.r2.data(); + for (size_t i = 0; i < std::size(iso.a); ++i) { + for (int k = 0; k < n; ++k) + x[k] = std::max(iso.b[i] * r2[k], -88.f); + for (int k = 0; k < n; ++k) + density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]); } - for (int k = 0; k < m; ++k) - data[p.idx[k0 + k]] += float(occ * density[k]); + } else { + for (size_t i = 0; i < std::size(aniso.a); ++i) { + for (int k = 0; k < n; ++k) { + const gemmi::Vec3f r(p.dx[k], p.dy[k], p.dz[k]); + x[k] = std::max(aniso.b[i].r_u_r(r), -88.f); + } + for (int k = 0; k < n; ++k) + density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]); + } + } + for (const DensityPoints::Run &run : p.runs) { + float *row = data + run.start; + for (int k = 0; k < run.n; ++k) + row[k] += float(occ * density[k]); + density += run.n; } } };