Model density: walk each row of an atom's box as one run of points

The points of a row within an atom's radius are one run (r2 falls and then rises along
the row), so RowRun() finds its ends from the roots of r2 = max_r2 and moves them onto the
points the exact single-precision test takes. The row's points are then filled without a
test in the loop, and AtomDensity adds them to the grid a run of consecutive grid points at
a time instead of by a gathered index per point. The fill and the adds now vectorise with
-march=x86-64-v3.

The points, their arithmetic and the order of the adds are unchanged, so the grid is the
same bit for bit (ModelDensityGPU_BitIdenticalToCPU holds); refinement results do not move.

The same run applied to GatherMapPoints() was slower: its rows are about ten points long,
and finding the run cost more than the well-predicted branch it removed. Vectorising the
gradient reductions by summing in lanes was faster (-16% in AtomGradientsFromMap) but
changes the summation order, which moved the optimiser's path and a final R-free by 0.003;
both were left out.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
This commit is contained in:
jungfrauandClaude Opus 5.5 committed 2026-11-16 17:14:54 +01:00
1 parent e1284f155a
commit f8eb9ede5c
1 file changed
+77 -51
@@ -178,24 +178,53 @@ DensityGeometry ModelDensityGeometry(const gemmi::Grid<float> &grid) {
static_cast<float>(m.a22), static_cast<float>(m.a23), static_cast<float>(m.a33)};
}
// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their index
// in the grid, squared distance and delta (the atom's centre minus the point, orthogonal).
// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their squared
// distance and delta (the atom's centre minus the point, orthogonal), and where they are in the grid - runs of
// consecutive points along u, a row's run split where it wraps. density and x are AtomDensity's scratch.
struct DensityPoints {
struct Run {
size_t start; // the grid index of the run's first point
int n;
};
int n = 0;
std::vector<size_t> idx;
std::vector<float> r2, dx, dy, dz;
std::vector<Run> runs;
std::vector<float> r2, dx, dy, dz, density, x;
void Clear(size_t capacity) {
n = 0;
if (idx.size() < capacity) {
idx.resize(capacity);
runs.clear();
if (r2.size() < capacity) {
r2.resize(capacity);
dx.resize(capacity);
dy.resize(capacity);
dz.resize(capacity);
density.resize(capacity);
x.resize(capacity);
}
}
};
// The points of one row of a density box, u_lo to u_hi, that the walk takes: those where r2 = yz + x x,
// x = m00 (nctr0 - u) + x_yz, is not above max_r2. r2 falls and then rises along the row, so they are one run, first to
// last (none where first > last). Its ends are put at the roots of r2 = max_r2, then moved onto the points the test
// itself takes, so the run is the walk's, point for point, and the loops over it need no test.
void RowRun(float m00, float nctr0, float x_yz, float yz, float max_r2, int u_lo, int u_hi, int &first, int &last) {
const auto inside = [&](int u) {
const float x = m00 * (nctr0 - static_cast<float>(u)) + x_yz;
return !(yz + x * x > max_r2);
};
const double centre = nctr0 + double(x_yz) / m00, half = std::sqrt(std::max(0.0, double(max_r2) - yz)) / m00;
first = std::max(u_lo, static_cast<int>(std::ceil(centre - half)));
last = std::min(u_hi, static_cast<int>(std::floor(centre + half)));
while (first <= last && !inside(first))
first++;
while (first > u_lo && inside(first - 1))
first--;
while (last < u_hi && inside(last + 1))
last++;
while (last >= first && !inside(last))
last--;
}
// UseAtomBoxesByPlane() for the density's boxes: the same planes, atoms and walk. A point at (u, v, w), the
// box's own (unwrapped) indices, is f = centre - (u, v, w) from the atom, and in A
// x = m00 fx + (m01 fy + m02 fz), y = m11 fy + m12 fz, z = m22 fz, r2 = (y y + z z) + x x,
@@ -220,7 +249,7 @@ void UseDensityBoxesByPlane(gemmi::Grid<float> &grid, const std::vector<DensityB
const int u_lo = b.c[0] - b.d[0], u_hi = b.c[0] + b.d[0];
const int v_lo = b.c[1] - b.d[1], v_hi = b.c[1] + b.d[1];
const int w_lo = b.c[2] - b.d[2], w_hi = b.c[2] + b.d[2];
const int u_0 = gemmi::modulo(u_lo, nu), v_0 = gemmi::modulo(v_lo, nv);
const int v_0 = gemmi::modulo(v_lo, nv);
points.Clear(static_cast<size_t>(2 * b.d[0] + 1) * (2 * b.d[1] + 1) * (1 + 2 * b.d[2] / nw));
int n = 0;
for (int w = w_lo + gemmi::modulo(plane - w_lo, nw); w <= w_hi; w += nw) {
@@ -233,22 +262,21 @@ void UseDensityBoxesByPlane(gemmi::Grid<float> &grid, const std::vector<DensityB
if (yz > b.max_r2)
continue; // so is every point of the row: x x only adds
const float x_yz = m.m01 * fy + m.m02 * fz;
int first, last;
RowRun(m.m00, b.nctr[0], x_yz, yz, b.max_r2, u_lo, u_hi, first, last);
for (int k = 0; k < last - first + 1; ++k) {
const float x = m.m00 * (b.nctr[0] - static_cast<float>(first + k)) + x_yz;
points.r2[n + k] = yz + x * x;
points.dx[n + k] = x;
points.dy[n + k] = y;
points.dz[n + k] = z;
}
const size_t row = grid.index_q(0, v_, plane);
for (int u = u_lo, u_ = u_0;;) {
const float x = m.m00 * (b.nctr[0] - static_cast<float>(u)) + x_yz;
const float r2 = yz + x * x;
if (!(r2 > b.max_r2)) {
points.idx[n] = row + u_;
points.r2[n] = r2;
points.dx[n] = x;
points.dy[n] = y;
points.dz[n] = z;
++n;
}
if (u >= u_hi)
break;
++u;
u_ = (u_ + 1 == nu ? 0 : u_ + 1);
for (int u = first; u <= last;) {
const int u_ = gemmi::modulo(u, nu), len = std::min(last - u + 1, nu - u_);
points.runs.push_back({row + u_, len});
u += len;
n += len;
}
}
}
@@ -310,37 +338,35 @@ struct AtomDensity {
// gemmi's ExpSum / ExpAnisoSum::calculate() for the points of a box, a Gaussian at a time: every
// point still sums its Gaussians in gemmi's order, starting from 0, and the loops over the points
// vectorise where gemmi's loop over one point's few Gaussians does not. The points are then added
// to the grid in the order given.
void operator()(float *data, const DensityPoints &p) const {
constexpr int BLOCK = 256;
// to the grid in the order given, a run of consecutive grid points at a time.
void operator()(float *data, DensityPoints &p) const {
const int n = p.n;
for (int k0 = 0; k0 < n; k0 += BLOCK) {
const int m = std::min(BLOCK, n - k0);
float density[BLOCK], x[BLOCK];
for (int k = 0; k < m; ++k)
density[k] = 0;
if (!is_aniso) {
float r2[BLOCK];
for (int k = 0; k < m; ++k)
r2[k] = p.r2[k0 + k];
for (size_t i = 0; i < std::size(iso.a); ++i) {
for (int k = 0; k < m; ++k)
x[k] = std::max(iso.b[i] * r2[k], -88.f);
for (int k = 0; k < m; ++k)
density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]);
}
} else {
for (size_t i = 0; i < std::size(aniso.a); ++i) {
for (int k = 0; k < m; ++k) {
const gemmi::Vec3f r(p.dx[k0 + k], p.dy[k0 + k], p.dz[k0 + k]);
x[k] = std::max(aniso.b[i].r_u_r(r), -88.f);
}
for (int k = 0; k < m; ++k)
density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]);
}
float *density = p.density.data(), *x = p.x.data();
for (int k = 0; k < n; ++k)
density[k] = 0;
if (!is_aniso) {
const float *r2 = p.r2.data();
for (size_t i = 0; i < std::size(iso.a); ++i) {
for (int k = 0; k < n; ++k)
x[k] = std::max(iso.b[i] * r2[k], -88.f);
for (int k = 0; k < n; ++k)
density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]);
}
for (int k = 0; k < m; ++k)
data[p.idx[k0 + k]] += float(occ * density[k]);
} else {
for (size_t i = 0; i < std::size(aniso.a); ++i) {
for (int k = 0; k < n; ++k) {
const gemmi::Vec3f r(p.dx[k], p.dy[k], p.dz[k]);
x[k] = std::max(aniso.b[i].r_u_r(r), -88.f);
}
for (int k = 0; k < n; ++k)
density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]);
}
}
for (const DensityPoints::Run &run : p.runs) {
float *row = data + run.start;
for (int k = 0; k < run.n; ++k)
row[k] += float(occ * density[k]);
density += run.n;
}
}
};