Model density: walk each row of an atom's box as one run of points
The points of a row within an atom's radius are one run (r2 falls and then rises along the row), so RowRun() finds its ends from the roots of r2 = max_r2 and moves them onto the points the exact single-precision test takes. The row's points are then filled without a test in the loop, and AtomDensity adds them to the grid a run of consecutive grid points at a time instead of by a gathered index per point. The fill and the adds now vectorise with -march=x86-64-v3. The points, their arithmetic and the order of the adds are unchanged, so the grid is the same bit for bit (ModelDensityGPU_BitIdenticalToCPU holds); refinement results do not move. The same run applied to GatherMapPoints() was slower: its rows are about ten points long, and finding the run cost more than the well-predicted branch it removed. Vectorising the gradient reductions by summing in lanes was faster (-16% in AtomGradientsFromMap) but changes the summation order, which moved the optimiser's path and a final R-free by 0.003; both were left out. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
This commit is contained in:
1 parent
e1284f155a
commit
f8eb9ede5c
1 file changed
+77
-51
@@ -178,24 +178,53 @@ DensityGeometry ModelDensityGeometry(const gemmi::Grid<float> &grid) {
|
||||
static_cast<float>(m.a22), static_cast<float>(m.a23), static_cast<float>(m.a33)};
|
||||
}
|
||||
|
||||
// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their index
|
||||
// in the grid, squared distance and delta (the atom's centre minus the point, orthogonal).
|
||||
// The points of one atom's box on one plane within its radius, in the order gemmi visits them: their squared
|
||||
// distance and delta (the atom's centre minus the point, orthogonal), and where they are in the grid - runs of
|
||||
// consecutive points along u, a row's run split where it wraps. density and x are AtomDensity's scratch.
|
||||
struct DensityPoints {
|
||||
struct Run {
|
||||
size_t start; // the grid index of the run's first point
|
||||
int n;
|
||||
};
|
||||
int n = 0;
|
||||
std::vector<size_t> idx;
|
||||
std::vector<float> r2, dx, dy, dz;
|
||||
std::vector<Run> runs;
|
||||
std::vector<float> r2, dx, dy, dz, density, x;
|
||||
void Clear(size_t capacity) {
|
||||
n = 0;
|
||||
if (idx.size() < capacity) {
|
||||
idx.resize(capacity);
|
||||
runs.clear();
|
||||
if (r2.size() < capacity) {
|
||||
r2.resize(capacity);
|
||||
dx.resize(capacity);
|
||||
dy.resize(capacity);
|
||||
dz.resize(capacity);
|
||||
density.resize(capacity);
|
||||
x.resize(capacity);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// The points of one row of a density box, u_lo to u_hi, that the walk takes: those where r2 = yz + x x,
|
||||
// x = m00 (nctr0 - u) + x_yz, is not above max_r2. r2 falls and then rises along the row, so they are one run, first to
|
||||
// last (none where first > last). Its ends are put at the roots of r2 = max_r2, then moved onto the points the test
|
||||
// itself takes, so the run is the walk's, point for point, and the loops over it need no test.
|
||||
void RowRun(float m00, float nctr0, float x_yz, float yz, float max_r2, int u_lo, int u_hi, int &first, int &last) {
|
||||
const auto inside = [&](int u) {
|
||||
const float x = m00 * (nctr0 - static_cast<float>(u)) + x_yz;
|
||||
return !(yz + x * x > max_r2);
|
||||
};
|
||||
const double centre = nctr0 + double(x_yz) / m00, half = std::sqrt(std::max(0.0, double(max_r2) - yz)) / m00;
|
||||
first = std::max(u_lo, static_cast<int>(std::ceil(centre - half)));
|
||||
last = std::min(u_hi, static_cast<int>(std::floor(centre + half)));
|
||||
while (first <= last && !inside(first))
|
||||
first++;
|
||||
while (first > u_lo && inside(first - 1))
|
||||
first--;
|
||||
while (last < u_hi && inside(last + 1))
|
||||
last++;
|
||||
while (last >= first && !inside(last))
|
||||
last--;
|
||||
}
|
||||
|
||||
// UseAtomBoxesByPlane() for the density's boxes: the same planes, atoms and walk. A point at (u, v, w), the
|
||||
// box's own (unwrapped) indices, is f = centre - (u, v, w) from the atom, and in A
|
||||
// x = m00 fx + (m01 fy + m02 fz), y = m11 fy + m12 fz, z = m22 fz, r2 = (y y + z z) + x x,
|
||||
@@ -220,7 +249,7 @@ void UseDensityBoxesByPlane(gemmi::Grid<float> &grid, const std::vector<DensityB
|
||||
const int u_lo = b.c[0] - b.d[0], u_hi = b.c[0] + b.d[0];
|
||||
const int v_lo = b.c[1] - b.d[1], v_hi = b.c[1] + b.d[1];
|
||||
const int w_lo = b.c[2] - b.d[2], w_hi = b.c[2] + b.d[2];
|
||||
const int u_0 = gemmi::modulo(u_lo, nu), v_0 = gemmi::modulo(v_lo, nv);
|
||||
const int v_0 = gemmi::modulo(v_lo, nv);
|
||||
points.Clear(static_cast<size_t>(2 * b.d[0] + 1) * (2 * b.d[1] + 1) * (1 + 2 * b.d[2] / nw));
|
||||
int n = 0;
|
||||
for (int w = w_lo + gemmi::modulo(plane - w_lo, nw); w <= w_hi; w += nw) {
|
||||
@@ -233,22 +262,21 @@ void UseDensityBoxesByPlane(gemmi::Grid<float> &grid, const std::vector<DensityB
|
||||
if (yz > b.max_r2)
|
||||
continue; // so is every point of the row: x x only adds
|
||||
const float x_yz = m.m01 * fy + m.m02 * fz;
|
||||
int first, last;
|
||||
RowRun(m.m00, b.nctr[0], x_yz, yz, b.max_r2, u_lo, u_hi, first, last);
|
||||
for (int k = 0; k < last - first + 1; ++k) {
|
||||
const float x = m.m00 * (b.nctr[0] - static_cast<float>(first + k)) + x_yz;
|
||||
points.r2[n + k] = yz + x * x;
|
||||
points.dx[n + k] = x;
|
||||
points.dy[n + k] = y;
|
||||
points.dz[n + k] = z;
|
||||
}
|
||||
const size_t row = grid.index_q(0, v_, plane);
|
||||
for (int u = u_lo, u_ = u_0;;) {
|
||||
const float x = m.m00 * (b.nctr[0] - static_cast<float>(u)) + x_yz;
|
||||
const float r2 = yz + x * x;
|
||||
if (!(r2 > b.max_r2)) {
|
||||
points.idx[n] = row + u_;
|
||||
points.r2[n] = r2;
|
||||
points.dx[n] = x;
|
||||
points.dy[n] = y;
|
||||
points.dz[n] = z;
|
||||
++n;
|
||||
}
|
||||
if (u >= u_hi)
|
||||
break;
|
||||
++u;
|
||||
u_ = (u_ + 1 == nu ? 0 : u_ + 1);
|
||||
for (int u = first; u <= last;) {
|
||||
const int u_ = gemmi::modulo(u, nu), len = std::min(last - u + 1, nu - u_);
|
||||
points.runs.push_back({row + u_, len});
|
||||
u += len;
|
||||
n += len;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -310,37 +338,35 @@ struct AtomDensity {
|
||||
// gemmi's ExpSum / ExpAnisoSum::calculate() for the points of a box, a Gaussian at a time: every
|
||||
// point still sums its Gaussians in gemmi's order, starting from 0, and the loops over the points
|
||||
// vectorise where gemmi's loop over one point's few Gaussians does not. The points are then added
|
||||
// to the grid in the order given.
|
||||
void operator()(float *data, const DensityPoints &p) const {
|
||||
constexpr int BLOCK = 256;
|
||||
// to the grid in the order given, a run of consecutive grid points at a time.
|
||||
void operator()(float *data, DensityPoints &p) const {
|
||||
const int n = p.n;
|
||||
for (int k0 = 0; k0 < n; k0 += BLOCK) {
|
||||
const int m = std::min(BLOCK, n - k0);
|
||||
float density[BLOCK], x[BLOCK];
|
||||
for (int k = 0; k < m; ++k)
|
||||
density[k] = 0;
|
||||
if (!is_aniso) {
|
||||
float r2[BLOCK];
|
||||
for (int k = 0; k < m; ++k)
|
||||
r2[k] = p.r2[k0 + k];
|
||||
for (size_t i = 0; i < std::size(iso.a); ++i) {
|
||||
for (int k = 0; k < m; ++k)
|
||||
x[k] = std::max(iso.b[i] * r2[k], -88.f);
|
||||
for (int k = 0; k < m; ++k)
|
||||
density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]);
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i < std::size(aniso.a); ++i) {
|
||||
for (int k = 0; k < m; ++k) {
|
||||
const gemmi::Vec3f r(p.dx[k0 + k], p.dy[k0 + k], p.dz[k0 + k]);
|
||||
x[k] = std::max(aniso.b[i].r_u_r(r), -88.f);
|
||||
}
|
||||
for (int k = 0; k < m; ++k)
|
||||
density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]);
|
||||
}
|
||||
float *density = p.density.data(), *x = p.x.data();
|
||||
for (int k = 0; k < n; ++k)
|
||||
density[k] = 0;
|
||||
if (!is_aniso) {
|
||||
const float *r2 = p.r2.data();
|
||||
for (size_t i = 0; i < std::size(iso.a); ++i) {
|
||||
for (int k = 0; k < n; ++k)
|
||||
x[k] = std::max(iso.b[i] * r2[k], -88.f);
|
||||
for (int k = 0; k < n; ++k)
|
||||
density[k] += iso.a[i] * gemmi::unsafe_expapprox(x[k]);
|
||||
}
|
||||
for (int k = 0; k < m; ++k)
|
||||
data[p.idx[k0 + k]] += float(occ * density[k]);
|
||||
} else {
|
||||
for (size_t i = 0; i < std::size(aniso.a); ++i) {
|
||||
for (int k = 0; k < n; ++k) {
|
||||
const gemmi::Vec3f r(p.dx[k], p.dy[k], p.dz[k]);
|
||||
x[k] = std::max(aniso.b[i].r_u_r(r), -88.f);
|
||||
}
|
||||
for (int k = 0; k < n; ++k)
|
||||
density[k] += aniso.a[i] * gemmi::unsafe_expapprox(x[k]);
|
||||
}
|
||||
}
|
||||
for (const DensityPoints::Run &run : p.runs) {
|
||||
float *row = data + run.start;
|
||||
for (int k = 0; k < run.n; ++k)
|
||||
row[k] += float(occ * density[k]);
|
||||
density += run.n;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user