Refinement CPU gradient: gather the box's points, then vectorisable loops

AtomGradientsFromMap() walked each atom's box and summed the Gaussians of every point in one scalar loop
(float -> double conversions, an integer modulo per point, a push of nine accumulators per Gaussian for an
anisotropic atom), which perf showed as 81% of a CPU refinement. The walk now gathers the points within the
radius (map value, place, r^2; the u index wrapped by a counter), the Gaussians' terms are worked out a
Gaussian at a time in loops the compiler vectorises (the exponent in a loop of its own, as the two together do
not vectorise), and the derivatives are then summed over the points in the walk's order, the Gaussians of each
in turn - the same terms in the same order, so the gradient is bit for bit the one before.

Microbenchmark (144^3 grid, 2658 atoms, 2442 of them anisotropic, one thread): 0.415 s -> 0.128 s per gradient.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-15 23:40:13 +01:00
1 parent 04baa16060
commit 9705845de7
1 file changed
+137 -78
@@ -761,35 +761,32 @@ std::unique_ptr<RefinementEngine> MakeRefinementEngineCPU(const gemmi::UnitCell
namespace {
// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q
// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) =
// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T,
// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element.
void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator<Table, float> &dc,
const gemmi::Atom &atom, double blur, std::array<double, 4> &grad, std::array<double, 6> *grad_u) {
using CReal = Table::Coef::coef_type;
const auto &coef = IonFormFactor<float>(atom.element.elem, atom.charge);
const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
const CReal b_max = AnisoRadiusB(aniso_b);
const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
const float max_r2 = static_cast<float>(radius * radius);
// The points of one atom's box (about the grid point c, c - d to c + d along each axis) within max_r2 of its centre
// nctr (in grid units), in the order of the walk: the map's value there and the place relative to the atom (the
// centre minus the point, orthogonal, and its square) - gathered for the loops over the points that follow, which the
// compiler vectorises, with the scratch they use.
struct MapPoints {
size_t n = 0; // the first n of each
std::vector<float> value, x, y, z, r2, exponent;
std::vector<double> s0, s1, terms;
};
void GatherMapPoints(const gemmi::Grid<float> &map, const float nctr[3], const int c[3], const int d[3], float max_r2,
MapPoints &p) {
const int nu = map.nu, nv = map.nv, nw = map.nw;
// As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision.
const gemmi::UpperTriangularMat33 &om = map.orth_n;
const float m00 = static_cast<float>(om.a11), m01 = static_cast<float>(om.a12), m02 = static_cast<float>(om.a13),
m11 = static_cast<float>(om.a22), m12 = static_cast<float>(om.a23), m22 = static_cast<float>(om.a33);
const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw};
float nctr[3];
int c[3], d[3];
for (int k = 0; k < 3; k++) {
nctr[k] = static_cast<float>(n3[k]);
c[k] = gemmi::iround(n3[k]);
const size_t box = static_cast<size_t>(2 * d[0] + 1) * (2 * d[1] + 1) * (2 * d[2] + 1);
if (p.value.size() < box) {
p.value.resize(box);
p.x.resize(box);
p.y.resize(box);
p.z.resize(box);
p.r2.resize(box);
}
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0};
size_t n = 0;
for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) {
const float fz = nctr[2] - static_cast<float>(w);
const float z = m22 * fz;
@@ -801,30 +798,101 @@ void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator
if (yz > max_r2)
continue;
const float x_yz = m01 * fy + m02 * fz;
const size_t row = static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_);
for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) {
const float *row = &map.data[static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_)];
for (int u = c[0] - d[0], u_ = gemmi::modulo(u, nu); u <= c[0] + d[0]; u++, u_ = (u_ + 1 == nu ? 0 : u_ + 1)) {
const float x = m00 * (nctr[0] - static_cast<float>(u)) + x_yz;
if (yz + x * x > max_r2)
const float r2 = yz + x * x;
if (r2 > max_r2)
continue;
const double value = map.data[row + gemmi::modulo(u, nu)];
for (size_t q = 0; q < std::size(precal.a); q++) {
const gemmi::SMat33<float> &m = precal.b[q];
const gemmi::Vec3f r(x, y, z);
const double e = value * precal.a[q] * gemmi::unsafe_expapprox(std::max(m.r_u_r(r), -88.f));
const double mr[3] = {m.u11 * x + m.u12 * y + m.u13 * z, m.u12 * x + m.u22 * y + m.u23 * z,
m.u13 * x + m.u23 * y + m.u33 * z};
for (int k = 0; k < 3; k++)
g[k] += e * mr[k];
gu[0] += e * (m.u11 + 2 * mr[0] * mr[0]);
gu[1] += e * (m.u22 + 2 * mr[1] * mr[1]);
gu[2] += e * (m.u33 + 2 * mr[2] * mr[2]);
gu[3] += e * (m.u12 + 2 * mr[0] * mr[1]);
gu[4] += e * (m.u13 + 2 * mr[0] * mr[2]);
gu[5] += e * (m.u23 + 2 * mr[1] * mr[2]);
}
p.value[n] = row[u_];
p.x[n] = x;
p.y[n] = y;
p.z[n] = z;
p.r2[n] = r2;
n++;
}
}
}
p.n = n;
}
// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q
// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) =
// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T,
// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element.
void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator<Table, float> &dc,
const gemmi::Atom &atom, double blur, MapPoints &p, std::array<double, 4> &grad,
std::array<double, 6> *grad_u) {
using CReal = Table::Coef::coef_type;
const auto &coef = IonFormFactor<float>(atom.element.elem, atom.charge);
const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
const CReal b_max = AnisoRadiusB(aniso_b);
const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
const float max_r2 = static_cast<float>(radius * radius);
const int nu = map.nu, nv = map.nv, nw = map.nw;
const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw};
float nctr[3];
int c[3], d[3];
for (int k = 0; k < 3; k++) {
nctr[k] = static_cast<float>(n3[k]);
c[k] = gemmi::iround(n3[k]);
}
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
GatherMapPoints(map, nctr, c, d, max_r2, p);
const size_t np = p.n, nq = std::size(precal.a);
// In blocks of points: per Gaussian q and point k the nine terms the derivatives sum, e (M_q r)_t and
// e (M_q + 2 (M_q r)(M_q r)^T)_tt' with e = value a_q exp(r^T M_q r), at (t nq + q) BLOCK + k, then summed over the
// points in the walk's order, the Gaussians of each in turn.
constexpr size_t BLOCK = 64;
p.terms.resize(9 * nq * BLOCK);
double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0};
for (size_t k0 = 0; k0 < np; k0 += BLOCK) {
const size_t nb = std::min(BLOCK, np - k0);
const float *x = &p.x[k0], *y = &p.y[k0], *z = &p.z[k0], *value = &p.value[k0];
for (size_t q = 0; q < nq; q++) {
const gemmi::SMat33<float> &m = precal.b[q];
const float a = precal.a[q];
float exponent[BLOCK];
for (size_t k = 0; k < nb; k++)
exponent[k] = std::max(m.r_u_r(gemmi::Vec3f(x[k], y[k], z[k])), -88.f);
double *t[9];
for (int c = 0; c < 9; c++)
t[c] = &p.terms[(c * nq + q) * BLOCK];
for (size_t k = 0; k < nb; k++) {
const double e = static_cast<double>(value[k]) * a * gemmi::unsafe_expapprox(exponent[k]);
const double mr0 = m.u11 * x[k] + m.u12 * y[k] + m.u13 * z[k];
const double mr1 = m.u12 * x[k] + m.u22 * y[k] + m.u23 * z[k];
const double mr2 = m.u13 * x[k] + m.u23 * y[k] + m.u33 * z[k];
t[0][k] = e * mr0;
t[1][k] = e * mr1;
t[2][k] = e * mr2;
t[3][k] = e * (m.u11 + 2 * mr0 * mr0);
t[4][k] = e * (m.u22 + 2 * mr1 * mr1);
t[5][k] = e * (m.u33 + 2 * mr2 * mr2);
t[6][k] = e * (m.u12 + 2 * mr0 * mr1);
t[7][k] = e * (m.u13 + 2 * mr0 * mr2);
t[8][k] = e * (m.u23 + 2 * mr1 * mr2);
}
}
for (size_t k = 0; k < nb; k++)
for (size_t q = 0; q < nq; q++) {
const double *t = &p.terms[q * BLOCK + k];
const size_t c = nq * BLOCK; // from one term to the next
g[0] += t[0];
g[1] += t[c];
g[2] += t[2 * c];
gu[0] += t[3 * c];
gu[1] += t[4 * c];
gu[2] += t[5 * c];
gu[3] += t[6 * c];
gu[4] += t[7 * c];
gu[5] += t[8 * c];
}
}
const double occ = atom.occ;
grad = {2 * occ * g[0], 2 * occ * g[1], 2 * occ * g[2], occ * (gu[0] + gu[1] + gu[2]) / gemmi::u_to_b()};
if (grad_u != nullptr)
@@ -845,16 +913,13 @@ void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &mod
if (grad_u != nullptr)
grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0});
const int nu = map.nu, nv = map.nv, nw = map.nw;
// As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision.
const gemmi::UpperTriangularMat33 &om = map.orth_n;
const float m00 = static_cast<float>(om.a11), m01 = static_cast<float>(om.a12), m02 = static_cast<float>(om.a13),
m11 = static_cast<float>(om.a22), m12 = static_cast<float>(om.a23), m22 = static_cast<float>(om.a33);
constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
ParallelChunks(static_cast<int>(atoms.size()), nthreads, [&](int lo, int hi) {
MapPoints p;
for (int i = lo; i < hi; i++) {
const gemmi::Atom &atom = *atoms[i];
if (atom.aniso.nonzero()) {
AnisoGradient(map, dc, atom, blur, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr);
AnisoGradient(map, dc, atom, blur, p, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr);
continue;
}
const float b = static_cast<float>(atom.b_iso + blur);
@@ -872,38 +937,32 @@ void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &mod
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
double gx = 0, gy = 0, gz = 0, gb = 0;
for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) {
const float fz = nctr[2] - static_cast<float>(w);
const float z = m22 * fz;
const int w_ = gemmi::modulo(w, nw);
for (int v = c[1] - d[1]; v <= c[1] + d[1]; v++) {
const float fy = nctr[1] - static_cast<float>(v);
const float y = m11 * fy + m12 * fz;
const float yz = y * y + z * z;
if (yz > max_r2)
continue;
const float x_yz = m01 * fy + m02 * fz;
const size_t row = static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_);
for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) {
const float x = m00 * (nctr[0] - static_cast<float>(u)) + x_yz;
const float r2 = yz + x * x;
if (r2 > max_r2)
continue;
const double value = map.data[row + gemmi::modulo(u, nu)];
double dr2 = 0, db = 0;
for (size_t q = 0; q < std::size(precal.a); q++) {
const double e = precal.a[q] * gemmi::unsafe_expapprox(std::max(precal.b[q] * r2, -88.f));
dr2 += e * precal.b[q];
db += e * precal.b[q] * (1.5 + precal.b[q] * r2);
}
gx += value * dr2 * x;
gy += value * dr2 * y;
gz += value * dr2 * z;
gb += value * db;
}
GatherMapPoints(map, nctr, c, d, max_r2, p);
const size_t np = p.n;
// the two sums over the Gaussians per point, a Gaussian at a time, each point still summing them in order
// from 0 (as AtomDensity does); the exponent in a loop of its own, as the two together do not vectorise
std::vector<double> &sum_dr2 = p.s0, &sum_db = p.s1;
sum_dr2.assign(np, 0.0);
sum_db.assign(np, 0.0);
p.exponent.resize(np);
for (size_t q = 0; q < std::size(precal.a); q++) {
const float a = precal.a[q], bq = precal.b[q];
for (size_t k = 0; k < np; k++)
p.exponent[k] = std::max(bq * p.r2[k], -88.f);
for (size_t k = 0; k < np; k++) {
const double e = a * gemmi::unsafe_expapprox(p.exponent[k]);
sum_dr2[k] += e * bq;
sum_db[k] += e * bq * (1.5 + bq * p.r2[k]);
}
}
double gx = 0, gy = 0, gz = 0, gb = 0;
for (size_t k = 0; k < np; k++) {
const double v = p.value[k];
gx += v * sum_dr2[k] * p.x[k];
gy += v * sum_dr2[k] * p.y[k];
gz += v * sum_dr2[k] * p.z[k];
gb += v * sum_db[k];
}
// rho = occ sum_q a_q exp(b_q r^2): d rho / d(centre) = occ 2 (sum a_q b_q e_q) (centre - point), and
// with b_q = -4 pi^2 / (B_q + B + blur), d rho / dB = occ sum a_q e_q b_q (1.5 + b_q r^2) / (4 pi^2).
grad[i] = {2.0 * atom.occ * gx, 2.0 * atom.occ * gy, 2.0 * atom.occ * gz, atom.occ * gb / FOUR_PI2};