From 9705845de7ac0794115d760e384720ca6beaa94d Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 15 Nov 2026 23:40:13 +0100 Subject: [PATCH] Refinement CPU gradient: gather the box's points, then vectorisable loops AtomGradientsFromMap() walked each atom's box and summed the Gaussians of every point in one scalar loop (float -> double conversions, an integer modulo per point, a push of nine accumulators per Gaussian for an anisotropic atom), which perf showed as 81% of a CPU refinement. The walk now gathers the points within the radius (map value, place, r^2; the u index wrapped by a counter), the Gaussians' terms are worked out a Gaussian at a time in loops the compiler vectorises (the exponent in a loop of its own, as the two together do not vectorise), and the derivatives are then summed over the points in the walk's order, the Gaussians of each in turn - the same terms in the same order, so the gradient is bit for bit the one before. Microbenchmark (144^3 grid, 2658 atoms, 2442 of them anisotropic, one thread): 0.415 s -> 0.128 s per gradient. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/RefinementXray.cpp | 215 +++++++++++------- 1 file changed, 137 insertions(+), 78 deletions(-) diff --git a/image_analysis/structure_refinement/RefinementXray.cpp b/image_analysis/structure_refinement/RefinementXray.cpp index 955e887d3..595771ede 100644 --- a/image_analysis/structure_refinement/RefinementXray.cpp +++ b/image_analysis/structure_refinement/RefinementXray.cpp @@ -761,35 +761,32 @@ std::unique_ptr MakeRefinementEngineCPU(const gemmi::UnitCell namespace { -// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q -// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) = -// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T, -// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element. -void AnisoGradient(const gemmi::Grid &map, const gemmi::DensityCalculator &dc, - const gemmi::Atom &atom, double blur, std::array &grad, std::array *grad_u) { - using CReal = Table::Coef::coef_type; - const auto &coef = IonFormFactor(atom.element.elem, atom.charge); - const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur)); - const CReal b_max = AnisoRadiusB(aniso_b); - const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max); - const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f); - const float max_r2 = static_cast(radius * radius); +// The points of one atom's box (about the grid point c, c - d to c + d along each axis) within max_r2 of its centre +// nctr (in grid units), in the order of the walk: the map's value there and the place relative to the atom (the +// centre minus the point, orthogonal, and its square) - gathered for the loops over the points that follow, which the +// compiler vectorises, with the scratch they use. +struct MapPoints { + size_t n = 0; // the first n of each + std::vector value, x, y, z, r2, exponent; + std::vector s0, s1, terms; +}; + +void GatherMapPoints(const gemmi::Grid &map, const float nctr[3], const int c[3], const int d[3], float max_r2, + MapPoints &p) { const int nu = map.nu, nv = map.nv, nw = map.nw; + // As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision. const gemmi::UpperTriangularMat33 &om = map.orth_n; const float m00 = static_cast(om.a11), m01 = static_cast(om.a12), m02 = static_cast(om.a13), m11 = static_cast(om.a22), m12 = static_cast(om.a23), m22 = static_cast(om.a33); - const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos); - const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw}; - float nctr[3]; - int c[3], d[3]; - for (int k = 0; k < 3; k++) { - nctr[k] = static_cast(n3[k]); - c[k] = gemmi::iround(n3[k]); + const size_t box = static_cast(2 * d[0] + 1) * (2 * d[1] + 1) * (2 * d[2] + 1); + if (p.value.size() < box) { + p.value.resize(box); + p.x.resize(box); + p.y.resize(box); + p.z.resize(box); + p.r2.resize(box); } - d[0] = std::min(static_cast(std::ceil(radius / map.spacing[0])), nu - 1); - d[1] = std::min(static_cast(std::ceil(radius / map.spacing[1])), nv - 1); - d[2] = std::min(static_cast(std::ceil(radius / map.spacing[2])), nw - 1); - double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0}; + size_t n = 0; for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) { const float fz = nctr[2] - static_cast(w); const float z = m22 * fz; @@ -801,30 +798,101 @@ void AnisoGradient(const gemmi::Grid &map, const gemmi::DensityCalculator if (yz > max_r2) continue; const float x_yz = m01 * fy + m02 * fz; - const size_t row = static_cast(nu) * (gemmi::modulo(v, nv) + static_cast(nv) * w_); - for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) { + const float *row = &map.data[static_cast(nu) * (gemmi::modulo(v, nv) + static_cast(nv) * w_)]; + for (int u = c[0] - d[0], u_ = gemmi::modulo(u, nu); u <= c[0] + d[0]; u++, u_ = (u_ + 1 == nu ? 0 : u_ + 1)) { const float x = m00 * (nctr[0] - static_cast(u)) + x_yz; - if (yz + x * x > max_r2) + const float r2 = yz + x * x; + if (r2 > max_r2) continue; - const double value = map.data[row + gemmi::modulo(u, nu)]; - for (size_t q = 0; q < std::size(precal.a); q++) { - const gemmi::SMat33 &m = precal.b[q]; - const gemmi::Vec3f r(x, y, z); - const double e = value * precal.a[q] * gemmi::unsafe_expapprox(std::max(m.r_u_r(r), -88.f)); - const double mr[3] = {m.u11 * x + m.u12 * y + m.u13 * z, m.u12 * x + m.u22 * y + m.u23 * z, - m.u13 * x + m.u23 * y + m.u33 * z}; - for (int k = 0; k < 3; k++) - g[k] += e * mr[k]; - gu[0] += e * (m.u11 + 2 * mr[0] * mr[0]); - gu[1] += e * (m.u22 + 2 * mr[1] * mr[1]); - gu[2] += e * (m.u33 + 2 * mr[2] * mr[2]); - gu[3] += e * (m.u12 + 2 * mr[0] * mr[1]); - gu[4] += e * (m.u13 + 2 * mr[0] * mr[2]); - gu[5] += e * (m.u23 + 2 * mr[1] * mr[2]); - } + p.value[n] = row[u_]; + p.x[n] = x; + p.y[n] = y; + p.z[n] = z; + p.r2[n] = r2; + n++; } } } + p.n = n; +} + +// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q +// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) = +// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T, +// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element. +void AnisoGradient(const gemmi::Grid &map, const gemmi::DensityCalculator &dc, + const gemmi::Atom &atom, double blur, MapPoints &p, std::array &grad, + std::array *grad_u) { + using CReal = Table::Coef::coef_type; + const auto &coef = IonFormFactor(atom.element.elem, atom.charge); + const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur)); + const CReal b_max = AnisoRadiusB(aniso_b); + const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max); + const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f); + const float max_r2 = static_cast(radius * radius); + const int nu = map.nu, nv = map.nv, nw = map.nw; + const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos); + const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw}; + float nctr[3]; + int c[3], d[3]; + for (int k = 0; k < 3; k++) { + nctr[k] = static_cast(n3[k]); + c[k] = gemmi::iround(n3[k]); + } + d[0] = std::min(static_cast(std::ceil(radius / map.spacing[0])), nu - 1); + d[1] = std::min(static_cast(std::ceil(radius / map.spacing[1])), nv - 1); + d[2] = std::min(static_cast(std::ceil(radius / map.spacing[2])), nw - 1); + GatherMapPoints(map, nctr, c, d, max_r2, p); + const size_t np = p.n, nq = std::size(precal.a); + // In blocks of points: per Gaussian q and point k the nine terms the derivatives sum, e (M_q r)_t and + // e (M_q + 2 (M_q r)(M_q r)^T)_tt' with e = value a_q exp(r^T M_q r), at (t nq + q) BLOCK + k, then summed over the + // points in the walk's order, the Gaussians of each in turn. + constexpr size_t BLOCK = 64; + p.terms.resize(9 * nq * BLOCK); + double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0}; + for (size_t k0 = 0; k0 < np; k0 += BLOCK) { + const size_t nb = std::min(BLOCK, np - k0); + const float *x = &p.x[k0], *y = &p.y[k0], *z = &p.z[k0], *value = &p.value[k0]; + for (size_t q = 0; q < nq; q++) { + const gemmi::SMat33 &m = precal.b[q]; + const float a = precal.a[q]; + float exponent[BLOCK]; + for (size_t k = 0; k < nb; k++) + exponent[k] = std::max(m.r_u_r(gemmi::Vec3f(x[k], y[k], z[k])), -88.f); + double *t[9]; + for (int c = 0; c < 9; c++) + t[c] = &p.terms[(c * nq + q) * BLOCK]; + for (size_t k = 0; k < nb; k++) { + const double e = static_cast(value[k]) * a * gemmi::unsafe_expapprox(exponent[k]); + const double mr0 = m.u11 * x[k] + m.u12 * y[k] + m.u13 * z[k]; + const double mr1 = m.u12 * x[k] + m.u22 * y[k] + m.u23 * z[k]; + const double mr2 = m.u13 * x[k] + m.u23 * y[k] + m.u33 * z[k]; + t[0][k] = e * mr0; + t[1][k] = e * mr1; + t[2][k] = e * mr2; + t[3][k] = e * (m.u11 + 2 * mr0 * mr0); + t[4][k] = e * (m.u22 + 2 * mr1 * mr1); + t[5][k] = e * (m.u33 + 2 * mr2 * mr2); + t[6][k] = e * (m.u12 + 2 * mr0 * mr1); + t[7][k] = e * (m.u13 + 2 * mr0 * mr2); + t[8][k] = e * (m.u23 + 2 * mr1 * mr2); + } + } + for (size_t k = 0; k < nb; k++) + for (size_t q = 0; q < nq; q++) { + const double *t = &p.terms[q * BLOCK + k]; + const size_t c = nq * BLOCK; // from one term to the next + g[0] += t[0]; + g[1] += t[c]; + g[2] += t[2 * c]; + gu[0] += t[3 * c]; + gu[1] += t[4 * c]; + gu[2] += t[5 * c]; + gu[3] += t[6 * c]; + gu[4] += t[7 * c]; + gu[5] += t[8 * c]; + } + } const double occ = atom.occ; grad = {2 * occ * g[0], 2 * occ * g[1], 2 * occ * g[2], occ * (gu[0] + gu[1] + gu[2]) / gemmi::u_to_b()}; if (grad_u != nullptr) @@ -845,16 +913,13 @@ void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &mod if (grad_u != nullptr) grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0}); const int nu = map.nu, nv = map.nv, nw = map.nw; - // As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision. - const gemmi::UpperTriangularMat33 &om = map.orth_n; - const float m00 = static_cast(om.a11), m01 = static_cast(om.a12), m02 = static_cast(om.a13), - m11 = static_cast(om.a22), m12 = static_cast(om.a23), m22 = static_cast(om.a33); constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI; ParallelChunks(static_cast(atoms.size()), nthreads, [&](int lo, int hi) { + MapPoints p; for (int i = lo; i < hi; i++) { const gemmi::Atom &atom = *atoms[i]; if (atom.aniso.nonzero()) { - AnisoGradient(map, dc, atom, blur, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr); + AnisoGradient(map, dc, atom, blur, p, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr); continue; } const float b = static_cast(atom.b_iso + blur); @@ -872,38 +937,32 @@ void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &mod d[0] = std::min(static_cast(std::ceil(radius / map.spacing[0])), nu - 1); d[1] = std::min(static_cast(std::ceil(radius / map.spacing[1])), nv - 1); d[2] = std::min(static_cast(std::ceil(radius / map.spacing[2])), nw - 1); - double gx = 0, gy = 0, gz = 0, gb = 0; - for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) { - const float fz = nctr[2] - static_cast(w); - const float z = m22 * fz; - const int w_ = gemmi::modulo(w, nw); - for (int v = c[1] - d[1]; v <= c[1] + d[1]; v++) { - const float fy = nctr[1] - static_cast(v); - const float y = m11 * fy + m12 * fz; - const float yz = y * y + z * z; - if (yz > max_r2) - continue; - const float x_yz = m01 * fy + m02 * fz; - const size_t row = static_cast(nu) * (gemmi::modulo(v, nv) + static_cast(nv) * w_); - for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) { - const float x = m00 * (nctr[0] - static_cast(u)) + x_yz; - const float r2 = yz + x * x; - if (r2 > max_r2) - continue; - const double value = map.data[row + gemmi::modulo(u, nu)]; - double dr2 = 0, db = 0; - for (size_t q = 0; q < std::size(precal.a); q++) { - const double e = precal.a[q] * gemmi::unsafe_expapprox(std::max(precal.b[q] * r2, -88.f)); - dr2 += e * precal.b[q]; - db += e * precal.b[q] * (1.5 + precal.b[q] * r2); - } - gx += value * dr2 * x; - gy += value * dr2 * y; - gz += value * dr2 * z; - gb += value * db; - } + GatherMapPoints(map, nctr, c, d, max_r2, p); + const size_t np = p.n; + // the two sums over the Gaussians per point, a Gaussian at a time, each point still summing them in order + // from 0 (as AtomDensity does); the exponent in a loop of its own, as the two together do not vectorise + std::vector &sum_dr2 = p.s0, &sum_db = p.s1; + sum_dr2.assign(np, 0.0); + sum_db.assign(np, 0.0); + p.exponent.resize(np); + for (size_t q = 0; q < std::size(precal.a); q++) { + const float a = precal.a[q], bq = precal.b[q]; + for (size_t k = 0; k < np; k++) + p.exponent[k] = std::max(bq * p.r2[k], -88.f); + for (size_t k = 0; k < np; k++) { + const double e = a * gemmi::unsafe_expapprox(p.exponent[k]); + sum_dr2[k] += e * bq; + sum_db[k] += e * bq * (1.5 + bq * p.r2[k]); } } + double gx = 0, gy = 0, gz = 0, gb = 0; + for (size_t k = 0; k < np; k++) { + const double v = p.value[k]; + gx += v * sum_dr2[k] * p.x[k]; + gy += v * sum_dr2[k] * p.y[k]; + gz += v * sum_dr2[k] * p.z[k]; + gb += v * sum_db[k]; + } // rho = occ sum_q a_q exp(b_q r^2): d rho / d(centre) = occ 2 (sum a_q b_q e_q) (centre - point), and // with b_q = -4 pi^2 / (B_q + B + blur), d rho / dB = occ sum a_q e_q b_q (1.5 + b_q r^2) / (4 pi^2). grad[i] = {2.0 * atom.occ * gx, 2.0 * atom.occ * gy, 2.0 * atom.occ * gz, atom.occ * gb / FOUR_PI2};