Refinement CPU gradient: gather the box's points, then vectorisable loops
AtomGradientsFromMap() walked each atom's box and summed the Gaussians of every point in one scalar loop (float -> double conversions, an integer modulo per point, a push of nine accumulators per Gaussian for an anisotropic atom), which perf showed as 81% of a CPU refinement. The walk now gathers the points within the radius (map value, place, r^2; the u index wrapped by a counter), the Gaussians' terms are worked out a Gaussian at a time in loops the compiler vectorises (the exponent in a loop of its own, as the two together do not vectorise), and the derivatives are then summed over the points in the walk's order, the Gaussians of each in turn - the same terms in the same order, so the gradient is bit for bit the one before. Microbenchmark (144^3 grid, 2658 atoms, 2442 of them anisotropic, one thread): 0.415 s -> 0.128 s per gradient. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
1 parent
04baa16060
commit
9705845de7
1 file changed
+137
-78
@@ -761,35 +761,32 @@ std::unique_ptr<RefinementEngine> MakeRefinementEngineCPU(const gemmi::UnitCell
|
||||
|
||||
namespace {
|
||||
|
||||
// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q
|
||||
// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) =
|
||||
// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T,
|
||||
// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element.
|
||||
void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator<Table, float> &dc,
|
||||
const gemmi::Atom &atom, double blur, std::array<double, 4> &grad, std::array<double, 6> *grad_u) {
|
||||
using CReal = Table::Coef::coef_type;
|
||||
const auto &coef = IonFormFactor<float>(atom.element.elem, atom.charge);
|
||||
const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
|
||||
const CReal b_max = AnisoRadiusB(aniso_b);
|
||||
const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
|
||||
const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
|
||||
const float max_r2 = static_cast<float>(radius * radius);
|
||||
// The points of one atom's box (about the grid point c, c - d to c + d along each axis) within max_r2 of its centre
|
||||
// nctr (in grid units), in the order of the walk: the map's value there and the place relative to the atom (the
|
||||
// centre minus the point, orthogonal, and its square) - gathered for the loops over the points that follow, which the
|
||||
// compiler vectorises, with the scratch they use.
|
||||
struct MapPoints {
|
||||
size_t n = 0; // the first n of each
|
||||
std::vector<float> value, x, y, z, r2, exponent;
|
||||
std::vector<double> s0, s1, terms;
|
||||
};
|
||||
|
||||
void GatherMapPoints(const gemmi::Grid<float> &map, const float nctr[3], const int c[3], const int d[3], float max_r2,
|
||||
MapPoints &p) {
|
||||
const int nu = map.nu, nv = map.nv, nw = map.nw;
|
||||
// As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision.
|
||||
const gemmi::UpperTriangularMat33 &om = map.orth_n;
|
||||
const float m00 = static_cast<float>(om.a11), m01 = static_cast<float>(om.a12), m02 = static_cast<float>(om.a13),
|
||||
m11 = static_cast<float>(om.a22), m12 = static_cast<float>(om.a23), m22 = static_cast<float>(om.a33);
|
||||
const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
|
||||
const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw};
|
||||
float nctr[3];
|
||||
int c[3], d[3];
|
||||
for (int k = 0; k < 3; k++) {
|
||||
nctr[k] = static_cast<float>(n3[k]);
|
||||
c[k] = gemmi::iround(n3[k]);
|
||||
const size_t box = static_cast<size_t>(2 * d[0] + 1) * (2 * d[1] + 1) * (2 * d[2] + 1);
|
||||
if (p.value.size() < box) {
|
||||
p.value.resize(box);
|
||||
p.x.resize(box);
|
||||
p.y.resize(box);
|
||||
p.z.resize(box);
|
||||
p.r2.resize(box);
|
||||
}
|
||||
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
|
||||
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
|
||||
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
|
||||
double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0};
|
||||
size_t n = 0;
|
||||
for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) {
|
||||
const float fz = nctr[2] - static_cast<float>(w);
|
||||
const float z = m22 * fz;
|
||||
@@ -801,30 +798,101 @@ void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator
|
||||
if (yz > max_r2)
|
||||
continue;
|
||||
const float x_yz = m01 * fy + m02 * fz;
|
||||
const size_t row = static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_);
|
||||
for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) {
|
||||
const float *row = &map.data[static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_)];
|
||||
for (int u = c[0] - d[0], u_ = gemmi::modulo(u, nu); u <= c[0] + d[0]; u++, u_ = (u_ + 1 == nu ? 0 : u_ + 1)) {
|
||||
const float x = m00 * (nctr[0] - static_cast<float>(u)) + x_yz;
|
||||
if (yz + x * x > max_r2)
|
||||
const float r2 = yz + x * x;
|
||||
if (r2 > max_r2)
|
||||
continue;
|
||||
const double value = map.data[row + gemmi::modulo(u, nu)];
|
||||
for (size_t q = 0; q < std::size(precal.a); q++) {
|
||||
const gemmi::SMat33<float> &m = precal.b[q];
|
||||
const gemmi::Vec3f r(x, y, z);
|
||||
const double e = value * precal.a[q] * gemmi::unsafe_expapprox(std::max(m.r_u_r(r), -88.f));
|
||||
const double mr[3] = {m.u11 * x + m.u12 * y + m.u13 * z, m.u12 * x + m.u22 * y + m.u23 * z,
|
||||
m.u13 * x + m.u23 * y + m.u33 * z};
|
||||
for (int k = 0; k < 3; k++)
|
||||
g[k] += e * mr[k];
|
||||
gu[0] += e * (m.u11 + 2 * mr[0] * mr[0]);
|
||||
gu[1] += e * (m.u22 + 2 * mr[1] * mr[1]);
|
||||
gu[2] += e * (m.u33 + 2 * mr[2] * mr[2]);
|
||||
gu[3] += e * (m.u12 + 2 * mr[0] * mr[1]);
|
||||
gu[4] += e * (m.u13 + 2 * mr[0] * mr[2]);
|
||||
gu[5] += e * (m.u23 + 2 * mr[1] * mr[2]);
|
||||
}
|
||||
p.value[n] = row[u_];
|
||||
p.x[n] = x;
|
||||
p.y[n] = y;
|
||||
p.z[n] = z;
|
||||
p.r2[n] = r2;
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
p.n = n;
|
||||
}
|
||||
|
||||
// AtomGradientsFromMap() for an anisotropic atom, gridded as PutModelDensityOnGrid() grids it: rho = occ sum_q a_q
|
||||
// exp(r^T M_q r), M_q = -4 pi^2 (B_q I + 8 pi^2 U + blur I)^-1 and r = centre - point. Then d rho / d(centre) =
|
||||
// occ sum_q 2 e_q M_q r, and, as d a_q / dU = -4 pi^2 a_q (B..)^-1 = a_q M_q and d(r^T M_q r) / dU = 2 (M_q r)(M_q r)^T,
|
||||
// d rho / dU = occ sum_q e_q (M_q + 2 (M_q r)(M_q r)^T) - an off-diagonal U_ij twice its matrix element.
|
||||
void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator<Table, float> &dc,
|
||||
const gemmi::Atom &atom, double blur, MapPoints &p, std::array<double, 4> &grad,
|
||||
std::array<double, 6> *grad_u) {
|
||||
using CReal = Table::Coef::coef_type;
|
||||
const auto &coef = IonFormFactor<float>(atom.element.elem, atom.charge);
|
||||
const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
|
||||
const CReal b_max = AnisoRadiusB(aniso_b);
|
||||
const double radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
|
||||
const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
|
||||
const float max_r2 = static_cast<float>(radius * radius);
|
||||
const int nu = map.nu, nv = map.nv, nw = map.nw;
|
||||
const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
|
||||
const double n3[3] = {fpos.x * nu, fpos.y * nv, fpos.z * nw};
|
||||
float nctr[3];
|
||||
int c[3], d[3];
|
||||
for (int k = 0; k < 3; k++) {
|
||||
nctr[k] = static_cast<float>(n3[k]);
|
||||
c[k] = gemmi::iround(n3[k]);
|
||||
}
|
||||
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
|
||||
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
|
||||
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
|
||||
GatherMapPoints(map, nctr, c, d, max_r2, p);
|
||||
const size_t np = p.n, nq = std::size(precal.a);
|
||||
// In blocks of points: per Gaussian q and point k the nine terms the derivatives sum, e (M_q r)_t and
|
||||
// e (M_q + 2 (M_q r)(M_q r)^T)_tt' with e = value a_q exp(r^T M_q r), at (t nq + q) BLOCK + k, then summed over the
|
||||
// points in the walk's order, the Gaussians of each in turn.
|
||||
constexpr size_t BLOCK = 64;
|
||||
p.terms.resize(9 * nq * BLOCK);
|
||||
double g[3] = {0, 0, 0}, gu[6] = {0, 0, 0, 0, 0, 0};
|
||||
for (size_t k0 = 0; k0 < np; k0 += BLOCK) {
|
||||
const size_t nb = std::min(BLOCK, np - k0);
|
||||
const float *x = &p.x[k0], *y = &p.y[k0], *z = &p.z[k0], *value = &p.value[k0];
|
||||
for (size_t q = 0; q < nq; q++) {
|
||||
const gemmi::SMat33<float> &m = precal.b[q];
|
||||
const float a = precal.a[q];
|
||||
float exponent[BLOCK];
|
||||
for (size_t k = 0; k < nb; k++)
|
||||
exponent[k] = std::max(m.r_u_r(gemmi::Vec3f(x[k], y[k], z[k])), -88.f);
|
||||
double *t[9];
|
||||
for (int c = 0; c < 9; c++)
|
||||
t[c] = &p.terms[(c * nq + q) * BLOCK];
|
||||
for (size_t k = 0; k < nb; k++) {
|
||||
const double e = static_cast<double>(value[k]) * a * gemmi::unsafe_expapprox(exponent[k]);
|
||||
const double mr0 = m.u11 * x[k] + m.u12 * y[k] + m.u13 * z[k];
|
||||
const double mr1 = m.u12 * x[k] + m.u22 * y[k] + m.u23 * z[k];
|
||||
const double mr2 = m.u13 * x[k] + m.u23 * y[k] + m.u33 * z[k];
|
||||
t[0][k] = e * mr0;
|
||||
t[1][k] = e * mr1;
|
||||
t[2][k] = e * mr2;
|
||||
t[3][k] = e * (m.u11 + 2 * mr0 * mr0);
|
||||
t[4][k] = e * (m.u22 + 2 * mr1 * mr1);
|
||||
t[5][k] = e * (m.u33 + 2 * mr2 * mr2);
|
||||
t[6][k] = e * (m.u12 + 2 * mr0 * mr1);
|
||||
t[7][k] = e * (m.u13 + 2 * mr0 * mr2);
|
||||
t[8][k] = e * (m.u23 + 2 * mr1 * mr2);
|
||||
}
|
||||
}
|
||||
for (size_t k = 0; k < nb; k++)
|
||||
for (size_t q = 0; q < nq; q++) {
|
||||
const double *t = &p.terms[q * BLOCK + k];
|
||||
const size_t c = nq * BLOCK; // from one term to the next
|
||||
g[0] += t[0];
|
||||
g[1] += t[c];
|
||||
g[2] += t[2 * c];
|
||||
gu[0] += t[3 * c];
|
||||
gu[1] += t[4 * c];
|
||||
gu[2] += t[5 * c];
|
||||
gu[3] += t[6 * c];
|
||||
gu[4] += t[7 * c];
|
||||
gu[5] += t[8 * c];
|
||||
}
|
||||
}
|
||||
const double occ = atom.occ;
|
||||
grad = {2 * occ * g[0], 2 * occ * g[1], 2 * occ * g[2], occ * (gu[0] + gu[1] + gu[2]) / gemmi::u_to_b()};
|
||||
if (grad_u != nullptr)
|
||||
@@ -845,16 +913,13 @@ void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &mod
|
||||
if (grad_u != nullptr)
|
||||
grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0});
|
||||
const int nu = map.nu, nv = map.nv, nw = map.nw;
|
||||
// As ModelGrid.cpp walks a density box: the orthogonalization over the grid size in single precision.
|
||||
const gemmi::UpperTriangularMat33 &om = map.orth_n;
|
||||
const float m00 = static_cast<float>(om.a11), m01 = static_cast<float>(om.a12), m02 = static_cast<float>(om.a13),
|
||||
m11 = static_cast<float>(om.a22), m12 = static_cast<float>(om.a23), m22 = static_cast<float>(om.a33);
|
||||
constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
|
||||
ParallelChunks(static_cast<int>(atoms.size()), nthreads, [&](int lo, int hi) {
|
||||
MapPoints p;
|
||||
for (int i = lo; i < hi; i++) {
|
||||
const gemmi::Atom &atom = *atoms[i];
|
||||
if (atom.aniso.nonzero()) {
|
||||
AnisoGradient(map, dc, atom, blur, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr);
|
||||
AnisoGradient(map, dc, atom, blur, p, grad[i], grad_u != nullptr ? &(*grad_u)[i] : nullptr);
|
||||
continue;
|
||||
}
|
||||
const float b = static_cast<float>(atom.b_iso + blur);
|
||||
@@ -872,38 +937,32 @@ void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &mod
|
||||
d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), nu - 1);
|
||||
d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), nv - 1);
|
||||
d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), nw - 1);
|
||||
double gx = 0, gy = 0, gz = 0, gb = 0;
|
||||
for (int w = c[2] - d[2]; w <= c[2] + d[2]; w++) {
|
||||
const float fz = nctr[2] - static_cast<float>(w);
|
||||
const float z = m22 * fz;
|
||||
const int w_ = gemmi::modulo(w, nw);
|
||||
for (int v = c[1] - d[1]; v <= c[1] + d[1]; v++) {
|
||||
const float fy = nctr[1] - static_cast<float>(v);
|
||||
const float y = m11 * fy + m12 * fz;
|
||||
const float yz = y * y + z * z;
|
||||
if (yz > max_r2)
|
||||
continue;
|
||||
const float x_yz = m01 * fy + m02 * fz;
|
||||
const size_t row = static_cast<size_t>(nu) * (gemmi::modulo(v, nv) + static_cast<size_t>(nv) * w_);
|
||||
for (int u = c[0] - d[0]; u <= c[0] + d[0]; u++) {
|
||||
const float x = m00 * (nctr[0] - static_cast<float>(u)) + x_yz;
|
||||
const float r2 = yz + x * x;
|
||||
if (r2 > max_r2)
|
||||
continue;
|
||||
const double value = map.data[row + gemmi::modulo(u, nu)];
|
||||
double dr2 = 0, db = 0;
|
||||
for (size_t q = 0; q < std::size(precal.a); q++) {
|
||||
const double e = precal.a[q] * gemmi::unsafe_expapprox(std::max(precal.b[q] * r2, -88.f));
|
||||
dr2 += e * precal.b[q];
|
||||
db += e * precal.b[q] * (1.5 + precal.b[q] * r2);
|
||||
}
|
||||
gx += value * dr2 * x;
|
||||
gy += value * dr2 * y;
|
||||
gz += value * dr2 * z;
|
||||
gb += value * db;
|
||||
}
|
||||
GatherMapPoints(map, nctr, c, d, max_r2, p);
|
||||
const size_t np = p.n;
|
||||
// the two sums over the Gaussians per point, a Gaussian at a time, each point still summing them in order
|
||||
// from 0 (as AtomDensity does); the exponent in a loop of its own, as the two together do not vectorise
|
||||
std::vector<double> &sum_dr2 = p.s0, &sum_db = p.s1;
|
||||
sum_dr2.assign(np, 0.0);
|
||||
sum_db.assign(np, 0.0);
|
||||
p.exponent.resize(np);
|
||||
for (size_t q = 0; q < std::size(precal.a); q++) {
|
||||
const float a = precal.a[q], bq = precal.b[q];
|
||||
for (size_t k = 0; k < np; k++)
|
||||
p.exponent[k] = std::max(bq * p.r2[k], -88.f);
|
||||
for (size_t k = 0; k < np; k++) {
|
||||
const double e = a * gemmi::unsafe_expapprox(p.exponent[k]);
|
||||
sum_dr2[k] += e * bq;
|
||||
sum_db[k] += e * bq * (1.5 + bq * p.r2[k]);
|
||||
}
|
||||
}
|
||||
double gx = 0, gy = 0, gz = 0, gb = 0;
|
||||
for (size_t k = 0; k < np; k++) {
|
||||
const double v = p.value[k];
|
||||
gx += v * sum_dr2[k] * p.x[k];
|
||||
gy += v * sum_dr2[k] * p.y[k];
|
||||
gz += v * sum_dr2[k] * p.z[k];
|
||||
gb += v * sum_db[k];
|
||||
}
|
||||
// rho = occ sum_q a_q exp(b_q r^2): d rho / d(centre) = occ 2 (sum a_q b_q e_q) (centre - point), and
|
||||
// with b_q = -4 pi^2 / (B_q + B + blur), d rho / dB = occ sum a_q e_q b_q (1.5 + b_q r^2) / (4 pi^2).
|
||||
grad[i] = {2.0 * atom.occ * gx, 2.0 * atom.occ * gy, 2.0 * atom.occ * gz, atom.occ * gb / FOUR_PI2};
|
||||
|
||||
Reference in new issue
Block a user