Refinement Newton: the restraints' Gauss-Newton product on the GPU
With the engine on a GPU, RestraintNormalMatrix hands At/Multiply/Diagonal/Blocks to
RestraintNormalMatrixGPU (RefinementNewtonGPU.{h,cpp,cu}), on the engine's card on a stream of
its own at the highest priority. J is built on the device, one thread per restraint from the
REFINE_HD terms in float, into a fixed layout worked out once from the set (rows the CPU drops,
E = 0, are kept as zeros). A product is two gathers, J v per row and J^T t per parameter atom,
without atomics, in a fixed order.
Precision: J in float, v/t/hv as double-float pairs (error-free product and 2Sum per term).
Plain float sums are not enough: J v is a small difference of large terms in the soft directions
the X-ray term decides, and with plain float the outcomes moved beyond the CPU's own noise on a
model with stiff restraints (weight choice 2/10 vs 8/10 at the CPU's value over jittered runs).
Float J with exact accumulation matched the CPU's spread, as do the double-float pairs.
On a large rotation set (GPU, Newton joint, relative stop 0.05): product thread-time 1008 s ->
38 s, jacobian 114 s -> 8 s, refine wall 191 s -> 32 s under load; evaluations and R-free within
the spread a 1e-12 relative jitter of the CPU product gives.
Interface: RestraintNormalMatrix now takes the Context (for the device) instead of n_atoms and
nthreads.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
1 parent
4dc4a47d2f
commit
5360629f4e
7 files changed
+602
-8
No files matched your search
@@ -842,7 +842,8 @@ pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472
|
||||
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
|
||||
|
||||
**Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an
|
||||
Apple GPU (`BraggExp.h`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the
|
||||
Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU
|
||||
(`RefinementNewtonGPU.cu`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the
|
||||
Elementary Functions" (1980), Prentice-Hall; the exact product (with Veltkamp's split) and Fast2Sum
|
||||
of T. J. Dekker, "A floating-point technique for extending the available precision" (1971), Numer.
|
||||
Math. 18, 224-242; and the 2Sum of D. E. Knuth, "The Art of Computer Programming, Vol. 2:
|
||||
|
||||
@@ -37,6 +37,9 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC
|
||||
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementGPU.cpp>
|
||||
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementGPU.cu>
|
||||
RefinementGPU.h RefinementGPUEngine.h
|
||||
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementNewtonGPU.cpp>
|
||||
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementNewtonGPU.cu>
|
||||
RefinementNewtonGPU.h
|
||||
${CMAKE_CURRENT_BINARY_DIR}/MonomerLibraryData.cpp)
|
||||
|
||||
# The subset of the CCP4 Monomer Library that refinement restrains with (monomers/, chosen by
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
#include "../../common/CUDAWrapper.h"
|
||||
#include "RefinementGPU.h"
|
||||
#include "RefinementNewtonGPU.h"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -1160,18 +1161,27 @@ std::vector<double> FlooredDiagonal(std::vector<double> d) {
|
||||
}
|
||||
|
||||
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
|
||||
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it.
|
||||
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
|
||||
// a GPU, on its card (RestraintNormalMatrixGPU).
|
||||
class RestraintNormalMatrix {
|
||||
public:
|
||||
RestraintNormalMatrix(const std::vector<int> ¶m_atoms, size_t n_atoms, bool xyz, size_t nthreads)
|
||||
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(nthreads),
|
||||
param_of_(n_atoms, -1) {
|
||||
RestraintNormalMatrix(const Context &ctx, const std::vector<int> ¶m_atoms, bool xyz)
|
||||
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(ctx.nthreads),
|
||||
param_of_(ctx.atoms.size(), -1) {
|
||||
for (size_t k = 0; k < param_atoms.size(); k++)
|
||||
param_of_[param_atoms[k]] = static_cast<int>(k);
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (ctx.engine->OnGPU())
|
||||
gpu_ = std::make_unique<RestraintNormalMatrixGPU>(ctx.device, param_atoms, ctx.atoms.size(), xyz);
|
||||
#endif
|
||||
}
|
||||
|
||||
void At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb) {
|
||||
const RefinementStage stage("newton.jacobian");
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (gpu_)
|
||||
return gpu_->At(set, xyzb);
|
||||
#endif
|
||||
jac_ = RestraintGaussNewton(set, xyzb);
|
||||
const size_t rows = jac_.start.size() - 1;
|
||||
row_of_.resize(jac_.atom.size());
|
||||
@@ -1199,6 +1209,10 @@ public:
|
||||
// hv = J^T J v: J v per row, then per parameter atom the sum over its entries
|
||||
void Multiply(const std::vector<double> &v, std::vector<double> &hv) {
|
||||
const RefinementStage stage("newton.product");
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (gpu_)
|
||||
return gpu_->Multiply(v, hv);
|
||||
#endif
|
||||
const int rows = static_cast<int>(t_.size());
|
||||
ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) {
|
||||
for (int r = lo; r < hi; r++) {
|
||||
@@ -1224,6 +1238,10 @@ public:
|
||||
|
||||
// Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only.
|
||||
std::vector<std::array<double, 6>> Blocks() const {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (gpu_)
|
||||
return gpu_->Blocks();
|
||||
#endif
|
||||
std::vector<std::array<double, 6>> b(n_params_, {0, 0, 0, 0, 0, 0});
|
||||
for (size_t k = 0; k < n_params_; k++)
|
||||
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) {
|
||||
@@ -1239,6 +1257,10 @@ public:
|
||||
}
|
||||
|
||||
std::vector<double> Diagonal() const {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (gpu_)
|
||||
return gpu_->Diagonal();
|
||||
#endif
|
||||
std::vector<double> d(per_ * n_params_, 0.0);
|
||||
for (size_t k = 0; k < n_params_; k++)
|
||||
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++)
|
||||
@@ -1254,6 +1276,9 @@ private:
|
||||
RestraintJacobian jac_;
|
||||
std::vector<int> row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none)
|
||||
std::vector<double> t_;
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
std::unique_ptr<RestraintNormalMatrixGPU> gpu_;
|
||||
#endif
|
||||
};
|
||||
|
||||
// z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate.
|
||||
@@ -1295,7 +1320,7 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto
|
||||
std::vector<double> h(n);
|
||||
for (size_t k = 0; k < n; k++)
|
||||
h[k] = xray[target.Atoms()[k / per]];
|
||||
RestraintNormalMatrix normal(target.Atoms(), ctx.atoms.size(), xyz, ctx.nthreads);
|
||||
RestraintNormalMatrix normal(ctx, target.Atoms(), xyz);
|
||||
std::vector<double> x_at = p;
|
||||
target.Set(p.data());
|
||||
normal.At(set, ctx.XyzB());
|
||||
@@ -1332,8 +1357,7 @@ void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vect
|
||||
const RestraintSet &set_b, double w, double adp_w, int iterations) {
|
||||
const size_t n = atoms.size();
|
||||
const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT;
|
||||
RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads),
|
||||
normal_b(atoms, ctx.atoms.size(), false, ctx.nthreads);
|
||||
RestraintNormalMatrix normal_xyz(ctx, atoms, true), normal_b(ctx, atoms, false);
|
||||
std::vector<double> x_at = target.Get();
|
||||
auto at = [&](const std::vector<double> &x) {
|
||||
target.Set(x.data());
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "RefinementNewtonGPU.h"
|
||||
|
||||
#include "RefinementRestraints.h"
|
||||
|
||||
// The host half: the index of J, as RestraintGaussNewton lays out its rows, with every row kept.
|
||||
void RestraintNormalMatrixGPU::At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb) {
|
||||
if (!indexed_) {
|
||||
std::vector<int> record_entry, start{0}, atom;
|
||||
for (const RestraintRecord &r : set.records) {
|
||||
record_entry.push_back(static_cast<int>(atom.size()));
|
||||
switch (static_cast<RestraintKind>(r.kind)) {
|
||||
case RestraintKind::Plane:
|
||||
// one row per atom of the plane, each over all of them
|
||||
for (int i = 0; i < r.atom[1]; i++) {
|
||||
for (int k = 0; k < r.atom[1]; k++)
|
||||
atom.push_back(set.plane_atoms[r.atom[0] + k]);
|
||||
start.push_back(static_cast<int>(atom.size()));
|
||||
}
|
||||
continue;
|
||||
case RestraintKind::Angle:
|
||||
atom.insert(atom.end(), r.atom, r.atom + 3);
|
||||
break;
|
||||
case RestraintKind::Torsion:
|
||||
case RestraintKind::Chiral:
|
||||
case RestraintKind::NcsDistance:
|
||||
atom.insert(atom.end(), r.atom, r.atom + 4);
|
||||
break;
|
||||
default: // Bond, Repulsion, BSimilarity
|
||||
atom.insert(atom.end(), r.atom, r.atom + 2);
|
||||
break;
|
||||
}
|
||||
start.push_back(static_cast<int>(atom.size()));
|
||||
}
|
||||
const size_t rows = start.size() - 1;
|
||||
std::vector<int> row_of(atom.size()), param(atom.size());
|
||||
for (size_t e = 0; e < atom.size(); e++)
|
||||
param[e] = param_of_[atom[e]];
|
||||
for (size_t r = 0; r < rows; r++)
|
||||
for (int e = start[r]; e < start[r + 1]; e++)
|
||||
row_of[e] = static_cast<int>(r);
|
||||
std::vector<int> entry_start(n_params_ + 1, 0);
|
||||
for (int k : param)
|
||||
if (k >= 0)
|
||||
entry_start[k + 1]++;
|
||||
for (size_t k = 0; k < n_params_; k++)
|
||||
entry_start[k + 1] += entry_start[k];
|
||||
std::vector<int> entry(entry_start.back());
|
||||
std::vector<int> fill(entry_start.begin(), entry_start.end() - 1);
|
||||
for (size_t e = 0; e < param.size(); e++)
|
||||
if (param[e] >= 0)
|
||||
entry[fill[param[e]]++] = static_cast<int>(e);
|
||||
Upload(set.records, set.plane_atoms, record_entry, start, param, row_of, entry_start, entry);
|
||||
indexed_ = true;
|
||||
}
|
||||
Fill(xyzb);
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
// The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per
|
||||
// parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in
|
||||
// float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of
|
||||
// H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with
|
||||
// plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise.
|
||||
|
||||
#include "RefinementNewtonGPU.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "../indexing/CUDAMemHelpers.h"
|
||||
#include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int THREADS = 256;
|
||||
using V3f = refine_math::V3<float>;
|
||||
|
||||
void cuda_err(cudaError_t val) {
|
||||
if (val != cudaSuccess)
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
|
||||
}
|
||||
|
||||
class DeviceGuard {
|
||||
public:
|
||||
explicit DeviceGuard(int device) {
|
||||
cuda_err(cudaGetDevice(&previous_));
|
||||
cuda_err(cudaSetDevice(device));
|
||||
}
|
||||
~DeviceGuard() { cudaSetDevice(previous_); }
|
||||
DeviceGuard(const DeviceGuard &) = delete;
|
||||
DeviceGuard &operator=(const DeviceGuard &) = delete;
|
||||
|
||||
private:
|
||||
int previous_ = 0;
|
||||
};
|
||||
|
||||
int GreatestStreamPriority() {
|
||||
int least = 0, greatest = 0;
|
||||
cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest));
|
||||
return greatest;
|
||||
}
|
||||
|
||||
int GridSize(int n) {
|
||||
return (n + THREADS - 1) / THREADS;
|
||||
}
|
||||
|
||||
__device__ V3f xyz(const float4 *p, int i) {
|
||||
return {p[i].x, p[i].y, p[i].z};
|
||||
}
|
||||
|
||||
__device__ float Get(const float4 &j, int c) {
|
||||
return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w;
|
||||
}
|
||||
|
||||
// sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math).
|
||||
// The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2
|
||||
__device__ void AddProduct(float2 &sum, float a, float2 b) {
|
||||
const float p = a * b.x;
|
||||
const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly
|
||||
const float s = sum.x + p;
|
||||
const float v = s - sum.x;
|
||||
const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly
|
||||
const float lo = s_err + sum.y + p_err + a * b.y;
|
||||
const float hi = s + lo;
|
||||
sum = make_float2(hi, lo - (hi - s));
|
||||
}
|
||||
|
||||
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
|
||||
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
|
||||
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
|
||||
const float4 *xyzb, float4 *j) {
|
||||
const int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= n)
|
||||
return;
|
||||
const RestraintRecord &r = records[i];
|
||||
float4 *out = j + record_entry[i];
|
||||
V3f g[4] = {};
|
||||
int na = 0;
|
||||
float e = 0;
|
||||
switch (static_cast<RestraintKind>(r.kind)) {
|
||||
case RestraintKind::Bond:
|
||||
e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g);
|
||||
na = 2;
|
||||
break;
|
||||
case RestraintKind::Angle:
|
||||
e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal,
|
||||
r.sigma, g);
|
||||
na = 3;
|
||||
break;
|
||||
case RestraintKind::Torsion:
|
||||
e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
|
||||
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g);
|
||||
na = 4;
|
||||
break;
|
||||
case RestraintKind::Chiral:
|
||||
e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
|
||||
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g);
|
||||
na = 4;
|
||||
break;
|
||||
case RestraintKind::NcsDistance:
|
||||
e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
|
||||
xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g);
|
||||
na = 4;
|
||||
break;
|
||||
case RestraintKind::Repulsion: {
|
||||
const V3f q = xyz(xyzb, r.atom[1]);
|
||||
const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0],
|
||||
r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1],
|
||||
r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]};
|
||||
e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g);
|
||||
const V3f gi = g[1];
|
||||
g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z,
|
||||
r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z,
|
||||
r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z};
|
||||
na = 2;
|
||||
break;
|
||||
}
|
||||
case RestraintKind::Plane: {
|
||||
V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS];
|
||||
const int m = r.atom[1];
|
||||
for (int k = 0; k < m; k++)
|
||||
px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]);
|
||||
refine_math::PlaneTerm(px, m, r.sigma, pg);
|
||||
V3f nv{0, 0, 0};
|
||||
for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n)
|
||||
if (refine_math::Norm(pg[k]) > refine_math::Norm(nv))
|
||||
nv = pg[k];
|
||||
const float norm = refine_math::Norm(nv);
|
||||
const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f;
|
||||
for (int a = 0; a < m; a++)
|
||||
for (int k = 0; k < m; k++) {
|
||||
const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m);
|
||||
out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f);
|
||||
}
|
||||
return;
|
||||
}
|
||||
case RestraintKind::BSimilarity: {
|
||||
float gi = 0, gj = 0;
|
||||
e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj);
|
||||
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
|
||||
out[0] = make_float4(0, 0, 0, f * gi);
|
||||
out[1] = make_float4(0, 0, 0, f * gj);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
|
||||
for (int k = 0; k < na; k++)
|
||||
out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f);
|
||||
}
|
||||
|
||||
// t_r = sum over the row's entries of J_e . v_k
|
||||
__global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v,
|
||||
float2 *t, int rows) {
|
||||
const int r = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (r >= rows)
|
||||
return;
|
||||
float2 dot = make_float2(0, 0);
|
||||
for (int e = start[r]; e < start[r + 1]; e++)
|
||||
if (const int k = param[e]; k >= 0)
|
||||
for (int c = 0; c < per; c++)
|
||||
AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]);
|
||||
t[r] = dot;
|
||||
}
|
||||
|
||||
// hv_k = sum over the parameter atom's entries of t_row J_e
|
||||
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
|
||||
int c0, const float2 *t, float2 *hv, int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
for (int c = 0; c < per; c++) {
|
||||
float2 sum = make_float2(0, 0);
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const int e = entry[q];
|
||||
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
|
||||
}
|
||||
hv[per * k + c] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d,
|
||||
int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
for (int c = 0; c < per; c++) {
|
||||
float sum = 0;
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const float x = Get(j[entry[q]], c0 + c);
|
||||
sum += x * x;
|
||||
}
|
||||
d[per * k + c] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void BlocksKernel(const int *entry_start, const int *entry, const float4 *j, float *b, int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
float s[6] = {0, 0, 0, 0, 0, 0};
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const float4 x = j[entry[q]];
|
||||
s[0] += x.x * x.x;
|
||||
s[1] += x.y * x.y;
|
||||
s[2] += x.z * x.z;
|
||||
s[3] += x.x * x.y;
|
||||
s[4] += x.x * x.z;
|
||||
s[5] += x.y * x.z;
|
||||
}
|
||||
for (int c = 0; c < 6; c++)
|
||||
b[6 * k + c] = s[c];
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
|
||||
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
|
||||
cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream));
|
||||
return d;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct RestraintNormalMatrixGPU::Device {
|
||||
int device;
|
||||
CudaStream stream;
|
||||
int records = 0, rows = 0;
|
||||
CudaDevicePtr<RestraintRecord> record;
|
||||
CudaDevicePtr<int> plane_atoms, record_entry, start, param, row_of, entry_start, entry;
|
||||
CudaDevicePtr<float4> xyzb, j;
|
||||
CudaDevicePtr<float2> t, v, hv;
|
||||
CudaDevicePtr<float> diagonal;
|
||||
PinnedVector<float2> host; // v and hv as float pairs
|
||||
PinnedVector<float4> host_xyzb;
|
||||
};
|
||||
|
||||
RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector<int> ¶m_atoms, size_t n_atoms,
|
||||
bool xyz)
|
||||
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) {
|
||||
for (size_t k = 0; k < param_atoms.size(); k++)
|
||||
param_of_[param_atoms[k]] = static_cast<int>(k);
|
||||
DeviceGuard guard(device);
|
||||
// At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the
|
||||
// X-ray evaluations of the other trials on the card.
|
||||
dev_ = std::unique_ptr<Device>(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())});
|
||||
const size_t n = std::max<size_t>(per_ * n_params_, 1);
|
||||
dev_->xyzb = CudaDevicePtr<float4>(std::max<size_t>(n_atoms, 1));
|
||||
dev_->host_xyzb.resize(n_atoms);
|
||||
dev_->v = CudaDevicePtr<float2>(n);
|
||||
dev_->hv = CudaDevicePtr<float2>(n);
|
||||
dev_->diagonal = CudaDevicePtr<float>(n);
|
||||
dev_->host.resize(n);
|
||||
}
|
||||
|
||||
RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() {
|
||||
DeviceGuard guard(dev_->device);
|
||||
dev_.reset();
|
||||
}
|
||||
|
||||
void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &records, const std::vector<int> &plane_atoms,
|
||||
const std::vector<int> &record_entry, const std::vector<int> &start,
|
||||
const std::vector<int> ¶m, const std::vector<int> &row_of,
|
||||
const std::vector<int> &entry_start, const std::vector<int> &entry) {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
d.records = static_cast<int>(records.size());
|
||||
d.rows = static_cast<int>(start.size()) - 1;
|
||||
d.record = Up(records, d.stream);
|
||||
d.plane_atoms = Up(plane_atoms, d.stream);
|
||||
d.record_entry = Up(record_entry, d.stream);
|
||||
d.start = Up(start, d.stream);
|
||||
d.param = Up(param, d.stream);
|
||||
d.row_of = Up(row_of, d.stream);
|
||||
d.entry_start = Up(entry_start, d.stream);
|
||||
d.entry = Up(entry, d.stream);
|
||||
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
|
||||
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
}
|
||||
|
||||
void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xyzb) {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
for (size_t a = 0; a < xyzb.size(); a++)
|
||||
d.host_xyzb[a] = make_float4(static_cast<float>(xyzb[a][0]), static_cast<float>(xyzb[a][1]),
|
||||
static_cast<float>(xyzb[a][2]), static_cast<float>(xyzb[a][3]));
|
||||
cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice,
|
||||
d.stream));
|
||||
if (d.records > 0)
|
||||
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
|
||||
d.record_entry, d.xyzb, d.j);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
}
|
||||
|
||||
void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vector<double> &hv) {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
const int n = static_cast<int>(n_params_);
|
||||
hv.resize(per_ * n_params_);
|
||||
if (n == 0)
|
||||
return;
|
||||
for (size_t i = 0; i < hv.size(); i++) {
|
||||
const float hi = static_cast<float>(v[i]);
|
||||
d.host[i] = make_float2(hi, static_cast<float>(v[i] - hi));
|
||||
}
|
||||
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
|
||||
if (d.rows > 0)
|
||||
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
|
||||
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
|
||||
d.hv, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
for (size_t i = 0; i < hv.size(); i++)
|
||||
hv[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
|
||||
}
|
||||
|
||||
std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
const int n = static_cast<int>(n_params_);
|
||||
std::vector<double> out(per_ * n_params_);
|
||||
if (n == 0)
|
||||
return out;
|
||||
std::vector<float> host(out.size());
|
||||
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
std::copy(host.begin(), host.end(), out.begin());
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::array<double, 6>> RestraintNormalMatrixGPU::Blocks() const {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
const int n = static_cast<int>(n_params_);
|
||||
std::vector<std::array<double, 6>> out(n_params_);
|
||||
if (n == 0)
|
||||
return out;
|
||||
CudaDevicePtr<float> b(6 * n_params_);
|
||||
std::vector<float> host(6 * n_params_);
|
||||
BlocksKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, b, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(host.data(), b.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
for (size_t k = 0; k < n_params_; k++)
|
||||
for (int c = 0; c < 6; c++)
|
||||
out[k][c] = host[6 * k + c];
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) of the Newton minimisation on a GPU (CUDA
|
||||
// builds only): RestraintNormalMatrix of Refinement.cpp on the card, in single precision. J is worked out on the device,
|
||||
// one thread per restraint, with the terms the engine evaluates the restraints with (RefinementMath.h); a product is two
|
||||
// gathers, J v per row and J^T t per parameter atom over its entries, each summed in a fixed order without atomics, in
|
||||
// double-float pairs (RefinementNewtonGPU.cu).
|
||||
//
|
||||
// Every restraint keeps its rows whatever its E: where RestraintGaussNewton drops a row (E = 0, a plane without a
|
||||
// normal) its entries here are zero, so the index of J is worked out once, at the first At(), and the set must stay
|
||||
// the same for the matrix's life.
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "RefinementMath.h" // RestraintRecord
|
||||
|
||||
struct RestraintSet;
|
||||
|
||||
class RestraintNormalMatrixGPU {
|
||||
public:
|
||||
// On the x y z (`xyz`) or the B of `param_atoms` of the model's `n_atoms`, on GPU `device`.
|
||||
RestraintNormalMatrixGPU(int device, const std::vector<int> ¶m_atoms, size_t n_atoms, bool xyz);
|
||||
~RestraintNormalMatrixGPU();
|
||||
|
||||
void At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb);
|
||||
// hv = J^T J v
|
||||
void Multiply(const std::vector<double> &v, std::vector<double> &hv);
|
||||
// Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only.
|
||||
std::vector<std::array<double, 6>> Blocks() const;
|
||||
std::vector<double> Diagonal() const;
|
||||
|
||||
private:
|
||||
// The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries
|
||||
// start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries.
|
||||
void Upload(const std::vector<RestraintRecord> &records, const std::vector<int> &plane_atoms,
|
||||
const std::vector<int> &record_entry, const std::vector<int> &start, const std::vector<int> ¶m,
|
||||
const std::vector<int> &row_of, const std::vector<int> &entry_start, const std::vector<int> &entry);
|
||||
// J at `xyzb` (x, y, z, B per atom), in float
|
||||
void Fill(const std::vector<std::array<double, 4>> &xyzb);
|
||||
|
||||
int per_, c0_;
|
||||
size_t n_params_;
|
||||
std::vector<int> param_of_;
|
||||
bool indexed_ = false;
|
||||
struct Device;
|
||||
std::unique_ptr<Device> dev_;
|
||||
};
|
||||
@@ -37,6 +37,7 @@
|
||||
#include "../image_analysis/structure_refinement/RefinementXray.h"
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
#include "../image_analysis/structure_refinement/RefinementGPU.h"
|
||||
#include "../image_analysis/structure_refinement/RefinementNewtonGPU.h"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -986,6 +987,104 @@ TEST_CASE("RefinementXray_gradient_CPU", "[Refinement]") {
|
||||
}
|
||||
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") {
|
||||
if (get_gpu_count() == 0)
|
||||
return;
|
||||
const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90);
|
||||
gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5);
|
||||
st.models[0] = shaken(st.models[0], 0.15, 3.0, 2);
|
||||
const RefinementTopology topo = BuildRefinementTopology(st);
|
||||
RestraintSet set = topo.geometry;
|
||||
set.Append(topo.adp);
|
||||
set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0));
|
||||
const std::vector<gemmi::Atom *> atoms = atoms_of(st.models[0]);
|
||||
set.Finalize(atoms.size());
|
||||
std::vector<std::array<double, 4>> xyzb;
|
||||
for (gemmi::Atom *a : atoms)
|
||||
xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso});
|
||||
// every atom but each fifth a parameter: the rest drop out
|
||||
std::vector<int> params, param_of(atoms.size(), -1);
|
||||
for (int a = 0; a < static_cast<int>(atoms.size()); a++)
|
||||
if (a % 5 != 4) {
|
||||
param_of[a] = static_cast<int>(params.size());
|
||||
params.push_back(a);
|
||||
}
|
||||
const RestraintJacobian jac = RestraintGaussNewton(set, xyzb);
|
||||
REQUIRE(jac.start.size() > 100);
|
||||
std::mt19937 gen(3);
|
||||
std::uniform_real_distribution<double> uniform(-1, 1);
|
||||
// a random v, and a smooth one - the same shift of every atom and a thousandth of it at random, J v then the small
|
||||
// difference of large terms
|
||||
for (const auto [xyz, smooth] : {std::pair{true, false}, {true, true}, {false, false}, {false, true}}) {
|
||||
CAPTURE(xyz, smooth);
|
||||
const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3;
|
||||
const size_t n = per * params.size();
|
||||
std::vector<double> v(n);
|
||||
for (size_t i = 0; i < n; i++)
|
||||
v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen);
|
||||
// J^T J v, the diagonal and the 3 x 3 blocks on the CPU, each entry in turn
|
||||
std::vector<double> hv(n, 0.0), diagonal(n, 0.0);
|
||||
std::vector<std::array<double, 6>> blocks(params.size(), {0, 0, 0, 0, 0, 0});
|
||||
for (size_t r = 0; r + 1 < jac.start.size(); r++) {
|
||||
double t = 0;
|
||||
for (int e = jac.start[r]; e < jac.start[r + 1]; e++)
|
||||
if (const int k = param_of[jac.atom[e]]; k >= 0)
|
||||
for (int c = 0; c < per; c++)
|
||||
t += jac.j[e][c0 + c] * v[per * k + c];
|
||||
for (int e = jac.start[r]; e < jac.start[r + 1]; e++)
|
||||
if (const int k = param_of[jac.atom[e]]; k >= 0)
|
||||
for (int c = 0; c < per; c++)
|
||||
hv[per * k + c] += t * jac.j[e][c0 + c];
|
||||
}
|
||||
for (size_t e = 0; e < jac.atom.size(); e++)
|
||||
if (const int k = param_of[jac.atom[e]]; k >= 0) {
|
||||
const std::array<double, 4> &j = jac.j[e];
|
||||
for (int c = 0; c < per; c++)
|
||||
diagonal[per * k + c] += j[c0 + c] * j[c0 + c];
|
||||
blocks[k][0] += j[0] * j[0];
|
||||
blocks[k][1] += j[1] * j[1];
|
||||
blocks[k][2] += j[2] * j[2];
|
||||
blocks[k][3] += j[0] * j[1];
|
||||
blocks[k][4] += j[0] * j[2];
|
||||
blocks[k][5] += j[1] * j[2];
|
||||
}
|
||||
RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz);
|
||||
gpu.At(set, xyzb);
|
||||
std::vector<double> hv_gpu;
|
||||
gpu.Multiply(v, hv_gpu);
|
||||
const std::vector<double> diagonal_gpu = gpu.Diagonal();
|
||||
REQUIRE(hv_gpu.size() == n);
|
||||
REQUIRE(diagonal_gpu.size() == n);
|
||||
// In float, from float positions: within a few float epsilons of the double product in norm (in refinement
|
||||
// its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself, each block
|
||||
// relative to its own size.
|
||||
double num = 0, den = 0, worst_diagonal = 0, worst_block = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]);
|
||||
den += hv[i] * hv[i];
|
||||
if (diagonal[i] > 0)
|
||||
worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]);
|
||||
}
|
||||
if (xyz) {
|
||||
const std::vector<std::array<double, 6>> blocks_gpu = gpu.Blocks();
|
||||
REQUIRE(blocks_gpu.size() == params.size());
|
||||
for (size_t k = 0; k < params.size(); k++) {
|
||||
double d2 = 0, b2 = 0;
|
||||
for (int c = 0; c < 6; c++) {
|
||||
d2 += (blocks_gpu[k][c] - blocks[k][c]) * (blocks_gpu[k][c] - blocks[k][c]);
|
||||
b2 += blocks[k][c] * blocks[k][c];
|
||||
}
|
||||
if (b2 > 0)
|
||||
worst_block = std::max(worst_block, std::sqrt(d2 / b2));
|
||||
}
|
||||
}
|
||||
CAPTURE(std::sqrt(num / den), worst_diagonal, worst_block);
|
||||
CHECK(std::sqrt(num / den) < 1e-5);
|
||||
CHECK(worst_diagonal < 1e-5);
|
||||
CHECK(worst_block < 1e-5);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") {
|
||||
if (get_gpu_count() == 0)
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user