Refinement Newton: the restraints' Gauss-Newton product on the GPU

With the engine on a GPU, RestraintNormalMatrix hands At/Multiply/Diagonal/Blocks to
RestraintNormalMatrixGPU (RefinementNewtonGPU.{h,cpp,cu}), on the engine's card on a stream of
its own at the highest priority. J is built on the device, one thread per restraint from the
REFINE_HD terms in float, into a fixed layout worked out once from the set (rows the CPU drops,
E = 0, are kept as zeros). A product is two gathers, J v per row and J^T t per parameter atom,
without atomics, in a fixed order.

Precision: J in float, v/t/hv as double-float pairs (error-free product and 2Sum per term).
Plain float sums are not enough: J v is a small difference of large terms in the soft directions
the X-ray term decides, and with plain float the outcomes moved beyond the CPU's own noise on a
model with stiff restraints (weight choice 2/10 vs 8/10 at the CPU's value over jittered runs).
Float J with exact accumulation matched the CPU's spread, as do the double-float pairs.

On a large rotation set (GPU, Newton joint, relative stop 0.05): product thread-time 1008 s ->
38 s, jacobian 114 s -> 8 s, refine wall 191 s -> 32 s under load; evaluations and R-free within
the spread a 1e-12 relative jitter of the CPU product gives.

Interface: RestraintNormalMatrix now takes the Context (for the device) instead of n_atoms and
nthreads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-16 10:38:47 +01:00
1 parent 4dc4a47d2f
commit 5360629f4e
7 files changed
+602 -8

No files matched your search

+2 -1
View File
@@ -842,7 +842,8 @@ pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472
[doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195).
**Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an
Apple GPU (`BraggExp.h`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the
Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU
(`RefinementNewtonGPU.cu`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the
Elementary Functions" (1980), Prentice-Hall; the exact product (with Veltkamp's split) and Fast2Sum
of T. J. Dekker, "A floating-point technique for extending the available precision" (1971), Numer.
Math. 18, 224-242; and the 2Sum of D. E. Knuth, "The Art of Computer Programming, Vol. 2:
@@ -37,6 +37,9 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementGPU.cpp>
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementGPU.cu>
RefinementGPU.h RefinementGPUEngine.h
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementNewtonGPU.cpp>
$<$<BOOL:${JFJOCH_CUDA_AVAILABLE}>:RefinementNewtonGPU.cu>
RefinementNewtonGPU.h
${CMAKE_CURRENT_BINARY_DIR}/MonomerLibraryData.cpp)
# The subset of the CCP4 Monomer Library that refinement restrains with (monomers/, chosen by
@@ -49,6 +49,7 @@
#ifdef JFJOCH_USE_CUDA
#include "../../common/CUDAWrapper.h"
#include "RefinementGPU.h"
#include "RefinementNewtonGPU.h"
#endif
namespace {
@@ -1160,18 +1161,27 @@ std::vector<double> FlooredDiagonal(std::vector<double> d) {
}
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it.
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
// a GPU, on its card (RestraintNormalMatrixGPU).
class RestraintNormalMatrix {
public:
RestraintNormalMatrix(const std::vector<int> &param_atoms, size_t n_atoms, bool xyz, size_t nthreads)
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(nthreads),
param_of_(n_atoms, -1) {
RestraintNormalMatrix(const Context &ctx, const std::vector<int> &param_atoms, bool xyz)
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(ctx.nthreads),
param_of_(ctx.atoms.size(), -1) {
for (size_t k = 0; k < param_atoms.size(); k++)
param_of_[param_atoms[k]] = static_cast<int>(k);
#ifdef JFJOCH_USE_CUDA
if (ctx.engine->OnGPU())
gpu_ = std::make_unique<RestraintNormalMatrixGPU>(ctx.device, param_atoms, ctx.atoms.size(), xyz);
#endif
}
void At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb) {
const RefinementStage stage("newton.jacobian");
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->At(set, xyzb);
#endif
jac_ = RestraintGaussNewton(set, xyzb);
const size_t rows = jac_.start.size() - 1;
row_of_.resize(jac_.atom.size());
@@ -1199,6 +1209,10 @@ public:
// hv = J^T J v: J v per row, then per parameter atom the sum over its entries
void Multiply(const std::vector<double> &v, std::vector<double> &hv) {
const RefinementStage stage("newton.product");
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->Multiply(v, hv);
#endif
const int rows = static_cast<int>(t_.size());
ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) {
for (int r = lo; r < hi; r++) {
@@ -1224,6 +1238,10 @@ public:
// Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only.
std::vector<std::array<double, 6>> Blocks() const {
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->Blocks();
#endif
std::vector<std::array<double, 6>> b(n_params_, {0, 0, 0, 0, 0, 0});
for (size_t k = 0; k < n_params_; k++)
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) {
@@ -1239,6 +1257,10 @@ public:
}
std::vector<double> Diagonal() const {
#ifdef JFJOCH_USE_CUDA
if (gpu_)
return gpu_->Diagonal();
#endif
std::vector<double> d(per_ * n_params_, 0.0);
for (size_t k = 0; k < n_params_; k++)
for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++)
@@ -1254,6 +1276,9 @@ private:
RestraintJacobian jac_;
std::vector<int> row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none)
std::vector<double> t_;
#ifdef JFJOCH_USE_CUDA
std::unique_ptr<RestraintNormalMatrixGPU> gpu_;
#endif
};
// z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate.
@@ -1295,7 +1320,7 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto
std::vector<double> h(n);
for (size_t k = 0; k < n; k++)
h[k] = xray[target.Atoms()[k / per]];
RestraintNormalMatrix normal(target.Atoms(), ctx.atoms.size(), xyz, ctx.nthreads);
RestraintNormalMatrix normal(ctx, target.Atoms(), xyz);
std::vector<double> x_at = p;
target.Set(p.data());
normal.At(set, ctx.XyzB());
@@ -1332,8 +1357,7 @@ void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vect
const RestraintSet &set_b, double w, double adp_w, int iterations) {
const size_t n = atoms.size();
const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT;
RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads),
normal_b(atoms, ctx.atoms.size(), false, ctx.nthreads);
RestraintNormalMatrix normal_xyz(ctx, atoms, true), normal_b(ctx, atoms, false);
std::vector<double> x_at = target.Get();
auto at = [&](const std::vector<double> &x) {
target.Set(x.data());
@@ -0,0 +1,59 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "RefinementNewtonGPU.h"
#include "RefinementRestraints.h"
// The host half: the index of J, as RestraintGaussNewton lays out its rows, with every row kept.
void RestraintNormalMatrixGPU::At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb) {
if (!indexed_) {
std::vector<int> record_entry, start{0}, atom;
for (const RestraintRecord &r : set.records) {
record_entry.push_back(static_cast<int>(atom.size()));
switch (static_cast<RestraintKind>(r.kind)) {
case RestraintKind::Plane:
// one row per atom of the plane, each over all of them
for (int i = 0; i < r.atom[1]; i++) {
for (int k = 0; k < r.atom[1]; k++)
atom.push_back(set.plane_atoms[r.atom[0] + k]);
start.push_back(static_cast<int>(atom.size()));
}
continue;
case RestraintKind::Angle:
atom.insert(atom.end(), r.atom, r.atom + 3);
break;
case RestraintKind::Torsion:
case RestraintKind::Chiral:
case RestraintKind::NcsDistance:
atom.insert(atom.end(), r.atom, r.atom + 4);
break;
default: // Bond, Repulsion, BSimilarity
atom.insert(atom.end(), r.atom, r.atom + 2);
break;
}
start.push_back(static_cast<int>(atom.size()));
}
const size_t rows = start.size() - 1;
std::vector<int> row_of(atom.size()), param(atom.size());
for (size_t e = 0; e < atom.size(); e++)
param[e] = param_of_[atom[e]];
for (size_t r = 0; r < rows; r++)
for (int e = start[r]; e < start[r + 1]; e++)
row_of[e] = static_cast<int>(r);
std::vector<int> entry_start(n_params_ + 1, 0);
for (int k : param)
if (k >= 0)
entry_start[k + 1]++;
for (size_t k = 0; k < n_params_; k++)
entry_start[k + 1] += entry_start[k];
std::vector<int> entry(entry_start.back());
std::vector<int> fill(entry_start.begin(), entry_start.end() - 1);
for (size_t e = 0; e < param.size(); e++)
if (param[e] >= 0)
entry[fill[param[e]]++] = static_cast<int>(e);
Upload(set.records, set.plane_atoms, record_entry, start, param, row_of, entry_start, entry);
indexed_ = true;
}
Fill(xyzb);
}
@@ -0,0 +1,355 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per
// parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in
// float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of
// H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with
// plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise.
#include "RefinementNewtonGPU.h"
#include <algorithm>
#include "../indexing/CUDAMemHelpers.h"
#include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS
namespace {
constexpr int THREADS = 256;
using V3f = refine_math::V3<float>;
void cuda_err(cudaError_t val) {
if (val != cudaSuccess)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
}
class DeviceGuard {
public:
explicit DeviceGuard(int device) {
cuda_err(cudaGetDevice(&previous_));
cuda_err(cudaSetDevice(device));
}
~DeviceGuard() { cudaSetDevice(previous_); }
DeviceGuard(const DeviceGuard &) = delete;
DeviceGuard &operator=(const DeviceGuard &) = delete;
private:
int previous_ = 0;
};
int GreatestStreamPriority() {
int least = 0, greatest = 0;
cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest));
return greatest;
}
int GridSize(int n) {
return (n + THREADS - 1) / THREADS;
}
__device__ V3f xyz(const float4 *p, int i) {
return {p[i].x, p[i].y, p[i].z};
}
__device__ float Get(const float4 &j, int c) {
return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w;
}
// sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math).
// The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2
__device__ void AddProduct(float2 &sum, float a, float2 b) {
const float p = a * b.x;
const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly
const float s = sum.x + p;
const float v = s - sum.x;
const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly
const float lo = s_err + sum.y + p_err + a * b.y;
const float hi = s + lo;
sum = make_float2(hi, lo - (hi - s));
}
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
const float4 *xyzb, float4 *j) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
const RestraintRecord &r = records[i];
float4 *out = j + record_entry[i];
V3f g[4] = {};
int na = 0;
float e = 0;
switch (static_cast<RestraintKind>(r.kind)) {
case RestraintKind::Bond:
e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g);
na = 2;
break;
case RestraintKind::Angle:
e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal,
r.sigma, g);
na = 3;
break;
case RestraintKind::Torsion:
e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g);
na = 4;
break;
case RestraintKind::Chiral:
e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g);
na = 4;
break;
case RestraintKind::NcsDistance:
e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g);
na = 4;
break;
case RestraintKind::Repulsion: {
const V3f q = xyz(xyzb, r.atom[1]);
const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0],
r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1],
r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]};
e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g);
const V3f gi = g[1];
g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z,
r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z,
r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z};
na = 2;
break;
}
case RestraintKind::Plane: {
V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS];
const int m = r.atom[1];
for (int k = 0; k < m; k++)
px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]);
refine_math::PlaneTerm(px, m, r.sigma, pg);
V3f nv{0, 0, 0};
for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n)
if (refine_math::Norm(pg[k]) > refine_math::Norm(nv))
nv = pg[k];
const float norm = refine_math::Norm(nv);
const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f;
for (int a = 0; a < m; a++)
for (int k = 0; k < m; k++) {
const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m);
out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f);
}
return;
}
case RestraintKind::BSimilarity: {
float gi = 0, gj = 0;
e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj);
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
out[0] = make_float4(0, 0, 0, f * gi);
out[1] = make_float4(0, 0, 0, f * gj);
return;
}
}
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
for (int k = 0; k < na; k++)
out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f);
}
// t_r = sum over the row's entries of J_e . v_k
__global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v,
float2 *t, int rows) {
const int r = blockIdx.x * blockDim.x + threadIdx.x;
if (r >= rows)
return;
float2 dot = make_float2(0, 0);
for (int e = start[r]; e < start[r + 1]; e++)
if (const int k = param[e]; k >= 0)
for (int c = 0; c < per; c++)
AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]);
t[r] = dot;
}
// hv_k = sum over the parameter atom's entries of t_row J_e
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
int c0, const float2 *t, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float2 sum = make_float2(0, 0);
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const int e = entry[q];
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
}
hv[per * k + c] = sum;
}
}
__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d,
int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float sum = 0;
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float x = Get(j[entry[q]], c0 + c);
sum += x * x;
}
d[per * k + c] = sum;
}
}
__global__ void BlocksKernel(const int *entry_start, const int *entry, const float4 *j, float *b, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
float s[6] = {0, 0, 0, 0, 0, 0};
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float4 x = j[entry[q]];
s[0] += x.x * x.x;
s[1] += x.y * x.y;
s[2] += x.z * x.z;
s[3] += x.x * x.y;
s[4] += x.x * x.z;
s[5] += x.y * x.z;
}
for (int c = 0; c < 6; c++)
b[6 * k + c] = s[c];
}
template <typename T>
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream));
return d;
}
} // namespace
struct RestraintNormalMatrixGPU::Device {
int device;
CudaStream stream;
int records = 0, rows = 0;
CudaDevicePtr<RestraintRecord> record;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, row_of, entry_start, entry;
CudaDevicePtr<float4> xyzb, j;
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
PinnedVector<float2> host; // v and hv as float pairs
PinnedVector<float4> host_xyzb;
};
RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector<int> &param_atoms, size_t n_atoms,
bool xyz)
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) {
for (size_t k = 0; k < param_atoms.size(); k++)
param_of_[param_atoms[k]] = static_cast<int>(k);
DeviceGuard guard(device);
// At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the
// X-ray evaluations of the other trials on the card.
dev_ = std::unique_ptr<Device>(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())});
const size_t n = std::max<size_t>(per_ * n_params_, 1);
dev_->xyzb = CudaDevicePtr<float4>(std::max<size_t>(n_atoms, 1));
dev_->host_xyzb.resize(n_atoms);
dev_->v = CudaDevicePtr<float2>(n);
dev_->hv = CudaDevicePtr<float2>(n);
dev_->diagonal = CudaDevicePtr<float>(n);
dev_->host.resize(n);
}
RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() {
DeviceGuard guard(dev_->device);
dev_.reset();
}
void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &records, const std::vector<int> &plane_atoms,
const std::vector<int> &record_entry, const std::vector<int> &start,
const std::vector<int> &param, const std::vector<int> &row_of,
const std::vector<int> &entry_start, const std::vector<int> &entry) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
d.records = static_cast<int>(records.size());
d.rows = static_cast<int>(start.size()) - 1;
d.record = Up(records, d.stream);
d.plane_atoms = Up(plane_atoms, d.stream);
d.record_entry = Up(record_entry, d.stream);
d.start = Up(start, d.stream);
d.param = Up(param, d.stream);
d.row_of = Up(row_of, d.stream);
d.entry_start = Up(entry_start, d.stream);
d.entry = Up(entry, d.stream);
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
cuda_err(cudaStreamSynchronize(d.stream));
}
void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xyzb) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
for (size_t a = 0; a < xyzb.size(); a++)
d.host_xyzb[a] = make_float4(static_cast<float>(xyzb[a][0]), static_cast<float>(xyzb[a][1]),
static_cast<float>(xyzb[a][2]), static_cast<float>(xyzb[a][3]));
cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice,
d.stream));
if (d.records > 0)
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
d.record_entry, d.xyzb, d.j);
cuda_err(cudaGetLastError());
cuda_err(cudaStreamSynchronize(d.stream));
}
void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vector<double> &hv) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(n_params_);
hv.resize(per_ * n_params_);
if (n == 0)
return;
for (size_t i = 0; i < hv.size(); i++) {
const float hi = static_cast<float>(v[i]);
d.host[i] = make_float2(hi, static_cast<float>(v[i] - hi));
}
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (size_t i = 0; i < hv.size(); i++)
hv[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
}
std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(n_params_);
std::vector<double> out(per_ * n_params_);
if (n == 0)
return out;
std::vector<float> host(out.size());
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
std::copy(host.begin(), host.end(), out.begin());
return out;
}
std::vector<std::array<double, 6>> RestraintNormalMatrixGPU::Blocks() const {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(n_params_);
std::vector<std::array<double, 6>> out(n_params_);
if (n == 0)
return out;
CudaDevicePtr<float> b(6 * n_params_);
std::vector<float> host(6 * n_params_);
BlocksKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, b, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(host.data(), b.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (size_t k = 0; k < n_params_; k++)
for (int c = 0; c < 6; c++)
out[k][c] = host[6 * k + c];
return out;
}
@@ -0,0 +1,53 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) of the Newton minimisation on a GPU (CUDA
// builds only): RestraintNormalMatrix of Refinement.cpp on the card, in single precision. J is worked out on the device,
// one thread per restraint, with the terms the engine evaluates the restraints with (RefinementMath.h); a product is two
// gathers, J v per row and J^T t per parameter atom over its entries, each summed in a fixed order without atomics, in
// double-float pairs (RefinementNewtonGPU.cu).
//
// Every restraint keeps its rows whatever its E: where RestraintGaussNewton drops a row (E = 0, a plane without a
// normal) its entries here are zero, so the index of J is worked out once, at the first At(), and the set must stay
// the same for the matrix's life.
#include <array>
#include <cstddef>
#include <memory>
#include <vector>
#include "RefinementMath.h" // RestraintRecord
struct RestraintSet;
class RestraintNormalMatrixGPU {
public:
// On the x y z (`xyz`) or the B of `param_atoms` of the model's `n_atoms`, on GPU `device`.
RestraintNormalMatrixGPU(int device, const std::vector<int> &param_atoms, size_t n_atoms, bool xyz);
~RestraintNormalMatrixGPU();
void At(const RestraintSet &set, const std::vector<std::array<double, 4>> &xyzb);
// hv = J^T J v
void Multiply(const std::vector<double> &v, std::vector<double> &hv);
// Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only.
std::vector<std::array<double, 6>> Blocks() const;
std::vector<double> Diagonal() const;
private:
// The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries
// start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries.
void Upload(const std::vector<RestraintRecord> &records, const std::vector<int> &plane_atoms,
const std::vector<int> &record_entry, const std::vector<int> &start, const std::vector<int> &param,
const std::vector<int> &row_of, const std::vector<int> &entry_start, const std::vector<int> &entry);
// J at `xyzb` (x, y, z, B per atom), in float
void Fill(const std::vector<std::array<double, 4>> &xyzb);
int per_, c0_;
size_t n_params_;
std::vector<int> param_of_;
bool indexed_ = false;
struct Device;
std::unique_ptr<Device> dev_;
};
+99
View File
@@ -37,6 +37,7 @@
#include "../image_analysis/structure_refinement/RefinementXray.h"
#ifdef JFJOCH_USE_CUDA
#include "../image_analysis/structure_refinement/RefinementGPU.h"
#include "../image_analysis/structure_refinement/RefinementNewtonGPU.h"
#endif
namespace {
@@ -986,6 +987,104 @@ TEST_CASE("RefinementXray_gradient_CPU", "[Refinement]") {
}
#ifdef JFJOCH_USE_CUDA
TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;
const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90);
gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5);
st.models[0] = shaken(st.models[0], 0.15, 3.0, 2);
const RefinementTopology topo = BuildRefinementTopology(st);
RestraintSet set = topo.geometry;
set.Append(topo.adp);
set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0));
const std::vector<gemmi::Atom *> atoms = atoms_of(st.models[0]);
set.Finalize(atoms.size());
std::vector<std::array<double, 4>> xyzb;
for (gemmi::Atom *a : atoms)
xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso});
// every atom but each fifth a parameter: the rest drop out
std::vector<int> params, param_of(atoms.size(), -1);
for (int a = 0; a < static_cast<int>(atoms.size()); a++)
if (a % 5 != 4) {
param_of[a] = static_cast<int>(params.size());
params.push_back(a);
}
const RestraintJacobian jac = RestraintGaussNewton(set, xyzb);
REQUIRE(jac.start.size() > 100);
std::mt19937 gen(3);
std::uniform_real_distribution<double> uniform(-1, 1);
// a random v, and a smooth one - the same shift of every atom and a thousandth of it at random, J v then the small
// difference of large terms
for (const auto [xyz, smooth] : {std::pair{true, false}, {true, true}, {false, false}, {false, true}}) {
CAPTURE(xyz, smooth);
const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3;
const size_t n = per * params.size();
std::vector<double> v(n);
for (size_t i = 0; i < n; i++)
v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen);
// J^T J v, the diagonal and the 3 x 3 blocks on the CPU, each entry in turn
std::vector<double> hv(n, 0.0), diagonal(n, 0.0);
std::vector<std::array<double, 6>> blocks(params.size(), {0, 0, 0, 0, 0, 0});
for (size_t r = 0; r + 1 < jac.start.size(); r++) {
double t = 0;
for (int e = jac.start[r]; e < jac.start[r + 1]; e++)
if (const int k = param_of[jac.atom[e]]; k >= 0)
for (int c = 0; c < per; c++)
t += jac.j[e][c0 + c] * v[per * k + c];
for (int e = jac.start[r]; e < jac.start[r + 1]; e++)
if (const int k = param_of[jac.atom[e]]; k >= 0)
for (int c = 0; c < per; c++)
hv[per * k + c] += t * jac.j[e][c0 + c];
}
for (size_t e = 0; e < jac.atom.size(); e++)
if (const int k = param_of[jac.atom[e]]; k >= 0) {
const std::array<double, 4> &j = jac.j[e];
for (int c = 0; c < per; c++)
diagonal[per * k + c] += j[c0 + c] * j[c0 + c];
blocks[k][0] += j[0] * j[0];
blocks[k][1] += j[1] * j[1];
blocks[k][2] += j[2] * j[2];
blocks[k][3] += j[0] * j[1];
blocks[k][4] += j[0] * j[2];
blocks[k][5] += j[1] * j[2];
}
RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz);
gpu.At(set, xyzb);
std::vector<double> hv_gpu;
gpu.Multiply(v, hv_gpu);
const std::vector<double> diagonal_gpu = gpu.Diagonal();
REQUIRE(hv_gpu.size() == n);
REQUIRE(diagonal_gpu.size() == n);
// In float, from float positions: within a few float epsilons of the double product in norm (in refinement
// its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself, each block
// relative to its own size.
double num = 0, den = 0, worst_diagonal = 0, worst_block = 0;
for (size_t i = 0; i < n; i++) {
num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]);
den += hv[i] * hv[i];
if (diagonal[i] > 0)
worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]);
}
if (xyz) {
const std::vector<std::array<double, 6>> blocks_gpu = gpu.Blocks();
REQUIRE(blocks_gpu.size() == params.size());
for (size_t k = 0; k < params.size(); k++) {
double d2 = 0, b2 = 0;
for (int c = 0; c < 6; c++) {
d2 += (blocks_gpu[k][c] - blocks[k][c]) * (blocks_gpu[k][c] - blocks[k][c]);
b2 += blocks[k][c] * blocks[k][c];
}
if (b2 > 0)
worst_block = std::max(worst_block, std::sqrt(d2 / b2));
}
}
CAPTURE(std::sqrt(num / den), worst_diagonal, worst_block);
CHECK(std::sqrt(num / den) < 1e-5);
CHECK(worst_diagonal < 1e-5);
CHECK(worst_block < 1e-5);
}
}
TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;