From 5360629f4e65e17ebc449db3e175c2316e0d56aa Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 10:38:47 +0100 Subject: [PATCH] Refinement Newton: the restraints' Gauss-Newton product on the GPU With the engine on a GPU, RestraintNormalMatrix hands At/Multiply/Diagonal/Blocks to RestraintNormalMatrixGPU (RefinementNewtonGPU.{h,cpp,cu}), on the engine's card on a stream of its own at the highest priority. J is built on the device, one thread per restraint from the REFINE_HD terms in float, into a fixed layout worked out once from the set (rows the CPU drops, E = 0, are kept as zeros). A product is two gathers, J v per row and J^T t per parameter atom, without atomics, in a fixed order. Precision: J in float, v/t/hv as double-float pairs (error-free product and 2Sum per term). Plain float sums are not enough: J v is a small difference of large terms in the soft directions the X-ray term decides, and with plain float the outcomes moved beyond the CPU's own noise on a model with stiff restraints (weight choice 2/10 vs 8/10 at the CPU's value over jittered runs). Float J with exact accumulation matched the CPU's spread, as do the double-float pairs. On a large rotation set (GPU, Newton joint, relative stop 0.05): product thread-time 1008 s -> 38 s, jacobian 114 s -> 8 s, refine wall 191 s -> 32 s under load; evaluations and R-free within the spread a 1e-12 relative jitter of the CPU product gives. Interface: RestraintNormalMatrix now takes the Context (for the device) instead of n_atoms and nthreads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 3 +- .../structure_refinement/CMakeLists.txt | 3 + .../structure_refinement/Refinement.cpp | 38 +- .../RefinementNewtonGPU.cpp | 59 +++ .../RefinementNewtonGPU.cu | 355 ++++++++++++++++++ .../RefinementNewtonGPU.h | 53 +++ tests/RefinementTest.cpp | 99 +++++ 7 files changed, 602 insertions(+), 8 deletions(-) create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.cpp create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.cu create mode 100644 image_analysis/structure_refinement/RefinementNewtonGPU.h diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index dd37ce457..83de168ef 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -842,7 +842,8 @@ pseudorandom number generators" (2014), Proc. OOPSLA 2014, 453-472 [doi:10.1145/2660193.2660195](https://doi.org/10.1145/2660193.2660195). **Error-free floating-point arithmetic** — the single-precision exp shared between the CPU and an -Apple GPU (`BraggExp.h`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the +Apple GPU (`BraggExp.h`) and the double-float sums of the refinement's Gauss-Newton products on the GPU +(`RefinementNewtonGPU.cu`): the argument reduction of W. J. Cody and W. Waite, "Software Manual for the Elementary Functions" (1980), Prentice-Hall; the exact product (with Veltkamp's split) and Fast2Sum of T. J. Dekker, "A floating-point technique for extending the available precision" (1971), Numer. Math. 18, 224-242; and the 2Sum of D. E. Knuth, "The Art of Computer Programming, Vol. 2: diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index 54e66051f..07cbcd4c6 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -37,6 +37,9 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC $<$:RefinementGPU.cpp> $<$:RefinementGPU.cu> RefinementGPU.h RefinementGPUEngine.h + $<$:RefinementNewtonGPU.cpp> + $<$:RefinementNewtonGPU.cu> + RefinementNewtonGPU.h ${CMAKE_CURRENT_BINARY_DIR}/MonomerLibraryData.cpp) # The subset of the CCP4 Monomer Library that refinement restrains with (monomers/, chosen by diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index bf14d0b03..b4a2afb08 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -49,6 +49,7 @@ #ifdef JFJOCH_USE_CUDA #include "../../common/CUDAWrapper.h" #include "RefinementGPU.h" +#include "RefinementNewtonGPU.h" #endif namespace { @@ -1160,18 +1161,27 @@ std::vector FlooredDiagonal(std::vector d) { } // The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the -// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. +// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on +// a GPU, on its card (RestraintNormalMatrixGPU). class RestraintNormalMatrix { public: - RestraintNormalMatrix(const std::vector ¶m_atoms, size_t n_atoms, bool xyz, size_t nthreads) - : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(nthreads), - param_of_(n_atoms, -1) { + RestraintNormalMatrix(const Context &ctx, const std::vector ¶m_atoms, bool xyz) + : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), nthreads_(ctx.nthreads), + param_of_(ctx.atoms.size(), -1) { for (size_t k = 0; k < param_atoms.size(); k++) param_of_[param_atoms[k]] = static_cast(k); +#ifdef JFJOCH_USE_CUDA + if (ctx.engine->OnGPU()) + gpu_ = std::make_unique(ctx.device, param_atoms, ctx.atoms.size(), xyz); +#endif } void At(const RestraintSet &set, const std::vector> &xyzb) { const RefinementStage stage("newton.jacobian"); +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->At(set, xyzb); +#endif jac_ = RestraintGaussNewton(set, xyzb); const size_t rows = jac_.start.size() - 1; row_of_.resize(jac_.atom.size()); @@ -1199,6 +1209,10 @@ public: // hv = J^T J v: J v per row, then per parameter atom the sum over its entries void Multiply(const std::vector &v, std::vector &hv) { const RefinementStage stage("newton.product"); +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Multiply(v, hv); +#endif const int rows = static_cast(t_.size()); ParallelChunks(rows, ThreadsForWork(rows, nthreads_, 16384), [&](int lo, int hi) { for (int r = lo; r < hi; r++) { @@ -1224,6 +1238,10 @@ public: // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. std::vector> Blocks() const { +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Blocks(); +#endif std::vector> b(n_params_, {0, 0, 0, 0, 0, 0}); for (size_t k = 0; k < n_params_; k++) for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) { @@ -1239,6 +1257,10 @@ public: } std::vector Diagonal() const { +#ifdef JFJOCH_USE_CUDA + if (gpu_) + return gpu_->Diagonal(); +#endif std::vector d(per_ * n_params_, 0.0); for (size_t k = 0; k < n_params_; k++) for (int q = entry_start_[k]; q < entry_start_[k + 1]; q++) @@ -1254,6 +1276,9 @@ private: RestraintJacobian jac_; std::vector row_of_, param_, entry_start_, entry_; // per entry its row and its parameter atom (-1: none) std::vector t_; +#ifdef JFJOCH_USE_CUDA + std::unique_ptr gpu_; +#endif }; // z = M^-1 r for M symmetric positive definite (xx yy zz xy xz yz), by its adjugate. @@ -1295,7 +1320,7 @@ void MinimiseAtomsNewton(Context &ctx, const AtomTarget &target, RefinementNewto std::vector h(n); for (size_t k = 0; k < n; k++) h[k] = xray[target.Atoms()[k / per]]; - RestraintNormalMatrix normal(target.Atoms(), ctx.atoms.size(), xyz, ctx.nthreads); + RestraintNormalMatrix normal(ctx, target.Atoms(), xyz); std::vector x_at = p; target.Set(p.data()); normal.At(set, ctx.XyzB()); @@ -1332,8 +1357,7 @@ void MinimiseJointNewton(Context &ctx, const XyzBTarget &target, const std::vect const RestraintSet &set_b, double w, double adp_w, int iterations) { const size_t n = atoms.size(); const double u2 = XyzBTarget::B_UNIT * XyzBTarget::B_UNIT; - RestraintNormalMatrix normal_xyz(atoms, ctx.atoms.size(), true, ctx.nthreads), - normal_b(atoms, ctx.atoms.size(), false, ctx.nthreads); + RestraintNormalMatrix normal_xyz(ctx, atoms, true), normal_b(ctx, atoms, false); std::vector x_at = target.Get(); auto at = [&](const std::vector &x) { target.Set(x.data()); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cpp b/image_analysis/structure_refinement/RefinementNewtonGPU.cpp new file mode 100644 index 000000000..990a3b692 --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cpp @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RefinementNewtonGPU.h" + +#include "RefinementRestraints.h" + +// The host half: the index of J, as RestraintGaussNewton lays out its rows, with every row kept. +void RestraintNormalMatrixGPU::At(const RestraintSet &set, const std::vector> &xyzb) { + if (!indexed_) { + std::vector record_entry, start{0}, atom; + for (const RestraintRecord &r : set.records) { + record_entry.push_back(static_cast(atom.size())); + switch (static_cast(r.kind)) { + case RestraintKind::Plane: + // one row per atom of the plane, each over all of them + for (int i = 0; i < r.atom[1]; i++) { + for (int k = 0; k < r.atom[1]; k++) + atom.push_back(set.plane_atoms[r.atom[0] + k]); + start.push_back(static_cast(atom.size())); + } + continue; + case RestraintKind::Angle: + atom.insert(atom.end(), r.atom, r.atom + 3); + break; + case RestraintKind::Torsion: + case RestraintKind::Chiral: + case RestraintKind::NcsDistance: + atom.insert(atom.end(), r.atom, r.atom + 4); + break; + default: // Bond, Repulsion, BSimilarity + atom.insert(atom.end(), r.atom, r.atom + 2); + break; + } + start.push_back(static_cast(atom.size())); + } + const size_t rows = start.size() - 1; + std::vector row_of(atom.size()), param(atom.size()); + for (size_t e = 0; e < atom.size(); e++) + param[e] = param_of_[atom[e]]; + for (size_t r = 0; r < rows; r++) + for (int e = start[r]; e < start[r + 1]; e++) + row_of[e] = static_cast(r); + std::vector entry_start(n_params_ + 1, 0); + for (int k : param) + if (k >= 0) + entry_start[k + 1]++; + for (size_t k = 0; k < n_params_; k++) + entry_start[k + 1] += entry_start[k]; + std::vector entry(entry_start.back()); + std::vector fill(entry_start.begin(), entry_start.end() - 1); + for (size_t e = 0; e < param.size(); e++) + if (param[e] >= 0) + entry[fill[param[e]]++] = static_cast(e); + Upload(set.records, set.plane_atoms, record_entry, start, param, row_of, entry_start, entry); + indexed_ = true; + } + Fill(xyzb); +} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu new file mode 100644 index 000000000..a3c61554e --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -0,0 +1,355 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per +// parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in +// float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of +// H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with +// plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise. + +#include "RefinementNewtonGPU.h" + +#include + +#include "../indexing/CUDAMemHelpers.h" +#include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS + +namespace { + +constexpr int THREADS = 256; +using V3f = refine_math::V3; + +void cuda_err(cudaError_t val) { + if (val != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); +} + +class DeviceGuard { +public: + explicit DeviceGuard(int device) { + cuda_err(cudaGetDevice(&previous_)); + cuda_err(cudaSetDevice(device)); + } + ~DeviceGuard() { cudaSetDevice(previous_); } + DeviceGuard(const DeviceGuard &) = delete; + DeviceGuard &operator=(const DeviceGuard &) = delete; + +private: + int previous_ = 0; +}; + +int GreatestStreamPriority() { + int least = 0, greatest = 0; + cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest)); + return greatest; +} + +int GridSize(int n) { + return (n + THREADS - 1) / THREADS; +} + +__device__ V3f xyz(const float4 *p, int i) { + return {p[i].x, p[i].y, p[i].z}; +} + +__device__ float Get(const float4 &j, int c) { + return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w; +} + +// sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math). +// The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2 +__device__ void AddProduct(float2 &sum, float a, float2 b) { + const float p = a * b.x; + const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly + const float s = sum.x + p; + const float v = s - sum.x; + const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly + const float lo = s_err + sum.y + p_err + a * b.y; + const float hi = s + lo; + sum = make_float2(hi, lo - (hi - s)); +} + +// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a +// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row. +__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry, + const float4 *xyzb, float4 *j) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) + return; + const RestraintRecord &r = records[i]; + float4 *out = j + record_entry[i]; + V3f g[4] = {}; + int na = 0; + float e = 0; + switch (static_cast(r.kind)) { + case RestraintKind::Bond: + e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g); + na = 2; + break; + case RestraintKind::Angle: + e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal, + r.sigma, g); + na = 3; + break; + case RestraintKind::Torsion: + e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g); + na = 4; + break; + case RestraintKind::Chiral: + e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g); + na = 4; + break; + case RestraintKind::NcsDistance: + e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), + xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g); + na = 4; + break; + case RestraintKind::Repulsion: { + const V3f q = xyz(xyzb, r.atom[1]); + const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0], + r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1], + r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]}; + e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g); + const V3f gi = g[1]; + g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z, + r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z, + r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z}; + na = 2; + break; + } + case RestraintKind::Plane: { + V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS]; + const int m = r.atom[1]; + for (int k = 0; k < m; k++) + px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]); + refine_math::PlaneTerm(px, m, r.sigma, pg); + V3f nv{0, 0, 0}; + for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n) + if (refine_math::Norm(pg[k]) > refine_math::Norm(nv)) + nv = pg[k]; + const float norm = refine_math::Norm(nv); + const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f; + for (int a = 0; a < m; a++) + for (int k = 0; k < m; k++) { + const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m); + out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f); + } + return; + } + case RestraintKind::BSimilarity: { + float gi = 0, gj = 0; + e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj); + const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; + out[0] = make_float4(0, 0, 0, f * gi); + out[1] = make_float4(0, 0, 0, f * gj); + return; + } + } + const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; + for (int k = 0; k < na; k++) + out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f); +} + +// t_r = sum over the row's entries of J_e . v_k +__global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v, + float2 *t, int rows) { + const int r = blockIdx.x * blockDim.x + threadIdx.x; + if (r >= rows) + return; + float2 dot = make_float2(0, 0); + for (int e = start[r]; e < start[r + 1]; e++) + if (const int k = param[e]; k >= 0) + for (int c = 0; c < per; c++) + AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]); + t[r] = dot; +} + +// hv_k = sum over the parameter atom's entries of t_row J_e +__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, + int c0, const float2 *t, float2 *hv, int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + for (int c = 0; c < per; c++) { + float2 sum = make_float2(0, 0); + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const int e = entry[q]; + AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); + } + hv[per * k + c] = sum; + } +} + +__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d, + int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + for (int c = 0; c < per; c++) { + float sum = 0; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float x = Get(j[entry[q]], c0 + c); + sum += x * x; + } + d[per * k + c] = sum; + } +} + +__global__ void BlocksKernel(const int *entry_start, const int *entry, const float4 *j, float *b, int n) { + const int k = blockIdx.x * blockDim.x + threadIdx.x; + if (k >= n) + return; + float s[6] = {0, 0, 0, 0, 0, 0}; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float4 x = j[entry[q]]; + s[0] += x.x * x.x; + s[1] += x.y * x.y; + s[2] += x.z * x.z; + s[3] += x.x * x.y; + s[4] += x.x * x.z; + s[5] += x.y * x.z; + } + for (int c = 0; c < 6; c++) + b[6 * k + c] = s[c]; +} + +template +CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { + CudaDevicePtr d(std::max(host.size(), 1)); + cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream)); + return d; +} + +} // namespace + +struct RestraintNormalMatrixGPU::Device { + int device; + CudaStream stream; + int records = 0, rows = 0; + CudaDevicePtr record; + CudaDevicePtr plane_atoms, record_entry, start, param, row_of, entry_start, entry; + CudaDevicePtr xyzb, j; + CudaDevicePtr t, v, hv; + CudaDevicePtr diagonal; + PinnedVector host; // v and hv as float pairs + PinnedVector host_xyzb; +}; + +RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector ¶m_atoms, size_t n_atoms, + bool xyz) + : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) { + for (size_t k = 0; k < param_atoms.size(); k++) + param_of_[param_atoms[k]] = static_cast(k); + DeviceGuard guard(device); + // At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the + // X-ray evaluations of the other trials on the card. + dev_ = std::unique_ptr(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())}); + const size_t n = std::max(per_ * n_params_, 1); + dev_->xyzb = CudaDevicePtr(std::max(n_atoms, 1)); + dev_->host_xyzb.resize(n_atoms); + dev_->v = CudaDevicePtr(n); + dev_->hv = CudaDevicePtr(n); + dev_->diagonal = CudaDevicePtr(n); + dev_->host.resize(n); +} + +RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() { + DeviceGuard guard(dev_->device); + dev_.reset(); +} + +void RestraintNormalMatrixGPU::Upload(const std::vector &records, const std::vector &plane_atoms, + const std::vector &record_entry, const std::vector &start, + const std::vector ¶m, const std::vector &row_of, + const std::vector &entry_start, const std::vector &entry) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + d.records = static_cast(records.size()); + d.rows = static_cast(start.size()) - 1; + d.record = Up(records, d.stream); + d.plane_atoms = Up(plane_atoms, d.stream); + d.record_entry = Up(record_entry, d.stream); + d.start = Up(start, d.stream); + d.param = Up(param, d.stream); + d.row_of = Up(row_of, d.stream); + d.entry_start = Up(entry_start, d.stream); + d.entry = Up(entry, d.stream); + d.j = CudaDevicePtr(std::max(param.size(), 1)); + d.t = CudaDevicePtr(std::max(d.rows, 1)); + cuda_err(cudaStreamSynchronize(d.stream)); +} + +void RestraintNormalMatrixGPU::Fill(const std::vector> &xyzb) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + for (size_t a = 0; a < xyzb.size(); a++) + d.host_xyzb[a] = make_float4(static_cast(xyzb[a][0]), static_cast(xyzb[a][1]), + static_cast(xyzb[a][2]), static_cast(xyzb[a][3])); + cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice, + d.stream)); + if (d.records > 0) + JacobianKernel<<>>(d.record, d.records, d.plane_atoms, + d.record_entry, d.xyzb, d.j); + cuda_err(cudaGetLastError()); + cuda_err(cudaStreamSynchronize(d.stream)); +} + +void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vector &hv) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + hv.resize(per_ * n_params_); + if (n == 0) + return; + for (size_t i = 0; i < hv.size(); i++) { + const float hi = static_cast(v[i]); + d.host[i] = make_float2(hi, static_cast(v[i] - hi)); + } + cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); + if (d.rows > 0) + RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); + TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, + d.hv, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + for (size_t i = 0; i < hv.size(); i++) + hv[i] = static_cast(d.host[i].x) + d.host[i].y; +} + +std::vector RestraintNormalMatrixGPU::Diagonal() const { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + std::vector out(per_ * n_params_); + if (n == 0) + return out; + std::vector host(out.size()); + DiagonalKernel<<>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + std::copy(host.begin(), host.end(), out.begin()); + return out; +} + +std::vector> RestraintNormalMatrixGPU::Blocks() const { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(n_params_); + std::vector> out(n_params_); + if (n == 0) + return out; + CudaDevicePtr b(6 * n_params_); + std::vector host(6 * n_params_); + BlocksKernel<<>>(d.entry_start, d.entry, d.j, b, n); + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(host.data(), b.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + for (size_t k = 0; k < n_params_; k++) + for (int c = 0; c < 6; c++) + out[k][c] = host[6 * k + c]; + return out; +} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h new file mode 100644 index 000000000..3136ed029 --- /dev/null +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) of the Newton minimisation on a GPU (CUDA +// builds only): RestraintNormalMatrix of Refinement.cpp on the card, in single precision. J is worked out on the device, +// one thread per restraint, with the terms the engine evaluates the restraints with (RefinementMath.h); a product is two +// gathers, J v per row and J^T t per parameter atom over its entries, each summed in a fixed order without atomics, in +// double-float pairs (RefinementNewtonGPU.cu). +// +// Every restraint keeps its rows whatever its E: where RestraintGaussNewton drops a row (E = 0, a plane without a +// normal) its entries here are zero, so the index of J is worked out once, at the first At(), and the set must stay +// the same for the matrix's life. + +#include +#include +#include +#include + +#include "RefinementMath.h" // RestraintRecord + +struct RestraintSet; + +class RestraintNormalMatrixGPU { +public: + // On the x y z (`xyz`) or the B of `param_atoms` of the model's `n_atoms`, on GPU `device`. + RestraintNormalMatrixGPU(int device, const std::vector ¶m_atoms, size_t n_atoms, bool xyz); + ~RestraintNormalMatrixGPU(); + + void At(const RestraintSet &set, const std::vector> &xyzb); + // hv = J^T J v + void Multiply(const std::vector &v, std::vector &hv); + // Per parameter atom the 3 x 3 block of J^T J on its x y z (xx yy zz xy xz yz); positions only. + std::vector> Blocks() const; + std::vector Diagonal() const; + +private: + // The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries + // start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries. + void Upload(const std::vector &records, const std::vector &plane_atoms, + const std::vector &record_entry, const std::vector &start, const std::vector ¶m, + const std::vector &row_of, const std::vector &entry_start, const std::vector &entry); + // J at `xyzb` (x, y, z, B per atom), in float + void Fill(const std::vector> &xyzb); + + int per_, c0_; + size_t n_params_; + std::vector param_of_; + bool indexed_ = false; + struct Device; + std::unique_ptr dev_; +}; diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index 7f84cf8f4..e34653877 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -37,6 +37,7 @@ #include "../image_analysis/structure_refinement/RefinementXray.h" #ifdef JFJOCH_USE_CUDA #include "../image_analysis/structure_refinement/RefinementGPU.h" +#include "../image_analysis/structure_refinement/RefinementNewtonGPU.h" #endif namespace { @@ -986,6 +987,104 @@ TEST_CASE("RefinementXray_gradient_CPU", "[Refinement]") { } #ifdef JFJOCH_USE_CUDA +TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { + if (get_gpu_count() == 0) + return; + const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); + gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); + st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); + const RefinementTopology topo = BuildRefinementTopology(st); + RestraintSet set = topo.geometry; + set.Append(topo.adp); + set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0)); + const std::vector atoms = atoms_of(st.models[0]); + set.Finalize(atoms.size()); + std::vector> xyzb; + for (gemmi::Atom *a : atoms) + xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); + // every atom but each fifth a parameter: the rest drop out + std::vector params, param_of(atoms.size(), -1); + for (int a = 0; a < static_cast(atoms.size()); a++) + if (a % 5 != 4) { + param_of[a] = static_cast(params.size()); + params.push_back(a); + } + const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); + REQUIRE(jac.start.size() > 100); + std::mt19937 gen(3); + std::uniform_real_distribution uniform(-1, 1); + // a random v, and a smooth one - the same shift of every atom and a thousandth of it at random, J v then the small + // difference of large terms + for (const auto [xyz, smooth] : {std::pair{true, false}, {true, true}, {false, false}, {false, true}}) { + CAPTURE(xyz, smooth); + const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3; + const size_t n = per * params.size(); + std::vector v(n); + for (size_t i = 0; i < n; i++) + v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen); + // J^T J v, the diagonal and the 3 x 3 blocks on the CPU, each entry in turn + std::vector hv(n, 0.0), diagonal(n, 0.0); + std::vector> blocks(params.size(), {0, 0, 0, 0, 0, 0}); + for (size_t r = 0; r + 1 < jac.start.size(); r++) { + double t = 0; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + t += jac.j[e][c0 + c] * v[per * k + c]; + for (int e = jac.start[r]; e < jac.start[r + 1]; e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) + for (int c = 0; c < per; c++) + hv[per * k + c] += t * jac.j[e][c0 + c]; + } + for (size_t e = 0; e < jac.atom.size(); e++) + if (const int k = param_of[jac.atom[e]]; k >= 0) { + const std::array &j = jac.j[e]; + for (int c = 0; c < per; c++) + diagonal[per * k + c] += j[c0 + c] * j[c0 + c]; + blocks[k][0] += j[0] * j[0]; + blocks[k][1] += j[1] * j[1]; + blocks[k][2] += j[2] * j[2]; + blocks[k][3] += j[0] * j[1]; + blocks[k][4] += j[0] * j[2]; + blocks[k][5] += j[1] * j[2]; + } + RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); + gpu.At(set, xyzb); + std::vector hv_gpu; + gpu.Multiply(v, hv_gpu); + const std::vector diagonal_gpu = gpu.Diagonal(); + REQUIRE(hv_gpu.size() == n); + REQUIRE(diagonal_gpu.size() == n); + // In float, from float positions: within a few float epsilons of the double product in norm (in refinement + // its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself, each block + // relative to its own size. + double num = 0, den = 0, worst_diagonal = 0, worst_block = 0; + for (size_t i = 0; i < n; i++) { + num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]); + den += hv[i] * hv[i]; + if (diagonal[i] > 0) + worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]); + } + if (xyz) { + const std::vector> blocks_gpu = gpu.Blocks(); + REQUIRE(blocks_gpu.size() == params.size()); + for (size_t k = 0; k < params.size(); k++) { + double d2 = 0, b2 = 0; + for (int c = 0; c < 6; c++) { + d2 += (blocks_gpu[k][c] - blocks[k][c]) * (blocks_gpu[k][c] - blocks[k][c]); + b2 += blocks[k][c] * blocks[k][c]; + } + if (b2 > 0) + worst_block = std::max(worst_block, std::sqrt(d2 / b2)); + } + } + CAPTURE(std::sqrt(num / den), worst_diagonal, worst_block); + CHECK(std::sqrt(num / den) < 1e-5); + CHECK(worst_diagonal < 1e-5); + CHECK(worst_block < 1e-5); + } +} + TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") { if (get_gpu_count() == 0) return;