Refinement on the GPU: the normal matrix's conjugate gradients on the card

The minimiser's initial inverse Hessian (20 preconditioned CG iterations per direction) ran on the host
apart from the J^T J product, which was uploaded and downloaded every iteration. Now the whole solve
runs on the card: the product with the X-ray diagonal, the per-residue Cholesky preconditioner (the
diagonal as 1x1 blocks for B), the vector updates and the dot products, the vectors and the scalars
staying on the device; one transfer in and one out per solve.

Single precision only, Metal-portable: every vector and sum in float pairs (double-float arithmetic),
the dot products in a fixed number of partial sums reduced in a fixed tree, no atomics, so a run is
reproducible. The U and TLS minimisers keep the CPU solve.

Test: RefinementNewton_solve_GPU_vs_CPU (device CG vs ConjugateGradients, and bitwise repeat).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-16 22:26:14 +01:00
1 parent 81fb4eafd1
commit 2ad8913b5d
4 files changed
+384 -16

No files matched your search

@@ -1013,20 +1013,19 @@ std::vector<double> FlooredDiagonal(std::vector<double> d) {
constexpr int NORMAL_CG_ITERATIONS = 20;
// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q)
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then hv = H v
// (product) and z = M^-1 r for a preconditioner M of it (precondition).
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then z = H^-1 q by
// NORMAL_CG_ITERATIONS conjugate gradients (solve: ConjugateGradients on the CPU, or on the card).
template <typename Target>
void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations,
const std::function<void(const std::vector<double> &, const std::vector<double> &)> &at,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
const std::function<void(const std::vector<double> &, std::vector<double> &)> &solve) {
std::vector<double> p = target.Get();
if (p.empty())
return;
minimiser.SetInverseHessian([&](const std::vector<double> &x, std::vector<double> &q) {
at(x, q);
std::vector<double> z;
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
solve(q, z);
q.swap(z);
});
minimiser.Minimise([&](const std::vector<double> &x, double &f, std::vector<double> &g) {
@@ -1037,6 +1036,15 @@ void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMini
target.Set(p.data()); // where the minimiser ended, not where its last line search probed
}
// The conjugate gradients on the CPU: hv = H v (product) and z = M^-1 r for a preconditioner M of H (precondition).
std::function<void(const std::vector<double> &, std::vector<double> &)>
NormalSolve(const Context &ctx, const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
return [&ctx, product, precondition](const std::vector<double> &q, std::vector<double> &z) {
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
}
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
// a GPU, on its card (RestraintNormalMatrixGPU).
@@ -1113,6 +1121,11 @@ public:
});
}
#ifdef JFJOCH_USE_CUDA
// The conjugate gradients on the card, the vectors kept there (RestraintNormalMatrixGPU::Solve)
RestraintNormalMatrixGPU *GPU() { return gpu_.get(); }
#endif
std::vector<double> Diagonal() const {
#ifdef JFJOCH_USE_CUDA
if (gpu_)
@@ -1182,6 +1195,23 @@ public:
ParallelFor(static_cast<int>(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); });
}
// The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and its factor, row by row.
void Factors(std::vector<int> &start, std::vector<int> &element, std::vector<double> &lower) const {
start.assign(1, 0);
element.clear();
lower.clear();
for (size_t g = 0; g < params_.size(); g++) {
for (int k : params_[g])
for (int c = 0; c < 3; c++)
element.push_back(3 * k + c);
start.push_back(static_cast<int>(element.size()));
const Eigen::MatrixXd l = llt_[g].matrixL();
for (int i = 0; i < l.rows(); i++)
for (int k = 0; k <= i; k++)
lower.push_back(l(i, k));
}
}
void Apply(const std::vector<double> &r, std::vector<double> &z, size_t nthreads) const {
z.resize(r.size());
ParallelFor(static_cast<int>(params_.size()), nthreads, [&](int g) {
@@ -1240,6 +1270,23 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
diagonal = FlooredDiagonal(diagonal);
if (xyz)
residues = std::make_unique<ResidueBlocks>(ctx, target.Atoms(), set, h);
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) { // the blocks, or the diagonal as blocks of one by its square root
std::vector<int> start, element;
std::vector<double> lower;
if (residues)
residues->Factors(start, element, lower);
else {
for (size_t k = 0; k < n; k++) {
start.push_back(static_cast<int>(k));
element.push_back(static_cast<int>(k));
lower.push_back(std::sqrt(diagonal[k]));
}
start.push_back(static_cast<int>(n));
}
normal.GPU()->SetSolve(h, start, element, lower);
}
#endif
}
if (x != x_at) {
x_at = x;
@@ -1259,7 +1306,16 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
}
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
auto solve = [&](const std::vector<double> &q, std::vector<double> &z) {
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) {
const RefinementStage stage("normal.solve");
return normal.GPU()->Solve(q, z, NORMAL_CG_ITERATIONS);
}
#endif
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, solve);
}
// Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their
@@ -1306,7 +1362,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim
z.resize(n);
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's
@@ -1364,7 +1420,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min
for (size_t k = 0; k < r.size(); k++)
z[k] = r[k] / diagonal[k];
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis
@@ -17,6 +17,7 @@
namespace {
constexpr int THREADS = 256;
constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order
using V3f = refine_math::V3<float>;
void cuda_err(cudaError_t val) {
@@ -69,6 +70,34 @@ __device__ void AddProduct(float2 &sum, float a, float2 b) {
sum = make_float2(hi, lo - (hi - s));
}
// The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b.
__device__ float2 Normalised(float s, float e) {
const float hi = s + e;
return make_float2(hi, e - (hi - s));
}
__device__ float2 Add(float2 a, float2 b) {
const float s = a.x + b.x;
const float v = s - a.x;
const float e = (a.x - (s - v)) + (b.x - v);
return Normalised(s, e + a.y + b.y);
}
__device__ float2 Mul(float2 a, float2 b) {
const float p = a.x * b.x;
return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x));
}
__device__ float2 Neg(float2 a) {
return make_float2(-a.x, -a.y);
}
__device__ float2 Div(float2 a, float2 b) {
const float q = a.x / b.x;
const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b)));
return Normalised(q, rest.x / b.x);
}
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
@@ -166,9 +195,9 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j,
t[r] = dot;
}
// hv_k = sum over the parameter atom's entries of t_row J_e
// hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
int c0, const float2 *t, float2 *hv, int n) {
int c0, const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
@@ -178,7 +207,7 @@ __global__ void TransposeProduct(const int *entry_start, const int *entry, const
const int e = entry[q];
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
}
hv[per * k + c] = sum;
hv[per * k + c] = h ? Add(sum, Mul(h[per * k + c], v[per * k + c])) : sum;
}
}
@@ -197,6 +226,105 @@ __global__ void DiagonalKernel(const int *entry_start, const int *entry, const f
}
}
// The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta
struct CGScalars {
float2 rz, a, beta;
int done; // p H p was not positive: x stays where it is
};
// The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size).
__device__ float2 BlockSum(float2 *shared, float2 value) {
const int t = threadIdx.x;
shared[t] = value;
__syncthreads();
for (int w = blockDim.x / 2; w > 0; w /= 2) {
if (t < w)
shared[t] = Add(shared[t], shared[t + w]);
__syncthreads();
}
return shared[0];
}
// u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements
__global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) {
__shared__ float2 shared[THREADS];
float2 sum = make_float2(0, 0);
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
sum = Add(sum, Mul(u[i], v[i]));
const float2 block = BlockSum(shared, sum);
if (threadIdx.x == 0)
partial[blockIdx.x] = block;
}
enum class DotUse { FirstRz, PHp, Rz };
// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta)
__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use) {
__shared__ float2 shared[DOT_BLOCKS];
const float2 dot = BlockSum(shared, partial[threadIdx.x]);
if (threadIdx.x != 0)
return;
switch (use) {
case DotUse::FirstRz:
s->rz = dot;
break;
case DotUse::PHp:
s->a = Div(s->rz, dot);
if (!(dot.x > 0))
s->done = 1;
break;
case DotUse::Rz:
s->beta = Div(dot, s->rz);
s->rz = dot;
break;
}
}
// x += a p, r -= a H p
__global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n || s->done)
return;
x[i] = Add(x[i], Mul(s->a, p[i]));
r[i] = Add(r[i], Neg(Mul(s->a, hp[i])));
}
// p = z + beta p
__global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n)
p[i] = Add(z[i], Mul(s->beta, p[i]));
}
// z = M^-1 r, one thread per block: L y = r forward, then L^T z = y back, y held in z; the factor's diagonal is
// stored as its inverse.
__global__ void BlockSolve(const int *start, const int *element, const int *factor_start, const float2 *factor,
const float2 *r, float2 *z, int blocks) {
const int g = blockIdx.x * blockDim.x + threadIdx.x;
if (g >= blocks)
return;
const int *e = element + start[g];
const int m = start[g + 1] - start[g];
const float2 *l = factor + factor_start[g];
for (int i = 0; i < m; i++) {
float2 sum = r[e[i]];
for (int k = 0; k < i; k++)
sum = Add(sum, Neg(Mul(l[i * (i + 1) / 2 + k], z[e[k]])));
z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]);
}
for (int i = m - 1; i >= 0; i--) {
float2 sum = z[e[i]];
for (int k = i + 1; k < m; k++)
sum = Add(sum, Neg(Mul(l[k * (k + 1) / 2 + i], z[e[k]])));
z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]);
}
}
float2 Split(double v) {
const float hi = static_cast<float>(v);
return make_float2(hi, static_cast<float>(v - hi));
}
template <typename T>
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
@@ -215,6 +343,11 @@ struct RestraintNormalMatrixGPU::Device {
CudaDevicePtr<float4> xyzb, j;
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
// the conjugate gradients: H's X-ray diagonal, M's blocks and factors, the vectors, the dot products' partial sums
int blocks = 0;
CudaDevicePtr<float2> h, factor, x, r, z, p, hp, partial;
CudaDevicePtr<int> block_start, block_element, factor_start;
CudaDevicePtr<CGScalars> scalars;
PinnedVector<float2> host; // v and hv as float pairs
PinnedVector<float4> host_xyzb;
};
@@ -234,6 +367,10 @@ RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector
dev_->v = CudaDevicePtr<float2>(n);
dev_->hv = CudaDevicePtr<float2>(n);
dev_->diagonal = CudaDevicePtr<float>(n);
for (CudaDevicePtr<float2> *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp})
*u = CudaDevicePtr<float2>(n);
dev_->partial = CudaDevicePtr<float2>(DOT_BLOCKS);
dev_->scalars = CudaDevicePtr<CGScalars>(1);
dev_->host.resize(n);
}
@@ -285,15 +422,13 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
hv.resize(per_ * n_params_);
if (n == 0)
return;
for (size_t i = 0; i < hv.size(); i++) {
const float hi = static_cast<float>(v[i]);
d.host[i] = make_float2(hi, static_cast<float>(v[i] - hi));
}
for (size_t i = 0; i < hv.size(); i++)
d.host[i] = Split(v[i]);
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
d.hv, n);
nullptr, nullptr, d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
@@ -317,3 +452,75 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
return out;
}
void RestraintNormalMatrixGPU::SetSolve(const std::vector<double> &h, const std::vector<int> &start,
const std::vector<int> &element, const std::vector<double> &lower) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
std::vector<float2> h2(h.size());
for (size_t i = 0; i < h.size(); i++)
h2[i] = Split(h[i]);
d.blocks = static_cast<int>(start.size()) - 1;
std::vector<int> factor_start{0};
std::vector<float2> factor(lower.size());
for (int g = 0; g < d.blocks; g++) {
const int m = start[g + 1] - start[g];
for (int i = 0; i < m; i++)
for (int k = 0; k <= i; k++) {
const size_t q = factor_start.back() + i * (i + 1) / 2 + k;
factor[q] = Split(i == k ? 1.0 / lower[q] : lower[q]);
}
factor_start.push_back(factor_start.back() + m * (m + 1) / 2);
}
d.h = Up(h2, d.stream);
d.factor = Up(factor, d.stream);
d.block_start = Up(start, d.stream);
d.block_element = Up(element, d.stream);
d.factor_start = Up(factor_start, d.stream);
cuda_err(cudaStreamSynchronize(d.stream));
}
// ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host
// before x.
void RestraintNormalMatrixGPU::Solve(const std::vector<double> &b, std::vector<double> &x, int iterations) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(per_ * n_params_);
x.assign(n, 0.0);
if (n == 0)
return;
auto precondition = [&]() { // z = M^-1 r
BlockSolve<<<GridSize(d.blocks), THREADS, 0, d.stream>>>(d.block_start, d.block_element, d.factor_start,
d.factor, d.r, d.z, d.blocks);
};
auto dot = [&](const float2 *u, const float2 *v, DotUse use) {
DotPartial<<<DOT_BLOCKS, THREADS, 0, d.stream>>>(u, v, n, d.partial);
DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use);
};
for (int i = 0; i < n; i++)
d.host[i] = Split(b[i]);
cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream));
cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream));
precondition();
cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream));
dot(d.r, d.z, DotUse::FirstRz);
for (int k = 0; k < iterations; k++) {
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows);
TransposeProduct<<<GridSize(static_cast<int>(n_params_)), THREADS, 0, d.stream>>>(
d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, d.h, d.p, d.hp, static_cast<int>(n_params_));
dot(d.p, d.hp, DotUse::PHp);
Step<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.p, d.hp, d.x, d.r, n);
if (k + 1 < iterations) { // the last direction would not be used
precondition();
dot(d.r, d.z, DotUse::Rz);
NextDirection<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.z, d.p, n);
}
}
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (int i = 0; i < n; i++)
x[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
}
@@ -34,6 +34,16 @@ public:
// The diagonal of J^T J.
std::vector<double> Diagonal() const;
// The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at
// the point last given to At(), preconditioned by a block-diagonal M given by its Cholesky factors: block g over
// the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, its lower factor L in `lower`, row
// by row (L_ij at i (i + 1) / 2 + j from the block's first). h and M stay until the next SetSolve().
void SetSolve(const std::vector<double> &h, const std::vector<int> &start, const std::vector<int> &element,
const std::vector<double> &lower);
// x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), the vectors on the card
// and every sum in float pairs, in a fixed order (RefinementNewtonGPU.cu).
void Solve(const std::vector<double> &b, std::vector<double> &x, int iterations);
private:
// The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries
// start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries.
+95
View File
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <Eigen/Dense>
#include <algorithm>
#include <atomic>
@@ -1112,6 +1113,100 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") {
}
}
// The conjugate gradients on the card against ConjugateGradients on the CPU, for H = J^T J + diag(h) preconditioned by
// blocks of H: two atoms' x y z per block for the positions, each B alone.
TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;
const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90);
gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5);
st.models[0] = shaken(st.models[0], 0.15, 3.0, 2);
const RefinementTopology topo = BuildRefinementTopology(st);
RestraintSet set = topo.geometry;
set.Append(topo.adp);
set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0));
const std::vector<gemmi::Atom *> atoms = atoms_of(st.models[0]);
set.Finalize(atoms.size());
std::vector<std::array<double, 4>> xyzb;
for (gemmi::Atom *a : atoms)
xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso});
std::vector<int> params, param_of(atoms.size(), -1);
for (int a = 0; a < static_cast<int>(atoms.size()); a++)
if (a % 5 != 4) {
param_of[a] = static_cast<int>(params.size());
params.push_back(a);
}
const RestraintJacobian jac = RestraintGaussNewton(set, xyzb);
std::mt19937 gen(5);
std::uniform_real_distribution<double> uniform(-1, 1);
for (const bool xyz : {true, false}) {
CAPTURE(xyz);
const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3;
const size_t n = per * params.size();
// H in full, the X-ray diagonal small beside the stiffest restraints
Eigen::MatrixXd hm = Eigen::MatrixXd::Zero(n, n);
for (size_t r = 0; r + 1 < jac.start.size(); r++)
for (int e1 = jac.start[r]; e1 < jac.start[r + 1]; e1++)
for (int e2 = jac.start[r]; e2 < jac.start[r + 1]; e2++) {
const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]];
if (k1 >= 0 && k2 >= 0)
for (int a = 0; a < per; a++)
for (int c = 0; c < per; c++)
hm(per * k1 + a, per * k2 + c) += jac.j[e1][c0 + a] * jac.j[e2][c0 + c];
}
std::vector<double> h(n), b(n);
for (size_t i = 0; i < n; i++) {
h[i] = xyz ? 10 + 5 * uniform(gen) : 0.01 + 0.005 * uniform(gen);
hm(i, i) += h[i];
b[i] = uniform(gen);
}
const int block = xyz ? 6 : 1;
std::vector<int> start{0}, element;
std::vector<double> lower;
std::vector<Eigen::LLT<Eigen::MatrixXd>> llt;
for (size_t first = 0; first < n; first += block) {
const int m = static_cast<int>(std::min<size_t>(block, n - first));
llt.emplace_back(hm.block(first, first, m, m));
const Eigen::MatrixXd l = llt.back().matrixL();
for (int i = 0; i < m; i++) {
element.push_back(static_cast<int>(first) + i);
for (int k = 0; k <= i; k++)
lower.push_back(l(i, k));
}
start.push_back(static_cast<int>(element.size()));
}
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
const Eigen::VectorXd out = hm * Eigen::Map<const Eigen::VectorXd>(v.data(), n);
hv.assign(out.data(), out.data() + n);
};
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
z.resize(n);
for (size_t g = 0; g < llt.size(); g++) {
const int m = start[g + 1] - start[g];
const Eigen::VectorXd zg = llt[g].solve(Eigen::Map<const Eigen::VectorXd>(r.data() + start[g], m));
std::copy(zg.data(), zg.data() + m, z.begin() + start[g]);
}
};
std::vector<double> x;
ConjugateGradients(product, precondition, b, x, 20, 0.0);
RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz);
gpu.At(set, xyzb);
gpu.SetSolve(h, start, element, lower);
std::vector<double> x_gpu, x_again;
gpu.Solve(b, x_gpu, 20);
gpu.Solve(b, x_again, 20);
REQUIRE(x_gpu.size() == n);
CHECK(x_gpu == x_again); // a fixed order: the same bits every time
double num = 0, den = 0;
for (size_t i = 0; i < n; i++) {
num += (x_gpu[i] - x[i]) * (x_gpu[i] - x[i]);
den += x[i] * x[i];
}
CAPTURE(std::sqrt(num / den));
CHECK(std::sqrt(num / den) < 1e-5);
}
}
TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;