From 2ad8913b5d6117bde1bb47900ebc44d8ea150bfe Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 16 Nov 2026 22:26:14 +0100 Subject: [PATCH 1/4] Refinement on the GPU: the normal matrix's conjugate gradients on the card The minimiser's initial inverse Hessian (20 preconditioned CG iterations per direction) ran on the host apart from the J^T J product, which was uploaded and downloaded every iteration. Now the whole solve runs on the card: the product with the X-ray diagonal, the per-residue Cholesky preconditioner (the diagonal as 1x1 blocks for B), the vector updates and the dot products, the vectors and the scalars staying on the device; one transfer in and one out per solve. Single precision only, Metal-portable: every vector and sum in float pairs (double-float arithmetic), the dot products in a fixed number of partial sums reduced in a fixed tree, no atomics, so a run is reproducible. The U and TLS minimisers keep the CPU solve. Test: RefinementNewton_solve_GPU_vs_CPU (device CG vs ConjugateGradients, and bitwise repeat). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 72 +++++- .../RefinementNewtonGPU.cu | 223 +++++++++++++++++- .../RefinementNewtonGPU.h | 10 + tests/RefinementTest.cpp | 95 ++++++++ 4 files changed, 384 insertions(+), 16 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index ae6ecbac0..ac5481cad 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1013,20 +1013,19 @@ std::vector FlooredDiagonal(std::vector d) { constexpr int NORMAL_CG_ITERATIONS = 20; // `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q) -// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then hv = H v -// (product) and z = M^-1 r for a preconditioner M of it (precondition). +// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then z = H^-1 q by +// NORMAL_CG_ITERATIONS conjugate gradients (solve: ConjugateGradients on the CPU, or on the card). template void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations, const std::function &, const std::vector &)> &at, - const std::function &, std::vector &)> &product, - const std::function &, std::vector &)> &precondition) { + const std::function &, std::vector &)> &solve) { std::vector p = target.Get(); if (p.empty()) return; minimiser.SetInverseHessian([&](const std::vector &x, std::vector &q) { at(x, q); std::vector z; - ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads); + solve(q, z); q.swap(z); }); minimiser.Minimise([&](const std::vector &x, double &f, std::vector &g) { @@ -1037,6 +1036,15 @@ void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMini target.Set(p.data()); // where the minimiser ended, not where its last line search probed } +// The conjugate gradients on the CPU: hv = H v (product) and z = M^-1 r for a preconditioner M of H (precondition). +std::function &, std::vector &)> +NormalSolve(const Context &ctx, const std::function &, std::vector &)> &product, + const std::function &, std::vector &)> &precondition) { + return [&ctx, product, precondition](const std::vector &q, std::vector &z) { + ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads); + }; +} + // The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the // point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on // a GPU, on its card (RestraintNormalMatrixGPU). @@ -1113,6 +1121,11 @@ public: }); } +#ifdef JFJOCH_USE_CUDA + // The conjugate gradients on the card, the vectors kept there (RestraintNormalMatrixGPU::Solve) + RestraintNormalMatrixGPU *GPU() { return gpu_.get(); } +#endif + std::vector Diagonal() const { #ifdef JFJOCH_USE_CUDA if (gpu_) @@ -1182,6 +1195,23 @@ public: ParallelFor(static_cast(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); }); } + // The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and its factor, row by row. + void Factors(std::vector &start, std::vector &element, std::vector &lower) const { + start.assign(1, 0); + element.clear(); + lower.clear(); + for (size_t g = 0; g < params_.size(); g++) { + for (int k : params_[g]) + for (int c = 0; c < 3; c++) + element.push_back(3 * k + c); + start.push_back(static_cast(element.size())); + const Eigen::MatrixXd l = llt_[g].matrixL(); + for (int i = 0; i < l.rows(); i++) + for (int k = 0; k <= i; k++) + lower.push_back(l(i, k)); + } + } + void Apply(const std::vector &r, std::vector &z, size_t nthreads) const { z.resize(r.size()); ParallelFor(static_cast(params_.size()), nthreads, [&](int g) { @@ -1240,6 +1270,23 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & diagonal = FlooredDiagonal(diagonal); if (xyz) residues = std::make_unique(ctx, target.Atoms(), set, h); +#ifdef JFJOCH_USE_CUDA + if (normal.GPU()) { // the blocks, or the diagonal as blocks of one by its square root + std::vector start, element; + std::vector lower; + if (residues) + residues->Factors(start, element, lower); + else { + for (size_t k = 0; k < n; k++) { + start.push_back(static_cast(k)); + element.push_back(static_cast(k)); + lower.push_back(std::sqrt(diagonal[k])); + } + start.push_back(static_cast(n)); + } + normal.GPU()->SetSolve(h, start, element, lower); + } +#endif } if (x != x_at) { x_at = x; @@ -1259,7 +1306,16 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; }); } }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); + auto solve = [&](const std::vector &q, std::vector &z) { +#ifdef JFJOCH_USE_CUDA + if (normal.GPU()) { + const RefinementStage stage("normal.solve"); + return normal.GPU()->Solve(q, z, NORMAL_CG_ITERATIONS); + } +#endif + ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads); + }; + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, solve); } // Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their @@ -1306,7 +1362,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim z.resize(n); refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; }); }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition)); } // The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's @@ -1364,7 +1420,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min for (size_t k = 0; k < r.size(); k++) z[k] = r[k] / diagonal[k]; }; - MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition); + MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition)); } // Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index a11281e46..ebc927691 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -17,6 +17,7 @@ namespace { constexpr int THREADS = 256; +constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order using V3f = refine_math::V3; void cuda_err(cudaError_t val) { @@ -69,6 +70,34 @@ __device__ void AddProduct(float2 &sum, float a, float2 b) { sum = make_float2(hi, lo - (hi - s)); } +// The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b. +__device__ float2 Normalised(float s, float e) { + const float hi = s + e; + return make_float2(hi, e - (hi - s)); +} + +__device__ float2 Add(float2 a, float2 b) { + const float s = a.x + b.x; + const float v = s - a.x; + const float e = (a.x - (s - v)) + (b.x - v); + return Normalised(s, e + a.y + b.y); +} + +__device__ float2 Mul(float2 a, float2 b) { + const float p = a.x * b.x; + return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x)); +} + +__device__ float2 Neg(float2 a) { + return make_float2(-a.x, -a.y); +} + +__device__ float2 Div(float2 a, float2 b) { + const float q = a.x / b.x; + const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b))); + return Normalised(q, rest.x / b.x); +} + // Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a // residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row. __global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry, @@ -166,9 +195,9 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j, t[r] = dot; } -// hv_k = sum over the parameter atom's entries of t_row J_e +// hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given __global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, - int c0, const float2 *t, float2 *hv, int n) { + int c0, const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; @@ -178,7 +207,7 @@ __global__ void TransposeProduct(const int *entry_start, const int *entry, const const int e = entry[q]; AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); } - hv[per * k + c] = sum; + hv[per * k + c] = h ? Add(sum, Mul(h[per * k + c], v[per * k + c])) : sum; } } @@ -197,6 +226,105 @@ __global__ void DiagonalKernel(const int *entry_start, const int *entry, const f } } +// The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta +struct CGScalars { + float2 rz, a, beta; + int done; // p H p was not positive: x stays where it is +}; + +// The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size). +__device__ float2 BlockSum(float2 *shared, float2 value) { + const int t = threadIdx.x; + shared[t] = value; + __syncthreads(); + for (int w = blockDim.x / 2; w > 0; w /= 2) { + if (t < w) + shared[t] = Add(shared[t], shared[t + w]); + __syncthreads(); + } + return shared[0]; +} + +// u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements +__global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) { + __shared__ float2 shared[THREADS]; + float2 sum = make_float2(0, 0); + for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS) + sum = Add(sum, Mul(u[i], v[i])); + const float2 block = BlockSum(shared, sum); + if (threadIdx.x == 0) + partial[blockIdx.x] = block; +} + +enum class DotUse { FirstRz, PHp, Rz }; + +// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta) +__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use) { + __shared__ float2 shared[DOT_BLOCKS]; + const float2 dot = BlockSum(shared, partial[threadIdx.x]); + if (threadIdx.x != 0) + return; + switch (use) { + case DotUse::FirstRz: + s->rz = dot; + break; + case DotUse::PHp: + s->a = Div(s->rz, dot); + if (!(dot.x > 0)) + s->done = 1; + break; + case DotUse::Rz: + s->beta = Div(dot, s->rz); + s->rz = dot; + break; + } +} + +// x += a p, r -= a H p +__global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n || s->done) + return; + x[i] = Add(x[i], Mul(s->a, p[i])); + r[i] = Add(r[i], Neg(Mul(s->a, hp[i]))); +} + +// p = z + beta p +__global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + p[i] = Add(z[i], Mul(s->beta, p[i])); +} + +// z = M^-1 r, one thread per block: L y = r forward, then L^T z = y back, y held in z; the factor's diagonal is +// stored as its inverse. +__global__ void BlockSolve(const int *start, const int *element, const int *factor_start, const float2 *factor, + const float2 *r, float2 *z, int blocks) { + const int g = blockIdx.x * blockDim.x + threadIdx.x; + if (g >= blocks) + return; + const int *e = element + start[g]; + const int m = start[g + 1] - start[g]; + const float2 *l = factor + factor_start[g]; + for (int i = 0; i < m; i++) { + float2 sum = r[e[i]]; + for (int k = 0; k < i; k++) + sum = Add(sum, Neg(Mul(l[i * (i + 1) / 2 + k], z[e[k]]))); + z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]); + } + for (int i = m - 1; i >= 0; i--) { + float2 sum = z[e[i]]; + for (int k = i + 1; k < m; k++) + sum = Add(sum, Neg(Mul(l[k * (k + 1) / 2 + i], z[e[k]]))); + z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]); + } +} + +float2 Split(double v) { + const float hi = static_cast(v); + return make_float2(hi, static_cast(v - hi)); +} + template CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { CudaDevicePtr d(std::max(host.size(), 1)); @@ -215,6 +343,11 @@ struct RestraintNormalMatrixGPU::Device { CudaDevicePtr xyzb, j; CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; + // the conjugate gradients: H's X-ray diagonal, M's blocks and factors, the vectors, the dot products' partial sums + int blocks = 0; + CudaDevicePtr h, factor, x, r, z, p, hp, partial; + CudaDevicePtr block_start, block_element, factor_start; + CudaDevicePtr scalars; PinnedVector host; // v and hv as float pairs PinnedVector host_xyzb; }; @@ -234,6 +367,10 @@ RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector dev_->v = CudaDevicePtr(n); dev_->hv = CudaDevicePtr(n); dev_->diagonal = CudaDevicePtr(n); + for (CudaDevicePtr *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp}) + *u = CudaDevicePtr(n); + dev_->partial = CudaDevicePtr(DOT_BLOCKS); + dev_->scalars = CudaDevicePtr(1); dev_->host.resize(n); } @@ -285,15 +422,13 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vecto hv.resize(per_ * n_params_); if (n == 0) return; - for (size_t i = 0; i < hv.size(); i++) { - const float hi = static_cast(v[i]); - d.host[i] = make_float2(hi, static_cast(v[i] - hi)); - } + for (size_t i = 0; i < hv.size(); i++) + d.host[i] = Split(v[i]); cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, - d.hv, n); + nullptr, nullptr, d.hv, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); @@ -317,3 +452,75 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { return out; } + +void RestraintNormalMatrixGPU::SetSolve(const std::vector &h, const std::vector &start, + const std::vector &element, const std::vector &lower) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + std::vector h2(h.size()); + for (size_t i = 0; i < h.size(); i++) + h2[i] = Split(h[i]); + d.blocks = static_cast(start.size()) - 1; + std::vector factor_start{0}; + std::vector factor(lower.size()); + for (int g = 0; g < d.blocks; g++) { + const int m = start[g + 1] - start[g]; + for (int i = 0; i < m; i++) + for (int k = 0; k <= i; k++) { + const size_t q = factor_start.back() + i * (i + 1) / 2 + k; + factor[q] = Split(i == k ? 1.0 / lower[q] : lower[q]); + } + factor_start.push_back(factor_start.back() + m * (m + 1) / 2); + } + d.h = Up(h2, d.stream); + d.factor = Up(factor, d.stream); + d.block_start = Up(start, d.stream); + d.block_element = Up(element, d.stream); + d.factor_start = Up(factor_start, d.stream); + cuda_err(cudaStreamSynchronize(d.stream)); +} + +// ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host +// before x. +void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector &x, int iterations) { + DeviceGuard guard(dev_->device); + Device &d = *dev_; + const int n = static_cast(per_ * n_params_); + x.assign(n, 0.0); + if (n == 0) + return; + auto precondition = [&]() { // z = M^-1 r + BlockSolve<<>>(d.block_start, d.block_element, d.factor_start, + d.factor, d.r, d.z, d.blocks); + }; + auto dot = [&](const float2 *u, const float2 *v, DotUse use) { + DotPartial<<>>(u, v, n, d.partial); + DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use); + }; + for (int i = 0; i < n; i++) + d.host[i] = Split(b[i]); + cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); + cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream)); + cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream)); + precondition(); + cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream)); + dot(d.r, d.z, DotUse::FirstRz); + for (int k = 0; k < iterations; k++) { + if (d.rows > 0) + RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows); + TransposeProduct<<(n_params_)), THREADS, 0, d.stream>>>( + d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, d.h, d.p, d.hp, static_cast(n_params_)); + dot(d.p, d.hp, DotUse::PHp); + Step<<>>(d.scalars, d.p, d.hp, d.x, d.r, n); + if (k + 1 < iterations) { // the last direction would not be used + precondition(); + dot(d.r, d.z, DotUse::Rz); + NextDirection<<>>(d.scalars, d.z, d.p, n); + } + } + cuda_err(cudaGetLastError()); + cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); + cuda_err(cudaStreamSynchronize(d.stream)); + for (int i = 0; i < n; i++) + x[i] = static_cast(d.host[i].x) + d.host[i].y; +} diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h index 317cfa8f9..3106e10f9 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.h +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -34,6 +34,16 @@ public: // The diagonal of J^T J. std::vector Diagonal() const; + // The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at + // the point last given to At(), preconditioned by a block-diagonal M given by its Cholesky factors: block g over + // the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, its lower factor L in `lower`, row + // by row (L_ij at i (i + 1) / 2 + j from the block's first). h and M stay until the next SetSolve(). + void SetSolve(const std::vector &h, const std::vector &start, const std::vector &element, + const std::vector &lower); + // x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), the vectors on the card + // and every sum in float pairs, in a fixed order (RefinementNewtonGPU.cu). + void Solve(const std::vector &b, std::vector &x, int iterations); + private: // The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries // start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries. diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index d2f5ac074..2d007bda9 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-only #include +#include #include #include @@ -1112,6 +1113,100 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { } } +// The conjugate gradients on the card against ConjugateGradients on the CPU, for H = J^T J + diag(h) preconditioned by +// blocks of H: two atoms' x y z per block for the positions, each B alone. +TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") { + if (get_gpu_count() == 0) + return; + const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); + gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); + st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); + const RefinementTopology topo = BuildRefinementTopology(st); + RestraintSet set = topo.geometry; + set.Append(topo.adp); + set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0)); + const std::vector atoms = atoms_of(st.models[0]); + set.Finalize(atoms.size()); + std::vector> xyzb; + for (gemmi::Atom *a : atoms) + xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); + std::vector params, param_of(atoms.size(), -1); + for (int a = 0; a < static_cast(atoms.size()); a++) + if (a % 5 != 4) { + param_of[a] = static_cast(params.size()); + params.push_back(a); + } + const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); + std::mt19937 gen(5); + std::uniform_real_distribution uniform(-1, 1); + for (const bool xyz : {true, false}) { + CAPTURE(xyz); + const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3; + const size_t n = per * params.size(); + // H in full, the X-ray diagonal small beside the stiffest restraints + Eigen::MatrixXd hm = Eigen::MatrixXd::Zero(n, n); + for (size_t r = 0; r + 1 < jac.start.size(); r++) + for (int e1 = jac.start[r]; e1 < jac.start[r + 1]; e1++) + for (int e2 = jac.start[r]; e2 < jac.start[r + 1]; e2++) { + const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]]; + if (k1 >= 0 && k2 >= 0) + for (int a = 0; a < per; a++) + for (int c = 0; c < per; c++) + hm(per * k1 + a, per * k2 + c) += jac.j[e1][c0 + a] * jac.j[e2][c0 + c]; + } + std::vector h(n), b(n); + for (size_t i = 0; i < n; i++) { + h[i] = xyz ? 10 + 5 * uniform(gen) : 0.01 + 0.005 * uniform(gen); + hm(i, i) += h[i]; + b[i] = uniform(gen); + } + const int block = xyz ? 6 : 1; + std::vector start{0}, element; + std::vector lower; + std::vector> llt; + for (size_t first = 0; first < n; first += block) { + const int m = static_cast(std::min(block, n - first)); + llt.emplace_back(hm.block(first, first, m, m)); + const Eigen::MatrixXd l = llt.back().matrixL(); + for (int i = 0; i < m; i++) { + element.push_back(static_cast(first) + i); + for (int k = 0; k <= i; k++) + lower.push_back(l(i, k)); + } + start.push_back(static_cast(element.size())); + } + auto product = [&](const std::vector &v, std::vector &hv) { + const Eigen::VectorXd out = hm * Eigen::Map(v.data(), n); + hv.assign(out.data(), out.data() + n); + }; + auto precondition = [&](const std::vector &r, std::vector &z) { + z.resize(n); + for (size_t g = 0; g < llt.size(); g++) { + const int m = start[g + 1] - start[g]; + const Eigen::VectorXd zg = llt[g].solve(Eigen::Map(r.data() + start[g], m)); + std::copy(zg.data(), zg.data() + m, z.begin() + start[g]); + } + }; + std::vector x; + ConjugateGradients(product, precondition, b, x, 20, 0.0); + RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); + gpu.At(set, xyzb); + gpu.SetSolve(h, start, element, lower); + std::vector x_gpu, x_again; + gpu.Solve(b, x_gpu, 20); + gpu.Solve(b, x_again, 20); + REQUIRE(x_gpu.size() == n); + CHECK(x_gpu == x_again); // a fixed order: the same bits every time + double num = 0, den = 0; + for (size_t i = 0; i < n; i++) { + num += (x_gpu[i] - x[i]) * (x_gpu[i] - x[i]); + den += x[i] * x[i]; + } + CAPTURE(std::sqrt(num / den)); + CHECK(std::sqrt(num / den) < 1e-5); + } +} + TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") { if (get_gpu_count() == 0) return; From a1838b023ab393da5a349f09a4b9c68ab4deca45 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 17 Nov 2026 09:06:53 +0100 Subject: [PATCH 2/4] Refinement on the GPU: the CG preconditioner as explicit block inverses The per-residue Cholesky substitutions ran one thread per residue, a chain of m^2 dependent steps each: on a small model the device solve took 15 ms against the host's few ms (cytc refinement 6.0 -> 9.7 s). Each block's M_g^-1 is now formed on the host in double (from the same LLT, symmetrised) and applied on the card one thread per element, a row of the inverse times the block's part of r in float pairs: cytc 5.1 s, 4ybb 497 -> 344 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 26 +++---- .../RefinementNewtonGPU.cu | 72 +++++++++---------- .../RefinementNewtonGPU.h | 8 +-- tests/RefinementTest.cpp | 11 +-- 4 files changed, 56 insertions(+), 61 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index ac5481cad..875342434 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1195,20 +1195,22 @@ public: ParallelFor(static_cast(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); }); } - // The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and its factor, row by row. - void Factors(std::vector &start, std::vector &element, std::vector &lower) const { + // The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and its inverse, row by row. + void Inverses(std::vector &start, std::vector &element, std::vector &inverse) const { start.assign(1, 0); element.clear(); - lower.clear(); + inverse.clear(); for (size_t g = 0; g < params_.size(); g++) { for (int k : params_[g]) for (int c = 0; c < 3; c++) element.push_back(3 * k + c); start.push_back(static_cast(element.size())); - const Eigen::MatrixXd l = llt_[g].matrixL(); - for (int i = 0; i < l.rows(); i++) - for (int k = 0; k <= i; k++) - lower.push_back(l(i, k)); + const int m = 3 * static_cast(params_[g].size()); + Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m)); + w = (0.5 * (w + w.transpose())).eval(); // symmetric to the last bit, as CG needs M + for (int i = 0; i < m; i++) + for (int k = 0; k < m; k++) + inverse.push_back(w(i, k)); } } @@ -1271,20 +1273,20 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser & if (xyz) residues = std::make_unique(ctx, target.Atoms(), set, h); #ifdef JFJOCH_USE_CUDA - if (normal.GPU()) { // the blocks, or the diagonal as blocks of one by its square root + if (normal.GPU()) { // the blocks, or the diagonal as blocks of one std::vector start, element; - std::vector lower; + std::vector inverse; if (residues) - residues->Factors(start, element, lower); + residues->Inverses(start, element, inverse); else { for (size_t k = 0; k < n; k++) { start.push_back(static_cast(k)); element.push_back(static_cast(k)); - lower.push_back(std::sqrt(diagonal[k])); + inverse.push_back(1.0 / diagonal[k]); } start.push_back(static_cast(n)); } - normal.GPU()->SetSolve(h, start, element, lower); + normal.GPU()->SetSolve(h, start, element, inverse); } #endif } diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index ebc927691..52ebc7250 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -296,28 +296,22 @@ __global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, in p[i] = Add(z[i], Mul(s->beta, p[i])); } -// z = M^-1 r, one thread per block: L y = r forward, then L^T z = y back, y held in z; the factor's diagonal is -// stored as its inverse. -__global__ void BlockSolve(const int *start, const int *element, const int *factor_start, const float2 *factor, - const float2 *r, float2 *z, int blocks) { - const int g = blockIdx.x * blockDim.x + threadIdx.x; - if (g >= blocks) +// z = M^-1 r, one thread per element (q: its place in the blocks' element list, of block g): its row of M_g^-1 times +// the block's part of r. An explicit inverse, not the substitutions of the Cholesky factor: those are a chain of m^2 +// dependent steps per block, a thread's worth of latency the card cannot hide. +__global__ void BlockApply(const int *block_of, const int *start, const int *element, const int *inverse_start, + const float2 *inverse, const float2 *r, float2 *z, int elements) { + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q >= elements) return; - const int *e = element + start[g]; + const int g = block_of[q]; const int m = start[g + 1] - start[g]; - const float2 *l = factor + factor_start[g]; - for (int i = 0; i < m; i++) { - float2 sum = r[e[i]]; - for (int k = 0; k < i; k++) - sum = Add(sum, Neg(Mul(l[i * (i + 1) / 2 + k], z[e[k]]))); - z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]); - } - for (int i = m - 1; i >= 0; i--) { - float2 sum = z[e[i]]; - for (int k = i + 1; k < m; k++) - sum = Add(sum, Neg(Mul(l[k * (k + 1) / 2 + i], z[e[k]]))); - z[e[i]] = Mul(sum, l[i * (i + 1) / 2 + i]); - } + const int *e = element + start[g]; + const float2 *w = inverse + inverse_start[g] + (q - start[g]) * m; + float2 sum = make_float2(0, 0); + for (int k = 0; k < m; k++) + sum = Add(sum, Mul(w[k], r[e[k]])); + z[element[q]] = sum; } float2 Split(double v) { @@ -343,10 +337,10 @@ struct RestraintNormalMatrixGPU::Device { CudaDevicePtr xyzb, j; CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; - // the conjugate gradients: H's X-ray diagonal, M's blocks and factors, the vectors, the dot products' partial sums - int blocks = 0; - CudaDevicePtr h, factor, x, r, z, p, hp, partial; - CudaDevicePtr block_start, block_element, factor_start; + // the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums + int elements = 0; + CudaDevicePtr h, inverse, x, r, z, p, hp, partial; + CudaDevicePtr block_of, block_start, block_element, inverse_start; CudaDevicePtr scalars; PinnedVector host; // v and hv as float pairs PinnedVector host_xyzb; @@ -454,29 +448,27 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { void RestraintNormalMatrixGPU::SetSolve(const std::vector &h, const std::vector &start, - const std::vector &element, const std::vector &lower) { + const std::vector &element, const std::vector &inverse) { DeviceGuard guard(dev_->device); Device &d = *dev_; - std::vector h2(h.size()); + std::vector h2(h.size()), inverse2(inverse.size()); for (size_t i = 0; i < h.size(); i++) h2[i] = Split(h[i]); - d.blocks = static_cast(start.size()) - 1; - std::vector factor_start{0}; - std::vector factor(lower.size()); - for (int g = 0; g < d.blocks; g++) { + for (size_t i = 0; i < inverse.size(); i++) + inverse2[i] = Split(inverse[i]); + std::vector block_of(element.size()), inverse_start{0}; + for (size_t g = 0; g + 1 < start.size(); g++) { const int m = start[g + 1] - start[g]; - for (int i = 0; i < m; i++) - for (int k = 0; k <= i; k++) { - const size_t q = factor_start.back() + i * (i + 1) / 2 + k; - factor[q] = Split(i == k ? 1.0 / lower[q] : lower[q]); - } - factor_start.push_back(factor_start.back() + m * (m + 1) / 2); + std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast(g)); + inverse_start.push_back(inverse_start.back() + m * m); } + d.elements = static_cast(element.size()); d.h = Up(h2, d.stream); - d.factor = Up(factor, d.stream); + d.inverse = Up(inverse2, d.stream); + d.block_of = Up(block_of, d.stream); d.block_start = Up(start, d.stream); d.block_element = Up(element, d.stream); - d.factor_start = Up(factor_start, d.stream); + d.inverse_start = Up(inverse_start, d.stream); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -490,8 +482,8 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector>>(d.block_start, d.block_element, d.factor_start, - d.factor, d.r, d.z, d.blocks); + BlockApply<<>>(d.block_of, d.block_start, d.block_element, + d.inverse_start, d.inverse, d.r, d.z, d.elements); }; auto dot = [&](const float2 *u, const float2 *v, DotUse use) { DotPartial<<>>(u, v, n, d.partial); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h index 3106e10f9..7475b9d1c 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.h +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -35,11 +35,11 @@ public: std::vector Diagonal() const; // The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at - // the point last given to At(), preconditioned by a block-diagonal M given by its Cholesky factors: block g over - // the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, its lower factor L in `lower`, row - // by row (L_ij at i (i + 1) / 2 + j from the block's first). h and M stay until the next SetSolve(). + // the point last given to At(), preconditioned by a block-diagonal M given by the inverses of its blocks: block g + // over the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, its M_g^-1 (symmetric) in + // `inverse`, in full, row by row. h and M stay until the next SetSolve(). void SetSolve(const std::vector &h, const std::vector &start, const std::vector &element, - const std::vector &lower); + const std::vector &inverse); // x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), the vectors on the card // and every sum in float pairs, in a fixed order (RefinementNewtonGPU.cu). void Solve(const std::vector &b, std::vector &x, int iterations); diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index 2d007bda9..34ab0c366 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -1162,16 +1162,17 @@ TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") { } const int block = xyz ? 6 : 1; std::vector start{0}, element; - std::vector lower; + std::vector inverse; std::vector> llt; for (size_t first = 0; first < n; first += block) { const int m = static_cast(std::min(block, n - first)); llt.emplace_back(hm.block(first, first, m, m)); - const Eigen::MatrixXd l = llt.back().matrixL(); + Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m)); + w = (0.5 * (w + w.transpose())).eval(); for (int i = 0; i < m; i++) { element.push_back(static_cast(first) + i); - for (int k = 0; k <= i; k++) - lower.push_back(l(i, k)); + for (int k = 0; k < m; k++) + inverse.push_back(w(i, k)); } start.push_back(static_cast(element.size())); } @@ -1191,7 +1192,7 @@ TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") { ConjugateGradients(product, precondition, b, x, 20, 0.0); RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); gpu.At(set, xyzb); - gpu.SetSolve(h, start, element, lower); + gpu.SetSolve(h, start, element, inverse); std::vector x_gpu, x_again; gpu.Solve(b, x_gpu, 20); gpu.Solve(b, x_again, 20); From 3c59c4fff2e277440255f7db25be45dc533f6b24 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 17 Nov 2026 11:38:51 +0100 Subject: [PATCH 3/4] Refinement minimiser: reorthogonalise the conjugate gradients' residuals The normal-matrix solve (20 fixed preconditioned CG iterations per L-BFGS direction) was forward unstable: run in lockstep on right-hand sides 1e-14 apart, two CPU solves in double drifted apart 5-7x per iteration from about the 5th iteration and ended up to 7% apart (myoglobin: 17% of the position solves above 1e-5). That is the loss of orthogonality of the recurrence in rounding once its first Ritz values converge, and it is what set the CPU and the GPU apart: the device solve matched the host to 1e-14 until the first such call and then differed by exactly that spread. It is not small curvature: an absolute floor on the X-ray diagonal (1e-3 x median, the review's remedy) changed nothing, and blocks of up to 8 residues in the preconditioner hardly anything. Each new residual is now made M^-1-orthogonal to all the earlier ones (modified Gram-Schmidt on r and z together), on the CPU and on the card. Measured on every solve of a refinement run (myoglobin, cytochrome C): a 1e-12 change of q now moves the result by 2e-12 (max 2e-11), was median 2e-7 / max 8e-2; device vs host median 4e-14, max 1e-11, was p90 2e-3 / max 6e-2. The residual after 20 iterations is the same (mean |r20|/|r0| 0.016 vs 0.017). Costs 2 x 20 stored vectors per solve. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- docs/ACKNOWLEDGEMENT.md | 7 ++- docs/CPU_DATA_ANALYSIS.md | 1 + docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 8 ++- .../RefinementMinimiser.h | 18 +++++++ .../RefinementNewtonGPU.cu | 54 +++++++++++++++---- .../RefinementNewtonGPU.h | 5 +- 6 files changed, 80 insertions(+), 13 deletions(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 8359d48d4..aa2328e36 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -603,7 +603,12 @@ initial inverse Hessian is the target's sparse normal matrix - the X-ray target' on the diagonal and the restraints' Gauss-Newton matrix in full - as REFMAC refines on its normal matrix (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients: M. R. Hestenes and E. Stiefel, "Methods of conjugate gradients for solving linear systems" (1952), J. Res. Natl. Bur. -Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). Each atom's +Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044), each residual +made orthogonal again to the earlier ones against the loss of orthogonality in rounding described by +C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix" (1976), +J. Inst. Maths Applics 18, 341-349 (DOI not verified here), and A. Greenbaum, "Behavior of +slightly perturbed Lanczos and conjugate-gradient recurrences" (1989), Linear Algebra Appl. 113, 7-63 +[doi:10.1016/0024-3795(89)90285-1](https://doi.org/10.1016/0024-3795(89)90285-1). Each atom's diagonal is raised to hold its step within the step bound, a trust region per atom after the damping of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" (1944), Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and D. W. diff --git a/docs/CPU_DATA_ANALYSIS.md b/docs/CPU_DATA_ANALYSIS.md index 503030a2f..84af30f96 100644 --- a/docs/CPU_DATA_ANALYSIS.md +++ b/docs/CPU_DATA_ANALYSIS.md @@ -73,6 +73,7 @@ The methods draw on, and in places reimplement, solutions from: - D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782 (the two-loop recursion of the minimiser, §15.6). - M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6). +- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the reorthogonalised conjugate gradients of the minimiser, §15.6). - G. N. Murshudov, P. Skubák, A. A. Lebedev et al., "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (the likelihood's D from the working set and its variance from the free, and the Kullback-Leibler B restraints, §15.3, §15.5). - P. Roversi, E. Blanc, S. Johnson & S. M. Lea, "Tetartohedral twinning could happen to you too", *Acta Cryst.* **D68** (2012), 418-424 (twin refinement with more than two domains, and the free set closed under the twin domains, §15.12). - K. A. Sultana, I. Alexeev, I. Kursula, P. Mäntsälä, J. Niemi & G. Schneider, "Structure determination by multiwavelength anomalous diffraction of aclacinomycin oxidoreductase: indications of multidomain pseudomerohedral twinning", *Acta Cryst.* **D63** (2007), 149-159 (a six-domain twin of a monoclinic crystal on a hexagonal metric, §15.12). diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 225816d63..6fde51f9f 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -320,7 +320,12 @@ leaves out. conjugate-gradient iterations - not to a tolerance, whose iteration count, and with it the direction, a rounding difference can flip - preconditioned per residue for the positions (the residue's own $J^\mathsf{T}J$ and X-ray diagonal, by Cholesky: the stiff bonds and angles inside a residue are solved there, what couples residues is - left to the iterations) and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser + left to the iterations) and by the diagonal elsewhere. Each new residual is made orthogonal again (in the + preconditioner's inner product) to all the earlier ones: the plain recurrence loses that orthogonality in + rounding once its first Ritz values converge (Paige; Greenbaum), and from then on a rounding difference grew + several-fold per iteration, so that two solves of right-hand sides $10^{-14}$ apart ended up to 7% apart - which + is what set the CPU and GPU results apart. With it the solve is a smooth function of its input, and the GPU's + float-pair solve agrees with the CPU's to $10^{-13}$. The Jacobian is taken again at each point the minimiser stands on. - **Its memory is kept from one macro-cycle to the next.** Each weight trial starts with an empty memory. @@ -790,6 +795,7 @@ What the refinement does not do yet - each a known gap, not a design choice: - N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10). - J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6). - M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6). +- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the loss of orthogonality the minimiser's conjugate gradients are kept from, §15.6). - K. Levenberg, "A method for the solution of certain non-linear problems in least squares", *Q. Appl. Math.* **2** (1944), 164-168, and D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters", *J. Soc. Ind. Appl. Math.* **11** (1963), 431-441 (the minimiser's trust floor per atom, §15.6). - A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7). - P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7). diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 6bedb58fc..0cfa14d4b 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -172,6 +172,12 @@ private: // z with H z = b to `tolerance` of |b| (at most `max_iterations`), by conjugate gradients preconditioned with M: // hv = H v (product), z = M^-1 r (precondition), both symmetric positive definite. Returns the iterations taken. // Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436 +// Each residual is made M^-1-orthogonal again to all the earlier ones (modified Gram-Schmidt, r and z = M^-1 r +// together). Without it the recurrence loses that orthogonality in rounding once the first Ritz values converge, and +// from then on amplifies a rounding difference several times per iteration: on the refinement's normal matrix two +// solves of right-hand sides 1e-14 apart ended up to 7% apart after 20 iterations, which set CPU and GPU apart. +// The loss of orthogonality after Paige (1976) J. Inst. Maths Applics 18, 341-349 and Greenbaum (1989) Linear +// Algebra Appl. 113, 7-63 inline int ConjugateGradients(const std::function &v, std::vector &hv)> &product, const std::function &r, std::vector &z)> &precondition, const std::vector &b, std::vector &x, int max_iterations, double tolerance, @@ -183,6 +189,8 @@ inline int ConjugateGradients(const std::function precondition(r, z); p = z; double rz = dot(r, z); + std::vector> rs{r}, zs{z}; // the residuals so far, and M^-1 of each + std::vector rzs{rz}; const double r0 = std::sqrt(dot(r, r)); int k = 0; while (k < max_iterations) { @@ -199,9 +207,19 @@ inline int ConjugateGradients(const std::function if (std::sqrt(dot(r, r)) < tolerance * r0) break; precondition(r, z); + for (size_t i = 0; i < rs.size(); i++) { + const double c = dot(r, zs[i]) / rzs[i]; + refine_vec::Each(n, nthreads, [&](size_t j) { + r[j] -= c * rs[i][j]; + z[j] -= c * zs[i][j]; + }); + } const double rz_new = dot(r, z); refine_vec::Each(n, nthreads, [&](size_t j) { p[j] = z[j] + rz_new / rz * p[j]; }); rz = rz_new; + rs.push_back(r); + zs.push_back(z); + rzs.push_back(rz); } return k; } diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index 52ebc7250..cd93c9c38 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -228,7 +228,7 @@ __global__ void DiagonalKernel(const int *entry_start, const int *entry, const f // The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta struct CGScalars { - float2 rz, a, beta; + float2 rz, a, beta, c; // c: the residual's component along an earlier one int done; // p H p was not positive: x stays where it is }; @@ -256,10 +256,11 @@ __global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *part partial[blockIdx.x] = block; } -enum class DotUse { FirstRz, PHp, Rz }; +enum class DotUse { FirstRz, PHp, Rz, Component }; -// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta) -__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use) { +// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp), as the next r M^-1 r (beta), each +// r M^-1 r kept as rz_of[j], or as r . z_j (c = r . z_j / rz_of[j]) +__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) { __shared__ float2 shared[DOT_BLOCKS]; const float2 dot = BlockSum(shared, partial[threadIdx.x]); if (threadIdx.x != 0) @@ -267,6 +268,7 @@ __global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use) { switch (use) { case DotUse::FirstRz: s->rz = dot; + rz_of[j] = dot; break; case DotUse::PHp: s->a = Div(s->rz, dot); @@ -276,6 +278,10 @@ __global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use) { case DotUse::Rz: s->beta = Div(dot, s->rz); s->rz = dot; + rz_of[j] = dot; + break; + case DotUse::Component: + s->c = Div(dot, rz_of[j]); break; } } @@ -289,6 +295,15 @@ __global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, floa r[i] = Add(r[i], Neg(Mul(s->a, hp[i]))); } +// r -= c r_j, z -= c z_j +__global__ void Project(const CGScalars *s, const float2 *rj, const float2 *zj, float2 *r, float2 *z, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) + return; + r[i] = Add(r[i], Neg(Mul(s->c, rj[i]))); + z[i] = Add(z[i], Neg(Mul(s->c, zj[i]))); +} + // p = z + beta p __global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -340,6 +355,8 @@ struct RestraintNormalMatrixGPU::Device { // the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums int elements = 0; CudaDevicePtr h, inverse, x, r, z, p, hp, partial; + CudaDevicePtr rs, zs, rz_of; // the residuals of a solve so far, M^-1 of each, r M^-1 r of each + int kept = 0; // how many of them rs and zs hold room for CudaDevicePtr block_of, block_start, block_element, inverse_start; CudaDevicePtr scalars; PinnedVector host; // v and hv as float pairs @@ -485,9 +502,21 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector>>(d.block_of, d.block_start, d.block_element, d.inverse_start, d.inverse, d.r, d.z, d.elements); }; - auto dot = [&](const float2 *u, const float2 *v, DotUse use) { + auto dot = [&](const float2 *u, const float2 *v, DotUse use, int j) { DotPartial<<>>(u, v, n, d.partial); - DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use); + DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j); + }; + if (d.kept < iterations) { + d.rs = CudaDevicePtr(static_cast(iterations) * n); + d.zs = CudaDevicePtr(static_cast(iterations) * n); + d.rz_of = CudaDevicePtr(iterations); + d.kept = iterations; + } + auto keep = [&](int j) { // r and z as the j-th + cuda_err(cudaMemcpyAsync(d.rs.get() + static_cast(j) * n, d.r.get(), n * sizeof(float2), + cudaMemcpyDeviceToDevice, d.stream)); + cuda_err(cudaMemcpyAsync(d.zs.get() + static_cast(j) * n, d.z.get(), n * sizeof(float2), + cudaMemcpyDeviceToDevice, d.stream)); }; for (int i = 0; i < n; i++) d.host[i] = Split(b[i]); @@ -496,18 +525,25 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows); TransposeProduct<<(n_params_)), THREADS, 0, d.stream>>>( d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, d.h, d.p, d.hp, static_cast(n_params_)); - dot(d.p, d.hp, DotUse::PHp); + dot(d.p, d.hp, DotUse::PHp, 0); Step<<>>(d.scalars, d.p, d.hp, d.x, d.r, n); if (k + 1 < iterations) { // the last direction would not be used precondition(); - dot(d.r, d.z, DotUse::Rz); + for (int j = 0; j <= k; j++) { // M^-1-orthogonal to the earlier residuals, as ConjugateGradients + const float2 *rj = d.rs.get() + static_cast(j) * n, *zj = d.zs.get() + static_cast(j) * n; + dot(d.r, zj, DotUse::Component, j); + Project<<>>(d.scalars, rj, zj, d.r, d.z, n); + } + dot(d.r, d.z, DotUse::Rz, k + 1); NextDirection<<>>(d.scalars, d.z, d.p, n); + keep(k + 1); } } cuda_err(cudaGetLastError()); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h index 7475b9d1c..bbd30ed4a 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.h +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -40,8 +40,9 @@ public: // `inverse`, in full, row by row. h and M stay until the next SetSolve(). void SetSolve(const std::vector &h, const std::vector &start, const std::vector &element, const std::vector &inverse); - // x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), the vectors on the card - // and every sum in float pairs, in a fixed order (RefinementNewtonGPU.cu). + // x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), each residual made + // M^-1-orthogonal to the earlier ones as there; the vectors on the card and every sum in float pairs, in a fixed + // order (RefinementNewtonGPU.cu). void Solve(const std::vector &b, std::vector &x, int iterations); private: From c582a4813104df27a2070831e46b9c363b41b3e0 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Tue, 17 Nov 2026 15:51:17 +0100 Subject: [PATCH 4/4] Refinement on the GPU: the reorthogonalisation in three kernels, and less memory The residuals' reorthogonalisation launched a dot and a projection per earlier residual (about 570 extra launches per solve): a solve took 5.7-10 ms on the small models against 1.5-3.8 ms before it. All coefficients are now taken at once from the same z (classical Gram-Schmidt: c_j = z . r_j / rz_j, on the CPU too), in one partial-sum kernel over all kept residuals, one finish and one projection: 1.9-4.4 ms per solve (myob 3.3, cytc 2.8, 7ris 2.6, 9zlo 1.9, 7l6j 4.4 ms; 4ybb 44 ms, as before the reorthogonalisation). Memory: the per-residue preconditioner keeps only the lower triangle of each block inverse, and the kept M^-1 r_j are in float (c_j comes from r_j, kept in pairs, so float's rounding of z_j enters only as c_j times it, far below the pairs). 4ybb peak GPU memory 14838 -> 14198 MiB (base 12982, without the reorthogonalisation 14038). Device vs host still median 3e-14, max 6e-12 (myob, cytc, every solve); a 1e-12 change of the right-hand side moves the solve by 2e-12. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- .../structure_refinement/Refinement.cpp | 8 +- .../RefinementMinimiser.h | 20 ++-- .../RefinementNewtonGPU.cu | 96 +++++++++++++------ .../RefinementNewtonGPU.h | 5 +- tests/RefinementTest.cpp | 5 +- 5 files changed, 85 insertions(+), 49 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 875342434..beee4c3a3 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1195,7 +1195,8 @@ public: ParallelFor(static_cast(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); }); } - // The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and its inverse, row by row. + // The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and the lower triangle of its + // inverse, row by row. void Inverses(std::vector &start, std::vector &element, std::vector &inverse) const { start.assign(1, 0); element.clear(); @@ -1206,10 +1207,9 @@ public: element.push_back(3 * k + c); start.push_back(static_cast(element.size())); const int m = 3 * static_cast(params_[g].size()); - Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m)); - w = (0.5 * (w + w.transpose())).eval(); // symmetric to the last bit, as CG needs M + const Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m)); for (int i = 0; i < m; i++) - for (int k = 0; k < m; k++) + for (int k = 0; k <= i; k++) inverse.push_back(w(i, k)); } } diff --git a/image_analysis/structure_refinement/RefinementMinimiser.h b/image_analysis/structure_refinement/RefinementMinimiser.h index 0cfa14d4b..ea4487b1c 100644 --- a/image_analysis/structure_refinement/RefinementMinimiser.h +++ b/image_analysis/structure_refinement/RefinementMinimiser.h @@ -172,8 +172,8 @@ private: // z with H z = b to `tolerance` of |b| (at most `max_iterations`), by conjugate gradients preconditioned with M: // hv = H v (product), z = M^-1 r (precondition), both symmetric positive definite. Returns the iterations taken. // Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436 -// Each residual is made M^-1-orthogonal again to all the earlier ones (modified Gram-Schmidt, r and z = M^-1 r -// together). Without it the recurrence loses that orthogonality in rounding once the first Ritz values converge, and +// Each residual is made M^-1-orthogonal again to all the earlier ones (Gram-Schmidt, every coefficient r_j M^-1 r / +// r_j M^-1 r_j = z . r_j / rz_j from the same z, then r and z = M^-1 r corrected together). Without it the recurrence loses that orthogonality in rounding once the first Ritz values converge, and // from then on amplifies a rounding difference several times per iteration: on the refinement's normal matrix two // solves of right-hand sides 1e-14 apart ended up to 7% apart after 20 iterations, which set CPU and GPU apart. // The loss of orthogonality after Paige (1976) J. Inst. Maths Applics 18, 341-349 and Greenbaum (1989) Linear @@ -207,13 +207,15 @@ inline int ConjugateGradients(const std::function if (std::sqrt(dot(r, r)) < tolerance * r0) break; precondition(r, z); - for (size_t i = 0; i < rs.size(); i++) { - const double c = dot(r, zs[i]) / rzs[i]; - refine_vec::Each(n, nthreads, [&](size_t j) { - r[j] -= c * rs[i][j]; - z[j] -= c * zs[i][j]; - }); - } + std::vector c(rs.size()); + for (size_t i = 0; i < rs.size(); i++) + c[i] = dot(z, rs[i]) / rzs[i]; + refine_vec::Each(n, nthreads, [&](size_t j) { + for (size_t i = 0; i < rs.size(); i++) { + r[j] -= c[i] * rs[i][j]; + z[j] -= c[i] * zs[i][j]; + } + }); const double rz_new = dot(r, z); refine_vec::Each(n, nthreads, [&](size_t j) { p[j] = z[j] + rz_new / rz * p[j]; }); rz = rz_new; diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index cd93c9c38..99e58e473 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -228,7 +228,7 @@ __global__ void DiagonalKernel(const int *entry_start, const int *entry, const f // The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta struct CGScalars { - float2 rz, a, beta, c; // c: the residual's component along an earlier one + float2 rz, a, beta; int done; // p H p was not positive: x stays where it is }; @@ -256,10 +256,10 @@ __global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *part partial[blockIdx.x] = block; } -enum class DotUse { FirstRz, PHp, Rz, Component }; +enum class DotUse { FirstRz, PHp, Rz }; -// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp), as the next r M^-1 r (beta), each -// r M^-1 r kept as rz_of[j], or as r . z_j (c = r . z_j / rz_of[j]) +// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta); each +// r M^-1 r kept as rz_of[j] __global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) { __shared__ float2 shared[DOT_BLOCKS]; const float2 dot = BlockSum(shared, partial[threadIdx.x]); @@ -280,9 +280,6 @@ __global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float s->rz = dot; rz_of[j] = dot; break; - case DotUse::Component: - s->c = Div(dot, rz_of[j]); - break; } } @@ -295,13 +292,49 @@ __global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, floa r[i] = Add(r[i], Neg(Mul(s->a, hp[i]))); } -// r -= c r_j, z -= c z_j -__global__ void Project(const CGScalars *s, const float2 *rj, const float2 *zj, float2 *r, float2 *z, int n) { +// z . r_j for every kept r_j (j = blockIdx.y), each in DOT_BLOCKS partial sums as DotPartial +__global__ void ComponentPartial(const float2 *z, const float2 *rs, int n, float2 *partial) { + __shared__ float2 shared[THREADS]; + const float2 *rj = rs + static_cast(blockIdx.y) * n; + float2 sum = make_float2(0, 0); + for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS) + sum = Add(sum, Mul(z[i], rj[i])); + const float2 block = BlockSum(shared, sum); + if (threadIdx.x == 0) + partial[blockIdx.y * DOT_BLOCKS + blockIdx.x] = block; +} + +// c_j = z . r_j / rz_of[j], block j +__global__ void ComponentFinish(const float2 *partial, const float2 *rz_of, float2 *c) { + __shared__ float2 shared[DOT_BLOCKS]; + const float2 dot = BlockSum(shared, partial[blockIdx.x * DOT_BLOCKS + threadIdx.x]); + if (threadIdx.x == 0) + c[blockIdx.x] = Div(dot, rz_of[blockIdx.x]); +} + +// r -= sum_j c_j r_j, z -= sum_j c_j z_j, over the `kept` earlier residuals in order. z_j is kept in float: c_j is +// the orthogonality lost in rounding, so what float leaves of c_j z_j is far below the pairs' precision (c_j itself +// is from r_j, in pairs). +__global__ void Project(const float2 *c, int kept, const float2 *rs, const float *zs, float2 *r, float2 *z, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; - r[i] = Add(r[i], Neg(Mul(s->c, rj[i]))); - z[i] = Add(z[i], Neg(Mul(s->c, zj[i]))); + float2 ri = r[i], zi = z[i]; + for (int j = 0; j < kept; j++) { + ri = Add(ri, Neg(Mul(c[j], rs[static_cast(j) * n + i]))); + zi = Add(zi, Neg(Mul(c[j], make_float2(zs[static_cast(j) * n + i], 0.0f)))); + } + r[i] = ri; + z[i] = zi; +} + +// r and z kept as the j-th: r in pairs, z in float +__global__ void Keep(const float2 *r, const float2 *z, float2 *rj, float *zj, int n) { + const int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= n) + return; + rj[i] = r[i]; + zj[i] = z[i].x; } // p = z + beta p @@ -322,10 +355,11 @@ __global__ void BlockApply(const int *block_of, const int *start, const int *ele const int g = block_of[q]; const int m = start[g + 1] - start[g]; const int *e = element + start[g]; - const float2 *w = inverse + inverse_start[g] + (q - start[g]) * m; + const float2 *w = inverse + inverse_start[g]; // the lower triangle, W_ik at i (i + 1) / 2 + k for k <= i + const int i = q - start[g]; float2 sum = make_float2(0, 0); for (int k = 0; k < m; k++) - sum = Add(sum, Mul(w[k], r[e[k]])); + sum = Add(sum, Mul(k <= i ? w[i * (i + 1) / 2 + k] : w[k * (k + 1) / 2 + i], r[e[k]])); z[element[q]] = sum; } @@ -355,8 +389,9 @@ struct RestraintNormalMatrixGPU::Device { // the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums int elements = 0; CudaDevicePtr h, inverse, x, r, z, p, hp, partial; - CudaDevicePtr rs, zs, rz_of; // the residuals of a solve so far, M^-1 of each, r M^-1 r of each - int kept = 0; // how many of them rs and zs hold room for + CudaDevicePtr rs, rz_of, c, components; // the residuals of a solve so far, r M^-1 r of each, the + CudaDevicePtr zs; // coefficients and their partial sums; M^-1 of each residual + int kept = 0; // how many residuals rs and zs hold room for CudaDevicePtr block_of, block_start, block_element, inverse_start; CudaDevicePtr scalars; PinnedVector host; // v and hv as float pairs @@ -477,7 +512,7 @@ void RestraintNormalMatrixGPU::SetSolve(const std::vector &h, const std: for (size_t g = 0; g + 1 < start.size(); g++) { const int m = start[g + 1] - start[g]; std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast(g)); - inverse_start.push_back(inverse_start.back() + m * m); + inverse_start.push_back(inverse_start.back() + m * (m + 1) / 2); } d.elements = static_cast(element.size()); d.h = Up(h2, d.stream); @@ -498,6 +533,14 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector(static_cast(iterations) * n); + d.zs = CudaDevicePtr(static_cast(iterations) * n); + d.rz_of = CudaDevicePtr(iterations); + d.c = CudaDevicePtr(iterations); + d.components = CudaDevicePtr(static_cast(iterations) * DOT_BLOCKS); + d.kept = iterations; + } auto precondition = [&]() { // z = M^-1 r BlockApply<<>>(d.block_of, d.block_start, d.block_element, d.inverse_start, d.inverse, d.r, d.z, d.elements); @@ -506,17 +549,9 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector>>(u, v, n, d.partial); DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j); }; - if (d.kept < iterations) { - d.rs = CudaDevicePtr(static_cast(iterations) * n); - d.zs = CudaDevicePtr(static_cast(iterations) * n); - d.rz_of = CudaDevicePtr(iterations); - d.kept = iterations; - } auto keep = [&](int j) { // r and z as the j-th - cuda_err(cudaMemcpyAsync(d.rs.get() + static_cast(j) * n, d.r.get(), n * sizeof(float2), - cudaMemcpyDeviceToDevice, d.stream)); - cuda_err(cudaMemcpyAsync(d.zs.get() + static_cast(j) * n, d.z.get(), n * sizeof(float2), - cudaMemcpyDeviceToDevice, d.stream)); + Keep<<>>(d.r, d.z, d.rs.get() + static_cast(j) * n, + d.zs.get() + static_cast(j) * n, n); }; for (int i = 0; i < n; i++) d.host[i] = Split(b[i]); @@ -536,11 +571,10 @@ void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector>>(d.scalars, d.p, d.hp, d.x, d.r, n); if (k + 1 < iterations) { // the last direction would not be used precondition(); - for (int j = 0; j <= k; j++) { // M^-1-orthogonal to the earlier residuals, as ConjugateGradients - const float2 *rj = d.rs.get() + static_cast(j) * n, *zj = d.zs.get() + static_cast(j) * n; - dot(d.r, zj, DotUse::Component, j); - Project<<>>(d.scalars, rj, zj, d.r, d.z, n); - } + // M^-1-orthogonal to the k + 1 earlier residuals, as ConjugateGradients + ComponentPartial<<>>(d.z, d.rs, n, d.components); + ComponentFinish<<>>(d.components, d.rz_of, d.c); + Project<<>>(d.c, k + 1, d.rs, d.zs, d.r, d.z, n); dot(d.r, d.z, DotUse::Rz, k + 1); NextDirection<<>>(d.scalars, d.z, d.p, n); keep(k + 1); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.h b/image_analysis/structure_refinement/RefinementNewtonGPU.h index bbd30ed4a..2b07898f3 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.h +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.h @@ -36,8 +36,9 @@ public: // The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at // the point last given to At(), preconditioned by a block-diagonal M given by the inverses of its blocks: block g - // over the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, its M_g^-1 (symmetric) in - // `inverse`, in full, row by row. h and M stay until the next SetSolve(). + // over the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, the lower triangle of its + // M_g^-1 (symmetric) in `inverse`, row by row (W_ik, k <= i, at i (i + 1) / 2 + k from the block's first). h and M + // stay until the next SetSolve(). void SetSolve(const std::vector &h, const std::vector &start, const std::vector &element, const std::vector &inverse); // x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), each residual made diff --git a/tests/RefinementTest.cpp b/tests/RefinementTest.cpp index 34ab0c366..d5f09d955 100644 --- a/tests/RefinementTest.cpp +++ b/tests/RefinementTest.cpp @@ -1167,11 +1167,10 @@ TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") { for (size_t first = 0; first < n; first += block) { const int m = static_cast(std::min(block, n - first)); llt.emplace_back(hm.block(first, first, m, m)); - Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m)); - w = (0.5 * (w + w.transpose())).eval(); + const Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m)); for (int i = 0; i < m; i++) { element.push_back(static_cast(first) + i); - for (int k = 0; k < m; k++) + for (int k = 0; k <= i; k++) inverse.push_back(w(i, k)); } start.push_back(static_cast(element.size()));