Merge branch 'rc188-g' into rc188-int

# Conflicts:
#	image_analysis/structure_refinement/RefinementNewtonGPU.cu
This commit is contained in:
leonarski_f committed 2026-11-17 16:01:35 +01:00
commit 809e178ef9
8 files changed
+486 -19

No files matched your search

+6 -1
View File
@@ -603,7 +603,12 @@ initial inverse Hessian is the target's sparse normal matrix - the X-ray target'
on the diagonal and the restraints' Gauss-Newton matrix in full - as REFMAC refines on its normal
matrix (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients: M. R. Hestenes and
E. Stiefel, "Methods of conjugate gradients for solving linear systems" (1952), J. Res. Natl. Bur.
Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). Each atom's
Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044), each residual
made orthogonal again to the earlier ones against the loss of orthogonality in rounding described by
C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix" (1976),
J. Inst. Maths Applics 18, 341-349 (DOI not verified here), and A. Greenbaum, "Behavior of
slightly perturbed Lanczos and conjugate-gradient recurrences" (1989), Linear Algebra Appl. 113, 7-63
[doi:10.1016/0024-3795(89)90285-1](https://doi.org/10.1016/0024-3795(89)90285-1). Each atom's
diagonal is raised to hold its step within the step bound, a trust region per atom after the damping
of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" (1944),
Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and D. W.
+1
View File
@@ -73,6 +73,7 @@ The methods draw on, and in places reimplement, solutions from:
- D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15).
- J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782 (the two-loop recursion of the minimiser, §15.6).
- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6).
- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the reorthogonalised conjugate gradients of the minimiser, §15.6).
- G. N. Murshudov, P. Skubák, A. A. Lebedev et al., "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (the likelihood's D from the working set and its variance from the free, and the Kullback-Leibler B restraints, §15.3, §15.5).
- P. Roversi, E. Blanc, S. Johnson & S. M. Lea, "Tetartohedral twinning could happen to you too", *Acta Cryst.* **D68** (2012), 418-424 (twin refinement with more than two domains, and the free set closed under the twin domains, §15.12).
- K. A. Sultana, I. Alexeev, I. Kursula, P. Mäntsälä, J. Niemi & G. Schneider, "Structure determination by multiwavelength anomalous diffraction of aclacinomycin oxidoreductase: indications of multidomain pseudomerohedral twinning", *Acta Cryst.* **D63** (2007), 149-159 (a six-domain twin of a monoclinic crystal on a hexagonal metric, §15.12).
+7 -1
View File
@@ -320,7 +320,12 @@ leaves out.
conjugate-gradient iterations - not to a tolerance, whose iteration count, and with it the direction, a rounding
difference can flip - preconditioned per residue for the positions (the residue's own $J^\mathsf{T}J$ and X-ray
diagonal, by Cholesky: the stiff bonds and angles inside a residue are solved there, what couples residues is
left to the iterations) and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser
left to the iterations) and by the diagonal elsewhere. Each new residual is made orthogonal again (in the
preconditioner's inner product) to all the earlier ones: the plain recurrence loses that orthogonality in
rounding once its first Ritz values converge (Paige; Greenbaum), and from then on a rounding difference grew
several-fold per iteration, so that two solves of right-hand sides $10^{-14}$ apart ended up to 7% apart - which
is what set the CPU and GPU results apart. With it the solve is a smooth function of its input, and the GPU's
float-pair solve agrees with the CPU's to $10^{-13}$. The Jacobian is taken again at each point the minimiser
stands on.
- **Its memory is kept from one macro-cycle to the next.** Each weight trial starts with an empty memory.
@@ -790,6 +795,7 @@ What the refinement does not do yet - each a known gap, not a design choice:
- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10).
- J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6).
- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6).
- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the loss of orthogonality the minimiser's conjugate gradients are kept from, §15.6).
- K. Levenberg, "A method for the solution of certain non-linear problems in least squares", *Q. Appl. Math.* **2** (1944), 164-168, and D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters", *J. Soc. Ind. Appl. Math.* **11** (1963), 431-441 (the minimiser's trust floor per atom, §15.6).
- A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7).
- P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7).
@@ -1013,20 +1013,19 @@ std::vector<double> FlooredDiagonal(std::vector<double> d) {
constexpr int NORMAL_CG_ITERATIONS = 20;
// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q)
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then hv = H v
// (product) and z = M^-1 r for a preconditioner M of it (precondition).
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then z = H^-1 q by
// NORMAL_CG_ITERATIONS conjugate gradients (solve: ConjugateGradients on the CPU, or on the card).
template <typename Target>
void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations,
const std::function<void(const std::vector<double> &, const std::vector<double> &)> &at,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
const std::function<void(const std::vector<double> &, std::vector<double> &)> &solve) {
std::vector<double> p = target.Get();
if (p.empty())
return;
minimiser.SetInverseHessian([&](const std::vector<double> &x, std::vector<double> &q) {
at(x, q);
std::vector<double> z;
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
solve(q, z);
q.swap(z);
});
minimiser.Minimise([&](const std::vector<double> &x, double &f, std::vector<double> &g) {
@@ -1037,6 +1036,15 @@ void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMini
target.Set(p.data()); // where the minimiser ended, not where its last line search probed
}
// The conjugate gradients on the CPU: hv = H v (product) and z = M^-1 r for a preconditioner M of H (precondition).
std::function<void(const std::vector<double> &, std::vector<double> &)>
NormalSolve(const Context &ctx, const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
return [&ctx, product, precondition](const std::vector<double> &q, std::vector<double> &z) {
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
}
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
// a GPU, on its card (RestraintNormalMatrixGPU).
@@ -1113,6 +1121,11 @@ public:
});
}
#ifdef JFJOCH_USE_CUDA
// The conjugate gradients on the card, the vectors kept there (RestraintNormalMatrixGPU::Solve)
RestraintNormalMatrixGPU *GPU() { return gpu_.get(); }
#endif
std::vector<double> Diagonal() const {
#ifdef JFJOCH_USE_CUDA
if (gpu_)
@@ -1182,6 +1195,25 @@ public:
ParallelFor(static_cast<int>(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); });
}
// The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and the lower triangle of its
// inverse, row by row.
void Inverses(std::vector<int> &start, std::vector<int> &element, std::vector<double> &inverse) const {
start.assign(1, 0);
element.clear();
inverse.clear();
for (size_t g = 0; g < params_.size(); g++) {
for (int k : params_[g])
for (int c = 0; c < 3; c++)
element.push_back(3 * k + c);
start.push_back(static_cast<int>(element.size()));
const int m = 3 * static_cast<int>(params_[g].size());
const Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m));
for (int i = 0; i < m; i++)
for (int k = 0; k <= i; k++)
inverse.push_back(w(i, k));
}
}
void Apply(const std::vector<double> &r, std::vector<double> &z, size_t nthreads) const {
z.resize(r.size());
ParallelFor(static_cast<int>(params_.size()), nthreads, [&](int g) {
@@ -1240,6 +1272,23 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
diagonal = FlooredDiagonal(diagonal);
if (xyz)
residues = std::make_unique<ResidueBlocks>(ctx, target.Atoms(), set, h);
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) { // the blocks, or the diagonal as blocks of one
std::vector<int> start, element;
std::vector<double> inverse;
if (residues)
residues->Inverses(start, element, inverse);
else {
for (size_t k = 0; k < n; k++) {
start.push_back(static_cast<int>(k));
element.push_back(static_cast<int>(k));
inverse.push_back(1.0 / diagonal[k]);
}
start.push_back(static_cast<int>(n));
}
normal.GPU()->SetSolve(h, start, element, inverse);
}
#endif
}
if (x != x_at) {
x_at = x;
@@ -1259,7 +1308,16 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
}
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
auto solve = [&](const std::vector<double> &q, std::vector<double> &z) {
#ifdef JFJOCH_USE_CUDA
if (normal.GPU()) {
const RefinementStage stage("normal.solve");
return normal.GPU()->Solve(q, z, NORMAL_CG_ITERATIONS);
}
#endif
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, solve);
}
// Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their
@@ -1306,7 +1364,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim
z.resize(n);
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's
@@ -1364,7 +1422,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min
for (size_t k = 0; k < r.size(); k++)
z[k] = r[k] / diagonal[k];
};
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
}
// Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis
@@ -172,6 +172,12 @@ private:
// z with H z = b to `tolerance` of |b| (at most `max_iterations`), by conjugate gradients preconditioned with M:
// hv = H v (product), z = M^-1 r (precondition), both symmetric positive definite. Returns the iterations taken.
// Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436
// Each residual is made M^-1-orthogonal again to all the earlier ones (Gram-Schmidt, every coefficient r_j M^-1 r /
// r_j M^-1 r_j = z . r_j / rz_j from the same z, then r and z = M^-1 r corrected together). Without it the recurrence loses that orthogonality in rounding once the first Ritz values converge, and
// from then on amplifies a rounding difference several times per iteration: on the refinement's normal matrix two
// solves of right-hand sides 1e-14 apart ended up to 7% apart after 20 iterations, which set CPU and GPU apart.
// The loss of orthogonality after Paige (1976) J. Inst. Maths Applics 18, 341-349 and Greenbaum (1989) Linear
// Algebra Appl. 113, 7-63
inline int ConjugateGradients(const std::function<void(const std::vector<double> &v, std::vector<double> &hv)> &product,
const std::function<void(const std::vector<double> &r, std::vector<double> &z)> &precondition,
const std::vector<double> &b, std::vector<double> &x, int max_iterations, double tolerance,
@@ -183,6 +189,8 @@ inline int ConjugateGradients(const std::function<void(const std::vector<double>
precondition(r, z);
p = z;
double rz = dot(r, z);
std::vector<std::vector<double>> rs{r}, zs{z}; // the residuals so far, and M^-1 of each
std::vector<double> rzs{rz};
const double r0 = std::sqrt(dot(r, r));
int k = 0;
while (k < max_iterations) {
@@ -199,9 +207,21 @@ inline int ConjugateGradients(const std::function<void(const std::vector<double>
if (std::sqrt(dot(r, r)) < tolerance * r0)
break;
precondition(r, z);
std::vector<double> c(rs.size());
for (size_t i = 0; i < rs.size(); i++)
c[i] = dot(z, rs[i]) / rzs[i];
refine_vec::Each(n, nthreads, [&](size_t j) {
for (size_t i = 0; i < rs.size(); i++) {
r[j] -= c[i] * rs[i][j];
z[j] -= c[i] * zs[i][j];
}
});
const double rz_new = dot(r, z);
refine_vec::Each(n, nthreads, [&](size_t j) { p[j] = z[j] + rz_new / rz * p[j]; });
rz = rz_new;
rs.push_back(r);
zs.push_back(z);
rzs.push_back(rz);
}
return k;
}
@@ -17,6 +17,7 @@
namespace {
constexpr int THREADS = 256;
constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order
using V3f = refine_math::V3<float>;
void cuda_err(cudaError_t val) {
@@ -69,6 +70,34 @@ __device__ void AddProduct(float2 &sum, float a, float2 b) {
sum = make_float2(hi, lo - (hi - s));
}
// The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b.
__device__ float2 Normalised(float s, float e) {
const float hi = s + e;
return make_float2(hi, e - (hi - s));
}
__device__ float2 Add(float2 a, float2 b) {
const float s = a.x + b.x;
const float v = s - a.x;
const float e = (a.x - (s - v)) + (b.x - v);
return Normalised(s, e + a.y + b.y);
}
__device__ float2 Mul(float2 a, float2 b) {
const float p = a.x * b.x;
return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x));
}
__device__ float2 Neg(float2 a) {
return make_float2(-a.x, -a.y);
}
__device__ float2 Div(float2 a, float2 b) {
const float q = a.x / b.x;
const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b)));
return Normalised(q, rest.x / b.x);
}
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
@@ -174,10 +203,10 @@ __global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int en
jt[q] = j[entry[q]];
}
// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components,
// each component summed over the entries in their order
// hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given: each entry read once for
// all of the atom's components, each component summed over the entries in their order
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
const float2 *t, float2 *hv, int n) {
const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
@@ -189,7 +218,7 @@ __global__ void TransposeProduct(const int *entry_start, const float4 *jt, const
AddProduct(sum[c], Get(jq, c0 + c), tr);
}
for (int c = 0; c < per; c++)
hv[per * k + c] = sum[c];
hv[per * k + c] = h ? Add(sum[c], Mul(h[per * k + c], v[per * k + c])) : sum[c];
}
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
@@ -206,6 +235,148 @@ __global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per
}
}
// The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta
struct CGScalars {
float2 rz, a, beta;
int done; // p H p was not positive: x stays where it is
};
// The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size).
__device__ float2 BlockSum(float2 *shared, float2 value) {
const int t = threadIdx.x;
shared[t] = value;
__syncthreads();
for (int w = blockDim.x / 2; w > 0; w /= 2) {
if (t < w)
shared[t] = Add(shared[t], shared[t + w]);
__syncthreads();
}
return shared[0];
}
// u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements
__global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) {
__shared__ float2 shared[THREADS];
float2 sum = make_float2(0, 0);
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
sum = Add(sum, Mul(u[i], v[i]));
const float2 block = BlockSum(shared, sum);
if (threadIdx.x == 0)
partial[blockIdx.x] = block;
}
enum class DotUse { FirstRz, PHp, Rz };
// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta); each
// r M^-1 r kept as rz_of[j]
__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) {
__shared__ float2 shared[DOT_BLOCKS];
const float2 dot = BlockSum(shared, partial[threadIdx.x]);
if (threadIdx.x != 0)
return;
switch (use) {
case DotUse::FirstRz:
s->rz = dot;
rz_of[j] = dot;
break;
case DotUse::PHp:
s->a = Div(s->rz, dot);
if (!(dot.x > 0))
s->done = 1;
break;
case DotUse::Rz:
s->beta = Div(dot, s->rz);
s->rz = dot;
rz_of[j] = dot;
break;
}
}
// x += a p, r -= a H p
__global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n || s->done)
return;
x[i] = Add(x[i], Mul(s->a, p[i]));
r[i] = Add(r[i], Neg(Mul(s->a, hp[i])));
}
// z . r_j for every kept r_j (j = blockIdx.y), each in DOT_BLOCKS partial sums as DotPartial
__global__ void ComponentPartial(const float2 *z, const float2 *rs, int n, float2 *partial) {
__shared__ float2 shared[THREADS];
const float2 *rj = rs + static_cast<size_t>(blockIdx.y) * n;
float2 sum = make_float2(0, 0);
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
sum = Add(sum, Mul(z[i], rj[i]));
const float2 block = BlockSum(shared, sum);
if (threadIdx.x == 0)
partial[blockIdx.y * DOT_BLOCKS + blockIdx.x] = block;
}
// c_j = z . r_j / rz_of[j], block j
__global__ void ComponentFinish(const float2 *partial, const float2 *rz_of, float2 *c) {
__shared__ float2 shared[DOT_BLOCKS];
const float2 dot = BlockSum(shared, partial[blockIdx.x * DOT_BLOCKS + threadIdx.x]);
if (threadIdx.x == 0)
c[blockIdx.x] = Div(dot, rz_of[blockIdx.x]);
}
// r -= sum_j c_j r_j, z -= sum_j c_j z_j, over the `kept` earlier residuals in order. z_j is kept in float: c_j is
// the orthogonality lost in rounding, so what float leaves of c_j z_j is far below the pairs' precision (c_j itself
// is from r_j, in pairs).
__global__ void Project(const float2 *c, int kept, const float2 *rs, const float *zs, float2 *r, float2 *z, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
float2 ri = r[i], zi = z[i];
for (int j = 0; j < kept; j++) {
ri = Add(ri, Neg(Mul(c[j], rs[static_cast<size_t>(j) * n + i])));
zi = Add(zi, Neg(Mul(c[j], make_float2(zs[static_cast<size_t>(j) * n + i], 0.0f))));
}
r[i] = ri;
z[i] = zi;
}
// r and z kept as the j-th: r in pairs, z in float
__global__ void Keep(const float2 *r, const float2 *z, float2 *rj, float *zj, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
rj[i] = r[i];
zj[i] = z[i].x;
}
// p = z + beta p
__global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n)
p[i] = Add(z[i], Mul(s->beta, p[i]));
}
// z = M^-1 r, one thread per element (q: its place in the blocks' element list, of block g): its row of M_g^-1 times
// the block's part of r. An explicit inverse, not the substitutions of the Cholesky factor: those are a chain of m^2
// dependent steps per block, a thread's worth of latency the card cannot hide.
__global__ void BlockApply(const int *block_of, const int *start, const int *element, const int *inverse_start,
const float2 *inverse, const float2 *r, float2 *z, int elements) {
const int q = blockIdx.x * blockDim.x + threadIdx.x;
if (q >= elements)
return;
const int g = block_of[q];
const int m = start[g + 1] - start[g];
const int *e = element + start[g];
const float2 *w = inverse + inverse_start[g]; // the lower triangle, W_ik at i (i + 1) / 2 + k for k <= i
const int i = q - start[g];
float2 sum = make_float2(0, 0);
for (int k = 0; k < m; k++)
sum = Add(sum, Mul(k <= i ? w[i * (i + 1) / 2 + k] : w[k * (k + 1) / 2 + i], r[e[k]]));
z[element[q]] = sum;
}
float2 Split(double v) {
const float hi = static_cast<float>(v);
return make_float2(hi, static_cast<float>(v - hi));
}
template <typename T>
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
@@ -224,6 +395,14 @@ struct RestraintNormalMatrixGPU::Device {
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
// the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums
int elements = 0;
CudaDevicePtr<float2> h, inverse, x, r, z, p, hp, partial;
CudaDevicePtr<float2> rs, rz_of, c, components; // the residuals of a solve so far, r M^-1 r of each, the
CudaDevicePtr<float> zs; // coefficients and their partial sums; M^-1 of each residual
int kept = 0; // how many residuals rs and zs hold room for
CudaDevicePtr<int> block_of, block_start, block_element, inverse_start;
CudaDevicePtr<CGScalars> scalars;
PinnedVector<float2> host; // v and hv as float pairs
PinnedVector<float4> host_xyzb;
};
@@ -243,6 +422,10 @@ RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector
dev_->v = CudaDevicePtr<float2>(n);
dev_->hv = CudaDevicePtr<float2>(n);
dev_->diagonal = CudaDevicePtr<float>(n);
for (CudaDevicePtr<float2> *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp})
*u = CudaDevicePtr<float2>(n);
dev_->partial = CudaDevicePtr<float2>(DOT_BLOCKS);
dev_->scalars = CudaDevicePtr<CGScalars>(1);
dev_->host.resize(n);
}
@@ -302,14 +485,13 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
hv.resize(per_ * n_params_);
if (n == 0)
return;
for (size_t i = 0; i < hv.size(); i++) {
const float hi = static_cast<float>(v[i]);
d.host[i] = make_float2(hi, static_cast<float>(v[i] - hi));
}
for (size_t i = 0; i < hv.size(); i++)
d.host[i] = Split(v[i]);
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, nullptr,
nullptr, d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
@@ -333,3 +515,91 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
return out;
}
void RestraintNormalMatrixGPU::SetSolve(const std::vector<double> &h, const std::vector<int> &start,
const std::vector<int> &element, const std::vector<double> &inverse) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
std::vector<float2> h2(h.size()), inverse2(inverse.size());
for (size_t i = 0; i < h.size(); i++)
h2[i] = Split(h[i]);
for (size_t i = 0; i < inverse.size(); i++)
inverse2[i] = Split(inverse[i]);
std::vector<int> block_of(element.size()), inverse_start{0};
for (size_t g = 0; g + 1 < start.size(); g++) {
const int m = start[g + 1] - start[g];
std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast<int>(g));
inverse_start.push_back(inverse_start.back() + m * (m + 1) / 2);
}
d.elements = static_cast<int>(element.size());
d.h = Up(h2, d.stream);
d.inverse = Up(inverse2, d.stream);
d.block_of = Up(block_of, d.stream);
d.block_start = Up(start, d.stream);
d.block_element = Up(element, d.stream);
d.inverse_start = Up(inverse_start, d.stream);
cuda_err(cudaStreamSynchronize(d.stream));
}
// ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host
// before x.
void RestraintNormalMatrixGPU::Solve(const std::vector<double> &b, std::vector<double> &x, int iterations) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(per_ * n_params_);
x.assign(n, 0.0);
if (n == 0)
return;
if (d.kept < iterations) {
d.rs = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * n);
d.zs = CudaDevicePtr<float>(static_cast<size_t>(iterations) * n);
d.rz_of = CudaDevicePtr<float2>(iterations);
d.c = CudaDevicePtr<float2>(iterations);
d.components = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * DOT_BLOCKS);
d.kept = iterations;
}
auto precondition = [&]() { // z = M^-1 r
BlockApply<<<GridSize(d.elements), THREADS, 0, d.stream>>>(d.block_of, d.block_start, d.block_element,
d.inverse_start, d.inverse, d.r, d.z, d.elements);
};
auto dot = [&](const float2 *u, const float2 *v, DotUse use, int j) {
DotPartial<<<DOT_BLOCKS, THREADS, 0, d.stream>>>(u, v, n, d.partial);
DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j);
};
auto keep = [&](int j) { // r and z as the j-th
Keep<<<GridSize(n), THREADS, 0, d.stream>>>(d.r, d.z, d.rs.get() + static_cast<size_t>(j) * n,
d.zs.get() + static_cast<size_t>(j) * n, n);
};
for (int i = 0; i < n; i++)
d.host[i] = Split(b[i]);
cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream));
cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream));
precondition();
cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream));
dot(d.r, d.z, DotUse::FirstRz, 0);
keep(0);
for (int k = 0; k < iterations; k++) {
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows);
TransposeProduct<<<GridSize(static_cast<int>(n_params_)), THREADS, 0, d.stream>>>(
d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.h, d.p, d.hp, static_cast<int>(n_params_));
dot(d.p, d.hp, DotUse::PHp, 0);
Step<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.p, d.hp, d.x, d.r, n);
if (k + 1 < iterations) { // the last direction would not be used
precondition();
// M^-1-orthogonal to the k + 1 earlier residuals, as ConjugateGradients
ComponentPartial<<<dim3(DOT_BLOCKS, k + 1), THREADS, 0, d.stream>>>(d.z, d.rs, n, d.components);
ComponentFinish<<<k + 1, DOT_BLOCKS, 0, d.stream>>>(d.components, d.rz_of, d.c);
Project<<<GridSize(n), THREADS, 0, d.stream>>>(d.c, k + 1, d.rs, d.zs, d.r, d.z, n);
dot(d.r, d.z, DotUse::Rz, k + 1);
NextDirection<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.z, d.p, n);
keep(k + 1);
}
}
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (int i = 0; i < n; i++)
x[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
}
@@ -34,6 +34,18 @@ public:
// The diagonal of J^T J.
std::vector<double> Diagonal() const;
// The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at
// the point last given to At(), preconditioned by a block-diagonal M given by the inverses of its blocks: block g
// over the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, the lower triangle of its
// M_g^-1 (symmetric) in `inverse`, row by row (W_ik, k <= i, at i (i + 1) / 2 + k from the block's first). h and M
// stay until the next SetSolve().
void SetSolve(const std::vector<double> &h, const std::vector<int> &start, const std::vector<int> &element,
const std::vector<double> &inverse);
// x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), each residual made
// M^-1-orthogonal to the earlier ones as there; the vectors on the card and every sum in float pairs, in a fixed
// order (RefinementNewtonGPU.cu).
void Solve(const std::vector<double> &b, std::vector<double> &x, int iterations);
private:
// The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries
// start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries.
+95
View File
@@ -2,6 +2,7 @@
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <Eigen/Dense>
#include <algorithm>
#include <atomic>
@@ -1112,6 +1113,100 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") {
}
}
// The conjugate gradients on the card against ConjugateGradients on the CPU, for H = J^T J + diag(h) preconditioned by
// blocks of H: two atoms' x y z per block for the positions, each B alone.
TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;
const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90);
gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5);
st.models[0] = shaken(st.models[0], 0.15, 3.0, 2);
const RefinementTopology topo = BuildRefinementTopology(st);
RestraintSet set = topo.geometry;
set.Append(topo.adp);
set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0));
const std::vector<gemmi::Atom *> atoms = atoms_of(st.models[0]);
set.Finalize(atoms.size());
std::vector<std::array<double, 4>> xyzb;
for (gemmi::Atom *a : atoms)
xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso});
std::vector<int> params, param_of(atoms.size(), -1);
for (int a = 0; a < static_cast<int>(atoms.size()); a++)
if (a % 5 != 4) {
param_of[a] = static_cast<int>(params.size());
params.push_back(a);
}
const RestraintJacobian jac = RestraintGaussNewton(set, xyzb);
std::mt19937 gen(5);
std::uniform_real_distribution<double> uniform(-1, 1);
for (const bool xyz : {true, false}) {
CAPTURE(xyz);
const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3;
const size_t n = per * params.size();
// H in full, the X-ray diagonal small beside the stiffest restraints
Eigen::MatrixXd hm = Eigen::MatrixXd::Zero(n, n);
for (size_t r = 0; r + 1 < jac.start.size(); r++)
for (int e1 = jac.start[r]; e1 < jac.start[r + 1]; e1++)
for (int e2 = jac.start[r]; e2 < jac.start[r + 1]; e2++) {
const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]];
if (k1 >= 0 && k2 >= 0)
for (int a = 0; a < per; a++)
for (int c = 0; c < per; c++)
hm(per * k1 + a, per * k2 + c) += jac.j[e1][c0 + a] * jac.j[e2][c0 + c];
}
std::vector<double> h(n), b(n);
for (size_t i = 0; i < n; i++) {
h[i] = xyz ? 10 + 5 * uniform(gen) : 0.01 + 0.005 * uniform(gen);
hm(i, i) += h[i];
b[i] = uniform(gen);
}
const int block = xyz ? 6 : 1;
std::vector<int> start{0}, element;
std::vector<double> inverse;
std::vector<Eigen::LLT<Eigen::MatrixXd>> llt;
for (size_t first = 0; first < n; first += block) {
const int m = static_cast<int>(std::min<size_t>(block, n - first));
llt.emplace_back(hm.block(first, first, m, m));
const Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m));
for (int i = 0; i < m; i++) {
element.push_back(static_cast<int>(first) + i);
for (int k = 0; k <= i; k++)
inverse.push_back(w(i, k));
}
start.push_back(static_cast<int>(element.size()));
}
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
const Eigen::VectorXd out = hm * Eigen::Map<const Eigen::VectorXd>(v.data(), n);
hv.assign(out.data(), out.data() + n);
};
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
z.resize(n);
for (size_t g = 0; g < llt.size(); g++) {
const int m = start[g + 1] - start[g];
const Eigen::VectorXd zg = llt[g].solve(Eigen::Map<const Eigen::VectorXd>(r.data() + start[g], m));
std::copy(zg.data(), zg.data() + m, z.begin() + start[g]);
}
};
std::vector<double> x;
ConjugateGradients(product, precondition, b, x, 20, 0.0);
RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz);
gpu.At(set, xyzb);
gpu.SetSolve(h, start, element, inverse);
std::vector<double> x_gpu, x_again;
gpu.Solve(b, x_gpu, 20);
gpu.Solve(b, x_again, 20);
REQUIRE(x_gpu.size() == n);
CHECK(x_gpu == x_again); // a fixed order: the same bits every time
double num = 0, den = 0;
for (size_t i = 0; i < n; i++) {
num += (x_gpu[i] - x[i]) * (x_gpu[i] - x[i]);
den += x[i] * x[i];
}
CAPTURE(std::sqrt(num / den));
CHECK(std::sqrt(num / den) < 1e-5);
}
}
TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") {
if (get_gpu_count() == 0)
return;