Merge branch 'rc188-g' into rc188-int
# Conflicts: # image_analysis/structure_refinement/RefinementNewtonGPU.cu
This commit is contained in:
commit
809e178ef9
8 files changed
+486
-19
No files matched your search
@@ -603,7 +603,12 @@ initial inverse Hessian is the target's sparse normal matrix - the X-ray target'
|
||||
on the diagonal and the restraints' Gauss-Newton matrix in full - as REFMAC refines on its normal
|
||||
matrix (Murshudov, Vagin & Dodson 1997, above), solved by conjugate gradients: M. R. Hestenes and
|
||||
E. Stiefel, "Methods of conjugate gradients for solving linear systems" (1952), J. Res. Natl. Bur.
|
||||
Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044). Each atom's
|
||||
Stand. 49, 409-436 [doi:10.6028/jres.049.044](https://doi.org/10.6028/jres.049.044), each residual
|
||||
made orthogonal again to the earlier ones against the loss of orthogonality in rounding described by
|
||||
C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix" (1976),
|
||||
J. Inst. Maths Applics 18, 341-349 (DOI not verified here), and A. Greenbaum, "Behavior of
|
||||
slightly perturbed Lanczos and conjugate-gradient recurrences" (1989), Linear Algebra Appl. 113, 7-63
|
||||
[doi:10.1016/0024-3795(89)90285-1](https://doi.org/10.1016/0024-3795(89)90285-1). Each atom's
|
||||
diagonal is raised to hold its step within the step bound, a trust region per atom after the damping
|
||||
of K. Levenberg, "A method for the solution of certain non-linear problems in least squares" (1944),
|
||||
Q. Appl. Math. 2, 164-168 [doi:10.1090/qam/10666](https://doi.org/10.1090/qam/10666), and D. W.
|
||||
|
||||
@@ -73,6 +73,7 @@ The methods draw on, and in places reimplement, solutions from:
|
||||
- D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15).
|
||||
- J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782 (the two-loop recursion of the minimiser, §15.6).
|
||||
- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6).
|
||||
- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the reorthogonalised conjugate gradients of the minimiser, §15.6).
|
||||
- G. N. Murshudov, P. Skubák, A. A. Lebedev et al., "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (the likelihood's D from the working set and its variance from the free, and the Kullback-Leibler B restraints, §15.3, §15.5).
|
||||
- P. Roversi, E. Blanc, S. Johnson & S. M. Lea, "Tetartohedral twinning could happen to you too", *Acta Cryst.* **D68** (2012), 418-424 (twin refinement with more than two domains, and the free set closed under the twin domains, §15.12).
|
||||
- K. A. Sultana, I. Alexeev, I. Kursula, P. Mäntsälä, J. Niemi & G. Schneider, "Structure determination by multiwavelength anomalous diffraction of aclacinomycin oxidoreductase: indications of multidomain pseudomerohedral twinning", *Acta Cryst.* **D63** (2007), 149-159 (a six-domain twin of a monoclinic crystal on a hexagonal metric, §15.12).
|
||||
|
||||
@@ -320,7 +320,12 @@ leaves out.
|
||||
conjugate-gradient iterations - not to a tolerance, whose iteration count, and with it the direction, a rounding
|
||||
difference can flip - preconditioned per residue for the positions (the residue's own $J^\mathsf{T}J$ and X-ray
|
||||
diagonal, by Cholesky: the stiff bonds and angles inside a residue are solved there, what couples residues is
|
||||
left to the iterations) and by the diagonal elsewhere. The Jacobian is taken again at each point the minimiser
|
||||
left to the iterations) and by the diagonal elsewhere. Each new residual is made orthogonal again (in the
|
||||
preconditioner's inner product) to all the earlier ones: the plain recurrence loses that orthogonality in
|
||||
rounding once its first Ritz values converge (Paige; Greenbaum), and from then on a rounding difference grew
|
||||
several-fold per iteration, so that two solves of right-hand sides $10^{-14}$ apart ended up to 7% apart - which
|
||||
is what set the CPU and GPU results apart. With it the solve is a smooth function of its input, and the GPU's
|
||||
float-pair solve agrees with the CPU's to $10^{-13}$. The Jacobian is taken again at each point the minimiser
|
||||
stands on.
|
||||
- **Its memory is kept from one macro-cycle to the next.** Each weight trial starts with an empty memory.
|
||||
|
||||
@@ -790,6 +795,7 @@ What the refinement does not do yet - each a known gap, not a design choice:
|
||||
- N. Echols et al., "Automated identification of elemental ions in macromolecular crystal structures", *Acta Cryst.* **D70** (2014), 1104-1114 (a light ion flagged by its density above the waters', §15.10).
|
||||
- J. Nocedal, "Updating quasi-Newton matrices with limited storage", *Math. Comp.* **35** (1980), 773-782, and D. C. Liu & J. Nocedal, "On the limited memory BFGS method for large scale optimization", *Math. Program.* **45** (1989), 503-528 (the minimiser, §15.6).
|
||||
- M. R. Hestenes & E. Stiefel, "Methods of conjugate gradients for solving linear systems", *J. Res. Natl. Bur. Stand.* **49** (1952), 409-436 (the normal equations of the minimiser, §15.6).
|
||||
- C. C. Paige, "Error analysis of the Lanczos algorithm for tridiagonalizing a symmetric matrix", *J. Inst. Maths Applics* **18** (1976), 341-349, and A. Greenbaum, "Behavior of slightly perturbed Lanczos and conjugate-gradient recurrences", *Linear Algebra Appl.* **113** (1989), 7-63 (the loss of orthogonality the minimiser's conjugate gradients are kept from, §15.6).
|
||||
- K. Levenberg, "A method for the solution of certain non-linear problems in least squares", *Q. Appl. Math.* **2** (1944), 164-168, and D. W. Marquardt, "An algorithm for least-squares estimation of nonlinear parameters", *J. Soc. Ind. Appl. Math.* **11** (1963), 431-441 (the minimiser's trust floor per atom, §15.6).
|
||||
- A. T. Brünger, M. Karplus & G. A. Petsko, "Crystallographic refinement by simulated annealing: application to crambin", *Acta Cryst.* **A45** (1989), 50-61 (the starting weight from the ratio of the gradients, §15.7).
|
||||
- P. V. Afonine et al., "Towards automated crystallographic structure refinement with phenix.refine", *Acta Cryst.* **D68** (2012), 352-367 (weights chosen among refinements by R-free within geometry bounds, and the overall B moved into the atoms, §15.3, §15.7).
|
||||
|
||||
@@ -1013,20 +1013,19 @@ std::vector<double> FlooredDiagonal(std::vector<double> d) {
|
||||
constexpr int NORMAL_CG_ITERATIONS = 20;
|
||||
|
||||
// `target` by `minimiser` (its memory from the call before) from where it is, H0 the inverse of the normal matrix: at(x, q)
|
||||
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then hv = H v
|
||||
// (product) and z = M^-1 r for a preconditioner M of it (precondition).
|
||||
// takes the matrix at x (where the minimiser stands; q the vector H0 is about to be applied to), then z = H^-1 q by
|
||||
// NORMAL_CG_ITERATIONS conjugate gradients (solve: ConjugateGradients on the CPU, or on the card).
|
||||
template <typename Target>
|
||||
void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMinimiser &minimiser, int iterations,
|
||||
const std::function<void(const std::vector<double> &, const std::vector<double> &)> &at,
|
||||
const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
|
||||
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
|
||||
const std::function<void(const std::vector<double> &, std::vector<double> &)> &solve) {
|
||||
std::vector<double> p = target.Get();
|
||||
if (p.empty())
|
||||
return;
|
||||
minimiser.SetInverseHessian([&](const std::vector<double> &x, std::vector<double> &q) {
|
||||
at(x, q);
|
||||
std::vector<double> z;
|
||||
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
|
||||
solve(q, z);
|
||||
q.swap(z);
|
||||
});
|
||||
minimiser.Minimise([&](const std::vector<double> &x, double &f, std::vector<double> &g) {
|
||||
@@ -1037,6 +1036,15 @@ void MinimiseWithNormalMatrix(Context &ctx, const Target &target, RefinementMini
|
||||
target.Set(p.data()); // where the minimiser ended, not where its last line search probed
|
||||
}
|
||||
|
||||
// The conjugate gradients on the CPU: hv = H v (product) and z = M^-1 r for a preconditioner M of H (precondition).
|
||||
std::function<void(const std::vector<double> &, std::vector<double> &)>
|
||||
NormalSolve(const Context &ctx, const std::function<void(const std::vector<double> &, std::vector<double> &)> &product,
|
||||
const std::function<void(const std::vector<double> &, std::vector<double> &)> &precondition) {
|
||||
return [&ctx, product, precondition](const std::vector<double> &q, std::vector<double> &z) {
|
||||
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
|
||||
};
|
||||
}
|
||||
|
||||
// The restraints' Gauss-Newton normal matrix J^T J (RestraintGaussNewton) on the x y z or the B of `param_atoms`, at the
|
||||
// point last given to At(); an atom that is not a parameter (a rider, an atom held) drops out of it. With the engine on
|
||||
// a GPU, on its card (RestraintNormalMatrixGPU).
|
||||
@@ -1113,6 +1121,11 @@ public:
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
// The conjugate gradients on the card, the vectors kept there (RestraintNormalMatrixGPU::Solve)
|
||||
RestraintNormalMatrixGPU *GPU() { return gpu_.get(); }
|
||||
#endif
|
||||
|
||||
std::vector<double> Diagonal() const {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (gpu_)
|
||||
@@ -1182,6 +1195,25 @@ public:
|
||||
ParallelFor(static_cast<int>(params_.size()), ctx.nthreads, [&](int g) { llt_[g].compute(block[g]); });
|
||||
}
|
||||
|
||||
// The blocks as RestraintNormalMatrixGPU::SetSolve takes them: per block its elements and the lower triangle of its
|
||||
// inverse, row by row.
|
||||
void Inverses(std::vector<int> &start, std::vector<int> &element, std::vector<double> &inverse) const {
|
||||
start.assign(1, 0);
|
||||
element.clear();
|
||||
inverse.clear();
|
||||
for (size_t g = 0; g < params_.size(); g++) {
|
||||
for (int k : params_[g])
|
||||
for (int c = 0; c < 3; c++)
|
||||
element.push_back(3 * k + c);
|
||||
start.push_back(static_cast<int>(element.size()));
|
||||
const int m = 3 * static_cast<int>(params_[g].size());
|
||||
const Eigen::MatrixXd w = llt_[g].solve(Eigen::MatrixXd::Identity(m, m));
|
||||
for (int i = 0; i < m; i++)
|
||||
for (int k = 0; k <= i; k++)
|
||||
inverse.push_back(w(i, k));
|
||||
}
|
||||
}
|
||||
|
||||
void Apply(const std::vector<double> &r, std::vector<double> &z, size_t nthreads) const {
|
||||
z.resize(r.size());
|
||||
ParallelFor(static_cast<int>(params_.size()), nthreads, [&](int g) {
|
||||
@@ -1240,6 +1272,23 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
|
||||
diagonal = FlooredDiagonal(diagonal);
|
||||
if (xyz)
|
||||
residues = std::make_unique<ResidueBlocks>(ctx, target.Atoms(), set, h);
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (normal.GPU()) { // the blocks, or the diagonal as blocks of one
|
||||
std::vector<int> start, element;
|
||||
std::vector<double> inverse;
|
||||
if (residues)
|
||||
residues->Inverses(start, element, inverse);
|
||||
else {
|
||||
for (size_t k = 0; k < n; k++) {
|
||||
start.push_back(static_cast<int>(k));
|
||||
element.push_back(static_cast<int>(k));
|
||||
inverse.push_back(1.0 / diagonal[k]);
|
||||
}
|
||||
start.push_back(static_cast<int>(n));
|
||||
}
|
||||
normal.GPU()->SetSolve(h, start, element, inverse);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
if (x != x_at) {
|
||||
x_at = x;
|
||||
@@ -1259,7 +1308,16 @@ void MinimiseAtoms(Context &ctx, const AtomTarget &target, RefinementMinimiser &
|
||||
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
|
||||
}
|
||||
};
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
|
||||
auto solve = [&](const std::vector<double> &q, std::vector<double> &z) {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (normal.GPU()) {
|
||||
const RefinementStage stage("normal.solve");
|
||||
return normal.GPU()->Solve(q, z, NORMAL_CG_ITERATIONS);
|
||||
}
|
||||
#endif
|
||||
ConjugateGradients(product, precondition, q, z, NORMAL_CG_ITERATIONS, 0.0, ctx.nthreads);
|
||||
};
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, solve);
|
||||
}
|
||||
|
||||
// Each atom's U (in 8 pi^2 U): H = diag(xray) per element + the U restraints' Hessian, exact by differences of their
|
||||
@@ -1306,7 +1364,7 @@ void MinimiseU(Context &ctx, const AdpTarget &target, RefinementMinimiser &minim
|
||||
z.resize(n);
|
||||
refine_vec::Each(n, ctx.nthreads, [&](size_t k) { z[k] = r[k] / diagonal[k]; });
|
||||
};
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
|
||||
}
|
||||
|
||||
// The TLS bodies: X-ray only, H = sum over each body's atoms of A^T C A, A the derivatives of the atom's U in the body's
|
||||
@@ -1364,7 +1422,7 @@ void MinimiseTls(Context &ctx, const AdpTarget &target, RefinementMinimiser &min
|
||||
for (size_t k = 0; k < r.size(); k++)
|
||||
z[k] = r[k] / diagonal[k];
|
||||
};
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, product, precondition);
|
||||
MinimiseWithNormalMatrix(ctx, target, minimiser, iterations, at, NormalSolve(ctx, product, precondition));
|
||||
}
|
||||
|
||||
// Groups of atoms that move as rigid bodies (one per chain) against the X-ray target: per body an angle-axis
|
||||
|
||||
@@ -172,6 +172,12 @@ private:
|
||||
// z with H z = b to `tolerance` of |b| (at most `max_iterations`), by conjugate gradients preconditioned with M:
|
||||
// hv = H v (product), z = M^-1 r (precondition), both symmetric positive definite. Returns the iterations taken.
|
||||
// Following Hestenes & Stiefel (1952) J. Res. Natl. Bur. Stand. 49, 409-436
|
||||
// Each residual is made M^-1-orthogonal again to all the earlier ones (Gram-Schmidt, every coefficient r_j M^-1 r /
|
||||
// r_j M^-1 r_j = z . r_j / rz_j from the same z, then r and z = M^-1 r corrected together). Without it the recurrence loses that orthogonality in rounding once the first Ritz values converge, and
|
||||
// from then on amplifies a rounding difference several times per iteration: on the refinement's normal matrix two
|
||||
// solves of right-hand sides 1e-14 apart ended up to 7% apart after 20 iterations, which set CPU and GPU apart.
|
||||
// The loss of orthogonality after Paige (1976) J. Inst. Maths Applics 18, 341-349 and Greenbaum (1989) Linear
|
||||
// Algebra Appl. 113, 7-63
|
||||
inline int ConjugateGradients(const std::function<void(const std::vector<double> &v, std::vector<double> &hv)> &product,
|
||||
const std::function<void(const std::vector<double> &r, std::vector<double> &z)> &precondition,
|
||||
const std::vector<double> &b, std::vector<double> &x, int max_iterations, double tolerance,
|
||||
@@ -183,6 +189,8 @@ inline int ConjugateGradients(const std::function<void(const std::vector<double>
|
||||
precondition(r, z);
|
||||
p = z;
|
||||
double rz = dot(r, z);
|
||||
std::vector<std::vector<double>> rs{r}, zs{z}; // the residuals so far, and M^-1 of each
|
||||
std::vector<double> rzs{rz};
|
||||
const double r0 = std::sqrt(dot(r, r));
|
||||
int k = 0;
|
||||
while (k < max_iterations) {
|
||||
@@ -199,9 +207,21 @@ inline int ConjugateGradients(const std::function<void(const std::vector<double>
|
||||
if (std::sqrt(dot(r, r)) < tolerance * r0)
|
||||
break;
|
||||
precondition(r, z);
|
||||
std::vector<double> c(rs.size());
|
||||
for (size_t i = 0; i < rs.size(); i++)
|
||||
c[i] = dot(z, rs[i]) / rzs[i];
|
||||
refine_vec::Each(n, nthreads, [&](size_t j) {
|
||||
for (size_t i = 0; i < rs.size(); i++) {
|
||||
r[j] -= c[i] * rs[i][j];
|
||||
z[j] -= c[i] * zs[i][j];
|
||||
}
|
||||
});
|
||||
const double rz_new = dot(r, z);
|
||||
refine_vec::Each(n, nthreads, [&](size_t j) { p[j] = z[j] + rz_new / rz * p[j]; });
|
||||
rz = rz_new;
|
||||
rs.push_back(r);
|
||||
zs.push_back(z);
|
||||
rzs.push_back(rz);
|
||||
}
|
||||
return k;
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
namespace {
|
||||
|
||||
constexpr int THREADS = 256;
|
||||
constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order
|
||||
using V3f = refine_math::V3<float>;
|
||||
|
||||
void cuda_err(cudaError_t val) {
|
||||
@@ -69,6 +70,34 @@ __device__ void AddProduct(float2 &sum, float a, float2 b) {
|
||||
sum = make_float2(hi, lo - (hi - s));
|
||||
}
|
||||
|
||||
// The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b.
|
||||
__device__ float2 Normalised(float s, float e) {
|
||||
const float hi = s + e;
|
||||
return make_float2(hi, e - (hi - s));
|
||||
}
|
||||
|
||||
__device__ float2 Add(float2 a, float2 b) {
|
||||
const float s = a.x + b.x;
|
||||
const float v = s - a.x;
|
||||
const float e = (a.x - (s - v)) + (b.x - v);
|
||||
return Normalised(s, e + a.y + b.y);
|
||||
}
|
||||
|
||||
__device__ float2 Mul(float2 a, float2 b) {
|
||||
const float p = a.x * b.x;
|
||||
return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x));
|
||||
}
|
||||
|
||||
__device__ float2 Neg(float2 a) {
|
||||
return make_float2(-a.x, -a.y);
|
||||
}
|
||||
|
||||
__device__ float2 Div(float2 a, float2 b) {
|
||||
const float q = a.x / b.x;
|
||||
const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b)));
|
||||
return Normalised(q, rest.x / b.x);
|
||||
}
|
||||
|
||||
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
|
||||
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
|
||||
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
|
||||
@@ -174,10 +203,10 @@ __global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int en
|
||||
jt[q] = j[entry[q]];
|
||||
}
|
||||
|
||||
// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components,
|
||||
// each component summed over the entries in their order
|
||||
// hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given: each entry read once for
|
||||
// all of the atom's components, each component summed over the entries in their order
|
||||
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
|
||||
const float2 *t, float2 *hv, int n) {
|
||||
const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
@@ -189,7 +218,7 @@ __global__ void TransposeProduct(const int *entry_start, const float4 *jt, const
|
||||
AddProduct(sum[c], Get(jq, c0 + c), tr);
|
||||
}
|
||||
for (int c = 0; c < per; c++)
|
||||
hv[per * k + c] = sum[c];
|
||||
hv[per * k + c] = h ? Add(sum[c], Mul(h[per * k + c], v[per * k + c])) : sum[c];
|
||||
}
|
||||
|
||||
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
|
||||
@@ -206,6 +235,148 @@ __global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per
|
||||
}
|
||||
}
|
||||
|
||||
// The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta
|
||||
struct CGScalars {
|
||||
float2 rz, a, beta;
|
||||
int done; // p H p was not positive: x stays where it is
|
||||
};
|
||||
|
||||
// The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size).
|
||||
__device__ float2 BlockSum(float2 *shared, float2 value) {
|
||||
const int t = threadIdx.x;
|
||||
shared[t] = value;
|
||||
__syncthreads();
|
||||
for (int w = blockDim.x / 2; w > 0; w /= 2) {
|
||||
if (t < w)
|
||||
shared[t] = Add(shared[t], shared[t + w]);
|
||||
__syncthreads();
|
||||
}
|
||||
return shared[0];
|
||||
}
|
||||
|
||||
// u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements
|
||||
__global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) {
|
||||
__shared__ float2 shared[THREADS];
|
||||
float2 sum = make_float2(0, 0);
|
||||
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
|
||||
sum = Add(sum, Mul(u[i], v[i]));
|
||||
const float2 block = BlockSum(shared, sum);
|
||||
if (threadIdx.x == 0)
|
||||
partial[blockIdx.x] = block;
|
||||
}
|
||||
|
||||
enum class DotUse { FirstRz, PHp, Rz };
|
||||
|
||||
// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta); each
|
||||
// r M^-1 r kept as rz_of[j]
|
||||
__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) {
|
||||
__shared__ float2 shared[DOT_BLOCKS];
|
||||
const float2 dot = BlockSum(shared, partial[threadIdx.x]);
|
||||
if (threadIdx.x != 0)
|
||||
return;
|
||||
switch (use) {
|
||||
case DotUse::FirstRz:
|
||||
s->rz = dot;
|
||||
rz_of[j] = dot;
|
||||
break;
|
||||
case DotUse::PHp:
|
||||
s->a = Div(s->rz, dot);
|
||||
if (!(dot.x > 0))
|
||||
s->done = 1;
|
||||
break;
|
||||
case DotUse::Rz:
|
||||
s->beta = Div(dot, s->rz);
|
||||
s->rz = dot;
|
||||
rz_of[j] = dot;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// x += a p, r -= a H p
|
||||
__global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) {
|
||||
const int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= n || s->done)
|
||||
return;
|
||||
x[i] = Add(x[i], Mul(s->a, p[i]));
|
||||
r[i] = Add(r[i], Neg(Mul(s->a, hp[i])));
|
||||
}
|
||||
|
||||
// z . r_j for every kept r_j (j = blockIdx.y), each in DOT_BLOCKS partial sums as DotPartial
|
||||
__global__ void ComponentPartial(const float2 *z, const float2 *rs, int n, float2 *partial) {
|
||||
__shared__ float2 shared[THREADS];
|
||||
const float2 *rj = rs + static_cast<size_t>(blockIdx.y) * n;
|
||||
float2 sum = make_float2(0, 0);
|
||||
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
|
||||
sum = Add(sum, Mul(z[i], rj[i]));
|
||||
const float2 block = BlockSum(shared, sum);
|
||||
if (threadIdx.x == 0)
|
||||
partial[blockIdx.y * DOT_BLOCKS + blockIdx.x] = block;
|
||||
}
|
||||
|
||||
// c_j = z . r_j / rz_of[j], block j
|
||||
__global__ void ComponentFinish(const float2 *partial, const float2 *rz_of, float2 *c) {
|
||||
__shared__ float2 shared[DOT_BLOCKS];
|
||||
const float2 dot = BlockSum(shared, partial[blockIdx.x * DOT_BLOCKS + threadIdx.x]);
|
||||
if (threadIdx.x == 0)
|
||||
c[blockIdx.x] = Div(dot, rz_of[blockIdx.x]);
|
||||
}
|
||||
|
||||
// r -= sum_j c_j r_j, z -= sum_j c_j z_j, over the `kept` earlier residuals in order. z_j is kept in float: c_j is
|
||||
// the orthogonality lost in rounding, so what float leaves of c_j z_j is far below the pairs' precision (c_j itself
|
||||
// is from r_j, in pairs).
|
||||
__global__ void Project(const float2 *c, int kept, const float2 *rs, const float *zs, float2 *r, float2 *z, int n) {
|
||||
const int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= n)
|
||||
return;
|
||||
float2 ri = r[i], zi = z[i];
|
||||
for (int j = 0; j < kept; j++) {
|
||||
ri = Add(ri, Neg(Mul(c[j], rs[static_cast<size_t>(j) * n + i])));
|
||||
zi = Add(zi, Neg(Mul(c[j], make_float2(zs[static_cast<size_t>(j) * n + i], 0.0f))));
|
||||
}
|
||||
r[i] = ri;
|
||||
z[i] = zi;
|
||||
}
|
||||
|
||||
// r and z kept as the j-th: r in pairs, z in float
|
||||
__global__ void Keep(const float2 *r, const float2 *z, float2 *rj, float *zj, int n) {
|
||||
const int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= n)
|
||||
return;
|
||||
rj[i] = r[i];
|
||||
zj[i] = z[i].x;
|
||||
}
|
||||
|
||||
// p = z + beta p
|
||||
__global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) {
|
||||
const int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i < n)
|
||||
p[i] = Add(z[i], Mul(s->beta, p[i]));
|
||||
}
|
||||
|
||||
// z = M^-1 r, one thread per element (q: its place in the blocks' element list, of block g): its row of M_g^-1 times
|
||||
// the block's part of r. An explicit inverse, not the substitutions of the Cholesky factor: those are a chain of m^2
|
||||
// dependent steps per block, a thread's worth of latency the card cannot hide.
|
||||
__global__ void BlockApply(const int *block_of, const int *start, const int *element, const int *inverse_start,
|
||||
const float2 *inverse, const float2 *r, float2 *z, int elements) {
|
||||
const int q = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (q >= elements)
|
||||
return;
|
||||
const int g = block_of[q];
|
||||
const int m = start[g + 1] - start[g];
|
||||
const int *e = element + start[g];
|
||||
const float2 *w = inverse + inverse_start[g]; // the lower triangle, W_ik at i (i + 1) / 2 + k for k <= i
|
||||
const int i = q - start[g];
|
||||
float2 sum = make_float2(0, 0);
|
||||
for (int k = 0; k < m; k++)
|
||||
sum = Add(sum, Mul(k <= i ? w[i * (i + 1) / 2 + k] : w[k * (k + 1) / 2 + i], r[e[k]]));
|
||||
z[element[q]] = sum;
|
||||
}
|
||||
|
||||
float2 Split(double v) {
|
||||
const float hi = static_cast<float>(v);
|
||||
return make_float2(hi, static_cast<float>(v - hi));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
|
||||
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
|
||||
@@ -224,6 +395,14 @@ struct RestraintNormalMatrixGPU::Device {
|
||||
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
|
||||
CudaDevicePtr<float2> t, v, hv;
|
||||
CudaDevicePtr<float> diagonal;
|
||||
// the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums
|
||||
int elements = 0;
|
||||
CudaDevicePtr<float2> h, inverse, x, r, z, p, hp, partial;
|
||||
CudaDevicePtr<float2> rs, rz_of, c, components; // the residuals of a solve so far, r M^-1 r of each, the
|
||||
CudaDevicePtr<float> zs; // coefficients and their partial sums; M^-1 of each residual
|
||||
int kept = 0; // how many residuals rs and zs hold room for
|
||||
CudaDevicePtr<int> block_of, block_start, block_element, inverse_start;
|
||||
CudaDevicePtr<CGScalars> scalars;
|
||||
PinnedVector<float2> host; // v and hv as float pairs
|
||||
PinnedVector<float4> host_xyzb;
|
||||
};
|
||||
@@ -243,6 +422,10 @@ RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector
|
||||
dev_->v = CudaDevicePtr<float2>(n);
|
||||
dev_->hv = CudaDevicePtr<float2>(n);
|
||||
dev_->diagonal = CudaDevicePtr<float>(n);
|
||||
for (CudaDevicePtr<float2> *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp})
|
||||
*u = CudaDevicePtr<float2>(n);
|
||||
dev_->partial = CudaDevicePtr<float2>(DOT_BLOCKS);
|
||||
dev_->scalars = CudaDevicePtr<CGScalars>(1);
|
||||
dev_->host.resize(n);
|
||||
}
|
||||
|
||||
@@ -302,14 +485,13 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
|
||||
hv.resize(per_ * n_params_);
|
||||
if (n == 0)
|
||||
return;
|
||||
for (size_t i = 0; i < hv.size(); i++) {
|
||||
const float hi = static_cast<float>(v[i]);
|
||||
d.host[i] = make_float2(hi, static_cast<float>(v[i] - hi));
|
||||
}
|
||||
for (size_t i = 0; i < hv.size(); i++)
|
||||
d.host[i] = Split(v[i]);
|
||||
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
|
||||
if (d.rows > 0)
|
||||
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
|
||||
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n);
|
||||
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, nullptr,
|
||||
nullptr, d.hv, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
@@ -333,3 +515,91 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
void RestraintNormalMatrixGPU::SetSolve(const std::vector<double> &h, const std::vector<int> &start,
|
||||
const std::vector<int> &element, const std::vector<double> &inverse) {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
std::vector<float2> h2(h.size()), inverse2(inverse.size());
|
||||
for (size_t i = 0; i < h.size(); i++)
|
||||
h2[i] = Split(h[i]);
|
||||
for (size_t i = 0; i < inverse.size(); i++)
|
||||
inverse2[i] = Split(inverse[i]);
|
||||
std::vector<int> block_of(element.size()), inverse_start{0};
|
||||
for (size_t g = 0; g + 1 < start.size(); g++) {
|
||||
const int m = start[g + 1] - start[g];
|
||||
std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast<int>(g));
|
||||
inverse_start.push_back(inverse_start.back() + m * (m + 1) / 2);
|
||||
}
|
||||
d.elements = static_cast<int>(element.size());
|
||||
d.h = Up(h2, d.stream);
|
||||
d.inverse = Up(inverse2, d.stream);
|
||||
d.block_of = Up(block_of, d.stream);
|
||||
d.block_start = Up(start, d.stream);
|
||||
d.block_element = Up(element, d.stream);
|
||||
d.inverse_start = Up(inverse_start, d.stream);
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
}
|
||||
|
||||
// ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host
|
||||
// before x.
|
||||
void RestraintNormalMatrixGPU::Solve(const std::vector<double> &b, std::vector<double> &x, int iterations) {
|
||||
DeviceGuard guard(dev_->device);
|
||||
Device &d = *dev_;
|
||||
const int n = static_cast<int>(per_ * n_params_);
|
||||
x.assign(n, 0.0);
|
||||
if (n == 0)
|
||||
return;
|
||||
if (d.kept < iterations) {
|
||||
d.rs = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * n);
|
||||
d.zs = CudaDevicePtr<float>(static_cast<size_t>(iterations) * n);
|
||||
d.rz_of = CudaDevicePtr<float2>(iterations);
|
||||
d.c = CudaDevicePtr<float2>(iterations);
|
||||
d.components = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * DOT_BLOCKS);
|
||||
d.kept = iterations;
|
||||
}
|
||||
auto precondition = [&]() { // z = M^-1 r
|
||||
BlockApply<<<GridSize(d.elements), THREADS, 0, d.stream>>>(d.block_of, d.block_start, d.block_element,
|
||||
d.inverse_start, d.inverse, d.r, d.z, d.elements);
|
||||
};
|
||||
auto dot = [&](const float2 *u, const float2 *v, DotUse use, int j) {
|
||||
DotPartial<<<DOT_BLOCKS, THREADS, 0, d.stream>>>(u, v, n, d.partial);
|
||||
DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j);
|
||||
};
|
||||
auto keep = [&](int j) { // r and z as the j-th
|
||||
Keep<<<GridSize(n), THREADS, 0, d.stream>>>(d.r, d.z, d.rs.get() + static_cast<size_t>(j) * n,
|
||||
d.zs.get() + static_cast<size_t>(j) * n, n);
|
||||
};
|
||||
for (int i = 0; i < n; i++)
|
||||
d.host[i] = Split(b[i]);
|
||||
cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
|
||||
cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream));
|
||||
cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream));
|
||||
precondition();
|
||||
cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream));
|
||||
dot(d.r, d.z, DotUse::FirstRz, 0);
|
||||
keep(0);
|
||||
for (int k = 0; k < iterations; k++) {
|
||||
if (d.rows > 0)
|
||||
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows);
|
||||
TransposeProduct<<<GridSize(static_cast<int>(n_params_)), THREADS, 0, d.stream>>>(
|
||||
d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.h, d.p, d.hp, static_cast<int>(n_params_));
|
||||
dot(d.p, d.hp, DotUse::PHp, 0);
|
||||
Step<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.p, d.hp, d.x, d.r, n);
|
||||
if (k + 1 < iterations) { // the last direction would not be used
|
||||
precondition();
|
||||
// M^-1-orthogonal to the k + 1 earlier residuals, as ConjugateGradients
|
||||
ComponentPartial<<<dim3(DOT_BLOCKS, k + 1), THREADS, 0, d.stream>>>(d.z, d.rs, n, d.components);
|
||||
ComponentFinish<<<k + 1, DOT_BLOCKS, 0, d.stream>>>(d.components, d.rz_of, d.c);
|
||||
Project<<<GridSize(n), THREADS, 0, d.stream>>>(d.c, k + 1, d.rs, d.zs, d.r, d.z, n);
|
||||
dot(d.r, d.z, DotUse::Rz, k + 1);
|
||||
NextDirection<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.z, d.p, n);
|
||||
keep(k + 1);
|
||||
}
|
||||
}
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
for (int i = 0; i < n; i++)
|
||||
x[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
|
||||
}
|
||||
@@ -34,6 +34,18 @@ public:
|
||||
// The diagonal of J^T J.
|
||||
std::vector<double> Diagonal() const;
|
||||
|
||||
// The conjugate gradients of ConjugateGradients (RefinementMinimiser.h) on the card, for H = J^T J + diag(h) at
|
||||
// the point last given to At(), preconditioned by a block-diagonal M given by the inverses of its blocks: block g
|
||||
// over the elements element[start[g]] ... element[start[g + 1] - 1] of the vector, the lower triangle of its
|
||||
// M_g^-1 (symmetric) in `inverse`, row by row (W_ik, k <= i, at i (i + 1) / 2 + k from the block's first). h and M
|
||||
// stay until the next SetSolve().
|
||||
void SetSolve(const std::vector<double> &h, const std::vector<int> &start, const std::vector<int> &element,
|
||||
const std::vector<double> &inverse);
|
||||
// x = H^-1 b by `iterations` of them from x = 0 (fewer only where p H p is not positive), each residual made
|
||||
// M^-1-orthogonal to the earlier ones as there; the vectors on the card and every sum in float pairs, in a fixed
|
||||
// order (RefinementNewtonGPU.cu).
|
||||
void Solve(const std::vector<double> &b, std::vector<double> &x, int iterations);
|
||||
|
||||
private:
|
||||
// The set and the index of J (RefinementNewtonGPU.cu): per restraint its first entry, per row its entries
|
||||
// start[r] ... start[r + 1], per entry its parameter atom (-1: none) and row, per parameter atom its entries.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
#include <Eigen/Dense>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
@@ -1112,6 +1113,100 @@ TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") {
|
||||
}
|
||||
}
|
||||
|
||||
// The conjugate gradients on the card against ConjugateGradients on the CPU, for H = J^T J + diag(h) preconditioned by
|
||||
// blocks of H: two atoms' x y z per block for the positions, each B alone.
|
||||
TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") {
|
||||
if (get_gpu_count() == 0)
|
||||
return;
|
||||
const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90);
|
||||
gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5);
|
||||
st.models[0] = shaken(st.models[0], 0.15, 3.0, 2);
|
||||
const RefinementTopology topo = BuildRefinementTopology(st);
|
||||
RestraintSet set = topo.geometry;
|
||||
set.Append(topo.adp);
|
||||
set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0));
|
||||
const std::vector<gemmi::Atom *> atoms = atoms_of(st.models[0]);
|
||||
set.Finalize(atoms.size());
|
||||
std::vector<std::array<double, 4>> xyzb;
|
||||
for (gemmi::Atom *a : atoms)
|
||||
xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso});
|
||||
std::vector<int> params, param_of(atoms.size(), -1);
|
||||
for (int a = 0; a < static_cast<int>(atoms.size()); a++)
|
||||
if (a % 5 != 4) {
|
||||
param_of[a] = static_cast<int>(params.size());
|
||||
params.push_back(a);
|
||||
}
|
||||
const RestraintJacobian jac = RestraintGaussNewton(set, xyzb);
|
||||
std::mt19937 gen(5);
|
||||
std::uniform_real_distribution<double> uniform(-1, 1);
|
||||
for (const bool xyz : {true, false}) {
|
||||
CAPTURE(xyz);
|
||||
const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3;
|
||||
const size_t n = per * params.size();
|
||||
// H in full, the X-ray diagonal small beside the stiffest restraints
|
||||
Eigen::MatrixXd hm = Eigen::MatrixXd::Zero(n, n);
|
||||
for (size_t r = 0; r + 1 < jac.start.size(); r++)
|
||||
for (int e1 = jac.start[r]; e1 < jac.start[r + 1]; e1++)
|
||||
for (int e2 = jac.start[r]; e2 < jac.start[r + 1]; e2++) {
|
||||
const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]];
|
||||
if (k1 >= 0 && k2 >= 0)
|
||||
for (int a = 0; a < per; a++)
|
||||
for (int c = 0; c < per; c++)
|
||||
hm(per * k1 + a, per * k2 + c) += jac.j[e1][c0 + a] * jac.j[e2][c0 + c];
|
||||
}
|
||||
std::vector<double> h(n), b(n);
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
h[i] = xyz ? 10 + 5 * uniform(gen) : 0.01 + 0.005 * uniform(gen);
|
||||
hm(i, i) += h[i];
|
||||
b[i] = uniform(gen);
|
||||
}
|
||||
const int block = xyz ? 6 : 1;
|
||||
std::vector<int> start{0}, element;
|
||||
std::vector<double> inverse;
|
||||
std::vector<Eigen::LLT<Eigen::MatrixXd>> llt;
|
||||
for (size_t first = 0; first < n; first += block) {
|
||||
const int m = static_cast<int>(std::min<size_t>(block, n - first));
|
||||
llt.emplace_back(hm.block(first, first, m, m));
|
||||
const Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m));
|
||||
for (int i = 0; i < m; i++) {
|
||||
element.push_back(static_cast<int>(first) + i);
|
||||
for (int k = 0; k <= i; k++)
|
||||
inverse.push_back(w(i, k));
|
||||
}
|
||||
start.push_back(static_cast<int>(element.size()));
|
||||
}
|
||||
auto product = [&](const std::vector<double> &v, std::vector<double> &hv) {
|
||||
const Eigen::VectorXd out = hm * Eigen::Map<const Eigen::VectorXd>(v.data(), n);
|
||||
hv.assign(out.data(), out.data() + n);
|
||||
};
|
||||
auto precondition = [&](const std::vector<double> &r, std::vector<double> &z) {
|
||||
z.resize(n);
|
||||
for (size_t g = 0; g < llt.size(); g++) {
|
||||
const int m = start[g + 1] - start[g];
|
||||
const Eigen::VectorXd zg = llt[g].solve(Eigen::Map<const Eigen::VectorXd>(r.data() + start[g], m));
|
||||
std::copy(zg.data(), zg.data() + m, z.begin() + start[g]);
|
||||
}
|
||||
};
|
||||
std::vector<double> x;
|
||||
ConjugateGradients(product, precondition, b, x, 20, 0.0);
|
||||
RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz);
|
||||
gpu.At(set, xyzb);
|
||||
gpu.SetSolve(h, start, element, inverse);
|
||||
std::vector<double> x_gpu, x_again;
|
||||
gpu.Solve(b, x_gpu, 20);
|
||||
gpu.Solve(b, x_again, 20);
|
||||
REQUIRE(x_gpu.size() == n);
|
||||
CHECK(x_gpu == x_again); // a fixed order: the same bits every time
|
||||
double num = 0, den = 0;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
num += (x_gpu[i] - x[i]) * (x_gpu[i] - x[i]);
|
||||
den += x[i] * x[i];
|
||||
}
|
||||
CAPTURE(std::sqrt(num / den));
|
||||
CHECK(std::sqrt(num / den) < 1e-5);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") {
|
||||
if (get_gpu_count() == 0)
|
||||
return;
|
||||
|
||||
Reference in new issue
Block a user