Files
leonarski_f 809e178ef9 Merge branch 'rc188-g' into rc188-int
# Conflicts:
#	image_analysis/structure_refinement/RefinementNewtonGPU.cu
2026-11-17 16:01:35 +01:00

606 lines
27 KiB
Plaintext

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per
// parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in
// float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of
// H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with
// plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise.
#include "RefinementNewtonGPU.h"
#include <algorithm>
#include "../indexing/CUDAMemHelpers.h"
#include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS
namespace {
constexpr int THREADS = 256;
constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order
using V3f = refine_math::V3<float>;
void cuda_err(cudaError_t val) {
if (val != cudaSuccess)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
}
class DeviceGuard {
public:
explicit DeviceGuard(int device) {
cuda_err(cudaGetDevice(&previous_));
cuda_err(cudaSetDevice(device));
}
~DeviceGuard() { cudaSetDevice(previous_); }
DeviceGuard(const DeviceGuard &) = delete;
DeviceGuard &operator=(const DeviceGuard &) = delete;
private:
int previous_ = 0;
};
int GreatestStreamPriority() {
int least = 0, greatest = 0;
cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest));
return greatest;
}
int GridSize(int n) {
return (n + THREADS - 1) / THREADS;
}
__device__ V3f xyz(const float4 *p, int i) {
return {p[i].x, p[i].y, p[i].z};
}
__device__ float Get(const float4 &j, int c) {
return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w;
}
// sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math).
// The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2
__device__ void AddProduct(float2 &sum, float a, float2 b) {
const float p = a * b.x;
const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly
const float s = sum.x + p;
const float v = s - sum.x;
const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly
const float lo = s_err + sum.y + p_err + a * b.y;
const float hi = s + lo;
sum = make_float2(hi, lo - (hi - s));
}
// The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b.
__device__ float2 Normalised(float s, float e) {
const float hi = s + e;
return make_float2(hi, e - (hi - s));
}
__device__ float2 Add(float2 a, float2 b) {
const float s = a.x + b.x;
const float v = s - a.x;
const float e = (a.x - (s - v)) + (b.x - v);
return Normalised(s, e + a.y + b.y);
}
__device__ float2 Mul(float2 a, float2 b) {
const float p = a.x * b.x;
return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x));
}
__device__ float2 Neg(float2 a) {
return make_float2(-a.x, -a.y);
}
__device__ float2 Div(float2 a, float2 b) {
const float q = a.x / b.x;
const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b)));
return Normalised(q, rest.x / b.x);
}
// Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a
// residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row.
__global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry,
const float4 *xyzb, float4 *j) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
const RestraintRecord &r = records[i];
float4 *out = j + record_entry[i];
V3f g[4] = {};
int na = 0;
float e = 0;
switch (static_cast<RestraintKind>(r.kind)) {
case RestraintKind::Bond:
e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g);
na = 2;
break;
case RestraintKind::Angle:
e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal,
r.sigma, g);
na = 3;
break;
case RestraintKind::Torsion:
e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g);
na = 4;
break;
case RestraintKind::Chiral:
e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g);
na = 4;
break;
case RestraintKind::NcsDistance:
e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]),
xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g);
na = 4;
break;
case RestraintKind::Repulsion: {
const V3f q = xyz(xyzb, r.atom[1]);
const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0],
r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1],
r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]};
e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g);
const V3f gi = g[1];
g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z,
r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z,
r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z};
na = 2;
break;
}
case RestraintKind::Plane: {
V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS];
const int m = r.atom[1];
for (int k = 0; k < m; k++)
px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]);
refine_math::PlaneTerm(px, m, r.sigma, pg);
V3f nv{0, 0, 0};
for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n)
if (refine_math::Norm(pg[k]) > refine_math::Norm(nv))
nv = pg[k];
const float norm = refine_math::Norm(nv);
const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f;
for (int a = 0; a < m; a++)
for (int k = 0; k < m; k++) {
const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m);
out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f);
}
return;
}
case RestraintKind::BSimilarity: {
float gi = 0, gj = 0;
e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj);
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
out[0] = make_float4(0, 0, 0, f * gi);
out[1] = make_float4(0, 0, 0, f * gj);
return;
}
}
const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f;
for (int k = 0; k < na; k++)
out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f);
}
// t_r = sum over the row's entries of J_e . v_k
__global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v,
float2 *t, int rows) {
const int r = blockIdx.x * blockDim.x + threadIdx.x;
if (r >= rows)
return;
float2 dot = make_float2(0, 0);
for (int e = start[r]; e < start[r + 1]; e++)
if (const int k = param[e]; k >= 0)
for (int c = 0; c < per; c++)
AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]);
t[r] = dot;
}
// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's
// entries read them one after another
__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) {
const int q = blockIdx.x * blockDim.x + threadIdx.x;
if (q < entries)
jt[q] = j[entry[q]];
}
// hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given: each entry read once for
// all of the atom's components, each component summed over the entries in their order
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)};
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float4 jq = jt[q];
const float2 tr = t[row_at[q]];
for (int c = 0; c < per; c++)
AddProduct(sum[c], Get(jq, c0 + c), tr);
}
for (int c = 0; c < per; c++)
hv[per * k + c] = h ? Add(sum[c], Mul(h[per * k + c], v[per * k + c])) : sum[c];
}
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float sum = 0;
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float x = Get(jt[q], c0 + c);
sum += x * x;
}
d[per * k + c] = sum;
}
}
// The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta
struct CGScalars {
float2 rz, a, beta;
int done; // p H p was not positive: x stays where it is
};
// The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size).
__device__ float2 BlockSum(float2 *shared, float2 value) {
const int t = threadIdx.x;
shared[t] = value;
__syncthreads();
for (int w = blockDim.x / 2; w > 0; w /= 2) {
if (t < w)
shared[t] = Add(shared[t], shared[t + w]);
__syncthreads();
}
return shared[0];
}
// u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements
__global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) {
__shared__ float2 shared[THREADS];
float2 sum = make_float2(0, 0);
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
sum = Add(sum, Mul(u[i], v[i]));
const float2 block = BlockSum(shared, sum);
if (threadIdx.x == 0)
partial[blockIdx.x] = block;
}
enum class DotUse { FirstRz, PHp, Rz };
// The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta); each
// r M^-1 r kept as rz_of[j]
__global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) {
__shared__ float2 shared[DOT_BLOCKS];
const float2 dot = BlockSum(shared, partial[threadIdx.x]);
if (threadIdx.x != 0)
return;
switch (use) {
case DotUse::FirstRz:
s->rz = dot;
rz_of[j] = dot;
break;
case DotUse::PHp:
s->a = Div(s->rz, dot);
if (!(dot.x > 0))
s->done = 1;
break;
case DotUse::Rz:
s->beta = Div(dot, s->rz);
s->rz = dot;
rz_of[j] = dot;
break;
}
}
// x += a p, r -= a H p
__global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n || s->done)
return;
x[i] = Add(x[i], Mul(s->a, p[i]));
r[i] = Add(r[i], Neg(Mul(s->a, hp[i])));
}
// z . r_j for every kept r_j (j = blockIdx.y), each in DOT_BLOCKS partial sums as DotPartial
__global__ void ComponentPartial(const float2 *z, const float2 *rs, int n, float2 *partial) {
__shared__ float2 shared[THREADS];
const float2 *rj = rs + static_cast<size_t>(blockIdx.y) * n;
float2 sum = make_float2(0, 0);
for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS)
sum = Add(sum, Mul(z[i], rj[i]));
const float2 block = BlockSum(shared, sum);
if (threadIdx.x == 0)
partial[blockIdx.y * DOT_BLOCKS + blockIdx.x] = block;
}
// c_j = z . r_j / rz_of[j], block j
__global__ void ComponentFinish(const float2 *partial, const float2 *rz_of, float2 *c) {
__shared__ float2 shared[DOT_BLOCKS];
const float2 dot = BlockSum(shared, partial[blockIdx.x * DOT_BLOCKS + threadIdx.x]);
if (threadIdx.x == 0)
c[blockIdx.x] = Div(dot, rz_of[blockIdx.x]);
}
// r -= sum_j c_j r_j, z -= sum_j c_j z_j, over the `kept` earlier residuals in order. z_j is kept in float: c_j is
// the orthogonality lost in rounding, so what float leaves of c_j z_j is far below the pairs' precision (c_j itself
// is from r_j, in pairs).
__global__ void Project(const float2 *c, int kept, const float2 *rs, const float *zs, float2 *r, float2 *z, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
float2 ri = r[i], zi = z[i];
for (int j = 0; j < kept; j++) {
ri = Add(ri, Neg(Mul(c[j], rs[static_cast<size_t>(j) * n + i])));
zi = Add(zi, Neg(Mul(c[j], make_float2(zs[static_cast<size_t>(j) * n + i], 0.0f))));
}
r[i] = ri;
z[i] = zi;
}
// r and z kept as the j-th: r in pairs, z in float
__global__ void Keep(const float2 *r, const float2 *z, float2 *rj, float *zj, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= n)
return;
rj[i] = r[i];
zj[i] = z[i].x;
}
// p = z + beta p
__global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i < n)
p[i] = Add(z[i], Mul(s->beta, p[i]));
}
// z = M^-1 r, one thread per element (q: its place in the blocks' element list, of block g): its row of M_g^-1 times
// the block's part of r. An explicit inverse, not the substitutions of the Cholesky factor: those are a chain of m^2
// dependent steps per block, a thread's worth of latency the card cannot hide.
__global__ void BlockApply(const int *block_of, const int *start, const int *element, const int *inverse_start,
const float2 *inverse, const float2 *r, float2 *z, int elements) {
const int q = blockIdx.x * blockDim.x + threadIdx.x;
if (q >= elements)
return;
const int g = block_of[q];
const int m = start[g + 1] - start[g];
const int *e = element + start[g];
const float2 *w = inverse + inverse_start[g]; // the lower triangle, W_ik at i (i + 1) / 2 + k for k <= i
const int i = q - start[g];
float2 sum = make_float2(0, 0);
for (int k = 0; k < m; k++)
sum = Add(sum, Mul(k <= i ? w[i * (i + 1) / 2 + k] : w[k * (k + 1) / 2 + i], r[e[k]]));
z[element[q]] = sum;
}
float2 Split(double v) {
const float hi = static_cast<float>(v);
return make_float2(hi, static_cast<float>(v - hi));
}
template <typename T>
CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
CudaDevicePtr<T> d(std::max<size_t>(host.size(), 1));
cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream));
return d;
}
} // namespace
struct RestraintNormalMatrixGPU::Device {
int device;
CudaStream stream;
int records = 0, rows = 0, entries = 0;
CudaDevicePtr<RestraintRecord> record;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, entry_start, entry, row_at;
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
// the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums
int elements = 0;
CudaDevicePtr<float2> h, inverse, x, r, z, p, hp, partial;
CudaDevicePtr<float2> rs, rz_of, c, components; // the residuals of a solve so far, r M^-1 r of each, the
CudaDevicePtr<float> zs; // coefficients and their partial sums; M^-1 of each residual
int kept = 0; // how many residuals rs and zs hold room for
CudaDevicePtr<int> block_of, block_start, block_element, inverse_start;
CudaDevicePtr<CGScalars> scalars;
PinnedVector<float2> host; // v and hv as float pairs
PinnedVector<float4> host_xyzb;
};
RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector<int> &param_atoms, size_t n_atoms,
bool xyz)
: per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) {
for (size_t k = 0; k < param_atoms.size(); k++)
param_of_[param_atoms[k]] = static_cast<int>(k);
DeviceGuard guard(device);
// At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the
// X-ray evaluations of the other trials on the card.
dev_ = std::unique_ptr<Device>(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())});
const size_t n = std::max<size_t>(per_ * n_params_, 1);
dev_->xyzb = CudaDevicePtr<float4>(std::max<size_t>(n_atoms, 1));
dev_->host_xyzb.resize(n_atoms);
dev_->v = CudaDevicePtr<float2>(n);
dev_->hv = CudaDevicePtr<float2>(n);
dev_->diagonal = CudaDevicePtr<float>(n);
for (CudaDevicePtr<float2> *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp})
*u = CudaDevicePtr<float2>(n);
dev_->partial = CudaDevicePtr<float2>(DOT_BLOCKS);
dev_->scalars = CudaDevicePtr<CGScalars>(1);
dev_->host.resize(n);
}
RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() {
DeviceGuard guard(dev_->device);
dev_.reset();
}
void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &records, const std::vector<int> &plane_atoms,
const std::vector<int> &record_entry, const std::vector<int> &start,
const std::vector<int> &param, const std::vector<int> &row_of,
const std::vector<int> &entry_start, const std::vector<int> &entry) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
d.records = static_cast<int>(records.size());
d.rows = static_cast<int>(start.size()) - 1;
d.record = Up(records, d.stream);
d.plane_atoms = Up(plane_atoms, d.stream);
d.record_entry = Up(record_entry, d.stream);
d.start = Up(start, d.stream);
d.param = Up(param, d.stream);
d.entry_start = Up(entry_start, d.stream);
d.entry = Up(entry, d.stream);
// the row of each entry of the transpose, in its order
std::vector<int> row_at(entry.size());
for (size_t q = 0; q < entry.size(); q++)
row_at[q] = row_of[entry[q]];
d.row_at = Up(row_at, d.stream);
d.entries = static_cast<int>(entry.size());
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
d.jt = CudaDevicePtr<float4>(std::max<size_t>(entry.size(), 1));
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
cuda_err(cudaStreamSynchronize(d.stream));
}
void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xyzb) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
for (size_t a = 0; a < xyzb.size(); a++)
d.host_xyzb[a] = make_float4(static_cast<float>(xyzb[a][0]), static_cast<float>(xyzb[a][1]),
static_cast<float>(xyzb[a][2]), static_cast<float>(xyzb[a][3]));
cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice,
d.stream));
if (d.records > 0)
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
d.record_entry, d.xyzb, d.j);
if (d.entries > 0)
TransposeJ<<<GridSize(d.entries), THREADS, 0, d.stream>>>(d.entry, d.j, d.jt, d.entries);
cuda_err(cudaGetLastError());
cuda_err(cudaStreamSynchronize(d.stream));
}
void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vector<double> &hv) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(n_params_);
hv.resize(per_ * n_params_);
if (n == 0)
return;
for (size_t i = 0; i < hv.size(); i++)
d.host[i] = Split(v[i]);
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, nullptr,
nullptr, d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (size_t i = 0; i < hv.size(); i++)
hv[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
}
std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(n_params_);
std::vector<double> out(per_ * n_params_);
if (n == 0)
return out;
std::vector<float> host(out.size());
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
std::copy(host.begin(), host.end(), out.begin());
return out;
}
void RestraintNormalMatrixGPU::SetSolve(const std::vector<double> &h, const std::vector<int> &start,
const std::vector<int> &element, const std::vector<double> &inverse) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
std::vector<float2> h2(h.size()), inverse2(inverse.size());
for (size_t i = 0; i < h.size(); i++)
h2[i] = Split(h[i]);
for (size_t i = 0; i < inverse.size(); i++)
inverse2[i] = Split(inverse[i]);
std::vector<int> block_of(element.size()), inverse_start{0};
for (size_t g = 0; g + 1 < start.size(); g++) {
const int m = start[g + 1] - start[g];
std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast<int>(g));
inverse_start.push_back(inverse_start.back() + m * (m + 1) / 2);
}
d.elements = static_cast<int>(element.size());
d.h = Up(h2, d.stream);
d.inverse = Up(inverse2, d.stream);
d.block_of = Up(block_of, d.stream);
d.block_start = Up(start, d.stream);
d.block_element = Up(element, d.stream);
d.inverse_start = Up(inverse_start, d.stream);
cuda_err(cudaStreamSynchronize(d.stream));
}
// ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host
// before x.
void RestraintNormalMatrixGPU::Solve(const std::vector<double> &b, std::vector<double> &x, int iterations) {
DeviceGuard guard(dev_->device);
Device &d = *dev_;
const int n = static_cast<int>(per_ * n_params_);
x.assign(n, 0.0);
if (n == 0)
return;
if (d.kept < iterations) {
d.rs = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * n);
d.zs = CudaDevicePtr<float>(static_cast<size_t>(iterations) * n);
d.rz_of = CudaDevicePtr<float2>(iterations);
d.c = CudaDevicePtr<float2>(iterations);
d.components = CudaDevicePtr<float2>(static_cast<size_t>(iterations) * DOT_BLOCKS);
d.kept = iterations;
}
auto precondition = [&]() { // z = M^-1 r
BlockApply<<<GridSize(d.elements), THREADS, 0, d.stream>>>(d.block_of, d.block_start, d.block_element,
d.inverse_start, d.inverse, d.r, d.z, d.elements);
};
auto dot = [&](const float2 *u, const float2 *v, DotUse use, int j) {
DotPartial<<<DOT_BLOCKS, THREADS, 0, d.stream>>>(u, v, n, d.partial);
DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j);
};
auto keep = [&](int j) { // r and z as the j-th
Keep<<<GridSize(n), THREADS, 0, d.stream>>>(d.r, d.z, d.rs.get() + static_cast<size_t>(j) * n,
d.zs.get() + static_cast<size_t>(j) * n, n);
};
for (int i = 0; i < n; i++)
d.host[i] = Split(b[i]);
cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream));
cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream));
precondition();
cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream));
dot(d.r, d.z, DotUse::FirstRz, 0);
keep(0);
for (int k = 0; k < iterations; k++) {
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows);
TransposeProduct<<<GridSize(static_cast<int>(n_params_)), THREADS, 0, d.stream>>>(
d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.h, d.p, d.hp, static_cast<int>(n_params_));
dot(d.p, d.hp, DotUse::PHp, 0);
Step<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.p, d.hp, d.x, d.r, n);
if (k + 1 < iterations) { // the last direction would not be used
precondition();
// M^-1-orthogonal to the k + 1 earlier residuals, as ConjugateGradients
ComponentPartial<<<dim3(DOT_BLOCKS, k + 1), THREADS, 0, d.stream>>>(d.z, d.rs, n, d.components);
ComponentFinish<<<k + 1, DOT_BLOCKS, 0, d.stream>>>(d.components, d.rz_of, d.c);
Project<<<GridSize(n), THREADS, 0, d.stream>>>(d.c, k + 1, d.rs, d.zs, d.r, d.z, n);
dot(d.r, d.z, DotUse::Rz, k + 1);
NextDirection<<<GridSize(n), THREADS, 0, d.stream>>>(d.scalars, d.z, d.p, n);
keep(k + 1);
}
}
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
for (int i = 0; i < n; i++)
x[i] = static_cast<double>(d.host[i].x) + d.host[i].y;
}