// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only // The device half of RestraintNormalMatrixGPU, in float: one thread per restraint for J, one per row of J or per // parameter atom for the products, each summing its terms in order. J is held in float; the products' v, t and hv in // float pairs (hi + lo, double-float arithmetic: an error-free sum and product per term), since the soft directions of // H - those the X-ray term decides, J v the small difference of large terms - need more than float's 24 bits: with // plain float sums the Newton steps on a model with stiff restraints differed from the CPU's by more than its noise. #include "RefinementNewtonGPU.h" #include #include "../indexing/CUDAMemHelpers.h" #include "RefinementGPUEngine.h" // MAX_PLANE_ATOMS namespace { constexpr int THREADS = 256; constexpr int DOT_BLOCKS = 128; // a dot product's partial sums: a fixed number, whatever the card, for a fixed order using V3f = refine_math::V3; void cuda_err(cudaError_t val) { if (val != cudaSuccess) throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); } class DeviceGuard { public: explicit DeviceGuard(int device) { cuda_err(cudaGetDevice(&previous_)); cuda_err(cudaSetDevice(device)); } ~DeviceGuard() { cudaSetDevice(previous_); } DeviceGuard(const DeviceGuard &) = delete; DeviceGuard &operator=(const DeviceGuard &) = delete; private: int previous_ = 0; }; int GreatestStreamPriority() { int least = 0, greatest = 0; cuda_err(cudaDeviceGetStreamPriorityRange(&least, &greatest)); return greatest; } int GridSize(int n) { return (n + THREADS - 1) / THREADS; } __device__ V3f xyz(const float4 *p, int i) { return {p[i].x, p[i].y, p[i].z}; } __device__ float Get(const float4 &j, int c) { return c == 0 ? j.x : c == 1 ? j.y : c == 2 ? j.z : j.w; } // sum += a (b.x + b.y), sum and b float pairs (hi in x, lo in y). Needs fma and no reassociation (no fast math). // The exact product and 2Sum following Dekker (1971) Numer. Math. 18, 224-242 and Knuth, TAOCP vol. 2, 4.2.2 __device__ void AddProduct(float2 &sum, float a, float2 b) { const float p = a * b.x; const float p_err = fmaf(a, b.x, -p); // a b.x = p + p_err exactly const float s = sum.x + p; const float v = s - sum.x; const float s_err = (sum.x - (s - v)) + (p - v); // sum.x + p = s + s_err exactly const float lo = s_err + sum.y + p_err + a * b.y; const float hi = s + lo; sum = make_float2(hi, lo - (hi - s)); } // The same double-float arithmetic on pairs throughout: a + b, a b, -a, a / b. __device__ float2 Normalised(float s, float e) { const float hi = s + e; return make_float2(hi, e - (hi - s)); } __device__ float2 Add(float2 a, float2 b) { const float s = a.x + b.x; const float v = s - a.x; const float e = (a.x - (s - v)) + (b.x - v); return Normalised(s, e + a.y + b.y); } __device__ float2 Mul(float2 a, float2 b) { const float p = a.x * b.x; return Normalised(p, fmaf(a.x, b.x, -p) + (a.x * b.y + a.y * b.x)); } __device__ float2 Neg(float2 a) { return make_float2(-a.x, -a.y); } __device__ float2 Div(float2 a, float2 b) { const float q = a.x / b.x; const float2 rest = Add(a, Neg(Mul(make_float2(q, 0.0f), b))); return Normalised(q, rest.x / b.x); } // Each restraint's rows of J (RestraintGaussNewton): E = residual^2, so J = g / sqrt(2 E); a plane's atoms each a // residual of their own, d r_i / d x_k = (delta_ik - 1/m) sqrt(2) normal / sigma. Zero where there is no row. __global__ void JacobianKernel(const RestraintRecord *records, int n, const int *plane_atoms, const int *record_entry, const float4 *xyzb, float4 *j) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; const RestraintRecord &r = records[i]; float4 *out = j + record_entry[i]; V3f g[4] = {}; int na = 0; float e = 0; switch (static_cast(r.kind)) { case RestraintKind::Bond: e = refine_math::BondTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), r.ideal, r.sigma, g); na = 2; break; case RestraintKind::Angle: e = refine_math::AngleTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), r.ideal, r.sigma, g); na = 3; break; case RestraintKind::Torsion: e = refine_math::TorsionTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), xyz(xyzb, r.atom[3]), r.ideal, r.sigma, r.period, g); na = 4; break; case RestraintKind::Chiral: e = refine_math::ChiralTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), xyz(xyzb, r.atom[3]), r.ideal, r.sigma, g); na = 4; break; case RestraintKind::NcsDistance: e = refine_math::NcsDistanceTerm(xyz(xyzb, r.atom[0]), xyz(xyzb, r.atom[1]), xyz(xyzb, r.atom[2]), xyz(xyzb, r.atom[3]), r.sigma, r.ideal, g); na = 4; break; case RestraintKind::Repulsion: { const V3f q = xyz(xyzb, r.atom[1]); const V3f image{r.rot[0] * q.x + r.rot[1] * q.y + r.rot[2] * q.z + r.tran[0], r.rot[3] * q.x + r.rot[4] * q.y + r.rot[5] * q.z + r.tran[1], r.rot[6] * q.x + r.rot[7] * q.y + r.rot[8] * q.z + r.tran[2]}; e = refine_math::RepulsionTerm(xyz(xyzb, r.atom[0]), image, r.ideal, r.sigma, g); const V3f gi = g[1]; g[1] = {r.rot[0] * gi.x + r.rot[3] * gi.y + r.rot[6] * gi.z, r.rot[1] * gi.x + r.rot[4] * gi.y + r.rot[7] * gi.z, r.rot[2] * gi.x + r.rot[5] * gi.y + r.rot[8] * gi.z}; na = 2; break; } case RestraintKind::Plane: { V3f px[RefinementGPUEngine::MAX_PLANE_ATOMS], pg[RefinementGPUEngine::MAX_PLANE_ATOMS]; const int m = r.atom[1]; for (int k = 0; k < m; k++) px[k] = xyz(xyzb, plane_atoms[r.atom[0] + k]); refine_math::PlaneTerm(px, m, r.sigma, pg); V3f nv{0, 0, 0}; for (int k = 0; k < m; k++) // the normal from the largest of the gradients (each is 2 r_k / sigma n) if (refine_math::Norm(pg[k]) > refine_math::Norm(nv)) nv = pg[k]; const float norm = refine_math::Norm(nv); const float c = norm > 0 ? sqrtf(2.0f) / (r.sigma * norm) : 0.0f; for (int a = 0; a < m; a++) for (int k = 0; k < m; k++) { const float f = c * ((a == k ? 1.0f : 0.0f) - 1.0f / m); out[a * m + k] = make_float4(f * nv.x, f * nv.y, f * nv.z, 0.0f); } return; } case RestraintKind::BSimilarity: { float gi = 0, gj = 0; e = refine_math::BSimilarityTerm(xyzb[r.atom[0]].w, xyzb[r.atom[1]].w, r.sigma, gi, gj); const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; out[0] = make_float4(0, 0, 0, f * gi); out[1] = make_float4(0, 0, 0, f * gj); return; } } const float f = e > 0 ? 1.0f / sqrtf(2.0f * e) : 0.0f; for (int k = 0; k < na; k++) out[k] = make_float4(f * g[k].x, f * g[k].y, f * g[k].z, 0.0f); } // t_r = sum over the row's entries of J_e . v_k __global__ void RowProduct(const int *start, const int *param, const float4 *j, int per, int c0, const float2 *v, float2 *t, int rows) { const int r = blockIdx.x * blockDim.x + threadIdx.x; if (r >= rows) return; float2 dot = make_float2(0, 0); for (int e = start[r]; e < start[r + 1]; e++) if (const int k = param[e]; k >= 0) for (int c = 0; c < per; c++) AddProduct(dot, Get(j[e], c0 + c), v[per * k + c]); t[r] = dot; } // J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's // entries read them one after another __global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) { const int q = blockIdx.x * blockDim.x + threadIdx.x; if (q < entries) jt[q] = j[entry[q]]; } // hv_k = sum over the parameter atom's entries of t_row J_e, then + h_k v_k where h is given: each entry read once for // all of the atom's components, each component summed over the entries in their order __global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0, const float2 *t, const float2 *h, const float2 *v, float2 *hv, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)}; for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { const float4 jq = jt[q]; const float2 tr = t[row_at[q]]; for (int c = 0; c < per; c++) AddProduct(sum[c], Get(jq, c0 + c), tr); } for (int c = 0; c < per; c++) hv[per * k + c] = h ? Add(sum[c], Mul(h[per * k + c], v[per * k + c])) : sum[c]; } __global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; for (int c = 0; c < per; c++) { float sum = 0; for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { const float x = Get(jt[q], c0 + c); sum += x * x; } d[per * k + c] = sum; } } // The conjugate gradients' scalars, kept on the card between the kernels: r M^-1 r, the step a and beta struct CGScalars { float2 rz, a, beta; int done; // p H p was not positive: x stays where it is }; // The sum of the block's `value`s, in a fixed tree (blockDim.x a power of two, `shared` of its size). __device__ float2 BlockSum(float2 *shared, float2 value) { const int t = threadIdx.x; shared[t] = value; __syncthreads(); for (int w = blockDim.x / 2; w > 0; w /= 2) { if (t < w) shared[t] = Add(shared[t], shared[t + w]); __syncthreads(); } return shared[0]; } // u . v in DOT_BLOCKS partial sums, each over a fixed stride of the elements __global__ void DotPartial(const float2 *u, const float2 *v, int n, float2 *partial) { __shared__ float2 shared[THREADS]; float2 sum = make_float2(0, 0); for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS) sum = Add(sum, Mul(u[i], v[i])); const float2 block = BlockSum(shared, sum); if (threadIdx.x == 0) partial[blockIdx.x] = block; } enum class DotUse { FirstRz, PHp, Rz }; // The partial sums' total, taken as the first r M^-1 r, as p H p (a = rz / pHp) or as the next r M^-1 r (beta); each // r M^-1 r kept as rz_of[j] __global__ void DotFinish(const float2 *partial, CGScalars *s, DotUse use, float2 *rz_of, int j) { __shared__ float2 shared[DOT_BLOCKS]; const float2 dot = BlockSum(shared, partial[threadIdx.x]); if (threadIdx.x != 0) return; switch (use) { case DotUse::FirstRz: s->rz = dot; rz_of[j] = dot; break; case DotUse::PHp: s->a = Div(s->rz, dot); if (!(dot.x > 0)) s->done = 1; break; case DotUse::Rz: s->beta = Div(dot, s->rz); s->rz = dot; rz_of[j] = dot; break; } } // x += a p, r -= a H p __global__ void Step(const CGScalars *s, const float2 *p, const float2 *hp, float2 *x, float2 *r, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n || s->done) return; x[i] = Add(x[i], Mul(s->a, p[i])); r[i] = Add(r[i], Neg(Mul(s->a, hp[i]))); } // z . r_j for every kept r_j (j = blockIdx.y), each in DOT_BLOCKS partial sums as DotPartial __global__ void ComponentPartial(const float2 *z, const float2 *rs, int n, float2 *partial) { __shared__ float2 shared[THREADS]; const float2 *rj = rs + static_cast(blockIdx.y) * n; float2 sum = make_float2(0, 0); for (int i = blockIdx.x * THREADS + threadIdx.x; i < n; i += DOT_BLOCKS * THREADS) sum = Add(sum, Mul(z[i], rj[i])); const float2 block = BlockSum(shared, sum); if (threadIdx.x == 0) partial[blockIdx.y * DOT_BLOCKS + blockIdx.x] = block; } // c_j = z . r_j / rz_of[j], block j __global__ void ComponentFinish(const float2 *partial, const float2 *rz_of, float2 *c) { __shared__ float2 shared[DOT_BLOCKS]; const float2 dot = BlockSum(shared, partial[blockIdx.x * DOT_BLOCKS + threadIdx.x]); if (threadIdx.x == 0) c[blockIdx.x] = Div(dot, rz_of[blockIdx.x]); } // r -= sum_j c_j r_j, z -= sum_j c_j z_j, over the `kept` earlier residuals in order. z_j is kept in float: c_j is // the orthogonality lost in rounding, so what float leaves of c_j z_j is far below the pairs' precision (c_j itself // is from r_j, in pairs). __global__ void Project(const float2 *c, int kept, const float2 *rs, const float *zs, float2 *r, float2 *z, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; float2 ri = r[i], zi = z[i]; for (int j = 0; j < kept; j++) { ri = Add(ri, Neg(Mul(c[j], rs[static_cast(j) * n + i]))); zi = Add(zi, Neg(Mul(c[j], make_float2(zs[static_cast(j) * n + i], 0.0f)))); } r[i] = ri; z[i] = zi; } // r and z kept as the j-th: r in pairs, z in float __global__ void Keep(const float2 *r, const float2 *z, float2 *rj, float *zj, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= n) return; rj[i] = r[i]; zj[i] = z[i].x; } // p = z + beta p __global__ void NextDirection(const CGScalars *s, const float2 *z, float2 *p, int n) { const int i = blockIdx.x * blockDim.x + threadIdx.x; if (i < n) p[i] = Add(z[i], Mul(s->beta, p[i])); } // z = M^-1 r, one thread per element (q: its place in the blocks' element list, of block g): its row of M_g^-1 times // the block's part of r. An explicit inverse, not the substitutions of the Cholesky factor: those are a chain of m^2 // dependent steps per block, a thread's worth of latency the card cannot hide. __global__ void BlockApply(const int *block_of, const int *start, const int *element, const int *inverse_start, const float2 *inverse, const float2 *r, float2 *z, int elements) { const int q = blockIdx.x * blockDim.x + threadIdx.x; if (q >= elements) return; const int g = block_of[q]; const int m = start[g + 1] - start[g]; const int *e = element + start[g]; const float2 *w = inverse + inverse_start[g]; // the lower triangle, W_ik at i (i + 1) / 2 + k for k <= i const int i = q - start[g]; float2 sum = make_float2(0, 0); for (int k = 0; k < m; k++) sum = Add(sum, Mul(k <= i ? w[i * (i + 1) / 2 + k] : w[k * (k + 1) / 2 + i], r[e[k]])); z[element[q]] = sum; } float2 Split(double v) { const float hi = static_cast(v); return make_float2(hi, static_cast(v - hi)); } template CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { CudaDevicePtr d(std::max(host.size(), 1)); cuda_err(cudaMemcpyAsync(d.get(), host.data(), host.size() * sizeof(T), cudaMemcpyHostToDevice, stream)); return d; } } // namespace struct RestraintNormalMatrixGPU::Device { int device; CudaStream stream; int records = 0, rows = 0, entries = 0; CudaDevicePtr record; CudaDevicePtr plane_atoms, record_entry, start, param, entry_start, entry, row_at; CudaDevicePtr xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ) CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; // the conjugate gradients: H's X-ray diagonal, M's blocks and their inverses, the vectors, the dot products' partial sums int elements = 0; CudaDevicePtr h, inverse, x, r, z, p, hp, partial; CudaDevicePtr rs, rz_of, c, components; // the residuals of a solve so far, r M^-1 r of each, the CudaDevicePtr zs; // coefficients and their partial sums; M^-1 of each residual int kept = 0; // how many residuals rs and zs hold room for CudaDevicePtr block_of, block_start, block_element, inverse_start; CudaDevicePtr scalars; PinnedVector host; // v and hv as float pairs PinnedVector host_xyzb; }; RestraintNormalMatrixGPU::RestraintNormalMatrixGPU(int device, const std::vector ¶m_atoms, size_t n_atoms, bool xyz) : per_(xyz ? 3 : 1), c0_(xyz ? 0 : 3), n_params_(param_atoms.size()), param_of_(n_atoms, -1) { for (size_t k = 0; k < param_atoms.size(); k++) param_of_[param_atoms[k]] = static_cast(k); DeviceGuard guard(device); // At the device's highest priority: a product is a few small kernels the minimiser waits for, queued beside the // X-ray evaluations of the other trials on the card. dev_ = std::unique_ptr(new Device{device, CudaStream(cudaStreamNonBlocking, GreatestStreamPriority())}); const size_t n = std::max(per_ * n_params_, 1); dev_->xyzb = CudaDevicePtr(std::max(n_atoms, 1)); dev_->host_xyzb.resize(n_atoms); dev_->v = CudaDevicePtr(n); dev_->hv = CudaDevicePtr(n); dev_->diagonal = CudaDevicePtr(n); for (CudaDevicePtr *u : {&dev_->x, &dev_->r, &dev_->z, &dev_->p, &dev_->hp}) *u = CudaDevicePtr(n); dev_->partial = CudaDevicePtr(DOT_BLOCKS); dev_->scalars = CudaDevicePtr(1); dev_->host.resize(n); } RestraintNormalMatrixGPU::~RestraintNormalMatrixGPU() { DeviceGuard guard(dev_->device); dev_.reset(); } void RestraintNormalMatrixGPU::Upload(const std::vector &records, const std::vector &plane_atoms, const std::vector &record_entry, const std::vector &start, const std::vector ¶m, const std::vector &row_of, const std::vector &entry_start, const std::vector &entry) { DeviceGuard guard(dev_->device); Device &d = *dev_; d.records = static_cast(records.size()); d.rows = static_cast(start.size()) - 1; d.record = Up(records, d.stream); d.plane_atoms = Up(plane_atoms, d.stream); d.record_entry = Up(record_entry, d.stream); d.start = Up(start, d.stream); d.param = Up(param, d.stream); d.entry_start = Up(entry_start, d.stream); d.entry = Up(entry, d.stream); // the row of each entry of the transpose, in its order std::vector row_at(entry.size()); for (size_t q = 0; q < entry.size(); q++) row_at[q] = row_of[entry[q]]; d.row_at = Up(row_at, d.stream); d.entries = static_cast(entry.size()); d.j = CudaDevicePtr(std::max(param.size(), 1)); d.jt = CudaDevicePtr(std::max(entry.size(), 1)); d.t = CudaDevicePtr(std::max(d.rows, 1)); cuda_err(cudaStreamSynchronize(d.stream)); } void RestraintNormalMatrixGPU::Fill(const std::vector> &xyzb) { DeviceGuard guard(dev_->device); Device &d = *dev_; for (size_t a = 0; a < xyzb.size(); a++) d.host_xyzb[a] = make_float4(static_cast(xyzb[a][0]), static_cast(xyzb[a][1]), static_cast(xyzb[a][2]), static_cast(xyzb[a][3])); cuda_err(cudaMemcpyAsync(d.xyzb.get(), d.host_xyzb.data(), xyzb.size() * sizeof(float4), cudaMemcpyHostToDevice, d.stream)); if (d.records > 0) JacobianKernel<<>>(d.record, d.records, d.plane_atoms, d.record_entry, d.xyzb, d.j); if (d.entries > 0) TransposeJ<<>>(d.entry, d.j, d.jt, d.entries); cuda_err(cudaGetLastError()); cuda_err(cudaStreamSynchronize(d.stream)); } void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vector &hv) { DeviceGuard guard(dev_->device); Device &d = *dev_; const int n = static_cast(n_params_); hv.resize(per_ * n_params_); if (n == 0) return; for (size_t i = 0; i < hv.size(); i++) d.host[i] = Split(v[i]); cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); TransposeProduct<<>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, nullptr, nullptr, d.hv, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); for (size_t i = 0; i < hv.size(); i++) hv[i] = static_cast(d.host[i].x) + d.host[i].y; } std::vector RestraintNormalMatrixGPU::Diagonal() const { DeviceGuard guard(dev_->device); Device &d = *dev_; const int n = static_cast(n_params_); std::vector out(per_ * n_params_); if (n == 0) return out; std::vector host(out.size()); DiagonalKernel<<>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); std::copy(host.begin(), host.end(), out.begin()); return out; } void RestraintNormalMatrixGPU::SetSolve(const std::vector &h, const std::vector &start, const std::vector &element, const std::vector &inverse) { DeviceGuard guard(dev_->device); Device &d = *dev_; std::vector h2(h.size()), inverse2(inverse.size()); for (size_t i = 0; i < h.size(); i++) h2[i] = Split(h[i]); for (size_t i = 0; i < inverse.size(); i++) inverse2[i] = Split(inverse[i]); std::vector block_of(element.size()), inverse_start{0}; for (size_t g = 0; g + 1 < start.size(); g++) { const int m = start[g + 1] - start[g]; std::fill(block_of.begin() + start[g], block_of.begin() + start[g + 1], static_cast(g)); inverse_start.push_back(inverse_start.back() + m * (m + 1) / 2); } d.elements = static_cast(element.size()); d.h = Up(h2, d.stream); d.inverse = Up(inverse2, d.stream); d.block_of = Up(block_of, d.stream); d.block_start = Up(start, d.stream); d.block_element = Up(element, d.stream); d.inverse_start = Up(inverse_start, d.stream); cuda_err(cudaStreamSynchronize(d.stream)); } // ConjugateGradients step by step, x, r, z, p and H p on the card; the scalars too, so nothing comes back to the host // before x. void RestraintNormalMatrixGPU::Solve(const std::vector &b, std::vector &x, int iterations) { DeviceGuard guard(dev_->device); Device &d = *dev_; const int n = static_cast(per_ * n_params_); x.assign(n, 0.0); if (n == 0) return; if (d.kept < iterations) { d.rs = CudaDevicePtr(static_cast(iterations) * n); d.zs = CudaDevicePtr(static_cast(iterations) * n); d.rz_of = CudaDevicePtr(iterations); d.c = CudaDevicePtr(iterations); d.components = CudaDevicePtr(static_cast(iterations) * DOT_BLOCKS); d.kept = iterations; } auto precondition = [&]() { // z = M^-1 r BlockApply<<>>(d.block_of, d.block_start, d.block_element, d.inverse_start, d.inverse, d.r, d.z, d.elements); }; auto dot = [&](const float2 *u, const float2 *v, DotUse use, int j) { DotPartial<<>>(u, v, n, d.partial); DotFinish<<<1, DOT_BLOCKS, 0, d.stream>>>(d.partial, d.scalars, use, d.rz_of, j); }; auto keep = [&](int j) { // r and z as the j-th Keep<<>>(d.r, d.z, d.rs.get() + static_cast(j) * n, d.zs.get() + static_cast(j) * n, n); }; for (int i = 0; i < n; i++) d.host[i] = Split(b[i]); cuda_err(cudaMemcpyAsync(d.r.get(), d.host.data(), n * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); cuda_err(cudaMemsetAsync(d.x.get(), 0, n * sizeof(float2), d.stream)); cuda_err(cudaMemsetAsync(d.scalars.get(), 0, sizeof(CGScalars), d.stream)); precondition(); cuda_err(cudaMemcpyAsync(d.p.get(), d.z.get(), n * sizeof(float2), cudaMemcpyDeviceToDevice, d.stream)); dot(d.r, d.z, DotUse::FirstRz, 0); keep(0); for (int k = 0; k < iterations; k++) { if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.p, d.t, d.rows); TransposeProduct<<(n_params_)), THREADS, 0, d.stream>>>( d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.h, d.p, d.hp, static_cast(n_params_)); dot(d.p, d.hp, DotUse::PHp, 0); Step<<>>(d.scalars, d.p, d.hp, d.x, d.r, n); if (k + 1 < iterations) { // the last direction would not be used precondition(); // M^-1-orthogonal to the k + 1 earlier residuals, as ConjugateGradients ComponentPartial<<>>(d.z, d.rs, n, d.components); ComponentFinish<<>>(d.components, d.rz_of, d.c); Project<<>>(d.c, k + 1, d.rs, d.zs, d.r, d.z, n); dot(d.r, d.z, DotUse::Rz, k + 1); NextDirection<<>>(d.scalars, d.z, d.p, n); keep(k + 1); } } cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.x.get(), n * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); for (int i = 0; i < n; i++) x[i] = static_cast(d.host[i].x) + d.host[i].y; }