Restraint normal matrix on the GPU: J^T t streamed in the parameter atoms' order

TransposeProduct summed each component over an atom's entries in a loop of its own, so every
entry's index, J and row were gathered once per component, in random sectors. Now the entries
are read once, each component's float-pair sum updated in the same order as before; and J is
copied into the transpose's order once per At() (TransposeJ, beside JacobianKernel in Fill),
with the row of each entry indexed once at Upload, so the product streams J and the rows and
gathers only t. The DiagonalKernel reads the same copy. Every sum keeps its terms and their
order: the products and the diagonal are bit-identical to the old kernel's (all four cases of
RefinementNewton_product_GPU_vs_CPU dumped and compared), at +20 B per entry of device memory.

On a 242k-atom model with 45 NCS copies, 4 x T4, the first ladder (4 trials per card): per
product, xyz without NCS 3.12 -> 0.89 ms, with NCS 11.1 -> 2.88 ms, B unchanged (0.44 ms);
TransposeJ 0.29 ms per At(). The refinement 1116 -> 1065 s, the refined model bit-identical, the
card peak 11.3 GB. Two small models: bit-identical, slightly faster.

Following the review in TPROD_REPORT.md (gather amplification diagnosis, options A and B).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
This commit is contained in:
jungfrauandClaude Opus 5.5 committed 2026-11-17 14:33:24 +01:00
1 parent 3e76b880df
commit 9d54253d2b
1 file changed
+36 -20
@@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j,
t[r] = dot;
}
// hv_k = sum over the parameter atom's entries of t_row J_e
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
int c0, const float2 *t, float2 *hv, int n) {
// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's
// entries read them one after another
__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) {
const int q = blockIdx.x * blockDim.x + threadIdx.x;
if (q < entries)
jt[q] = j[entry[q]];
}
// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components,
// each component summed over the entries in their order
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
const float2 *t, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float2 sum = make_float2(0, 0);
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const int e = entry[q];
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
}
hv[per * k + c] = sum;
float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)};
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float4 jq = jt[q];
const float2 tr = t[row_at[q]];
for (int c = 0; c < per; c++)
AddProduct(sum[c], Get(jq, c0 + c), tr);
}
for (int c = 0; c < per; c++)
hv[per * k + c] = sum[c];
}
__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d,
int n) {
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float sum = 0;
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float x = Get(j[entry[q]], c0 + c);
const float x = Get(jt[q], c0 + c);
sum += x * x;
}
d[per * k + c] = sum;
@@ -209,10 +218,10 @@ CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
struct RestraintNormalMatrixGPU::Device {
int device;
CudaStream stream;
int records = 0, rows = 0;
int records = 0, rows = 0, entries = 0;
CudaDevicePtr<RestraintRecord> record;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, row_of, entry_start, entry;
CudaDevicePtr<float4> xyzb, j;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, entry_start, entry, row_at;
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
PinnedVector<float2> host; // v and hv as float pairs
@@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &record
d.record_entry = Up(record_entry, d.stream);
d.start = Up(start, d.stream);
d.param = Up(param, d.stream);
d.row_of = Up(row_of, d.stream);
d.entry_start = Up(entry_start, d.stream);
d.entry = Up(entry, d.stream);
// the row of each entry of the transpose, in its order
std::vector<int> row_at(entry.size());
for (size_t q = 0; q < entry.size(); q++)
row_at[q] = row_of[entry[q]];
d.row_at = Up(row_at, d.stream);
d.entries = static_cast<int>(entry.size());
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
d.jt = CudaDevicePtr<float4>(std::max<size_t>(entry.size(), 1));
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
cuda_err(cudaStreamSynchronize(d.stream));
}
@@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xy
if (d.records > 0)
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
d.record_entry, d.xyzb, d.j);
if (d.entries > 0)
TransposeJ<<<GridSize(d.entries), THREADS, 0, d.stream>>>(d.entry, d.j, d.jt, d.entries);
cuda_err(cudaGetLastError());
cuda_err(cudaStreamSynchronize(d.stream));
}
@@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
d.hv, n);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
@@ -309,7 +325,7 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
if (n == 0)
return out;
std::vector<float> host(out.size());
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n);
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));