Restraint normal matrix on the GPU: J^T t streamed in the parameter atoms' order
TransposeProduct summed each component over an atom's entries in a loop of its own, so every entry's index, J and row were gathered once per component, in random sectors. Now the entries are read once, each component's float-pair sum updated in the same order as before; and J is copied into the transpose's order once per At() (TransposeJ, beside JacobianKernel in Fill), with the row of each entry indexed once at Upload, so the product streams J and the rows and gathers only t. The DiagonalKernel reads the same copy. Every sum keeps its terms and their order: the products and the diagonal are bit-identical to the old kernel's (all four cases of RefinementNewton_product_GPU_vs_CPU dumped and compared), at +20 B per entry of device memory. On a 242k-atom model with 45 NCS copies, 4 x T4, the first ladder (4 trials per card): per product, xyz without NCS 3.12 -> 0.89 ms, with NCS 11.1 -> 2.88 ms, B unchanged (0.44 ms); TransposeJ 0.29 ms per At(). The refinement 1116 -> 1065 s, the refined model bit-identical, the card peak 11.3 GB. Two small models: bit-identical, slightly faster. Following the review in TPROD_REPORT.md (gather amplification diagnosis, options A and B). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
This commit is contained in:
1 parent
3e76b880df
commit
9d54253d2b
1 file changed
+36
-20
@@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j,
|
||||
t[r] = dot;
|
||||
}
|
||||
|
||||
// hv_k = sum over the parameter atom's entries of t_row J_e
|
||||
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
|
||||
int c0, const float2 *t, float2 *hv, int n) {
|
||||
// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's
|
||||
// entries read them one after another
|
||||
__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) {
|
||||
const int q = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (q < entries)
|
||||
jt[q] = j[entry[q]];
|
||||
}
|
||||
|
||||
// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components,
|
||||
// each component summed over the entries in their order
|
||||
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
|
||||
const float2 *t, float2 *hv, int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
for (int c = 0; c < per; c++) {
|
||||
float2 sum = make_float2(0, 0);
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const int e = entry[q];
|
||||
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
|
||||
}
|
||||
hv[per * k + c] = sum;
|
||||
float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)};
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const float4 jq = jt[q];
|
||||
const float2 tr = t[row_at[q]];
|
||||
for (int c = 0; c < per; c++)
|
||||
AddProduct(sum[c], Get(jq, c0 + c), tr);
|
||||
}
|
||||
for (int c = 0; c < per; c++)
|
||||
hv[per * k + c] = sum[c];
|
||||
}
|
||||
|
||||
__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d,
|
||||
int n) {
|
||||
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
|
||||
const int k = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (k >= n)
|
||||
return;
|
||||
for (int c = 0; c < per; c++) {
|
||||
float sum = 0;
|
||||
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
|
||||
const float x = Get(j[entry[q]], c0 + c);
|
||||
const float x = Get(jt[q], c0 + c);
|
||||
sum += x * x;
|
||||
}
|
||||
d[per * k + c] = sum;
|
||||
@@ -209,10 +218,10 @@ CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
|
||||
struct RestraintNormalMatrixGPU::Device {
|
||||
int device;
|
||||
CudaStream stream;
|
||||
int records = 0, rows = 0;
|
||||
int records = 0, rows = 0, entries = 0;
|
||||
CudaDevicePtr<RestraintRecord> record;
|
||||
CudaDevicePtr<int> plane_atoms, record_entry, start, param, row_of, entry_start, entry;
|
||||
CudaDevicePtr<float4> xyzb, j;
|
||||
CudaDevicePtr<int> plane_atoms, record_entry, start, param, entry_start, entry, row_at;
|
||||
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
|
||||
CudaDevicePtr<float2> t, v, hv;
|
||||
CudaDevicePtr<float> diagonal;
|
||||
PinnedVector<float2> host; // v and hv as float pairs
|
||||
@@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &record
|
||||
d.record_entry = Up(record_entry, d.stream);
|
||||
d.start = Up(start, d.stream);
|
||||
d.param = Up(param, d.stream);
|
||||
d.row_of = Up(row_of, d.stream);
|
||||
d.entry_start = Up(entry_start, d.stream);
|
||||
d.entry = Up(entry, d.stream);
|
||||
// the row of each entry of the transpose, in its order
|
||||
std::vector<int> row_at(entry.size());
|
||||
for (size_t q = 0; q < entry.size(); q++)
|
||||
row_at[q] = row_of[entry[q]];
|
||||
d.row_at = Up(row_at, d.stream);
|
||||
d.entries = static_cast<int>(entry.size());
|
||||
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
|
||||
d.jt = CudaDevicePtr<float4>(std::max<size_t>(entry.size(), 1));
|
||||
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
}
|
||||
@@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xy
|
||||
if (d.records > 0)
|
||||
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
|
||||
d.record_entry, d.xyzb, d.j);
|
||||
if (d.entries > 0)
|
||||
TransposeJ<<<GridSize(d.entries), THREADS, 0, d.stream>>>(d.entry, d.j, d.jt, d.entries);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
}
|
||||
@@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
|
||||
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
|
||||
if (d.rows > 0)
|
||||
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
|
||||
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
|
||||
d.hv, n);
|
||||
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
@@ -309,7 +325,7 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
|
||||
if (n == 0)
|
||||
return out;
|
||||
std::vector<float> host(out.size());
|
||||
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n);
|
||||
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n);
|
||||
cuda_err(cudaGetLastError());
|
||||
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
|
||||
cuda_err(cudaStreamSynchronize(d.stream));
|
||||
|
||||
Reference in new issue
Block a user