diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index a11281e46..c738326e9 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j, t[r] = dot; } -// hv_k = sum over the parameter atom's entries of t_row J_e -__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, - int c0, const float2 *t, float2 *hv, int n) { +// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's +// entries read them one after another +__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) { + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q < entries) + jt[q] = j[entry[q]]; +} + +// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components, +// each component summed over the entries in their order +__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0, + const float2 *t, float2 *hv, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; - for (int c = 0; c < per; c++) { - float2 sum = make_float2(0, 0); - for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const int e = entry[q]; - AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); - } - hv[per * k + c] = sum; + float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)}; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float4 jq = jt[q]; + const float2 tr = t[row_at[q]]; + for (int c = 0; c < per; c++) + AddProduct(sum[c], Get(jq, c0 + c), tr); } + for (int c = 0; c < per; c++) + hv[per * k + c] = sum[c]; } -__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d, - int n) { +__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; for (int c = 0; c < per; c++) { float sum = 0; for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const float x = Get(j[entry[q]], c0 + c); + const float x = Get(jt[q], c0 + c); sum += x * x; } d[per * k + c] = sum; @@ -209,10 +218,10 @@ CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { struct RestraintNormalMatrixGPU::Device { int device; CudaStream stream; - int records = 0, rows = 0; + int records = 0, rows = 0, entries = 0; CudaDevicePtr record; - CudaDevicePtr plane_atoms, record_entry, start, param, row_of, entry_start, entry; - CudaDevicePtr xyzb, j; + CudaDevicePtr plane_atoms, record_entry, start, param, entry_start, entry, row_at; + CudaDevicePtr xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ) CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; PinnedVector host; // v and hv as float pairs @@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector &record d.record_entry = Up(record_entry, d.stream); d.start = Up(start, d.stream); d.param = Up(param, d.stream); - d.row_of = Up(row_of, d.stream); d.entry_start = Up(entry_start, d.stream); d.entry = Up(entry, d.stream); + // the row of each entry of the transpose, in its order + std::vector row_at(entry.size()); + for (size_t q = 0; q < entry.size(); q++) + row_at[q] = row_of[entry[q]]; + d.row_at = Up(row_at, d.stream); + d.entries = static_cast(entry.size()); d.j = CudaDevicePtr(std::max(param.size(), 1)); + d.jt = CudaDevicePtr(std::max(entry.size(), 1)); d.t = CudaDevicePtr(std::max(d.rows, 1)); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector> &xy if (d.records > 0) JacobianKernel<<>>(d.record, d.records, d.plane_atoms, d.record_entry, d.xyzb, d.j); + if (d.entries > 0) + TransposeJ<<>>(d.entry, d.j, d.jt, d.entries); cuda_err(cudaGetLastError()); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vecto cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); - TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, - d.hv, n); + TransposeProduct<<>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); @@ -309,7 +325,7 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { if (n == 0) return out; std::vector host(out.size()); - DiagonalKernel<<>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n); + DiagonalKernel<<>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream));