diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 2bf5d0840..f7d460acd 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3849,13 +3849,19 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector evaluator; #ifdef JFJOCH_USE_CUDA - // on the card where its states fit beside the engine - decided up front on the card's total, as the - // engines are; on the CPU otherwise, rather than failing at an allocation inside a sweep + // on every card, each with a part of the reflections, where a part's states fit beside the engines - + // decided up front on the card's total, as the engines are; on the CPU otherwise, rather than failing at + // an allocation inside a sweep if (ctx.engine->OnGPU()) { const size_t ops = FieldOperators(sg).size(); - const size_t wanted = SideChainFieldDeviceBytes(fr.Size(), ops), card = get_gpu_total_memory(ctx.device); + const int cards = get_gpu_count(); + const size_t wanted = SideChainFieldDeviceBytes(fr.Size() / cards + 1, ops), + card = get_gpu_total_memory(ctx.device); + std::vector devices; + for (int k = 0; k < cards; k++) + devices.push_back((ctx.device + k) % cards); if (wanted < card / 2) - evaluator = MakeSideChainFieldEvaluatorGPU(ctx.device, fr, FieldOperators(sg), elements); + evaluator = MakeSideChainFieldEvaluatorGPUs(devices, fr, FieldOperators(sg), elements); else logger.Info("Side-chain field on the CPU: {} MiB wanted on the card of {} MiB", wanted >> 20, card >> 20); diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index a11281e46..c738326e9 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j, t[r] = dot; } -// hv_k = sum over the parameter atom's entries of t_row J_e -__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, - int c0, const float2 *t, float2 *hv, int n) { +// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's +// entries read them one after another +__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) { + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q < entries) + jt[q] = j[entry[q]]; +} + +// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components, +// each component summed over the entries in their order +__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0, + const float2 *t, float2 *hv, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; - for (int c = 0; c < per; c++) { - float2 sum = make_float2(0, 0); - for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const int e = entry[q]; - AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); - } - hv[per * k + c] = sum; + float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)}; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float4 jq = jt[q]; + const float2 tr = t[row_at[q]]; + for (int c = 0; c < per; c++) + AddProduct(sum[c], Get(jq, c0 + c), tr); } + for (int c = 0; c < per; c++) + hv[per * k + c] = sum[c]; } -__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d, - int n) { +__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; for (int c = 0; c < per; c++) { float sum = 0; for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const float x = Get(j[entry[q]], c0 + c); + const float x = Get(jt[q], c0 + c); sum += x * x; } d[per * k + c] = sum; @@ -209,10 +218,10 @@ CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { struct RestraintNormalMatrixGPU::Device { int device; CudaStream stream; - int records = 0, rows = 0; + int records = 0, rows = 0, entries = 0; CudaDevicePtr record; - CudaDevicePtr plane_atoms, record_entry, start, param, row_of, entry_start, entry; - CudaDevicePtr xyzb, j; + CudaDevicePtr plane_atoms, record_entry, start, param, entry_start, entry, row_at; + CudaDevicePtr xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ) CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; PinnedVector host; // v and hv as float pairs @@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector &record d.record_entry = Up(record_entry, d.stream); d.start = Up(start, d.stream); d.param = Up(param, d.stream); - d.row_of = Up(row_of, d.stream); d.entry_start = Up(entry_start, d.stream); d.entry = Up(entry, d.stream); + // the row of each entry of the transpose, in its order + std::vector row_at(entry.size()); + for (size_t q = 0; q < entry.size(); q++) + row_at[q] = row_of[entry[q]]; + d.row_at = Up(row_at, d.stream); + d.entries = static_cast(entry.size()); d.j = CudaDevicePtr(std::max(param.size(), 1)); + d.jt = CudaDevicePtr(std::max(entry.size(), 1)); d.t = CudaDevicePtr(std::max(d.rows, 1)); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector> &xy if (d.records > 0) JacobianKernel<<>>(d.record, d.records, d.plane_atoms, d.record_entry, d.xyzb, d.j); + if (d.entries > 0) + TransposeJ<<>>(d.entry, d.j, d.jt, d.entries); cuda_err(cudaGetLastError()); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vecto cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); - TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, - d.hv, n); + TransposeProduct<<>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); @@ -309,7 +325,7 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { if (n == 0) return out; std::vector host(out.size()); - DiagonalKernel<<>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n); + DiagonalKernel<<>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp index 84a518fad..15e29d8e4 100644 --- a/image_analysis/structure_refinement/SideChainField.cpp +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -368,6 +369,80 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) { // the states of one variable: within the sweep budget where the copies move alone, MAX_STATES otherwise return rows * (16 * n_ops + 48) + 8 * std::max(static_cast(SWEEP_BUDGET), MAX_STATES * rows); } + +namespace { + +// The GPU evaluator over several cards, each holding a contiguous part of the reflections. -ln L is a sum over the +// reflections, so the cards evaluate their parts side by side and each combination's value is the sum of theirs, added +// in card order (the last bits differ from one card's, whose sum is not cut where the parts meet). +class EvaluatorCards : public SideChainFieldEvaluator { +public: + EvaluatorCards(const std::vector &devices, const FieldReflections &refl, + const std::vector> &ops, const std::vector &elements) { + const size_t n = devices.size(), rows = refl.Size(); + for (size_t k = 0; k <= n; k++) + begin_.push_back(rows * k / n); + for (size_t k = 0; k < n; k++) + parts_.push_back(MakeSideChainFieldEvaluatorGPU(devices[k], Slice(refl, begin_[k], begin_[k + 1]), ops, + elements)); + } + + void SetTotal(const std::vector> &f) override { + for (size_t k = 0; k < parts_.size(); k++) + parts_[k]->SetTotal(std::vector>(f.begin() + begin_[k], f.begin() + begin_[k + 1])); + } + + std::vector Evaluate(const FieldStates &states) override { + std::vector>> on; + for (auto &p : parts_) + on.push_back(std::async(std::launch::async, [&p, &states] { return p->Evaluate(states); })); + std::vector nll = on[0].get(); + for (size_t k = 1; k < on.size(); k++) { + const std::vector part = on[k].get(); + for (size_t c = 0; c < nll.size(); c++) + nll[c] += part[c]; + } + return nll; + } + + void Move(const FieldStates &states, int to) override { + std::vector> on; + for (auto &p : parts_) + on.push_back(std::async(std::launch::async, [&p, &states, to] { p->Move(states, to); })); + for (auto &f : on) + f.get(); + } + + std::vector> Contribution(const FieldStates &states, int s) override { + std::vector> f; + for (auto &p : parts_) { + const std::vector> part = p->Contribution(states, s); + f.insert(f.end(), part.begin(), part.end()); + } + return f; + } + +private: + static FieldReflections Slice(const FieldReflections &r, size_t lo, size_t hi) { + auto part = [lo, hi](const auto &v) { return std::decay_t(v.begin() + lo, v.begin() + hi); }; + return {part(r.hkl), part(r.stol2), part(r.fo), part(r.d), part(r.var), part(r.centric), part(r.work), + part(r.k_total), part(r.fmask)}; + } + + std::vector begin_; // the first row of each card's part, and the end + std::vector> parts_; +}; + +} // namespace + +std::unique_ptr MakeSideChainFieldEvaluatorGPUs(const std::vector &devices, + const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements) { + if (devices.size() == 1) + return MakeSideChainFieldEvaluatorGPU(devices[0], refl, ops, elements); + return std::make_unique(devices, refl, ops, elements); +} #endif std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h index 409b3a896..e5be5fec1 100644 --- a/image_analysis/structure_refinement/SideChainField.h +++ b/image_analysis/structure_refinement/SideChainField.h @@ -104,6 +104,12 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops); std::unique_ptr MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl, const std::vector> &ops, const std::vector &elements); +// On every card of `devices`, the reflections split among them in contiguous parts that the cards evaluate side by +// side; on one card, the evaluator above. +std::unique_ptr MakeSideChainFieldEvaluatorGPUs(const std::vector &devices, + const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements); #endif // A residue the field moved: its atoms (index in the model's atom list) at their new positions, or absent. diff --git a/image_analysis/structure_refinement/SideChainFieldGPU.cu b/image_analysis/structure_refinement/SideChainFieldGPU.cu index 37f042cc4..fe47eb352 100644 --- a/image_analysis/structure_refinement/SideChainFieldGPU.cu +++ b/image_analysis/structure_refinement/SideChainFieldGPU.cu @@ -156,6 +156,8 @@ public: cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2))); } ~EvaluatorGPU() override { + int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread + cudaGetDevice(&previous); cudaSetDevice(device_); for (void *p : {static_cast(hop_), static_cast(stol2_), static_cast(ff_), static_cast(fo_), static_cast(d_), static_cast(var_), @@ -165,6 +167,7 @@ public: static_cast(combo_states_), static_cast(combo_start_)}) cudaFree(p); cudaStreamDestroy(stream_); + cudaSetDevice(previous); } void SetTotal(const std::vector> &f) override { diff --git a/tests/SideChainFieldTest.cpp b/tests/SideChainFieldTest.cpp index 0a050764a..0d2fe3f8f 100644 --- a/tests/SideChainFieldTest.cpp +++ b/tests/SideChainFieldTest.cpp @@ -194,6 +194,17 @@ TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") { REQUIRE(g.moves.size() == 1); CHECK(g.moves[0].chi_after == r.moves[0].chi_after); CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4)); + // on every card, the reflections split among them: the same move, the energy summed in another order + int n = 0; + cudaGetDeviceCount(&n); + std::vector cards; + for (int k = 0; k < n; k++) + cards.push_back(k); + auto split = MakeSideChainFieldEvaluatorGPUs(cards, fr, ops, ff); + const SideChainFieldResult s = run(*split); + REQUIRE(s.moves.size() == 1); + CHECK(s.moves[0].chi_after == g.moves[0].chi_after); + CHECK(s.energy_after == Catch::Approx(g.energy_after).epsilon(1e-9)); #endif } }