From f07493c9bfdbd2660cd70b836c9587d63cca9e01 Mon Sep 17 00:00:00 2001 From: jungfrau Date: Tue, 17 Nov 2026 11:27:24 +0100 Subject: [PATCH 1/3] Side-chain field: the reflections split across every card The field's search is iterated conditional modes over the residues, one after another, and each step evaluates -ln L over every working reflection for each combination of states. On a large model it ran on the main engine's card alone while the others idled: a 242k-atom model with 45 NCS copies and 663k reflections spent 1743 s of a 2392 s refinement in it on one of four T4s. -ln L is a sum over the reflections, so MakeSideChainFieldEvaluatorGPUs gives each card a contiguous part of them and adds the parts' sums in card order. The search, its order and its decisions are unchanged; only the last bits of each sum move, where the parts meet. One card keeps the evaluator as it was. On that model, 4 x T4: the field 1743 -> 456 s, the refinement 2392 -> 1119 s, the refined model bit-identical (same R-free, same md5). Two small models refine bit-identically, in the same time. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL --- .../structure_refinement/Refinement.cpp | 14 +++- .../structure_refinement/SideChainField.cpp | 75 +++++++++++++++++++ .../structure_refinement/SideChainField.h | 6 ++ tests/SideChainFieldTest.cpp | 11 +++ 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 2bf5d0840..f7d460acd 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3849,13 +3849,19 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector evaluator; #ifdef JFJOCH_USE_CUDA - // on the card where its states fit beside the engine - decided up front on the card's total, as the - // engines are; on the CPU otherwise, rather than failing at an allocation inside a sweep + // on every card, each with a part of the reflections, where a part's states fit beside the engines - + // decided up front on the card's total, as the engines are; on the CPU otherwise, rather than failing at + // an allocation inside a sweep if (ctx.engine->OnGPU()) { const size_t ops = FieldOperators(sg).size(); - const size_t wanted = SideChainFieldDeviceBytes(fr.Size(), ops), card = get_gpu_total_memory(ctx.device); + const int cards = get_gpu_count(); + const size_t wanted = SideChainFieldDeviceBytes(fr.Size() / cards + 1, ops), + card = get_gpu_total_memory(ctx.device); + std::vector devices; + for (int k = 0; k < cards; k++) + devices.push_back((ctx.device + k) % cards); if (wanted < card / 2) - evaluator = MakeSideChainFieldEvaluatorGPU(ctx.device, fr, FieldOperators(sg), elements); + evaluator = MakeSideChainFieldEvaluatorGPUs(devices, fr, FieldOperators(sg), elements); else logger.Info("Side-chain field on the CPU: {} MiB wanted on the card of {} MiB", wanted >> 20, card >> 20); diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp index 84a518fad..15e29d8e4 100644 --- a/image_analysis/structure_refinement/SideChainField.cpp +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -368,6 +369,80 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) { // the states of one variable: within the sweep budget where the copies move alone, MAX_STATES otherwise return rows * (16 * n_ops + 48) + 8 * std::max(static_cast(SWEEP_BUDGET), MAX_STATES * rows); } + +namespace { + +// The GPU evaluator over several cards, each holding a contiguous part of the reflections. -ln L is a sum over the +// reflections, so the cards evaluate their parts side by side and each combination's value is the sum of theirs, added +// in card order (the last bits differ from one card's, whose sum is not cut where the parts meet). +class EvaluatorCards : public SideChainFieldEvaluator { +public: + EvaluatorCards(const std::vector &devices, const FieldReflections &refl, + const std::vector> &ops, const std::vector &elements) { + const size_t n = devices.size(), rows = refl.Size(); + for (size_t k = 0; k <= n; k++) + begin_.push_back(rows * k / n); + for (size_t k = 0; k < n; k++) + parts_.push_back(MakeSideChainFieldEvaluatorGPU(devices[k], Slice(refl, begin_[k], begin_[k + 1]), ops, + elements)); + } + + void SetTotal(const std::vector> &f) override { + for (size_t k = 0; k < parts_.size(); k++) + parts_[k]->SetTotal(std::vector>(f.begin() + begin_[k], f.begin() + begin_[k + 1])); + } + + std::vector Evaluate(const FieldStates &states) override { + std::vector>> on; + for (auto &p : parts_) + on.push_back(std::async(std::launch::async, [&p, &states] { return p->Evaluate(states); })); + std::vector nll = on[0].get(); + for (size_t k = 1; k < on.size(); k++) { + const std::vector part = on[k].get(); + for (size_t c = 0; c < nll.size(); c++) + nll[c] += part[c]; + } + return nll; + } + + void Move(const FieldStates &states, int to) override { + std::vector> on; + for (auto &p : parts_) + on.push_back(std::async(std::launch::async, [&p, &states, to] { p->Move(states, to); })); + for (auto &f : on) + f.get(); + } + + std::vector> Contribution(const FieldStates &states, int s) override { + std::vector> f; + for (auto &p : parts_) { + const std::vector> part = p->Contribution(states, s); + f.insert(f.end(), part.begin(), part.end()); + } + return f; + } + +private: + static FieldReflections Slice(const FieldReflections &r, size_t lo, size_t hi) { + auto part = [lo, hi](const auto &v) { return std::decay_t(v.begin() + lo, v.begin() + hi); }; + return {part(r.hkl), part(r.stol2), part(r.fo), part(r.d), part(r.var), part(r.centric), part(r.work), + part(r.k_total), part(r.fmask)}; + } + + std::vector begin_; // the first row of each card's part, and the end + std::vector> parts_; +}; + +} // namespace + +std::unique_ptr MakeSideChainFieldEvaluatorGPUs(const std::vector &devices, + const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements) { + if (devices.size() == 1) + return MakeSideChainFieldEvaluatorGPU(devices[0], refl, ops, elements); + return std::make_unique(devices, refl, ops, elements); +} #endif std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h index 409b3a896..e5be5fec1 100644 --- a/image_analysis/structure_refinement/SideChainField.h +++ b/image_analysis/structure_refinement/SideChainField.h @@ -104,6 +104,12 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops); std::unique_ptr MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl, const std::vector> &ops, const std::vector &elements); +// On every card of `devices`, the reflections split among them in contiguous parts that the cards evaluate side by +// side; on one card, the evaluator above. +std::unique_ptr MakeSideChainFieldEvaluatorGPUs(const std::vector &devices, + const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements); #endif // A residue the field moved: its atoms (index in the model's atom list) at their new positions, or absent. diff --git a/tests/SideChainFieldTest.cpp b/tests/SideChainFieldTest.cpp index 0a050764a..0d2fe3f8f 100644 --- a/tests/SideChainFieldTest.cpp +++ b/tests/SideChainFieldTest.cpp @@ -194,6 +194,17 @@ TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") { REQUIRE(g.moves.size() == 1); CHECK(g.moves[0].chi_after == r.moves[0].chi_after); CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4)); + // on every card, the reflections split among them: the same move, the energy summed in another order + int n = 0; + cudaGetDeviceCount(&n); + std::vector cards; + for (int k = 0; k < n; k++) + cards.push_back(k); + auto split = MakeSideChainFieldEvaluatorGPUs(cards, fr, ops, ff); + const SideChainFieldResult s = run(*split); + REQUIRE(s.moves.size() == 1); + CHECK(s.moves[0].chi_after == g.moves[0].chi_after); + CHECK(s.energy_after == Catch::Approx(g.energy_after).epsilon(1e-9)); #endif } } From 3e76b880df038a831dadd91b2df52f2b291275b0 Mon Sep 17 00:00:00 2001 From: jungfrau Date: Tue, 17 Nov 2026 14:09:27 +0100 Subject: [PATCH 2/3] Side-chain field GPU evaluator: give back the caller's device when it goes ~EvaluatorGPU switched to its card to free its buffers and left the thread there. With the field on every card (f07493c9b) the evaluators are destroyed on the refinement's thread, which was then left on the last card: the engines the second round makes again (a raised B ceiling) went with it, so the main engine landed on a card already holding its share of slots. Measured on a 242k-atom model on 4 x T4, per card, against the footprint OwnSlotPlaces plans (11290 MiB on every card): with the fix the untwinned arm peaks at 11188 / 11124 / 10836 / 10996 MiB (the field's search adds about 450 MiB per card, freed after it); without it card 3 reached 14295 MiB, the main engine there beside the card's slots. The plan is an upper bound once the engine stays on its card. The refined model is bit-identical, with the same 16 trials at a time in both arms (1116 s). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL --- image_analysis/structure_refinement/SideChainFieldGPU.cu | 3 +++ 1 file changed, 3 insertions(+) diff --git a/image_analysis/structure_refinement/SideChainFieldGPU.cu b/image_analysis/structure_refinement/SideChainFieldGPU.cu index 37f042cc4..fe47eb352 100644 --- a/image_analysis/structure_refinement/SideChainFieldGPU.cu +++ b/image_analysis/structure_refinement/SideChainFieldGPU.cu @@ -156,6 +156,8 @@ public: cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2))); } ~EvaluatorGPU() override { + int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread + cudaGetDevice(&previous); cudaSetDevice(device_); for (void *p : {static_cast(hop_), static_cast(stol2_), static_cast(ff_), static_cast(fo_), static_cast(d_), static_cast(var_), @@ -165,6 +167,7 @@ public: static_cast(combo_states_), static_cast(combo_start_)}) cudaFree(p); cudaStreamDestroy(stream_); + cudaSetDevice(previous); } void SetTotal(const std::vector> &f) override { From 9d54253d2bce26ba81e2dddbabd55f8729bdd421 Mon Sep 17 00:00:00 2001 From: jungfrau Date: Tue, 17 Nov 2026 14:33:24 +0100 Subject: [PATCH 3/3] Restraint normal matrix on the GPU: J^T t streamed in the parameter atoms' order TransposeProduct summed each component over an atom's entries in a loop of its own, so every entry's index, J and row were gathered once per component, in random sectors. Now the entries are read once, each component's float-pair sum updated in the same order as before; and J is copied into the transpose's order once per At() (TransposeJ, beside JacobianKernel in Fill), with the row of each entry indexed once at Upload, so the product streams J and the rows and gathers only t. The DiagonalKernel reads the same copy. Every sum keeps its terms and their order: the products and the diagonal are bit-identical to the old kernel's (all four cases of RefinementNewton_product_GPU_vs_CPU dumped and compared), at +20 B per entry of device memory. On a 242k-atom model with 45 NCS copies, 4 x T4, the first ladder (4 trials per card): per product, xyz without NCS 3.12 -> 0.89 ms, with NCS 11.1 -> 2.88 ms, B unchanged (0.44 ms); TransposeJ 0.29 ms per At(). The refinement 1116 -> 1065 s, the refined model bit-identical, the card peak 11.3 GB. Two small models: bit-identical, slightly faster. Following the review in TPROD_REPORT.md (gather amplification diagnosis, options A and B). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL --- .../RefinementNewtonGPU.cu | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/image_analysis/structure_refinement/RefinementNewtonGPU.cu b/image_analysis/structure_refinement/RefinementNewtonGPU.cu index a11281e46..c738326e9 100644 --- a/image_analysis/structure_refinement/RefinementNewtonGPU.cu +++ b/image_analysis/structure_refinement/RefinementNewtonGPU.cu @@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j, t[r] = dot; } -// hv_k = sum over the parameter atom's entries of t_row J_e -__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per, - int c0, const float2 *t, float2 *hv, int n) { +// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's +// entries read them one after another +__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) { + const int q = blockIdx.x * blockDim.x + threadIdx.x; + if (q < entries) + jt[q] = j[entry[q]]; +} + +// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components, +// each component summed over the entries in their order +__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0, + const float2 *t, float2 *hv, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; - for (int c = 0; c < per; c++) { - float2 sum = make_float2(0, 0); - for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const int e = entry[q]; - AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]); - } - hv[per * k + c] = sum; + float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)}; + for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { + const float4 jq = jt[q]; + const float2 tr = t[row_at[q]]; + for (int c = 0; c < per; c++) + AddProduct(sum[c], Get(jq, c0 + c), tr); } + for (int c = 0; c < per; c++) + hv[per * k + c] = sum[c]; } -__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d, - int n) { +__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) { const int k = blockIdx.x * blockDim.x + threadIdx.x; if (k >= n) return; for (int c = 0; c < per; c++) { float sum = 0; for (int q = entry_start[k]; q < entry_start[k + 1]; q++) { - const float x = Get(j[entry[q]], c0 + c); + const float x = Get(jt[q], c0 + c); sum += x * x; } d[per * k + c] = sum; @@ -209,10 +218,10 @@ CudaDevicePtr Up(const std::vector &host, cudaStream_t stream) { struct RestraintNormalMatrixGPU::Device { int device; CudaStream stream; - int records = 0, rows = 0; + int records = 0, rows = 0, entries = 0; CudaDevicePtr record; - CudaDevicePtr plane_atoms, record_entry, start, param, row_of, entry_start, entry; - CudaDevicePtr xyzb, j; + CudaDevicePtr plane_atoms, record_entry, start, param, entry_start, entry, row_at; + CudaDevicePtr xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ) CudaDevicePtr t, v, hv; CudaDevicePtr diagonal; PinnedVector host; // v and hv as float pairs @@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector &record d.record_entry = Up(record_entry, d.stream); d.start = Up(start, d.stream); d.param = Up(param, d.stream); - d.row_of = Up(row_of, d.stream); d.entry_start = Up(entry_start, d.stream); d.entry = Up(entry, d.stream); + // the row of each entry of the transpose, in its order + std::vector row_at(entry.size()); + for (size_t q = 0; q < entry.size(); q++) + row_at[q] = row_of[entry[q]]; + d.row_at = Up(row_at, d.stream); + d.entries = static_cast(entry.size()); d.j = CudaDevicePtr(std::max(param.size(), 1)); + d.jt = CudaDevicePtr(std::max(entry.size(), 1)); d.t = CudaDevicePtr(std::max(d.rows, 1)); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector> &xy if (d.records > 0) JacobianKernel<<>>(d.record, d.records, d.plane_atoms, d.record_entry, d.xyzb, d.j); + if (d.entries > 0) + TransposeJ<<>>(d.entry, d.j, d.jt, d.entries); cuda_err(cudaGetLastError()); cuda_err(cudaStreamSynchronize(d.stream)); } @@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector &v, std::vecto cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream)); if (d.rows > 0) RowProduct<<>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows); - TransposeProduct<<>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t, - d.hv, n); + TransposeProduct<<>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream)); @@ -309,7 +325,7 @@ std::vector RestraintNormalMatrixGPU::Diagonal() const { if (n == 0) return out; std::vector host(out.size()); - DiagonalKernel<<>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n); + DiagonalKernel<<>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n); cuda_err(cudaGetLastError()); cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream)); cuda_err(cudaStreamSynchronize(d.stream));