Merge branch 'rc188-m' into rc188-int

This commit is contained in:
leonarski_f committed 2026-11-17 15:04:53 +01:00
commit d7a2fee624
6 files changed
+141 -24

No files matched your search

@@ -3849,13 +3849,19 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
}
std::unique_ptr<SideChainFieldEvaluator> evaluator;
#ifdef JFJOCH_USE_CUDA
// on the card where its states fit beside the engine - decided up front on the card's total, as the
// engines are; on the CPU otherwise, rather than failing at an allocation inside a sweep
// on every card, each with a part of the reflections, where a part's states fit beside the engines -
// decided up front on the card's total, as the engines are; on the CPU otherwise, rather than failing at
// an allocation inside a sweep
if (ctx.engine->OnGPU()) {
const size_t ops = FieldOperators(sg).size();
const size_t wanted = SideChainFieldDeviceBytes(fr.Size(), ops), card = get_gpu_total_memory(ctx.device);
const int cards = get_gpu_count();
const size_t wanted = SideChainFieldDeviceBytes(fr.Size() / cards + 1, ops),
card = get_gpu_total_memory(ctx.device);
std::vector<int> devices;
for (int k = 0; k < cards; k++)
devices.push_back((ctx.device + k) % cards);
if (wanted < card / 2)
evaluator = MakeSideChainFieldEvaluatorGPU(ctx.device, fr, FieldOperators(sg), elements);
evaluator = MakeSideChainFieldEvaluatorGPUs(devices, fr, FieldOperators(sg), elements);
else
logger.Info("Side-chain field on the CPU: {} MiB wanted on the card of {} MiB", wanted >> 20,
card >> 20);
@@ -166,31 +166,40 @@ __global__ void RowProduct(const int *start, const int *param, const float4 *j,
t[r] = dot;
}
// hv_k = sum over the parameter atom's entries of t_row J_e
__global__ void TransposeProduct(const int *entry_start, const int *entry, const int *row_of, const float4 *j, int per,
int c0, const float2 *t, float2 *hv, int n) {
// J in the parameter atoms' order: jt_q = J of entry q of the transpose (entry[q]), so that the products over an atom's
// entries read them one after another
__global__ void TransposeJ(const int *entry, const float4 *j, float4 *jt, int entries) {
const int q = blockIdx.x * blockDim.x + threadIdx.x;
if (q < entries)
jt[q] = j[entry[q]];
}
// hv_k = sum over the parameter atom's entries of t_row J_e: each entry read once for all of the atom's components,
// each component summed over the entries in their order
__global__ void TransposeProduct(const int *entry_start, const float4 *jt, const int *row_at, int per, int c0,
const float2 *t, float2 *hv, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float2 sum = make_float2(0, 0);
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const int e = entry[q];
AddProduct(sum, Get(j[e], c0 + c), t[row_of[e]]);
}
hv[per * k + c] = sum;
float2 sum[3] = {make_float2(0, 0), make_float2(0, 0), make_float2(0, 0)};
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float4 jq = jt[q];
const float2 tr = t[row_at[q]];
for (int c = 0; c < per; c++)
AddProduct(sum[c], Get(jq, c0 + c), tr);
}
for (int c = 0; c < per; c++)
hv[per * k + c] = sum[c];
}
__global__ void DiagonalKernel(const int *entry_start, const int *entry, const float4 *j, int per, int c0, float *d,
int n) {
__global__ void DiagonalKernel(const int *entry_start, const float4 *jt, int per, int c0, float *d, int n) {
const int k = blockIdx.x * blockDim.x + threadIdx.x;
if (k >= n)
return;
for (int c = 0; c < per; c++) {
float sum = 0;
for (int q = entry_start[k]; q < entry_start[k + 1]; q++) {
const float x = Get(j[entry[q]], c0 + c);
const float x = Get(jt[q], c0 + c);
sum += x * x;
}
d[per * k + c] = sum;
@@ -209,10 +218,10 @@ CudaDevicePtr<T> Up(const std::vector<T> &host, cudaStream_t stream) {
struct RestraintNormalMatrixGPU::Device {
int device;
CudaStream stream;
int records = 0, rows = 0;
int records = 0, rows = 0, entries = 0;
CudaDevicePtr<RestraintRecord> record;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, row_of, entry_start, entry;
CudaDevicePtr<float4> xyzb, j;
CudaDevicePtr<int> plane_atoms, record_entry, start, param, entry_start, entry, row_at;
CudaDevicePtr<float4> xyzb, j, jt; // J per entry in row order, and in the parameter atoms' order (TransposeJ)
CudaDevicePtr<float2> t, v, hv;
CudaDevicePtr<float> diagonal;
PinnedVector<float2> host; // v and hv as float pairs
@@ -255,10 +264,16 @@ void RestraintNormalMatrixGPU::Upload(const std::vector<RestraintRecord> &record
d.record_entry = Up(record_entry, d.stream);
d.start = Up(start, d.stream);
d.param = Up(param, d.stream);
d.row_of = Up(row_of, d.stream);
d.entry_start = Up(entry_start, d.stream);
d.entry = Up(entry, d.stream);
// the row of each entry of the transpose, in its order
std::vector<int> row_at(entry.size());
for (size_t q = 0; q < entry.size(); q++)
row_at[q] = row_of[entry[q]];
d.row_at = Up(row_at, d.stream);
d.entries = static_cast<int>(entry.size());
d.j = CudaDevicePtr<float4>(std::max<size_t>(param.size(), 1));
d.jt = CudaDevicePtr<float4>(std::max<size_t>(entry.size(), 1));
d.t = CudaDevicePtr<float2>(std::max(d.rows, 1));
cuda_err(cudaStreamSynchronize(d.stream));
}
@@ -274,6 +289,8 @@ void RestraintNormalMatrixGPU::Fill(const std::vector<std::array<double, 4>> &xy
if (d.records > 0)
JacobianKernel<<<GridSize(d.records), THREADS, 0, d.stream>>>(d.record, d.records, d.plane_atoms,
d.record_entry, d.xyzb, d.j);
if (d.entries > 0)
TransposeJ<<<GridSize(d.entries), THREADS, 0, d.stream>>>(d.entry, d.j, d.jt, d.entries);
cuda_err(cudaGetLastError());
cuda_err(cudaStreamSynchronize(d.stream));
}
@@ -292,8 +309,7 @@ void RestraintNormalMatrixGPU::Multiply(const std::vector<double> &v, std::vecto
cuda_err(cudaMemcpyAsync(d.v.get(), d.host.data(), hv.size() * sizeof(float2), cudaMemcpyHostToDevice, d.stream));
if (d.rows > 0)
RowProduct<<<GridSize(d.rows), THREADS, 0, d.stream>>>(d.start, d.param, d.j, per_, c0_, d.v, d.t, d.rows);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.row_of, d.j, per_, c0_, d.t,
d.hv, n);
TransposeProduct<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, d.row_at, per_, c0_, d.t, d.hv, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(d.host.data(), d.hv.get(), hv.size() * sizeof(float2), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
@@ -309,7 +325,7 @@ std::vector<double> RestraintNormalMatrixGPU::Diagonal() const {
if (n == 0)
return out;
std::vector<float> host(out.size());
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.entry, d.j, per_, c0_, d.diagonal, n);
DiagonalKernel<<<GridSize(n), THREADS, 0, d.stream>>>(d.entry_start, d.jt, per_, c0_, d.diagonal, n);
cuda_err(cudaGetLastError());
cuda_err(cudaMemcpyAsync(host.data(), d.diagonal.get(), host.size() * sizeof(float), cudaMemcpyDeviceToHost, d.stream));
cuda_err(cudaStreamSynchronize(d.stream));
@@ -5,6 +5,7 @@
#include <algorithm>
#include <cmath>
#include <future>
#include <map>
#include <optional>
#include <set>
@@ -368,6 +369,80 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) {
// the states of one variable: within the sweep budget where the copies move alone, MAX_STATES otherwise
return rows * (16 * n_ops + 48) + 8 * std::max(static_cast<size_t>(SWEEP_BUDGET), MAX_STATES * rows);
}
namespace {
// The GPU evaluator over several cards, each holding a contiguous part of the reflections. -ln L is a sum over the
// reflections, so the cards evaluate their parts side by side and each combination's value is the sum of theirs, added
// in card order (the last bits differ from one card's, whose sum is not cut where the parts meet).
class EvaluatorCards : public SideChainFieldEvaluator {
public:
EvaluatorCards(const std::vector<int> &devices, const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops, const std::vector<FormFactor> &elements) {
const size_t n = devices.size(), rows = refl.Size();
for (size_t k = 0; k <= n; k++)
begin_.push_back(rows * k / n);
for (size_t k = 0; k < n; k++)
parts_.push_back(MakeSideChainFieldEvaluatorGPU(devices[k], Slice(refl, begin_[k], begin_[k + 1]), ops,
elements));
}
void SetTotal(const std::vector<std::complex<float>> &f) override {
for (size_t k = 0; k < parts_.size(); k++)
parts_[k]->SetTotal(std::vector<std::complex<float>>(f.begin() + begin_[k], f.begin() + begin_[k + 1]));
}
std::vector<double> Evaluate(const FieldStates &states) override {
std::vector<std::future<std::vector<double>>> on;
for (auto &p : parts_)
on.push_back(std::async(std::launch::async, [&p, &states] { return p->Evaluate(states); }));
std::vector<double> nll = on[0].get();
for (size_t k = 1; k < on.size(); k++) {
const std::vector<double> part = on[k].get();
for (size_t c = 0; c < nll.size(); c++)
nll[c] += part[c];
}
return nll;
}
void Move(const FieldStates &states, int to) override {
std::vector<std::future<void>> on;
for (auto &p : parts_)
on.push_back(std::async(std::launch::async, [&p, &states, to] { p->Move(states, to); }));
for (auto &f : on)
f.get();
}
std::vector<std::complex<float>> Contribution(const FieldStates &states, int s) override {
std::vector<std::complex<float>> f;
for (auto &p : parts_) {
const std::vector<std::complex<float>> part = p->Contribution(states, s);
f.insert(f.end(), part.begin(), part.end());
}
return f;
}
private:
static FieldReflections Slice(const FieldReflections &r, size_t lo, size_t hi) {
auto part = [lo, hi](const auto &v) { return std::decay_t<decltype(v)>(v.begin() + lo, v.begin() + hi); };
return {part(r.hkl), part(r.stol2), part(r.fo), part(r.d), part(r.var), part(r.centric), part(r.work),
part(r.k_total), part(r.fmask)};
}
std::vector<size_t> begin_; // the first row of each card's part, and the end
std::vector<std::unique_ptr<SideChainFieldEvaluator>> parts_;
};
} // namespace
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorGPUs(const std::vector<int> &devices,
const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements) {
if (devices.size() == 1)
return MakeSideChainFieldEvaluatorGPU(devices[0], refl, ops, elements);
return std::make_unique<EvaluatorCards>(devices, refl, ops, elements);
}
#endif
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl,
@@ -104,6 +104,12 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops);
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements);
// On every card of `devices`, the reflections split among them in contiguous parts that the cards evaluate side by
// side; on one card, the evaluator above.
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorGPUs(const std::vector<int> &devices,
const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements);
#endif
// A residue the field moved: its atoms (index in the model's atom list) at their new positions, or absent.
@@ -156,6 +156,8 @@ public:
cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2)));
}
~EvaluatorGPU() override {
int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread
cudaGetDevice(&previous);
cudaSetDevice(device_);
for (void *p : {static_cast<void *>(hop_), static_cast<void *>(stol2_), static_cast<void *>(ff_),
static_cast<void *>(fo_), static_cast<void *>(d_), static_cast<void *>(var_),
@@ -165,6 +167,7 @@ public:
static_cast<void *>(combo_states_), static_cast<void *>(combo_start_)})
cudaFree(p);
cudaStreamDestroy(stream_);
cudaSetDevice(previous);
}
void SetTotal(const std::vector<std::complex<float>> &f) override {
+11
View File
@@ -194,6 +194,17 @@ TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") {
REQUIRE(g.moves.size() == 1);
CHECK(g.moves[0].chi_after == r.moves[0].chi_after);
CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4));
// on every card, the reflections split among them: the same move, the energy summed in another order
int n = 0;
cudaGetDeviceCount(&n);
std::vector<int> cards;
for (int k = 0; k < n; k++)
cards.push_back(k);
auto split = MakeSideChainFieldEvaluatorGPUs(cards, fr, ops, ff);
const SideChainFieldResult s = run(*split);
REQUIRE(s.moves.size() == 1);
CHECK(s.moves[0].chi_after == g.moves[0].chi_after);
CHECK(s.energy_after == Catch::Approx(g.energy_after).epsilon(1e-9));
#endif
}
}