~EvaluatorGPU switched to its card to free its buffers and left the thread there. With the
field on every card (f07493c9b) the evaluators are destroyed on the refinement's thread, which
was then left on the last card: the engines the second round makes again (a raised B ceiling)
went with it, so the main engine landed on a card already holding its share of slots.
Measured on a 242k-atom model on 4 x T4, per card, against the footprint OwnSlotPlaces plans
(11290 MiB on every card): with the fix the untwinned arm peaks at 11188 / 11124 / 10836 /
10996 MiB (the field's search adds about 450 MiB per card, freed after it); without it card 3
reached 14295 MiB, the main engine there beside the card's slots. The plan is an upper bound once
the engine stays on its card. The refined model is bit-identical, with the same 16 trials at a
time in both arms (1116 s).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJz9sA2BQBEgKkuVuL
288 lines
13 KiB
Plaintext
288 lines
13 KiB
Plaintext
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
// The side-chain field's evaluator on the GPU (SideChainField.h): the direct summation of a state's atoms over the
|
|
// group's operators at every row, and -ln L over the working rows of the total moved by it. Every sum runs in a
|
|
// fixed order: the same states give the same energies, bit for bit.
|
|
|
|
#include <cuda_runtime.h>
|
|
|
|
#include "../../common/JFJochException.h"
|
|
#include "RefinementMath.h"
|
|
#include "SideChainField.h"
|
|
|
|
namespace {
|
|
|
|
constexpr float TWO_PI = 6.283185307179586f;
|
|
constexpr int THREADS = 256;
|
|
|
|
void cuda_err(cudaError_t val) {
|
|
if (val != cudaSuccess)
|
|
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
|
|
}
|
|
|
|
class DeviceGuard {
|
|
public:
|
|
explicit DeviceGuard(int device) {
|
|
cuda_err(cudaGetDevice(&previous_));
|
|
cuda_err(cudaSetDevice(device));
|
|
}
|
|
~DeviceGuard() { cudaSetDevice(previous_); }
|
|
DeviceGuard(const DeviceGuard &) = delete;
|
|
DeviceGuard &operator=(const DeviceGuard &) = delete;
|
|
|
|
private:
|
|
int previous_ = 0;
|
|
};
|
|
|
|
struct Atom {
|
|
float x, y, z, b, occ;
|
|
int element;
|
|
};
|
|
|
|
// dF of every state at every row: thread per row, block row y the state. `hop`: per row and operator R^T h and
|
|
// h . t; `ff`: the form-factor table, 9 per element.
|
|
__global__ void contribution_kernel(int rows, int n_ops, const float4 *hop, const float *stol2, const float *ff,
|
|
const Atom *atoms, const int *start, float2 *df) {
|
|
const int m = blockIdx.x * blockDim.x + threadIdx.x;
|
|
const int s = blockIdx.y;
|
|
if (m >= rows)
|
|
return;
|
|
const float st = stol2[m];
|
|
float re = 0, im = 0;
|
|
for (int a = start[s]; a < start[s + 1]; a++) {
|
|
const Atom at = atoms[a];
|
|
const float *c = ff + 9 * at.element;
|
|
float f = c[8];
|
|
for (int i = 0; i < 4; i++)
|
|
f += c[i] * expf(-c[4 + i] * st);
|
|
f *= at.occ * expf(-at.b * st);
|
|
for (int k = 0; k < n_ops; k++) {
|
|
const float4 h = hop[m * n_ops + k];
|
|
float sn, cs;
|
|
sincosf(TWO_PI * (h.x * at.x + h.y * at.y + h.z * at.z + h.w), &sn, &cs);
|
|
re += f * cs;
|
|
im += f * sn;
|
|
}
|
|
}
|
|
df[s * rows + m] = make_float2(re, im);
|
|
}
|
|
|
|
// -ln L over the working rows per combination: one block per combination, its threads over the rows in strides,
|
|
// reduced in a fixed order. F = total - sum of the current states' dF + sum of the combination's.
|
|
__global__ void nll_kernel(int rows, const float2 *total, const float2 *df, const int *current, int n_current,
|
|
const int *combo_states, const int *combo_start, const float *fo, const float *d,
|
|
const float *var, const char *centric, const char *work, const float *k_total,
|
|
const float2 *fmask, double *nll) {
|
|
__shared__ double partial[THREADS];
|
|
const int c = blockIdx.x;
|
|
double sum = 0;
|
|
for (int m = threadIdx.x; m < rows; m += THREADS) {
|
|
if (!work[m])
|
|
continue;
|
|
float re = total[m].x + fmask[m].x, im = total[m].y + fmask[m].y;
|
|
for (int i = 0; i < n_current; i++) {
|
|
re -= df[current[i] * rows + m].x;
|
|
im -= df[current[i] * rows + m].y;
|
|
}
|
|
for (int k = combo_start[c]; k < combo_start[c + 1]; k++) {
|
|
re += df[combo_states[k] * rows + m].x;
|
|
im += df[combo_states[k] * rows + m].y;
|
|
}
|
|
const double fc = k_total[m] * sqrt(static_cast<double>(re) * re + static_cast<double>(im) * im);
|
|
double dfc;
|
|
sum += refine_math::AmplitudeNegLogLikelihood<double>(static_cast<double>(fo[m]), fc, static_cast<double>(d[m]),
|
|
static_cast<double>(var[m]), centric[m] != 0, dfc);
|
|
}
|
|
partial[threadIdx.x] = sum;
|
|
__syncthreads();
|
|
for (int stride = THREADS / 2; stride > 0; stride /= 2) {
|
|
if (threadIdx.x < stride)
|
|
partial[threadIdx.x] += partial[threadIdx.x + stride];
|
|
__syncthreads();
|
|
}
|
|
if (threadIdx.x == 0)
|
|
nll[c] = partial[0];
|
|
}
|
|
|
|
__global__ void move_kernel(int rows, const float2 *df, const int *current, int n_current, const int *combo_states,
|
|
int from, int to, float2 *total) {
|
|
const int m = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (m >= rows)
|
|
return;
|
|
float re = total[m].x, im = total[m].y;
|
|
for (int i = 0; i < n_current; i++) {
|
|
re -= df[current[i] * rows + m].x;
|
|
im -= df[current[i] * rows + m].y;
|
|
}
|
|
for (int k = from; k < to; k++) {
|
|
re += df[combo_states[k] * rows + m].x;
|
|
im += df[combo_states[k] * rows + m].y;
|
|
}
|
|
total[m] = make_float2(re, im);
|
|
}
|
|
|
|
class EvaluatorGPU : public SideChainFieldEvaluator {
|
|
public:
|
|
EvaluatorGPU(int device, const FieldReflections &refl, const std::vector<std::array<float, 12>> &ops,
|
|
const std::vector<FormFactor> &elements)
|
|
: device_(device), rows_(static_cast<int>(refl.Size())), n_ops_(static_cast<int>(ops.size())) {
|
|
DeviceGuard guard(device_);
|
|
std::vector<float4> hop(refl.Size() * ops.size());
|
|
for (size_t m = 0; m < refl.Size(); m++) {
|
|
const std::array<float, 3> &h = refl.hkl[m];
|
|
for (size_t k = 0; k < ops.size(); k++) {
|
|
const std::array<float, 12> &op = ops[k];
|
|
hop[m * ops.size() + k] = make_float4(op[0] * h[0] + op[3] * h[1] + op[6] * h[2],
|
|
op[1] * h[0] + op[4] * h[1] + op[7] * h[2],
|
|
op[2] * h[0] + op[5] * h[1] + op[8] * h[2],
|
|
op[9] * h[0] + op[10] * h[1] + op[11] * h[2]);
|
|
}
|
|
}
|
|
std::vector<float> ff;
|
|
for (const FormFactor &e : elements)
|
|
ff.insert(ff.end(), e.begin(), e.end());
|
|
cuda_err(cudaStreamCreate(&stream_));
|
|
hop_ = Upload(hop);
|
|
stol2_ = Upload(refl.stol2);
|
|
ff_ = Upload(ff);
|
|
fo_ = Upload(refl.fo);
|
|
d_ = Upload(refl.d);
|
|
var_ = Upload(refl.var);
|
|
centric_ = Upload(refl.centric);
|
|
work_ = Upload(refl.work);
|
|
k_total_ = Upload(refl.k_total);
|
|
fmask_ = reinterpret_cast<float2 *>(Upload(refl.fmask));
|
|
cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2)));
|
|
}
|
|
~EvaluatorGPU() override {
|
|
int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread
|
|
cudaGetDevice(&previous);
|
|
cudaSetDevice(device_);
|
|
for (void *p : {static_cast<void *>(hop_), static_cast<void *>(stol2_), static_cast<void *>(ff_),
|
|
static_cast<void *>(fo_), static_cast<void *>(d_), static_cast<void *>(var_),
|
|
static_cast<void *>(centric_), static_cast<void *>(work_), static_cast<void *>(k_total_),
|
|
static_cast<void *>(fmask_), static_cast<void *>(total_), static_cast<void *>(atoms_),
|
|
static_cast<void *>(start_), static_cast<void *>(df_), static_cast<void *>(nll_),
|
|
static_cast<void *>(combo_states_), static_cast<void *>(combo_start_)})
|
|
cudaFree(p);
|
|
cudaStreamDestroy(stream_);
|
|
cudaSetDevice(previous);
|
|
}
|
|
|
|
void SetTotal(const std::vector<std::complex<float>> &f) override {
|
|
DeviceGuard guard(device_);
|
|
cuda_err(cudaMemcpyAsync(total_, f.data(), rows_ * sizeof(float2), cudaMemcpyHostToDevice, stream_));
|
|
cuda_err(cudaStreamSynchronize(stream_));
|
|
}
|
|
|
|
std::vector<double> Evaluate(const FieldStates &states) override {
|
|
DeviceGuard guard(device_);
|
|
Contributions(states);
|
|
const int n = static_cast<int>(states.Combos());
|
|
nll_kernel<<<n, THREADS, 0, stream_>>>(rows_, total_, df_, current_, static_cast<int>(states.current.size()),
|
|
combo_states_, combo_start_, fo_, d_, var_, centric_, work_, k_total_,
|
|
fmask_, nll_);
|
|
cuda_err(cudaGetLastError());
|
|
std::vector<double> nll(n);
|
|
cuda_err(cudaMemcpyAsync(nll.data(), nll_, n * sizeof(double), cudaMemcpyDeviceToHost, stream_));
|
|
cuda_err(cudaStreamSynchronize(stream_));
|
|
return nll;
|
|
}
|
|
|
|
void Move(const FieldStates &states, int to) override {
|
|
DeviceGuard guard(device_);
|
|
Contributions(states);
|
|
move_kernel<<<(rows_ + THREADS - 1) / THREADS, THREADS, 0, stream_>>>(
|
|
rows_, df_, current_, static_cast<int>(states.current.size()), combo_states_, states.combo_start[to],
|
|
states.combo_start[to + 1], total_);
|
|
cuda_err(cudaGetLastError());
|
|
cuda_err(cudaStreamSynchronize(stream_));
|
|
}
|
|
|
|
std::vector<std::complex<float>> Contribution(const FieldStates &states, int s) override {
|
|
DeviceGuard guard(device_);
|
|
Contributions(states);
|
|
std::vector<std::complex<float>> f(rows_);
|
|
cuda_err(cudaMemcpyAsync(f.data(), df_ + static_cast<size_t>(s) * rows_, rows_ * sizeof(float2),
|
|
cudaMemcpyDeviceToHost, stream_));
|
|
cuda_err(cudaStreamSynchronize(stream_));
|
|
return f;
|
|
}
|
|
|
|
private:
|
|
template <typename T>
|
|
T *Upload(const std::vector<T> &v) {
|
|
T *p = nullptr;
|
|
cuda_err(cudaMalloc(&p, v.size() * sizeof(T)));
|
|
cuda_err(cudaMemcpyAsync(p, v.data(), v.size() * sizeof(T), cudaMemcpyHostToDevice, stream_));
|
|
cuda_err(cudaStreamSynchronize(stream_));
|
|
return p;
|
|
}
|
|
// dF of every state of `states` into df_ (grown as needed).
|
|
void Contributions(const FieldStates &states) {
|
|
const int n = static_cast<int>(states.Count());
|
|
if (states.atoms.size() > atoms_capacity_) {
|
|
cudaFree(atoms_);
|
|
atoms_capacity_ = states.atoms.size();
|
|
cuda_err(cudaMalloc(&atoms_, atoms_capacity_ * sizeof(Atom)));
|
|
}
|
|
if (static_cast<size_t>(n) > states_capacity_) {
|
|
cudaFree(start_);
|
|
cudaFree(df_);
|
|
states_capacity_ = n;
|
|
cuda_err(cudaMalloc(&start_, (states_capacity_ + 1) * sizeof(int)));
|
|
cuda_err(cudaMalloc(&df_, states_capacity_ * rows_ * sizeof(float2)));
|
|
}
|
|
const size_t combos = states.Combos(), lists = states.combo_states.size() + states.current.size();
|
|
if (combos + 1 > combos_capacity_) {
|
|
cudaFree(combo_start_);
|
|
cudaFree(nll_);
|
|
combos_capacity_ = combos + 1;
|
|
cuda_err(cudaMalloc(&combo_start_, combos_capacity_ * sizeof(int)));
|
|
cuda_err(cudaMalloc(&nll_, combos_capacity_ * sizeof(double)));
|
|
}
|
|
if (lists > lists_capacity_) {
|
|
cudaFree(combo_states_);
|
|
lists_capacity_ = lists;
|
|
cuda_err(cudaMalloc(&combo_states_, lists_capacity_ * sizeof(int)));
|
|
}
|
|
current_ = combo_states_ + states.combo_states.size();
|
|
if (!states.combo_states.empty())
|
|
cuda_err(cudaMemcpyAsync(combo_states_, states.combo_states.data(), states.combo_states.size() * sizeof(int),
|
|
cudaMemcpyHostToDevice, stream_));
|
|
if (!states.current.empty())
|
|
cuda_err(cudaMemcpyAsync(current_, states.current.data(), states.current.size() * sizeof(int),
|
|
cudaMemcpyHostToDevice, stream_));
|
|
cuda_err(cudaMemcpyAsync(combo_start_, states.combo_start.data(), (combos + 1) * sizeof(int),
|
|
cudaMemcpyHostToDevice, stream_));
|
|
static_assert(sizeof(Atom) == sizeof(FieldAtom));
|
|
if (!states.atoms.empty())
|
|
cuda_err(cudaMemcpyAsync(atoms_, states.atoms.data(), states.atoms.size() * sizeof(Atom),
|
|
cudaMemcpyHostToDevice, stream_));
|
|
cuda_err(cudaMemcpyAsync(start_, states.start.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream_));
|
|
const dim3 grid((rows_ + THREADS - 1) / THREADS, n);
|
|
contribution_kernel<<<grid, THREADS, 0, stream_>>>(rows_, n_ops_, hop_, stol2_, ff_, atoms_, start_, df_);
|
|
cuda_err(cudaGetLastError());
|
|
}
|
|
|
|
int device_, rows_, n_ops_;
|
|
cudaStream_t stream_ = nullptr;
|
|
float4 *hop_ = nullptr;
|
|
float *stol2_ = nullptr, *ff_ = nullptr, *fo_ = nullptr, *d_ = nullptr, *var_ = nullptr, *k_total_ = nullptr;
|
|
char *centric_ = nullptr, *work_ = nullptr;
|
|
float2 *fmask_ = nullptr, *total_ = nullptr, *df_ = nullptr;
|
|
Atom *atoms_ = nullptr;
|
|
int *start_ = nullptr, *combo_states_ = nullptr, *current_ = nullptr, *combo_start_ = nullptr;
|
|
double *nll_ = nullptr;
|
|
size_t atoms_capacity_ = 0, states_capacity_ = 0, combos_capacity_ = 0, lists_capacity_ = 0;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl,
|
|
const std::vector<std::array<float, 12>> &ops,
|
|
const std::vector<FormFactor> &elements) {
|
|
return std::make_unique<EvaluatorGPU>(device, refl, ops, elements);
|
|
}
|