// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only // The side-chain field's evaluator on the GPU (SideChainField.h): the direct summation of a state's atoms over the // group's operators at every row, and -ln L over the working rows of the total moved by it. Every sum runs in a // fixed order: the same states give the same energies, bit for bit. #include #include "../../common/JFJochException.h" #include "RefinementMath.h" #include "SideChainField.h" namespace { constexpr float TWO_PI = 6.283185307179586f; constexpr int THREADS = 256; void cuda_err(cudaError_t val) { if (val != cudaSuccess) throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); } class DeviceGuard { public: explicit DeviceGuard(int device) { cuda_err(cudaGetDevice(&previous_)); cuda_err(cudaSetDevice(device)); } ~DeviceGuard() { cudaSetDevice(previous_); } DeviceGuard(const DeviceGuard &) = delete; DeviceGuard &operator=(const DeviceGuard &) = delete; private: int previous_ = 0; }; struct Atom { float x, y, z, b, occ; int element; }; // dF of every state at every row: thread per row, block row y the state. `hop`: per row and operator R^T h and // h . t; `ff`: the form-factor table, 9 per element. __global__ void contribution_kernel(int rows, int n_ops, const float4 *hop, const float *stol2, const float *ff, const Atom *atoms, const int *start, float2 *df) { const int m = blockIdx.x * blockDim.x + threadIdx.x; const int s = blockIdx.y; if (m >= rows) return; const float st = stol2[m]; float re = 0, im = 0; for (int a = start[s]; a < start[s + 1]; a++) { const Atom at = atoms[a]; const float *c = ff + 9 * at.element; float f = c[8]; for (int i = 0; i < 4; i++) f += c[i] * expf(-c[4 + i] * st); f *= at.occ * expf(-at.b * st); for (int k = 0; k < n_ops; k++) { const float4 h = hop[m * n_ops + k]; float sn, cs; sincosf(TWO_PI * (h.x * at.x + h.y * at.y + h.z * at.z + h.w), &sn, &cs); re += f * cs; im += f * sn; } } df[s * rows + m] = make_float2(re, im); } // -ln L over the working rows per combination: one block per combination, its threads over the rows in strides, // reduced in a fixed order. F = total - sum of the current states' dF + sum of the combination's. __global__ void nll_kernel(int rows, const float2 *total, const float2 *df, const int *current, int n_current, const int *combo_states, const int *combo_start, const float *fo, const float *d, const float *var, const char *centric, const char *work, const float *k_total, const float2 *fmask, double *nll) { __shared__ double partial[THREADS]; const int c = blockIdx.x; double sum = 0; for (int m = threadIdx.x; m < rows; m += THREADS) { if (!work[m]) continue; float re = total[m].x + fmask[m].x, im = total[m].y + fmask[m].y; for (int i = 0; i < n_current; i++) { re -= df[current[i] * rows + m].x; im -= df[current[i] * rows + m].y; } for (int k = combo_start[c]; k < combo_start[c + 1]; k++) { re += df[combo_states[k] * rows + m].x; im += df[combo_states[k] * rows + m].y; } const double fc = k_total[m] * sqrt(static_cast(re) * re + static_cast(im) * im); double dfc; sum += refine_math::AmplitudeNegLogLikelihood(static_cast(fo[m]), fc, static_cast(d[m]), static_cast(var[m]), centric[m] != 0, dfc); } partial[threadIdx.x] = sum; __syncthreads(); for (int stride = THREADS / 2; stride > 0; stride /= 2) { if (threadIdx.x < stride) partial[threadIdx.x] += partial[threadIdx.x + stride]; __syncthreads(); } if (threadIdx.x == 0) nll[c] = partial[0]; } __global__ void move_kernel(int rows, const float2 *df, const int *current, int n_current, const int *combo_states, int from, int to, float2 *total) { const int m = blockIdx.x * blockDim.x + threadIdx.x; if (m >= rows) return; float re = total[m].x, im = total[m].y; for (int i = 0; i < n_current; i++) { re -= df[current[i] * rows + m].x; im -= df[current[i] * rows + m].y; } for (int k = from; k < to; k++) { re += df[combo_states[k] * rows + m].x; im += df[combo_states[k] * rows + m].y; } total[m] = make_float2(re, im); } class EvaluatorGPU : public SideChainFieldEvaluator { public: EvaluatorGPU(int device, const FieldReflections &refl, const std::vector> &ops, const std::vector &elements) : device_(device), rows_(static_cast(refl.Size())), n_ops_(static_cast(ops.size())) { DeviceGuard guard(device_); std::vector hop(refl.Size() * ops.size()); for (size_t m = 0; m < refl.Size(); m++) { const std::array &h = refl.hkl[m]; for (size_t k = 0; k < ops.size(); k++) { const std::array &op = ops[k]; hop[m * ops.size() + k] = make_float4(op[0] * h[0] + op[3] * h[1] + op[6] * h[2], op[1] * h[0] + op[4] * h[1] + op[7] * h[2], op[2] * h[0] + op[5] * h[1] + op[8] * h[2], op[9] * h[0] + op[10] * h[1] + op[11] * h[2]); } } std::vector ff; for (const FormFactor &e : elements) ff.insert(ff.end(), e.begin(), e.end()); cuda_err(cudaStreamCreate(&stream_)); hop_ = Upload(hop); stol2_ = Upload(refl.stol2); ff_ = Upload(ff); fo_ = Upload(refl.fo); d_ = Upload(refl.d); var_ = Upload(refl.var); centric_ = Upload(refl.centric); work_ = Upload(refl.work); k_total_ = Upload(refl.k_total); fmask_ = reinterpret_cast(Upload(refl.fmask)); cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2))); } ~EvaluatorGPU() override { int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread cudaGetDevice(&previous); cudaSetDevice(device_); for (void *p : {static_cast(hop_), static_cast(stol2_), static_cast(ff_), static_cast(fo_), static_cast(d_), static_cast(var_), static_cast(centric_), static_cast(work_), static_cast(k_total_), static_cast(fmask_), static_cast(total_), static_cast(atoms_), static_cast(start_), static_cast(df_), static_cast(nll_), static_cast(combo_states_), static_cast(combo_start_)}) cudaFree(p); cudaStreamDestroy(stream_); cudaSetDevice(previous); } void SetTotal(const std::vector> &f) override { DeviceGuard guard(device_); cuda_err(cudaMemcpyAsync(total_, f.data(), rows_ * sizeof(float2), cudaMemcpyHostToDevice, stream_)); cuda_err(cudaStreamSynchronize(stream_)); } std::vector Evaluate(const FieldStates &states) override { DeviceGuard guard(device_); Contributions(states); const int n = static_cast(states.Combos()); nll_kernel<<>>(rows_, total_, df_, current_, static_cast(states.current.size()), combo_states_, combo_start_, fo_, d_, var_, centric_, work_, k_total_, fmask_, nll_); cuda_err(cudaGetLastError()); std::vector nll(n); cuda_err(cudaMemcpyAsync(nll.data(), nll_, n * sizeof(double), cudaMemcpyDeviceToHost, stream_)); cuda_err(cudaStreamSynchronize(stream_)); return nll; } void Move(const FieldStates &states, int to) override { DeviceGuard guard(device_); Contributions(states); move_kernel<<<(rows_ + THREADS - 1) / THREADS, THREADS, 0, stream_>>>( rows_, df_, current_, static_cast(states.current.size()), combo_states_, states.combo_start[to], states.combo_start[to + 1], total_); cuda_err(cudaGetLastError()); cuda_err(cudaStreamSynchronize(stream_)); } std::vector> Contribution(const FieldStates &states, int s) override { DeviceGuard guard(device_); Contributions(states); std::vector> f(rows_); cuda_err(cudaMemcpyAsync(f.data(), df_ + static_cast(s) * rows_, rows_ * sizeof(float2), cudaMemcpyDeviceToHost, stream_)); cuda_err(cudaStreamSynchronize(stream_)); return f; } private: template T *Upload(const std::vector &v) { T *p = nullptr; cuda_err(cudaMalloc(&p, v.size() * sizeof(T))); cuda_err(cudaMemcpyAsync(p, v.data(), v.size() * sizeof(T), cudaMemcpyHostToDevice, stream_)); cuda_err(cudaStreamSynchronize(stream_)); return p; } // dF of every state of `states` into df_ (grown as needed). void Contributions(const FieldStates &states) { const int n = static_cast(states.Count()); if (states.atoms.size() > atoms_capacity_) { cudaFree(atoms_); atoms_capacity_ = states.atoms.size(); cuda_err(cudaMalloc(&atoms_, atoms_capacity_ * sizeof(Atom))); } if (static_cast(n) > states_capacity_) { cudaFree(start_); cudaFree(df_); states_capacity_ = n; cuda_err(cudaMalloc(&start_, (states_capacity_ + 1) * sizeof(int))); cuda_err(cudaMalloc(&df_, states_capacity_ * rows_ * sizeof(float2))); } const size_t combos = states.Combos(), lists = states.combo_states.size() + states.current.size(); if (combos + 1 > combos_capacity_) { cudaFree(combo_start_); cudaFree(nll_); combos_capacity_ = combos + 1; cuda_err(cudaMalloc(&combo_start_, combos_capacity_ * sizeof(int))); cuda_err(cudaMalloc(&nll_, combos_capacity_ * sizeof(double))); } if (lists > lists_capacity_) { cudaFree(combo_states_); lists_capacity_ = lists; cuda_err(cudaMalloc(&combo_states_, lists_capacity_ * sizeof(int))); } current_ = combo_states_ + states.combo_states.size(); if (!states.combo_states.empty()) cuda_err(cudaMemcpyAsync(combo_states_, states.combo_states.data(), states.combo_states.size() * sizeof(int), cudaMemcpyHostToDevice, stream_)); if (!states.current.empty()) cuda_err(cudaMemcpyAsync(current_, states.current.data(), states.current.size() * sizeof(int), cudaMemcpyHostToDevice, stream_)); cuda_err(cudaMemcpyAsync(combo_start_, states.combo_start.data(), (combos + 1) * sizeof(int), cudaMemcpyHostToDevice, stream_)); static_assert(sizeof(Atom) == sizeof(FieldAtom)); if (!states.atoms.empty()) cuda_err(cudaMemcpyAsync(atoms_, states.atoms.data(), states.atoms.size() * sizeof(Atom), cudaMemcpyHostToDevice, stream_)); cuda_err(cudaMemcpyAsync(start_, states.start.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream_)); const dim3 grid((rows_ + THREADS - 1) / THREADS, n); contribution_kernel<<>>(rows_, n_ops_, hop_, stol2_, ff_, atoms_, start_, df_); cuda_err(cudaGetLastError()); } int device_, rows_, n_ops_; cudaStream_t stream_ = nullptr; float4 *hop_ = nullptr; float *stol2_ = nullptr, *ff_ = nullptr, *fo_ = nullptr, *d_ = nullptr, *var_ = nullptr, *k_total_ = nullptr; char *centric_ = nullptr, *work_ = nullptr; float2 *fmask_ = nullptr, *total_ = nullptr, *df_ = nullptr; Atom *atoms_ = nullptr; int *start_ = nullptr, *combo_states_ = nullptr, *current_ = nullptr, *combo_start_ = nullptr; double *nll_ = nullptr; size_t atoms_capacity_ = 0, states_capacity_ = 0, combos_capacity_ = 0, lists_capacity_ = 0; }; } // namespace std::unique_ptr MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl, const std::vector> &ops, const std::vector &elements) { return std::make_unique(device, refl, ops, elements); }