Files
leonarski_f 4809c1622c
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 16m43s
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 28m46s
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 32m5s
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 39m44s
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m7s
Build Packages / Create release (push) Successful in 8s
Build Packages / Build documentation (push) Successful in 49s
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 22m30s
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 31m54s
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 11m29s
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 23m51s
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 26m7s
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 29m38s
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 28m6s
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 29m44s
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 33m22s
Build Packages / Generate python client (push) Successful in 47s
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 34m6s
Build Packages / Unit tests (push) Successful in 1h2m47s
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 35m59s
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 34m8s
Build Packages / Validation Battery (push) Successful in 38m31s
v1.0.0-rc.187 (#98)
* Rugnux: Refinement several times faster, one protocol on the CPU and the GPU, parts of it on the Apple GPU; its random draws are the same on every platform.
* Rugnux: Refinement follows common practice: the free set decides only the restraint weights and the displacement model (isotropic, TLS or anisotropic, by Hamilton's test), everything else is decided on the working set; lower R-free for placed and predicted models.
* Rugnux: Restraint dictionaries via `--restraints` (or among the inputs) and more built-in ligands; a model whose space group is a subgroup of the data's is refined in its own group; reflections plus a model select `--mode refine` on their own.
* Rugnux: `-C` fixes the crystal system on the axes as typed; `-S` alone fixes the space group only, the cell is still found and reduced; a space group or cell stored in the input file is ignored, with a line in the log.
* Rugnux: More reliable multi-sweep merging; `--mode scale` merges several stills runs as one; `_plot.txt` and the report name each input; a second crystal form is reported for stills given a cell.
* jfjoch_broker (experimental): Live Rugnux on FPGA detectors (JUNGFRAU) takes the same path as a DECTRIS stream, rotation and serial alike, model step included, with analysis threads per GPU; frames waiting for the analysis can be viewed; no host stage drops a frame - a source faster than the analysis is held at the card.
* jfjoch_broker: Summation of more than 256 frames per image is split between the card and the host (`summation_mode` is then `hybrid`), and live Rugnux runs on such collections; pedestal runs take a receiver of their own.
* jfjoch_broker (experimental): `stills_downsample` analyses 1 of every N serial images (chosen by a hash; the first 2000 all; the others written without analysis; reset to 1 by `/initialize`); `max_spot_count` left out means 250 for per-image indexing and up to 1000 for live stills; the live status gives the merge before the model work, the fitted resolution and the anomalous peak height of each unexplained blob.
* jfjoch_broker: Without a `pcie` entry the broker runs on a mock of the card instead of the HLS simulation; its spot, ROI and azimuthal results are zero then.
* Web frontend: Redesigned Rugnux live page; Vite 8, TypeScript 6, Mol* 5.13 and dependency security updates.
* Tools: `jfjoch_fpga_stills_test` measures live stills on the cards with a stored dataset; `jfjoch_fpga_test` loses the HLS-simulation option.
* Other: FFTW no longer fetched or linked (the CPU FFT indexer keeps the name `FFTW`); PCIe driver now GPL-2.0-or-later and third-party notices complete; several crash fixes.

Reviewed-on: #98
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-11-19 20:49:28 +01:00

288 lines
13 KiB
Plaintext

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The side-chain field's evaluator on the GPU (SideChainField.h): the direct summation of a state's atoms over the
// group's operators at every row, and -ln L over the working rows of the total moved by it. Every sum runs in a
// fixed order: the same states give the same energies, bit for bit.
#include <cuda_runtime.h>
#include "../../common/JFJochException.h"
#include "RefinementMath.h"
#include "SideChainField.h"
namespace {
constexpr float TWO_PI = 6.283185307179586f;
constexpr int THREADS = 256;
void cuda_err(cudaError_t val) {
if (val != cudaSuccess)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val));
}
class DeviceGuard {
public:
explicit DeviceGuard(int device) {
cuda_err(cudaGetDevice(&previous_));
cuda_err(cudaSetDevice(device));
}
~DeviceGuard() { cudaSetDevice(previous_); }
DeviceGuard(const DeviceGuard &) = delete;
DeviceGuard &operator=(const DeviceGuard &) = delete;
private:
int previous_ = 0;
};
struct Atom {
float x, y, z, b, occ;
int element;
};
// dF of every state at every row: thread per row, block row y the state. `hop`: per row and operator R^T h and
// h . t; `ff`: the form-factor table, 9 per element.
__global__ void contribution_kernel(int rows, int n_ops, const float4 *hop, const float *stol2, const float *ff,
const Atom *atoms, const int *start, float2 *df) {
const int m = blockIdx.x * blockDim.x + threadIdx.x;
const int s = blockIdx.y;
if (m >= rows)
return;
const float st = stol2[m];
float re = 0, im = 0;
for (int a = start[s]; a < start[s + 1]; a++) {
const Atom at = atoms[a];
const float *c = ff + 9 * at.element;
float f = c[8];
for (int i = 0; i < 4; i++)
f += c[i] * expf(-c[4 + i] * st);
f *= at.occ * expf(-at.b * st);
for (int k = 0; k < n_ops; k++) {
const float4 h = hop[m * n_ops + k];
float sn, cs;
sincosf(TWO_PI * (h.x * at.x + h.y * at.y + h.z * at.z + h.w), &sn, &cs);
re += f * cs;
im += f * sn;
}
}
df[s * rows + m] = make_float2(re, im);
}
// -ln L over the working rows per combination: one block per combination, its threads over the rows in strides,
// reduced in a fixed order. F = total - sum of the current states' dF + sum of the combination's.
__global__ void nll_kernel(int rows, const float2 *total, const float2 *df, const int *current, int n_current,
const int *combo_states, const int *combo_start, const float *fo, const float *d,
const float *var, const char *centric, const char *work, const float *k_total,
const float2 *fmask, double *nll) {
__shared__ double partial[THREADS];
const int c = blockIdx.x;
double sum = 0;
for (int m = threadIdx.x; m < rows; m += THREADS) {
if (!work[m])
continue;
float re = total[m].x + fmask[m].x, im = total[m].y + fmask[m].y;
for (int i = 0; i < n_current; i++) {
re -= df[current[i] * rows + m].x;
im -= df[current[i] * rows + m].y;
}
for (int k = combo_start[c]; k < combo_start[c + 1]; k++) {
re += df[combo_states[k] * rows + m].x;
im += df[combo_states[k] * rows + m].y;
}
const double fc = k_total[m] * sqrt(static_cast<double>(re) * re + static_cast<double>(im) * im);
double dfc;
sum += refine_math::AmplitudeNegLogLikelihood<double>(static_cast<double>(fo[m]), fc, static_cast<double>(d[m]),
static_cast<double>(var[m]), centric[m] != 0, dfc);
}
partial[threadIdx.x] = sum;
__syncthreads();
for (int stride = THREADS / 2; stride > 0; stride /= 2) {
if (threadIdx.x < stride)
partial[threadIdx.x] += partial[threadIdx.x + stride];
__syncthreads();
}
if (threadIdx.x == 0)
nll[c] = partial[0];
}
__global__ void move_kernel(int rows, const float2 *df, const int *current, int n_current, const int *combo_states,
int from, int to, float2 *total) {
const int m = blockIdx.x * blockDim.x + threadIdx.x;
if (m >= rows)
return;
float re = total[m].x, im = total[m].y;
for (int i = 0; i < n_current; i++) {
re -= df[current[i] * rows + m].x;
im -= df[current[i] * rows + m].y;
}
for (int k = from; k < to; k++) {
re += df[combo_states[k] * rows + m].x;
im += df[combo_states[k] * rows + m].y;
}
total[m] = make_float2(re, im);
}
class EvaluatorGPU : public SideChainFieldEvaluator {
public:
EvaluatorGPU(int device, const FieldReflections &refl, const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements)
: device_(device), rows_(static_cast<int>(refl.Size())), n_ops_(static_cast<int>(ops.size())) {
DeviceGuard guard(device_);
std::vector<float4> hop(refl.Size() * ops.size());
for (size_t m = 0; m < refl.Size(); m++) {
const std::array<float, 3> &h = refl.hkl[m];
for (size_t k = 0; k < ops.size(); k++) {
const std::array<float, 12> &op = ops[k];
hop[m * ops.size() + k] = make_float4(op[0] * h[0] + op[3] * h[1] + op[6] * h[2],
op[1] * h[0] + op[4] * h[1] + op[7] * h[2],
op[2] * h[0] + op[5] * h[1] + op[8] * h[2],
op[9] * h[0] + op[10] * h[1] + op[11] * h[2]);
}
}
std::vector<float> ff;
for (const FormFactor &e : elements)
ff.insert(ff.end(), e.begin(), e.end());
cuda_err(cudaStreamCreate(&stream_));
hop_ = Upload(hop);
stol2_ = Upload(refl.stol2);
ff_ = Upload(ff);
fo_ = Upload(refl.fo);
d_ = Upload(refl.d);
var_ = Upload(refl.var);
centric_ = Upload(refl.centric);
work_ = Upload(refl.work);
k_total_ = Upload(refl.k_total);
fmask_ = reinterpret_cast<float2 *>(Upload(refl.fmask));
cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2)));
}
~EvaluatorGPU() override {
int previous = 0; // the caller's device kept: the field is destroyed on the refinement's thread
cudaGetDevice(&previous);
cudaSetDevice(device_);
for (void *p : {static_cast<void *>(hop_), static_cast<void *>(stol2_), static_cast<void *>(ff_),
static_cast<void *>(fo_), static_cast<void *>(d_), static_cast<void *>(var_),
static_cast<void *>(centric_), static_cast<void *>(work_), static_cast<void *>(k_total_),
static_cast<void *>(fmask_), static_cast<void *>(total_), static_cast<void *>(atoms_),
static_cast<void *>(start_), static_cast<void *>(df_), static_cast<void *>(nll_),
static_cast<void *>(combo_states_), static_cast<void *>(combo_start_)})
cudaFree(p);
cudaStreamDestroy(stream_);
cudaSetDevice(previous);
}
void SetTotal(const std::vector<std::complex<float>> &f) override {
DeviceGuard guard(device_);
cuda_err(cudaMemcpyAsync(total_, f.data(), rows_ * sizeof(float2), cudaMemcpyHostToDevice, stream_));
cuda_err(cudaStreamSynchronize(stream_));
}
std::vector<double> Evaluate(const FieldStates &states) override {
DeviceGuard guard(device_);
Contributions(states);
const int n = static_cast<int>(states.Combos());
nll_kernel<<<n, THREADS, 0, stream_>>>(rows_, total_, df_, current_, static_cast<int>(states.current.size()),
combo_states_, combo_start_, fo_, d_, var_, centric_, work_, k_total_,
fmask_, nll_);
cuda_err(cudaGetLastError());
std::vector<double> nll(n);
cuda_err(cudaMemcpyAsync(nll.data(), nll_, n * sizeof(double), cudaMemcpyDeviceToHost, stream_));
cuda_err(cudaStreamSynchronize(stream_));
return nll;
}
void Move(const FieldStates &states, int to) override {
DeviceGuard guard(device_);
Contributions(states);
move_kernel<<<(rows_ + THREADS - 1) / THREADS, THREADS, 0, stream_>>>(
rows_, df_, current_, static_cast<int>(states.current.size()), combo_states_, states.combo_start[to],
states.combo_start[to + 1], total_);
cuda_err(cudaGetLastError());
cuda_err(cudaStreamSynchronize(stream_));
}
std::vector<std::complex<float>> Contribution(const FieldStates &states, int s) override {
DeviceGuard guard(device_);
Contributions(states);
std::vector<std::complex<float>> f(rows_);
cuda_err(cudaMemcpyAsync(f.data(), df_ + static_cast<size_t>(s) * rows_, rows_ * sizeof(float2),
cudaMemcpyDeviceToHost, stream_));
cuda_err(cudaStreamSynchronize(stream_));
return f;
}
private:
template <typename T>
T *Upload(const std::vector<T> &v) {
T *p = nullptr;
cuda_err(cudaMalloc(&p, v.size() * sizeof(T)));
cuda_err(cudaMemcpyAsync(p, v.data(), v.size() * sizeof(T), cudaMemcpyHostToDevice, stream_));
cuda_err(cudaStreamSynchronize(stream_));
return p;
}
// dF of every state of `states` into df_ (grown as needed).
void Contributions(const FieldStates &states) {
const int n = static_cast<int>(states.Count());
if (states.atoms.size() > atoms_capacity_) {
cudaFree(atoms_);
atoms_capacity_ = states.atoms.size();
cuda_err(cudaMalloc(&atoms_, atoms_capacity_ * sizeof(Atom)));
}
if (static_cast<size_t>(n) > states_capacity_) {
cudaFree(start_);
cudaFree(df_);
states_capacity_ = n;
cuda_err(cudaMalloc(&start_, (states_capacity_ + 1) * sizeof(int)));
cuda_err(cudaMalloc(&df_, states_capacity_ * rows_ * sizeof(float2)));
}
const size_t combos = states.Combos(), lists = states.combo_states.size() + states.current.size();
if (combos + 1 > combos_capacity_) {
cudaFree(combo_start_);
cudaFree(nll_);
combos_capacity_ = combos + 1;
cuda_err(cudaMalloc(&combo_start_, combos_capacity_ * sizeof(int)));
cuda_err(cudaMalloc(&nll_, combos_capacity_ * sizeof(double)));
}
if (lists > lists_capacity_) {
cudaFree(combo_states_);
lists_capacity_ = lists;
cuda_err(cudaMalloc(&combo_states_, lists_capacity_ * sizeof(int)));
}
current_ = combo_states_ + states.combo_states.size();
if (!states.combo_states.empty())
cuda_err(cudaMemcpyAsync(combo_states_, states.combo_states.data(), states.combo_states.size() * sizeof(int),
cudaMemcpyHostToDevice, stream_));
if (!states.current.empty())
cuda_err(cudaMemcpyAsync(current_, states.current.data(), states.current.size() * sizeof(int),
cudaMemcpyHostToDevice, stream_));
cuda_err(cudaMemcpyAsync(combo_start_, states.combo_start.data(), (combos + 1) * sizeof(int),
cudaMemcpyHostToDevice, stream_));
static_assert(sizeof(Atom) == sizeof(FieldAtom));
if (!states.atoms.empty())
cuda_err(cudaMemcpyAsync(atoms_, states.atoms.data(), states.atoms.size() * sizeof(Atom),
cudaMemcpyHostToDevice, stream_));
cuda_err(cudaMemcpyAsync(start_, states.start.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream_));
const dim3 grid((rows_ + THREADS - 1) / THREADS, n);
contribution_kernel<<<grid, THREADS, 0, stream_>>>(rows_, n_ops_, hop_, stol2_, ff_, atoms_, start_, df_);
cuda_err(cudaGetLastError());
}
int device_, rows_, n_ops_;
cudaStream_t stream_ = nullptr;
float4 *hop_ = nullptr;
float *stol2_ = nullptr, *ff_ = nullptr, *fo_ = nullptr, *d_ = nullptr, *var_ = nullptr, *k_total_ = nullptr;
char *centric_ = nullptr, *work_ = nullptr;
float2 *fmask_ = nullptr, *total_ = nullptr, *df_ = nullptr;
Atom *atoms_ = nullptr;
int *start_ = nullptr, *combo_states_ = nullptr, *current_ = nullptr, *combo_start_ = nullptr;
double *nll_ = nullptr;
size_t atoms_capacity_ = 0, states_capacity_ = 0, combos_capacity_ = 0, lists_capacity_ = 0;
};
} // namespace
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements) {
return std::make_unique<EvaluatorGPU>(device, refl, ops, elements);
}