diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md index 60f8ddf21..af83088ac 100644 --- a/docs/GPU_DATA_ANALYSIS.md +++ b/docs/GPU_DATA_ANALYSIS.md @@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato ## Apple GPUs (Metal) -A macOS build has no CUDA, and four engines run on the Apple GPU instead (none under +A macOS build has no CUDA, and five engines run on the Apple GPU instead (none under `rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning): - **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its @@ -362,6 +362,9 @@ A macOS build has no CUDA, and four engines run on the Apple GPU instead (none u engine's gradient over the FFT method's map, each atom's box walked by one SIMD group as under CUDA; the CPU's points, summed in single precision, so the gradient is the CPU's to a few parts in 1e5 (a macOS CI step checks it against the CPU's). It halves the time of a refinement on the CPU engine. +- **Side-chain field** (`SideChainFieldMetal.mm`, `.metal`) - the direct sums of every side-chain state's + structure factors over the reflections, one thread per reflection and state; the CPU's sums **to the bit** + (the B factor's exp is `BraggExp.h` on both), which a macOS CI step checks. - **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the rotation function's trial orientations scored from a copy of the molecular transform; equal to the CPU's to rounding. diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index f513855cf..f3012ba4f 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -121,5 +121,21 @@ IF (APPLE) AtomGradientMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp) SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc) SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.metal PROPERTIES HEADER_FILE_ONLY ON) + # The side-chain field's direct sums (SideChainFieldMetal.h), carried in the same way, after the exp the CPU's + # sums take too (bragg_integration/BraggExp.h). + FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h exp_text) + FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal text) + SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal + ${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h) + FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp + "// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n" + "extern const char *const side_chain_field_metal_source;\n" + "const char *const side_chain_field_metal_source = R\"FIELD_METAL(\n#include \nusing namespace metal;\n#pragma clang fp contract(off)\n${exp_text}\n${text})FIELD_METAL\";\n") + CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp + ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp COPYONLY) + TARGET_SOURCES(JFJochStructureRefinement PRIVATE SideChainFieldMetal.mm SideChainFieldMetal.h + SideChainFieldMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp) + SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc) + SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.metal PROPERTIES HEADER_FILE_ONLY ON) TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation") ENDIF() diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 924ed0803..bef6fc53f 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3444,7 +3444,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector> f_total(ctx.fcalc.begin(), ctx.fcalc.begin() + ctx.refl.Size()); evaluator->SetTotal(f_total); // the copies of each residue, from the NCS anchors (corresponding C-alpha atoms): per residue, numbered diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp index 8c3dca856..84a518fad 100644 --- a/image_analysis/structure_refinement/SideChainField.cpp +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -13,9 +13,16 @@ #include #include "../../common/ParallelFor.h" +#include "../bragg_integration/BraggExp.h" #include "RefinementMath.h" #include "SideChainRotamers.h" +#ifdef __APPLE__ +#include "../../common/JFJochException.h" +#include "../../common/Logger.h" +#include "SideChainFieldMetal.h" +#endif + namespace { constexpr double REPULSION_SIGMA = 0.2; // A, the refinement's (RefinementRestraints.cpp) @@ -100,7 +107,7 @@ inline void SinCos2Pi(float v, float &sn, float &cs) { class EvaluatorCPU : public SideChainFieldEvaluator { public: EvaluatorCPU(const FieldReflections &refl, const std::vector> &ops, - const std::vector &elements, size_t nthreads) + const std::vector &elements, size_t nthreads, [[maybe_unused]] bool gpu) : refl_(refl), elements_(elements), nthreads_(nthreads), n_ops_(ops.size()) { // per operator and row (k * rows + m): R^T h and h . t, so that the phase of an atom at x is // 2 pi (R^T h . x + h . t) @@ -126,14 +133,29 @@ public: f += ff[i] * std::exp(-ff[4 + i] * refl.stol2[m]); f0_[e * rows + m] = f; } +#ifdef __APPLE__ + if (gpu) + try { + metal_ = SideChainFieldMetal::Make(rows, n_ops_, hx_, hy_, hz_, ht_, f0_, refl.stol2); + } catch (const JFJochException &e) { + MetalFailed(e); + } +#endif } void SetTotal(const std::vector> &f) override { total_ = f; } // In single precision, as the GPU's contribution_kernel: per atom of the state its form factor at every row of // a block, then per operator the phase's sine and cosine (SinCos2Pi) - loops over the rows that the compiler - // vectorises. + // vectorises. The B factor's exp is BraggExp, which an Apple GPU computes to the same bits. std::vector> Contribution(const FieldStates &states, int s) override { +#ifdef __APPLE__ + if (metal_) { + std::vector>> one(1); + if (ContributionsMetal(states, s, one)) + return one[0]; + } +#endif const size_t rows = refl_.Size(); std::vector> f(rows); constexpr size_t BLOCK = 256; @@ -145,7 +167,7 @@ public: const FieldAtom &at = states.atoms[a]; const float *f0 = &f0_[at.element * rows + lo]; for (size_t m = 0; m < n; m++) - fa[m] = f0[m] * at.occ * std::exp(-at.b * refl_.stol2[lo + m]); + fa[m] = f0[m] * at.occ * BraggExp(-at.b * refl_.stol2[lo + m]); for (size_t k = 0; k < n_ops_; k++) { const size_t o = k * rows + lo; for (size_t m = 0; m < n; m++) @@ -204,10 +226,39 @@ public: private: std::vector>> All(const FieldStates &states) { std::vector>> df; - for (size_t s = 0; s < states.Count(); s++) +#ifdef __APPLE__ + // on the GPU in batches of states, so that its buffer stays small beside the states' own + constexpr size_t BATCH = 32; + for (size_t s = 0; metal_ && s < states.Count(); s += BATCH) { + std::vector>> batch(std::min(BATCH, states.Count() - s)); + if (!ContributionsMetal(states, static_cast(s), batch)) + break; + for (auto &f : batch) + df.push_back(std::move(f)); + } +#endif + for (size_t s = df.size(); s < states.Count(); s++) df.push_back(Contribution(states, static_cast(s))); return df; } +#ifdef __APPLE__ + // Contribution() of the states from `first` on, one per element of `out`, on the Apple GPU (SideChainFieldMetal.h): + // the same values, bit for bit. False where it failed: the CPU from then on, with one warning. + bool ContributionsMetal(const FieldStates &states, int first, std::vector>> &out) { + try { + metal_->Contributions(states, first, out); + return true; + } catch (const JFJochException &e) { + MetalFailed(e); + return false; + } + } + void MetalFailed(const JFJochException &e) { + Logger("Rugnux").Warning("Side-chain field: the Apple GPU failed, on the CPU from here on: {}", e.what()); + metal_.reset(); + } + std::unique_ptr metal_; +#endif double Term(size_t m, std::complex f_atoms) const { const double fc = std::abs(refl_.k_total[m] * (f_atoms + refl_.fmask[m])); double dfc = 0; @@ -322,8 +373,8 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) { std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, const std::vector> &ops, const std::vector &elements, - size_t nthreads) { - return std::make_unique(refl, ops, elements, nthreads); + size_t nthreads, bool gpu) { + return std::make_unique(refl, ops, elements, nthreads, gpu); } std::vector FieldAtomsOf(const gemmi::Residue &res, const SideChain &sc, const std::vector &p, diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h index be098f2c7..409b3a896 100644 --- a/image_analysis/structure_refinement/SideChainField.h +++ b/image_analysis/structure_refinement/SideChainField.h @@ -91,10 +91,12 @@ public: // The group's operators as the evaluator takes them: rotation (row-major) and translation, fractional. std::vector> FieldOperators(const gemmi::SpaceGroup &sg); +// With `gpu`, the direct sums on the Apple GPU where there is one (SideChainFieldMetal.h, macOS): the same values, +// bit for bit. std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, const std::vector> &ops, const std::vector &elements, - size_t nthreads); + size_t nthreads, bool gpu = false); #ifdef JFJOCH_USE_CUDA // The device memory the GPU evaluator takes at most for `rows` reflections under `n_ops` operators, bytes. size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops); diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.h b/image_analysis/structure_refinement/SideChainFieldMetal.h new file mode 100644 index 000000000..4eea7ad92 --- /dev/null +++ b/image_analysis/structure_refinement/SideChainFieldMetal.h @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// The side-chain field's direct sums (SideChainField.cpp, EvaluatorCPU::Contribution) on an Apple GPU (macOS only): +// one thread per reflection and state, as SideChainFieldGPU.cu's contribution_kernel does under CUDA, with the CPU's +// single-precision arithmetic in its order (SideChainFieldMetal.metal spells it out). tests/SideChainFieldMetalTest.cpp +// holds it to the CPU's. + +#include +#include +#include +#include + +#include "SideChainField.h" // FieldStates + +class SideChainFieldMetal { +public: + // The evaluator's tables, kept on the GPU: per operator and row (k * rows + m) R^T h and h . t, per element and + // row the form factor, per row stol2. Null where there is no Metal device the process may use (rugnux --no-gpu + // hides it). Throws JFJochException on a Metal failure. + static std::unique_ptr Make(size_t rows, size_t n_ops, const std::vector &hx, + const std::vector &hy, const std::vector &hz, + const std::vector &ht, const std::vector &f0, + const std::vector &stol2); + ~SideChainFieldMetal(); + // The contributions of states first .. first + out.size() - 1, each `rows` values into its vector. + void Contributions(const FieldStates &states, int first, std::vector>> &out); + // How many states the GPU has summed in this process. + static size_t States(); + + struct Impl; + +private: + explicit SideChainFieldMetal(std::unique_ptr impl); + std::unique_ptr impl_; +}; diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.metal b/image_analysis/structure_refinement/SideChainFieldMetal.metal new file mode 100644 index 000000000..7621a884f --- /dev/null +++ b/image_analysis/structure_refinement/SideChainFieldMetal.metal @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// The side-chain field's direct sums (SideChainFieldMetal.h), compiled when first used from this source, which +// CMakeLists.txt copies into the binary after bragg_integration/BraggExp.h - with fast math off and no a*b+c +// contracted, as on the host, so that every operation below rounds as EvaluatorCPU::Contribution's does. + +#include +using namespace metal; + +#pragma clang fp contract(off) + +struct Atom { // FieldAtom + float x, y, z, b, occ; + int element; +}; + +struct Params { + uint rows, n_ops, first; +}; + +// SideChainField.cpp's SinCos2Pi(): sin and cos of 2 pi v, v taken to [-1/2, 1/2] by its nearest integer, the half +// angle by its Taylor series, then the double-angle formulas. +static void sincos_2pi(float v, thread float &sn, thread float &cs) { + const float x = 3.14159265f * (v - rint(v)); + const float x2 = x * x; + const float s = x * (1 + x2 * (-1.f / 6 + x2 * (1.f / 120 + x2 * (-1.f / 5040 + x2 * (1.f / 362880 + x2 * (-1.f / 39916800)))))); + const float c = 1 + x2 * (-1.f / 2 + x2 * (1.f / 24 + x2 * (-1.f / 720 + x2 * (1.f / 40320 + x2 * (-1.f / 3628800 + x2 * (1.f / 479001600)))))); + sn = 2 * s * c; + cs = c * c - s * s; +} + +// One thread per row m and state first + s: the state's atoms in order, each over the operators in order. +kernel void contribution(device const float *hx [[buffer(0)]], device const float *hy [[buffer(1)]], + device const float *hz [[buffer(2)]], device const float *ht [[buffer(3)]], + device const float *f0 [[buffer(4)]], device const float *stol2 [[buffer(5)]], + device const Atom *atoms [[buffer(6)]], device const int *start [[buffer(7)]], + constant Params &p [[buffer(8)]], device float2 *out [[buffer(9)]], + uint2 gid [[thread_position_in_grid]]) { + const uint m = gid.x, s = gid.y; + if (m >= p.rows) + return; + const float st = stol2[m]; + float re = 0, im = 0; + for (int a = start[p.first + s]; a < start[p.first + s + 1]; a++) { + const Atom at = atoms[a]; + const float fa = f0[uint(at.element) * p.rows + m] * at.occ * BraggExp(-at.b * st); + for (uint k = 0; k < p.n_ops; k++) { + const uint o = k * p.rows + m; + const float v = hx[o] * at.x + hy[o] * at.y + hz[o] * at.z + ht[o]; + float sn, cs; + sincos_2pi(v, sn, cs); + re += fa * cs; + im += fa * sn; + } + } + out[s * p.rows + m] = float2(re, im); +} diff --git a/image_analysis/structure_refinement/SideChainFieldMetal.mm b/image_analysis/structure_refinement/SideChainFieldMetal.mm new file mode 100644 index 000000000..b8cfd41ee --- /dev/null +++ b/image_analysis/structure_refinement/SideChainFieldMetal.mm @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#import + +#include "SideChainFieldMetal.h" + +#include +#include +#include +#include +#include +#include + +#include "../../common/CUDAWrapper.h" // gpus_are_hidden +#include "../../common/JFJochException.h" + +// The kernel's source, SideChainFieldMetal.metal, copied in at configure time (CMakeLists.txt). +extern const char *const side_chain_field_metal_source; + +namespace { + +std::atomic states_on_gpu{0}; + +static_assert(sizeof(FieldAtom) == 6 * 4, "the kernel's Atom is six words"); + +struct Kernel { + id device = nil; // nil: no Metal device + id pipeline = nil; + id queue = nil; +}; + +// Compiled once per process, the first time it is asked for. +const Kernel &GetKernel() { + static const Kernel kernel = [] { + Kernel k; + k.device = MTLCreateSystemDefaultDevice(); + if (k.device == nil) + return k; + MTLCompileOptions *options = [MTLCompileOptions new]; + // Fast math off: nothing reassociated or approximated, as on the CPU. + options.fastMathEnabled = NO; + options.languageVersion = MTLLanguageVersion3_0; + NSError *error = nil; + id library = [k.device newLibraryWithSource:@(side_chain_field_metal_source) options:options + error:&error]; + if (library == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + std::string("side-chain field kernel: ") + error.localizedDescription.UTF8String); + id function = [library newFunctionWithName:@"contribution"]; + k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error]; + if (k.pipeline == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + std::string("contribution: ") + error.localizedDescription.UTF8String); + k.queue = [k.device newCommandQueue]; + return k; + }(); + return kernel; +} + +id NewBuffer(id device, size_t bytes) { + id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared]; + if (b == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, "side-chain field: out of GPU memory"); + return b; +} + +id Upload(id device, const std::vector &v) { + id b = NewBuffer(device, v.size() * sizeof(float)); + std::memcpy(b.contents, v.data(), v.size() * sizeof(float)); + return b; +} + +struct Params { + uint32_t rows, n_ops, first; +}; + +} // namespace + +struct SideChainFieldMetal::Impl { + uint32_t rows, n_ops; + id hx, hy, hz, ht, f0, stol2; +}; + +SideChainFieldMetal::SideChainFieldMetal(std::unique_ptr impl) : impl_(std::move(impl)) {} + +SideChainFieldMetal::~SideChainFieldMetal() = default; + +std::unique_ptr SideChainFieldMetal::Make(size_t rows, size_t n_ops, const std::vector &hx, + const std::vector &hy, + const std::vector &hz, + const std::vector &ht, + const std::vector &f0, + const std::vector &stol2) { + @autoreleasepool { + if (gpus_are_hidden() || rows * std::max(n_ops, 1) > std::numeric_limits::max()) + return nullptr; + const Kernel &k = GetKernel(); + if (k.device == nil) + return nullptr; + auto impl = std::make_unique(); + impl->rows = static_cast(rows); + impl->n_ops = static_cast(n_ops); + impl->hx = Upload(k.device, hx); + impl->hy = Upload(k.device, hy); + impl->hz = Upload(k.device, hz); + impl->ht = Upload(k.device, ht); + impl->f0 = Upload(k.device, f0); + impl->stol2 = Upload(k.device, stol2); + return std::unique_ptr(new SideChainFieldMetal(std::move(impl))); + } +} + +void SideChainFieldMetal::Contributions(const FieldStates &states, int first, + std::vector>> &out) { + // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none. + @autoreleasepool { + const Kernel &k = GetKernel(); + const size_t rows = impl_->rows, count = out.size(); + if (count == 0) + return; + id d_atoms = NewBuffer(k.device, states.atoms.size() * sizeof(FieldAtom)); + std::memcpy(d_atoms.contents, states.atoms.data(), states.atoms.size() * sizeof(FieldAtom)); + id d_start = NewBuffer(k.device, states.start.size() * sizeof(int)); + std::memcpy(d_start.contents, states.start.data(), states.start.size() * sizeof(int)); + id d_out = NewBuffer(k.device, count * rows * sizeof(std::complex)); + const Params p{impl_->rows, impl_->n_ops, static_cast(first)}; + + id cb = [k.queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:k.pipeline]; + id tables[] = {impl_->hx, impl_->hy, impl_->hz, impl_->ht, impl_->f0, impl_->stol2}; + for (int i = 0; i < 6; i++) + [e setBuffer:tables[i] offset:0 atIndex:i]; + [e setBuffer:d_atoms offset:0 atIndex:6]; + [e setBuffer:d_start offset:0 atIndex:7]; + [e setBytes:&p length:sizeof(p) atIndex:8]; + [e setBuffer:d_out offset:0 atIndex:9]; + [e dispatchThreads:MTLSizeMake(rows, count, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)]; + [e endEncoding]; + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + cb.error ? cb.error.localizedDescription.UTF8String + : "side-chain field: command buffer failed"); + const auto *f = static_cast *>(d_out.contents); + for (size_t s = 0; s < count; s++) + out[s].assign(f + s * rows, f + (s + 1) * rows); + states_on_gpu += count; + } +} + +size_t SideChainFieldMetal::States() { + return states_on_gpu; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ce0055c8f..860a589a1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL MolecularReplacementMetalTest.cpp ModelDensityMetalTest.cpp AtomGradientMetalTest.cpp + SideChainFieldMetalTest.cpp WriteReflectionsTest.cpp LoadReferenceMtzFreeFlagsTest.cpp LoadReferenceSfCifTest.cpp diff --git a/tests/SideChainFieldMetalTest.cpp b/tests/SideChainFieldMetalTest.cpp new file mode 100644 index 000000000..8266114aa --- /dev/null +++ b/tests/SideChainFieldMetalTest.cpp @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#ifdef __APPLE__ + +#include +#include +#include +#include + +#include "gemmi/it92.hpp" +#include "gemmi/symmetry.hpp" +#include "../image_analysis/structure_refinement/SideChainField.h" +#include "../image_analysis/structure_refinement/SideChainFieldMetal.h" + +// The side-chain field's direct sums on the Apple GPU (SideChainFieldMetal.h) are the CPU's, bit for bit, so the +// contributions are compared as bit patterns and the energies exactly. A Mac without a Metal device fails rather than +// skips, as in ModelDensityMetalTest.cpp. + +TEST_CASE("SideChainFieldMetal_BitIdenticalToCPU", "[portable][Metal]") { + std::mt19937 rng(23); + std::uniform_real_distribution unit(0.0f, 1.0f); + std::uniform_int_distribution index(-25, 25); + // reflections of a 50 A cell to about 2 A, in no particular order, a tenth of them free + FieldReflections fr; + for (int i = 0; i < 20000; i++) { + const float h = static_cast(index(rng)), k = static_cast(index(rng)), l = static_cast(index(rng)); + fr.hkl.push_back({h, k, l}); + fr.stol2.push_back((h * h + k * k + l * l) / (4.0f * 50 * 50)); + fr.fo.push_back(100 * unit(rng)); + fr.d.push_back(0.9f); + fr.var.push_back(50 + 100 * unit(rng)); + fr.centric.push_back(i % 7 == 0); + fr.work.push_back(i % 10 != 0); + fr.k_total.push_back(1.0f); + fr.fmask.push_back({10 * unit(rng), 10 * unit(rng)}); + } + const std::vector elements = {gemmi::IT92::get(gemmi::El::C, 0).coefs, + gemmi::IT92::get(gemmi::El::N, 0).coefs, + gemmi::IT92::get(gemmi::El::O, 0).coefs}; + const auto ops = FieldOperators(*gemmi::find_spacegroup_by_name("P 21 21 21")); + // 70 states of six atoms each - more than one batch - and combinations of one and of two states + FieldStates states; + for (int s = 0; s < 70; s++) { + std::vector atoms; + for (int a = 0; a < 6; a++) + atoms.push_back({unit(rng), unit(rng), unit(rng), 10 + 50 * unit(rng), 0.5f + 0.5f * unit(rng), a % 3}); + states.Add(atoms); + } + states.current = {0}; + for (int c = 0; c < 69; c++) + states.AddCombo(c % 2 == 0 ? std::vector{c} : std::vector{c, c + 1}); + + auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, false); + auto gpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, true); + const size_t before = SideChainFieldMetal::States(); + size_t different = 0; + for (int s : {0, 33, 69}) { + const std::vector> c = cpu->Contribution(states, s), g = gpu->Contribution(states, s); + REQUIRE(c.size() == g.size()); + for (size_t m = 0; m < c.size(); m++) + different += std::memcmp(&c[m], &g[m], sizeof(c[m])) != 0; + } + REQUIRE(SideChainFieldMetal::States() == before + 3); // the GPU summed them + CHECK(different == 0); + + const std::vector> total = cpu->Contribution(states, 0); + cpu->SetTotal(total); + gpu->SetTotal(total); + const std::vector ec = cpu->Evaluate(states), eg = gpu->Evaluate(states); + REQUIRE(SideChainFieldMetal::States() == before + 3 + 70); + CHECK(ec == eg); +} + +#endif