Merge branch 'rc188-metal' into rc188-merge

This commit is contained in:
leonarski_f committed 2026-11-16 08:58:06 +01:00
commit f7a2fd948b
23 files changed
+1270 -21

No files matched your search

+3 -3
View File
@@ -79,7 +79,7 @@ These live in the source tree (see the path) rather than being fetched; traccc,
| [traccc (ACTS)](https://github.com/acts-project/traccc) | `image_analysis/spot_finding/StrongPixelSet.cpp`, `SpotExtractorGPU.cu` | CERN, for the benefit of the ACTS project | MPL-2.0 | [traccc.txt](licenses/traccc.txt) |
| [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (adapted) | `image_analysis/indexing/FFBIDXIndexerCPU.cpp` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](licenses/fast-feedback-indexer.txt) |
| [Ceres Solver](https://github.com/ceres-solver/ceres-solver) (adapted) | `image_analysis/geom_refinement/LMSolver.h`, `LMSolver.cpp` | Google Inc. | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) |
| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) |
| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal`, `AtomGradientMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) |
| [Durin](https://github.com/DiamondLightSource/durin) (adapted) | `xds-plugin/plugin.h`, `plugin.cpp` | Diamond Light Source Ltd.; Global Phasing Ltd. | BSD-3-Clause | [durin.txt](licenses/durin.txt) |
| [xbflash.qspi](https://github.com/Xilinx/XRT) | `tools/xbflash.qspi/` | Xilinx / AMD | Apache-2.0 | [xbflash-qspi.txt](licenses/xbflash-qspi.txt) |
| [wingetopt](https://github.com/alex85k/wingetopt) | `tools/wingetopt/` | Todd C. Miller; The NetBSD Foundation | ISC AND BSD-2-Clause | [wingetopt.txt](licenses/wingetopt.txt) |
@@ -153,8 +153,8 @@ served frontend, so the shipped web UI carries its own attribution.
`FFBIDXIndexerCPU.cpp` ports its CUDA kernels to the CPU. The file names the origin at the top and
is covered by `licenses/fast-feedback-indexer.txt`.
* **GEMMI** is also vendored (table above); separately, `ModelScaling.cpp`, `ModelMaskGPU.cu`,
`ModelStructureFactorsGPU.cpp` and `ModelDensityMetal.metal` in `image_analysis/structure_refinement/`
follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
`ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` and `AtomGradientMetal.metal` in
`image_analysis/structure_refinement/` follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code.
Each names the GEMMI source it follows and is covered by `licenses/gemmi.txt`.
* **Durin** (`xds-plugin/plugin.h`/`.cpp`): the XDS plugin's entry points are based on the Durin plugin
of Diamond Light Source and Global Phasing; both files name the origin at the top and are covered by
+10 -1
View File
@@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato
## Apple GPUs (Metal)
A macOS build has no CUDA, and three engines run on the Apple GPU instead (none under
A macOS build has no CUDA, and six engines run on the Apple GPU instead (none under
`rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning):
- **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its
@@ -358,6 +358,15 @@ A macOS build has no CUDA, and three engines run on the Apple GPU instead (none
for model validation and molecular replacement, as a gather over bricks of the grid as under CUDA;
the same grid as the CPU's **to the bit**.
[§14.1](CPU_DATA_ANALYSIS_DECISIONS.md#141-model-structure-factors)
- **Refinement gradient** (`AtomGradientMetal.mm`, `.metal`) - the sums of the CPU refinement
engine's gradient over the FFT method's map, each atom's box walked by one SIMD group as under CUDA;
the CPU's points, summed in single precision, so the gradient is the CPU's to a few parts in 1e5
(a macOS CI step checks it against the CPU's). It halves the time of a refinement on the CPU engine.
- **Side-chain field** (`SideChainFieldMetal.mm`, `.metal`) - the direct sums of every side-chain state's
structure factors over the reflections, one thread per reflection and state; the CPU's sums **to the bit**
(the B factor's exp is `BraggExp.h` on both), which a macOS CI step checks.
- **Bulk-solvent mask shrink** (`ShrinkMaskMetal.mm`, `.metal`) - the mask's shrink by the 0.8 Å radius,
one thread per grid point; the same mask as the CPU's **to the bit**, which a macOS CI step checks.
- **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the
rotation function's trial orientations scored from a copy of the molecular transform; equal to the
CPU's to rounding.
@@ -0,0 +1,31 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
// AtomGradientsFromMap()'s sums (RefinementXray.cpp) on an Apple GPU (macOS only): every atom's box walked against
// the FFT method's map, one SIMD group of 32 threads per atom, as RefinementGPU.cu's atom_gradient_kernel does under
// CUDA. The points, Gaussians and arithmetic are the CPU's, but the sums are in single precision and in another
// order, so the gradient is the CPU's to a few parts in 1e5, not bit for bit. tests/AtomGradientMetalTest.cpp holds
// it to the CPU's.
#include <cstddef>
#include <vector>
#include "ModelDensityMetal.h" // ModelDensityMetalAtom, ModelDensityMetalGrid
// Per atom the nine sums of AtomGradientsFromMap() before occupancy and the constants: an isotropic atom's
// sum v dr2 x, y, z and sum v db (s[0..3]); an anisotropic atom's sum e (M r)_t (s[0..2]) and
// sum e (M + 2 (M r)(M r)^T) as u11 u22 u33 u12 u13 u23 (s[3..8]).
struct AtomGradientMetalSums {
float s[9];
};
// The sums of `atoms` (b with the addend 0, as the gradient takes them) over `map` (nu nv nw floats, u fastest) into
// `out`, one per atom. False where the GPU does not take it - no Metal device the process may use (rugnux --no-gpu
// hides it), or an atom whose box is wider than the cell; `out` is then untouched. Throws JFJochException on a
// Metal failure.
bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector<ModelDensityMetalAtom> &atoms,
const float *map, std::vector<AtomGradientMetalSums> &out);
// How many gradients the GPU has summed in this process.
size_t AtomGradientMetalCalls();
@@ -0,0 +1,134 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only AND MPL-2.0
// The refinement gradient's sums over the map (AtomGradientMetal.h), compiled when it is first used from this source,
// which CMakeLists.txt copies into the binary. The place of a point and the test against the radius are spelled as
// GatherMapPoints() (RefinementXray.cpp) spells them, with no a*b+c contracted, so that the GPU takes the CPU's points.
#include <metal_stdlib>
using namespace metal;
#pragma clang fp contract(off)
struct Atom { // ModelDensityMetalAtom
float nctr[3];
int c[3], d[3];
float max_r2;
float occ;
int aniso;
float a[5];
float b[5][6];
};
struct Grid { // ModelDensityMetalGrid
int nu, nv, nw;
float m00, m01, m02, m11, m12, m22;
};
struct Sums { // AtomGradientMetalSums
float s[9];
};
// gemmi's unsafe_expapprox() (formfact.hpp, MPL-2.0; https://github.com/project-gemmi/gemmi), as in
// ModelDensityMetal.metal.
static float expapprox(float x) {
const float val = 12102203.1615614f * x + 1065353216.f;
const int vali = static_cast<int>(val);
const int xu1 = vali & 0x7F800000;
const int xu2 = (vali & 0x7FFFFF) | 0x3F800000;
const float a = as_type<float>(xu1);
const float b = as_type<float>(xu2);
return a * (0.509871020f + b * (0.312146713f + b * (0.166617139f + b *
(-2.190619930e-3f + b * 1.3555747234e-2f))));
}
static int wrap(int i, int n) {
const int m = i % n;
return m < 0 ? m + n : m;
}
// One SIMD group of 32 threads per atom: the lanes take the box's points in turn, u fastest, each summing its own,
// and the group's sums are added at the end.
kernel void atom_gradient(device const Atom *atoms [[buffer(0)]], constant Grid &g [[buffer(1)]],
device const float *map [[buffer(2)]], device Sums *out [[buffer(3)]],
constant uint &n [[buffer(4)]],
uint group [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]],
uint groups_per_tg [[simdgroups_per_threadgroup]], uint lane [[thread_index_in_simdgroup]]) {
const uint i = group * groups_per_tg + sg;
if (i >= n)
return;
device const Atom &at = atoms[i];
const int span[3] = {2 * at.d[0] + 1, 2 * at.d[1] + 1, 2 * at.d[2] + 1};
const int lo[3] = {at.c[0] - at.d[0], at.c[1] - at.d[1], at.c[2] - at.d[2]};
const int total = span[0] * span[1] * span[2];
float s[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
// the lane's point, as offsets in the box, moved on by 32 points at a time
int iu = int(lane), iv = 0, iw = 0;
while (iu >= span[0]) {
iu -= span[0];
if (++iv == span[1]) {
iv = 0;
iw++;
}
}
for (int t = int(lane); t < total; t += 32) {
const int u = lo[0] + iu, v = lo[1] + iv, w = lo[2] + iw;
iu += 32;
while (iu >= span[0]) {
iu -= span[0];
if (++iv == span[1]) {
iv = 0;
iw++;
}
}
// GatherMapPoints()'s place and test
const float fz = at.nctr[2] - float(w);
const float z = g.m22 * fz;
const float fy = at.nctr[1] - float(v);
const float y = g.m11 * fy + g.m12 * fz;
const float yz = y * y + z * z;
const float x_yz = g.m01 * fy + g.m02 * fz;
const float x = g.m00 * (at.nctr[0] - float(u)) + x_yz;
const float r2 = yz + x * x;
if (r2 > at.max_r2)
continue;
const float value = map[uint(wrap(u, g.nu)) + uint(g.nu) * (uint(wrap(v, g.nv)) + uint(g.nv) * uint(wrap(w, g.nw)))];
if (at.aniso == 0) {
float dr2 = 0, db = 0;
for (int q = 0; q < 5; q++) {
const float b = at.b[q][0];
const float e = at.a[q] * expapprox(max(b * r2, -88.f));
dr2 += e * b;
db += e * b * (1.5f + b * r2);
}
s[0] += value * dr2 * x;
s[1] += value * dr2 * y;
s[2] += value * dr2 * z;
s[3] += value * db;
} else {
for (int q = 0; q < 5; q++) {
device const float *m = at.b[q]; // u11 u22 u33 u12 u13 u23
// gemmi's SMat33::r_u_r(), in its order
const float rur = x * x * m[0] + y * y * m[1] + z * z * m[2] + 2 * (x * y * m[3] + x * z * m[4] + y * z * m[5]);
const float e = value * at.a[q] * expapprox(max(rur, -88.f));
const float mr0 = m[0] * x + m[3] * y + m[4] * z;
const float mr1 = m[3] * x + m[1] * y + m[5] * z;
const float mr2 = m[4] * x + m[5] * y + m[2] * z;
s[0] += e * mr0;
s[1] += e * mr1;
s[2] += e * mr2;
s[3] += e * (m[0] + 2 * mr0 * mr0);
s[4] += e * (m[1] + 2 * mr1 * mr1);
s[5] += e * (m[2] + 2 * mr2 * mr2);
s[6] += e * (m[3] + 2 * mr0 * mr1);
s[7] += e * (m[4] + 2 * mr0 * mr2);
s[8] += e * (m[5] + 2 * mr1 * mr2);
}
}
}
for (int k = 0; k < 9; k++)
s[k] = simd_sum(s[k]);
if (lane == 0)
for (int k = 0; k < 9; k++)
out[i].s[k] = s[k];
}
@@ -0,0 +1,136 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#import <Metal/Metal.h>
#include "AtomGradientMetal.h"
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <limits>
#include <string>
#include <unistd.h>
#include "../../common/CUDAWrapper.h" // gpus_are_hidden
#include "../../common/JFJochException.h"
// The kernel's source, AtomGradientMetal.metal, copied in at configure time (CMakeLists.txt).
extern const char *const atom_gradient_metal_source;
namespace {
constexpr int ATOMS_PER_THREADGROUP = 8; // SIMD groups of 32 threads
std::atomic<size_t> calls_on_gpu{0};
static_assert(sizeof(AtomGradientMetalSums) == 9 * 4);
struct Kernel {
id<MTLDevice> device = nil; // nil: no Metal device
id<MTLComputePipelineState> pipeline = nil;
id<MTLCommandQueue> queue = nil;
};
// Compiled once per process, the first time it is asked for.
const Kernel &GetKernel() {
static const Kernel kernel = [] {
Kernel k;
k.device = MTLCreateSystemDefaultDevice();
if (k.device == nil)
return k;
MTLCompileOptions *options = [MTLCompileOptions new];
// Fast math off, so that a point's place rounds as on the CPU and the GPU takes the CPU's points.
options.fastMathEnabled = NO;
options.languageVersion = MTLLanguageVersion3_0;
NSError *error = nil;
id<MTLLibrary> library = [k.device newLibraryWithSource:@(atom_gradient_metal_source) options:options
error:&error];
if (library == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("atom gradient kernel: ") + error.localizedDescription.UTF8String);
id<MTLFunction> function = [library newFunctionWithName:@"atom_gradient"];
k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
if (k.pipeline == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("atom_gradient: ") + error.localizedDescription.UTF8String);
k.queue = [k.device newCommandQueue];
return k;
}();
return kernel;
}
id<MTLBuffer> NewBuffer(id<MTLDevice> device, size_t bytes) {
id<MTLBuffer> b = [device newBufferWithLength:std::max<size_t>(bytes, 4) options:MTLResourceStorageModeShared];
if (b == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError, "atom gradient: out of GPU memory");
return b;
}
} // namespace
bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector<ModelDensityMetalAtom> &atoms,
const float *map, std::vector<AtomGradientMetalSums> &out) {
// In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
@autoreleasepool {
const size_t points = static_cast<size_t>(grid.nu) * grid.nv * grid.nw;
if (gpus_are_hidden() || points > std::numeric_limits<uint32_t>::max() ||
atoms.size() > std::numeric_limits<uint32_t>::max())
return false;
const int n3[3] = {grid.nu, grid.nv, grid.nw};
for (const ModelDensityMetalAtom &a : atoms)
for (int k = 0; k < 3; k++)
if (2 * a.d[k] + 1 > n3[k])
return false; // a box wider than the cell: the CPU's walk adds both images of a point
const Kernel &k = GetKernel();
if (k.device == nil)
return false;
if (atoms.empty()) {
out.clear();
return true;
}
// The map as it is where its pages can be handed to the GPU, a copy otherwise.
const size_t page = static_cast<size_t>(getpagesize());
const size_t bytes = points * sizeof(float);
id<MTLBuffer> d_map = nil;
if (reinterpret_cast<uintptr_t>(map) % page == 0)
d_map = [k.device newBufferWithBytesNoCopy:const_cast<float *>(map) length:(bytes + page - 1) / page * page
options:MTLResourceStorageModeShared deallocator:nil];
if (d_map == nil) {
d_map = NewBuffer(k.device, bytes);
std::memcpy(d_map.contents, map, bytes);
}
id<MTLBuffer> d_atoms = NewBuffer(k.device, atoms.size() * sizeof(ModelDensityMetalAtom));
std::memcpy(d_atoms.contents, atoms.data(), atoms.size() * sizeof(ModelDensityMetalAtom));
id<MTLBuffer> d_out = NewBuffer(k.device, atoms.size() * sizeof(AtomGradientMetalSums));
const uint32_t n = static_cast<uint32_t>(atoms.size());
id<MTLCommandBuffer> cb = [k.queue commandBuffer];
id<MTLComputeCommandEncoder> e = [cb computeCommandEncoder];
[e setComputePipelineState:k.pipeline];
[e setBuffer:d_atoms offset:0 atIndex:0];
[e setBytes:&grid length:sizeof(grid) atIndex:1];
[e setBuffer:d_map offset:0 atIndex:2];
[e setBuffer:d_out offset:0 atIndex:3];
[e setBytes:&n length:sizeof(n) atIndex:4];
[e dispatchThreadgroups:MTLSizeMake((n + ATOMS_PER_THREADGROUP - 1) / ATOMS_PER_THREADGROUP, 1, 1)
threadsPerThreadgroup:MTLSizeMake(32 * ATOMS_PER_THREADGROUP, 1, 1)];
[e endEncoding];
[cb commit];
[cb waitUntilCompleted];
if (cb.status != MTLCommandBufferStatusCompleted)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
cb.error ? cb.error.localizedDescription.UTF8String
: "atom gradient: command buffer failed");
out.resize(atoms.size());
std::memcpy(out.data(), d_out.contents, atoms.size() * sizeof(AtomGradientMetalSums));
calls_on_gpu++;
return true;
}
}
size_t AtomGradientMetalCalls() {
return calls_on_gpu;
}
@@ -108,5 +108,47 @@ IF (APPLE)
ModelDensityMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/ModelDensityMetalSource.cpp)
SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
# The refinement gradient's sums over the map (AtomGradientMetal.h), carried in the same way.
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal text)
SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp
"// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
"extern const char *const atom_gradient_metal_source;\n"
"const char *const atom_gradient_metal_source = R\"GRADIENT_METAL(\n${text})GRADIENT_METAL\";\n")
CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp
${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp COPYONLY)
TARGET_SOURCES(JFJochStructureRefinement PRIVATE AtomGradientMetal.mm AtomGradientMetal.h
AtomGradientMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp)
SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
# The side-chain field's direct sums (SideChainFieldMetal.h), carried in the same way, after the exp the CPU's
# sums take too (bragg_integration/BraggExp.h).
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h exp_text)
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal text)
SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/SideChainFieldMetal.metal
${CMAKE_CURRENT_SOURCE_DIR}/../bragg_integration/BraggExp.h)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp
"// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
"extern const char *const side_chain_field_metal_source;\n"
"const char *const side_chain_field_metal_source = R\"FIELD_METAL(\n#include <metal_stdlib>\nusing namespace metal;\n#pragma clang fp contract(off)\n${exp_text}\n${text})FIELD_METAL\";\n")
CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp.tmp
${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp COPYONLY)
TARGET_SOURCES(JFJochStructureRefinement PRIVATE SideChainFieldMetal.mm SideChainFieldMetal.h
SideChainFieldMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/SideChainFieldMetalSource.cpp)
SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
SET_SOURCE_FILES_PROPERTIES(SideChainFieldMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
# The bulk-solvent mask's shrink (ShrinkMaskMetal.h), carried in the same way.
FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/ShrinkMaskMetal.metal text)
SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/ShrinkMaskMetal.metal)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp.tmp
"// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n"
"extern const char *const shrink_mask_metal_source;\n"
"const char *const shrink_mask_metal_source = R\"SHRINK_METAL(\n${text})SHRINK_METAL\";\n")
CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp.tmp
${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp COPYONLY)
TARGET_SOURCES(JFJochStructureRefinement PRIVATE ShrinkMaskMetal.mm ShrinkMaskMetal.h
ShrinkMaskMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/ShrinkMaskMetalSource.cpp)
SET_SOURCE_FILES_PROPERTIES(ShrinkMaskMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc)
SET_SOURCE_FILES_PROPERTIES(ShrinkMaskMetal.metal PROPERTIES HEADER_FILE_ONLY ON)
TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation")
ENDIF()
@@ -18,6 +18,7 @@
#include "../../common/JFJochException.h"
#include "../../common/Logger.h"
#include "ModelDensityMetal.h"
#include "ShrinkMaskMetal.h"
#endif
namespace {
@@ -469,6 +470,11 @@ void PutModelDensityOnGrid(gemmi::DensityCalculator<Table, float> &dc, const gem
namespace {
#ifdef __APPLE__
// From the first time the Apple GPU fails to shrink a mask on, with one warning, the CPU does it.
std::atomic<bool> shrink_metal_failed{false};
#endif
// gemmi's SolventMasker::shrink() - set_margin_around(r, 1, -1), then the margin set to 1 - with the planes of the
// grid spread over the threads, the same mask bit for bit. gemmi walks the unmasked points and marks the masked
// points within r of each: those among the nearest neighbours, and, where one of those was masked, the rest.
@@ -476,7 +482,7 @@ namespace {
// of the marks does not matter: here each masked point reads, from the mask as it was, whether gemmi would have
// marked it - an unmasked nearest neighbour, or an unmasked point further out within r that has a masked nearest
// neighbour of its own (the offsets are symmetric, so the points that reach q are the points q reaches).
void ShrinkMask(gemmi::Grid<float> &grid, double r, size_t nthreads) {
void ShrinkMask(gemmi::Grid<float> &grid, double r, size_t nthreads, [[maybe_unused]] bool gpu) {
// the two rings of offsets, as set_margin_around() builds them: a positive offset stored wrapped (u - nu), so
// that a point's coordinate plus an offset stays within [-n, n), which is what index_near_zero() takes
const int du = static_cast<int>(std::floor(r / grid.spacing[0])), dv = static_cast<int>(std::floor(r / grid.spacing[1])),
@@ -499,6 +505,19 @@ void ShrinkMask(gemmi::Grid<float> &grid, double r, size_t nthreads) {
return in[grid.index_near_zero(u + d[0], v + d[1], w + d[2])] >= 1.0f;
};
std::vector<float> out(in.size());
#ifdef __APPLE__
if (gpu && !shrink_metal_failed) {
try {
if (ShrinkMaskMetal(grid.nu, grid.nv, grid.nw, near, far, in.data(), out.data())) {
grid.data = std::move(out);
return;
}
} catch (const JFJochException &e) {
if (!shrink_metal_failed.exchange(true))
Logger("Rugnux").Warning("Solvent mask: the Apple GPU failed, on the CPU from here on: {}", e.what());
}
}
#endif
ParallelChunks(grid.nw, nthreads, [&](int lo, int hi) {
for (int w = lo; w < hi; ++w)
for (int v = 0; v < grid.nv; ++v)
@@ -532,7 +551,7 @@ void ShrinkMask(gemmi::Grid<float> &grid, double r, size_t nthreads) {
// rshrink of each point (set_margin_around), and on a grid whose spacing is coarser than rshrink along all
// three axes there are none - which is most rigid-body zones - yet gemmi's still walks the whole grid.
void PutMaskOnGrid(gemmi::Grid<float> &grid, const gemmi::Model &model, const std::vector<size_t> &orbit_leaders,
size_t nthreads) {
size_t nthreads, bool gpu) {
const gemmi::SolventMasker masker(gemmi::AtomicRadiiSet::Refmac);
masker.clear(grid);
std::vector<AtomBox> boxes;
@@ -556,7 +575,7 @@ void PutMaskOnGrid(gemmi::Grid<float> &grid, const gemmi::Model &model, const st
for (int i = 0; i < 3; i++)
shrink_has_offsets = shrink_has_offsets || static_cast<int>(std::floor(masker.rshrink / grid.spacing[i])) > 0;
if (shrink_has_offsets)
ShrinkMask(grid, masker.rshrink, nthreads);
ShrinkMask(grid, masker.rshrink, nthreads, gpu);
}
gemmi::Grid<float> SolventMaskGrid(const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min,
@@ -57,9 +57,10 @@ void PutModelDensityOnGrid(gemmi::DensityCalculator<gemmi::IT92<float>, float> &
const std::vector<size_t> &orbit_leaders, size_t nthreads, bool gpu = true,
bool ion_charges = false);
// `grid` sized, with its cell and group set.
// `grid` sized, with its cell and group set. On macOS the Apple GPU shrinks the mask (ShrinkMaskMetal.h), to the
// same bits; `gpu` false keeps it on the CPU.
void PutMaskOnGrid(gemmi::Grid<float> &grid, const gemmi::Model &model, const std::vector<size_t> &orbit_leaders,
size_t nthreads);
size_t nthreads, bool gpu = true);
// The step a bulk-solvent mask's grid is held below. The mask's shrink (0.8 A, the Refmac radii) reaches no grid point
// on a coarser grid and is skipped, and the macromolecule keeps the whole 1.0 A probe: some 5-10% of the cell's solvent
@@ -3444,7 +3444,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector<Me
}
#endif
if (!evaluator)
evaluator = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(sg), elements, ctx.nthreads);
evaluator = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(sg), elements, ctx.nthreads, settings.gpu);
std::vector<std::complex<float>> f_total(ctx.fcalc.begin(), ctx.fcalc.begin() + ctx.refl.Size());
evaluator->SetTotal(f_total);
// the copies of each residue, from the NCS anchors (corresponding C-alpha atoms): per residue, numbered
@@ -16,12 +16,19 @@
#include "gemmi/math.hpp"
#include "gemmi/twin.hpp"
#include "../../common/JFJochException.h"
#include "../../common/Logger.h"
#include "../../common/ParallelFor.h"
#include "ModelFFT.h"
#include "ModelGrid.h"
#include "RefinementMath.h"
#ifdef __APPLE__
#include <atomic>
#include "AtomGradientMetal.h"
#endif
namespace {
using Table = gemmi::IT92<float>;
@@ -230,7 +237,7 @@ public:
map_.data[i] *= norm;
});
const bool aniso = AnyAniso();
AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr);
AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr, true);
if (!aniso)
grad_u_.clear();
return sum;
@@ -919,10 +926,97 @@ void AnisoGradient(const gemmi::Grid<float> &map, const gemmi::DensityCalculator
*grad_u = {occ * gu[0], occ * gu[1], occ * gu[2], 2 * occ * gu[3], 2 * occ * gu[4], 2 * occ * gu[5]};
}
#ifdef __APPLE__
// AtomGradientsFromMap() with the sums over the map on the Apple GPU (AtomGradientMetal.h): each atom's box, radius and
// Gaussians as the CPU's loops below take them, then the sums turned into the derivatives as there. False where the
// GPU does not take it - and from the first time it fails on, with one warning, as the CPU then does the same work.
std::atomic<bool> gradient_metal_failed{false};
bool AtomGradientsFromMapMetal(const gemmi::Grid<float> &map, const gemmi::DensityCalculator<Table, float> &dc,
const std::vector<const gemmi::Atom *> &atoms, double blur, size_t nthreads,
std::vector<std::array<double, 4>> &grad, std::vector<std::array<double, 6>> *grad_u) {
if (gradient_metal_failed)
return false;
using CReal = Table::Coef::coef_type;
const gemmi::UpperTriangularMat33 &om = map.orth_n;
const ModelDensityMetalGrid g{map.nu, map.nv, map.nw, static_cast<float>(om.a11), static_cast<float>(om.a12),
static_cast<float>(om.a13), static_cast<float>(om.a22), static_cast<float>(om.a23),
static_cast<float>(om.a33)};
const int n = static_cast<int>(atoms.size());
std::vector<ModelDensityMetalAtom> in(n);
ParallelChunks(n, nthreads, [&](int lo, int hi) {
for (int i = lo; i < hi; i++) {
const gemmi::Atom &atom = *atoms[i];
ModelDensityMetalAtom &a = in[i];
const auto &coef = IonFormFactor<float>(atom.element.elem, atom.charge);
double radius;
a.aniso = atom.aniso.nonzero() ? 1 : 0;
for (auto &b : a.b)
std::fill(std::begin(b), std::end(b), 0.0f);
if (!a.aniso) {
const float b = static_cast<float>(atom.b_iso + blur);
const auto precal = coef.precalculate_density_iso(b, 0.0f);
radius = dc.estimate_radius(precal, b);
for (int q = 0; q < 5; q++) {
a.a[q] = precal.a[q];
a.b[q][0] = precal.b[q];
}
} else {
const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur));
const CReal b_max = AnisoRadiusB(aniso_b);
radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max);
const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f);
for (int q = 0; q < 5; q++) {
const gemmi::SMat33<float> &m = precal.b[q];
a.a[q] = precal.a[q];
const float row[6] = {m.u11, m.u22, m.u33, m.u12, m.u13, m.u23};
std::copy(std::begin(row), std::end(row), a.b[q]);
}
}
const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos);
const double n3[3] = {fpos.x * map.nu, fpos.y * map.nv, fpos.z * map.nw};
for (int k = 0; k < 3; k++) {
a.nctr[k] = static_cast<float>(n3[k]);
a.c[k] = gemmi::iround(n3[k]);
}
a.d[0] = std::min(static_cast<int>(std::ceil(radius / map.spacing[0])), map.nu - 1);
a.d[1] = std::min(static_cast<int>(std::ceil(radius / map.spacing[1])), map.nv - 1);
a.d[2] = std::min(static_cast<int>(std::ceil(radius / map.spacing[2])), map.nw - 1);
a.max_r2 = static_cast<float>(radius * radius);
a.occ = atom.occ;
}
});
std::vector<AtomGradientMetalSums> sums;
try {
if (!AtomGradientMetal(g, in, map.data.data(), sums))
return false;
} catch (const JFJochException &e) {
if (!gradient_metal_failed.exchange(true))
Logger("Rugnux").Warning("Refinement gradient: the Apple GPU failed, on the CPU from here on: {}", e.what());
return false;
}
constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
for (int i = 0; i < n; i++) {
const float *s = sums[i].s;
const double occ = atoms[i]->occ;
if (!in[i].aniso) {
grad[i] = {2.0 * occ * s[0], 2.0 * occ * s[1], 2.0 * occ * s[2], occ * s[3] / FOUR_PI2};
continue;
}
grad[i] = {2 * occ * s[0], 2 * occ * s[1], 2 * occ * s[2],
occ * (double(s[3]) + s[4] + s[5]) / gemmi::u_to_b()};
if (grad_u != nullptr)
(*grad_u)[i] = {occ * s[3], occ * s[4], occ * s[5], 2 * occ * s[6], 2 * occ * s[7], 2 * occ * s[8]};
}
return true;
}
#endif
} // namespace
void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &model, double blur, size_t nthreads,
std::vector<std::array<double, 4>> &grad, std::vector<std::array<double, 6>> *grad_u) {
std::vector<std::array<double, 4>> &grad, std::vector<std::array<double, 6>> *grad_u,
[[maybe_unused]] bool gpu) {
const gemmi::DensityCalculator<Table, float> dc = Calculator(map.unit_cell, *map.spacegroup, 1.0, blur);
std::vector<const gemmi::Atom *> atoms;
for (const gemmi::Chain &ch : model.chains)
@@ -932,6 +1026,10 @@ void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &mod
grad.assign(atoms.size(), {0, 0, 0, 0});
if (grad_u != nullptr)
grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0});
#ifdef __APPLE__
if (gpu && AtomGradientsFromMapMetal(map, dc, atoms, blur, nthreads, grad, grad_u))
return;
#endif
const int nu = map.nu, nv = map.nv, nw = map.nw;
constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI;
// in blocks of atoms taken as the threads come free: an anisotropic atom, or one of high B, costs several times
@@ -288,7 +288,9 @@ std::unique_ptr<RefinementEngine> MakeRefinementEngineCPU(const gemmi::UnitCell
// derivatives: dT/d(x, y, z, B) per atom. `map` is u fastest on the grid of RigidBodyZoneGrid(), the atoms
// gridded with `blur`. Exposed for the tests. An atom with a nonzero aniso is gridded anisotropically, as
// PutModelDensityOnGrid() grids it: its B derivative is that along U + dB / (8 pi^2) I, and with `grad_u` (resized
// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23.
// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23. With `gpu`, on the Apple GPU
// where there is one (AtomGradientMetal.h, macOS): the same points summed in single precision, the CPU's gradient to
// a few parts in 1e5 rather than bit for bit.
void AtomGradientsFromMap(const gemmi::Grid<float> &map, const gemmi::Model &model, double blur, size_t nthreads,
std::vector<std::array<double, 4>> &grad,
std::vector<std::array<double, 6>> *grad_u = nullptr);
std::vector<std::array<double, 6>> *grad_u = nullptr, bool gpu = false);
@@ -0,0 +1,20 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
// The bulk-solvent mask's shrink (ModelGrid.cpp, ShrinkMask) on an Apple GPU (macOS only): one thread per point, the
// CPU's rule read from the mask as it was - the same mask, bit for bit. tests/ShrinkMaskMetalTest.cpp holds it to the
// CPU's.
#include <array>
#include <cstddef>
#include <vector>
// `in` shrunk into `out` (nu nv nw floats each, u fastest), with the CPU's offsets: `near`, then `far`, each stored
// wrapped (a positive offset less the grid's size). False where the GPU does not take it - no Metal device the process
// may use (rugnux --no-gpu hides it); `out` is then untouched. Throws JFJochException on a Metal failure.
bool ShrinkMaskMetal(int nu, int nv, int nw, const std::vector<std::array<int, 3>> &near,
const std::vector<std::array<int, 3>> &far, const float *in, float *out);
// How many masks the GPU has shrunk in this process.
size_t ShrinkMaskMetalCalls();
@@ -0,0 +1,52 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The bulk-solvent mask's shrink (ShrinkMaskMetal.h), compiled when first used from this source, which
// CMakeLists.txt copies into the binary.
#include <metal_stdlib>
using namespace metal;
struct Params {
int nu, nv, nw;
int n_near, n_far;
};
// gemmi's Grid::index_near_zero(): a coordinate in [-n, n) taken into the grid.
static uint index_near_zero(int u, int v, int w, constant Params &p) {
u += u < 0 ? p.nu : 0;
v += v < 0 ? p.nv : 0;
w += w < 0 ? p.nw : 0;
return uint(u) + uint(p.nu) * (uint(v) + uint(p.nv) * uint(w));
}
// One thread per point, ModelGrid.cpp's ShrinkMask() rule read from the mask as it was: a masked point becomes
// solvent where a near offset is solvent, or where a far offset is a solvent point with a masked near offset of its
// own. `offsets` holds the near ones, then the far ones, each stored wrapped as the CPU stores it.
kernel void shrink_mask(device const float *in [[buffer(0)]], device float *out [[buffer(1)]],
device const int3 *offsets [[buffer(2)]], constant Params &p [[buffer(3)]],
uint i [[thread_position_in_grid]]) {
if (i >= uint(p.nu) * uint(p.nv) * uint(p.nw))
return;
const float x = in[i];
if (x >= 1.0f) {
out[i] = x;
return;
}
const int u = int(i % uint(p.nu)), v = int(i / uint(p.nu) % uint(p.nv)), w = int(i / (uint(p.nu) * uint(p.nv)));
bool margin = false;
for (int k = 0; k < p.n_near && !margin; k++)
margin = in[index_near_zero(u + offsets[k].x, v + offsets[k].y, w + offsets[k].z, p)] >= 1.0f;
for (int k = p.n_near; k < p.n_near + p.n_far && !margin; k++) {
const int3 d = offsets[k];
if (!(in[index_near_zero(u + d.x, v + d.y, w + d.z, p)] >= 1.0f))
continue;
int pu = u + d.x, pv = v + d.y, pw = w + d.z; // back into [0, n)
pu += pu < 0 ? p.nu : 0;
pv += pv < 0 ? p.nv : 0;
pw += pw < 0 ? p.nw : 0;
for (int j = 0; j < p.n_near && !margin; j++)
margin = !(in[index_near_zero(pu + offsets[j].x, pv + offsets[j].y, pw + offsets[j].z, p)] >= 1.0f);
}
out[i] = margin ? 1.0f : x;
}
@@ -0,0 +1,138 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#import <Metal/Metal.h>
#include "ShrinkMaskMetal.h"
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <limits>
#include <string>
#include <unistd.h>
#include "../../common/CUDAWrapper.h" // gpus_are_hidden
#include "../../common/JFJochException.h"
// The kernel's source, ShrinkMaskMetal.metal, copied in at configure time (CMakeLists.txt).
extern const char *const shrink_mask_metal_source;
namespace {
std::atomic<size_t> calls_on_gpu{0};
struct Kernel {
id<MTLDevice> device = nil; // nil: no Metal device
id<MTLComputePipelineState> pipeline = nil;
id<MTLCommandQueue> queue = nil;
};
// Compiled once per process, the first time it is asked for.
const Kernel &GetKernel() {
static const Kernel kernel = [] {
Kernel k;
k.device = MTLCreateSystemDefaultDevice();
if (k.device == nil)
return k;
MTLCompileOptions *options = [MTLCompileOptions new];
options.languageVersion = MTLLanguageVersion3_0;
NSError *error = nil;
id<MTLLibrary> library = [k.device newLibraryWithSource:@(shrink_mask_metal_source) options:options
error:&error];
if (library == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("mask shrink kernel: ") + error.localizedDescription.UTF8String);
id<MTLFunction> function = [library newFunctionWithName:@"shrink_mask"];
k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
if (k.pipeline == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("shrink_mask: ") + error.localizedDescription.UTF8String);
k.queue = [k.device newCommandQueue];
return k;
}();
return kernel;
}
id<MTLBuffer> NewBuffer(id<MTLDevice> device, size_t bytes) {
id<MTLBuffer> b = [device newBufferWithLength:std::max<size_t>(bytes, 4) options:MTLResourceStorageModeShared];
if (b == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError, "mask shrink: out of GPU memory");
return b;
}
// `data` as it is where its pages can be handed to the GPU, nil otherwise.
id<MTLBuffer> InPlace(id<MTLDevice> device, const float *data, size_t bytes) {
const size_t page = static_cast<size_t>(getpagesize());
if (reinterpret_cast<uintptr_t>(data) % page != 0)
return nil;
return [device newBufferWithBytesNoCopy:const_cast<float *>(data) length:(bytes + page - 1) / page * page
options:MTLResourceStorageModeShared deallocator:nil];
}
struct Params {
int32_t nu, nv, nw, n_near, n_far;
};
} // namespace
bool ShrinkMaskMetal(int nu, int nv, int nw, const std::vector<std::array<int, 3>> &near,
const std::vector<std::array<int, 3>> &far, const float *in, float *out) {
// In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
@autoreleasepool {
const size_t points = static_cast<size_t>(nu) * nv * nw;
if (gpus_are_hidden() || points > std::numeric_limits<uint32_t>::max())
return false;
const Kernel &k = GetKernel();
if (k.device == nil)
return false;
const size_t bytes = points * sizeof(float);
id<MTLBuffer> d_in = InPlace(k.device, in, bytes);
if (d_in == nil) {
d_in = NewBuffer(k.device, bytes);
std::memcpy(d_in.contents, in, bytes);
}
id<MTLBuffer> d_out = InPlace(k.device, out, bytes);
const bool out_in_place = d_out != nil;
if (!out_in_place)
d_out = NewBuffer(k.device, bytes);
// the offsets as int3, which Metal pads to four ints
id<MTLBuffer> d_offsets = NewBuffer(k.device, (near.size() + far.size()) * 4 * sizeof(int32_t));
auto *o = static_cast<int32_t *>(d_offsets.contents);
size_t n = 0;
for (const auto *v : {&near, &far})
for (const std::array<int, 3> &d : *v) {
o[4 * n] = d[0];
o[4 * n + 1] = d[1];
o[4 * n + 2] = d[2];
o[4 * n + 3] = 0;
n++;
}
const Params p{nu, nv, nw, static_cast<int32_t>(near.size()), static_cast<int32_t>(far.size())};
id<MTLCommandBuffer> cb = [k.queue commandBuffer];
id<MTLComputeCommandEncoder> e = [cb computeCommandEncoder];
[e setComputePipelineState:k.pipeline];
[e setBuffer:d_in offset:0 atIndex:0];
[e setBuffer:d_out offset:0 atIndex:1];
[e setBuffer:d_offsets offset:0 atIndex:2];
[e setBytes:&p length:sizeof(p) atIndex:3];
[e dispatchThreads:MTLSizeMake(points, 1, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
[e endEncoding];
[cb commit];
[cb waitUntilCompleted];
if (cb.status != MTLCommandBufferStatusCompleted)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
cb.error ? cb.error.localizedDescription.UTF8String
: "mask shrink: command buffer failed");
if (!out_in_place)
std::memcpy(out, d_out.contents, bytes);
calls_on_gpu++;
return true;
}
}
size_t ShrinkMaskMetalCalls() {
return calls_on_gpu;
}
@@ -13,9 +13,16 @@
#include <gemmi/neighbor.hpp>
#include "../../common/ParallelFor.h"
#include "../bragg_integration/BraggExp.h"
#include "RefinementMath.h"
#include "SideChainRotamers.h"
#ifdef __APPLE__
#include "../../common/JFJochException.h"
#include "../../common/Logger.h"
#include "SideChainFieldMetal.h"
#endif
namespace {
constexpr double REPULSION_SIGMA = 0.2; // A, the refinement's (RefinementRestraints.cpp)
@@ -100,7 +107,7 @@ inline void SinCos2Pi(float v, float &sn, float &cs) {
class EvaluatorCPU : public SideChainFieldEvaluator {
public:
EvaluatorCPU(const FieldReflections &refl, const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements, size_t nthreads)
const std::vector<FormFactor> &elements, size_t nthreads, [[maybe_unused]] bool gpu)
: refl_(refl), elements_(elements), nthreads_(nthreads), n_ops_(ops.size()) {
// per operator and row (k * rows + m): R^T h and h . t, so that the phase of an atom at x is
// 2 pi (R^T h . x + h . t)
@@ -126,14 +133,29 @@ public:
f += ff[i] * std::exp(-ff[4 + i] * refl.stol2[m]);
f0_[e * rows + m] = f;
}
#ifdef __APPLE__
if (gpu)
try {
metal_ = SideChainFieldMetal::Make(rows, n_ops_, hx_, hy_, hz_, ht_, f0_, refl.stol2);
} catch (const JFJochException &e) {
MetalFailed(e);
}
#endif
}
void SetTotal(const std::vector<std::complex<float>> &f) override { total_ = f; }
// In single precision, as the GPU's contribution_kernel: per atom of the state its form factor at every row of
// a block, then per operator the phase's sine and cosine (SinCos2Pi) - loops over the rows that the compiler
// vectorises.
// vectorises. The B factor's exp is BraggExp, which an Apple GPU computes to the same bits.
std::vector<std::complex<float>> Contribution(const FieldStates &states, int s) override {
#ifdef __APPLE__
if (metal_) {
std::vector<std::vector<std::complex<float>>> one(1);
if (ContributionsMetal(states, s, one))
return one[0];
}
#endif
const size_t rows = refl_.Size();
std::vector<std::complex<float>> f(rows);
constexpr size_t BLOCK = 256;
@@ -145,7 +167,7 @@ public:
const FieldAtom &at = states.atoms[a];
const float *f0 = &f0_[at.element * rows + lo];
for (size_t m = 0; m < n; m++)
fa[m] = f0[m] * at.occ * std::exp(-at.b * refl_.stol2[lo + m]);
fa[m] = f0[m] * at.occ * BraggExp(-at.b * refl_.stol2[lo + m]);
for (size_t k = 0; k < n_ops_; k++) {
const size_t o = k * rows + lo;
for (size_t m = 0; m < n; m++)
@@ -204,10 +226,39 @@ public:
private:
std::vector<std::vector<std::complex<float>>> All(const FieldStates &states) {
std::vector<std::vector<std::complex<float>>> df;
for (size_t s = 0; s < states.Count(); s++)
#ifdef __APPLE__
// on the GPU in batches of states, so that its buffer stays small beside the states' own
constexpr size_t BATCH = 32;
for (size_t s = 0; metal_ && s < states.Count(); s += BATCH) {
std::vector<std::vector<std::complex<float>>> batch(std::min(BATCH, states.Count() - s));
if (!ContributionsMetal(states, static_cast<int>(s), batch))
break;
for (auto &f : batch)
df.push_back(std::move(f));
}
#endif
for (size_t s = df.size(); s < states.Count(); s++)
df.push_back(Contribution(states, static_cast<int>(s)));
return df;
}
#ifdef __APPLE__
// Contribution() of the states from `first` on, one per element of `out`, on the Apple GPU (SideChainFieldMetal.h):
// the same values, bit for bit. False where it failed: the CPU from then on, with one warning.
bool ContributionsMetal(const FieldStates &states, int first, std::vector<std::vector<std::complex<float>>> &out) {
try {
metal_->Contributions(states, first, out);
return true;
} catch (const JFJochException &e) {
MetalFailed(e);
return false;
}
}
void MetalFailed(const JFJochException &e) {
Logger("Rugnux").Warning("Side-chain field: the Apple GPU failed, on the CPU from here on: {}", e.what());
metal_.reset();
}
std::unique_ptr<SideChainFieldMetal> metal_;
#endif
double Term(size_t m, std::complex<float> f_atoms) const {
const double fc = std::abs(refl_.k_total[m] * (f_atoms + refl_.fmask[m]));
double dfc = 0;
@@ -322,8 +373,8 @@ size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops) {
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements,
size_t nthreads) {
return std::make_unique<EvaluatorCPU>(refl, ops, elements, nthreads);
size_t nthreads, bool gpu) {
return std::make_unique<EvaluatorCPU>(refl, ops, elements, nthreads, gpu);
}
std::vector<FieldAtom> FieldAtomsOf(const gemmi::Residue &res, const SideChain &sc, const std::vector<gemmi::Position> &p,
@@ -91,10 +91,12 @@ public:
// The group's operators as the evaluator takes them: rotation (row-major) and translation, fractional.
std::vector<std::array<float, 12>> FieldOperators(const gemmi::SpaceGroup &sg);
// With `gpu`, the direct sums on the Apple GPU where there is one (SideChainFieldMetal.h, macOS): the same values,
// bit for bit.
std::unique_ptr<SideChainFieldEvaluator> MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl,
const std::vector<std::array<float, 12>> &ops,
const std::vector<FormFactor> &elements,
size_t nthreads);
size_t nthreads, bool gpu = false);
#ifdef JFJOCH_USE_CUDA
// The device memory the GPU evaluator takes at most for `rows` reflections under `n_ops` operators, bytes.
size_t SideChainFieldDeviceBytes(size_t rows, size_t n_ops);
@@ -0,0 +1,38 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
// The side-chain field's direct sums (SideChainField.cpp, EvaluatorCPU::Contribution) on an Apple GPU (macOS only):
// one thread per reflection and state, as SideChainFieldGPU.cu's contribution_kernel does under CUDA, with the CPU's
// single-precision arithmetic in its order (SideChainFieldMetal.metal spells it out). tests/SideChainFieldMetalTest.cpp
// holds it to the CPU's.
#include <complex>
#include <cstddef>
#include <memory>
#include <vector>
#include "SideChainField.h" // FieldStates
class SideChainFieldMetal {
public:
// The evaluator's tables, kept on the GPU: per operator and row (k * rows + m) R^T h and h . t, per element and
// row the form factor, per row stol2. Null where there is no Metal device the process may use (rugnux --no-gpu
// hides it). Throws JFJochException on a Metal failure.
static std::unique_ptr<SideChainFieldMetal> Make(size_t rows, size_t n_ops, const std::vector<float> &hx,
const std::vector<float> &hy, const std::vector<float> &hz,
const std::vector<float> &ht, const std::vector<float> &f0,
const std::vector<float> &stol2);
~SideChainFieldMetal();
// The contributions of states first .. first + out.size() - 1, each `rows` values into its vector.
void Contributions(const FieldStates &states, int first, std::vector<std::vector<std::complex<float>>> &out);
// How many states the GPU has summed in this process.
static size_t States();
struct Impl;
private:
explicit SideChainFieldMetal(std::unique_ptr<Impl> impl);
std::unique_ptr<Impl> impl_;
};
@@ -0,0 +1,58 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
// The side-chain field's direct sums (SideChainFieldMetal.h), compiled when first used from this source, which
// CMakeLists.txt copies into the binary after bragg_integration/BraggExp.h - with fast math off and no a*b+c
// contracted, as on the host, so that every operation below rounds as EvaluatorCPU::Contribution's does.
#include <metal_stdlib>
using namespace metal;
#pragma clang fp contract(off)
struct Atom { // FieldAtom
float x, y, z, b, occ;
int element;
};
struct Params {
uint rows, n_ops, first;
};
// SideChainField.cpp's SinCos2Pi(): sin and cos of 2 pi v, v taken to [-1/2, 1/2] by its nearest integer, the half
// angle by its Taylor series, then the double-angle formulas.
static void sincos_2pi(float v, thread float &sn, thread float &cs) {
const float x = 3.14159265f * (v - rint(v));
const float x2 = x * x;
const float s = x * (1 + x2 * (-1.f / 6 + x2 * (1.f / 120 + x2 * (-1.f / 5040 + x2 * (1.f / 362880 + x2 * (-1.f / 39916800))))));
const float c = 1 + x2 * (-1.f / 2 + x2 * (1.f / 24 + x2 * (-1.f / 720 + x2 * (1.f / 40320 + x2 * (-1.f / 3628800 + x2 * (1.f / 479001600))))));
sn = 2 * s * c;
cs = c * c - s * s;
}
// One thread per row m and state first + s: the state's atoms in order, each over the operators in order.
kernel void contribution(device const float *hx [[buffer(0)]], device const float *hy [[buffer(1)]],
device const float *hz [[buffer(2)]], device const float *ht [[buffer(3)]],
device const float *f0 [[buffer(4)]], device const float *stol2 [[buffer(5)]],
device const Atom *atoms [[buffer(6)]], device const int *start [[buffer(7)]],
constant Params &p [[buffer(8)]], device float2 *out [[buffer(9)]],
uint2 gid [[thread_position_in_grid]]) {
const uint m = gid.x, s = gid.y;
if (m >= p.rows)
return;
const float st = stol2[m];
float re = 0, im = 0;
for (int a = start[p.first + s]; a < start[p.first + s + 1]; a++) {
const Atom at = atoms[a];
const float fa = f0[uint(at.element) * p.rows + m] * at.occ * BraggExp(-at.b * st);
for (uint k = 0; k < p.n_ops; k++) {
const uint o = k * p.rows + m;
const float v = hx[o] * at.x + hy[o] * at.y + hz[o] * at.z + ht[o];
float sn, cs;
sincos_2pi(v, sn, cs);
re += fa * cs;
im += fa * sn;
}
}
out[s * p.rows + m] = float2(re, im);
}
@@ -0,0 +1,156 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#import <Metal/Metal.h>
#include "SideChainFieldMetal.h"
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <limits>
#include <string>
#include "../../common/CUDAWrapper.h" // gpus_are_hidden
#include "../../common/JFJochException.h"
// The kernel's source, SideChainFieldMetal.metal, copied in at configure time (CMakeLists.txt).
extern const char *const side_chain_field_metal_source;
namespace {
std::atomic<size_t> states_on_gpu{0};
static_assert(sizeof(FieldAtom) == 6 * 4, "the kernel's Atom is six words");
struct Kernel {
id<MTLDevice> device = nil; // nil: no Metal device
id<MTLComputePipelineState> pipeline = nil;
id<MTLCommandQueue> queue = nil;
};
// Compiled once per process, the first time it is asked for.
const Kernel &GetKernel() {
static const Kernel kernel = [] {
Kernel k;
k.device = MTLCreateSystemDefaultDevice();
if (k.device == nil)
return k;
MTLCompileOptions *options = [MTLCompileOptions new];
// Fast math off: nothing reassociated or approximated, as on the CPU.
options.fastMathEnabled = NO;
options.languageVersion = MTLLanguageVersion3_0;
NSError *error = nil;
id<MTLLibrary> library = [k.device newLibraryWithSource:@(side_chain_field_metal_source) options:options
error:&error];
if (library == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("side-chain field kernel: ") + error.localizedDescription.UTF8String);
id<MTLFunction> function = [library newFunctionWithName:@"contribution"];
k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
if (k.pipeline == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("contribution: ") + error.localizedDescription.UTF8String);
k.queue = [k.device newCommandQueue];
return k;
}();
return kernel;
}
id<MTLBuffer> NewBuffer(id<MTLDevice> device, size_t bytes) {
id<MTLBuffer> b = [device newBufferWithLength:std::max<size_t>(bytes, 4) options:MTLResourceStorageModeShared];
if (b == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError, "side-chain field: out of GPU memory");
return b;
}
id<MTLBuffer> Upload(id<MTLDevice> device, const std::vector<float> &v) {
id<MTLBuffer> b = NewBuffer(device, v.size() * sizeof(float));
std::memcpy(b.contents, v.data(), v.size() * sizeof(float));
return b;
}
struct Params {
uint32_t rows, n_ops, first;
};
} // namespace
struct SideChainFieldMetal::Impl {
uint32_t rows, n_ops;
id<MTLBuffer> hx, hy, hz, ht, f0, stol2;
};
SideChainFieldMetal::SideChainFieldMetal(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
SideChainFieldMetal::~SideChainFieldMetal() = default;
std::unique_ptr<SideChainFieldMetal> SideChainFieldMetal::Make(size_t rows, size_t n_ops, const std::vector<float> &hx,
const std::vector<float> &hy,
const std::vector<float> &hz,
const std::vector<float> &ht,
const std::vector<float> &f0,
const std::vector<float> &stol2) {
@autoreleasepool {
if (gpus_are_hidden() || rows * std::max<size_t>(n_ops, 1) > std::numeric_limits<uint32_t>::max())
return nullptr;
const Kernel &k = GetKernel();
if (k.device == nil)
return nullptr;
auto impl = std::make_unique<Impl>();
impl->rows = static_cast<uint32_t>(rows);
impl->n_ops = static_cast<uint32_t>(n_ops);
impl->hx = Upload(k.device, hx);
impl->hy = Upload(k.device, hy);
impl->hz = Upload(k.device, hz);
impl->ht = Upload(k.device, ht);
impl->f0 = Upload(k.device, f0);
impl->stol2 = Upload(k.device, stol2);
return std::unique_ptr<SideChainFieldMetal>(new SideChainFieldMetal(std::move(impl)));
}
}
void SideChainFieldMetal::Contributions(const FieldStates &states, int first,
std::vector<std::vector<std::complex<float>>> &out) {
// In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
@autoreleasepool {
const Kernel &k = GetKernel();
const size_t rows = impl_->rows, count = out.size();
if (count == 0)
return;
id<MTLBuffer> d_atoms = NewBuffer(k.device, states.atoms.size() * sizeof(FieldAtom));
std::memcpy(d_atoms.contents, states.atoms.data(), states.atoms.size() * sizeof(FieldAtom));
id<MTLBuffer> d_start = NewBuffer(k.device, states.start.size() * sizeof(int));
std::memcpy(d_start.contents, states.start.data(), states.start.size() * sizeof(int));
id<MTLBuffer> d_out = NewBuffer(k.device, count * rows * sizeof(std::complex<float>));
const Params p{impl_->rows, impl_->n_ops, static_cast<uint32_t>(first)};
id<MTLCommandBuffer> cb = [k.queue commandBuffer];
id<MTLComputeCommandEncoder> e = [cb computeCommandEncoder];
[e setComputePipelineState:k.pipeline];
id<MTLBuffer> tables[] = {impl_->hx, impl_->hy, impl_->hz, impl_->ht, impl_->f0, impl_->stol2};
for (int i = 0; i < 6; i++)
[e setBuffer:tables[i] offset:0 atIndex:i];
[e setBuffer:d_atoms offset:0 atIndex:6];
[e setBuffer:d_start offset:0 atIndex:7];
[e setBytes:&p length:sizeof(p) atIndex:8];
[e setBuffer:d_out offset:0 atIndex:9];
[e dispatchThreads:MTLSizeMake(rows, count, 1) threadsPerThreadgroup:MTLSizeMake(256, 1, 1)];
[e endEncoding];
[cb commit];
[cb waitUntilCompleted];
if (cb.status != MTLCommandBufferStatusCompleted)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
cb.error ? cb.error.localizedDescription.UTF8String
: "side-chain field: command buffer failed");
const auto *f = static_cast<const std::complex<float> *>(d_out.contents);
for (size_t s = 0; s < count; s++)
out[s].assign(f + s * rows, f + (s + 1) * rows);
states_on_gpu += count;
}
}
size_t SideChainFieldMetal::States() {
return states_on_gpu;
}
+102
View File
@@ -0,0 +1,102 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#ifdef __APPLE__
#include <cmath>
#include <random>
#include <vector>
#include <gemmi/model.hpp>
#include "../image_analysis/structure_refinement/AtomGradientMetal.h"
#include "../image_analysis/structure_refinement/RefinementXray.h"
#include "../image_analysis/structure_refinement/RigidBodyRefine.h"
// The refinement gradient's sums on the Apple GPU (AtomGradientMetal.h) against the CPU's: the same points, summed
// in single precision on the GPU and in double on the CPU. A Mac without a Metal device fails rather than skips, as
// in ModelDensityMetalTest.cpp.
namespace {
// Atoms scattered over the whole cell - so that boxes wrap at every face - of a few elements, a quarter of them
// anisotropic, with occupancies below one.
gemmi::Model random_atoms(const gemmi::UnitCell &cell) {
std::mt19937 rng(17);
std::uniform_real_distribution<double> frac(0.0, 1.0), b(8.0, 60.0), occ(0.3, 1.0), u(0.05, 0.4);
const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S};
gemmi::Model model(1);
gemmi::Chain chain("A");
gemmi::Residue res;
res.name = "UNK";
res.seqid = gemmi::SeqId(1, ' ');
for (int i = 0; i < 2000; i++) {
gemmi::Atom a;
a.name = "X";
a.element = elements[i % 4];
a.pos = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng)));
a.b_iso = static_cast<float>(b(rng));
a.occ = static_cast<float>(occ(rng));
if (i % 4 == 3) {
const float d1 = static_cast<float>(u(rng)), d2 = static_cast<float>(u(rng)),
d3 = static_cast<float>(u(rng));
a.aniso = {d1, d2, d3, 0.3f * std::min(d1, d2), -0.2f * std::min(d1, d3), 0.1f * std::min(d2, d3)};
}
res.atoms.push_back(a);
}
chain.residues.push_back(res);
model.chains.push_back(chain);
return model;
}
// The largest difference of a derivative between the two, over the largest of that derivative on the CPU.
template <size_t N>
double worst(const std::vector<std::array<double, N>> &cpu, const std::vector<std::array<double, N>> &gpu) {
double out = 0;
for (size_t c = 0; c < N; c++) {
double max_g = 0, max_d = 0;
for (size_t i = 0; i < cpu.size(); i++) {
max_g = std::max(max_g, std::fabs(cpu[i][c]));
max_d = std::max(max_d, std::fabs(gpu[i][c] - cpu[i][c]));
}
out = std::max(out, max_d / max_g);
}
return out;
}
void check_gradient(const gemmi::UnitCell &cell, const char *group, double d_min) {
const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group);
gemmi::Grid<float> map = RigidBodyZoneGrid(cell, sg, d_min);
map.data.resize(map.point_count());
// a smooth map with both signs, as the FFT method's derivative map is
for (int w = 0; w < map.nw; w++)
for (int v = 0; v < map.nv; v++)
for (int u = 0; u < map.nu; u++)
map.data[map.index_q(u, v, w)] = static_cast<float>(
std::sin(2 * M_PI * (3.0 * u / map.nu + 1.0 * v / map.nv)) +
std::cos(2 * M_PI * (2.0 * v / map.nv - 4.0 * w / map.nw)) +
0.5 * std::sin(2 * M_PI * (5.0 * u / map.nu + 3.0 * w / map.nw)));
const gemmi::Model model = random_atoms(cell);
const double blur = 10.0;
std::vector<std::array<double, 4>> gc, gg;
std::vector<std::array<double, 6>> uc, ug;
AtomGradientsFromMap(map, model, blur, 4, gc, &uc, false);
const size_t before = AtomGradientMetalCalls();
AtomGradientsFromMap(map, model, blur, 4, gg, &ug, true);
REQUIRE(AtomGradientMetalCalls() == before + 1); // the GPU summed it
REQUIRE(gg.size() == gc.size());
const double g = worst(gc, gg), u = worst(uc, ug);
CAPTURE(group, g, u);
CHECK(g < 1e-5);
CHECK(u < 5e-5);
}
}
TEST_CASE("AtomGradientMetal_MatchesCPU", "[portable][Metal]") {
// Triclinic, so that every term of the orthogonalization is there; then an orthorhombic group.
check_gradient(gemmi::UnitCell(41, 46, 52, 81, 96, 107), "P 1", 2.0);
check_gradient(gemmi::UnitCell(50, 56, 63, 90, 90, 90), "P 21 21 21", 1.6);
}
#endif
+3
View File
@@ -35,6 +35,9 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
BraggIntegrationEngineMetalTest.cpp
MolecularReplacementMetalTest.cpp
ModelDensityMetalTest.cpp
AtomGradientMetalTest.cpp
SideChainFieldMetalTest.cpp
ShrinkMaskMetalTest.cpp
WriteReflectionsTest.cpp
LoadReferenceMtzFreeFlagsTest.cpp
LoadReferenceSfCifTest.cpp
+80
View File
@@ -0,0 +1,80 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#ifdef __APPLE__
#include <cstring>
#include <random>
#include <vector>
#include <gemmi/model.hpp>
#include "../image_analysis/structure_refinement/ModelGrid.h"
#include "../image_analysis/structure_refinement/RefinementXray.h"
#include "../image_analysis/structure_refinement/ShrinkMaskMetal.h"
// The bulk-solvent mask shrunk on the Apple GPU (ShrinkMaskMetal.h) is the CPU's, bit for bit. A Mac without a
// Metal device fails rather than skips, as in ModelDensityMetalTest.cpp.
namespace {
// Atoms in clusters over the cell - so that the mask has an edge to shrink everywhere and boxes wrap at every face.
gemmi::Model clustered_atoms(const gemmi::UnitCell &cell) {
std::mt19937 rng(29);
std::uniform_real_distribution<double> frac(0.0, 1.0);
std::normal_distribution<double> spread(0.0, 2.5);
const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S};
gemmi::Model model(1);
gemmi::Chain chain("A");
gemmi::Residue res;
res.name = "UNK";
res.seqid = gemmi::SeqId(1, ' ');
for (int c = 0; c < 40; c++) {
const gemmi::Position centre = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng)));
for (int i = 0; i < 50; i++) {
gemmi::Atom a;
a.name = "X";
a.element = elements[i % 4];
a.pos = centre + gemmi::Position(spread(rng), spread(rng), spread(rng));
a.b_iso = 20.0f;
a.occ = 1.0f;
res.atoms.push_back(a);
}
}
chain.residues.push_back(res);
model.chains.push_back(chain);
return model;
}
void check_same_mask(const gemmi::UnitCell &cell, const char *group, double d_min) {
const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group);
gemmi::Grid<float> cpu = BulkSolventMask::MaskGrid(cell, sg, d_min);
cpu.data.resize(cpu.point_count());
gemmi::Grid<float> gpu = cpu;
const std::vector<size_t> leaders = OrbitLeaders(cpu, 4);
const gemmi::Model model = clustered_atoms(cell);
PutMaskOnGrid(cpu, model, leaders, 4, false);
const size_t before = ShrinkMaskMetalCalls();
PutMaskOnGrid(gpu, model, leaders, 4, true);
REQUIRE(ShrinkMaskMetalCalls() == before + 1); // the GPU shrank it
REQUIRE(gpu.data.size() == cpu.data.size());
size_t different = 0, solvent = 0;
for (size_t i = 0; i < cpu.data.size(); i++) {
different += std::memcmp(&cpu.data[i], &gpu.data[i], sizeof(float)) != 0;
solvent += cpu.data[i] >= 1.0f;
}
CAPTURE(group, solvent, cpu.data.size());
CHECK(solvent > cpu.data.size() / 10);
CHECK(solvent < cpu.data.size() * 9 / 10);
CHECK(different == 0);
}
}
TEST_CASE("ShrinkMaskMetal_BitIdenticalToCPU", "[portable][Metal]") {
// Triclinic, so that the offsets are not symmetric in the axes; then a group with symmetry.
check_same_mask(gemmi::UnitCell(41, 46, 52, 81, 96, 107), "P 1", 2.0);
check_same_mask(gemmi::UnitCell(50, 56, 63, 90, 90, 90), "P 21 21 21", 1.6);
}
#endif
+77
View File
@@ -0,0 +1,77 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#ifdef __APPLE__
#include <complex>
#include <cstring>
#include <random>
#include <vector>
#include "gemmi/it92.hpp"
#include "gemmi/symmetry.hpp"
#include "../image_analysis/structure_refinement/SideChainField.h"
#include "../image_analysis/structure_refinement/SideChainFieldMetal.h"
// The side-chain field's direct sums on the Apple GPU (SideChainFieldMetal.h) are the CPU's, bit for bit, so the
// contributions are compared as bit patterns and the energies exactly. A Mac without a Metal device fails rather than
// skips, as in ModelDensityMetalTest.cpp.
TEST_CASE("SideChainFieldMetal_BitIdenticalToCPU", "[portable][Metal]") {
std::mt19937 rng(23);
std::uniform_real_distribution<float> unit(0.0f, 1.0f);
std::uniform_int_distribution<int> index(-25, 25);
// reflections of a 50 A cell to about 2 A, in no particular order, a tenth of them free
FieldReflections fr;
for (int i = 0; i < 20000; i++) {
const float h = static_cast<float>(index(rng)), k = static_cast<float>(index(rng)), l = static_cast<float>(index(rng));
fr.hkl.push_back({h, k, l});
fr.stol2.push_back((h * h + k * k + l * l) / (4.0f * 50 * 50));
fr.fo.push_back(100 * unit(rng));
fr.d.push_back(0.9f);
fr.var.push_back(50 + 100 * unit(rng));
fr.centric.push_back(i % 7 == 0);
fr.work.push_back(i % 10 != 0);
fr.k_total.push_back(1.0f);
fr.fmask.push_back({10 * unit(rng), 10 * unit(rng)});
}
const std::vector<FormFactor> elements = {gemmi::IT92<float>::get(gemmi::El::C, 0).coefs,
gemmi::IT92<float>::get(gemmi::El::N, 0).coefs,
gemmi::IT92<float>::get(gemmi::El::O, 0).coefs};
const auto ops = FieldOperators(*gemmi::find_spacegroup_by_name("P 21 21 21"));
// 70 states of six atoms each - more than one batch - and combinations of one and of two states
FieldStates states;
for (int s = 0; s < 70; s++) {
std::vector<FieldAtom> atoms;
for (int a = 0; a < 6; a++)
atoms.push_back({unit(rng), unit(rng), unit(rng), 10 + 50 * unit(rng), 0.5f + 0.5f * unit(rng), a % 3});
states.Add(atoms);
}
states.current = {0};
for (int c = 0; c < 69; c++)
states.AddCombo(c % 2 == 0 ? std::vector<int>{c} : std::vector<int>{c, c + 1});
auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, false);
auto gpu = MakeSideChainFieldEvaluatorCPU(fr, ops, elements, 4, true);
const size_t before = SideChainFieldMetal::States();
size_t different = 0;
for (int s : {0, 33, 69}) {
const std::vector<std::complex<float>> c = cpu->Contribution(states, s), g = gpu->Contribution(states, s);
REQUIRE(c.size() == g.size());
for (size_t m = 0; m < c.size(); m++)
different += std::memcmp(&c[m], &g[m], sizeof(c[m])) != 0;
}
REQUIRE(SideChainFieldMetal::States() == before + 3); // the GPU summed them
CHECK(different == 0);
const std::vector<std::complex<float>> total = cpu->Contribution(states, 0);
cpu->SetTotal(total);
gpu->SetTotal(total);
const std::vector<double> ec = cpu->Evaluate(states), eg = gpu->Evaluate(states);
REQUIRE(SideChainFieldMetal::States() == before + 3 + 70);
CHECK(ec == eg);
}
#endif