Files
Filip LeonarskiandClaude Opus 5.5 88330f395f Refinement gradient on the Apple GPU: the sums over the derivative map in Metal
RESULT-CHANGING (ulp level; owner approved for Metal ports where exactness costs speed)

AtomGradientsFromMap() takes a `gpu` flag; the CPU engine sets it, and on macOS the sums of every
atom's box against the FFT method's map then run on the Apple GPU (AtomGradientMetal.mm/.metal):
one SIMD group of 32 threads per atom, the lanes taking the box's points in turn, as
RefinementGPU.cu's atom_gradient_kernel does under CUDA. Each atom's box, radius and Gaussians are
the CPU's (built on the host as the CPU loops build them), and a point's place and the radius test
are spelled as GatherMapPoints() spells them with fast math and contraction off, so the GPU takes
the same points; the sums are in single precision and in another order, so the gradient is not the
CPU's bit for bit. Where the GPU does not take it (rugnux --no-gpu, no device, a box wider than the
cell) or fails (one warning, then the CPU from there on) the CPU loops run as before.

Parity (AtomGradientMetal_MatchesCPU, [portable][Metal], 2000 atoms, a quarter anisotropic, P1
triclinic and P212121): worst |GPU - CPU| over the largest |CPU| of that derivative 0.6-1.0e-6 for
x, y, z, B and 2.1-7.6e-6 for U; held to 1e-5 and 5e-5.

M6 Mac mini, rugnux --mode refine, an open-arm set (9zlo, its merged MTZ and refined model):
90.6 s -> 45.8 s wall; R-work 0.2697 both, R-free 0.2936 -> 0.2938. With --no-gpu the refined model
is byte-identical to the build before (115 s: --no-gpu also takes the density gather off the GPU).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
2026-11-16 07:21:12 +01:00

137 lines
5.8 KiB
Plaintext

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#import <Metal/Metal.h>
#include "AtomGradientMetal.h"
#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <limits>
#include <string>
#include <unistd.h>
#include "../../common/CUDAWrapper.h" // gpus_are_hidden
#include "../../common/JFJochException.h"
// The kernel's source, AtomGradientMetal.metal, copied in at configure time (CMakeLists.txt).
extern const char *const atom_gradient_metal_source;
namespace {
constexpr int ATOMS_PER_THREADGROUP = 8; // SIMD groups of 32 threads
std::atomic<size_t> calls_on_gpu{0};
static_assert(sizeof(AtomGradientMetalSums) == 9 * 4);
struct Kernel {
id<MTLDevice> device = nil; // nil: no Metal device
id<MTLComputePipelineState> pipeline = nil;
id<MTLCommandQueue> queue = nil;
};
// Compiled once per process, the first time it is asked for.
const Kernel &GetKernel() {
static const Kernel kernel = [] {
Kernel k;
k.device = MTLCreateSystemDefaultDevice();
if (k.device == nil)
return k;
MTLCompileOptions *options = [MTLCompileOptions new];
// Fast math off, so that a point's place rounds as on the CPU and the GPU takes the CPU's points.
options.fastMathEnabled = NO;
options.languageVersion = MTLLanguageVersion3_0;
NSError *error = nil;
id<MTLLibrary> library = [k.device newLibraryWithSource:@(atom_gradient_metal_source) options:options
error:&error];
if (library == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("atom gradient kernel: ") + error.localizedDescription.UTF8String);
id<MTLFunction> function = [library newFunctionWithName:@"atom_gradient"];
k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error];
if (k.pipeline == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
std::string("atom_gradient: ") + error.localizedDescription.UTF8String);
k.queue = [k.device newCommandQueue];
return k;
}();
return kernel;
}
id<MTLBuffer> NewBuffer(id<MTLDevice> device, size_t bytes) {
id<MTLBuffer> b = [device newBufferWithLength:std::max<size_t>(bytes, 4) options:MTLResourceStorageModeShared];
if (b == nil)
throw JFJochException(JFJochExceptionCategory::GPUMetalError, "atom gradient: out of GPU memory");
return b;
}
} // namespace
bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector<ModelDensityMetalAtom> &atoms,
const float *map, std::vector<AtomGradientMetalSums> &out) {
// In a pool of its own, as ModelDensityMetal(): the threads this runs on have none.
@autoreleasepool {
const size_t points = static_cast<size_t>(grid.nu) * grid.nv * grid.nw;
if (gpus_are_hidden() || points > std::numeric_limits<uint32_t>::max() ||
atoms.size() > std::numeric_limits<uint32_t>::max())
return false;
const int n3[3] = {grid.nu, grid.nv, grid.nw};
for (const ModelDensityMetalAtom &a : atoms)
for (int k = 0; k < 3; k++)
if (2 * a.d[k] + 1 > n3[k])
return false; // a box wider than the cell: the CPU's walk adds both images of a point
const Kernel &k = GetKernel();
if (k.device == nil)
return false;
if (atoms.empty()) {
out.clear();
return true;
}
// The map as it is where its pages can be handed to the GPU, a copy otherwise.
const size_t page = static_cast<size_t>(getpagesize());
const size_t bytes = points * sizeof(float);
id<MTLBuffer> d_map = nil;
if (reinterpret_cast<uintptr_t>(map) % page == 0)
d_map = [k.device newBufferWithBytesNoCopy:const_cast<float *>(map) length:(bytes + page - 1) / page * page
options:MTLResourceStorageModeShared deallocator:nil];
if (d_map == nil) {
d_map = NewBuffer(k.device, bytes);
std::memcpy(d_map.contents, map, bytes);
}
id<MTLBuffer> d_atoms = NewBuffer(k.device, atoms.size() * sizeof(ModelDensityMetalAtom));
std::memcpy(d_atoms.contents, atoms.data(), atoms.size() * sizeof(ModelDensityMetalAtom));
id<MTLBuffer> d_out = NewBuffer(k.device, atoms.size() * sizeof(AtomGradientMetalSums));
const uint32_t n = static_cast<uint32_t>(atoms.size());
id<MTLCommandBuffer> cb = [k.queue commandBuffer];
id<MTLComputeCommandEncoder> e = [cb computeCommandEncoder];
[e setComputePipelineState:k.pipeline];
[e setBuffer:d_atoms offset:0 atIndex:0];
[e setBytes:&grid length:sizeof(grid) atIndex:1];
[e setBuffer:d_map offset:0 atIndex:2];
[e setBuffer:d_out offset:0 atIndex:3];
[e setBytes:&n length:sizeof(n) atIndex:4];
[e dispatchThreadgroups:MTLSizeMake((n + ATOMS_PER_THREADGROUP - 1) / ATOMS_PER_THREADGROUP, 1, 1)
threadsPerThreadgroup:MTLSizeMake(32 * ATOMS_PER_THREADGROUP, 1, 1)];
[e endEncoding];
[cb commit];
[cb waitUntilCompleted];
if (cb.status != MTLCommandBufferStatusCompleted)
throw JFJochException(JFJochExceptionCategory::GPUMetalError,
cb.error ? cb.error.localizedDescription.UTF8String
: "atom gradient: command buffer failed");
out.resize(atoms.size());
std::memcpy(out.data(), d_out.contents, atoms.size() * sizeof(AtomGradientMetalSums));
calls_on_gpu++;
return true;
}
}
size_t AtomGradientMetalCalls() {
return calls_on_gpu;
}