// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #import #include "AtomGradientMetal.h" #include #include #include #include #include #include #include #include "../../common/CUDAWrapper.h" // gpus_are_hidden #include "../../common/JFJochException.h" // The kernel's source, AtomGradientMetal.metal, copied in at configure time (CMakeLists.txt). extern const char *const atom_gradient_metal_source; namespace { constexpr int ATOMS_PER_THREADGROUP = 8; // SIMD groups of 32 threads std::atomic calls_on_gpu{0}; static_assert(sizeof(AtomGradientMetalSums) == 9 * 4); struct Kernel { id device = nil; // nil: no Metal device id pipeline = nil; id queue = nil; }; // Compiled once per process, the first time it is asked for. const Kernel &GetKernel() { static const Kernel kernel = [] { Kernel k; k.device = MTLCreateSystemDefaultDevice(); if (k.device == nil) return k; MTLCompileOptions *options = [MTLCompileOptions new]; // Fast math off, so that a point's place rounds as on the CPU and the GPU takes the CPU's points. options.fastMathEnabled = NO; options.languageVersion = MTLLanguageVersion3_0; NSError *error = nil; id library = [k.device newLibraryWithSource:@(atom_gradient_metal_source) options:options error:&error]; if (library == nil) throw JFJochException(JFJochExceptionCategory::GPUMetalError, std::string("atom gradient kernel: ") + error.localizedDescription.UTF8String); id function = [library newFunctionWithName:@"atom_gradient"]; k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error]; if (k.pipeline == nil) throw JFJochException(JFJochExceptionCategory::GPUMetalError, std::string("atom_gradient: ") + error.localizedDescription.UTF8String); k.queue = [k.device newCommandQueue]; return k; }(); return kernel; } id NewBuffer(id device, size_t bytes) { id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared]; if (b == nil) throw JFJochException(JFJochExceptionCategory::GPUMetalError, "atom gradient: out of GPU memory"); return b; } } // namespace bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector &atoms, const float *map, std::vector &out) { // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none. @autoreleasepool { const size_t points = static_cast(grid.nu) * grid.nv * grid.nw; if (gpus_are_hidden() || points > std::numeric_limits::max() || atoms.size() > std::numeric_limits::max()) return false; const int n3[3] = {grid.nu, grid.nv, grid.nw}; for (const ModelDensityMetalAtom &a : atoms) for (int k = 0; k < 3; k++) if (2 * a.d[k] + 1 > n3[k]) return false; // a box wider than the cell: the CPU's walk adds both images of a point const Kernel &k = GetKernel(); if (k.device == nil) return false; if (atoms.empty()) { out.clear(); return true; } // The map as it is where its pages can be handed to the GPU, a copy otherwise. const size_t page = static_cast(getpagesize()); const size_t bytes = points * sizeof(float); id d_map = nil; if (reinterpret_cast(map) % page == 0) d_map = [k.device newBufferWithBytesNoCopy:const_cast(map) length:(bytes + page - 1) / page * page options:MTLResourceStorageModeShared deallocator:nil]; if (d_map == nil) { d_map = NewBuffer(k.device, bytes); std::memcpy(d_map.contents, map, bytes); } id d_atoms = NewBuffer(k.device, atoms.size() * sizeof(ModelDensityMetalAtom)); std::memcpy(d_atoms.contents, atoms.data(), atoms.size() * sizeof(ModelDensityMetalAtom)); id d_out = NewBuffer(k.device, atoms.size() * sizeof(AtomGradientMetalSums)); const uint32_t n = static_cast(atoms.size()); id cb = [k.queue commandBuffer]; id e = [cb computeCommandEncoder]; [e setComputePipelineState:k.pipeline]; [e setBuffer:d_atoms offset:0 atIndex:0]; [e setBytes:&grid length:sizeof(grid) atIndex:1]; [e setBuffer:d_map offset:0 atIndex:2]; [e setBuffer:d_out offset:0 atIndex:3]; [e setBytes:&n length:sizeof(n) atIndex:4]; [e dispatchThreadgroups:MTLSizeMake((n + ATOMS_PER_THREADGROUP - 1) / ATOMS_PER_THREADGROUP, 1, 1) threadsPerThreadgroup:MTLSizeMake(32 * ATOMS_PER_THREADGROUP, 1, 1)]; [e endEncoding]; [cb commit]; [cb waitUntilCompleted]; if (cb.status != MTLCommandBufferStatusCompleted) throw JFJochException(JFJochExceptionCategory::GPUMetalError, cb.error ? cb.error.localizedDescription.UTF8String : "atom gradient: command buffer failed"); out.resize(atoms.size()); std::memcpy(out.data(), d_out.contents, atoms.size() * sizeof(AtomGradientMetalSums)); calls_on_gpu++; return true; } } size_t AtomGradientMetalCalls() { return calls_on_gpu; }