diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index eb27cc3b6..807d735a5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -80,7 +80,7 @@ These live in the source tree (see the path) rather than being fetched; traccc, | [traccc (ACTS)](https://github.com/acts-project/traccc) | `image_analysis/spot_finding/StrongPixelSet.cpp`, `SpotExtractorGPU.cu` | CERN, for the benefit of the ACTS project | MPL-2.0 | [traccc.txt](licenses/traccc.txt) | | [fast-feedback-indexer](https://github.com/paulscherrerinstitute/fast-feedback-indexer) (adapted) | `image_analysis/indexing/FFBIDXIndexerCPU.cpp` | PSI | BSD-3-Clause | [fast-feedback-indexer.txt](licenses/fast-feedback-indexer.txt) | | [Ceres Solver](https://github.com/ceres-solver/ceres-solver) (adapted) | `image_analysis/geom_refinement/LMSolver.h`, `LMSolver.cpp` | Google Inc. | BSD-3-Clause | [ceres-solver.txt](licenses/ceres-solver.txt) | -| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) | +| [GEMMI](https://github.com/project-gemmi/gemmi) (adapted) | `image_analysis/structure_refinement/ModelScaling.cpp`, `ModelMaskGPU.cu`, `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal`, `AtomGradientMetal.metal` | Global Phasing Ltd. | MPL-2.0 | [gemmi.txt](licenses/gemmi.txt) | | [Durin](https://github.com/DiamondLightSource/durin) (adapted) | `xds-plugin/plugin.h`, `plugin.cpp` | Diamond Light Source Ltd.; Global Phasing Ltd. | BSD-3-Clause | [durin.txt](licenses/durin.txt) | | [xbflash.qspi](https://github.com/Xilinx/XRT) | `tools/xbflash.qspi/` | Xilinx / AMD | Apache-2.0 | [xbflash-qspi.txt](licenses/xbflash-qspi.txt) | | [wingetopt](https://github.com/alex85k/wingetopt) | `tools/wingetopt/` | Todd C. Miller; The NetBSD Foundation | ISC AND BSD-2-Clause | [wingetopt.txt](licenses/wingetopt.txt) | @@ -154,8 +154,8 @@ served frontend, so the shipped web UI carries its own attribution. `FFBIDXIndexerCPU.cpp` ports its CUDA kernels to the CPU. The file names the origin at the top and is covered by `licenses/fast-feedback-indexer.txt`. * **GEMMI** is also vendored (table above); separately, `ModelScaling.cpp`, `ModelMaskGPU.cu`, - `ModelStructureFactorsGPU.cpp` and `ModelDensityMetal.metal` in `image_analysis/structure_refinement/` - follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code. + `ModelStructureFactorsGPU.cpp`, `ModelDensityMetal.metal` and `AtomGradientMetal.metal` in + `image_analysis/structure_refinement/` follow GEMMI's scaling, Levenberg-Marquardt, solvent-mask, reciprocal-grid and form-factor code. Each names the GEMMI source it follows and is covered by `licenses/gemmi.txt`. * **Durin** (`xds-plugin/plugin.h`/`.cpp`): the XDS plugin's entry points are based on the Durin plugin of Diamond Light Source and Global Phasing; both files name the origin at the top and are covered by diff --git a/docs/GPU_DATA_ANALYSIS.md b/docs/GPU_DATA_ANALYSIS.md index 82343dbdb..60f8ddf21 100644 --- a/docs/GPU_DATA_ANALYSIS.md +++ b/docs/GPU_DATA_ANALYSIS.md @@ -342,7 +342,7 @@ mask 15-18% at low resolution. Inside a full `rugnux --model` run of a 5,000-ato ## Apple GPUs (Metal) -A macOS build has no CUDA, and three engines run on the Apple GPU instead (none under +A macOS build has no CUDA, and four engines run on the Apple GPU instead (none under `rugnux --no-gpu`; where the GPU fails, the CPU takes over with a warning): - **Bragg integration** (`BraggIntegrationEngineMetal.mm`, `.metal`) - the CPU engine with its @@ -358,6 +358,10 @@ A macOS build has no CUDA, and three engines run on the Apple GPU instead (none for model validation and molecular replacement, as a gather over bricks of the grid as under CUDA; the same grid as the CPU's **to the bit**. [ยง14.1](CPU_DATA_ANALYSIS_DECISIONS.md#141-model-structure-factors) +- **Refinement gradient** (`AtomGradientMetal.mm`, `.metal`) - the sums of the CPU refinement + engine's gradient over the FFT method's map, each atom's box walked by one SIMD group as under CUDA; + the CPU's points, summed in single precision, so the gradient is the CPU's to a few parts in 1e5 + (a macOS CI step checks it against the CPU's). It halves the time of a refinement on the CPU engine. - **Molecular-replacement rotation function** (`MolecularReplacementMetal.mm`, `.metal`) - the rotation function's trial orientations scored from a copy of the molecular transform; equal to the CPU's to rounding. diff --git a/image_analysis/structure_refinement/AtomGradientMetal.h b/image_analysis/structure_refinement/AtomGradientMetal.h new file mode 100644 index 000000000..0e1ce706a --- /dev/null +++ b/image_analysis/structure_refinement/AtomGradientMetal.h @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// AtomGradientsFromMap()'s sums (RefinementXray.cpp) on an Apple GPU (macOS only): every atom's box walked against +// the FFT method's map, one SIMD group of 32 threads per atom, as RefinementGPU.cu's atom_gradient_kernel does under +// CUDA. The points, Gaussians and arithmetic are the CPU's, but the sums are in single precision and in another +// order, so the gradient is the CPU's to a few parts in 1e5, not bit for bit. tests/AtomGradientMetalTest.cpp holds +// it to the CPU's. + +#include +#include + +#include "ModelDensityMetal.h" // ModelDensityMetalAtom, ModelDensityMetalGrid + +// Per atom the nine sums of AtomGradientsFromMap() before occupancy and the constants: an isotropic atom's +// sum v dr2 x, y, z and sum v db (s[0..3]); an anisotropic atom's sum e (M r)_t (s[0..2]) and +// sum e (M + 2 (M r)(M r)^T) as u11 u22 u33 u12 u13 u23 (s[3..8]). +struct AtomGradientMetalSums { + float s[9]; +}; + +// The sums of `atoms` (b with the addend 0, as the gradient takes them) over `map` (nu nv nw floats, u fastest) into +// `out`, one per atom. False where the GPU does not take it - no Metal device the process may use (rugnux --no-gpu +// hides it), or an atom whose box is wider than the cell; `out` is then untouched. Throws JFJochException on a +// Metal failure. +bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector &atoms, + const float *map, std::vector &out); +// How many gradients the GPU has summed in this process. +size_t AtomGradientMetalCalls(); diff --git a/image_analysis/structure_refinement/AtomGradientMetal.metal b/image_analysis/structure_refinement/AtomGradientMetal.metal new file mode 100644 index 000000000..d2ffc4ed6 --- /dev/null +++ b/image_analysis/structure_refinement/AtomGradientMetal.metal @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only AND MPL-2.0 + +// The refinement gradient's sums over the map (AtomGradientMetal.h), compiled when it is first used from this source, +// which CMakeLists.txt copies into the binary. The place of a point and the test against the radius are spelled as +// GatherMapPoints() (RefinementXray.cpp) spells them, with no a*b+c contracted, so that the GPU takes the CPU's points. + +#include +using namespace metal; + +#pragma clang fp contract(off) + +struct Atom { // ModelDensityMetalAtom + float nctr[3]; + int c[3], d[3]; + float max_r2; + float occ; + int aniso; + float a[5]; + float b[5][6]; +}; + +struct Grid { // ModelDensityMetalGrid + int nu, nv, nw; + float m00, m01, m02, m11, m12, m22; +}; + +struct Sums { // AtomGradientMetalSums + float s[9]; +}; + +// gemmi's unsafe_expapprox() (formfact.hpp, MPL-2.0; https://github.com/project-gemmi/gemmi), as in +// ModelDensityMetal.metal. +static float expapprox(float x) { + const float val = 12102203.1615614f * x + 1065353216.f; + const int vali = static_cast(val); + const int xu1 = vali & 0x7F800000; + const int xu2 = (vali & 0x7FFFFF) | 0x3F800000; + const float a = as_type(xu1); + const float b = as_type(xu2); + return a * (0.509871020f + b * (0.312146713f + b * (0.166617139f + b * + (-2.190619930e-3f + b * 1.3555747234e-2f)))); +} + +static int wrap(int i, int n) { + const int m = i % n; + return m < 0 ? m + n : m; +} + +// One SIMD group of 32 threads per atom: the lanes take the box's points in turn, u fastest, each summing its own, +// and the group's sums are added at the end. +kernel void atom_gradient(device const Atom *atoms [[buffer(0)]], constant Grid &g [[buffer(1)]], + device const float *map [[buffer(2)]], device Sums *out [[buffer(3)]], + constant uint &n [[buffer(4)]], + uint group [[threadgroup_position_in_grid]], uint sg [[simdgroup_index_in_threadgroup]], + uint groups_per_tg [[simdgroups_per_threadgroup]], uint lane [[thread_index_in_simdgroup]]) { + const uint i = group * groups_per_tg + sg; + if (i >= n) + return; + device const Atom &at = atoms[i]; + const int span[3] = {2 * at.d[0] + 1, 2 * at.d[1] + 1, 2 * at.d[2] + 1}; + const int lo[3] = {at.c[0] - at.d[0], at.c[1] - at.d[1], at.c[2] - at.d[2]}; + const int total = span[0] * span[1] * span[2]; + float s[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0}; + // the lane's point, as offsets in the box, moved on by 32 points at a time + int iu = int(lane), iv = 0, iw = 0; + while (iu >= span[0]) { + iu -= span[0]; + if (++iv == span[1]) { + iv = 0; + iw++; + } + } + for (int t = int(lane); t < total; t += 32) { + const int u = lo[0] + iu, v = lo[1] + iv, w = lo[2] + iw; + iu += 32; + while (iu >= span[0]) { + iu -= span[0]; + if (++iv == span[1]) { + iv = 0; + iw++; + } + } + // GatherMapPoints()'s place and test + const float fz = at.nctr[2] - float(w); + const float z = g.m22 * fz; + const float fy = at.nctr[1] - float(v); + const float y = g.m11 * fy + g.m12 * fz; + const float yz = y * y + z * z; + const float x_yz = g.m01 * fy + g.m02 * fz; + const float x = g.m00 * (at.nctr[0] - float(u)) + x_yz; + const float r2 = yz + x * x; + if (r2 > at.max_r2) + continue; + const float value = map[uint(wrap(u, g.nu)) + uint(g.nu) * (uint(wrap(v, g.nv)) + uint(g.nv) * uint(wrap(w, g.nw)))]; + if (at.aniso == 0) { + float dr2 = 0, db = 0; + for (int q = 0; q < 5; q++) { + const float b = at.b[q][0]; + const float e = at.a[q] * expapprox(max(b * r2, -88.f)); + dr2 += e * b; + db += e * b * (1.5f + b * r2); + } + s[0] += value * dr2 * x; + s[1] += value * dr2 * y; + s[2] += value * dr2 * z; + s[3] += value * db; + } else { + for (int q = 0; q < 5; q++) { + device const float *m = at.b[q]; // u11 u22 u33 u12 u13 u23 + // gemmi's SMat33::r_u_r(), in its order + const float rur = x * x * m[0] + y * y * m[1] + z * z * m[2] + 2 * (x * y * m[3] + x * z * m[4] + y * z * m[5]); + const float e = value * at.a[q] * expapprox(max(rur, -88.f)); + const float mr0 = m[0] * x + m[3] * y + m[4] * z; + const float mr1 = m[3] * x + m[1] * y + m[5] * z; + const float mr2 = m[4] * x + m[5] * y + m[2] * z; + s[0] += e * mr0; + s[1] += e * mr1; + s[2] += e * mr2; + s[3] += e * (m[0] + 2 * mr0 * mr0); + s[4] += e * (m[1] + 2 * mr1 * mr1); + s[5] += e * (m[2] + 2 * mr2 * mr2); + s[6] += e * (m[3] + 2 * mr0 * mr1); + s[7] += e * (m[4] + 2 * mr0 * mr2); + s[8] += e * (m[5] + 2 * mr1 * mr2); + } + } + } + for (int k = 0; k < 9; k++) + s[k] = simd_sum(s[k]); + if (lane == 0) + for (int k = 0; k < 9; k++) + out[i].s[k] = s[k]; +} diff --git a/image_analysis/structure_refinement/AtomGradientMetal.mm b/image_analysis/structure_refinement/AtomGradientMetal.mm new file mode 100644 index 000000000..780187a2d --- /dev/null +++ b/image_analysis/structure_refinement/AtomGradientMetal.mm @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#import + +#include "AtomGradientMetal.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "../../common/CUDAWrapper.h" // gpus_are_hidden +#include "../../common/JFJochException.h" + +// The kernel's source, AtomGradientMetal.metal, copied in at configure time (CMakeLists.txt). +extern const char *const atom_gradient_metal_source; + +namespace { + +constexpr int ATOMS_PER_THREADGROUP = 8; // SIMD groups of 32 threads + +std::atomic calls_on_gpu{0}; + +static_assert(sizeof(AtomGradientMetalSums) == 9 * 4); + +struct Kernel { + id device = nil; // nil: no Metal device + id pipeline = nil; + id queue = nil; +}; + +// Compiled once per process, the first time it is asked for. +const Kernel &GetKernel() { + static const Kernel kernel = [] { + Kernel k; + k.device = MTLCreateSystemDefaultDevice(); + if (k.device == nil) + return k; + MTLCompileOptions *options = [MTLCompileOptions new]; + // Fast math off, so that a point's place rounds as on the CPU and the GPU takes the CPU's points. + options.fastMathEnabled = NO; + options.languageVersion = MTLLanguageVersion3_0; + NSError *error = nil; + id library = [k.device newLibraryWithSource:@(atom_gradient_metal_source) options:options + error:&error]; + if (library == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + std::string("atom gradient kernel: ") + error.localizedDescription.UTF8String); + id function = [library newFunctionWithName:@"atom_gradient"]; + k.pipeline = [k.device newComputePipelineStateWithFunction:function error:&error]; + if (k.pipeline == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + std::string("atom_gradient: ") + error.localizedDescription.UTF8String); + k.queue = [k.device newCommandQueue]; + return k; + }(); + return kernel; +} + +id NewBuffer(id device, size_t bytes) { + id b = [device newBufferWithLength:std::max(bytes, 4) options:MTLResourceStorageModeShared]; + if (b == nil) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, "atom gradient: out of GPU memory"); + return b; +} + +} // namespace + +bool AtomGradientMetal(const ModelDensityMetalGrid &grid, const std::vector &atoms, + const float *map, std::vector &out) { + // In a pool of its own, as ModelDensityMetal(): the threads this runs on have none. + @autoreleasepool { + const size_t points = static_cast(grid.nu) * grid.nv * grid.nw; + if (gpus_are_hidden() || points > std::numeric_limits::max() || + atoms.size() > std::numeric_limits::max()) + return false; + const int n3[3] = {grid.nu, grid.nv, grid.nw}; + for (const ModelDensityMetalAtom &a : atoms) + for (int k = 0; k < 3; k++) + if (2 * a.d[k] + 1 > n3[k]) + return false; // a box wider than the cell: the CPU's walk adds both images of a point + const Kernel &k = GetKernel(); + if (k.device == nil) + return false; + if (atoms.empty()) { + out.clear(); + return true; + } + + // The map as it is where its pages can be handed to the GPU, a copy otherwise. + const size_t page = static_cast(getpagesize()); + const size_t bytes = points * sizeof(float); + id d_map = nil; + if (reinterpret_cast(map) % page == 0) + d_map = [k.device newBufferWithBytesNoCopy:const_cast(map) length:(bytes + page - 1) / page * page + options:MTLResourceStorageModeShared deallocator:nil]; + if (d_map == nil) { + d_map = NewBuffer(k.device, bytes); + std::memcpy(d_map.contents, map, bytes); + } + id d_atoms = NewBuffer(k.device, atoms.size() * sizeof(ModelDensityMetalAtom)); + std::memcpy(d_atoms.contents, atoms.data(), atoms.size() * sizeof(ModelDensityMetalAtom)); + id d_out = NewBuffer(k.device, atoms.size() * sizeof(AtomGradientMetalSums)); + const uint32_t n = static_cast(atoms.size()); + + id cb = [k.queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:k.pipeline]; + [e setBuffer:d_atoms offset:0 atIndex:0]; + [e setBytes:&grid length:sizeof(grid) atIndex:1]; + [e setBuffer:d_map offset:0 atIndex:2]; + [e setBuffer:d_out offset:0 atIndex:3]; + [e setBytes:&n length:sizeof(n) atIndex:4]; + [e dispatchThreadgroups:MTLSizeMake((n + ATOMS_PER_THREADGROUP - 1) / ATOMS_PER_THREADGROUP, 1, 1) + threadsPerThreadgroup:MTLSizeMake(32 * ATOMS_PER_THREADGROUP, 1, 1)]; + [e endEncoding]; + [cb commit]; + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) + throw JFJochException(JFJochExceptionCategory::GPUMetalError, + cb.error ? cb.error.localizedDescription.UTF8String + : "atom gradient: command buffer failed"); + out.resize(atoms.size()); + std::memcpy(out.data(), d_out.contents, atoms.size() * sizeof(AtomGradientMetalSums)); + calls_on_gpu++; + return true; + } +} + +size_t AtomGradientMetalCalls() { + return calls_on_gpu; +} diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index 54e66051f..f513855cf 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -108,5 +108,18 @@ IF (APPLE) ModelDensityMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/ModelDensityMetalSource.cpp) SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc) SET_SOURCE_FILES_PROPERTIES(ModelDensityMetal.metal PROPERTIES HEADER_FILE_ONLY ON) + # The refinement gradient's sums over the map (AtomGradientMetal.h), carried in the same way. + FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal text) + SET_PROPERTY(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/AtomGradientMetal.metal) + FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp + "// Generated from image_analysis/structure_refinement by its CMakeLists.txt - do not edit.\n" + "extern const char *const atom_gradient_metal_source;\n" + "const char *const atom_gradient_metal_source = R\"GRADIENT_METAL(\n${text})GRADIENT_METAL\";\n") + CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp.tmp + ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp COPYONLY) + TARGET_SOURCES(JFJochStructureRefinement PRIVATE AtomGradientMetal.mm AtomGradientMetal.h + AtomGradientMetal.metal ${CMAKE_CURRENT_BINARY_DIR}/AtomGradientMetalSource.cpp) + SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.mm PROPERTIES COMPILE_OPTIONS -fobjc-arc) + SET_SOURCE_FILES_PROPERTIES(AtomGradientMetal.metal PROPERTIES HEADER_FILE_ONLY ON) TARGET_LINK_LIBRARIES(JFJochStructureRefinement "-framework Metal" "-framework Foundation") ENDIF() diff --git a/image_analysis/structure_refinement/RefinementXray.cpp b/image_analysis/structure_refinement/RefinementXray.cpp index 3544710fb..956d7c9ae 100644 --- a/image_analysis/structure_refinement/RefinementXray.cpp +++ b/image_analysis/structure_refinement/RefinementXray.cpp @@ -16,12 +16,19 @@ #include "gemmi/math.hpp" #include "gemmi/twin.hpp" +#include "../../common/JFJochException.h" #include "../../common/Logger.h" #include "../../common/ParallelFor.h" #include "ModelFFT.h" #include "ModelGrid.h" #include "RefinementMath.h" +#ifdef __APPLE__ +#include + +#include "AtomGradientMetal.h" +#endif + namespace { using Table = gemmi::IT92; @@ -230,7 +237,7 @@ public: map_.data[i] *= norm; }); const bool aniso = AnyAniso(); - AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr); + AtomGradientsFromMap(map_, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr, true); if (!aniso) grad_u_.clear(); return sum; @@ -919,10 +926,97 @@ void AnisoGradient(const gemmi::Grid &map, const gemmi::DensityCalculator *grad_u = {occ * gu[0], occ * gu[1], occ * gu[2], 2 * occ * gu[3], 2 * occ * gu[4], 2 * occ * gu[5]}; } +#ifdef __APPLE__ +// AtomGradientsFromMap() with the sums over the map on the Apple GPU (AtomGradientMetal.h): each atom's box, radius and +// Gaussians as the CPU's loops below take them, then the sums turned into the derivatives as there. False where the +// GPU does not take it - and from the first time it fails on, with one warning, as the CPU then does the same work. +std::atomic gradient_metal_failed{false}; + +bool AtomGradientsFromMapMetal(const gemmi::Grid &map, const gemmi::DensityCalculator &dc, + const std::vector &atoms, double blur, size_t nthreads, + std::vector> &grad, std::vector> *grad_u) { + if (gradient_metal_failed) + return false; + using CReal = Table::Coef::coef_type; + const gemmi::UpperTriangularMat33 &om = map.orth_n; + const ModelDensityMetalGrid g{map.nu, map.nv, map.nw, static_cast(om.a11), static_cast(om.a12), + static_cast(om.a13), static_cast(om.a22), static_cast(om.a23), + static_cast(om.a33)}; + const int n = static_cast(atoms.size()); + std::vector in(n); + ParallelChunks(n, nthreads, [&](int lo, int hi) { + for (int i = lo; i < hi; i++) { + const gemmi::Atom &atom = *atoms[i]; + ModelDensityMetalAtom &a = in[i]; + const auto &coef = IonFormFactor(atom.element.elem, atom.charge); + double radius; + a.aniso = atom.aniso.nonzero() ? 1 : 0; + for (auto &b : a.b) + std::fill(std::begin(b), std::end(b), 0.0f); + if (!a.aniso) { + const float b = static_cast(atom.b_iso + blur); + const auto precal = coef.precalculate_density_iso(b, 0.0f); + radius = dc.estimate_radius(precal, b); + for (int q = 0; q < 5; q++) { + a.a[q] = precal.a[q]; + a.b[q][0] = precal.b[q]; + } + } else { + const auto aniso_b = atom.aniso.scaled(CReal(gemmi::u_to_b())).added_kI(CReal(blur)); + const CReal b_max = AnisoRadiusB(aniso_b); + radius = dc.estimate_radius(coef.precalculate_density_iso(b_max, 0.0f), b_max); + const auto precal = coef.precalculate_density_aniso_b(aniso_b, 0.0f); + for (int q = 0; q < 5; q++) { + const gemmi::SMat33 &m = precal.b[q]; + a.a[q] = precal.a[q]; + const float row[6] = {m.u11, m.u22, m.u33, m.u12, m.u13, m.u23}; + std::copy(std::begin(row), std::end(row), a.b[q]); + } + } + const gemmi::Fractional fpos = map.unit_cell.fractionalize(atom.pos); + const double n3[3] = {fpos.x * map.nu, fpos.y * map.nv, fpos.z * map.nw}; + for (int k = 0; k < 3; k++) { + a.nctr[k] = static_cast(n3[k]); + a.c[k] = gemmi::iround(n3[k]); + } + a.d[0] = std::min(static_cast(std::ceil(radius / map.spacing[0])), map.nu - 1); + a.d[1] = std::min(static_cast(std::ceil(radius / map.spacing[1])), map.nv - 1); + a.d[2] = std::min(static_cast(std::ceil(radius / map.spacing[2])), map.nw - 1); + a.max_r2 = static_cast(radius * radius); + a.occ = atom.occ; + } + }); + std::vector sums; + try { + if (!AtomGradientMetal(g, in, map.data.data(), sums)) + return false; + } catch (const JFJochException &e) { + if (!gradient_metal_failed.exchange(true)) + Logger("Rugnux").Warning("Refinement gradient: the Apple GPU failed, on the CPU from here on: {}", e.what()); + return false; + } + constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI; + for (int i = 0; i < n; i++) { + const float *s = sums[i].s; + const double occ = atoms[i]->occ; + if (!in[i].aniso) { + grad[i] = {2.0 * occ * s[0], 2.0 * occ * s[1], 2.0 * occ * s[2], occ * s[3] / FOUR_PI2}; + continue; + } + grad[i] = {2 * occ * s[0], 2 * occ * s[1], 2 * occ * s[2], + occ * (double(s[3]) + s[4] + s[5]) / gemmi::u_to_b()}; + if (grad_u != nullptr) + (*grad_u)[i] = {occ * s[3], occ * s[4], occ * s[5], 2 * occ * s[6], 2 * occ * s[7], 2 * occ * s[8]}; + } + return true; +} +#endif + } // namespace void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads, - std::vector> &grad, std::vector> *grad_u) { + std::vector> &grad, std::vector> *grad_u, + [[maybe_unused]] bool gpu) { const gemmi::DensityCalculator dc = Calculator(map.unit_cell, *map.spacegroup, 1.0, blur); std::vector atoms; for (const gemmi::Chain &ch : model.chains) @@ -932,6 +1026,10 @@ void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &mod grad.assign(atoms.size(), {0, 0, 0, 0}); if (grad_u != nullptr) grad_u->assign(atoms.size(), {0, 0, 0, 0, 0, 0}); +#ifdef __APPLE__ + if (gpu && AtomGradientsFromMapMetal(map, dc, atoms, blur, nthreads, grad, grad_u)) + return; +#endif const int nu = map.nu, nv = map.nv, nw = map.nw; constexpr double FOUR_PI2 = 4 * refine_math::PI * refine_math::PI; // in blocks of atoms taken as the threads come free: an anisotropic atom, or one of high B, costs several times diff --git a/image_analysis/structure_refinement/RefinementXray.h b/image_analysis/structure_refinement/RefinementXray.h index 66bf1dd44..3294fcc19 100644 --- a/image_analysis/structure_refinement/RefinementXray.h +++ b/image_analysis/structure_refinement/RefinementXray.h @@ -288,7 +288,9 @@ std::unique_ptr MakeRefinementEngineCPU(const gemmi::UnitCell // derivatives: dT/d(x, y, z, B) per atom. `map` is u fastest on the grid of RigidBodyZoneGrid(), the atoms // gridded with `blur`. Exposed for the tests. An atom with a nonzero aniso is gridded anisotropically, as // PutModelDensityOnGrid() grids it: its B derivative is that along U + dB / (8 pi^2) I, and with `grad_u` (resized -// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23. +// to the atoms, zero for the isotropic ones) its derivatives in U11 U22 U33 U12 U13 U23. With `gpu`, on the Apple GPU +// where there is one (AtomGradientMetal.h, macOS): the same points summed in single precision, the CPU's gradient to +// a few parts in 1e5 rather than bit for bit. void AtomGradientsFromMap(const gemmi::Grid &map, const gemmi::Model &model, double blur, size_t nthreads, std::vector> &grad, - std::vector> *grad_u = nullptr); + std::vector> *grad_u = nullptr, bool gpu = false); diff --git a/tests/AtomGradientMetalTest.cpp b/tests/AtomGradientMetalTest.cpp new file mode 100644 index 000000000..0ed3040dc --- /dev/null +++ b/tests/AtomGradientMetalTest.cpp @@ -0,0 +1,102 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#ifdef __APPLE__ + +#include +#include +#include + +#include + +#include "../image_analysis/structure_refinement/AtomGradientMetal.h" +#include "../image_analysis/structure_refinement/RefinementXray.h" +#include "../image_analysis/structure_refinement/RigidBodyRefine.h" + +// The refinement gradient's sums on the Apple GPU (AtomGradientMetal.h) against the CPU's: the same points, summed +// in single precision on the GPU and in double on the CPU. A Mac without a Metal device fails rather than skips, as +// in ModelDensityMetalTest.cpp. + +namespace { + // Atoms scattered over the whole cell - so that boxes wrap at every face - of a few elements, a quarter of them + // anisotropic, with occupancies below one. + gemmi::Model random_atoms(const gemmi::UnitCell &cell) { + std::mt19937 rng(17); + std::uniform_real_distribution frac(0.0, 1.0), b(8.0, 60.0), occ(0.3, 1.0), u(0.05, 0.4); + const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S}; + gemmi::Model model(1); + gemmi::Chain chain("A"); + gemmi::Residue res; + res.name = "UNK"; + res.seqid = gemmi::SeqId(1, ' '); + for (int i = 0; i < 2000; i++) { + gemmi::Atom a; + a.name = "X"; + a.element = elements[i % 4]; + a.pos = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng))); + a.b_iso = static_cast(b(rng)); + a.occ = static_cast(occ(rng)); + if (i % 4 == 3) { + const float d1 = static_cast(u(rng)), d2 = static_cast(u(rng)), + d3 = static_cast(u(rng)); + a.aniso = {d1, d2, d3, 0.3f * std::min(d1, d2), -0.2f * std::min(d1, d3), 0.1f * std::min(d2, d3)}; + } + res.atoms.push_back(a); + } + chain.residues.push_back(res); + model.chains.push_back(chain); + return model; + } + + // The largest difference of a derivative between the two, over the largest of that derivative on the CPU. + template + double worst(const std::vector> &cpu, const std::vector> &gpu) { + double out = 0; + for (size_t c = 0; c < N; c++) { + double max_g = 0, max_d = 0; + for (size_t i = 0; i < cpu.size(); i++) { + max_g = std::max(max_g, std::fabs(cpu[i][c])); + max_d = std::max(max_d, std::fabs(gpu[i][c] - cpu[i][c])); + } + out = std::max(out, max_d / max_g); + } + return out; + } + + void check_gradient(const gemmi::UnitCell &cell, const char *group, double d_min) { + const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); + gemmi::Grid map = RigidBodyZoneGrid(cell, sg, d_min); + map.data.resize(map.point_count()); + // a smooth map with both signs, as the FFT method's derivative map is + for (int w = 0; w < map.nw; w++) + for (int v = 0; v < map.nv; v++) + for (int u = 0; u < map.nu; u++) + map.data[map.index_q(u, v, w)] = static_cast( + std::sin(2 * M_PI * (3.0 * u / map.nu + 1.0 * v / map.nv)) + + std::cos(2 * M_PI * (2.0 * v / map.nv - 4.0 * w / map.nw)) + + 0.5 * std::sin(2 * M_PI * (5.0 * u / map.nu + 3.0 * w / map.nw))); + const gemmi::Model model = random_atoms(cell); + const double blur = 10.0; + std::vector> gc, gg; + std::vector> uc, ug; + AtomGradientsFromMap(map, model, blur, 4, gc, &uc, false); + const size_t before = AtomGradientMetalCalls(); + AtomGradientsFromMap(map, model, blur, 4, gg, &ug, true); + REQUIRE(AtomGradientMetalCalls() == before + 1); // the GPU summed it + REQUIRE(gg.size() == gc.size()); + const double g = worst(gc, gg), u = worst(uc, ug); + CAPTURE(group, g, u); + CHECK(g < 1e-5); + CHECK(u < 5e-5); + } +} + +TEST_CASE("AtomGradientMetal_MatchesCPU", "[portable][Metal]") { + // Triclinic, so that every term of the orthogonalization is there; then an orthorhombic group. + check_gradient(gemmi::UnitCell(41, 46, 52, 81, 96, 107), "P 1", 2.0); + check_gradient(gemmi::UnitCell(50, 56, 63, 90, 90, 90), "P 21 21 21", 1.6); +} + +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 17b290a3e..ce0055c8f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,6 +35,7 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL BraggIntegrationEngineMetalTest.cpp MolecularReplacementMetalTest.cpp ModelDensityMetalTest.cpp + AtomGradientMetalTest.cpp WriteReflectionsTest.cpp LoadReferenceMtzFreeFlagsTest.cpp LoadReferenceSfCifTest.cpp