From 168a4e64a10acd23daaeebea81cda420c298a690 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 18 Oct 2026 12:51:48 +0200 Subject: [PATCH] MolecularReplacement: prefetch the transform in the CPU rotation function, output unchanged The CPU rotation function interpolates the molecular transform at points scattered over its grid, four cache lines per point. Once the grid is far larger than the caches (835 MB for 4nwv: 16.9e9 interpolations over 17255 orientations) the loop waits on memory, at ~92 ns per interpolation per core on an Apple M6. MolecularTransform::Prefetch() asks for the rows a point's corners lie on, 4 reflections ahead of the one being computed, through one portable helper (__builtin_prefetch under GCC/Clang, nothing elsewhere). Same values, same order, so the output is bit-identical. Only for a transform of 256 MB or more: a smaller one sits mostly in the caches, where the extra address arithmetic only costs. In a standalone copy of the loop on the M6: 30% slower at 13-90 MB, even at ~130-175 MB, 1.5x faster at 400 MB, 1.75-1.8x at 835 MB (prefetch distance 4 to 8 equally good). M6 Mac mini, 4nwv, alternating runs on a quiet machine, against the previous commit: - rotation function 139.1/138.8 s -> 79.1/79.3/78.9 s - wall 400.7/415.7 s -> 338.0/343.1/345.6 s - --no-gpu: 450.4 s -> 385.4 s (rotation 140.3 -> 84.1 s) - 9s02 (14k reflections, 13 MB transform, below the threshold): rotation 2.83 s -> 2.98 s, wall 79.4 -> 80.4 s p.hkl, p_model.cif/pdb, the maps and the MTZ/CIF reflection data are byte-identical in every run (the files differ only in the git tag and command line in their headers). [MolecularReplacement] tests pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018My7tqt8a4LyKKjP3FhScT --- .../MolecularReplacement.cpp | 40 +++++++++++++++++++ .../MolecularReplacement.h | 2 + 2 files changed, 42 insertions(+) diff --git a/image_analysis/structure_refinement/MolecularReplacement.cpp b/image_analysis/structure_refinement/MolecularReplacement.cpp index 5d976310f..e5fc047e9 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.cpp +++ b/image_analysis/structure_refinement/MolecularReplacement.cpp @@ -58,6 +58,11 @@ constexpr double SEARCH_D_MIN = 4.0; constexpr double SEARCH_D_MAX = 12.0; constexpr int SEARCH_BINS = 10; constexpr double ROTATION_STEP_DEG = 6.0; +// How many reflections ahead the rotation function prefetches the transform (4 to 8 are equally fast), and +// from what size of the transform on: a smaller one is mostly in the caches, where prefetching only costs +// (on an Apple M6: 30% slower at 50 MB, even at ~150 MB, 1.75x faster at 835 MB). +constexpr size_t ROTATION_PREFETCH_AHEAD = 4; +constexpr size_t ROTATION_PREFETCH_MIN_BYTES = 256 * 1024 * 1024; // Orientations the translation function is run for, per merge, and placements per space group taken // through the rigid body. constexpr int TRANSLATION_ROTATIONS = 50; @@ -1199,6 +1204,35 @@ std::complex MolecularTransform::At(const gemmi::Vec3 &s) const { return out * std::exp(blur_ * s.length_sq() / 4.0); } +namespace { + +// A hint that the memory at p is read soon; a no-op where the compiler has no such builtin. +inline void PrefetchRead(const void *p) { +#if defined(__GNUC__) || defined(__clang__) + __builtin_prefetch(p); +#else + (void) p; +#endif +} + +} // namespace + +void MolecularTransform::Prefetch(const gemmi::Vec3 &s) const { + const int i0[3] = {static_cast(std::floor(s.x * box_)), static_cast(std::floor(s.y * box_)), + static_cast(std::floor(s.z * box_))}; + // At()'s corners lie on four rows of the grid (a corner and its neighbour along u share one). + for (int dy = 0; dy < 2; dy++) + for (int dz = 0; dz < 2; dz++) { + int i = i0[0], j = i0[1] + dy, l = i0[2] + dz; + if (l < 0) { + i = -i; j = -j; l = -l; + } + if (i < 0 || i >= grid_.nu) { i %= grid_.nu; if (i < 0) i += grid_.nu; } + if (j < 0 || j >= grid_.nv) { j %= grid_.nv; if (j < 0) j += grid_.nv; } + PrefetchRead(&grid_.data[i + static_cast(grid_.nu) * (j + static_cast(grid_.nv) * l)]); + } +} + // --- the observations --- SearchReflections MakeSearchReflections(const std::vector &merged, const gemmi::UnitCell &cell, @@ -1278,6 +1312,7 @@ std::vector RotationFunction(const MolecularTransform &ft, const SearchR for (double e2 : refl.e2) w_ss += (e2 - w_mean) * (e2 - w_mean); + const bool prefetch = ft.Grid().data.size() * sizeof(ft.Grid().data[0]) >= ROTATION_PREFETCH_MIN_BYTES; std::vector out(rotations.size(), 0.0); ParallelFor(static_cast(rotations.size()), nthreads, [&](int r) { // The model turned by R has F_R(s) = F_model(R^T s). @@ -1285,6 +1320,11 @@ std::vector RotationFunction(const MolecularTransform &ft, const SearchR std::vector ic(n, 0.0), bin_sum(refl.bins, 0.0); std::vector bin_n(refl.bins, 0); for (size_t i = 0; i < n; i++) { + // The transform is read at points scattered over a grid far larger than the caches, so this loop + // waits on memory: the points of a reflection a few ahead are fetched while this one is computed. + if (prefetch && i + ROTATION_PREFETCH_AHEAD < n) + for (size_t j = 0; j < nops; j++) + ft.Prefetch(rt.multiply(s[(i + ROTATION_PREFETCH_AHEAD) * nops + j])); for (size_t j = 0; j < nops; j++) ic[i] += std::norm(ft.At(rt.multiply(s[i * nops + j]))); bin_sum[refl.bin[i]] += ic[i]; diff --git a/image_analysis/structure_refinement/MolecularReplacement.h b/image_analysis/structure_refinement/MolecularReplacement.h index 8b1a0543a..8c90974a3 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.h +++ b/image_analysis/structure_refinement/MolecularReplacement.h @@ -92,6 +92,8 @@ public: MolecularTransform(const SearchModel &model, double d_min, size_t nthreads, size_t gpu_budget = 0); // F at the Cartesian reciprocal vector s (1/A), interpolated trilinearly. std::complex At(const gemmi::Vec3 &s) const; + // Asks the CPU to fetch the grid points At(s) reads, without waiting for them: a hint, nothing is computed. + void Prefetch(const gemmi::Vec3 &s) const; // The grid At() interpolates (half-l, XYZ), its box edge in A and the blur it is unblurred by. const gemmi::FPhiGrid &Grid() const { return grid_; } double Box() const { return box_; }