diff --git a/image_analysis/structure_refinement/MolecularReplacement.cpp b/image_analysis/structure_refinement/MolecularReplacement.cpp index 5d976310f..e5fc047e9 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.cpp +++ b/image_analysis/structure_refinement/MolecularReplacement.cpp @@ -58,6 +58,11 @@ constexpr double SEARCH_D_MIN = 4.0; constexpr double SEARCH_D_MAX = 12.0; constexpr int SEARCH_BINS = 10; constexpr double ROTATION_STEP_DEG = 6.0; +// How many reflections ahead the rotation function prefetches the transform (4 to 8 are equally fast), and +// from what size of the transform on: a smaller one is mostly in the caches, where prefetching only costs +// (on an Apple M6: 30% slower at 50 MB, even at ~150 MB, 1.75x faster at 835 MB). +constexpr size_t ROTATION_PREFETCH_AHEAD = 4; +constexpr size_t ROTATION_PREFETCH_MIN_BYTES = 256 * 1024 * 1024; // Orientations the translation function is run for, per merge, and placements per space group taken // through the rigid body. constexpr int TRANSLATION_ROTATIONS = 50; @@ -1199,6 +1204,35 @@ std::complex MolecularTransform::At(const gemmi::Vec3 &s) const { return out * std::exp(blur_ * s.length_sq() / 4.0); } +namespace { + +// A hint that the memory at p is read soon; a no-op where the compiler has no such builtin. +inline void PrefetchRead(const void *p) { +#if defined(__GNUC__) || defined(__clang__) + __builtin_prefetch(p); +#else + (void) p; +#endif +} + +} // namespace + +void MolecularTransform::Prefetch(const gemmi::Vec3 &s) const { + const int i0[3] = {static_cast(std::floor(s.x * box_)), static_cast(std::floor(s.y * box_)), + static_cast(std::floor(s.z * box_))}; + // At()'s corners lie on four rows of the grid (a corner and its neighbour along u share one). + for (int dy = 0; dy < 2; dy++) + for (int dz = 0; dz < 2; dz++) { + int i = i0[0], j = i0[1] + dy, l = i0[2] + dz; + if (l < 0) { + i = -i; j = -j; l = -l; + } + if (i < 0 || i >= grid_.nu) { i %= grid_.nu; if (i < 0) i += grid_.nu; } + if (j < 0 || j >= grid_.nv) { j %= grid_.nv; if (j < 0) j += grid_.nv; } + PrefetchRead(&grid_.data[i + static_cast(grid_.nu) * (j + static_cast(grid_.nv) * l)]); + } +} + // --- the observations --- SearchReflections MakeSearchReflections(const std::vector &merged, const gemmi::UnitCell &cell, @@ -1278,6 +1312,7 @@ std::vector RotationFunction(const MolecularTransform &ft, const SearchR for (double e2 : refl.e2) w_ss += (e2 - w_mean) * (e2 - w_mean); + const bool prefetch = ft.Grid().data.size() * sizeof(ft.Grid().data[0]) >= ROTATION_PREFETCH_MIN_BYTES; std::vector out(rotations.size(), 0.0); ParallelFor(static_cast(rotations.size()), nthreads, [&](int r) { // The model turned by R has F_R(s) = F_model(R^T s). @@ -1285,6 +1320,11 @@ std::vector RotationFunction(const MolecularTransform &ft, const SearchR std::vector ic(n, 0.0), bin_sum(refl.bins, 0.0); std::vector bin_n(refl.bins, 0); for (size_t i = 0; i < n; i++) { + // The transform is read at points scattered over a grid far larger than the caches, so this loop + // waits on memory: the points of a reflection a few ahead are fetched while this one is computed. + if (prefetch && i + ROTATION_PREFETCH_AHEAD < n) + for (size_t j = 0; j < nops; j++) + ft.Prefetch(rt.multiply(s[(i + ROTATION_PREFETCH_AHEAD) * nops + j])); for (size_t j = 0; j < nops; j++) ic[i] += std::norm(ft.At(rt.multiply(s[i * nops + j]))); bin_sum[refl.bin[i]] += ic[i]; diff --git a/image_analysis/structure_refinement/MolecularReplacement.h b/image_analysis/structure_refinement/MolecularReplacement.h index 8b1a0543a..8c90974a3 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.h +++ b/image_analysis/structure_refinement/MolecularReplacement.h @@ -92,6 +92,8 @@ public: MolecularTransform(const SearchModel &model, double d_min, size_t nthreads, size_t gpu_budget = 0); // F at the Cartesian reciprocal vector s (1/A), interpolated trilinearly. std::complex At(const gemmi::Vec3 &s) const; + // Asks the CPU to fetch the grid points At(s) reads, without waiting for them: a hint, nothing is computed. + void Prefetch(const gemmi::Vec3 &s) const; // The grid At() interpolates (half-l, XYZ), its box edge in A and the blur it is unblurred by. const gemmi::FPhiGrid &Grid() const { return grid_; } double Box() const { return box_; }