MolecularReplacement: prefetch the transform in the CPU rotation function, output unchanged
The CPU rotation function interpolates the molecular transform at points scattered over its grid, four cache lines per point. Once the grid is far larger than the caches (835 MB for 4nwv: 16.9e9 interpolations over 17255 orientations) the loop waits on memory, at ~92 ns per interpolation per core on an Apple M6. MolecularTransform::Prefetch() asks for the rows a point's corners lie on, 4 reflections ahead of the one being computed, through one portable helper (__builtin_prefetch under GCC/Clang, nothing elsewhere). Same values, same order, so the output is bit-identical. Only for a transform of 256 MB or more: a smaller one sits mostly in the caches, where the extra address arithmetic only costs. In a standalone copy of the loop on the M6: 30% slower at 13-90 MB, even at ~130-175 MB, 1.5x faster at 400 MB, 1.75-1.8x at 835 MB (prefetch distance 4 to 8 equally good). M6 Mac mini, 4nwv, alternating runs on a quiet machine, against the previous commit: - rotation function 139.1/138.8 s -> 79.1/79.3/78.9 s - wall 400.7/415.7 s -> 338.0/343.1/345.6 s - --no-gpu: 450.4 s -> 385.4 s (rotation 140.3 -> 84.1 s) - 9s02 (14k reflections, 13 MB transform, below the threshold): rotation 2.83 s -> 2.98 s, wall 79.4 -> 80.4 s p.hkl, p_model.cif/pdb, the maps and the MTZ/CIF reflection data are byte-identical in every run (the files differ only in the git tag and command line in their headers). [MolecularReplacement] tests pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018My7tqt8a4LyKKjP3FhScT
This commit is contained in:
1 parent
14ee27f362
commit
168a4e64a1
2 files changed
+42
No files matched your search
@@ -58,6 +58,11 @@ constexpr double SEARCH_D_MIN = 4.0;
|
||||
constexpr double SEARCH_D_MAX = 12.0;
|
||||
constexpr int SEARCH_BINS = 10;
|
||||
constexpr double ROTATION_STEP_DEG = 6.0;
|
||||
// How many reflections ahead the rotation function prefetches the transform (4 to 8 are equally fast), and
|
||||
// from what size of the transform on: a smaller one is mostly in the caches, where prefetching only costs
|
||||
// (on an Apple M6: 30% slower at 50 MB, even at ~150 MB, 1.75x faster at 835 MB).
|
||||
constexpr size_t ROTATION_PREFETCH_AHEAD = 4;
|
||||
constexpr size_t ROTATION_PREFETCH_MIN_BYTES = 256 * 1024 * 1024;
|
||||
// Orientations the translation function is run for, per merge, and placements per space group taken
|
||||
// through the rigid body.
|
||||
constexpr int TRANSLATION_ROTATIONS = 50;
|
||||
@@ -1199,6 +1204,35 @@ std::complex<double> MolecularTransform::At(const gemmi::Vec3 &s) const {
|
||||
return out * std::exp(blur_ * s.length_sq() / 4.0);
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
// A hint that the memory at p is read soon; a no-op where the compiler has no such builtin.
|
||||
inline void PrefetchRead(const void *p) {
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
__builtin_prefetch(p);
|
||||
#else
|
||||
(void) p;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void MolecularTransform::Prefetch(const gemmi::Vec3 &s) const {
|
||||
const int i0[3] = {static_cast<int>(std::floor(s.x * box_)), static_cast<int>(std::floor(s.y * box_)),
|
||||
static_cast<int>(std::floor(s.z * box_))};
|
||||
// At()'s corners lie on four rows of the grid (a corner and its neighbour along u share one).
|
||||
for (int dy = 0; dy < 2; dy++)
|
||||
for (int dz = 0; dz < 2; dz++) {
|
||||
int i = i0[0], j = i0[1] + dy, l = i0[2] + dz;
|
||||
if (l < 0) {
|
||||
i = -i; j = -j; l = -l;
|
||||
}
|
||||
if (i < 0 || i >= grid_.nu) { i %= grid_.nu; if (i < 0) i += grid_.nu; }
|
||||
if (j < 0 || j >= grid_.nv) { j %= grid_.nv; if (j < 0) j += grid_.nv; }
|
||||
PrefetchRead(&grid_.data[i + static_cast<size_t>(grid_.nu) * (j + static_cast<size_t>(grid_.nv) * l)]);
|
||||
}
|
||||
}
|
||||
|
||||
// --- the observations ---
|
||||
|
||||
SearchReflections MakeSearchReflections(const std::vector<MergedReflection> &merged, const gemmi::UnitCell &cell,
|
||||
@@ -1278,6 +1312,7 @@ std::vector<double> RotationFunction(const MolecularTransform &ft, const SearchR
|
||||
for (double e2 : refl.e2)
|
||||
w_ss += (e2 - w_mean) * (e2 - w_mean);
|
||||
|
||||
const bool prefetch = ft.Grid().data.size() * sizeof(ft.Grid().data[0]) >= ROTATION_PREFETCH_MIN_BYTES;
|
||||
std::vector<double> out(rotations.size(), 0.0);
|
||||
ParallelFor(static_cast<int>(rotations.size()), nthreads, [&](int r) {
|
||||
// The model turned by R has F_R(s) = F_model(R^T s).
|
||||
@@ -1285,6 +1320,11 @@ std::vector<double> RotationFunction(const MolecularTransform &ft, const SearchR
|
||||
std::vector<double> ic(n, 0.0), bin_sum(refl.bins, 0.0);
|
||||
std::vector<int> bin_n(refl.bins, 0);
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
// The transform is read at points scattered over a grid far larger than the caches, so this loop
|
||||
// waits on memory: the points of a reflection a few ahead are fetched while this one is computed.
|
||||
if (prefetch && i + ROTATION_PREFETCH_AHEAD < n)
|
||||
for (size_t j = 0; j < nops; j++)
|
||||
ft.Prefetch(rt.multiply(s[(i + ROTATION_PREFETCH_AHEAD) * nops + j]));
|
||||
for (size_t j = 0; j < nops; j++)
|
||||
ic[i] += std::norm(ft.At(rt.multiply(s[i * nops + j])));
|
||||
bin_sum[refl.bin[i]] += ic[i];
|
||||
|
||||
Reference in new issue
Block a user