Files
leonarski_fandClaude Opus 5.5 04baa16060 CPU FFTs on pocketfft instead of FFTW (prototype)
Every CPU FFT site - the FFT indexer's batched 1D r2c, the beam-centre 2D r2c/c2r,
the model-path 3D transforms (MapToFPhi, MapFromFPhi(Execute), MapToFPhiInParallel),
the tNCS Patterson c2r and the viewer's 1D chart - goes through common/CpuFFT, a
plan-free wrapper over the pocketfft header already vendored with gemmi. The header
is included under a namespace of its own: gemmi includes it with
POCKETFFT_NO_MULTITHREADING, and the linker otherwise merges the two sets of
templates (MapToFPhiInParallel with 4 threads gave garbage until it was renamed).
MapToFPhiInParallel uses pocketfft's own threads (one call per stage) instead of a
ParallelFor over FFTW plans. The programs no longer link fftw3f; FFTW stays fetched
for jfjoch_test only, where [fft] checks pocketfft against it (rel. error < 1e-5).

Measured (Ryzen 5950X, --no-gpu -N 16, one run each, against rc187 aa898ac0f):
cytc_x10sa --model --no-refine 100.7 -> 99.5 s; euxfel83 first 6 files --no-refine
81.7 -> 83.3 s (noise level); full cytc refinement 1514 -> ~1399 s. Merged results
identical (same hkl md5, SG, cell, R-work/R-free/CC of model validation).
Microbench, single thread, FFTW_ESTIMATE as the call sites plan: pocketfft 0.4-0.6x
FFTW's time on the 2D/3D shapes; with FFTW_MEASURE the two are equal on 3D.
Indexer length 3142 = 2 x 1571 is 2.4x SLOWER on pocketfft (Bluestein); a smooth
length (3150) is 0.06 s vs FFTW's 0.25 s at 3142.

BeamCenterFFTGPU_MatchesTheCPUEngine: gate mismatches over 4M centres on the
threshold 7 (FFTW) -> 17 (pocketfft); bound raised to 30. pocketfft's float error
is FFTW's or lower (rms 1.4e-7 vs 1.5e-7 against double).
Known: "A near-extinct class is reported as a sub-lattice" [tncs] fails - its
synthetic reads modulation 20.0x against a 20.0x gate; FFT rounding tips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
2026-11-15 21:23:40 +01:00

96 lines
4.0 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "BeamCenterFFTCPU.h"
#include <algorithm>
#include <complex>
#include <cstring>
#include <future>
#include <memory>
#include "../../common/CpuFFT.h"
namespace {
// One real-to-complex 2D FFT workspace of fixed padded size, reused for every forward transform.
class Rfft2 {
int64_t ny, nx, nxc;
std::vector<float> real;
public:
std::vector<std::complex<float>> spectrum;
Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) {
real.resize(static_cast<size_t>(ny * nx));
spectrum.resize(static_cast<size_t>(ny * nxc));
}
// Zero-pad `src` (height x width) into the workspace and return its spectrum.
std::vector<std::complex<float>> Forward(const std::vector<float> &src, int64_t height,
int64_t width) {
std::fill(real.begin(), real.end(), 0.0f);
for (int64_t y = 0; y < height; y++)
std::memcpy(&real[static_cast<size_t>(y * nx)], &src[static_cast<size_t>(y * width)],
sizeof(float) * static_cast<size_t>(width));
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, real.data(), spectrum.data());
return spectrum; // copy: the workspace is reused
}
// Inverse-transform the elementwise product u*v, normalised, cropped to out_h x out_w.
std::vector<float> InverseProduct(const std::vector<std::complex<float>> &u,
const std::vector<std::complex<float>> &v, int64_t out_h,
int64_t out_w) {
const float norm = 1.0f / (static_cast<float>(ny) * static_cast<float>(nx));
for (size_t i = 0; i < spectrum.size(); i++)
spectrum[i] = u[i] * v[i];
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, spectrum.data(), real.data());
std::vector<float> out(static_cast<size_t>(out_h * out_w));
for (int64_t y = 0; y < out_h; y++)
for (int64_t x = 0; x < out_w; x++)
out[static_cast<size_t>(y * out_w + x)] =
real[static_cast<size_t>(y * nx + x)] * norm;
return out;
}
};
} // namespace
BeamCenterConvSurfaces2D BeamCenterFFTCPU::PointSurfaces(const std::vector<float> &a,
const std::vector<float> &m, int64_t h,
int64_t w) {
// Two transforms at a time, each on a workspace of its own. A workspace is padded to twice the
// detector each way, so on a large detector the four workspaces and three spectra that the seven
// transforms used to hold at once were gigabytes. In order: the forward transforms of the image
// and the mask; the two inverse ones that read the image's spectrum, which is then handed back;
// the squared image's forward transform beside the mask's own inverse; and the last inverse.
const int64_t ny = BeamCenterFFTPadSize(2 * h), nx = BeamCenterFFTPadSize(2 * w);
Rfft2 ws0(ny, nx), ws1(ny, nx);
std::vector<std::complex<float>> A, M;
{
auto fA = std::async(std::launch::async, [&] { return ws0.Forward(a, h, w); });
M = ws1.Forward(m, h, w);
A = fA.get();
}
BeamCenterConvSurfaces2D out;
{
auto fC = std::async(std::launch::async, [&] { return ws0.InverseProduct(A, A, 2 * h, 2 * w); });
out.S = ws1.InverseProduct(A, M, 2 * h, 2 * w);
out.C = fC.get();
}
std::vector<std::complex<float>>().swap(A);
std::vector<float> a2(a.size());
for (size_t i = 0; i < a2.size(); i++)
a2[i] = a[i] * a[i];
{
auto fD = std::async(std::launch::async, [&] { return ws1.InverseProduct(M, M, 2 * h, 2 * w); });
const auto A2 = ws0.Forward(a2, h, w);
out.Q = ws0.InverseProduct(A2, M, 2 * h, 2 * w);
out.D = fD.get();
}
return out;
}