Every CPU FFT site - the FFT indexer's batched 1D r2c, the beam-centre 2D r2c/c2r,
the model-path 3D transforms (MapToFPhi, MapFromFPhi(Execute), MapToFPhiInParallel),
the tNCS Patterson c2r and the viewer's 1D chart - goes through common/CpuFFT, a
plan-free wrapper over the pocketfft header already vendored with gemmi. The header
is included under a namespace of its own: gemmi includes it with
POCKETFFT_NO_MULTITHREADING, and the linker otherwise merges the two sets of
templates (MapToFPhiInParallel with 4 threads gave garbage until it was renamed).
MapToFPhiInParallel uses pocketfft's own threads (one call per stage) instead of a
ParallelFor over FFTW plans. The programs no longer link fftw3f; FFTW stays fetched
for jfjoch_test only, where [fft] checks pocketfft against it (rel. error < 1e-5).
Measured (Ryzen 5950X, --no-gpu -N 16, one run each, against rc187 aa898ac0f):
cytc_x10sa --model --no-refine 100.7 -> 99.5 s; euxfel83 first 6 files --no-refine
81.7 -> 83.3 s (noise level); full cytc refinement 1514 -> ~1399 s. Merged results
identical (same hkl md5, SG, cell, R-work/R-free/CC of model validation).
Microbench, single thread, FFTW_ESTIMATE as the call sites plan: pocketfft 0.4-0.6x
FFTW's time on the 2D/3D shapes; with FFTW_MEASURE the two are equal on 3D.
Indexer length 3142 = 2 x 1571 is 2.4x SLOWER on pocketfft (Bluestein); a smooth
length (3150) is 0.06 s vs FFTW's 0.25 s at 3142.
BeamCenterFFTGPU_MatchesTheCPUEngine: gate mismatches over 4M centres on the
threshold 7 (FFTW) -> 17 (pocketfft); bound raised to 30. pocketfft's float error
is FFTW's or lower (rms 1.4e-7 vs 1.5e-7 against double).
Known: "A near-extinct class is reported as a sub-lattice" [tncs] fails - its
synthetic reads modulation 20.0x against a 20.0x gate; FFT rounding tips it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
96 lines
4.0 KiB
C++
96 lines
4.0 KiB
C++
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include "BeamCenterFFTCPU.h"
|
|
|
|
#include <algorithm>
|
|
#include <complex>
|
|
#include <cstring>
|
|
#include <future>
|
|
#include <memory>
|
|
|
|
#include "../../common/CpuFFT.h"
|
|
|
|
namespace {
|
|
|
|
// One real-to-complex 2D FFT workspace of fixed padded size, reused for every forward transform.
|
|
class Rfft2 {
|
|
int64_t ny, nx, nxc;
|
|
std::vector<float> real;
|
|
|
|
public:
|
|
std::vector<std::complex<float>> spectrum;
|
|
|
|
Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) {
|
|
real.resize(static_cast<size_t>(ny * nx));
|
|
spectrum.resize(static_cast<size_t>(ny * nxc));
|
|
}
|
|
|
|
// Zero-pad `src` (height x width) into the workspace and return its spectrum.
|
|
std::vector<std::complex<float>> Forward(const std::vector<float> &src, int64_t height,
|
|
int64_t width) {
|
|
std::fill(real.begin(), real.end(), 0.0f);
|
|
for (int64_t y = 0; y < height; y++)
|
|
std::memcpy(&real[static_cast<size_t>(y * nx)], &src[static_cast<size_t>(y * width)],
|
|
sizeof(float) * static_cast<size_t>(width));
|
|
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, real.data(), spectrum.data());
|
|
return spectrum; // copy: the workspace is reused
|
|
}
|
|
|
|
// Inverse-transform the elementwise product u*v, normalised, cropped to out_h x out_w.
|
|
std::vector<float> InverseProduct(const std::vector<std::complex<float>> &u,
|
|
const std::vector<std::complex<float>> &v, int64_t out_h,
|
|
int64_t out_w) {
|
|
const float norm = 1.0f / (static_cast<float>(ny) * static_cast<float>(nx));
|
|
for (size_t i = 0; i < spectrum.size(); i++)
|
|
spectrum[i] = u[i] * v[i];
|
|
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, spectrum.data(), real.data());
|
|
std::vector<float> out(static_cast<size_t>(out_h * out_w));
|
|
for (int64_t y = 0; y < out_h; y++)
|
|
for (int64_t x = 0; x < out_w; x++)
|
|
out[static_cast<size_t>(y * out_w + x)] =
|
|
real[static_cast<size_t>(y * nx + x)] * norm;
|
|
return out;
|
|
}
|
|
};
|
|
|
|
} // namespace
|
|
|
|
BeamCenterConvSurfaces2D BeamCenterFFTCPU::PointSurfaces(const std::vector<float> &a,
|
|
const std::vector<float> &m, int64_t h,
|
|
int64_t w) {
|
|
// Two transforms at a time, each on a workspace of its own. A workspace is padded to twice the
|
|
// detector each way, so on a large detector the four workspaces and three spectra that the seven
|
|
// transforms used to hold at once were gigabytes. In order: the forward transforms of the image
|
|
// and the mask; the two inverse ones that read the image's spectrum, which is then handed back;
|
|
// the squared image's forward transform beside the mask's own inverse; and the last inverse.
|
|
const int64_t ny = BeamCenterFFTPadSize(2 * h), nx = BeamCenterFFTPadSize(2 * w);
|
|
Rfft2 ws0(ny, nx), ws1(ny, nx);
|
|
|
|
std::vector<std::complex<float>> A, M;
|
|
{
|
|
auto fA = std::async(std::launch::async, [&] { return ws0.Forward(a, h, w); });
|
|
M = ws1.Forward(m, h, w);
|
|
A = fA.get();
|
|
}
|
|
|
|
BeamCenterConvSurfaces2D out;
|
|
{
|
|
auto fC = std::async(std::launch::async, [&] { return ws0.InverseProduct(A, A, 2 * h, 2 * w); });
|
|
out.S = ws1.InverseProduct(A, M, 2 * h, 2 * w);
|
|
out.C = fC.get();
|
|
}
|
|
std::vector<std::complex<float>>().swap(A);
|
|
|
|
std::vector<float> a2(a.size());
|
|
for (size_t i = 0; i < a2.size(); i++)
|
|
a2[i] = a[i] * a[i];
|
|
{
|
|
auto fD = std::async(std::launch::async, [&] { return ws1.InverseProduct(M, M, 2 * h, 2 * w); });
|
|
const auto A2 = ws0.Forward(a2, h, w);
|
|
out.Q = ws0.InverseProduct(A2, M, 2 * h, 2 * w);
|
|
out.D = fD.get();
|
|
}
|
|
return out;
|
|
}
|