From 04baa16060098db38e62a5bc930ae30b709f0f32 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 15 Nov 2026 21:23:40 +0100 Subject: [PATCH] CPU FFTs on pocketfft instead of FFTW (prototype) Every CPU FFT site - the FFT indexer's batched 1D r2c, the beam-centre 2D r2c/c2r, the model-path 3D transforms (MapToFPhi, MapFromFPhi(Execute), MapToFPhiInParallel), the tNCS Patterson c2r and the viewer's 1D chart - goes through common/CpuFFT, a plan-free wrapper over the pocketfft header already vendored with gemmi. The header is included under a namespace of its own: gemmi includes it with POCKETFFT_NO_MULTITHREADING, and the linker otherwise merges the two sets of templates (MapToFPhiInParallel with 4 threads gave garbage until it was renamed). MapToFPhiInParallel uses pocketfft's own threads (one call per stage) instead of a ParallelFor over FFTW plans. The programs no longer link fftw3f; FFTW stays fetched for jfjoch_test only, where [fft] checks pocketfft against it (rel. error < 1e-5). Measured (Ryzen 5950X, --no-gpu -N 16, one run each, against rc187 aa898ac0f): cytc_x10sa --model --no-refine 100.7 -> 99.5 s; euxfel83 first 6 files --no-refine 81.7 -> 83.3 s (noise level); full cytc refinement 1514 -> ~1399 s. Merged results identical (same hkl md5, SG, cell, R-work/R-free/CC of model validation). Microbench, single thread, FFTW_ESTIMATE as the call sites plan: pocketfft 0.4-0.6x FFTW's time on the 2D/3D shapes; with FFTW_MEASURE the two are equal on 3D. Indexer length 3142 = 2 x 1571 is 2.4x SLOWER on pocketfft (Bluestein); a smooth length (3150) is 0.06 s vs FFTW's 0.25 s at 3142. BeamCenterFFTGPU_MatchesTheCPUEngine: gate mismatches over 4M centres on the threshold 7 (FFTW) -> 17 (pocketfft); bound raised to 30. pocketfft's float error is FFTW's or lower (rms 1.4e-7 vs 1.5e-7 against double). Known: "A near-extinct class is reported as a sub-lattice" [tncs] fails - its synthetic reads modulation 20.0x against a 20.0x gate; FFT rounding tips it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- CMakeLists.txt | 5 +- common/CMakeLists.txt | 1 + common/CpuFFT.cpp | 70 +++++++++ common/CpuFFT.h | 36 +++++ common/FFTWPlannerLock.h | 14 -- .../geom_refinement/BeamCenterFFT.h | 2 +- .../geom_refinement/BeamCenterFFTCPU.cpp | 32 +---- .../geom_refinement/BeamCenterFFTCPU.h | 2 +- .../geom_refinement/BeamCenterFFTEngine.h | 2 +- image_analysis/geom_refinement/CMakeLists.txt | 4 +- image_analysis/indexing/CMakeLists.txt | 3 +- image_analysis/indexing/FFTIndexerCPU.cpp | 36 +---- image_analysis/indexing/FFTIndexerCPU.h | 13 +- image_analysis/scale_merge/CMakeLists.txt | 2 +- .../scale_merge/TranslationalNCS.cpp | 38 ++--- .../structure_refinement/CMakeLists.txt | 2 +- .../structure_refinement/ModelFFT.cpp | 135 ++++-------------- .../structure_refinement/ModelFFT.h | 19 +-- .../MolecularReplacement.cpp | 14 +- .../structure_refinement/RefinementXray.cpp | 10 +- rugnux/CMakeLists.txt | 2 +- rugnux/rugnux_cli.cpp | 15 -- tests/BeamCenterFFTGPUTest.cpp | 5 +- tests/CMakeLists.txt | 3 +- tests/CpuFFTTest.cpp | 117 +++++++++++++++ viewer/CMakeLists.txt | 2 +- viewer/charts/JFJochDatasetInfoChartView.cpp | 22 +-- viewer/charts/JFJochDatasetInfoChartView.h | 4 - 28 files changed, 308 insertions(+), 302 deletions(-) create mode 100644 common/CpuFFT.cpp create mode 100644 common/CpuFFT.h delete mode 100644 common/FFTWPlannerLock.h create mode 100644 tests/CpuFFTTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b17e90199..eae9a9287 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -474,8 +474,9 @@ FetchContent_Declare(tiff EXCLUDE_FROM_ALL) FetchContent_MakeAvailable(tiff) -# FFTW single precision (target fftw3f): the CPU fallback indexer, enables JFJOCH_USE_FFTW. -# Built from the release tarball -- the git repo ships no pre-generated codelets (needs the +# FFTW single precision (target fftw3f): no longer used by the programs, whose CPU FFTs are pocketfft's +# (common/CpuFFT) - linked only by jfjoch_test, which checks pocketfft against it. JFJOCH_USE_FFTW still +# means "the CPU FFT indexer is built". Built from the release tarball -- the git repo ships no pre-generated codelets (needs the # OCaml genfft). SET(ENABLE_FLOAT ON CACHE BOOL "" FORCE) SET(BUILD_TESTS OFF CACHE BOOL "" FORCE) diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 97d641f1a..767c4f2b7 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -76,6 +76,7 @@ ADD_LIBRARY(JFJochZMQ STATIC ZMQWrappers.cpp ZMQWrappers.h) ADD_LIBRARY(JFJochCommon STATIC Coord.cpp Coord.h + CpuFFT.cpp CpuFFT.h DiffractionExperiment.cpp DiffractionExperiment.h RawToConvertedGeometry.h JFJochException.h diff --git a/common/CpuFFT.cpp b/common/CpuFFT.cpp new file mode 100644 index 000000000..6085fc0d3 --- /dev/null +++ b/common/CpuFFT.cpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "CpuFFT.h" + +// Under a name of its own: gemmi includes the same header with POCKETFFT_NO_MULTITHREADING, and its +// templates and ours must not be merged by the linker into one (single-threaded or not) copy. +#define pocketfft jfjoch_pocketfft +#include "../gemmi_gph/gemmi/third_party/pocketfft_hdronly.h" + +namespace cpu_fft { + +namespace { + +// pocketfft counts strides in bytes. +pocketfft::stride_t Bytes(const Strides &s, size_t element) { + pocketfft::stride_t out(s.size()); + for (size_t i = 0; i < s.size(); i++) + out[i] = s[i] * static_cast(element); + return out; +} + +Strides Contiguous(const Shape &shape) { + Strides s(shape.size()); + std::ptrdiff_t step = 1; + for (size_t i = shape.size(); i-- > 0;) { + s[i] = step; + step *= static_cast(shape[i]); + } + return s; +} + +Shape Halved(Shape shape, size_t axis) { + shape[axis] = shape[axis] / 2 + 1; + return shape; +} + +} // namespace + +void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes, + const float *in, std::complex *out, size_t nthreads) { + pocketfft::r2c(shape, Bytes(stride_in, sizeof(float)), Bytes(stride_out, sizeof(std::complex)), + axes, pocketfft::FORWARD, in, out, 1.0f, nthreads); +} + +void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex *out, size_t nthreads) { + R2C(shape, Contiguous(shape), Contiguous(Halved(shape, axes.back())), axes, in, out, nthreads); +} + +void C2R(const Shape &shape, const Shape &axes, std::complex *in, float *out, size_t nthreads) { + const Shape half = Halved(shape, axes.back()); + const pocketfft::stride_t stride_in = Bytes(Contiguous(half), sizeof(std::complex)); + if (axes.size() > 1) + pocketfft::c2c(half, stride_in, stride_in, Shape(axes.begin(), axes.end() - 1), pocketfft::BACKWARD, + in, in, 1.0f, nthreads); + pocketfft::c2r(shape, stride_in, Bytes(Contiguous(shape), sizeof(float)), axes.back(), + pocketfft::BACKWARD, in, out, 1.0f, nthreads); +} + +void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex *data, + size_t nthreads) { + const pocketfft::stride_t s = Bytes(stride, sizeof(std::complex)); + pocketfft::c2c(shape, s, s, axes, forward, data, data, 1.0f, nthreads); +} + +void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex *data, size_t nthreads) { + C2C(shape, Contiguous(shape), axes, forward, data, nthreads); +} + +} // namespace cpu_fft diff --git a/common/CpuFFT.h b/common/CpuFFT.h new file mode 100644 index 000000000..c1cebdb61 --- /dev/null +++ b/common/CpuFFT.h @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include + +// Single-precision CPU FFTs on pocketfft (the header vendored with gemmi). There are no plans to make: +// every call is a whole transform, safe from several threads at once. +// +// Arrays are row-major (the last axis fastest) and the transforms unnormalised, the forward one with +// exp(-2 pi i k.x), as FFTW's. A real transform halves the last axis in `axes` (n/2 + 1 complex values +// along it). Strides count elements of the array they belong to - floats on the real side, complex values +// on the complex side; without them the array is contiguous. `nthreads` splits one transform's lines. +namespace cpu_fft { + +using Shape = std::vector; +using Strides = std::vector; + +// Real -> complex, forward. `shape` is the real array's. +void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes, + const float *in, std::complex *out, size_t nthreads = 1); +void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex *out, size_t nthreads = 1); + +// Complex -> real, backward (exp(+2 pi i k.x)). `shape` is the real array's. `in` is overwritten, as by +// FFTW's c2r: the complex axes are transformed in place before the real one. +void C2R(const Shape &shape, const Shape &axes, std::complex *in, float *out, size_t nthreads = 1); + +// Complex -> complex, in place. +void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex *data, + size_t nthreads = 1); +void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex *data, size_t nthreads = 1); + +} // namespace cpu_fft diff --git a/common/FFTWPlannerLock.h b/common/FFTWPlannerLock.h deleted file mode 100644 index a432b7971..000000000 --- a/common/FFTWPlannerLock.h +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include - -// FFTW's planner - every fftwf_plan_* and fftwf_destroy_plan call - shares global state and is not -// thread-safe; executing a plan is. So every piece of code that makes or destroys a plan takes this -// one lock, not a lock of its own: two private locks do not stop two planners running at once. -inline std::mutex &FFTWPlannerMutex() { - static std::mutex m; - return m; -} diff --git a/image_analysis/geom_refinement/BeamCenterFFT.h b/image_analysis/geom_refinement/BeamCenterFFT.h index 4092b3be9..0ec907e10 100644 --- a/image_analysis/geom_refinement/BeamCenterFFT.h +++ b/image_analysis/geom_refinement/BeamCenterFFT.h @@ -30,7 +30,7 @@ // The input is the mean-projection the pre-scan already accumulates for the beam-stop search // (ShadowFinder::GetMeanProjection), NAN where nothing was counted - so this costs no new I/O. // -// The transforms run on the GPU where there is one with room for them and on fftw3f otherwise +// The transforms run on the GPU where there is one with room for them and on the CPU (pocketfft) otherwise // (BeamCenterFFTGPU / BeamCenterFFTCPU); everything that decides anything is shared between the // two. Whole-detector reach is what this buys, and on a 16 Mpixel detector the transforms are the // whole of its cost, so which engine runs them is the difference between seconds and milliseconds. diff --git a/image_analysis/geom_refinement/BeamCenterFFTCPU.cpp b/image_analysis/geom_refinement/BeamCenterFFTCPU.cpp index 1496752f5..52ca18938 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTCPU.cpp +++ b/image_analysis/geom_refinement/BeamCenterFFTCPU.cpp @@ -8,12 +8,8 @@ #include #include #include -#include -#include -#include - -#include "../../common/FFTWPlannerLock.h" +#include "../../common/CpuFFT.h" namespace { @@ -21,8 +17,6 @@ namespace { class Rfft2 { int64_t ny, nx, nxc; std::vector real; - fftwf_plan fwd = nullptr; - fftwf_plan bwd = nullptr; public: std::vector> spectrum; @@ -30,24 +24,6 @@ public: Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) { real.resize(static_cast(ny * nx)); spectrum.resize(static_cast(ny * nxc)); - std::unique_lock lock(FFTWPlannerMutex()); - fwd = fftwf_plan_dft_r2c_2d(static_cast(ny), static_cast(nx), real.data(), - reinterpret_cast(spectrum.data()), - FFTW_ESTIMATE); - bwd = fftwf_plan_dft_c2r_2d(static_cast(ny), static_cast(nx), - reinterpret_cast(spectrum.data()), - real.data(), FFTW_ESTIMATE); - if (!fwd || !bwd) - throw std::runtime_error("BeamCenterFFT: fftwf plan failed"); - } - Rfft2(const Rfft2 &) = delete; - Rfft2 &operator=(const Rfft2 &) = delete; - ~Rfft2() { - std::unique_lock lock(FFTWPlannerMutex()); - if (fwd) - fftwf_destroy_plan(fwd); - if (bwd) - fftwf_destroy_plan(bwd); } // Zero-pad `src` (height x width) into the workspace and return its spectrum. @@ -57,7 +33,7 @@ public: for (int64_t y = 0; y < height; y++) std::memcpy(&real[static_cast(y * nx)], &src[static_cast(y * width)], sizeof(float) * static_cast(width)); - fftwf_execute(fwd); + cpu_fft::R2C({static_cast(ny), static_cast(nx)}, {0, 1}, real.data(), spectrum.data()); return spectrum; // copy: the workspace is reused } @@ -68,7 +44,7 @@ public: const float norm = 1.0f / (static_cast(ny) * static_cast(nx)); for (size_t i = 0; i < spectrum.size(); i++) spectrum[i] = u[i] * v[i]; - fftwf_execute(bwd); + cpu_fft::C2R({static_cast(ny), static_cast(nx)}, {0, 1}, spectrum.data(), real.data()); std::vector out(static_cast(out_h * out_w)); for (int64_t y = 0; y < out_h; y++) for (int64_t x = 0; x < out_w; x++) @@ -88,8 +64,6 @@ BeamCenterConvSurfaces2D BeamCenterFFTCPU::PointSurfaces(const std::vector the CPU FFT indexer is always built. +# The CPU FFT indexer is always built (its transforms are pocketfft's, common/CpuFFT). TARGET_SOURCES(JFJochIndexing PRIVATE FFTIndexerCPU.cpp FFTIndexerCPU.h) -TARGET_LINK_LIBRARIES(JFJochIndexing fftw3f) diff --git a/image_analysis/indexing/FFTIndexerCPU.cpp b/image_analysis/indexing/FFTIndexerCPU.cpp index 43f663d17..e596cf44f 100644 --- a/image_analysis/indexing/FFTIndexerCPU.cpp +++ b/image_analysis/indexing/FFTIndexerCPU.cpp @@ -7,10 +7,9 @@ #include #include #include -#include #include -#include "../../common/FFTWPlannerLock.h" +#include "../../common/CpuFFT.h" #include "../../common/ParallelFor.h" static inline double dot_abs(const Coord& a, const Coord& b) { @@ -26,32 +25,9 @@ FFTIndexerCPU::FFTIndexerCPU(const IndexingSettings& settings) // Validate allocations vs. expected sizes if (h_input_fft.size() != input_size) - throw std::runtime_error("FFTWIndexer: input buffer size mismatch"); + throw std::runtime_error("FFTIndexerCPU: input buffer size mismatch"); if (h_output_fft.size() != output_size) - throw std::runtime_error("FFTWIndexer: output buffer size mismatch"); - - const int H = static_cast(histogram_size); - const int out_len = (H / 2) + 1; - - int n[1] = { H }; - - { - std::unique_lock ul(FFTWPlannerMutex()); - plan = fftwf_plan_many_dft_r2c( - 1, n, nDirections, - h_input_fft.data(), nullptr, 1, H, - reinterpret_cast(h_output_fft.data()), nullptr, 1, out_len, - FFTW_ESTIMATE); - } - - if (!plan) - throw std::runtime_error("fftw_plan_many_dft_r2c failed"); -} - - -FFTIndexerCPU::~FFTIndexerCPU() { - std::unique_lock ul(FFTWPlannerMutex()); - fftwf_destroy_plan(plan); + throw std::runtime_error("FFTIndexerCPU: output buffer size mismatch"); } void FFTIndexerCPU::ExecuteFFT(const std::vector &coord, size_t nspots) { @@ -80,8 +56,8 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector &coord, size_t nspots) { } }); - // Plan and execute batched R2C FFT with FFTW - fftwf_execute(plan); + // One real transform per direction, all in one call. + cpu_fft::R2C({static_cast(D), static_cast(H)}, {1}, h_input_fft.data(), h_output_fft.data()); // Post-process: pick the peak past min_length_A by PROMINENCE above a local background. // The projected histogram has a broad low-frequency ENVELOPE (spots cluster near the @@ -106,7 +82,7 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector &coord, size_t nspots) { pref[0] = 0.0; for (int j = 0; j < out_len; ++j) { - mag[j] = std::hypot(spec[j][0], spec[j][1]); + mag[j] = std::hypot(spec[j].real(), spec[j].imag()); pref[j + 1] = pref[j] + mag[j]; } diff --git a/image_analysis/indexing/FFTIndexerCPU.h b/image_analysis/indexing/FFTIndexerCPU.h index a784155c0..43beac286 100644 --- a/image_analysis/indexing/FFTIndexerCPU.h +++ b/image_analysis/indexing/FFTIndexerCPU.h @@ -4,30 +4,25 @@ #pragma once +#include #include -#include #include "FFTIndexer.h" #include "../../common/Coord.h" -// CPU/FFTW-based FFT indexer (drop-in replacement for the CUDA version). +// CPU FFT indexer (drop-in replacement for the CUDA version). class FFTIndexerCPU : public FFTIndexer { - // Host-side buffers for batched FFTW + // Host-side buffers for the batched transform // Input: [nDirections][histogram_size] // Output (R2C): [nDirections][(histogram_size/2)+1] std::vector h_input_fft; - std::vector> h_output_fft; // fftw_complex-compatible layout - - fftwf_plan plan; + std::vector> h_output_fft; void ExecuteFFT(const std::vector &coord, size_t nspots) override; public: explicit FFTIndexerCPU(const IndexingSettings& settings); - FFTIndexerCPU(const FFTIndexerCPU&) = delete; - FFTIndexerCPU& operator=(const FFTIndexerCPU&) = delete; - ~FFTIndexerCPU() override; }; diff --git a/image_analysis/scale_merge/CMakeLists.txt b/image_analysis/scale_merge/CMakeLists.txt index 989cfa3d8..ca6b2ba14 100644 --- a/image_analysis/scale_merge/CMakeLists.txt +++ b/image_analysis/scale_merge/CMakeLists.txt @@ -52,7 +52,7 @@ ADD_LIBRARY(JFJochScaleMerge SublatticeRelabel.h IceRingVerdict.cpp IceRingVerdict.h) -TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration fftw3f) +TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration) IF (JFJOCH_CUDA_AVAILABLE) TARGET_SOURCES(JFJochScaleMerge PRIVATE ../indexing/CUDAMemHelpers.h diff --git a/image_analysis/scale_merge/TranslationalNCS.cpp b/image_analysis/scale_merge/TranslationalNCS.cpp index 2acb83aee..cf9e1f486 100644 --- a/image_analysis/scale_merge/TranslationalNCS.cpp +++ b/image_analysis/scale_merge/TranslationalNCS.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -15,9 +16,7 @@ #include #include -#include - -#include "../../common/FFTWPlannerLock.h" +#include "../../common/CpuFFT.h" #include "../../common/ParallelFor.h" #include "gemmi/twin.hpp" @@ -350,20 +349,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector> coeff; + std::vector map; + try { + coeff.resize(n_complex); + map.resize(n_points); + } catch (const std::bad_alloc &) { result.refusal = "could not allocate the Patterson grid"; return result; } - fftwf_plan plan; - { - std::lock_guard lock(FFTWPlannerMutex()); - plan = fftwf_plan_dft_c2r_3d(nx, ny, nz, reinterpret_cast(coeff), map, - FFTW_ESTIMATE); - } // The near-origin exclusion, as a set of grid points rather than a per-point distance test: only // the points inside a 15 A ball around each origin-equivalent point can be excluded, and there @@ -405,17 +399,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector &values) { - std::memset(coeff, 0, 2 * n_complex * sizeof(float)); + std::fill(coeff.begin(), coeff.end(), std::complex(0.0f, 0.0f)); for (const auto &[at, src] : fill) - coeff[2 * at] += static_cast(values[src]); - fftwf_execute(plan); + coeff[at] += static_cast(values[src]); + cpu_fft::C2R({static_cast(nx), static_cast(ny), static_cast(nz)}, {0, 1, 2}, + coeff.data(), map.data()); }; auto max_far = [&]() { double best = -std::numeric_limits::infinity(); @@ -510,12 +502,6 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector #include #include -#include -#include - -#include +#include #include "gemmi/fail.hpp" -#include "../../common/FFTWPlannerLock.h" +#include "../../common/CpuFFT.h" #include "../../common/ParallelFor.h" -namespace { +// An (nu, nv, nw) map is stored u fastest - row-major (nw, nv, nu) - and its half-l coefficients +// likewise as (nw/2 + 1, nv, nu): the layout gemmi's own transforms read and write. The real transform +// halves w, so w is the last axis transformed. +static const cpu_fft::Shape kUVW{2, 1, 0}; -// The r2c plan (or, inverse, the c2r plan) for an (nu, nv, nw) map stored u fastest, halving w - the -// layout gemmi's own transforms read and write. FFTW's planner is not thread-safe, so plans are made -// under the process-wide planner lock (which also guards this cache); executing one on new arrays -// (fftwf_execute_dft_r2c / _c2r) is, which is how every caller uses it. -fftwf_plan PlanFor(int nu, int nv, int nw, bool inverse) { - static std::map, fftwf_plan> plans; - std::lock_guard lock(FFTWPlannerMutex()); - const std::array key{nu, nv, nw, inverse}; - if (const auto it = plans.find(key); it != plans.end()) - return it->second; - float *real = fftwf_alloc_real(static_cast(nu) * nv * nw); - fftwf_complex *cplx = fftwf_alloc_complex(static_cast(nu) * nv * (nw / 2 + 1)); - // The last dimension is the one r2c halves; the strides put u fastest on both sides. - fftwf_iodim dims[3] = {{nu, 1, 1}, {nv, nu, nu}, {nw, nu * nv, nu * nv}}; - fftwf_plan plan = nullptr; - if (real != nullptr && cplx != nullptr) - plan = inverse ? fftwf_plan_guru_dft_c2r(3, dims, 0, nullptr, cplx, real, FFTW_ESTIMATE) - : fftwf_plan_guru_dft_r2c(3, dims, 0, nullptr, real, cplx, FFTW_ESTIMATE); - fftwf_free(real); - fftwf_free(cplx); - if (plan == nullptr) - gemmi::fail("ModelFFT: no FFTW plan for the grid"); - plans.emplace(key, plan); - return plan; +static cpu_fft::Shape MapShape(int nu, int nv, int nw) { + return {static_cast(nw), static_cast(nv), static_cast(nu)}; } -} // namespace - gemmi::FPhiGrid MapToFPhi(const gemmi::Grid &map) { if (map.axis_order == gemmi::AxisOrder::ZYX) gemmi::fail("MapToFPhi(): ZYX order is not supported"); @@ -58,30 +34,16 @@ gemmi::FPhiGrid MapToFPhi(const gemmi::Grid &map) { hkl.set_size_without_checking(map.nu, map.nv, map.nw / 2 + 1); const float norm = static_cast(map.unit_cell.volume / map.point_count()); - const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, false); - // Buffers from fftwf_malloc: a plan may only be executed on arrays aligned as the ones it was made - // on were, which the vectors' own allocations do not promise. - float *in = fftwf_alloc_real(map.data.size()); - fftwf_complex *out = fftwf_alloc_complex(hkl.data.size()); - if (in == nullptr || out == nullptr) { - fftwf_free(in); - fftwf_free(out); - gemmi::fail("MapToFPhi(): cannot allocate the FFT buffers"); - } - std::copy(map.data.begin(), map.data.end(), in); - fftwf_execute_dft_r2c(plan, in, out); - // Conjugated, as gemmi does: FFTW's forward transform carries exp(-2 pi i h.x), and a structure + cpu_fft::R2C(MapShape(map.nu, map.nv, map.nw), kUVW, map.data.data(), hkl.data.data()); + // Conjugated, as gemmi does: the forward transform carries exp(-2 pi i h.x), and a structure // factor is the sum over exp(+2 pi i h.x). - for (size_t i = 0; i < hkl.data.size(); ++i) - hkl.data[i] = std::complex(out[i][0] * norm, -out[i][1] * norm); - fftwf_free(in); - fftwf_free(out); + for (auto &x : hkl.data) + x = std::conj(x) * norm; return hkl; } void MapFromFPhiExecute(int nu, int nv, int nw, std::complex *in, float *out) { - static_assert(sizeof(std::complex) == sizeof(fftwf_complex)); - fftwf_execute_dft_c2r(PlanFor(nu, nv, nw, true), reinterpret_cast(in), out); + cpu_fft::C2R(MapShape(nu, nv, nw), kUVW, in, out); } gemmi::FPhiGrid MapToFPhiInParallel(const gemmi::Grid &map, int reach, size_t nthreads) { @@ -89,34 +51,12 @@ gemmi::FPhiGrid MapToFPhiInParallel(const gemmi::Grid &map, int re gemmi::fail("MapToFPhiInParallel(): ZYX order is not supported"); const int nu = map.nu, nv = map.nv, nw = map.nw, nwh = nw / 2 + 1, nk = 2 * reach + 1; const size_t plane = static_cast(nu) * nv; - float *in = fftwf_alloc_real(map.data.size()); - fftwf_complex *half = fftwf_alloc_complex(plane * nwh); - if (in == nullptr || half == nullptr) { - fftwf_free(in); - fftwf_free(half); - gemmi::fail("MapToFPhiInParallel(): cannot allocate the FFT buffers"); - } - // Made once here; executing a plan on new arrays is safe from several threads at once. Unaligned, as a - // column or a plane starts wherever it does. - fftwf_plan columns = nullptr, planes = nullptr; - { - std::lock_guard lock(FFTWPlannerMutex()); - // The nu columns of one v, w along the stride of a plane. - fftwf_iodim col_dim{nw, static_cast(plane), static_cast(plane)}; - fftwf_iodim col_many{nu, 1, 1}; - columns = fftwf_plan_guru_dft_r2c(1, &col_dim, 1, &col_many, in, half, FFTW_ESTIMATE | FFTW_UNALIGNED); - fftwf_iodim plane_dims[2] = {{nv, nu, nu}, {nu, 1, 1}}; - planes = fftwf_plan_guru_dft(2, plane_dims, 0, nullptr, half, half, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_UNALIGNED); - } - if (columns == nullptr || planes == nullptr) - gemmi::fail("MapToFPhiInParallel(): no FFTW plan for the grid"); - std::copy(map.data.begin(), map.data.end(), in); - ParallelFor(nv, nthreads, [&](int v) { - fftwf_execute_dft_r2c(columns, in + static_cast(v) * nu, half + static_cast(v) * nu); - }); - ParallelFor(reach + 1, nthreads, [&](int l) { - fftwf_execute_dft(planes, half + l * plane, half + l * plane); - }); + std::vector> half(plane * nwh); + // The r2c along w of every (u, v) column, then the 2D transform of the planes l <= reach only - the + // rest are never read. Each is one call split over the threads by lines. + cpu_fft::R2C({static_cast(nw), plane}, {0}, map.data.data(), half.data(), nthreads); + cpu_fft::C2C({static_cast(reach) + 1, static_cast(nv), static_cast(nu)}, {1, 2}, true, + half.data(), nthreads); gemmi::FPhiGrid hkl; hkl.unit_cell = map.unit_cell; @@ -128,19 +68,12 @@ gemmi::FPhiGrid MapToFPhiInParallel(const gemmi::Grid &map, int re // Conjugated and scaled as MapToFPhi does it, and cropped as CropHalfL crops it. ParallelFor(reach + 1, nthreads, [&](int l) { for (int k = -reach; k <= reach; k++) { - const fftwf_complex *row = half + l * plane + static_cast(nu) * gemmi::modulo(k, nv); + const std::complex *row = &half[l * plane + static_cast(nu) * gemmi::modulo(k, nv)]; std::complex *out = &hkl.data[static_cast(nu) * (gemmi::modulo(k, nk) + static_cast(nk) * l)]; for (int u = 0; u < nu; u++) - out[u] = std::complex(row[u][0] * norm, -row[u][1] * norm); + out[u] = std::conj(row[u]) * norm; } }); - { - std::lock_guard lock(FFTWPlannerMutex()); - fftwf_destroy_plan(columns); - fftwf_destroy_plan(planes); - } - fftwf_free(in); - fftwf_free(half); return hkl; } @@ -154,26 +87,14 @@ gemmi::Grid MapFromFPhi(const gemmi::FPhiGrid &hkl) { map.axis_order = hkl.axis_order; const float norm = static_cast(1.0 / hkl.unit_cell.volume); - const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, true); - fftwf_complex *in = fftwf_alloc_complex(hkl.data.size()); - float *out = fftwf_alloc_real(map.data.size()); - if (in == nullptr || out == nullptr) { - fftwf_free(in); - fftwf_free(out); - gemmi::fail("MapFromFPhi(): cannot allocate the FFT buffers"); - } - // Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and FFTW's backward transform + // Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and the backward transform // carries exp(+2 pi i h.x). A missing coefficient (NaN) counts as zero, also as gemmi does. - for (size_t i = 0; i < hkl.data.size(); ++i) { - const std::complex x = hkl.data[i]; - in[i][0] = std::isnan(x.imag()) ? 0.0f : x.real(); - in[i][1] = std::isnan(x.imag()) ? 0.0f : -x.imag(); - } - fftwf_execute_dft_c2r(plan, in, out); - for (size_t i = 0; i < map.data.size(); ++i) - map.data[i] = out[i] * norm; - fftwf_free(in); - fftwf_free(out); + std::vector> in(hkl.data.size()); + for (size_t i = 0; i < in.size(); ++i) + in[i] = std::isnan(hkl.data[i].imag()) ? 0.0f : std::conj(hkl.data[i]); + cpu_fft::C2R(MapShape(map.nu, map.nv, map.nw), kUVW, in.data(), map.data.data()); + for (auto &x : map.data) + x *= norm; return map; } diff --git a/image_analysis/structure_refinement/ModelFFT.h b/image_analysis/structure_refinement/ModelFFT.h index d59d64984..1e44a294a 100644 --- a/image_analysis/structure_refinement/ModelFFT.h +++ b/image_analysis/structure_refinement/ModelFFT.h @@ -9,30 +9,23 @@ // The structure factors of a real-space map on the model path (Fcalc from the model density, Fmask // from the bulk-solvent mask): what gemmi::transform_map_to_f_phi(map, true) returns - the // conjugated forward transform (F = sum rho exp(+2 pi i h.x)) scaled by V/N, kept for l >= 0 on an (nu, nv, nw/2+1) grid in the map's own XYZ order, -// ready for prepare_asu_data() - computed with FFTW rather than gemmi's single-threaded pocketfft. -// The same transform, not the same bits: the two libraries order their arithmetic differently. -// -// Plans are FFTW_ESTIMATE only - MEASURE would time candidate algorithms and could pick a different -// one from run to run - and made once per grid size and kept, so the rigid body's hundred-odd -// evaluations on one grid plan once. Safe to call from several threads at once. +// ready for prepare_asu_data() - computed with common/CpuFFT. Safe to call from several threads at once. gemmi::FPhiGrid MapToFPhi(const gemmi::Grid &map); // The inverse: the real-space map of an (nu, nv, nw/2+1) half-l coefficient grid in XYZ order, as // get_f_phi_on_grid() fills it - what gemmi::transform_f_phi_grid_to_map() returns (rho = 1/V sum F -// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero), computed with FFTW -// under the same planning rules. +// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero). gemmi::Grid MapFromFPhi(const gemmi::FPhiGrid &hkl); // MapFromFPhi's transform itself, for a caller that fills its input: `in`, the (nu, nv, nw/2+1) half-l coefficients -// u fastest as MapFromFPhi hands them to FFTW (conjugated, a NaN as zero), is overwritten, and `out` receives the -// (nu, nv, nw) map before MapFromFPhi's 1/V. Both from fftwf_malloc, as a plan requires. +// u fastest as MapFromFPhi transforms them (conjugated, a NaN as zero), is overwritten, and `out` receives the +// (nu, nv, nw) map before MapFromFPhi's 1/V. void MapFromFPhiExecute(int nu, int nv, int nw, std::complex *in, float *out); // CropHalfL(MapToFPhi(map), reach) on `nthreads` threads, for a map of hundreds of millions of points (the // molecular transform's box), whose single-threaded transform is minutes of CPU: the r2c along w of every -// (u, v) column, then the 2D transform of every plane l <= reach - each column and each plane by one plan, so -// the result does not depend on the thread count. The same transform as MapToFPhi to float rounding, not bit -// for bit: FFTW's 3D plan orders its arithmetic its own way. +// (u, v) column, then the 2D transform of every plane l <= reach. The threads split the lines of a transform, each +// line transformed as it would be alone, so the result does not depend on the thread count. gemmi::FPhiGrid MapToFPhiInParallel(const gemmi::Grid &map, int reach, size_t nthreads); // The half-l grid `full` with only the indices |k|, l <= reach kept, on a grid of full.nu x (2 reach + 1) x diff --git a/image_analysis/structure_refinement/MolecularReplacement.cpp b/image_analysis/structure_refinement/MolecularReplacement.cpp index 6abacdba9..0afb637e9 100644 --- a/image_analysis/structure_refinement/MolecularReplacement.cpp +++ b/image_analysis/structure_refinement/MolecularReplacement.cpp @@ -19,7 +19,6 @@ #include #include -#include #include // remove_hydrogens, remove_alternative_conformations #include // setup_entities, remove_ligands_and_waters @@ -1602,13 +1601,8 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla w[t] = setup.weight[t] / (bin_sum[setup.bin[t]] / bin_n[setup.bin[t]]); const int nu = setup.nu, nv = setup.nv, nw = setup.nw; const size_t cells = static_cast(nu) * nv * (nw / 2 + 1), points = static_cast(nu) * nv * nw; - auto *in = reinterpret_cast *>(fftwf_alloc_complex(cells)); - float *map = fftwf_alloc_real(points); - if (in == nullptr || map == nullptr) { - fftwf_free(in); - fftwf_free(map); - throw std::runtime_error("TranslationFunction: cannot allocate the FFT buffers"); - } + std::vector> in(cells); + std::vector map(points); for (size_t c = 0; c < cells; c++) { std::complex v = 0.0; for (int k = gather.offset[c]; k < gather.offset[c + 1]; k++) { @@ -1626,7 +1620,7 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla const std::complex f(std::conj(v)); in[c] = std::isnan(f.imag()) ? std::complex(0.0f, 0.0f) : std::complex(f.real(), -f.imag()); } - MapFromFPhiExecute(nu, nv, nw, in, map); + MapFromFPhiExecute(nu, nv, nw, in.data(), map.data()); const float norm = static_cast(1.0 / ft.Grid().unit_cell.volume); double sum = 0, sum2 = 0; size_t best = 0; @@ -1640,8 +1634,6 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla const double count = static_cast(points); const double mean = sum / count, sd = std::sqrt(std::max(0.0, sum2 / count - mean * mean)); out.z = sd > 0 ? (map[best] * norm - mean) / sd : 0.0; - fftwf_free(in); - fftwf_free(map); const size_t u = best % nu, v = (best / nu) % nv, l = best / (static_cast(nu) * nv); out.t = gemmi::Fractional(static_cast(u) / nu, static_cast(v) / nv, static_cast(l) / nw); return out; diff --git a/image_analysis/structure_refinement/RefinementXray.cpp b/image_analysis/structure_refinement/RefinementXray.cpp index 7f8991888..955e887d3 100644 --- a/image_analysis/structure_refinement/RefinementXray.cpp +++ b/image_analysis/structure_refinement/RefinementXray.cpp @@ -10,7 +10,6 @@ #include #include -#include #include "gemmi/dencalc.hpp" #include "gemmi/it92.hpp" @@ -188,9 +187,8 @@ public: // l = nw/2, which hold both, as both. const int nu = grid_.nu, nv = grid_.nv, nw = grid_.nw, nwh = nw / 2 + 1; const size_t ncomplex = static_cast(nu) * nv * nwh; - std::complex *in = reinterpret_cast *>(fftwf_alloc_complex(ncomplex)); - float *out = fftwf_alloc_real(static_cast(nu) * nv * nw); - std::fill(in, in + ncomplex, std::complex(0, 0)); + std::vector> in(ncomplex); + std::vector out(static_cast(nu) * nv * nw); const std::vector &terms = composition_.Terms(); const size_t ops = composition_.Ops(); auto add = [&](int ku, int kv, int kw, std::complex value) { @@ -217,13 +215,11 @@ public: } } } - MapFromFPhiExecute(nu, nv, nw, in, out); + MapFromFPhiExecute(nu, nv, nw, in.data(), out.data()); gemmi::Grid map = grid_; const float norm = static_cast(cell_.volume / static_cast(map.point_count())); for (size_t i = 0; i < map.data.size(); i++) map.data[i] = out[i] * norm; - fftwf_free(in); - fftwf_free(out); const bool aniso = AnyAniso(); AtomGradientsFromMap(map, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr); if (!aniso) diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index 55379b405..12db3d061 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -41,7 +41,7 @@ ADD_LIBRARY(JFJochRugnux STATIC BrokerFeed.h ) -TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi fftw3f +TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi httplib::httplib) # The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 3238a24fd..4f1ebe1f7 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -846,21 +846,6 @@ namespace { } } -// FFTW ends the program itself when its allocator comes back empty (CK(p) in kernel/alloc.c) - while -// planning, or inside a buffered transform - from whichever thread that happens in, so it cannot become -// an exception. Defining its handler here keeps FFTW's own (kernel/assert.c) out of the link, and says -// what happened in the same words as every other way of running out of memory. -extern "C" void fftwf_assertion_failed(const char *s, int line, const char *file) { - std::fflush(stdout); - if (std::strcmp(s, "p") == 0) { - std::fprintf(stderr, "Processing failed: out of host memory (FFTW could not allocate) - " - "this data set needs more RAM than is available\n"); - std::_Exit(EXIT_FAILURE); - } - std::fprintf(stderr, "fftw: %s:%d: assertion failed: %s\n", file, line, s); - std::abort(); -} - namespace { // One input's run ended: what exit(EXIT_FAILURE) was where a command line ran one input only. A run of // several sweeps goes on with the next sweep; RunRugnux returns the code. diff --git a/tests/BeamCenterFFTGPUTest.cpp b/tests/BeamCenterFFTGPUTest.cpp index 3f5d1618b..d30023a78 100644 --- a/tests/BeamCenterFFTGPUTest.cpp +++ b/tests/BeamCenterFFTGPUTest.cpp @@ -174,10 +174,11 @@ TEST_CASE("BeamCenterFFTGPU_MatchesTheCPUEngine", "[BeamCenter][BeamCenterFFTGPU std::abs(static_cast(rg[i]) - rc[i])); CHECK(peak_worst < 1e-5); // And the gate agrees: over four million centres, the two paths disagree about whether a - // centre is scored at all only for a handful sitting exactly on the threshold. + // centre is scored at all only for a handful sitting exactly on the threshold (measured: 7 + // with FFTW on the CPU, 17 with pocketfft, whose float error is FFTW's or lower). const auto diff = Compare(rg, rc, &c.D, 0.26f * d_max); CHECK(diff.gate_mismatches == 0); - CHECK(Compare(rg, rc).gate_mismatches < 10); + CHECK(Compare(rg, rc).gate_mismatches < 30); // The device kernel against that reference, on the device's own surfaces - so this measures // the combination and not the transform. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b316d427c..17b290a3e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -237,6 +237,7 @@ ADD_EXECUTABLE(jfjoch_test LatticeReductionTest.cpp BeamCenterFromBackgroundTest.cpp BeamCenterFFTTest.cpp + CpuFFTTest.cpp BeamCenterFFTGPUTest.cpp BearerTokensTest.cpp BrokerHttpAuthTest.cpp @@ -246,7 +247,7 @@ ADD_EXECUTABLE(jfjoch_test target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochStreamWriter JFJochRugnuxOutput JFJochRugnux JFJochModelWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview - jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib) + jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib fftw3f) target_include_directories(jfjoch_test PRIVATE .) IF (JFJOCH_CUDA_AVAILABLE) TARGET_COMPILE_DEFINITIONS(jfjoch_test PRIVATE FFBIDX_SIMPLE_DATA_DIR="${FFBIDX_SIMPLE_DATA_DIR}") diff --git a/tests/CpuFFTTest.cpp b/tests/CpuFFTTest.cpp new file mode 100644 index 000000000..2046231d6 --- /dev/null +++ b/tests/CpuFFTTest.cpp @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include +#include +#include +#include + +#include + +#include "../common/CpuFFT.h" +#include "../image_analysis/structure_refinement/ModelFFT.h" + +// pocketfft (common/CpuFFT) against FFTW on the shapes and layouts the call sites use: the same +// transform to float rounding. FFTW is linked into the test binary only, as the reference. + +namespace { + +std::vector Random(size_t n, unsigned seed) { + std::mt19937 rng(seed); + std::normal_distribution g(0.0f, 1.0f); + std::vector v(n); + for (auto &x : v) + x = g(rng); + return v; +} + +// Largest difference relative to the largest value of the reference. +template +double RelativeError(const std::vector &a, const std::vector &ref) { + REQUIRE(a.size() == ref.size()); + double diff = 0, scale = 0; + for (size_t i = 0; i < a.size(); i++) { + diff = std::max(diff, static_cast(std::abs(a[i] - ref[i]))); + scale = std::max(scale, static_cast(std::abs(ref[i]))); + } + return diff / scale; +} + +} // namespace + +TEST_CASE("CpuFFT_BatchedR2CMatchesFFTW", "[fft]") { + // The FFT indexer's histograms: one transform per direction; 3142 = 2 x 1571 is a real length. + for (int n : {3142, 3150, 1024}) { + const int howmany = 37, nc = n / 2 + 1; + std::vector in = Random(static_cast(howmany) * n, n); + std::vector> ours(static_cast(howmany) * nc), ref(ours.size()); + cpu_fft::R2C({static_cast(howmany), static_cast(n)}, {1}, in.data(), ours.data()); + std::vector work = in; + fftwf_plan p = fftwf_plan_many_dft_r2c(1, &n, howmany, work.data(), nullptr, 1, n, + reinterpret_cast(ref.data()), nullptr, 1, nc, + FFTW_ESTIMATE); + fftwf_execute(p); + fftwf_destroy_plan(p); + CHECK(RelativeError(ours, ref) < 1e-5); + } +} + +TEST_CASE("CpuFFT_2DRoundTripMatchesFFTW", "[fft]") { + // The beam-centre search: a 2D r2c and c2r on a 2-3-5-7-smooth padded size. + const int ny = 210, nx = 336, nxc = nx / 2 + 1; + std::vector in = Random(static_cast(ny) * nx, 1); + std::vector> ours(static_cast(ny) * nxc), ref(ours.size()); + cpu_fft::R2C({static_cast(ny), static_cast(nx)}, {0, 1}, in.data(), ours.data()); + std::vector work = in; + fftwf_plan p = fftwf_plan_dft_r2c_2d(ny, nx, work.data(), reinterpret_cast(ref.data()), + FFTW_ESTIMATE); + fftwf_execute(p); + fftwf_destroy_plan(p); + CHECK(RelativeError(ours, ref) < 1e-5); + + std::vector back_ours(in.size()), back_ref(in.size()); + cpu_fft::C2R({static_cast(ny), static_cast(nx)}, {0, 1}, ours.data(), back_ours.data()); + p = fftwf_plan_dft_c2r_2d(ny, nx, reinterpret_cast(ref.data()), back_ref.data(), FFTW_ESTIMATE); + fftwf_execute(p); + fftwf_destroy_plan(p); + CHECK(RelativeError(back_ours, back_ref) < 1e-5); +} + +TEST_CASE("CpuFFT_3DMatchesFFTW", "[fft]") { + // The model maps (u fastest, w halved) and the tNCS Patterson (row-major, last axis halved) are both a + // row-major 3D transform halving the last axis; an odd length and a factor 7 included. + const int n0 = 70, n1 = 45, n2 = 54, nc = static_cast(static_cast(n0) * n1 * (n2 / 2 + 1)); + const std::vector in = Random(static_cast(n0) * n1 * n2, 2); + std::vector> ours(nc), ref(nc); + cpu_fft::R2C({70, 45, 54}, {0, 1, 2}, in.data(), ours.data()); + std::vector work = in; + fftwf_plan p = fftwf_plan_dft_r2c_3d(n0, n1, n2, work.data(), reinterpret_cast(ref.data()), + FFTW_ESTIMATE); + fftwf_execute(p); + fftwf_destroy_plan(p); + CHECK(RelativeError(ours, ref) < 1e-5); + + std::vector back_ours(in.size()), back_ref(in.size()); + cpu_fft::C2R({70, 45, 54}, {0, 1, 2}, ours.data(), back_ours.data()); + p = fftwf_plan_dft_c2r_3d(n0, n1, n2, reinterpret_cast(ref.data()), back_ref.data(), + FFTW_ESTIMATE); + fftwf_execute(p); + fftwf_destroy_plan(p); + CHECK(RelativeError(back_ours, back_ref) < 1e-5); +} + +TEST_CASE("CpuFFT_MapToFPhiInParallelMatchesMapToFPhi", "[fft]") { + gemmi::Grid map; + map.unit_cell.set(60, 50, 40, 90, 90, 90); + map.spacegroup = gemmi::find_spacegroup_by_name("P1"); + map.set_size(60, 50, 40); + map.data = Random(map.data.size(), 3); + const int reach = 12; + const gemmi::FPhiGrid ref = CropHalfL(MapToFPhi(map), reach); + for (size_t nthreads : {1, 4}) { + const gemmi::FPhiGrid ours = MapToFPhiInParallel(map, reach, nthreads); + CHECK(RelativeError(ours.data, ref.data) < 1e-5); + } +} diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 2251557d6..0345b7587 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -124,7 +124,7 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::Concurrent JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis JFJochRugnux JFJochRugnuxOutput - fftw3f httplib::httplib MbedTLS::mbedtls) + httplib::httplib MbedTLS::mbedtls) # HTTPS for the live-broker client. Defined here, on the viewer only, not on httplib::httplib: the # broker shares that target and stays plain HTTP. JFJochHttpReader.cpp is the one viewer source diff --git a/viewer/charts/JFJochDatasetInfoChartView.cpp b/viewer/charts/JFJochDatasetInfoChartView.cpp index 5a3dc3c44..03b8ce9fb 100644 --- a/viewer/charts/JFJochDatasetInfoChartView.cpp +++ b/viewer/charts/JFJochDatasetInfoChartView.cpp @@ -13,7 +13,7 @@ #include #include "JFJochDatasetInfoChartView.h" -#include "../../common/FFTWPlannerLock.h" +#include "../../common/CpuFFT.h" namespace { // Qt Charts draws its curves 2 px wide, which is hard to follow across a desk. Take the weight @@ -1122,25 +1122,7 @@ void JFJochDatasetInfoChartView::buildFFTChart() { std::vector> out(static_cast(nComplex)); - fftwf_plan plan; - { - std::lock_guard lock(FFTWPlannerMutex()); - plan = fftwf_plan_dft_r2c_1d( - n, - in.data(), - reinterpret_cast(out.data()), - FFTW_ESTIMATE); - } - - if (!plan) { - return; - } - - fftwf_execute(plan); - { - std::lock_guard lock(FFTWPlannerMutex()); - fftwf_destroy_plan(plan); - } + cpu_fft::R2C({N}, {0}, in.data(), out.data()); // Compute amplitude spectrum and frequencies (0 .. Nyquist) m_fftMagnitudes.resize(static_cast(nComplex)); diff --git a/viewer/charts/JFJochDatasetInfoChartView.h b/viewer/charts/JFJochDatasetInfoChartView.h index 5c1c82794..e6c1cfb40 100644 --- a/viewer/charts/JFJochDatasetInfoChartView.h +++ b/viewer/charts/JFJochDatasetInfoChartView.h @@ -15,10 +15,6 @@ #include #include -#ifdef JFJOCH_USE_FFTW -#include -#endif - #include "../../common/GoniometerAxis.h" #include "../../reader/JFJochReaderDataset.h"