CPU FFTs on pocketfft instead of FFTW (prototype)

Every CPU FFT site - the FFT indexer's batched 1D r2c, the beam-centre 2D r2c/c2r,
the model-path 3D transforms (MapToFPhi, MapFromFPhi(Execute), MapToFPhiInParallel),
the tNCS Patterson c2r and the viewer's 1D chart - goes through common/CpuFFT, a
plan-free wrapper over the pocketfft header already vendored with gemmi. The header
is included under a namespace of its own: gemmi includes it with
POCKETFFT_NO_MULTITHREADING, and the linker otherwise merges the two sets of
templates (MapToFPhiInParallel with 4 threads gave garbage until it was renamed).
MapToFPhiInParallel uses pocketfft's own threads (one call per stage) instead of a
ParallelFor over FFTW plans. The programs no longer link fftw3f; FFTW stays fetched
for jfjoch_test only, where [fft] checks pocketfft against it (rel. error < 1e-5).

Measured (Ryzen 5950X, --no-gpu -N 16, one run each, against rc187 aa898ac0f):
cytc_x10sa --model --no-refine 100.7 -> 99.5 s; euxfel83 first 6 files --no-refine
81.7 -> 83.3 s (noise level); full cytc refinement 1514 -> ~1399 s. Merged results
identical (same hkl md5, SG, cell, R-work/R-free/CC of model validation).
Microbench, single thread, FFTW_ESTIMATE as the call sites plan: pocketfft 0.4-0.6x
FFTW's time on the 2D/3D shapes; with FFTW_MEASURE the two are equal on 3D.
Indexer length 3142 = 2 x 1571 is 2.4x SLOWER on pocketfft (Bluestein); a smooth
length (3150) is 0.06 s vs FFTW's 0.25 s at 3142.

BeamCenterFFTGPU_MatchesTheCPUEngine: gate mismatches over 4M centres on the
threshold 7 (FFTW) -> 17 (pocketfft); bound raised to 30. pocketfft's float error
is FFTW's or lower (rms 1.4e-7 vs 1.5e-7 against double).
Known: "A near-extinct class is reported as a sub-lattice" [tncs] fails - its
synthetic reads modulation 20.0x against a 20.0x gate; FFT rounding tips it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-15 21:23:40 +01:00
1 parent aa898ac0f2
commit 04baa16060
28 files changed
+308 -302

No files matched your search

+3 -2
View File
@@ -474,8 +474,9 @@ FetchContent_Declare(tiff
EXCLUDE_FROM_ALL)
FetchContent_MakeAvailable(tiff)
# FFTW single precision (target fftw3f): the CPU fallback indexer, enables JFJOCH_USE_FFTW.
# Built from the release tarball -- the git repo ships no pre-generated codelets (needs the
# FFTW single precision (target fftw3f): no longer used by the programs, whose CPU FFTs are pocketfft's
# (common/CpuFFT) - linked only by jfjoch_test, which checks pocketfft against it. JFJOCH_USE_FFTW still
# means "the CPU FFT indexer is built". Built from the release tarball -- the git repo ships no pre-generated codelets (needs the
# OCaml genfft).
SET(ENABLE_FLOAT ON CACHE BOOL "" FORCE)
SET(BUILD_TESTS OFF CACHE BOOL "" FORCE)
+1
View File
@@ -76,6 +76,7 @@ ADD_LIBRARY(JFJochZMQ STATIC ZMQWrappers.cpp ZMQWrappers.h)
ADD_LIBRARY(JFJochCommon STATIC
Coord.cpp Coord.h
CpuFFT.cpp CpuFFT.h
DiffractionExperiment.cpp DiffractionExperiment.h
RawToConvertedGeometry.h
JFJochException.h
+70
View File
@@ -0,0 +1,70 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "CpuFFT.h"
// Under a name of its own: gemmi includes the same header with POCKETFFT_NO_MULTITHREADING, and its
// templates and ours must not be merged by the linker into one (single-threaded or not) copy.
#define pocketfft jfjoch_pocketfft
#include "../gemmi_gph/gemmi/third_party/pocketfft_hdronly.h"
namespace cpu_fft {
namespace {
// pocketfft counts strides in bytes.
pocketfft::stride_t Bytes(const Strides &s, size_t element) {
pocketfft::stride_t out(s.size());
for (size_t i = 0; i < s.size(); i++)
out[i] = s[i] * static_cast<std::ptrdiff_t>(element);
return out;
}
Strides Contiguous(const Shape &shape) {
Strides s(shape.size());
std::ptrdiff_t step = 1;
for (size_t i = shape.size(); i-- > 0;) {
s[i] = step;
step *= static_cast<std::ptrdiff_t>(shape[i]);
}
return s;
}
Shape Halved(Shape shape, size_t axis) {
shape[axis] = shape[axis] / 2 + 1;
return shape;
}
} // namespace
void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes,
const float *in, std::complex<float> *out, size_t nthreads) {
pocketfft::r2c<float>(shape, Bytes(stride_in, sizeof(float)), Bytes(stride_out, sizeof(std::complex<float>)),
axes, pocketfft::FORWARD, in, out, 1.0f, nthreads);
}
void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex<float> *out, size_t nthreads) {
R2C(shape, Contiguous(shape), Contiguous(Halved(shape, axes.back())), axes, in, out, nthreads);
}
void C2R(const Shape &shape, const Shape &axes, std::complex<float> *in, float *out, size_t nthreads) {
const Shape half = Halved(shape, axes.back());
const pocketfft::stride_t stride_in = Bytes(Contiguous(half), sizeof(std::complex<float>));
if (axes.size() > 1)
pocketfft::c2c<float>(half, stride_in, stride_in, Shape(axes.begin(), axes.end() - 1), pocketfft::BACKWARD,
in, in, 1.0f, nthreads);
pocketfft::c2r<float>(shape, stride_in, Bytes(Contiguous(shape), sizeof(float)), axes.back(),
pocketfft::BACKWARD, in, out, 1.0f, nthreads);
}
void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex<float> *data,
size_t nthreads) {
const pocketfft::stride_t s = Bytes(stride, sizeof(std::complex<float>));
pocketfft::c2c<float>(shape, s, s, axes, forward, data, data, 1.0f, nthreads);
}
void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex<float> *data, size_t nthreads) {
C2C(shape, Contiguous(shape), axes, forward, data, nthreads);
}
} // namespace cpu_fft
+36
View File
@@ -0,0 +1,36 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <complex>
#include <cstddef>
#include <vector>
// Single-precision CPU FFTs on pocketfft (the header vendored with gemmi). There are no plans to make:
// every call is a whole transform, safe from several threads at once.
//
// Arrays are row-major (the last axis fastest) and the transforms unnormalised, the forward one with
// exp(-2 pi i k.x), as FFTW's. A real transform halves the last axis in `axes` (n/2 + 1 complex values
// along it). Strides count elements of the array they belong to - floats on the real side, complex values
// on the complex side; without them the array is contiguous. `nthreads` splits one transform's lines.
namespace cpu_fft {
using Shape = std::vector<size_t>;
using Strides = std::vector<std::ptrdiff_t>;
// Real -> complex, forward. `shape` is the real array's.
void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes,
const float *in, std::complex<float> *out, size_t nthreads = 1);
void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex<float> *out, size_t nthreads = 1);
// Complex -> real, backward (exp(+2 pi i k.x)). `shape` is the real array's. `in` is overwritten, as by
// FFTW's c2r: the complex axes are transformed in place before the real one.
void C2R(const Shape &shape, const Shape &axes, std::complex<float> *in, float *out, size_t nthreads = 1);
// Complex -> complex, in place.
void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex<float> *data,
size_t nthreads = 1);
void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex<float> *data, size_t nthreads = 1);
} // namespace cpu_fft
-14
View File
@@ -1,14 +0,0 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <mutex>
// FFTW's planner - every fftwf_plan_* and fftwf_destroy_plan call - shares global state and is not
// thread-safe; executing a plan is. So every piece of code that makes or destroys a plan takes this
// one lock, not a lock of its own: two private locks do not stop two planners running at once.
inline std::mutex &FFTWPlannerMutex() {
static std::mutex m;
return m;
}
@@ -30,7 +30,7 @@
// The input is the mean-projection the pre-scan already accumulates for the beam-stop search
// (ShadowFinder::GetMeanProjection), NAN where nothing was counted - so this costs no new I/O.
//
// The transforms run on the GPU where there is one with room for them and on fftw3f otherwise
// The transforms run on the GPU where there is one with room for them and on the CPU (pocketfft) otherwise
// (BeamCenterFFTGPU / BeamCenterFFTCPU); everything that decides anything is shared between the
// two. Whole-detector reach is what this buys, and on a 16 Mpixel detector the transforms are the
// whole of its cost, so which engine runs them is the difference between seconds and milliseconds.
@@ -8,12 +8,8 @@
#include <cstring>
#include <future>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <fftw3.h>
#include "../../common/FFTWPlannerLock.h"
#include "../../common/CpuFFT.h"
namespace {
@@ -21,8 +17,6 @@ namespace {
class Rfft2 {
int64_t ny, nx, nxc;
std::vector<float> real;
fftwf_plan fwd = nullptr;
fftwf_plan bwd = nullptr;
public:
std::vector<std::complex<float>> spectrum;
@@ -30,24 +24,6 @@ public:
Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) {
real.resize(static_cast<size_t>(ny * nx));
spectrum.resize(static_cast<size_t>(ny * nxc));
std::unique_lock lock(FFTWPlannerMutex());
fwd = fftwf_plan_dft_r2c_2d(static_cast<int>(ny), static_cast<int>(nx), real.data(),
reinterpret_cast<fftwf_complex *>(spectrum.data()),
FFTW_ESTIMATE);
bwd = fftwf_plan_dft_c2r_2d(static_cast<int>(ny), static_cast<int>(nx),
reinterpret_cast<fftwf_complex *>(spectrum.data()),
real.data(), FFTW_ESTIMATE);
if (!fwd || !bwd)
throw std::runtime_error("BeamCenterFFT: fftwf plan failed");
}
Rfft2(const Rfft2 &) = delete;
Rfft2 &operator=(const Rfft2 &) = delete;
~Rfft2() {
std::unique_lock lock(FFTWPlannerMutex());
if (fwd)
fftwf_destroy_plan(fwd);
if (bwd)
fftwf_destroy_plan(bwd);
}
// Zero-pad `src` (height x width) into the workspace and return its spectrum.
@@ -57,7 +33,7 @@ public:
for (int64_t y = 0; y < height; y++)
std::memcpy(&real[static_cast<size_t>(y * nx)], &src[static_cast<size_t>(y * width)],
sizeof(float) * static_cast<size_t>(width));
fftwf_execute(fwd);
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, real.data(), spectrum.data());
return spectrum; // copy: the workspace is reused
}
@@ -68,7 +44,7 @@ public:
const float norm = 1.0f / (static_cast<float>(ny) * static_cast<float>(nx));
for (size_t i = 0; i < spectrum.size(); i++)
spectrum[i] = u[i] * v[i];
fftwf_execute(bwd);
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, spectrum.data(), real.data());
std::vector<float> out(static_cast<size_t>(out_h * out_w));
for (int64_t y = 0; y < out_h; y++)
for (int64_t x = 0; x < out_w; x++)
@@ -88,8 +64,6 @@ BeamCenterConvSurfaces2D BeamCenterFFTCPU::PointSurfaces(const std::vector<float
// transforms used to hold at once were gigabytes. In order: the forward transforms of the image
// and the mask; the two inverse ones that read the image's spectrum, which is then handed back;
// the squared image's forward transform beside the mask's own inverse; and the last inverse.
// A workspace is built exactly as a single one was - its own buffers and its own plan for them -
// so every transform is the one it was, bit for bit.
const int64_t ny = BeamCenterFFTPadSize(2 * h), nx = BeamCenterFFTPadSize(2 * w);
Rfft2 ws0(ny, nx), ws1(ny, nx);
@@ -5,7 +5,7 @@
#include "BeamCenterFFTEngine.h"
// The beam-centre transforms on fftw3f. Always built (FFTW is fetched unconditionally), and the
// The beam-centre transforms on the CPU (pocketfft, common/CpuFFT). Always built, and the
// fallback wherever there is no GPU to run them on.
class BeamCenterFFTCPU : public BeamCenterFFTEngine {
public:
@@ -8,7 +8,7 @@
#include "BeamCenterFFT.h"
// The transform half of BeamCenterFFTScore, with a CPU (fftw3f) and a GPU (cuFFT) implementation
// The transform half of BeamCenterFFTScore, with a CPU (pocketfft) and a GPU (cuFFT) implementation
// chosen the way the FFT indexer's are. Everything that decides anything - the preparation of the
// image, the masked Pearson combination, the shortlist and the margins - is shared, so the two
// engines can differ only in how the four convolutions are computed. That is what makes them
@@ -40,9 +40,7 @@ ADD_LIBRARY(JFJochGeomRefinement STATIC
CellEsd.h
)
# fftw3f: BeamCenterFFT scores every candidate centre with one transform set (always fetched,
# same target the CPU FFT indexer links), and is the fallback wherever there is no GPU.
TARGET_LINK_LIBRARIES(JFJochGeomRefinement Ceres::ceres Eigen3::Eigen JFJochCommon fftw3f)
TARGET_LINK_LIBRARIES(JFJochGeomRefinement Ceres::ceres Eigen3::Eigen JFJochCommon)
IF (JFJOCH_CUDA_AVAILABLE)
TARGET_SOURCES(JFJochGeomRefinement PRIVATE BeamCenterFFTGPU.cu BeamCenterFFTGPU.h
+1 -2
View File
@@ -72,6 +72,5 @@ IF (NOT MSVC)
SET_SOURCE_FILES_PROPERTIES(FFBIDXIndexerCPU.cpp PROPERTIES COMPILE_OPTIONS "-fno-math-errno;-fno-trapping-math")
ENDIF()
# FFTW (fftw3f) is always available via FetchContent -> the CPU FFT indexer is always built.
# The CPU FFT indexer is always built (its transforms are pocketfft's, common/CpuFFT).
TARGET_SOURCES(JFJochIndexing PRIVATE FFTIndexerCPU.cpp FFTIndexerCPU.h)
TARGET_LINK_LIBRARIES(JFJochIndexing fftw3f)
+6 -30
View File
@@ -7,10 +7,9 @@
#include <algorithm>
#include <stdexcept>
#include <cassert>
#include <mutex>
#include <vector>
#include "../../common/FFTWPlannerLock.h"
#include "../../common/CpuFFT.h"
#include "../../common/ParallelFor.h"
static inline double dot_abs(const Coord& a, const Coord& b) {
@@ -26,32 +25,9 @@ FFTIndexerCPU::FFTIndexerCPU(const IndexingSettings& settings)
// Validate allocations vs. expected sizes
if (h_input_fft.size() != input_size)
throw std::runtime_error("FFTWIndexer: input buffer size mismatch");
throw std::runtime_error("FFTIndexerCPU: input buffer size mismatch");
if (h_output_fft.size() != output_size)
throw std::runtime_error("FFTWIndexer: output buffer size mismatch");
const int H = static_cast<int>(histogram_size);
const int out_len = (H / 2) + 1;
int n[1] = { H };
{
std::unique_lock ul(FFTWPlannerMutex());
plan = fftwf_plan_many_dft_r2c(
1, n, nDirections,
h_input_fft.data(), nullptr, 1, H,
reinterpret_cast<fftwf_complex*>(h_output_fft.data()), nullptr, 1, out_len,
FFTW_ESTIMATE);
}
if (!plan)
throw std::runtime_error("fftw_plan_many_dft_r2c failed");
}
FFTIndexerCPU::~FFTIndexerCPU() {
std::unique_lock ul(FFTWPlannerMutex());
fftwf_destroy_plan(plan);
throw std::runtime_error("FFTIndexerCPU: output buffer size mismatch");
}
void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
@@ -80,8 +56,8 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
}
});
// Plan and execute batched R2C FFT with FFTW
fftwf_execute(plan);
// One real transform per direction, all in one call.
cpu_fft::R2C({static_cast<size_t>(D), static_cast<size_t>(H)}, {1}, h_input_fft.data(), h_output_fft.data());
// Post-process: pick the peak past min_length_A by PROMINENCE above a local background.
// The projected histogram has a broad low-frequency ENVELOPE (spots cluster near the
@@ -106,7 +82,7 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
pref[0] = 0.0;
for (int j = 0; j < out_len; ++j) {
mag[j] = std::hypot(spec[j][0], spec[j][1]);
mag[j] = std::hypot(spec[j].real(), spec[j].imag());
pref[j + 1] = pref[j] + mag[j];
}
+4 -9
View File
@@ -4,30 +4,25 @@
#pragma once
#include <complex>
#include <vector>
#include <fftw3.h>
#include "FFTIndexer.h"
#include "../../common/Coord.h"
// CPU/FFTW-based FFT indexer (drop-in replacement for the CUDA version).
// CPU FFT indexer (drop-in replacement for the CUDA version).
class FFTIndexerCPU : public FFTIndexer {
// Host-side buffers for batched FFTW
// Host-side buffers for the batched transform
// Input: [nDirections][histogram_size]
// Output (R2C): [nDirections][(histogram_size/2)+1]
std::vector<float> h_input_fft;
std::vector<std::array<float,2>> h_output_fft; // fftw_complex-compatible layout
fftwf_plan plan;
std::vector<std::complex<float>> h_output_fft;
void ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) override;
public:
explicit FFTIndexerCPU(const IndexingSettings& settings);
FFTIndexerCPU(const FFTIndexerCPU&) = delete;
FFTIndexerCPU& operator=(const FFTIndexerCPU&) = delete;
~FFTIndexerCPU() override;
};
+1 -1
View File
@@ -52,7 +52,7 @@ ADD_LIBRARY(JFJochScaleMerge
SublatticeRelabel.h
IceRingVerdict.cpp
IceRingVerdict.h)
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration fftw3f)
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration)
IF (JFJOCH_CUDA_AVAILABLE)
TARGET_SOURCES(JFJochScaleMerge PRIVATE ../indexing/CUDAMemHelpers.h
+12 -26
View File
@@ -5,6 +5,7 @@
#include <algorithm>
#include <cmath>
#include <complex>
#include <cstring>
#include <iomanip>
#include <limits>
@@ -15,9 +16,7 @@
#include <tuple>
#include <unordered_set>
#include <fftw3.h>
#include "../../common/FFTWPlannerLock.h"
#include "../../common/CpuFFT.h"
#include "../../common/ParallelFor.h"
#include "gemmi/twin.hpp"
@@ -350,20 +349,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
}
}
float *coeff = fftwf_alloc_real(2 * n_complex);
float *map = fftwf_alloc_real(n_points);
if (coeff == nullptr || map == nullptr) {
if (coeff) fftwf_free(coeff);
if (map) fftwf_free(map);
std::vector<std::complex<float>> coeff;
std::vector<float> map;
try {
coeff.resize(n_complex);
map.resize(n_points);
} catch (const std::bad_alloc &) {
result.refusal = "could not allocate the Patterson grid";
return result;
}
fftwf_plan plan;
{
std::lock_guard lock(FFTWPlannerMutex());
plan = fftwf_plan_dft_c2r_3d(nx, ny, nz, reinterpret_cast<fftwf_complex *>(coeff), map,
FFTW_ESTIMATE);
}
// The near-origin exclusion, as a set of grid points rather than a per-point distance test: only
// the points inside a 15 A ball around each origin-equivalent point can be excluded, and there
@@ -405,17 +399,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
os << "every point of the cell lies within " << kMinPeakDistanceA
<< " A of a lattice point - a small-molecule cell, with no room for a translation between molecules";
result.refusal = os.str();
fftwf_destroy_plan(plan);
fftwf_free(coeff);
fftwf_free(map);
return result;
}
auto transform = [&](const std::vector<double> &values) {
std::memset(coeff, 0, 2 * n_complex * sizeof(float));
std::fill(coeff.begin(), coeff.end(), std::complex<float>(0.0f, 0.0f));
for (const auto &[at, src] : fill)
coeff[2 * at] += static_cast<float>(values[src]);
fftwf_execute(plan);
coeff[at] += static_cast<float>(values[src]);
cpu_fft::C2R({static_cast<size_t>(nx), static_cast<size_t>(ny), static_cast<size_t>(nz)}, {0, 1, 2},
coeff.data(), map.data());
};
auto max_far = [&]() {
double best = -std::numeric_limits<double>::infinity();
@@ -510,12 +502,6 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
}
}
{
std::lock_guard lock(FFTWPlannerMutex());
fftwf_destroy_plan(plan);
}
fftwf_free(coeff);
fftwf_free(map);
result.measurable = true;
// ------------------------------------------------------------------ (b) the modulation depth
@@ -76,7 +76,7 @@ TARGET_INCLUDE_DIRECTORIES(JFJochStructureRefinement PRIVATE ${CMAKE_CURRENT_SOU
# JFJochIndexing for CUDAMemHelpers.h and, under CUDA, to put cuFFT (RigidBodyGPU.cu) after these
# objects on the link line.
TARGET_LINK_LIBRARIES(JFJochStructureRefinement JFJochScaleMerge JFJochIndexing JFJochCommon gemmi fftw3f)
TARGET_LINK_LIBRARIES(JFJochStructureRefinement JFJochScaleMerge JFJochIndexing JFJochCommon gemmi)
IF (APPLE)
# The rotation function on the Apple GPU (MolecularReplacementMetal.h). Its kernel is compiled when it is first
+28 -107
View File
@@ -7,46 +7,22 @@
#include <array>
#include <cmath>
#include <complex>
#include <map>
#include <mutex>
#include <fftw3.h>
#include <vector>
#include "gemmi/fail.hpp"
#include "../../common/FFTWPlannerLock.h"
#include "../../common/CpuFFT.h"
#include "../../common/ParallelFor.h"
namespace {
// An (nu, nv, nw) map is stored u fastest - row-major (nw, nv, nu) - and its half-l coefficients
// likewise as (nw/2 + 1, nv, nu): the layout gemmi's own transforms read and write. The real transform
// halves w, so w is the last axis transformed.
static const cpu_fft::Shape kUVW{2, 1, 0};
// The r2c plan (or, inverse, the c2r plan) for an (nu, nv, nw) map stored u fastest, halving w - the
// layout gemmi's own transforms read and write. FFTW's planner is not thread-safe, so plans are made
// under the process-wide planner lock (which also guards this cache); executing one on new arrays
// (fftwf_execute_dft_r2c / _c2r) is, which is how every caller uses it.
fftwf_plan PlanFor(int nu, int nv, int nw, bool inverse) {
static std::map<std::array<int, 4>, fftwf_plan> plans;
std::lock_guard lock(FFTWPlannerMutex());
const std::array<int, 4> key{nu, nv, nw, inverse};
if (const auto it = plans.find(key); it != plans.end())
return it->second;
float *real = fftwf_alloc_real(static_cast<size_t>(nu) * nv * nw);
fftwf_complex *cplx = fftwf_alloc_complex(static_cast<size_t>(nu) * nv * (nw / 2 + 1));
// The last dimension is the one r2c halves; the strides put u fastest on both sides.
fftwf_iodim dims[3] = {{nu, 1, 1}, {nv, nu, nu}, {nw, nu * nv, nu * nv}};
fftwf_plan plan = nullptr;
if (real != nullptr && cplx != nullptr)
plan = inverse ? fftwf_plan_guru_dft_c2r(3, dims, 0, nullptr, cplx, real, FFTW_ESTIMATE)
: fftwf_plan_guru_dft_r2c(3, dims, 0, nullptr, real, cplx, FFTW_ESTIMATE);
fftwf_free(real);
fftwf_free(cplx);
if (plan == nullptr)
gemmi::fail("ModelFFT: no FFTW plan for the grid");
plans.emplace(key, plan);
return plan;
static cpu_fft::Shape MapShape(int nu, int nv, int nw) {
return {static_cast<size_t>(nw), static_cast<size_t>(nv), static_cast<size_t>(nu)};
}
} // namespace
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map) {
if (map.axis_order == gemmi::AxisOrder::ZYX)
gemmi::fail("MapToFPhi(): ZYX order is not supported");
@@ -58,30 +34,16 @@ gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map) {
hkl.set_size_without_checking(map.nu, map.nv, map.nw / 2 + 1);
const float norm = static_cast<float>(map.unit_cell.volume / map.point_count());
const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, false);
// Buffers from fftwf_malloc: a plan may only be executed on arrays aligned as the ones it was made
// on were, which the vectors' own allocations do not promise.
float *in = fftwf_alloc_real(map.data.size());
fftwf_complex *out = fftwf_alloc_complex(hkl.data.size());
if (in == nullptr || out == nullptr) {
fftwf_free(in);
fftwf_free(out);
gemmi::fail("MapToFPhi(): cannot allocate the FFT buffers");
}
std::copy(map.data.begin(), map.data.end(), in);
fftwf_execute_dft_r2c(plan, in, out);
// Conjugated, as gemmi does: FFTW's forward transform carries exp(-2 pi i h.x), and a structure
cpu_fft::R2C(MapShape(map.nu, map.nv, map.nw), kUVW, map.data.data(), hkl.data.data());
// Conjugated, as gemmi does: the forward transform carries exp(-2 pi i h.x), and a structure
// factor is the sum over exp(+2 pi i h.x).
for (size_t i = 0; i < hkl.data.size(); ++i)
hkl.data[i] = std::complex<float>(out[i][0] * norm, -out[i][1] * norm);
fftwf_free(in);
fftwf_free(out);
for (auto &x : hkl.data)
x = std::conj(x) * norm;
return hkl;
}
void MapFromFPhiExecute(int nu, int nv, int nw, std::complex<float> *in, float *out) {
static_assert(sizeof(std::complex<float>) == sizeof(fftwf_complex));
fftwf_execute_dft_c2r(PlanFor(nu, nv, nw, true), reinterpret_cast<fftwf_complex *>(in), out);
cpu_fft::C2R(MapShape(nu, nv, nw), kUVW, in, out);
}
gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int reach, size_t nthreads) {
@@ -89,34 +51,12 @@ gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int re
gemmi::fail("MapToFPhiInParallel(): ZYX order is not supported");
const int nu = map.nu, nv = map.nv, nw = map.nw, nwh = nw / 2 + 1, nk = 2 * reach + 1;
const size_t plane = static_cast<size_t>(nu) * nv;
float *in = fftwf_alloc_real(map.data.size());
fftwf_complex *half = fftwf_alloc_complex(plane * nwh);
if (in == nullptr || half == nullptr) {
fftwf_free(in);
fftwf_free(half);
gemmi::fail("MapToFPhiInParallel(): cannot allocate the FFT buffers");
}
// Made once here; executing a plan on new arrays is safe from several threads at once. Unaligned, as a
// column or a plane starts wherever it does.
fftwf_plan columns = nullptr, planes = nullptr;
{
std::lock_guard lock(FFTWPlannerMutex());
// The nu columns of one v, w along the stride of a plane.
fftwf_iodim col_dim{nw, static_cast<int>(plane), static_cast<int>(plane)};
fftwf_iodim col_many{nu, 1, 1};
columns = fftwf_plan_guru_dft_r2c(1, &col_dim, 1, &col_many, in, half, FFTW_ESTIMATE | FFTW_UNALIGNED);
fftwf_iodim plane_dims[2] = {{nv, nu, nu}, {nu, 1, 1}};
planes = fftwf_plan_guru_dft(2, plane_dims, 0, nullptr, half, half, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_UNALIGNED);
}
if (columns == nullptr || planes == nullptr)
gemmi::fail("MapToFPhiInParallel(): no FFTW plan for the grid");
std::copy(map.data.begin(), map.data.end(), in);
ParallelFor(nv, nthreads, [&](int v) {
fftwf_execute_dft_r2c(columns, in + static_cast<size_t>(v) * nu, half + static_cast<size_t>(v) * nu);
});
ParallelFor(reach + 1, nthreads, [&](int l) {
fftwf_execute_dft(planes, half + l * plane, half + l * plane);
});
std::vector<std::complex<float>> half(plane * nwh);
// The r2c along w of every (u, v) column, then the 2D transform of the planes l <= reach only - the
// rest are never read. Each is one call split over the threads by lines.
cpu_fft::R2C({static_cast<size_t>(nw), plane}, {0}, map.data.data(), half.data(), nthreads);
cpu_fft::C2C({static_cast<size_t>(reach) + 1, static_cast<size_t>(nv), static_cast<size_t>(nu)}, {1, 2}, true,
half.data(), nthreads);
gemmi::FPhiGrid<float> hkl;
hkl.unit_cell = map.unit_cell;
@@ -128,19 +68,12 @@ gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int re
// Conjugated and scaled as MapToFPhi does it, and cropped as CropHalfL crops it.
ParallelFor(reach + 1, nthreads, [&](int l) {
for (int k = -reach; k <= reach; k++) {
const fftwf_complex *row = half + l * plane + static_cast<size_t>(nu) * gemmi::modulo(k, nv);
const std::complex<float> *row = &half[l * plane + static_cast<size_t>(nu) * gemmi::modulo(k, nv)];
std::complex<float> *out = &hkl.data[static_cast<size_t>(nu) * (gemmi::modulo(k, nk) + static_cast<size_t>(nk) * l)];
for (int u = 0; u < nu; u++)
out[u] = std::complex<float>(row[u][0] * norm, -row[u][1] * norm);
out[u] = std::conj(row[u]) * norm;
}
});
{
std::lock_guard lock(FFTWPlannerMutex());
fftwf_destroy_plan(columns);
fftwf_destroy_plan(planes);
}
fftwf_free(in);
fftwf_free(half);
return hkl;
}
@@ -154,26 +87,14 @@ gemmi::Grid<float> MapFromFPhi(const gemmi::FPhiGrid<float> &hkl) {
map.axis_order = hkl.axis_order;
const float norm = static_cast<float>(1.0 / hkl.unit_cell.volume);
const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, true);
fftwf_complex *in = fftwf_alloc_complex(hkl.data.size());
float *out = fftwf_alloc_real(map.data.size());
if (in == nullptr || out == nullptr) {
fftwf_free(in);
fftwf_free(out);
gemmi::fail("MapFromFPhi(): cannot allocate the FFT buffers");
}
// Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and FFTW's backward transform
// Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and the backward transform
// carries exp(+2 pi i h.x). A missing coefficient (NaN) counts as zero, also as gemmi does.
for (size_t i = 0; i < hkl.data.size(); ++i) {
const std::complex<float> x = hkl.data[i];
in[i][0] = std::isnan(x.imag()) ? 0.0f : x.real();
in[i][1] = std::isnan(x.imag()) ? 0.0f : -x.imag();
}
fftwf_execute_dft_c2r(plan, in, out);
for (size_t i = 0; i < map.data.size(); ++i)
map.data[i] = out[i] * norm;
fftwf_free(in);
fftwf_free(out);
std::vector<std::complex<float>> in(hkl.data.size());
for (size_t i = 0; i < in.size(); ++i)
in[i] = std::isnan(hkl.data[i].imag()) ? 0.0f : std::conj(hkl.data[i]);
cpu_fft::C2R(MapShape(map.nu, map.nv, map.nw), kUVW, in.data(), map.data.data());
for (auto &x : map.data)
x *= norm;
return map;
}
+6 -13
View File
@@ -9,30 +9,23 @@
// The structure factors of a real-space map on the model path (Fcalc from the model density, Fmask
// from the bulk-solvent mask): what gemmi::transform_map_to_f_phi(map, true) returns - the
// conjugated forward transform (F = sum rho exp(+2 pi i h.x)) scaled by V/N, kept for l >= 0 on an (nu, nv, nw/2+1) grid in the map's own XYZ order,
// ready for prepare_asu_data() - computed with FFTW rather than gemmi's single-threaded pocketfft.
// The same transform, not the same bits: the two libraries order their arithmetic differently.
//
// Plans are FFTW_ESTIMATE only - MEASURE would time candidate algorithms and could pick a different
// one from run to run - and made once per grid size and kept, so the rigid body's hundred-odd
// evaluations on one grid plan once. Safe to call from several threads at once.
// ready for prepare_asu_data() - computed with common/CpuFFT. Safe to call from several threads at once.
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map);
// The inverse: the real-space map of an (nu, nv, nw/2+1) half-l coefficient grid in XYZ order, as
// get_f_phi_on_grid() fills it - what gemmi::transform_f_phi_grid_to_map() returns (rho = 1/V sum F
// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero), computed with FFTW
// under the same planning rules.
// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero).
gemmi::Grid<float> MapFromFPhi(const gemmi::FPhiGrid<float> &hkl);
// MapFromFPhi's transform itself, for a caller that fills its input: `in`, the (nu, nv, nw/2+1) half-l coefficients
// u fastest as MapFromFPhi hands them to FFTW (conjugated, a NaN as zero), is overwritten, and `out` receives the
// (nu, nv, nw) map before MapFromFPhi's 1/V. Both from fftwf_malloc, as a plan requires.
// u fastest as MapFromFPhi transforms them (conjugated, a NaN as zero), is overwritten, and `out` receives the
// (nu, nv, nw) map before MapFromFPhi's 1/V.
void MapFromFPhiExecute(int nu, int nv, int nw, std::complex<float> *in, float *out);
// CropHalfL(MapToFPhi(map), reach) on `nthreads` threads, for a map of hundreds of millions of points (the
// molecular transform's box), whose single-threaded transform is minutes of CPU: the r2c along w of every
// (u, v) column, then the 2D transform of every plane l <= reach - each column and each plane by one plan, so
// the result does not depend on the thread count. The same transform as MapToFPhi to float rounding, not bit
// for bit: FFTW's 3D plan orders its arithmetic its own way.
// (u, v) column, then the 2D transform of every plane l <= reach. The threads split the lines of a transform, each
// line transformed as it would be alone, so the result does not depend on the thread count.
gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int reach, size_t nthreads);
// The half-l grid `full` with only the indices |k|, l <= reach kept, on a grid of full.nu x (2 reach + 1) x
@@ -19,7 +19,6 @@
#include <random>
#include <unordered_map>
#include <fftw3.h>
#include <gemmi/modify.hpp> // remove_hydrogens, remove_alternative_conformations
#include <gemmi/polyheur.hpp> // setup_entities, remove_ligands_and_waters
@@ -1602,13 +1601,8 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
w[t] = setup.weight[t] / (bin_sum[setup.bin[t]] / bin_n[setup.bin[t]]);
const int nu = setup.nu, nv = setup.nv, nw = setup.nw;
const size_t cells = static_cast<size_t>(nu) * nv * (nw / 2 + 1), points = static_cast<size_t>(nu) * nv * nw;
auto *in = reinterpret_cast<std::complex<float> *>(fftwf_alloc_complex(cells));
float *map = fftwf_alloc_real(points);
if (in == nullptr || map == nullptr) {
fftwf_free(in);
fftwf_free(map);
throw std::runtime_error("TranslationFunction: cannot allocate the FFT buffers");
}
std::vector<std::complex<float>> in(cells);
std::vector<float> map(points);
for (size_t c = 0; c < cells; c++) {
std::complex<double> v = 0.0;
for (int k = gather.offset[c]; k < gather.offset[c + 1]; k++) {
@@ -1626,7 +1620,7 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
const std::complex<float> f(std::conj(v));
in[c] = std::isnan(f.imag()) ? std::complex<float>(0.0f, 0.0f) : std::complex<float>(f.real(), -f.imag());
}
MapFromFPhiExecute(nu, nv, nw, in, map);
MapFromFPhiExecute(nu, nv, nw, in.data(), map.data());
const float norm = static_cast<float>(1.0 / ft.Grid().unit_cell.volume);
double sum = 0, sum2 = 0;
size_t best = 0;
@@ -1640,8 +1634,6 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
const double count = static_cast<double>(points);
const double mean = sum / count, sd = std::sqrt(std::max(0.0, sum2 / count - mean * mean));
out.z = sd > 0 ? (map[best] * norm - mean) / sd : 0.0;
fftwf_free(in);
fftwf_free(map);
const size_t u = best % nu, v = (best / nu) % nv, l = best / (static_cast<size_t>(nu) * nv);
out.t = gemmi::Fractional(static_cast<double>(u) / nu, static_cast<double>(v) / nv, static_cast<double>(l) / nw);
return out;
@@ -10,7 +10,6 @@
#include <numeric>
#include <set>
#include <fftw3.h>
#include "gemmi/dencalc.hpp"
#include "gemmi/it92.hpp"
@@ -188,9 +187,8 @@ public:
// l = nw/2, which hold both, as both.
const int nu = grid_.nu, nv = grid_.nv, nw = grid_.nw, nwh = nw / 2 + 1;
const size_t ncomplex = static_cast<size_t>(nu) * nv * nwh;
std::complex<float> *in = reinterpret_cast<std::complex<float> *>(fftwf_alloc_complex(ncomplex));
float *out = fftwf_alloc_real(static_cast<size_t>(nu) * nv * nw);
std::fill(in, in + ncomplex, std::complex<float>(0, 0));
std::vector<std::complex<float>> in(ncomplex);
std::vector<float> out(static_cast<size_t>(nu) * nv * nw);
const std::vector<SymmetryComposition::Term> &terms = composition_.Terms();
const size_t ops = composition_.Ops();
auto add = [&](int ku, int kv, int kw, std::complex<double> value) {
@@ -217,13 +215,11 @@ public:
}
}
}
MapFromFPhiExecute(nu, nv, nw, in, out);
MapFromFPhiExecute(nu, nv, nw, in.data(), out.data());
gemmi::Grid<float> map = grid_;
const float norm = static_cast<float>(cell_.volume / static_cast<double>(map.point_count()));
for (size_t i = 0; i < map.data.size(); i++)
map.data[i] = out[i] * norm;
fftwf_free(in);
fftwf_free(out);
const bool aniso = AnyAniso();
AtomGradientsFromMap(map, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr);
if (!aniso)
+1 -1
View File
@@ -41,7 +41,7 @@ ADD_LIBRARY(JFJochRugnux STATIC
BrokerFeed.h
)
TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi fftw3f
TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi
httplib::httplib)
# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux
-15
View File
@@ -846,21 +846,6 @@ namespace {
}
}
// FFTW ends the program itself when its allocator comes back empty (CK(p) in kernel/alloc.c) - while
// planning, or inside a buffered transform - from whichever thread that happens in, so it cannot become
// an exception. Defining its handler here keeps FFTW's own (kernel/assert.c) out of the link, and says
// what happened in the same words as every other way of running out of memory.
extern "C" void fftwf_assertion_failed(const char *s, int line, const char *file) {
std::fflush(stdout);
if (std::strcmp(s, "p") == 0) {
std::fprintf(stderr, "Processing failed: out of host memory (FFTW could not allocate) - "
"this data set needs more RAM than is available\n");
std::_Exit(EXIT_FAILURE);
}
std::fprintf(stderr, "fftw: %s:%d: assertion failed: %s\n", file, line, s);
std::abort();
}
namespace {
// One input's run ended: what exit(EXIT_FAILURE) was where a command line ran one input only. A run of
// several sweeps goes on with the next sweep; RunRugnux returns the code.
+3 -2
View File
@@ -174,10 +174,11 @@ TEST_CASE("BeamCenterFFTGPU_MatchesTheCPUEngine", "[BeamCenter][BeamCenterFFTGPU
std::abs(static_cast<double>(rg[i]) - rc[i]));
CHECK(peak_worst < 1e-5);
// And the gate agrees: over four million centres, the two paths disagree about whether a
// centre is scored at all only for a handful sitting exactly on the threshold.
// centre is scored at all only for a handful sitting exactly on the threshold (measured: 7
// with FFTW on the CPU, 17 with pocketfft, whose float error is FFTW's or lower).
const auto diff = Compare(rg, rc, &c.D, 0.26f * d_max);
CHECK(diff.gate_mismatches == 0);
CHECK(Compare(rg, rc).gate_mismatches < 10);
CHECK(Compare(rg, rc).gate_mismatches < 30);
// The device kernel against that reference, on the device's own surfaces - so this measures
// the combination and not the transform.
+2 -1
View File
@@ -237,6 +237,7 @@ ADD_EXECUTABLE(jfjoch_test
LatticeReductionTest.cpp
BeamCenterFromBackgroundTest.cpp
BeamCenterFFTTest.cpp
CpuFFTTest.cpp
BeamCenterFFTGPUTest.cpp
BearerTokensTest.cpp
BrokerHttpAuthTest.cpp
@@ -246,7 +247,7 @@ ADD_EXECUTABLE(jfjoch_test
target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochStreamWriter
JFJochRugnuxOutput JFJochRugnux JFJochModelWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview
jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib)
jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib fftw3f)
target_include_directories(jfjoch_test PRIVATE .)
IF (JFJOCH_CUDA_AVAILABLE)
TARGET_COMPILE_DEFINITIONS(jfjoch_test PRIVATE FFBIDX_SIMPLE_DATA_DIR="${FFBIDX_SIMPLE_DATA_DIR}")
+117
View File
@@ -0,0 +1,117 @@
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <algorithm>
#include <complex>
#include <random>
#include <vector>
#include <fftw3.h>
#include "../common/CpuFFT.h"
#include "../image_analysis/structure_refinement/ModelFFT.h"
// pocketfft (common/CpuFFT) against FFTW on the shapes and layouts the call sites use: the same
// transform to float rounding. FFTW is linked into the test binary only, as the reference.
namespace {
std::vector<float> Random(size_t n, unsigned seed) {
std::mt19937 rng(seed);
std::normal_distribution<float> g(0.0f, 1.0f);
std::vector<float> v(n);
for (auto &x : v)
x = g(rng);
return v;
}
// Largest difference relative to the largest value of the reference.
template <typename T>
double RelativeError(const std::vector<T> &a, const std::vector<T> &ref) {
REQUIRE(a.size() == ref.size());
double diff = 0, scale = 0;
for (size_t i = 0; i < a.size(); i++) {
diff = std::max(diff, static_cast<double>(std::abs(a[i] - ref[i])));
scale = std::max(scale, static_cast<double>(std::abs(ref[i])));
}
return diff / scale;
}
} // namespace
TEST_CASE("CpuFFT_BatchedR2CMatchesFFTW", "[fft]") {
// The FFT indexer's histograms: one transform per direction; 3142 = 2 x 1571 is a real length.
for (int n : {3142, 3150, 1024}) {
const int howmany = 37, nc = n / 2 + 1;
std::vector<float> in = Random(static_cast<size_t>(howmany) * n, n);
std::vector<std::complex<float>> ours(static_cast<size_t>(howmany) * nc), ref(ours.size());
cpu_fft::R2C({static_cast<size_t>(howmany), static_cast<size_t>(n)}, {1}, in.data(), ours.data());
std::vector<float> work = in;
fftwf_plan p = fftwf_plan_many_dft_r2c(1, &n, howmany, work.data(), nullptr, 1, n,
reinterpret_cast<fftwf_complex *>(ref.data()), nullptr, 1, nc,
FFTW_ESTIMATE);
fftwf_execute(p);
fftwf_destroy_plan(p);
CHECK(RelativeError(ours, ref) < 1e-5);
}
}
TEST_CASE("CpuFFT_2DRoundTripMatchesFFTW", "[fft]") {
// The beam-centre search: a 2D r2c and c2r on a 2-3-5-7-smooth padded size.
const int ny = 210, nx = 336, nxc = nx / 2 + 1;
std::vector<float> in = Random(static_cast<size_t>(ny) * nx, 1);
std::vector<std::complex<float>> ours(static_cast<size_t>(ny) * nxc), ref(ours.size());
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, in.data(), ours.data());
std::vector<float> work = in;
fftwf_plan p = fftwf_plan_dft_r2c_2d(ny, nx, work.data(), reinterpret_cast<fftwf_complex *>(ref.data()),
FFTW_ESTIMATE);
fftwf_execute(p);
fftwf_destroy_plan(p);
CHECK(RelativeError(ours, ref) < 1e-5);
std::vector<float> back_ours(in.size()), back_ref(in.size());
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, ours.data(), back_ours.data());
p = fftwf_plan_dft_c2r_2d(ny, nx, reinterpret_cast<fftwf_complex *>(ref.data()), back_ref.data(), FFTW_ESTIMATE);
fftwf_execute(p);
fftwf_destroy_plan(p);
CHECK(RelativeError(back_ours, back_ref) < 1e-5);
}
TEST_CASE("CpuFFT_3DMatchesFFTW", "[fft]") {
// The model maps (u fastest, w halved) and the tNCS Patterson (row-major, last axis halved) are both a
// row-major 3D transform halving the last axis; an odd length and a factor 7 included.
const int n0 = 70, n1 = 45, n2 = 54, nc = static_cast<int>(static_cast<size_t>(n0) * n1 * (n2 / 2 + 1));
const std::vector<float> in = Random(static_cast<size_t>(n0) * n1 * n2, 2);
std::vector<std::complex<float>> ours(nc), ref(nc);
cpu_fft::R2C({70, 45, 54}, {0, 1, 2}, in.data(), ours.data());
std::vector<float> work = in;
fftwf_plan p = fftwf_plan_dft_r2c_3d(n0, n1, n2, work.data(), reinterpret_cast<fftwf_complex *>(ref.data()),
FFTW_ESTIMATE);
fftwf_execute(p);
fftwf_destroy_plan(p);
CHECK(RelativeError(ours, ref) < 1e-5);
std::vector<float> back_ours(in.size()), back_ref(in.size());
cpu_fft::C2R({70, 45, 54}, {0, 1, 2}, ours.data(), back_ours.data());
p = fftwf_plan_dft_c2r_3d(n0, n1, n2, reinterpret_cast<fftwf_complex *>(ref.data()), back_ref.data(),
FFTW_ESTIMATE);
fftwf_execute(p);
fftwf_destroy_plan(p);
CHECK(RelativeError(back_ours, back_ref) < 1e-5);
}
TEST_CASE("CpuFFT_MapToFPhiInParallelMatchesMapToFPhi", "[fft]") {
gemmi::Grid<float> map;
map.unit_cell.set(60, 50, 40, 90, 90, 90);
map.spacegroup = gemmi::find_spacegroup_by_name("P1");
map.set_size(60, 50, 40);
map.data = Random(map.data.size(), 3);
const int reach = 12;
const gemmi::FPhiGrid<float> ref = CropHalfL(MapToFPhi(map), reach);
for (size_t nthreads : {1, 4}) {
const gemmi::FPhiGrid<float> ours = MapToFPhiInParallel(map, reach, nthreads);
CHECK(RelativeError(ours.data, ref.data) < 1e-5);
}
}
+1 -1
View File
@@ -124,7 +124,7 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView
TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::Concurrent
JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis JFJochRugnux JFJochRugnuxOutput
fftw3f httplib::httplib MbedTLS::mbedtls)
httplib::httplib MbedTLS::mbedtls)
# HTTPS for the live-broker client. Defined here, on the viewer only, not on httplib::httplib: the
# broker shares that target and stays plain HTTP. JFJochHttpReader.cpp is the one viewer source
+2 -20
View File
@@ -13,7 +13,7 @@
#include <complex>
#include "JFJochDatasetInfoChartView.h"
#include "../../common/FFTWPlannerLock.h"
#include "../../common/CpuFFT.h"
namespace {
// Qt Charts draws its curves 2 px wide, which is hard to follow across a desk. Take the weight
@@ -1122,25 +1122,7 @@ void JFJochDatasetInfoChartView::buildFFTChart() {
std::vector<std::complex<float>> out(static_cast<size_t>(nComplex));
fftwf_plan plan;
{
std::lock_guard lock(FFTWPlannerMutex());
plan = fftwf_plan_dft_r2c_1d(
n,
in.data(),
reinterpret_cast<fftwf_complex *>(out.data()),
FFTW_ESTIMATE);
}
if (!plan) {
return;
}
fftwf_execute(plan);
{
std::lock_guard lock(FFTWPlannerMutex());
fftwf_destroy_plan(plan);
}
cpu_fft::R2C({N}, {0}, in.data(), out.data());
// Compute amplitude spectrum and frequencies (0 .. Nyquist)
m_fftMagnitudes.resize(static_cast<size_t>(nComplex));
@@ -15,10 +15,6 @@
#include <QPointer>
#include <QColor>
#ifdef JFJOCH_USE_FFTW
#include <fftw3.h>
#endif
#include "../../common/GoniometerAxis.h"
#include "../../reader/JFJochReaderDataset.h"