CPU FFTs on pocketfft instead of FFTW (prototype)
Every CPU FFT site - the FFT indexer's batched 1D r2c, the beam-centre 2D r2c/c2r,
the model-path 3D transforms (MapToFPhi, MapFromFPhi(Execute), MapToFPhiInParallel),
the tNCS Patterson c2r and the viewer's 1D chart - goes through common/CpuFFT, a
plan-free wrapper over the pocketfft header already vendored with gemmi. The header
is included under a namespace of its own: gemmi includes it with
POCKETFFT_NO_MULTITHREADING, and the linker otherwise merges the two sets of
templates (MapToFPhiInParallel with 4 threads gave garbage until it was renamed).
MapToFPhiInParallel uses pocketfft's own threads (one call per stage) instead of a
ParallelFor over FFTW plans. The programs no longer link fftw3f; FFTW stays fetched
for jfjoch_test only, where [fft] checks pocketfft against it (rel. error < 1e-5).
Measured (Ryzen 5950X, --no-gpu -N 16, one run each, against rc187 aa898ac0f):
cytc_x10sa --model --no-refine 100.7 -> 99.5 s; euxfel83 first 6 files --no-refine
81.7 -> 83.3 s (noise level); full cytc refinement 1514 -> ~1399 s. Merged results
identical (same hkl md5, SG, cell, R-work/R-free/CC of model validation).
Microbench, single thread, FFTW_ESTIMATE as the call sites plan: pocketfft 0.4-0.6x
FFTW's time on the 2D/3D shapes; with FFTW_MEASURE the two are equal on 3D.
Indexer length 3142 = 2 x 1571 is 2.4x SLOWER on pocketfft (Bluestein); a smooth
length (3150) is 0.06 s vs FFTW's 0.25 s at 3142.
BeamCenterFFTGPU_MatchesTheCPUEngine: gate mismatches over 4M centres on the
threshold 7 (FFTW) -> 17 (pocketfft); bound raised to 30. pocketfft's float error
is FFTW's or lower (rms 1.4e-7 vs 1.5e-7 against double).
Known: "A near-extinct class is reported as a sub-lattice" [tncs] fails - its
synthetic reads modulation 20.0x against a 20.0x gate; FFT rounding tips it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK
This commit is contained in:
1 parent
aa898ac0f2
commit
04baa16060
28 files changed
+308
-302
No files matched your search
+3
-2
@@ -474,8 +474,9 @@ FetchContent_Declare(tiff
|
||||
EXCLUDE_FROM_ALL)
|
||||
FetchContent_MakeAvailable(tiff)
|
||||
|
||||
# FFTW single precision (target fftw3f): the CPU fallback indexer, enables JFJOCH_USE_FFTW.
|
||||
# Built from the release tarball -- the git repo ships no pre-generated codelets (needs the
|
||||
# FFTW single precision (target fftw3f): no longer used by the programs, whose CPU FFTs are pocketfft's
|
||||
# (common/CpuFFT) - linked only by jfjoch_test, which checks pocketfft against it. JFJOCH_USE_FFTW still
|
||||
# means "the CPU FFT indexer is built". Built from the release tarball -- the git repo ships no pre-generated codelets (needs the
|
||||
# OCaml genfft).
|
||||
SET(ENABLE_FLOAT ON CACHE BOOL "" FORCE)
|
||||
SET(BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
|
||||
@@ -76,6 +76,7 @@ ADD_LIBRARY(JFJochZMQ STATIC ZMQWrappers.cpp ZMQWrappers.h)
|
||||
|
||||
ADD_LIBRARY(JFJochCommon STATIC
|
||||
Coord.cpp Coord.h
|
||||
CpuFFT.cpp CpuFFT.h
|
||||
DiffractionExperiment.cpp DiffractionExperiment.h
|
||||
RawToConvertedGeometry.h
|
||||
JFJochException.h
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include "CpuFFT.h"
|
||||
|
||||
// Under a name of its own: gemmi includes the same header with POCKETFFT_NO_MULTITHREADING, and its
|
||||
// templates and ours must not be merged by the linker into one (single-threaded or not) copy.
|
||||
#define pocketfft jfjoch_pocketfft
|
||||
#include "../gemmi_gph/gemmi/third_party/pocketfft_hdronly.h"
|
||||
|
||||
namespace cpu_fft {
|
||||
|
||||
namespace {
|
||||
|
||||
// pocketfft counts strides in bytes.
|
||||
pocketfft::stride_t Bytes(const Strides &s, size_t element) {
|
||||
pocketfft::stride_t out(s.size());
|
||||
for (size_t i = 0; i < s.size(); i++)
|
||||
out[i] = s[i] * static_cast<std::ptrdiff_t>(element);
|
||||
return out;
|
||||
}
|
||||
|
||||
Strides Contiguous(const Shape &shape) {
|
||||
Strides s(shape.size());
|
||||
std::ptrdiff_t step = 1;
|
||||
for (size_t i = shape.size(); i-- > 0;) {
|
||||
s[i] = step;
|
||||
step *= static_cast<std::ptrdiff_t>(shape[i]);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Shape Halved(Shape shape, size_t axis) {
|
||||
shape[axis] = shape[axis] / 2 + 1;
|
||||
return shape;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes,
|
||||
const float *in, std::complex<float> *out, size_t nthreads) {
|
||||
pocketfft::r2c<float>(shape, Bytes(stride_in, sizeof(float)), Bytes(stride_out, sizeof(std::complex<float>)),
|
||||
axes, pocketfft::FORWARD, in, out, 1.0f, nthreads);
|
||||
}
|
||||
|
||||
void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex<float> *out, size_t nthreads) {
|
||||
R2C(shape, Contiguous(shape), Contiguous(Halved(shape, axes.back())), axes, in, out, nthreads);
|
||||
}
|
||||
|
||||
void C2R(const Shape &shape, const Shape &axes, std::complex<float> *in, float *out, size_t nthreads) {
|
||||
const Shape half = Halved(shape, axes.back());
|
||||
const pocketfft::stride_t stride_in = Bytes(Contiguous(half), sizeof(std::complex<float>));
|
||||
if (axes.size() > 1)
|
||||
pocketfft::c2c<float>(half, stride_in, stride_in, Shape(axes.begin(), axes.end() - 1), pocketfft::BACKWARD,
|
||||
in, in, 1.0f, nthreads);
|
||||
pocketfft::c2r<float>(shape, stride_in, Bytes(Contiguous(shape), sizeof(float)), axes.back(),
|
||||
pocketfft::BACKWARD, in, out, 1.0f, nthreads);
|
||||
}
|
||||
|
||||
void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex<float> *data,
|
||||
size_t nthreads) {
|
||||
const pocketfft::stride_t s = Bytes(stride, sizeof(std::complex<float>));
|
||||
pocketfft::c2c<float>(shape, s, s, axes, forward, data, data, 1.0f, nthreads);
|
||||
}
|
||||
|
||||
void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex<float> *data, size_t nthreads) {
|
||||
C2C(shape, Contiguous(shape), axes, forward, data, nthreads);
|
||||
}
|
||||
|
||||
} // namespace cpu_fft
|
||||
@@ -0,0 +1,36 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <complex>
|
||||
#include <cstddef>
|
||||
#include <vector>
|
||||
|
||||
// Single-precision CPU FFTs on pocketfft (the header vendored with gemmi). There are no plans to make:
|
||||
// every call is a whole transform, safe from several threads at once.
|
||||
//
|
||||
// Arrays are row-major (the last axis fastest) and the transforms unnormalised, the forward one with
|
||||
// exp(-2 pi i k.x), as FFTW's. A real transform halves the last axis in `axes` (n/2 + 1 complex values
|
||||
// along it). Strides count elements of the array they belong to - floats on the real side, complex values
|
||||
// on the complex side; without them the array is contiguous. `nthreads` splits one transform's lines.
|
||||
namespace cpu_fft {
|
||||
|
||||
using Shape = std::vector<size_t>;
|
||||
using Strides = std::vector<std::ptrdiff_t>;
|
||||
|
||||
// Real -> complex, forward. `shape` is the real array's.
|
||||
void R2C(const Shape &shape, const Strides &stride_in, const Strides &stride_out, const Shape &axes,
|
||||
const float *in, std::complex<float> *out, size_t nthreads = 1);
|
||||
void R2C(const Shape &shape, const Shape &axes, const float *in, std::complex<float> *out, size_t nthreads = 1);
|
||||
|
||||
// Complex -> real, backward (exp(+2 pi i k.x)). `shape` is the real array's. `in` is overwritten, as by
|
||||
// FFTW's c2r: the complex axes are transformed in place before the real one.
|
||||
void C2R(const Shape &shape, const Shape &axes, std::complex<float> *in, float *out, size_t nthreads = 1);
|
||||
|
||||
// Complex -> complex, in place.
|
||||
void C2C(const Shape &shape, const Strides &stride, const Shape &axes, bool forward, std::complex<float> *data,
|
||||
size_t nthreads = 1);
|
||||
void C2C(const Shape &shape, const Shape &axes, bool forward, std::complex<float> *data, size_t nthreads = 1);
|
||||
|
||||
} // namespace cpu_fft
|
||||
@@ -1,14 +0,0 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <mutex>
|
||||
|
||||
// FFTW's planner - every fftwf_plan_* and fftwf_destroy_plan call - shares global state and is not
|
||||
// thread-safe; executing a plan is. So every piece of code that makes or destroys a plan takes this
|
||||
// one lock, not a lock of its own: two private locks do not stop two planners running at once.
|
||||
inline std::mutex &FFTWPlannerMutex() {
|
||||
static std::mutex m;
|
||||
return m;
|
||||
}
|
||||
@@ -30,7 +30,7 @@
|
||||
// The input is the mean-projection the pre-scan already accumulates for the beam-stop search
|
||||
// (ShadowFinder::GetMeanProjection), NAN where nothing was counted - so this costs no new I/O.
|
||||
//
|
||||
// The transforms run on the GPU where there is one with room for them and on fftw3f otherwise
|
||||
// The transforms run on the GPU where there is one with room for them and on the CPU (pocketfft) otherwise
|
||||
// (BeamCenterFFTGPU / BeamCenterFFTCPU); everything that decides anything is shared between the
|
||||
// two. Whole-detector reach is what this buys, and on a 16 Mpixel detector the transforms are the
|
||||
// whole of its cost, so which engine runs them is the difference between seconds and milliseconds.
|
||||
|
||||
@@ -8,12 +8,8 @@
|
||||
#include <cstring>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
#include "../../common/CpuFFT.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -21,8 +17,6 @@ namespace {
|
||||
class Rfft2 {
|
||||
int64_t ny, nx, nxc;
|
||||
std::vector<float> real;
|
||||
fftwf_plan fwd = nullptr;
|
||||
fftwf_plan bwd = nullptr;
|
||||
|
||||
public:
|
||||
std::vector<std::complex<float>> spectrum;
|
||||
@@ -30,24 +24,6 @@ public:
|
||||
Rfft2(int64_t ny_, int64_t nx_) : ny(ny_), nx(nx_), nxc(nx_ / 2 + 1) {
|
||||
real.resize(static_cast<size_t>(ny * nx));
|
||||
spectrum.resize(static_cast<size_t>(ny * nxc));
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
fwd = fftwf_plan_dft_r2c_2d(static_cast<int>(ny), static_cast<int>(nx), real.data(),
|
||||
reinterpret_cast<fftwf_complex *>(spectrum.data()),
|
||||
FFTW_ESTIMATE);
|
||||
bwd = fftwf_plan_dft_c2r_2d(static_cast<int>(ny), static_cast<int>(nx),
|
||||
reinterpret_cast<fftwf_complex *>(spectrum.data()),
|
||||
real.data(), FFTW_ESTIMATE);
|
||||
if (!fwd || !bwd)
|
||||
throw std::runtime_error("BeamCenterFFT: fftwf plan failed");
|
||||
}
|
||||
Rfft2(const Rfft2 &) = delete;
|
||||
Rfft2 &operator=(const Rfft2 &) = delete;
|
||||
~Rfft2() {
|
||||
std::unique_lock lock(FFTWPlannerMutex());
|
||||
if (fwd)
|
||||
fftwf_destroy_plan(fwd);
|
||||
if (bwd)
|
||||
fftwf_destroy_plan(bwd);
|
||||
}
|
||||
|
||||
// Zero-pad `src` (height x width) into the workspace and return its spectrum.
|
||||
@@ -57,7 +33,7 @@ public:
|
||||
for (int64_t y = 0; y < height; y++)
|
||||
std::memcpy(&real[static_cast<size_t>(y * nx)], &src[static_cast<size_t>(y * width)],
|
||||
sizeof(float) * static_cast<size_t>(width));
|
||||
fftwf_execute(fwd);
|
||||
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, real.data(), spectrum.data());
|
||||
return spectrum; // copy: the workspace is reused
|
||||
}
|
||||
|
||||
@@ -68,7 +44,7 @@ public:
|
||||
const float norm = 1.0f / (static_cast<float>(ny) * static_cast<float>(nx));
|
||||
for (size_t i = 0; i < spectrum.size(); i++)
|
||||
spectrum[i] = u[i] * v[i];
|
||||
fftwf_execute(bwd);
|
||||
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, spectrum.data(), real.data());
|
||||
std::vector<float> out(static_cast<size_t>(out_h * out_w));
|
||||
for (int64_t y = 0; y < out_h; y++)
|
||||
for (int64_t x = 0; x < out_w; x++)
|
||||
@@ -88,8 +64,6 @@ BeamCenterConvSurfaces2D BeamCenterFFTCPU::PointSurfaces(const std::vector<float
|
||||
// transforms used to hold at once were gigabytes. In order: the forward transforms of the image
|
||||
// and the mask; the two inverse ones that read the image's spectrum, which is then handed back;
|
||||
// the squared image's forward transform beside the mask's own inverse; and the last inverse.
|
||||
// A workspace is built exactly as a single one was - its own buffers and its own plan for them -
|
||||
// so every transform is the one it was, bit for bit.
|
||||
const int64_t ny = BeamCenterFFTPadSize(2 * h), nx = BeamCenterFFTPadSize(2 * w);
|
||||
Rfft2 ws0(ny, nx), ws1(ny, nx);
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
#include "BeamCenterFFTEngine.h"
|
||||
|
||||
// The beam-centre transforms on fftw3f. Always built (FFTW is fetched unconditionally), and the
|
||||
// The beam-centre transforms on the CPU (pocketfft, common/CpuFFT). Always built, and the
|
||||
// fallback wherever there is no GPU to run them on.
|
||||
class BeamCenterFFTCPU : public BeamCenterFFTEngine {
|
||||
public:
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
#include "BeamCenterFFT.h"
|
||||
|
||||
// The transform half of BeamCenterFFTScore, with a CPU (fftw3f) and a GPU (cuFFT) implementation
|
||||
// The transform half of BeamCenterFFTScore, with a CPU (pocketfft) and a GPU (cuFFT) implementation
|
||||
// chosen the way the FFT indexer's are. Everything that decides anything - the preparation of the
|
||||
// image, the masked Pearson combination, the shortlist and the margins - is shared, so the two
|
||||
// engines can differ only in how the four convolutions are computed. That is what makes them
|
||||
|
||||
@@ -40,9 +40,7 @@ ADD_LIBRARY(JFJochGeomRefinement STATIC
|
||||
CellEsd.h
|
||||
)
|
||||
|
||||
# fftw3f: BeamCenterFFT scores every candidate centre with one transform set (always fetched,
|
||||
# same target the CPU FFT indexer links), and is the fallback wherever there is no GPU.
|
||||
TARGET_LINK_LIBRARIES(JFJochGeomRefinement Ceres::ceres Eigen3::Eigen JFJochCommon fftw3f)
|
||||
TARGET_LINK_LIBRARIES(JFJochGeomRefinement Ceres::ceres Eigen3::Eigen JFJochCommon)
|
||||
|
||||
IF (JFJOCH_CUDA_AVAILABLE)
|
||||
TARGET_SOURCES(JFJochGeomRefinement PRIVATE BeamCenterFFTGPU.cu BeamCenterFFTGPU.h
|
||||
|
||||
@@ -72,6 +72,5 @@ IF (NOT MSVC)
|
||||
SET_SOURCE_FILES_PROPERTIES(FFBIDXIndexerCPU.cpp PROPERTIES COMPILE_OPTIONS "-fno-math-errno;-fno-trapping-math")
|
||||
ENDIF()
|
||||
|
||||
# FFTW (fftw3f) is always available via FetchContent -> the CPU FFT indexer is always built.
|
||||
# The CPU FFT indexer is always built (its transforms are pocketfft's, common/CpuFFT).
|
||||
TARGET_SOURCES(JFJochIndexing PRIVATE FFTIndexerCPU.cpp FFTIndexerCPU.h)
|
||||
TARGET_LINK_LIBRARIES(JFJochIndexing fftw3f)
|
||||
@@ -7,10 +7,9 @@
|
||||
#include <algorithm>
|
||||
#include <stdexcept>
|
||||
#include <cassert>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
#include "../../common/CpuFFT.h"
|
||||
#include "../../common/ParallelFor.h"
|
||||
|
||||
static inline double dot_abs(const Coord& a, const Coord& b) {
|
||||
@@ -26,32 +25,9 @@ FFTIndexerCPU::FFTIndexerCPU(const IndexingSettings& settings)
|
||||
|
||||
// Validate allocations vs. expected sizes
|
||||
if (h_input_fft.size() != input_size)
|
||||
throw std::runtime_error("FFTWIndexer: input buffer size mismatch");
|
||||
throw std::runtime_error("FFTIndexerCPU: input buffer size mismatch");
|
||||
if (h_output_fft.size() != output_size)
|
||||
throw std::runtime_error("FFTWIndexer: output buffer size mismatch");
|
||||
|
||||
const int H = static_cast<int>(histogram_size);
|
||||
const int out_len = (H / 2) + 1;
|
||||
|
||||
int n[1] = { H };
|
||||
|
||||
{
|
||||
std::unique_lock ul(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_many_dft_r2c(
|
||||
1, n, nDirections,
|
||||
h_input_fft.data(), nullptr, 1, H,
|
||||
reinterpret_cast<fftwf_complex*>(h_output_fft.data()), nullptr, 1, out_len,
|
||||
FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
if (!plan)
|
||||
throw std::runtime_error("fftw_plan_many_dft_r2c failed");
|
||||
}
|
||||
|
||||
|
||||
FFTIndexerCPU::~FFTIndexerCPU() {
|
||||
std::unique_lock ul(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
throw std::runtime_error("FFTIndexerCPU: output buffer size mismatch");
|
||||
}
|
||||
|
||||
void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
|
||||
@@ -80,8 +56,8 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
|
||||
}
|
||||
});
|
||||
|
||||
// Plan and execute batched R2C FFT with FFTW
|
||||
fftwf_execute(plan);
|
||||
// One real transform per direction, all in one call.
|
||||
cpu_fft::R2C({static_cast<size_t>(D), static_cast<size_t>(H)}, {1}, h_input_fft.data(), h_output_fft.data());
|
||||
|
||||
// Post-process: pick the peak past min_length_A by PROMINENCE above a local background.
|
||||
// The projected histogram has a broad low-frequency ENVELOPE (spots cluster near the
|
||||
@@ -106,7 +82,7 @@ void FFTIndexerCPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
|
||||
|
||||
pref[0] = 0.0;
|
||||
for (int j = 0; j < out_len; ++j) {
|
||||
mag[j] = std::hypot(spec[j][0], spec[j][1]);
|
||||
mag[j] = std::hypot(spec[j].real(), spec[j].imag());
|
||||
pref[j + 1] = pref[j] + mag[j];
|
||||
}
|
||||
|
||||
|
||||
@@ -4,30 +4,25 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <complex>
|
||||
#include <vector>
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "FFTIndexer.h"
|
||||
|
||||
#include "../../common/Coord.h"
|
||||
|
||||
// CPU/FFTW-based FFT indexer (drop-in replacement for the CUDA version).
|
||||
// CPU FFT indexer (drop-in replacement for the CUDA version).
|
||||
class FFTIndexerCPU : public FFTIndexer {
|
||||
// Host-side buffers for batched FFTW
|
||||
// Host-side buffers for the batched transform
|
||||
// Input: [nDirections][histogram_size]
|
||||
// Output (R2C): [nDirections][(histogram_size/2)+1]
|
||||
std::vector<float> h_input_fft;
|
||||
std::vector<std::array<float,2>> h_output_fft; // fftw_complex-compatible layout
|
||||
|
||||
fftwf_plan plan;
|
||||
std::vector<std::complex<float>> h_output_fft;
|
||||
|
||||
void ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) override;
|
||||
|
||||
public:
|
||||
explicit FFTIndexerCPU(const IndexingSettings& settings);
|
||||
FFTIndexerCPU(const FFTIndexerCPU&) = delete;
|
||||
FFTIndexerCPU& operator=(const FFTIndexerCPU&) = delete;
|
||||
~FFTIndexerCPU() override;
|
||||
};
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ ADD_LIBRARY(JFJochScaleMerge
|
||||
SublatticeRelabel.h
|
||||
IceRingVerdict.cpp
|
||||
IceRingVerdict.h)
|
||||
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration fftw3f)
|
||||
TARGET_LINK_LIBRARIES(JFJochScaleMerge Ceres::ceres Eigen3::Eigen JFJochCommon JFJochBraggIntegration)
|
||||
|
||||
IF (JFJOCH_CUDA_AVAILABLE)
|
||||
TARGET_SOURCES(JFJochScaleMerge PRIVATE ../indexing/CUDAMemHelpers.h
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <complex>
|
||||
#include <cstring>
|
||||
#include <iomanip>
|
||||
#include <limits>
|
||||
@@ -15,9 +16,7 @@
|
||||
#include <tuple>
|
||||
#include <unordered_set>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
#include "../../common/CpuFFT.h"
|
||||
#include "../../common/ParallelFor.h"
|
||||
#include "gemmi/twin.hpp"
|
||||
|
||||
@@ -350,20 +349,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
|
||||
}
|
||||
}
|
||||
|
||||
float *coeff = fftwf_alloc_real(2 * n_complex);
|
||||
float *map = fftwf_alloc_real(n_points);
|
||||
if (coeff == nullptr || map == nullptr) {
|
||||
if (coeff) fftwf_free(coeff);
|
||||
if (map) fftwf_free(map);
|
||||
std::vector<std::complex<float>> coeff;
|
||||
std::vector<float> map;
|
||||
try {
|
||||
coeff.resize(n_complex);
|
||||
map.resize(n_points);
|
||||
} catch (const std::bad_alloc &) {
|
||||
result.refusal = "could not allocate the Patterson grid";
|
||||
return result;
|
||||
}
|
||||
fftwf_plan plan;
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_dft_c2r_3d(nx, ny, nz, reinterpret_cast<fftwf_complex *>(coeff), map,
|
||||
FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
// The near-origin exclusion, as a set of grid points rather than a per-point distance test: only
|
||||
// the points inside a 15 A ball around each origin-equivalent point can be excluded, and there
|
||||
@@ -405,17 +399,15 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
|
||||
os << "every point of the cell lies within " << kMinPeakDistanceA
|
||||
<< " A of a lattice point - a small-molecule cell, with no room for a translation between molecules";
|
||||
result.refusal = os.str();
|
||||
fftwf_destroy_plan(plan);
|
||||
fftwf_free(coeff);
|
||||
fftwf_free(map);
|
||||
return result;
|
||||
}
|
||||
|
||||
auto transform = [&](const std::vector<double> &values) {
|
||||
std::memset(coeff, 0, 2 * n_complex * sizeof(float));
|
||||
std::fill(coeff.begin(), coeff.end(), std::complex<float>(0.0f, 0.0f));
|
||||
for (const auto &[at, src] : fill)
|
||||
coeff[2 * at] += static_cast<float>(values[src]);
|
||||
fftwf_execute(plan);
|
||||
coeff[at] += static_cast<float>(values[src]);
|
||||
cpu_fft::C2R({static_cast<size_t>(nx), static_cast<size_t>(ny), static_cast<size_t>(nz)}, {0, 1, 2},
|
||||
coeff.data(), map.data());
|
||||
};
|
||||
auto max_far = [&]() {
|
||||
double best = -std::numeric_limits<double>::infinity();
|
||||
@@ -510,12 +502,6 @@ TranslationalNCSResult AnalyzeTranslationalNCS(const std::vector<MergedReflectio
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
}
|
||||
fftwf_free(coeff);
|
||||
fftwf_free(map);
|
||||
result.measurable = true;
|
||||
|
||||
// ------------------------------------------------------------------ (b) the modulation depth
|
||||
|
||||
@@ -76,7 +76,7 @@ TARGET_INCLUDE_DIRECTORIES(JFJochStructureRefinement PRIVATE ${CMAKE_CURRENT_SOU
|
||||
|
||||
# JFJochIndexing for CUDAMemHelpers.h and, under CUDA, to put cuFFT (RigidBodyGPU.cu) after these
|
||||
# objects on the link line.
|
||||
TARGET_LINK_LIBRARIES(JFJochStructureRefinement JFJochScaleMerge JFJochIndexing JFJochCommon gemmi fftw3f)
|
||||
TARGET_LINK_LIBRARIES(JFJochStructureRefinement JFJochScaleMerge JFJochIndexing JFJochCommon gemmi)
|
||||
|
||||
IF (APPLE)
|
||||
# The rotation function on the Apple GPU (MolecularReplacementMetal.h). Its kernel is compiled when it is first
|
||||
|
||||
@@ -7,46 +7,22 @@
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <complex>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
|
||||
#include <fftw3.h>
|
||||
#include <vector>
|
||||
|
||||
#include "gemmi/fail.hpp"
|
||||
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
#include "../../common/CpuFFT.h"
|
||||
#include "../../common/ParallelFor.h"
|
||||
|
||||
namespace {
|
||||
// An (nu, nv, nw) map is stored u fastest - row-major (nw, nv, nu) - and its half-l coefficients
|
||||
// likewise as (nw/2 + 1, nv, nu): the layout gemmi's own transforms read and write. The real transform
|
||||
// halves w, so w is the last axis transformed.
|
||||
static const cpu_fft::Shape kUVW{2, 1, 0};
|
||||
|
||||
// The r2c plan (or, inverse, the c2r plan) for an (nu, nv, nw) map stored u fastest, halving w - the
|
||||
// layout gemmi's own transforms read and write. FFTW's planner is not thread-safe, so plans are made
|
||||
// under the process-wide planner lock (which also guards this cache); executing one on new arrays
|
||||
// (fftwf_execute_dft_r2c / _c2r) is, which is how every caller uses it.
|
||||
fftwf_plan PlanFor(int nu, int nv, int nw, bool inverse) {
|
||||
static std::map<std::array<int, 4>, fftwf_plan> plans;
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
const std::array<int, 4> key{nu, nv, nw, inverse};
|
||||
if (const auto it = plans.find(key); it != plans.end())
|
||||
return it->second;
|
||||
float *real = fftwf_alloc_real(static_cast<size_t>(nu) * nv * nw);
|
||||
fftwf_complex *cplx = fftwf_alloc_complex(static_cast<size_t>(nu) * nv * (nw / 2 + 1));
|
||||
// The last dimension is the one r2c halves; the strides put u fastest on both sides.
|
||||
fftwf_iodim dims[3] = {{nu, 1, 1}, {nv, nu, nu}, {nw, nu * nv, nu * nv}};
|
||||
fftwf_plan plan = nullptr;
|
||||
if (real != nullptr && cplx != nullptr)
|
||||
plan = inverse ? fftwf_plan_guru_dft_c2r(3, dims, 0, nullptr, cplx, real, FFTW_ESTIMATE)
|
||||
: fftwf_plan_guru_dft_r2c(3, dims, 0, nullptr, real, cplx, FFTW_ESTIMATE);
|
||||
fftwf_free(real);
|
||||
fftwf_free(cplx);
|
||||
if (plan == nullptr)
|
||||
gemmi::fail("ModelFFT: no FFTW plan for the grid");
|
||||
plans.emplace(key, plan);
|
||||
return plan;
|
||||
static cpu_fft::Shape MapShape(int nu, int nv, int nw) {
|
||||
return {static_cast<size_t>(nw), static_cast<size_t>(nv), static_cast<size_t>(nu)};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map) {
|
||||
if (map.axis_order == gemmi::AxisOrder::ZYX)
|
||||
gemmi::fail("MapToFPhi(): ZYX order is not supported");
|
||||
@@ -58,30 +34,16 @@ gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map) {
|
||||
hkl.set_size_without_checking(map.nu, map.nv, map.nw / 2 + 1);
|
||||
const float norm = static_cast<float>(map.unit_cell.volume / map.point_count());
|
||||
|
||||
const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, false);
|
||||
// Buffers from fftwf_malloc: a plan may only be executed on arrays aligned as the ones it was made
|
||||
// on were, which the vectors' own allocations do not promise.
|
||||
float *in = fftwf_alloc_real(map.data.size());
|
||||
fftwf_complex *out = fftwf_alloc_complex(hkl.data.size());
|
||||
if (in == nullptr || out == nullptr) {
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
gemmi::fail("MapToFPhi(): cannot allocate the FFT buffers");
|
||||
}
|
||||
std::copy(map.data.begin(), map.data.end(), in);
|
||||
fftwf_execute_dft_r2c(plan, in, out);
|
||||
// Conjugated, as gemmi does: FFTW's forward transform carries exp(-2 pi i h.x), and a structure
|
||||
cpu_fft::R2C(MapShape(map.nu, map.nv, map.nw), kUVW, map.data.data(), hkl.data.data());
|
||||
// Conjugated, as gemmi does: the forward transform carries exp(-2 pi i h.x), and a structure
|
||||
// factor is the sum over exp(+2 pi i h.x).
|
||||
for (size_t i = 0; i < hkl.data.size(); ++i)
|
||||
hkl.data[i] = std::complex<float>(out[i][0] * norm, -out[i][1] * norm);
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
for (auto &x : hkl.data)
|
||||
x = std::conj(x) * norm;
|
||||
return hkl;
|
||||
}
|
||||
|
||||
void MapFromFPhiExecute(int nu, int nv, int nw, std::complex<float> *in, float *out) {
|
||||
static_assert(sizeof(std::complex<float>) == sizeof(fftwf_complex));
|
||||
fftwf_execute_dft_c2r(PlanFor(nu, nv, nw, true), reinterpret_cast<fftwf_complex *>(in), out);
|
||||
cpu_fft::C2R(MapShape(nu, nv, nw), kUVW, in, out);
|
||||
}
|
||||
|
||||
gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int reach, size_t nthreads) {
|
||||
@@ -89,34 +51,12 @@ gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int re
|
||||
gemmi::fail("MapToFPhiInParallel(): ZYX order is not supported");
|
||||
const int nu = map.nu, nv = map.nv, nw = map.nw, nwh = nw / 2 + 1, nk = 2 * reach + 1;
|
||||
const size_t plane = static_cast<size_t>(nu) * nv;
|
||||
float *in = fftwf_alloc_real(map.data.size());
|
||||
fftwf_complex *half = fftwf_alloc_complex(plane * nwh);
|
||||
if (in == nullptr || half == nullptr) {
|
||||
fftwf_free(in);
|
||||
fftwf_free(half);
|
||||
gemmi::fail("MapToFPhiInParallel(): cannot allocate the FFT buffers");
|
||||
}
|
||||
// Made once here; executing a plan on new arrays is safe from several threads at once. Unaligned, as a
|
||||
// column or a plane starts wherever it does.
|
||||
fftwf_plan columns = nullptr, planes = nullptr;
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
// The nu columns of one v, w along the stride of a plane.
|
||||
fftwf_iodim col_dim{nw, static_cast<int>(plane), static_cast<int>(plane)};
|
||||
fftwf_iodim col_many{nu, 1, 1};
|
||||
columns = fftwf_plan_guru_dft_r2c(1, &col_dim, 1, &col_many, in, half, FFTW_ESTIMATE | FFTW_UNALIGNED);
|
||||
fftwf_iodim plane_dims[2] = {{nv, nu, nu}, {nu, 1, 1}};
|
||||
planes = fftwf_plan_guru_dft(2, plane_dims, 0, nullptr, half, half, FFTW_FORWARD, FFTW_ESTIMATE | FFTW_UNALIGNED);
|
||||
}
|
||||
if (columns == nullptr || planes == nullptr)
|
||||
gemmi::fail("MapToFPhiInParallel(): no FFTW plan for the grid");
|
||||
std::copy(map.data.begin(), map.data.end(), in);
|
||||
ParallelFor(nv, nthreads, [&](int v) {
|
||||
fftwf_execute_dft_r2c(columns, in + static_cast<size_t>(v) * nu, half + static_cast<size_t>(v) * nu);
|
||||
});
|
||||
ParallelFor(reach + 1, nthreads, [&](int l) {
|
||||
fftwf_execute_dft(planes, half + l * plane, half + l * plane);
|
||||
});
|
||||
std::vector<std::complex<float>> half(plane * nwh);
|
||||
// The r2c along w of every (u, v) column, then the 2D transform of the planes l <= reach only - the
|
||||
// rest are never read. Each is one call split over the threads by lines.
|
||||
cpu_fft::R2C({static_cast<size_t>(nw), plane}, {0}, map.data.data(), half.data(), nthreads);
|
||||
cpu_fft::C2C({static_cast<size_t>(reach) + 1, static_cast<size_t>(nv), static_cast<size_t>(nu)}, {1, 2}, true,
|
||||
half.data(), nthreads);
|
||||
|
||||
gemmi::FPhiGrid<float> hkl;
|
||||
hkl.unit_cell = map.unit_cell;
|
||||
@@ -128,19 +68,12 @@ gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int re
|
||||
// Conjugated and scaled as MapToFPhi does it, and cropped as CropHalfL crops it.
|
||||
ParallelFor(reach + 1, nthreads, [&](int l) {
|
||||
for (int k = -reach; k <= reach; k++) {
|
||||
const fftwf_complex *row = half + l * plane + static_cast<size_t>(nu) * gemmi::modulo(k, nv);
|
||||
const std::complex<float> *row = &half[l * plane + static_cast<size_t>(nu) * gemmi::modulo(k, nv)];
|
||||
std::complex<float> *out = &hkl.data[static_cast<size_t>(nu) * (gemmi::modulo(k, nk) + static_cast<size_t>(nk) * l)];
|
||||
for (int u = 0; u < nu; u++)
|
||||
out[u] = std::complex<float>(row[u][0] * norm, -row[u][1] * norm);
|
||||
out[u] = std::conj(row[u]) * norm;
|
||||
}
|
||||
});
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(columns);
|
||||
fftwf_destroy_plan(planes);
|
||||
}
|
||||
fftwf_free(in);
|
||||
fftwf_free(half);
|
||||
return hkl;
|
||||
}
|
||||
|
||||
@@ -154,26 +87,14 @@ gemmi::Grid<float> MapFromFPhi(const gemmi::FPhiGrid<float> &hkl) {
|
||||
map.axis_order = hkl.axis_order;
|
||||
const float norm = static_cast<float>(1.0 / hkl.unit_cell.volume);
|
||||
|
||||
const fftwf_plan plan = PlanFor(map.nu, map.nv, map.nw, true);
|
||||
fftwf_complex *in = fftwf_alloc_complex(hkl.data.size());
|
||||
float *out = fftwf_alloc_real(map.data.size());
|
||||
if (in == nullptr || out == nullptr) {
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
gemmi::fail("MapFromFPhi(): cannot allocate the FFT buffers");
|
||||
}
|
||||
// Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and FFTW's backward transform
|
||||
// Conjugated, as gemmi does: rho(x) = 1/V sum F exp(-2 pi i h.x), and the backward transform
|
||||
// carries exp(+2 pi i h.x). A missing coefficient (NaN) counts as zero, also as gemmi does.
|
||||
for (size_t i = 0; i < hkl.data.size(); ++i) {
|
||||
const std::complex<float> x = hkl.data[i];
|
||||
in[i][0] = std::isnan(x.imag()) ? 0.0f : x.real();
|
||||
in[i][1] = std::isnan(x.imag()) ? 0.0f : -x.imag();
|
||||
}
|
||||
fftwf_execute_dft_c2r(plan, in, out);
|
||||
for (size_t i = 0; i < map.data.size(); ++i)
|
||||
map.data[i] = out[i] * norm;
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
std::vector<std::complex<float>> in(hkl.data.size());
|
||||
for (size_t i = 0; i < in.size(); ++i)
|
||||
in[i] = std::isnan(hkl.data[i].imag()) ? 0.0f : std::conj(hkl.data[i]);
|
||||
cpu_fft::C2R(MapShape(map.nu, map.nv, map.nw), kUVW, in.data(), map.data.data());
|
||||
for (auto &x : map.data)
|
||||
x *= norm;
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
@@ -9,30 +9,23 @@
|
||||
// The structure factors of a real-space map on the model path (Fcalc from the model density, Fmask
|
||||
// from the bulk-solvent mask): what gemmi::transform_map_to_f_phi(map, true) returns - the
|
||||
// conjugated forward transform (F = sum rho exp(+2 pi i h.x)) scaled by V/N, kept for l >= 0 on an (nu, nv, nw/2+1) grid in the map's own XYZ order,
|
||||
// ready for prepare_asu_data() - computed with FFTW rather than gemmi's single-threaded pocketfft.
|
||||
// The same transform, not the same bits: the two libraries order their arithmetic differently.
|
||||
//
|
||||
// Plans are FFTW_ESTIMATE only - MEASURE would time candidate algorithms and could pick a different
|
||||
// one from run to run - and made once per grid size and kept, so the rigid body's hundred-odd
|
||||
// evaluations on one grid plan once. Safe to call from several threads at once.
|
||||
// ready for prepare_asu_data() - computed with common/CpuFFT. Safe to call from several threads at once.
|
||||
gemmi::FPhiGrid<float> MapToFPhi(const gemmi::Grid<float> &map);
|
||||
|
||||
// The inverse: the real-space map of an (nu, nv, nw/2+1) half-l coefficient grid in XYZ order, as
|
||||
// get_f_phi_on_grid() fills it - what gemmi::transform_f_phi_grid_to_map() returns (rho = 1/V sum F
|
||||
// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero), computed with FFTW
|
||||
// under the same planning rules.
|
||||
// exp(-2 pi i h.x) on an (nu, nv, 2*(nw-1)) grid, NaN coefficients read as zero).
|
||||
gemmi::Grid<float> MapFromFPhi(const gemmi::FPhiGrid<float> &hkl);
|
||||
|
||||
// MapFromFPhi's transform itself, for a caller that fills its input: `in`, the (nu, nv, nw/2+1) half-l coefficients
|
||||
// u fastest as MapFromFPhi hands them to FFTW (conjugated, a NaN as zero), is overwritten, and `out` receives the
|
||||
// (nu, nv, nw) map before MapFromFPhi's 1/V. Both from fftwf_malloc, as a plan requires.
|
||||
// u fastest as MapFromFPhi transforms them (conjugated, a NaN as zero), is overwritten, and `out` receives the
|
||||
// (nu, nv, nw) map before MapFromFPhi's 1/V.
|
||||
void MapFromFPhiExecute(int nu, int nv, int nw, std::complex<float> *in, float *out);
|
||||
|
||||
// CropHalfL(MapToFPhi(map), reach) on `nthreads` threads, for a map of hundreds of millions of points (the
|
||||
// molecular transform's box), whose single-threaded transform is minutes of CPU: the r2c along w of every
|
||||
// (u, v) column, then the 2D transform of every plane l <= reach - each column and each plane by one plan, so
|
||||
// the result does not depend on the thread count. The same transform as MapToFPhi to float rounding, not bit
|
||||
// for bit: FFTW's 3D plan orders its arithmetic its own way.
|
||||
// (u, v) column, then the 2D transform of every plane l <= reach. The threads split the lines of a transform, each
|
||||
// line transformed as it would be alone, so the result does not depend on the thread count.
|
||||
gemmi::FPhiGrid<float> MapToFPhiInParallel(const gemmi::Grid<float> &map, int reach, size_t nthreads);
|
||||
|
||||
// The half-l grid `full` with only the indices |k|, l <= reach kept, on a grid of full.nu x (2 reach + 1) x
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#include <random>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include <gemmi/modify.hpp> // remove_hydrogens, remove_alternative_conformations
|
||||
#include <gemmi/polyheur.hpp> // setup_entities, remove_ligands_and_waters
|
||||
@@ -1602,13 +1601,8 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
|
||||
w[t] = setup.weight[t] / (bin_sum[setup.bin[t]] / bin_n[setup.bin[t]]);
|
||||
const int nu = setup.nu, nv = setup.nv, nw = setup.nw;
|
||||
const size_t cells = static_cast<size_t>(nu) * nv * (nw / 2 + 1), points = static_cast<size_t>(nu) * nv * nw;
|
||||
auto *in = reinterpret_cast<std::complex<float> *>(fftwf_alloc_complex(cells));
|
||||
float *map = fftwf_alloc_real(points);
|
||||
if (in == nullptr || map == nullptr) {
|
||||
fftwf_free(in);
|
||||
fftwf_free(map);
|
||||
throw std::runtime_error("TranslationFunction: cannot allocate the FFT buffers");
|
||||
}
|
||||
std::vector<std::complex<float>> in(cells);
|
||||
std::vector<float> map(points);
|
||||
for (size_t c = 0; c < cells; c++) {
|
||||
std::complex<double> v = 0.0;
|
||||
for (int k = gather.offset[c]; k < gather.offset[c + 1]; k++) {
|
||||
@@ -1626,7 +1620,7 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
|
||||
const std::complex<float> f(std::conj(v));
|
||||
in[c] = std::isnan(f.imag()) ? std::complex<float>(0.0f, 0.0f) : std::complex<float>(f.real(), -f.imag());
|
||||
}
|
||||
MapFromFPhiExecute(nu, nv, nw, in, map);
|
||||
MapFromFPhiExecute(nu, nv, nw, in.data(), map.data());
|
||||
const float norm = static_cast<float>(1.0 / ft.Grid().unit_cell.volume);
|
||||
double sum = 0, sum2 = 0;
|
||||
size_t best = 0;
|
||||
@@ -1640,8 +1634,6 @@ TranslationPeak translation_function(const MolecularTransform &ft, const Transla
|
||||
const double count = static_cast<double>(points);
|
||||
const double mean = sum / count, sd = std::sqrt(std::max(0.0, sum2 / count - mean * mean));
|
||||
out.z = sd > 0 ? (map[best] * norm - mean) / sd : 0.0;
|
||||
fftwf_free(in);
|
||||
fftwf_free(map);
|
||||
const size_t u = best % nu, v = (best / nu) % nv, l = best / (static_cast<size_t>(nu) * nv);
|
||||
out.t = gemmi::Fractional(static_cast<double>(u) / nu, static_cast<double>(v) / nv, static_cast<double>(l) / nw);
|
||||
return out;
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
#include <numeric>
|
||||
#include <set>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "gemmi/dencalc.hpp"
|
||||
#include "gemmi/it92.hpp"
|
||||
@@ -188,9 +187,8 @@ public:
|
||||
// l = nw/2, which hold both, as both.
|
||||
const int nu = grid_.nu, nv = grid_.nv, nw = grid_.nw, nwh = nw / 2 + 1;
|
||||
const size_t ncomplex = static_cast<size_t>(nu) * nv * nwh;
|
||||
std::complex<float> *in = reinterpret_cast<std::complex<float> *>(fftwf_alloc_complex(ncomplex));
|
||||
float *out = fftwf_alloc_real(static_cast<size_t>(nu) * nv * nw);
|
||||
std::fill(in, in + ncomplex, std::complex<float>(0, 0));
|
||||
std::vector<std::complex<float>> in(ncomplex);
|
||||
std::vector<float> out(static_cast<size_t>(nu) * nv * nw);
|
||||
const std::vector<SymmetryComposition::Term> &terms = composition_.Terms();
|
||||
const size_t ops = composition_.Ops();
|
||||
auto add = [&](int ku, int kv, int kw, std::complex<double> value) {
|
||||
@@ -217,13 +215,11 @@ public:
|
||||
}
|
||||
}
|
||||
}
|
||||
MapFromFPhiExecute(nu, nv, nw, in, out);
|
||||
MapFromFPhiExecute(nu, nv, nw, in.data(), out.data());
|
||||
gemmi::Grid<float> map = grid_;
|
||||
const float norm = static_cast<float>(cell_.volume / static_cast<double>(map.point_count()));
|
||||
for (size_t i = 0; i < map.data.size(); i++)
|
||||
map.data[i] = out[i] * norm;
|
||||
fftwf_free(in);
|
||||
fftwf_free(out);
|
||||
const bool aniso = AnyAniso();
|
||||
AtomGradientsFromMap(map, model, blur_, nthreads_, *grad, aniso ? &grad_u_ : nullptr);
|
||||
if (!aniso)
|
||||
|
||||
@@ -41,7 +41,7 @@ ADD_LIBRARY(JFJochRugnux STATIC
|
||||
BrokerFeed.h
|
||||
)
|
||||
|
||||
TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi fftw3f
|
||||
TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPreview gemmi
|
||||
httplib::httplib)
|
||||
|
||||
# The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux
|
||||
|
||||
@@ -846,21 +846,6 @@ namespace {
|
||||
}
|
||||
}
|
||||
|
||||
// FFTW ends the program itself when its allocator comes back empty (CK(p) in kernel/alloc.c) - while
|
||||
// planning, or inside a buffered transform - from whichever thread that happens in, so it cannot become
|
||||
// an exception. Defining its handler here keeps FFTW's own (kernel/assert.c) out of the link, and says
|
||||
// what happened in the same words as every other way of running out of memory.
|
||||
extern "C" void fftwf_assertion_failed(const char *s, int line, const char *file) {
|
||||
std::fflush(stdout);
|
||||
if (std::strcmp(s, "p") == 0) {
|
||||
std::fprintf(stderr, "Processing failed: out of host memory (FFTW could not allocate) - "
|
||||
"this data set needs more RAM than is available\n");
|
||||
std::_Exit(EXIT_FAILURE);
|
||||
}
|
||||
std::fprintf(stderr, "fftw: %s:%d: assertion failed: %s\n", file, line, s);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
namespace {
|
||||
// One input's run ended: what exit(EXIT_FAILURE) was where a command line ran one input only. A run of
|
||||
// several sweeps goes on with the next sweep; RunRugnux returns the code.
|
||||
|
||||
@@ -174,10 +174,11 @@ TEST_CASE("BeamCenterFFTGPU_MatchesTheCPUEngine", "[BeamCenter][BeamCenterFFTGPU
|
||||
std::abs(static_cast<double>(rg[i]) - rc[i]));
|
||||
CHECK(peak_worst < 1e-5);
|
||||
// And the gate agrees: over four million centres, the two paths disagree about whether a
|
||||
// centre is scored at all only for a handful sitting exactly on the threshold.
|
||||
// centre is scored at all only for a handful sitting exactly on the threshold (measured: 7
|
||||
// with FFTW on the CPU, 17 with pocketfft, whose float error is FFTW's or lower).
|
||||
const auto diff = Compare(rg, rc, &c.D, 0.26f * d_max);
|
||||
CHECK(diff.gate_mismatches == 0);
|
||||
CHECK(Compare(rg, rc).gate_mismatches < 10);
|
||||
CHECK(Compare(rg, rc).gate_mismatches < 30);
|
||||
|
||||
// The device kernel against that reference, on the device's own surfaces - so this measures
|
||||
// the combination and not the transform.
|
||||
|
||||
@@ -237,6 +237,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
LatticeReductionTest.cpp
|
||||
BeamCenterFromBackgroundTest.cpp
|
||||
BeamCenterFFTTest.cpp
|
||||
CpuFFTTest.cpp
|
||||
BeamCenterFFTGPUTest.cpp
|
||||
BearerTokensTest.cpp
|
||||
BrokerHttpAuthTest.cpp
|
||||
@@ -246,7 +247,7 @@ ADD_EXECUTABLE(jfjoch_test
|
||||
|
||||
target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochStreamWriter
|
||||
JFJochRugnuxOutput JFJochRugnux JFJochModelWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview
|
||||
jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib)
|
||||
jfjoch_xds_plugin ZLIB::ZLIB httplib::httplib fftw3f)
|
||||
target_include_directories(jfjoch_test PRIVATE .)
|
||||
IF (JFJOCH_CUDA_AVAILABLE)
|
||||
TARGET_COMPILE_DEFINITIONS(jfjoch_test PRIVATE FFBIDX_SIMPLE_DATA_DIR="${FFBIDX_SIMPLE_DATA_DIR}")
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <complex>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
#include <fftw3.h>
|
||||
|
||||
#include "../common/CpuFFT.h"
|
||||
#include "../image_analysis/structure_refinement/ModelFFT.h"
|
||||
|
||||
// pocketfft (common/CpuFFT) against FFTW on the shapes and layouts the call sites use: the same
|
||||
// transform to float rounding. FFTW is linked into the test binary only, as the reference.
|
||||
|
||||
namespace {
|
||||
|
||||
std::vector<float> Random(size_t n, unsigned seed) {
|
||||
std::mt19937 rng(seed);
|
||||
std::normal_distribution<float> g(0.0f, 1.0f);
|
||||
std::vector<float> v(n);
|
||||
for (auto &x : v)
|
||||
x = g(rng);
|
||||
return v;
|
||||
}
|
||||
|
||||
// Largest difference relative to the largest value of the reference.
|
||||
template <typename T>
|
||||
double RelativeError(const std::vector<T> &a, const std::vector<T> &ref) {
|
||||
REQUIRE(a.size() == ref.size());
|
||||
double diff = 0, scale = 0;
|
||||
for (size_t i = 0; i < a.size(); i++) {
|
||||
diff = std::max(diff, static_cast<double>(std::abs(a[i] - ref[i])));
|
||||
scale = std::max(scale, static_cast<double>(std::abs(ref[i])));
|
||||
}
|
||||
return diff / scale;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("CpuFFT_BatchedR2CMatchesFFTW", "[fft]") {
|
||||
// The FFT indexer's histograms: one transform per direction; 3142 = 2 x 1571 is a real length.
|
||||
for (int n : {3142, 3150, 1024}) {
|
||||
const int howmany = 37, nc = n / 2 + 1;
|
||||
std::vector<float> in = Random(static_cast<size_t>(howmany) * n, n);
|
||||
std::vector<std::complex<float>> ours(static_cast<size_t>(howmany) * nc), ref(ours.size());
|
||||
cpu_fft::R2C({static_cast<size_t>(howmany), static_cast<size_t>(n)}, {1}, in.data(), ours.data());
|
||||
std::vector<float> work = in;
|
||||
fftwf_plan p = fftwf_plan_many_dft_r2c(1, &n, howmany, work.data(), nullptr, 1, n,
|
||||
reinterpret_cast<fftwf_complex *>(ref.data()), nullptr, 1, nc,
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_execute(p);
|
||||
fftwf_destroy_plan(p);
|
||||
CHECK(RelativeError(ours, ref) < 1e-5);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("CpuFFT_2DRoundTripMatchesFFTW", "[fft]") {
|
||||
// The beam-centre search: a 2D r2c and c2r on a 2-3-5-7-smooth padded size.
|
||||
const int ny = 210, nx = 336, nxc = nx / 2 + 1;
|
||||
std::vector<float> in = Random(static_cast<size_t>(ny) * nx, 1);
|
||||
std::vector<std::complex<float>> ours(static_cast<size_t>(ny) * nxc), ref(ours.size());
|
||||
cpu_fft::R2C({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, in.data(), ours.data());
|
||||
std::vector<float> work = in;
|
||||
fftwf_plan p = fftwf_plan_dft_r2c_2d(ny, nx, work.data(), reinterpret_cast<fftwf_complex *>(ref.data()),
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_execute(p);
|
||||
fftwf_destroy_plan(p);
|
||||
CHECK(RelativeError(ours, ref) < 1e-5);
|
||||
|
||||
std::vector<float> back_ours(in.size()), back_ref(in.size());
|
||||
cpu_fft::C2R({static_cast<size_t>(ny), static_cast<size_t>(nx)}, {0, 1}, ours.data(), back_ours.data());
|
||||
p = fftwf_plan_dft_c2r_2d(ny, nx, reinterpret_cast<fftwf_complex *>(ref.data()), back_ref.data(), FFTW_ESTIMATE);
|
||||
fftwf_execute(p);
|
||||
fftwf_destroy_plan(p);
|
||||
CHECK(RelativeError(back_ours, back_ref) < 1e-5);
|
||||
}
|
||||
|
||||
TEST_CASE("CpuFFT_3DMatchesFFTW", "[fft]") {
|
||||
// The model maps (u fastest, w halved) and the tNCS Patterson (row-major, last axis halved) are both a
|
||||
// row-major 3D transform halving the last axis; an odd length and a factor 7 included.
|
||||
const int n0 = 70, n1 = 45, n2 = 54, nc = static_cast<int>(static_cast<size_t>(n0) * n1 * (n2 / 2 + 1));
|
||||
const std::vector<float> in = Random(static_cast<size_t>(n0) * n1 * n2, 2);
|
||||
std::vector<std::complex<float>> ours(nc), ref(nc);
|
||||
cpu_fft::R2C({70, 45, 54}, {0, 1, 2}, in.data(), ours.data());
|
||||
std::vector<float> work = in;
|
||||
fftwf_plan p = fftwf_plan_dft_r2c_3d(n0, n1, n2, work.data(), reinterpret_cast<fftwf_complex *>(ref.data()),
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_execute(p);
|
||||
fftwf_destroy_plan(p);
|
||||
CHECK(RelativeError(ours, ref) < 1e-5);
|
||||
|
||||
std::vector<float> back_ours(in.size()), back_ref(in.size());
|
||||
cpu_fft::C2R({70, 45, 54}, {0, 1, 2}, ours.data(), back_ours.data());
|
||||
p = fftwf_plan_dft_c2r_3d(n0, n1, n2, reinterpret_cast<fftwf_complex *>(ref.data()), back_ref.data(),
|
||||
FFTW_ESTIMATE);
|
||||
fftwf_execute(p);
|
||||
fftwf_destroy_plan(p);
|
||||
CHECK(RelativeError(back_ours, back_ref) < 1e-5);
|
||||
}
|
||||
|
||||
TEST_CASE("CpuFFT_MapToFPhiInParallelMatchesMapToFPhi", "[fft]") {
|
||||
gemmi::Grid<float> map;
|
||||
map.unit_cell.set(60, 50, 40, 90, 90, 90);
|
||||
map.spacegroup = gemmi::find_spacegroup_by_name("P1");
|
||||
map.set_size(60, 50, 40);
|
||||
map.data = Random(map.data.size(), 3);
|
||||
const int reach = 12;
|
||||
const gemmi::FPhiGrid<float> ref = CropHalfL(MapToFPhi(map), reach);
|
||||
for (size_t nthreads : {1, 4}) {
|
||||
const gemmi::FPhiGrid<float> ours = MapToFPhiInParallel(map, reach, nthreads);
|
||||
CHECK(RelativeError(ours.data, ref.data) < 1e-5);
|
||||
}
|
||||
}
|
||||
@@ -124,7 +124,7 @@ ADD_EXECUTABLE(jfjoch_viewer jfjoch_viewer.cpp JFJochViewerWindow.cpp JFJochView
|
||||
|
||||
TARGET_LINK_LIBRARIES(jfjoch_viewer Qt6::Core Qt6::Gui Qt6::Widgets Qt6::Charts Qt6::Concurrent
|
||||
JFJochReader JFJochLogger JFJochCommon JFJochWriter JFJochImageAnalysis JFJochRugnux JFJochRugnuxOutput
|
||||
fftw3f httplib::httplib MbedTLS::mbedtls)
|
||||
httplib::httplib MbedTLS::mbedtls)
|
||||
|
||||
# HTTPS for the live-broker client. Defined here, on the viewer only, not on httplib::httplib: the
|
||||
# broker shares that target and stays plain HTTP. JFJochHttpReader.cpp is the one viewer source
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#include <complex>
|
||||
|
||||
#include "JFJochDatasetInfoChartView.h"
|
||||
#include "../../common/FFTWPlannerLock.h"
|
||||
#include "../../common/CpuFFT.h"
|
||||
|
||||
namespace {
|
||||
// Qt Charts draws its curves 2 px wide, which is hard to follow across a desk. Take the weight
|
||||
@@ -1122,25 +1122,7 @@ void JFJochDatasetInfoChartView::buildFFTChart() {
|
||||
|
||||
std::vector<std::complex<float>> out(static_cast<size_t>(nComplex));
|
||||
|
||||
fftwf_plan plan;
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
plan = fftwf_plan_dft_r2c_1d(
|
||||
n,
|
||||
in.data(),
|
||||
reinterpret_cast<fftwf_complex *>(out.data()),
|
||||
FFTW_ESTIMATE);
|
||||
}
|
||||
|
||||
if (!plan) {
|
||||
return;
|
||||
}
|
||||
|
||||
fftwf_execute(plan);
|
||||
{
|
||||
std::lock_guard lock(FFTWPlannerMutex());
|
||||
fftwf_destroy_plan(plan);
|
||||
}
|
||||
cpu_fft::R2C({N}, {0}, in.data(), out.data());
|
||||
|
||||
// Compute amplitude spectrum and frequencies (0 .. Nyquist)
|
||||
m_fftMagnitudes.resize(static_cast<size_t>(nComplex));
|
||||
|
||||
@@ -15,10 +15,6 @@
|
||||
#include <QPointer>
|
||||
#include <QColor>
|
||||
|
||||
#ifdef JFJOCH_USE_FFTW
|
||||
#include <fftw3.h>
|
||||
#endif
|
||||
|
||||
#include "../../common/GoniometerAxis.h"
|
||||
#include "../../reader/JFJochReaderDataset.h"
|
||||
|
||||
|
||||
Reference in new issue
Block a user