From c3cefebe8fc15be58f54e748d9add17f377ace37 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 13:48:39 +0200 Subject: [PATCH 1/2] FFT indexer GPU peak search: four threads per direction, output unchanged The peak search gave each of the 16384 directions to one thread, a chain of dependent FP64 additions over 1572 bins - latency-bound on a card that holds far more threads. Each direction's spectrum is now split into four consecutive runs of bins, one thread each, and the runs' first-best peaks are combined in order with the same strict comparison, so the first bin of highest prominence is still the one returned. A run after the first sums its starting window afresh rather than carrying the running sum in. The double sums of float magnitudes are exact unless a magnitude is below ~1e-9 of its window sum, so the background is the same double: 0 of 491520 directions differ in a synthetic still/rotation harness, and the hkl of 3000 stills frames (-X fft) is byte-identical. Kernel (isolated, RTX 5080): 0.87 -> 0.65 ms per call. The stills de-novo loop gains ~4% (128 -> 133 frames/s, ABAB, load ~10): the FFT chain as a whole is bound by GPU memory bandwidth under the indexer pool's concurrency, so even a background in float (6x faster kernel, not adopted) gives only ~9%. Rotation first pass: FFT kernels are ~3% of the indexing step. Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/indexing/FFTIndexerGPU.cu | 138 ++++++++++++----------- 1 file changed, 73 insertions(+), 65 deletions(-) diff --git a/image_analysis/indexing/FFTIndexerGPU.cu b/image_analysis/indexing/FFTIndexerGPU.cu index 99ecbd4d2..418b06f00 100644 --- a/image_analysis/indexing/FFTIndexerGPU.cu +++ b/image_analysis/indexing/FFTIndexerGPU.cu @@ -40,6 +40,16 @@ __global__ void magnitude_transpose(const cufftComplex *__restrict__ d_output, } } +// The peak search splits each direction's spectrum into PEAK_CHUNKS consecutive runs of bins and gives each +// run to its own thread, which keeps a running window sum over its run; one thread per direction leaves the +// card nearly empty (16384 threads), and this search, a chain of dependent FP64 additions per thread, is +// then latency-bound. A block holds 32 directions (x, so the bin-major loads stay coalesced) times PEAK_CHUNKS. +// A run after the first sums its starting window afresh instead of carrying the sum in from the bins before +// it. That is the same double wherever the sums are exact, and they are: a window of float magnitudes fits +// a double's 53 bits unless one of them is below ~1e-9 of the window sum (no difference over 3000 stills +// frames and a rotation-style synthetic set). More runs buy nothing: the card's FP64 rate is then the limit. +constexpr int PEAK_CHUNKS = 4; + __global__ void calculate_fft_result( const float *__restrict__ mag_t, const float max_length_A, @@ -48,86 +58,83 @@ __global__ void calculate_fft_result( const int bg_half, const int directions_size, FFTResult *d_results) { - int i = blockIdx.x * blockDim.x + threadIdx.x; // Get thread index + __shared__ float run_prom[PEAK_CHUNKS][32]; + __shared__ float run_len[PEAK_CHUNKS][32]; + + const int i = blockIdx.x * 32 + threadIdx.x; + const int run = threadIdx.y; + float best_prom = 0.0f; + float best_len = -1.0f; if (i < directions_size) { const int out_len = (histogram_size / 2) + 1; // |transform| of bin k of this direction, from the bin-major array (magnitude_transpose) const auto mag_at = [&](int k) { return mag_t[static_cast(k) * directions_size + i]; }; - float len_coeff = 2.0f * max_length_A / static_cast(histogram_size); + const float len_coeff = 2.0f * max_length_A / static_cast(histogram_size); // Pick the peak by PROMINENCE above a running-mean background of half-width bg_half: // the projected histogram has a broad low-frequency envelope whose magnitude can // exceed the true lattice peaks, so a plain argmax|spec| returns a short envelope // vector on weak/pink-beam frames. Subtracting the local mean removes that envelope // while sharp lattice peaks keep their height (mirrors FFTIndexerCPU). - float best_prom = 0.0f; - FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1}; - // bg is consider()'s background, the window mean - formed by whichever loop below applies. - const auto consider = [&](int j, float bg) { - const float len = len_coeff * static_cast(j); - if (len <= min_length_A) return; + // + // The window has a constant width, slid inward at the ends instead of truncated (see + // FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - otherwise + // gets a one-sided background and a biased prominence. A spectrum shorter than the window + // averages over all of itself. + const int width = min(2 * bg_half, out_len - 1); + const int lo_max = out_len - 1 - width; + const auto window_lo = [&](int j) { return min(max(j - bg_half, 0), lo_max); }; - const float mag = mag_at(j); - const float prom = mag - bg; + // The window width is fixed, so the mean divides by one number throughout. A double division + // is a long iteration on the FP64 units a consumer card has few of; multiplying by the correctly + // rounded reciprocal and correcting the quotient once with its exact remainder gives the + // correctly rounded quotient - the same double the division returns (Markstein (1990) IBM J. + // Res. Dev. 34, 111-119). + const double count = width + 1; + const double inv_count = 1.0 / count; + + const int j0 = (out_len * run) / PEAK_CHUNKS; + const int j1 = (out_len * (run + 1)) / PEAK_CHUNKS; + int lo = window_lo(j0); + double winsum = 0.0; + for (int k = lo; k <= lo + width; ++k) + winsum += mag_at(k); + + for (int j = j0; j < j1; ++j) { + if (window_lo(j) > lo) { + winsum += mag_at(lo + width + 1); + winsum -= mag_at(lo); + ++lo; + } + const float len = len_coeff * static_cast(j); + if (len <= min_length_A) + continue; + const double q = __dmul_rn(winsum, inv_count); + const float bg = static_cast(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q)); + const float prom = mag_at(j) - bg; if (prom > best_prom) { best_prom = prom; - result.magnitude = prom; - result.length = len; - } - }; - - if (out_len - 1 >= 2 * bg_half) { - // Where the spectrum holds a whole window, the window of the general loop below is - // [0, 2 bg_half] until it can be centred, centred while it fits, and the last 2 bg_half + 1 - // bins after that: the running sum takes the same additions and subtractions in the same - // order, so it is bit for bit the same number. Spelled out, the loads no longer wait on - // data-dependent loops and the compiler can issue them ahead, which is what the search - - // latency-bound, one dependent chain per direction - is short of. - // The window width is fixed here, so the mean divides by one number throughout. A double - // division is a long iteration on the FP64 units a consumer card has few of, and it was a - // quarter of this kernel's time; multiplying by the correctly rounded reciprocal and correcting the - // quotient once with its exact remainder gives the correctly rounded quotient - the same - // double the division returns (Markstein (1990) IBM J. Res. Dev. 34, 111-119). - const double count = 2 * bg_half + 1; - const double inv_count = 1.0 / count; - double winsum = 0.0; - for (int k = 0; k <= 2 * bg_half; ++k) - winsum += mag_at(k); -#pragma unroll 4 - for (int j = 0; j < out_len; ++j) { - if (j > bg_half && j + bg_half <= out_len - 1) { - winsum += mag_at(j + bg_half); - winsum -= mag_at(j - bg_half - 1); - } - const double q = __dmul_rn(winsum, inv_count); - consider(j, static_cast(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q))); - } - } else { - double winsum = 0.0; - int wlo = 0; - int whi = min(bg_half, out_len - 1); - for (int k = wlo; k <= whi; ++k) - winsum += mag_at(k); - - for (int j = 0; j < out_len; ++j) { - // Constant-width window, slid inward at the ends instead of truncated (see - // FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - - // otherwise gets a one-sided background and a biased prominence. Both bounds stay - // monotonically non-decreasing in j, so the running sum below is still valid. - int want_lo = j - bg_half; - int want_hi = j + bg_half; - if (want_lo < 0) { want_hi = min(out_len - 1, want_hi - want_lo); want_lo = 0; } - if (want_hi > out_len - 1) { - want_lo = max(0, want_lo - (want_hi - (out_len - 1))); - want_hi = out_len - 1; - } - while (whi < want_hi) { ++whi; winsum += mag_at(whi); } - while (wlo < want_lo) { winsum -= mag_at(wlo); ++wlo; } - consider(j, static_cast(winsum / static_cast(whi - wlo + 1))); + best_len = len; } } - d_results[i] = result; // Store the result + } + + run_prom[run][threadIdx.x] = best_prom; + run_len[run][threadIdx.x] = best_len; + __syncthreads(); + + // Each run kept its first bin of highest prominence; taking the runs in order with the same strict + // comparison gives the direction's first bin of highest prominence. + if (run == 0 && i < directions_size) { + FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1}; + for (int r = 0; r < PEAK_CHUNKS; ++r) { + if (run_prom[r][threadIdx.x] > result.magnitude) { + result.magnitude = run_prom[r][threadIdx.x]; + result.length = run_len[r][threadIdx.x]; + } + } + d_results[i] = result; } } @@ -308,7 +315,8 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector &coord, size_t nspots) { magnitude_transpose<<>>(d_output_fft, out_len, direction_vectors.size(), d_mag_t); cuda_err(cudaGetLastError()); - calculate_fft_result<<>>(d_mag_t, + const int peak_grid = (static_cast(direction_vectors.size()) + 31) / 32; + calculate_fft_result<<>>(d_mag_t, max_length_A, min_length_A, histogram_size, bg_half, direction_vectors.size(), d_result_fft); From 90ba15871f2ee1a83a75b8258a9c09f7db676eed Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 16:40:34 +0200 Subject: [PATCH 2/2] CudaFFTPlan: wait for the plan's NULL-stream setup before it is used, output unchanged A transform launched on a non-blocking stream straight after cufftPlanMany could read the plan before its setup, still queued on the NULL stream, had finished: Xid 31, a read at a fixed near-null address, "an illegal memory access" in the indexer threads. It showed at transform length 3072, in 3 of 4 runs of one rotation set where a pool built its FFT indexers at the start of integration (6 of 28 ci-tier sets, 9 of 25 null-model runs). It went away under CUDA_LAUNCH_BLOCKING=1 and with a cudaStreamSynchronize(NULL) after the plan (0 of 11 runs). The 3142 length the indexer used so far never hit it, but nothing ordered it either. Plans are built once per indexer, so the wait costs nothing measurable. The other cuFFT users (beam-centre FFT, model structure factors, rigid body) create raw plans and are not covered by this wrapper. Co-Authored-By: Claude Opus 5.5 (1M context) --- image_analysis/indexing/CUDAMemHelpers.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 0aa741556..6d2db7ad0 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -128,6 +128,13 @@ public: throw JFJochException(JFJochExceptionCategory::GPUCUDAError, GPUMemoryFailure("Creating the CUFFT plan (cufftPlanMany)", 0, true, ("cuFFT error " + std::to_string(r)).c_str())); + // The plan's setup can still be queued on the NULL stream when cufftPlanMany returns, and a + // non-blocking stream (CudaStream) does not wait for it: a transform launched on one straight + // away read the plan's tables before they were there - an illegal address in 3 of 4 runs where + // a rotation pool built its FFT indexers at transform length 3072. Waiting here orders it. + if (cudaStreamSynchronize(nullptr) != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, + "Creating the CUFFT plan: setup on the NULL stream failed"); } // Convenience overload for vector input