Merge fftpeak exact commits into rc175: GPU peak search four threads per direction; CudaFFTPlan waits for cuFFT's NULL-stream plan setup (latent race)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-14 17:37:54 +02:00
commit 3f25bbf003
2 files changed
+80 -65

No files matched your search

+7
View File
@@ -128,6 +128,13 @@ public:
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
GPUMemoryFailure("Creating the CUFFT plan (cufftPlanMany)", 0, true,
("cuFFT error " + std::to_string(r)).c_str()));
// The plan's setup can still be queued on the NULL stream when cufftPlanMany returns, and a
// non-blocking stream (CudaStream) does not wait for it: a transform launched on one straight
// away read the plan's tables before they were there - an illegal address in 3 of 4 runs where
// a rotation pool built its FFT indexers at transform length 3072. Waiting here orders it.
if (cudaStreamSynchronize(nullptr) != cudaSuccess)
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
"Creating the CUFFT plan: setup on the NULL stream failed");
}
// Convenience overload for vector input
+73 -65
View File
@@ -40,6 +40,16 @@ __global__ void magnitude_transpose(const cufftComplex *__restrict__ d_output,
}
}
// The peak search splits each direction's spectrum into PEAK_CHUNKS consecutive runs of bins and gives each
// run to its own thread, which keeps a running window sum over its run; one thread per direction leaves the
// card nearly empty (16384 threads), and this search, a chain of dependent FP64 additions per thread, is
// then latency-bound. A block holds 32 directions (x, so the bin-major loads stay coalesced) times PEAK_CHUNKS.
// A run after the first sums its starting window afresh instead of carrying the sum in from the bins before
// it. That is the same double wherever the sums are exact, and they are: a window of float magnitudes fits
// a double's 53 bits unless one of them is below ~1e-9 of the window sum (no difference over 3000 stills
// frames and a rotation-style synthetic set). More runs buy nothing: the card's FP64 rate is then the limit.
constexpr int PEAK_CHUNKS = 4;
__global__ void calculate_fft_result(
const float *__restrict__ mag_t,
const float max_length_A,
@@ -48,86 +58,83 @@ __global__ void calculate_fft_result(
const int bg_half,
const int directions_size,
FFTResult *d_results) {
int i = blockIdx.x * blockDim.x + threadIdx.x; // Get thread index
__shared__ float run_prom[PEAK_CHUNKS][32];
__shared__ float run_len[PEAK_CHUNKS][32];
const int i = blockIdx.x * 32 + threadIdx.x;
const int run = threadIdx.y;
float best_prom = 0.0f;
float best_len = -1.0f;
if (i < directions_size) {
const int out_len = (histogram_size / 2) + 1;
// |transform| of bin k of this direction, from the bin-major array (magnitude_transpose)
const auto mag_at = [&](int k) { return mag_t[static_cast<size_t>(k) * directions_size + i]; };
float len_coeff = 2.0f * max_length_A / static_cast<float>(histogram_size);
const float len_coeff = 2.0f * max_length_A / static_cast<float>(histogram_size);
// Pick the peak by PROMINENCE above a running-mean background of half-width bg_half:
// the projected histogram has a broad low-frequency envelope whose magnitude can
// exceed the true lattice peaks, so a plain argmax|spec| returns a short envelope
// vector on weak/pink-beam frames. Subtracting the local mean removes that envelope
// while sharp lattice peaks keep their height (mirrors FFTIndexerCPU).
float best_prom = 0.0f;
FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1};
// bg is consider()'s background, the window mean - formed by whichever loop below applies.
const auto consider = [&](int j, float bg) {
const float len = len_coeff * static_cast<float>(j);
if (len <= min_length_A) return;
//
// The window has a constant width, slid inward at the ends instead of truncated (see
// FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - otherwise
// gets a one-sided background and a biased prominence. A spectrum shorter than the window
// averages over all of itself.
const int width = min(2 * bg_half, out_len - 1);
const int lo_max = out_len - 1 - width;
const auto window_lo = [&](int j) { return min(max(j - bg_half, 0), lo_max); };
const float mag = mag_at(j);
const float prom = mag - bg;
// The window width is fixed, so the mean divides by one number throughout. A double division
// is a long iteration on the FP64 units a consumer card has few of; multiplying by the correctly
// rounded reciprocal and correcting the quotient once with its exact remainder gives the
// correctly rounded quotient - the same double the division returns (Markstein (1990) IBM J.
// Res. Dev. 34, 111-119).
const double count = width + 1;
const double inv_count = 1.0 / count;
const int j0 = (out_len * run) / PEAK_CHUNKS;
const int j1 = (out_len * (run + 1)) / PEAK_CHUNKS;
int lo = window_lo(j0);
double winsum = 0.0;
for (int k = lo; k <= lo + width; ++k)
winsum += mag_at(k);
for (int j = j0; j < j1; ++j) {
if (window_lo(j) > lo) {
winsum += mag_at(lo + width + 1);
winsum -= mag_at(lo);
++lo;
}
const float len = len_coeff * static_cast<float>(j);
if (len <= min_length_A)
continue;
const double q = __dmul_rn(winsum, inv_count);
const float bg = static_cast<float>(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q));
const float prom = mag_at(j) - bg;
if (prom > best_prom) {
best_prom = prom;
result.magnitude = prom;
result.length = len;
}
};
if (out_len - 1 >= 2 * bg_half) {
// Where the spectrum holds a whole window, the window of the general loop below is
// [0, 2 bg_half] until it can be centred, centred while it fits, and the last 2 bg_half + 1
// bins after that: the running sum takes the same additions and subtractions in the same
// order, so it is bit for bit the same number. Spelled out, the loads no longer wait on
// data-dependent loops and the compiler can issue them ahead, which is what the search -
// latency-bound, one dependent chain per direction - is short of.
// The window width is fixed here, so the mean divides by one number throughout. A double
// division is a long iteration on the FP64 units a consumer card has few of, and it was a
// quarter of this kernel's time; multiplying by the correctly rounded reciprocal and correcting the
// quotient once with its exact remainder gives the correctly rounded quotient - the same
// double the division returns (Markstein (1990) IBM J. Res. Dev. 34, 111-119).
const double count = 2 * bg_half + 1;
const double inv_count = 1.0 / count;
double winsum = 0.0;
for (int k = 0; k <= 2 * bg_half; ++k)
winsum += mag_at(k);
#pragma unroll 4
for (int j = 0; j < out_len; ++j) {
if (j > bg_half && j + bg_half <= out_len - 1) {
winsum += mag_at(j + bg_half);
winsum -= mag_at(j - bg_half - 1);
}
const double q = __dmul_rn(winsum, inv_count);
consider(j, static_cast<float>(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q)));
}
} else {
double winsum = 0.0;
int wlo = 0;
int whi = min(bg_half, out_len - 1);
for (int k = wlo; k <= whi; ++k)
winsum += mag_at(k);
for (int j = 0; j < out_len; ++j) {
// Constant-width window, slid inward at the ends instead of truncated (see
// FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit -
// otherwise gets a one-sided background and a biased prominence. Both bounds stay
// monotonically non-decreasing in j, so the running sum below is still valid.
int want_lo = j - bg_half;
int want_hi = j + bg_half;
if (want_lo < 0) { want_hi = min(out_len - 1, want_hi - want_lo); want_lo = 0; }
if (want_hi > out_len - 1) {
want_lo = max(0, want_lo - (want_hi - (out_len - 1)));
want_hi = out_len - 1;
}
while (whi < want_hi) { ++whi; winsum += mag_at(whi); }
while (wlo < want_lo) { winsum -= mag_at(wlo); ++wlo; }
consider(j, static_cast<float>(winsum / static_cast<double>(whi - wlo + 1)));
best_len = len;
}
}
d_results[i] = result; // Store the result
}
run_prom[run][threadIdx.x] = best_prom;
run_len[run][threadIdx.x] = best_len;
__syncthreads();
// Each run kept its first bin of highest prominence; taking the runs in order with the same strict
// comparison gives the direction's first bin of highest prominence.
if (run == 0 && i < directions_size) {
FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1};
for (int r = 0; r < PEAK_CHUNKS; ++r) {
if (run_prom[r][threadIdx.x] > result.magnitude) {
result.magnitude = run_prom[r][threadIdx.x];
result.length = run_len[r][threadIdx.x];
}
}
d_results[i] = result;
}
}
@@ -308,7 +315,8 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
magnitude_transpose<<<tr_grid, dim3(32, 8), 0, stream>>>(d_output_fft, out_len,
direction_vectors.size(), d_mag_t);
cuda_err(cudaGetLastError());
calculate_fft_result<<<l_gridDim, l_blockDim, 0, stream>>>(d_mag_t,
const int peak_grid = (static_cast<int>(direction_vectors.size()) + 31) / 32;
calculate_fft_result<<<peak_grid, dim3(32, PEAK_CHUNKS), 0, stream>>>(d_mag_t,
max_length_A, min_length_A, histogram_size,
bg_half,
direction_vectors.size(), d_result_fft);