Merge fftpeak exact commits into rc175: GPU peak search four threads per direction; CudaFFTPlan waits for cuFFT's NULL-stream plan setup (latent race)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
3f25bbf003
2 files changed
+80
-65
No files matched your search
@@ -128,6 +128,13 @@ public:
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
|
||||
GPUMemoryFailure("Creating the CUFFT plan (cufftPlanMany)", 0, true,
|
||||
("cuFFT error " + std::to_string(r)).c_str()));
|
||||
// The plan's setup can still be queued on the NULL stream when cufftPlanMany returns, and a
|
||||
// non-blocking stream (CudaStream) does not wait for it: a transform launched on one straight
|
||||
// away read the plan's tables before they were there - an illegal address in 3 of 4 runs where
|
||||
// a rotation pool built its FFT indexers at transform length 3072. Waiting here orders it.
|
||||
if (cudaStreamSynchronize(nullptr) != cudaSuccess)
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
|
||||
"Creating the CUFFT plan: setup on the NULL stream failed");
|
||||
}
|
||||
|
||||
// Convenience overload for vector input
|
||||
|
||||
@@ -40,6 +40,16 @@ __global__ void magnitude_transpose(const cufftComplex *__restrict__ d_output,
|
||||
}
|
||||
}
|
||||
|
||||
// The peak search splits each direction's spectrum into PEAK_CHUNKS consecutive runs of bins and gives each
|
||||
// run to its own thread, which keeps a running window sum over its run; one thread per direction leaves the
|
||||
// card nearly empty (16384 threads), and this search, a chain of dependent FP64 additions per thread, is
|
||||
// then latency-bound. A block holds 32 directions (x, so the bin-major loads stay coalesced) times PEAK_CHUNKS.
|
||||
// A run after the first sums its starting window afresh instead of carrying the sum in from the bins before
|
||||
// it. That is the same double wherever the sums are exact, and they are: a window of float magnitudes fits
|
||||
// a double's 53 bits unless one of them is below ~1e-9 of the window sum (no difference over 3000 stills
|
||||
// frames and a rotation-style synthetic set). More runs buy nothing: the card's FP64 rate is then the limit.
|
||||
constexpr int PEAK_CHUNKS = 4;
|
||||
|
||||
__global__ void calculate_fft_result(
|
||||
const float *__restrict__ mag_t,
|
||||
const float max_length_A,
|
||||
@@ -48,86 +58,83 @@ __global__ void calculate_fft_result(
|
||||
const int bg_half,
|
||||
const int directions_size,
|
||||
FFTResult *d_results) {
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x; // Get thread index
|
||||
__shared__ float run_prom[PEAK_CHUNKS][32];
|
||||
__shared__ float run_len[PEAK_CHUNKS][32];
|
||||
|
||||
const int i = blockIdx.x * 32 + threadIdx.x;
|
||||
const int run = threadIdx.y;
|
||||
float best_prom = 0.0f;
|
||||
float best_len = -1.0f;
|
||||
|
||||
if (i < directions_size) {
|
||||
const int out_len = (histogram_size / 2) + 1;
|
||||
// |transform| of bin k of this direction, from the bin-major array (magnitude_transpose)
|
||||
const auto mag_at = [&](int k) { return mag_t[static_cast<size_t>(k) * directions_size + i]; };
|
||||
float len_coeff = 2.0f * max_length_A / static_cast<float>(histogram_size);
|
||||
const float len_coeff = 2.0f * max_length_A / static_cast<float>(histogram_size);
|
||||
|
||||
// Pick the peak by PROMINENCE above a running-mean background of half-width bg_half:
|
||||
// the projected histogram has a broad low-frequency envelope whose magnitude can
|
||||
// exceed the true lattice peaks, so a plain argmax|spec| returns a short envelope
|
||||
// vector on weak/pink-beam frames. Subtracting the local mean removes that envelope
|
||||
// while sharp lattice peaks keep their height (mirrors FFTIndexerCPU).
|
||||
float best_prom = 0.0f;
|
||||
FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1};
|
||||
// bg is consider()'s background, the window mean - formed by whichever loop below applies.
|
||||
const auto consider = [&](int j, float bg) {
|
||||
const float len = len_coeff * static_cast<float>(j);
|
||||
if (len <= min_length_A) return;
|
||||
//
|
||||
// The window has a constant width, slid inward at the ends instead of truncated (see
|
||||
// FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - otherwise
|
||||
// gets a one-sided background and a biased prominence. A spectrum shorter than the window
|
||||
// averages over all of itself.
|
||||
const int width = min(2 * bg_half, out_len - 1);
|
||||
const int lo_max = out_len - 1 - width;
|
||||
const auto window_lo = [&](int j) { return min(max(j - bg_half, 0), lo_max); };
|
||||
|
||||
const float mag = mag_at(j);
|
||||
const float prom = mag - bg;
|
||||
// The window width is fixed, so the mean divides by one number throughout. A double division
|
||||
// is a long iteration on the FP64 units a consumer card has few of; multiplying by the correctly
|
||||
// rounded reciprocal and correcting the quotient once with its exact remainder gives the
|
||||
// correctly rounded quotient - the same double the division returns (Markstein (1990) IBM J.
|
||||
// Res. Dev. 34, 111-119).
|
||||
const double count = width + 1;
|
||||
const double inv_count = 1.0 / count;
|
||||
|
||||
const int j0 = (out_len * run) / PEAK_CHUNKS;
|
||||
const int j1 = (out_len * (run + 1)) / PEAK_CHUNKS;
|
||||
int lo = window_lo(j0);
|
||||
double winsum = 0.0;
|
||||
for (int k = lo; k <= lo + width; ++k)
|
||||
winsum += mag_at(k);
|
||||
|
||||
for (int j = j0; j < j1; ++j) {
|
||||
if (window_lo(j) > lo) {
|
||||
winsum += mag_at(lo + width + 1);
|
||||
winsum -= mag_at(lo);
|
||||
++lo;
|
||||
}
|
||||
const float len = len_coeff * static_cast<float>(j);
|
||||
if (len <= min_length_A)
|
||||
continue;
|
||||
const double q = __dmul_rn(winsum, inv_count);
|
||||
const float bg = static_cast<float>(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q));
|
||||
const float prom = mag_at(j) - bg;
|
||||
if (prom > best_prom) {
|
||||
best_prom = prom;
|
||||
result.magnitude = prom;
|
||||
result.length = len;
|
||||
}
|
||||
};
|
||||
|
||||
if (out_len - 1 >= 2 * bg_half) {
|
||||
// Where the spectrum holds a whole window, the window of the general loop below is
|
||||
// [0, 2 bg_half] until it can be centred, centred while it fits, and the last 2 bg_half + 1
|
||||
// bins after that: the running sum takes the same additions and subtractions in the same
|
||||
// order, so it is bit for bit the same number. Spelled out, the loads no longer wait on
|
||||
// data-dependent loops and the compiler can issue them ahead, which is what the search -
|
||||
// latency-bound, one dependent chain per direction - is short of.
|
||||
// The window width is fixed here, so the mean divides by one number throughout. A double
|
||||
// division is a long iteration on the FP64 units a consumer card has few of, and it was a
|
||||
// quarter of this kernel's time; multiplying by the correctly rounded reciprocal and correcting the
|
||||
// quotient once with its exact remainder gives the correctly rounded quotient - the same
|
||||
// double the division returns (Markstein (1990) IBM J. Res. Dev. 34, 111-119).
|
||||
const double count = 2 * bg_half + 1;
|
||||
const double inv_count = 1.0 / count;
|
||||
double winsum = 0.0;
|
||||
for (int k = 0; k <= 2 * bg_half; ++k)
|
||||
winsum += mag_at(k);
|
||||
#pragma unroll 4
|
||||
for (int j = 0; j < out_len; ++j) {
|
||||
if (j > bg_half && j + bg_half <= out_len - 1) {
|
||||
winsum += mag_at(j + bg_half);
|
||||
winsum -= mag_at(j - bg_half - 1);
|
||||
}
|
||||
const double q = __dmul_rn(winsum, inv_count);
|
||||
consider(j, static_cast<float>(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q)));
|
||||
}
|
||||
} else {
|
||||
double winsum = 0.0;
|
||||
int wlo = 0;
|
||||
int whi = min(bg_half, out_len - 1);
|
||||
for (int k = wlo; k <= whi; ++k)
|
||||
winsum += mag_at(k);
|
||||
|
||||
for (int j = 0; j < out_len; ++j) {
|
||||
// Constant-width window, slid inward at the ends instead of truncated (see
|
||||
// FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit -
|
||||
// otherwise gets a one-sided background and a biased prominence. Both bounds stay
|
||||
// monotonically non-decreasing in j, so the running sum below is still valid.
|
||||
int want_lo = j - bg_half;
|
||||
int want_hi = j + bg_half;
|
||||
if (want_lo < 0) { want_hi = min(out_len - 1, want_hi - want_lo); want_lo = 0; }
|
||||
if (want_hi > out_len - 1) {
|
||||
want_lo = max(0, want_lo - (want_hi - (out_len - 1)));
|
||||
want_hi = out_len - 1;
|
||||
}
|
||||
while (whi < want_hi) { ++whi; winsum += mag_at(whi); }
|
||||
while (wlo < want_lo) { winsum -= mag_at(wlo); ++wlo; }
|
||||
consider(j, static_cast<float>(winsum / static_cast<double>(whi - wlo + 1)));
|
||||
best_len = len;
|
||||
}
|
||||
}
|
||||
d_results[i] = result; // Store the result
|
||||
}
|
||||
|
||||
run_prom[run][threadIdx.x] = best_prom;
|
||||
run_len[run][threadIdx.x] = best_len;
|
||||
__syncthreads();
|
||||
|
||||
// Each run kept its first bin of highest prominence; taking the runs in order with the same strict
|
||||
// comparison gives the direction's first bin of highest prominence.
|
||||
if (run == 0 && i < directions_size) {
|
||||
FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1};
|
||||
for (int r = 0; r < PEAK_CHUNKS; ++r) {
|
||||
if (run_prom[r][threadIdx.x] > result.magnitude) {
|
||||
result.magnitude = run_prom[r][threadIdx.x];
|
||||
result.length = run_len[r][threadIdx.x];
|
||||
}
|
||||
}
|
||||
d_results[i] = result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,7 +315,8 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector<Coord> &coord, size_t nspots) {
|
||||
magnitude_transpose<<<tr_grid, dim3(32, 8), 0, stream>>>(d_output_fft, out_len,
|
||||
direction_vectors.size(), d_mag_t);
|
||||
cuda_err(cudaGetLastError());
|
||||
calculate_fft_result<<<l_gridDim, l_blockDim, 0, stream>>>(d_mag_t,
|
||||
const int peak_grid = (static_cast<int>(direction_vectors.size()) + 31) / 32;
|
||||
calculate_fft_result<<<peak_grid, dim3(32, PEAK_CHUNKS), 0, stream>>>(d_mag_t,
|
||||
max_length_A, min_length_A, histogram_size,
|
||||
bg_half,
|
||||
direction_vectors.size(), d_result_fft);
|
||||
|
||||
Reference in new issue
Block a user