diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 0aa741556..6d2db7ad0 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -128,6 +128,13 @@ public: throw JFJochException(JFJochExceptionCategory::GPUCUDAError, GPUMemoryFailure("Creating the CUFFT plan (cufftPlanMany)", 0, true, ("cuFFT error " + std::to_string(r)).c_str())); + // The plan's setup can still be queued on the NULL stream when cufftPlanMany returns, and a + // non-blocking stream (CudaStream) does not wait for it: a transform launched on one straight + // away read the plan's tables before they were there - an illegal address in 3 of 4 runs where + // a rotation pool built its FFT indexers at transform length 3072. Waiting here orders it. + if (cudaStreamSynchronize(nullptr) != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, + "Creating the CUFFT plan: setup on the NULL stream failed"); } // Convenience overload for vector input diff --git a/image_analysis/indexing/FFTIndexerGPU.cu b/image_analysis/indexing/FFTIndexerGPU.cu index 99ecbd4d2..418b06f00 100644 --- a/image_analysis/indexing/FFTIndexerGPU.cu +++ b/image_analysis/indexing/FFTIndexerGPU.cu @@ -40,6 +40,16 @@ __global__ void magnitude_transpose(const cufftComplex *__restrict__ d_output, } } +// The peak search splits each direction's spectrum into PEAK_CHUNKS consecutive runs of bins and gives each +// run to its own thread, which keeps a running window sum over its run; one thread per direction leaves the +// card nearly empty (16384 threads), and this search, a chain of dependent FP64 additions per thread, is +// then latency-bound. A block holds 32 directions (x, so the bin-major loads stay coalesced) times PEAK_CHUNKS. +// A run after the first sums its starting window afresh instead of carrying the sum in from the bins before +// it. That is the same double wherever the sums are exact, and they are: a window of float magnitudes fits +// a double's 53 bits unless one of them is below ~1e-9 of the window sum (no difference over 3000 stills +// frames and a rotation-style synthetic set). More runs buy nothing: the card's FP64 rate is then the limit. +constexpr int PEAK_CHUNKS = 4; + __global__ void calculate_fft_result( const float *__restrict__ mag_t, const float max_length_A, @@ -48,86 +58,83 @@ __global__ void calculate_fft_result( const int bg_half, const int directions_size, FFTResult *d_results) { - int i = blockIdx.x * blockDim.x + threadIdx.x; // Get thread index + __shared__ float run_prom[PEAK_CHUNKS][32]; + __shared__ float run_len[PEAK_CHUNKS][32]; + + const int i = blockIdx.x * 32 + threadIdx.x; + const int run = threadIdx.y; + float best_prom = 0.0f; + float best_len = -1.0f; if (i < directions_size) { const int out_len = (histogram_size / 2) + 1; // |transform| of bin k of this direction, from the bin-major array (magnitude_transpose) const auto mag_at = [&](int k) { return mag_t[static_cast(k) * directions_size + i]; }; - float len_coeff = 2.0f * max_length_A / static_cast(histogram_size); + const float len_coeff = 2.0f * max_length_A / static_cast(histogram_size); // Pick the peak by PROMINENCE above a running-mean background of half-width bg_half: // the projected histogram has a broad low-frequency envelope whose magnitude can // exceed the true lattice peaks, so a plain argmax|spec| returns a short envelope // vector on weak/pink-beam frames. Subtracting the local mean removes that envelope // while sharp lattice peaks keep their height (mirrors FFTIndexerCPU). - float best_prom = 0.0f; - FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1}; - // bg is consider()'s background, the window mean - formed by whichever loop below applies. - const auto consider = [&](int j, float bg) { - const float len = len_coeff * static_cast(j); - if (len <= min_length_A) return; + // + // The window has a constant width, slid inward at the ends instead of truncated (see + // FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - otherwise + // gets a one-sided background and a biased prominence. A spectrum shorter than the window + // averages over all of itself. + const int width = min(2 * bg_half, out_len - 1); + const int lo_max = out_len - 1 - width; + const auto window_lo = [&](int j) { return min(max(j - bg_half, 0), lo_max); }; - const float mag = mag_at(j); - const float prom = mag - bg; + // The window width is fixed, so the mean divides by one number throughout. A double division + // is a long iteration on the FP64 units a consumer card has few of; multiplying by the correctly + // rounded reciprocal and correcting the quotient once with its exact remainder gives the + // correctly rounded quotient - the same double the division returns (Markstein (1990) IBM J. + // Res. Dev. 34, 111-119). + const double count = width + 1; + const double inv_count = 1.0 / count; + + const int j0 = (out_len * run) / PEAK_CHUNKS; + const int j1 = (out_len * (run + 1)) / PEAK_CHUNKS; + int lo = window_lo(j0); + double winsum = 0.0; + for (int k = lo; k <= lo + width; ++k) + winsum += mag_at(k); + + for (int j = j0; j < j1; ++j) { + if (window_lo(j) > lo) { + winsum += mag_at(lo + width + 1); + winsum -= mag_at(lo); + ++lo; + } + const float len = len_coeff * static_cast(j); + if (len <= min_length_A) + continue; + const double q = __dmul_rn(winsum, inv_count); + const float bg = static_cast(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q)); + const float prom = mag_at(j) - bg; if (prom > best_prom) { best_prom = prom; - result.magnitude = prom; - result.length = len; - } - }; - - if (out_len - 1 >= 2 * bg_half) { - // Where the spectrum holds a whole window, the window of the general loop below is - // [0, 2 bg_half] until it can be centred, centred while it fits, and the last 2 bg_half + 1 - // bins after that: the running sum takes the same additions and subtractions in the same - // order, so it is bit for bit the same number. Spelled out, the loads no longer wait on - // data-dependent loops and the compiler can issue them ahead, which is what the search - - // latency-bound, one dependent chain per direction - is short of. - // The window width is fixed here, so the mean divides by one number throughout. A double - // division is a long iteration on the FP64 units a consumer card has few of, and it was a - // quarter of this kernel's time; multiplying by the correctly rounded reciprocal and correcting the - // quotient once with its exact remainder gives the correctly rounded quotient - the same - // double the division returns (Markstein (1990) IBM J. Res. Dev. 34, 111-119). - const double count = 2 * bg_half + 1; - const double inv_count = 1.0 / count; - double winsum = 0.0; - for (int k = 0; k <= 2 * bg_half; ++k) - winsum += mag_at(k); -#pragma unroll 4 - for (int j = 0; j < out_len; ++j) { - if (j > bg_half && j + bg_half <= out_len - 1) { - winsum += mag_at(j + bg_half); - winsum -= mag_at(j - bg_half - 1); - } - const double q = __dmul_rn(winsum, inv_count); - consider(j, static_cast(__fma_rn(__fma_rn(-q, count, winsum), inv_count, q))); - } - } else { - double winsum = 0.0; - int wlo = 0; - int whi = min(bg_half, out_len - 1); - for (int k = wlo; k <= whi; ++k) - winsum += mag_at(k); - - for (int j = 0; j < out_len; ++j) { - // Constant-width window, slid inward at the ends instead of truncated (see - // FFTIndexerCPU): a peak within bg_half of either end - where the LONGEST cells sit - - // otherwise gets a one-sided background and a biased prominence. Both bounds stay - // monotonically non-decreasing in j, so the running sum below is still valid. - int want_lo = j - bg_half; - int want_hi = j + bg_half; - if (want_lo < 0) { want_hi = min(out_len - 1, want_hi - want_lo); want_lo = 0; } - if (want_hi > out_len - 1) { - want_lo = max(0, want_lo - (want_hi - (out_len - 1))); - want_hi = out_len - 1; - } - while (whi < want_hi) { ++whi; winsum += mag_at(whi); } - while (wlo < want_lo) { winsum -= mag_at(wlo); ++wlo; } - consider(j, static_cast(winsum / static_cast(whi - wlo + 1))); + best_len = len; } } - d_results[i] = result; // Store the result + } + + run_prom[run][threadIdx.x] = best_prom; + run_len[run][threadIdx.x] = best_len; + __syncthreads(); + + // Each run kept its first bin of highest prominence; taking the runs in order with the same strict + // comparison gives the direction's first bin of highest prominence. + if (run == 0 && i < directions_size) { + FFTResult result{.magnitude = 0.0f, .direction = i, .length = -1}; + for (int r = 0; r < PEAK_CHUNKS; ++r) { + if (run_prom[r][threadIdx.x] > result.magnitude) { + result.magnitude = run_prom[r][threadIdx.x]; + result.length = run_len[r][threadIdx.x]; + } + } + d_results[i] = result; } } @@ -308,7 +315,8 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector &coord, size_t nspots) { magnitude_transpose<<>>(d_output_fft, out_len, direction_vectors.size(), d_mag_t); cuda_err(cudaGetLastError()); - calculate_fft_result<<>>(d_mag_t, + const int peak_grid = (static_cast(direction_vectors.size()) + 31) / 32; + calculate_fft_result<<>>(d_mag_t, max_length_A, min_length_A, histogram_size, bg_half, direction_vectors.size(), d_result_fft);