From 65805cc3b2874d0191f2611e29c49d31c3e95d9c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 27 Sep 2026 11:40:55 +0200 Subject: [PATCH] BeamCenterFFTGPU: allocate synchronously, the work is on the NULL stream Every kernel, cuFFT plan and copy in the beam-centre FFT engine is queued on the legacy NULL stream, but its buffers came from the stream-ordered pool, whose cudaFreeAsync is ordered on the thread's non-blocking allocation stream - not after the NULL stream. The free therefore completed at once while PointShortlist's last Suppress (and, depending on what the plan teardown syncs, the last inverse transform) was still writing, and the pool was free to hand those bytes to another worker or return them to the driver. The capture runs in the background into the first pass of the main run, exactly when the spot engines are allocating their buffers, and on a loaded shared card its NULL-stream work lags. That is the "illegal memory access" seen 0.3-0.5 s after "Processing ... images" in four runs across three binaries and two datasets: a sticky error, reported by whichever worker synchronised next - the bslz4 device decode, whose host fallback cannot recover from it. compute-sanitizer --track-stream-ordered-races all over the [BeamCenterFFTGPU] tests: 16502 use-after-free reports before, 0 after; on a rugnux run it flagged the 294 MB transform buffers freed under a cuFFT kernel. myob output (hkl, mtz, P1 mtz, unmerged mtz) byte-identical. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- .../geom_refinement/BeamCenterFFTGPU.cu | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu index 93e0b77d1..98a7e8a88 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu +++ b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu @@ -22,6 +22,14 @@ constexpr int ACCUMULATE_THREADS = 128; // far above what the card needs to stay busy. constexpr int SHORTLIST_BLOCKS = 1024; +// Every buffer here is read by work on the legacy NULL stream - the kernels, the cuFFT plans and the +// copies below are all queued there - so none of them may come from the pool. A pooled buffer is +// freed with cudaFreeAsync on the thread's allocation stream, which is non-blocking and so not +// ordered after the NULL stream: the free completed at once while a Suppress or an inverse transform +// was still writing, and the pool handed those bytes to the analysis engines being built on other +// workers at the same moment, or unmapped them. cudaFree synchronises the device first. +constexpr CudaAlloc ALLOC = CudaAlloc::Synchronous; + int Blocks(int64_t n) { return static_cast((n + BLOCK - 1) / BLOCK); } void Check(cudaError_t err, const char *what) { @@ -238,8 +246,8 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto PointSurfacesDevice out; out.nx = nx; - out.real = CudaDevicePtr(nreal); - CudaDevicePtr d_a(ncomplex), d_m(ncomplex), d_prod(ncomplex); + out.real = CudaDevicePtr(nreal, ALLOC); + CudaDevicePtr d_a(ncomplex, ALLOC), d_m(ncomplex, ALLOC), d_prod(ncomplex, ALLOC); CudaFFTPlan fwd(2, Dims(ny, nx), Dims(ny, nx), 1, static_cast(nreal), Dims(ny, nxc), 1, static_cast(ncomplex), CUFFT_R2C, 1); @@ -267,7 +275,7 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto Check(cufftExecC2R(inv, d_prod, out.real), "BeamCenterFFT: inverse transform failed"); }; auto take_corner = [&](CudaDevicePtr &dst) { - dst = CudaDevicePtr(ncrop); + dst = CudaDevicePtr(ncrop, ALLOC); CopyScaledCorner<<>>(out.real.get(), nx, 2 * h, 2 * w, norm, dst.get()); CheckLastKernel("BeamCenterFFT: surface scaling failed"); @@ -294,8 +302,8 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto void ScorePoint(PointSurfacesDevice &surf, int64_t h, int64_t w, float min_pair_fraction, double global_variance) { const int64_t ncrop = 4 * h * w; - CudaDevicePtr block_value(SHORTLIST_BLOCKS); - CudaDevicePtr block_index(SHORTLIST_BLOCKS); + CudaDevicePtr block_value(SHORTLIST_BLOCKS, ALLOC); + CudaDevicePtr block_index(SHORTLIST_BLOCKS, ALLOC); BlockBest<<>>(surf.D.get(), ncrop, block_value.get(), block_index.get()); CheckLastKernel("BeamCenterFFT: pair-count maximum failed"); @@ -367,8 +375,8 @@ BeamCenterFFTGPU::PointShortlist(const std::vector &a, const std::vector< const int64_t surface_h = 2 * h, surface_w = 2 * w; const int64_t n = surface_h * surface_w; const int64_t d = std::lround(2.0f * settings.nms_radius_pxl); - CudaDevicePtr block_value(SHORTLIST_BLOCKS); - CudaDevicePtr block_index(SHORTLIST_BLOCKS); + CudaDevicePtr block_value(SHORTLIST_BLOCKS, ALLOC); + CudaDevicePtr block_index(SHORTLIST_BLOCKS, ALLOC); std::vector values(SHORTLIST_BLOCKS); std::vector indices(SHORTLIST_BLOCKS); @@ -416,10 +424,10 @@ BeamCenterConvSurfaces1D BeamCenterFFTGPU::LineSurfaces(const std::vector const int64_t nc = nfft / 2 + 1; BeamCenterConvSurfaces1D out; - CudaDevicePtr accC(nc), accS(nc), accQ(nc), accD(nc); + CudaDevicePtr accC(nc, ALLOC), accS(nc, ALLOC), accQ(nc, ALLOC), accD(nc, ALLOC); { - CudaDevicePtr d_img(h * w), d_in(nfft * nbatch); - CudaDevicePtr d_a(nbatch * nc), d_m(nbatch * nc), d_a2(nbatch * nc); + CudaDevicePtr d_img(h * w, ALLOC), d_in(nfft * nbatch, ALLOC); + CudaDevicePtr d_a(nbatch * nc, ALLOC), d_m(nbatch * nc, ALLOC), d_a2(nbatch * nc, ALLOC); CudaFFTPlan fwd(1, Dims(nfft), Dims(nfft), 1, static_cast(nfft), Dims(nc), 1, static_cast(nc), CUFFT_R2C, static_cast(nbatch)); @@ -447,8 +455,8 @@ BeamCenterConvSurfaces1D BeamCenterFFTGPU::LineSurfaces(const std::vector CheckLastKernel("BeamCenterFFT: spectrum accumulation failed"); } - CudaDevicePtr d_pack(4 * nc); - CudaDevicePtr d_out(4 * nfft); + CudaDevicePtr d_pack(4 * nc, ALLOC); + CudaDevicePtr d_out(4 * nfft, ALLOC); PackAccumulators<<>>(accC, accS, accQ, accD, nc, d_pack.get()); CheckLastKernel("BeamCenterFFT: accumulator packing failed"); CudaFFTPlan inv(1, Dims(nfft), Dims(nc), 1, static_cast(nc), Dims(nfft), 1,