diff --git a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu index 93e0b77d1..98a7e8a88 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu +++ b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu @@ -22,6 +22,14 @@ constexpr int ACCUMULATE_THREADS = 128; // far above what the card needs to stay busy. constexpr int SHORTLIST_BLOCKS = 1024; +// Every buffer here is read by work on the legacy NULL stream - the kernels, the cuFFT plans and the +// copies below are all queued there - so none of them may come from the pool. A pooled buffer is +// freed with cudaFreeAsync on the thread's allocation stream, which is non-blocking and so not +// ordered after the NULL stream: the free completed at once while a Suppress or an inverse transform +// was still writing, and the pool handed those bytes to the analysis engines being built on other +// workers at the same moment, or unmapped them. cudaFree synchronises the device first. +constexpr CudaAlloc ALLOC = CudaAlloc::Synchronous; + int Blocks(int64_t n) { return static_cast((n + BLOCK - 1) / BLOCK); } void Check(cudaError_t err, const char *what) { @@ -238,8 +246,8 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto PointSurfacesDevice out; out.nx = nx; - out.real = CudaDevicePtr(nreal); - CudaDevicePtr d_a(ncomplex), d_m(ncomplex), d_prod(ncomplex); + out.real = CudaDevicePtr(nreal, ALLOC); + CudaDevicePtr d_a(ncomplex, ALLOC), d_m(ncomplex, ALLOC), d_prod(ncomplex, ALLOC); CudaFFTPlan fwd(2, Dims(ny, nx), Dims(ny, nx), 1, static_cast(nreal), Dims(ny, nxc), 1, static_cast(ncomplex), CUFFT_R2C, 1); @@ -267,7 +275,7 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto Check(cufftExecC2R(inv, d_prod, out.real), "BeamCenterFFT: inverse transform failed"); }; auto take_corner = [&](CudaDevicePtr &dst) { - dst = CudaDevicePtr(ncrop); + dst = CudaDevicePtr(ncrop, ALLOC); CopyScaledCorner<<>>(out.real.get(), nx, 2 * h, 2 * w, norm, dst.get()); CheckLastKernel("BeamCenterFFT: surface scaling failed"); @@ -294,8 +302,8 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto void ScorePoint(PointSurfacesDevice &surf, int64_t h, int64_t w, float min_pair_fraction, double global_variance) { const int64_t ncrop = 4 * h * w; - CudaDevicePtr block_value(SHORTLIST_BLOCKS); - CudaDevicePtr block_index(SHORTLIST_BLOCKS); + CudaDevicePtr block_value(SHORTLIST_BLOCKS, ALLOC); + CudaDevicePtr block_index(SHORTLIST_BLOCKS, ALLOC); BlockBest<<>>(surf.D.get(), ncrop, block_value.get(), block_index.get()); CheckLastKernel("BeamCenterFFT: pair-count maximum failed"); @@ -367,8 +375,8 @@ BeamCenterFFTGPU::PointShortlist(const std::vector &a, const std::vector< const int64_t surface_h = 2 * h, surface_w = 2 * w; const int64_t n = surface_h * surface_w; const int64_t d = std::lround(2.0f * settings.nms_radius_pxl); - CudaDevicePtr block_value(SHORTLIST_BLOCKS); - CudaDevicePtr block_index(SHORTLIST_BLOCKS); + CudaDevicePtr block_value(SHORTLIST_BLOCKS, ALLOC); + CudaDevicePtr block_index(SHORTLIST_BLOCKS, ALLOC); std::vector values(SHORTLIST_BLOCKS); std::vector indices(SHORTLIST_BLOCKS); @@ -416,10 +424,10 @@ BeamCenterConvSurfaces1D BeamCenterFFTGPU::LineSurfaces(const std::vector const int64_t nc = nfft / 2 + 1; BeamCenterConvSurfaces1D out; - CudaDevicePtr accC(nc), accS(nc), accQ(nc), accD(nc); + CudaDevicePtr accC(nc, ALLOC), accS(nc, ALLOC), accQ(nc, ALLOC), accD(nc, ALLOC); { - CudaDevicePtr d_img(h * w), d_in(nfft * nbatch); - CudaDevicePtr d_a(nbatch * nc), d_m(nbatch * nc), d_a2(nbatch * nc); + CudaDevicePtr d_img(h * w, ALLOC), d_in(nfft * nbatch, ALLOC); + CudaDevicePtr d_a(nbatch * nc, ALLOC), d_m(nbatch * nc, ALLOC), d_a2(nbatch * nc, ALLOC); CudaFFTPlan fwd(1, Dims(nfft), Dims(nfft), 1, static_cast(nfft), Dims(nc), 1, static_cast(nc), CUFFT_R2C, static_cast(nbatch)); @@ -447,8 +455,8 @@ BeamCenterConvSurfaces1D BeamCenterFFTGPU::LineSurfaces(const std::vector CheckLastKernel("BeamCenterFFT: spectrum accumulation failed"); } - CudaDevicePtr d_pack(4 * nc); - CudaDevicePtr d_out(4 * nfft); + CudaDevicePtr d_pack(4 * nc, ALLOC); + CudaDevicePtr d_out(4 * nfft, ALLOC); PackAccumulators<<>>(accC, accS, accQ, accD, nc, d_pack.get()); CheckLastKernel("BeamCenterFFT: accumulator packing failed"); CudaFFTPlan inv(1, Dims(nfft), Dims(nc), 1, static_cast(nc), Dims(nfft), 1,