diff --git a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu index 3b49d6eae..bdf7ba5a4 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu +++ b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu @@ -210,7 +210,7 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto auto inverse_product = [&](const cufftComplex *u, const cufftComplex *v, cufftComplex *dst) { MultiplySpectra<<>>(u, v, dst, ncomplex); CheckLastKernel("BeamCenterFFT: spectrum product failed"); - Check(cufftExecC2R(inv, dst, out.real), "BeamCenterFFT: inverse transform failed"); + Check(inv.C2R(dst, out.real), "BeamCenterFFT: inverse transform failed"); }; auto take_corner = [&](CudaDevicePtr &dst) { dst = CudaDevicePtr(ncrop, ALLOC); @@ -222,11 +222,11 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto // a.a goes where m's spectrum will be, before it is there; a.m in the place of a's spectrum, the last // to read it; m.m there too; and the spectrum of a^2 there after that. upload(a); - Check(cufftExecR2C(fwd, out.real, d_a), "BeamCenterFFT: forward transform failed"); + Check(fwd.R2C(out.real, d_a), "BeamCenterFFT: forward transform failed"); inverse_product(d_a, d_a, d_m); take_corner(out.C); upload(m); - Check(cufftExecR2C(fwd, out.real, d_m), "BeamCenterFFT: forward transform failed"); + Check(fwd.R2C(out.real, d_m), "BeamCenterFFT: forward transform failed"); inverse_product(d_a, d_m, d_a); take_corner(out.S); inverse_product(d_m, d_m, d_a); @@ -234,7 +234,7 @@ PointSurfacesDevice TransformPoint(const std::vector &a, const std::vecto upload(a); SquareInPlace<<>>(out.real.get(), nreal); CheckLastKernel("BeamCenterFFT: squaring failed"); - Check(cufftExecR2C(fwd, out.real, d_a), "BeamCenterFFT: forward transform failed"); + Check(fwd.R2C(out.real, d_a), "BeamCenterFFT: forward transform failed"); inverse_product(d_a, d_m, d_a); ScaleCorner<<>>(out.real.get(), nx, 2 * h, 2 * w, norm); CheckLastKernel("BeamCenterFFT: surface scaling failed"); diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 91b052439..fcbe55d72 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -348,33 +349,45 @@ struct CudaFFTShape { int batch = 1; }; -// While a CudaFFTPlanScope is alive, cuFFT plans are kept when their user is done with them, by device -// and shape, and lent to the next user of that shape; when the last scope ends they are destroyed, and -// outside any scope a plan is destroyed as soon as its user is done. Making a plan is costly and -// serialised: cuFFT holds a process-wide lock while it loads the plan's kernel modules (each plan loads -// its own, two plans of one shape included), which took over a second a plan on a busy card. The modules -// are PTX on every platform, compiled by the driver - ~1 ms from its JIT cache, 60-130 ms without one -// (CudaJitCache.h); and it frees and loads them all again for the next plan of the same shape. A plan is never shared by two users at once - cuFFT is thread safe only across distinct -// plans. +// A cuFFT plan is made once per device and shape and shared by all its users at once: each user +// transforms on its own stream with its own work area (CudaFFTPlan), and a transform sets both and +// launches under the plan's lock, since cuFFT is thread safe only across distinct plans. While a +// CudaFFTPlanScope is alive a plan is kept when its last user is done, for the next user of that shape; +// when the last scope ends they are destroyed, and outside any scope a plan is destroyed as soon as its +// last user is done. +// +// Making a plan is costly and serialised: cuFFT holds a process-wide lock while it loads the plan's kernel +// modules, and it ships those as PTX, compiled for the card at every plan made - two plans of one shape +// included. The CUDA JIT cache makes that ~1 ms a plan; without one (CUDA_CACHE_DISABLE, no writable +// $HOME; see CudaJitCache.h) it is 20-250 ms a plan whatever the size, on every card and every cuFFT version tried; on a +// busy card it took over a second. So the number of plans made is what counts, not their size. // // In `rugnux` the scope is one processing run, so nothing is kept from one run for the next, nor past the -// device reset at the end of a run; jfjoch_broker, which resets no device, keeps one for its lifetime. A kept plan holds no work area: the user supplies one (CudaFFTPlan), -// so an idle plan holds only its tables and modules. +// device reset at the end of a run; jfjoch_broker, which resets no device, keeps one for its lifetime. A +// plan holds no work area: each user supplies one, so a kept plan holds only its tables and modules. namespace jfjoch_cufft_plans { struct Plan { cufftHandle handle = 0; size_t work_bytes = 0; }; - struct Idle { + // A plan and its users; its handle is used only under `exec`. + struct Shared { + Plan plan; + std::mutex exec; + int users = 0; + }; + + struct Kept { std::mutex m; - std::map, std::vector> plans; // by key() + std::mutex make; // one plan made at a time, so that two users of a new shape make it once + std::map, std::unique_ptr> plans; // by key() int scopes = 0; }; - inline Idle &idle() { - static Idle *i = new Idle; - return *i; + inline Kept &kept_plans() { + static Kept *k = new Kept; + return *k; } // The current device and the shape. @@ -422,18 +435,27 @@ namespace jfjoch_cufft_plans { return p; } - // An idle plan of this key, or a new one. - inline Plan borrow(const std::vector &k, const CudaFFTShape &s) { - { - std::lock_guard lock(idle().m); - auto it = idle().plans.find(k); - if (it != idle().plans.end() && !it->second.empty()) { - const Plan p = it->second.back(); - it->second.pop_back(); - return p; - } - } - return make(s); + // The plan of this key, made if there is none, with one more user. + inline Shared *borrow(const std::vector &k, const CudaFFTShape &s) { + Kept &kp = kept_plans(); + auto find = [&]() -> Shared * { + std::lock_guard lock(kp.m); + auto it = kp.plans.find(k); + if (it == kp.plans.end()) + return nullptr; + it->second->users++; + return it->second.get(); + }; + if (Shared *p = find()) + return p; + std::lock_guard make_lock(kp.make); + if (Shared *p = find()) + return p; + auto p = std::make_unique(); + p->plan = make(s); + p->users = 1; + std::lock_guard lock(kp.m); + return kp.plans.emplace(k, std::move(p)).first->second.get(); } // On the plan's own device (the first element of its key). @@ -445,24 +467,24 @@ namespace jfjoch_cufft_plans { cudaSetDevice(current); } - inline void give_back(const std::vector &k, const Plan &p) { + inline void give_back(const std::vector &k) { + Plan p; { - std::lock_guard lock(idle().m); - if (idle().scopes > 0) { - idle().plans[k].push_back(p); + Kept &kp = kept_plans(); + std::lock_guard lock(kp.m); + auto it = kp.plans.find(k); + if (--it->second->users > 0 || kp.scopes > 0) return; - } + p = it->second->plan; + kp.plans.erase(it); } destroy(k, p); } - // How many plans are kept, idle. + // How many plans there are, in use or kept. inline size_t kept() { - std::lock_guard lock(idle().m); - size_t n = 0; - for (const auto &[k, plans] : idle().plans) - n += plans.size(); - return n; + std::lock_guard lock(kept_plans().m); + return kept_plans().plans.size(); } } @@ -470,68 +492,93 @@ namespace jfjoch_cufft_plans { class CudaFFTPlanScope { public: CudaFFTPlanScope() { - std::lock_guard lock(jfjoch_cufft_plans::idle().m); - jfjoch_cufft_plans::idle().scopes++; + std::lock_guard lock(jfjoch_cufft_plans::kept_plans().m); + jfjoch_cufft_plans::kept_plans().scopes++; } + // The plans nobody uses go; one still in use goes when its last user is done. ~CudaFFTPlanScope() { - std::map, std::vector> plans; + std::vector, jfjoch_cufft_plans::Plan>> unused; { - std::lock_guard lock(jfjoch_cufft_plans::idle().m); - if (--jfjoch_cufft_plans::idle().scopes == 0) - plans.swap(jfjoch_cufft_plans::idle().plans); + jfjoch_cufft_plans::Kept &kp = jfjoch_cufft_plans::kept_plans(); + std::lock_guard lock(kp.m); + if (--kp.scopes == 0) + for (auto it = kp.plans.begin(); it != kp.plans.end();) { + if (it->second->users == 0) { + unused.emplace_back(it->first, it->second->plan); + it = kp.plans.erase(it); + } else + ++it; + } } - for (const auto &[k, list] : plans) - for (const auto &p : list) - jfjoch_cufft_plans::destroy(k, p); + for (const auto &[k, p] : unused) + jfjoch_cufft_plans::destroy(k, p); } CudaFFTPlanScope(const CudaFFTPlanScope &) = delete; CudaFFTPlanScope &operator=(const CudaFFTPlanScope &) = delete; }; -// A cuFFT plan of `shape` on the current device, borrowed from the plans kept above (or made) for as long -// as this lives, its transforms on `stream`. Its work area is `work_area` (at least WorkBytes() long), or, given -// none, one of its own. The plan goes back once the stream has finished with it. +// The cuFFT plan of `shape` on the current device (see above), used for as long as this lives, its +// transforms on `stream`. Its work area is `work_area` (at least WorkBytes() long), or, given none, one of +// its own. class CudaFFTPlan { std::vector key_; - jfjoch_cufft_plans::Plan plan_; + jfjoch_cufft_plans::Shared *plan_ = nullptr; cudaStream_t stream_ = nullptr; + void *work_area_ = nullptr; CudaDevicePtr work_; void release() { - if (!plan_.handle) + if (!plan_) return; cudaStreamSynchronize(stream_); work_ = CudaDevicePtr(); - jfjoch_cufft_plans::give_back(key_, plan_); - plan_ = {}; + jfjoch_cufft_plans::give_back(key_); + plan_ = nullptr; + } + + // The plan set to this user's work area and stream, under its lock for the transform `f` launches. + template + cufftResult Exec(F f) const { + std::lock_guard lock(plan_->exec); + cufftResult r = cufftSetStream(plan_->plan.handle, stream_); + if (r == CUFFT_SUCCESS) + r = cufftSetWorkArea(plan_->plan.handle, work_area_); + if (r == CUFFT_SUCCESS) + r = f(plan_->plan.handle); + return r; } public: CudaFFTPlan() = default; explicit CudaFFTPlan(const CudaFFTShape &shape, cudaStream_t stream = nullptr, void *work_area = nullptr) - : key_(jfjoch_cufft_plans::key(shape)), plan_(jfjoch_cufft_plans::borrow(key_, shape)), stream_(stream) { + : key_(jfjoch_cufft_plans::key(shape)), plan_(jfjoch_cufft_plans::borrow(key_, shape)), stream_(stream), + work_area_(work_area) { try { - if (work_area == nullptr) { - work_ = CudaDevicePtr(std::max(plan_.work_bytes, 1)); - work_area = work_; + if (work_area_ == nullptr) { + work_ = CudaDevicePtr(std::max(plan_->plan.work_bytes, 1)); + work_area_ = work_; } - if (cufftSetWorkArea(plan_.handle, work_area) != CUFFT_SUCCESS - || cufftSetStream(plan_.handle, stream_) != CUFFT_SUCCESS) - throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Setting up the CUFFT plan failed"); } catch (...) { release(); throw; } } - // The work area a plan of `shape` on the current device needs - read from the kept plan, which is - // made here if there is none, so that within a scope the plan asked is the plan used. + // The work area a plan of `shape` on the current device needs - read from the plan, which is made here + // if there is none, so that within a scope the plan asked is the plan used. static size_t WorkBytes(const CudaFFTShape &shape) { const std::vector k = jfjoch_cufft_plans::key(shape); - const jfjoch_cufft_plans::Plan p = jfjoch_cufft_plans::borrow(k, shape); - jfjoch_cufft_plans::give_back(k, p); - return p.work_bytes; + const size_t bytes = jfjoch_cufft_plans::borrow(k, shape)->plan.work_bytes; + jfjoch_cufft_plans::give_back(k); + return bytes; + } + + cufftResult R2C(cufftReal *in, cufftComplex *out) const { + return Exec([&](cufftHandle h) { return cufftExecR2C(h, in, out); }); + } + cufftResult C2R(cufftComplex *in, cufftReal *out) const { + return Exec([&](cufftHandle h) { return cufftExecC2R(h, in, out); }); } ~CudaFFTPlan() { @@ -539,8 +586,9 @@ public: } // Move-only type CudaFFTPlan(CudaFFTPlan&& other) noexcept - : key_(std::move(other.key_)), plan_(other.plan_), stream_(other.stream_), work_(std::move(other.work_)) { - other.plan_ = {}; + : key_(std::move(other.key_)), plan_(other.plan_), stream_(other.stream_), work_area_(other.work_area_), + work_(std::move(other.work_)) { + other.plan_ = nullptr; } CudaFFTPlan& operator=(CudaFFTPlan&& other) noexcept { if (this != &other) { @@ -548,17 +596,18 @@ public: key_ = std::move(other.key_); plan_ = other.plan_; stream_ = other.stream_; + work_area_ = other.work_area_; work_ = std::move(other.work_); - other.plan_ = {}; + other.plan_ = nullptr; } return *this; } CudaFFTPlan(const CudaFFTPlan&) = delete; CudaFFTPlan& operator=(const CudaFFTPlan&) = delete; - size_t WorkBytes() const { return plan_.work_bytes; } - operator cufftHandle() const { return plan_.handle; } - cufftHandle get() const { return plan_.handle; } + size_t WorkBytes() const { return plan_->plan.work_bytes; } + // Which plan this is - for tests; a transform goes through R2C/C2R. + cufftHandle Handle() const { return plan_->plan.handle; } }; // Page-locked host memory is slow to get and to give back: cudaHostAlloc and cudaFreeHost map and lock diff --git a/image_analysis/indexing/FFTIndexerGPU.cu b/image_analysis/indexing/FFTIndexerGPU.cu index cc72cd2a4..234820928 100644 --- a/image_analysis/indexing/FFTIndexerGPU.cu +++ b/image_analysis/indexing/FFTIndexerGPU.cu @@ -312,7 +312,7 @@ void FFTIndexerGPU::ExecuteFFT(const std::vector &coord, size_t nspots) { } cuda_err(cudaGetLastError()); - cuda_err(cufftExecR2C(n == FFT_BATCH_DIRECTIONS || n == nDirections ? plan : plan_last, + cuda_err((n == FFT_BATCH_DIRECTIONS || n == nDirections ? plan : plan_last).R2C( d_input_fft, d_output_fft)); const dim3 tr_grid((out_len + 31) / 32, (n + 31) / 32); diff --git a/image_analysis/structure_refinement/ModelStructureFactorsGPU.cu b/image_analysis/structure_refinement/ModelStructureFactorsGPU.cu index aa1b036d0..274214da8 100644 --- a/image_analysis/structure_refinement/ModelStructureFactorsGPU.cu +++ b/image_analysis/structure_refinement/ModelStructureFactorsGPU.cu @@ -280,7 +280,7 @@ void ModelStructureFactorsGPUEngine::Compute(const std::vector cudaMemcpyHostToDevice, e.stream)); e.density->Compute(e.pos, e.grid); - cufft_err(cufftExecR2C(e.forward, e.grid, reinterpret_cast(e.spectrum.get()))); + cufft_err(e.forward.R2C(e.grid, reinterpret_cast(e.spectrum.get()))); if (rows > 0) { compose_kernel<<>>(e.spectrum, e.Norm(), g.nu, g.nv, g.nw, e.ops, static_cast(e.setup.sym_ops.size()), e.setup.den, @@ -289,7 +289,7 @@ void ModelStructureFactorsGPUEngine::Compute(const std::vector } e.mask->Compute(e.mask_atoms, static_cast(mask_atoms.size()), e.grid); - cufft_err(cufftExecR2C(e.forward, e.grid, reinterpret_cast(e.spectrum.get()))); + cufft_err(e.forward.R2C(e.grid, reinterpret_cast(e.spectrum.get()))); if (rows > 0) { fmask_kernel<<>>(e.spectrum, e.Norm(), e.row_hkl, rows, g.nu, g.nv, g.nw, e.fmask); @@ -320,7 +320,7 @@ void ModelStructureFactorsGPUEngine::Map(int nu, int nv, int nw, double volume, e.stream)); conjugate_kernel<<>>(e.spectrum, complex_points); cuda_err(cudaGetLastError()); - cufft_err(cufftExecC2R(plan, reinterpret_cast(e.spectrum.get()), + cufft_err(plan.C2R(reinterpret_cast(e.spectrum.get()), reinterpret_cast(e.spectrum.get()))); transpose_kernel<<>>(reinterpret_cast(e.spectrum.get()), nu, nv, nw, static_cast(1.0 / volume), e.grid); diff --git a/image_analysis/structure_refinement/MolecularReplacementGPU.cu b/image_analysis/structure_refinement/MolecularReplacementGPU.cu index f405855ca..bb1fe7b88 100644 --- a/image_analysis/structure_refinement/MolecularReplacementGPU.cu +++ b/image_analysis/structure_refinement/MolecularReplacementGPU.cu @@ -386,7 +386,7 @@ gemmi::FPhiGrid MapToFPhiGPU(const gemmi::Grid &map, int reach) { CudaDevicePtr d_half(static_cast(map.nu / 2 + 1) * map.nv * map.nw); CudaDevicePtr d_out(hkl.data.size()); CudaFFTPlan plan(CudaFFTShape{.n = {map.nw, map.nv, map.nu}, .type = CUFFT_R2C}, stream); - cufft_err(cufftExecR2C(plan, d_map.get(), d_half.get())); + cufft_err(plan.R2C(d_map.get(), d_half.get())); half_u_to_half_w<<((hkl.data.size() + THREADS - 1) / THREADS), THREADS, 0, stream>>>( d_half, map.nu, map.nv, map.nw, reach, norm, d_out); cuda_err(cudaGetLastError()); @@ -658,9 +658,8 @@ std::vector MolecularReplacementGPU::TranslationFunction( CudaDevicePtr d_best(nstat); CudaDevicePtr d_best_index(nstat); - const cufftHandle plan = p.plan; + const CudaFFTPlan &plan = p.plan; // on `stream` { - cufft_err(cufftSetStream(plan, stream)); for (size_t r = 0; r < rotations.size(); r++) { const Rot rot = transposed(rotations[r]); translation_copies<<<(terms + THREADS - 1) / THREADS, THREADS, 0, stream>>>( @@ -673,7 +672,7 @@ std::vector MolecularReplacementGPU::TranslationFunction( translation_gather_cross<<((cells + THREADS - 1) / THREADS), THREADS, 0, stream>>>( p.cross_offset, p.cross_entry, copies, scaled, d_known, p.cross_term, p.cross_op, p.cross_conj, nops, cells, coef); - cufft_err(cufftExecC2R(plan, reinterpret_cast(coef.get()), map.get())); + cufft_err(plan.C2R(reinterpret_cast(coef.get()), map.get())); translation_stats<<>>( map, points, d_sum.get() + r * stat_blocks, d_sum2.get() + r * stat_blocks, d_best.get() + r * stat_blocks, d_best_index.get() + r * stat_blocks); diff --git a/image_analysis/structure_refinement/RigidBodyGPU.cu b/image_analysis/structure_refinement/RigidBodyGPU.cu index 962221384..54a26a2a3 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.cu +++ b/image_analysis/structure_refinement/RigidBodyGPU.cu @@ -295,10 +295,9 @@ int blocks(size_t n) { return static_cast((n + THREADS - 1) / THREADS); } -// An (nu, nv, nw) R2C transform, u fastest, `batch` grids back to back. -CudaFFTShape GridShape(int nu, int nv, int nw, int batch) { - return {.n = {nw, nv, nu}, .idist = nu * nv * nw, .odist = (nu / 2 + 1) * nv * nw, .type = CUFFT_R2C, - .batch = batch}; +// An (nu, nv, nw) R2C transform, u fastest. +CudaFFTShape GridShape(int nu, int nv, int nw) { + return {.n = {nw, nv, nu}, .type = CUFFT_R2C}; } int GreatestStreamPriority() { @@ -356,7 +355,9 @@ struct RigidBodyGPUEngineImpl { CudaDevicePtr point_fcmol, point_fmask; CudaDevicePtr products, x; - std::map, CudaFFTPlan> plans; + // The zone grid's transform, one grid at a time: a batch of three gives the same bits, but would be one more + // plan per zone and card to make (CudaFFTPlan). + CudaFFTPlan plan; std::unique_ptr density; std::unique_ptr mask; std::unique_ptr scale; @@ -422,15 +423,10 @@ struct RigidBodyGPUEngineImpl { cudaStreamSynchronize(stream); } - cufftHandle Plan(int batch) { - const std::array k{geom.nu, geom.nv, geom.nw, batch}; - auto it = plans.find(k); - if (it != plans.end()) - return it->second; - CudaFFTPlan plan(GridShape(geom.nu, geom.nv, geom.nw, batch), stream, fft_work); - if (plan.WorkBytes() > cap.fft_work_bytes) - throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "rigid body: cuFFT work area over the reserve"); - return plans.emplace(k, std::move(plan)).first->second; + // Grid `slot`'s transform into its spectrum. + void Transform(int slot) { + cufft_err(plan.R2C(grid.get() + slot * GridPoints(), + reinterpret_cast(spectrum.get()) + slot * ComplexPoints())); } size_t GridPoints() const { return static_cast(geom.nu) * geom.nv * geom.nw; } @@ -489,10 +485,7 @@ int RigidBodyGPUEngine::CurrentDevice() { } size_t RigidBodyGPUEngine::FFTWorkBytes(int nu, int nv, int nw) { - size_t largest = 0; - for (int batch : {1, 3}) - largest = std::max(largest, CudaFFTPlan::WorkBytes(GridShape(nu, nv, nw, batch))); - return largest; + return CudaFFTPlan::WorkBytes(GridShape(nu, nv, nw)); } size_t RigidBodyGPUEngine::DeviceBytes(const RigidBodyGPUCapacity &c) { @@ -596,10 +589,9 @@ void RigidBodyGPUEngine::SetZone(const RigidBodyGPUZone &zone) { e.mask->SetGrid(mask_grid, images); e.density->SetAtoms(DensityGrid(zone), zone.atoms); e.scale->SetPoints(zone.point_hkl, zone.point_stol2, zone.point_fobs, zone.point_sigma, zone.constraints, zone.frac); - // The previous zone's plans go back to be lent to another engine; this one takes the zone's own. - e.plans.clear(); - e.Plan(1); - e.Plan(3); + e.plan = CudaFFTPlan(GridShape(e.geom.nu, e.geom.nv, e.geom.nw), e.stream, e.fft_work); + if (e.plan.WorkBytes() > e.cap.fft_work_bytes) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "rigid body: cuFFT work area over the reserve"); cuda_err(cudaStreamSynchronize(e.stream)); } @@ -607,7 +599,7 @@ void RigidBodyGPUEngine::Fcalc(const double rotation[9], const double translatio RigidBodyGPUEngineImpl &e = *impl_; const DeviceGuard guard(e.device); e.Density(rotation, translation, 0); - cufft_err(cufftExecR2C(e.Plan(1), e.grid, reinterpret_cast(e.spectrum.get()))); + e.Transform(0); e.Compose(0, e.f, e.df_dt); } @@ -633,7 +625,7 @@ void RigidBodyGPUEngine::SetFixedCopies(const std::vector cuda_err(cudaGetLastError()); if (c < fixed.size()) { e.density->Compute(e.pos, e.grid.get()); - cufft_err(cufftExecR2C(e.Plan(1), e.grid, reinterpret_cast(e.spectrum.get()))); + cufft_err(e.plan.R2C(e.grid, reinterpret_cast(e.spectrum.get()))); e.has_fixed = false; e.Compose(0, e.shifted, nullptr); add_kernel<<>>(e.shifted, static_cast(e.rows), e.fixed); @@ -644,7 +636,7 @@ void RigidBodyGPUEngine::SetFixedCopies(const std::vector cufftComplex *mask_spectrum = reinterpret_cast(e.spectrum.get() + e.ComplexPoints()); e.mask->Compute(reinterpret_cast(e.copies_mask_atoms.get()), static_cast(copies * e.n_mask_atoms), mask_grid); - cufft_err(cufftExecR2C(e.Plan(1), mask_grid, mask_spectrum)); + cufft_err(e.plan.R2C(mask_grid, mask_spectrum)); fmask_kernel<<>>(reinterpret_cast(mask_spectrum), e.Norm(), e.row_hkl, static_cast(e.rows), e.geom.nu, e.geom.nv, e.geom.nw, e.fmask); @@ -661,7 +653,7 @@ void RigidBodyGPUEngine::Fmask() { // The mask atoms Fcalc() placed, four doubles each, as ModelMaskAtom lays them out. e.mask->Compute(reinterpret_cast(e.mask_atoms.get()), static_cast(e.n_mask_atoms), mask_grid); - cufft_err(cufftExecR2C(e.Plan(1), mask_grid, mask_spectrum)); + cufft_err(e.plan.R2C(mask_grid, mask_spectrum)); fmask_kernel<<>>(reinterpret_cast(mask_spectrum), e.Norm(), e.row_hkl, static_cast(e.rows), e.geom.nu, e.geom.nv, e.geom.nw, e.fmask); @@ -720,7 +712,8 @@ void RigidBodyGPUEngine::Jacobian(const double rotation[3][9], const double tran const DeviceGuard guard(e.device); for (int j = 0; j < 3; j++) e.Density(rotation[j], translation[j], j); - cufft_err(cufftExecR2C(e.Plan(3), e.grid, reinterpret_cast(e.spectrum.get()))); + for (int j = 0; j < 3; j++) + e.Transform(j); for (int j = 0; j < 3; j++) e.Compose(j, e.shifted.get() + j * e.rows, nullptr); diff --git a/image_analysis/structure_refinement/RigidBodyGPUEngine.h b/image_analysis/structure_refinement/RigidBodyGPUEngine.h index 1c1859e2c..683c5522b 100644 --- a/image_analysis/structure_refinement/RigidBodyGPUEngine.h +++ b/image_analysis/structure_refinement/RigidBodyGPUEngine.h @@ -83,7 +83,7 @@ class RigidBodyGPUEngine { public: // The bytes an engine of this capacity reserves on the device. static size_t DeviceBytes(const RigidBodyGPUCapacity &capacity); - // The largest cuFFT work area a (nu, nv, nw) grid needs, batch 1 or 3. + // The cuFFT work area a (nu, nv, nw) grid needs. static size_t FFTWorkBytes(int nu, int nv, int nw); // (brick, atom) pairs of the gather over at most, for a zone's atoms on its grid. static size_t PairBound(const RigidBodyGPUZone &zone); diff --git a/tests/CUDAMemHelpersTest.cpp b/tests/CUDAMemHelpersTest.cpp index 9c70f0c0f..6f5cf08fd 100644 --- a/tests/CUDAMemHelpersTest.cpp +++ b/tests/CUDAMemHelpersTest.cpp @@ -138,11 +138,11 @@ TEST_CASE("CudaPinnedPool_ReusedByTheNextEngine", "[CUDAMemHelpers]") { CHECK(cuda_release_pinned_pool() == 0); } -// Within a scope a cuFFT plan is kept once made and lent to one user at a time: two users at once get -// two plans, the next one after them gets one of those back, and a transform on a lent plan - another -// stream, a work area of the caller's - gives the same result bit for bit. When the scope ends, every -// kept plan is destroyed; outside a scope nothing is kept. -TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") { +// Within a scope a cuFFT plan is made once per shape and shared: two users at once get the same plan, +// each transforming on its own stream with its own work area, bit for bit as on a plan of its own; the +// plan is kept when they are done, and the next user gets it. When the scope ends, every kept plan is +// destroyed; outside a scope a plan goes with its last user. +TEST_CASE("CudaFFTPlan_SharedAndKept", "[CUDAMemHelpers]") { if (get_gpu_count() == 0) SKIP("No CUDA GPU present"); REQUIRE(jfjoch_cufft_plans::kept() == 0); @@ -154,11 +154,11 @@ TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") { for (size_t i = 0; i < input.size(); i++) input[i] = static_cast((i * 7919) % 1013) - 500.0f; CudaDevicePtr d_in(input.size()); - CudaDevicePtr d_out(nc * batch); REQUIRE(cudaMemcpy(d_in, input.data(), input.size() * sizeof(float), cudaMemcpyHostToDevice) == cudaSuccess); auto transform = [&](const CudaFFTPlan &plan, cudaStream_t stream) { - REQUIRE(cufftExecR2C(plan, d_in, d_out) == CUFFT_SUCCESS); + CudaDevicePtr d_out(nc * batch); + REQUIRE(plan.R2C(d_in, d_out) == CUFFT_SUCCESS); std::vector out(nc * batch); REQUIRE(cudaMemcpyAsync(out.data(), d_out, out.size() * sizeof(cufftComplex), cudaMemcpyDeviceToHost, stream) == cudaSuccess); @@ -166,30 +166,34 @@ TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") { return out; }; - CudaStream stream_a, stream_c; + CudaStream stream_a, stream_b, stream_c; std::vector first; - cufftHandle handle_a = 0; + cufftHandle handle = 0; { CudaFFTPlan a(shape, stream_a); - CudaFFTPlan b(shape); - handle_a = a.get(); - CHECK(handle_a != b.get()); + CudaFFTPlan b(shape, stream_b); + handle = a.Handle(); + CHECK(b.Handle() == handle); + CHECK(jfjoch_cufft_plans::kept() == 1); first = transform(a, stream_a); + const std::vector second = transform(b, stream_b); + CHECK(memcmp(second.data(), first.data(), first.size() * sizeof(cufftComplex)) == 0); } + CHECK(jfjoch_cufft_plans::kept() == 1); const size_t work = CudaFFTPlan::WorkBytes(shape); CudaDevicePtr work_area(std::max(work, 1)); CudaFFTPlan c(shape, stream_c, work_area); - CHECK(c.get() == handle_a); + CHECK(c.Handle() == handle); CHECK(c.WorkBytes() == work); const std::vector again = transform(c, stream_c); REQUIRE(again.size() == first.size()); CHECK(memcmp(again.data(), first.data(), first.size() * sizeof(cufftComplex)) == 0); - CHECK(jfjoch_cufft_plans::kept() == 1); // b; a is lent to c - c = CudaFFTPlan(); - CHECK(jfjoch_cufft_plans::kept() == 2); + // The scope ends while c still uses the plan: it goes with c. scope.reset(); + CHECK(jfjoch_cufft_plans::kept() == 1); + c = CudaFFTPlan(); CHECK(jfjoch_cufft_plans::kept() == 0); CHECK(CudaFFTPlan::WorkBytes(shape) == work);