CudaFFTPlan: one cuFFT plan per card and shape, shared by its users; rigid body transforms one grid at a time
Every cuFFT plan has the driver compile cuFFT's PTX kernels: cufftMakePlanMany -> cuModuleLoadData -> the PTX JIT, under cuFFT's process-wide lock. A second plan of the same shape pays again, even while the first is alive. The CUDA JIT cache makes that ~1 ms. Without one (CUDA_CACHE_DISABLE, no writable $HOME) a plan costs 20-250 ms. That holds for every size (64^3 and 1D 256 included), every transform type and batch, on sm_75 and sm_120, with cuFFT 11.2, 12.0 and 12.4. No CUFFT_* setting changes it, so a choice of grid sizes cannot avoid it; only the number of plans made can be cut. Before this change, a plan was lent to one user at a time, so each concurrent user of a shape on a card made its own: 9-24 rigid-body engines, the translation slices and the score engines. Now a plan is made once per card and shape and shared. Each user keeps its own stream and work area, and CudaFFTPlan::R2C/C2R set both and launch under the plan's mutex. Shared plans give the same bits as private ones: 8 threads on one plan with their own work areas, 9 shapes x 160 transforms, 0 mismatches on RTX 5080 and T4. A shared work area, the negative control, gives 2-13 mismatches per shape where the plan has a work area. The rigid-body engine also dropped its batch-3 plan: the Jacobian transforms its three grids one at a time on the batch-1 plan. That gives the same bits (10 zone-grid shapes, sm_75 and sm_120) and one plan less per zone and card. The cytochrome C and thaumatin MR test sets, AF models (MR step, s): 4x T4 cytc cache on 2.22 -> 2.22 cache off 8.41 -> 4.00 plans 86 -> 24 4x T4 thau cache on 1.58 -> 1.61 cache off 11.13 -> 3.08 plans 131 -> 23 1x T4 cytc cache on 2.87 -> 2.81 cache off 8.82 -> 3.84 plans 80 -> 12 1x T4 thau cache on 2.41 -> 2.39 cache off 10.42 -> 3.17 plans 104 -> 11 RTX5080 cytc cache on 1.41 -> 1.12 cache off 6.5-10.7 -> 2.02 RTX5080 thau cache on 0.90 -> 0.85 cache off 8.03 -> 1.60 Outputs are bit-identical before and after, cache on or off: model, merged hkl, maps, and every MR and validation log line. The mtz/cif differ only in the command line they record. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4
This commit is contained in:
1 parent
6936b04d5f
commit
97ef0936b2
8 files changed
+173
-128
No files matched your search
@@ -138,11 +138,11 @@ TEST_CASE("CudaPinnedPool_ReusedByTheNextEngine", "[CUDAMemHelpers]") {
|
||||
CHECK(cuda_release_pinned_pool() == 0);
|
||||
}
|
||||
|
||||
// Within a scope a cuFFT plan is kept once made and lent to one user at a time: two users at once get
|
||||
// two plans, the next one after them gets one of those back, and a transform on a lent plan - another
|
||||
// stream, a work area of the caller's - gives the same result bit for bit. When the scope ends, every
|
||||
// kept plan is destroyed; outside a scope nothing is kept.
|
||||
TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") {
|
||||
// Within a scope a cuFFT plan is made once per shape and shared: two users at once get the same plan,
|
||||
// each transforming on its own stream with its own work area, bit for bit as on a plan of its own; the
|
||||
// plan is kept when they are done, and the next user gets it. When the scope ends, every kept plan is
|
||||
// destroyed; outside a scope a plan goes with its last user.
|
||||
TEST_CASE("CudaFFTPlan_SharedAndKept", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("No CUDA GPU present");
|
||||
REQUIRE(jfjoch_cufft_plans::kept() == 0);
|
||||
@@ -154,11 +154,11 @@ TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") {
|
||||
for (size_t i = 0; i < input.size(); i++)
|
||||
input[i] = static_cast<float>((i * 7919) % 1013) - 500.0f;
|
||||
CudaDevicePtr<float> d_in(input.size());
|
||||
CudaDevicePtr<cufftComplex> d_out(nc * batch);
|
||||
REQUIRE(cudaMemcpy(d_in, input.data(), input.size() * sizeof(float), cudaMemcpyHostToDevice) == cudaSuccess);
|
||||
|
||||
auto transform = [&](const CudaFFTPlan &plan, cudaStream_t stream) {
|
||||
REQUIRE(cufftExecR2C(plan, d_in, d_out) == CUFFT_SUCCESS);
|
||||
CudaDevicePtr<cufftComplex> d_out(nc * batch);
|
||||
REQUIRE(plan.R2C(d_in, d_out) == CUFFT_SUCCESS);
|
||||
std::vector<cufftComplex> out(nc * batch);
|
||||
REQUIRE(cudaMemcpyAsync(out.data(), d_out, out.size() * sizeof(cufftComplex), cudaMemcpyDeviceToHost,
|
||||
stream) == cudaSuccess);
|
||||
@@ -166,30 +166,34 @@ TEST_CASE("CudaFFTPlan_KeptAndLentToOneUserAtATime", "[CUDAMemHelpers]") {
|
||||
return out;
|
||||
};
|
||||
|
||||
CudaStream stream_a, stream_c;
|
||||
CudaStream stream_a, stream_b, stream_c;
|
||||
std::vector<cufftComplex> first;
|
||||
cufftHandle handle_a = 0;
|
||||
cufftHandle handle = 0;
|
||||
{
|
||||
CudaFFTPlan a(shape, stream_a);
|
||||
CudaFFTPlan b(shape);
|
||||
handle_a = a.get();
|
||||
CHECK(handle_a != b.get());
|
||||
CudaFFTPlan b(shape, stream_b);
|
||||
handle = a.Handle();
|
||||
CHECK(b.Handle() == handle);
|
||||
CHECK(jfjoch_cufft_plans::kept() == 1);
|
||||
first = transform(a, stream_a);
|
||||
const std::vector<cufftComplex> second = transform(b, stream_b);
|
||||
CHECK(memcmp(second.data(), first.data(), first.size() * sizeof(cufftComplex)) == 0);
|
||||
}
|
||||
CHECK(jfjoch_cufft_plans::kept() == 1);
|
||||
|
||||
const size_t work = CudaFFTPlan::WorkBytes(shape);
|
||||
CudaDevicePtr<char> work_area(std::max<size_t>(work, 1));
|
||||
CudaFFTPlan c(shape, stream_c, work_area);
|
||||
CHECK(c.get() == handle_a);
|
||||
CHECK(c.Handle() == handle);
|
||||
CHECK(c.WorkBytes() == work);
|
||||
const std::vector<cufftComplex> again = transform(c, stream_c);
|
||||
REQUIRE(again.size() == first.size());
|
||||
CHECK(memcmp(again.data(), first.data(), first.size() * sizeof(cufftComplex)) == 0);
|
||||
CHECK(jfjoch_cufft_plans::kept() == 1); // b; a is lent to c
|
||||
|
||||
c = CudaFFTPlan();
|
||||
CHECK(jfjoch_cufft_plans::kept() == 2);
|
||||
// The scope ends while c still uses the plan: it goes with c.
|
||||
scope.reset();
|
||||
CHECK(jfjoch_cufft_plans::kept() == 1);
|
||||
c = CudaFFTPlan();
|
||||
CHECK(jfjoch_cufft_plans::kept() == 0);
|
||||
|
||||
CHECK(CudaFFTPlan::WorkBytes(shape) == work);
|
||||
|
||||
Reference in new issue
Block a user