rugnux live: keep FFT indexers and cuFFT plans for the broker's lifetime, built at START
jfjoch_broker opened no FFTPlanReuse, so every cuFFT plan of a live run was made on first use and destroyed after it, and every pass (first pass, measured-centre arm, short-axis pool, pass 2) built its GPU FFT indexers afresh. Now: - FFTIndexerReuse: while one lives, a pool worker borrows a kept GPU FFT indexer of its card and construction settings instead of building one, and gives it back when it ends or is released. - The rugnux receiver holds an FFTIndexerReuse and an FFTPlanReuse for the broker's lifetime. - At START, beside the run (which is waiting for its first frames), WarmUpRugnuxLiveGPU builds on every card the first-pass and short-axis indexers and the beam-centre capture's plan pair. The CLI opens no FFTIndexerReuse and is unchanged. Results identical: CLI p.hkl md5 equal on two sets (CLI and --simulate-live --fast-live); broker replay, 3 collections each, MTZ data identical. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4
This commit is contained in:
1 parent
8261b7354e
commit
19ee20fbe8
7 files changed
+237
-5
No files matched your search
@@ -389,6 +389,64 @@ TEST_CASE("IndexerThreadPool_ReleaseIndexersGivesTheCardBack", "[CUDAMemHelpers]
|
||||
}
|
||||
}
|
||||
|
||||
// jfjoch_broker keeps the FFT indexers from one rugnux pass and collection to the next: a pool's worker
|
||||
// borrows the one kept on its card instead of building its own, gives it back when the pool ends, and
|
||||
// the card gets the memory back only with the last FFTIndexerReuse.
|
||||
TEST_CASE("IndexerThreadPool_FFTIndexerKeptAcrossPools", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("No CUDA GPU present");
|
||||
|
||||
IndexingSettings settings;
|
||||
settings.Algorithm(IndexingAlgorithmEnum::FFT).IndexingThreads(1);
|
||||
DiffractionExperiment x;
|
||||
x.ImportIndexingSettings(settings);
|
||||
const std::vector<Coord> recip{{0.1, 0, 0}, {0, 0.1, 0}, {0, 0, 0.1}};
|
||||
|
||||
const auto pooled = [] {
|
||||
uint64_t sum = 0;
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
cudaMemPool_t pool = nullptr;
|
||||
uint64_t used = 0;
|
||||
if (cudaDeviceGetDefaultMemPool(&pool, d) == cudaSuccess
|
||||
&& cudaMemPoolGetAttribute(pool, cudaMemPoolAttrUsedMemCurrent, &used) == cudaSuccess)
|
||||
sum += used;
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
|
||||
IndexerResult fresh;
|
||||
{
|
||||
IndexerThreadPool pool(settings, IndexerConstruction::OnFirstUse);
|
||||
fresh = pool.Run(x, recip);
|
||||
REQUIRE(fresh.error == nullptr);
|
||||
}
|
||||
|
||||
const uint64_t before = pooled();
|
||||
{
|
||||
FFTIndexerReuse reuse;
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
ScopedGpuPin pin(d);
|
||||
KeepFFTIndexers(settings, 1);
|
||||
}
|
||||
const uint64_t kept = pooled();
|
||||
REQUIRE(kept > before + (uint64_t(100) << 20));
|
||||
for (int i = 0; i < 2; i++) {
|
||||
IndexerThreadPool pool(settings, IndexerConstruction::OnFirstUse);
|
||||
const IndexerResult r = pool.Run(x, recip);
|
||||
CHECK(pooled() == kept);
|
||||
CHECK(r.error == nullptr);
|
||||
CHECK(r.executed == fresh.executed);
|
||||
CHECK(r.lattice.size() == fresh.lattice.size());
|
||||
}
|
||||
CHECK(pooled() == kept);
|
||||
}
|
||||
CHECK(pooled() <= before + (uint64_t(16) << 20));
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
ScopedGpuPin pin(d);
|
||||
cuda_trim_allocation_pool();
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// A merge that finds the card full waits for the GPU work running beside it: until the last one
|
||||
|
||||
Reference in new issue
Block a user