GPU memory over the run: FFT indexers given back after the last indexing; speculative probe takes turns with the merge on a small card
- IndexerThreadPool::ReleaseIndexers(): each worker drops the indexers it has built (once its current frame is done) and the allocation pool is trimmed, so their memory goes back to the card, not only to the pool. A later frame builds them again, as on first use. A rotation pass calls it once its first pass is done and again when the leftover-lattice census - the last of its indexing - ends. Measured on the RTX 5080: the 1.24 GB (pre-pass) / 0.62 GB (later passes) the pass's pool held through the image pass and the scaling is gone; 4nwv's pre-pass image pass peaks 5.97 instead of 7.18 GB of device memory. - GPUMerge: the main line holds one through ScaleMergeAndSymmetry. The speculative geometry probe (GPUWorkBeside(after_merges=true)) waits for it on a card below the take-turns size (12 GB), where the probe's FFT indexers and image workers beside the merge's arrays would overfill the card - on a Windows (WDDM) card that does not fail an allocation but pages it and crawls. The merge waits for nothing, so nothing the merge waits on (the rotation-scale walk's probes) can wait in a circle with it. No change on a 16 GB card. Output unchanged: p.hkl byte-identical on cytc_x10sa, myob_x10sa, 4nwv and 8qaw on this card; also identical with the card's total read as 8.55 GB (10 / 9 image workers instead of 16) and with 4 image workers, so the worker count does not change the result. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
640b2e464a
commit
1c3a2e202c
7 files changed
+174
-10
No files matched your search
@@ -4,6 +4,7 @@
|
||||
#include <catch2/catch_all.hpp>
|
||||
#include "../common/CUDAWrapper.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
|
||||
@@ -329,6 +330,65 @@ TEST_CASE("GetGpuFreeMemory_IntegratedCountsThePageCache", "[CUDAMemHelpers]") {
|
||||
CHECK(double(free_bytes) == Catch::Approx(double(cuda_free)).margin(SLACK));
|
||||
}
|
||||
|
||||
|
||||
// The pool's indexers go back to the card - not only to the allocation pool - when it is asked, and the
|
||||
// next frame builds them again.
|
||||
TEST_CASE("IndexerThreadPool_ReleaseIndexersGivesTheCardBack", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("No CUDA GPU present");
|
||||
|
||||
IndexingSettings settings;
|
||||
settings.Algorithm(IndexingAlgorithmEnum::FFT).IndexingThreads(1);
|
||||
DiffractionExperiment x;
|
||||
x.ImportIndexingSettings(settings);
|
||||
const std::vector<Coord> recip{{0.1, 0, 0}, {0, 0.1, 0}, {0, 0, 0.1}};
|
||||
|
||||
// What the allocation pools of all cards hand out: an indexer worker takes its card round-robin.
|
||||
const auto pooled = [] {
|
||||
uint64_t sum = 0;
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
cudaMemPool_t pool = nullptr;
|
||||
uint64_t used = 0;
|
||||
if (cudaDeviceGetDefaultMemPool(&pool, d) == cudaSuccess
|
||||
&& cudaMemPoolGetAttribute(pool, cudaMemPoolAttrUsedMemCurrent, &used) == cudaSuccess)
|
||||
sum += used;
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
const auto reserved = [] {
|
||||
uint64_t sum = 0;
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
cudaMemPool_t pool = nullptr;
|
||||
uint64_t r = 0;
|
||||
if (cudaDeviceGetDefaultMemPool(&pool, d) == cudaSuccess
|
||||
&& cudaMemPoolGetAttribute(pool, cudaMemPoolAttrReservedMemCurrent, &r) == cudaSuccess)
|
||||
sum += r;
|
||||
}
|
||||
return sum;
|
||||
};
|
||||
const uint64_t before = pooled();
|
||||
const uint64_t reserved_before = reserved();
|
||||
{
|
||||
IndexerThreadPool pool(settings, IndexerConstruction::OnFirstUse);
|
||||
const IndexerResult first = pool.Run(x, recip);
|
||||
REQUIRE(first.error == nullptr);
|
||||
REQUIRE(pooled() > before + (uint64_t(100) << 20));
|
||||
|
||||
pool.ReleaseIndexers();
|
||||
CHECK(pooled() <= before + (uint64_t(16) << 20));
|
||||
CHECK(reserved() <= reserved_before + (uint64_t(64) << 20));
|
||||
|
||||
const IndexerResult again = pool.Run(x, recip);
|
||||
CHECK(again.error == nullptr);
|
||||
CHECK(again.executed == first.executed);
|
||||
CHECK(again.lattice.size() == first.lattice.size());
|
||||
}
|
||||
for (int d = 0; d < get_gpu_count(); d++) {
|
||||
ScopedGpuPin pin(d);
|
||||
cuda_trim_allocation_pool();
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// A merge that finds the card full waits for the GPU work running beside it: until the last one
|
||||
@@ -347,3 +407,26 @@ TEST_CASE("GPUWorkBeside_WaitEndsWithTheWork", "[CUDAMemHelpers]") {
|
||||
CHECK(wait_for_gpu_work_beside(std::chrono::seconds(60)));
|
||||
ends.join();
|
||||
}
|
||||
|
||||
// The speculative geometry probe waits for the main line's merge on a card too small for both, and on a
|
||||
// larger one runs beside it; the rest of the work beside never waits for a merge.
|
||||
TEST_CASE("GPUWorkBeside_AfterMergesTakesTurnsOnASmallCard", "[CUDAMemHelpers]") {
|
||||
// As take_turns() in CUDAWrapper.cpp.
|
||||
const bool small = get_gpu_count() > 0 && get_gpu_total_memory(0) < 12'000'000'000ull;
|
||||
|
||||
std::optional<GPUMerge> merge(std::in_place);
|
||||
{ GPUWorkBeside other; } // does not wait
|
||||
|
||||
std::atomic<bool> started = false;
|
||||
std::thread probe([&started] {
|
||||
GPUWorkBeside beside(/*after_merges=*/true);
|
||||
started = true;
|
||||
});
|
||||
if (small) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(200));
|
||||
CHECK_FALSE(started);
|
||||
}
|
||||
merge.reset();
|
||||
probe.join();
|
||||
CHECK(started);
|
||||
}
|
||||
Reference in new issue
Block a user