diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 5d3f45ecc..2cb53c297 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -28,50 +28,62 @@ std::string get_gpu_description() { } namespace { - // How many of one kind of GPU work are running, for a thread to wait until none is. - struct GPUWorkCount { - std::mutex m; - std::condition_variable none; - int n = 0; - void Add() { - std::lock_guard lock(m); - ++n; + // How many probe passes (GPUWorkBeside) and main-line image passes (GPUImagePass) are running, for a + // thread to wait until none of one kind is. + std::mutex gpu_work_m; + std::condition_variable gpu_work_changed; + int gpu_work_beside_n = 0, gpu_image_passes_n = 0; + + // A card too small for a probe pass beside the main line's image pass: measured on a 16 Mpx detector, + // the two together held 8.6 GB. On such a card each of the two waits, before it starts, for the other + // kind to end - so they take turns, and only when each runs depends on the card, never what it + // computes. Neither waits once it is running, so nothing waits in a circle. + constexpr size_t TAKE_TURNS_BELOW_BYTES = 12'000'000'000ull; + + bool take_turns() { + static const bool small = get_gpu_count() > 0 && get_gpu_total_memory(0) < TAKE_TURNS_BELOW_BYTES; + return small; + } + + void remove(int &n) { + { + std::lock_guard lock(gpu_work_m); + --n; } - void Remove() { - { - std::lock_guard lock(m); - --n; - } - none.notify_all(); - } - }; - GPUWorkCount gpu_work_beside, gpu_image_passes; + gpu_work_changed.notify_all(); + } } GPUWorkBeside::GPUWorkBeside() { - gpu_work_beside.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); + ++gpu_work_beside_n; } GPUWorkBeside::~GPUWorkBeside() { - gpu_work_beside.Remove(); + remove(gpu_work_beside_n); } bool wait_for_gpu_work_beside(std::chrono::seconds timeout) { - std::unique_lock lock(gpu_work_beside.m); - return gpu_work_beside.none.wait_for(lock, timeout, [] { return gpu_work_beside.n == 0; }); + std::unique_lock lock(gpu_work_m); + return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; }); } GPUImagePass::GPUImagePass() { - gpu_image_passes.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_work_beside_n == 0; }); + ++gpu_image_passes_n; } GPUImagePass::~GPUImagePass() { - gpu_image_passes.Remove(); + remove(gpu_image_passes_n); } void wait_for_gpu_image_passes() { - std::unique_lock lock(gpu_image_passes.m); - gpu_image_passes.none.wait(lock, [] { return gpu_image_passes.n == 0; }); + std::unique_lock lock(gpu_work_m); + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); } #ifndef JFJOCH_USE_CUDA diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index a61937f5a..429443b59 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -89,7 +89,8 @@ void cuda_throw_if_context_lost(); // GPU work that runs beside the main line of a process and gives all of its device memory back when it // ends: a probe pass run on a copy of the run while the run itself scales and merges. Hold one for as -// long as that work runs. Build-independent - it only counts. +// long as that work runs. On a card too small for both, it waits first for the main line's image passes +// to end (GPUImagePass). Build-independent - it only counts. class GPUWorkBeside { public: GPUWorkBeside(); @@ -103,7 +104,8 @@ bool wait_for_gpu_work_beside(std::chrono::seconds timeout); // The image pass of a run's main line - not of a probe holding a GPUWorkBeside: the stretch that holds // the most of the card, and that never waits for a probe, so a probe that does not fit beside it can -// wait for it to end. Hold one for as long as the image pass runs. Build-independent - it only counts. +// wait for it to end. Hold one for as long as the image pass runs. On a card too small for both, it +// waits first for the work beside to end. Build-independent - it only counts. class GPUImagePass { public: GPUImagePass();