From 39b488e323e3e10b620cdc0921af825f555d4b1a Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 10:50:56 +0200 Subject: [PATCH] Probe passes and the main image pass take turns on a card under 12 GB On a 16 GB card a probe pass (GPUWorkBeside) runs its image work beside the main line's image pass (GPUImagePass). Measured on a 16 Mpx detector with nsys, the two together held 8.6 GB of device buffers (two sets of 24 per-worker 67 MB image buffers plus five 0.6 GB FFT indexers), so on an 8 GB card the refined-geometry pass ran out of memory ("Allocating GPU memory (53.0 MB) failed"). Below 12 GB of card memory each of the two now waits, before it starts, for the other kind to end. Only when each runs depends on the card, never what it computes; neither waits once running, so nothing waits in a circle. On a 16 GB card nothing changes. Co-Authored-By: Claude Opus 5.5 (1M context) --- common/CUDAWrapper.cpp | 62 +++++++++++++++++++++++++----------------- common/CUDAWrapper.h | 6 ++-- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 5d3f45ecc..2cb53c297 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -28,50 +28,62 @@ std::string get_gpu_description() { } namespace { - // How many of one kind of GPU work are running, for a thread to wait until none is. - struct GPUWorkCount { - std::mutex m; - std::condition_variable none; - int n = 0; - void Add() { - std::lock_guard lock(m); - ++n; + // How many probe passes (GPUWorkBeside) and main-line image passes (GPUImagePass) are running, for a + // thread to wait until none of one kind is. + std::mutex gpu_work_m; + std::condition_variable gpu_work_changed; + int gpu_work_beside_n = 0, gpu_image_passes_n = 0; + + // A card too small for a probe pass beside the main line's image pass: measured on a 16 Mpx detector, + // the two together held 8.6 GB. On such a card each of the two waits, before it starts, for the other + // kind to end - so they take turns, and only when each runs depends on the card, never what it + // computes. Neither waits once it is running, so nothing waits in a circle. + constexpr size_t TAKE_TURNS_BELOW_BYTES = 12'000'000'000ull; + + bool take_turns() { + static const bool small = get_gpu_count() > 0 && get_gpu_total_memory(0) < TAKE_TURNS_BELOW_BYTES; + return small; + } + + void remove(int &n) { + { + std::lock_guard lock(gpu_work_m); + --n; } - void Remove() { - { - std::lock_guard lock(m); - --n; - } - none.notify_all(); - } - }; - GPUWorkCount gpu_work_beside, gpu_image_passes; + gpu_work_changed.notify_all(); + } } GPUWorkBeside::GPUWorkBeside() { - gpu_work_beside.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); + ++gpu_work_beside_n; } GPUWorkBeside::~GPUWorkBeside() { - gpu_work_beside.Remove(); + remove(gpu_work_beside_n); } bool wait_for_gpu_work_beside(std::chrono::seconds timeout) { - std::unique_lock lock(gpu_work_beside.m); - return gpu_work_beside.none.wait_for(lock, timeout, [] { return gpu_work_beside.n == 0; }); + std::unique_lock lock(gpu_work_m); + return gpu_work_changed.wait_for(lock, timeout, [] { return gpu_work_beside_n == 0; }); } GPUImagePass::GPUImagePass() { - gpu_image_passes.Add(); + std::unique_lock lock(gpu_work_m); + if (take_turns()) + gpu_work_changed.wait(lock, [] { return gpu_work_beside_n == 0; }); + ++gpu_image_passes_n; } GPUImagePass::~GPUImagePass() { - gpu_image_passes.Remove(); + remove(gpu_image_passes_n); } void wait_for_gpu_image_passes() { - std::unique_lock lock(gpu_image_passes.m); - gpu_image_passes.none.wait(lock, [] { return gpu_image_passes.n == 0; }); + std::unique_lock lock(gpu_work_m); + gpu_work_changed.wait(lock, [] { return gpu_image_passes_n == 0; }); } #ifndef JFJOCH_USE_CUDA diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index a61937f5a..429443b59 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -89,7 +89,8 @@ void cuda_throw_if_context_lost(); // GPU work that runs beside the main line of a process and gives all of its device memory back when it // ends: a probe pass run on a copy of the run while the run itself scales and merges. Hold one for as -// long as that work runs. Build-independent - it only counts. +// long as that work runs. On a card too small for both, it waits first for the main line's image passes +// to end (GPUImagePass). Build-independent - it only counts. class GPUWorkBeside { public: GPUWorkBeside(); @@ -103,7 +104,8 @@ bool wait_for_gpu_work_beside(std::chrono::seconds timeout); // The image pass of a run's main line - not of a probe holding a GPUWorkBeside: the stretch that holds // the most of the card, and that never waits for a probe, so a probe that does not fit beside it can -// wait for it to end. Hold one for as long as the image pass runs. Build-independent - it only counts. +// wait for it to end. Hold one for as long as the image pass runs. On a card too small for both, it +// waits first for the work beside to end. Build-independent - it only counts. class GPUImagePass { public: GPUImagePass();