From 5de7189643e456ab445b1add3cb3194038ade07c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 14 Oct 2026 21:28:38 +0200 Subject: [PATCH] GPU or CPU decided up front from the card's total memory, never from what is free - Scaling: the 80%-of-card check runs right after the device ingest's sort and cut, where the observation and distinct-hkl counts are first known, instead of after the geometry smoothing; same counts, same decision, but a run that goes to the CPU no longer does the device smoothing (6.9 s on an 8 GB card for a 33 M observation set) or holds its memory meanwhile. - Rigid body: engines budgeted on a quarter of the card's total (the validation's plan), not min(total/4, free - 1 GB). A busy card no longer sends the rigid body to the CPU (measured on an 8 GB card: ~80 s and a different fit); engine allocations wait for GPU work beside the run, and an engine that cannot be set up fails the validation (RigidBodyGPUFailure) instead of falling back to the CPU. - Beam-centre FFT: GPU when its footprint is at most half of the card's total, otherwise CPU, logged; no catch-and-rescore on the CPU after a GPU failure. - FitPerFrameG reserves its per-frame vector (CPU scaling waited on the heap lock on Windows). - get_gpu_free_memory(): cudaMemGetInfo, except on an integrated GPU under Linux, where free also counts MemAvailable less a tenth of the total. Used only where free memory is waited on (cuda_memory_available) or quoted in out-of-memory messages; it decides nothing. The two card-filling [CUDAMemHelpers] tests skip on integrated GPUs. - Out-of-memory messages point at rugnux --no-gpu instead of "the CPU build". Co-Authored-By: Claude Opus 5.5 --- common/CUDAWrapper.cpp | 4 ++ common/CUDAWrapper.cu | 33 +++++++++++++ common/CUDAWrapper.h | 8 +++ .../geom_refinement/BeamCenterFFT.cpp | 15 ++---- .../geom_refinement/BeamCenterFFTGPU.cu | 8 +-- .../geom_refinement/BeamCenterFFTGPU.h | 6 +-- image_analysis/indexing/CUDAMemHelpers.h | 6 +-- image_analysis/scale_merge/ErrorModelGPU.cu | 2 +- image_analysis/scale_merge/ErrorModelGPU.h | 2 +- image_analysis/scale_merge/PairSortGPU.cu | 2 +- image_analysis/scale_merge/PairSortGPU.h | 2 +- .../scale_merge/RotationScaleMerge.cpp | 49 +++++++++++-------- .../scale_merge/RotationScaleMerge.h | 3 ++ .../scale_merge/RotationScaleMergeGPU.cu | 2 +- .../scale_merge/WilsonOutliersGPU.cu | 2 +- .../scale_merge/WilsonOutliersGPU.h | 2 +- .../structure_refinement/RigidBodyGPU.cpp | 21 ++++---- .../structure_refinement/RigidBodyGPU.cu | 7 +-- .../structure_refinement/RigidBodyGPU.h | 6 +-- .../structure_refinement/RigidBodyGPUEngine.h | 1 - rugnux/RugnuxScaleMerge.cpp | 2 +- tests/CUDAMemHelpersTest.cpp | 30 ++++++++++++ tests/ModelValidationTest.cpp | 9 +--- 23 files changed, 143 insertions(+), 79 deletions(-) diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 0da306fcf..585f8d939 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -103,6 +103,10 @@ size_t get_gpu_total_memory(int32_t) { return 0; } +bool get_gpu_free_memory(size_t &, size_t &) { + return false; +} + void set_gpu(int32_t dev_id) {} void pin_gpu() {} diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 4df5a9daa..c6ce3abf4 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only +#include #include +#include #include #include #include @@ -66,6 +68,37 @@ size_t get_gpu_total_memory(int32_t dev_id) { return prop.totalGlobalMem; } +bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes) { + if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) + return false; +#ifdef __linux__ + // Asked once: whether each card shares the host's memory. + static const std::vector integrated = [] { + std::vector v(std::max(get_gpu_count(), 0)); + for (int i = 0; i < static_cast(v.size()); i++) + if (cudaDeviceGetAttribute(&v[i], cudaDevAttrIntegrated, i) != cudaSuccess) { + v[i] = 0; + cudaGetLastError(); + } + return v; + }(); + int device = 0; + if (cudaGetDevice(&device) != cudaSuccess || device >= static_cast(integrated.size()) || !integrated[device]) + return true; + std::ifstream meminfo("/proc/meminfo"); + std::string line; + while (std::getline(meminfo, line)) + if (line.rfind("MemAvailable:", 0) == 0) { + const size_t available = std::stoull(line.substr(13)) * 1024; // given in kB + const size_t headroom = total_bytes / 10; // the host's own processes live in the same memory + if (available > headroom) + free_bytes = std::max(free_bytes, std::min(available - headroom, total_bytes)); + break; + } +#endif + return true; +} + void set_gpu(int32_t dev_id) { auto dev_count = get_gpu_count(); diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index 76f5cf23a..d5f923523 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -29,6 +29,14 @@ std::string get_gpu_description(); // device cannot be queried. size_t get_gpu_total_memory(int32_t dev_id); +// Free and total memory of the current device in bytes, as cudaMemGetInfo gives them - except on a GPU +// that shares the host's memory (cudaDevAttrIntegrated: GB10, Jetson), whose cudaMemGetInfo free is the +// host's MemFree and leaves out the page cache the kernel hands back on demand. There, on Linux, free is +// the host's MemAvailable less a tenth of the total, at most the total. For waiting on memory and for +// messages only: what runs on the GPU is decided on the total (get_gpu_total_memory), never on this. +// False without CUDA, or when the device cannot be queried. +bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes); + void set_gpu(int32_t dev_id); // Pin the calling thread to the next GPU in round-robin order, using a process-wide counter diff --git a/image_analysis/geom_refinement/BeamCenterFFT.cpp b/image_analysis/geom_refinement/BeamCenterFFT.cpp index ff8726b66..5a7a02001 100644 --- a/image_analysis/geom_refinement/BeamCenterFFT.cpp +++ b/image_analysis/geom_refinement/BeamCenterFFT.cpp @@ -250,18 +250,13 @@ BeamCenterFFTResult BeamCenterFFTScore(int64_t width, int64_t height, const std::vector &mean, const BeamCenterFFTSettings &settings) { #ifdef JFJOCH_USE_CUDA - if (get_gpu_count() > 0 && BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) { - BeamCenterFFTGPU gpu; - try { + if (get_gpu_count() > 0) { + if (BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) { + BeamCenterFFTGPU gpu; return BeamCenterFFTScore(width, height, mean, settings, gpu); - } catch (const JFJochException &e) { - // The card is shared with this run's analysis workers, so the memory the check above - // saw can be gone by the time it is asked for. A capture is not worth failing a run - // over: score it on the CPU instead - unless the device is gone. - spdlog::warn("Beam centre FFT: GPU scoring failed ({}), scoring on the CPU", e.what()); - cuda_throw_if_context_lost(); - cuda_clear_error(); // handled - see cuda_clear_error() } + spdlog::info("Beam centre FFT: scoring on the CPU - on the GPU it needs {:.2f} GB, over half of the card", + BeamCenterFFTGPU::DeviceMemoryNeeded(width, height) / 1e9); } #endif BeamCenterFFTCPU cpu; diff --git a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu index 98a7e8a88..1f8ddd1c3 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTGPU.cu +++ b/image_analysis/geom_refinement/BeamCenterFFTGPU.cu @@ -516,10 +516,6 @@ size_t BeamCenterFFTGPU::DeviceMemoryNeeded(int64_t width, int64_t height) { bool BeamCenterFFTGPU::FitsInDeviceMemory(int64_t width, int64_t height) { size_t free_bytes = 0, total_bytes = 0; - if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) - return false; - // Headroom for the allocator's own rounding, and for whatever else lands on a shared card - // between this question and the allocations it is asked about. - constexpr size_t HEADROOM = 512ull << 20; - return DeviceMemoryNeeded(width, height) + HEADROOM <= free_bytes; + Check(cudaMemGetInfo(&free_bytes, &total_bytes), "Beam centre FFT: reading the device memory"); + return DeviceMemoryNeeded(width, height) <= total_bytes / 2; } diff --git a/image_analysis/geom_refinement/BeamCenterFFTGPU.h b/image_analysis/geom_refinement/BeamCenterFFTGPU.h index f47d8eb81..57ddd19c1 100644 --- a/image_analysis/geom_refinement/BeamCenterFFTGPU.h +++ b/image_analysis/geom_refinement/BeamCenterFFTGPU.h @@ -42,8 +42,8 @@ public: // Peak device memory the two phases need for an image of this size: buffers plus the cuFFT // plans' own workspace. [[nodiscard]] static size_t DeviceMemoryNeeded(int64_t width, int64_t height); - // Whether the visible device has that much free right now. The card is shared - with the - // analysis workers of this run and, on a development box, with whatever else is on it - so a - // capture that would not fit runs on the CPU instead of taking the card down with it. + // Whether that is at most half of the current device's TOTAL memory - the rest is the analysis + // workers' beside it. Not what is free at the moment, so the same image on the same card always + // takes the same path; a capture over it runs on the CPU. [[nodiscard]] static bool FitsInDeviceMemory(int64_t width, int64_t height); }; diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 6d2db7ad0..93cee089f 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -27,7 +27,7 @@ inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_ if (!out_of_memory) return msg; size_t free_b = 0, total_b = 0; - if (cudaMemGetInfo(&free_b, &total_b) == cudaSuccess) { + if (get_gpu_free_memory(free_b, total_b)) { std::snprintf(buf, sizeof buf, ", with %.1f of the %.1f GB card free", free_b / 1e9, total_b / 1e9); msg += buf; } @@ -265,7 +265,7 @@ inline void cuda_trim_allocation_pool() { } // Whether `bytes` more device memory can be had on the current device, for a step whose buffers are -// sized by the data set and which ends the run when they do not fit. cudaMemGetInfo counts what the +// sized by the data set and which ends the run when they do not fit. The free memory counts what the // pool holds with no buffer in it (up to CUDA_MEM_POOL_RELEASE_THRESHOLD) as taken, so that is handed // back before the step gives up; and GPU work running beside it in this process (GPUWorkBeside) is // waited for, as its memory comes back when it ends. Only when the step runs depends on this, never @@ -273,7 +273,7 @@ inline void cuda_trim_allocation_pool() { inline bool cuda_memory_available(size_t bytes, size_t &free_bytes) { const auto read_free = [&] { size_t total_bytes = 0; - if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess) + if (!get_gpu_free_memory(free_bytes, total_bytes)) throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Failed to read free device memory"); return bytes <= free_bytes; }; diff --git a/image_analysis/scale_merge/ErrorModelGPU.cu b/image_analysis/scale_merge/ErrorModelGPU.cu index cd8c5a4c4..ef1302b34 100644 --- a/image_analysis/scale_merge/ErrorModelGPU.cu +++ b/image_analysis/scale_merge/ErrorModelGPU.cu @@ -96,7 +96,7 @@ ErrorModelFit FitErrorModelGPU(const std::vector &pool) { throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Error model: " + std::to_string(pool.size()) + " samples need " + std::to_string(BYTES_PER_SAMPLE * pool.size() >> 20) + " MiB of GPU memory, " - + std::to_string(free_bytes >> 20) + " MiB free - run the CPU build"); + + std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)"); CudaStream stream; CudaDevicePtr d_pool(pool.size(), ALLOC); diff --git a/image_analysis/scale_merge/ErrorModelGPU.h b/image_analysis/scale_merge/ErrorModelGPU.h index 31ee96f1c..ea3785efa 100644 --- a/image_analysis/scale_merge/ErrorModelGPU.h +++ b/image_analysis/scale_merge/ErrorModelGPU.h @@ -12,5 +12,5 @@ // (ErrorModelUpdate). What differs is the order each bin's two sums are added in - a fixed tree here, // the order nth_element left the bin in on the host - so a and b^2 can differ from the host fit in // their last bits. Deterministic: the same pool gives the same fit on every run. Only built with CUDA; -// a device without room for about 72 bytes a sample throws (run on the CPU build then). +// a device without room for about 72 bytes a sample throws (run on the CPU then: rugnux --no-gpu). ErrorModelFit FitErrorModelGPU(const std::vector &pool); diff --git a/image_analysis/scale_merge/PairSortGPU.cu b/image_analysis/scale_merge/PairSortGPU.cu index 50ef96811..c4a013890 100644 --- a/image_analysis/scale_merge/PairSortGPU.cu +++ b/image_analysis/scale_merge/PairSortGPU.cu @@ -51,7 +51,7 @@ void SortPairsGPU(std::vector> &v) { throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Sorting " + std::to_string(v.size()) + " pairs needs " + std::to_string(BYTES_PER_PAIR * v.size() >> 20) + " MiB of GPU memory, " - + std::to_string(free_bytes >> 20) + " MiB free - run the CPU build"); + + std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)"); const unsigned blocks = static_cast((n + THREADS - 1) / THREADS); CudaStream stream; CudaDevicePtr pair(n, ALLOC), out(n, ALLOC); diff --git a/image_analysis/scale_merge/PairSortGPU.h b/image_analysis/scale_merge/PairSortGPU.h index 85adda070..5ba4bf1b6 100644 --- a/image_analysis/scale_merge/PairSortGPU.h +++ b/image_analysis/scale_merge/PairSortGPU.h @@ -8,5 +8,5 @@ // Sort pairs of finite doubles ascending, first by .first then by .second, on the GPU. Two pairs that // compare equal are the same two numbers, so this is the sequence std::sort leaves, to the bit. Only -// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU build then). +// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU then: rugnux --no-gpu). void SortPairsGPU(std::vector> &v); diff --git a/image_analysis/scale_merge/RotationScaleMerge.cpp b/image_analysis/scale_merge/RotationScaleMerge.cpp index 6840ec09e..88e37bdb3 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.cpp +++ b/image_analysis/scale_merge/RotationScaleMerge.cpp @@ -638,6 +638,10 @@ void RotationScaleMerge::IngestOnDevice() { rawrun_k = std::move(layout.run_k); rawrun_l = std::move(layout.run_l); rawrun_d = std::move(layout.run_d); + // The scaling's footprint is known from here on: where the card cannot take it, the card's work + // ends before the smoothing. + if (ScaleOnCPUWhereTooLarge()) + return; const SmoothedGeometry sg = SmoothedFrameGeometry(); int64_t n_resolved = 0; @@ -688,6 +692,23 @@ void RotationScaleMerge::IngestOnDevice() { corr_ingested.resize(n); gpu_->IngestFinish(corr_ingested.data()); } + +// The scaling's footprint on the card is known once the ingest has counted the observations and the +// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run +// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the card +// but not on the moment. +bool RotationScaleMerge::ScaleOnCPUWhereTooLarge() { + const size_t card = get_gpu_total_memory(gpu_device); + if (GPUScalingPeakBytes() <= CPU_SCALING_ABOVE_CARD_FRACTION * card) + return false; + logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about " + "{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead", + n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9, + 100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9); + scale_on_cpu_ = true; + gpu_active_ = false; + return true; +} #endif bool RotationScaleMerge::OnGPU() const { @@ -749,26 +770,13 @@ void RotationScaleMerge::IngestDevice(const RotationScaleMerge *observations_fro if (gpu_active_ && observations_from) gpu_->CopyObservationsFrom(*observations_from->gpu_); else if (resident_ingest) - IngestOnDevice(); - // The scaling's footprint on the card is known once the ingest has counted the observations and the - // distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run - // scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the - // card but not on the moment. The CPU path builds its observations on the host, so a resident - // ingest is made again there. - if (gpu_active_ && !observations_from) { - const size_t card = get_gpu_total_memory(gpu_device); - if (GPUScalingPeakBytes() > CPU_SCALING_ABOVE_CARD_FRACTION * card) { - logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about " - "{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead", - n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9, - 100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9); - scale_on_cpu_ = true; - gpu_active_ = false; - if (resident_ingest) { - gpu_.reset(); - IngestHost(); - } - } + IngestOnDevice(); // decides ScaleOnCPUWhereTooLarge() itself, before its smoothing + else if (gpu_active_) + ScaleOnCPUWhereTooLarge(); + // The CPU path builds its observations on the host, so a resident ingest is made again there. + if (scale_on_cpu_ && resident_ingest) { + gpu_.reset(); + IngestHost(); } if (gpu_active_ && !resident_ingest && !observations_from) { const int n = static_cast(partials.size()); @@ -1669,6 +1677,7 @@ void RotationScaleMerge::FitPerFrameG(const std::vector &obs, const std::vect const int n_fr = static_cast(fstart.size()); ParallelFor(n_fr, nthreads, [&](int f) { std::vector so; + so.reserve(fcount[f]); // grown one by one, every thread queued on the heap's lock for (int i = fstart[f]; i < fstart[f] + fcount[f]; ++i) { const auto &o = obs[i]; if (o.group < 0) continue; diff --git a/image_analysis/scale_merge/RotationScaleMerge.h b/image_analysis/scale_merge/RotationScaleMerge.h index 34baebcd9..5dd3a0290 100644 --- a/image_analysis/scale_merge/RotationScaleMerge.h +++ b/image_analysis/scale_merge/RotationScaleMerge.h @@ -602,6 +602,9 @@ private: // The resident path's ingest (see IngestDevice): the observations and the raw-hkl runs built on the // card from the reflections, exactly as BuildInRangeObservations builds them on the host. void IngestOnDevice(); + // Where the scaling's estimated peak is over CPU_SCALING_ABOVE_CARD_FRACTION of the card: says so, + // gives the device up (scale_on_cpu_) and returns true. Call once the ingest has counted. + bool ScaleOnCPUWhereTooLarge(); #endif // Compute the dense ASU-group id for the current space group by grouping the (pre-sorted) raw-hkl diff --git a/image_analysis/scale_merge/RotationScaleMergeGPU.cu b/image_analysis/scale_merge/RotationScaleMergeGPU.cu index 2582589b5..4756fd580 100644 --- a/image_analysis/scale_merge/RotationScaleMergeGPU.cu +++ b/image_analysis/scale_merge/RotationScaleMergeGPU.cu @@ -1139,7 +1139,7 @@ namespace { // this process running on the GPU any more: what holds the rest is this run or another program. [[noreturn]] void ThrowOutOfGPUMemory(int64_t n_obs, size_t bytes) { size_t free_bytes = 0, total_bytes = 0; - cudaMemGetInfo(&free_bytes, &total_bytes); + get_gpu_free_memory(free_bytes, total_bytes); cudaGetLastError(); char msg[768]; snprintf(msg, sizeof(msg), diff --git a/image_analysis/scale_merge/WilsonOutliersGPU.cu b/image_analysis/scale_merge/WilsonOutliersGPU.cu index b877d07a7..f3fd947e7 100644 --- a/image_analysis/scale_merge/WilsonOutliersGPU.cu +++ b/image_analysis/scale_merge/WilsonOutliersGPU.cu @@ -234,7 +234,7 @@ WilsonOutlierResult WilsonOutliersGPU(const std::vector &obs, throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Wilson outlier test: " + std::to_string(obs.size()) + " observations need " + std::to_string(BYTES_PER_OBS * obs.size() >> 20) + " MiB of GPU memory, " - + std::to_string(free_bytes >> 20) + " MiB free - run the CPU build"); + + std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)"); CudaStream stream; CudaDevicePtr d_obs(obs.size(), ALLOC); diff --git a/image_analysis/scale_merge/WilsonOutliersGPU.h b/image_analysis/scale_merge/WilsonOutliersGPU.h index f0309fa96..2e33b129f 100644 --- a/image_analysis/scale_merge/WilsonOutliersGPU.h +++ b/image_analysis/scale_merge/WilsonOutliersGPU.h @@ -13,5 +13,5 @@ // logarithm are taken on the host. Only built with CUDA. // // Device memory is about 96 bytes per observation at the peak; a device that cannot hold it throws -// (run on the CPU build then). +// (run on the CPU then: rugnux --no-gpu). WilsonOutlierResult WilsonOutliersGPU(const std::vector &obs, double alpha); diff --git a/image_analysis/structure_refinement/RigidBodyGPU.cpp b/image_analysis/structure_refinement/RigidBodyGPU.cpp index 99509714b..bc5632942 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.cpp +++ b/image_analysis/structure_refinement/RigidBodyGPU.cpp @@ -157,15 +157,16 @@ std::unique_ptr RigidBodyGPUPool::Create(const gemmi::Model &m cap.terms = max_observations * ops; const size_t bytes = RigidBodyGPUEngine::DeviceBytes(cap); - size_t free = 0, total = 0; - RigidBodyGPUEngine::MemoryInfo(free, total); - constexpr size_t HEADROOM = 1ull << 30; - const size_t budget = std::min(total / 4, free > HEADROOM ? free - HEADROOM : 0); + // A quarter of the card's TOTAL memory - the validation's plan (StructureFactorsGPU) - and not what + // happens to be free: the same input on the same card always gets the same engines. A card that + // is merely busy is waited for by the engines' allocations, never left for the CPU. + const int device = RigidBodyGPUEngine::CurrentDevice(); + const size_t total = get_gpu_total_memory(device); + const size_t budget = total / 4; const size_t fit = bytes > 0 ? budget / bytes : 0; const size_t want = std::min(fit, std::max(max_engines, 1)); // Engine i on the i-th card from the calling thread's, so a validation uses every card there is; // the engines give the same bits on any card of one model, so this changes only where they run. - const int device = RigidBodyGPUEngine::CurrentDevice(); const int cards = get_gpu_count(); pool->device_ = device; pool->planned_bytes_ = bytes * std::max(max_engines, 1); @@ -174,8 +175,8 @@ std::unique_ptr RigidBodyGPUPool::Create(const gemmi::Model &m pool->engines_.push_back(std::make_unique(cap, (device + static_cast(i)) % cards)); set_gpu(device); // each engine was made on its own card if (pool->engines_.empty()) { - logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB and the " - "budget is {:.0f} MB", bytes / 1e6, budget / 1e6); + logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB, over a " + "quarter of the card's {:.0f} MB", bytes / 1e6, total / 1e6); return nullptr; } for (auto &e : pool->engines_) @@ -184,11 +185,7 @@ std::unique_ptr RigidBodyGPUPool::Create(const gemmi::Model &m bytes / 1e6); return pool; } catch (const JFJochException &e) { - logger.Warning("Model validation: the rigid body runs on the CPU - the GPU engines could not be set up ({})", - e.what()); - cuda_throw_if_context_lost(); - cuda_clear_error(); - return nullptr; + throw RigidBodyGPUFailure(e.what()); } } diff --git a/image_analysis/structure_refinement/RigidBodyGPU.cu b/image_analysis/structure_refinement/RigidBodyGPU.cu index b5a40a784..231d7d0b8 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.cu +++ b/image_analysis/structure_refinement/RigidBodyGPU.cu @@ -367,7 +367,8 @@ struct RigidBodyGPUEngineImpl { explicit RigidBodyGPUEngineImpl(const RigidBodyGPUCapacity &c, int dev) : device(dev), cap(c) { - const auto sync = CudaAlloc::Synchronous; + // A card busy with a probe pass beside the run is waited for, not given up on (RigidBodyGPUPool). + const auto sync = CudaAlloc::SynchronousAfterWorkBeside; const size_t na = std::max(cap.atoms, 1); rel = CudaDevicePtr(3 * na, sync); pos = CudaDevicePtr(na, sync); @@ -484,10 +485,6 @@ bool RigidBodyGPUEngine::Supports(const RigidBodyGPUZone &zone) { return ModelDensityGPU::Supports(DensityGrid(zone), zone.atoms); } -void RigidBodyGPUEngine::MemoryInfo(size_t &free, size_t &total) { - cuda_err(cudaMemGetInfo(&free, &total)); -} - int RigidBodyGPUEngine::CurrentDevice() { int device = 0; cuda_err(cudaGetDevice(&device)); diff --git a/image_analysis/structure_refinement/RigidBodyGPU.h b/image_analysis/structure_refinement/RigidBodyGPU.h index c2b8cb37f..628e8bc39 100644 --- a/image_analysis/structure_refinement/RigidBodyGPU.h +++ b/image_analysis/structure_refinement/RigidBodyGPU.h @@ -39,9 +39,9 @@ public: // radii, which the placement does not change - is worked out once per zone and kept. class RigidBodyGPUPool { public: - // Null where there is no GPU, where not even one engine fits the budget - a quarter of the card, - // and never the last gigabyte of what is free, since the merge may be running beside it - or where - // the cell is too small for the gather (an atom's box wider than the cell). Logged either way. + // Null where there is no GPU, where not even one engine fits the budget - a quarter of the card's + // total memory - or where the cell is too small for the gather (an atom's box wider than the cell). + // Logged either way. Engines that cannot be set up throw RigidBodyGPUFailure. // `max_observations`: the most working reflections any fit will be given in its finest zone. // `max_engines`: at most this many, however much memory there is. static std::unique_ptr Create(const gemmi::Model &model, const gemmi::UnitCell &cell, diff --git a/image_analysis/structure_refinement/RigidBodyGPUEngine.h b/image_analysis/structure_refinement/RigidBodyGPUEngine.h index e68ad8e51..1c1859e2c 100644 --- a/image_analysis/structure_refinement/RigidBodyGPUEngine.h +++ b/image_analysis/structure_refinement/RigidBodyGPUEngine.h @@ -93,7 +93,6 @@ public: // Whether the gather reproduces gemmi's box walk on this zone: every atom's box narrower than the cell, // so that no point is reached by two images of one atom. static bool Supports(const RigidBodyGPUZone &zone); - static void MemoryInfo(size_t &free, size_t &total); static int CurrentDevice(); RigidBodyGPUEngine(const RigidBodyGPUCapacity &capacity, int device); diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index a36205b67..5fe49147c 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -4281,7 +4281,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { Step("Molecular replacement"); TimingMark mark(logger, "model prior"); // As before the validation above, which a predicted model or --mr does not run: the engine's card - // memory goes back before the search sizes its rigid-body engines from what the card has free. + // memory goes back before the search reserves its rigid-body engines on the card. if (!ledger_ahead.valid()) rsm.reset(); std::vector merges{{"MERGE", &sm.merged, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}}; diff --git a/tests/CUDAMemHelpersTest.cpp b/tests/CUDAMemHelpersTest.cpp index c349d9d88..f3c38f1f9 100644 --- a/tests/CUDAMemHelpersTest.cpp +++ b/tests/CUDAMemHelpersTest.cpp @@ -17,6 +17,14 @@ #include "../image_analysis/indexing/CUDAMemHelpers.h" #include "../image_analysis/indexing/IndexerThreadPool.h" +// Whether the current GPU shares the host's memory (GB10, Jetson). A test that fills the card until an +// allocation fails would there fill the host's RAM - every process on the machine runs out with it. +static bool gpu_is_integrated() { + int device = 0, integrated = 0; + return cudaGetDevice(&device) == cudaSuccess + && cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, device) == cudaSuccess && integrated; +} + // The broker starts fresh worker threads for every data collection. A stream per thread ever started // is a device-memory leak that ends a long-running broker, so threads that follow one another have to // end up on the same stream. @@ -134,6 +142,8 @@ TEST_CASE("CudaPinnedPool_ReusedByTheNextEngine", "[CUDAMemHelpers]") { TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]") { if (get_gpu_count() == 0) SKIP("No CUDA GPU present"); + if (gpu_is_integrated()) + SKIP("Fills the card, which on an integrated GPU is the host's memory"); IndexingSettings settings; settings.Algorithm(IndexingAlgorithmEnum::FFT).IndexingThreads(1); @@ -212,6 +222,8 @@ TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers] TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]") { if (get_gpu_count() == 0) SKIP("no GPU"); + if (gpu_is_integrated()) + SKIP("Fills the card, which on an integrated GPU is the host's memory"); std::vector> held; const auto fill = [&] { for (size_t chunk = size_t(256) << 20; chunk >= (size_t(1) << 20); chunk /= 2) @@ -239,6 +251,24 @@ TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]") cuda_trim_allocation_pool(); } +// The free memory waited on is cudaMemGetInfo's on a discrete card. On one that shares the host's memory +// (GB10, Jetson) it also counts the page cache the host hands back, less a tenth of the card for the host: +// never less than cudaMemGetInfo's. Allocations between the readings move them, hence the 1 GB slack. +TEST_CASE("GetGpuFreeMemory_IntegratedCountsThePageCache", "[CUDAMemHelpers]") { + if (get_gpu_count() == 0) + SKIP("No CUDA GPU present"); + constexpr double SLACK = 1e9; + size_t free_bytes = 0, total_bytes = 0; + REQUIRE(get_gpu_free_memory(free_bytes, total_bytes)); + size_t cuda_free = 0, cuda_total = 0; + REQUIRE(cudaMemGetInfo(&cuda_free, &cuda_total) == cudaSuccess); + CHECK(total_bytes == cuda_total); + CHECK(free_bytes <= total_bytes); + CHECK(double(free_bytes) >= double(cuda_free) - SLACK); + if (!gpu_is_integrated()) + CHECK(double(free_bytes) == Catch::Approx(double(cuda_free)).margin(SLACK)); +} + #endif // A merge that finds the card full waits for the GPU work running beside it: until the last one diff --git a/tests/ModelValidationTest.cpp b/tests/ModelValidationTest.cpp index 2af466c49..2d0c0cbef 100644 --- a/tests/ModelValidationTest.cpp +++ b/tests/ModelValidationTest.cpp @@ -1182,18 +1182,11 @@ TEST_CASE("ModelValidation_RigidBodyTranslationDerivativeIsExact", "[ModelValida #ifdef JFJOCH_USE_CUDA namespace { - // A pool of `engines`, or a SKIP where there is none because the card is busy - over half its memory - // taken by something else, where the pool falls back to the CPU by design. Anywhere else no pool fails. + // A pool of `engines`: decided on the card's total memory, so there is one wherever there is a GPU. std::unique_ptr TestPool(const gemmi::Model &model, const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min, size_t observations, size_t engines, Logger &logger) { auto pool = RigidBodyGPUPool::Create(model, cell, sg, d_min, observations, engines, logger); - if (!pool) { - size_t free = 0, total = 0; - RigidBodyGPUEngine::MemoryInfo(free, total); - if (free < total / 2) - SKIP("No GPU engine: the card is busy (" << free / 1000000 << " of " << total / 1000000 << " MB free)"); - } REQUIRE(pool); return pool; }