Merge gpucpu-decide into rc175: GPU or CPU decided up front from the card's total memory, never from what is free

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-14 22:12:44 +02:00
commit 7f57cfb8d4
23 files changed
+143 -79

No files matched your search

+4
View File
@@ -103,6 +103,10 @@ size_t get_gpu_total_memory(int32_t) {
return 0;
}
bool get_gpu_free_memory(size_t &, size_t &) {
return false;
}
void set_gpu(int32_t dev_id) {}
void pin_gpu() {}
+33
View File
@@ -1,7 +1,9 @@
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <algorithm>
#include <fstream>
#include <string>
#include <thread>
#include <atomic>
#include <mutex>
@@ -66,6 +68,37 @@ size_t get_gpu_total_memory(int32_t dev_id) {
return prop.totalGlobalMem;
}
bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes) {
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
return false;
#ifdef __linux__
// Asked once: whether each card shares the host's memory.
static const std::vector<int> integrated = [] {
std::vector<int> v(std::max(get_gpu_count(), 0));
for (int i = 0; i < static_cast<int>(v.size()); i++)
if (cudaDeviceGetAttribute(&v[i], cudaDevAttrIntegrated, i) != cudaSuccess) {
v[i] = 0;
cudaGetLastError();
}
return v;
}();
int device = 0;
if (cudaGetDevice(&device) != cudaSuccess || device >= static_cast<int>(integrated.size()) || !integrated[device])
return true;
std::ifstream meminfo("/proc/meminfo");
std::string line;
while (std::getline(meminfo, line))
if (line.rfind("MemAvailable:", 0) == 0) {
const size_t available = std::stoull(line.substr(13)) * 1024; // given in kB
const size_t headroom = total_bytes / 10; // the host's own processes live in the same memory
if (available > headroom)
free_bytes = std::max(free_bytes, std::min(available - headroom, total_bytes));
break;
}
#endif
return true;
}
void set_gpu(int32_t dev_id) {
auto dev_count = get_gpu_count();
+8
View File
@@ -29,6 +29,14 @@ std::string get_gpu_description();
// device cannot be queried.
size_t get_gpu_total_memory(int32_t dev_id);
// Free and total memory of the current device in bytes, as cudaMemGetInfo gives them - except on a GPU
// that shares the host's memory (cudaDevAttrIntegrated: GB10, Jetson), whose cudaMemGetInfo free is the
// host's MemFree and leaves out the page cache the kernel hands back on demand. There, on Linux, free is
// the host's MemAvailable less a tenth of the total, at most the total. For waiting on memory and for
// messages only: what runs on the GPU is decided on the total (get_gpu_total_memory), never on this.
// False without CUDA, or when the device cannot be queried.
bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes);
void set_gpu(int32_t dev_id);
// Pin the calling thread to the next GPU in round-robin order, using a process-wide counter
@@ -250,18 +250,13 @@ BeamCenterFFTResult BeamCenterFFTScore(int64_t width, int64_t height,
const std::vector<float> &mean,
const BeamCenterFFTSettings &settings) {
#ifdef JFJOCH_USE_CUDA
if (get_gpu_count() > 0 && BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) {
BeamCenterFFTGPU gpu;
try {
if (get_gpu_count() > 0) {
if (BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) {
BeamCenterFFTGPU gpu;
return BeamCenterFFTScore(width, height, mean, settings, gpu);
} catch (const JFJochException &e) {
// The card is shared with this run's analysis workers, so the memory the check above
// saw can be gone by the time it is asked for. A capture is not worth failing a run
// over: score it on the CPU instead - unless the device is gone.
spdlog::warn("Beam centre FFT: GPU scoring failed ({}), scoring on the CPU", e.what());
cuda_throw_if_context_lost();
cuda_clear_error(); // handled - see cuda_clear_error()
}
spdlog::info("Beam centre FFT: scoring on the CPU - on the GPU it needs {:.2f} GB, over half of the card",
BeamCenterFFTGPU::DeviceMemoryNeeded(width, height) / 1e9);
}
#endif
BeamCenterFFTCPU cpu;
@@ -516,10 +516,6 @@ size_t BeamCenterFFTGPU::DeviceMemoryNeeded(int64_t width, int64_t height) {
bool BeamCenterFFTGPU::FitsInDeviceMemory(int64_t width, int64_t height) {
size_t free_bytes = 0, total_bytes = 0;
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
return false;
// Headroom for the allocator's own rounding, and for whatever else lands on a shared card
// between this question and the allocations it is asked about.
constexpr size_t HEADROOM = 512ull << 20;
return DeviceMemoryNeeded(width, height) + HEADROOM <= free_bytes;
Check(cudaMemGetInfo(&free_bytes, &total_bytes), "Beam centre FFT: reading the device memory");
return DeviceMemoryNeeded(width, height) <= total_bytes / 2;
}
@@ -42,8 +42,8 @@ public:
// Peak device memory the two phases need for an image of this size: buffers plus the cuFFT
// plans' own workspace.
[[nodiscard]] static size_t DeviceMemoryNeeded(int64_t width, int64_t height);
// Whether the visible device has that much free right now. The card is shared - with the
// analysis workers of this run and, on a development box, with whatever else is on it - so a
// capture that would not fit runs on the CPU instead of taking the card down with it.
// Whether that is at most half of the current device's TOTAL memory - the rest is the analysis
// workers' beside it. Not what is free at the moment, so the same image on the same card always
// takes the same path; a capture over it runs on the CPU.
[[nodiscard]] static bool FitsInDeviceMemory(int64_t width, int64_t height);
};
+3 -3
View File
@@ -27,7 +27,7 @@ inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_
if (!out_of_memory)
return msg;
size_t free_b = 0, total_b = 0;
if (cudaMemGetInfo(&free_b, &total_b) == cudaSuccess) {
if (get_gpu_free_memory(free_b, total_b)) {
std::snprintf(buf, sizeof buf, ", with %.1f of the %.1f GB card free", free_b / 1e9, total_b / 1e9);
msg += buf;
}
@@ -265,7 +265,7 @@ inline void cuda_trim_allocation_pool() {
}
// Whether `bytes` more device memory can be had on the current device, for a step whose buffers are
// sized by the data set and which ends the run when they do not fit. cudaMemGetInfo counts what the
// sized by the data set and which ends the run when they do not fit. The free memory counts what the
// pool holds with no buffer in it (up to CUDA_MEM_POOL_RELEASE_THRESHOLD) as taken, so that is handed
// back before the step gives up; and GPU work running beside it in this process (GPUWorkBeside) is
// waited for, as its memory comes back when it ends. Only when the step runs depends on this, never
@@ -273,7 +273,7 @@ inline void cuda_trim_allocation_pool() {
inline bool cuda_memory_available(size_t bytes, size_t &free_bytes) {
const auto read_free = [&] {
size_t total_bytes = 0;
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
if (!get_gpu_free_memory(free_bytes, total_bytes))
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Failed to read free device memory");
return bytes <= free_bytes;
};
+1 -1
View File
@@ -96,7 +96,7 @@ ErrorModelFit FitErrorModelGPU(const std::vector<ErrorModelSample> &pool) {
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
"Error model: " + std::to_string(pool.size()) + " samples need "
+ std::to_string(BYTES_PER_SAMPLE * pool.size() >> 20) + " MiB of GPU memory, "
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
CudaStream stream;
CudaDevicePtr<ErrorModelSample> d_pool(pool.size(), ALLOC);
+1 -1
View File
@@ -12,5 +12,5 @@
// (ErrorModelUpdate). What differs is the order each bin's two sums are added in - a fixed tree here,
// the order nth_element left the bin in on the host - so a and b^2 can differ from the host fit in
// their last bits. Deterministic: the same pool gives the same fit on every run. Only built with CUDA;
// a device without room for about 72 bytes a sample throws (run on the CPU build then).
// a device without room for about 72 bytes a sample throws (run on the CPU then: rugnux --no-gpu).
ErrorModelFit FitErrorModelGPU(const std::vector<ErrorModelSample> &pool);
+1 -1
View File
@@ -51,7 +51,7 @@ void SortPairsGPU(std::vector<std::pair<double, double>> &v) {
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
"Sorting " + std::to_string(v.size()) + " pairs needs "
+ std::to_string(BYTES_PER_PAIR * v.size() >> 20) + " MiB of GPU memory, "
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
const unsigned blocks = static_cast<unsigned>((n + THREADS - 1) / THREADS);
CudaStream stream;
CudaDevicePtr<double2> pair(n, ALLOC), out(n, ALLOC);
+1 -1
View File
@@ -8,5 +8,5 @@
// Sort pairs of finite doubles ascending, first by .first then by .second, on the GPU. Two pairs that
// compare equal are the same two numbers, so this is the sequence std::sort leaves, to the bit. Only
// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU build then).
// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU then: rugnux --no-gpu).
void SortPairsGPU(std::vector<std::pair<double, double>> &v);
@@ -638,6 +638,10 @@ void RotationScaleMerge::IngestOnDevice() {
rawrun_k = std::move(layout.run_k);
rawrun_l = std::move(layout.run_l);
rawrun_d = std::move(layout.run_d);
// The scaling's footprint is known from here on: where the card cannot take it, the card's work
// ends before the smoothing.
if (ScaleOnCPUWhereTooLarge())
return;
const SmoothedGeometry sg = SmoothedFrameGeometry();
int64_t n_resolved = 0;
@@ -688,6 +692,23 @@ void RotationScaleMerge::IngestOnDevice() {
corr_ingested.resize(n);
gpu_->IngestFinish(corr_ingested.data());
}
// The scaling's footprint on the card is known once the ingest has counted the observations and the
// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run
// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the card
// but not on the moment.
bool RotationScaleMerge::ScaleOnCPUWhereTooLarge() {
const size_t card = get_gpu_total_memory(gpu_device);
if (GPUScalingPeakBytes() <= CPU_SCALING_ABOVE_CARD_FRACTION * card)
return false;
logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about "
"{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead",
n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9,
100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9);
scale_on_cpu_ = true;
gpu_active_ = false;
return true;
}
#endif
bool RotationScaleMerge::OnGPU() const {
@@ -749,26 +770,13 @@ void RotationScaleMerge::IngestDevice(const RotationScaleMerge *observations_fro
if (gpu_active_ && observations_from)
gpu_->CopyObservationsFrom(*observations_from->gpu_);
else if (resident_ingest)
IngestOnDevice();
// The scaling's footprint on the card is known once the ingest has counted the observations and the
// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run
// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the
// card but not on the moment. The CPU path builds its observations on the host, so a resident
// ingest is made again there.
if (gpu_active_ && !observations_from) {
const size_t card = get_gpu_total_memory(gpu_device);
if (GPUScalingPeakBytes() > CPU_SCALING_ABOVE_CARD_FRACTION * card) {
logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about "
"{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead",
n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9,
100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9);
scale_on_cpu_ = true;
gpu_active_ = false;
if (resident_ingest) {
gpu_.reset();
IngestHost();
}
}
IngestOnDevice(); // decides ScaleOnCPUWhereTooLarge() itself, before its smoothing
else if (gpu_active_)
ScaleOnCPUWhereTooLarge();
// The CPU path builds its observations on the host, so a resident ingest is made again there.
if (scale_on_cpu_ && resident_ingest) {
gpu_.reset();
IngestHost();
}
if (gpu_active_ && !resident_ingest && !observations_from) {
const int n = static_cast<int>(partials.size());
@@ -1669,6 +1677,7 @@ void RotationScaleMerge::FitPerFrameG(const std::vector<O> &obs, const std::vect
const int n_fr = static_cast<int>(fstart.size());
ParallelFor(n_fr, nthreads, [&](int f) {
std::vector<ScaleObs> so;
so.reserve(fcount[f]); // grown one by one, every thread queued on the heap's lock
for (int i = fstart[f]; i < fstart[f] + fcount[f]; ++i) {
const auto &o = obs[i];
if (o.group < 0) continue;
@@ -602,6 +602,9 @@ private:
// The resident path's ingest (see IngestDevice): the observations and the raw-hkl runs built on the
// card from the reflections, exactly as BuildInRangeObservations builds them on the host.
void IngestOnDevice();
// Where the scaling's estimated peak is over CPU_SCALING_ABOVE_CARD_FRACTION of the card: says so,
// gives the device up (scale_on_cpu_) and returns true. Call once the ingest has counted.
bool ScaleOnCPUWhereTooLarge();
#endif
// Compute the dense ASU-group id for the current space group by grouping the (pre-sorted) raw-hkl
@@ -1139,7 +1139,7 @@ namespace {
// this process running on the GPU any more: what holds the rest is this run or another program.
[[noreturn]] void ThrowOutOfGPUMemory(int64_t n_obs, size_t bytes) {
size_t free_bytes = 0, total_bytes = 0;
cudaMemGetInfo(&free_bytes, &total_bytes);
get_gpu_free_memory(free_bytes, total_bytes);
cudaGetLastError();
char msg[768];
snprintf(msg, sizeof(msg),
@@ -234,7 +234,7 @@ WilsonOutlierResult WilsonOutliersGPU(const std::vector<WilsonObservation> &obs,
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
"Wilson outlier test: " + std::to_string(obs.size()) + " observations need "
+ std::to_string(BYTES_PER_OBS * obs.size() >> 20) + " MiB of GPU memory, "
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
CudaStream stream;
CudaDevicePtr<WilsonObservation> d_obs(obs.size(), ALLOC);
@@ -13,5 +13,5 @@
// logarithm are taken on the host. Only built with CUDA.
//
// Device memory is about 96 bytes per observation at the peak; a device that cannot hold it throws
// (run on the CPU build then).
// (run on the CPU then: rugnux --no-gpu).
WilsonOutlierResult WilsonOutliersGPU(const std::vector<WilsonObservation> &obs, double alpha);
@@ -157,15 +157,16 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
cap.terms = max_observations * ops;
const size_t bytes = RigidBodyGPUEngine::DeviceBytes(cap);
size_t free = 0, total = 0;
RigidBodyGPUEngine::MemoryInfo(free, total);
constexpr size_t HEADROOM = 1ull << 30;
const size_t budget = std::min(total / 4, free > HEADROOM ? free - HEADROOM : 0);
// A quarter of the card's TOTAL memory - the validation's plan (StructureFactorsGPU) - and not what
// happens to be free: the same input on the same card always gets the same engines. A card that
// is merely busy is waited for by the engines' allocations, never left for the CPU.
const int device = RigidBodyGPUEngine::CurrentDevice();
const size_t total = get_gpu_total_memory(device);
const size_t budget = total / 4;
const size_t fit = bytes > 0 ? budget / bytes : 0;
const size_t want = std::min<size_t>(fit, std::max<size_t>(max_engines, 1));
// Engine i on the i-th card from the calling thread's, so a validation uses every card there is;
// the engines give the same bits on any card of one model, so this changes only where they run.
const int device = RigidBodyGPUEngine::CurrentDevice();
const int cards = get_gpu_count();
pool->device_ = device;
pool->planned_bytes_ = bytes * std::max<size_t>(max_engines, 1);
@@ -174,8 +175,8 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
pool->engines_.push_back(std::make_unique<RigidBodyGPUEngine>(cap, (device + static_cast<int>(i)) % cards));
set_gpu(device); // each engine was made on its own card
if (pool->engines_.empty()) {
logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB and the "
"budget is {:.0f} MB", bytes / 1e6, budget / 1e6);
logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB, over a "
"quarter of the card's {:.0f} MB", bytes / 1e6, total / 1e6);
return nullptr;
}
for (auto &e : pool->engines_)
@@ -184,11 +185,7 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
bytes / 1e6);
return pool;
} catch (const JFJochException &e) {
logger.Warning("Model validation: the rigid body runs on the CPU - the GPU engines could not be set up ({})",
e.what());
cuda_throw_if_context_lost();
cuda_clear_error();
return nullptr;
throw RigidBodyGPUFailure(e.what());
}
}
@@ -367,7 +367,8 @@ struct RigidBodyGPUEngineImpl {
explicit RigidBodyGPUEngineImpl(const RigidBodyGPUCapacity &c, int dev)
: device(dev), cap(c) {
const auto sync = CudaAlloc::Synchronous;
// A card busy with a probe pass beside the run is waited for, not given up on (RigidBodyGPUPool).
const auto sync = CudaAlloc::SynchronousAfterWorkBeside;
const size_t na = std::max<size_t>(cap.atoms, 1);
rel = CudaDevicePtr<double>(3 * na, sync);
pos = CudaDevicePtr<float4>(na, sync);
@@ -484,10 +485,6 @@ bool RigidBodyGPUEngine::Supports(const RigidBodyGPUZone &zone) {
return ModelDensityGPU::Supports(DensityGrid(zone), zone.atoms);
}
void RigidBodyGPUEngine::MemoryInfo(size_t &free, size_t &total) {
cuda_err(cudaMemGetInfo(&free, &total));
}
int RigidBodyGPUEngine::CurrentDevice() {
int device = 0;
cuda_err(cudaGetDevice(&device));
@@ -39,9 +39,9 @@ public:
// radii, which the placement does not change - is worked out once per zone and kept.
class RigidBodyGPUPool {
public:
// Null where there is no GPU, where not even one engine fits the budget - a quarter of the card,
// and never the last gigabyte of what is free, since the merge may be running beside it - or where
// the cell is too small for the gather (an atom's box wider than the cell). Logged either way.
// Null where there is no GPU, where not even one engine fits the budget - a quarter of the card's
// total memory - or where the cell is too small for the gather (an atom's box wider than the cell).
// Logged either way. Engines that cannot be set up throw RigidBodyGPUFailure.
// `max_observations`: the most working reflections any fit will be given in its finest zone.
// `max_engines`: at most this many, however much memory there is.
static std::unique_ptr<RigidBodyGPUPool> Create(const gemmi::Model &model, const gemmi::UnitCell &cell,
@@ -93,7 +93,6 @@ public:
// Whether the gather reproduces gemmi's box walk on this zone: every atom's box narrower than the cell,
// so that no point is reached by two images of one atom.
static bool Supports(const RigidBodyGPUZone &zone);
static void MemoryInfo(size_t &free, size_t &total);
static int CurrentDevice();
RigidBodyGPUEngine(const RigidBodyGPUCapacity &capacity, int device);
+1 -1
View File
@@ -4281,7 +4281,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
Step("Molecular replacement");
TimingMark mark(logger, "model prior");
// As before the validation above, which a predicted model or --mr does not run: the engine's card
// memory goes back before the search sizes its rigid-body engines from what the card has free.
// memory goes back before the search reserves its rigid-body engines on the card.
if (!ledger_ahead.valid())
rsm.reset();
std::vector<MRMerge> merges{{"MERGE", &sm.merged, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}};
+30
View File
@@ -17,6 +17,14 @@
#include "../image_analysis/indexing/CUDAMemHelpers.h"
#include "../image_analysis/indexing/IndexerThreadPool.h"
// Whether the current GPU shares the host's memory (GB10, Jetson). A test that fills the card until an
// allocation fails would there fill the host's RAM - every process on the machine runs out with it.
static bool gpu_is_integrated() {
int device = 0, integrated = 0;
return cudaGetDevice(&device) == cudaSuccess
&& cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, device) == cudaSuccess && integrated;
}
// The broker starts fresh worker threads for every data collection. A stream per thread ever started
// is a device-memory leak that ends a long-running broker, so threads that follow one another have to
// end up on the same stream.
@@ -134,6 +142,8 @@ TEST_CASE("CudaPinnedPool_ReusedByTheNextEngine", "[CUDAMemHelpers]") {
TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]") {
if (get_gpu_count() == 0)
SKIP("No CUDA GPU present");
if (gpu_is_integrated())
SKIP("Fills the card, which on an integrated GPU is the host's memory");
IndexingSettings settings;
settings.Algorithm(IndexingAlgorithmEnum::FFT).IndexingThreads(1);
@@ -212,6 +222,8 @@ TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]
TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]") {
if (get_gpu_count() == 0)
SKIP("no GPU");
if (gpu_is_integrated())
SKIP("Fills the card, which on an integrated GPU is the host's memory");
std::vector<CudaDevicePtr<uint8_t>> held;
const auto fill = [&] {
for (size_t chunk = size_t(256) << 20; chunk >= (size_t(1) << 20); chunk /= 2)
@@ -239,6 +251,24 @@ TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]")
cuda_trim_allocation_pool();
}
// The free memory waited on is cudaMemGetInfo's on a discrete card. On one that shares the host's memory
// (GB10, Jetson) it also counts the page cache the host hands back, less a tenth of the card for the host:
// never less than cudaMemGetInfo's. Allocations between the readings move them, hence the 1 GB slack.
TEST_CASE("GetGpuFreeMemory_IntegratedCountsThePageCache", "[CUDAMemHelpers]") {
if (get_gpu_count() == 0)
SKIP("No CUDA GPU present");
constexpr double SLACK = 1e9;
size_t free_bytes = 0, total_bytes = 0;
REQUIRE(get_gpu_free_memory(free_bytes, total_bytes));
size_t cuda_free = 0, cuda_total = 0;
REQUIRE(cudaMemGetInfo(&cuda_free, &cuda_total) == cudaSuccess);
CHECK(total_bytes == cuda_total);
CHECK(free_bytes <= total_bytes);
CHECK(double(free_bytes) >= double(cuda_free) - SLACK);
if (!gpu_is_integrated())
CHECK(double(free_bytes) == Catch::Approx(double(cuda_free)).margin(SLACK));
}
#endif
// A merge that finds the card full waits for the GPU work running beside it: until the last one
+1 -8
View File
@@ -1182,18 +1182,11 @@ TEST_CASE("ModelValidation_RigidBodyTranslationDerivativeIsExact", "[ModelValida
#ifdef JFJOCH_USE_CUDA
namespace {
// A pool of `engines`, or a SKIP where there is none because the card is busy - over half its memory
// taken by something else, where the pool falls back to the CPU by design. Anywhere else no pool fails.
// A pool of `engines`: decided on the card's total memory, so there is one wherever there is a GPU.
std::unique_ptr<RigidBodyGPUPool> TestPool(const gemmi::Model &model, const gemmi::UnitCell &cell,
const gemmi::SpaceGroup &sg, double d_min, size_t observations,
size_t engines, Logger &logger) {
auto pool = RigidBodyGPUPool::Create(model, cell, sg, d_min, observations, engines, logger);
if (!pool) {
size_t free = 0, total = 0;
RigidBodyGPUEngine::MemoryInfo(free, total);
if (free < total / 2)
SKIP("No GPU engine: the card is busy (" << free / 1000000 << " of " << total / 1000000 << " MB free)");
}
REQUIRE(pool);
return pool;
}