Merge gpucpu-decide into rc175: GPU or CPU decided up front from the card's total memory, never from what is free
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
commit
7f57cfb8d4
23 files changed
+143
-79
No files matched your search
@@ -103,6 +103,10 @@ size_t get_gpu_total_memory(int32_t) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool get_gpu_free_memory(size_t &, size_t &) {
|
||||
return false;
|
||||
}
|
||||
|
||||
void set_gpu(int32_t dev_id) {}
|
||||
|
||||
void pin_gpu() {}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
||||
// SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
#include <algorithm>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
@@ -66,6 +68,37 @@ size_t get_gpu_total_memory(int32_t dev_id) {
|
||||
return prop.totalGlobalMem;
|
||||
}
|
||||
|
||||
bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes) {
|
||||
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
|
||||
return false;
|
||||
#ifdef __linux__
|
||||
// Asked once: whether each card shares the host's memory.
|
||||
static const std::vector<int> integrated = [] {
|
||||
std::vector<int> v(std::max(get_gpu_count(), 0));
|
||||
for (int i = 0; i < static_cast<int>(v.size()); i++)
|
||||
if (cudaDeviceGetAttribute(&v[i], cudaDevAttrIntegrated, i) != cudaSuccess) {
|
||||
v[i] = 0;
|
||||
cudaGetLastError();
|
||||
}
|
||||
return v;
|
||||
}();
|
||||
int device = 0;
|
||||
if (cudaGetDevice(&device) != cudaSuccess || device >= static_cast<int>(integrated.size()) || !integrated[device])
|
||||
return true;
|
||||
std::ifstream meminfo("/proc/meminfo");
|
||||
std::string line;
|
||||
while (std::getline(meminfo, line))
|
||||
if (line.rfind("MemAvailable:", 0) == 0) {
|
||||
const size_t available = std::stoull(line.substr(13)) * 1024; // given in kB
|
||||
const size_t headroom = total_bytes / 10; // the host's own processes live in the same memory
|
||||
if (available > headroom)
|
||||
free_bytes = std::max(free_bytes, std::min(available - headroom, total_bytes));
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
|
||||
void set_gpu(int32_t dev_id) {
|
||||
auto dev_count = get_gpu_count();
|
||||
|
||||
|
||||
@@ -29,6 +29,14 @@ std::string get_gpu_description();
|
||||
// device cannot be queried.
|
||||
size_t get_gpu_total_memory(int32_t dev_id);
|
||||
|
||||
// Free and total memory of the current device in bytes, as cudaMemGetInfo gives them - except on a GPU
|
||||
// that shares the host's memory (cudaDevAttrIntegrated: GB10, Jetson), whose cudaMemGetInfo free is the
|
||||
// host's MemFree and leaves out the page cache the kernel hands back on demand. There, on Linux, free is
|
||||
// the host's MemAvailable less a tenth of the total, at most the total. For waiting on memory and for
|
||||
// messages only: what runs on the GPU is decided on the total (get_gpu_total_memory), never on this.
|
||||
// False without CUDA, or when the device cannot be queried.
|
||||
bool get_gpu_free_memory(size_t &free_bytes, size_t &total_bytes);
|
||||
|
||||
void set_gpu(int32_t dev_id);
|
||||
|
||||
// Pin the calling thread to the next GPU in round-robin order, using a process-wide counter
|
||||
|
||||
@@ -250,18 +250,13 @@ BeamCenterFFTResult BeamCenterFFTScore(int64_t width, int64_t height,
|
||||
const std::vector<float> &mean,
|
||||
const BeamCenterFFTSettings &settings) {
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
if (get_gpu_count() > 0 && BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) {
|
||||
BeamCenterFFTGPU gpu;
|
||||
try {
|
||||
if (get_gpu_count() > 0) {
|
||||
if (BeamCenterFFTGPU::FitsInDeviceMemory(width, height)) {
|
||||
BeamCenterFFTGPU gpu;
|
||||
return BeamCenterFFTScore(width, height, mean, settings, gpu);
|
||||
} catch (const JFJochException &e) {
|
||||
// The card is shared with this run's analysis workers, so the memory the check above
|
||||
// saw can be gone by the time it is asked for. A capture is not worth failing a run
|
||||
// over: score it on the CPU instead - unless the device is gone.
|
||||
spdlog::warn("Beam centre FFT: GPU scoring failed ({}), scoring on the CPU", e.what());
|
||||
cuda_throw_if_context_lost();
|
||||
cuda_clear_error(); // handled - see cuda_clear_error()
|
||||
}
|
||||
spdlog::info("Beam centre FFT: scoring on the CPU - on the GPU it needs {:.2f} GB, over half of the card",
|
||||
BeamCenterFFTGPU::DeviceMemoryNeeded(width, height) / 1e9);
|
||||
}
|
||||
#endif
|
||||
BeamCenterFFTCPU cpu;
|
||||
|
||||
@@ -516,10 +516,6 @@ size_t BeamCenterFFTGPU::DeviceMemoryNeeded(int64_t width, int64_t height) {
|
||||
|
||||
bool BeamCenterFFTGPU::FitsInDeviceMemory(int64_t width, int64_t height) {
|
||||
size_t free_bytes = 0, total_bytes = 0;
|
||||
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
|
||||
return false;
|
||||
// Headroom for the allocator's own rounding, and for whatever else lands on a shared card
|
||||
// between this question and the allocations it is asked about.
|
||||
constexpr size_t HEADROOM = 512ull << 20;
|
||||
return DeviceMemoryNeeded(width, height) + HEADROOM <= free_bytes;
|
||||
Check(cudaMemGetInfo(&free_bytes, &total_bytes), "Beam centre FFT: reading the device memory");
|
||||
return DeviceMemoryNeeded(width, height) <= total_bytes / 2;
|
||||
}
|
||||
@@ -42,8 +42,8 @@ public:
|
||||
// Peak device memory the two phases need for an image of this size: buffers plus the cuFFT
|
||||
// plans' own workspace.
|
||||
[[nodiscard]] static size_t DeviceMemoryNeeded(int64_t width, int64_t height);
|
||||
// Whether the visible device has that much free right now. The card is shared - with the
|
||||
// analysis workers of this run and, on a development box, with whatever else is on it - so a
|
||||
// capture that would not fit runs on the CPU instead of taking the card down with it.
|
||||
// Whether that is at most half of the current device's TOTAL memory - the rest is the analysis
|
||||
// workers' beside it. Not what is free at the moment, so the same image on the same card always
|
||||
// takes the same path; a capture over it runs on the CPU.
|
||||
[[nodiscard]] static bool FitsInDeviceMemory(int64_t width, int64_t height);
|
||||
};
|
||||
@@ -27,7 +27,7 @@ inline std::string GPUMemoryFailure(const char *what, size_t bytes, bool out_of_
|
||||
if (!out_of_memory)
|
||||
return msg;
|
||||
size_t free_b = 0, total_b = 0;
|
||||
if (cudaMemGetInfo(&free_b, &total_b) == cudaSuccess) {
|
||||
if (get_gpu_free_memory(free_b, total_b)) {
|
||||
std::snprintf(buf, sizeof buf, ", with %.1f of the %.1f GB card free", free_b / 1e9, total_b / 1e9);
|
||||
msg += buf;
|
||||
}
|
||||
@@ -265,7 +265,7 @@ inline void cuda_trim_allocation_pool() {
|
||||
}
|
||||
|
||||
// Whether `bytes` more device memory can be had on the current device, for a step whose buffers are
|
||||
// sized by the data set and which ends the run when they do not fit. cudaMemGetInfo counts what the
|
||||
// sized by the data set and which ends the run when they do not fit. The free memory counts what the
|
||||
// pool holds with no buffer in it (up to CUDA_MEM_POOL_RELEASE_THRESHOLD) as taken, so that is handed
|
||||
// back before the step gives up; and GPU work running beside it in this process (GPUWorkBeside) is
|
||||
// waited for, as its memory comes back when it ends. Only when the step runs depends on this, never
|
||||
@@ -273,7 +273,7 @@ inline void cuda_trim_allocation_pool() {
|
||||
inline bool cuda_memory_available(size_t bytes, size_t &free_bytes) {
|
||||
const auto read_free = [&] {
|
||||
size_t total_bytes = 0;
|
||||
if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
|
||||
if (!get_gpu_free_memory(free_bytes, total_bytes))
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Failed to read free device memory");
|
||||
return bytes <= free_bytes;
|
||||
};
|
||||
|
||||
@@ -96,7 +96,7 @@ ErrorModelFit FitErrorModelGPU(const std::vector<ErrorModelSample> &pool) {
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
|
||||
"Error model: " + std::to_string(pool.size()) + " samples need "
|
||||
+ std::to_string(BYTES_PER_SAMPLE * pool.size() >> 20) + " MiB of GPU memory, "
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
|
||||
|
||||
CudaStream stream;
|
||||
CudaDevicePtr<ErrorModelSample> d_pool(pool.size(), ALLOC);
|
||||
|
||||
@@ -12,5 +12,5 @@
|
||||
// (ErrorModelUpdate). What differs is the order each bin's two sums are added in - a fixed tree here,
|
||||
// the order nth_element left the bin in on the host - so a and b^2 can differ from the host fit in
|
||||
// their last bits. Deterministic: the same pool gives the same fit on every run. Only built with CUDA;
|
||||
// a device without room for about 72 bytes a sample throws (run on the CPU build then).
|
||||
// a device without room for about 72 bytes a sample throws (run on the CPU then: rugnux --no-gpu).
|
||||
ErrorModelFit FitErrorModelGPU(const std::vector<ErrorModelSample> &pool);
|
||||
@@ -51,7 +51,7 @@ void SortPairsGPU(std::vector<std::pair<double, double>> &v) {
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
|
||||
"Sorting " + std::to_string(v.size()) + " pairs needs "
|
||||
+ std::to_string(BYTES_PER_PAIR * v.size() >> 20) + " MiB of GPU memory, "
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
|
||||
const unsigned blocks = static_cast<unsigned>((n + THREADS - 1) / THREADS);
|
||||
CudaStream stream;
|
||||
CudaDevicePtr<double2> pair(n, ALLOC), out(n, ALLOC);
|
||||
|
||||
@@ -8,5 +8,5 @@
|
||||
|
||||
// Sort pairs of finite doubles ascending, first by .first then by .second, on the GPU. Two pairs that
|
||||
// compare equal are the same two numbers, so this is the sequence std::sort leaves, to the bit. Only
|
||||
// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU build then).
|
||||
// built with CUDA; a device without room for about 40 bytes a pair throws (run on the CPU then: rugnux --no-gpu).
|
||||
void SortPairsGPU(std::vector<std::pair<double, double>> &v);
|
||||
@@ -638,6 +638,10 @@ void RotationScaleMerge::IngestOnDevice() {
|
||||
rawrun_k = std::move(layout.run_k);
|
||||
rawrun_l = std::move(layout.run_l);
|
||||
rawrun_d = std::move(layout.run_d);
|
||||
// The scaling's footprint is known from here on: where the card cannot take it, the card's work
|
||||
// ends before the smoothing.
|
||||
if (ScaleOnCPUWhereTooLarge())
|
||||
return;
|
||||
|
||||
const SmoothedGeometry sg = SmoothedFrameGeometry();
|
||||
int64_t n_resolved = 0;
|
||||
@@ -688,6 +692,23 @@ void RotationScaleMerge::IngestOnDevice() {
|
||||
corr_ingested.resize(n);
|
||||
gpu_->IngestFinish(corr_ingested.data());
|
||||
}
|
||||
|
||||
// The scaling's footprint on the card is known once the ingest has counted the observations and the
|
||||
// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run
|
||||
// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the card
|
||||
// but not on the moment.
|
||||
bool RotationScaleMerge::ScaleOnCPUWhereTooLarge() {
|
||||
const size_t card = get_gpu_total_memory(gpu_device);
|
||||
if (GPUScalingPeakBytes() <= CPU_SCALING_ABOVE_CARD_FRACTION * card)
|
||||
return false;
|
||||
logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about "
|
||||
"{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead",
|
||||
n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9,
|
||||
100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9);
|
||||
scale_on_cpu_ = true;
|
||||
gpu_active_ = false;
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool RotationScaleMerge::OnGPU() const {
|
||||
@@ -749,26 +770,13 @@ void RotationScaleMerge::IngestDevice(const RotationScaleMerge *observations_fro
|
||||
if (gpu_active_ && observations_from)
|
||||
gpu_->CopyObservationsFrom(*observations_from->gpu_);
|
||||
else if (resident_ingest)
|
||||
IngestOnDevice();
|
||||
// The scaling's footprint on the card is known once the ingest has counted the observations and the
|
||||
// distinct hkl, before anything is scaled. Where it is clearly more than the card can take, the run
|
||||
// scales on the CPU - decided here, once, on the card's total memory, so the answer depends on the
|
||||
// card but not on the moment. The CPU path builds its observations on the host, so a resident
|
||||
// ingest is made again there.
|
||||
if (gpu_active_ && !observations_from) {
|
||||
const size_t card = get_gpu_total_memory(gpu_device);
|
||||
if (GPUScalingPeakBytes() > CPU_SCALING_ABOVE_CARD_FRACTION * card) {
|
||||
logger->Warning("RotationScaleMerge: scaling {} observations of {} distinct hkl on the GPU needs about "
|
||||
"{:.1f} GB, over {:.0f}% of the card's {:.1f} GB - scaling on the CPU instead",
|
||||
n_partials_obs, rawrun_start.size(), GPUScalingPeakBytes() / 1e9,
|
||||
100.0 * CPU_SCALING_ABOVE_CARD_FRACTION, card / 1e9);
|
||||
scale_on_cpu_ = true;
|
||||
gpu_active_ = false;
|
||||
if (resident_ingest) {
|
||||
gpu_.reset();
|
||||
IngestHost();
|
||||
}
|
||||
}
|
||||
IngestOnDevice(); // decides ScaleOnCPUWhereTooLarge() itself, before its smoothing
|
||||
else if (gpu_active_)
|
||||
ScaleOnCPUWhereTooLarge();
|
||||
// The CPU path builds its observations on the host, so a resident ingest is made again there.
|
||||
if (scale_on_cpu_ && resident_ingest) {
|
||||
gpu_.reset();
|
||||
IngestHost();
|
||||
}
|
||||
if (gpu_active_ && !resident_ingest && !observations_from) {
|
||||
const int n = static_cast<int>(partials.size());
|
||||
@@ -1669,6 +1677,7 @@ void RotationScaleMerge::FitPerFrameG(const std::vector<O> &obs, const std::vect
|
||||
const int n_fr = static_cast<int>(fstart.size());
|
||||
ParallelFor(n_fr, nthreads, [&](int f) {
|
||||
std::vector<ScaleObs> so;
|
||||
so.reserve(fcount[f]); // grown one by one, every thread queued on the heap's lock
|
||||
for (int i = fstart[f]; i < fstart[f] + fcount[f]; ++i) {
|
||||
const auto &o = obs[i];
|
||||
if (o.group < 0) continue;
|
||||
|
||||
@@ -602,6 +602,9 @@ private:
|
||||
// The resident path's ingest (see IngestDevice): the observations and the raw-hkl runs built on the
|
||||
// card from the reflections, exactly as BuildInRangeObservations builds them on the host.
|
||||
void IngestOnDevice();
|
||||
// Where the scaling's estimated peak is over CPU_SCALING_ABOVE_CARD_FRACTION of the card: says so,
|
||||
// gives the device up (scale_on_cpu_) and returns true. Call once the ingest has counted.
|
||||
bool ScaleOnCPUWhereTooLarge();
|
||||
#endif
|
||||
|
||||
// Compute the dense ASU-group id for the current space group by grouping the (pre-sorted) raw-hkl
|
||||
|
||||
@@ -1139,7 +1139,7 @@ namespace {
|
||||
// this process running on the GPU any more: what holds the rest is this run or another program.
|
||||
[[noreturn]] void ThrowOutOfGPUMemory(int64_t n_obs, size_t bytes) {
|
||||
size_t free_bytes = 0, total_bytes = 0;
|
||||
cudaMemGetInfo(&free_bytes, &total_bytes);
|
||||
get_gpu_free_memory(free_bytes, total_bytes);
|
||||
cudaGetLastError();
|
||||
char msg[768];
|
||||
snprintf(msg, sizeof(msg),
|
||||
|
||||
@@ -234,7 +234,7 @@ WilsonOutlierResult WilsonOutliersGPU(const std::vector<WilsonObservation> &obs,
|
||||
throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
|
||||
"Wilson outlier test: " + std::to_string(obs.size()) + " observations need "
|
||||
+ std::to_string(BYTES_PER_OBS * obs.size() >> 20) + " MiB of GPU memory, "
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run the CPU build");
|
||||
+ std::to_string(free_bytes >> 20) + " MiB free - run on the CPU (rugnux --no-gpu)");
|
||||
|
||||
CudaStream stream;
|
||||
CudaDevicePtr<WilsonObservation> d_obs(obs.size(), ALLOC);
|
||||
|
||||
@@ -13,5 +13,5 @@
|
||||
// logarithm are taken on the host. Only built with CUDA.
|
||||
//
|
||||
// Device memory is about 96 bytes per observation at the peak; a device that cannot hold it throws
|
||||
// (run on the CPU build then).
|
||||
// (run on the CPU then: rugnux --no-gpu).
|
||||
WilsonOutlierResult WilsonOutliersGPU(const std::vector<WilsonObservation> &obs, double alpha);
|
||||
@@ -157,15 +157,16 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
|
||||
cap.terms = max_observations * ops;
|
||||
const size_t bytes = RigidBodyGPUEngine::DeviceBytes(cap);
|
||||
|
||||
size_t free = 0, total = 0;
|
||||
RigidBodyGPUEngine::MemoryInfo(free, total);
|
||||
constexpr size_t HEADROOM = 1ull << 30;
|
||||
const size_t budget = std::min(total / 4, free > HEADROOM ? free - HEADROOM : 0);
|
||||
// A quarter of the card's TOTAL memory - the validation's plan (StructureFactorsGPU) - and not what
|
||||
// happens to be free: the same input on the same card always gets the same engines. A card that
|
||||
// is merely busy is waited for by the engines' allocations, never left for the CPU.
|
||||
const int device = RigidBodyGPUEngine::CurrentDevice();
|
||||
const size_t total = get_gpu_total_memory(device);
|
||||
const size_t budget = total / 4;
|
||||
const size_t fit = bytes > 0 ? budget / bytes : 0;
|
||||
const size_t want = std::min<size_t>(fit, std::max<size_t>(max_engines, 1));
|
||||
// Engine i on the i-th card from the calling thread's, so a validation uses every card there is;
|
||||
// the engines give the same bits on any card of one model, so this changes only where they run.
|
||||
const int device = RigidBodyGPUEngine::CurrentDevice();
|
||||
const int cards = get_gpu_count();
|
||||
pool->device_ = device;
|
||||
pool->planned_bytes_ = bytes * std::max<size_t>(max_engines, 1);
|
||||
@@ -174,8 +175,8 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
|
||||
pool->engines_.push_back(std::make_unique<RigidBodyGPUEngine>(cap, (device + static_cast<int>(i)) % cards));
|
||||
set_gpu(device); // each engine was made on its own card
|
||||
if (pool->engines_.empty()) {
|
||||
logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB and the "
|
||||
"budget is {:.0f} MB", bytes / 1e6, budget / 1e6);
|
||||
logger.Info("Model validation: the rigid body runs on the CPU - one GPU engine needs {:.0f} MB, over a "
|
||||
"quarter of the card's {:.0f} MB", bytes / 1e6, total / 1e6);
|
||||
return nullptr;
|
||||
}
|
||||
for (auto &e : pool->engines_)
|
||||
@@ -184,11 +185,7 @@ std::unique_ptr<RigidBodyGPUPool> RigidBodyGPUPool::Create(const gemmi::Model &m
|
||||
bytes / 1e6);
|
||||
return pool;
|
||||
} catch (const JFJochException &e) {
|
||||
logger.Warning("Model validation: the rigid body runs on the CPU - the GPU engines could not be set up ({})",
|
||||
e.what());
|
||||
cuda_throw_if_context_lost();
|
||||
cuda_clear_error();
|
||||
return nullptr;
|
||||
throw RigidBodyGPUFailure(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -367,7 +367,8 @@ struct RigidBodyGPUEngineImpl {
|
||||
|
||||
explicit RigidBodyGPUEngineImpl(const RigidBodyGPUCapacity &c, int dev)
|
||||
: device(dev), cap(c) {
|
||||
const auto sync = CudaAlloc::Synchronous;
|
||||
// A card busy with a probe pass beside the run is waited for, not given up on (RigidBodyGPUPool).
|
||||
const auto sync = CudaAlloc::SynchronousAfterWorkBeside;
|
||||
const size_t na = std::max<size_t>(cap.atoms, 1);
|
||||
rel = CudaDevicePtr<double>(3 * na, sync);
|
||||
pos = CudaDevicePtr<float4>(na, sync);
|
||||
@@ -484,10 +485,6 @@ bool RigidBodyGPUEngine::Supports(const RigidBodyGPUZone &zone) {
|
||||
return ModelDensityGPU::Supports(DensityGrid(zone), zone.atoms);
|
||||
}
|
||||
|
||||
void RigidBodyGPUEngine::MemoryInfo(size_t &free, size_t &total) {
|
||||
cuda_err(cudaMemGetInfo(&free, &total));
|
||||
}
|
||||
|
||||
int RigidBodyGPUEngine::CurrentDevice() {
|
||||
int device = 0;
|
||||
cuda_err(cudaGetDevice(&device));
|
||||
|
||||
@@ -39,9 +39,9 @@ public:
|
||||
// radii, which the placement does not change - is worked out once per zone and kept.
|
||||
class RigidBodyGPUPool {
|
||||
public:
|
||||
// Null where there is no GPU, where not even one engine fits the budget - a quarter of the card,
|
||||
// and never the last gigabyte of what is free, since the merge may be running beside it - or where
|
||||
// the cell is too small for the gather (an atom's box wider than the cell). Logged either way.
|
||||
// Null where there is no GPU, where not even one engine fits the budget - a quarter of the card's
|
||||
// total memory - or where the cell is too small for the gather (an atom's box wider than the cell).
|
||||
// Logged either way. Engines that cannot be set up throw RigidBodyGPUFailure.
|
||||
// `max_observations`: the most working reflections any fit will be given in its finest zone.
|
||||
// `max_engines`: at most this many, however much memory there is.
|
||||
static std::unique_ptr<RigidBodyGPUPool> Create(const gemmi::Model &model, const gemmi::UnitCell &cell,
|
||||
|
||||
@@ -93,7 +93,6 @@ public:
|
||||
// Whether the gather reproduces gemmi's box walk on this zone: every atom's box narrower than the cell,
|
||||
// so that no point is reached by two images of one atom.
|
||||
static bool Supports(const RigidBodyGPUZone &zone);
|
||||
static void MemoryInfo(size_t &free, size_t &total);
|
||||
static int CurrentDevice();
|
||||
|
||||
RigidBodyGPUEngine(const RigidBodyGPUCapacity &capacity, int device);
|
||||
|
||||
@@ -4281,7 +4281,7 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) {
|
||||
Step("Molecular replacement");
|
||||
TimingMark mark(logger, "model prior");
|
||||
// As before the validation above, which a predicted model or --mr does not run: the engine's card
|
||||
// memory goes back before the search sizes its rigid-body engines from what the card has free.
|
||||
// memory goes back before the search reserves its rigid-body engines on the card.
|
||||
if (!ledger_ahead.valid())
|
||||
rsm.reset();
|
||||
std::vector<MRMerge> merges{{"MERGE", &sm.merged, *result.consensus_cell, experiment_.GetSpaceGroupOrP1()}};
|
||||
|
||||
@@ -17,6 +17,14 @@
|
||||
#include "../image_analysis/indexing/CUDAMemHelpers.h"
|
||||
#include "../image_analysis/indexing/IndexerThreadPool.h"
|
||||
|
||||
// Whether the current GPU shares the host's memory (GB10, Jetson). A test that fills the card until an
|
||||
// allocation fails would there fill the host's RAM - every process on the machine runs out with it.
|
||||
static bool gpu_is_integrated() {
|
||||
int device = 0, integrated = 0;
|
||||
return cudaGetDevice(&device) == cudaSuccess
|
||||
&& cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated, device) == cudaSuccess && integrated;
|
||||
}
|
||||
|
||||
// The broker starts fresh worker threads for every data collection. A stream per thread ever started
|
||||
// is a device-memory leak that ends a long-running broker, so threads that follow one another have to
|
||||
// end up on the same stream.
|
||||
@@ -134,6 +142,8 @@ TEST_CASE("CudaPinnedPool_ReusedByTheNextEngine", "[CUDAMemHelpers]") {
|
||||
TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("No CUDA GPU present");
|
||||
if (gpu_is_integrated())
|
||||
SKIP("Fills the card, which on an integrated GPU is the host's memory");
|
||||
|
||||
IndexingSettings settings;
|
||||
settings.Algorithm(IndexingAlgorithmEnum::FFT).IndexingThreads(1);
|
||||
@@ -212,6 +222,8 @@ TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]
|
||||
TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("no GPU");
|
||||
if (gpu_is_integrated())
|
||||
SKIP("Fills the card, which on an integrated GPU is the host's memory");
|
||||
std::vector<CudaDevicePtr<uint8_t>> held;
|
||||
const auto fill = [&] {
|
||||
for (size_t chunk = size_t(256) << 20; chunk >= (size_t(1) << 20); chunk /= 2)
|
||||
@@ -239,6 +251,24 @@ TEST_CASE("CudaDevicePtr_AfterWorkBesideWaitsForItsMemory", "[CUDAMemHelpers]")
|
||||
cuda_trim_allocation_pool();
|
||||
}
|
||||
|
||||
// The free memory waited on is cudaMemGetInfo's on a discrete card. On one that shares the host's memory
|
||||
// (GB10, Jetson) it also counts the page cache the host hands back, less a tenth of the card for the host:
|
||||
// never less than cudaMemGetInfo's. Allocations between the readings move them, hence the 1 GB slack.
|
||||
TEST_CASE("GetGpuFreeMemory_IntegratedCountsThePageCache", "[CUDAMemHelpers]") {
|
||||
if (get_gpu_count() == 0)
|
||||
SKIP("No CUDA GPU present");
|
||||
constexpr double SLACK = 1e9;
|
||||
size_t free_bytes = 0, total_bytes = 0;
|
||||
REQUIRE(get_gpu_free_memory(free_bytes, total_bytes));
|
||||
size_t cuda_free = 0, cuda_total = 0;
|
||||
REQUIRE(cudaMemGetInfo(&cuda_free, &cuda_total) == cudaSuccess);
|
||||
CHECK(total_bytes == cuda_total);
|
||||
CHECK(free_bytes <= total_bytes);
|
||||
CHECK(double(free_bytes) >= double(cuda_free) - SLACK);
|
||||
if (!gpu_is_integrated())
|
||||
CHECK(double(free_bytes) == Catch::Approx(double(cuda_free)).margin(SLACK));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// A merge that finds the card full waits for the GPU work running beside it: until the last one
|
||||
|
||||
@@ -1182,18 +1182,11 @@ TEST_CASE("ModelValidation_RigidBodyTranslationDerivativeIsExact", "[ModelValida
|
||||
|
||||
#ifdef JFJOCH_USE_CUDA
|
||||
namespace {
|
||||
// A pool of `engines`, or a SKIP where there is none because the card is busy - over half its memory
|
||||
// taken by something else, where the pool falls back to the CPU by design. Anywhere else no pool fails.
|
||||
// A pool of `engines`: decided on the card's total memory, so there is one wherever there is a GPU.
|
||||
std::unique_ptr<RigidBodyGPUPool> TestPool(const gemmi::Model &model, const gemmi::UnitCell &cell,
|
||||
const gemmi::SpaceGroup &sg, double d_min, size_t observations,
|
||||
size_t engines, Logger &logger) {
|
||||
auto pool = RigidBodyGPUPool::Create(model, cell, sg, d_min, observations, engines, logger);
|
||||
if (!pool) {
|
||||
size_t free = 0, total = 0;
|
||||
RigidBodyGPUEngine::MemoryInfo(free, total);
|
||||
if (free < total / 2)
|
||||
SKIP("No GPU engine: the card is busy (" << free / 1000000 << " of " << total / 1000000 << " MB free)");
|
||||
}
|
||||
REQUIRE(pool);
|
||||
return pool;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user