Jungfraujoch/image_analysis/indexing/CUDAMemHelpers.h

// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#pragma once

#include <cuda_runtime.h>
#include <cufft.h>
#include <map>
#include <mutex>
#include <stdexcept>
#include <vector>
#include "../common/JFJochException.h"
#include "../common/CUDAWrapper.h"

class CudaStream {
    cudaStream_t stream_ = nullptr;
public:
    // Non-blocking by default: a stream created with cudaStreamDefault synchronises against the legacy
    // NULL stream, so any NULL-stream operation anywhere in the process serialises every worker's GPU
    // work against every other's. With one engine per worker thread that costs most of the parallelism.
    CudaStream(unsigned int flags = cudaStreamNonBlocking, int priority = 0) {
        if (cudaStreamCreateWithPriority(&stream_, flags, priority) != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
                                  "Failed to create CUDA stream");
    }
    ~CudaStream() {
        if (stream_) cudaStreamDestroy(stream_);
    }
    // Move-only type
    CudaStream(CudaStream&& other) noexcept : stream_(other.stream_) { other.stream_ = nullptr; }
    CudaStream& operator=(CudaStream&& other) noexcept {
        if (this != &other) {
            if (stream_) cudaStreamDestroy(stream_);
            stream_ = other.stream_;
            other.stream_ = nullptr;
        }
        return *this;
    }
    CudaStream(const CudaStream&) = delete;
    CudaStream& operator=(const CudaStream&) = delete;

    operator cudaStream_t() const { return stream_; }
    cudaStream_t get() const { return stream_; }
};

// A timing event, so a phase that is queued on a stream can still report how long the device spent
// on it. cudaEventDisableTiming is deliberately NOT used - timing is the whole point here.
class CudaEvent {
    cudaEvent_t event_ = nullptr;
public:
    CudaEvent() {
        if (cudaEventCreate(&event_) != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
                                  "Failed to create CUDA event");
    }
    ~CudaEvent() {
        if (event_) cudaEventDestroy(event_);
    }
    CudaEvent(CudaEvent&& other) noexcept : event_(other.event_) { other.event_ = nullptr; }
    CudaEvent& operator=(CudaEvent&& other) noexcept {
        if (this != &other) {
            if (event_) cudaEventDestroy(event_);
            event_ = other.event_;
            other.event_ = nullptr;
        }
        return *this;
    }
    CudaEvent(const CudaEvent&) = delete;
    CudaEvent& operator=(const CudaEvent&) = delete;

    operator cudaEvent_t() const { return event_; }
    cudaEvent_t get() const { return event_; }
};

class CudaFFTPlan {
    cufftHandle plan_ = 0;
public:
    CudaFFTPlan() = default;

    CudaFFTPlan(
            int rank,
            const int* n,
            const int* inembed, int istride, int idist,
            const int* onembed, int ostride, int odist,
            cufftType type, int batch)
    {
        if (cufftPlanMany(&plan_, rank, const_cast<int*>(n),
                          const_cast<int*>(inembed), istride, idist,
                          const_cast<int*>(onembed), ostride, odist,
                          type, batch) != CUFFT_SUCCESS)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
                                  "Failed to create CUFFT plan with cufftPlanMany");
    }

    // Convenience overload for vector input
    CudaFFTPlan(
            int rank,
            const std::vector<int>& n,
            const std::vector<int>& inembed, int istride, int idist,
            const std::vector<int>& onembed, int ostride, int odist,
            cufftType type, int batch)
            : CudaFFTPlan(rank, n.data(), inembed.data(), istride, idist, onembed.data(), ostride, odist, type, batch)
    {}

    ~CudaFFTPlan() {
        if (plan_) cufftDestroy(plan_);
    }
    // Move-only type
    CudaFFTPlan(CudaFFTPlan&& other) noexcept : plan_(other.plan_) { other.plan_ = 0; }
    CudaFFTPlan& operator=(CudaFFTPlan&& other) noexcept {
        if (this != &other) {
            if (plan_) cufftDestroy(plan_);
            plan_ = other.plan_;
            other.plan_ = 0;
        }
        return *this;
    }
    CudaFFTPlan(const CudaFFTPlan&) = delete;
    CudaFFTPlan& operator=(const CudaFFTPlan&) = delete;

    operator cufftHandle() const { return plan_; }
    cufftHandle get() const { return plan_; }
};

// How much freed memory the pool keeps rather than returning it to the driver. Returning it puts the
// next allocation straight back on the path this exists to avoid, so hold the per-worker engine
// buffers; but the card also has to fit the merge, which asks for several gigabytes of its own after
// the image loop, so do not hold everything.
constexpr uint64_t CUDA_MEM_POOL_RELEASE_THRESHOLD = 1ull << 30;

// The stream device allocations are ordered on: one per (thread, device), borrowed on first use and
// handed back when the thread exits. The streams themselves are never destroyed.
//
// cudaMalloc and cudaFree are on CUDA's implicit-synchronization list - each one synchronises the
// device across every stream - so one analysis engine per worker thread, each making a few dozen of
// them, serialises every worker against every other AND stalls the workers already processing
// images. cudaMallocAsync/cudaFreeAsync take the stream-ordered path instead and do not.
//
// The stream is deliberately not owned by the engine: cudaFreeAsync must be ordered on a stream that
// is still alive, and an engine's own stream can be destroyed before the buffers it allocated. It is
// keyed by device because a thread pinned to one GPU must not order work on another's stream.
//
// Nor is it owned by the thread, for the same reason: a buffer can outlive the thread that allocated
// it - the shadow finder builds its accumulator on a throw-away thread and frees it from another,
// long after - so the thread only borrows the stream. But it does hand it back. The broker starts
// fresh worker threads for every data collection, and a stream holds about half a megabyte of device
// memory, so a stream per thread ever started is a leak that a long-running broker does not survive.
// Handed back, the streams number as many as the threads that were ever alive at once.
namespace jfjoch_cuda_allocation_streams {
    struct Idle {
        std::mutex m;
        std::map<int, std::vector<CudaStream>> streams;   // by device
    };

    // Never destroyed: a thread can still be handing a stream back while the process exits, and
    // destroying streams during static destruction would call into a runtime that may be gone.
    inline Idle &idle() {
        static Idle *i = new Idle;
        return *i;
    }

    struct Borrowed {
        std::map<int, CudaStream> streams;   // by device
        ~Borrowed() {
            std::lock_guard lock(idle().m);
            for (auto &[device, stream] : streams)
                idle().streams[device].push_back(std::move(stream));
        }
    };
}

inline cudaStream_t cuda_allocation_stream() {
    int device = 0;
    if (cudaGetDevice(&device) != cudaSuccess)
        return nullptr;
    thread_local jfjoch_cuda_allocation_streams::Borrowed borrowed;
    const auto it = borrowed.streams.find(device);
    if (it != borrowed.streams.end())
        return it->second;

    auto &idle = jfjoch_cuda_allocation_streams::idle();
    {
        std::lock_guard lock(idle.m);
        auto &streams = idle.streams[device];
        if (!streams.empty()) {
            CudaStream stream = std::move(streams.back());
            streams.pop_back();
            return borrowed.streams.emplace(device, std::move(stream)).first->second;
        }
    }

    try {
        CudaStream stream;
        cudaMemPool_t pool = nullptr;
        if (cudaDeviceGetDefaultMemPool(&pool, device) == cudaSuccess) {
            uint64_t threshold = CUDA_MEM_POOL_RELEASE_THRESHOLD;
            cudaMemPoolSetAttribute(pool, cudaMemPoolAttrReleaseThreshold, &threshold);
        }
        return borrowed.streams.emplace(device, std::move(stream)).first->second;
    } catch (const JFJochException &) {
        // No stream: the caller allocates synchronously instead, so this is handled - see
        // CudaDevicePtr for why it must then not stay behind as the thread's last error.
        cudaGetLastError();
        return nullptr;
    }
}

// Return the current device's pool memory that no buffer holds to the driver. Frees ordered on a
// stream count only once the stream has reached them, hence the device-wide wait first.
inline void cuda_trim_allocation_pool() {
    int device = 0;
    cudaMemPool_t pool = nullptr;
    if (cudaGetDevice(&device) != cudaSuccess || cudaDeviceGetDefaultMemPool(&pool, device) != cudaSuccess) {
        cudaGetLastError();
        return;
    }
    cudaDeviceSynchronize();
    cudaMemPoolTrimTo(pool, 0);
    cudaGetLastError();
}

// Whether `bytes` more device memory can be had on the current device, for a step whose buffers are
// sized by the data set and which ends the run when they do not fit. cudaMemGetInfo counts what the
// pool holds with no buffer in it (up to CUDA_MEM_POOL_RELEASE_THRESHOLD) as taken, so that is handed
// back before the step gives up; and GPU work running beside it in this process (GPUWorkBeside) is
// waited for, as its memory comes back when it ends. Only when the step runs depends on this, never
// what it computes. `free_bytes` is the last reading, for the message of a step that does not fit.
inline bool cuda_memory_available(size_t bytes, size_t &free_bytes) {
    const auto read_free = [&] {
        size_t total_bytes = 0;
        if (cudaMemGetInfo(&free_bytes, &total_bytes) != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "Failed to read free device memory");
        return bytes <= free_bytes;
    };
    if (read_free())
        return true;
    cuda_trim_allocation_pool();
    if (read_free())
        return true;
    // That work is a probe pass of seconds, so this long means it is stuck: the step fails as it would
    // have without it.
    wait_for_gpu_work_beside(std::chrono::minutes(10));
    cuda_trim_allocation_pool();
    return read_free();
}

// Which allocator a device buffer uses. Pooled is what every per-worker buffer wants. Synchronous is
// for a buffer that is freed by a thread other than the one whose work read it: cudaFreeAsync orders
// the free on the freeing side's allocation stream, which says nothing about kernels queued
// elsewhere, while cudaFree synchronises the whole device and therefore cannot be early.
enum class CudaAlloc { Pooled, Synchronous };

template <typename T>
class CudaDevicePtr {
    T* ptr_ = nullptr;
    // The stream the allocation was ordered on, and the one the free must be ordered on. Null when
    // the pool was not used, so the two always pair up.
    cudaStream_t stream_ = nullptr;

    void release() {
        if (!ptr_) return;
        if (stream_) cudaFreeAsync(ptr_, stream_);
        else         cudaFree(ptr_);
        ptr_ = nullptr;
    }

public:
    CudaDevicePtr() = default;

    explicit CudaDevicePtr(size_t count, CudaAlloc alloc = CudaAlloc::Pooled) {
        if (alloc == CudaAlloc::Pooled)
            stream_ = cuda_allocation_stream();
        if (stream_) {
            if (cudaMallocAsync(&ptr_, count * sizeof(T), stream_) == cudaSuccess) {
                // The allocation is ordered on this stream, so anything that wants to use the memory
                // from another stream has to be ordered after it. Waiting for it once here is that
                // ordering, and it leaves the pointer as freely usable as cudaMalloc's would have been.
                cudaStreamSynchronize(stream_);
                return;
            }
            // The pool failing is handled - by the synchronous allocation below - so it must not stay
            // behind as the thread's last error. Every kernel launch is followed by cudaGetLastError(),
            // which would report it there: an "out of memory" thrown over an image whose buffers were
            // all allocated and whose kernel launched.
            cudaGetLastError();
            stream_ = nullptr;
        }
        if (cudaMalloc(&ptr_, count * sizeof(T)) == cudaSuccess)
            return;
        // cudaMalloc cannot take memory the pool is holding: what pooled buffers handed back stays
        // reserved by the pool, up to gigabytes after the image loop's engines are gone, until the
        // pool is trimmed. Hand it back to the driver and try once more; the failed attempt must not
        // stay behind as the last error (see above).
        cudaGetLastError();
        cuda_trim_allocation_pool();
        if (cudaMalloc(&ptr_, count * sizeof(T)) != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError,
                                  "Failed to allocate device memory");
    }
    ~CudaDevicePtr() {
        release();
    }
    // Move-only type
    CudaDevicePtr(CudaDevicePtr&& other) noexcept : ptr_(other.ptr_), stream_(other.stream_) {
        other.ptr_ = nullptr;
        other.stream_ = nullptr;
    }
    CudaDevicePtr& operator=(CudaDevicePtr&& other) noexcept {
        if (this != &other) {
            release();
            ptr_ = other.ptr_;
            stream_ = other.stream_;
            other.ptr_ = nullptr;
            other.stream_ = nullptr;
        }
        return *this;
    }
    CudaDevicePtr(const CudaDevicePtr&) = delete;
    CudaDevicePtr& operator=(const CudaDevicePtr&) = delete;

    T* get() const { return ptr_; }
    operator T*() const { return ptr_; }
};

template <typename T>
class CudaHostPtr {
    T* ptr_ = nullptr;
public:
    CudaHostPtr() = default;
    explicit CudaHostPtr(size_t count) {
        if (cudaMallocHost(&ptr_, count * sizeof(T)) != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::MemAllocFailed,
                                  "Failed to allocate pinned host memory");
    }
    ~CudaHostPtr() {
        if (ptr_) cudaFreeHost(ptr_);
    }
    // Move-only type
    CudaHostPtr(CudaHostPtr&& other) noexcept : ptr_(other.ptr_) { other.ptr_ = nullptr; }
    CudaHostPtr& operator=(CudaHostPtr&& other) noexcept {
        if (this != &other) {
            if (ptr_) cudaFreeHost(ptr_);
            ptr_ = other.ptr_;
            other.ptr_ = nullptr;
        }
        return *this;
    }
    CudaHostPtr(const CudaHostPtr&) = delete;
    CudaHostPtr& operator=(const CudaHostPtr&) = delete;

    T* get() const { return ptr_; }
    operator T*() const { return ptr_; }
};

template <typename T>
class CudaRegisteredVector {
    std::vector<T>* vec_ = nullptr;
    bool registered_ = false;

    static void registerPtr(void* ptr, size_t bytes, unsigned int flags) {
        cudaError_t err = cudaHostRegister(ptr, bytes, flags);
        if (err != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "cudaHostRegister failed");
    }
    static void unregisterPtr(void* ptr) {
        cudaError_t err = cudaHostUnregister(ptr);
        if (err != cudaSuccess)
            throw JFJochException(JFJochExceptionCategory::GPUCUDAError, "cudaHostUnregister failed");
    }
    // Unchecked, for the destructor and the move-assignment: both are noexcept, so throwing out of
    // them aborts the process instead of reporting the failure - and teardown is exactly where
    // cudaHostUnregister fails (after a device reset, or while another exception is unwinding).
    // The other destructors in this header ignore their teardown status for the same reason.
    static void unregisterPtrNoThrow(void* ptr) noexcept {
        cudaHostUnregister(ptr);
    }

public:
    // Non-owning wrapper. Does NOT provide accessors to the vector.
    CudaRegisteredVector() = default;

    CudaRegisteredVector(std::vector<T>& vec, unsigned int flags = cudaHostRegisterDefault)
        : vec_(&vec)
    {
        if (!vec.empty()) {
            registerPtr(vec.data(), vec.size() * sizeof(T), flags);
            registered_ = true;
        }
    }

    ~CudaRegisteredVector() {
        if (registered_ && vec_ && !vec_->empty()) {
            unregisterPtrNoThrow(vec_->data());
        }
    }

    // Move-only
    CudaRegisteredVector(CudaRegisteredVector&& other) noexcept
        : vec_(other.vec_), registered_(other.registered_) {
        other.vec_ = nullptr;
        other.registered_ = false;
    }

    CudaRegisteredVector& operator=(CudaRegisteredVector&& other) noexcept {
        if (this != &other) {
            // Clean current registration
            if (registered_ && vec_ && !vec_->empty()) {
                unregisterPtrNoThrow(vec_->data());
            }
            vec_ = other.vec_;
            registered_ = other.registered_;
            other.vec_ = nullptr;
            other.registered_ = false;
        }
        return *this;
    }

    CudaRegisteredVector(const CudaRegisteredVector&) = delete;
    CudaRegisteredVector& operator=(const CudaRegisteredVector&) = delete;

    // Re-register after vector capacity/size change. Caller must ensure
    // the vector is not registered at the moment of mutation.
    void rebind(std::vector<T>& vec, unsigned int flags = cudaHostRegisterDefault) {
        // Unregister previous if needed
        if (registered_ && vec_ && !vec_->empty()) {
            unregisterPtr(vec_->data());
        }
        vec_ = &vec;
        if (!vec.empty()) {
            registerPtr(vec.data(), vec.size() * sizeof(T), flags);
            registered_ = true;
        } else {
            registered_ = false;
        }
    }

    // Explicit unregister (optional).
    void unregister() {
        if (registered_ && vec_ && !vec_->empty()) {
            unregisterPtr(vec_->data());
            registered_ = false;
        }
    }

    bool isRegistered() const { return registered_; }
};