diff --git a/common/CUDAWrapper.cpp b/common/CUDAWrapper.cpp index 4c6f53f25..19838a246 100644 --- a/common/CUDAWrapper.cpp +++ b/common/CUDAWrapper.cpp @@ -75,6 +75,10 @@ void enable_gpu_numa_binding() {} void set_gpu_blocking_sync() {} +void create_gpu_contexts(int32_t) {} + +void release_gpu_contexts() {} + void cuda_clear_error() {} void cuda_throw_if_context_lost() {} diff --git a/common/CUDAWrapper.cu b/common/CUDAWrapper.cu index 440f5eb60..f150cf101 100644 --- a/common/CUDAWrapper.cu +++ b/common/CUDAWrapper.cu @@ -2,10 +2,13 @@ // SPDX-License-Identifier: GPL-3.0-only #include +#include #include #include #include +#include + #include "CUDAWrapper.h" #include "ThreadAffinity.h" #include "JFJochException.h" @@ -116,16 +119,45 @@ void pin_gpu() { } void set_gpu_blocking_sync() { - const int32_t count = get_gpu_count(); - for (int32_t i = 0; i < count; i++) { - if (cudaSetDevice(i) != cudaSuccess) - continue; - // cudaErrorSetOnActiveProcess where a context exists already: it keeps its flags. - if (cudaSetDeviceFlags(cudaDeviceScheduleBlockingSync) != cudaSuccess) - cudaGetLastError(); + // Through the driver's primary-context flags, which a context created later takes: the runtime's + // cudaSetDevice creates the device's context, serially, and on four cards that was most of a + // second before the run had started. The function is taken from the driver at run time, so the + // program does not link against it and still starts on a machine without one. + void *fn = nullptr; + cudaDriverEntryPointQueryResult found{}; + if (get_gpu_count() == 0 + || cudaGetDriverEntryPointByVersion("cuDevicePrimaryCtxSetFlags", &fn, 12000, cudaEnableDefault, + &found) != cudaSuccess + || found != cudaDriverEntryPointSuccess) { + cudaGetLastError(); + return; } - if (count > 0) - cudaSetDevice(0); + const auto set_flags = reinterpret_cast(fn); + for (int32_t i = 0; i < get_gpu_count(); i++) + set_flags(i, CU_CTX_SCHED_BLOCKING_SYNC); +} + +void create_gpu_contexts(int32_t first_device) { + std::vector threads; + for (int32_t i = first_device; i < get_gpu_count(); i++) + threads.emplace_back([i] { + // A device that fails here fails again, with its error, where it is used. + if (cudaSetDevice(i) != cudaSuccess || cudaFree(nullptr) != cudaSuccess) + cudaGetLastError(); + }); + for (auto &t : threads) + t.join(); +} + +void release_gpu_contexts() { + std::vector threads; + for (int32_t i = 0; i < get_gpu_count(); i++) + threads.emplace_back([i] { + if (cudaSetDevice(i) != cudaSuccess || cudaDeviceReset() != cudaSuccess) + cudaGetLastError(); + }); + for (auto &t : threads) + t.join(); } void cuda_clear_error() { diff --git a/common/CUDAWrapper.h b/common/CUDAWrapper.h index d713176f3..b7405aca6 100644 --- a/common/CUDAWrapper.h +++ b/common/CUDAWrapper.h @@ -50,9 +50,19 @@ private: // Have every GPU's host threads BLOCK in a synchronisation (cudaDeviceScheduleBlockingSync) instead // of spinning on a core until the device finishes. Must be called before anything creates a CUDA -// context; a device that already has one keeps the flags it was created with. No-op without CUDA. +// context; it creates none itself. No-op without CUDA. void set_gpu_blocking_sync(); +// Create the CUDA context of every device from first_device on, one thread per device, and return +// when they all exist - for a caller that runs it beside other work, so that the first use of a card +// does not wait for its context. No-op without CUDA. +void create_gpu_contexts(int32_t first_device); + +// Destroy every device's CUDA context, one thread per device, for a program about to exit with no +// CUDA work left: at exit the runtime destroys them one after another, a fifth of a second per card. +// No-op without CUDA. +void release_gpu_contexts(); + // Drop the error CUDA has recorded for the calling thread. Call it where a CUDA failure has been // HANDLED - a device route that fell back to the host, an indexing attempt whose failure was turned // into a result - because the error otherwise stays as the thread's last error and the next diff --git a/reader/SMV.cpp b/reader/SMV.cpp index c497afb36..16c30011c 100644 --- a/reader/SMV.cpp +++ b/reader/SMV.cpp @@ -15,6 +15,13 @@ #include "../common/JFJochException.h" +#ifndef _WIN32 +#include +#include +#include +#include +#endif + namespace smv { namespace { @@ -369,18 +376,49 @@ Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vec throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, path + ": image is larger than the buffer given for it"); + const size_t image_bytes = npixel * sizeof(uint16_t); + const auto too_short = [&] { + return JFJochException(JFJochExceptionCategory::InputParameterInvalid, + path + ": file ends before the image does"); + }; +#ifdef _WIN32 std::ifstream f(path, std::ios::binary); if (!f) throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path); f.seekg(static_cast(h.data_offset), std::ios::beg); - scratch.resize(npixel * sizeof(uint16_t)); + scratch.resize(image_bytes); f.read(reinterpret_cast(scratch.data()), static_cast(scratch.size())); if (static_cast(f.gcount()) != scratch.size()) - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - path + ": file ends before the image does"); + throw too_short(); + const uint8_t *src = scratch.data(); +#else + // Mapped rather than read: the pixels are converted straight out of the page cache, where a read + // first copies the whole frame into a buffer. The image loop that reads a sweep - again in every + // pass, and in every round of the geometry walk - is bound by memory traffic, and the copy adds the + // whole file to it: measured on a 9 Mpx sweep of 360 frames, mapping took a seventh off the run. + (void) scratch; + const int fd = open(path.c_str(), O_RDONLY); + if (fd < 0) + throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot open " + path); + struct stat st{}; + if (fstat(fd, &st) != 0 || static_cast(st.st_size) < h.data_offset + image_bytes) { + close(fd); + throw too_short(); + } + const size_t mapped_bytes = h.data_offset + image_bytes; + void *mapped = mmap(nullptr, mapped_bytes, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (mapped == MAP_FAILED) + throw JFJochException(JFJochExceptionCategory::MockFileOpenError, "Cannot map " + path); + struct Unmap { + void *p; + size_t bytes; + ~Unmap() { munmap(p, bytes); } + } unmap{mapped, mapped_bytes}; + const uint8_t *src = static_cast(mapped) + h.data_offset; +#endif // Stored unsigned 16-bit, handed out as the signed 32-bit the rest of the code reads. - const uint8_t *src = scratch.data(); if (h.little_endian) { for (size_t i = 0; i < npixel; i++) out[i] = static_cast(static_cast(src[2 * i] | (src[2 * i + 1] << 8))); diff --git a/reader/SMV.h b/reader/SMV.h index 9bbf3a14d..708a5d17a 100644 --- a/reader/SMV.h +++ b/reader/SMV.h @@ -85,7 +85,8 @@ Header ReadHeader(const std::string &path); Header Read(const std::string &path, std::vector &out); // The same, widening the stored pixels into memory the caller already has. scratch carries the -// raw bytes between frames so a worker walking a sweep allocates once. +// raw bytes between frames so a worker walking a sweep allocates once - on Windows only; elsewhere the +// file is mapped and scratch is not used. Header ReadInto(const std::string &path, int32_t *out, size_t capacity, std::vector &scratch); } // namespace smv diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index ac5475419..4fb623ca4 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -1589,6 +1589,10 @@ static int RunRugnux(int argc, char **argv) { // Off the critical path, so cuFFT's one-time initialisation overlaps the file open, the mask read // and the beam-stop pre-scan instead of landing inside the first pass. auto cufft_warmup = std::async(std::launch::async, WarmUpCuFFT); + // The same for the CUDA contexts of the other cards (the warm-up above creates card 0's), which + // the pre-scan does not use: created one after another at start-up, they cost a fifth of a second + // per card before the run began. + auto context_warmup = std::async(std::launch::async, create_gpu_contexts, 1); // 1. Read Input File. An HDF5 master, one PILATUS miniCBF frame, or one marCCD frame - naming // any frame of a sweep reads the whole sweep, which is how these are archived and how a user @@ -3221,7 +3225,10 @@ int main(int argc, char **argv) { // run, a fifth of all CPU time went to that spinning, with no change in wall time either way. set_gpu_blocking_sync(); enable_gpu_numa_binding(); - return RunRugnux(argc, argv); + const int ret = RunRugnux(argc, argv); + // Everything that used a card has been destroyed with the run. + release_gpu_contexts(); + return ret; } catch (const std::exception &e) { Logger("rugnux").Error("{}", FailureText(e)); return EXIT_FAILURE;