diff --git a/broker/jfjoch_broker.cpp b/broker/jfjoch_broker.cpp index 29fca9145..852b9972b 100644 --- a/broker/jfjoch_broker.cpp +++ b/broker/jfjoch_broker.cpp @@ -9,10 +9,12 @@ #include +#include "../common/CudaJitCache.h" #include "../common/GitInfo.h" #include "../common/Logger.h" #include "../common/print_license.h" #include "../detector_control/DectrisSimplonClient.h" +#include "../image_analysis/indexing/IndexerThreadPool.h" #include "../writer/HDF5Objects.h" #include "JFJochBrokerHttp.h" @@ -64,6 +66,9 @@ int main(int argc, char **argv) { Logger logger("jfjoch_broker"); logger.Info("jfjoch_broker {} (git {}), built with {}", jfjoch_version(), jfjoch_git_sha1().substr(0, 6), jfjoch_build_info(true)); + // cuFFT's initialisation, and card 0's context, ahead of the first collection; the plan it times shows + // whether the CUDA JIT cache works - without one, every plan the analysis makes is compiled again. + LogCudaJitCache(logger, WarmUpCuFFT()); org::openapitools::server::model::Jfjoch_settings settings; std::ifstream file(argv[1]); diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index bac24abd8..e91172f1d 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -86,6 +86,7 @@ ADD_LIBRARY(JFJochCommon STATIC DetectorModuleGeometry.cpp DetectorModuleGeometry.h DetectorSetup.h DetectorSetup.cpp ZeroCopyReturnValue.h Histogram.h DiffractionGeometry.h CUDAWrapper.cpp CUDAWrapper.h + CudaJitCache.cpp CudaJitCache.h HostMemory.cpp HostMemory.h ThreadAffinity.cpp ThreadAffinity.h ADUHistogram.cpp ADUHistogram.h diff --git a/common/CudaJitCache.cpp b/common/CudaJitCache.cpp new file mode 100644 index 000000000..6967a72d3 --- /dev/null +++ b/common/CudaJitCache.cpp @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "CudaJitCache.h" + +#include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#include +#include +#endif + +#ifdef __linux__ +namespace { + // statfs f_type of the filesystems a home directory is likely to be on. A network one is not wrong in + // itself, but each cuFFT plan opens and locks the cache about ten times: at 1 ms a round trip that is + // 15-20 ms a plan, at 5 ms as slow as no cache. + struct FsType { + long magic; + const char *name; + }; + + constexpr FsType fs_types[] = { + {0x6969, "nfs"}, {0x5346414F, "afs"}, {0x6B414653, "afs"}, {0x47504653, "gpfs"}, + {0x0BD00BD0, "lustre"}, {0x00C36400, "ceph"}, {0x19830326, "beegfs"}, {0xFF534D42, "cifs"}, + {0xFE534D42, "smb2"}, {0x517B, "smb"}, {0x01021997, "9p"}, {0x65735546, "fuse"}, + {0x58465342, "xfs"}, {0xEF53, "ext4"}, {0x9123683E, "btrfs"}, {0x01021994, "tmpfs"}, + {0x794C7630, "overlay"}, {0x2FC12FC1, "zfs"}, + }; +} +#endif + +CudaJitCache InspectCudaJitCache() { + CudaJitCache c; + const char *path = std::getenv("CUDA_CACHE_PATH"); + const char *disable = std::getenv("CUDA_CACHE_DISABLE"); + const char *maxsize = std::getenv("CUDA_CACHE_MAXSIZE"); + + if (path != nullptr && *path != '\0') + c.path = path; +#ifdef _WIN32 + else + c.path = "%APPDATA%\\NVIDIA\\ComputeCache"; +#else + else if (const char *home = std::getenv("HOME"); home != nullptr && *home != '\0') + c.path = std::string(home) + "/.nv/ComputeCache"; +#endif + + if (disable != nullptr && std::string(disable) == "1") + c.problem = "CUDA_CACHE_DISABLE=1 turns it off"; + else if (maxsize != nullptr && std::strtoull(maxsize, nullptr, 10) < 64 * 1024 * 1024) + c.problem = fmt::format("CUDA_CACHE_MAXSIZE={} keeps too little of it", maxsize); + else if (c.path.empty()) + c.problem = "neither CUDA_CACHE_PATH nor HOME is set (a systemd service without User= has no HOME)"; + +#ifdef __linux__ + if (c.path.empty()) + return c; + // The driver makes the directory when it is missing, so the nearest one that exists must be writable. + std::error_code ec; + std::filesystem::path dir = c.path; + while (!std::filesystem::exists(dir, ec) && dir.has_parent_path() && dir != dir.parent_path()) + dir = dir.parent_path(); + + struct statfs s{}; + if (statfs(dir.c_str(), &s) == 0) { + c.filesystem = fmt::format("{:#x}", static_cast(s.f_type)); + for (const auto &t: fs_types) + if (static_cast(s.f_type) == static_cast(t.magic)) + c.filesystem = t.name; + } + + if (!c.problem.empty()) + return c; + if (access(dir.c_str(), W_OK | X_OK) != 0) { + c.problem = fmt::format("{} is not writable ({})", dir.string(), std::strerror(errno)); + return c; + } + // Each lookup opens the index read-write and read-locks it with fcntl: where either fails (a read-only + // cache, a filesystem without POSIX locks) the driver compiles every time, though the cache is there. + const std::string index = c.path + "/index"; + if (std::filesystem::exists(index, ec)) { + const int fd = open(index.c_str(), O_RDWR); + if (fd < 0) { + c.problem = fmt::format("its index cannot be opened for writing ({})", std::strerror(errno)); + return c; + } + struct flock lock{}; + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + if (fcntl(fd, F_SETLK, &lock) != 0 && errno != EAGAIN && errno != EACCES) + c.problem = fmt::format("its index cannot be locked ({})", std::strerror(errno)); + lock.l_type = F_UNLCK; + fcntl(fd, F_SETLK, &lock); + close(fd); + } +#endif + return c; +} + +void LogCudaJitCache(Logger &logger, double plan_ms) { + if (std::isnan(plan_ms)) + return; + const CudaJitCache c = InspectCudaJitCache(); + // Measured: 0.4-1.7 ms from a working cache, 60-130 ms with none. + const bool slow = plan_ms > 10.0; + const std::string where = c.path.empty() ? std::string("none") + : c.filesystem.empty() ? c.path : fmt::format("{} ({})", c.path, c.filesystem); + if (c.problem.empty() && !slow) { + logger.Info("CUDA JIT cache {}: a cuFFT plan takes {:.1f} ms", where, plan_ms); + return; + } + logger.Warning("CUDA JIT cache {}: {}a cuFFT plan takes {:.0f} ms{}, which can make molecular " + "replacement several times slower. Give the process a cache " + "on a local disk that only its user can write: CUDA_CACHE_PATH=, for instance " + "Environment=CUDA_CACHE_PATH=/var/cache/jfjoch/nv in a systemd unit", + where, c.problem.empty() ? std::string() : c.problem + "; ", plan_ms, + slow ? " (~1 ms from a working cache)" : ""); +} diff --git a/common/CudaJitCache.h b/common/CudaJitCache.h new file mode 100644 index 000000000..72eb2168f --- /dev/null +++ b/common/CudaJitCache.h @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include + +#include "Logger.h" + +// cuFFT ships its transforms as PTX (cuFFT 11 and 12 alike, for every card), so every plan it makes has +// the driver compile kernels - and makes them again for a second plan of the same shape. The driver keeps +// what it compiled in its JIT cache, CUDA_CACHE_PATH or else $HOME/.nv/ComputeCache: from there a plan +// takes about 1 ms, without it 60-130 ms, all under cuFFT's process-wide lock. A run that makes a hundred +// plans (molecular replacement) is then seconds slower, with nothing else to show for it. +struct CudaJitCache { + std::string path; // where the driver keeps it; empty when it keeps none + std::string filesystem; // the type of the filesystem it is on (Linux only) + std::string problem; // why the driver cannot use it, or only slowly; empty when none is seen +}; + +CudaJitCache InspectCudaJitCache(); + +// One line for a program's start: the cache, and `plan_ms` - how long one cuFFT plan took to make once +// cuFFT was initialised (WarmUpCuFFT). A warning, saying what to do, when the cache is unusable or a plan +// is slow. Nothing when plan_ms is NaN: there is no GPU, so no cuFFT. +void LogCudaJitCache(Logger &logger, double plan_ms); diff --git a/image_analysis/indexing/CUDAMemHelpers.h b/image_analysis/indexing/CUDAMemHelpers.h index 5c9d680d7..91b052439 100644 --- a/image_analysis/indexing/CUDAMemHelpers.h +++ b/image_analysis/indexing/CUDAMemHelpers.h @@ -352,9 +352,9 @@ struct CudaFFTShape { // and shape, and lent to the next user of that shape; when the last scope ends they are destroyed, and // outside any scope a plan is destroyed as soon as its user is done. Making a plan is costly and // serialised: cuFFT holds a process-wide lock while it loads the plan's kernel modules (each plan loads -// its own, two plans of one shape included), which took over a second a plan on a busy card, and on -// Windows compiles them from PTX first; and it frees and loads them all again for the next plan of the -// same shape. A plan is never shared by two users at once - cuFFT is thread safe only across distinct +// its own, two plans of one shape included), which took over a second a plan on a busy card. The modules +// are PTX on every platform, compiled by the driver - ~1 ms from its JIT cache, 60-130 ms without one +// (CudaJitCache.h); and it frees and loads them all again for the next plan of the same shape. A plan is never shared by two users at once - cuFFT is thread safe only across distinct // plans. // // In `rugnux` the scope is one processing run, so nothing is kept from one run for the next, nor past the diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 53f0b2d82..bd8e2696d 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -3,6 +3,9 @@ #include "IndexerThreadPool.h" +#include +#include +#include #include #include @@ -18,13 +21,27 @@ #include "FFTIndexerCPU.h" #endif -void WarmUpCuFFT() { +double WarmUpCuFFT() { #ifdef JFJOCH_USE_CUDA if (get_gpu_count() == 0) - return; - cufftHandle plan = 0; - if (cufftPlan1d(&plan, 1024, CUFFT_C2C, 1) == CUFFT_SUCCESS) + return NAN; + // The first plan pays the initialisation, and compiles its kernels if the JIT cache does not have them + // yet; the next ones show what a plan costs from then on. The fastest of three, so that a stall + // beside (another card's context being made) is not taken for a slow plan. + double fastest_ms = INFINITY; + for (int i = 0; i < 4; i++) { + const auto t0 = std::chrono::steady_clock::now(); + cufftHandle plan = 0; + if (cufftPlan3d(&plan, 64, 64, 64, CUFFT_R2C) != CUFFT_SUCCESS) + return NAN; cufftDestroy(plan); + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (i > 0) + fastest_ms = std::min(fastest_ms, ms); + } + return fastest_ms; +#else + return NAN; #endif } diff --git a/image_analysis/indexing/IndexerThreadPool.h b/image_analysis/indexing/IndexerThreadPool.h index 61b6a145a..023f6971d 100644 --- a/image_analysis/indexing/IndexerThreadPool.h +++ b/image_analysis/indexing/IndexerThreadPool.h @@ -37,8 +37,10 @@ enum class IndexerConstruction { Preconstruct, OnFirstUse }; // Force cuFFT's one-time library initialisation - 0.3 s measured, paid by whichever call happens to // be first - so it can be got out of the way on a background thread at startup rather than landing in -// the middle of the first pass with nothing overlapping it. No-op without a GPU. -void WarmUpCuFFT(); +// the middle of the first pass with nothing overlapping it. Returns how long a 3D plan takes to make +// after that, in ms - about 1 from a working CUDA JIT cache, 60 or more without (see CudaJitCache.h) - +// and NaN without a GPU. +double WarmUpCuFFT(); // cuFFT plans are kept and reused for as long as one of these lives, and destroyed when the last one // ends (CudaFFTPlanScope): one per processing run in `rugnux`, so no plan outlives the run that made it; diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 0f28f5403..1e546c24b 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -37,6 +37,7 @@ #include #include +#include "../common/CudaJitCache.h" #include "../common/Logger.h" #include "../common/ThreadAffinity.h" #include "../common/GitInfo.h" @@ -1701,8 +1702,9 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { logger.Info("Using space group {} (number {})", space_group->hm, space_group->number); // Off the critical path, so cuFFT's one-time initialisation overlaps the file open, the mask read - // and the beam-stop pre-scan instead of landing inside the first pass. - auto cufft_warmup = std::async(std::launch::async, WarmUpCuFFT); + // and the beam-stop pre-scan instead of landing inside the first pass. It also times a plan, which + // shows whether the CUDA JIT cache works. + auto cufft_warmup = std::async(std::launch::async, [&logger] { LogCudaJitCache(logger, WarmUpCuFFT()); }); // The same for the CUDA contexts of the other cards (the warm-up above creates card 0's), which // the pre-scan does not use: created one after another at start-up, they cost a fifth of a second // per card before the run began.