jfjoch_broker and rugnux check the CUDA JIT cache at start and warn when cuFFT plans are slow

cuFFT ships its transforms as PTX for every card (cuFFT 11 and 12 alike), so each plan has the driver
compile kernels, and a second plan of the same shape compiles them again. From the driver's JIT cache
that costs ~0.5 ms a plan; without it 45-130 ms, under cuFFT's process-wide lock - which, over the
~100 plans of molecular replacement, matches the stage pattern of the slow MR in the production broker.

The cache is unusable in more ways than CUDA_CACHE_DISABLE: with no HOME at all (a systemd system
service without User= gets none, and the driver does not fall back to the passwd entry), on a read-only
or unwritable directory, on a filesystem whose fcntl locks fail (NFS without locking) - each measured
as slow as no cache - or on a slow network filesystem (each plan opens and locks the cache ~10 times).

- common/CudaJitCache: the path the driver uses (CUDA_CACHE_PATH, else $HOME/.nv/ComputeCache), its
  filesystem type, whether it is writable, whether its index opens read-write and takes the driver's
  fcntl read lock, CUDA_CACHE_DISABLE and a CUDA_CACHE_MAXSIZE below 64 MiB. One line: info when fine,
  a warning with the cause and the remedy (CUDA_CACHE_PATH on a local disk) when not, or when a plan
  takes over 10 ms.
- WarmUpCuFFT returns the fastest of three 64^3 R2C plan makes after a first one that pays the
  initialisation (and a cold-cache compile): ~2 ms in all with a cache, ~0.3 s without.
- jfjoch_broker runs it after its version line (0.15 s at start, cuFFT init and card 0's context
  included); rugnux on its existing background warm-up thread.
- CUDAMemHelpers.h: the modules are PTX on every platform, not only on Windows.

No result changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4

broker/jfjoch_broker.cpp

@@ -9,10 +9,12 @@
 
 #include <nlohmann/json.hpp>
 
+#include "../common/CudaJitCache.h"
 #include "../common/GitInfo.h"
 #include "../common/Logger.h"
 #include "../common/print_license.h"
 #include "../detector_control/DectrisSimplonClient.h"
+#include "../image_analysis/indexing/IndexerThreadPool.h"
 #include "../writer/HDF5Objects.h"
 
 #include "JFJochBrokerHttp.h"
@@ -64,6 +66,9 @@ int main(int argc, char **argv) {
     Logger logger("jfjoch_broker");
     logger.Info("jfjoch_broker {} (git {}), built with {}", jfjoch_version(), jfjoch_git_sha1().substr(0, 6),
                 jfjoch_build_info(true));
+    // cuFFT's initialisation, and card 0's context, ahead of the first collection; the plan it times shows
+    // whether the CUDA JIT cache works - without one, every plan the analysis makes is compiled again.
+    LogCudaJitCache(logger, WarmUpCuFFT());
 
     org::openapitools::server::model::Jfjoch_settings settings;
     std::ifstream file(argv[1]);

common/CMakeLists.txt

@@ -86,6 +86,7 @@ ADD_LIBRARY(JFJochCommon STATIC
         DetectorModuleGeometry.cpp DetectorModuleGeometry.h
         DetectorSetup.h DetectorSetup.cpp ZeroCopyReturnValue.h Histogram.h DiffractionGeometry.h
         CUDAWrapper.cpp CUDAWrapper.h
+        CudaJitCache.cpp CudaJitCache.h
         HostMemory.cpp HostMemory.h
         ThreadAffinity.cpp ThreadAffinity.h
         ADUHistogram.cpp ADUHistogram.h

common/CudaJitCache.cpp

@@ -0,0 +1,124 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#include "CudaJitCache.h"
+
+#include <cmath>
+#include <cstdlib>
+#include <filesystem>
+
+#ifdef __linux__
+#include <cerrno>
+#include <cstring>
+#include <fcntl.h>
+#include <sys/vfs.h>
+#include <unistd.h>
+#endif
+
+#ifdef __linux__
+namespace {
+    // statfs f_type of the filesystems a home directory is likely to be on. A network one is not wrong in
+    // itself, but each cuFFT plan opens and locks the cache about ten times: at 1 ms a round trip that is
+    // 15-20 ms a plan, at 5 ms as slow as no cache.
+    struct FsType {
+        long magic;
+        const char *name;
+    };
+
+    constexpr FsType fs_types[] = {
+        {0x6969, "nfs"}, {0x5346414F, "afs"}, {0x6B414653, "afs"}, {0x47504653, "gpfs"},
+        {0x0BD00BD0, "lustre"}, {0x00C36400, "ceph"}, {0x19830326, "beegfs"}, {0xFF534D42, "cifs"},
+        {0xFE534D42, "smb2"}, {0x517B, "smb"}, {0x01021997, "9p"}, {0x65735546, "fuse"},
+        {0x58465342, "xfs"}, {0xEF53, "ext4"}, {0x9123683E, "btrfs"}, {0x01021994, "tmpfs"},
+        {0x794C7630, "overlay"}, {0x2FC12FC1, "zfs"},
+    };
+}
+#endif
+
+CudaJitCache InspectCudaJitCache() {
+    CudaJitCache c;
+    const char *path = std::getenv("CUDA_CACHE_PATH");
+    const char *disable = std::getenv("CUDA_CACHE_DISABLE");
+    const char *maxsize = std::getenv("CUDA_CACHE_MAXSIZE");
+
+    if (path != nullptr && *path != '\0')
+        c.path = path;
+#ifdef _WIN32
+    else
+        c.path = "%APPDATA%\\NVIDIA\\ComputeCache";
+#else
+    else if (const char *home = std::getenv("HOME"); home != nullptr && *home != '\0')
+        c.path = std::string(home) + "/.nv/ComputeCache";
+#endif
+
+    if (disable != nullptr && std::string(disable) == "1")
+        c.problem = "CUDA_CACHE_DISABLE=1 turns it off";
+    else if (maxsize != nullptr && std::strtoull(maxsize, nullptr, 10) < 64 * 1024 * 1024)
+        c.problem = fmt::format("CUDA_CACHE_MAXSIZE={} keeps too little of it", maxsize);
+    else if (c.path.empty())
+        c.problem = "neither CUDA_CACHE_PATH nor HOME is set (a systemd service without User= has no HOME)";
+
+#ifdef __linux__
+    if (c.path.empty())
+        return c;
+    // The driver makes the directory when it is missing, so the nearest one that exists must be writable.
+    std::error_code ec;
+    std::filesystem::path dir = c.path;
+    while (!std::filesystem::exists(dir, ec) && dir.has_parent_path() && dir != dir.parent_path())
+        dir = dir.parent_path();
+
+    struct statfs s{};
+    if (statfs(dir.c_str(), &s) == 0) {
+        c.filesystem = fmt::format("{:#x}", static_cast<unsigned long>(s.f_type));
+        for (const auto &t: fs_types)
+            if (static_cast<unsigned long>(s.f_type) == static_cast<unsigned long>(t.magic))
+                c.filesystem = t.name;
+    }
+
+    if (!c.problem.empty())
+        return c;
+    if (access(dir.c_str(), W_OK | X_OK) != 0) {
+        c.problem = fmt::format("{} is not writable ({})", dir.string(), std::strerror(errno));
+        return c;
+    }
+    // Each lookup opens the index read-write and read-locks it with fcntl: where either fails (a read-only
+    // cache, a filesystem without POSIX locks) the driver compiles every time, though the cache is there.
+    const std::string index = c.path + "/index";
+    if (std::filesystem::exists(index, ec)) {
+        const int fd = open(index.c_str(), O_RDWR);
+        if (fd < 0) {
+            c.problem = fmt::format("its index cannot be opened for writing ({})", std::strerror(errno));
+            return c;
+        }
+        struct flock lock{};
+        lock.l_type = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+        if (fcntl(fd, F_SETLK, &lock) != 0 && errno != EAGAIN && errno != EACCES)
+            c.problem = fmt::format("its index cannot be locked ({})", std::strerror(errno));
+        lock.l_type = F_UNLCK;
+        fcntl(fd, F_SETLK, &lock);
+        close(fd);
+    }
+#endif
+    return c;
+}
+
+void LogCudaJitCache(Logger &logger, double plan_ms) {
+    if (std::isnan(plan_ms))
+        return;
+    const CudaJitCache c = InspectCudaJitCache();
+    // Measured: 0.4-1.7 ms from a working cache, 60-130 ms with none.
+    const bool slow = plan_ms > 10.0;
+    const std::string where = c.path.empty() ? std::string("none")
+            : c.filesystem.empty() ? c.path : fmt::format("{} ({})", c.path, c.filesystem);
+    if (c.problem.empty() && !slow) {
+        logger.Info("CUDA JIT cache {}: a cuFFT plan takes {:.1f} ms", where, plan_ms);
+        return;
+    }
+    logger.Warning("CUDA JIT cache {}: {}a cuFFT plan takes {:.0f} ms{}, which can make molecular "
+                   "replacement several times slower. Give the process a cache "
+                   "on a local disk that only its user can write: CUDA_CACHE_PATH=<dir>, for instance "
+                   "Environment=CUDA_CACHE_PATH=/var/cache/jfjoch/nv in a systemd unit",
+                   where, c.problem.empty() ? std::string() : c.problem + "; ", plan_ms,
+                   slow ? " (~1 ms from a working cache)" : "");
+}

common/CudaJitCache.h

@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
+// SPDX-License-Identifier: GPL-3.0-only
+
+#pragma once
+
+#include <string>
+
+#include "Logger.h"
+
+// cuFFT ships its transforms as PTX (cuFFT 11 and 12 alike, for every card), so every plan it makes has
+// the driver compile kernels - and makes them again for a second plan of the same shape. The driver keeps
+// what it compiled in its JIT cache, CUDA_CACHE_PATH or else $HOME/.nv/ComputeCache: from there a plan
+// takes about 1 ms, without it 60-130 ms, all under cuFFT's process-wide lock. A run that makes a hundred
+// plans (molecular replacement) is then seconds slower, with nothing else to show for it.
+struct CudaJitCache {
+    std::string path;         // where the driver keeps it; empty when it keeps none
+    std::string filesystem;   // the type of the filesystem it is on (Linux only)
+    std::string problem;      // why the driver cannot use it, or only slowly; empty when none is seen
+};
+
+CudaJitCache InspectCudaJitCache();
+
+// One line for a program's start: the cache, and `plan_ms` - how long one cuFFT plan took to make once
+// cuFFT was initialised (WarmUpCuFFT). A warning, saying what to do, when the cache is unusable or a plan
+// is slow. Nothing when plan_ms is NaN: there is no GPU, so no cuFFT.
+void LogCudaJitCache(Logger &logger, double plan_ms);

image_analysis/indexing/CUDAMemHelpers.h

@@ -352,9 +352,9 @@ struct CudaFFTShape {
 // and shape, and lent to the next user of that shape; when the last scope ends they are destroyed, and
 // outside any scope a plan is destroyed as soon as its user is done. Making a plan is costly and
 // serialised: cuFFT holds a process-wide lock while it loads the plan's kernel modules (each plan loads
-// its own, two plans of one shape included), which took over a second a plan on a busy card, and on
-// Windows compiles them from PTX first; and it frees and loads them all again for the next plan of the
-// same shape. A plan is never shared by two users at once - cuFFT is thread safe only across distinct
+// its own, two plans of one shape included), which took over a second a plan on a busy card. The modules
+// are PTX on every platform, compiled by the driver - ~1 ms from its JIT cache, 60-130 ms without one
+// (CudaJitCache.h); and it frees and loads them all again for the next plan of the same shape. A plan is never shared by two users at once - cuFFT is thread safe only across distinct
 // plans.
 //
 // In `rugnux` the scope is one processing run, so nothing is kept from one run for the next, nor past the

image_analysis/indexing/IndexerThreadPool.cpp

@@ -3,6 +3,9 @@
 
 #include "IndexerThreadPool.h"
 
+#include <algorithm>
+#include <chrono>
+#include <cmath>
 #include <map>
 #include <tuple>
 
@@ -18,13 +21,27 @@
 #include "FFTIndexerCPU.h"
 #endif
 
-void WarmUpCuFFT() {
+double WarmUpCuFFT() {
 #ifdef JFJOCH_USE_CUDA
     if (get_gpu_count() == 0)
-        return;
-    cufftHandle plan = 0;
-    if (cufftPlan1d(&plan, 1024, CUFFT_C2C, 1) == CUFFT_SUCCESS)
+        return NAN;
+    // The first plan pays the initialisation, and compiles its kernels if the JIT cache does not have them
+    // yet; the next ones show what a plan costs from then on. The fastest of three, so that a stall
+    // beside (another card's context being made) is not taken for a slow plan.
+    double fastest_ms = INFINITY;
+    for (int i = 0; i < 4; i++) {
+        const auto t0 = std::chrono::steady_clock::now();
+        cufftHandle plan = 0;
+        if (cufftPlan3d(&plan, 64, 64, 64, CUFFT_R2C) != CUFFT_SUCCESS)
+            return NAN;
         cufftDestroy(plan);
+        const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
+        if (i > 0)
+            fastest_ms = std::min(fastest_ms, ms);
+    }
+    return fastest_ms;
+#else
+    return NAN;
 #endif
 }
 

image_analysis/indexing/IndexerThreadPool.h

@@ -37,8 +37,10 @@ enum class IndexerConstruction { Preconstruct, OnFirstUse };
 
 // Force cuFFT's one-time library initialisation - 0.3 s measured, paid by whichever call happens to
 // be first - so it can be got out of the way on a background thread at startup rather than landing in
-// the middle of the first pass with nothing overlapping it. No-op without a GPU.
-void WarmUpCuFFT();
+// the middle of the first pass with nothing overlapping it. Returns how long a 3D plan takes to make
+// after that, in ms - about 1 from a working CUDA JIT cache, 60 or more without (see CudaJitCache.h) -
+// and NaN without a GPU.
+double WarmUpCuFFT();
 
 // cuFFT plans are kept and reused for as long as one of these lives, and destroyed when the last one
 // ends (CudaFFTPlanScope): one per processing run in `rugnux`, so no plan outlives the run that made it;

rugnux/rugnux_cli.cpp

@@ -37,6 +37,7 @@
 #include <spdlog/sinks/ringbuffer_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 
+#include "../common/CudaJitCache.h"
 #include "../common/Logger.h"
 #include "../common/ThreadAffinity.h"
 #include "../common/GitInfo.h"
@@ -1701,8 +1702,9 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) {
         logger.Info("Using space group {} (number {})", space_group->hm, space_group->number);
 
     // Off the critical path, so cuFFT's one-time initialisation overlaps the file open, the mask read
-    // and the beam-stop pre-scan instead of landing inside the first pass.
-    auto cufft_warmup = std::async(std::launch::async, WarmUpCuFFT);
+    // and the beam-stop pre-scan instead of landing inside the first pass. It also times a plan, which
+    // shows whether the CUDA JIT cache works.
+    auto cufft_warmup = std::async(std::launch::async, [&logger] { LogCudaJitCache(logger, WarmUpCuFFT()); });
     // The same for the CUDA contexts of the other cards (the warm-up above creates card 0's), which
     // the pre-scan does not use: created one after another at start-up, they cost a fifth of a second
     // per card before the run began.