From 52f94d29538be7dc880e863ff4e87f30c3511f3b Mon Sep 17 00:00:00 2001
From: Filip Leonarski <filip.leonarski@psi.ch>
Date: Sun, 11 Oct 2026 16:39:43 +0200
Subject: [PATCH] tests: FFT-indexer admission test fills every card, right
 before each Run

IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside filled only the test thread's
card, but an indexer worker takes its card round-robin (pin_gpu: a process-wide
counter modulo the device count). On the CI runner, a container given all of the
host's GPUs, the probe's worker could land on a card nobody had filled, build its
indexer and fail "CHECK(probe.Run(x, recip).error != nullptr)"; the two checks that
expect a wait passed there for the same wrong reason. On a one-card workstation the
worker is always on device 0, so it never showed.

Every card is now filled, each under its own device, and released the same way, and
the fill is topped up right before each Run so memory given back in between by
anything else does not let the indexer fit. The pools are trimmed on every card at
the end. What the test checks is unchanged.

Run 20x alone and with the [CUDAMemHelpers],[AdaptiveSpotFinderGPU] group, also with
another process holding 4 GB of the card: all pass.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---
 tests/CUDAMemHelpersTest.cpp | 52 +++++++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/tests/CUDAMemHelpersTest.cpp b/tests/CUDAMemHelpersTest.cpp
index e095333a7..2d0047319 100644
--- a/tests/CUDAMemHelpersTest.cpp
+++ b/tests/CUDAMemHelpersTest.cpp
@@ -141,55 +141,69 @@ TEST_CASE("IndexerThreadPool_FFTIndexerWaitsForGPUWorkBeside", "[CUDAMemHelpers]
     x.ImportIndexingSettings(settings);
     const std::vector<Coord> recip{{0.1, 0, 0}, {0, 0.1, 0}, {0, 0, 0.1}};
 
-    // The whole card, held by the work beside.
-    std::optional<std::vector<CudaDevicePtr<uint8_t>>> held;
-    const auto fill = [&held] {
-        held.emplace();
-        for (size_t chunk = size_t(256) << 20; chunk >= (size_t(1) << 20); chunk /= 2)
-            while (true) {
-                try {
-                    held->emplace_back(chunk, CudaAlloc::Synchronous);
-                } catch (const JFJochException &) {
-                    break;
+    // Every card, held by the work beside: an indexer worker takes its card round-robin (pin_gpu), so on
+    // a machine with several the one it lands on is not the test thread's. Topped up right before each
+    // Run, so memory something else gave back in between does not let the indexer fit.
+    const int gpus = get_gpu_count();
+    std::vector<std::vector<CudaDevicePtr<uint8_t>>> held(gpus);
+    const auto fill = [&] {
+        for (int d = 0; d < gpus; d++) {
+            ScopedGpuPin pin(d);
+            for (size_t chunk = size_t(256) << 20; chunk >= (size_t(1) << 20); chunk /= 2)
+                while (true) {
+                    try {
+                        held[d].emplace_back(chunk, CudaAlloc::Synchronous);
+                    } catch (const JFJochException &) {
+                        break;
+                    }
                 }
-            }
+        }
+    };
+    const auto release = [&] {
+        for (int d = 0; d < gpus; d++) {
+            ScopedGpuPin pin(d);
+            held[d].clear();
+        }
     };
     // The pools' indexers go back to the allocation pool when they are destroyed; trim it after them, so
-    // the next test finds the card free.
+    // the next test finds the cards free.
     {
         IndexerThreadPool probe(settings, IndexerConstruction::OnFirstUse, /*beside=*/true);
         IndexerThreadPool pool(settings, IndexerConstruction::OnFirstUse);
 
-        fill();
         {
             GPUWorkBeside work;
+            fill();
             CHECK(probe.Run(x, recip).error != nullptr);
         }
         {
             std::optional<GPUImagePass> image_pass(std::in_place);
+            GPUWorkBeside work;
+            fill();
             std::thread ends([&] {
                 std::this_thread::sleep_for(std::chrono::milliseconds(500));
-                held.reset();
+                release();
                 image_pass.reset();
             });
-            GPUWorkBeside work;
             CHECK(probe.Run(x, recip).error == nullptr);
             ends.join();
         }
-
-        fill();
         {
             std::optional<GPUWorkBeside> work(std::in_place);
+            fill();
             std::thread ends([&] {
                 std::this_thread::sleep_for(std::chrono::milliseconds(500));
-                held.reset();
+                release();
                 work.reset();
             });
             CHECK(pool.Run(x, recip).error == nullptr);
             ends.join();
         }
     }
-    cuda_trim_allocation_pool();
+    for (int d = 0; d < gpus; d++) {
+        ScopedGpuPin pin(d);
+        cuda_trim_allocation_pool();
+    }
 }
 
 #endif