From 8f47fb756bef406bfcedd8e0b3786519b1ee9a51 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sun, 8 Nov 2026 09:51:04 +0100 Subject: [PATCH] JFJochReceiverService::Indexing: kept FFT indexers of the old settings are released Since 0df766328 the broker keeps an FFTIndexerReuse for its whole life, so the broker's own indexing pool (Lite and FPGA receivers) hands its GPU FFT indexers back to the reuse cache instead of freeing them. The cache is keyed on the FFT settings and never shrank: every change of the indexing settings through the API parked another 0.6-0.9 GB per worker on the card, until a pool build ran out of GPU memory. (Before 0df766328 the same happened from the first live rotation run on.) The service now releases the kept indexers when it rebuilds the pool; rugnux live keeps new ones at its next run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- image_analysis/indexing/IndexerThreadPool.cpp | 13 +++++++++++++ image_analysis/indexing/IndexerThreadPool.h | 3 +++ receiver/JFJochReceiverService.cpp | 1 + tests/CUDAMemHelpersTest.cpp | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/image_analysis/indexing/IndexerThreadPool.cpp b/image_analysis/indexing/IndexerThreadPool.cpp index 25b761483..013ea4e51 100644 --- a/image_analysis/indexing/IndexerThreadPool.cpp +++ b/image_analysis/indexing/IndexerThreadPool.cpp @@ -59,6 +59,7 @@ FFTPlanReuse::~FFTPlanReuse() = default; FFTIndexerReuse::FFTIndexerReuse() = default; FFTIndexerReuse::~FFTIndexerReuse() = default; void KeepFFTIndexers(const IndexingSettings &, int) {} +void ReleaseKeptFFTIndexers() {} #endif #ifdef JFJOCH_USE_CUDA @@ -152,6 +153,18 @@ static std::unique_ptr MakeFFTIndexerGPU(const IndexingSettings &settin return std::make_unique(settings); } +void ReleaseKeptFFTIndexers() { + std::map>> idle; + { + std::lock_guard lock(kept_fft_indexers().m); + idle.swap(kept_fft_indexers().idle); + } + for (auto &[key, list] : idle) { + ScopedGpuPin pin(std::get<0>(key)); + list.clear(); + } +} + void KeepFFTIndexers(const IndexingSettings &settings, int count) { if (get_gpu_count() == 0) return; diff --git a/image_analysis/indexing/IndexerThreadPool.h b/image_analysis/indexing/IndexerThreadPool.h index 7e3b7fdd6..3f2f62b48 100644 --- a/image_analysis/indexing/IndexerThreadPool.h +++ b/image_analysis/indexing/IndexerThreadPool.h @@ -74,6 +74,9 @@ public: // destroyed). Nothing without a GPU. void KeepFFTIndexers(const IndexingSettings &settings, int count); +// Destroys the kept FFT indexers that no pool is using: a settings change leaves them keyed on the old settings. +void ReleaseKeptFFTIndexers(); + class IndexerThread { struct TaskInput { const DiffractionExperiment &experiment; diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index df3bd3345..ec1c575c7 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -433,6 +433,7 @@ JFJochReceiverService &JFJochReceiverService::Indexing(const IndexingSettings &i logger.Info("Resetting indexing thread pool"); indexer_thread_pool.reset(); + ReleaseKeptFFTIndexers(); if (input.GetAlgorithm() != IndexingAlgorithmEnum::None) { logger.Info("Creating indexing thread pool..."); diff --git a/tests/CUDAMemHelpersTest.cpp b/tests/CUDAMemHelpersTest.cpp index 6f5cf08fd..a01e008e9 100644 --- a/tests/CUDAMemHelpersTest.cpp +++ b/tests/CUDAMemHelpersTest.cpp @@ -443,6 +443,10 @@ TEST_CASE("IndexerThreadPool_FFTIndexerKeptAcrossPools", "[CUDAMemHelpers]") { CHECK(r.lattice.size() == fresh.lattice.size()); } CHECK(pooled() == kept); + + // A settings change in the broker: the kept indexers go while the reuse scope lives on + ReleaseKeptFFTIndexers(); + CHECK(pooled() <= before + (uint64_t(16) << 20)); } CHECK(pooled() <= before + (uint64_t(16) << 20)); for (int d = 0; d < get_gpu_count(); d++) {