From bb6354252c0931f75436e65cd5ba734d900e264b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 16:13:40 +0100 Subject: [PATCH] Live stills: analysis threads per GPU as offline rugnux; stills test indexer pool 8 The live stills analysis took min(16, threads) threads whatever the cards. It now takes what offline rugnux's per-image loop takes (ImageWorkers): 16 a card, fewer on a card too small for 16 workers' engines (half its memory, about 24 bytes a detector pixel each, GPUWorkersPerCard), at most the threads given; without a GPU, the threads given. Both now come from free functions in Rugnux.h that the Rugnux members call. The indexer pool stays the broker's (indexing settings thread_count, default 4): it is shared with the per-image analysis and its GPU indexers are built once for the broker's life. jfjoch_fpga_stills_test -t defaults to 8, as offline rugnux on stills (StillsIndexerThreads). On this host (one GPU) the thread count does not change (16); on a server with 4 cards it is 64, at most -N. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/JFJochReceiverRugnuxStills.cpp | 4 +++- rugnux/Rugnux.h | 6 ++++++ rugnux/RugnuxImagePass.cpp | 21 +++++++++++++++------ tools/jfjoch_fpga_stills_test.cpp | 4 ++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/receiver/JFJochReceiverRugnuxStills.cpp b/receiver/JFJochReceiverRugnuxStills.cpp index 59b634bdb..a4133b54a 100644 --- a/receiver/JFJochReceiverRugnuxStills.cpp +++ b/receiver/JFJochReceiverRugnuxStills.cpp @@ -8,6 +8,7 @@ #include "../common/time_utc.h" #include "../frame_serialize/CBORStream2Serializer.h" +#include "../rugnux/Rugnux.h" #include "../rugnux/RugnuxDefaults.h" #include "RugnuxStructure.h" #include "StillsSelection.h" @@ -39,7 +40,8 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime downsample(in_downsample), live_status(in_live_status), indexing_thread_pool(in_indexing_thread_pool), - stills_nthreads(std::min(16, in_forward_nthreads)) { + // As offline rugnux's per-image loop: 16 a card, at most the threads given + stills_nthreads(ImageWorkers(in_experiment.GetPixelsNum(), static_cast(in_forward_nthreads))) { StartThreads(); } diff --git a/rugnux/Rugnux.h b/rugnux/Rugnux.h index 519c68ade..28d0d9e27 100644 --- a/rugnux/Rugnux.h +++ b/rugnux/Rugnux.h @@ -83,6 +83,12 @@ enum class ProcessMode { // Where the per-image analysis of an output pass goes as it is made: rugnux and the viewer put it in // _process.h5 (ProcessFileSink, RugnuxOutputFiles.h). Called from one thread at a time. +// The image workers a card takes: as many as keep half of its memory (about 24 bytes a detector pixel each), at most +// `most` (Rugnux::GPUWorkersPerCard); and those of a per-image loop: 16 a card, at most nthreads, nthreads without a +// GPU (Rugnux::ImageWorkers). Also the live stills analysis's. +int GPUWorkersPerCard(int64_t pixels, int most); +int ImageWorkers(int64_t pixels, int nthreads); + class RugnuxImageSink { public: virtual ~RugnuxImageSink() = default; diff --git a/rugnux/RugnuxImagePass.cpp b/rugnux/RugnuxImagePass.cpp index 872ea5145..cdb00ddb6 100644 --- a/rugnux/RugnuxImagePass.cpp +++ b/rugnux/RugnuxImagePass.cpp @@ -88,13 +88,16 @@ using namespace rugnux_internal; // detector, measured at 12-22 bytes a pixel on four data sets - and the workers keep to half of the card, // so a small card gets fewer of them. How many there are changes only how fast a loop runs, never what it // computes. -int Rugnux::GPUWorkersPerCard(int most) const { +int GPUWorkersPerCard(int64_t pixels, int most) { constexpr int64_t WORKER_BYTES_PER_PIXEL = 24; - const auto fit = static_cast(get_gpu_total_memory(0) / 2) - / (WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum()); + const auto fit = static_cast(get_gpu_total_memory(0) / 2) / (WORKER_BYTES_PER_PIXEL * pixels); return static_cast(std::clamp(fit, 1, most)); } +int Rugnux::GPUWorkersPerCard(int most) const { + return ::GPUWorkersPerCard(experiment_.GetPixelsNum(), most); +} + // Every worker submits its own kernels to a card, and a card runs out of room to accept them long before it // runs out of work: measured on two GPUs, the image loop's own time falls to sixteen workers and then rises // again, so forty-eight is slower than sixteen on a 16 Mpx set. Sixteen is where that measurement put the @@ -103,11 +106,17 @@ int Rugnux::GPUWorkersPerCard(int most) const { // was left at its default - an explicit -N is a deliberate instruction and is obeyed, which is what a // previous attempt at this got wrong. A worker also holds GPU memory, so a small card gets fewer of them // (GPUWorkersPerCard). -int Rugnux::ImageWorkers() const { +int ImageWorkers(int64_t pixels, int nthreads) { const int gpus = get_gpu_count(); - if (!config_.nthreads_auto || gpus == 0) + if (gpus == 0) + return nthreads; + return std::min(nthreads, GPUWorkersPerCard(pixels, 16) * gpus); +} + +int Rugnux::ImageWorkers() const { + if (!config_.nthreads_auto) return config_.nthreads; - return std::min(config_.nthreads, GPUWorkersPerCard(16) * gpus); + return ::ImageWorkers(experiment_.GetPixelsNum(), config_.nthreads); } void Rugnux::ImagePass(PipelineLocals &p) { diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index 32552fd30..46f20f0bb 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -39,7 +39,7 @@ void print_usage(Logger &logger) { logger.Info(" -C Unit cell (default: the file's)"); logger.Info(" -S Space group of the collection (a space group stored in the file is ignored)"); logger.Info(" -X Indexing (auto|fft|fftw|ffbidx), auto is default"); - logger.Info(" -t Indexing thread pool size (default: 4)"); + logger.Info(" -t Indexing thread pool size (default: 8, as offline rugnux on stills; the broker's default is 4)"); logger.Info(" -r Rates the source is paced at, one collection each (default: 0 = as fast as it goes); the"); logger.Info(" host takes frame i no earlier than i/rate, the card waits meanwhile: blocking runs only"); logger.Info(" -p FPGAImagePuller slots (default 128)"); @@ -323,7 +323,7 @@ int main(int argc, char **argv) { std::optional nimages; std::optional cell; IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto; - uint16_t indexing_threads = 4; + uint16_t indexing_threads = 8; // as offline rugnux's stills (Rugnux::StillsIndexerThreads) size_t puller_slots = FPGAImagePuller::DefaultSlots; std::optional mock_card_MiB; bool verbose = false;