Live stills: analysis threads per GPU as offline rugnux; stills test indexer pool 8
The live stills analysis took min(16, threads) threads whatever the cards. It now takes what offline rugnux's per-image loop takes (ImageWorkers): 16 a card, fewer on a card too small for 16 workers' engines (half its memory, about 24 bytes a detector pixel each, GPUWorkersPerCard), at most the threads given; without a GPU, the threads given. Both now come from free functions in Rugnux.h that the Rugnux members call. The indexer pool stays the broker's (indexing settings thread_count, default 4): it is shared with the per-image analysis and its GPU indexers are built once for the broker's life. jfjoch_fpga_stills_test -t defaults to 8, as offline rugnux on stills (StillsIndexerThreads). On this host (one GPU) the thread count does not change (16); on a server with 4 cards it is 64, at most -N. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
1 parent
1b76b4539e
commit
bb6354252c
4 files changed
+26
-9
No files matched your search
@@ -8,6 +8,7 @@
|
||||
|
||||
#include "../common/time_utc.h"
|
||||
#include "../frame_serialize/CBORStream2Serializer.h"
|
||||
#include "../rugnux/Rugnux.h"
|
||||
#include "../rugnux/RugnuxDefaults.h"
|
||||
#include "RugnuxStructure.h"
|
||||
#include "StillsSelection.h"
|
||||
@@ -39,7 +40,8 @@ JFJochReceiverRugnuxStills::JFJochReceiverRugnuxStills(const DiffractionExperime
|
||||
downsample(in_downsample),
|
||||
live_status(in_live_status),
|
||||
indexing_thread_pool(in_indexing_thread_pool),
|
||||
stills_nthreads(std::min<int64_t>(16, in_forward_nthreads)) {
|
||||
// As offline rugnux's per-image loop: 16 a card, at most the threads given
|
||||
stills_nthreads(ImageWorkers(in_experiment.GetPixelsNum(), static_cast<int>(in_forward_nthreads))) {
|
||||
StartThreads();
|
||||
}
|
||||
|
||||
|
||||
@@ -83,6 +83,12 @@ enum class ProcessMode {
|
||||
|
||||
// Where the per-image analysis of an output pass goes as it is made: rugnux and the viewer put it in
|
||||
// <prefix>_process.h5 (ProcessFileSink, RugnuxOutputFiles.h). Called from one thread at a time.
|
||||
// The image workers a card takes: as many as keep half of its memory (about 24 bytes a detector pixel each), at most
|
||||
// `most` (Rugnux::GPUWorkersPerCard); and those of a per-image loop: 16 a card, at most nthreads, nthreads without a
|
||||
// GPU (Rugnux::ImageWorkers). Also the live stills analysis's.
|
||||
int GPUWorkersPerCard(int64_t pixels, int most);
|
||||
int ImageWorkers(int64_t pixels, int nthreads);
|
||||
|
||||
class RugnuxImageSink {
|
||||
public:
|
||||
virtual ~RugnuxImageSink() = default;
|
||||
|
||||
@@ -88,13 +88,16 @@ using namespace rugnux_internal;
|
||||
// detector, measured at 12-22 bytes a pixel on four data sets - and the workers keep to half of the card,
|
||||
// so a small card gets fewer of them. How many there are changes only how fast a loop runs, never what it
|
||||
// computes.
|
||||
int Rugnux::GPUWorkersPerCard(int most) const {
|
||||
int GPUWorkersPerCard(int64_t pixels, int most) {
|
||||
constexpr int64_t WORKER_BYTES_PER_PIXEL = 24;
|
||||
const auto fit = static_cast<int64_t>(get_gpu_total_memory(0) / 2)
|
||||
/ (WORKER_BYTES_PER_PIXEL * experiment_.GetPixelsNum());
|
||||
const auto fit = static_cast<int64_t>(get_gpu_total_memory(0) / 2) / (WORKER_BYTES_PER_PIXEL * pixels);
|
||||
return static_cast<int>(std::clamp<int64_t>(fit, 1, most));
|
||||
}
|
||||
|
||||
int Rugnux::GPUWorkersPerCard(int most) const {
|
||||
return ::GPUWorkersPerCard(experiment_.GetPixelsNum(), most);
|
||||
}
|
||||
|
||||
// Every worker submits its own kernels to a card, and a card runs out of room to accept them long before it
|
||||
// runs out of work: measured on two GPUs, the image loop's own time falls to sixteen workers and then rises
|
||||
// again, so forty-eight is slower than sixteen on a 16 Mpx set. Sixteen is where that measurement put the
|
||||
@@ -103,11 +106,17 @@ int Rugnux::GPUWorkersPerCard(int most) const {
|
||||
// was left at its default - an explicit -N is a deliberate instruction and is obeyed, which is what a
|
||||
// previous attempt at this got wrong. A worker also holds GPU memory, so a small card gets fewer of them
|
||||
// (GPUWorkersPerCard).
|
||||
int Rugnux::ImageWorkers() const {
|
||||
int ImageWorkers(int64_t pixels, int nthreads) {
|
||||
const int gpus = get_gpu_count();
|
||||
if (!config_.nthreads_auto || gpus == 0)
|
||||
if (gpus == 0)
|
||||
return nthreads;
|
||||
return std::min(nthreads, GPUWorkersPerCard(pixels, 16) * gpus);
|
||||
}
|
||||
|
||||
int Rugnux::ImageWorkers() const {
|
||||
if (!config_.nthreads_auto)
|
||||
return config_.nthreads;
|
||||
return std::min(config_.nthreads, GPUWorkersPerCard(16) * gpus);
|
||||
return ::ImageWorkers(experiment_.GetPixelsNum(), config_.nthreads);
|
||||
}
|
||||
|
||||
void Rugnux::ImagePass(PipelineLocals &p) {
|
||||
|
||||
@@ -39,7 +39,7 @@ void print_usage(Logger &logger) {
|
||||
logger.Info(" -C<a,b,c,alpha,beta,gamma> Unit cell (default: the file's)");
|
||||
logger.Info(" -S<group> Space group of the collection (a space group stored in the file is ignored)");
|
||||
logger.Info(" -X<txt> Indexing (auto|fft|fftw|ffbidx), auto is default");
|
||||
logger.Info(" -t<num> Indexing thread pool size (default: 4)");
|
||||
logger.Info(" -t<num> Indexing thread pool size (default: 8, as offline rugnux on stills; the broker's default is 4)");
|
||||
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes); the");
|
||||
logger.Info(" host takes frame i no earlier than i/rate, the card waits meanwhile: blocking runs only");
|
||||
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
|
||||
@@ -323,7 +323,7 @@ int main(int argc, char **argv) {
|
||||
std::optional<size_t> nimages;
|
||||
std::optional<UnitCell> cell;
|
||||
IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto;
|
||||
uint16_t indexing_threads = 4;
|
||||
uint16_t indexing_threads = 8; // as offline rugnux's stills (Rugnux::StillsIndexerThreads)
|
||||
size_t puller_slots = FPGAImagePuller::DefaultSlots;
|
||||
std::optional<uint32_t> mock_card_MiB;
|
||||
bool verbose = false;
|
||||
|
||||
Reference in new issue
Block a user