From 3ac1fd657f358ef9221cea25840d980df006e93e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:09:53 +0100 Subject: [PATCH 1/3] CMake: cuFFT linked statically wherever the toolkit has the static library The server build linked libcufft.so, so any executable copied to another machine needed the CUDA toolkit library beside it; only the portable builds linked libcufft_static.a. Now every build does where CUDA::cufft_static exists (Linux); Windows, which has none, keeps the DLL as before. libcufft_static.a needs a device-link step in every executable that links it. The four per-target CUDA_RESOLVE_DEVICE_SYMBOLS settings (rugnux, jfjoch_viewer, jfjoch_portable_test, rugnux_refine_test) become one pass at the end of the top-level CMakeLists over the executables and shared libraries of the source tree (fetched dependencies left alone). Executables grow by about 290 MB (rugnux 193 -> 486 MB, jfjoch_broker 488 MB, jfjoch_fpga_stills_test 484 MB); ldd no longer lists libcufft. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- CMakeLists.txt | 26 +++++++++++++++++++ image_analysis/geom_refinement/CMakeLists.txt | 2 +- image_analysis/indexing/CMakeLists.txt | 8 +++--- rugnux/CMakeLists.txt | 8 ------ tests/CMakeLists.txt | 4 --- tools/CMakeLists.txt | 4 --- viewer/CMakeLists.txt | 8 ------ 7 files changed, 31 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 88e03e81b..4626a2392 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -656,6 +656,32 @@ IF (NOT JFJOCH_WRITER_ONLY AND NOT JFJOCH_PORTABLE_ONLY) INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend/dist/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch ) ENDIF() +# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable +# linking it needs a CUDA device-link step -- without it the host link fails on an undefined +# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which +# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving +# the host link driver alone, so the -march/-flto flags CI passes still apply. Set on every executable +# and shared library of this source tree, as nearly all of them link the image analysis. +FUNCTION(JFJOCH_RESOLVE_DEVICE_SYMBOLS dir) + GET_PROPERTY(_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS) + FOREACH(_target IN LISTS _targets) + GET_TARGET_PROPERTY(_type ${_target} TYPE) + IF (_type MATCHES "EXECUTABLE|SHARED_LIBRARY|MODULE_LIBRARY") + SET_TARGET_PROPERTIES(${_target} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) + ENDIF() + ENDFOREACH() + GET_PROPERTY(_subdirs DIRECTORY ${dir} PROPERTY SUBDIRECTORIES) + FOREACH(_subdir IN LISTS _subdirs) + STRING(FIND "${_subdir}" "${CMAKE_SOURCE_DIR}/" _position) + IF (_position EQUAL 0) # not the fetched dependencies + JFJOCH_RESOLVE_DEVICE_SYMBOLS(${_subdir}) + ENDIF() + ENDFOREACH() +ENDFUNCTION() +IF (JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) + JFJOCH_RESOLVE_DEVICE_SYMBOLS(${CMAKE_SOURCE_DIR}) +ENDIF() + # Windows has no static cuFFT (the toolkit ships no cufft_static.lib), so there the indexer links # the import library and the self-contained products must carry the DLL beside the executable -- # the same idea the Linux builds no longer need, since they link libcufft_static.a. CUDA 13 keeps diff --git a/image_analysis/geom_refinement/CMakeLists.txt b/image_analysis/geom_refinement/CMakeLists.txt index a82fee507..74659539b 100644 --- a/image_analysis/geom_refinement/CMakeLists.txt +++ b/image_analysis/geom_refinement/CMakeLists.txt @@ -47,7 +47,7 @@ IF (JFJOCH_CUDA_AVAILABLE) BeamCenterBackgroundGPU.cu BeamCenterBackgroundGPU.h) # Same static/dynamic cuFFT choice as the FFT indexer, and for the same reasons - see the long # note in image_analysis/indexing/CMakeLists.txt. - IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static) + IF (TARGET CUDA::cufft_static) TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft_static) ELSE() TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft) diff --git a/image_analysis/indexing/CMakeLists.txt b/image_analysis/indexing/CMakeLists.txt index e7d787197..66f005b10 100644 --- a/image_analysis/indexing/CMakeLists.txt +++ b/image_analysis/indexing/CMakeLists.txt @@ -51,10 +51,10 @@ IF (JFJOCH_CUDA_AVAILABLE) # and the dynamic library is the only option; the top-level CMakeLists installs the DLL beside # the executable for the self-contained builds. Test the target rather than the platform, so a # toolkit that gains or loses the static library is handled without another special case. - # Both halves matter: the server build keeps the shared library (its .deb/.rpm take CUDA from the - # distro, and every executable there -- broker, tests, tools -- would otherwise need the device - # link below), and Windows has no static cuFFT to link even for the portable products. - IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static) + # The server build links it statically too, so its executables (broker, tools copied to another + # machine) need no CUDA toolkit library; the top-level CMakeLists gives every executable the device + # link this needs. + IF (TARGET CUDA::cufft_static) TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft_static) ELSE() TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft) diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index 12db3d061..132d93764 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -66,14 +66,6 @@ IF(APPLE AND JFJOCH_MACOS_SIGNING_IDENTITY) COMPONENT rugnux) ENDIF() -# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable -# linking it needs a CUDA device-link step -- without it the host link fails on an undefined -# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which -# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving -# the host link driver alone, so the -march/-flto flags CI passes still apply. -IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) - SET_TARGET_PROPERTIES(rugnux PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) -ENDIF() # On Windows this CLI gets getopt/getopt_long from the vendored wingetopt shim (libc has none). diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f30cb28c6..ed7af1f68 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,10 +47,6 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis JFJochWriter JFJochAPIConvert JFJochCommon) target_include_directories(jfjoch_portable_test PRIVATE .) -# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically. -IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) - SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) -ENDIF() # The CPU vs GPU test of the fast-feedback indexer reads the library's own test spot lists IF (JFJOCH_CUDA_AVAILABLE) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 9fdf2e2a9..74d302a1c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -18,10 +18,6 @@ TARGET_LINK_LIBRARIES(jfjoch_extract_hkl JFJochReader) # measuring the refinement on battery outputs without processing anything again. ADD_EXECUTABLE(rugnux_refine_test rugnux_refine_test.cpp) TARGET_LINK_LIBRARIES(rugnux_refine_test JFJochStructureRefinement JFJochCommon gemmi) -# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically. -IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) - SET_TARGET_PROPERTIES(rugnux_refine_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) -ENDIF() # In-place re-compress /entry/data/data of a _data file from bitshuffle/LZ4 to bitshuffle/zstd. ADD_EXECUTABLE(jfjoch_recompress jfjoch_recompress.cpp) diff --git a/viewer/CMakeLists.txt b/viewer/CMakeLists.txt index 0345b7587..ec62a3b68 100644 --- a/viewer/CMakeLists.txt +++ b/viewer/CMakeLists.txt @@ -184,14 +184,6 @@ IF(APPLE) PATTERN "COLLECT.sh" EXCLUDE) ENDIF() -# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable -# linking it needs a CUDA device-link step -- without it the host link fails on an undefined -# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which -# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving -# the host link driver alone, so the -march/-flto flags CI passes still apply. -IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static) - SET_TARGET_PROPERTIES(jfjoch_viewer PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) -ENDIF() IF(JFJOCH_VIEWER_DBUS) From efb676520734e382c80517011d1799fd6581afc5 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:09:53 +0100 Subject: [PATCH 2/3] jfjoch_fpga_stills_test: -d (not blocking, as the broker), -v card progress -d runs the puller as the broker does, dropping what the host has no room for, to compare with the default blocking run. -v logs each card's slowest and fastest module frame every 0.5 s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- tools/jfjoch_fpga_stills_test.cpp | 35 +++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index 11843b7bd..09e0eacd6 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -7,9 +7,11 @@ // The generator cycles through the images its memory holds, so images repeat: throughput, latency and indexing rate // are meaningful, the merge and its statistics are not. +#include #include #include #include +#include #include #include "../acquisition_device/PCIExpressDevice.h" @@ -28,7 +30,7 @@ void print_usage(Logger &logger) { logger.Info("loaded into the cards' internal generators, which cycle through them: images repeat, so throughput, latency"); logger.Info("and indexing rate are meaningful, the merge and its statistics are not. See docs/TOOLS.md."); logger.Info("Options:"); - logger.Info(" -v Verbose"); + logger.Info(" -v Verbose, with each card's progress every 0.5 s"); logger.Info(" -i Images in the run (default: as many as the file has)"); logger.Info(" -s Number of data streams (cards)"); logger.Info(" -N Number of image processing threads (default 64)"); @@ -38,6 +40,7 @@ void print_usage(Logger &logger) { logger.Info(" -t Indexing thread pool size (default: 4)"); logger.Info(" -r Rates the source is paced at, one collection each (default: 0 = as fast as it goes)"); logger.Info(" -p FPGAImagePuller slots (default 128)"); + logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped"); logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller (not blocking: frames"); logger.Info(" past what its analysis holds are lost)"); logger.Info(" -K{} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB ="); @@ -85,7 +88,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file, std::optional nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB, const IndexingSettings &indexing, const std::optional &cell, size_t puller_slots, - bool fpga_receiver, const std::vector &rates) { + bool fpga_receiver, const std::vector &rates, bool dropping) { RegisterHDF5Filter(); JFJochHDF5Reader reader; reader.ReadFile(file); @@ -165,10 +168,11 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s // Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer, // the cards hold their frames and their generators stall. JFJochReceiverFPGA's live stills cannot block. service.UseFPGAImagePuller(fpga_receiver ? FPGAImagePullerUse::Never : FPGAImagePullerUse::RugnuxLive, - puller_slots, true, rate); + puller_slots, !dropping, rate); logger.Info(""); logger.Info("=== {}, {} ===", fpga_receiver ? "JFJochReceiverFPGA (not blocking)" - : fmt::format("FPGAImagePuller ({} slots, blocking)", puller_slots), + : fmt::format("FPGAImagePuller ({} slots, {})", puller_slots, + dropping ? "dropping" : "blocking"), (rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes")); rusage usage_start{}; @@ -176,7 +180,22 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s const auto wall_start = std::chrono::steady_clock::now(); service.Start(x, mask, nullptr); + // -v: each card's progress, the frames of its fastest and slowest module written to the host + std::atomic running = true; + std::thread progress([&] { + while (running) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + std::string line; + for (int i = 0; i < x.GetDataStreamsNum(); i++) + line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(), + aq_devices[i].Counters().GetFastestFrameNumber()); + logger.Debug("{:.1f} s{}", std::chrono::duration(std::chrono::steady_clock::now() + - wall_start).count(), line); + } + }); const auto output = service.Stop(); + running = false; + progress.join(); const auto live = service.GetRugnuxStatus(); const double wall_time = std::chrono::duration(std::chrono::steady_clock::now() - wall_start).count(); @@ -274,9 +293,10 @@ int main(int argc, char **argv) { std::optional mock_card_MiB; bool verbose = false; std::vector rates = {0}; + bool dropping = false; int opt; - while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:FK::")) != -1) { + while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dFK::")) != -1) { switch (opt) { case 'v': verbose = true; @@ -329,6 +349,9 @@ int main(int argc, char **argv) { case 'F': fpga_receiver = true; break; + case 'd': + dropping = true; + break; case 'K': mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64; break; @@ -364,7 +387,7 @@ int main(int argc, char **argv) { bool ret = false; try { ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB, - i_settings, cell, puller_slots, fpga_receiver, rates); + i_settings, cell, puller_slots, fpga_receiver, rates, dropping); } catch (std::exception &e) { logger.Error(e.what()); } From 3a5391dd65152cf7fabf0f593239f27a044d655c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 10:20:23 +0100 Subject: [PATCH 3/3] FPGAImagePuller: a module behind another card waits as long as the slots hold A module was given up for a frame once ANY card was MissingAfterFrames (52) frames further on. The cards run on their own, so a card a little behind lost whole modules from every image: on the mock, 9M on 2 and 4 cards not blocking, the cards drift 50-500 frames apart (each mock card is two CPU threads; none of them ever waited for host buffers, so the drift is the cards' own speed, not the host) and the efficiency was 0.44-0.61. Now a module is missing when its OWN card is MissingAfterFrames further on, as the device itself counts, and behind another card only once that card is max_slots ahead - past that, its images would be dropped for want of a slot anyway. A dead card is still noticed, after max_slots frames. Mock, 9M, 1500 images, not blocking: 2 cards 0.905 (drift up to ~120), 4 cards with 512 slots 0.978 (drift ~380), 4 cards with 128 slots 0.487 (drift ~500, now whole images dropped for want of a slot rather than modules missing). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/FPGAImagePuller.cpp | 3 ++- receiver/FPGAImagePuller.h | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index 26eacc4c0..5d0d884a4 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -147,7 +147,8 @@ void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, // A frame is complete once the module is two further on, as in JFJochReceiverFPGA if ((static_cast(counters.GetCurrFrameNumber(module_number)) >= frame + 2) || counters.IsAcquisitionFinished() || failed - || (!blocking && (FastestFrame() >= frame + 2 + MissingAfterFrames))) + || (!blocking && (static_cast(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames)) + || (!blocking && (FastestFrame() >= frame + 2 + static_cast(max_slots)))) return; std::this_thread::sleep_for(std::chrono::milliseconds(1)); } diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index 844a839f5..d5d90a16e 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -24,7 +24,9 @@ class FPGAImagePuller : public ImagePuller { public: static constexpr size_t DefaultSlots = 128; - // A module this far behind the fastest one is taken as missing from that frame, as the device itself does + // A module this far behind the fastest one of its card is taken as missing from that frame, as the device itself + // does. Behind another card it is waited for as long as the slots can hold the images started meanwhile: the + // cards run on their own and one can be ahead, but past max_slots its images would be dropped anyway. static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2; private: struct Slot {