Merge commit '3a5391dd6' into rc188-l

This commit is contained in:
leonarski_f committed 2026-11-18 10:57:27 +01:00
commit 466edb7415
10 files changed
+65 -37

No files matched your search

+26
View File
@@ -656,6 +656,32 @@ IF (NOT JFJOCH_WRITER_ONLY AND NOT JFJOCH_PORTABLE_ONLY)
INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend/dist/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch )
ENDIF()
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
# the host link driver alone, so the -march/-flto flags CI passes still apply. Set on every executable
# and shared library of this source tree, as nearly all of them link the image analysis.
FUNCTION(JFJOCH_RESOLVE_DEVICE_SYMBOLS dir)
GET_PROPERTY(_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS)
FOREACH(_target IN LISTS _targets)
GET_TARGET_PROPERTY(_type ${_target} TYPE)
IF (_type MATCHES "EXECUTABLE|SHARED_LIBRARY|MODULE_LIBRARY")
SET_TARGET_PROPERTIES(${_target} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
ENDFOREACH()
GET_PROPERTY(_subdirs DIRECTORY ${dir} PROPERTY SUBDIRECTORIES)
FOREACH(_subdir IN LISTS _subdirs)
STRING(FIND "${_subdir}" "${CMAKE_SOURCE_DIR}/" _position)
IF (_position EQUAL 0) # not the fetched dependencies
JFJOCH_RESOLVE_DEVICE_SYMBOLS(${_subdir})
ENDIF()
ENDFOREACH()
ENDFUNCTION()
IF (JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
JFJOCH_RESOLVE_DEVICE_SYMBOLS(${CMAKE_SOURCE_DIR})
ENDIF()
# Windows has no static cuFFT (the toolkit ships no cufft_static.lib), so there the indexer links
# the import library and the self-contained products must carry the DLL beside the executable --
# the same idea the Linux builds no longer need, since they link libcufft_static.a. CUDA 13 keeps
@@ -47,7 +47,7 @@ IF (JFJOCH_CUDA_AVAILABLE)
BeamCenterBackgroundGPU.cu BeamCenterBackgroundGPU.h)
# Same static/dynamic cuFFT choice as the FFT indexer, and for the same reasons - see the long
# note in image_analysis/indexing/CMakeLists.txt.
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
IF (TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft_static)
ELSE()
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft)
+4 -4
View File
@@ -51,10 +51,10 @@ IF (JFJOCH_CUDA_AVAILABLE)
# and the dynamic library is the only option; the top-level CMakeLists installs the DLL beside
# the executable for the self-contained builds. Test the target rather than the platform, so a
# toolkit that gains or loses the static library is handled without another special case.
# Both halves matter: the server build keeps the shared library (its .deb/.rpm take CUDA from the
# distro, and every executable there -- broker, tests, tools -- would otherwise need the device
# link below), and Windows has no static cuFFT to link even for the portable products.
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
# The server build links it statically too, so its executables (broker, tools copied to another
# machine) need no CUDA toolkit library; the top-level CMakeLists gives every executable the device
# link this needs.
IF (TARGET CUDA::cufft_static)
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft_static)
ELSE()
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft)
+2 -1
View File
@@ -147,7 +147,8 @@ void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number,
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
if ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) >= frame + 2)
|| counters.IsAcquisitionFinished() || failed
|| (!blocking && (FastestFrame() >= frame + 2 + MissingAfterFrames)))
|| (!blocking && (static_cast<int64_t>(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames))
|| (!blocking && (FastestFrame() >= frame + 2 + static_cast<int64_t>(max_slots))))
return;
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
+3 -1
View File
@@ -24,7 +24,9 @@
class FPGAImagePuller : public ImagePuller {
public:
static constexpr size_t DefaultSlots = 128;
// A module this far behind the fastest one is taken as missing from that frame, as the device itself does
// A module this far behind the fastest one of its card is taken as missing from that frame, as the device itself
// does. Behind another card it is waited for as long as the slots can hold the images started meanwhile: the
// cards run on their own and one can be ahead, but past max_slots its images would be dropped anyway.
static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2;
private:
struct Slot {
-8
View File
@@ -66,14 +66,6 @@ IF(APPLE AND JFJOCH_MACOS_SIGNING_IDENTITY)
COMPONENT rugnux)
ENDIF()
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
# the host link driver alone, so the -march/-flto flags CI passes still apply.
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
SET_TARGET_PROPERTIES(rugnux PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
# On Windows this CLI gets getopt/getopt_long from the vendored wingetopt shim (libc has none).
-4
View File
@@ -47,10 +47,6 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis
JFJochWriter JFJochAPIConvert JFJochCommon)
target_include_directories(jfjoch_portable_test PRIVATE .)
# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically.
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
# The CPU vs GPU test of the fast-feedback indexer reads the library's own test spot lists
IF (JFJOCH_CUDA_AVAILABLE)
-4
View File
@@ -18,10 +18,6 @@ TARGET_LINK_LIBRARIES(jfjoch_extract_hkl JFJochReader)
# measuring the refinement on battery outputs without processing anything again.
ADD_EXECUTABLE(rugnux_refine_test rugnux_refine_test.cpp)
TARGET_LINK_LIBRARIES(rugnux_refine_test JFJochStructureRefinement JFJochCommon gemmi)
# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically.
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
SET_TARGET_PROPERTIES(rugnux_refine_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
# In-place re-compress /entry/data/data of a _data file from bitshuffle/LZ4 to bitshuffle/zstd.
ADD_EXECUTABLE(jfjoch_recompress jfjoch_recompress.cpp)
+29 -6
View File
@@ -7,9 +7,11 @@
// The generator cycles through the images its memory holds, so images repeat: throughput, latency and indexing rate
// are meaningful, the merge and its statistics are not.
#include <atomic>
#include <getopt.h>
#include <iostream>
#include <sstream>
#include <thread>
#include <sys/resource.h>
#include "../acquisition_device/PCIExpressDevice.h"
@@ -28,7 +30,7 @@ void print_usage(Logger &logger) {
logger.Info("loaded into the cards' internal generators, which cycle through them: images repeat, so throughput, latency");
logger.Info("and indexing rate are meaningful, the merge and its statistics are not. See docs/TOOLS.md.");
logger.Info("Options:");
logger.Info(" -v Verbose");
logger.Info(" -v Verbose, with each card's progress every 0.5 s");
logger.Info(" -i<num> Images in the run (default: as many as the file has)");
logger.Info(" -s<num> Number of data streams (cards)");
logger.Info(" -N<num> Number of image processing threads (default 64)");
@@ -38,6 +40,7 @@ void print_usage(Logger &logger) {
logger.Info(" -t<num> Indexing thread pool size (default: 4)");
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes)");
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped");
logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller (not blocking: frames");
logger.Info(" past what its analysis holds are lost)");
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
@@ -85,7 +88,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, size_t puller_slots,
bool fpga_receiver, const std::vector<double> &rates) {
bool fpga_receiver, const std::vector<double> &rates, bool dropping) {
RegisterHDF5Filter();
JFJochHDF5Reader reader;
reader.ReadFile(file);
@@ -165,10 +168,11 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
// Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer,
// the cards hold their frames and their generators stall. JFJochReceiverFPGA's live stills cannot block.
service.UseFPGAImagePuller(fpga_receiver ? FPGAImagePullerUse::Never : FPGAImagePullerUse::RugnuxLive,
puller_slots, true, rate);
puller_slots, !dropping, rate);
logger.Info("");
logger.Info("=== {}, {} ===", fpga_receiver ? "JFJochReceiverFPGA (not blocking)"
: fmt::format("FPGAImagePuller ({} slots, blocking)", puller_slots),
: fmt::format("FPGAImagePuller ({} slots, {})", puller_slots,
dropping ? "dropping" : "blocking"),
(rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes"));
rusage usage_start{};
@@ -176,7 +180,22 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
const auto wall_start = std::chrono::steady_clock::now();
service.Start(x, mask, nullptr);
// -v: each card's progress, the frames of its fastest and slowest module written to the host
std::atomic<bool> running = true;
std::thread progress([&] {
while (running) {
std::this_thread::sleep_for(std::chrono::milliseconds(500));
std::string line;
for (int i = 0; i < x.GetDataStreamsNum(); i++)
line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
aq_devices[i].Counters().GetFastestFrameNumber());
logger.Debug("{:.1f} s{}", std::chrono::duration<double>(std::chrono::steady_clock::now()
- wall_start).count(), line);
}
});
const auto output = service.Stop();
running = false;
progress.join();
const auto live = service.GetRugnuxStatus();
const double wall_time = std::chrono::duration<double>(std::chrono::steady_clock::now() - wall_start).count();
@@ -274,9 +293,10 @@ int main(int argc, char **argv) {
std::optional<uint32_t> mock_card_MiB;
bool verbose = false;
std::vector<double> rates = {0};
bool dropping = false;
int opt;
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:FK::")) != -1) {
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dFK::")) != -1) {
switch (opt) {
case 'v':
verbose = true;
@@ -329,6 +349,9 @@ int main(int argc, char **argv) {
case 'F':
fpga_receiver = true;
break;
case 'd':
dropping = true;
break;
case 'K':
mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64;
break;
@@ -364,7 +387,7 @@ int main(int argc, char **argv) {
bool ret = false;
try {
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
i_settings, cell, puller_slots, fpga_receiver, rates);
i_settings, cell, puller_slots, fpga_receiver, rates, dropping);
} catch (std::exception &e) {
logger.Error(e.what());
}
-8
View File
@@ -184,14 +184,6 @@ IF(APPLE)
PATTERN "COLLECT.sh" EXCLUDE)
ENDIF()
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
# the host link driver alone, so the -march/-flto flags CI passes still apply.
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
SET_TARGET_PROPERTIES(jfjoch_viewer PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
ENDIF()
IF(JFJOCH_VIEWER_DBUS)