Merge commit '3a5391dd6' into rc188-l
This commit is contained in:
commit
466edb7415
10 files changed
+65
-37
No files matched your search
@@ -656,6 +656,32 @@ IF (NOT JFJOCH_WRITER_ONLY AND NOT JFJOCH_PORTABLE_ONLY)
|
||||
INSTALL(DIRECTORY ${CMAKE_SOURCE_DIR}/frontend/dist/ DESTINATION share/jfjoch/frontend COMPONENT jfjoch )
|
||||
ENDIF()
|
||||
|
||||
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
|
||||
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
|
||||
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
|
||||
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
|
||||
# the host link driver alone, so the -march/-flto flags CI passes still apply. Set on every executable
|
||||
# and shared library of this source tree, as nearly all of them link the image analysis.
|
||||
FUNCTION(JFJOCH_RESOLVE_DEVICE_SYMBOLS dir)
|
||||
GET_PROPERTY(_targets DIRECTORY ${dir} PROPERTY BUILDSYSTEM_TARGETS)
|
||||
FOREACH(_target IN LISTS _targets)
|
||||
GET_TARGET_PROPERTY(_type ${_target} TYPE)
|
||||
IF (_type MATCHES "EXECUTABLE|SHARED_LIBRARY|MODULE_LIBRARY")
|
||||
SET_TARGET_PROPERTIES(${_target} PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
ENDIF()
|
||||
ENDFOREACH()
|
||||
GET_PROPERTY(_subdirs DIRECTORY ${dir} PROPERTY SUBDIRECTORIES)
|
||||
FOREACH(_subdir IN LISTS _subdirs)
|
||||
STRING(FIND "${_subdir}" "${CMAKE_SOURCE_DIR}/" _position)
|
||||
IF (_position EQUAL 0) # not the fetched dependencies
|
||||
JFJOCH_RESOLVE_DEVICE_SYMBOLS(${_subdir})
|
||||
ENDIF()
|
||||
ENDFOREACH()
|
||||
ENDFUNCTION()
|
||||
IF (JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
|
||||
JFJOCH_RESOLVE_DEVICE_SYMBOLS(${CMAKE_SOURCE_DIR})
|
||||
ENDIF()
|
||||
|
||||
# Windows has no static cuFFT (the toolkit ships no cufft_static.lib), so there the indexer links
|
||||
# the import library and the self-contained products must carry the DLL beside the executable --
|
||||
# the same idea the Linux builds no longer need, since they link libcufft_static.a. CUDA 13 keeps
|
||||
|
||||
@@ -47,7 +47,7 @@ IF (JFJOCH_CUDA_AVAILABLE)
|
||||
BeamCenterBackgroundGPU.cu BeamCenterBackgroundGPU.h)
|
||||
# Same static/dynamic cuFFT choice as the FFT indexer, and for the same reasons - see the long
|
||||
# note in image_analysis/indexing/CMakeLists.txt.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
|
||||
IF (TARGET CUDA::cufft_static)
|
||||
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft_static)
|
||||
ELSE()
|
||||
TARGET_LINK_LIBRARIES(JFJochGeomRefinement CUDA::cufft)
|
||||
|
||||
@@ -51,10 +51,10 @@ IF (JFJOCH_CUDA_AVAILABLE)
|
||||
# and the dynamic library is the only option; the top-level CMakeLists installs the DLL beside
|
||||
# the executable for the self-contained builds. Test the target rather than the platform, so a
|
||||
# toolkit that gains or loses the static library is handled without another special case.
|
||||
# Both halves matter: the server build keeps the shared library (its .deb/.rpm take CUDA from the
|
||||
# distro, and every executable there -- broker, tests, tools -- would otherwise need the device
|
||||
# link below), and Windows has no static cuFFT to link even for the portable products.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND TARGET CUDA::cufft_static)
|
||||
# The server build links it statically too, so its executables (broker, tools copied to another
|
||||
# machine) need no CUDA toolkit library; the top-level CMakeLists gives every executable the device
|
||||
# link this needs.
|
||||
IF (TARGET CUDA::cufft_static)
|
||||
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft_static)
|
||||
ELSE()
|
||||
TARGET_LINK_LIBRARIES(JFJochIndexing fast_indexer_static CUDA::cufft)
|
||||
|
||||
@@ -147,7 +147,8 @@ void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number,
|
||||
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
|
||||
if ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) >= frame + 2)
|
||||
|| counters.IsAcquisitionFinished() || failed
|
||||
|| (!blocking && (FastestFrame() >= frame + 2 + MissingAfterFrames)))
|
||||
|| (!blocking && (static_cast<int64_t>(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames))
|
||||
|| (!blocking && (FastestFrame() >= frame + 2 + static_cast<int64_t>(max_slots))))
|
||||
return;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
}
|
||||
|
||||
@@ -24,7 +24,9 @@
|
||||
class FPGAImagePuller : public ImagePuller {
|
||||
public:
|
||||
static constexpr size_t DefaultSlots = 128;
|
||||
// A module this far behind the fastest one is taken as missing from that frame, as the device itself does
|
||||
// A module this far behind the fastest one of its card is taken as missing from that frame, as the device itself
|
||||
// does. Behind another card it is waited for as long as the slots can hold the images started meanwhile: the
|
||||
// cards run on their own and one can be ahead, but past max_slots its images would be dropped anyway.
|
||||
static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2;
|
||||
private:
|
||||
struct Slot {
|
||||
|
||||
@@ -66,14 +66,6 @@ IF(APPLE AND JFJOCH_MACOS_SIGNING_IDENTITY)
|
||||
COMPONENT rugnux)
|
||||
ENDIF()
|
||||
|
||||
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
|
||||
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
|
||||
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
|
||||
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
|
||||
# the host link driver alone, so the -march/-flto flags CI passes still apply.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
|
||||
SET_TARGET_PROPERTIES(rugnux PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
ENDIF()
|
||||
|
||||
|
||||
# On Windows this CLI gets getopt/getopt_long from the vendored wingetopt shim (libc has none).
|
||||
|
||||
@@ -47,10 +47,6 @@ ADD_EXECUTABLE(jfjoch_portable_test EXCLUDE_FROM_ALL
|
||||
target_link_libraries(jfjoch_portable_test Catch2WithMain JFJochRugnux JFJochReader JFJochImageAnalysis
|
||||
JFJochWriter JFJochAPIConvert JFJochCommon)
|
||||
target_include_directories(jfjoch_portable_test PRIVATE .)
|
||||
# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
|
||||
SET_TARGET_PROPERTIES(jfjoch_portable_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
ENDIF()
|
||||
|
||||
# The CPU vs GPU test of the fast-feedback indexer reads the library's own test spot lists
|
||||
IF (JFJOCH_CUDA_AVAILABLE)
|
||||
|
||||
@@ -18,10 +18,6 @@ TARGET_LINK_LIBRARIES(jfjoch_extract_hkl JFJochReader)
|
||||
# measuring the refinement on battery outputs without processing anything again.
|
||||
ADD_EXECUTABLE(rugnux_refine_test rugnux_refine_test.cpp)
|
||||
TARGET_LINK_LIBRARIES(rugnux_refine_test JFJochStructureRefinement JFJochCommon gemmi)
|
||||
# Same device-link step as rugnux (see rugnux/CMakeLists.txt): a portable CUDA build links cuFFT statically.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
|
||||
SET_TARGET_PROPERTIES(rugnux_refine_test PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
ENDIF()
|
||||
|
||||
# In-place re-compress /entry/data/data of a _data file from bitshuffle/LZ4 to bitshuffle/zstd.
|
||||
ADD_EXECUTABLE(jfjoch_recompress jfjoch_recompress.cpp)
|
||||
|
||||
@@ -7,9 +7,11 @@
|
||||
// The generator cycles through the images its memory holds, so images repeat: throughput, latency and indexing rate
|
||||
// are meaningful, the merge and its statistics are not.
|
||||
|
||||
#include <atomic>
|
||||
#include <getopt.h>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <sys/resource.h>
|
||||
|
||||
#include "../acquisition_device/PCIExpressDevice.h"
|
||||
@@ -28,7 +30,7 @@ void print_usage(Logger &logger) {
|
||||
logger.Info("loaded into the cards' internal generators, which cycle through them: images repeat, so throughput, latency");
|
||||
logger.Info("and indexing rate are meaningful, the merge and its statistics are not. See docs/TOOLS.md.");
|
||||
logger.Info("Options:");
|
||||
logger.Info(" -v Verbose");
|
||||
logger.Info(" -v Verbose, with each card's progress every 0.5 s");
|
||||
logger.Info(" -i<num> Images in the run (default: as many as the file has)");
|
||||
logger.Info(" -s<num> Number of data streams (cards)");
|
||||
logger.Info(" -N<num> Number of image processing threads (default 64)");
|
||||
@@ -38,6 +40,7 @@ void print_usage(Logger &logger) {
|
||||
logger.Info(" -t<num> Indexing thread pool size (default: 4)");
|
||||
logger.Info(" -r<Hz,...> Rates the source is paced at, one collection each (default: 0 = as fast as it goes)");
|
||||
logger.Info(" -p<num> FPGAImagePuller slots (default 128)");
|
||||
logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped");
|
||||
logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller (not blocking: frames");
|
||||
logger.Info(" past what its analysis holds are lost)");
|
||||
logger.Info(" -K{<MiB>} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB =");
|
||||
@@ -85,7 +88,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp
|
||||
bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file,
|
||||
std::optional<size_t> nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB,
|
||||
const IndexingSettings &indexing, const std::optional<UnitCell> &cell, size_t puller_slots,
|
||||
bool fpga_receiver, const std::vector<double> &rates) {
|
||||
bool fpga_receiver, const std::vector<double> &rates, bool dropping) {
|
||||
RegisterHDF5Filter();
|
||||
JFJochHDF5Reader reader;
|
||||
reader.ReadFile(file);
|
||||
@@ -165,10 +168,11 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
// Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer,
|
||||
// the cards hold their frames and their generators stall. JFJochReceiverFPGA's live stills cannot block.
|
||||
service.UseFPGAImagePuller(fpga_receiver ? FPGAImagePullerUse::Never : FPGAImagePullerUse::RugnuxLive,
|
||||
puller_slots, true, rate);
|
||||
puller_slots, !dropping, rate);
|
||||
logger.Info("");
|
||||
logger.Info("=== {}, {} ===", fpga_receiver ? "JFJochReceiverFPGA (not blocking)"
|
||||
: fmt::format("FPGAImagePuller ({} slots, blocking)", puller_slots),
|
||||
: fmt::format("FPGAImagePuller ({} slots, {})", puller_slots,
|
||||
dropping ? "dropping" : "blocking"),
|
||||
(rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes"));
|
||||
|
||||
rusage usage_start{};
|
||||
@@ -176,7 +180,22 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
const auto wall_start = std::chrono::steady_clock::now();
|
||||
|
||||
service.Start(x, mask, nullptr);
|
||||
// -v: each card's progress, the frames of its fastest and slowest module written to the host
|
||||
std::atomic<bool> running = true;
|
||||
std::thread progress([&] {
|
||||
while (running) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
||||
std::string line;
|
||||
for (int i = 0; i < x.GetDataStreamsNum(); i++)
|
||||
line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
|
||||
aq_devices[i].Counters().GetFastestFrameNumber());
|
||||
logger.Debug("{:.1f} s{}", std::chrono::duration<double>(std::chrono::steady_clock::now()
|
||||
- wall_start).count(), line);
|
||||
}
|
||||
});
|
||||
const auto output = service.Stop();
|
||||
running = false;
|
||||
progress.join();
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
|
||||
const double wall_time = std::chrono::duration<double>(std::chrono::steady_clock::now() - wall_start).count();
|
||||
@@ -274,9 +293,10 @@ int main(int argc, char **argv) {
|
||||
std::optional<uint32_t> mock_card_MiB;
|
||||
bool verbose = false;
|
||||
std::vector<double> rates = {0};
|
||||
bool dropping = false;
|
||||
|
||||
int opt;
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:FK::")) != -1) {
|
||||
while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dFK::")) != -1) {
|
||||
switch (opt) {
|
||||
case 'v':
|
||||
verbose = true;
|
||||
@@ -329,6 +349,9 @@ int main(int argc, char **argv) {
|
||||
case 'F':
|
||||
fpga_receiver = true;
|
||||
break;
|
||||
case 'd':
|
||||
dropping = true;
|
||||
break;
|
||||
case 'K':
|
||||
mock_card_MiB = (optarg != nullptr) ? atol(optarg) : 64;
|
||||
break;
|
||||
@@ -364,7 +387,7 @@ int main(int argc, char **argv) {
|
||||
bool ret = false;
|
||||
try {
|
||||
ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB,
|
||||
i_settings, cell, puller_slots, fpga_receiver, rates);
|
||||
i_settings, cell, puller_slots, fpga_receiver, rates, dropping);
|
||||
} catch (std::exception &e) {
|
||||
logger.Error(e.what());
|
||||
}
|
||||
|
||||
@@ -184,14 +184,6 @@ IF(APPLE)
|
||||
PATTERN "COLLECT.sh" EXCLUDE)
|
||||
ENDIF()
|
||||
|
||||
# libcufft_static.a carries a relocatable-device-code object (separate_callback.o), so an executable
|
||||
# linking it needs a CUDA device-link step -- without it the host link fails on an undefined
|
||||
# __cudaRegisterLinkedBinary_* symbol. CUDA 13 no longer ships libcufft_static_nocallback.a, which
|
||||
# used to be the way around it. CUDA_RESOLVE_DEVICE_SYMBOLS makes CMake emit that step while leaving
|
||||
# the host link driver alone, so the -march/-flto flags CI passes still apply.
|
||||
IF (JFJOCH_PORTABLE_ONLY AND JFJOCH_CUDA_AVAILABLE AND TARGET CUDA::cufft_static)
|
||||
SET_TARGET_PROPERTIES(jfjoch_viewer PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON)
|
||||
ENDIF()
|
||||
|
||||
|
||||
IF(JFJOCH_VIEWER_DBUS)
|
||||
|
||||
Reference in new issue
Block a user