From f2e4dc63b4bc847edf99f323f73018493c321d39 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 12:19:07 +0100 Subject: [PATCH] Receivers: rugnux live only through FPGAImagePuller, pedestal runs on JFJochReceiverPedestal - JFJochReceiverFPGA no longer runs rugnux: its live stills (StillsGPUAnalysisFPGA, GPUSink, the merged data in END) and its pedestal code go; the receiver is the one of rc186 (rc186-old), which differs from what it replaces only in these two parts. - JFJochReceiverPedestal back from rc186 (242587c58, 7080109a4) unchanged: raw frames from the devices into JFPedestalCalc, nothing on any socket. In rc186 it took every PSI pedestal run directly from the devices, not through the puller; so it does here. - Dispatch for a PSI detector: pedestal -> JFJochReceiverPedestal; rugnux live on (Standard mode, no host summation) -> FPGAImagePuller and JFJochReceiverRugnuxStills or ...Rotation; otherwise JFJochReceiverFPGA, unchanged. With host summation rugnux live is not run (a warning in the status). - The FPGAImagePullerUse enum goes: FPGAImagePullerSettings(slots, blocking, rate, always), where always (every collection through the puller, without rugnux to JFJochReceiverLite) is kept for the tests comparing the puller with JFJochReceiverFPGA pixel for pixel. - jfjoch_fpga_stills_test loses -F; StillsGPUAnalysisFPGATest goes, the GPU-stills processing tests run the puller path only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- docs/TOOLS.md | 7 +- receiver/CMakeLists.txt | 2 +- receiver/JFJochReceiverFPGA.cpp | 362 ++------------------- receiver/JFJochReceiverFPGA.h | 31 +- receiver/JFJochReceiverPedestal.cpp | 242 ++++++++++++++ receiver/JFJochReceiverPedestal.h | 63 ++++ receiver/JFJochReceiverService.cpp | 34 +- receiver/JFJochReceiverService.h | 15 +- receiver/StillsGPUAnalysisFPGA.cpp | 419 ------------------------- receiver/StillsGPUAnalysisFPGA.h | 151 --------- tests/CMakeLists.txt | 1 - tests/FPGAImagePullerTest.cpp | 12 +- tests/JFJochReceiverProcessingTest.cpp | 6 - tests/StillsGPUAnalysisFPGATest.cpp | 127 -------- tools/jfjoch_fpga_stills_test.cpp | 21 +- 15 files changed, 381 insertions(+), 1112 deletions(-) create mode 100644 receiver/JFJochReceiverPedestal.cpp create mode 100644 receiver/JFJochReceiverPedestal.h delete mode 100644 receiver/StillsGPUAnalysisFPGA.cpp delete mode 100644 receiver/StillsGPUAnalysisFPGA.h delete mode 100644 tests/StillsGPUAnalysisFPGATest.cpp diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 56cfe734f..30da17140 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -74,7 +74,7 @@ synthesis C model on the CPU, so no FPGA device is required. Measures rugnux live stills on the FPGA card(s), with a stored dataset standing in for the detector: ``` -jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C a,b,c,al,be,ga] [-X ] [-r ,...] [-p ] [-F] [-K[]] +jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C a,b,c,al,be,ga] [-X ] [-r ,...] [-p ] [-d] [-v] [-K[]] ``` - The dataset must be a stored JUNGFRAU dataset with its modules assembled (1030 x 514 each, any placement @@ -87,9 +87,8 @@ jfjoch_fpga_stills_test [-i ] [-s ] [-N ] [-B ] [-C one is saturated or the sum exceeds 16 bits it is saturated. - The frames are loaded into the card's internal frame generator and the run is made with the JUNGFRAU conversion off, so the card returns them verbatim; the host assembles them as signed 16-bit photon - counts and rugnux live stills analyses them through `FPGAImagePuller` (`-F`: through - `JFJochReceiverFPGA`'s own live stills, for comparison; that path does not block: frames past what its - analysis holds are lost and counted). Beam centre, distance, energy and the cell + counts and rugnux live stills analyses them through `FPGAImagePuller`, as the broker does with rugnux live + on (`-d`: not blocking, as the broker runs; `-v`: each card's progress every 0.5 s). Beam centre, distance, energy and the cell come from the file (`-C` overrides the cell; without one the frames are indexed de novo). Nothing is written. - Each card's generator holds module frames of its own modules only: as many images as its memory holds diff --git a/receiver/CMakeLists.txt b/receiver/CMakeLists.txt index c49af596f..5cc13a491 100644 --- a/receiver/CMakeLists.txt +++ b/receiver/CMakeLists.txt @@ -7,6 +7,7 @@ ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverService.cpp JFJochReceiverService.h JFJochReceiverTest.cpp JFJochReceiverTest.h JFJochReceiverFPGA.cpp JFJochReceiverFPGA.h + JFJochReceiverPedestal.cpp JFJochReceiverPedestal.h ImageMetadata.cpp ImageMetadata.h LossyFilter.cpp @@ -21,7 +22,6 @@ ADD_LIBRARY(JFJochReceiver STATIC JFJochReceiverRugnuxStills.h JFJochReceiverRugnuxStills.cpp StillsPrePass.h StillsPrePass.cpp StillsGPUAnalysis.h StillsGPUAnalysis.cpp - StillsGPUAnalysisFPGA.h StillsGPUAnalysisFPGA.cpp RugnuxLiveStatus.h RugnuxLiveStatus.cpp RugnuxStructure.h RugnuxStructure.cpp JFJochReceiver.cpp diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 163eaa553..09bc6d2a1 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -3,14 +3,9 @@ #include "JFJochReceiverFPGA.h" -#include "../image_analysis/WriteReflections.h" - #include #include "ImageMetadata.h" -#include "JFJochReceiverLite.h" -#include "../common/CUDAWrapper.h" -#include "../image_analysis/MXAnalysisWithoutFPGA.h" JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_pixel_mask, @@ -24,9 +19,7 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen ImageBuffer &in_send_buf_ctrl, ZMQPreviewSocket *in_zmq_preview_socket, ZMQMetadataSocket *in_zmq_metadata_socket, - IndexerThreadPool *indexing_thread_pool, - bool in_gpu_analysis, - RugnuxLiveStatus *in_rugnux_status) + IndexerThreadPool *indexing_thread_pool) : JFJochReceiver(in_experiment, in_send_buf_ctrl, in_image_sender, @@ -44,13 +37,9 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen ndatastreams(experiment.GetDataStreamsNum()), frame_transformation_nthreads( in_forward_and_sum_nthreads), - pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4), summation_nthreads(2), - frame_transformation_ready((experiment.GetImageNum() > 0) ? frame_transformation_nthreads : 0), - data_acquisition_ready(ndatastreams), - gpu_analysis(in_gpu_analysis && experiment.GetImageNum() > 0 - && experiment.GetDetectorMode() == DetectorMode::Standard), - rugnux_status(in_rugnux_status) { + frame_transformation_ready(frame_transformation_nthreads), + data_acquisition_ready(ndatastreams) { // The FPGA integration core has a fixed bin-count limit. Reject configurations it cannot // handle before starting acquisition, unless azimuthal integration is forced onto the CPU. const auto &azint_settings = experiment.GetAzimuthalIntegrationSettings(); @@ -92,92 +81,23 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen logger.Debug("Data acquisition devices ready"); - if ((experiment.GetDetectorMode() == DetectorMode::PedestalG0) - || (experiment.GetDetectorMode() == DetectorMode::PedestalG1) - || (experiment.GetDetectorMode() == DetectorMode::PedestalG2)) { - if (experiment.GetImageNum() > 0) { - throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, - "Saving and calculating pedestal is not supported for the time being"); - } + SendStartMessage(); + SendCalibration(); - switch (experiment.GetDetectorMode()) { - case DetectorMode::PedestalG1: - only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2); - break; - case DetectorMode::PedestalG2: - only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2); - break; - default: - only_2nd_sc_pedestal = false; - break; - } + for (int i = 0; i < experiment.GetImageNum(); i++) + images_to_go.Put(i); - int64_t pedestal_count = (only_2nd_sc_pedestal) - ? experiment.GetModulesNum() - : experiment.GetModulesNum() * experiment.GetStorageCellNumber(); - for (int i = 0; i < pedestal_count; i++) - pedestal.emplace_back(std::make_unique(experiment)); - - for (int s = 0; s < experiment.GetStorageCellNumber(); s++) { - bool ignore = only_2nd_sc_pedestal && (s == 0); - for (int d = 0; d < ndatastreams; d++) - for (int m = 0; m < experiment.GetModulesNum(d); m++) - for (int n = 0; n < pedestal_nthreads; n++) - frame_transformation_futures.emplace_back(std::async(std::launch::async, - &JFJochReceiverFPGA::MeasurePedestalThread, - this, d, m, s, n, ignore)); - } - logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads); - } else if (experiment.GetImageNum() > 0) { - if (gpu_analysis) { - // The start message and the calibration go out once the pre-pass is done. - gpu = std::make_unique( - experiment, pixel_mask, MakeStartMessage(experiment, pixel_mask, *az_int_mapping), - indexing_thread_pool, std::min(16, frame_transformation_nthreads), - // Half the image buffer for the frames waiting, the other half for those sent. - image_buffer.GetStatus().total_slots / 2, logger, - [this](const StillsGPUAnalysisFPGA::Setup &s) { - // The profiles are binned on the pre-pass's geometry. - plots.Setup(s.experiment, *s.mapping); - SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping)); - SendCalibration(); - }, - [this](DataMessage &m, const AzimuthalIntegrationProfile &p) { GPUSink(m, p); }, - [this] { - // The broker's settings, with the spot finder rugnux uses on stills (the pre-pass - // decided the centre and mask with it): adaptive threshold, single-pixel spots kept. - // With the broker's default fixed threshold 3 points fewer frames index. - // Every frame is integrated: integration is rugnux's, not the broker's to switch off. - SpotFindingSettings s = GetSpotFindingSettings(); - s.adaptive_threshold = true; - s.min_pix_per_spot = std::nullopt; - s.quick_integration = true; - return s; - }, - // The start message did not go out (or the analysis could not be set up): nothing can - // be written, so the collection stops, as it does on an error in a frame thread. - [this](const JFJochException &e) { Cancel(e); }, - rugnux_status); - } else { - SendStartMessage(); - SendCalibration(); - } - - for (int i = 0; i < experiment.GetImageNum(); i++) - images_to_go.Put(i); - - // Setup frames summation and forwarding - for (uint32_t i = 0; i < frame_transformation_nthreads; i++) { - auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread, - this, i); - frame_transformation_futures.emplace_back(std::move(handle)); - } - - logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads); - frame_transformation_ready.wait(); - logger.Debug("Image compression/forwarding threads ready"); + // Setup frames summation and forwarding + for (uint32_t i = 0; i < frame_transformation_nthreads; i++) { + auto handle = std::async(std::launch::async, &JFJochReceiverFPGA::FrameTransformationThread, + this, i); + frame_transformation_futures.emplace_back(std::move(handle)); } + logger.Debug("Image compression/forwarding threads started ({} threads)", frame_transformation_nthreads); + frame_transformation_ready.wait(); + logger.Debug("Image compression/forwarding threads ready"); + for (int d = 0; d < ndatastreams; d++) data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverFPGA::AcquireThread, this, d)); @@ -251,49 +171,11 @@ void JFJochReceiverFPGA::AcquireThread(uint16_t data_stream) { } -void JFJochReceiverFPGA::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell, - uint32_t threadid, bool ignore) { - JFPedestalCalc pedestal_calc(experiment); - - uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber(); - uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads; - uint32_t storage_cell_header = UINT32_MAX; - - try { - for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) { - // Frame will be processed only if one already collects frame+2 - acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number); - if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) { - auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number); - // Partial packets will bring more problems, than benefit - pedestal_calc.AnalyzeImage((uint16_t *) output->pixels); - storage_cell_header = (output->module_statistics.debug >> 8) & 0xF; - } - acquisition_device[data_stream].FrameBufferRelease(frame, module_number); - - UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number)); - current_status.SetProgress(GetProgress()); - current_status.SetStatus(GetStatus()); - } - - uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number; - if (!only_2nd_sc_pedestal) - offset += experiment.GetModulesNum() * storage_cell; - - if (!ignore) - *pedestal[offset] += pedestal_calc; - } catch (const JFJochException &e) { - Cancel(e); - } - logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done", - data_stream, module_number, storage_cell, storage_cell_header); -} - -int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream, - int64_t image_number, - uint16_t module_number, - uint32_t threadid, - ModuleSummation &summation) { +void JFJochReceiverFPGA::SummationThread(uint16_t data_stream, + int64_t image_number, + uint16_t module_number, + uint32_t threadid, + ModuleSummation &summation) { ModuleSummation local_summation(experiment); int64_t starting_frame = image_number * experiment.GetSummation(); @@ -317,7 +199,6 @@ int64_t JFJochReceiverFPGA::SummationThread(uint16_t data_stream, if (!summation.empty()) summation.AddFPGAOutput(local_summation.GetOutput(), 4); - return 0; } void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { @@ -380,7 +261,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { for (int i = 0; i < experiment.GetModulesNum(); i++) summation.emplace_back(std::make_unique(experiment)); - std::vector > futures; + std::vector> futures; for (int d = 0; d < ndatastreams; d++) { for (int m = 0; m < experiment.GetModulesNum(d); m++) { @@ -455,41 +336,6 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { continue; } - message.image = CompressedImage(transformation.GetImage(), - experiment.GetPixelsNum() * experiment.GetByteDepthImage(), - experiment.GetXPixelsNum(), - experiment.GetYPixelsNum(), - experiment.GetImageMode(), - CompressionAlgorithm::NO_COMPRESSION); - - if (gpu_analysis) { - // Compressed here into a slot of the image buffer, where it waits for the GPU analysis; GPUSink - // writes it, with its analysis, to a second slot, which is sent. A frame that finds no slot - // free, or as many frames waiting as the analysis is to hold, is dropped, as the receiver drops - // any frame that finds the image buffer full. - ++images_collected; - uncompressed_size += experiment.GetModulesNum() * RAW_MODULE_SIZE * experiment.GetByteDepthImage(); - auto held = std::make_shared(); - held->slot = image_buffer.GetImageSlot(); - if (held->slot == nullptr) { - writer_queue_full = true; - continue; - } - const size_t size = transformation.CompressImage(static_cast(held->slot->GetImage()), - experiment.GetImageBufferLocationSize()); - held->message = std::move(message); - held->message.image = CompressedImage(held->slot->GetImage(), size, - experiment.GetXPixelsNum(), experiment.GetYPixelsNum(), - experiment.GetImageMode(), - experiment.GetCompressionAlgorithm()); - if (!gpu->Hold(held)) { - writer_queue_full = true; - continue; - } - compressed_size += size; - continue; - } - // No-op unless the CPU backend is forced; fills az_int_profile_image from the assembled image analyzer->RunAzimuthalIntegration(transformation.GetImage(), az_int_profile_image); @@ -616,12 +462,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { } float JFJochReceiverFPGA::GetEfficiency() const { - - uint64_t expected_packets; - if (experiment.GetImageNum() == 0) - expected_packets = expected_packets_per_image * experiment.GetFrameNum(); - else - expected_packets = expected_packets_per_image * experiment.GetImageNum(); + uint64_t expected_packets = expected_packets_per_image * experiment.GetImageNum(); uint64_t received_packets = 0; for (int d = 0; d < ndatastreams; d++) { @@ -652,43 +493,29 @@ float JFJochReceiverFPGA::GetProgress() const { if (experiment.IsCPUSummation()) frames *= experiment.GetSummation(); - if (frames == 0) - frames = experiment.GetFrameNum(); - if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished())) + if (acquisition_device[0].Counters().IsAcquisitionFinished()) return 1.0; return static_cast(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast(frames); } +void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) { + AddIndexingToEndMessage(message, indexer); +} + void JFJochReceiverFPGA::FinalizeMeasurement() { - if (!frame_transformation_futures.empty()) { - for (auto &future: frame_transformation_futures) - future.get(); + for (auto &future: frame_transformation_futures) + future.get(); - logger.Debug("All processing threads done"); - } - - // Where the start message never went out (or the analysis could not be set up), there is no end - // message either: the devices are stopped all the same, and the error is the collection's result. - std::exception_ptr gpu_error; - if (gpu) { - try { - gpu->Finish(); - } catch (const std::exception &) { - gpu_error = std::current_exception(); - } - } + logger.Debug("All processing threads done"); current_status.SetProgress(1.0); current_status.SetStatus(GetStatus()); - if (!gpu_error) - SendEndMessage(); + SendEndMessage(); - if (experiment.GetImageNum() > 0) { - for (int d = 0; d < ndatastreams; d++) - acquisition_device[d].Cancel(); - } + for (int d = 0; d < ndatastreams; d++) + acquisition_device[d].Cancel(); end_time = std::chrono::system_clock::now(); @@ -702,18 +529,10 @@ void JFJochReceiverFPGA::FinalizeMeasurement() { throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Send commands not finalized in " + std::to_string(SendFinalizeTimeout.count()) + " seconds"); } - if (gpu_error) { - current_status.SetProgress({}); - current_status.SetStatus(GetStatus()); - std::rethrow_exception(gpu_error); - } logger.Info("Receiving data done"); if (push_images_to_writer) writer_error = image_pusher.Finalize(); - // Last image -> all written, as the rugnux receiver reports it - if (gpu) - gpu->Written(); current_status.SetProgress({}); current_status.SetStatus(GetStatus()); @@ -758,29 +577,10 @@ JFJochReceiverOutput JFJochReceiverFPGA::GetFinalStatistics() const { ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m)); } } - RetrievePedestal(ret.pedestal_result); return ret; } -void JFJochReceiverFPGA::RetrievePedestal(std::vector &output) const { - time_t curr_time = std::chrono::system_clock::to_time_t(start_time); - for (const auto &pc: pedestal) { - JFModulePedestal mp; - if (experiment.GetDetectorMode() == DetectorMode::PedestalG0) - pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT); - else - pc->Export(mp); - mp.SetCollectionTime(curr_time); - output.emplace_back(std::move(mp)); - } -} - void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) { - if (experiment.IsPedestalRun()) { - acquisition_device[data_stream].InitializeEmptyPixelMask(experiment); - return; // No calibration loaded for pedestal - } - if (calibration != nullptr) acquisition_device[data_stream].InitializeCalibration(experiment, *calibration); @@ -793,97 +593,3 @@ void JFJochReceiverFPGA::LoadCalibrationToFPGA(uint16_t data_stream) { // Initialize data processing acquisition_device[data_stream].InitializeDataProcessing(experiment, *az_int_mapping); } - -void JFJochReceiverFPGA::GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile) { - try { - const auto status = image_buffer.GetSlotStatus(); - message.receiver_buf_available = status.available_slots; - message.receiver_buf_in_preparation = status.preparation_slots; - message.receiver_buf_in_sending = status.sending_slots; - if (message.image.GetCompressedSize() > 0) - message.compression_ratio = static_cast(experiment.GetPixelsNum() * experiment.GetByteDepthImage()) - / static_cast(message.image.GetCompressedSize()); - - if (message.image_collection_efficiency == 1.0f) { - saturated_pixels.Add(message.saturated_pixel_count); - error_pixels.Add(message.error_pixel_count); - if (message.roi.contains("beam")) { - roi_beam_npixel.Add(message.roi["beam"].pixels); - roi_beam_sum.Add(message.roi["beam"].sum); - } - } - - scan_result.Add(message); - - if (!serialmx_filter.ApplyFilter(message)) - ++images_skipped; - else if (auto loc = image_buffer.GetImageSlot(); loc == nullptr) - writer_queue_full = true; - else { - auto writer_buffer = (uint8_t *) loc->GetImage(); - CBORStream2Serializer serializer(writer_buffer, experiment.GetImageBufferLocationSize()); - if (JFJochReceiverLite::SerializeImageOrDrop(serializer, message, *loc, - experiment.GetImageBufferLocationSize(), logger)) { - loc->SetImageNumber(message.number); - loc->SetImageSize(serializer.GetBufferSize()); - loc->SetIndexed(message.indexing_result.value_or(false)); - loc->ReadyToSend(); - - if (zmq_preview_socket != nullptr) - zmq_preview_socket->SendImage(writer_buffer, serializer.GetBufferSize()); - if (zmq_metadata_socket != nullptr) - zmq_metadata_socket->AddDataMessage(message); - - if (push_images_to_writer) { - if (image_pusher.SendImage(*loc)) - ++images_sent; - } else - loc->release(); - UpdateMaxImageSent(message.number); - } - } - - plots.Add(message, profile); - current_status.SetProgress(GetProgress()); - current_status.SetStatus(GetStatus()); - } catch (const JFJochException &e) { - logger.ErrorException(e); - Cancel(e); - } catch (const std::exception &e) { - const JFJochException error(JFJochExceptionCategory::WrongDAQState, - fmt::format("Image {} not written - {}", message.number, e.what())); - logger.ErrorException(error); - Cancel(error); - } -} - -void JFJochReceiverFPGA::AddToEndMessage(EndMessage &message) { - AddIndexingToEndMessage(message, indexer); - if (!gpu) - return; - message.unit_cell = gpu->GetConsensusUnitCell(); - message.refined_beam_center_x = gpu->GetBeamX_pxl(); - message.refined_beam_center_y = gpu->GetBeamY_pxl(); - message.refined_detector_distance_mm = gpu->GetDetectorDistance_mm(); - // The merged data, for the writer to put beside the master file. - if (const auto &m = gpu->GetMerged(); m && gpu->GetMergeExperiment().GetUnitCell()) - message.merged = MergedMtzTable(m->merge.merged, *gpu->GetMergeExperiment().GetUnitCell(), - gpu->GetMergeExperiment(), OutputProvenance{}); - if (!message.merged) - return; - // Every pusher serialises END into a buffer of MESSAGE_SIZE_FOR_START_END, and an END that does not fit there - // fails the collection. So it is serialised here first, as JFJochReceiverRugnuxRotation does; one that does not fit - // goes without the merged data. - const auto scratch = std::make_unique_for_overwrite(MESSAGE_SIZE_FOR_START_END); - CBORStream2Serializer serializer(scratch.get(), MESSAGE_SIZE_FOR_START_END); - try { - serializer.SerializeSequenceEnd(message); - } catch (const JFJochException &e) { - const std::string text = fmt::format("the merged data do not fit in the end message ({} MiB) and are not " - "written", MESSAGE_SIZE_FOR_START_END / (1024 * 1024)); - logger.Error("GPU analysis: {} - {}", text, e.what()); - if (rugnux_status) - rugnux_status->Warning(text); - message.merged.reset(); - } -} diff --git a/receiver/JFJochReceiverFPGA.h b/receiver/JFJochReceiverFPGA.h index d5afc4c73..5186ad6fa 100644 --- a/receiver/JFJochReceiverFPGA.h +++ b/receiver/JFJochReceiverFPGA.h @@ -3,7 +3,6 @@ #pragma once -#include #include #include #include @@ -21,18 +20,14 @@ #include "../common/ImageBuffer.h" #include "../common/PixelMask.h" -#include "../image_analysis/spot_finding/StrongPixelSet.h" #include "../common/AzimuthalIntegrationMapping.h" #include "../common/AzimuthalIntegrationProfile.h" #include "../jungfrau/JFCalibration.h" #include "../common/ADUHistogram.h" -#include "../common/Plot.h" #include "../common/JFJochReceiverPlots.h" -#include "../preview/PreviewImage.h" #include "LossyFilter.h" #include "JFJochReceiverCurrentStatus.h" -#include "../jungfrau/JFPedestalCalc.h" #include "../preview/ZMQPreviewSocket.h" #include "../preview/ZMQMetadataSocket.h" #include "ImageMetadata.h" @@ -40,15 +35,12 @@ #include "../image_analysis/MXAnalysisAfterFPGA.h" #include "../common/MovingAverage.h" #include "JFJochReceiverOutput.h" -#include "StillsGPUAnalysisFPGA.h" class JFJochReceiverFPGA : public JFJochReceiver { const JFCalibration *calibration; IndexAndRefine indexer; - std::vector> pedestal; - std::vector roi_map; std::vector> frame_transformation_futures; @@ -62,11 +54,9 @@ class JFJochReceiverFPGA : public JFJochReceiver { uint16_t ndatastreams{0}; const int64_t frame_transformation_nthreads; - const int64_t pedestal_nthreads; const int64_t summation_nthreads; int64_t expected_packets_per_image; - bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells std::latch frame_transformation_ready; std::latch data_acquisition_ready; @@ -75,31 +65,18 @@ class JFJochReceiverFPGA : public JFJochReceiver { void LoadCalibrationToFPGA(uint16_t data_stream); void FrameTransformationThread(uint32_t threadid); - void MeasurePedestalThread(uint16_t data_stream, - uint16_t module_number, - uint16_t storage_cell, - uint32_t threadid, - bool ignore); - int64_t SummationThread(uint16_t data_stream, + void SummationThread(uint16_t data_stream, int64_t image_number, uint16_t module_number, uint32_t threadid, ModuleSummation &summation); void Cancel(const JFJochException &e) override; - void FinalizeMeasurement(); - - // Serial data with rugnux on: the converted frames are analysed on the GPU (StillsGPUAnalysisFPGA), and - // the start message waits for its pre-pass. - const bool gpu_analysis; - RugnuxLiveStatus *const rugnux_status; - std::unique_ptr gpu; - void GPUSink(DataMessage &message, const AzimuthalIntegrationProfile &profile); void AddToEndMessage(EndMessage &message) override; + void FinalizeMeasurement(); void SendCalibration(); void SendPedestal(const std::string &prefix, const std::vector &v, int gain, int sc); - void RetrievePedestal(std::vector &output) const; public: JFJochReceiverFPGA(const DiffractionExperiment& experiment, const PixelMask &pixel_mask, @@ -113,9 +90,7 @@ public: ImageBuffer &image_buffer, ZMQPreviewSocket *zmq_preview_socket, ZMQMetadataSocket *zmq_metadata_socket, - IndexerThreadPool *indexing_thread_pool, - bool gpu_analysis = false, - RugnuxLiveStatus *rugnux_status = nullptr); + IndexerThreadPool *indexing_thread_pool); ~JFJochReceiverFPGA(); JFJochReceiverFPGA(const JFJochReceiverFPGA &other) = delete; JFJochReceiverFPGA& operator=(const JFJochReceiverFPGA &other) = delete; diff --git a/receiver/JFJochReceiverPedestal.cpp b/receiver/JFJochReceiverPedestal.cpp new file mode 100644 index 000000000..e44a3b87d --- /dev/null +++ b/receiver/JFJochReceiverPedestal.cpp @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "JFJochReceiverPedestal.h" + +JFJochReceiverPedestal::JFJochReceiverPedestal(const DiffractionExperiment &in_experiment, + const PixelMask &in_pixel_mask, + AcquisitionDeviceGroup &in_aq_device, + ImagePusher &in_image_sender, + Logger &in_logger, + const SpotFindingSettings &in_spot_finding_settings, + JFJochReceiverCurrentStatus &in_current_status, + JFJochReceiverPlots &in_plots, + ImageBuffer &in_send_buf_ctrl) + : JFJochReceiver(in_experiment, + in_send_buf_ctrl, + in_image_sender, + in_current_status, + in_plots, + in_spot_finding_settings, + in_logger, + in_pixel_mask), + acquisition_device(in_aq_device), + ndatastreams(experiment.GetDataStreamsNum()), + pedestal_nthreads((experiment.GetStorageCellNumber() > 2) ? 1 : 4), + data_acquisition_ready(ndatastreams) { + image_buffer.StartMeasurement(experiment); + + if (acquisition_device.size() < ndatastreams) + throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, + "Number of acquisition devices has to match data streams"); + + expected_packets_per_image = 0; + for (int d = 0; d < ndatastreams; d++) { + acquisition_device[d].PrepareAction(experiment); + expected_packets_per_image += acquisition_device[d].Counters().GetExpectedPacketsPerImage(); + + logger.Debug("Acquisition device {} prepared", d); + } + + logger.Debug("Data acquisition devices ready"); + + switch (experiment.GetDetectorMode()) { + case DetectorMode::PedestalG1: + only_2nd_sc_pedestal = !experiment.IsFixedGainG1() && (experiment.GetStorageCellNumber() == 2); + break; + case DetectorMode::PedestalG2: + only_2nd_sc_pedestal = (experiment.GetStorageCellNumber() == 2); + break; + default: + only_2nd_sc_pedestal = false; + break; + } + + int64_t pedestal_count = (only_2nd_sc_pedestal) + ? experiment.GetModulesNum() + : experiment.GetModulesNum() * experiment.GetStorageCellNumber(); + for (int i = 0; i < pedestal_count; i++) + pedestal.emplace_back(std::make_unique(experiment)); + + for (int s = 0; s < experiment.GetStorageCellNumber(); s++) { + bool ignore = only_2nd_sc_pedestal && (s == 0); + for (int d = 0; d < ndatastreams; d++) + for (int m = 0; m < experiment.GetModulesNum(d); m++) + for (int n = 0; n < pedestal_nthreads; n++) + pedestal_futures.emplace_back(std::async(std::launch::async, + &JFJochReceiverPedestal::MeasurePedestalThread, + this, d, m, s, n, ignore)); + } + logger.Debug("Pedestal threads ready ({} threads/module*SC)", pedestal_nthreads); + + for (int d = 0; d < ndatastreams; d++) + data_acquisition_futures.emplace_back(std::async(std::launch::async, &JFJochReceiverPedestal::AcquireThread, + this, d)); + + data_acquisition_ready.wait(); + start_time = std::chrono::system_clock::now(); + measurement = std::async(std::launch::async, &JFJochReceiverPedestal::FinalizeMeasurement, this); + logger.Info("Receiving data started"); +} + +void JFJochReceiverPedestal::AcquireThread(uint16_t data_stream) { + try { + acquisition_device[data_stream].InitializeEmptyPixelMask(experiment); + logger.Debug("Device thread {} start action", data_stream); + acquisition_device[data_stream].StartAction(experiment); + } catch (const JFJochException &e) { + Cancel(e); + data_acquisition_ready.count_down(); + logger.ErrorException(e); + logger.Warning("Device thread {} done due to an error", data_stream); + return; + } + + data_acquisition_ready.count_down(); + + try { + logger.Debug("Device thread {} wait for action complete", data_stream); + acquisition_device[data_stream].WaitForActionComplete(); + } catch (const JFJochException &e) { + logger.ErrorException(e); + Cancel(e); + logger.ErrorException(e); + logger.Warning("Device thread {} done due to an error", data_stream); + return; + } + logger.Debug("Device thread {} done", data_stream); +} + +void JFJochReceiverPedestal::MeasurePedestalThread(uint16_t data_stream, uint16_t module_number, uint16_t storage_cell, + uint32_t threadid, bool ignore) { + JFPedestalCalc pedestal_calc(experiment); + + uint64_t starting_frame = storage_cell + threadid * experiment.GetStorageCellNumber(); + uint64_t frame_stride = experiment.GetStorageCellNumber() * pedestal_nthreads; + uint32_t storage_cell_header = UINT32_MAX; + + try { + for (size_t frame = starting_frame; frame < experiment.GetFrameNum(); frame += frame_stride) { + // Frame will be processed only if one already collects frame+2 + acquisition_device[data_stream].Counters().WaitForFrame(frame + 2, module_number); + if (acquisition_device[data_stream].Counters().IsFullModuleCollected(frame, module_number) && !ignore) { + auto output = acquisition_device[data_stream].GetDeviceOutput(frame, module_number); + // Partial packets will bring more problems, than benefit + pedestal_calc.AnalyzeImage((uint16_t *) output->pixels); + storage_cell_header = (output->module_statistics.debug >> 8) & 0xF; + } + acquisition_device[data_stream].FrameBufferRelease(frame, module_number); + + UpdateMaxDelay(acquisition_device[data_stream].Counters().CalculateDelay(frame, module_number)); + current_status.SetProgress(GetProgress()); + current_status.SetStatus(GetStatus()); + } + + uint64_t offset = experiment.GetFirstModuleOfDataStream(data_stream) + module_number; + if (!only_2nd_sc_pedestal) + offset += experiment.GetModulesNum() * storage_cell; + + if (!ignore) + *pedestal[offset] += pedestal_calc; + } catch (const JFJochException &e) { + Cancel(e); + } + logger.Debug("Pedestal calculation thread for data stream {} module {} storage cell {} -> header {} done", + data_stream, module_number, storage_cell, storage_cell_header); +} + +float JFJochReceiverPedestal::GetEfficiency() const { + uint64_t expected_packets = expected_packets_per_image * experiment.GetFrameNum(); + uint64_t received_packets = 0; + + for (int d = 0; d < ndatastreams; d++) { + received_packets += acquisition_device[d].Counters().GetTotalPackets(); + } + + if ((expected_packets == received_packets) || (expected_packets == 0)) + return 1.0; + return received_packets / static_cast(expected_packets); +} + +void JFJochReceiverPedestal::Cancel(bool silent) { + JFJochReceiver::Cancel(silent); + + for (int d = 0; d < ndatastreams; d++) + acquisition_device[d].Cancel(); +} + +void JFJochReceiverPedestal::Cancel(const JFJochException &e) { + JFJochReceiver::Cancel(e); + + for (int d = 0; d < ndatastreams; d++) + acquisition_device[d].Cancel(); +} + +float JFJochReceiverPedestal::GetProgress() const { + int64_t frames = experiment.GetFrameNum(); + if ((frames == 0) || (acquisition_device[0].Counters().IsAcquisitionFinished())) + return 1.0; + + return static_cast(acquisition_device[0].Counters().GetSlowestFrameNumber()) / static_cast(frames); +} + +void JFJochReceiverPedestal::FinalizeMeasurement() { + for (auto &future: pedestal_futures) + future.get(); + + logger.Debug("All processing threads done"); + + current_status.SetProgress(1.0); + current_status.SetStatus(GetStatus()); + + end_time = std::chrono::system_clock::now(); + + for (auto &future: data_acquisition_futures) + future.get(); + + logger.Debug("Devices stopped"); + logger.Info("Receiving data done"); + + current_status.SetProgress({}); + current_status.SetStatus(GetStatus()); +} + +void JFJochReceiverPedestal::StopReceiver() { + if (measurement.valid()) { + measurement.get(); + logger.Debug("Receiver stopped"); + } +} + +JFJochReceiverPedestal::~JFJochReceiverPedestal() { + try { + if (measurement.valid()) + measurement.get(); + } catch (const std::exception &e) { + logger.Error("ReceiverPedestal teardown failed: {}", e.what()); + } catch (...) { + logger.Error("ReceiverPedestal teardown failed with unknown exception"); + } +} + +JFJochReceiverOutput JFJochReceiverPedestal::GetFinalStatistics() const { + JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics(); + for (int d = 0; d < ndatastreams; d++) { + for (int m = 0; m < acquisition_device[d].Counters().GetModuleNumber(); m++) { + ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule()); + ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m)); + } + } + + time_t curr_time = std::chrono::system_clock::to_time_t(start_time); + for (const auto &pc: pedestal) { + JFModulePedestal mp; + if (experiment.GetDetectorMode() == DetectorMode::PedestalG0) + pc->Export(mp, PEDESTAL_G0_WRONG_GAIN_ALLOWED_COUNT); + else + pc->Export(mp); + mp.SetCollectionTime(curr_time); + ret.pedestal_result.emplace_back(std::move(mp)); + } + return ret; +} diff --git a/receiver/JFJochReceiverPedestal.h b/receiver/JFJochReceiverPedestal.h new file mode 100644 index 000000000..af423ec70 --- /dev/null +++ b/receiver/JFJochReceiverPedestal.h @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include +#include +#include + +#include "JFJochReceiver.h" +#include "../acquisition_device/AcquisitionDeviceGroup.h" +#include "../jungfrau/JFPedestalCalc.h" + +// A pedestal run: raw frames from the devices go into JFPedestalCalc; no image buffer slots, nothing goes out on any socket +class JFJochReceiverPedestal : public JFJochReceiver { + std::vector> pedestal; + + std::vector> pedestal_futures; + std::vector> data_acquisition_futures; + + std::future measurement; + + AcquisitionDeviceGroup &acquisition_device; + uint16_t ndatastreams{0}; + + const int64_t pedestal_nthreads; + + int64_t expected_packets_per_image; + bool only_2nd_sc_pedestal = false; // special mode used for pedestal G1/G2 with storage cells + + std::latch data_acquisition_ready; + + void AcquireThread(uint16_t data_stream); + void MeasurePedestalThread(uint16_t data_stream, + uint16_t module_number, + uint16_t storage_cell, + uint32_t threadid, + bool ignore); + void Cancel(const JFJochException &e) override; + void FinalizeMeasurement(); +public: + JFJochReceiverPedestal(const DiffractionExperiment &experiment, + const PixelMask &pixel_mask, + AcquisitionDeviceGroup &acquisition_devices, + ImagePusher &image_pusher, + Logger &logger, + const SpotFindingSettings &spot_finding_settings, + JFJochReceiverCurrentStatus ¤t_status, + JFJochReceiverPlots &plots, + ImageBuffer &image_buffer); + ~JFJochReceiverPedestal() override; + JFJochReceiverPedestal(const JFJochReceiverPedestal &other) = delete; + JFJochReceiverPedestal &operator=(const JFJochReceiverPedestal &other) = delete; + + void StopReceiver() override; + + float GetEfficiency() const override; + void Cancel(bool silent) override; + float GetProgress() const override; + + JFJochReceiverOutput GetFinalStatistics() const override; +}; diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index 54a6eb618..378b8a722 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -9,6 +9,7 @@ #include "JFJochReceiverFPGA.h" #include "JFJochReceiverLite.h" #include "JFJochReceiverDarkMask.h" +#include "JFJochReceiverPedestal.h" #include "../preview/JFJochJPEG.h" #include "../preview/JFJochTIFF.h" @@ -38,16 +39,16 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share return *this; } -JFJochReceiverService &JFJochReceiverService::UseFPGAImagePuller(FPGAImagePullerUse use, size_t slots, bool blocking, - double frame_rate_hz) { +JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t slots, bool blocking, double frame_rate_hz, + bool always) { std::unique_lock ul_state(state_mutex); if (state != ReceiverState::Idle) throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Cannot change the receiver path during data collection"); - fpga_image_puller = use; fpga_image_puller_slots = slots; fpga_image_puller_blocking = blocking; fpga_image_puller_rate_hz = frame_rate_hz; + fpga_image_puller_always = always; return *this; } @@ -56,19 +57,12 @@ double JFJochReceiverService::GetFPGAImagePullerSlotWaitTime_s() const { return fpga_puller ? fpga_puller->GetSlotWaitTime_s() : 0.0; } -// Pedestal runs and summation on the host need JFJochReceiverFPGA +// A PSI collection analysed by rugnux live; summation on the host needs JFJochReceiverFPGA bool JFJochReceiverService::FPGAImagePullerFor(const DiffractionExperiment &experiment) const { if ((experiment.GetDetectorType() == DetectorType::DECTRIS) || experiment.IsPedestalRun() || experiment.IsCPUSummation() || (experiment.GetImageNum() <= 0)) return false; - switch (fpga_image_puller) { - case FPGAImagePullerUse::Always: - return true; - case FPGAImagePullerUse::RugnuxLive: - return rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard); - default: - return false; - } + return fpga_image_puller_always || (rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard)); } RugnuxLiveSnapshot JFJochReceiverService::GetRugnuxStatus() const { @@ -155,7 +149,15 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, puller = fpga_puller; } - if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) { + if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && experiment.IsPedestalRun()) { + receiver = std::make_unique(experiment, pixel_mask, aq_devices, image_pusher, + logger, spot_finding_settings, receiver_status, + plots, image_buffer); + } else if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) { + if (rugnux_enable && experiment.IsCPUSummation()) { + analysis_warning = "no rugnux analysis: summation on the host"; + logger.Warning("{}", analysis_warning); + } // Summation on the host runs threads of its own for each image auto fpga_nthreads = nthreads; if (experiment.IsCPUSummation()) @@ -171,11 +173,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, image_buffer, zmq_preview_socket.get(), zmq_metadata_socket.get(), - indexer_thread_pool.get(), - // Serial data with rugnux on: the frames are - // analysed on the GPU, after a pre-pass. - rugnux_enable && !RugnuxRotationData(experiment), - &rugnux_status); + indexer_thread_pool.get()); } else { if (puller) image_puller = puller; diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index c8a6a7f65..2590a30fc 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -20,9 +20,6 @@ struct RugnuxModel; -// RugnuxLive: collections analysed by rugnux live; Always: also the others, through JFJochReceiverLite; -// Never: all on JFJochReceiverFPGA, its own live stills included -enum class FPGAImagePullerUse { RugnuxLive, Always, Never }; class JFJochReceiverService { std::unique_ptr receiver; @@ -43,8 +40,8 @@ class JFJochReceiverService { RugnuxLiveStatus rugnux_status; // of the last collection, kept after it ends std::string analysis_warning; // why the collection was not analysed as asked, for the broker's status - // Which PSI collections go through FPGAImagePuller and the stream receivers - FPGAImagePullerUse fpga_image_puller = FPGAImagePullerUse::RugnuxLive; + // A PSI collection analysed by rugnux live goes through FPGAImagePuller to the stream receivers + bool fpga_image_puller_always = false; size_t fpga_image_puller_slots = FPGAImagePuller::DefaultSlots; bool fpga_image_puller_blocking = false; double fpga_image_puller_rate_hz = 0; @@ -77,9 +74,11 @@ public: double deadline_s, bool refine_solvent = false, bool force_still = false); RugnuxLiveSnapshot GetRugnuxStatus() const; void ClearRugnuxStatus(); // back to Idle, as before any collection - // blocking, frame_rate_hz: see FPGAImagePuller - JFJochReceiverService& UseFPGAImagePuller(FPGAImagePullerUse use, size_t slots = FPGAImagePuller::DefaultSlots, - bool blocking = false, double frame_rate_hz = 0); + // slots, blocking, frame_rate_hz: see FPGAImagePuller. always: every PSI collection but pedestal and host + // summation through the puller, without rugnux to JFJochReceiverLite (for tests comparing it with + // JFJochReceiverFPGA) + JFJochReceiverService& FPGAImagePullerSettings(size_t slots = FPGAImagePuller::DefaultSlots, bool blocking = false, + double frame_rate_hz = 0, bool always = false); // Of the last collection through FPGAImagePuller: FPGAImagePuller::GetSlotWaitTime_s, 0 without one double GetFPGAImagePullerSlotWaitTime_s() const; JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); diff --git a/receiver/StillsGPUAnalysisFPGA.cpp b/receiver/StillsGPUAnalysisFPGA.cpp deleted file mode 100644 index 41b4dc7c0..000000000 --- a/receiver/StillsGPUAnalysisFPGA.cpp +++ /dev/null @@ -1,419 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include "StillsGPUAnalysisFPGA.h" - -#include - -#include "../common/CUDAWrapper.h" -#include "../common/ThreadAffinity.h" -#include "../image_analysis/MXAnalysisWithoutFPGA.h" -#include "../rugnux/RugnuxDefaults.h" - -StillsGPUAnalysisFPGA::StillsGPUAnalysisFPGA(const DiffractionExperiment &in_experiment, const PixelMask &in_mask, - const StartMessage &in_start, IndexerThreadPool *in_indexing_thread_pool, - int64_t nthreads, int64_t in_max_held, Logger &in_logger, Ready in_ready, - Sink in_sink, Settings in_settings, Failed in_failed, - RugnuxLiveStatus *in_status, bool in_merge) - : experiment(in_experiment), mask(in_mask), start(in_start), indexing_thread_pool(in_indexing_thread_pool), - logger(in_logger), ready(std::move(in_ready)), sink(std::move(in_sink)), settings(std::move(in_settings)), - setup_failed(std::move(in_failed)), status(in_status), merge_frames(in_merge), - max_held(std::max(1, in_max_held)), queue(max_held) { - live.images_needed = std::min(PREPASS_IMAGES, experiment.GetImageNum()); - live.file_beam_x_pxl = experiment.GetBeamX_pxl(); - live.file_beam_y_pxl = experiment.GetBeamY_pxl(); - live.file_distance_mm = experiment.GetDetectorDistance_mm(); - if (status) { - status->Start(experiment.GetImageNum(), - static_cast(experiment.GetImageNum()) - * std::chrono::duration(experiment.GetImageTime()).count(), - std::chrono::steady_clock::now()); - status->Step("Pre-pass"); - Publish(); - } - prepass_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::PrePassThread, this); - for (int64_t i = 0; i < nthreads; i++) - analysis_futures.emplace_back(std::async(std::launch::async, &StillsGPUAnalysisFPGA::AnalysisThread, this)); - logger.Info("GPU analysis: {} threads, at least the first {} images held for the pre-pass", nthreads, PREPASS_IMAGES); -} - -StillsGPUAnalysisFPGA::~StillsGPUAnalysisFPGA() { - try { - if (!finished) - Finish(); - } catch (const std::exception &e) { - logger.Error("GPU analysis: {}", e.what()); - } - if (status && outcome) - status->Finish(outcome->first, outcome->second); -} - -void StillsGPUAnalysisFPGA::AnalysisError(const std::string &text) { - logger.Error("GPU analysis: {}", text); - std::unique_lock ul(error_mutex); - if (analysis_error.empty()) - analysis_error = text; -} - -void StillsGPUAnalysisFPGA::Publish() { - std::unique_lock ul(live_mutex); - if (live.prepass == RugnuxLivePrePass::Done || live.prepass == RugnuxLivePrePass::Failed) { - const double t = std::chrono::duration(std::chrono::steady_clock::now() - ready_time).count(); - if (t > 0 && live.images_analysed > 0) - live.analysis_rate_hz = static_cast(live.images_analysed) / t; - } - status->Stills(live); -} - -bool StillsGPUAnalysisFPGA::Hold(const std::shared_ptr &image) { - if (status) - status->ImageReceived(std::chrono::steady_clock::now()); - if (!queue.Put(image)) { - // As many frames wait as the buffer is to hold: the pre-pass runs on what it has, for the last time. - StartPrePass(); - return false; - } - { - std::unique_lock ul(prepass_mutex); - if (!prepass_over) { - prepass_images[image->message.number] = image; - std::unique_lock ul_live(live_mutex); - live.images_held = static_cast(prepass_images.size()); - } - } - prepass_cv.notify_all(); - if (status) - Publish(); - return true; -} - -void StillsGPUAnalysisFPGA::StartPrePass() { - { - std::unique_lock ul(prepass_mutex); - prepass_last = true; - } - prepass_cv.notify_all(); -} - -void StillsGPUAnalysisFPGA::Finish() { - finished = true; - StartPrePass(); - prepass_future.get(); - for (size_t i = 0; i < analysis_futures.size(); i++) - queue.PutBlocking(nullptr); - for (auto &f: analysis_futures) - f.get(); - if (status) { - last_image = status->LastImage(); - Publish(); - } - if (merge_future.valid()) { - if (status) - status->Step("Merge"); - merge_queue.Put(-1); - merge_future.get(); - if (merged && status) { - PublishMerge(*merged, "the collection"); - status->Headline(merge_experiment.GetSpaceGroupOrP1().xhm(), merged->merge.statistics.overall.d_min, - std::nullopt); - } - } - if (status) { - if (setup_error) - status->Finish(RugnuxLiveState::Failed, "GPU analysis setup failed"); - else { - // Reported by Written, once everything is written - the end message and the writer go on meanwhile. - status->Step("Writing"); - std::unique_lock ul(error_mutex); - if (analysis_error.empty()) - outcome.emplace(RugnuxLiveState::Done, ""); - else - outcome.emplace(RugnuxLiveState::Failed, analysis_error); - } - } - if (setup_error) - std::rethrow_exception(setup_error); -} - -void StillsGPUAnalysisFPGA::Written() { - if (!status || !outcome) - return; - if (last_image) - status->Latency(std::chrono::duration(std::chrono::steady_clock::now() - last_image.value()).count()); - status->Finish(outcome->first, outcome->second); - outcome.reset(); -} - -void StillsGPUAnalysisFPGA::PrePassThread() { - const auto masked = [](const PixelMask &m) { - return std::count_if(m.GetMask().begin(), m.GetMask().end(), [](uint32_t v) { return v != 0; }); - }; - setup.experiment = experiment; - setup.mask = mask; - const int64_t all_images = experiment.GetImageNum(); - std::optional result; - std::chrono::steady_clock::time_point prepass_start = std::chrono::steady_clock::now(); - // At a low hit rate the first frames may not be enough to fit the geometry on, or may not hold a crystal: with - // a cell to fit, the pre-pass runs again on all that is held once that has doubled, until the fit has its - // frames, or the collection ended, or the sample is as large as it may grow (the last try): bounded so that - // the last run is short and the other half of the queue takes the frames arriving meanwhile. - const int64_t max_sample = std::min(PREPASS_MAX_IMAGES, max_held / 2); - for (int64_t needed = std::min(PREPASS_IMAGES, all_images); ;) { - std::map> sample; - bool last; - { - std::unique_lock ul(prepass_mutex); - prepass_cv.wait(ul, [&] { - return prepass_last || static_cast(prepass_images.size()) >= needed; - }); - // Frames that arrived during the previous run do not make this one larger than the bound. - for (const auto &[n, image]: prepass_images) { - if (static_cast(sample.size()) >= std::max(needed, max_sample)) - break; - sample.emplace(n, image); - } - last = prepass_last || static_cast(sample.size()) >= std::min(all_images, max_sample); - } - if (sample.empty()) - break; - { - std::unique_lock ul(live_mutex); - live.prepass = RugnuxLivePrePass::Running; - } - if (status) - Publish(); - try { - const auto t0 = std::chrono::steady_clock::now(); - auto r = RunStillsPrePass(start, sample, static_cast(std::thread::hardware_concurrency())); - logger.Info("Pre-pass on {} images ({:.1f} s): beam centre ({:.2f}, {:.2f}) -> ({:.2f}, {:.2f}) px, " - "distance {:.2f} -> {:.2f} mm on {} frames, masked pixels {} -> {}", r.images, - std::chrono::duration(std::chrono::steady_clock::now() - t0).count(), - experiment.GetBeamX_pxl(), experiment.GetBeamY_pxl(), r.beam_x_pxl, r.beam_y_pxl, - experiment.GetDetectorDistance_mm(), r.distance_mm, r.geometry_frames, - masked(mask), masked(r.mask)); - result = std::move(r); - } catch (const std::exception &e) { - logger.Warning("Pre-pass on {} images failed - {}", sample.size(), e.what()); - } - if (last || !experiment.GetUnitCell() - || (result && result->geometry_frames >= RUGNUX_REFINE_GEOMETRY_FRAMES)) - break; - needed = std::min({2 * static_cast(sample.size()), all_images, max_sample}); - std::unique_lock ul(live_mutex); - live.prepass = RugnuxLivePrePass::Holding; - live.images_needed = needed; - } - { - // The frames are analysed from here on: their slots go back to the buffer once they are sent. - std::unique_lock ul(prepass_mutex); - prepass_over = true; - prepass_images.clear(); - } - const bool failed = !result; - int geometry_frames = 0; - if (result) { - const auto &r = *result; - setup.experiment.BeamX_pxl(r.beam_x_pxl).BeamY_pxl(r.beam_y_pxl).DetectorDistance_mm(r.distance_mm); - setup.mask = r.mask; - geometry_frames = r.geometry_frames; - const auto &m = r.mask.GetMask(); - std::unique_lock ul(live_mutex); - live.beam_x_pxl = r.beam_x_pxl; - live.beam_y_pxl = r.beam_y_pxl; - live.distance_mm = r.distance_mm; - live.shadow_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) { - return (v & (1u << PixelMask::BeamStopPixelBit)) != 0; - }); - live.bad_pixels = std::count_if(m.begin(), m.end(), [](uint32_t v) { - return (v & (1u << PixelMask::HotPixelBit)) != 0; - }); - } else - logger.Warning("Pre-pass failed, the images are analysed with the stated beam centre and mask"); - try { - setup.mapping = std::make_unique(setup.experiment, setup.mask); - // Retained for the merge, which takes each image's reflections as soon as it is analysed. - indexer = std::make_unique(setup.experiment, indexing_thread_pool, merge_frames, true); - if (geometry_frames == 0) { - const std::string text = fmt::format( - "The pre-pass could not refine the geometry (too few frames indexed in what the buffer holds, or no unit cell given): the " - "whole run is analysed at the distance in the header, {:.2f} mm, beam centre ({:.2f}, {:.2f}) px", - setup.experiment.GetDetectorDistance_mm(), setup.experiment.GetBeamX_pxl(), - setup.experiment.GetBeamY_pxl()); - logger.Warning("{}", text); - if (status) - status->Note(text); - } - if (merge_frames) { - merge_experiment = setup.experiment; - merge_experiment.ImportScalingSettings(RugnuxDefaultScalingSettings(false)); - merge = std::make_unique(merge_experiment, experiment.GetImageNum(), - setup.experiment.GetUnitCell(), std::nullopt, - setup.experiment.IsDetectIceRings(), - SpotFindingSettings().ice_ring_width_Q_recipA, logger); - merge_future = std::async(std::launch::async, &StillsGPUAnalysisFPGA::MergeThread, this); - } - ready(setup); - } catch (const std::exception &e) { - // Nothing can be written without the start message: the frames are dropped (AnalysisThread), and - // the collection is stopped. - const auto *j = dynamic_cast(&e); - const JFJochException error = j ? *j : JFJochException(JFJochExceptionCategory::WrongDAQState, - std::string("GPU analysis setup failed: ") + e.what()); - logger.ErrorException(error); - setup_error = j ? std::current_exception() : std::make_exception_ptr(error); - if (setup_failed) - setup_failed(error); - prepass_done.count_down(); - return; - } - { - std::unique_lock ul(live_mutex); - live.prepass = failed ? RugnuxLivePrePass::Failed : RugnuxLivePrePass::Done; - live.prepass_s = std::chrono::duration(std::chrono::steady_clock::now() - prepass_start).count(); - ready_time = std::chrono::steady_clock::now(); - } - if (status) { - status->Step("GPU analysis"); - Publish(); - } - prepass_done.count_down(); -} - -void StillsGPUAnalysisFPGA::AnalysisThread() { - try { - pin_gpu(); - } catch (const std::exception &e) { - logger.Warning("Error pinning GPU {}", e.what()); - } - - prepass_done.wait(); - - if (setup_error) { - while (queue.GetBlocking()) {} - return; - } - - // A thread whose analysis cannot be built still passes its frames on, unanalysed. - std::unique_ptr analysis; - try { - analysis = std::make_unique(setup.experiment, *setup.mapping, setup.mask, - *indexer, /*enable_fused_adaptive_gpu=*/true); - } catch (const std::exception &e) { - AnalysisError(std::string("analysis thread could not start, its frames are written unanalysed - ") - + e.what()); - } - - while (auto held = queue.GetBlocking()) { - DataMessage &message = held->message; - std::optional profile; - const auto start_time = std::chrono::steady_clock::now(); - profile.emplace(*setup.mapping); - try { - if (analysis) - analysis->Analyze(message, *profile, settings()); - } catch (const JFJochException &e) { - // The frame is still written, without its analysis. - logger.ErrorException(e); - } catch (const std::exception &e) { - AnalysisError(fmt::format("image {} written unanalysed - {}", message.number, e.what())); - } - message.processing_time_s = std::chrono::duration(std::chrono::steady_clock::now() - - start_time).count(); - try { - sink(message, *profile); - if (merge) - merge_queue.Put(message.number); - if (status) { - int64_t analysed; - { - std::unique_lock ul(live_mutex); - analysed = ++live.images_analysed; - if (message.indexing_result.value_or(false)) - ++live.images_indexed; - } - status->Progress(analysed, experiment.GetImageNum()); - Publish(); - } - } catch (const std::exception &e) { - // The thread goes on with the next frame, so that the queue is drained. - AnalysisError(fmt::format("image {} lost - {}", message.number, e.what())); - } - } -} - -std::optional StillsGPUAnalysisFPGA::GetConsensusUnitCell() const { - return indexer->GetConsensusUnitCell(); -} - -float StillsGPUAnalysisFPGA::GetBeamX_pxl() const { - return setup.experiment.GetBeamX_pxl(); -} - -float StillsGPUAnalysisFPGA::GetBeamY_pxl() const { - return setup.experiment.GetBeamY_pxl(); -} - -float StillsGPUAnalysisFPGA::GetDetectorDistance_mm() const { - return setup.experiment.GetDetectorDistance_mm(); -} - -const std::optional &StillsGPUAnalysisFPGA::GetMerged() const { - return merged; -} - -const DiffractionExperiment &StillsGPUAnalysisFPGA::GetMergeExperiment() const { - return merge_experiment; -} - -void StillsGPUAnalysisFPGA::MergeThread() { - constexpr int64_t PREPARE_EVERY = 200; // frames between two Prepare calls after a checkpoint - const int64_t images = experiment.GetImageNum(); - // The cell merged in, as rugnux merges it: the consensus of the frames indexed by the first merge (the - // collection's cell only picks the frames that are merged). - bool cell_set = false; - const auto have_cell = [&] { - if (!cell_set) - if (const auto c = indexer->GetConsensusUnitCell()) { - merge_experiment.SetUnitCell(*c); - merge->Cell(*c); - cell_set = true; - } - return cell_set; - }; - size_t next = 0; - try { - const auto add = [&](int64_t n, IntegrationOutcome &&o) { - merge->Add(n, std::move(o)); - if (next < CHECKPOINTS.size() - && merge->Added() >= std::llround(CHECKPOINTS[next] * static_cast(images))) { - if (have_cell()) - PublishMerge(merge->Checkpoint(MERGE_THREADS), - fmt::format("{:.0f}% of the images (provisional)", 100.0 * CHECKPOINTS[next])); - ++next; - } else if (next > 0 && merge->Added() % PREPARE_EVERY == 0) - merge->Prepare(MERGE_THREADS); - }; - for (int64_t n = merge_queue.GetBlocking(); n >= 0; n = merge_queue.GetBlocking()) - add(n, indexer->TakeIntegrationOutcome(n)); - if (have_cell()) { - merged = merge->Final(static_cast(AvailableCpus())); - logger.Info("Stills merge after the last image: {:.2f} s", merged->seconds); - } - } catch (const std::exception &e) { - logger.Warning("Stills merge failed: {}", e.what()); - merged.reset(); - } -} - -void StillsGPUAnalysisFPGA::PublishMerge(const StillsMergeOutput &out, const std::string &title) { - const auto &o = out.merge.statistics.overall; - const std::string text = fmt::format("Merge of {}: {} crystals, {:.2f} A, CC1/2 {:.3f}, R_split {:.3f}, " - "completeness {:.1f}%, {:.1f} ({:.1f} s)", title, out.images, - o.d_min, o.cc_half, o.r_split, - o.possible_unique_reflections > 0 - ? 100.0 * o.unique_reflections / o.possible_unique_reflections : 0.0, - o.mean_i_over_sigma, out.seconds); - logger.Info("{}", text); - if (status) - status->Note(text); -} diff --git a/receiver/StillsGPUAnalysisFPGA.h b/receiver/StillsGPUAnalysisFPGA.h deleted file mode 100644 index 0773af0af..000000000 --- a/receiver/StillsGPUAnalysisFPGA.h +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "StillsPrePass.h" -#include "RugnuxLiveStatus.h" -#include "../common/DiffractionExperiment.h" -#include "../common/JFJochException.h" -#include "../common/AzimuthalIntegrationMapping.h" -#include "../common/AzimuthalIntegrationProfile.h" -#include "../common/Logger.h" -#include "../common/ThreadSafeFIFO.h" -#include "../image_analysis/IndexAndRefine.h" -#include "../rugnux/StillsLiveMerge.h" - -// JFJochReceiverFPGA's live stills, kept as they are at rc.185 while the stream receivers take StillsGPUAnalysis. -// Serial frames analysed on the GPU as the DECTRIS ones are (MXAnalysisWithoutFPGA), after a pre-pass on -// the first PREPASS_IMAGES (RunStillsPrePass) has decided the beam centre and the mask they are analysed -// with. Every frame given before the pre-pass is done is held; `ready` is called once it is, with the -// geometry and mask the frames are then analysed at, before any of them is analysed; `sink` gets each -// frame once analysed, from one of the analysis threads. Where `status` is given, the run is reported there -// (pre-pass, its decisions, the analysis). -// At most `max_held` frames wait for analysis: Hold refuses one more, so the slots held stay bounded. -// Where the setup after the pre-pass fails (`ready` included), `failed` is called at once, the frames are -// dropped unanalysed and Finish throws the error. An analysis thread that fails passes its frames on -// unanalysed, and the run is reported as failed. -// The indexed frames are merged as `rugnux --mode scale` merges them (StillsLiveMerge), beside the analysis: -// a provisional merge at each of CHECKPOINTS of the images, on MERGE_THREADS threads, reported as a step -// of the status, and the merge of the whole collection once the last frame is analysed, on every thread, -// before Finish returns (GetMerged). The cell is the run's consensus at the first merge, as rugnux merges in -// it (the collection's cell picks the frames merged); the space group the collection's, else P1. -// merge = false leaves it out. -// With a cell given, the pre-pass runs again on all the frames held each time they have doubled, until the geometry -// fit has its frames, the collection ends or the sample reaches PREPASS_MAX_IMAGES or half of `max_held`; the -// analysis starts after that and catches up from the frames held. The bound keeps the last run short and leaves -// room in the queue for the frames arriving meanwhile, which are otherwise not written. The geometry is fixed when the pre-pass ends: every frame is analysed and merged at it. Where the pre-pass could -// not refine it (too few of the frames it had indexed: a small image buffer, a serial hit rate, a chip whose first -// frames are empty; or no cell given), the header's distance stays for the whole run, with a warning in the log -// and the status. -class StillsGPUAnalysisFPGA { -public: - // Enough frames for the distance to be refined at a serial hit rate: of 100 frames of a collection indexed - // at 18%, 7 indexed at a distance 2.6% off and the geometry refinement did not run; of 500, 34 did and - // refined it to 0.4%. They wait in the image buffer, which has to hold them in half its slots. - constexpr static int64_t PREPASS_IMAGES = 500; - constexpr static int64_t PREPASS_MAX_IMAGES = 4 * PREPASS_IMAGES; - constexpr static std::array CHECKPOINTS = {0.5, 0.75}; - constexpr static int MERGE_THREADS = 8; - - struct Setup { - DiffractionExperiment experiment; - PixelMask mask; - std::unique_ptr mapping; - }; - using Ready = std::function; - using Sink = std::function; - using Settings = std::function; - using Failed = std::function; - - // `start`: the collection's start message, as it would be sent without the pre-pass. - StillsGPUAnalysisFPGA(const DiffractionExperiment &experiment, const PixelMask &mask, const StartMessage &start, - IndexerThreadPool *indexing_thread_pool, int64_t nthreads, int64_t max_held, Logger &logger, - Ready ready, Sink sink, Settings settings, Failed failed = {}, - RugnuxLiveStatus *status = nullptr, bool merge = true); - ~StillsGPUAnalysisFPGA(); - - // False where `max_held` frames already wait: the frame is not taken. - bool Hold(const std::shared_ptr &image); - // No frame comes after this: the pre-pass runs on what came, if it has not, and every frame is - // analysed before it returns. Throws where the setup after the pre-pass failed. - void Finish(); - // After Finish, once the end message is out and the writer is done: the status gets the latency from the last - // image and then the run's outcome. Without it the outcome is reported when the analysis goes, with no latency. - void Written(); - - // After Finish. - [[nodiscard]] std::optional GetConsensusUnitCell() const; - [[nodiscard]] float GetBeamX_pxl() const; - [[nodiscard]] float GetBeamY_pxl() const; - [[nodiscard]] float GetDetectorDistance_mm() const; - // The merge of the collection, where one was made. - [[nodiscard]] const std::optional &GetMerged() const; - [[nodiscard]] const DiffractionExperiment &GetMergeExperiment() const; - -private: - const DiffractionExperiment &experiment; - const PixelMask &mask; - const StartMessage start; - IndexerThreadPool *indexing_thread_pool; - Logger &logger; - Ready ready; - Sink sink; - Settings settings; - Failed setup_failed; - RugnuxLiveStatus *status; - const bool merge_frames; - - std::mutex live_mutex; - RugnuxLiveStills live; - std::chrono::steady_clock::time_point ready_time; - void Publish(); - // Kept by Finish for Written - std::optional last_image; - std::optional> outcome; - - Setup setup; - std::unique_ptr indexer; - - std::mutex prepass_mutex; - std::condition_variable prepass_cv; - bool prepass_last = false; // what is held is all there will be - bool prepass_over = false; - std::map> prepass_images; - std::latch prepass_done{1}; - std::exception_ptr setup_error; // set before prepass_done - - std::mutex error_mutex; - std::string analysis_error; // the first, where an analysis thread failed - void AnalysisError(const std::string &text); - - const int64_t max_held; - ThreadSafeFIFO> queue; - std::future prepass_future; - std::vector> analysis_futures; - bool finished = false; - - // The frames analysed, by number, for the merge (-1 after the last); it takes their outcomes from the indexer. - ThreadSafeFIFO merge_queue; - std::unique_ptr merge; // after the indexer: its reflections are in the indexer's arena - DiffractionExperiment merge_experiment; - std::optional merged; - std::future merge_future; - - void StartPrePass(); - void PrePassThread(); - void AnalysisThread(); - void MergeThread(); - void PublishMerge(const StillsMergeOutput &out, const std::string &title); -}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f30cb28c6..d120fcab0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -132,7 +132,6 @@ ADD_EXECUTABLE(jfjoch_test RugnuxLiveTriageTest.cpp RugnuxScaleInputsTest.cpp StillsGPUAnalysisTest.cpp - StillsGPUAnalysisFPGATest.cpp RugnuxStructureTest.cpp SweepLayoutTest.cpp BrukerReaderTest.cpp diff --git a/tests/FPGAImagePullerTest.cpp b/tests/FPGAImagePullerTest.cpp index 37027ba1d..9cc618646 100644 --- a/tests/FPGAImagePullerTest.cpp +++ b/tests/FPGAImagePullerTest.cpp @@ -165,7 +165,7 @@ namespace { JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4); service.SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(puller ? FPGAImagePullerUse::Always : FPGAImagePullerUse::Never); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller); service.RugnuxLive(rugnux, nullptr, 2, 30); PixelMask mask(x); @@ -419,7 +419,7 @@ TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") { CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(puller ? FPGAImagePullerUse::Always : FPGAImagePullerUse::Never); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller); PixelMask mask(x); service.Start(x, mask, nullptr); const auto out = service.Stop(); @@ -541,7 +541,7 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Receiver", "[FPGAImagePuller]") { pusher.delay = 3s; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(1).SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(FPGAImagePullerUse::Always, 2); + service.FPGAImagePullerSettings(2, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); @@ -654,7 +654,7 @@ TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") { CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(FPGAImagePullerUse::Always); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); @@ -689,7 +689,7 @@ TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(FPGAImagePullerUse::Always); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); @@ -860,7 +860,7 @@ TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") { HDF5FilePusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); - service.UseFPGAImagePuller(FPGAImagePullerUse::Always); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); diff --git a/tests/JFJochReceiverProcessingTest.cpp b/tests/JFJochReceiverProcessingTest.cpp index 9012745f0..312a92dd5 100644 --- a/tests/JFJochReceiverProcessingTest.cpp +++ b/tests/JFJochReceiverProcessingTest.cpp @@ -157,8 +157,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_crystal_spot_and_index_GPUStills", "[JFJoch service.Indexing(experiment.GetIndexingSettings()); // Serial data with rugnux on: the converted frames are analysed on the GPU, after the pre-pass. service.RugnuxLive(true, nullptr, 4, 60); - // The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills - service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never)); // No progress value at the start of measurement REQUIRE(!service.GetProgress().has_value()); @@ -236,8 +234,6 @@ TEST_CASE("JFJochIntegrationTest_ZMQ_GPUStills_writer_down", "[JFJochReceiver]") service.NumThreads(4); service.Indexing(experiment.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); - // The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills - service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never)); service.Start(experiment, pixel_mask, nullptr); REQUIRE_THROWS_AS(service.Stop(), JFJochException); @@ -279,8 +275,6 @@ TEST_CASE("JFJochIntegrationTest_GPUStills_cancel_during_prepass", "[JFJochRecei service.NumThreads(4); service.Indexing(experiment.GetIndexingSettings()); service.RugnuxLive(true, nullptr, 4, 60); - // The default (FPGAImagePuller and JFJochReceiverRugnuxStills) and JFJochReceiverFPGA's own live stills - service.UseFPGAImagePuller(GENERATE(FPGAImagePullerUse::RugnuxLive, FPGAImagePullerUse::Never)); service.Start(experiment, pixel_mask, nullptr); std::this_thread::sleep_for(std::chrono::seconds(3)); diff --git a/tests/StillsGPUAnalysisFPGATest.cpp b/tests/StillsGPUAnalysisFPGATest.cpp deleted file mode 100644 index a2129bf4e..000000000 --- a/tests/StillsGPUAnalysisFPGATest.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute -// SPDX-License-Identifier: GPL-3.0-only - -#include - -#include - -#include "../common/ImageBuffer.h" -#include "../receiver/StillsGPUAnalysisFPGA.h" - -namespace { - DiffractionExperiment StillsExperiment() { - DiffractionExperiment x(DetJF(1)); - x.ImagesPerTrigger(StillsGPUAnalysisFPGA::PREPASS_IMAGES + 200).NumTriggers(1).DetectorDistance_mm(100).BeamX_pxl(500).BeamY_pxl(250) - .IncidentEnergy_keV(12.4); - return x; - } - - // A frame after the pre-pass's sample (numbers from PREPASS_IMAGES on are never in it), in a slot of `buffer`. - std::shared_ptr Frame(ImageBuffer &buffer, int64_t number) { - auto held = std::make_shared(); - held->slot = buffer.GetImageSlot(); - REQUIRE(held->slot != nullptr); - held->message.number = StillsGPUAnalysisFPGA::PREPASS_IMAGES + number; - held->message.image_collection_efficiency = 1.0f; - return held; - } - - StartMessage Start(const DiffractionExperiment &x) { - StartMessage start; - x.FillMessage(start); - return start; - } -} - -TEST_CASE("StillsGPUAnalysisFPGA_StartFailureReleasesThreads", "[StillsGPUAnalysisFPGA]") { - Logger logger(Catch::getResultCapture().getCurrentTestName()); - const DiffractionExperiment x = StillsExperiment(); - const PixelMask mask(x); - ImageBuffer buffer(64 * 1024 * 1024); - buffer.StartMeasurement(x); - RugnuxLiveStatus status; - - std::atomic failed = 0, sunk = 0; - { - StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 4, 100, logger, - [](const StillsGPUAnalysisFPGA::Setup &) { - throw JFJochException(JFJochExceptionCategory::ZeroMQ, "Timeout on pushing start message"); - }, - [&](DataMessage &, const AzimuthalIntegrationProfile &) { ++sunk; }, - [] { return DiffractionExperiment::DefaultDataProcessingSettings(); }, - [&](const JFJochException &) { ++failed; }, &status); - for (int i = 0; i < 3; i++) - REQUIRE(gpu.Hold(Frame(buffer, i))); - REQUIRE_THROWS_AS(gpu.Finish(), JFJochException); - } - CHECK(failed == 1); - CHECK(sunk == 0); - CHECK(status.Get().state == RugnuxLiveState::Failed); - // Every slot the frames were held in is back. - CHECK(buffer.GetAvailSlots() == buffer.GetStatus().total_slots); -} - -TEST_CASE("StillsGPUAnalysisFPGA_HoldIsBounded", "[StillsGPUAnalysisFPGA]") { - Logger logger(Catch::getResultCapture().getCurrentTestName()); - const DiffractionExperiment x = StillsExperiment(); - const PixelMask mask(x); - ImageBuffer buffer(64 * 1024 * 1024); - buffer.StartMeasurement(x); - REQUIRE(buffer.GetStatus().total_slots >= 4); - - StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 2, 3, logger, - [](const StillsGPUAnalysisFPGA::Setup &) { - throw JFJochException(JFJochExceptionCategory::ZeroMQ, "no writer"); - }, - [](DataMessage &, const AzimuthalIntegrationProfile &) {}, - [] { return DiffractionExperiment::DefaultDataProcessingSettings(); }); - // The analysis waits for the pre-pass, which waits for its sample: nothing is taken off the queue. - for (int i = 0; i < 3; i++) - REQUIRE(gpu.Hold(Frame(buffer, i))); - { - const auto refused = Frame(buffer, 3); - CHECK(!gpu.Hold(refused)); - } - CHECK(buffer.GetStatus().preparation_slots == 3); - REQUIRE_THROWS(gpu.Finish()); - CHECK(buffer.GetStatus().preparation_slots == 0); -} - -TEST_CASE("StillsGPUAnalysisFPGA_FailingFrameDoesNotStopAnalysis", "[StillsGPUAnalysisFPGA]") { - Logger logger(Catch::getResultCapture().getCurrentTestName()); - const DiffractionExperiment x = StillsExperiment(); - const PixelMask mask(x); - ImageBuffer buffer(64 * 1024 * 1024); - buffer.StartMeasurement(x); - RugnuxLiveStatus status; - - std::atomic sunk = 0; - StillsGPUAnalysisFPGA gpu(x, mask, Start(x), nullptr, 1, 100, logger, - [](const StillsGPUAnalysisFPGA::Setup &) {}, - [&](DataMessage &m, const AzimuthalIntegrationProfile &) { - ++sunk; - if (m.number == StillsGPUAnalysisFPGA::PREPASS_IMAGES) - throw std::runtime_error("not a JFJochException"); - }, - [] { return DiffractionExperiment::DefaultDataProcessingSettings(); }, - {}, &status); - // The frames carry no image, so their analysis fails too - they are passed on all the same. - for (int i = 0; i < 3; i++) - REQUIRE(gpu.Hold(Frame(buffer, i))); - REQUIRE_NOTHROW(gpu.Finish()); - CHECK(sunk == 3); - // The outcome waits for everything to be written, and comes after the latency. - CHECK(status.Get().state == RugnuxLiveState::Running); - CHECK(!status.Get().last_image_to_written_s); - gpu.Written(); - const auto s = status.Get(); - CHECK(s.state == RugnuxLiveState::Failed); - CHECK(!s.message.empty()); - REQUIRE(s.last_image_to_written_s); - REQUIRE(s.last_image_s); - CHECK(s.elapsed_s >= s.last_image_s.value() + s.last_image_to_written_s.value() - 1e-6); - REQUIRE(!s.steps.empty()); - CHECK(s.steps.back().title == "Writing"); - CHECK(s.steps.back().duration_s); - CHECK(buffer.GetStatus().preparation_slots == 0); -} diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index 09e0eacd6..040b1f375 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -41,8 +41,6 @@ void print_usage(Logger &logger) { logger.Info(" -r Rates the source is paced at, one collection each (default: 0 = as fast as it goes)"); logger.Info(" -p FPGAImagePuller slots (default 128)"); logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped"); - logger.Info(" -F JFJochReceiverFPGA's own live stills instead of FPGAImagePuller (not blocking: frames"); - logger.Info(" past what its analysis holds are lost)"); logger.Info(" -K{} Mock of the card instead of /dev/jfjoch* (optional: its generator memory, default 64 MiB ="); logger.Info(" 128 module frames)"); } @@ -88,7 +86,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file, std::optional nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB, const IndexingSettings &indexing, const std::optional &cell, size_t puller_slots, - bool fpga_receiver, const std::vector &rates, bool dropping) { + const std::vector &rates, bool dropping) { RegisterHDF5Filter(); JFJochHDF5Reader reader; reader.ReadFile(file); @@ -166,13 +164,10 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s for (const double rate: rates) { // Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer, - // the cards hold their frames and their generators stall. JFJochReceiverFPGA's live stills cannot block. - service.UseFPGAImagePuller(fpga_receiver ? FPGAImagePullerUse::Never : FPGAImagePullerUse::RugnuxLive, - puller_slots, !dropping, rate); + // the cards hold their frames and their generators stall + service.FPGAImagePullerSettings(puller_slots, !dropping, rate); logger.Info(""); - logger.Info("=== {}, {} ===", fpga_receiver ? "JFJochReceiverFPGA (not blocking)" - : fmt::format("FPGAImagePuller ({} slots, {})", puller_slots, - dropping ? "dropping" : "blocking"), + logger.Info("=== FPGAImagePuller ({} slots, {}), {} ===", puller_slots, dropping ? "dropping" : "blocking", (rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes")); rusage usage_start{}; @@ -289,14 +284,13 @@ int main(int argc, char **argv) { IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto; uint16_t indexing_threads = 4; size_t puller_slots = FPGAImagePuller::DefaultSlots; - bool fpga_receiver = false; std::optional mock_card_MiB; bool verbose = false; std::vector rates = {0}; bool dropping = false; int opt; - while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dFK::")) != -1) { + while ((opt = getopt(argc, argv, "vi:s:N:B:C:X:t:r:p:dK::")) != -1) { switch (opt) { case 'v': verbose = true; @@ -346,9 +340,6 @@ int main(int argc, char **argv) { case 'p': puller_slots = atol(optarg); break; - case 'F': - fpga_receiver = true; - break; case 'd': dropping = true; break; @@ -387,7 +378,7 @@ int main(int argc, char **argv) { bool ret = false; try { ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB, - i_settings, cell, puller_slots, fpga_receiver, rates, dropping); + i_settings, cell, puller_slots, rates, dropping); } catch (std::exception &e) { logger.Error(e.what()); }