From 4e8ccc2b60ac323fb579ecff8a232f4d7a90d3ef Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 18:49:54 +0100 Subject: [PATCH] Hybrid summation: the card sums the largest divisor up to 256, the host the rest; through FPGAImagePuller too For a summation of N frames the card sums d, the largest divisor of N not above 256 (GetFPGASummation; 256 itself is now summed on the card), and the host the N/d card images of each image (GetHostSummation): 1000 = 250 x 4, a prime above 256 = 1 x N. CPUSummation(true) still puts all of it on the host. The card writes 32-bit where it sums and the host adds to it. FPGAImagePuller now takes summation on the host, so rugnux live runs on such collections. Each image thread keeps one int32 module accumulator per module (ModuleSummation, saturating, error pixels carried, as JFJochReceiverFPGA and the HLS do), takes the card's images of its image in order, all modules of each before the next, adds each into the accumulator and returns the card's buffer; the summed modules then go into the thread's image as before. A module missing from any card image is missing from the image. The card's per-frame spots and ROI sums were already not read on this path; the module statistics are those of the first card image with error, saturated, packet counts and pixel sum added (ModuleSummation now adds the pixel sum too). Packets expected per image and the not-blocking give-up distance are in card images (x N/d). JFJochReceiverFPGA follows the same split (it has to: the card's summation is set by the experiment), summing N/d card images instead of N frames; its end-of-run expected packets no longer multiply the run's total by the summation a second time. OpenAPI image_time_us and the broker docs describe the split; C++ model, frontend client and python client docs regenerated. Tests: the divisor rule, and N = 256 (card only), 1000 (250 x 4) and 257 (host only) on the mock through both receivers, pixels against a sum done in the test with error and saturated pixels and a 32-bit overflow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3 --- broker/gen/model/Dataset_settings.h | 2 +- broker/jfjoch_api.yaml | 6 +- common/DiffractionExperiment.cpp | 21 ++- common/DiffractionExperiment.h | 5 + common/ModuleSummation.cpp | 8 + common/ModuleSummation.h | 2 + docs/JFJOCH_BROKER.md | 14 +- docs/python_client/docs/DatasetSettings.md | 2 +- frontend/src/client/types.gen.ts | 12 +- receiver/FPGAImagePuller.cpp | 65 ++++++-- receiver/FPGAImagePuller.h | 3 + receiver/JFJochReceiverFPGA.cpp | 16 +- receiver/JFJochReceiverService.cpp | 8 +- tests/CMakeLists.txt | 1 + tests/DiffractionExperimentTest.cpp | 36 +++- tests/FPGAImagePullerSummationTest.cpp | 181 +++++++++++++++++++++ tests/FPGAImagePullerTest.cpp | 6 +- 17 files changed, 331 insertions(+), 57 deletions(-) create mode 100644 tests/FPGAImagePullerSummationTest.cpp diff --git a/broker/gen/model/Dataset_settings.h b/broker/gen/model/Dataset_settings.h index ba91eefb2..a91c0d4a5 100644 --- a/broker/gen/model/Dataset_settings.h +++ b/broker/gen/model/Dataset_settings.h @@ -81,7 +81,7 @@ public: bool ntriggerIsSet() const; void unsetNtrigger(); /// - /// Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 255 frames are summed on the FPGA; a longer image time is summed on the host, which is slower and runs no rugnux live analysis. In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. + /// Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of frames is summed on the host entirely). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. /// int64_t getImageTimeUs() const; void setImageTimeUs(int64_t const value); diff --git a/broker/jfjoch_api.yaml b/broker/jfjoch_api.yaml index 5abb78447..fdd8e0095 100644 --- a/broker/jfjoch_api.yaml +++ b/broker/jfjoch_api.yaml @@ -409,8 +409,10 @@ components: description: | Image time. If not provided (or zero value) the frame time is assumed as default. - For JUNGFRAU image time must be multiple of frame time. Up to 255 frames are summed on the FPGA; - a longer image time is summed on the host, which is slower and runs no rugnux live analysis. + For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. + Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image + (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of + frames is summed on the host entirely). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. beam_x_pxl: diff --git a/common/DiffractionExperiment.cpp b/common/DiffractionExperiment.cpp index b4a20a48b..a653b3462 100644 --- a/common/DiffractionExperiment.cpp +++ b/common/DiffractionExperiment.cpp @@ -881,14 +881,23 @@ int64_t DiffractionExperiment::GetSummation() const { } int64_t DiffractionExperiment::GetFPGASummation() const { - if (IsCPUSummation()) + const int64_t n = GetSummation(); + if (cpu_summation) return 1; - return GetSummation(); + for (int64_t d = std::min(n, MAX_FPGA_SUMMATION); d > 1; d--) + if (n % d == 0) + return d; + return 1; +} + +int64_t DiffractionExperiment::GetHostSummation() const { + return GetSummation() / GetFPGASummation(); } int64_t DiffractionExperiment::GetByteDepthFPGA() const { + // Summed on the card as well: 32-bit, the host adds to it if (IsCPUSummation()) - return 2; + return (GetFPGASummation() > 1) ? 4 : 2; return GetByteDepthImage(); } @@ -1638,11 +1647,7 @@ DiffractionExperiment &DiffractionExperiment::ApplyPixelMask(bool input) { } bool DiffractionExperiment::IsCPUSummation() const { - if (summation == 1) - return false; - if (summation >= MAX_FPGA_SUMMATION) - return true; - return cpu_summation; + return GetHostSummation() > 1; } DiffractionExperiment & DiffractionExperiment::ElectronSource(bool input) { diff --git a/common/DiffractionExperiment.h b/common/DiffractionExperiment.h index c55444a93..197455730 100644 --- a/common/DiffractionExperiment.h +++ b/common/DiffractionExperiment.h @@ -357,7 +357,10 @@ public: std::optional GetSpaceGroupNumber() const; bool GetSaveCalibration() const; int64_t GetSummation() const; + // The card sums the largest divisor of the summation up to MAX_FPGA_SUMMATION, the host the card's images of + // one image: GetSummation() = GetFPGASummation() * GetHostSummation() int64_t GetFPGASummation() const; + int64_t GetHostSummation() const; std::string GetSampleName() const; float GetIncidentEnergy_keV() const; float GetWavelength_A() const; @@ -424,7 +427,9 @@ public: [[nodiscard]] bool IsMaskPixelsWithoutG0() const; [[nodiscard]] bool IsApplyPixelMask() const; + // All of the summation on the host (tests) DiffractionExperiment& CPUSummation(bool input); + // The host sums part of the image (GetHostSummation() > 1) [[nodiscard]] bool IsCPUSummation() const; [[nodiscard]] FileWriterFormat GetFileWriterFormat() const; diff --git a/common/ModuleSummation.cpp b/common/ModuleSummation.cpp index 50866ae77..b05310401 100644 --- a/common/ModuleSummation.cpp +++ b/common/ModuleSummation.cpp @@ -17,6 +17,13 @@ ModuleSummation::ModuleSummation(bool in_pixel_signed, uint32_t in_fpga_depth_by throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "FPGA byte depth can be only 1, 2 or 4"); output = std::make_unique(); + Clear(); +} + +void ModuleSummation::Clear() { + std::unique_lock ul(module_summation_mutex); + first = true; + is_empty = false; memset(output.get(), 0, sizeof(DeviceOutput)); } @@ -69,6 +76,7 @@ void ModuleSummation::AddFPGAOutput(const DeviceOutput &input, const std::option output->module_statistics.err_pixels += input.module_statistics.err_pixels; output->module_statistics.saturated_pixels += input.module_statistics.saturated_pixels; output->module_statistics.packet_count += input.module_statistics.packet_count; + output->module_statistics.pixel_sum += input.module_statistics.pixel_sum; } // The only function not handled is spot finding diff --git a/common/ModuleSummation.h b/common/ModuleSummation.h index 31d63b178..cf59a3ec3 100644 --- a/common/ModuleSummation.h +++ b/common/ModuleSummation.h @@ -23,6 +23,8 @@ public: void AddFPGAOutput(const DeviceOutput& input, const std::optional &in_fpga_depth_bytes = {}); void AddEmptyOutput(); + // Empty, for the next image + void Clear(); [[nodiscard]] const DeviceOutput& GetOutput() const; [[nodiscard]] DeviceOutput& GetOutput(); diff --git a/docs/JFJOCH_BROKER.md b/docs/JFJOCH_BROKER.md index 43e3b31f4..0bef59630 100644 --- a/docs/JFJOCH_BROKER.md +++ b/docs/JFJOCH_BROKER.md @@ -221,8 +221,8 @@ goniometer and the rugnux live setting: | Input | Run | Receiver | What it does | |---|---|---|---| | FPGA cards (PSI JUNGFRAU, PSI EIGER) | pedestal run | `JFJochReceiverPedestal` | the raw frames into the pedestal and its RMS; nothing on any stream | -| FPGA cards | rugnux live off, or summation on the host | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer | -| FPGA cards | rugnux live on | `FPGAImagePuller`, then `JFJochReceiverRugnuxRotation` or `JFJochReceiverRugnuxStills` (`JFJochReceiverLite` for a rotation run the image buffer cannot hold) | the cards' frames assembled and compressed on the host into a stream as a DECTRIS detector gives, then taken as the DECTRIS rows below; the pedestal and gain calibration is not written to the file on this path | +| FPGA cards | rugnux live off | `JFJochReceiverFPGA` | the card's converted pixels, spots, azimuthal and ROI sums into the image; summation on the host above what the card sums; indexing of each image; the calibration to the writer | +| FPGA cards | rugnux live on | `FPGAImagePuller`, then `JFJochReceiverRugnuxRotation` or `JFJochReceiverRugnuxStills` (`JFJochReceiverLite` for a rotation run the image buffer cannot hold) | the cards' frames assembled (and summed above what the cards sum) and compressed on the host into a stream as a DECTRIS detector gives, then taken as the DECTRIS rows below; the pedestal and gain calibration is not written to the file on this path | | DECTRIS stream | dark-mask sequence | `JFJochReceiverDarkMask` | the dark frames into the mask; nothing on any stream | | DECTRIS stream | rugnux live off, or a rotation run the image buffer cannot hold | `JFJochReceiverLite` | each image on the GPU: masking, azimuthal and ROI integration, spot finding, indexing | | DECTRIS stream | rugnux live on, rotation that fits the image buffer | `JFJochReceiverRugnuxRotation` | images forwarded as they come; rugnux reads the collection back from the image buffer and runs the whole pipeline | @@ -236,13 +236,19 @@ built for earlier indexing settings are released when the settings change. `azim moves the azimuthal integration of the FPGA workflow from the card to the host, which a profile with more bins than the card holds needs. +A card sums up to 256 frames into an image. For more, it sums the largest number of frames up to 256 that divides +the frames of an image (250 of 1000) and the host adds up the card's images (4 of 250 each), in 32 bits, saturating; +a pixel in error or saturated in any of them is so in the sum, and a module missing from any of them is missing +from the image. A prime number of frames above 256 is summed on the host entirely, at the cost of reading every +frame. + ## Rugnux live mode With rugnux live mode on, collections from a DECTRIS detector are processed inside the broker while they arrive: rotation collections by rugnux, and serial (stills) collections on the GPU with the geometry and mask a rugnux pre-pass decided (see [Serial collections](#serial-collections-experimental) below). In the FPGA workflow -(PSI JUNGFRAU and EIGER) the frames reach the same receivers through `FPGAImagePuller`, rotation and serial alike; -with summation on the host the collection runs image by image, as with the mode off. It is off by default; the +(PSI JUNGFRAU and EIGER) the frames reach the same receivers through `FPGAImagePuller`, rotation and serial alike. +It is off by default; the configuration file switches it on with `"rugnux": {"enable": true, "reserved_cpus": 4}`, where `reserved_cpus` is the number of CPU threads left to the receiver and the image pusher. diff --git a/docs/python_client/docs/DatasetSettings.md b/docs/python_client/docs/DatasetSettings.md index 9e1e77451..ad29b5dd6 100644 --- a/docs/python_client/docs/DatasetSettings.md +++ b/docs/python_client/docs/DatasetSettings.md @@ -7,7 +7,7 @@ Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- **images_per_trigger** | **int** | For standard synchrotron data collection - this is number of images collected per one TTL trigger For XFEL (pulsed source) - this number is ignored and set to 1 For storage cell mode - this number is ignored and set to number of storage cells | [optional] [default to 1] **ntrigger** | **int** | Number of TTL trigger that the detector is expected to receive during data collection | [optional] [default to 1] -**image_time_us** | **int** | Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 255 frames are summed on the FPGA; a longer image time is summed on the host, which is slower and runs no rugnux live analysis. In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. | [optional] +**image_time_us** | **int** | Image time. If not provided (or zero value) the frame time is assumed as default. For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of frames is summed on the host entirely). In XFEL mode: summation happens for frames collected with multiple triggers. Ignored for storage cells and if raw data are saved. | [optional] **beam_x_pxl** | **float** | /entry/detector/beam_center_x in NXmx Beam center in X direction [pixels] | **beam_y_pxl** | **float** | /entry/detector/beam_center_y in NXmx Beam center in X direction [pixels] | **detector_distance_mm** | **float** | /entry/detector/distance in NXmx Detector distance [mm] | diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index 0473e22cc..757c8a5fe 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -166,8 +166,10 @@ export type dataset_settings = { /** * Image time. * If not provided (or zero value) the frame time is assumed as default. - * For JUNGFRAU image time must be multiple of frame time. Up to 255 frames are summed on the FPGA; - * a longer image time is summed on the host, which is slower and runs no rugnux live analysis. + * For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. + * Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image + * (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of + * frames is summed on the host entirely). * In XFEL mode: summation happens for frames collected with multiple triggers. * Ignored for storage cells and if raw data are saved. * @@ -2642,8 +2644,10 @@ export type dataset_settingsWritable = { /** * Image time. * If not provided (or zero value) the frame time is assumed as default. - * For JUNGFRAU image time must be multiple of frame time. Up to 255 frames are summed on the FPGA; - * a longer image time is summed on the host, which is slower and runs no rugnux live analysis. + * For JUNGFRAU image time must be multiple of frame time. Up to 256 frames are summed on the FPGA. + * Above that the FPGA sums the largest number of frames up to 256 that divides the frames of an image + * (250 of 1000) and the host adds up the rest; the more the host sums, the slower (a prime number of + * frames is summed on the host entirely). * In XFEL mode: summation happens for frames collected with multiple triggers. * Ignored for storage cells and if raw data are saved. * diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index 948e02acb..7ec665024 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -18,7 +18,8 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con devices(in_devices), queue(in_queue), blocking(in_blocking), - frame_rate_hz(in_frame_rate_hz) { + frame_rate_hz(in_frame_rate_hz), + host_summation(in_experiment.GetHostSummation()) { const int ndatastreams = experiment.GetDataStreamsNum(); if (devices.size() < ndatastreams) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, @@ -32,7 +33,7 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con given_up[d].resize(card_experiment.GetModulesNum(d)); devices[d].PrepareAction(card_experiment); // Of the modules of the image, not the dummies - expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() + expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() * host_summation * experiment.GetModulesNum(d) / card_experiment.GetModulesNum(d); } @@ -171,7 +172,7 @@ bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, || counters.IsAcquisitionFinished() || failed) break; if (!blocking && ((static_cast(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames) - || (FastestFrame() >= frame + 2 + static_cast(queue)))) + || (FastestFrame() >= frame + 2 + static_cast(queue) * host_summation))) break; std::this_thread::sleep_for(std::chrono::milliseconds(1)); } @@ -185,6 +186,11 @@ bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, void FPGAImagePuller::ImageThread() { // One image of its own, assembled and compressed here FrameTransformation transformation(experiment); + // Summation on the host: the cards' images of one image added up, one module each, as in JFJochReceiverFPGA + std::vector> summation; + if (host_summation > 1) + for (int m = 0; m < experiment.GetModulesNum(); m++) + summation.emplace_back(std::make_unique(experiment)); try { for (int64_t image = next_to_take++; image < experiment.GetImageNum() && !failed; image = next_to_take++) { if (frame_rate_hz > 0) @@ -194,24 +200,47 @@ void FPGAImagePuller::ImageThread() { // Not blocking, an image the receiver has no room for is not assembled: its frames go back at once const bool drop = !blocking && (GetCurrentFifoUtilization() >= queue); ImageMetadata metadata(experiment); - for (int d = 0; d < experiment.GetDataStreamsNum(); d++) { - auto &device = devices[d]; - for (int m = 0; m < card_experiment.GetModulesNum(d); m++) { - const bool collected = WaitForFrame(d, m, image); - // A dummy module (CardExperiment): its frames only go back to the card - if (!drop && m < experiment.GetModulesNum(d)) { - if (collected) { - const DeviceOutput *output = device.GetDeviceOutput(image, m); - metadata.Process(output); - adu_histogram[experiment.GetFirstModuleOfDataStream(d) + m]->Add(*output); - transformation.ProcessModule(output, d); - } else - transformation.FillNotCollectedModule(m, d); + auto take_module = [&](const DeviceOutput *output, int d, int m) { + metadata.Process(output); + adu_histogram[experiment.GetFirstModuleOfDataStream(d) + m]->Add(*output); + transformation.ProcessModule(output, d); + }; + for (auto &s: summation) + s->Clear(); + // The card's images of this image in their order, all modules of each before the next, as they come + for (int64_t frame = image * host_summation; frame < (image + 1) * host_summation; frame++) { + for (int d = 0; d < experiment.GetDataStreamsNum(); d++) { + auto &device = devices[d]; + for (int m = 0; m < card_experiment.GetModulesNum(d); m++) { + const bool collected = WaitForFrame(d, m, frame); + // A dummy module (CardExperiment): its frames only go back to the card + if (!drop && m < experiment.GetModulesNum(d)) { + if (!summation.empty()) { + auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m]; + if (collected) + s.AddFPGAOutput(*device.GetDeviceOutput(frame, m)); + else + s.AddEmptyOutput(); + } else if (collected) + take_module(device.GetDeviceOutput(frame, m), d, m); + else + transformation.FillNotCollectedModule(m, d); + } + if (collected) + device.FrameBufferRelease(frame, m); } - if (collected) - device.FrameBufferRelease(image, m); } } + // A module missing from any of the card's images is missing from the image + if (!drop && !summation.empty()) + for (int d = 0; d < experiment.GetDataStreamsNum(); d++) + for (int m = 0; m < experiment.GetModulesNum(d); m++) { + const auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m]; + if (s.empty()) + transformation.FillNotCollectedModule(m, d); + else + take_module(&s.GetOutput(), d, m); + } DataMessage message{}; message.number = image; diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index 9a8bc31fb..da757a4e6 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -16,6 +16,7 @@ #include "ImageMetadata.h" #include "../acquisition_device/AcquisitionDeviceGroup.h" #include "../common/ADUHistogram.h" +#include "../common/ModuleSummation.h" #include "../image_puller/ImagePuller.h" #include "../jungfrau/JFCalibration.h" @@ -41,6 +42,8 @@ private: // behind the others, so the cards keep their frames and stall at the source. const bool blocking; const double frame_rate_hz; // images taken from the cards at most at this rate (0: as they come) + // The cards' images making one image, summed here (DiffractionExperiment::GetHostSummation) + const int64_t host_summation; std::chrono::steady_clock::time_point start_time; std::atomic queue_wait_ns = 0; uint64_t expected_packets_per_image = 0; diff --git a/receiver/JFJochReceiverFPGA.cpp b/receiver/JFJochReceiverFPGA.cpp index 09bc6d2a1..89ab51f90 100644 --- a/receiver/JFJochReceiverFPGA.cpp +++ b/receiver/JFJochReceiverFPGA.cpp @@ -77,7 +77,7 @@ JFJochReceiverFPGA::JFJochReceiverFPGA(const DiffractionExperiment &in_experimen } if (experiment.IsCPUSummation()) - expected_packets_per_image *= experiment.GetSummation(); + expected_packets_per_image *= experiment.GetHostSummation(); logger.Debug("Data acquisition devices ready"); @@ -177,9 +177,9 @@ void JFJochReceiverFPGA::SummationThread(uint16_t data_stream, uint32_t threadid, ModuleSummation &summation) { ModuleSummation local_summation(experiment); - int64_t starting_frame = image_number * experiment.GetSummation(); + int64_t starting_frame = image_number * experiment.GetHostSummation(); - for (int64_t i = threadid; i < experiment.GetSummation(); i += summation_nthreads) { + for (int64_t i = threadid; i < experiment.GetHostSummation(); i += summation_nthreads) { const int64_t frame = starting_frame + i; // Frame will be processed only if one already collects frame+2 @@ -229,7 +229,7 @@ void JFJochReceiverFPGA::FrameTransformationThread(uint32_t threadid) { try { int64_t expected_frame = image_number; if (experiment.IsCPUSummation()) - expected_frame *= experiment.GetSummation(); + expected_frame *= experiment.GetHostSummation(); logger.Debug("Frame transformation thread - trying to get image {}", expected_frame); // If data acquisition is finished and fastest frame for the first device is behind @@ -491,7 +491,7 @@ void JFJochReceiverFPGA::Cancel(const JFJochException &e) { float JFJochReceiverFPGA::GetProgress() const { int64_t frames = experiment.GetImageNum(); if (experiment.IsCPUSummation()) - frames *= experiment.GetSummation(); + frames *= experiment.GetHostSummation(); if (acquisition_device[0].Counters().IsAcquisitionFinished()) return 1.0; @@ -570,10 +570,8 @@ JFJochReceiverOutput JFJochReceiverFPGA::GetFinalStatistics() const { JFJochReceiverOutput ret = JFJochReceiver::GetFinalStatistics(); for (int d = 0; d < ndatastreams; d++) { for (int m = 0; m < acquisition_device[d].Counters().GetModuleNumber(); m++) { - if (experiment.IsCPUSummation()) - ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule() * experiment.GetSummation()); - else - ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule()); + // All the frames of the collection, whoever sums them + ret.expected_packets.push_back(acquisition_device[d].Counters().GetTotalExpectedPacketsPerModule()); ret.received_packets.push_back(acquisition_device[d].Counters().GetTotalPackets(m)); } } diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index 50d9d892d..251ded179 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -59,10 +59,10 @@ double JFJochReceiverService::GetFPGAImagePullerQueueWaitTime_s() const { return fpga_puller ? fpga_puller->GetQueueWaitTime_s() : 0.0; } -// A PSI collection analysed by rugnux live; summation on the host needs JFJochReceiverFPGA +// A PSI collection analysed by rugnux live bool JFJochReceiverService::FPGAImagePullerFor(const DiffractionExperiment &experiment) const { if ((experiment.GetDetectorType() == DetectorType::DECTRIS) || experiment.IsPedestalRun() - || experiment.IsCPUSummation() || (experiment.GetImageNum() <= 0)) + || (experiment.GetImageNum() <= 0)) return false; return fpga_image_puller_always || (rugnux_enable && (experiment.GetDetectorMode() == DetectorMode::Standard)); } @@ -157,10 +157,6 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, logger, spot_finding_settings, receiver_status, plots, image_buffer); } else if ((experiment.GetDetectorType() != DetectorType::DECTRIS) && !fpga_puller) { - if (rugnux_enable && experiment.IsCPUSummation()) { - analysis_warning = "no rugnux analysis: summation on the host"; - logger.Warning("{}", analysis_warning); - } // Summation on the host runs threads of its own for each image auto fpga_nthreads = nthreads; if (experiment.IsCPUSummation()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3ae22959c..d7b1cd12b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -155,6 +155,7 @@ ADD_EXECUTABLE(jfjoch_test JFJochReceiverLiteTest.cpp JFJochReceiverRugnuxTest.cpp FPGAImagePullerTest.cpp + FPGAImagePullerSummationTest.cpp StillsSelectionTest.cpp HoldArenaTest.cpp GridScanSettingsTest.cpp diff --git a/tests/DiffractionExperimentTest.cpp b/tests/DiffractionExperimentTest.cpp index a5e2c172b..93c7090a0 100644 --- a/tests/DiffractionExperimentTest.cpp +++ b/tests/DiffractionExperimentTest.cpp @@ -783,10 +783,44 @@ TEST_CASE("DiffractionExperiment_Summation","[DiffractionExperiment]") { REQUIRE_NOTHROW(x.Summation(500)); CHECK(x.GetByteDepthImage() == 4); CHECK(x.GetSummation() == 500); - CHECK(x.GetFPGASummation() == 1); + CHECK(x.GetFPGASummation() == 250); + CHECK(x.GetHostSummation() == 2); CHECK(x.IsCPUSummation()); } +// The card sums the largest divisor up to 256, the host the rest +TEST_CASE("DiffractionExperiment_Summation_CardAndHost","[DiffractionExperiment]") { + DiffractionExperiment x; + for (const auto &[n, card]: std::vector>{ + {1, 1}, {7, 7}, {255, 255}, {256, 256}, {257, 1}, {512, 256}, {1000, 250}, {1024, 256}, {300, 150}, + {514, 2}, {1009, 1}}) { + INFO("summation " << n); + x.Summation(n); + CHECK(x.GetSummation() == n); + CHECK(x.GetFPGASummation() == card); + CHECK(x.GetHostSummation() == n / card); + CHECK(x.IsCPUSummation() == (card != n)); + } + + // 32-bit from the card where it sums as well; single frames at 16 bits + x.Summation(256); + CHECK(x.GetByteDepthFPGA() == 4); + x.Summation(1000); + CHECK(x.GetByteDepthFPGA() == 4); + CHECK(x.GetByteDepthImage() == 4); + x.Summation(257); + CHECK(x.GetByteDepthFPGA() == 2); + CHECK(x.GetByteDepthImage() == 4); + + // All of it on the host + x.Summation(7).CPUSummation(true); + CHECK(x.GetFPGASummation() == 1); + CHECK(x.GetHostSummation() == 7); + CHECK(x.IsCPUSummation()); + x.Summation(1); + CHECK(!x.IsCPUSummation()); +} + TEST_CASE("DiffractionExperiment_DetectorModuleHostname","[DiffractionExperiment]") { std::vector h = {"mx1", "mx2", "mx3"}; DiffractionExperiment x(DetectorSetup(DetectorGeometryModular(3), diff --git a/tests/FPGAImagePullerSummationTest.cpp b/tests/FPGAImagePullerSummationTest.cpp new file mode 100644 index 000000000..f8c8ef986 --- /dev/null +++ b/tests/FPGAImagePullerSummationTest.cpp @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Summation split between the card and the host, on the mock of the card: the card sums the largest divisor of the +// summation up to 256, the host the card's images of one image. The pixels of both receivers are checked against +// a sum done here. + +#include + +#include +#include + +#include "../frame_serialize/CBORStream2Deserializer.h" +#include "../image_pusher/ImagePusher.h" +#include "../receiver/FPGAImagePuller.h" +#include "../receiver/FrameTransformation.h" +#include "../receiver/JFJochReceiverService.h" +#include "MockCardOutput.h" + +namespace { + struct SumReceived { + DataMessage message; + std::vector pixels; + }; + + class SumPusher : public ImagePusher { + public: + std::mutex m; + std::map images; + + void StartDataCollection(StartMessage &) override {} + bool EndDataCollection(const EndMessage &) override { return true; } + bool SendCalibration(const CompressedImage &) override { return true; } + bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override { + auto d = CBORStream2Deserialize(image_data, image_size); + SumReceived r; + r.message = d->data_message.value(); + r.message.image.GetUncompressed(r.pixels); + r.message.image = CompressedImage(); + std::unique_lock ul(m); + images.emplace(image_number, std::move(r)); + return true; + } + std::string PrintSetup() const override { return "summation"; } + ImagePusherType GetType() const override { return ImagePusherType::Test; } + }; + + DiffractionExperiment SumExperiment(int64_t summation, int64_t images) { + DiffractionExperiment x(DetJF(2)); + x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).InternalPacketGeneratorImages(1) + .ImagesPerTrigger(images).ImagesPerFile(10).IncidentEnergy_keV(12.4) + .DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600); + x.Conversion(); + x.Summation(summation).Compression(CompressionAlgorithm::BSHUF_LZ4); + return x; + } + + constexpr size_t ErrPixel = 100, SatPixel = 200, LargePixel = 300; + constexpr int64_t CardImageCount = 5; + + // What the card writes for its images, in its pixel format: a pattern, one pixel in error in the second image, one + // saturated in the third, and one large enough that three of them saturate in 32 bits + std::vector CardSums(const DiffractionExperiment &x) { + const bool wide = x.GetByteDepthFPGA() == 4; + std::vector ret; + std::vector values(RAW_MODULE_SIZE); + for (int64_t f = 0; f < CardImageCount; f++) { + CardImage image; + for (int m = 0; m < x.GetModulesNum(); m++) { + for (size_t i = 0; i < RAW_MODULE_SIZE; i++) + values[i] = (f * 7 + m * 3 + static_cast(i)) % 1000; + values[ErrPixel] = (f == 1) ? (wide ? INT32_MIN : INT16_MIN) : 5; + values[SatPixel] = (f == 2) ? (wide ? INT32_MAX : INT16_MAX) : 5; + values[LargePixel] = wide ? INT32_MAX / 3 + 1 : 30000; + auto output = std::make_unique(); + StorePixels(*output, x, values); + output->module_statistics.pixel_sum = 1000 * f + m; + image.push_back(std::move(output)); + } + ret.push_back(std::move(image)); + } + return ret; + } + + template + int32_t Pixel(const DeviceOutput &output, size_t i) { + return reinterpret_cast(output.pixels)[i]; + } + + // The image summed here from the card's images, by ModuleSummation's rule: an error in any is an error, then a + // saturated one or a sum reaching INT32_MAX is saturated. Assembled by FrameTransformation, as the receivers do. + std::vector Reference(const DiffractionExperiment &x, const std::vector &cards, int64_t image) { + const bool wide = x.GetByteDepthFPGA() == 4; + const int64_t err_src = wide ? INT32_MIN : INT16_MIN; + const int64_t sat_src = wide ? INT32_MAX : INT16_MAX; + const int64_t h = x.GetHostSummation(); + + FrameTransformation transformation(x); + for (int m = 0; m < x.GetModulesNum(); m++) { + auto output = std::make_unique(); + memset(output.get(), 0, sizeof(DeviceOutput)); + output->module_statistics.module_number = m; + auto sum = reinterpret_cast(output->pixels); + for (size_t i = 0; i < RAW_MODULE_SIZE; i++) { + int64_t acc = 0; + for (int64_t k = 0; k < h; k++) { + const auto &card = *cards[(image * h + k) % CardImageCount][m]; + const int64_t v = wide ? Pixel(card, i) : Pixel(card, i); + if (v == err_src || acc == INT32_MIN) + acc = INT32_MIN; + else if (v == sat_src || acc == INT32_MAX || acc + v >= INT32_MAX) + acc = INT32_MAX; + else + acc += v; + } + sum[i] = static_cast(acc); + } + transformation.ProcessModule(output.get(), 0); + } + auto p = static_cast(transformation.GetImage()); + return {p, p + x.GetPixelsNum() * x.GetByteDepthImage()}; + } + + std::map Run(const DiffractionExperiment &x, bool puller) { + auto devices = MockDevices(x, CardSums(x)); + Logger logger("FPGAImagePullerSummation"); + SumPusher pusher; + JFJochReceiverService service(devices, logger, pusher, 256); + SpotFindingSettings spots = DiffractionExperiment::DefaultDataProcessingSettings(); + spots.indexing = false; + service.NumThreads(4).SetSpotFindingSettings(spots); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, puller); + PixelMask mask(x); + const JFCalibration calib = GeneratePedestalCalibration(x); + service.Start(x, mask, &calib); + const auto out = service.Stop(); + CHECK(out.efficiency == 1.0f); + CHECK(out.received_packets.empty() == puller); // only the FPGA receiver fills these + return std::move(pusher.images); + } + + void CheckSummation(int64_t summation, int64_t card, int64_t host) { + const auto x = SumExperiment(summation, 2); + REQUIRE(x.GetFPGASummation() == card); + REQUIRE(x.GetHostSummation() == host); + REQUIRE(x.GetByteDepthImage() == 4); + const auto cards = CardSums(x); + + for (const bool puller: {false, true}) { + INFO("puller " << puller); + const auto images = Run(x, puller); + REQUIRE(images.size() == 2); + for (const auto &[n, r]: images) { + INFO("image " << n); + CHECK(r.message.packets_received == r.message.packets_expected); + CHECK(r.message.packets_expected == 512 * summation * x.GetModulesNum()); + CHECK(r.pixels == Reference(x, cards, n)); + // From the card's module statistics: only the FPGA receiver keeps them + if (!puller) { + int64_t pixel_sum = 0; + for (int64_t k = 0; k < host; k++) + for (int m = 0; m < x.GetModulesNum(); m++) + pixel_sum += 1000 * ((n * host + k) % CardImageCount) + m; + CHECK(r.message.pixel_sum == pixel_sum); + } + } + } + } +} + +TEST_CASE("FPGAImagePuller_Summation256_CardOnly", "[FPGAImagePuller]") { + CheckSummation(256, 256, 1); +} + +TEST_CASE("FPGAImagePuller_Summation1000_Hybrid", "[FPGAImagePuller]") { + CheckSummation(1000, 250, 4); +} + +TEST_CASE("FPGAImagePuller_Summation257_HostOnly", "[FPGAImagePuller]") { + CheckSummation(257, 1, 257); +} diff --git a/tests/FPGAImagePullerTest.cpp b/tests/FPGAImagePullerTest.cpp index 35bee6f41..fee2d6f08 100644 --- a/tests/FPGAImagePullerTest.cpp +++ b/tests/FPGAImagePullerTest.cpp @@ -739,8 +739,8 @@ TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") { } } -// Summation above the card's limit never takes the puller -TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") { +// Summation on the host takes the puller as well +TEST_CASE("FPGAImagePuller_HostSummation", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 2, 1); x.Conversion(); x.Summation(2).CPUSummation(true).Compression(CompressionAlgorithm::BSHUF_LZ4); @@ -757,7 +757,7 @@ TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") service.Start(x, mask, &calib); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); - CHECK(!out.received_packets.empty()); // only the FPGA receiver fills these + CHECK(out.received_packets.empty()); // only the FPGA receiver fills these CHECK(out.status.images_collected == 2); }