// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only // The same frames through JFJochReceiverFPGA and through FPGAImagePuller + JFJochReceiverLite, on the mock of the // card. The pixels must be the same; the analysis differs by design, and its numbers are printed. #include #include #include #include #include #include #include #include "../compression/JFJochDecompress.h" #include "../frame_serialize/CBORStream2Deserializer.h" #include "../image_pusher/HDF5FilePusher.h" #include "../image_pusher/ImagePusher.h" #include "../writer/HDF5Objects.h" #include "../writer/StreamWriter.h" #include "../receiver/FPGAImagePuller.h" #include "../receiver/JFJochReceiverRugnuxRotation.h" #include "../receiver/JFJochReceiverService.h" #include "../rugnux/RugnuxDefaults.h" #include "MockCardOutput.h" using namespace std::chrono_literals; namespace { struct Received { DataMessage message; std::vector compressed; }; class CollectingPusher : public ImagePusher { public: std::mutex m; std::map images; std::vector arrival; std::vector sequence; EndMessage end; std::chrono::milliseconds delay{0}; void StartDataCollection(StartMessage &) override { std::unique_lock ul(m); sequence.emplace_back("start"); } bool EndDataCollection(const EndMessage &message) override { std::unique_lock ul(m); sequence.emplace_back("end"); end = message; return true; } bool SendCalibration(const CompressedImage &c) override { std::unique_lock ul(m); sequence.emplace_back("calibration " + c.GetChannel()); return true; } bool SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override { std::this_thread::sleep_for(delay); auto d = CBORStream2Deserialize(image_data, image_size); Received r; r.message = d->data_message.value(); const uint8_t *p = r.message.image.GetCompressed(); r.compressed.assign(p, p + r.message.image.GetCompressedSize()); r.message.image = CompressedImage(r.compressed.data(), r.compressed.size(), r.message.image.GetWidth(), r.message.image.GetHeight(), r.message.image.GetMode(), r.message.image.GetCompressionAlgorithm()); std::unique_lock ul(m); sequence.emplace_back("image"); arrival.push_back(image_number); images.emplace(image_number, std::move(r)); return true; } std::string PrintSetup() const override { return "collecting"; } ImagePusherType GetType() const override { return ImagePusherType::Test; } }; float Pedestal(int module, int pixel) { return 3000 + 10 * module + pixel % 50; } JFCalibration Calibration(const DiffractionExperiment &x) { JFCalibration ret(x); std::vector g0(RAW_MODULE_SIZE), g1(RAW_MODULE_SIZE), g2(RAW_MODULE_SIZE); for (int s = 0; s < x.GetStorageCellNumber(); s++) for (int m = 0; m < x.GetModulesNum(); m++) { for (int i = 0; i < RAW_MODULE_SIZE; i++) { g0[i] = Pedestal(m, i); g1[i] = 15000 + 10 * m + i % 50; g2[i] = 14000 + 10 * m + i % 50; } ret.Pedestal(m, 0, s).LoadPedestal(g0); ret.Pedestal(m, 1, s).LoadPedestal(g1); ret.Pedestal(m, 2, s).LoadPedestal(g2); } return ret; } // Raw detector frame of a module: background of a few photons, some spots, one saturated and one bad // pixel, different for each frame. std::vector Frame(int64_t frame, int module, bool jungfrau) { std::vector ret(RAW_MODULE_SIZE); std::mt19937 g(static_cast(frame * 100 + module)); std::uniform_int_distribution background(0, jungfrau ? 300 : 3); for (int i = 0; i < RAW_MODULE_SIZE; i++) ret[i] = (jungfrau ? static_cast(Pedestal(module, i)) : 0) + background(g); std::uniform_int_distribution where_x(20, RAW_MODULE_COLS - 20), where_y(20, RAW_MODULE_LINES - 20); for (int s = 0; s < 40; s++) { const int x0 = where_x(g), y0 = where_y(g); for (int y = y0; y < y0 + 3; y++) for (int x = x0; x < x0 + 3; x++) ret[y * RAW_MODULE_COLS + x] = jungfrau ? 12000 : 2000; } ret[50 * RAW_MODULE_COLS + 500] = jungfrau ? 0xC000 : 0xFFFE; ret[60 * RAW_MODULE_COLS + 600] = 0xFFFF; return ret; } SpotFindingSettings Spots() { SpotFindingSettings s = DiffractionExperiment::DefaultDataProcessingSettings(); s.signal_to_noise_threshold = 3; s.photon_count_threshold = 2; s.min_pix_per_spot = 1; s.indexing = false; return s; } // What the card writes for the first frames: their conversion std::vector CardImages(const DiffractionExperiment &x, int64_t distinct_frames) { const bool jungfrau = x.GetDetectorType() == DetectorType::JUNGFRAU; PixelMask mask(x); const JFCalibration calib = Calibration(x); std::vector ret; for (int64_t f = 0; f < distinct_frames; f++) { std::vector raw; for (int m = 0; m < x.GetModulesNum(); m++) { const auto frame = Frame(f, m, jungfrau); raw.insert(raw.end(), frame.begin(), frame.end()); } ret.push_back(ConvertedImage(x, calib, mask, raw)); } return ret; } // The card writing zeros AcquisitionDeviceGroup Devices(const DiffractionExperiment &x) { return MockDevices(x, {}); } struct PathResult { std::map images; std::vector arrival; std::vector sequence; EndMessage end; JFJochReceiverOutput output; }; PathResult Run(const DiffractionExperiment &x, bool puller, int64_t distinct_frames, bool rugnux = false) { auto devices = MockDevices(x, CardImages(x, distinct_frames)); Logger logger("FPGAImagePullerTest"); CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4); service.SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(puller); service.RugnuxLive(rugnux, nullptr, 2, 30); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); PathResult ret; ret.output = service.Stop(); ret.images = std::move(pusher.images); ret.arrival = std::move(pusher.arrival); ret.sequence = std::move(pusher.sequence); ret.end = pusher.end; return ret; } std::vector Pixels(const CompressedImage &image) { std::vector ret; image.GetUncompressed(ret); return ret; } template void Row(const std::string &name, const std::optional &a, const std::optional &b) { if (a == b) return; std::cout << " differs " << name << ": FPGA "; if (a) std::cout << *a; else std::cout << "-"; std::cout << " puller "; if (b) std::cout << *b; else std::cout << "-"; std::cout << std::endl; } // Must be the same: asserted. Analysis: printed for the first image and summed over all. void Compare(const std::string &name, const PathResult &a, const PathResult &b) { std::cout << "=== " << name << std::endl; REQUIRE(a.images.size() == b.images.size()); REQUIRE(!a.images.empty()); int64_t spots_a = 0, spots_b = 0, sat_a = 0, sat_b = 0, err_a = 0, err_b = 0, strong_a = 0, strong_b = 0; double dot = 0, norm_a = 0, norm_b = 0, ratio_sum = 0; size_t ratio_n = 0, bins_a = 0, bins_b = 0; for (const auto &[n, ia]: a.images) { INFO("image " << n); REQUIRE(b.images.contains(n)); const auto &ib = b.images.at(n); const auto &ma = ia.message; const auto &mb = ib.message; CHECK(ma.image.GetMode() == mb.image.GetMode()); CHECK(ma.image.GetCompressionAlgorithm() == mb.image.GetCompressionAlgorithm()); CHECK(ma.image.GetWidth() == mb.image.GetWidth()); CHECK(ma.image.GetHeight() == mb.image.GetHeight()); CHECK(Pixels(ma.image) == Pixels(mb.image)); CHECK(ia.compressed.size() == ib.compressed.size()); // From the source CHECK(ma.number == mb.number); CHECK(ma.original_number == mb.original_number); CHECK(ma.packets_expected == mb.packets_expected); CHECK(ma.packets_received == mb.packets_received); CHECK(ma.image_collection_efficiency == mb.image_collection_efficiency); CHECK(ma.timestamp == mb.timestamp); CHECK(ma.timestamp_base == mb.timestamp_base); CHECK(ma.exptime == mb.exptime); CHECK(ma.exptime_base == mb.exptime_base); CHECK(ma.jf_info == mb.jf_info); CHECK(ma.storage_cell == mb.storage_cell); CHECK(ma.xfel_pulse_id == mb.xfel_pulse_id); CHECK(ma.xfel_event_code == mb.xfel_event_code); CHECK(ma.pixel_sum == mb.pixel_sum); CHECK(ma.user_data == mb.user_data); CHECK(ma.run_number == mb.run_number); CHECK(ma.run_name == mb.run_name); if (n == a.images.begin()->first) { Row("saturated_pixel_count", ma.saturated_pixel_count, mb.saturated_pixel_count); Row("error_pixel_count", ma.error_pixel_count, mb.error_pixel_count); Row("min_viable_pixel_value", ma.min_viable_pixel_value, mb.min_viable_pixel_value); Row("max_viable_pixel_value", ma.max_viable_pixel_value, mb.max_viable_pixel_value); Row("strong_pixel_count", ma.strong_pixel_count, mb.strong_pixel_count); Row("spot_count", ma.spot_count, mb.spot_count); Row("bkg_estimate", ma.bkg_estimate, mb.bkg_estimate); Row("ice_ring_score", ma.ice_ring_score, mb.ice_ring_score); Row("receiver_aq_dev_delay", ma.receiver_aq_dev_delay, mb.receiver_aq_dev_delay); Row("receiver_buf_in_preparation", ma.receiver_buf_in_preparation, mb.receiver_buf_in_preparation); Row("receiver_buf_in_sending", ma.receiver_buf_in_sending, mb.receiver_buf_in_sending); std::cout << " compression_ratio FPGA " << ma.compression_ratio.value_or(0) << " puller " << mb.compression_ratio.value_or(0) << std::endl; } spots_a += ma.spots.size(); spots_b += mb.spots.size(); sat_a += ma.saturated_pixel_count.value_or(0); sat_b += mb.saturated_pixel_count.value_or(0); err_a += ma.error_pixel_count.value_or(0); err_b += mb.error_pixel_count.value_or(0); strong_a += ma.strong_pixel_count.value_or(0); strong_b += mb.strong_pixel_count.value_or(0); bins_a = ma.az_int_profile.size(); bins_b = mb.az_int_profile.size(); for (size_t i = 0; i < std::min(bins_a, bins_b); i++) { if (std::isfinite(ma.az_int_profile[i]) && std::isfinite(mb.az_int_profile[i])) { dot += ma.az_int_profile[i] * mb.az_int_profile[i]; norm_a += ma.az_int_profile[i] * ma.az_int_profile[i]; norm_b += mb.az_int_profile[i] * mb.az_int_profile[i]; if (ma.az_int_profile[i] != 0) { ratio_sum += mb.az_int_profile[i] / ma.az_int_profile[i]; ratio_n++; } } } } std::cout << " over " << a.images.size() << " images:" << " spots FPGA " << spots_a << " puller " << spots_b << " | strong " << strong_a << " / " << strong_b << " | saturated " << sat_a << " / " << sat_b << " | error " << err_a << " / " << err_b << " | az bins " << bins_a << " / " << bins_b << " cosine " << (norm_a * norm_b > 0 ? dot / std::sqrt(norm_a * norm_b) : 0) << " mean ratio puller/FPGA " << (ratio_n > 0 ? ratio_sum / ratio_n : 0) << std::endl; const auto &sa = a.output.status; const auto &sb = b.output.status; std::cout << " status: efficiency " << a.output.efficiency << " / " << b.output.efficiency << " | collected " << sa.images_collected << " / " << sb.images_collected << " | sent " << sa.images_sent << " / " << sb.images_sent << " | dropped " << sa.images_dropped << " / " << sb.images_dropped << " | max_image_sent " << sa.max_image_number_sent << " / " << sb.max_image_number_sent << " | compressed_size " << sa.compressed_size << " / " << sb.compressed_size << " | ratio " << sa.compressed_ratio.value_or(0) << " / " << sb.compressed_ratio.value_or(0) << " | max_delay " << sa.max_receive_delay.value_or(0) << " / " << sb.max_receive_delay.value_or(0) << std::endl; CHECK(a.output.efficiency == b.output.efficiency); CHECK(sa.images_collected == sb.images_collected); CHECK(sa.images_sent == sb.images_sent); CHECK(sb.images_dropped == 0); CHECK(sa.max_image_number_sent == sb.max_image_number_sent); CHECK(sa.compressed_size == sb.compressed_size); CHECK(a.end.images_collected_count == b.end.images_collected_count); CHECK(a.end.images_sent_to_write_count == b.end.images_sent_to_write_count); CHECK(a.end.max_image_number == b.end.max_image_number); CHECK(a.end.adu_histogram == b.end.adu_histogram); // Calibration goes to the writer only from the FPGA receiver CHECK(a.sequence.front() == "start"); CHECK(b.sequence.front() == "start"); CHECK(a.sequence.back() == "end"); CHECK(b.sequence.back() == "end"); auto sequence_a = a.sequence; std::erase_if(sequence_a, [](const std::string &s) { return s.starts_with("calibration"); }); CHECK(sequence_a == b.sequence); int64_t displacement = 0; for (size_t i = 0; i < b.arrival.size(); i++) displacement = std::max(displacement, std::abs(b.arrival[i] - static_cast(i))); std::cout << " max displacement at the pusher (Lite's threads): " << displacement << " images" << std::endl; } void CompareBoth(const std::string &name, const DiffractionExperiment &x, int64_t distinct_frames) { Compare(name, Run(x, false, distinct_frames), Run(x, true, distinct_frames)); } DiffractionExperiment Base(const DetectorSetup &detector, int64_t images, int64_t distinct_frames) { DiffractionExperiment x(detector); x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(true).InternalPacketGeneratorImages(distinct_frames) .ImagesPerTrigger(images).ImagesPerFile(10).IncidentEnergy_keV(12.4) .DetectorDistance_mm(75).BeamX_pxl(500).BeamY_pxl(600); return x; } } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 6, 3); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); CompareBoth("JF conversion, int16, BSHUF_LZ4", x, 3); } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_ApplyMask_ROI", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 4, 2); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_ZSTD).ApplyPixelMask(true); x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}}); const auto a = Run(x, false, 2); const auto b = Run(x, true, 2); Compare("JF conversion, applied mask, ROI, BSHUF_ZSTD", a, b); const auto &ra = a.images.begin()->second.message.roi.at("box"); const auto &rb = b.images.begin()->second.message.roi.at("box"); std::cout << " ROI sum " << ra.sum << " / " << rb.sum << " pixels " << ra.pixels << " / " << rb.pixels << " max " << ra.max_count << " / " << rb.max_count << std::endl; } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Unsigned", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 4, 2); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false); CompareBoth("JF conversion, uint16, BSHUF_LZ4", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_Summation", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 3, 2); x.Conversion(); x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4); CompareBoth("JF conversion, card sums 2 frames, int32", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Conversion_NoGeometry", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 4, 2); x.Conversion(); x.GeometryTransformation(false).Compression(CompressionAlgorithm::BSHUF_LZ4); CompareBoth("JF conversion, no geometry transform", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 4, 2); x.Raw(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); CompareBoth("JF raw, BSHUF_LZ4", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_JF_Raw_NoCompression", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 3, 1); x.Raw(); x.Compression(CompressionAlgorithm::NO_COMPRESSION); CompareBoth("JF raw, no compression", x, 1); } TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Conversion", "[FPGAImagePuller]") { auto x = Base(DetEIGER(2), 4, 2); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); CompareBoth("EIGER assembled, 16 bit, BSHUF_LZ4", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_Raw", "[FPGAImagePuller]") { auto x = Base(DetEIGER(2), 4, 2); x.Raw(); x.Compression(CompressionAlgorithm::BSHUF_ZSTD); CompareBoth("EIGER raw layout, 16 bit, BSHUF_ZSTD", x, 2); } TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") { auto x = Base(DetEIGER(2), 3, 1); x.Compression(CompressionAlgorithm::BSHUF_LZ4); x.EigerBitDepth(32); // The 16-bit raw frames do not map onto a 32-bit stream: the card writes zeros auto devices = Devices(x); Logger logger("FPGAImagePuller_EIGER32"); for (const bool puller: {false, true}) { CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(puller); PixelMask mask(x); service.Start(x, mask, nullptr); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); CHECK(pusher.images.size() == 3); CHECK(pusher.images.at(0).message.image.GetMode() == x.GetImageMode()); } } // Nobody takes the images: the slots run out and later images are dropped whole, without any waiting. TEST_CASE("FPGAImagePuller_SlowConsumer_Puller", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 20, 1); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); auto devices = Devices(x); Logger logger("FPGAImagePuller_SlowConsumer"); PixelMask mask(x); const JFCalibration calib = Calibration(x); FPGAImagePuller puller(x, mask, &calib, devices, logger, 2, 4); // Not read until the card is through, however long the simulation takes while (!devices[0].Counters().IsAcquisitionFinished()) std::this_thread::sleep_for(10ms); std::vector numbers; bool end = false; const auto t0 = std::chrono::steady_clock::now(); while (!end && std::chrono::steady_clock::now() - t0 < 60s) { const auto m = puller.PollImage(10ms); if (!m) continue; if (m->cbor->data_message) numbers.push_back(m->cbor->data_message->number); end = m->cbor->end_message.has_value(); } CHECK(end); puller.Stop(); std::cout << "slow consumer, puller alone: " << numbers.size() << " images, " << puller.GetDroppedImages() << " dropped" << std::endl; CHECK(numbers.size() + puller.GetDroppedImages() == 20); CHECK(puller.GetDroppedImages() > 0); for (size_t i = 1; i < numbers.size(); i++) CHECK(numbers[i] > numbers[i - 1]); } // The same behind Lite: a writer that takes its time keeps Lite's only thread, and the drops reach the statistics. TEST_CASE("FPGAImagePuller_SlowConsumer_Receiver", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 24, 1); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); auto devices = Devices(x); Logger logger("FPGAImagePuller_SlowReceiver"); CollectingPusher pusher; pusher.delay = 3s; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(1).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true, 2); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); const auto out = service.Stop(); std::cout << "slow consumer, Lite: collected " << out.status.images_collected << " dropped " << out.status.images_dropped << " efficiency " << out.efficiency << std::endl; CHECK(out.status.images_dropped > 0); CHECK(out.status.images_collected + out.status.images_dropped == 24); CHECK(out.efficiency == Catch::Approx(static_cast(out.status.images_collected) / 24.0f)); CHECK(pusher.sequence.back() == "end"); } // A module frame missing in the middle, and a module that dies: the images keep coming with no help // from the end of the run, and the missing module is filled, as in JFJochReceiverFPGA. TEST_CASE("FPGAImagePuller_MissingModule", "[FPGAImagePuller]") { constexpr int64_t images = 62; constexpr int64_t dies_at = 3; auto x = Base(DetJF(2), images, 1); x.UseInternalPacketGenerator(false).Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); auto device = std::make_unique(0, 64); SetOutputs(*device, x, 0, CardImages(x, 1)); for (int64_t f = 0; f < images; f++) for (int m = 0; m < 2; m++) { if ((m == 1) && (f == 1 || f >= dies_at)) continue; device->CreateJFPackets(x, f + 1, 1, m, nullptr); } AcquisitionDeviceGroup devices; devices.Add(std::move(device)); Logger logger("FPGAImagePuller_MissingModule"); PixelMask mask(x); const JFCalibration calib = Calibration(x); FPGAImagePuller puller(x, mask, &calib, devices, logger, 2); // No final packet and no cancel yet: images must come through anyway, up to the lag behind the card std::vector numbers; std::vector efficiency; bool stop_requested = false, end = false; size_t images_before_cancel = 0; const auto t0 = std::chrono::steady_clock::now(); auto last = t0; while (!end && std::chrono::steady_clock::now() - t0 < 900s) { const auto m = puller.PollImage(10ms); if (!m) { if (!stop_requested && std::chrono::steady_clock::now() - last > 5s) { stop_requested = true; std::cout << "dead module, before cancel: fastest " << devices[0].Counters().GetFastestFrameNumber() << " module 0 at " << devices[0].Counters().GetCurrFrameNumber(0) << " module 1 at " << devices[0].Counters().GetCurrFrameNumber(1) << " images so far " << numbers.size() << std::endl; puller.Cancel(); } continue; } last = std::chrono::steady_clock::now(); if (m->cbor->data_message) { const auto &d = *m->cbor->data_message; numbers.push_back(d.number); efficiency.push_back(d.image_collection_efficiency.value_or(-1)); if (!stop_requested) images_before_cancel = numbers.size(); if (d.number == 1) { // Module 1 absent from this frame: its pixels are the fill value std::vector tmp; const auto *p = reinterpret_cast(d.image.GetUncompressedPtr(tmp)); CHECK(p[x.GetPixel0OfModuleConv(1)] == x.GetImageFillValue()); CHECK(p[x.GetPixel0OfModuleConv(0)] != x.GetImageFillValue()); } } end = m->cbor->end_message.has_value(); } CHECK(end); REQUIRE_NOTHROW(puller.Stop()); std::cout << "dead module: " << numbers.size() << " images, before any cancel: " << images_before_cancel << std::endl; // Image i is let go once the card reports frame i + 2 + MissingAfterFrames of the other module CHECK(images_before_cancel >= 4); REQUIRE(numbers.size() >= 4); for (size_t i = 0; i < numbers.size(); i++) CHECK(numbers[i] == static_cast(i)); CHECK(efficiency[0] == 1.0f); CHECK(efficiency[1] == 0.5f); // Frame 2 of module 1 is sent, but the card holds a module's last frames until newer ones push them // out: given up on, it comes with the cancel. Its buffer went back. CHECK(efficiency[3] == 0.5f); CHECK(efficiency.back() == 0.5f); } TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") { for (const bool silent: {false, true}) { auto x = Base(DetJF(2), 6, 1); x.UseInternalPacketGenerator(false).Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); auto device = std::make_unique(0, 64); SetOutputs(*device, x, 0, CardImages(x, 3)); for (int64_t f = 0; f < 3; f++) for (int m = 0; m < 2; m++) device->CreateJFPackets(x, f + 1, 1, m, nullptr); AcquisitionDeviceGroup devices; devices.Add(std::move(device)); Logger logger("FPGAImagePuller_Cancel"); CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); std::this_thread::sleep_for(2s); service.Cancel(silent); const auto out = service.Stop(); std::cout << "cancel silent=" << silent << ": cancelled " << out.status.cancelled << " collected " << out.status.images_collected << " sent " << out.status.images_sent << " efficiency " << out.efficiency << std::endl; CHECK(out.status.cancelled == !silent); CHECK(pusher.sequence.front() == "start"); CHECK(pusher.sequence.back() == "end"); CHECK(out.status.images_collected <= 3); if (silent) { CHECK(out.status.images_collected == 3); CHECK(out.efficiency == Catch::Approx(0.5f)); } } } // Default stays the FPGA receiver; pedestal and summation above the card's limit never take the puller TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 2, 1); x.Conversion(); x.Summation(2).CPUSummation(true).Compression(CompressionAlgorithm::BSHUF_LZ4); REQUIRE(x.IsCPUSummation()); auto devices = Devices(x); Logger logger("FPGAImagePuller_HostSummation"); CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); CHECK(!out.received_packets.empty()); // only the FPGA receiver fills these CHECK(out.status.images_collected == 2); } // JFJochReceiverRugnuxRotation takes the same stream, with its own forwarding thread. TEST_CASE("FPGAImagePuller_RugnuxReceiver", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 6, 3); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt)); x.FilePrefix("fpga_puller_rugnux"); REQUIRE(RugnuxRotationData(x)); auto devices = MockDevices(x, CardImages(x, 3)); Logger logger("FPGAImagePuller_Rugnux"); CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true); service.RugnuxLive(true, nullptr, 2, 30); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); const auto out = service.Stop(); std::cout << "rugnux receiver: efficiency " << out.efficiency << " sent " << out.status.images_sent << " warning '" << out.analysis_warning << "'" << std::endl; CHECK(out.efficiency == 1.0f); CHECK(out.status.images_sent == 6); CHECK(pusher.images.size() == 6); CHECK(pusher.sequence.front() == "start"); CHECK(pusher.sequence.back() == "end"); CHECK(pusher.end.adu_histogram.contains("module0")); CHECK_FALSE(pusher.sequence[1].starts_with("calibration")); // the writer gets calibration only from the FPGA receiver const auto a = Pixels(pusher.images.at(0).message.image); const auto b = Pixels(pusher.images.at(3).message.image); CHECK(a == b); // the card output repeats after the third frame } // In an unsigned image the card gives saturated and error pixels one value: the counts are the card's, on every receiver. TEST_CASE("FPGAImagePuller_SaturatedAndErrorCountsFromTheCard", "[FPGAImagePuller]") { auto x = Base(DetJF(2), 4, 2); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(16).PixelSigned(false).FilePrefix("fpga_puller_counts"); const auto card = Run(x, false, 2); PathResult puller; SECTION("JFJochReceiverLite") { puller = Run(x, true, 2); } SECTION("JFJochReceiverRugnuxStills") { puller = Run(x, true, 2, true); } SECTION("JFJochReceiverRugnuxRotation") { auto rotation = x; rotation.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt)); puller = Run(rotation, true, 2, true); } REQUIRE(card.images.size() == 4); REQUIRE(puller.images.size() == 4); int64_t saturated = 0, error = 0; for (const auto &[n, image]: puller.images) { INFO("image " << n); CHECK(image.message.saturated_pixel_count == card.images.at(n).message.saturated_pixel_count); CHECK(image.message.error_pixel_count == card.images.at(n).message.error_pixel_count); saturated += image.message.saturated_pixel_count.value_or(0); error += image.message.error_pixel_count.value_or(0); } CHECK(saturated == 8); CHECK(error == 8); } // Rotation forwards the images unanalysed, so nothing would replace what the card's ROI map, not loaded for this run, gave. TEST_CASE("FPGAImagePuller_NoStaleROI", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 4, 1); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4); x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt)); x.FilePrefix("fpga_puller_roi"); x.ROI().SetROI(ROIDefinition{.boxes = {ROIBox("box", 380, 450, 280, 320)}, .circles = {}, .azimuthal = {}}); auto devices = Devices(x); Logger logger("FPGAImagePuller_NoStaleROI"); CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true); service.RugnuxLive(true, nullptr, 2, 30); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); service.Stop(); REQUIRE(pusher.images.size() == 4); for (const auto &[n, image]: pusher.images) CHECK(image.message.roi.empty()); } // An image is many frames: efficiency and progress count images. TEST_CASE("FPGAImagePuller_RugnuxReceiver_Summation", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 3, 2); x.Conversion(); x.Summation(2).Compression(CompressionAlgorithm::BSHUF_LZ4); x.Goniometer(GoniometerAxis("omega", 0, 0.1, Coord(-1, 0, 0), std::nullopt)); x.FilePrefix("fpga_puller_rugnux_sum"); REQUIRE(x.GetImageNum() == 3); REQUIRE(x.GetFrameNum() == 6); auto devices = Devices(x); Logger logger("FPGAImagePuller_RugnuxSum"); CollectingPusher pusher; ImageBuffer buffer(256 * 1024 * 1024); JFJochReceiverCurrentStatus status; JFJochReceiverPlots plots; RugnuxLiveStatus live; PixelMask mask(x); const JFCalibration calib = Calibration(x); FPGAImagePuller puller(x, mask, &calib, devices, logger, 2); JFJochReceiverRugnuxRotation receiver(x, mask, puller, pusher, logger, 2, 2, nullptr, 30, live, Spots(), status, plots, buffer); receiver.StopReceiver(); puller.Stop(); CHECK(receiver.GetEfficiency() == 1.0f); CHECK(receiver.GetProgress() == 1.0f); CHECK(pusher.images.size() == 3); } // The real writer: the master file states what the card did, not what a DECTRIS stream implies. TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") { RegisterHDF5Filter(); auto x = Base(DetJF(2), 5, 2); x.Conversion(); x.Compression(CompressionAlgorithm::BSHUF_LZ4).ApplyPixelMask(true).ImagesPerFile(2) .FilePrefix("fpga_image_puller_writer").SetFileWriterFormat(FileWriterFormat::NXmxVDS) .OverwriteExistingFiles(true).SaveCalibration(true); auto devices = Devices(x); Logger logger("FPGAImagePuller_Writer"); HDF5FilePusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); service.UseFPGAImagePuller(true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); const auto out = service.Stop(); CHECK(out.efficiency == 1.0f); CHECK(out.writer_err.empty()); CHECK(out.status.images_sent == 5); HDF5ReadOnlyFile master("fpga_image_puller_writer_master.h5"); CHECK(master.GetBool("/entry/instrument/detector/pixel_mask_applied")); HDF5DataSet dataset(master, "/entry/data/data"); HDF5DataSpace space(dataset); CHECK(space.GetDimensions()[0] == 5); CHECK(space.GetDimensions()[1] == x.GetYPixelsNum()); }