// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only // Host-side measurements of FPGAImagePuller without the HLS simulation: a device that hands out already // converted module frames at a chosen rate. Hidden: run by hand, with FPGA_PULLER_* in the environment. #include #include #include #include #include #include #include #include "../acquisition_device/AcquisitionDevice.h" #include "../compression/JFJochDecompress.h" #include "../frame_serialize/CBORStream2Deserializer.h" #include "../image_pusher/ImagePusher.h" #include "../receiver/FPGAImagePuller.h" #include "../receiver/JFJochReceiverService.h" namespace { int64_t Env(const char *name, int64_t def) { const char *v = std::getenv(name); return v ? std::atoll(v) : def; } // Converted frame of a module: sparse Poisson photons plus a few spots std::vector PhotonFrame(uint32_t seed, double photons_per_pixel) { std::vector ret(RAW_MODULE_SIZE); std::mt19937 g(seed); std::poisson_distribution p(photons_per_pixel); for (auto &v: ret) v = static_cast(p(g)); std::uniform_int_distribution where(0, RAW_MODULE_SIZE - 1); for (int i = 0; i < 2000; i++) ret[where(g)] = 500; return ret; } // Like the card, it writes the frame into a buffer it was given and reports the handle; with no buffer // given back in time the frame is lost. The pixels are put in the buffers beforehand, so that the // device itself costs nothing: the rate is limited by what is measured, not by this thread. class ReplayDevice : public AcquisitionDevice { std::vector> storage; std::thread generator; std::atomic stop{false}; const int64_t rate_hz; int64_t nimages = 0, nmodules = 0; std::function present; void HW_RunInternalGenerator(const FrameGeneratorConfig &) override {} void HW_SetSpotFinderParameters(const SpotFinderParameters &) override {} void Run() { const auto t0 = std::chrono::steady_clock::now(); for (int64_t f = 0; (f < nimages) && !stop; f++) { std::this_thread::sleep_until(t0 + std::chrono::nanoseconds(f * 1000000000 / rate_hz)); for (int m = 0; m < nmodules; m++) { if (!present(f, m)) continue; WorkRequest wr{}; if (work_request_queue.Get(wr) == 0) { lost++; continue; } auto &s = GetDeviceOutput(static_cast(wr.handle))->module_statistics; s.frame_number = f; s.module_number = m; s.packet_count = 512; s.timestamp = f * 100; s.exptime = 100; Completion c{}; c.type = Completion::Type::Image; c.handle = wr.handle; work_completion_queue.Put(c); } } Completion c{}; c.type = Completion::Type::End; work_completion_queue.Put(c); } void Start(const DiffractionExperiment &x, uint32_t) override { if (generator.joinable()) generator.join(); nimages = x.GetImageNum(); nmodules = x.GetModulesNum(data_stream); stop = false; lost = 0; Completion c{}; c.type = Completion::Type::Start; work_completion_queue.Put(c); generator = std::thread(&ReplayDevice::Run, this); } public: std::atomic lost{0}; ReplayDevice(uint16_t stream, size_t nbuffers, int64_t in_rate_hz, double photons_per_pixel, std::function in_present = [](int64_t, int) { return true; }) : AcquisitionDevice(stream), rate_hz(in_rate_hz), present(std::move(in_present)) { max_modules = 32; std::vector> patterns; for (int i = 0; i < 16; i++) patterns.push_back(PhotonFrame(i + 1, photons_per_pixel)); for (size_t i = 0; i < nbuffers; i++) { storage.emplace_back(new uint8_t[FPGA_BUFFER_LOCATION_SIZE]); memset(storage.back().get(), 0, FPGA_BUFFER_LOCATION_SIZE); buffer_device.push_back(reinterpret_cast(storage.back().get())); memcpy(buffer_device.back()->pixels, patterns[i % patterns.size()].data(), RAW_MODULE_SIZE * 2); } } ~ReplayDevice() override { stop = true; if (generator.joinable()) generator.join(); } void Cancel() override { stop = true; } uint32_t GetExpectedDescriptorsPerModule() const override { return 1; } }; double CpuSeconds() { rusage r{}; getrusage(RUSAGE_SELF, &r); return static_cast(r.ru_utime.tv_sec + r.ru_stime.tv_sec) + static_cast(r.ru_utime.tv_usec + r.ru_stime.tv_usec) * 1e-6; } DiffractionExperiment Detector(int64_t modules, int64_t images, CompressionAlgorithm algorithm) { DiffractionExperiment x(DetJF(modules)); x.Conversion(); x.PedestalG0Frames(0).NumTriggers(1).UseInternalPacketGenerator(false).ImagesPerTrigger(images) .ImagesPerFile(1000).IncidentEnergy_keV(12.4).DetectorDistance_mm(75) .BeamX_pxl(500).BeamY_pxl(600).Compression(algorithm); x.DataStreams(1); return x; } CompressionAlgorithm Algorithm() { switch (Env("FPGA_PULLER_ALG", 0)) { case 1: return CompressionAlgorithm::BSHUF_ZSTD; case 2: return CompressionAlgorithm::NO_COMPRESSION; default: return CompressionAlgorithm::BSHUF_LZ4; } } class TimingPusher : public ImagePusher { public: std::mutex m; std::vector decode_s, preprocess_s; void StartDataCollection(StartMessage &) override {} bool EndDataCollection(const EndMessage &) override { return true; } bool SendCalibration(const CompressedImage &) override { return true; } bool SendImage(const uint8_t *data, size_t size, int64_t) override { auto d = CBORStream2Deserialize(data, size); std::unique_lock ul(m); decode_s.push_back(d->data_message->compression_time_s.value_or(NAN)); preprocess_s.push_back(d->data_message->preprocessing_time_s.value_or(NAN)); return true; } std::string PrintSetup() const override { return "timing"; } ImagePusherType GetType() const override { return ImagePusherType::Test; } }; float Median(std::vector v) { if (v.empty()) return NAN; std::sort(v.begin(), v.end()); return v[v.size() / 2]; } } // One thread, one image: what assembly and compression cost per image. TEST_CASE("FPGAImagePullerPerf_SingleThread", "[.FPGAImagePullerPerf]") { for (const double photons: {0.1, 1.0}) { for (const int64_t modules: {2, 18}) { for (const auto alg: {CompressionAlgorithm::BSHUF_LZ4, CompressionAlgorithm::BSHUF_ZSTD}) { auto x = Detector(modules, 1, alg); std::vector> frames; for (int64_t m = 0; m < modules; m++) frames.push_back(PhotonFrame(m + 1, photons)); FrameTransformation t(x); std::vector out(MaxCompressedSize(alg, x.GetPixelsNum(), 2)); const int rep = 10; double assemble = 0, compress = 0, decompress = 0; size_t size = 0; std::vector decoded; for (int r = 0; r < rep; r++) { const auto t0 = std::chrono::steady_clock::now(); for (int64_t m = 0; m < modules; m++) t.ProcessModule(frames[m].data(), m, 0); const auto t1 = std::chrono::steady_clock::now(); size = t.CompressImage(out.data(), out.size()); const auto t2 = std::chrono::steady_clock::now(); CompressedImage(out.data(), size, x.GetXPixelsNum(), x.GetYPixelsNum(), x.GetImageMode(), alg) .GetUncompressed(decoded); const auto t3 = std::chrono::steady_clock::now(); assemble += std::chrono::duration(t1 - t0).count(); compress += std::chrono::duration(t2 - t1).count(); decompress += std::chrono::duration(t3 - t2).count(); } std::cout << "single thread: " << modules << " modules, " << photons << " ph/pixel, " << (alg == CompressionAlgorithm::BSHUF_LZ4 ? "BSHUF_LZ4" : "BSHUF_ZSTD") << ": raw " << x.GetPixelsNum() * 2 / 1000000.0 << " MB -> " << size / 1000000.0 << " MB" << "; assemble " << assemble / rep * 1e3 << " ms, compress " << compress / rep * 1e3 << " ms, host decompress " << decompress / rep * 1e3 << " ms" << std::endl; } } } } // The whole puller at a rate, the consumer letting go of each image at once. TEST_CASE("FPGAImagePullerPerf_Rate", "[.FPGAImagePullerPerf]") { const int64_t modules = Env("FPGA_PULLER_MODULES", 2); const int64_t rate = Env("FPGA_PULLER_RATE", 100); const int64_t images = Env("FPGA_PULLER_IMAGES", 500); const int64_t threads = Env("FPGA_PULLER_THREADS", 8); const double photons = static_cast(Env("FPGA_PULLER_MILLIPHOTONS", 100)) * 1e-3; auto x = Detector(modules, images, Algorithm()); AcquisitionDeviceGroup devices; auto device = std::make_unique(0, 512, rate, photons); auto *replay = device.get(); devices.Add(std::move(device)); Logger logger("FPGAImagePullerPerf"); PixelMask mask(x); const double cpu0 = CpuSeconds(); const auto t0 = std::chrono::steady_clock::now(); FPGAImagePuller puller(x, mask, nullptr, devices, logger, threads); int64_t received = 0, last = -1; bool in_order = true, end = false; size_t compressed = 0; while (!end) { const auto m = puller.PollImage(std::chrono::milliseconds(10)); if (!m) continue; if (m->cbor->data_message) { received++; in_order = in_order && (m->cbor->data_message->number > last); last = m->cbor->data_message->number; compressed += m->cbor->data_message->image.GetCompressedSize(); } end = m->cbor->end_message.has_value(); } const double wall = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); puller.Stop(); const double cpu = CpuSeconds() - cpu0; std::cout << "rate: " << modules << " modules, " << rate << " Hz asked, " << threads << " compression threads, " << photons << " ph/pixel: " << received << "/" << images << " images in " << wall << " s = " << received / wall << " Hz, dropped " << puller.GetDroppedImages() << ", lost at the device " << replay->lost.load() << " module frames, in order " << in_order << ", " << compressed / std::max(received, 1) / 1000000.0 << " MB/image, CPU " << cpu / wall << " cores" << std::endl; } // Through Lite, on whatever device the analysis picks: what decoding (or just getting the image across) costs. TEST_CASE("FPGAImagePullerPerf_Lite", "[.FPGAImagePullerPerf]") { const int64_t modules = Env("FPGA_PULLER_MODULES", 2); const int64_t rate = Env("FPGA_PULLER_RATE", 20); const int64_t images = Env("FPGA_PULLER_IMAGES", 100); const int64_t threads = Env("FPGA_PULLER_THREADS", 2); const double photons = static_cast(Env("FPGA_PULLER_MILLIPHOTONS", 100)) * 1e-3; auto x = Detector(modules, images, Algorithm()); AcquisitionDeviceGroup devices; devices.Add(std::make_unique(0, 512, rate, photons)); Logger logger("FPGAImagePullerPerf"); TimingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 2048); service.NumThreads(threads); SpotFindingSettings s = DiffractionExperiment::DefaultDataProcessingSettings(); s.enable = false; s.indexing = false; service.SetSpotFindingSettings(s); service.UseFPGAImagePuller(true); PixelMask mask(x); service.Start(x, mask, nullptr); const auto out = service.Stop(); std::cout << "lite: " << modules << " modules, " << rate << " Hz, " << threads << " threads, algorithm " << Env("FPGA_PULLER_ALG", 0) << ", " << photons << " ph/pixel: collected " << out.status.images_collected << " dropped " << out.status.images_dropped << " sent " << out.status.images_sent << "; ratio " << out.status.compressed_ratio.value_or(0) << "; median compression_time_s (decode on the analysis device) " << Median(pusher.decode_s) * 1e3 << " ms, preprocessing_time_s (rest of getting it into the preprocessed buffer) " << Median(pusher.preprocess_s) * 1e3 << " ms" << std::endl; } namespace { struct Stream { std::vector numbers; std::vector efficiency; bool start_first = false, end = false; }; // Reads until the end message, or until the deadline when `until_image` has not come by then Stream ReadAll(FPGAImagePuller &puller) { Stream ret; bool first = true; const auto t0 = std::chrono::steady_clock::now(); while (!ret.end && (std::chrono::steady_clock::now() - t0 < std::chrono::seconds(120))) { const auto m = puller.PollImage(std::chrono::milliseconds(10)); if (!m) continue; if (first) ret.start_first = m->cbor->start_message.has_value(); first = false; if (m->cbor->data_message) { ret.numbers.push_back(m->cbor->data_message->number); ret.efficiency.push_back(m->cbor->data_message->image_collection_efficiency.value_or(-1)); } ret.end = m->cbor->end_message.has_value(); } return ret; } DiffractionExperiment TwoCards(int64_t images) { auto x = Detector(4, images, CompressionAlgorithm::BSHUF_LZ4); x.DataStreams(2); return x; } } // Replay rather than simulation: the HLS simulation keeps its state in statics, so only one card at a time. TEST_CASE("FPGAImagePuller_Replay_TwoCards_InOrder", "[FPGAImagePuller]") { const auto x = TwoCards(300); REQUIRE(x.GetDataStreamsNum() == 2); AcquisitionDeviceGroup devices; devices.Add(std::make_unique(0, 64, 500, 0.1)); devices.Add(std::make_unique(1, 64, 500, 0.1)); Logger logger("FPGAImagePuller_TwoCards"); PixelMask mask(x); FPGAImagePuller puller(x, mask, nullptr, devices, logger, 8); const auto s = ReadAll(puller); REQUIRE_NOTHROW(puller.Stop()); CHECK(s.start_first); CHECK(s.end); REQUIRE(s.numbers.size() == 300); for (int i = 0; i < 300; i++) CHECK(s.numbers[i] == i); CHECK(puller.GetDroppedImages() == 0); } // A module that dies: nobody stops the run, the images behind it keep coming in order, with the module // taken as missing once the others are far enough ahead. TEST_CASE("FPGAImagePuller_Replay_DeadModule", "[FPGAImagePuller]") { const auto x = Detector(2, 300, CompressionAlgorithm::BSHUF_LZ4); AcquisitionDeviceGroup devices; devices.Add(std::make_unique(0, 64, 200, 0.1, [](int64_t f, int m) { return (m == 0) || (f < 100); })); Logger logger("FPGAImagePuller_DeadModule"); PixelMask mask(x); FPGAImagePuller puller(x, mask, nullptr, devices, logger, 4); const auto s = ReadAll(puller); REQUIRE_NOTHROW(puller.Stop()); CHECK(s.end); REQUIRE(s.numbers.size() == 300); for (int i = 0; i < 300; i++) { CHECK(s.numbers[i] == i); CHECK(s.efficiency[i] == ((i < 100) ? 1.0f : 0.5f)); } CHECK(puller.GetDroppedImages() == 0); } // A whole card dies, the other one goes on: as above, with no module of that card ever reporting again, // until the card is told to stop. TEST_CASE("FPGAImagePuller_Replay_DeadCard", "[FPGAImagePuller]") { const auto x = TwoCards(200); AcquisitionDeviceGroup devices; devices.Add(std::make_unique(0, 64, 200, 0.1)); devices.Add(std::make_unique(1, 64, 200, 0.1, [](int64_t f, int) { return f < 20; })); Logger logger("FPGAImagePuller_DeadCard"); PixelMask mask(x); FPGAImagePuller puller(x, mask, nullptr, devices, logger, 4); const auto s = ReadAll(puller); REQUIRE_NOTHROW(puller.Stop()); CHECK(s.end); REQUIRE(s.numbers.size() == 200); for (int i = 0; i < 200; i++) CHECK(s.numbers[i] == i); CHECK(s.efficiency[150] == 0.5f); CHECK(puller.GetDroppedImages() == 0); } TEST_CASE("FPGAImagePuller_Replay_9M", "[FPGAImagePuller]") { const auto x = Detector(18, 100, CompressionAlgorithm::BSHUF_LZ4); AcquisitionDeviceGroup devices; auto device = std::make_unique(0, 512, 100, 0.1); auto *replay = device.get(); devices.Add(std::move(device)); Logger logger("FPGAImagePuller_FewBuffers"); PixelMask mask(x); FPGAImagePuller puller(x, mask, nullptr, devices, logger, 8); const auto s = ReadAll(puller); REQUIRE_NOTHROW(puller.Stop()); std::cout << "9M replay at 100 Hz: " << s.numbers.size() << " images, dropped " << puller.GetDroppedImages() << ", module frames lost at the device " << replay->lost.load() << std::endl; CHECK(s.end); CHECK(s.numbers.size() + puller.GetDroppedImages() == 100); }