From 4c8af0dbc1de12a8621a8ac78986d4297170c058 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 18 Nov 2026 09:43:57 +0100 Subject: [PATCH] FPGAImagePuller: frames given up on go back to the card while a module waits A module waiting for its next frame released the late frames of images it had given up on only after the wait. With one card more than MissingAfterFrames behind the others, its buffers filled with such frames, the card could not write the frame waited for, and the run hung (seen with two mock cards under load). The release now happens inside the wait. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7 --- receiver/FPGAImagePuller.cpp | 30 +++++++++++++++++------------- receiver/FPGAImagePuller.h | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index f8244c995..3bc8b3aec 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -130,14 +130,23 @@ int64_t FPGAImagePuller::FastestFrame() const { return ret; } -// Polled: a dead card has to be noticed from the other cards' frames -void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) const { - const auto &counters = devices[data_stream].Counters(); - // A frame is complete once the module is two further on, as in JFJochReceiverFPGA - while ((static_cast(counters.GetCurrFrameNumber(module_number)) < frame + 2) - && !counters.IsAcquisitionFinished() && !failed - && (FastestFrame() < frame + 2 + MissingAfterFrames)) +// Polled: a dead card has to be noticed from the other cards' frames. +// A frame given up on can still arrive, late: its buffer goes back to the card, also while waiting, as the card +// may need it to write the frame waited for. +void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame, int64_t &oldest_held) { + auto &device = devices[data_stream]; + const auto &counters = device.Counters(); + while (true) { + while ((oldest_held < frame) + && (oldest_held < static_cast(counters.GetCurrFrameNumber(module_number)))) + device.FrameBufferRelease(oldest_held++, module_number); + // A frame is complete once the module is two further on, as in JFJochReceiverFPGA + if ((static_cast(counters.GetCurrFrameNumber(module_number)) >= frame + 2) + || counters.IsAcquisitionFinished() || failed + || (FastestFrame() >= frame + 2 + MissingAfterFrames)) + return; std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } } // The last frame any card got; waits for all cards, so every module stops at the same image @@ -185,7 +194,7 @@ void FPGAImagePuller::ModuleThread(uint16_t data_stream, uint16_t module_number) int64_t oldest_held = 0; for (int64_t image = 0; image < experiment.GetImageNum(); image++) { - WaitForFrame(data_stream, module_number, image); + WaitForFrame(data_stream, module_number, image, oldest_held); if (failed) break; @@ -195,11 +204,6 @@ void FPGAImagePuller::ModuleThread(uint16_t data_stream, uint16_t module_number) && (image > LastFrame())) break; - // A frame given up on can still arrive, late: its buffer goes back to the card. - while ((oldest_held < image) - && (oldest_held < static_cast(counters.GetCurrFrameNumber(module_number)))) - device.FrameBufferRelease(oldest_held++, module_number); - const bool collected = counters.IsAnyPacketCollected(image, module_number); if (const auto slot = GetSlot(image)) { if (collected) { diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index e02f272d1..422f60c87 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -72,7 +72,7 @@ private: void CompressThread(); void FinalizeMeasurement(); - void WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) const; + void WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame, int64_t &oldest_held); int64_t FastestFrame() const; int64_t LastFrame() const; std::shared_ptr GetSlot(int64_t image_number);