FPGAImagePuller: one way through - no host stage discards an image

Images now always wait for room in the queue to the receiver (128 compressed
images, Queue), holding the cards' buffers, so the cards wait for host
buffers and frames are lost only at the card, as the card's generator and a
detector do. Removed what served only particular tests: the not-blocking
path that dropped an image at a full queue (and the dropped count, which
JFJochReceiverStream now reports as the base's 0), the host pacing of the
source (frame_rate_hz), the queue-wait time, the queue size argument, the
cross-card give-up (FastestFrame) with its late-frame bookkeeping
(given_up, ReleaseLateFrames), and the dummy-module padding of the cards
(CardExperiment, modules_per_card). The device itself gives up on a module
lagging behind its card past ThresholdFramesLost and returns a frame that
comes later; MissingAfterFrames stays beyond that, as the header says.

FPGAImagePullerSettings is reduced to the one bool that sends every
collection through the puller (tests comparing it with JFJochReceiverFPGA).
IsBlocking() stays, now always true: JFJochReceiverRugnuxStills' hold waits
on it, and that wait is changed on its own branch.

The tests and jfjoch_fpga_stills_test are adapted in the commits that follow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-18 19:53:42 +01:00
1 parent b7771ca1d5
commit 83fa6daa2f
6 files changed
+65 -189

No files matched your search

-2
View File
@@ -76,8 +76,6 @@ public:
virtual void Disconnect() = 0;
// Images the puller had no room for, which the receiver therefore never gets.
virtual size_t GetDroppedImages() const { return 0; }
// The source waits for the receiver rather than lose an image: the receiver waits for room instead of dropping it
virtual bool IsBlocking() const { return false; }
};
+44 -121
View File
@@ -9,16 +9,11 @@
FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask,
const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices,
Logger &in_logger, int64_t image_threads, size_t in_queue,
bool in_blocking, double in_frame_rate_hz, int64_t modules_per_card)
: ImagePuller(in_queue + 2), // the images waiting, the start and the end message
Logger &in_logger, int64_t image_threads)
: ImagePuller(Queue + 2), // the images waiting, the start and the end message
experiment(in_experiment),
card_experiment(CardExperiment(in_experiment, modules_per_card)),
logger(in_logger),
devices(in_devices),
queue(in_queue),
blocking(in_blocking),
frame_rate_hz(in_frame_rate_hz),
host_summation(in_experiment.GetHostSummation()) {
const int ndatastreams = experiment.GetDataStreamsNum();
if (devices.size() < ndatastreams)
@@ -28,17 +23,12 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con
for (int m = 0; m < experiment.GetModulesNum(); m++)
adu_histogram.emplace_back(std::make_unique<ADUHistogram>());
given_up.resize(ndatastreams);
for (int d = 0; d < ndatastreams; d++) {
given_up[d].resize(card_experiment.GetModulesNum(d));
devices[d].PrepareAction(card_experiment);
// Of the modules of the image, not the dummies
expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() * host_summation
* experiment.GetModulesNum(d) / card_experiment.GetModulesNum(d);
devices[d].PrepareAction(experiment);
expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() * host_summation;
}
// The calibration and the mask are of the modules of the image
if ((experiment.GetDetectorType() != DetectorType::JUNGFRAU) || (modules_per_card > 0))
if (experiment.GetDetectorType() != DetectorType::JUNGFRAU)
calibration = nullptr;
// The receiver's own start message is built from its experiment; this one only says the run has begun.
@@ -47,7 +37,6 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con
outside_fifo.Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(start),
.received = std::chrono::steady_clock::now()});
start_time = std::chrono::steady_clock::now();
for (int i = 0; i < image_threads; i++)
image_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::ImageThread, this));
@@ -78,16 +67,6 @@ FPGAImagePuller::~FPGAImagePuller() {
}
}
DiffractionExperiment FPGAImagePuller::CardExperiment(const DiffractionExperiment &x, int64_t modules_per_card) {
if (modules_per_card <= 0)
return x;
DiffractionExperiment ret(x);
const DetectorSettings settings = x.GetDetectorSettings();
ret.Detector(DetJF(static_cast<int32_t>(modules_per_card * x.GetDataStreamsNum())));
ret.ImportDetectorSettings(settings);
return ret;
}
void FPGAImagePuller::SetError(std::exception_ptr e) {
std::unique_lock ul(error_mutex);
if (!error)
@@ -95,17 +74,17 @@ void FPGAImagePuller::SetError(std::exception_ptr e) {
failed = true;
}
void FPGAImagePuller::Cancel() {
// An image waiting for room in the queue is dropped from here on: after a cancel the receiver takes no more images
cancelled = true;
CancelCards();
}
void FPGAImagePuller::CancelCards() {
void FPGAImagePuller::StopCards() {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
devices[d].Cancel();
}
void FPGAImagePuller::Cancel() {
cancelled = true;
StopCards();
}
// Cancels first: a receiver that stopped on an error takes no more images, which would hold the threads otherwise
void FPGAImagePuller::Stop() {
Cancel();
std::call_once(finish_once, [this] { finish_future.get(); });
@@ -121,14 +100,11 @@ void FPGAImagePuller::AcquireThread(uint16_t data_stream, const PixelMask &mask,
// Only what the card needs for the pixels: its analysis results are not read.
if (calibration != nullptr)
device.InitializeCalibration(experiment, *calibration);
if (card_experiment.GetModulesNum() == experiment.GetModulesNum())
device.InitializePixelMask(experiment, mask);
else
device.InitializeEmptyPixelMask(card_experiment);
device.StartAction(card_experiment);
device.InitializePixelMask(experiment, mask);
device.StartAction(experiment);
} catch (...) {
SetError(std::current_exception());
CancelCards(); // the image threads waiting on the other cards have to come out as well
StopCards(); // the image threads waiting on the other cards have to come out as well
ready.count_down();
return;
}
@@ -138,49 +114,20 @@ void FPGAImagePuller::AcquireThread(uint16_t data_stream, const PixelMask &mask,
device.WaitForActionComplete();
} catch (...) {
SetError(std::current_exception());
CancelCards();
StopCards();
}
}
int64_t FPGAImagePuller::FastestFrame() const {
int64_t ret = 0;
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
ret = std::max(ret, static_cast<int64_t>(devices[d].Counters().GetFastestFrameNumber()));
return ret;
}
// The buffers of frames given up on that have come since go back to the card, which may need them for the frames
// waited for
void FPGAImagePuller::ReleaseLateFrames(uint16_t data_stream, uint16_t module_number) {
std::unique_lock ul(given_up_mutex);
auto &frames = given_up[data_stream][module_number];
std::erase_if(frames, [&](int64_t f) {
if (!devices[data_stream].Counters().IsAnyPacketCollected(f, module_number))
return false;
devices[data_stream].FrameBufferRelease(f, module_number);
return true;
});
}
// Polled: a dead card has to be noticed from the other cards' frames
// Polled: a dead card has to be noticed from the other cards' frames. A frame given up on that comes later is below
// the device's slowest frame then (MissingAfterFrames), and the device returns its buffer itself.
bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) {
const auto &counters = devices[data_stream].Counters();
while (true) {
ReleaseLateFrames(data_stream, module_number);
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
if ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) >= frame + 2)
|| counters.IsAcquisitionFinished() || failed)
break;
if (!blocking && ((static_cast<int64_t>(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames)
|| (FastestFrame() >= frame + 2 + static_cast<int64_t>(queue) * host_summation)))
break;
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
while ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) < frame + 2)
&& !counters.IsAcquisitionFinished() && !failed
&& (static_cast<int64_t>(counters.GetFastestFrameNumber()) < frame + 2 + MissingAfterFrames))
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
if (counters.IsAnyPacketCollected(frame, module_number))
return true;
std::unique_lock ul(given_up_mutex);
given_up[data_stream][module_number].push_back(frame);
return false;
return counters.IsAnyPacketCollected(frame, module_number);
}
void FPGAImagePuller::ImageThread() {
@@ -193,12 +140,6 @@ void FPGAImagePuller::ImageThread() {
summation.emplace_back(std::make_unique<ModuleSummation>(experiment));
try {
for (int64_t image = next_to_take++; image < experiment.GetImageNum() && !failed; image = next_to_take++) {
if (frame_rate_hz > 0)
std::this_thread::sleep_until(start_time + std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::duration<double>(static_cast<double>(image) / frame_rate_hz)));
// Not blocking, an image the receiver has no room for is not assembled: its frames go back at once
const bool drop = !blocking && (GetCurrentFifoUtilization() >= queue);
ImageMetadata metadata(experiment);
auto take_module = [&](const DeviceOutput *output, int d, int m) {
metadata.Process(output);
@@ -211,28 +152,25 @@ void FPGAImagePuller::ImageThread() {
for (int64_t frame = image * host_summation; frame < (image + 1) * host_summation; frame++) {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++) {
auto &device = devices[d];
for (int m = 0; m < card_experiment.GetModulesNum(d); m++) {
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const bool collected = WaitForFrame(d, m, frame);
// A dummy module (CardExperiment): its frames only go back to the card
if (!drop && m < experiment.GetModulesNum(d)) {
if (!summation.empty()) {
auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (collected)
s.AddFPGAOutput(*device.GetDeviceOutput(frame, m));
else
s.AddEmptyOutput();
} else if (collected)
take_module(device.GetDeviceOutput(frame, m), d, m);
if (!summation.empty()) {
auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (collected)
s.AddFPGAOutput(*device.GetDeviceOutput(frame, m));
else
transformation.FillNotCollectedModule(m, d);
}
s.AddEmptyOutput();
} else if (collected)
take_module(device.GetDeviceOutput(frame, m), d, m);
else
transformation.FillNotCollectedModule(m, d);
if (collected)
device.FrameBufferRelease(frame, m);
}
}
}
// A module missing from any of the card's images is missing from the image
if (!drop && !summation.empty())
if (!summation.empty())
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
@@ -246,10 +184,8 @@ void FPGAImagePuller::ImageThread() {
message.number = image;
message.original_number = image;
metadata.Export(message, expected_packets_per_image);
// No module of it came, or no room for it
if (drop || (message.image_collection_efficiency == 0.0f)) {
if (drop)
dropped_images++;
// No module of it came
if (message.image_collection_efficiency == 0.0f) {
Emit(image, std::nullopt);
continue;
}
@@ -272,7 +208,7 @@ void FPGAImagePuller::ImageThread() {
}
} catch (...) {
SetError(std::current_exception());
CancelCards();
StopCards();
}
}
@@ -284,9 +220,7 @@ void FPGAImagePuller::Emit(int64_t image_number, std::optional<ImagePullerOutput
it = images_done.erase(it), next_image++) {
if (it->second) {
it->second->received = std::chrono::steady_clock::now();
// Blocking, the image waits for room; not blocking, it is dropped
if (!(blocking ? PutWhileNotCancelled(*it->second) : outside_fifo.Put(*it->second)))
dropped_images++;
Put(*it->second);
}
}
}
@@ -295,7 +229,7 @@ void FPGAImagePuller::FinalizeMeasurement() {
for (auto &f: image_futures)
f.get();
CancelCards();
StopCards();
for (auto &f: acquire_futures)
f.get();
@@ -303,28 +237,17 @@ void FPGAImagePuller::FinalizeMeasurement() {
{
std::unique_lock ul(order_mutex);
for (auto &[n, output]: images_done)
if (output && !PutWhileNotCancelled(*output))
dropped_images++;
if (output)
Put(*output);
images_done.clear();
}
if (dropped_images > 0)
logger.Warning("FPGA image puller: {} images dropped, the receiver's queue of {} images full", dropped_images.load(),
queue);
EndMessage end{};
for (size_t i = 0; i < adu_histogram.size(); i++)
end.adu_histogram["module" + std::to_string(i)] = adu_histogram[i]->GetHistogram();
PutWhileNotCancelled({.cbor = std::make_shared<CBORStream2DeserializerOutput>(end),
.received = std::chrono::steady_clock::now()});
Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(end), .received = std::chrono::steady_clock::now()});
}
// Waits for room in the queue, as long as the receiver takes from it: after a cancel it no longer does
bool FPGAImagePuller::PutWhileNotCancelled(const ImagePullerOutput &output) {
const auto t0 = std::chrono::steady_clock::now();
bool ret = true;
while (ret && !outside_fifo.PutTimeout(output, std::chrono::milliseconds(100)))
ret = !cancelled;
queue_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now() - t0).count();
return ret;
void FPGAImagePuller::Put(const ImagePullerOutput &output) {
while (!outside_fifo.PutTimeout(output, std::chrono::milliseconds(100)) && !cancelled);
}
+16 -39
View File
@@ -23,38 +23,24 @@
// The images of a collection from the FPGA cards, as a detector stream, in image order. As JFJochReceiverFPGA: each
// thread takes the next image, waits for its modules in the cards' buffers, assembles them into an image of its own
// and compresses it; the image leaves as its compressed bytes, and the cards' buffers go back. The cards' buffers are
// the only window over the images in flight.
// the only window over the images in flight. No image is discarded here: an image waits for room in the queue to the
// receiver, holding the cards' buffers meanwhile, so the cards wait for the host. Images are lost only at the card.
class FPGAImagePuller : public ImagePuller {
public:
// Images compressed and waiting for the receiver
static constexpr size_t DefaultQueue = 128;
// A module this far behind the fastest one of its card is taken as missing from that frame, as the device itself
// does. Behind another card it is waited for until that card is `queue` images ahead (not blocking).
static constexpr size_t Queue = 128;
// A module this far behind the fastest one of its card is taken as missing from that frame. It must stay beyond
// AcquisitionCounters::ThresholdFramesLost: the device then returns that frame's buffer itself if it comes later.
static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2;
private:
const DiffractionExperiment experiment;
// What the cards run: the experiment, or with dummy modules (CardExperiment)
const DiffractionExperiment card_experiment;
Logger &logger;
AcquisitionDeviceGroup &devices;
const size_t queue;
// Blocking: an image waits for room in the queue instead of being dropped, and no module is given up for being
// behind the others, so the cards keep their frames and stall at the source.
const bool blocking;
const double frame_rate_hz; // images taken from the cards at most at this rate (0: as they come)
// The cards' images making one image, summed here (DiffractionExperiment::GetHostSummation)
const int64_t host_summation;
std::chrono::steady_clock::time_point start_time;
std::atomic<int64_t> queue_wait_ns = 0;
uint64_t expected_packets_per_image = 0;
std::atomic<int64_t> next_to_take = 0;
std::atomic<size_t> dropped_images = 0;
// Frames given up on that may still arrive, by card and module: their buffers go back once they do
std::mutex given_up_mutex;
std::vector<std::vector<std::vector<int64_t>>> given_up;
void ReleaseLateFrames(uint16_t data_stream, uint16_t module_number);
// What is done, waiting for the images before it. No value: that image will not come.
std::mutex order_mutex;
@@ -66,7 +52,7 @@ private:
std::mutex error_mutex;
std::exception_ptr error;
std::atomic<bool> failed = false; // a failed card may never report finished: nothing waits for it then
std::atomic<bool> cancelled = false; // Cancel(): no image waits for the receiver any more
std::atomic<bool> cancelled = false; // Cancel(): the receiver takes no more images, nothing waits for it
void SetError(std::exception_ptr e);
std::vector<std::future<void>> acquire_futures;
@@ -78,34 +64,25 @@ private:
void ImageThread();
void FinalizeMeasurement();
// False where the frame is given up on
// False where the module is missing from the frame
bool WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame);
int64_t FastestFrame() const;
void Emit(int64_t image_number, std::optional<ImagePullerOutput> output);
bool PutWhileNotCancelled(const ImagePullerOutput &output);
void CancelCards();
// Waits for room in the queue, as long as the receiver takes from it: after a cancel it no longer does
void Put(const ImagePullerOutput &output);
public:
// Arms the cards; when it returns the detector can be started.
FPGAImagePuller(const DiffractionExperiment &experiment, const PixelMask &mask, const JFCalibration *calibration,
AcquisitionDeviceGroup &devices, Logger &logger, int64_t image_threads,
size_t queue = DefaultQueue, bool blocking = false, double frame_rate_hz = 0,
int64_t modules_per_card = 0);
// The experiment the cards run where each carries modules_per_card modules (0: as `x` splits them): each card's
// modules of `x` first, in their order, then dummy modules the generator sends and the puller does not read.
// For the internal generator, whose speed goes with the modules of the card: cards of different module counts
// run apart, where a detector's modules keep in step.
static DiffractionExperiment CardExperiment(const DiffractionExperiment &x, int64_t modules_per_card);
AcquisitionDeviceGroup &devices, Logger &logger, int64_t image_threads);
~FPGAImagePuller() override;
void Disconnect() override {}
size_t GetDroppedImages() const override { return dropped_images; }
bool IsBlocking() const override { return blocking; }
// Time the image threads waited for room in the queue, summed over them
double GetQueueWaitTime_s() const { return static_cast<double>(queue_wait_ns) * 1e-9; }
// Its images wait for the receiver, so the receiver's hold waits as well (JFJochReceiverRugnuxStills)
bool IsBlocking() const override { return true; }
// The collection is over, whatever the cards have got: what is on its way still comes, then the end message.
// The cards finish: what they have got still comes, then the end message.
void StopCards();
// As StopCards(), and the receiver takes no more images: nothing waits for it any more.
void Cancel();
// After the receiver is done: stops the cards, waits for the threads, throws what went wrong with the cards.
// After the receiver is done: cancels, waits for the threads, throws what went wrong with the cards.
void Stop();
};
+2 -14
View File
@@ -40,25 +40,15 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share
return *this;
}
JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t queue, bool blocking, double frame_rate_hz,
bool always, int64_t modules_per_card) {
JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(bool always) {
std::unique_lock ul_state(state_mutex);
if (state != ReceiverState::Idle)
throw JFJochException(JFJochExceptionCategory::WrongDAQState,
"Cannot change the receiver path during data collection");
fpga_image_puller_queue = queue;
fpga_image_puller_blocking = blocking;
fpga_image_puller_rate_hz = frame_rate_hz;
fpga_image_puller_always = always;
fpga_image_puller_modules_per_card = modules_per_card;
return *this;
}
double JFJochReceiverService::GetFPGAImagePullerQueueWaitTime_s() const {
std::unique_lock ul(state_mutex);
return fpga_puller ? fpga_puller->GetQueueWaitTime_s() : 0.0;
}
// A PSI collection analysed by rugnux live
bool JFJochReceiverService::FPGAImagePullerFor(const DiffractionExperiment &experiment) const {
if ((experiment.GetDetectorType() == DetectorType::DECTRIS) || experiment.IsPedestalRun()
@@ -146,9 +136,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment,
// A PSI detector's frames through FPGAImagePuller go to the stream receivers, as a DECTRIS stream does
if (FPGAImagePullerFor(experiment)) {
fpga_puller = std::make_shared<FPGAImagePuller>(experiment, pixel_mask, calibration, aq_devices, logger,
nthreads, fpga_image_puller_queue,
fpga_image_puller_blocking, fpga_image_puller_rate_hz,
fpga_image_puller_modules_per_card);
nthreads);
puller = fpga_puller;
}
+3 -12
View File
@@ -43,10 +43,6 @@ class JFJochReceiverService {
// A PSI collection analysed by rugnux live goes through FPGAImagePuller to the stream receivers
bool fpga_image_puller_always = false;
int64_t fpga_image_puller_modules_per_card = 0;
size_t fpga_image_puller_queue = FPGAImagePuller::DefaultQueue;
bool fpga_image_puller_blocking = false;
double fpga_image_puller_rate_hz = 0;
std::shared_ptr<FPGAImagePuller> fpga_puller;
bool FPGAImagePullerFor(const DiffractionExperiment &experiment) const;
@@ -77,14 +73,9 @@ public:
int64_t stills_downsample = 1);
RugnuxLiveSnapshot GetRugnuxStatus() const;
void ClearRugnuxStatus(); // back to Idle, as before any collection
// queue, blocking, frame_rate_hz, modules_per_card: see FPGAImagePuller. always: every PSI collection but pedestal
// and host summation through the puller, without rugnux to JFJochReceiverLite (for tests comparing it with
// JFJochReceiverFPGA)
JFJochReceiverService& FPGAImagePullerSettings(size_t queue = FPGAImagePuller::DefaultQueue, bool blocking = false,
double frame_rate_hz = 0, bool always = false,
int64_t modules_per_card = 0);
// Of the last collection through FPGAImagePuller: FPGAImagePuller::GetQueueWaitTime_s, 0 without one
double GetFPGAImagePullerQueueWaitTime_s() const;
// Every PSI collection but a pedestal run through FPGAImagePuller, without rugnux to JFJochReceiverLite (for tests
// comparing it with JFJochReceiverFPGA)
JFJochReceiverService& FPGAImagePullerSettings(bool always);
JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional<int32_t> &watermark = {});
JFJochReceiverService& MetadataSocket(const std::string &addr);
-1
View File
@@ -57,7 +57,6 @@ protected:
virtual void Cut(const std::string &reason) {} // the analysis that goes on after the images, if there is one
void AddToEndMessage(EndMessage &message) override;
uint64_t GetImagesDropped() const override { return image_puller.GetDroppedImages(); }
public:
// Takes what the detector's start message decides into the experiment.
static void ApplyStartMessage(DiffractionExperiment &experiment, const StartMessage &message);