Files
leonarski_fandClaude Opus 5.5 d279137b2e FPGAImagePuller: images the cards never got to are skipped; the puller always stops
An image whose frames no card got to (all cards finished, every one's fastest
frame before the image's first) is now let go at once, as JFJochReceiverFPGA
does, instead of being assembled from missing modules: an abort early in a
long run took minutes to work through the rest while Stop() waited, and a
silent end with fewer frames than asked for left the receiver without
messages long enough to declare the end message lost.

JFJochReceiverService::FinalizeMeasurement stops the puller even when the
receiver ends with an error, so its threads do not hold the cards' buffers,
and the cards stay armed, until the next run; the receiver's error is
rethrown first.

Nits: WaitForFrame's comment no longer refers to the removed cross-card
give-up; emplace for try_emplace; jfjoch_fpga_stills_test no longer prints
the always-0 dropped count; docs/TOOLS.md names the writer exception (an
analysed image without an image-buffer slot within the wait is not written,
writer_queue_full).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQrPqZ52Tp6rzzZGxf3
2026-11-18 20:29:29 +01:00

269 lines
12 KiB
C++

// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include "FPGAImagePuller.h"
#include <thread>
#include <utility>
FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask,
const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices,
Logger &in_logger, int64_t image_threads)
: ImagePuller(Queue + 2), // the images waiting, the start and the end message
experiment(in_experiment),
logger(in_logger),
devices(in_devices),
host_summation(in_experiment.GetHostSummation()) {
const int ndatastreams = experiment.GetDataStreamsNum();
if (devices.size() < ndatastreams)
throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
"Number of acquisition devices has to match data streams");
for (int m = 0; m < experiment.GetModulesNum(); m++)
adu_histogram.emplace_back(std::make_unique<ADUHistogram>());
for (int d = 0; d < ndatastreams; d++) {
devices[d].PrepareAction(experiment);
expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() * host_summation;
}
if (experiment.GetDetectorType() != DetectorType::JUNGFRAU)
calibration = nullptr;
// The receiver's own start message is built from its experiment; this one only says the run has begun.
StartMessage start{};
experiment.FillMessage(start);
outside_fifo.Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(start),
.received = std::chrono::steady_clock::now()});
for (int i = 0; i < image_threads; i++)
image_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::ImageThread, this));
std::latch ready(ndatastreams);
for (int d = 0; d < ndatastreams; d++)
acquire_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::AcquireThread, this,
static_cast<uint16_t>(d), std::cref(mask), calibration,
std::ref(ready)));
ready.wait();
finish_future = std::async(std::launch::async, &FPGAImagePuller::FinalizeMeasurement, this);
if (failed) {
try {
Stop();
} catch (...) {
outside_fifo.Clear(); // the destructor does not run
throw;
}
}
}
FPGAImagePuller::~FPGAImagePuller() {
try {
Stop();
} catch (const std::exception &e) {
logger.Error("FPGA image puller teardown failed: {}", e.what());
}
}
void FPGAImagePuller::SetError(std::exception_ptr e) {
std::unique_lock ul(error_mutex);
if (!error)
error = e;
failed = true;
}
void FPGAImagePuller::StopCards() {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
devices[d].Cancel();
}
void FPGAImagePuller::Cancel() {
cancelled = true;
StopCards();
}
// Cancels first: a receiver that stopped on an error takes no more images, which would hold the threads otherwise
void FPGAImagePuller::Stop() {
Cancel();
std::call_once(finish_once, [this] { finish_future.get(); });
std::unique_lock ul(error_mutex);
if (error)
std::rethrow_exception(std::exchange(error, nullptr));
}
void FPGAImagePuller::AcquireThread(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration,
std::latch &ready) {
auto &device = devices[data_stream];
try {
// Only what the card needs for the pixels: its analysis results are not read.
if (calibration != nullptr)
device.InitializeCalibration(experiment, *calibration);
device.InitializePixelMask(experiment, mask);
device.StartAction(experiment);
} catch (...) {
SetError(std::current_exception());
StopCards(); // the image threads waiting on the other cards have to come out as well
ready.count_down();
return;
}
ready.count_down();
try {
device.WaitForActionComplete();
} catch (...) {
SetError(std::current_exception());
StopCards();
}
}
// Polled: a card that fails or finishes ends the wait. A frame given up on that comes later is below the device's
// slowest frame then (MissingAfterFrames), and the device returns its buffer itself.
bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) {
const auto &counters = devices[data_stream].Counters();
// A frame is complete once the module is two further on, as in JFJochReceiverFPGA
while ((static_cast<int64_t>(counters.GetCurrFrameNumber(module_number)) < frame + 2)
&& !counters.IsAcquisitionFinished() && !failed
&& (static_cast<int64_t>(counters.GetFastestFrameNumber()) < frame + 2 + MissingAfterFrames))
std::this_thread::sleep_for(std::chrono::milliseconds(1));
return counters.IsAnyPacketCollected(frame, module_number);
}
bool FPGAImagePuller::CardsFinishedBefore(int64_t frame) const {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++) {
const auto &counters = devices[d].Counters();
if (!counters.IsAcquisitionFinished() || (static_cast<int64_t>(counters.GetFastestFrameNumber()) >= frame))
return false;
}
return true;
}
void FPGAImagePuller::ImageThread() {
// One image of its own, assembled and compressed here
FrameTransformation transformation(experiment);
// Summation on the host: the cards' images of one image added up, one module each, as in JFJochReceiverFPGA
std::vector<std::unique_ptr<ModuleSummation>> summation;
if (host_summation > 1)
for (int m = 0; m < experiment.GetModulesNum(); m++)
summation.emplace_back(std::make_unique<ModuleSummation>(experiment));
try {
for (int64_t image = next_to_take++; image < experiment.GetImageNum() && !failed; image = next_to_take++) {
// The cards are through and none got as far as this image (a cancel, or fewer frames than asked for):
// it will not come, and is not assembled from missing modules either - as JFJochReceiverFPGA
if (CardsFinishedBefore(image * host_summation)) {
Emit(image, std::nullopt);
continue;
}
ImageMetadata metadata(experiment);
auto take_module = [&](const DeviceOutput *output, int d, int m) {
metadata.Process(output);
adu_histogram[experiment.GetFirstModuleOfDataStream(d) + m]->Add(*output);
transformation.ProcessModule(output, d);
};
for (auto &s: summation)
s->Clear();
// The card's images of this image in their order, all modules of each before the next, as they come
for (int64_t frame = image * host_summation; frame < (image + 1) * host_summation; frame++) {
for (int d = 0; d < experiment.GetDataStreamsNum(); d++) {
auto &device = devices[d];
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const bool collected = WaitForFrame(d, m, frame);
if (!summation.empty()) {
auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (collected)
s.AddFPGAOutput(*device.GetDeviceOutput(frame, m));
else
s.AddEmptyOutput();
} else if (collected)
take_module(device.GetDeviceOutput(frame, m), d, m);
else
transformation.FillNotCollectedModule(m, d);
if (collected)
device.FrameBufferRelease(frame, m);
}
}
}
// A module missing from any of the card's images is missing from the image
if (!summation.empty())
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
for (int m = 0; m < experiment.GetModulesNum(d); m++) {
const auto &s = *summation[experiment.GetFirstModuleOfDataStream(d) + m];
if (s.empty())
transformation.FillNotCollectedModule(m, d);
else
take_module(&s.GetOutput(), d, m);
}
DataMessage message{};
message.number = image;
message.original_number = image;
metadata.Export(message, expected_packets_per_image);
// No module of it came
if (message.image_collection_efficiency == 0.0f) {
Emit(image, std::nullopt);
continue;
}
// The card's spot finder and ROI map were not set up for this run; the receiver reports its own.
message.strong_pixel_count.reset();
message.roi.clear();
// The compressed image, copied once into a buffer of the output's own
const CompressedImage compressed = transformation.GetCompressedImage();
auto bytes = std::make_shared<std::vector<uint8_t>>(compressed.GetCompressed(),
compressed.GetCompressed() + compressed.GetCompressedSize());
message.image = CompressedImage(bytes->data(), bytes->size(), compressed.GetWidth(), compressed.GetHeight(),
compressed.GetMode(), compressed.GetCompressionAlgorithm());
ImagePullerOutput output;
output.cbor = std::make_shared<CBORStream2DeserializerOutput>(message);
output.raw = std::shared_ptr<ReceiveBufferRegion>(new ReceiveBufferRegion{bytes->data(), bytes->size()},
[bytes](ReceiveBufferRegion *region) { delete region; });
Emit(image, std::move(output));
}
} catch (...) {
SetError(std::current_exception());
StopCards();
}
}
// Whichever thread finishes the image next in line lets it, and those waiting behind it, out.
void FPGAImagePuller::Emit(int64_t image_number, std::optional<ImagePullerOutput> output) {
std::unique_lock ul(order_mutex);
images_done.emplace(image_number, std::move(output));
for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image);
it = images_done.erase(it), next_image++) {
if (it->second) {
it->second->received = std::chrono::steady_clock::now();
Put(*it->second);
}
}
}
void FPGAImagePuller::FinalizeMeasurement() {
for (auto &f: image_futures)
f.get();
StopCards();
for (auto &f: acquire_futures)
f.get();
// Only an error leaves images unfinished, holding back those behind them
{
std::unique_lock ul(order_mutex);
for (auto &[n, output]: images_done)
if (output)
Put(*output);
images_done.clear();
}
EndMessage end{};
for (size_t i = 0; i < adu_histogram.size(); i++)
end.adu_histogram["module" + std::to_string(i)] = adu_histogram[i]->GetHistogram();
Put({.cbor = std::make_shared<CBORStream2DeserializerOutput>(end), .received = std::chrono::steady_clock::now()});
}
void FPGAImagePuller::Put(const ImagePullerOutput &output) {
while (!outside_fifo.PutTimeout(output, std::chrono::milliseconds(100)) && !cancelled);
}