FPGAImagePuller: no lock-order inversion between images_mutex and order_mutex; cancel ends a blocking wait

The hang of a non-blocking run on four cards (backtrace from the server): a
module thread dropping an image called Emit from GetSlot with images_mutex
held and waited for order_mutex, while a compress thread in Emit, holding
order_mutex, destroyed the last copy of an image - the receiver had already
taken and dropped the copy just queued - and its deleter (ReturnSlot) waited
for images_mutex. Only a dropping run goes through that branch of GetSlot.

- GetSlot emits a dropped image after releasing images_mutex.
- Emit and the flush at the end keep the copies of what they let out until
  order_mutex is released, so no slot goes back under it.
- Cancel() sets `cancelled`: an image waiting for a slot in blocking mode is
  dropped instead, as after a cancel nobody may take the images holding the
  slots (FPGAImagePuller_Blocking_CancelWithoutConsumer hung before, passes now).
- FPGAImagePuller_DroppingNoLockInversion: 4 modules, 2 slots, 8 compress
  threads, a consumer that pauses so that images are dropped and frees them at
  once; under a time limit. The race window is a few instructions wide: on the
  mock it did not hang before the fix either, so it guards the lock order rather
  than reproducing the server's hang.
- jfjoch_fpga_stills_test -v: the receiver's and the stills analysis' counts and
  whether each card is done, beside the cards' progress, so that a run that stops
  shows which part waits.

The 4 h hang of a blocking two-card mock run (2026-11-18 09:29) was the binary
before 4c8af0dbc (late frames of a lagging card not released while a module
waited); the same run now takes 17-45 s (4 repeats).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-18 14:33:52 +01:00
1 parent 536f83e28f
commit 31759791df
4 files changed
+118 -32

No files matched your search

+44 -30
View File
@@ -90,6 +90,9 @@ void FPGAImagePuller::SetError(std::exception_ptr e) {
}
void FPGAImagePuller::Cancel() {
// An image waiting for a slot is dropped from here on: after a cancel nobody may take the images that hold them
cancelled = true;
slot_returned.notify_all();
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
devices[d].Cancel();
}
@@ -163,34 +166,40 @@ int64_t FPGAImagePuller::LastFrame() const {
}
std::shared_ptr<FPGAImagePuller::Slot> FPGAImagePuller::GetSlot(int64_t image_number) {
std::unique_lock ul(images_mutex);
std::shared_ptr<Slot> slot;
while (!slot) {
if (image_dropped[image_number])
return nullptr;
// Another module may have started the image while this one waited
if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end())
return it->second;
if (!free_slots.empty()) {
slot = free_slots.back();
free_slots.pop_back();
} else if (blocking && !failed) {
const auto t0 = std::chrono::steady_clock::now();
slot_returned.wait_for(ul, std::chrono::milliseconds(10));
slot_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now() - t0).count();
} else {
image_dropped[image_number] = 1;
dropped_images++;
Emit(image_number, std::nullopt);
return nullptr;
{
std::unique_lock ul(images_mutex);
std::shared_ptr<Slot> slot;
while (!slot) {
if (image_dropped[image_number])
return nullptr;
// Another module may have started the image while this one waited
if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end())
return it->second;
if (!free_slots.empty()) {
slot = free_slots.back();
free_slots.pop_back();
} else if (blocking && !failed && !cancelled) {
const auto t0 = std::chrono::steady_clock::now();
slot_returned.wait_for(ul, std::chrono::milliseconds(10));
slot_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now() - t0).count();
} else {
image_dropped[image_number] = 1;
dropped_images++;
break;
}
}
if (slot) {
slot->image_number = image_number;
slot->metadata = std::make_unique<ImageMetadata>(experiment);
slot->modules_left = static_cast<int>(experiment.GetModulesNum());
images_in_progress[image_number] = slot;
return slot;
}
}
slot->image_number = image_number;
slot->metadata = std::make_unique<ImageMetadata>(experiment);
slot->modules_left = static_cast<int>(experiment.GetModulesNum());
images_in_progress[image_number] = slot;
return slot;
// Not under images_mutex: Emit takes order_mutex, and a slot going back from under order_mutex takes images_mutex
Emit(image_number, std::nullopt);
return nullptr;
}
void FPGAImagePuller::ReturnSlot(const std::shared_ptr<Slot> &slot) {
@@ -291,13 +300,17 @@ void FPGAImagePuller::CompressThread() {
// Whichever thread finishes the image next in line lets it, and those waiting behind it, out.
void FPGAImagePuller::Emit(int64_t image_number, std::optional<ImagePullerOutput> output) {
// The copies of the images let out here go only once order_mutex is released (the lock is declared after them, and
// `output` outlives both): the last copy of an image returns its slot, under images_mutex
std::vector<ImagePullerOutput> released;
std::unique_lock ul(order_mutex);
images_done.emplace(image_number, std::move(output));
images_done.try_emplace(image_number, output);
for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image);
it = images_done.erase(it), next_image++) {
if (it->second) {
it->second->received = std::chrono::steady_clock::now();
outside_fifo.Put(std::move(*it->second));
outside_fifo.Put(*it->second);
released.push_back(std::move(*it->second));
}
}
}
@@ -316,11 +329,12 @@ void FPGAImagePuller::FinalizeMeasurement() {
// Only an error leaves images unfinished, holding back those behind them
{
std::map<int64_t, std::optional<ImagePullerOutput>> left; // destroyed after order_mutex, as in Emit
std::unique_lock ul(order_mutex);
for (auto &[n, output]: images_done)
if (output)
outside_fifo.Put(std::move(*output));
images_done.clear();
outside_fifo.Put(*output);
left.swap(images_done);
}
if (dropped_images > 0)
+1
View File
@@ -69,6 +69,7 @@ private:
std::mutex error_mutex;
std::exception_ptr error;
std::atomic<bool> failed = false; // a failed card may never report finished: nothing waits for it then
std::atomic<bool> cancelled = false; // Cancel(): no image waits for a slot any more
void SetError(std::exception_ptr e);
std::vector<std::future<void>> acquire_futures;
+62
View File
@@ -7,6 +7,7 @@
#include <catch2/catch_all.hpp>
#include <cmath>
#include <future>
#include <iostream>
#include <map>
#include <mutex>
@@ -500,6 +501,67 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Blocking", "[FPGAImagePuller]") {
CHECK(numbers[i] == static_cast<int64_t>(i));
}
// Blocking, and nobody takes the images: a cancel must still end the puller (the images waiting for a slot are
// dropped). Run on a thread of its own, so that a hang fails the test instead of stopping the run.
TEST_CASE("FPGAImagePuller_Blocking_CancelWithoutConsumer", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 20, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
auto logger = std::make_shared<Logger>("FPGAImagePuller_Blocking_Cancel");
auto mask = std::make_shared<PixelMask>(x);
auto puller = std::make_shared<FPGAImagePuller>(x, *mask, nullptr, *devices, *logger, 2, 2, true);
std::this_thread::sleep_for(1s);
auto stopped = std::make_shared<std::promise<void>>();
auto done = stopped->get_future();
std::thread([puller, devices, logger, mask, stopped] {
puller->Cancel();
try {
puller->Stop();
} catch (...) {}
stopped->set_value();
}).detach();
REQUIRE(done.wait_for(30s) == std::future_status::ready);
CHECK(puller->GetDroppedImages() > 0);
}
// Not blocking, with drops: a module thread drops an image (GetSlot, images_mutex) while a compress thread lets an
// image out (Emit, order_mutex) and the consumer frees images at once, so a slot can go back from inside Emit. The
// two locks must never be held the other way round. Run on a thread of its own, so that a hang fails the test.
TEST_CASE("FPGAImagePuller_DroppingNoLockInversion", "[FPGAImagePuller]") {
auto x = Base(DetJF(4), 3000, 1);
x.Conversion();
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
auto logger = std::make_shared<Logger>("FPGAImagePuller_DroppingNoLockInversion");
auto mask = std::make_shared<PixelMask>(x);
auto finished = std::make_shared<std::promise<size_t>>();
auto done = finished->get_future();
std::thread([x, devices, logger, mask, finished] {
FPGAImagePuller puller(x, *mask, nullptr, *devices, *logger, 8, 2);
size_t images = 0;
bool end = false;
while (!end) {
// Now and then the consumer pauses, so that the slots run out and images are dropped
if (images % 7 == 0)
std::this_thread::sleep_for(1ms);
const auto m = puller.PollImage(10ms);
if (m && m->cbor->data_message)
images++;
end = m && m->cbor->end_message.has_value();
}
puller.Stop();
std::cout << "dropping, no lock inversion: " << images << " images, " << puller.GetDroppedImages()
<< " dropped" << std::endl;
finished->set_value(images + puller.GetDroppedImages());
}).detach();
REQUIRE(done.wait_for(120s) == std::future_status::ready);
CHECK(done.get() <= 3000);
}
// Paced: the images are taken from the card at most at the rate given
TEST_CASE("FPGAImagePuller_Paced", "[FPGAImagePuller]") {
auto x = Base(DetJF(1), 10, 1);
+11 -2
View File
@@ -189,8 +189,17 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
std::this_thread::sleep_for(std::chrono::milliseconds(500));
std::string line;
for (int i = 0; i < x.GetDataStreamsNum(); i++)
line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
aq_devices[i].Counters().GetFastestFrameNumber());
line += fmt::format(" card {}: {} .. {}{}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
aq_devices[i].Counters().GetFastestFrameNumber(),
aq_devices[i].Counters().IsAcquisitionFinished() ? " done" : "");
// Where a run stands, so that one that stops shows which part waits
if (const auto st = service.GetStatus())
line += fmt::format(" receiver: collected {} sent {} dropped {}", st->images_collected,
st->images_sent, st->images_dropped);
const auto live = service.GetRugnuxStatus();
if (live.stills)
line += fmt::format(" stills: held {} analysed {} pre-pass {}", live.stills->images_held,
live.stills->images_analysed, static_cast<int>(live.stills->prepass));
logger.Debug("{:.1f} s{}", std::chrono::duration<double>(std::chrono::steady_clock::now()
- wall_start).count(), line);
}