FPGAImagePuller: no lock-order inversion between images_mutex and order_mutex; cancel ends a blocking wait
The hang of a non-blocking run on four cards (backtrace from the server): a
module thread dropping an image called Emit from GetSlot with images_mutex
held and waited for order_mutex, while a compress thread in Emit, holding
order_mutex, destroyed the last copy of an image - the receiver had already
taken and dropped the copy just queued - and its deleter (ReturnSlot) waited
for images_mutex. Only a dropping run goes through that branch of GetSlot.
- GetSlot emits a dropped image after releasing images_mutex.
- Emit and the flush at the end keep the copies of what they let out until
order_mutex is released, so no slot goes back under it.
- Cancel() sets `cancelled`: an image waiting for a slot in blocking mode is
dropped instead, as after a cancel nobody may take the images holding the
slots (FPGAImagePuller_Blocking_CancelWithoutConsumer hung before, passes now).
- FPGAImagePuller_DroppingNoLockInversion: 4 modules, 2 slots, 8 compress
threads, a consumer that pauses so that images are dropped and frees them at
once; under a time limit. The race window is a few instructions wide: on the
mock it did not hang before the fix either, so it guards the lock order rather
than reproducing the server's hang.
- jfjoch_fpga_stills_test -v: the receiver's and the stills analysis' counts and
whether each card is done, beside the cards' progress, so that a run that stops
shows which part waits.
The 4 h hang of a blocking two-card mock run (2026-11-18 09:29) was the binary
before 4c8af0dbc (late frames of a lagging card not released while a module
waited); the same run now takes 17-45 s (4 repeats).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UqXYdHHZ4cogS1mrGUtfN7
This commit is contained in:
1 parent
536f83e28f
commit
31759791df
4 files changed
+118
-32
No files matched your search
@@ -90,6 +90,9 @@ void FPGAImagePuller::SetError(std::exception_ptr e) {
|
||||
}
|
||||
|
||||
void FPGAImagePuller::Cancel() {
|
||||
// An image waiting for a slot is dropped from here on: after a cancel nobody may take the images that hold them
|
||||
cancelled = true;
|
||||
slot_returned.notify_all();
|
||||
for (int d = 0; d < experiment.GetDataStreamsNum(); d++)
|
||||
devices[d].Cancel();
|
||||
}
|
||||
@@ -163,34 +166,40 @@ int64_t FPGAImagePuller::LastFrame() const {
|
||||
}
|
||||
|
||||
std::shared_ptr<FPGAImagePuller::Slot> FPGAImagePuller::GetSlot(int64_t image_number) {
|
||||
std::unique_lock ul(images_mutex);
|
||||
std::shared_ptr<Slot> slot;
|
||||
while (!slot) {
|
||||
if (image_dropped[image_number])
|
||||
return nullptr;
|
||||
// Another module may have started the image while this one waited
|
||||
if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end())
|
||||
return it->second;
|
||||
if (!free_slots.empty()) {
|
||||
slot = free_slots.back();
|
||||
free_slots.pop_back();
|
||||
} else if (blocking && !failed) {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
slot_returned.wait_for(ul, std::chrono::milliseconds(10));
|
||||
slot_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - t0).count();
|
||||
} else {
|
||||
image_dropped[image_number] = 1;
|
||||
dropped_images++;
|
||||
Emit(image_number, std::nullopt);
|
||||
return nullptr;
|
||||
{
|
||||
std::unique_lock ul(images_mutex);
|
||||
std::shared_ptr<Slot> slot;
|
||||
while (!slot) {
|
||||
if (image_dropped[image_number])
|
||||
return nullptr;
|
||||
// Another module may have started the image while this one waited
|
||||
if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end())
|
||||
return it->second;
|
||||
if (!free_slots.empty()) {
|
||||
slot = free_slots.back();
|
||||
free_slots.pop_back();
|
||||
} else if (blocking && !failed && !cancelled) {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
slot_returned.wait_for(ul, std::chrono::milliseconds(10));
|
||||
slot_wait_ns += std::chrono::duration_cast<std::chrono::nanoseconds>(
|
||||
std::chrono::steady_clock::now() - t0).count();
|
||||
} else {
|
||||
image_dropped[image_number] = 1;
|
||||
dropped_images++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (slot) {
|
||||
slot->image_number = image_number;
|
||||
slot->metadata = std::make_unique<ImageMetadata>(experiment);
|
||||
slot->modules_left = static_cast<int>(experiment.GetModulesNum());
|
||||
images_in_progress[image_number] = slot;
|
||||
return slot;
|
||||
}
|
||||
}
|
||||
slot->image_number = image_number;
|
||||
slot->metadata = std::make_unique<ImageMetadata>(experiment);
|
||||
slot->modules_left = static_cast<int>(experiment.GetModulesNum());
|
||||
images_in_progress[image_number] = slot;
|
||||
return slot;
|
||||
// Not under images_mutex: Emit takes order_mutex, and a slot going back from under order_mutex takes images_mutex
|
||||
Emit(image_number, std::nullopt);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void FPGAImagePuller::ReturnSlot(const std::shared_ptr<Slot> &slot) {
|
||||
@@ -291,13 +300,17 @@ void FPGAImagePuller::CompressThread() {
|
||||
|
||||
// Whichever thread finishes the image next in line lets it, and those waiting behind it, out.
|
||||
void FPGAImagePuller::Emit(int64_t image_number, std::optional<ImagePullerOutput> output) {
|
||||
// The copies of the images let out here go only once order_mutex is released (the lock is declared after them, and
|
||||
// `output` outlives both): the last copy of an image returns its slot, under images_mutex
|
||||
std::vector<ImagePullerOutput> released;
|
||||
std::unique_lock ul(order_mutex);
|
||||
images_done.emplace(image_number, std::move(output));
|
||||
images_done.try_emplace(image_number, output);
|
||||
for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image);
|
||||
it = images_done.erase(it), next_image++) {
|
||||
if (it->second) {
|
||||
it->second->received = std::chrono::steady_clock::now();
|
||||
outside_fifo.Put(std::move(*it->second));
|
||||
outside_fifo.Put(*it->second);
|
||||
released.push_back(std::move(*it->second));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -316,11 +329,12 @@ void FPGAImagePuller::FinalizeMeasurement() {
|
||||
|
||||
// Only an error leaves images unfinished, holding back those behind them
|
||||
{
|
||||
std::map<int64_t, std::optional<ImagePullerOutput>> left; // destroyed after order_mutex, as in Emit
|
||||
std::unique_lock ul(order_mutex);
|
||||
for (auto &[n, output]: images_done)
|
||||
if (output)
|
||||
outside_fifo.Put(std::move(*output));
|
||||
images_done.clear();
|
||||
outside_fifo.Put(*output);
|
||||
left.swap(images_done);
|
||||
}
|
||||
|
||||
if (dropped_images > 0)
|
||||
|
||||
@@ -69,6 +69,7 @@ private:
|
||||
std::mutex error_mutex;
|
||||
std::exception_ptr error;
|
||||
std::atomic<bool> failed = false; // a failed card may never report finished: nothing waits for it then
|
||||
std::atomic<bool> cancelled = false; // Cancel(): no image waits for a slot any more
|
||||
void SetError(std::exception_ptr e);
|
||||
|
||||
std::vector<std::future<void>> acquire_futures;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <catch2/catch_all.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <future>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
@@ -500,6 +501,67 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Blocking", "[FPGAImagePuller]") {
|
||||
CHECK(numbers[i] == static_cast<int64_t>(i));
|
||||
}
|
||||
|
||||
// Blocking, and nobody takes the images: a cancel must still end the puller (the images waiting for a slot are
|
||||
// dropped). Run on a thread of its own, so that a hang fails the test instead of stopping the run.
|
||||
TEST_CASE("FPGAImagePuller_Blocking_CancelWithoutConsumer", "[FPGAImagePuller]") {
|
||||
auto x = Base(DetJF(1), 20, 1);
|
||||
x.Conversion();
|
||||
x.Compression(CompressionAlgorithm::BSHUF_LZ4);
|
||||
|
||||
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
|
||||
auto logger = std::make_shared<Logger>("FPGAImagePuller_Blocking_Cancel");
|
||||
auto mask = std::make_shared<PixelMask>(x);
|
||||
auto puller = std::make_shared<FPGAImagePuller>(x, *mask, nullptr, *devices, *logger, 2, 2, true);
|
||||
std::this_thread::sleep_for(1s);
|
||||
|
||||
auto stopped = std::make_shared<std::promise<void>>();
|
||||
auto done = stopped->get_future();
|
||||
std::thread([puller, devices, logger, mask, stopped] {
|
||||
puller->Cancel();
|
||||
try {
|
||||
puller->Stop();
|
||||
} catch (...) {}
|
||||
stopped->set_value();
|
||||
}).detach();
|
||||
REQUIRE(done.wait_for(30s) == std::future_status::ready);
|
||||
CHECK(puller->GetDroppedImages() > 0);
|
||||
}
|
||||
|
||||
// Not blocking, with drops: a module thread drops an image (GetSlot, images_mutex) while a compress thread lets an
|
||||
// image out (Emit, order_mutex) and the consumer frees images at once, so a slot can go back from inside Emit. The
|
||||
// two locks must never be held the other way round. Run on a thread of its own, so that a hang fails the test.
|
||||
TEST_CASE("FPGAImagePuller_DroppingNoLockInversion", "[FPGAImagePuller]") {
|
||||
auto x = Base(DetJF(4), 3000, 1);
|
||||
x.Conversion();
|
||||
x.Compression(CompressionAlgorithm::NO_COMPRESSION);
|
||||
|
||||
auto devices = std::make_shared<AcquisitionDeviceGroup>(Devices(x));
|
||||
auto logger = std::make_shared<Logger>("FPGAImagePuller_DroppingNoLockInversion");
|
||||
auto mask = std::make_shared<PixelMask>(x);
|
||||
auto finished = std::make_shared<std::promise<size_t>>();
|
||||
auto done = finished->get_future();
|
||||
std::thread([x, devices, logger, mask, finished] {
|
||||
FPGAImagePuller puller(x, *mask, nullptr, *devices, *logger, 8, 2);
|
||||
size_t images = 0;
|
||||
bool end = false;
|
||||
while (!end) {
|
||||
// Now and then the consumer pauses, so that the slots run out and images are dropped
|
||||
if (images % 7 == 0)
|
||||
std::this_thread::sleep_for(1ms);
|
||||
const auto m = puller.PollImage(10ms);
|
||||
if (m && m->cbor->data_message)
|
||||
images++;
|
||||
end = m && m->cbor->end_message.has_value();
|
||||
}
|
||||
puller.Stop();
|
||||
std::cout << "dropping, no lock inversion: " << images << " images, " << puller.GetDroppedImages()
|
||||
<< " dropped" << std::endl;
|
||||
finished->set_value(images + puller.GetDroppedImages());
|
||||
}).detach();
|
||||
REQUIRE(done.wait_for(120s) == std::future_status::ready);
|
||||
CHECK(done.get() <= 3000);
|
||||
}
|
||||
|
||||
// Paced: the images are taken from the card at most at the rate given
|
||||
TEST_CASE("FPGAImagePuller_Paced", "[FPGAImagePuller]") {
|
||||
auto x = Base(DetJF(1), 10, 1);
|
||||
|
||||
@@ -189,8 +189,17 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(500));
|
||||
std::string line;
|
||||
for (int i = 0; i < x.GetDataStreamsNum(); i++)
|
||||
line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
|
||||
aq_devices[i].Counters().GetFastestFrameNumber());
|
||||
line += fmt::format(" card {}: {} .. {}{}", i, aq_devices[i].Counters().GetSlowestFrameNumber(),
|
||||
aq_devices[i].Counters().GetFastestFrameNumber(),
|
||||
aq_devices[i].Counters().IsAcquisitionFinished() ? " done" : "");
|
||||
// Where a run stands, so that one that stops shows which part waits
|
||||
if (const auto st = service.GetStatus())
|
||||
line += fmt::format(" receiver: collected {} sent {} dropped {}", st->images_collected,
|
||||
st->images_sent, st->images_dropped);
|
||||
const auto live = service.GetRugnuxStatus();
|
||||
if (live.stills)
|
||||
line += fmt::format(" stills: held {} analysed {} pre-pass {}", live.stills->images_held,
|
||||
live.stills->images_analysed, static_cast<int>(live.stills->prepass));
|
||||
logger.Debug("{:.1f} s{}", std::chrono::duration<double>(std::chrono::steady_clock::now()
|
||||
- wall_start).count(), line);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user