diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index 5d0d884a4..b23af6da5 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -90,6 +90,9 @@ void FPGAImagePuller::SetError(std::exception_ptr e) { } void FPGAImagePuller::Cancel() { + // An image waiting for a slot is dropped from here on: after a cancel nobody may take the images that hold them + cancelled = true; + slot_returned.notify_all(); for (int d = 0; d < experiment.GetDataStreamsNum(); d++) devices[d].Cancel(); } @@ -163,34 +166,40 @@ int64_t FPGAImagePuller::LastFrame() const { } std::shared_ptr FPGAImagePuller::GetSlot(int64_t image_number) { - std::unique_lock ul(images_mutex); - std::shared_ptr slot; - while (!slot) { - if (image_dropped[image_number]) - return nullptr; - // Another module may have started the image while this one waited - if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end()) - return it->second; - if (!free_slots.empty()) { - slot = free_slots.back(); - free_slots.pop_back(); - } else if (blocking && !failed) { - const auto t0 = std::chrono::steady_clock::now(); - slot_returned.wait_for(ul, std::chrono::milliseconds(10)); - slot_wait_ns += std::chrono::duration_cast( - std::chrono::steady_clock::now() - t0).count(); - } else { - image_dropped[image_number] = 1; - dropped_images++; - Emit(image_number, std::nullopt); - return nullptr; + { + std::unique_lock ul(images_mutex); + std::shared_ptr slot; + while (!slot) { + if (image_dropped[image_number]) + return nullptr; + // Another module may have started the image while this one waited + if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end()) + return it->second; + if (!free_slots.empty()) { + slot = free_slots.back(); + free_slots.pop_back(); + } else if (blocking && !failed && !cancelled) { + const auto t0 = std::chrono::steady_clock::now(); + slot_returned.wait_for(ul, std::chrono::milliseconds(10)); + slot_wait_ns += std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + } else { + image_dropped[image_number] = 1; + dropped_images++; + break; + } + } + if (slot) { + slot->image_number = image_number; + slot->metadata = std::make_unique(experiment); + slot->modules_left = static_cast(experiment.GetModulesNum()); + images_in_progress[image_number] = slot; + return slot; } } - slot->image_number = image_number; - slot->metadata = std::make_unique(experiment); - slot->modules_left = static_cast(experiment.GetModulesNum()); - images_in_progress[image_number] = slot; - return slot; + // Not under images_mutex: Emit takes order_mutex, and a slot going back from under order_mutex takes images_mutex + Emit(image_number, std::nullopt); + return nullptr; } void FPGAImagePuller::ReturnSlot(const std::shared_ptr &slot) { @@ -291,13 +300,17 @@ void FPGAImagePuller::CompressThread() { // Whichever thread finishes the image next in line lets it, and those waiting behind it, out. void FPGAImagePuller::Emit(int64_t image_number, std::optional output) { + // The copies of the images let out here go only once order_mutex is released (the lock is declared after them, and + // `output` outlives both): the last copy of an image returns its slot, under images_mutex + std::vector released; std::unique_lock ul(order_mutex); - images_done.emplace(image_number, std::move(output)); + images_done.try_emplace(image_number, output); for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image); it = images_done.erase(it), next_image++) { if (it->second) { it->second->received = std::chrono::steady_clock::now(); - outside_fifo.Put(std::move(*it->second)); + outside_fifo.Put(*it->second); + released.push_back(std::move(*it->second)); } } } @@ -316,11 +329,12 @@ void FPGAImagePuller::FinalizeMeasurement() { // Only an error leaves images unfinished, holding back those behind them { + std::map> left; // destroyed after order_mutex, as in Emit std::unique_lock ul(order_mutex); for (auto &[n, output]: images_done) if (output) - outside_fifo.Put(std::move(*output)); - images_done.clear(); + outside_fifo.Put(*output); + left.swap(images_done); } if (dropped_images > 0) diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index d5d90a16e..599abfce7 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -69,6 +69,7 @@ private: std::mutex error_mutex; std::exception_ptr error; std::atomic failed = false; // a failed card may never report finished: nothing waits for it then + std::atomic cancelled = false; // Cancel(): no image waits for a slot any more void SetError(std::exception_ptr e); std::vector> acquire_futures; diff --git a/tests/FPGAImagePullerTest.cpp b/tests/FPGAImagePullerTest.cpp index 9cc618646..178248b91 100644 --- a/tests/FPGAImagePullerTest.cpp +++ b/tests/FPGAImagePullerTest.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -500,6 +501,67 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Blocking", "[FPGAImagePuller]") { CHECK(numbers[i] == static_cast(i)); } +// Blocking, and nobody takes the images: a cancel must still end the puller (the images waiting for a slot are +// dropped). Run on a thread of its own, so that a hang fails the test instead of stopping the run. +TEST_CASE("FPGAImagePuller_Blocking_CancelWithoutConsumer", "[FPGAImagePuller]") { + auto x = Base(DetJF(1), 20, 1); + x.Conversion(); + x.Compression(CompressionAlgorithm::BSHUF_LZ4); + + auto devices = std::make_shared(Devices(x)); + auto logger = std::make_shared("FPGAImagePuller_Blocking_Cancel"); + auto mask = std::make_shared(x); + auto puller = std::make_shared(x, *mask, nullptr, *devices, *logger, 2, 2, true); + std::this_thread::sleep_for(1s); + + auto stopped = std::make_shared>(); + auto done = stopped->get_future(); + std::thread([puller, devices, logger, mask, stopped] { + puller->Cancel(); + try { + puller->Stop(); + } catch (...) {} + stopped->set_value(); + }).detach(); + REQUIRE(done.wait_for(30s) == std::future_status::ready); + CHECK(puller->GetDroppedImages() > 0); +} + +// Not blocking, with drops: a module thread drops an image (GetSlot, images_mutex) while a compress thread lets an +// image out (Emit, order_mutex) and the consumer frees images at once, so a slot can go back from inside Emit. The +// two locks must never be held the other way round. Run on a thread of its own, so that a hang fails the test. +TEST_CASE("FPGAImagePuller_DroppingNoLockInversion", "[FPGAImagePuller]") { + auto x = Base(DetJF(4), 3000, 1); + x.Conversion(); + x.Compression(CompressionAlgorithm::NO_COMPRESSION); + + auto devices = std::make_shared(Devices(x)); + auto logger = std::make_shared("FPGAImagePuller_DroppingNoLockInversion"); + auto mask = std::make_shared(x); + auto finished = std::make_shared>(); + auto done = finished->get_future(); + std::thread([x, devices, logger, mask, finished] { + FPGAImagePuller puller(x, *mask, nullptr, *devices, *logger, 8, 2); + size_t images = 0; + bool end = false; + while (!end) { + // Now and then the consumer pauses, so that the slots run out and images are dropped + if (images % 7 == 0) + std::this_thread::sleep_for(1ms); + const auto m = puller.PollImage(10ms); + if (m && m->cbor->data_message) + images++; + end = m && m->cbor->end_message.has_value(); + } + puller.Stop(); + std::cout << "dropping, no lock inversion: " << images << " images, " << puller.GetDroppedImages() + << " dropped" << std::endl; + finished->set_value(images + puller.GetDroppedImages()); + }).detach(); + REQUIRE(done.wait_for(120s) == std::future_status::ready); + CHECK(done.get() <= 3000); +} + // Paced: the images are taken from the card at most at the rate given TEST_CASE("FPGAImagePuller_Paced", "[FPGAImagePuller]") { auto x = Base(DetJF(1), 10, 1); diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index a7359dcf7..76e654b4b 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -189,8 +189,17 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s std::this_thread::sleep_for(std::chrono::milliseconds(500)); std::string line; for (int i = 0; i < x.GetDataStreamsNum(); i++) - line += fmt::format(" card {}: {} .. {}", i, aq_devices[i].Counters().GetSlowestFrameNumber(), - aq_devices[i].Counters().GetFastestFrameNumber()); + line += fmt::format(" card {}: {} .. {}{}", i, aq_devices[i].Counters().GetSlowestFrameNumber(), + aq_devices[i].Counters().GetFastestFrameNumber(), + aq_devices[i].Counters().IsAcquisitionFinished() ? " done" : ""); + // Where a run stands, so that one that stops shows which part waits + if (const auto st = service.GetStatus()) + line += fmt::format(" receiver: collected {} sent {} dropped {}", st->images_collected, + st->images_sent, st->images_dropped); + const auto live = service.GetRugnuxStatus(); + if (live.stills) + line += fmt::format(" stills: held {} analysed {} pre-pass {}", live.stills->images_held, + live.stills->images_analysed, static_cast(live.stills->prepass)); logger.Debug("{:.1f} s{}", std::chrono::duration(std::chrono::steady_clock::now() - wall_start).count(), line); }