diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index c017651f1..948e02acb 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -9,17 +9,16 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask, const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices, - Logger &in_logger, int64_t compression_threads, size_t in_max_slots, + Logger &in_logger, int64_t image_threads, size_t in_queue, bool in_blocking, double in_frame_rate_hz, int64_t modules_per_card) - : ImagePuller(in_max_slots + 2), // as many images as slots, the start and the end message + : ImagePuller(in_queue + 2), // the images waiting, the start and the end message experiment(in_experiment), card_experiment(CardExperiment(in_experiment, modules_per_card)), logger(in_logger), devices(in_devices), - max_slots(in_max_slots), + queue(in_queue), blocking(in_blocking), - frame_rate_hz(in_frame_rate_hz), - image_dropped(experiment.GetImageNum(), 0) { + frame_rate_hz(in_frame_rate_hz) { const int ndatastreams = experiment.GetDataStreamsNum(); if (devices.size() < ndatastreams) throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, @@ -28,7 +27,9 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con for (int m = 0; m < experiment.GetModulesNum(); m++) adu_histogram.emplace_back(std::make_unique()); + given_up.resize(ndatastreams); for (int d = 0; d < ndatastreams; d++) { + given_up[d].resize(card_experiment.GetModulesNum(d)); devices[d].PrepareAction(card_experiment); // Of the modules of the image, not the dummies expected_packets_per_image += devices[d].Counters().GetExpectedPacketsPerImage() @@ -39,10 +40,6 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con if ((experiment.GetDetectorType() != DetectorType::JUNGFRAU) || (modules_per_card > 0)) calibration = nullptr; - // Making a slot is slow and its buffers are written once here: none is made, nor first touched, during the run - for (size_t i = 0; i < max_slots; i++) - free_slots.push_back(std::make_shared(experiment)); - // The receiver's own start message is built from its experiment; this one only says the run has begun. StartMessage start{}; experiment.FillMessage(start); @@ -50,12 +47,8 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con .received = std::chrono::steady_clock::now()}); start_time = std::chrono::steady_clock::now(); - for (int i = 0; i < compression_threads; i++) - compress_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::CompressThread, this)); - - for (int d = 0; d < ndatastreams; d++) - for (int m = 0; m < card_experiment.GetModulesNum(d); m++) - module_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::ModuleThread, this, d, m)); + for (int i = 0; i < image_threads; i++) + image_futures.emplace_back(std::async(std::launch::async, &FPGAImagePuller::ImageThread, this)); std::latch ready(ndatastreams); for (int d = 0; d < ndatastreams; d++) @@ -82,8 +75,6 @@ FPGAImagePuller::~FPGAImagePuller() { } catch (const std::exception &e) { logger.Error("FPGA image puller teardown failed: {}", e.what()); } - // These hold slots, which go back to this object - outside_fifo.Clear(); } DiffractionExperiment FPGAImagePuller::CardExperiment(const DiffractionExperiment &x, int64_t modules_per_card) { @@ -104,10 +95,8 @@ void FPGAImagePuller::SetError(std::exception_ptr e) { } void FPGAImagePuller::Cancel() { - // An image waiting for a slot or for room in the queue is dropped from here on: after a cancel the receiver takes - // no more images + // An image waiting for room in the queue is dropped from here on: after a cancel the receiver takes no more images cancelled = true; - slot_returned.notify_all(); CancelCards(); } @@ -138,7 +127,7 @@ void FPGAImagePuller::AcquireThread(uint16_t data_stream, const PixelMask &mask, device.StartAction(card_experiment); } catch (...) { SetError(std::current_exception()); - CancelCards(); // the module threads of the other cards have to come out as well + CancelCards(); // the image threads waiting on the other cards have to come out as well ready.count_down(); return; } @@ -159,153 +148,90 @@ int64_t FPGAImagePuller::FastestFrame() const { return ret; } -// Polled: a dead card has to be noticed from the other cards' frames. -// A frame given up on can still arrive, late: its buffer goes back to the card, also while waiting, as the card -// may need it to write the frame waited for. -void FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame, int64_t &oldest_held) { - auto &device = devices[data_stream]; - const auto &counters = device.Counters(); +// The buffers of frames given up on that have come since go back to the card, which may need them for the frames +// waited for +void FPGAImagePuller::ReleaseLateFrames(uint16_t data_stream, uint16_t module_number) { + std::unique_lock ul(given_up_mutex); + auto &frames = given_up[data_stream][module_number]; + std::erase_if(frames, [&](int64_t f) { + if (!devices[data_stream].Counters().IsAnyPacketCollected(f, module_number)) + return false; + devices[data_stream].FrameBufferRelease(f, module_number); + return true; + }); +} + +// Polled: a dead card has to be noticed from the other cards' frames +bool FPGAImagePuller::WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) { + const auto &counters = devices[data_stream].Counters(); while (true) { - while ((oldest_held < frame) - && (oldest_held < static_cast(counters.GetCurrFrameNumber(module_number)))) - device.FrameBufferRelease(oldest_held++, module_number); + ReleaseLateFrames(data_stream, module_number); // A frame is complete once the module is two further on, as in JFJochReceiverFPGA if ((static_cast(counters.GetCurrFrameNumber(module_number)) >= frame + 2) - || counters.IsAcquisitionFinished() || failed - || (!blocking && (static_cast(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames)) - || (!blocking && (FastestFrame() >= frame + 2 + static_cast(max_slots)))) - return; + || counters.IsAcquisitionFinished() || failed) + break; + if (!blocking && ((static_cast(counters.GetFastestFrameNumber()) >= frame + 2 + MissingAfterFrames) + || (FastestFrame() >= frame + 2 + static_cast(queue)))) + break; std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + if (counters.IsAnyPacketCollected(frame, module_number)) + return true; + std::unique_lock ul(given_up_mutex); + given_up[data_stream][module_number].push_back(frame); + return false; } -// The last frame any card got; waits for all cards, so every module stops at the same image -int64_t FPGAImagePuller::LastFrame() const { - for (int d = 0; d < experiment.GetDataStreamsNum(); d++) - while (!devices[d].Counters().IsAcquisitionFinished() && !failed) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - return FastestFrame(); -} - -std::shared_ptr FPGAImagePuller::GetSlot(int64_t image_number) { - { - std::unique_lock ul(images_mutex); - std::shared_ptr slot; - while (!slot) { - if (image_dropped[image_number]) - return nullptr; - // Another module may have started the image while this one waited - if (const auto it = images_in_progress.find(image_number); it != images_in_progress.end()) - return it->second; - if (!free_slots.empty()) { - slot = free_slots.back(); - free_slots.pop_back(); - } else if (blocking && !failed && !cancelled) { - const auto t0 = std::chrono::steady_clock::now(); - slot_returned.wait_for(ul, std::chrono::milliseconds(10)); - slot_wait_ns += std::chrono::duration_cast( - std::chrono::steady_clock::now() - t0).count(); - } else { - image_dropped[image_number] = 1; - dropped_images++; - break; - } - } - if (slot) { - slot->image_number = image_number; - slot->metadata = std::make_unique(experiment); - slot->modules_left = static_cast(experiment.GetModulesNum()); - images_in_progress[image_number] = slot; - return slot; - } - } - // Not under images_mutex: Emit takes order_mutex, and a slot going back from under order_mutex takes images_mutex - Emit(image_number, std::nullopt); - return nullptr; -} - -void FPGAImagePuller::ReturnSlot(const std::shared_ptr &slot) { - { - std::unique_lock ul(images_mutex); - free_slots.push_back(slot); - } - slot_returned.notify_one(); -} - -void FPGAImagePuller::ModuleThread(uint16_t data_stream, uint16_t module_number) { +void FPGAImagePuller::ImageThread() { + // One image of its own, assembled and compressed here + FrameTransformation transformation(experiment); try { - auto &device = devices[data_stream]; - const auto &counters = device.Counters(); - const size_t module_abs = experiment.GetFirstModuleOfDataStream(data_stream) + module_number; - int64_t oldest_held = 0; - - for (int64_t image = 0; image < experiment.GetImageNum(); image++) { + for (int64_t image = next_to_take++; image < experiment.GetImageNum() && !failed; image = next_to_take++) { if (frame_rate_hz > 0) std::this_thread::sleep_until(start_time + std::chrono::duration_cast( std::chrono::duration(static_cast(image) / frame_rate_hz))); - WaitForFrame(data_stream, module_number, image, oldest_held); - if (failed) - break; - - if (counters.IsAcquisitionFinished() - && (image > static_cast(counters.GetFastestFrameNumber())) - && (image > LastFrame())) - break; - - const bool collected = counters.IsAnyPacketCollected(image, module_number); - // A dummy module (CardExperiment): its frames only go back to the card - const bool dummy = module_number >= experiment.GetModulesNum(data_stream); - if (const auto slot = dummy ? nullptr : GetSlot(image)) { - if (collected) { - const DeviceOutput *output = device.GetDeviceOutput(image, module_number); - slot->metadata->Process(output); - adu_histogram[module_abs]->Add(*output); - slot->transformation.ProcessModule(output, data_stream); - } else - slot->transformation.FillNotCollectedModule(module_number, data_stream); - - if (--slot->modules_left == 0) { - { - std::unique_lock ul(images_mutex); - images_in_progress.erase(image); + // Not blocking, an image the receiver has no room for is not assembled: its frames go back at once + const bool drop = !blocking && (GetCurrentFifoUtilization() >= queue); + ImageMetadata metadata(experiment); + for (int d = 0; d < experiment.GetDataStreamsNum(); d++) { + auto &device = devices[d]; + for (int m = 0; m < card_experiment.GetModulesNum(d); m++) { + const bool collected = WaitForFrame(d, m, image); + // A dummy module (CardExperiment): its frames only go back to the card + if (!drop && m < experiment.GetModulesNum(d)) { + if (collected) { + const DeviceOutput *output = device.GetDeviceOutput(image, m); + metadata.Process(output); + adu_histogram[experiment.GetFirstModuleOfDataStream(d) + m]->Add(*output); + transformation.ProcessModule(output, d); + } else + transformation.FillNotCollectedModule(m, d); } - images_to_compress.Put(slot); + if (collected) + device.FrameBufferRelease(image, m); } } - if (collected) - device.FrameBufferRelease(image, module_number); - } - } catch (...) { - SetError(std::current_exception()); - CancelCards(); - } -} -void FPGAImagePuller::CompressThread() { - while (const auto slot = images_to_compress.GetBlocking()) { - const int64_t image_number = slot->image_number; - try { DataMessage message{}; - message.number = image_number; - message.original_number = image_number; - slot->metadata->Export(message, expected_packets_per_image); + message.number = image; + message.original_number = image; + metadata.Export(message, expected_packets_per_image); + // No module of it came, or no room for it + if (drop || (message.image_collection_efficiency == 0.0f)) { + if (drop) + dropped_images++; + Emit(image, std::nullopt); + continue; + } // The card's spot finder and ROI map were not set up for this run; the receiver reports its own. message.strong_pixel_count.reset(); message.roi.clear(); - if (message.image_collection_efficiency == 0.0f) { // no module of it came - ReturnSlot(slot); - Emit(image_number, std::nullopt); - continue; - } - - // The compressed image goes to a buffer of the output's own, and the slot back at once: the slots are - // held only while an image is assembled and compressed, not while it waits for the receiver - const CompressedImage compressed = slot->transformation.GetCompressedImage(); + // The compressed image, copied once into a buffer of the output's own + const CompressedImage compressed = transformation.GetCompressedImage(); auto bytes = std::make_shared>(compressed.GetCompressed(), compressed.GetCompressed() + compressed.GetCompressedSize()); - ReturnSlot(slot); message.image = CompressedImage(bytes->data(), bytes->size(), compressed.GetWidth(), compressed.GetHeight(), compressed.GetMode(), compressed.GetCompressionAlgorithm()); @@ -313,42 +239,31 @@ void FPGAImagePuller::CompressThread() { output.cbor = std::make_shared(message); output.raw = std::shared_ptr(new ReceiveBufferRegion{bytes->data(), bytes->size()}, [bytes](ReceiveBufferRegion *region) { delete region; }); - Emit(image_number, std::move(output)); - } catch (...) { - SetError(std::current_exception()); - CancelCards(); - ReturnSlot(slot); - Emit(image_number, std::nullopt); + Emit(image, std::move(output)); } + } catch (...) { + SetError(std::current_exception()); + CancelCards(); } } // Whichever thread finishes the image next in line lets it, and those waiting behind it, out. void FPGAImagePuller::Emit(int64_t image_number, std::optional output) { - // The copies of the images let out here go only once order_mutex is released (the lock is declared after them, and - // `output` outlives both): the last copy of an image returns its slot, under images_mutex - std::vector released; std::unique_lock ul(order_mutex); - images_done.try_emplace(image_number, output); + images_done.try_emplace(image_number, std::move(output)); for (auto it = images_done.begin(); (it != images_done.end()) && (it->first == next_image); it = images_done.erase(it), next_image++) { if (it->second) { it->second->received = std::chrono::steady_clock::now(); - // The queue holds as many images as there are slots: blocking, the image waits for room; not - // blocking, it is dropped + // Blocking, the image waits for room; not blocking, it is dropped if (!(blocking ? PutWhileNotCancelled(*it->second) : outside_fifo.Put(*it->second))) dropped_images++; - released.push_back(std::move(*it->second)); } } } void FPGAImagePuller::FinalizeMeasurement() { - for (auto &f: module_futures) - f.get(); - for (size_t i = 0; i < compress_futures.size(); i++) - images_to_compress.Put(nullptr); - for (auto &f: compress_futures) + for (auto &f: image_futures) f.get(); CancelCards(); @@ -357,16 +272,16 @@ void FPGAImagePuller::FinalizeMeasurement() { // Only an error leaves images unfinished, holding back those behind them { - std::map> left; // destroyed after order_mutex, as in Emit std::unique_lock ul(order_mutex); for (auto &[n, output]: images_done) if (output && !PutWhileNotCancelled(*output)) dropped_images++; - left.swap(images_done); + images_done.clear(); } if (dropped_images > 0) - logger.Warning("FPGA image puller: {} images dropped, no free slot", dropped_images.load()); + logger.Warning("FPGA image puller: {} images dropped, the receiver's queue of {} images full", dropped_images.load(), + queue); EndMessage end{}; for (size_t i = 0; i < adu_histogram.size(); i++) @@ -377,8 +292,10 @@ void FPGAImagePuller::FinalizeMeasurement() { // Waits for room in the queue, as long as the receiver takes from it: after a cancel it no longer does bool FPGAImagePuller::PutWhileNotCancelled(const ImagePullerOutput &output) { - while (!outside_fifo.PutTimeout(output, std::chrono::milliseconds(100))) - if (cancelled) - return false; - return true; + const auto t0 = std::chrono::steady_clock::now(); + bool ret = true; + while (ret && !outside_fifo.PutTimeout(output, std::chrono::milliseconds(100))) + ret = !cancelled; + queue_wait_ns += std::chrono::duration_cast(std::chrono::steady_clock::now() - t0).count(); + return ret; } diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index f34d862f2..9a8bc31fb 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -4,7 +4,6 @@ #pragma once #include -#include #include #include #include @@ -20,84 +19,73 @@ #include "../image_puller/ImagePuller.h" #include "../jungfrau/JFCalibration.h" -// The images of a collection from the FPGA cards, as a detector stream: assembled and compressed here, in image order +// The images of a collection from the FPGA cards, as a detector stream, in image order. As JFJochReceiverFPGA: each +// thread takes the next image, waits for its modules in the cards' buffers, assembles them into an image of its own +// and compresses it; the image leaves as its compressed bytes, and the cards' buffers go back. The cards' buffers are +// the only window over the images in flight. class FPGAImagePuller : public ImagePuller { public: - static constexpr size_t DefaultSlots = 128; + // Images compressed and waiting for the receiver + static constexpr size_t DefaultQueue = 128; // A module this far behind the fastest one of its card is taken as missing from that frame, as the device itself - // does. Behind another card it is waited for as long as the slots can hold the images started meanwhile: the - // cards run on their own and one can be ahead, but past max_slots its images would be dropped anyway. + // does. Behind another card it is waited for until that card is `queue` images ahead (not blocking). static constexpr int64_t MissingAfterFrames = AcquisitionCounters::ThresholdFramesLost + 2; private: - struct Slot { - explicit Slot(const DiffractionExperiment &experiment) : transformation(experiment) { - transformation.ReserveCompressedBuffer(); - } - FrameTransformation transformation; - std::unique_ptr metadata; - std::atomic modules_left = 0; - int64_t image_number = 0; - }; - const DiffractionExperiment experiment; // What the cards run: the experiment, or with dummy modules (CardExperiment) const DiffractionExperiment card_experiment; Logger &logger; AcquisitionDeviceGroup &devices; - const size_t max_slots; - // Blocking: an image waits for a free slot instead of being dropped, and no module is given up for being behind - // the others, so the cards hold their frames and stall at the source. + const size_t queue; + // Blocking: an image waits for room in the queue instead of being dropped, and no module is given up for being + // behind the others, so the cards keep their frames and stall at the source. const bool blocking; const double frame_rate_hz; // images taken from the cards at most at this rate (0: as they come) std::chrono::steady_clock::time_point start_time; - std::condition_variable slot_returned; - std::atomic slot_wait_ns = 0; + std::atomic queue_wait_ns = 0; uint64_t expected_packets_per_image = 0; - std::mutex images_mutex; - std::vector> free_slots; - std::map> images_in_progress; - std::vector image_dropped; + std::atomic next_to_take = 0; std::atomic dropped_images = 0; + // Frames given up on that may still arrive, by card and module: their buffers go back once they do + std::mutex given_up_mutex; + std::vector>> given_up; + void ReleaseLateFrames(uint16_t data_stream, uint16_t module_number); + // What is done, waiting for the images before it. No value: that image will not come. std::mutex order_mutex; std::map> images_done; int64_t next_image = 0; - ThreadSafeFIFO> images_to_compress; std::vector> adu_histogram; std::mutex error_mutex; std::exception_ptr error; std::atomic failed = false; // a failed card may never report finished: nothing waits for it then - std::atomic cancelled = false; // Cancel(): no image waits for a slot any more + std::atomic cancelled = false; // Cancel(): no image waits for the receiver any more void SetError(std::exception_ptr e); std::vector> acquire_futures; - std::vector> module_futures; - std::vector> compress_futures; + std::vector> image_futures; std::future finish_future; std::once_flag finish_once; void AcquireThread(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration, std::latch &ready); - void ModuleThread(uint16_t data_stream, uint16_t module_number); - void CompressThread(); + void ImageThread(); void FinalizeMeasurement(); - void WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame, int64_t &oldest_held); + // False where the frame is given up on + bool WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame); int64_t FastestFrame() const; - int64_t LastFrame() const; - std::shared_ptr GetSlot(int64_t image_number); - void ReturnSlot(const std::shared_ptr &slot); void Emit(int64_t image_number, std::optional output); bool PutWhileNotCancelled(const ImagePullerOutput &output); void CancelCards(); public: // Arms the cards; when it returns the detector can be started. FPGAImagePuller(const DiffractionExperiment &experiment, const PixelMask &mask, const JFCalibration *calibration, - AcquisitionDeviceGroup &devices, Logger &logger, int64_t compression_threads, - size_t max_slots = DefaultSlots, bool blocking = false, double frame_rate_hz = 0, + AcquisitionDeviceGroup &devices, Logger &logger, int64_t image_threads, + size_t queue = DefaultQueue, bool blocking = false, double frame_rate_hz = 0, int64_t modules_per_card = 0); // The experiment the cards run where each carries modules_per_card modules (0: as `x` splits them): each card's @@ -110,8 +98,8 @@ public: void Disconnect() override {} size_t GetDroppedImages() const override { return dropped_images; } bool IsBlocking() const override { return blocking; } - // Time the module threads waited for a free slot, summed over them - double GetSlotWaitTime_s() const { return static_cast(slot_wait_ns) * 1e-9; } + // Time the image threads waited for room in the queue, summed over them + double GetQueueWaitTime_s() const { return static_cast(queue_wait_ns) * 1e-9; } // The collection is over, whatever the cards have got: what is on its way still comes, then the end message. void Cancel(); diff --git a/receiver/JFJochReceiverService.cpp b/receiver/JFJochReceiverService.cpp index 988171da7..50d9d892d 100644 --- a/receiver/JFJochReceiverService.cpp +++ b/receiver/JFJochReceiverService.cpp @@ -40,13 +40,13 @@ JFJochReceiverService &JFJochReceiverService::RugnuxLive(bool enable, std::share return *this; } -JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t slots, bool blocking, double frame_rate_hz, +JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t queue, bool blocking, double frame_rate_hz, bool always, int64_t modules_per_card) { std::unique_lock ul_state(state_mutex); if (state != ReceiverState::Idle) throw JFJochException(JFJochExceptionCategory::WrongDAQState, "Cannot change the receiver path during data collection"); - fpga_image_puller_slots = slots; + fpga_image_puller_queue = queue; fpga_image_puller_blocking = blocking; fpga_image_puller_rate_hz = frame_rate_hz; fpga_image_puller_always = always; @@ -54,9 +54,9 @@ JFJochReceiverService &JFJochReceiverService::FPGAImagePullerSettings(size_t slo return *this; } -double JFJochReceiverService::GetFPGAImagePullerSlotWaitTime_s() const { +double JFJochReceiverService::GetFPGAImagePullerQueueWaitTime_s() const { std::unique_lock ul(state_mutex); - return fpga_puller ? fpga_puller->GetSlotWaitTime_s() : 0.0; + return fpga_puller ? fpga_puller->GetQueueWaitTime_s() : 0.0; } // A PSI collection analysed by rugnux live; summation on the host needs JFJochReceiverFPGA @@ -146,7 +146,7 @@ void JFJochReceiverService::Start(const DiffractionExperiment &experiment, // A PSI detector's frames through FPGAImagePuller go to the stream receivers, as a DECTRIS stream does if (FPGAImagePullerFor(experiment)) { fpga_puller = std::make_shared(experiment, pixel_mask, calibration, aq_devices, logger, - nthreads, fpga_image_puller_slots, + nthreads, fpga_image_puller_queue, fpga_image_puller_blocking, fpga_image_puller_rate_hz, fpga_image_puller_modules_per_card); puller = fpga_puller; diff --git a/receiver/JFJochReceiverService.h b/receiver/JFJochReceiverService.h index 3ac3cbed2..86672a0c0 100644 --- a/receiver/JFJochReceiverService.h +++ b/receiver/JFJochReceiverService.h @@ -44,7 +44,7 @@ class JFJochReceiverService { // A PSI collection analysed by rugnux live goes through FPGAImagePuller to the stream receivers bool fpga_image_puller_always = false; int64_t fpga_image_puller_modules_per_card = 0; - size_t fpga_image_puller_slots = FPGAImagePuller::DefaultSlots; + size_t fpga_image_puller_queue = FPGAImagePuller::DefaultQueue; bool fpga_image_puller_blocking = false; double fpga_image_puller_rate_hz = 0; std::shared_ptr fpga_puller; @@ -77,14 +77,14 @@ public: int64_t stills_downsample = 1); RugnuxLiveSnapshot GetRugnuxStatus() const; void ClearRugnuxStatus(); // back to Idle, as before any collection - // slots, blocking, frame_rate_hz, modules_per_card: see FPGAImagePuller. always: every PSI collection but pedestal + // queue, blocking, frame_rate_hz, modules_per_card: see FPGAImagePuller. always: every PSI collection but pedestal // and host summation through the puller, without rugnux to JFJochReceiverLite (for tests comparing it with // JFJochReceiverFPGA) - JFJochReceiverService& FPGAImagePullerSettings(size_t slots = FPGAImagePuller::DefaultSlots, bool blocking = false, + JFJochReceiverService& FPGAImagePullerSettings(size_t queue = FPGAImagePuller::DefaultQueue, bool blocking = false, double frame_rate_hz = 0, bool always = false, int64_t modules_per_card = 0); - // Of the last collection through FPGAImagePuller: FPGAImagePuller::GetSlotWaitTime_s, 0 without one - double GetFPGAImagePullerSlotWaitTime_s() const; + // Of the last collection through FPGAImagePuller: FPGAImagePuller::GetQueueWaitTime_s, 0 without one + double GetFPGAImagePullerQueueWaitTime_s() const; JFJochReceiverService& PreviewSocket(const std::string &addr, const std::optional &watermark = {}); JFJochReceiverService& MetadataSocket(const std::string &addr); diff --git a/tests/FPGAImagePullerTest.cpp b/tests/FPGAImagePullerTest.cpp index 178248b91..35bee6f41 100644 --- a/tests/FPGAImagePullerTest.cpp +++ b/tests/FPGAImagePullerTest.cpp @@ -166,7 +166,7 @@ namespace { JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4); service.SetSpotFindingSettings(Spots()); - service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, puller); service.RugnuxLive(rugnux, nullptr, 2, 30); PixelMask mask(x); @@ -420,7 +420,7 @@ TEST_CASE("FPGAImagePuller_SameAsFPGA_EIGER_32bit", "[FPGAImagePuller]") { CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); - service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, puller); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, puller); PixelMask mask(x); service.Start(x, mask, nullptr); const auto out = service.Stop(); @@ -495,7 +495,7 @@ TEST_CASE("FPGAImagePuller_SlowConsumer_Blocking", "[FPGAImagePuller]") { CHECK(end); puller.Stop(); CHECK(puller.GetDroppedImages() == 0); - CHECK(puller.GetSlotWaitTime_s() > 0.5); + CHECK(puller.GetQueueWaitTime_s() > 0.5); REQUIRE(numbers.size() == 20); for (size_t i = 0; i < numbers.size(); i++) CHECK(numbers[i] == static_cast(i)); @@ -716,7 +716,7 @@ TEST_CASE("FPGAImagePuller_Cancel", "[FPGAImagePuller]") { CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); - service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); @@ -751,7 +751,7 @@ TEST_CASE("FPGAImagePuller_NotForPedestalAndHostSummation", "[FPGAImagePuller]") CollectingPusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(4).SetSpotFindingSettings(Spots()); - service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); @@ -922,7 +922,7 @@ TEST_CASE("FPGAImagePuller_Writer", "[FPGAImagePuller]") { HDF5FilePusher pusher; JFJochReceiverService service(devices, logger, pusher, 256); service.NumThreads(2).SetSpotFindingSettings(Spots()); - service.FPGAImagePullerSettings(FPGAImagePuller::DefaultSlots, false, 0, true); + service.FPGAImagePullerSettings(FPGAImagePuller::DefaultQueue, false, 0, true); PixelMask mask(x); const JFCalibration calib = Calibration(x); service.Start(x, mask, &calib); diff --git a/tools/jfjoch_fpga_stills_test.cpp b/tools/jfjoch_fpga_stills_test.cpp index 38cf67851..4fd6a8282 100644 --- a/tools/jfjoch_fpga_stills_test.cpp +++ b/tools/jfjoch_fpga_stills_test.cpp @@ -42,7 +42,7 @@ void print_usage(Logger &logger) { logger.Info(" -t Indexing thread pool size (default: 8, as offline rugnux on stills; the broker's default is 4)"); logger.Info(" -r Rates the source is paced at, one collection each (default: 0 = as fast as it goes); the"); logger.Info(" host takes frame i no earlier than i/rate, the card waits meanwhile: blocking runs only"); - logger.Info(" -p FPGAImagePuller slots (default 128)"); + logger.Info(" -p Images FPGAImagePuller keeps compressed for the receiver (default 128)"); logger.Info(" -U No dummy modules: where the cards carry different numbers of modules, they run apart"); logger.Info(" -k Analyse 1 of every images on average (rugnux live stills_downsample, default 1)"); logger.Info(" -d Not blocking, as the broker runs: images the host has no room for are dropped. The cards'"); @@ -92,7 +92,7 @@ int64_t GeneratorImages(AcquisitionDeviceGroup &aq_devices, const DiffractionExp // A stored dataset through the card and rugnux live stills; returns false where the run failed bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::string &file, std::optional nimages, uint16_t nstreams, uint16_t nthreads, size_t send_buffer_size_MiB, - const IndexingSettings &indexing, const std::optional &cell, size_t puller_slots, + const IndexingSettings &indexing, const std::optional &cell, size_t puller_queue, const std::vector &rates, bool dropping, bool pad, bool verbose, int64_t downsample, const std::optional &space_group) { RegisterHDF5Filter(); @@ -195,9 +195,9 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s for (const double rate: rates) { // Blocking at source: the puller waits for a free slot and the receiver for room in the image buffer, // the cards hold their frames and their generators stall - service.FPGAImagePullerSettings(puller_slots, !dropping, rate, false, modules_per_card); + service.FPGAImagePullerSettings(puller_queue, !dropping, rate, false, modules_per_card); logger.Info(""); - logger.Info("=== FPGAImagePuller ({} slots, {}), {} ===", puller_slots, dropping ? "dropping" : "blocking", + logger.Info("=== FPGAImagePuller (queue {}, {}), {} ===", puller_queue, dropping ? "dropping" : "blocking", (rate > 0) ? fmt::format("source paced at {:.1f} Hz", rate) : std::string("source as fast as it goes")); rusage usage_start{}; @@ -251,7 +251,7 @@ bool RunDataset(Logger &logger, AcquisitionDeviceGroup &aq_devices, const std::s // threads waited for a free slot, as a fraction of that span (mean over the modules) const double delivery_s = live.last_image_s.value_or(0); const double delivered_hz = (delivery_s > 0) ? static_cast(output.status.images_collected - 1) / delivery_s : 0; - const double blocked = (delivery_s > 0) ? service.GetFPGAImagePullerSlotWaitTime_s() / (delivery_s * x.GetModulesNum()) : 0; + const double blocked = (delivery_s > 0) ? service.GetFPGAImagePullerQueueWaitTime_s() / delivery_s : 0; logger.Info("Images expected {} collected {} sent {} dropped {} efficiency {:.4f}", x.GetImageNum(), output.status.images_collected, output.status.images_sent, @@ -325,7 +325,7 @@ int main(int argc, char **argv) { std::optional cell; IndexingAlgorithmEnum indexing = IndexingAlgorithmEnum::Auto; uint16_t indexing_threads = 8; // as offline rugnux's stills (Rugnux::StillsIndexerThreads) - size_t puller_slots = FPGAImagePuller::DefaultSlots; + size_t puller_queue = FPGAImagePuller::DefaultQueue; std::optional mock_card_MiB; bool verbose = false; std::vector rates = {0}; @@ -383,7 +383,7 @@ int main(int argc, char **argv) { break; } case 'p': - puller_slots = atol(optarg); + puller_queue = atol(optarg); break; case 'd': dropping = true; @@ -445,7 +445,7 @@ int main(int argc, char **argv) { bool ret = false; try { ret = RunDataset(logger, aq_devices, argv[optind], nimages, nstreams, nthreads, send_buffer_size_MiB, - i_settings, cell, puller_slots, rates, dropping, pad, verbose, downsample, space_group); + i_settings, cell, puller_queue, rates, dropping, pad, verbose, downsample, space_group); } catch (std::exception &e) { logger.Error(e.what()); }