From 25ac0071b2b7297f6423806060bb832d733dce27 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 7 Nov 2026 16:19:44 +0100 Subject: [PATCH] Calibration goes to the writer only from JFJochReceiverFPGA Pedestal and RMS images are very bulky, so the FPGA image puller no longer prepares them and Lite and Rugnux no longer forward calibration messages. The DECTRIS stream does not carry any, so nothing is lost there. The card itself still gets its calibration through InitializeCalibration, which is how it converts; only the copy for the writer is gone on the puller path, also when SaveCalibration is set. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK --- image_puller/ImagePuller.h | 3 --- receiver/FPGAImagePuller.cpp | 25 ------------------------- receiver/FPGAImagePuller.h | 5 ----- receiver/JFJochReceiverLite.cpp | 7 ------- receiver/JFJochReceiverRugnux.cpp | 22 ---------------------- receiver/JFJochReceiverRugnux.h | 5 ----- tests/FPGAImagePullerTest.cpp | 8 +++++--- tests/JFJochReceiverRugnuxTest.cpp | 13 ++----------- 8 files changed, 7 insertions(+), 81 deletions(-) diff --git a/image_puller/ImagePuller.h b/image_puller/ImagePuller.h index a7e679ecb..ae09ff9da 100644 --- a/image_puller/ImagePuller.h +++ b/image_puller/ImagePuller.h @@ -83,9 +83,6 @@ public: virtual void Disconnect() = 0; - // To go to the writer right after the start message, before any image: what a detector stream sends - // as calibration messages can reach the writer after the first image, where it is refused. - virtual std::vector GetCalibration() const { return {}; } // Images the puller had no room for, which the receiver therefore never gets. virtual size_t GetDroppedImages() const { return 0; } }; diff --git a/receiver/FPGAImagePuller.cpp b/receiver/FPGAImagePuller.cpp index 4a4d81a22..fa6740bf1 100644 --- a/receiver/FPGAImagePuller.cpp +++ b/receiver/FPGAImagePuller.cpp @@ -6,7 +6,6 @@ #include #include -#include "JFJochCompressor.h" FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask, const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices, @@ -32,8 +31,6 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con if (experiment.GetDetectorType() != DetectorType::JUNGFRAU) calibration = nullptr; - if (calibration != nullptr && experiment.GetSaveCalibration()) - PrepareCalibration(*calibration); // Making a slot takes tens of milliseconds on a large detector, under the lock all modules go through: // the first ones are ready when the frames start. The rest only if the receiver falls behind. @@ -104,28 +101,6 @@ void FPGAImagePuller::Stop() { std::rethrow_exception(std::exchange(error, nullptr)); } -void FPGAImagePuller::PrepareCalibration(const JFCalibration &calibration) { - JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); - - const auto add = [&](const std::string &channel, std::vector v) { - calibration_data.emplace_back(std::move(v)); - calibration_images.emplace_back(calibration_data.back().data(), calibration_data.back().size(), - RAW_MODULE_COLS, experiment.GetModulesNum() * RAW_MODULE_LINES, - CompressedImageMode::Uint16, CompressionAlgorithm::BSHUF_LZ4, channel); - }; - for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) { - for (int gain = 0; gain < 3; gain++) { - if (experiment.IsFixedGainG1() && (gain != 1)) - continue; - const std::string suffix = (experiment.GetStorageCellNumber() > 1) - ? fmt::format("_g{:d}_sc{:d}", gain, sc) - : fmt::format("_g{:d}", gain); - add("pedestal" + suffix, compressor.Compress(calibration.GetPedestal(gain, sc))); - add("pedestal_rms" + suffix, compressor.Compress(calibration.GetPedestalRMS(gain, sc))); - } - } -} - void FPGAImagePuller::Acquire(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration, std::latch &ready) { auto &device = devices[data_stream]; diff --git a/receiver/FPGAImagePuller.h b/receiver/FPGAImagePuller.h index 711b36cf4..e25d2d2a1 100644 --- a/receiver/FPGAImagePuller.h +++ b/receiver/FPGAImagePuller.h @@ -64,9 +64,6 @@ private: ThreadSafeFIFO> images_to_compress; std::vector> adu_histogram; - std::vector> calibration_data; - std::vector calibration_images; - std::mutex error_mutex; std::exception_ptr error; std::atomic failed = false; // a failed card may never report finished: nothing waits for it then @@ -83,7 +80,6 @@ private: void CompressImages(); void Finish(); - void PrepareCalibration(const JFCalibration &calibration); void WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) const; int64_t FastestFrame() const; int64_t LastFrame() const; @@ -98,7 +94,6 @@ public: ~FPGAImagePuller() override; void Disconnect() override {} - std::vector GetCalibration() const override { return calibration_images; } size_t GetDroppedImages() const override { return dropped_images; } // The collection is over, whatever the cards have got: what is on its way still comes, then the end message. diff --git a/receiver/JFJochReceiverLite.cpp b/receiver/JFJochReceiverLite.cpp index 1a7dd726d..a58410c5a 100644 --- a/receiver/JFJochReceiverLite.cpp +++ b/receiver/JFJochReceiverLite.cpp @@ -156,9 +156,6 @@ void JFJochReceiverLite::MeasurementThread() { start_time = std::chrono::system_clock::now(); // Send new start message out SendStartMessage(); - if (push_images_to_writer) - for (const auto &c: image_puller.GetCalibration()) - image_pusher.SendCalibration(c); measurement_started.count_down(); } catch (const JFJochException &e) { @@ -326,10 +323,6 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) { end_adu_histogram = msg->cbor->end_message->adu_histogram; end_message_received = true; logger.Debug("Thread {} End message received in JFJochReceiverLite", id); - } else if (msg->cbor->calibration) { - // Calibration messages are just forwarded - if (push_images_to_writer) - image_pusher.SendCalibration(msg->cbor->calibration.value()); } else if (msg->cbor->data_message) { auto start = std::chrono::high_resolution_clock::now(); DataMessage data_msg = msg->cbor->data_message.value(); diff --git a/receiver/JFJochReceiverRugnux.cpp b/receiver/JFJochReceiverRugnux.cpp index 01feb229e..6538fc356 100644 --- a/receiver/JFJochReceiverRugnux.cpp +++ b/receiver/JFJochReceiverRugnux.cpp @@ -281,9 +281,6 @@ void JFJochReceiverRugnux::MeasurementThread() { StartStills(); else { SendStartMessage(); - if (push_images_to_writer) - for (const auto &c: image_puller.GetCalibration()) - image_pusher.SendCalibration(c); // Rugnux reads the images back from the buffer, so it needs one slot per image: a slot taken // again would hand it a later image. Without that the collection is only forwarded. @@ -397,13 +394,7 @@ void JFJochReceiverRugnux::StartStills() { [this](const StillsGPUAnalysis::Setup &s) { // The profiles are binned on the pre-pass's geometry. plots.Setup(s.experiment, *s.mapping); - std::unique_lock ul(calibration_m); SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping)); - start_sent = true; - if (push_images_to_writer) - for (const auto &c: calibration_held) - image_pusher.SendCalibration(c.cbor->calibration.value()); - calibration_held.clear(); }, [this](DataMessage &m, const AzimuthalIntegrationProfile &p) { SendAnalysedImage(m, p); }, [this] { @@ -506,14 +497,6 @@ void JFJochReceiverRugnux::HoldStillsImage(const DataMessage &message) { stills->Hold(held); } -void JFJochReceiverRugnux::StillsCalibration(const ImagePullerOutput &msg) { - std::unique_lock ul(calibration_m); - if (!start_sent) - calibration_held.push_back(msg); - else if (push_images_to_writer) - image_pusher.SendCalibration(msg.cbor->calibration.value()); -} - ZeroCopyReturnValue *JFJochReceiverRugnux::AnalysedImageSlot() { // An analysed frame is not dropped because the writer lags behind the catch-up from a full buffer: the analysis // waits for the slots being sent to come back, and the frames arriving meanwhile find the buffer full. A @@ -797,13 +780,8 @@ void JFJochReceiverRugnux::ForwardThread(uint32_t id) { continue; if (msg->cbor->end_message) { end_message_received = true; - } else if (stills_run && msg->cbor->calibration) { - StillsCalibration(*msg); } else if (stills_run && msg->cbor->data_message) { HoldStillsImage(msg->cbor->data_message.value()); - } else if (msg->cbor->calibration) { - if (push_images_to_writer) - image_pusher.SendCalibration(msg->cbor->calibration.value()); } else if (msg->cbor->data_message) { live_status.ImageReceived(msg->received); DataMessage data_msg = msg->cbor->data_message.value(); diff --git a/receiver/JFJochReceiverRugnux.h b/receiver/JFJochReceiverRugnux.h index c6f226477..1065f449c 100644 --- a/receiver/JFJochReceiverRugnux.h +++ b/receiver/JFJochReceiverRugnux.h @@ -61,16 +61,11 @@ class JFJochReceiverRugnux : public JFJochReceiver { IndexerThreadPool *const indexing_thread_pool; const int64_t stills_nthreads; int64_t stills_keep_free = 0; // slots frames do not take, for the analysis threads to send with - // Calibration that came while the pre-pass held the start message back. - std::mutex calibration_m; - std::vector calibration_held; - bool start_sent = false; std::unique_ptr stills; // under rugnux_m until the frames are in, for Cut void StartStills(); void StillsMeasurement(); void HoldStillsImage(const DataMessage &message); - void StillsCalibration(const ImagePullerOutput &msg); ZeroCopyReturnValue *AnalysedImageSlot(); void SendAnalysedImage(DataMessage &message, const AzimuthalIntegrationProfile &profile); diff --git a/tests/FPGAImagePullerTest.cpp b/tests/FPGAImagePullerTest.cpp index da2aff5ce..7ad94e9fc 100644 --- a/tests/FPGAImagePullerTest.cpp +++ b/tests/FPGAImagePullerTest.cpp @@ -303,12 +303,14 @@ namespace { CHECK(a.end.max_image_number == b.end.max_image_number); CHECK(a.end.adu_histogram == b.end.adu_histogram); - // Calibration between start and the first image, in the order the FPGA path sends it + // Calibration goes to the writer only from the FPGA receiver CHECK(a.sequence.front() == "start"); CHECK(b.sequence.front() == "start"); CHECK(a.sequence.back() == "end"); CHECK(b.sequence.back() == "end"); - CHECK(a.sequence == b.sequence); + auto sequence_a = a.sequence; + std::erase_if(sequence_a, [](const std::string &s) { return s.starts_with("calibration"); }); + CHECK(sequence_a == b.sequence); int64_t displacement = 0; for (size_t i = 0; i < b.arrival.size(); i++) @@ -667,7 +669,7 @@ TEST_CASE("FPGAImagePuller_RugnuxReceiver", "[FPGAImagePuller]") { CHECK(pusher.images.size() == 6); CHECK(pusher.sequence.front() == "start"); CHECK(pusher.sequence.back() == "end"); - CHECK(pusher.sequence[1].starts_with("calibration")); + CHECK_FALSE(pusher.sequence[1].starts_with("calibration")); // the writer gets calibration only from the FPGA receiver const auto a = Pixels(pusher.images.at(0).message.image); const auto b = Pixels(pusher.images.at(3).message.image); CHECK(a == b); // the third frame in the generator's memory repeats diff --git a/tests/JFJochReceiverRugnuxTest.cpp b/tests/JFJochReceiverRugnuxTest.cpp index 0f030e89d..316b5a9da 100644 --- a/tests/JFJochReceiverRugnuxTest.cpp +++ b/tests/JFJochReceiverRugnuxTest.cpp @@ -102,13 +102,6 @@ namespace { } } - void Calibration() { - static const std::vector pedestal(4, 1); - auto cbor = std::make_shared(); - cbor->calibration = CompressedImage(pedestal, 2, 2); - puller->Put(ImagePullerOutput{.cbor = cbor}); - } - void End() { puller->Put(ImagePullerOutput{.cbor = std::make_shared(EndMessage{})}); } @@ -140,8 +133,6 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") { PixelMask mask(x); Stream stream(x, images); - // Calibration ahead of the frames waits for the start message, which waits for the pre-pass. - stream.Calibration(); stream.Frames(0, images); stream.End(); @@ -163,8 +154,8 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") { CHECK(out.status.indexing_rate.value() > 0.5f); CHECK(out.scan_result.images.size() == images); - // The writer sees the start message before the calibration and the frames, the end message last. - CHECK(pusher.Events() == std::vector{"start", "calibration", "image", "end"}); + // The writer sees the start message before the frames, the end message last. + CHECK(pusher.Events() == std::vector{"start", "image", "end"}); auto sent = pusher.Images(); std::sort(sent.begin(), sent.end()); REQUIRE(sent.size() == images);