mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 93 Wiki Activity

Calibration goes to the writer only from JFJochReceiverFPGA

Browse Source 
Pedestal and RMS images are very bulky, so the FPGA image puller no longer prepares them and Lite and
Rugnux no longer forward calibration messages. The DECTRIS stream does not carry any, so nothing is
lost there. The card itself still gets its calibration through InitializeCalibration, which is how it
converts; only the copy for the writer is gone on the puller path, also when SaveCalibration is set.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HVeDQsU5a8AJUdJPgWevAK
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-11-07 16:19:44 +01:00
1 parent 15266b21ee
commit 25ac0071b2
8 files changed
+7 -81
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

image_puller/ImagePuller.h
-3
View File
@@ -83,9 +83,6 @@ public:
virtual void Disconnect() = 0;
// To go to the writer right after the start message, before any image: what a detector stream sends
// as calibration messages can reach the writer after the first image, where it is refused.
virtual std::vector<CompressedImage> GetCalibration() const { return {}; }
// Images the puller had no room for, which the receiver therefore never gets.
virtual size_t GetDroppedImages() const { return 0; }
};
receiver/FPGAImagePuller.cpp
-25
View File
@@ -6,7 +6,6 @@
#include <thread>
#include <utility>
#include "JFJochCompressor.h"
FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, const PixelMask &mask,
const JFCalibration *calibration, AcquisitionDeviceGroup &in_devices,
@@ -32,8 +31,6 @@ FPGAImagePuller::FPGAImagePuller(const DiffractionExperiment &in_experiment, con
if (experiment.GetDetectorType() != DetectorType::JUNGFRAU)
calibration = nullptr;
if (calibration != nullptr && experiment.GetSaveCalibration())
PrepareCalibration(*calibration);
// Making a slot takes tens of milliseconds on a large detector, under the lock all modules go through:
// the first ones are ready when the frames start. The rest only if the receiver falls behind.
@@ -104,28 +101,6 @@ void FPGAImagePuller::Stop() {
std::rethrow_exception(std::exchange(error, nullptr));
}
void FPGAImagePuller::PrepareCalibration(const JFCalibration &calibration) {
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
const auto add = [&](const std::string &channel, std::vector<uint8_t> v) {
calibration_data.emplace_back(std::move(v));
calibration_images.emplace_back(calibration_data.back().data(), calibration_data.back().size(),
RAW_MODULE_COLS, experiment.GetModulesNum() * RAW_MODULE_LINES,
CompressedImageMode::Uint16, CompressionAlgorithm::BSHUF_LZ4, channel);
};
for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) {
for (int gain = 0; gain < 3; gain++) {
if (experiment.IsFixedGainG1() && (gain != 1))
continue;
const std::string suffix = (experiment.GetStorageCellNumber() > 1)
? fmt::format("_g{:d}_sc{:d}", gain, sc)
: fmt::format("_g{:d}", gain);
add("pedestal" + suffix, compressor.Compress(calibration.GetPedestal(gain, sc)));
add("pedestal_rms" + suffix, compressor.Compress(calibration.GetPedestalRMS(gain, sc)));
}
}
}
void FPGAImagePuller::Acquire(uint16_t data_stream, const PixelMask &mask, const JFCalibration *calibration,
std::latch &ready) {
auto &device = devices[data_stream];
receiver/FPGAImagePuller.h
-5
View File
@@ -64,9 +64,6 @@ private:
ThreadSafeFIFO<std::shared_ptr<Slot>> images_to_compress;
std::vector<std::unique_ptr<ADUHistogram>> adu_histogram;
std::vector<std::vector<uint8_t>> calibration_data;
std::vector<CompressedImage> calibration_images;
std::mutex error_mutex;
std::exception_ptr error;
std::atomic<bool> failed = false; // a failed card may never report finished: nothing waits for it then
@@ -83,7 +80,6 @@ private:
void CompressImages();
void Finish();
void PrepareCalibration(const JFCalibration &calibration);
void WaitForFrame(uint16_t data_stream, uint16_t module_number, int64_t frame) const;
int64_t FastestFrame() const;
int64_t LastFrame() const;
@@ -98,7 +94,6 @@ public:
~FPGAImagePuller() override;
void Disconnect() override {}
std::vector<CompressedImage> GetCalibration() const override { return calibration_images; }
size_t GetDroppedImages() const override { return dropped_images; }
// The collection is over, whatever the cards have got: what is on its way still comes, then the end message.
receiver/JFJochReceiverLite.cpp
-7
View File
@@ -156,9 +156,6 @@ void JFJochReceiverLite::MeasurementThread() {
start_time = std::chrono::system_clock::now();
// Send new start message out
SendStartMessage();
if (push_images_to_writer)
for (const auto &c: image_puller.GetCalibration())
image_pusher.SendCalibration(c);
measurement_started.count_down();
} catch (const JFJochException &e) {
@@ -326,10 +323,6 @@ void JFJochReceiverLite::DataAnalysisThread(uint32_t id) {
end_adu_histogram = msg->cbor->end_message->adu_histogram;
end_message_received = true;
logger.Debug("Thread {} End message received in JFJochReceiverLite", id);
} else if (msg->cbor->calibration) {
// Calibration messages are just forwarded
if (push_images_to_writer)
image_pusher.SendCalibration(msg->cbor->calibration.value());
} else if (msg->cbor->data_message) {
auto start = std::chrono::high_resolution_clock::now();
DataMessage data_msg = msg->cbor->data_message.value();
receiver/JFJochReceiverRugnux.cpp
-22
View File
@@ -281,9 +281,6 @@ void JFJochReceiverRugnux::MeasurementThread() {
StartStills();
else {
SendStartMessage();
if (push_images_to_writer)
for (const auto &c: image_puller.GetCalibration())
image_pusher.SendCalibration(c);
// Rugnux reads the images back from the buffer, so it needs one slot per image: a slot taken
// again would hand it a later image. Without that the collection is only forwarded.
@@ -397,13 +394,7 @@ void JFJochReceiverRugnux::StartStills() {
[this](const StillsGPUAnalysis::Setup &s) {
// The profiles are binned on the pre-pass's geometry.
plots.Setup(s.experiment, *s.mapping);
std::unique_lock ul(calibration_m);
SendStartMessage(MakeStartMessage(s.experiment, s.mask, *s.mapping));
start_sent = true;
if (push_images_to_writer)
for (const auto &c: calibration_held)
image_pusher.SendCalibration(c.cbor->calibration.value());
calibration_held.clear();
},
[this](DataMessage &m, const AzimuthalIntegrationProfile &p) { SendAnalysedImage(m, p); },
[this] {
@@ -506,14 +497,6 @@ void JFJochReceiverRugnux::HoldStillsImage(const DataMessage &message) {
stills->Hold(held);
}
void JFJochReceiverRugnux::StillsCalibration(const ImagePullerOutput &msg) {
std::unique_lock ul(calibration_m);
if (!start_sent)
calibration_held.push_back(msg);
else if (push_images_to_writer)
image_pusher.SendCalibration(msg.cbor->calibration.value());
}
ZeroCopyReturnValue *JFJochReceiverRugnux::AnalysedImageSlot() {
// An analysed frame is not dropped because the writer lags behind the catch-up from a full buffer: the analysis
// waits for the slots being sent to come back, and the frames arriving meanwhile find the buffer full. A
@@ -797,13 +780,8 @@ void JFJochReceiverRugnux::ForwardThread(uint32_t id) {
continue;
if (msg->cbor->end_message) {
end_message_received = true;
} else if (stills_run && msg->cbor->calibration) {
StillsCalibration(*msg);
} else if (stills_run && msg->cbor->data_message) {
HoldStillsImage(msg->cbor->data_message.value());
} else if (msg->cbor->calibration) {
if (push_images_to_writer)
image_pusher.SendCalibration(msg->cbor->calibration.value());
} else if (msg->cbor->data_message) {
live_status.ImageReceived(msg->received);
DataMessage data_msg = msg->cbor->data_message.value();
receiver/JFJochReceiverRugnux.h
-5
View File
@@ -61,16 +61,11 @@ class JFJochReceiverRugnux : public JFJochReceiver {
IndexerThreadPool *const indexing_thread_pool;
const int64_t stills_nthreads;
int64_t stills_keep_free = 0; // slots frames do not take, for the analysis threads to send with
// Calibration that came while the pre-pass held the start message back.
std::mutex calibration_m;
std::vector<ImagePullerOutput> calibration_held;
bool start_sent = false;
std::unique_ptr<StillsGPUAnalysis> stills; // under rugnux_m until the frames are in, for Cut
void StartStills();
void StillsMeasurement();
void HoldStillsImage(const DataMessage &message);
void StillsCalibration(const ImagePullerOutput &msg);
ZeroCopyReturnValue *AnalysedImageSlot();
void SendAnalysedImage(DataMessage &message, const AzimuthalIntegrationProfile &profile);
tests/FPGAImagePullerTest.cpp
+5 -3
View File
@@ -303,12 +303,14 @@ namespace {
CHECK(a.end.max_image_number == b.end.max_image_number);
CHECK(a.end.adu_histogram == b.end.adu_histogram);
// Calibration between start and the first image, in the order the FPGA path sends it
// Calibration goes to the writer only from the FPGA receiver
CHECK(a.sequence.front() == "start");
CHECK(b.sequence.front() == "start");
CHECK(a.sequence.back() == "end");
CHECK(b.sequence.back() == "end");
CHECK(a.sequence == b.sequence);
auto sequence_a = a.sequence;
std::erase_if(sequence_a, [](const std::string &s) { return s.starts_with("calibration"); });
CHECK(sequence_a == b.sequence);
int64_t displacement = 0;
for (size_t i = 0; i < b.arrival.size(); i++)
@@ -667,7 +669,7 @@ TEST_CASE("FPGAImagePuller_RugnuxReceiver", "[FPGAImagePuller]") {
CHECK(pusher.images.size() == 6);
CHECK(pusher.sequence.front() == "start");
CHECK(pusher.sequence.back() == "end");
CHECK(pusher.sequence[1].starts_with("calibration"));
CHECK_FALSE(pusher.sequence[1].starts_with("calibration")); // the writer gets calibration only from the FPGA receiver
const auto a = Pixels(pusher.images.at(0).message.image);
const auto b = Pixels(pusher.images.at(3).message.image);
CHECK(a == b); // the third frame in the generator's memory repeats
tests/JFJochReceiverRugnuxTest.cpp
+2 -11
View File
@@ -102,13 +102,6 @@ namespace {
}
}
void Calibration() {
static const std::vector<uint16_t> pedestal(4, 1);
auto cbor = std::make_shared<CBORStream2DeserializerOutput>();
cbor->calibration = CompressedImage(pedestal, 2, 2);
puller->Put(ImagePullerOutput{.cbor = cbor});
}
void End() {
puller->Put(ImagePullerOutput{.cbor = std::make_shared<CBORStream2DeserializerOutput>(EndMessage{})});
}
@@ -140,8 +133,6 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
PixelMask mask(x);
Stream stream(x, images);
// Calibration ahead of the frames waits for the start message, which waits for the pre-pass.
stream.Calibration();
stream.Frames(0, images);
stream.End();
@@ -163,8 +154,8 @@ TEST_CASE("JFJochReceiverRugnux_Stills", "[JFJochReceiverRugnux]") {
CHECK(out.status.indexing_rate.value() > 0.5f);
CHECK(out.scan_result.images.size() == images);
// The writer sees the start message before the calibration and the frames, the end message last.
CHECK(pusher.Events() == std::vector<std::string>{"start", "calibration", "image", "end"});
// The writer sees the start message before the frames, the end message last.
CHECK(pusher.Events() == std::vector<std::string>{"start", "image", "end"});
auto sent = pusher.Images();
std::sort(sent.begin(), sent.end());
REQUIRE(sent.size() == images);