From a7c5d7a89b1ee901dd51001b3f1c73069ed81342 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 10 Aug 2026 05:52:44 +0200 Subject: [PATCH] CBOR: carry the reflection's background variance var_bkg was added to Reflection and to the HDF5 writer but never to the CBOR reflection map, so it survived only where rugnux drives the writer in-process. Everything that reaches the writer over the wire - i.e. every acquisition the broker records - wrote /entry/reflections/*/background_variance as an array of zeros, presented as a measured quantity, and re-processing such a file fed the merge a non-signal variance of zero. Encode and decode it. The key is optional on both sides, so a stream from an older version still reads and one from this version still reads on an older client. The round-trip test only checked h, k, l, the predicted position and d - which is why a missing float was invisible. It now gives every field a distinct value and checks all of them, so the next field added to Reflection and forgotten here fails immediately. Co-Authored-By: Claude Opus 5 (1M context) --- docs/CBOR.md | 1 + frame_serialize/CBORStream2Deserializer.cpp | 2 ++ frame_serialize/CBORStream2Serializer.cpp | 1 + tests/CBORTest.cpp | 34 +++++++++++++++++++-- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/CBOR.md b/docs/CBOR.md index 3536f9d6..93f6ae24 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -155,6 +155,7 @@ See [DECTRIS documentation](https://github.com/dectris/documentation/tree/main/s | - d | float | resolution \[Angstrom\] | | | | - I | float | integrated intensity (photons) | | | | - bkg | float | mean background value (photons) | | | +| - var_bkg | float | non-signal (background) part of sigma^2, carried to the merge (photons^2) | | | | - sigma | float | standard deviation, estimated from counting statistics (photons) | | | | - image | float | image number (present for each spot) | | | | - rp | float | Distance to Ewald sphere \[Angstrom^-1\] | | | diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 7acc12dd..6ba60651 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -509,6 +509,8 @@ namespace { r.I = GetCBORFloat(map_value); else if (key == "bkg") r.bkg = GetCBORFloat(map_value); + else if (key == "var_bkg") + r.var_bkg = GetCBORFloat(map_value); else if (key == "sigma") r.sigma = GetCBORFloat(map_value); else if (key == "image") diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index 70e29d9f..21056641 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -267,6 +267,7 @@ inline void CBOR_ENC(CborEncoder &encoder, const Reflection& r) { CBOR_ENC(mapEncoder, "d", r.d); CBOR_ENC(mapEncoder, "I", r.I); CBOR_ENC(mapEncoder, "bkg", r.bkg); + CBOR_ENC(mapEncoder, "var_bkg", r.var_bkg); CBOR_ENC(mapEncoder, "sigma", r.sigma); CBOR_ENC(mapEncoder, "image", r.image_number); CBOR_ENC(mapEncoder, "rp", r.dist_ewald); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index b7df0480..08be6b06 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -969,20 +969,35 @@ TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { TEST_CASE("CBORSerialize_Image_Reflections") { // Prepare a few reflections with distinct values std::vector refs_in; + // Every float is given a distinct value: a field the encoder forgets reads back as 0 and fails + // here. var_bkg was added to Reflection and to the HDF5 writer but not to the CBOR map, so every + // broker-written file carried a zeroed background_variance until this test was widened to see it. refs_in.push_back(Reflection{ .h = 1, .k = 0, .l = -1, + .image_number = 789.0f, .delta_phi_deg = 0.125f, .predicted_x = 1024.5f, .predicted_y = 768.25f, - .d = 2.345f + .observed_x = 1024.75f, .observed_y = 768.5f, + .d = 2.345f, .I = 1234.5f, .bkg = 3.25f, .var_bkg = 41.5f, .sigma = 37.125f, + .dist_ewald = 0.0025f, .rlp = 1.75f, .partiality = 0.875f, .zeta = 0.625f, + .image_scale_corr = 1.125f }); refs_in.push_back(Reflection{ .h = -3, .k = 2, .l = 7, + .image_number = 790.0f, .delta_phi_deg = -0.25f, .predicted_x = 5.0f, .predicted_y = 10.0f, - .d = 4.0f + .observed_x = 5.5f, .observed_y = 10.5f, + .d = 4.0f, .I = 12.0f, .bkg = 0.5f, .var_bkg = 6.75f, .sigma = 4.25f, + .dist_ewald = -0.001f, .rlp = 2.5f, .partiality = 0.5f, .zeta = 0.25f, + .image_scale_corr = 0.875f }); refs_in.push_back(Reflection{ .h = 0, .k = 0, .l = 1, + .image_number = 791.0f, .delta_phi_deg = 0.0f, .predicted_x = 0.0f, .predicted_y = 0.0f, - .d = 10.0f + .observed_x = 0.25f, .observed_y = 0.5f, + .d = 10.0f, .I = -1.5f, .bkg = 0.125f, .var_bkg = 0.875f, .sigma = 1.5f, + .dist_ewald = 0.0f, .rlp = 1.0f, .partiality = 1.0f, .zeta = 1.0f, + .image_scale_corr = 1.0f }); // Minimal DataMessage carrying reflections @@ -1020,8 +1035,21 @@ TEST_CASE("CBORSerialize_Image_Reflections") { CHECK(b.predicted_x == Catch::Approx(a.predicted_x).margin(1e-6f)); CHECK(b.predicted_y == Catch::Approx(a.predicted_y).margin(1e-6f)); + CHECK(b.observed_x == Catch::Approx(a.observed_x).margin(1e-6f)); + CHECK(b.observed_y == Catch::Approx(a.observed_y).margin(1e-6f)); CHECK(b.d == Catch::Approx(a.d).margin(1e-6f)); + CHECK(b.image_number == Catch::Approx(a.image_number).margin(1e-6f)); + CHECK(b.delta_phi_deg == Catch::Approx(a.delta_phi_deg).margin(1e-6f)); + CHECK(b.I == Catch::Approx(a.I).margin(1e-6f)); + CHECK(b.bkg == Catch::Approx(a.bkg).margin(1e-6f)); + CHECK(b.var_bkg == Catch::Approx(a.var_bkg).margin(1e-6f)); + CHECK(b.sigma == Catch::Approx(a.sigma).margin(1e-6f)); + CHECK(b.dist_ewald == Catch::Approx(a.dist_ewald).margin(1e-6f)); + CHECK(b.rlp == Catch::Approx(a.rlp).margin(1e-6f)); + CHECK(b.partiality == Catch::Approx(a.partiality).margin(1e-6f)); + CHECK(b.zeta == Catch::Approx(a.zeta).margin(1e-6f)); + CHECK(b.image_scale_corr == Catch::Approx(a.image_scale_corr).margin(1e-6f)); } }