v1.0.0-rc.148 (#58)

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

* jfjoch_broker: Improve azimuthal integration (add <I^2> calculation)
* jfjoch_broker: Fixes around indexing, aiming to handle multi-lattice crystals (work in progress, it is not fully integrated)
* jfjoch_writer: Save mean(I), stddev(I), and count(I) for each azimuthal bin

Reviewed-on: #58

frame_serialize/CBORStream2Serializer.cpp

@@ -153,6 +153,11 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<fl
     cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) v.data(), v.size() * sizeof(float)));
 }
 
+inline void CBOR_ENC_FLOAT_ARRAY_NOKEY(CborEncoder &encoder, const std::vector<float>& v) {
+    cborErr(cbor_encode_tag(&encoder, TagFloatLE));
+    cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) v.data(), v.size() * sizeof(float)));
+}
+
 inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<uint8_t>& v) {
     cborErr(cbor_encode_text_stringz(&encoder, key));
     cborErr(cbor_encode_tag(&encoder, TagUnsignedInt8Bit));
@@ -224,6 +229,7 @@ inline void CBOR_ENC(CborEncoder &encoder, const SpotToSave& spot) {
     CBOR_ENC(mapEncoder, "maxc", spot.maxc);
     CBOR_ENC(mapEncoder, "ice_ring", spot.ice_ring);
     CBOR_ENC(mapEncoder, "indexed", spot.indexed);
+    CBOR_ENC(mapEncoder, "latt", spot.lattice);
     CBOR_ENC(mapEncoder, "image", spot.image);
     if (spot.indexed) {
         CBOR_ENC(mapEncoder, "h", spot.h);
@@ -423,6 +429,23 @@ inline void CBOR_ENC_PIXEL_MASK(CborEncoder &encoder, const StartMessage &msg) {
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 }
 
+inline void CBOR_ENC_AZINT_MAP(CborEncoder &encoder, const StartMessage &msg) {
+    if (msg.az_int_map.empty())
+        return;
+
+    if (msg.az_int_map.size() != msg.image_size_x * msg.image_size_y)
+        throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
+                              "Mismatch in size of pixel mask");
+
+    JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
+    auto mask_compressed = compressor.Compress(msg.az_int_map);
+    CompressedImage image(mask_compressed.data(), mask_compressed.size(),
+                          msg.image_size_x, msg.image_size_y,
+                          CompressedImageMode::Uint16,
+                          CompressionAlgorithm::BSHUF_LZ4, "az_int_map");
+    CBOR_ENC_2D_TYPED_ARRAY(encoder, image);
+}
+
 inline void CBOR_ENC(CborEncoder &encoder, const char* key, const UnitCell &val) {
     CborEncoder mapEncoder;
 
@@ -637,10 +660,11 @@ void CBORStream2Serializer::SerializeSequenceStart(const StartMessage& message)
     CBOR_ENC(mapEncoder, "geometry_transformation_enabled", message.geometry_transformation_enabled);
 
     CBOR_ENC_PIXEL_MASK(mapEncoder, message);
+    CBOR_ENC_AZINT_MAP(mapEncoder, message);
 
     CBOR_ENC(mapEncoder, "channels", message.channels);
     CBOR_ENC(mapEncoder, "max_spot_count", message.max_spot_count);
-
+    CBOR_ENC(mapEncoder, "max_extra_lattices", message.max_extra_lattices);
     CBOR_ENC(mapEncoder, "storage_cell_number", message.storage_cell_number);
     CBOR_ENC_RATIONAL(mapEncoder, "storage_cell_delay", message.storage_cell_delay_ns, 1000*1000*1000UL);
     CBOR_ENC(mapEncoder, "threshold_energy", message.threshold_energy);
@@ -698,7 +722,14 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
     CBOR_ENC(mapEncoder, "rotation_lattice_type", message.rotation_lattice_type);
     if (message.rotation_lattice.has_value())
         CBOR_ENC(mapEncoder, "rotation_lattice", message.rotation_lattice->GetVector());
-
+    if (!message.rotation_extra_lattices.empty()) {
+        CborEncoder arrayEncoder;
+        cborErr(cbor_encode_text_stringz(&mapEncoder, "rotation_extra_lattices"));
+        cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, message.rotation_extra_lattices.size()));
+        for (const auto &el : message.rotation_extra_lattices)
+            CBOR_ENC_FLOAT_ARRAY_NOKEY(arrayEncoder, el.GetVector());
+        cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder));
+    }
     CBOR_ENC(mapEncoder, "data_collection_efficiency_image", message.data_collection_efficiency);
     CBOR_ENC(mapEncoder, "spot_count", message.spot_count);
     CBOR_ENC(mapEncoder, "spot_count_ice_ring", message.spot_count_ice_ring);
@@ -714,6 +745,7 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
     CBOR_ENC(mapEncoder, "max_viable_pixel_value", message.max_viable_pixel_value);
     CBOR_ENC(mapEncoder, "saturated_pixel_count", message.saturated_pixel_count);
     CBOR_ENC(mapEncoder, "error_pixel_count", message.error_pixel_count);
+    CBOR_ENC(mapEncoder, "indexed_lattice_count", message.indexed_lattice_count);
     CBOR_ENC(mapEncoder, "image_scale_factor", message.image_scale_factor);
     CBOR_ENC(mapEncoder, "image_scale_cc", message.image_scale_cc);
     CBOR_ENC(mapEncoder, "image_scale_mosaicity", message.image_scale_mosaicity);
@@ -742,9 +774,22 @@ void CBORStream2Serializer::SerializeImageInternal(CborEncoder &mapEncoder, cons
     CBOR_ENC(mapEncoder, "spot_count_low_res", message.spot_count_low_res);
     CBOR_ENC(mapEncoder, "spot_count_indexed", message.spot_count_indexed);
     CBOR_ENC(mapEncoder, "az_int_profile", message.az_int_profile);
+    if (!message.az_int_profile_std.empty())
+        CBOR_ENC(mapEncoder, "az_int_profile_std", message.az_int_profile_std);
+    CBOR_ENC(mapEncoder, "az_int_profile_count", message.az_int_profile_count);
+
     CBOR_ENC(mapEncoder, "indexing_result", message.indexing_result);
+    CBOR_ENC(mapEncoder, "indexing_lattice_count", message.indexing_lattice_count);
     if (message.indexing_lattice)
         CBOR_ENC(mapEncoder, "indexing_lattice", message.indexing_lattice->GetVector());
+    if (!message.indexing_extra_lattices.empty()) {
+        CborEncoder arrayEncoder;
+        cborErr(cbor_encode_text_stringz(&mapEncoder, "indexing_extra_lattices"));
+        cborErr(cbor_encoder_create_array(&mapEncoder, &arrayEncoder, message.indexing_extra_lattices.size()));
+        for (const auto &el : message.indexing_extra_lattices)
+            CBOR_ENC_FLOAT_ARRAY_NOKEY(arrayEncoder, el.GetVector());
+        cborErr(cbor_encoder_close_container(&mapEncoder, &arrayEncoder));
+    }
     CBOR_ENC(mapEncoder, "profile_radius", message.profile_radius);
     CBOR_ENC(mapEncoder, "mosaicity", message.mosaicity_deg);
     CBOR_ENC(mapEncoder, "integrated_reflections", message.integrated_reflections);