diff --git a/common/ZMQImagePusher.cpp b/common/ZMQImagePusher.cpp index 6b7681bf..c57339d5 100644 --- a/common/ZMQImagePusher.cpp +++ b/common/ZMQImagePusher.cpp @@ -56,7 +56,7 @@ void ZMQImagePusher::SendImage(const uint8_t *image_data, size_t image_size, int } void ZMQImagePusher::StartDataCollection(const StartMessage& message) { - std::vector serialization_buffer(80*1024*1024); + std::vector serialization_buffer(message.approx_size); JFJochFrameSerializer serializer(serialization_buffer.data(), serialization_buffer.size()); // 80 MiB should be safe even for 16M if (message.data_file_count < 1) diff --git a/frame_serialize/JFJochFrameDeserializer.cpp b/frame_serialize/JFJochFrameDeserializer.cpp index 226c13b0..5cdd38f3 100644 --- a/frame_serialize/JFJochFrameDeserializer.cpp +++ b/frame_serialize/JFJochFrameDeserializer.cpp @@ -441,19 +441,10 @@ void JFJochFrameDeserializer::ProcessPixelMaskElement(CborValue &value) { cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); - CBORImage cbor_multidim_array; - GetCBORMultidimTypedArray(cbor_multidim_array, map_value); - - if (cbor_multidim_array.size != cbor_multidim_array.xpixel * cbor_multidim_array.ypixel - * cbor_multidim_array.pixel_depth_bytes) - throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask size mismatch"); - - if (cbor_multidim_array.pixel_depth_bytes != sizeof(uint32_t)) - throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask must be 32-bit"); - - std::vector v(cbor_multidim_array.xpixel * cbor_multidim_array.ypixel); - memcpy(v.data(), cbor_multidim_array.data, cbor_multidim_array.size); - start_message.pixel_mask[key] = v; + CBORImage image; + image.channel = key; + GetCBORMultidimTypedArray(image, map_value); + start_message.pixel_mask.push_back(image); } cborErr(cbor_value_leave_container(&value, &map_value)); } diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index 832eee07..0d3ffc90 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -126,30 +126,6 @@ inline void CBOR_ENC_2D_TYPED_ARRAY(CborEncoder &encoder, const CBORImage& image cborErr(cbor_encoder_close_container(&encoder, &arrayEncoder)); } -inline void CBOR_ENC_PIXEL_MASK(CborEncoder &encoder, const char* key, - const std::map> &pixel_mask, - size_t xpixel, size_t ypixel) { - CborEncoder mapEncoder; - cborErr(cbor_encode_text_stringz(&encoder, key)); - cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, pixel_mask.size())); - for (auto &[pixel_mask_key, pixel_mask_array]: pixel_mask) { - CBORImage image{ - .data = reinterpret_cast(pixel_mask_array.data()), - .size = pixel_mask_array.size() * sizeof(uint32_t), - .xpixel = xpixel, - .ypixel = ypixel, - .pixel_depth_bytes = sizeof(uint32_t), - .pixel_is_signed = false, - .pixel_is_float = false, - .algorithm = CompressionAlgorithm::NO_COMPRESSION, - .channel = pixel_mask_key - }; - CBOR_ENC_2D_TYPED_ARRAY(mapEncoder, image); - } - - cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); -} - inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector& v) { CborEncoder arrayEncoder; @@ -392,7 +368,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC_GONIOMETER_MAP(mapEncoder, "goniometer", message.goniometer); CBOR_ENC_USER_DATA(mapEncoder, message); - CBOR_ENC_PIXEL_MASK(mapEncoder, "pixel_mask", message.pixel_mask, message.image_size_x, message.image_size_y); + CBOR_ENC(mapEncoder, "pixel_mask", message.pixel_mask); CBOR_ENC_CHANNELS(mapEncoder, "channels", message.channels); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); diff --git a/frame_serialize/StartMessage.h b/frame_serialize/StartMessage.h index cfa11c94..dece5f8d 100644 --- a/frame_serialize/StartMessage.h +++ b/frame_serialize/StartMessage.h @@ -53,7 +53,6 @@ struct StartMessage { uint64_t storage_cell_number; // user data - std::map> pixel_mask; bool pixel_mask_enabled; std::string arm_date; @@ -82,7 +81,21 @@ struct StartMessage { std::vector rad_int_bin_to_q; std::vector rad_int_solid_angle_corr; + std::vector pixel_mask; std::vector calibration; + + size_t approx_size = 1024*1024; + + // Use function below to update approx_size + void AddPixelMask(CBORImage image) { + approx_size += image.size; + pixel_mask.emplace_back(std::move(image)); + } + + void AddCalibration(CBORImage image) { + approx_size += image.size; + calibration.emplace_back(std::move(image)); + } }; #endif //JUNGFRAUJOCH_STARTMESSAGE_H diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index bda0b76b..a8d24072 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -153,8 +153,47 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings, experiment.FillMessage(message); message.arm_date = time_UTC(std::chrono::system_clock::now()); - if (calib) - message.pixel_mask["sc0"] = calib->CalculateNexusMask(experiment, 0); + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + std::vector pixel_mask; + std::vector > pedestal; + + if (calib) { + size_t xpixel = experiment.GetXPixelsNum(); + size_t ypixel = experiment.GetYPixelsNum(); + + pixel_mask = compressor.Compress(calib->CalculateNexusMask(experiment, 0)); + message.AddPixelMask(CBORImage{ + .data = pixel_mask.data(), + .size = pixel_mask.size(), + .xpixel = (size_t) xpixel, + .ypixel = (size_t) ypixel, + .pixel_depth_bytes = 4, + .pixel_is_signed = false, + .pixel_is_float = false, + .algorithm = CompressionAlgorithm::BSHUF_LZ4, + .channel = "sc0" + }); + for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) { + for (int gain = 0; gain < 3; gain++) { + auto tmp = compressor.Compress(calib->GetPedestal(gain, sc)); + pedestal.emplace_back(tmp); + std::string channel = "pedestal_G" + std::to_string(gain)+ "_sc" + std::to_string(sc); + + CBORImage image{ + .data = pedestal.at(pedestal.size()-1).data(), + .size = pedestal.at(pedestal.size()-1).size(), + .xpixel = (size_t) xpixel, + .ypixel = (size_t) ypixel, + .pixel_depth_bytes = 2, + .pixel_is_signed = false, + .pixel_is_float = false, + .algorithm = CompressionAlgorithm::BSHUF_LZ4, + .channel = channel + }; + message.AddCalibration(image); + } + } + } if (rad_int_mapping) { message.rad_int_bin_number = rad_int_mapping->GetBinNumber(); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index a335a211..ddb0881c 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -62,7 +62,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") { .rad_int_bin_to_q = {0.1, 0.2, 0.3, 0.5}, .rad_int_solid_angle_corr = {10, 20, 30, 50} }; - message.pixel_mask["sc0"] = std::vector(456*457, 15); REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); @@ -72,7 +71,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") { StartMessage output_message; REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage()); - REQUIRE(output_message.pixel_mask.contains("sc0")); CHECK(output_message.data_file_count == message.data_file_count); CHECK(output_message.detector_distance == Approx(message.detector_distance)); CHECK(output_message.beam_center_x == Approx(message.beam_center_x)); @@ -91,7 +89,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") { CHECK(output_message.pixel_size_y == Approx(message.pixel_size_y)); CHECK(output_message.sensor_thickness == Approx(message.sensor_thickness)); CHECK(output_message.sensor_material == message.sensor_material); - CHECK(output_message.pixel_mask == message.pixel_mask); CHECK(output_message.pixel_mask_enabled == message.pixel_mask_enabled); CHECK(output_message.compression_algorithm == message.compression_algorithm); CHECK(output_message.compression_block_size == message.compression_block_size); @@ -127,6 +124,42 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") { } +TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") { + std::vector buffer(8*1024*1024); + JFJochFrameSerializer serializer(buffer.data(), buffer.size()); + + std::vector mask(456*457, 15); + + CBORImage image_mask { + .data = reinterpret_cast(mask.data()), + .size = 456 * 457 * sizeof(uint32_t), + .xpixel = 456, + .ypixel = 457, + .pixel_depth_bytes = 4, + .pixel_is_signed = false, + .pixel_is_float = false, + .algorithm = CompressionAlgorithm::NO_COMPRESSION, + .channel = "sc0" + }; + + StartMessage message {}; + + message.AddPixelMask(image_mask); + REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); + + JFJochFrameDeserializer deserializer; + REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize())); + REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START); + + StartMessage output_message; + REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage()); + REQUIRE(output_message.pixel_mask.size() == 1); + CHECK(!output_message.pixel_mask[0].pixel_is_float); + CHECK(output_message.pixel_mask[0].xpixel == 456); + CHECK(output_message.pixel_mask[0].ypixel == 457); + CHECK(output_message.pixel_mask[0].channel == "sc0"); + CHECK(memcmp(output_message.pixel_mask[0].data, mask.data(), mask.size() * sizeof(float)) == 0); +} TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") { std::vector buffer(8*1024*1024); @@ -152,7 +185,6 @@ TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") { .channel = "calib1" }; - CBORImage image2 { .data = reinterpret_cast(calib2.data()), .size = 16 * 16 * sizeof(float), @@ -165,10 +197,10 @@ TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") { .channel = "calib2" }; - StartMessage message { - .calibration = {image2, image1} - }; + StartMessage message {}; + message.AddCalibration(image2); + message.AddCalibration(image1); REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); JFJochFrameDeserializer deserializer; @@ -615,7 +647,21 @@ inline bool CmpString(const char *str1, const std::string& str2) { TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") { std::vector buffer(8*1024*1024); -JFJochFrameSerializer serializer(buffer.data(), buffer.size()); + JFJochFrameSerializer serializer(buffer.data(), buffer.size()); + + std::vector mask(456*457, 15); + + CBORImage image_mask { + .data = reinterpret_cast(mask.data()), + .size = 456 * 457 * sizeof(uint32_t), + .xpixel = 456, + .ypixel = 457, + .pixel_depth_bytes = 4, + .pixel_is_signed = false, + .pixel_is_float = false, + .algorithm = CompressionAlgorithm::NO_COMPRESSION, + .channel = "sc0" + }; StartMessage message { .data_file_count = 3, @@ -664,7 +710,8 @@ JFJochFrameSerializer serializer(buffer.data(), buffer.size()); .rad_int_bin_number = 35, .summation = 567 }; - message.pixel_mask["sc0"] = std::vector(456*457, 15); + + message.AddPixelMask(image_mask); REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); @@ -693,8 +740,7 @@ JFJochFrameSerializer serializer(buffer.data(), buffer.size()); CHECK(msg2->pixel_mask.ptr[0].pixel_mask.dim[0] == 457); CHECK(msg2->pixel_mask.ptr[0].pixel_mask.dim[1] == 456); CHECK(msg2->pixel_mask.ptr[0].pixel_mask.array.data.len == 456 * 457 * sizeof(uint32_t)); - CHECK(memcmp(msg2->pixel_mask.ptr[0].pixel_mask.array.data.ptr, message.pixel_mask["sc0"].data(), - 456 * 457 * sizeof(uint32_t)) == 0); + CHECK(memcmp(msg2->pixel_mask.ptr[0].pixel_mask.array.data.ptr, mask.data(), 456 * 457 * sizeof(uint32_t)) == 0); CHECK(msg2->pixel_mask.ptr[0].pixel_mask.array.tag == TagUnsignedInt32BitLE); stream2_free_msg(msg); } diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index 68f82595..3c99acbb 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -192,14 +192,33 @@ void HDF5Metadata::DetectorModule(HDF5File *hdf5_file, const std::string &name, "", "", "translation", {0,0,0}); } +void HDF5Metadata::SaveCBORImage(HDF5File *hdf5_file, const std::string &hdf5_path, const CBORImage &image) { + std::vector dims = {image.xpixel, image.ypixel}; + + HDF5DataType data_type(image.pixel_depth_bytes, image.pixel_is_signed); + HDF5Dcpl dcpl; + + dcpl.SetCompression(image.algorithm, H5Tget_size(data_type.GetID()), 0); + dcpl.SetChunking(dims); + + HDF5DataSpace data_space(dims); + auto dataset = std::make_unique(*hdf5_file, hdf5_path, data_type, data_space, dcpl); + + if (image.algorithm == CompressionAlgorithm::NO_COMPRESSION) + dataset->Write(data_type, image.data); + else + dataset->WriteDirectChunk(image.data, image.size, {0,0}); +} + void HDF5Metadata::Calibration(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { - if (start.pixel_mask.count("sc0") == 1) { - auto v = start.pixel_mask.at("sc0"); - std::vector dims = {start.image_size_y, start.image_size_x}; - SaveVector(*hdf5_file, "/entry/instrument/detector/pixel_mask", v, dims, CompressionAlgorithm::BSHUF_LZ4); + if (!start.pixel_mask.empty()) { + SaveCBORImage(hdf5_file, "/entry/instrument/detector/pixel_mask", start.pixel_mask[0]); hdf5_file->HardLink("/entry/instrument/detector/pixel_mask", "/entry/instrument/detector/detectorSpecific/pixel_mask"); } + + for (const auto &i: start.calibration) + SaveCBORImage(hdf5_file, "/entry/instrument/detector/detectorSpecific/" + i.channel, i); } void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) { diff --git a/writer/HDF5NXmx.h b/writer/HDF5NXmx.h index 05516b87..291fb895 100644 --- a/writer/HDF5NXmx.h +++ b/writer/HDF5NXmx.h @@ -27,6 +27,7 @@ namespace HDF5Metadata { void Mask(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end); void Calibration(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end); + void SaveCBORImage(HDF5File *hdf5_file, const std::string& hdf5_path, const CBORImage &image); void RadInt(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end); std::string DataFileName(const std::string& prefix, int64_t file_number); }