mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 43 Wiki Activity

JFJochFrameSerializer: Using CBORImage for mask and calibration + writing calibration in HDF5 file

Browse Source
This commit is contained in:
leonarski_f
2023-06-28 17:14:09 +02:00
parent e472047839
commit 0904e1f198
8 changed files with 142 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/ZMQImagePusher.cpp
View File

@@ -56,7 +56,7 @@ void ZMQImagePusher::SendImage(const uint8_t *image_data, size_t image_size, int
}
void ZMQImagePusher::StartDataCollection(const StartMessage& message) {
std::vector<uint8_t> serialization_buffer(80*1024*1024);
std::vector<uint8_t> serialization_buffer(message.approx_size);
JFJochFrameSerializer serializer(serialization_buffer.data(), serialization_buffer.size()); // 80 MiB should be safe even for 16M
if (message.data_file_count < 1)

17
frame_serialize/JFJochFrameDeserializer.cpp
View File

@@ -441,19 +441,10 @@ void JFJochFrameDeserializer::ProcessPixelMaskElement(CborValue &value) {
cborErr(cbor_value_enter_container(&value, &map_value));
while (! cbor_value_at_end(&map_value)) {
auto key = GetCBORString(map_value);
CBORImage cbor_multidim_array;
GetCBORMultidimTypedArray(cbor_multidim_array, map_value);
if (cbor_multidim_array.size != cbor_multidim_array.xpixel * cbor_multidim_array.ypixel
* cbor_multidim_array.pixel_depth_bytes)
throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask size mismatch");
if (cbor_multidim_array.pixel_depth_bytes != sizeof(uint32_t))
throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask must be 32-bit");
std::vector<uint32_t> v(cbor_multidim_array.xpixel * cbor_multidim_array.ypixel);
memcpy(v.data(), cbor_multidim_array.data, cbor_multidim_array.size);
start_message.pixel_mask[key] = v;
CBORImage image;
image.channel = key;
GetCBORMultidimTypedArray(image, map_value);
start_message.pixel_mask.push_back(image);
}
cborErr(cbor_value_leave_container(&value, &map_value));
}

26
frame_serialize/JFJochFrameSerializer.cpp
View File

@@ -126,30 +126,6 @@ inline void CBOR_ENC_2D_TYPED_ARRAY(CborEncoder &encoder, const CBORImage& image
cborErr(cbor_encoder_close_container(&encoder, &arrayEncoder));
}
inline void CBOR_ENC_PIXEL_MASK(CborEncoder &encoder, const char* key,
const std::map<std::string, std::vector<uint32_t>> &pixel_mask,
size_t xpixel, size_t ypixel) {
CborEncoder mapEncoder;
cborErr(cbor_encode_text_stringz(&encoder, key));
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, pixel_mask.size()));
for (auto &[pixel_mask_key, pixel_mask_array]: pixel_mask) {
CBORImage image{
.data = reinterpret_cast<const uint8_t *>(pixel_mask_array.data()),
.size = pixel_mask_array.size() * sizeof(uint32_t),
.xpixel = xpixel,
.ypixel = ypixel,
.pixel_depth_bytes = sizeof(uint32_t),
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::NO_COMPRESSION,
.channel = pixel_mask_key
};
CBOR_ENC_2D_TYPED_ARRAY(mapEncoder, image);
}
cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
}
inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<float>& v) {
CborEncoder arrayEncoder;
@@ -392,7 +368,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message)
CBOR_ENC_GONIOMETER_MAP(mapEncoder, "goniometer", message.goniometer);
CBOR_ENC_USER_DATA(mapEncoder, message);
CBOR_ENC_PIXEL_MASK(mapEncoder, "pixel_mask", message.pixel_mask, message.image_size_x, message.image_size_y);
CBOR_ENC(mapEncoder, "pixel_mask", message.pixel_mask);
CBOR_ENC_CHANNELS(mapEncoder, "channels", message.channels);
cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));

15
frame_serialize/StartMessage.h
View File

@@ -53,7 +53,6 @@ struct StartMessage {
uint64_t storage_cell_number; // user data
std::map<std::string, std::vector<uint32_t>> pixel_mask;
bool pixel_mask_enabled;
std::string arm_date;
@@ -82,7 +81,21 @@ struct StartMessage {
std::vector<float> rad_int_bin_to_q;
std::vector<float> rad_int_solid_angle_corr;
std::vector<CBORImage> pixel_mask;
std::vector<CBORImage> calibration;
size_t approx_size = 1024*1024;
// Use function below to update approx_size
void AddPixelMask(CBORImage image) {
approx_size += image.size;
pixel_mask.emplace_back(std::move(image));
}
void AddCalibration(CBORImage image) {
approx_size += image.size;
calibration.emplace_back(std::move(image));
}
};
#endif //JUNGFRAUJOCH_STARTMESSAGE_H

43
receiver/JFJochReceiver.cpp
View File

@@ -153,8 +153,47 @@ JFJochReceiver::JFJochReceiver(const JFJochProtoBuf::ReceiverInput &settings,
experiment.FillMessage(message);
message.arm_date = time_UTC(std::chrono::system_clock::now());
if (calib)
message.pixel_mask["sc0"] = calib->CalculateNexusMask(experiment, 0);
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> pixel_mask;
std::vector<std::vector<uint8_t> > pedestal;
if (calib) {
size_t xpixel = experiment.GetXPixelsNum();
size_t ypixel = experiment.GetYPixelsNum();
pixel_mask = compressor.Compress(calib->CalculateNexusMask(experiment, 0));
message.AddPixelMask(CBORImage{
.data = pixel_mask.data(),
.size = pixel_mask.size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 4,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = "sc0"
});
for (int sc = 0; sc < experiment.GetStorageCellNumber(); sc++) {
for (int gain = 0; gain < 3; gain++) {
auto tmp = compressor.Compress(calib->GetPedestal(gain, sc));
pedestal.emplace_back(tmp);
std::string channel = "pedestal_G" + std::to_string(gain)+ "_sc" + std::to_string(sc);
CBORImage image{
.data = pedestal.at(pedestal.size()-1).data(),
.size = pedestal.at(pedestal.size()-1).size(),
.xpixel = (size_t) xpixel,
.ypixel = (size_t) ypixel,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = channel
};
message.AddCalibration(image);
}
}
}
if (rad_int_mapping) {
message.rad_int_bin_number = rad_int_mapping->GetBinNumber();

68
tests/CBORTest.cpp
View File

@@ -62,7 +62,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
.rad_int_bin_to_q = {0.1, 0.2, 0.3, 0.5},
.rad_int_solid_angle_corr = {10, 20, 30, 50}
};
message.pixel_mask["sc0"] = std::vector<uint32_t>(456*457, 15);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
@@ -72,7 +71,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
StartMessage output_message;
REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage());
REQUIRE(output_message.pixel_mask.contains("sc0"));
CHECK(output_message.data_file_count == message.data_file_count);
CHECK(output_message.detector_distance == Approx(message.detector_distance));
CHECK(output_message.beam_center_x == Approx(message.beam_center_x));
@@ -91,7 +89,6 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
CHECK(output_message.pixel_size_y == Approx(message.pixel_size_y));
CHECK(output_message.sensor_thickness == Approx(message.sensor_thickness));
CHECK(output_message.sensor_material == message.sensor_material);
CHECK(output_message.pixel_mask == message.pixel_mask);
CHECK(output_message.pixel_mask_enabled == message.pixel_mask_enabled);
CHECK(output_message.compression_algorithm == message.compression_algorithm);
CHECK(output_message.compression_block_size == message.compression_block_size);
@@ -127,6 +124,42 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") {
}
TEST_CASE("CBORSerialize_Start_PixelMask", "[CBOR]") {
std::vector<uint8_t> buffer(8*1024*1024);
JFJochFrameSerializer serializer(buffer.data(), buffer.size());
std::vector<uint32_t> mask(456*457, 15);
CBORImage image_mask {
.data = reinterpret_cast<uint8_t *>(mask.data()),
.size = 456 * 457 * sizeof(uint32_t),
.xpixel = 456,
.ypixel = 457,
.pixel_depth_bytes = 4,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::NO_COMPRESSION,
.channel = "sc0"
};
StartMessage message {};
message.AddPixelMask(image_mask);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
JFJochFrameDeserializer deserializer;
REQUIRE_NOTHROW(deserializer.Process(buffer.data(), serializer.GetBufferSize()));
REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START);
StartMessage output_message;
REQUIRE_NOTHROW(output_message = deserializer.GetStartMessage());
REQUIRE(output_message.pixel_mask.size() == 1);
CHECK(!output_message.pixel_mask[0].pixel_is_float);
CHECK(output_message.pixel_mask[0].xpixel == 456);
CHECK(output_message.pixel_mask[0].ypixel == 457);
CHECK(output_message.pixel_mask[0].channel == "sc0");
CHECK(memcmp(output_message.pixel_mask[0].data, mask.data(), mask.size() * sizeof(float)) == 0);
}
TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") {
std::vector<uint8_t> buffer(8*1024*1024);
@@ -152,7 +185,6 @@ TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") {
.channel = "calib1"
};
CBORImage image2 {
.data = reinterpret_cast<uint8_t *>(calib2.data()),
.size = 16 * 16 * sizeof(float),
@@ -165,10 +197,10 @@ TEST_CASE("CBORSerialize_Start_Calibration", "[CBOR]") {
.channel = "calib2"
};
StartMessage message {
.calibration = {image2, image1}
};
StartMessage message {};
message.AddCalibration(image2);
message.AddCalibration(image1);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
JFJochFrameDeserializer deserializer;
@@ -615,7 +647,21 @@ inline bool CmpString(const char *str1, const std::string& str2) {
TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") {
std::vector<uint8_t> buffer(8*1024*1024);
JFJochFrameSerializer serializer(buffer.data(), buffer.size());
JFJochFrameSerializer serializer(buffer.data(), buffer.size());
std::vector<uint32_t> mask(456*457, 15);
CBORImage image_mask {
.data = reinterpret_cast<uint8_t *>(mask.data()),
.size = 456 * 457 * sizeof(uint32_t),
.xpixel = 456,
.ypixel = 457,
.pixel_depth_bytes = 4,
.pixel_is_signed = false,
.pixel_is_float = false,
.algorithm = CompressionAlgorithm::NO_COMPRESSION,
.channel = "sc0"
};
StartMessage message {
.data_file_count = 3,
@@ -664,7 +710,8 @@ JFJochFrameSerializer serializer(buffer.data(), buffer.size());
.rad_int_bin_number = 35,
.summation = 567
};
message.pixel_mask["sc0"] = std::vector<uint32_t>(456*457, 15);
message.AddPixelMask(image_mask);
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
@@ -693,8 +740,7 @@ JFJochFrameSerializer serializer(buffer.data(), buffer.size());
CHECK(msg2->pixel_mask.ptr[0].pixel_mask.dim[0] == 457);
CHECK(msg2->pixel_mask.ptr[0].pixel_mask.dim[1] == 456);
CHECK(msg2->pixel_mask.ptr[0].pixel_mask.array.data.len == 456 * 457 * sizeof(uint32_t));
CHECK(memcmp(msg2->pixel_mask.ptr[0].pixel_mask.array.data.ptr, message.pixel_mask["sc0"].data(),
456 * 457 * sizeof(uint32_t)) == 0);
CHECK(memcmp(msg2->pixel_mask.ptr[0].pixel_mask.array.data.ptr, mask.data(), 456 * 457 * sizeof(uint32_t)) == 0);
CHECK(msg2->pixel_mask.ptr[0].pixel_mask.array.tag == TagUnsignedInt32BitLE);
stream2_free_msg(msg);
}

27
writer/HDF5NXmx.cpp
View File

@@ -192,14 +192,33 @@ void HDF5Metadata::DetectorModule(HDF5File *hdf5_file, const std::string &name,
"", "", "translation", {0,0,0});
}
void HDF5Metadata::SaveCBORImage(HDF5File *hdf5_file, const std::string &hdf5_path, const CBORImage &image) {
std::vector<hsize_t> dims = {image.xpixel, image.ypixel};
HDF5DataType data_type(image.pixel_depth_bytes, image.pixel_is_signed);
HDF5Dcpl dcpl;
dcpl.SetCompression(image.algorithm, H5Tget_size(data_type.GetID()), 0);
dcpl.SetChunking(dims);
HDF5DataSpace data_space(dims);
auto dataset = std::make_unique<HDF5DataSet>(*hdf5_file, hdf5_path, data_type, data_space, dcpl);
if (image.algorithm == CompressionAlgorithm::NO_COMPRESSION)
dataset->Write(data_type, image.data);
else
dataset->WriteDirectChunk(image.data, image.size, {0,0});
}
void HDF5Metadata::Calibration(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) {
if (start.pixel_mask.count("sc0") == 1) {
auto v = start.pixel_mask.at("sc0");
std::vector<hsize_t> dims = {start.image_size_y, start.image_size_x};
SaveVector(*hdf5_file, "/entry/instrument/detector/pixel_mask", v, dims, CompressionAlgorithm::BSHUF_LZ4);
if (!start.pixel_mask.empty()) {
SaveCBORImage(hdf5_file, "/entry/instrument/detector/pixel_mask", start.pixel_mask[0]);
hdf5_file->HardLink("/entry/instrument/detector/pixel_mask",
"/entry/instrument/detector/detectorSpecific/pixel_mask");
}
for (const auto &i: start.calibration)
SaveCBORImage(hdf5_file, "/entry/instrument/detector/detectorSpecific/" + i.channel, i);
}
void HDF5Metadata::LinkToData(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end) {

1
writer/HDF5NXmx.h
View File

@@ -27,6 +27,7 @@ namespace HDF5Metadata {
void Mask(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end);
void Calibration(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end);
void SaveCBORImage(HDF5File *hdf5_file, const std::string& hdf5_path, const CBORImage &image);
void RadInt(HDF5File *hdf5_file, const StartMessage &start, const EndMessage &end);
std::string DataFileName(const std::string& prefix, int64_t file_number);
}