From af2ed525a0d90633d9b4c6667fc9879afe65da35 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 1 May 2023 19:01:12 +0200 Subject: [PATCH 1/4] JFJochFrameSerializer: for the buffer return pointer + size, not vector reference (allows for more flexibility) --- common/ImagePusher.cpp | 12 ------- common/ImagePusher.h | 4 +-- common/TestImagePusher.cpp | 4 +-- common/TestImagePusher.h | 2 +- common/ZMQImagePusher.cpp | 12 +++---- common/ZMQImagePusher.h | 2 +- frame_serialize/JFJochFrameDeserializer.cpp | 6 +++- frame_serialize/JFJochFrameDeserializer.h | 1 + frame_serialize/JFJochFrameSerializer.cpp | 28 +++++++-------- frame_serialize/JFJochFrameSerializer.h | 4 ++- image_analysis/fast-feedback-indexer | 2 +- receiver/JFJochReceiver.cpp | 2 +- tests/CBORTest.cpp | 39 +++++++-------------- tests/ZMQImagePusherTest.cpp | 6 ++-- writer/jfjoch_writer_test.cpp | 2 +- 15 files changed, 50 insertions(+), 76 deletions(-) diff --git a/common/ImagePusher.cpp b/common/ImagePusher.cpp index 15335931..64c9f54f 100644 --- a/common/ImagePusher.cpp +++ b/common/ImagePusher.cpp @@ -15,15 +15,3 @@ void PrepareCBORImage(DataMessage& message, message.image.algorithm = experiment.GetCompressionAlgorithmEnum(); message.image.channel = "default"; } - - -void PrepareDataMessageSpots(DataMessage& message, - const std::vector& spots) { - message.spots.clear(); - for (const auto & spot : spots) - message.spots.push_back(spot); -} - -void ImagePusher::SendData(const std::vector &serialized_image, int64_t image_number) { - SendDataInternal(serialized_image, image_number); -} \ No newline at end of file diff --git a/common/ImagePusher.h b/common/ImagePusher.h index 2a16c632..7d2f4ee9 100644 --- a/common/ImagePusher.h +++ b/common/ImagePusher.h @@ -18,12 +18,10 @@ void PrepareCBORImage(DataMessage& message, void *image, size_t image_size); class ImagePusher { -protected: - virtual void SendDataInternal(const std::vector& serialized_image, int64_t image_number) = 0; public: virtual void StartDataCollection(const StartMessage& message) = 0; virtual void EndDataCollection(const EndMessage& message) = 0; - void SendData(const std::vector& serialized_image, int64_t image_number); + virtual void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) = 0; }; diff --git a/common/TestImagePusher.cpp b/common/TestImagePusher.cpp index e4331ab1..2bd2254a 100644 --- a/common/TestImagePusher.cpp +++ b/common/TestImagePusher.cpp @@ -29,14 +29,14 @@ void TestImagePusher::EndDataCollection(const EndMessage& message) { is_running = false; } -void TestImagePusher::SendDataInternal(const std::vector &serialized_image, int64_t image_number) { +void TestImagePusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) { std::unique_lock ul(m); frame_counter++; if (image_number == image_id) { JFJochFrameDeserializer deserializer; - deserializer.Process(serialized_image); + deserializer.Process(image_data, image_size); auto image_array = deserializer.GetDataMessage(); receiver_generated_image.resize(image_array.image.size); diff --git a/common/TestImagePusher.h b/common/TestImagePusher.h index 906348c3..47d89ba2 100644 --- a/common/TestImagePusher.h +++ b/common/TestImagePusher.h @@ -19,8 +19,8 @@ class TestImagePusher : public ImagePusher { bool correct_sequence = true; bool is_running = false; size_t frame_counter = 0; - void SendDataInternal(const std::vector &serialized_image, int64_t image_number) override; public: + void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; explicit TestImagePusher(int64_t image_number); void StartDataCollection(const StartMessage& message) override; void EndDataCollection(const EndMessage& message) override; diff --git a/common/ZMQImagePusher.cpp b/common/ZMQImagePusher.cpp index bf6bf01a..96537e5b 100644 --- a/common/ZMQImagePusher.cpp +++ b/common/ZMQImagePusher.cpp @@ -40,11 +40,11 @@ ZMQImagePusher::ZMQImagePusher(const std::vector &addr, } } -void ZMQImagePusher::SendDataInternal(const std::vector &serialized_image, int64_t image_number) { +void ZMQImagePusher::SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) { if (sockets.empty()) return; auto socket_number = (image_number % file_count) % sockets.size(); - sockets[socket_number]->Send(serialized_image.data(), serialized_image.size()); + sockets[socket_number]->Send(image_data, image_size); } void ZMQImagePusher::StartDataCollection(const StartMessage& message) { @@ -56,9 +56,9 @@ void ZMQImagePusher::StartDataCollection(const StartMessage& message) { file_count = message.data_file_count; serializer.SerializeSequenceStart(message); - auto &buffer = serializer.GetBuffer(); + for (const auto &s: sockets) - s->Send(buffer.data(), buffer.size(), true); + s->Send(serializer.GetBuffer(), serializer.GetBufferSize(), true); } @@ -69,9 +69,7 @@ void ZMQImagePusher::EndDataCollection(const EndMessage& message) { for (const auto &s: sockets) { serializer.SerializeSequenceEnd(end_message); - auto &buffer = serializer.GetBuffer(); - - s->Send(buffer.data(), buffer.size(), true); + s->Send(serializer.GetBuffer(), serializer.GetBufferSize(), true); end_message.write_master_file = false; } } \ No newline at end of file diff --git a/common/ZMQImagePusher.h b/common/ZMQImagePusher.h index cea7fb13..ccdaa544 100644 --- a/common/ZMQImagePusher.h +++ b/common/ZMQImagePusher.h @@ -16,8 +16,8 @@ class ZMQImagePusher : public ImagePusher { std::vector> contexts; std::vector> sockets; int64_t file_count = 1; - void SendDataInternal(const std::vector& serialized_image, int64_t image_number) override; public: + void SendImage(const uint8_t *image_data, size_t image_size, int64_t image_number) override; ZMQImagePusher(ZMQContext &context, const std::vector& addr, int32_t send_buffer_high_watermark = -1, int32_t send_buffer_size = -1); // High performance implementation, where each socket has dedicated ZMQ context diff --git a/frame_serialize/JFJochFrameDeserializer.cpp b/frame_serialize/JFJochFrameDeserializer.cpp index f3b62e2a..a7c46eb5 100644 --- a/frame_serialize/JFJochFrameDeserializer.cpp +++ b/frame_serialize/JFJochFrameDeserializer.cpp @@ -603,12 +603,16 @@ void JFJochFrameDeserializer::ProcessImageData(CborValue &value) { } void JFJochFrameDeserializer::Process(const std::vector &buffer) { + Process(buffer.data(), buffer.size()); +} + +void JFJochFrameDeserializer::Process(const uint8_t *msg, size_t msg_size) { std::unique_lock ul(m); data_message = DataMessage(); CborParser parser; CborValue value; - cborErr(cbor_parser_init(buffer.data(), buffer.size(), 0, &parser, &value)); + cborErr(cbor_parser_init(msg, msg_size, 0, &parser, &value)); if (GetCBORTag(value) != CborSignatureTag) throw JFJochException(JFJochExceptionCategory::CBORError, "CBOR must start with dedicated tag"); diff --git a/frame_serialize/JFJochFrameDeserializer.h b/frame_serialize/JFJochFrameDeserializer.h index 90011f1b..42172498 100644 --- a/frame_serialize/JFJochFrameDeserializer.h +++ b/frame_serialize/JFJochFrameDeserializer.h @@ -47,6 +47,7 @@ private: bool ProcessEndMessageElement(CborValue &value); public: void Process(const std::vector& buffer); + void Process(const uint8_t *msg, size_t msg_size); [[nodiscard]] Type GetType() const; [[nodiscard]] EndMessage GetEndMessage() const; [[nodiscard]] StartMessage GetStartMessage() const; diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index 62e935ce..5dbd51ac 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -52,9 +52,9 @@ void CBOR_ENC_COMPRESSED(CborEncoder &encoder, const void *image, size_t image_size, CompressionAlgorithm algorithm, size_t elem_size) { - if (algorithm == CompressionAlgorithm::NO_COMPRESSION) { + if (algorithm == CompressionAlgorithm::NO_COMPRESSION) cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) image, image_size)); - } else { + else { cbor_encode_tag(&encoder, TagDECTRISCompression); CborEncoder arrayEncoder; @@ -288,16 +288,19 @@ inline void CBOR_ENC_USER_DATA(CborEncoder &encoder, const StartMessage& message } JFJochFrameSerializer::JFJochFrameSerializer(size_t in_max_buffer_size) : -max_buffer_size(in_max_buffer_size) { - buffer.reserve(in_max_buffer_size); +max_buffer_size(in_max_buffer_size), curr_size(0) { + buffer.resize(max_buffer_size); } -const std::vector &JFJochFrameSerializer::GetBuffer() const { - return buffer; +const uint8_t *JFJochFrameSerializer::GetBuffer() const { + return buffer.data(); +} + +size_t JFJochFrameSerializer::GetBufferSize() const { + return curr_size; } void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) { - buffer.resize(max_buffer_size); CborEncoder encoder, mapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); @@ -342,13 +345,10 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC_CHANNELS(mapEncoder, "channels", message.channels); cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); - - buffer.resize(cbor_encoder_get_buffer_size(&encoder, buffer.data())); + curr_size = cbor_encoder_get_buffer_size(&encoder, buffer.data()); } void JFJochFrameSerializer::SerializeSequenceEnd(const EndMessage& message) { - buffer.resize(max_buffer_size); - CborEncoder encoder, mapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); cborErr(cbor_encode_tag(&encoder,CborSignatureTag )); @@ -365,12 +365,10 @@ void JFJochFrameSerializer::SerializeSequenceEnd(const EndMessage& message) { cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); - buffer.resize(cbor_encoder_get_buffer_size(&encoder, buffer.data())); + curr_size = cbor_encoder_get_buffer_size(&encoder, buffer.data()); } void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { - buffer.resize(max_buffer_size); - CborEncoder encoder, mapEncoder, userDataMapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); @@ -397,5 +395,5 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { cborErr(cbor_encoder_close_container(&encoder, &mapEncoder)); - buffer.resize(cbor_encoder_get_buffer_size(&encoder, buffer.data())); + curr_size = cbor_encoder_get_buffer_size(&encoder, buffer.data()); } diff --git a/frame_serialize/JFJochFrameSerializer.h b/frame_serialize/JFJochFrameSerializer.h index ecc28706..251f407b 100644 --- a/frame_serialize/JFJochFrameSerializer.h +++ b/frame_serialize/JFJochFrameSerializer.h @@ -15,9 +15,11 @@ class JFJochFrameSerializer { std::vector buffer; size_t max_buffer_size; + size_t curr_size; public: explicit JFJochFrameSerializer(size_t buffer_size); - [[nodiscard]] const std::vector &GetBuffer() const; + [[nodiscard]] const uint8_t *GetBuffer() const; + [[nodiscard]] size_t GetBufferSize() const; void SerializeSequenceStart(const StartMessage& message); void SerializeSequenceEnd(const EndMessage& message); void SerializeImage(const DataMessage& message); diff --git a/image_analysis/fast-feedback-indexer b/image_analysis/fast-feedback-indexer index 111b4975..6eccc70b 160000 --- a/image_analysis/fast-feedback-indexer +++ b/image_analysis/fast-feedback-indexer @@ -1 +1 @@ -Subproject commit 111b4975cadc013ad98fb982dfc3d252c9721cb5 +Subproject commit 6eccc70b524ba7a0207aa2c93b85769c1c9f5a62 diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 88c4d658..7bb5591c 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -413,7 +413,7 @@ int64_t JFJochReceiver::FrameTransformationThread() { if (push_images_to_writer) { PrepareCBORImage(message, experiment, writer_buffer.data(), image_size); serializer.SerializeImage(message); - image_pusher.SendData(serializer.GetBuffer(), image_number); + image_pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), image_number); } UpdateMaxImage(image_number); diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index 59dfa809..b22bb1dc 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -63,10 +63,8 @@ TEST_CASE("CBORSerialize_Start", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); - auto image = serializer.GetBuffer(); - JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(image)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::START); StartMessage output_message; @@ -137,11 +135,9 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") { }; REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); - - auto image = serializer.GetBuffer(); - + JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(image)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::END); EndMessage output_message{}; @@ -188,10 +184,9 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") { }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto serialized = serializer.GetBuffer(); JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(serialized)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); auto image_array = deserializer.GetDataMessage(); @@ -241,13 +236,11 @@ TEST_CASE("CBORSerialize_Image_2", "[CBOR]") { }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto serialized = serializer.GetBuffer(); JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(serialized)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); - auto image_array = deserializer.GetDataMessage(); REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION); REQUIRE(image_array.image.xpixel == 1024); @@ -288,10 +281,9 @@ TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto serialized = serializer.GetBuffer(); JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(serialized)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); auto image_array = deserializer.GetDataMessage(); @@ -331,10 +323,8 @@ TEST_CASE("CBORSerialize_Image_Rad_Int_Profile", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto serialized = serializer.GetBuffer(); - JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(serialized)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); auto image_array = deserializer.GetDataMessage(); @@ -374,10 +364,8 @@ TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto serialized = serializer.GetBuffer(); - JFJochFrameDeserializer deserializer; - REQUIRE_NOTHROW(deserializer.Process(serialized)); + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); auto image_array = deserializer.GetDataMessage(); @@ -452,10 +440,9 @@ TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); - auto image = serializer.GetBuffer(); stream2_msg *msg; - auto ret = stream2_parse_msg(image.data(), image.size(), &msg); + auto ret = stream2_parse_msg(serializer.GetBuffer(), serializer.GetBufferSize(), &msg); REQUIRE(ret == STREAM2_OK); CHECK(msg->type == STREAM2_MSG_START); auto msg2 = (stream2_start_msg *) msg; @@ -499,10 +486,9 @@ TEST_CASE("CBORSerialize_End_stream2", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); - auto image = serializer.GetBuffer(); stream2_msg *msg; - auto ret = stream2_parse_msg(image.data(), image.size(), &msg); + auto ret = stream2_parse_msg(serializer.GetBuffer(), serializer.GetBufferSize(), &msg); REQUIRE(ret == STREAM2_OK); CHECK(msg->type == STREAM2_MSG_END); auto msg2 = (stream2_end_msg *) msg; @@ -546,11 +532,10 @@ TEST_CASE("CBORSerialize_Image_compressed_stream2", "[CBOR]") { }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); - auto cbor_image = serializer.GetBuffer(); stream2_msg *msg; - auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg); + auto ret = stream2_parse_msg(serializer.GetBuffer(), serializer.GetBufferSize(), &msg); REQUIRE(ret == STREAM2_OK); CHECK(msg->type == STREAM2_MSG_IMAGE); auto msg2 = (stream2_image_msg *) msg; @@ -601,7 +586,7 @@ TEST_CASE("CBORSerialize_Image_uncompressed_stream2", "[CBOR]") { stream2_msg *msg; - auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg); + auto ret = stream2_parse_msg(serializer.GetBuffer(), serializer.GetBufferSize(), &msg); REQUIRE(ret == STREAM2_OK); CHECK(msg->type == STREAM2_MSG_IMAGE); auto msg2 = (stream2_image_msg *) msg; diff --git a/tests/ZMQImagePusherTest.cpp b/tests/ZMQImagePusherTest.cpp index 2dda7ecf..68d080d8 100644 --- a/tests/ZMQImagePusherTest.cpp +++ b/tests/ZMQImagePusherTest.cpp @@ -97,7 +97,7 @@ TEST_CASE("ZMQImageCommTest_1Writer","[ZeroMQ]") { data_message.number = i; PrepareCBORImage(data_message, x, image1.data() + i * x.GetPixelsNum(), x.GetPixelsNum() * sizeof(uint16_t)); serializer.SerializeImage(data_message); - pusher.SendData(serializer.GetBuffer(), i); + pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), i); } pusher.EndDataCollection(end_message); @@ -172,7 +172,7 @@ TEST_CASE("ZMQImageCommTest_2Writers","[ZeroMQ]") { data_message.number = i; PrepareCBORImage(data_message, x, image1.data() + i * x.GetPixelsNum(), x.GetPixelsNum() * sizeof(uint16_t)); serializer.SerializeImage(data_message); - pusher.SendData(serializer.GetBuffer(), i); + pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), i); } pusher.EndDataCollection(end_message); @@ -260,7 +260,7 @@ TEST_CASE("ZMQImageCommTest_4Writers","[ZeroMQ]") { data_message.number = i; PrepareCBORImage(data_message, x, image1.data() + i * x.GetPixelsNum(), x.GetPixelsNum() * sizeof(uint16_t)); serializer.SerializeImage(data_message); - pusher.SendData(serializer.GetBuffer(), i); + pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), i); } pusher.EndDataCollection(end_message); diff --git a/writer/jfjoch_writer_test.cpp b/writer/jfjoch_writer_test.cpp index ea55712a..90d8c7c5 100644 --- a/writer/jfjoch_writer_test.cpp +++ b/writer/jfjoch_writer_test.cpp @@ -102,7 +102,7 @@ int main(int argc, char **argv) { data_message.number = i; PrepareCBORImage(data_message, x, output[i % nimages_in_file].data(), output_size[i % nimages_in_file]); serializer.SerializeImage(data_message); - pusher.SendData(serializer.GetBuffer(), i); + pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), i); } EndMessage end_message{}; From dce186f9f98da5dc1a07f018b5912b79c8a8333c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 1 May 2023 21:09:03 +0200 Subject: [PATCH 2/4] JFJochFrameSerializer: add functions to append image to buffer --- frame_serialize/JFJochFrameSerializer.cpp | 30 +++++++++++++- frame_serialize/JFJochFrameSerializer.h | 4 ++ tests/CBORTest.cpp | 48 +++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index 5dbd51ac..51d59eb8 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -82,8 +82,8 @@ inline void CBOR_ENC_MULTIDIM_TYPED_ARRAY(CborEncoder &encoder, const char* key, const void *image, size_t image_size, size_t xpixel, size_t ypixel, CompressionAlgorithm algorithm, size_t elem_size, bool elem_sign) { - if ((algorithm == CompressionAlgorithm::NO_COMPRESSION) && (xpixel * ypixel != image_size / elem_size)) - throw JFJochException(JFJochExceptionCategory::CBORError, "Mismatch in array size"); + //if ((algorithm == CompressionAlgorithm::NO_COMPRESSION) && (xpixel * ypixel != image_size / elem_size)) + // throw JFJochException(JFJochExceptionCategory::CBORError, "Mismatch in array size"); CborEncoder arrayEncoder, arrayEncoder_2; cborErr(cbor_encode_text_stringz(&encoder, key)); @@ -296,10 +296,35 @@ const uint8_t *JFJochFrameSerializer::GetBuffer() const { return buffer.data(); } +uint8_t *JFJochFrameSerializer::GetImageAppendLocation() { + return buffer.data() + curr_size + sizeof(size_t); +} + size_t JFJochFrameSerializer::GetBufferSize() const { return curr_size; } +size_t JFJochFrameSerializer::GetRemainingBuffer() const { + return max_buffer_size - curr_size; +} + +void JFJochFrameSerializer::AppendImage(size_t image_size) { + if (curr_size + image_size + sizeof(size_t) >= max_buffer_size) + throw JFJochException(JFJochExceptionCategory::CBORError, "No space to extend the image"); + curr_size--; + buffer[curr_size] = 0x40 | 27; + curr_size++; +#ifdef LITTLE_ENDIAN + size_t image_size_be = __builtin_bswap64(image_size); +#else + size_t image_size_be = image_size; +#endif + memcpy(buffer.data() + curr_size, &image_size_be, sizeof(size_t)); + curr_size += sizeof(size_t); + curr_size += image_size; +} + + void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) { CborEncoder encoder, mapEncoder; @@ -397,3 +422,4 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { curr_size = cbor_encoder_get_buffer_size(&encoder, buffer.data()); } + diff --git a/frame_serialize/JFJochFrameSerializer.h b/frame_serialize/JFJochFrameSerializer.h index 251f407b..fc48c895 100644 --- a/frame_serialize/JFJochFrameSerializer.h +++ b/frame_serialize/JFJochFrameSerializer.h @@ -19,10 +19,14 @@ class JFJochFrameSerializer { public: explicit JFJochFrameSerializer(size_t buffer_size); [[nodiscard]] const uint8_t *GetBuffer() const; + [[nodiscard]] uint8_t *GetImageAppendLocation(); [[nodiscard]] size_t GetBufferSize() const; + [[nodiscard]] size_t GetRemainingBuffer() const; void SerializeSequenceStart(const StartMessage& message); void SerializeSequenceEnd(const EndMessage& message); void SerializeImage(const DataMessage& message); + void AppendImage(size_t image_size); + }; diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index b22bb1dc..e82c91a6 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -254,6 +254,54 @@ TEST_CASE("CBORSerialize_Image_2", "[CBOR]") { REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0); } +TEST_CASE("CBORSerialize_Image_Append", "[CBOR]") { + JFJochFrameSerializer serializer(8*1024*1024); + + std::vector spots; + + std::vector test(512*1024); + for (int i = 0; i < test.size(); i++) + test[i] = (i * 253 + 56) % 256; + + CBORImage image { + .data = nullptr, + .size = 0, + .xpixel = 1024, + .ypixel = 512, + .pixel_depth_bytes = 1, + .pixel_is_signed = false, + .algorithm = CompressionAlgorithm::NO_COMPRESSION, + .channel = "default" + }; + + DataMessage message { + .number = 480, + .image = image, + .spots = spots, + .indexing_result = 3 + }; + + REQUIRE_NOTHROW(serializer.SerializeImage(message)); + memcpy(serializer.GetImageAppendLocation(), test.data(), 512*1024); + REQUIRE_NOTHROW(serializer.AppendImage(512*1024)); + + JFJochFrameDeserializer deserializer; + REQUIRE_NOTHROW(deserializer.Process(serializer.GetBuffer(), serializer.GetBufferSize())); + REQUIRE(deserializer.GetType() == JFJochFrameDeserializer::Type::IMAGE); + + auto image_array = deserializer.GetDataMessage(); + REQUIRE(image_array.image.algorithm == CompressionAlgorithm::NO_COMPRESSION); + REQUIRE(image_array.image.xpixel == 1024); + REQUIRE(image_array.image.ypixel == 512); + REQUIRE(image_array.image.pixel_depth_bytes == 1); + REQUIRE(!image_array.image.pixel_is_signed); + REQUIRE(image_array.image.channel == "default"); + REQUIRE(image_array.image.size == test.size()); + REQUIRE(image_array.indexing_result == message.indexing_result); + REQUIRE(image_array.number == 480); + REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0); +} + TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { JFJochFrameSerializer serializer(8*1024*1024); From 9d5be2a4e9c27413aded51f5cc34e2fbfc3e19d5 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 1 May 2023 21:27:08 +0200 Subject: [PATCH 3/4] FrameTransformation: Split packing and compression --- common/FrameTransformation.cpp | 15 +++----- common/FrameTransformation.h | 7 +--- receiver/JFJochReceiver.cpp | 6 +-- tests/FrameTransformationTest.cpp | 63 ++++++++++++------------------- tools/CompressionBenchmark.cpp | 8 ++-- tools/HDF5DatasetWriteTest.cpp | 4 +- writer/jfjoch_writer_test.cpp | 4 +- 7 files changed, 44 insertions(+), 63 deletions(-) diff --git a/common/FrameTransformation.cpp b/common/FrameTransformation.cpp index 05a05e2a..924a29a2 100644 --- a/common/FrameTransformation.cpp +++ b/common/FrameTransformation.cpp @@ -40,11 +40,6 @@ FrameTransformation::FrameTransformation(const DiffractionExperiment &in_experim } } -FrameTransformation& FrameTransformation::SetOutput(void *output) { - standard_output = (char *) output; - return *this; -} - template void AddToFramesSum(Td *destination, const int16_t *source) { for (int i = 0; i < RAW_MODULE_SIZE; i++) { if ((source[i] == INT16_MIN) || (destination[i] == INT32_MIN)) @@ -90,9 +85,7 @@ void FrameTransformation::Generate16BitPreview() { } } -size_t FrameTransformation::PackStandardOutput() { - if (standard_output == nullptr) - throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "Default stream output not initialized"); +void FrameTransformation::Pack() { if (summation > 1) { PackSummation(); @@ -114,7 +107,11 @@ size_t FrameTransformation::PackStandardOutput() { experiment.GetYPixelsNumFullImage(), INT16_MIN, INT16_MAX); } - return compressor.Compress(standard_output, precompression_buffer.data(), + +} + +size_t FrameTransformation::SaveCompressedImage(void *output) { + return compressor.Compress((char *) output, precompression_buffer.data(), experiment.GetPixelsNum(), pixel_depth); } diff --git a/common/FrameTransformation.h b/common/FrameTransformation.h index ad463ce7..13442a2d 100644 --- a/common/FrameTransformation.h +++ b/common/FrameTransformation.h @@ -16,8 +16,6 @@ class FrameTransformation { std::vector precompression_buffer; std::vector image16bit; - char *standard_output = nullptr; - const size_t summation; const size_t pixel_depth; @@ -27,12 +25,11 @@ class FrameTransformation { void Generate16BitPreview(); public: FrameTransformation(const DiffractionExperiment &experiment); - FrameTransformation& SetOutput(void *output); void ProcessModule(const int16_t *input, uint16_t module_number, int data_stream); void ProcessModule(JFConversion &conv, const int16_t *input, uint16_t module_number, int data_stream); - size_t PackStandardOutput(); // transfer summed image to converted coordinates, clear summation buffer, compress - + void Pack(); // transfer summed image to converted coordinates, clear summation buffer + size_t SaveCompressedImage(void *output); int16_t *GetPreview16BitImage(); }; diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index 7bb5591c..f55c967c 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -286,8 +286,6 @@ int64_t JFJochReceiver::FrameTransformationThread() { int64_t max_thread_delay = 0; uint64_t image_number; - transformation.SetOutput(writer_buffer.data()); - frame_transformation_ready->CountDown(); std::unique_ptr indexer; @@ -354,7 +352,7 @@ int64_t JFJochReceiver::FrameTransformationThread() { } if (send_image) { - size_t image_size = transformation.PackStandardOutput(); + transformation.Pack(); std::vector spots; auto local_data_processing_settings = GetDataProcessingSettings(); @@ -410,6 +408,8 @@ int64_t JFJochReceiver::FrameTransformationThread() { AddRadialIntegrationProfile(message.rad_int_profile); } + size_t image_size = transformation.SaveCompressedImage(writer_buffer.data()); + if (push_images_to_writer) { PrepareCBORImage(message, experiment, writer_buffer.data(), image_size); serializer.SerializeImage(message); diff --git a/tests/FrameTransformationTest.cpp b/tests/FrameTransformationTest.cpp index 64ec9fd1..5b02bc04 100644 --- a/tests/FrameTransformationTest.cpp +++ b/tests/FrameTransformationTest.cpp @@ -39,14 +39,12 @@ TEST_CASE("FrameTransformation_Raw_NoCompression" ,"") { std::vector output(experiment.GetPixelsNum()); - transformation.SetOutput(output.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } - - REQUIRE(transformation.PackStandardOutput() == experiment.GetPixelDepth() * experiment.GetPixelsNum()); + transformation.Pack(); + REQUIRE(transformation.SaveCompressedImage(output.data()) == experiment.GetPixelDepth() * experiment.GetPixelsNum()); uint32_t diff_0 = 0; uint32_t diff_1 = 0; @@ -81,14 +79,13 @@ TEST_CASE("FrameTransformation_Converted_NoCompression" ,"") { std::vector output(experiment.GetPixelsNum()); - transformation.SetOutput(output.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } - REQUIRE(transformation.PackStandardOutput() == experiment.GetPixelDepth() * experiment.GetPixelsNum()); + transformation.Pack(); + REQUIRE(transformation.SaveCompressedImage(output.data()) == experiment.GetPixelDepth() * experiment.GetPixelsNum()); REQUIRE(input_0[511*1024] == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 0]); REQUIRE(input_0[511*1024+256]/2 == output[CONVERTED_MODULE_SIZE * (2 * nmodules - 2) + 258]); @@ -134,15 +131,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_lz4" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); output_compressed.resize(compressed_size); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); @@ -195,16 +191,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -257,15 +251,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_bin_2x2" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -305,16 +298,14 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_rle" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -369,8 +360,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); @@ -392,7 +381,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation" ,"") { } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -440,8 +430,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_bin_2x2" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - for (int frame = 0; frame < experiment.GetSummation(); frame++) { for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); @@ -450,7 +438,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_bin_2x2" ,"") { } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -493,15 +482,14 @@ TEST_CASE("FrameTransformation_Converted_16bit_preview_summation" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data(), 0, 0)); size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -561,9 +549,7 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - - for (int i = 0; i < nmodules; i++) { + for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } @@ -574,7 +560,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") { } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); @@ -596,7 +583,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_2frames" ,"") { REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); @@ -644,8 +632,6 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_negatives" ,"") { std::vector output_compressed(experiment.GetMaxCompressedSize()); - transformation.SetOutput(output_compressed.data()); - for (int i = 0; i < nmodules; i++) { REQUIRE_NOTHROW(transformation.ProcessModule(input_0.data() + i * RAW_MODULE_SIZE, i, 0)); REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); @@ -666,7 +652,8 @@ TEST_CASE("FrameTransformation_Converted_bshuf_zstd_summation_negatives" ,"") { REQUIRE_NOTHROW(transformation.ProcessModule(input_1.data() + i * RAW_MODULE_SIZE, i, 1)); } size_t compressed_size; - REQUIRE_NOTHROW(compressed_size = transformation.PackStandardOutput()); + transformation.Pack(); + REQUIRE_NOTHROW(compressed_size = transformation.SaveCompressedImage(output_compressed.data())); REQUIRE(bshuf_read_uint64_BE(output_compressed.data()) == experiment.GetPixelsNum() * experiment.GetPixelDepth()); REQUIRE(bshuf_read_uint32_BE(output_compressed.data()+8) == JFJochBitShuffleCompressor::DefaultBlockSize * experiment.GetPixelDepth()); diff --git a/tools/CompressionBenchmark.cpp b/tools/CompressionBenchmark.cpp index 4646ca8c..fcf37e21 100644 --- a/tools/CompressionBenchmark.cpp +++ b/tools/CompressionBenchmark.cpp @@ -21,11 +21,11 @@ std::string CheckCompression(const DiffractionExperiment &x, size_t nimages, con auto start_time = std::chrono::system_clock::now(); for (int i = 0; i < nimages; i++) { - transformation.SetOutput(output.data() + i * x.GetMaxCompressedSize()); for (int j = 0; j < x.GetModulesNum(); j++ ) { transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0); } - compressed_size += transformation.PackStandardOutput(); + transformation.Pack(); + compressed_size += transformation.SaveCompressedImage(output.data() + i * x.GetMaxCompressedSize()); } auto end_time = std::chrono::system_clock::now(); @@ -47,11 +47,11 @@ std::string CheckDecompression(const DiffractionExperiment &x, size_t nimages, c std::vector compressed_size(nimages); for (int i = 0; i < nimages; i++) { - transformation.SetOutput(output[i].data()); for (int j = 0; j < x.GetModulesNum(); j++ ) { transformation.ProcessModule(image.data() + (j + i * x.GetModulesNum()) * RAW_MODULE_SIZE, j, 0); } - compressed_size[i] = transformation.PackStandardOutput(); + transformation.Pack(); + compressed_size[i] = transformation.SaveCompressedImage(output[i].data()); output[i].resize(compressed_size[i]); } diff --git a/tools/HDF5DatasetWriteTest.cpp b/tools/HDF5DatasetWriteTest.cpp index d6e4bf9e..7fcac68f 100644 --- a/tools/HDF5DatasetWriteTest.cpp +++ b/tools/HDF5DatasetWriteTest.cpp @@ -87,10 +87,10 @@ int main(int argc, char **argv) { for (auto &i: output) i.resize(x.GetMaxCompressedSize()); for (int i = 0; i < nimages; i++) { - transformation.SetOutput(output[i].data()); for (int j = 0; j < 8; j++) transformation.ProcessModule(image + (i * x.GetModulesNum() + j) * RAW_MODULE_SIZE, j, 0); - output_size[i] = transformation.PackStandardOutput(); + transformation.Pack(); + output_size[i] = transformation.SaveCompressedImage(output[i].data()); } x.ImagesPerTrigger(nimages_out); diff --git a/writer/jfjoch_writer_test.cpp b/writer/jfjoch_writer_test.cpp index 90d8c7c5..f81f1c28 100644 --- a/writer/jfjoch_writer_test.cpp +++ b/writer/jfjoch_writer_test.cpp @@ -79,10 +79,10 @@ int main(int argc, char **argv) { image_tmp_raw.data(), image_tmp_conv.data()); - transformation.SetOutput(output[i].data()); for (int j = 0; j < x.GetModulesNum(); j++) transformation.ProcessModule(image_tmp_raw.data() + j * RAW_MODULE_SIZE, j, 0); - output_size[i] = transformation.PackStandardOutput(); + transformation.Pack(); + output_size[i] = transformation.SaveCompressedImage((uint8_t *) output[i].data()); } logger.Info("Sending {} images", nimages_out); From ef7b6e84d76d88d389409ac27bb7653027905ae3 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 1 May 2023 21:37:55 +0200 Subject: [PATCH 4/4] JFJochReceiver: Compress directly to CBOR format --- receiver/JFJochReceiver.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/receiver/JFJochReceiver.cpp b/receiver/JFJochReceiver.cpp index f55c967c..c099f46b 100644 --- a/receiver/JFJochReceiver.cpp +++ b/receiver/JFJochReceiver.cpp @@ -408,16 +408,16 @@ int64_t JFJochReceiver::FrameTransformationThread() { AddRadialIntegrationProfile(message.rad_int_profile); } - size_t image_size = transformation.SaveCompressedImage(writer_buffer.data()); - if (push_images_to_writer) { - PrepareCBORImage(message, experiment, writer_buffer.data(), image_size); + PrepareCBORImage(message, experiment, nullptr, 0); serializer.SerializeImage(message); + size_t image_size = transformation.SaveCompressedImage(serializer.GetImageAppendLocation()); + serializer.AppendImage(image_size); image_pusher.SendImage(serializer.GetBuffer(), serializer.GetBufferSize(), image_number); + compressed_size += image_size; } UpdateMaxImage(image_number); - compressed_size += image_size; images_sent++; } } catch (const JFJochException &e) { Abort(e); }