diff --git a/compression/JFJochCompressor.cpp b/compression/JFJochCompressor.cpp index 99d603b1..1263f251 100644 --- a/compression/JFJochCompressor.cpp +++ b/compression/JFJochCompressor.cpp @@ -72,7 +72,6 @@ size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size if (tmp_space.size() < DefaultBlockSize * elem_size) tmp_space.resize(DefaultBlockSize * elem_size); - size_t num_full_blocks = nelements / DefaultBlockSize; size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize; size_t compressed_size = 12; diff --git a/frame_serialize/CMakeLists.txt b/frame_serialize/CMakeLists.txt index f72fac12..999a530a 100644 --- a/frame_serialize/CMakeLists.txt +++ b/frame_serialize/CMakeLists.txt @@ -2,6 +2,7 @@ ADD_LIBRARY(FrameSerialize STATIC JFJochFrameSerializer.cpp JFJochFrameSerializer.h JFJochFrameDeserializer.cpp JFJochFrameDeserializer.h ImageMessage.h + tinycbor/src/cborparser_dup_string.c tinycbor/src/cborencoder.c tinycbor/src/cborencoder_close_container_checked.c tinycbor/src/cborencoder_float.c @@ -10,4 +11,5 @@ ADD_LIBRARY(FrameSerialize STATIC tinycbor/src/cborpretty.c tinycbor/src/cborerrorstrings.c tinycbor/src/cbor.h - tinycbor/src/tinycbor-version.h CborErr.h StartMessage.h EndMessage.h CborUtil.h) + tinycbor/src/tinycbor-version.h CborErr.h StartMessage.h EndMessage.h CborUtil.h + stream2.h stream2.c) diff --git a/frame_serialize/CborUtil.h b/frame_serialize/CborUtil.h index cb016f54..9f5c29a0 100644 --- a/frame_serialize/CborUtil.h +++ b/frame_serialize/CborUtil.h @@ -6,8 +6,7 @@ #include "tinycbor/src/cbor.h" -constexpr const CborTag SelfDescribedCBOR = 55799; -constexpr const CborTag TagMultiDimArray = 40; +constexpr const CborTag TagMultiDimArray = 40; constexpr const CborTag TagDECTRISCompression = 56500; constexpr const CborTag TagFloatLE = 0b01010101; diff --git a/frame_serialize/EndMessage.h b/frame_serialize/EndMessage.h index ac96ff0d..2527bb2a 100644 --- a/frame_serialize/EndMessage.h +++ b/frame_serialize/EndMessage.h @@ -12,6 +12,9 @@ struct EndMessage { bool write_master_file; std::string end_date; + + std::string series_unique_id; + uint64_t series_id; }; #endif //JUNGFRAUJOCH_ENDMESSAGE_H diff --git a/frame_serialize/ImageMessage.h b/frame_serialize/ImageMessage.h index f6b579dd..c0f1a586 100644 --- a/frame_serialize/ImageMessage.h +++ b/frame_serialize/ImageMessage.h @@ -31,6 +31,9 @@ struct DataMessage { uint64_t bunch_id; uint32_t jf_info; uint64_t timestamp; + + std::string series_unique_id; + uint64_t series_id; }; #endif //JUNGFRAUJOCH_IMAGEMESSAGE_H diff --git a/frame_serialize/JFJochFrameDeserializer.cpp b/frame_serialize/JFJochFrameDeserializer.cpp index 13bb11ab..f3b62e2a 100644 --- a/frame_serialize/JFJochFrameDeserializer.cpp +++ b/frame_serialize/JFJochFrameDeserializer.cpp @@ -65,6 +65,19 @@ inline CborTag GetCBORTag(CborValue &value) { return tmp; } +inline std::string GetCBORDateTime(CborValue &value) { + auto tag = GetCBORTag(value); + if (tag == CborDateTimeStringTag) + return GetCBORString(value); + else if (tag == CborUnixTime_tTag) { + time_t t = GetCBORUInt(value); + char buf1[255]; + strftime(buf1, sizeof(buf1), "%FT%T", gmtime(&t)); + return std::string(buf1) + "Z"; + } else + throw JFJochException(JFJochExceptionCategory::CBORError, "Time/date tag error"); +} + inline uint64_t GetCBORArrayLen(CborValue &value) { if (!cbor_value_is_array(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); @@ -355,6 +368,10 @@ bool JFJochFrameDeserializer::ProcessImageMessageElement(CborValue &value) { data_message.number = GetCBORInt(value); else if (key == "data") ProcessImageData(value); + else if (key == "series_unique_id") + data_message.series_unique_id = GetCBORString(value); + else if (key == "series_id") + data_message.series_id = GetCBORUInt(value); else if (key == "user_data") ProcessImageMessageUserDataElement(value); else @@ -491,9 +508,9 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) { start_message.detector_distance = GetCBORFloat(value); else if (key == "number_of_images") start_message.number_of_images = GetCBORUInt(value); - else if (key == "xpixel") + else if (key == "image_size_x") start_message.image_size_x = GetCBORUInt(value); - else if (key == "ypixel") + else if (key == "image_size_y") start_message.image_size_y = GetCBORUInt(value); else if (key == "incident_energy") start_message.incident_energy = GetCBORFloat(value); @@ -520,7 +537,7 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) { else if (key == "series_unique_id") start_message.series_unique_id = GetCBORString(value); else if (key == "series_id") - start_message.series_id = GetCBORInt(value); + start_message.series_id = GetCBORUInt(value); else if (key == "pixel_mask") ProcessPixelMaskElement(value); else if (key == "channels") @@ -532,7 +549,7 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) { else if (key == "pixel_mask_enabled") start_message.pixel_mask_enabled = GetCBORBool(value); else if (key == "arm_date") - start_message.arm_date = GetCBORString(value); + start_message.arm_date = GetCBORDateTime(value); else if (key == "user_data") ProcessStartMessageUserDataElement(value); else @@ -556,6 +573,11 @@ bool JFJochFrameDeserializer::ProcessEndMessageElement(CborValue &value) { end_message.write_master_file = GetCBORBool(value); else if (key == "end_date") end_message.end_date = GetCBORString(value); + else if (key == "series_unique_id") + end_message.series_unique_id = GetCBORString(value); + else if (key == "series_id") + end_message.series_id = GetCBORUInt(value); + else cbor_value_advance(&value); return true; @@ -588,7 +610,7 @@ std::unique_lock ul(m); CborValue value; cborErr(cbor_parser_init(buffer.data(), buffer.size(), 0, &parser, &value)); - if (GetCBORTag(value) != SelfDescribedCBOR) + if (GetCBORTag(value) != CborSignatureTag) throw JFJochException(JFJochExceptionCategory::CBORError, "CBOR must start with dedicated tag"); diff --git a/frame_serialize/JFJochFrameSerializer.cpp b/frame_serialize/JFJochFrameSerializer.cpp index bb897b90..90717bea 100644 --- a/frame_serialize/JFJochFrameSerializer.cpp +++ b/frame_serialize/JFJochFrameSerializer.cpp @@ -19,6 +19,15 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::string &v cborErr(cbor_encode_text_stringz(&encoder, value.c_str())); } +inline void CBOR_ENC_DATE(CborEncoder &encoder, const char* key, const std::string &value) { + cborErr(cbor_encode_text_stringz(&encoder, key)); + cbor_encode_tag(&encoder, CborDateTimeStringTag); + if (value.empty()) + cborErr(cbor_encode_text_stringz(&encoder, ".")); + else + cborErr(cbor_encode_text_stringz(&encoder, value.c_str())); +} + inline void CBOR_ENC(CborEncoder &encoder, const char* key, float value) { cborErr(cbor_encode_text_stringz(&encoder, key)); cborErr(cbor_encode_float(&encoder, value)); @@ -88,28 +97,25 @@ inline void CBOR_ENC_MULTIDIM_TYPED_ARRAY(CborEncoder &encoder, const char* key, cborErr(cbor_encoder_close_container(&arrayEncoder, &arrayEncoder_2)); CborTag typed_array_tag; - if (algorithm != CompressionAlgorithm::NO_COMPRESSION) - typed_array_tag = TagUnsignedInt8Bit; - else { - if (elem_sign) { - if (elem_size == 4) - typed_array_tag = TagSignedInt32BitLE; - else if (elem_size == 2) - typed_array_tag = TagSignedInt16BitLE; - else if (elem_size == 1) - typed_array_tag = TagSignedInt8Bit; - else - throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported"); - } else { - if (elem_size == 4) - typed_array_tag = TagUnsignedInt32BitLE; - else if (elem_size == 2) - typed_array_tag = TagUnsignedInt16BitLE; - else if (elem_size == 1) - typed_array_tag = TagUnsignedInt8Bit; - else - throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported"); - } + + if (elem_sign) { + if (elem_size == 4) + typed_array_tag = TagSignedInt32BitLE; + else if (elem_size == 2) + typed_array_tag = TagSignedInt16BitLE; + else if (elem_size == 1) + typed_array_tag = TagSignedInt8Bit; + else + throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported"); + } else { + if (elem_size == 4) + typed_array_tag = TagUnsignedInt32BitLE; + else if (elem_size == 2) + typed_array_tag = TagUnsignedInt16BitLE; + else if (elem_size == 1) + typed_array_tag = TagUnsignedInt8Bit; + else + throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported"); } cbor_encode_tag(&arrayEncoder, typed_array_tag); @@ -295,7 +301,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) CborEncoder encoder, mapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); - cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR )); + cborErr(cbor_encode_tag(&encoder, CborSignatureTag )); size_t elements = 27; cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, elements)); @@ -306,8 +312,8 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC(mapEncoder, "beam_center_y", message.beam_center_y); CBOR_ENC(mapEncoder, "number_of_images", message.number_of_images); - CBOR_ENC(mapEncoder, "xpixel", message.image_size_x); - CBOR_ENC(mapEncoder, "ypixel", message.image_size_y); + CBOR_ENC(mapEncoder, "image_size_x", message.image_size_x); + CBOR_ENC(mapEncoder, "image_size_y", message.image_size_y); CBOR_ENC(mapEncoder, "incident_energy", message.incident_energy); CBOR_ENC(mapEncoder, "incident_wavelength", message.incident_wavelength); @@ -321,7 +327,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message) CBOR_ENC(mapEncoder, "pixel_size_y", message.pixel_size_y); CBOR_ENC(mapEncoder, "sensor_thickness", message.sensor_thickness); CBOR_ENC(mapEncoder, "sensor_material", message.sensor_material); - CBOR_ENC(mapEncoder, "arm_date", message.arm_date); + CBOR_ENC_DATE(mapEncoder, "arm_date", message.arm_date); CBOR_ENC(mapEncoder, "pixel_mask_enabled", message.pixel_mask_enabled); CBOR_ENC(mapEncoder, "detector_description", message.detector_description); CBOR_ENC(mapEncoder, "detector_serial_number", message.detector_serial_number); @@ -345,11 +351,12 @@ void JFJochFrameSerializer::SerializeSequenceEnd(const EndMessage& message) { CborEncoder encoder, mapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); - cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR )); - cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 6)); + cborErr(cbor_encode_tag(&encoder,CborSignatureTag )); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 8)); CBOR_ENC(mapEncoder, "type", "end"); - + CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id); + CBOR_ENC(mapEncoder, "series_id", message.series_id); CBOR_ENC(mapEncoder, "number_of_images", message.number_of_images); CBOR_ENC(mapEncoder, "max_receiver_delay", message.max_receiver_delay); CBOR_ENC(mapEncoder, "receiver_efficiency", message.efficiency); @@ -367,10 +374,12 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) { CborEncoder encoder, mapEncoder, userDataMapEncoder; cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0); - cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR )); - cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 4)); + cborErr(cbor_encode_tag(&encoder, CborSignatureTag )); + cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 6)); CBOR_ENC(mapEncoder, "type", "image"); + CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id); + CBOR_ENC(mapEncoder, "series_id", message.series_id); CBOR_ENC(mapEncoder, "image_id", message.number); CBOR_ENC(mapEncoder, "data", message.image); diff --git a/frame_serialize/StartMessage.h b/frame_serialize/StartMessage.h index a83dcb99..c77330fa 100644 --- a/frame_serialize/StartMessage.h +++ b/frame_serialize/StartMessage.h @@ -65,7 +65,7 @@ struct StartMessage { std::string detector_description; std::string detector_serial_number; std::string series_unique_id; - int64_t series_id; + uint64_t series_id; std::map goniometer; float detector_translation[3]; diff --git a/frame_serialize/stream2.c b/frame_serialize/stream2.c new file mode 100644 index 00000000..3f6c50b5 --- /dev/null +++ b/frame_serialize/stream2.c @@ -0,0 +1,953 @@ +// DECTRIS proprietary license + +#define __STDC_WANT_IEC_60559_TYPES_EXT__ +#include "stream2.h" + +#include +#include +#if FLT16_MANT_DIG > 0 || __FLT16_MANT_DIG__ > 0 +#else +#include +#if defined(_MSC_VER) +#include +#endif +#endif +#include +#include +#include + +#include "tinycbor/src/cbor.h" + +enum { MAX_KEY_LEN = 64 }; + +static const CborTag MULTI_DIMENSIONAL_ARRAY_ROW_MAJOR = 40; +static const CborTag DECTRIS_COMPRESSION = 56500; + +static uint64_t read_u64_be(const uint8_t* buf) { + return ((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) | + ((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) | + ((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) | + ((uint64_t)buf[6] << 8) | (uint64_t)buf[7]; +} + +static enum stream2_result CBOR_RESULT(CborError e) { + switch (e) { + case CborNoError: + return STREAM2_OK; + case CborErrorUnknownLength: + case CborErrorDataTooLarge: + return STREAM2_ERROR_PARSE; + case CborErrorOutOfMemory: + return STREAM2_ERROR_OUT_OF_MEMORY; + default: + return STREAM2_ERROR_DECODE; + } +} + +static enum stream2_result consume_byte_string_nocopy(const CborValue* it, + const uint8_t** bstr, + size_t* bstr_len, + CborValue* next) { + enum stream2_result r; + + assert(cbor_value_is_byte_string(it)); + + if ((r = CBOR_RESULT(cbor_value_get_string_length(it, bstr_len)))) + return r; + + const uint8_t* ptr = cbor_value_get_next_byte(it); + assert(*ptr >= 0x40 && *ptr <= 0x5b); + switch (*ptr++) { + case 0x58: + ptr += 1; + break; + case 0x59: + ptr += 2; + break; + case 0x5a: + ptr += 4; + break; + case 0x5b: + ptr += 8; + break; + } + *bstr = ptr; + + if (next) { + *next = *it; + if ((r = CBOR_RESULT(cbor_value_advance(next)))) + return r; + } + return STREAM2_OK; +} + +static enum stream2_result parse_key(CborValue* it, char key[MAX_KEY_LEN]) { + if (!cbor_value_is_text_string(it)) + return STREAM2_ERROR_PARSE; + + size_t key_len = MAX_KEY_LEN; + CborError e = cbor_value_copy_text_string(it, key, &key_len, it); + if (e == CborErrorOutOfMemory || key_len == MAX_KEY_LEN) { + // The key is longer than any we suppport. Return the empty key which + // should be handled by the caller like other unknown keys. + key[0] = '\0'; + return STREAM2_OK; + } + return CBOR_RESULT(e); +} + +static enum stream2_result parse_tag(CborValue* it, CborTag* value) { + enum stream2_result r; + + if (!cbor_value_is_tag(it)) + return STREAM2_ERROR_PARSE; + + if ((r = CBOR_RESULT(cbor_value_get_tag(it, value)))) + return r; + + return CBOR_RESULT(cbor_value_advance_fixed(it)); +} + +static enum stream2_result parse_bool(CborValue* it, bool* value) { + enum stream2_result r; + + if (!cbor_value_is_boolean(it)) + return STREAM2_ERROR_PARSE; + + if ((r = CBOR_RESULT(cbor_value_get_boolean(it, value)))) + return r; + + return CBOR_RESULT(cbor_value_advance_fixed(it)); +} + +static float half_to_float(uint16_t x) { +#if FLT16_MANT_DIG > 0 || __FLT16_MANT_DIG__ > 0 + _Float16 f; + memcpy(&f, &x, 2); + return (float)f; +#else + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x))); +#endif +} + +static enum stream2_result parse_double(CborValue* it, double* value) { + enum stream2_result r; + + if (cbor_value_is_half_float(it)) { + uint16_t h; + if ((r = CBOR_RESULT(cbor_value_get_half_float(it, &h)))) + return r; + *value = (double)half_to_float(h); + } else if (cbor_value_is_float(it)) { + float f; + if ((r = CBOR_RESULT(cbor_value_get_float(it, &f)))) + return r; + *value = (double)f; + } else if (cbor_value_is_double(it)) { + if ((r = CBOR_RESULT(cbor_value_get_double(it, value)))) + return r; + } else { + return STREAM2_ERROR_PARSE; + } + + return CBOR_RESULT(cbor_value_advance_fixed(it)); +} + +static enum stream2_result parse_uint64(CborValue* it, uint64_t* value) { + enum stream2_result r; + + if (!cbor_value_is_unsigned_integer(it)) + return STREAM2_ERROR_PARSE; + + if ((r = CBOR_RESULT(cbor_value_get_uint64(it, value)))) + return r; + + return CBOR_RESULT(cbor_value_advance_fixed(it)); +} + +static enum stream2_result parse_text_string(CborValue* it, char** tstr) { + if (!cbor_value_is_text_string(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + return CBOR_RESULT(cbor_value_dup_text_string(it, tstr, &len, it)); +} + +static enum stream2_result parse_array_2_uint64(CborValue* it, + uint64_t array[2]) { + enum stream2_result r; + + if (!cbor_value_is_array(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len)))) + return r; + + if (len != 2) + return STREAM2_ERROR_PARSE; + + CborValue elt; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_uint64(&elt, &array[i]))) + return r; + } + + return CBOR_RESULT(cbor_value_leave_container(it, &elt)); +} + +static enum stream2_result parse_dectris_compression( + CborValue* it, + struct stream2_compression* compression, + const uint8_t** bstr, + size_t* bstr_len) { + enum stream2_result r; + + CborTag tag; + if ((r = parse_tag(it, &tag))) + return r; + + if (tag != DECTRIS_COMPRESSION) + return STREAM2_ERROR_PARSE; + + if (!cbor_value_is_array(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len)))) + return r; + + if (len != 3) + return STREAM2_ERROR_PARSE; + + CborValue elt; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt)))) + return r; + + if ((r = parse_text_string(&elt, &compression->algorithm))) + return r; + + if ((r = parse_uint64(&elt, &compression->elem_size))) + return r; + + if (!cbor_value_is_byte_string(&elt)) + return STREAM2_ERROR_PARSE; + + if ((r = consume_byte_string_nocopy(&elt, bstr, bstr_len, &elt))) + return r; + + // https://github.com/dectris/compression/blob/v0.2.3/src/compression.c#L42 + if (strcmp(compression->algorithm, "bslz4") == 0 || + strcmp(compression->algorithm, "lz4") == 0) + { + if (*bstr_len < 12) + return STREAM2_ERROR_DECODE; + compression->orig_size = read_u64_be(*bstr); + } else { + return STREAM2_ERROR_NOT_IMPLEMENTED; + } + + return CBOR_RESULT(cbor_value_leave_container(it, &elt)); +} + +static enum stream2_result parse_bytes(CborValue* it, + struct stream2_bytes* bytes) { + enum stream2_result r; + + if (cbor_value_is_tag(it)) { + CborTag tag; + if ((r = CBOR_RESULT(cbor_value_get_tag(it, &tag)))) + return r; + + if (tag == DECTRIS_COMPRESSION) { + return parse_dectris_compression(it, &bytes->compression, + &bytes->ptr, &bytes->len); + } else { + return STREAM2_ERROR_PARSE; + } + } + + if (!cbor_value_is_byte_string(it)) + return STREAM2_ERROR_PARSE; + + bytes->compression.algorithm = NULL; + bytes->compression.elem_size = 0; + bytes->compression.orig_size = 0; + + return consume_byte_string_nocopy(it, &bytes->ptr, &bytes->len, it); +} + +// Parses a typed array from [RFC 8746 section 2]. +// +// [RFC 8746 section 2]: +// https://www.rfc-editor.org/rfc/rfc8746.html#name-typed-arrays +static enum stream2_result parse_typed_array(CborValue* it, + struct stream2_typed_array* array, + uint64_t* len) { + enum stream2_result r; + + if ((r = parse_tag(it, &array->tag))) + return r; + + if ((r = parse_bytes(it, &array->data))) + return r; + + uint64_t elem_size; + if ((r = stream2_typed_array_elem_size(array, &elem_size))) + return r; + + uint64_t size; + if (array->data.compression.algorithm == NULL) + size = array->data.len; + else + size = array->data.compression.orig_size; + + if (size % elem_size != 0) + return STREAM2_ERROR_PARSE; + + *len = size / elem_size; + return STREAM2_OK; +} + +// Parses a multi-dimensional array from [RFC 8746 section 3.1.1]. +// +// [RFC 8746 section 3.1.1]: +// https://www.rfc-editor.org/rfc/rfc8746.html#name-row-major-order +static enum stream2_result parse_multidim_array( + CborValue* it, + struct stream2_multidim_array* multidim) { + enum stream2_result r; + + CborTag tag; + if ((r = parse_tag(it, &tag))) + return r; + + if (tag != MULTI_DIMENSIONAL_ARRAY_ROW_MAJOR) + return STREAM2_ERROR_PARSE; + + if (!cbor_value_is_array(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len)))) + return r; + + if (len != 2) + return STREAM2_ERROR_PARSE; + + CborValue elt; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt)))) + return r; + + if ((r = parse_array_2_uint64(&elt, multidim->dim))) + return r; + + uint64_t array_len; + if ((r = parse_typed_array(&elt, &multidim->array, &array_len))) + return r; + + if (multidim->dim[0] * multidim->dim[1] != array_len) + return STREAM2_ERROR_PARSE; + + return CBOR_RESULT(cbor_value_leave_container(it, &elt)); +} + +static enum stream2_result parse_goniometer_axis( + CborValue* it, + struct stream2_goniometer_axis* axis) { + enum stream2_result r; + + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + while (cbor_value_is_valid(&field)) { + char key[MAX_KEY_LEN]; + if ((r = parse_key(&field, key))) + return r; + + if ((r = CBOR_RESULT(cbor_value_skip_tag(&field)))) + return r; + + if (strcmp(key, "increment") == 0) { + if ((r = parse_double(&field, &axis->increment))) + return r; + } else if (strcmp(key, "start") == 0) { + if ((r = parse_double(&field, &axis->start))) + return r; + } else { + if ((r = CBOR_RESULT(cbor_value_advance(&field)))) + return r; + } + } + + return CBOR_RESULT(cbor_value_leave_container(it, &field)); +} + +static enum stream2_result parse_goniometer( + CborValue* it, + struct stream2_goniometer* goniometer) { + enum stream2_result r; + + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + while (cbor_value_is_valid(&field)) { + char key[MAX_KEY_LEN]; + if ((r = parse_key(&field, key))) + return r; + + if ((r = CBOR_RESULT(cbor_value_skip_tag(&field)))) + return r; + + if (strcmp(key, "chi") == 0) { + if ((r = parse_goniometer_axis(&field, &goniometer->chi))) + return r; + } else if (strcmp(key, "kappa") == 0) { + if ((r = parse_goniometer_axis(&field, &goniometer->kappa))) + return r; + } else if (strcmp(key, "omega") == 0) { + if ((r = parse_goniometer_axis(&field, &goniometer->omega))) + return r; + } else if (strcmp(key, "phi") == 0) { + if ((r = parse_goniometer_axis(&field, &goniometer->phi))) + return r; + } else if (strcmp(key, "two_theta") == 0) { + if ((r = parse_goniometer_axis(&field, &goniometer->two_theta))) + return r; + } else { + if ((r = CBOR_RESULT(cbor_value_advance(&field)))) + return r; + } + } + + return CBOR_RESULT(cbor_value_leave_container(it, &field)); +} + +static enum stream2_result parse_user_data( + CborValue* it, + struct stream2_user_data* user_data) { + enum stream2_result r; + + const uint8_t* ptr = cbor_value_get_next_byte(it); + + if ((r = CBOR_RESULT(cbor_value_advance(it)))) + return r; + + user_data->ptr = ptr; + user_data->len = cbor_value_get_next_byte(it) - ptr; + + return STREAM2_OK; +} + +static enum stream2_result parse_start_msg(CborValue* it, + struct stream2_msg** msg_out) { + enum stream2_result r; + + struct stream2_start_msg* msg = calloc(1, sizeof(struct stream2_start_msg)); + *msg_out = (struct stream2_msg*)msg; + if (msg == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->type = STREAM2_MSG_START; + msg->countrate_correction_lookup_table.tag = UINT64_MAX; + + while (cbor_value_is_valid(it)) { + char key[MAX_KEY_LEN]; + if ((r = parse_key(it, key))) + return r; + + // skip any tag for a value, except where verified + if (strcmp(key, "countrate_correction_lookup_table") != 0) { + if ((r = CBOR_RESULT(cbor_value_skip_tag(it)))) + return r; + } + + if (strcmp(key, "series_id") == 0) { + if ((r = parse_uint64(it, &msg->series_id))) + return r; + } else if (strcmp(key, "series_unique_id") == 0) { + if ((r = parse_text_string(it, &msg->series_unique_id))) + return r; + } else if (strcmp(key, "arm_date") == 0) { + if ((r = parse_text_string(it, &msg->arm_date))) + return r; + } else if (strcmp(key, "beam_center_x") == 0) { + if ((r = parse_double(it, &msg->beam_center_x))) + return r; + } else if (strcmp(key, "beam_center_y") == 0) { + if ((r = parse_double(it, &msg->beam_center_y))) + return r; + } else if (strcmp(key, "channels") == 0) { + if (!cbor_value_is_array(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len)))) + return r; + + msg->channels.ptr = calloc(len, sizeof(char*)); + if (msg->channels.ptr == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->channels.len = len; + + CborValue elt; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_text_string(&elt, &msg->channels.ptr[i]))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &elt)))) + return r; + } else if (strcmp(key, "count_time") == 0) { + if ((r = parse_double(it, &msg->count_time))) + return r; + } else if (strcmp(key, "countrate_correction_enabled") == 0) { + if ((r = parse_bool(it, &msg->countrate_correction_enabled))) + return r; + } else if (strcmp(key, "countrate_correction_lookup_table") == 0) { + uint64_t len; + if ((r = parse_typed_array( + it, &msg->countrate_correction_lookup_table, &len))) + return r; + } else if (strcmp(key, "detector_description") == 0) { + if ((r = parse_text_string(it, &msg->detector_description))) + return r; + } else if (strcmp(key, "detector_serial_number") == 0) { + if ((r = parse_text_string(it, &msg->detector_serial_number))) + return r; + } else if (strcmp(key, "detector_translation") == 0) { + if (!cbor_value_is_array(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len)))) + return r; + + if (len != 3) + return STREAM2_ERROR_PARSE; + + CborValue elt; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_double(&elt, &msg->detector_translation[i]))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &elt)))) + return r; + } else if (strcmp(key, "flatfield") == 0) { + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len)))) + return r; + + msg->flatfield.ptr = calloc(len, sizeof(struct stream2_flatfield)); + if (msg->flatfield.ptr == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->flatfield.len = len; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_text_string(&field, + &msg->flatfield.ptr[i].channel))) + return r; + + if ((r = parse_multidim_array( + &field, &msg->flatfield.ptr[i].flatfield))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field)))) + return r; + } else if (strcmp(key, "flatfield_enabled") == 0) { + if ((r = parse_bool(it, &msg->flatfield_enabled))) + return r; + } else if (strcmp(key, "frame_time") == 0) { + if ((r = parse_double(it, &msg->frame_time))) + return r; + } else if (strcmp(key, "goniometer") == 0) { + if ((r = parse_goniometer(it, &msg->goniometer))) + return r; + } else if (strcmp(key, "image_size_x") == 0) { + if ((r = parse_uint64(it, &msg->image_size_x))) + return r; + } else if (strcmp(key, "image_size_y") == 0) { + if ((r = parse_uint64(it, &msg->image_size_y))) + return r; + } else if (strcmp(key, "incident_energy") == 0) { + if ((r = parse_double(it, &msg->incident_energy))) + return r; + } else if (strcmp(key, "incident_wavelength") == 0) { + if ((r = parse_double(it, &msg->incident_wavelength))) + return r; + } else if (strcmp(key, "number_of_images") == 0) { + if ((r = parse_uint64(it, &msg->number_of_images))) + return r; + } else if (strcmp(key, "pixel_mask") == 0) { + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len)))) + return r; + + msg->pixel_mask.ptr = + calloc(len, sizeof(struct stream2_pixel_mask)); + if (msg->pixel_mask.ptr == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->pixel_mask.len = len; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_text_string(&field, + &msg->pixel_mask.ptr[i].channel))) + return r; + + if ((r = parse_multidim_array( + &field, &msg->pixel_mask.ptr[i].pixel_mask))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field)))) + return r; + } else if (strcmp(key, "pixel_mask_enabled") == 0) { + if ((r = parse_bool(it, &msg->pixel_mask_enabled))) + return r; + } else if (strcmp(key, "pixel_size_x") == 0) { + if ((r = parse_double(it, &msg->pixel_size_x))) + return r; + } else if (strcmp(key, "pixel_size_y") == 0) { + if ((r = parse_double(it, &msg->pixel_size_y))) + return r; + } else if (strcmp(key, "saturation_value") == 0) { + if ((r = parse_uint64(it, &msg->saturation_value))) + return r; + } else if (strcmp(key, "sensor_material") == 0) { + if ((r = parse_text_string(it, &msg->sensor_material))) + return r; + } else if (strcmp(key, "sensor_thickness") == 0) { + if ((r = parse_double(it, &msg->sensor_thickness))) + return r; + } else if (strcmp(key, "threshold_energy") == 0) { + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len)))) + return r; + + msg->threshold_energy.ptr = + calloc(len, sizeof(struct stream2_threshold_energy)); + if (msg->threshold_energy.ptr == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->threshold_energy.len = len; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_text_string( + &field, &msg->threshold_energy.ptr[i].channel))) + return r; + + if ((r = parse_double(&field, + &msg->threshold_energy.ptr[i].energy))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field)))) + return r; + } else if (strcmp(key, "user_data") == 0) { + if ((r = parse_user_data(it, &msg->user_data))) + return r; + } else if (strcmp(key, "virtual_pixel_interpolation_enabled") == 0) { + if ((r = parse_bool(it, &msg->virtual_pixel_interpolation_enabled))) + return r; + } else { + if ((r = CBOR_RESULT(cbor_value_advance(it)))) + return r; + } + } + return STREAM2_OK; +} + +static enum stream2_result parse_image_msg(CborValue* it, + struct stream2_msg** msg_out) { + enum stream2_result r; + + struct stream2_image_msg* msg = calloc(1, sizeof(struct stream2_image_msg)); + *msg_out = (struct stream2_msg*)msg; + if (msg == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->type = STREAM2_MSG_IMAGE; + + while (cbor_value_is_valid(it)) { + char key[MAX_KEY_LEN]; + if ((r = parse_key(it, key))) + return r; + + if ((r = CBOR_RESULT(cbor_value_skip_tag(it)))) + return r; + + if (strcmp(key, "series_id") == 0) { + if ((r = parse_uint64(it, &msg->series_id))) + return r; + } else if (strcmp(key, "series_unique_id") == 0) { + if ((r = parse_text_string(it, &msg->series_unique_id))) + return r; + } else if (strcmp(key, "image_id") == 0) { + if ((r = parse_uint64(it, &msg->image_id))) + return r; + } else if (strcmp(key, "real_time") == 0) { + if ((r = parse_array_2_uint64(it, msg->real_time))) + return r; + } else if (strcmp(key, "series_date") == 0) { + if ((r = parse_text_string(it, &msg->series_date))) + return r; + } else if (strcmp(key, "start_time") == 0) { + if ((r = parse_array_2_uint64(it, msg->start_time))) + return r; + } else if (strcmp(key, "stop_time") == 0) { + if ((r = parse_array_2_uint64(it, msg->stop_time))) + return r; + } else if (strcmp(key, "user_data") == 0) { + if ((r = parse_user_data(it, &msg->user_data))) + return r; + } else if (strcmp(key, "data") == 0) { + if (!cbor_value_is_map(it)) + return STREAM2_ERROR_PARSE; + + size_t len; + if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len)))) + return r; + + msg->data.ptr = calloc(len, sizeof(struct stream2_image_data)); + if (msg->data.ptr == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->data.len = len; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field)))) + return r; + + for (size_t i = 0; i < len; i++) { + if ((r = parse_text_string(&field, &msg->data.ptr[i].channel))) + return r; + + if ((r = parse_multidim_array(&field, &msg->data.ptr[i].data))) + return r; + } + + if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field)))) + return r; + } else { + if ((r = CBOR_RESULT(cbor_value_advance(it)))) + return r; + } + } + return STREAM2_OK; +} + +static enum stream2_result parse_end_msg(CborValue* it, + struct stream2_msg** msg_out) { + enum stream2_result r; + + struct stream2_end_msg* msg = calloc(1, sizeof(struct stream2_end_msg)); + *msg_out = (struct stream2_msg*)msg; + if (msg == NULL) + return STREAM2_ERROR_OUT_OF_MEMORY; + + msg->type = STREAM2_MSG_END; + + while (cbor_value_is_valid(it)) { + char key[MAX_KEY_LEN]; + if ((r = parse_key(it, key))) + return r; + + if ((r = CBOR_RESULT(cbor_value_skip_tag(it)))) + return r; + + if (strcmp(key, "series_id") == 0) { + if ((r = parse_uint64(it, &msg->series_id))) + return r; + } else if (strcmp(key, "series_unique_id") == 0) { + if ((r = parse_text_string(it, &msg->series_unique_id))) + return r; + } else { + if ((r = CBOR_RESULT(cbor_value_advance(it)))) + return r; + } + } + return STREAM2_OK; +} + +static enum stream2_result parse_msg_type(CborValue* it, + char type[MAX_KEY_LEN]) { + enum stream2_result r; + + char key[MAX_KEY_LEN]; + if ((r = parse_key(it, key))) + return r; + + if (strcmp(key, "type") != 0) + return STREAM2_ERROR_PARSE; + + if ((r = CBOR_RESULT(cbor_value_skip_tag(it)))) + return r; + + return parse_key(it, type); +} + +static enum stream2_result parse_msg(const uint8_t* buffer, + size_t size, + struct stream2_msg** msg_out) { + enum stream2_result r; + + // https://www.rfc-editor.org/rfc/rfc8949.html#name-self-described-cbor + const uint8_t MAGIC[3] = {0xd9, 0xd9, 0xf7}; + if (size < sizeof(MAGIC) || memcmp(buffer, MAGIC, sizeof(MAGIC)) != 0) + return STREAM2_ERROR_SIGNATURE; + + buffer += sizeof(MAGIC); + size -= sizeof(MAGIC); + + CborParser parser; + CborValue it; + if ((r = CBOR_RESULT(cbor_parser_init(buffer, size, 0, &parser, &it)))) + return r; + + if (!cbor_value_is_map(&it)) + return STREAM2_ERROR_PARSE; + + CborValue field; + if ((r = CBOR_RESULT(cbor_value_enter_container(&it, &field)))) + return r; + + char type[MAX_KEY_LEN]; + if ((r = parse_msg_type(&field, type))) + return r; + + if (strcmp(type, "start") == 0) { + if ((r = parse_start_msg(&field, msg_out))) + return r; + } else if (strcmp(type, "image") == 0) { + if ((r = parse_image_msg(&field, msg_out))) + return r; + } else if (strcmp(type, "end") == 0) { + if ((r = parse_end_msg(&field, msg_out))) + return r; + } else { + return STREAM2_ERROR_PARSE; + } + + return CBOR_RESULT(cbor_value_leave_container(&it, &field)); +} + +enum stream2_result stream2_parse_msg(const uint8_t* buffer, + size_t size, + struct stream2_msg** msg_out) { + enum stream2_result r; + + *msg_out = NULL; + if ((r = parse_msg(buffer, size, msg_out))) { + if (*msg_out) { + stream2_free_msg(*msg_out); + *msg_out = NULL; + } + return r; + } + return STREAM2_OK; +} + +static void free_start_msg(struct stream2_start_msg* msg) { + free(msg->arm_date); + for (size_t i = 0; i < msg->channels.len; i++) + free(msg->channels.ptr[i]); + free(msg->channels.ptr); + free(msg->countrate_correction_lookup_table.data.compression.algorithm); + free(msg->detector_description); + free(msg->detector_serial_number); + for (size_t i = 0; i < msg->flatfield.len; i++) { + free(msg->flatfield.ptr[i].channel); + free(msg->flatfield.ptr[i].flatfield.array.data.compression.algorithm); + } + free(msg->flatfield.ptr); + for (size_t i = 0; i < msg->pixel_mask.len; i++) { + free(msg->pixel_mask.ptr[i].channel); + free(msg->pixel_mask.ptr[i] + .pixel_mask.array.data.compression.algorithm); + } + free(msg->pixel_mask.ptr); + free(msg->sensor_material); + for (size_t i = 0; i < msg->threshold_energy.len; i++) + free(msg->threshold_energy.ptr[i].channel); + free(msg->threshold_energy.ptr); +} + +static void free_image_msg(struct stream2_image_msg* msg) { + free(msg->series_date); + for (size_t i = 0; i < msg->data.len; i++) { + free(msg->data.ptr[i].channel); + free(msg->data.ptr[i].data.array.data.compression.algorithm); + } + free(msg->data.ptr); +} + +void stream2_free_msg(struct stream2_msg* msg) { + switch (msg->type) { + case STREAM2_MSG_START: + free_start_msg((struct stream2_start_msg*)msg); + break; + case STREAM2_MSG_IMAGE: + free_image_msg((struct stream2_image_msg*)msg); + break; + case STREAM2_MSG_END: + break; + } + free(msg->series_unique_id); + free(msg); +} + +enum stream2_result stream2_typed_array_elem_size( + const struct stream2_typed_array* array, + uint64_t* elem_size) { + // https://www.rfc-editor.org/rfc/rfc8746.html#name-types-of-numbers + if (array->tag >= 64 && array->tag <= 87) { + const uint64_t f = (array->tag >> 4) & 1; + const uint64_t ll = array->tag & 3; + *elem_size = 1 << (f + ll); + return STREAM2_OK; + } + return STREAM2_ERROR_NOT_IMPLEMENTED; +} diff --git a/frame_serialize/stream2.h b/frame_serialize/stream2.h new file mode 100644 index 00000000..03399a87 --- /dev/null +++ b/frame_serialize/stream2.h @@ -0,0 +1,213 @@ +// DECTRIS proprietary license +#pragma once + +#include +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +enum stream2_result { + STREAM2_OK = 0, + STREAM2_ERROR_OUT_OF_MEMORY, + STREAM2_ERROR_SIGNATURE, + STREAM2_ERROR_DECODE, + STREAM2_ERROR_PARSE, + STREAM2_ERROR_NOT_IMPLEMENTED, +}; + +// https://github.com/dectris/documentation/blob/main/cbor/dectris-compression-tag.md +struct stream2_compression { + // Name of compression algorithm used. + char* algorithm; + // Element size if required for decompression, reserved otherwise. + // Required by algorithm "bslz4". + uint64_t elem_size; + // Uncompressed size of the data. + uint64_t orig_size; +}; + +// Represents a byte string, possibly compressed. +struct stream2_bytes { + const uint8_t* ptr; + size_t len; + struct stream2_compression compression; +}; + +// CBOR tags used with typed arrays. +// +// https://www.rfc-editor.org/rfc/rfc8746.html#tab-tag-values +enum stream2_typed_array_tag { + STREAM2_TYPED_ARRAY_UINT8 = 64, + STREAM2_TYPED_ARRAY_UINT16_LITTLE_ENDIAN = 69, + STREAM2_TYPED_ARRAY_UINT32_LITTLE_ENDIAN = 70, + STREAM2_TYPED_ARRAY_FLOAT32_LITTLE_ENDIAN = 85, +}; + +// A typed array defined in RFC 8746 section 2. +// +// https://www.rfc-editor.org/rfc/rfc8746.html#name-typed-arrays +struct stream2_typed_array { + // CBOR tag of the typed array. + uint64_t tag; + // Byte representation of the array elements. + struct stream2_bytes data; +}; + +// A multi-dimensional array defined in RFC 8746 section 3.1. +// +// The array is always a typed array of row-major order. +// +// https://www.rfc-editor.org/rfc/rfc8746.html#name-multi-dimensional-array +// https://www.rfc-editor.org/rfc/rfc8746.html#name-row-major-order +struct stream2_multidim_array { + uint64_t dim[2]; + struct stream2_typed_array array; +}; + +struct stream2_array_text_string { + char** ptr; + size_t len; +}; + +struct stream2_flatfield { + char* channel; + struct stream2_multidim_array flatfield; +}; + +struct stream2_flatfield_map { + struct stream2_flatfield* ptr; + size_t len; +}; + +struct stream2_goniometer_axis { + double increment; + double start; +}; + +struct stream2_goniometer { + struct stream2_goniometer_axis chi; + struct stream2_goniometer_axis kappa; + struct stream2_goniometer_axis omega; + struct stream2_goniometer_axis phi; + struct stream2_goniometer_axis two_theta; +}; + +struct stream2_image_data { + char* channel; + struct stream2_multidim_array data; +}; + +struct stream2_image_data_map { + struct stream2_image_data* ptr; + size_t len; +}; + +struct stream2_pixel_mask { + char* channel; + struct stream2_multidim_array pixel_mask; +}; + +struct stream2_pixel_mask_map { + struct stream2_pixel_mask* ptr; + size_t len; +}; + +struct stream2_threshold_energy { + char* channel; + double energy; +}; + +struct stream2_threshold_energy_map { + struct stream2_threshold_energy* ptr; + size_t len; +}; + +struct stream2_user_data { + const uint8_t* ptr; + size_t len; +}; + +enum stream2_msg_type { + STREAM2_MSG_START, + STREAM2_MSG_IMAGE, + STREAM2_MSG_END, +}; + +struct stream2_msg { + enum stream2_msg_type type; + uint64_t series_id; + char* series_unique_id; +}; + +struct stream2_start_msg { + enum stream2_msg_type type; + uint64_t series_id; + char* series_unique_id; + + char* arm_date; + double beam_center_x; + double beam_center_y; + struct stream2_array_text_string channels; + double count_time; + bool countrate_correction_enabled; + struct stream2_typed_array countrate_correction_lookup_table; + char* detector_description; + char* detector_serial_number; + double detector_translation[3]; + struct stream2_flatfield_map flatfield; + bool flatfield_enabled; + double frame_time; + struct stream2_goniometer goniometer; + uint64_t image_size_x; + uint64_t image_size_y; + double incident_energy; + double incident_wavelength; + uint64_t number_of_images; + struct stream2_pixel_mask_map pixel_mask; + bool pixel_mask_enabled; + double pixel_size_x; + double pixel_size_y; + uint64_t saturation_value; + char* sensor_material; + double sensor_thickness; + struct stream2_threshold_energy_map threshold_energy; + struct stream2_user_data user_data; + bool virtual_pixel_interpolation_enabled; +}; + +struct stream2_image_msg { + enum stream2_msg_type type; + uint64_t series_id; + char* series_unique_id; + + uint64_t image_id; + uint64_t real_time[2]; + char* series_date; + uint64_t start_time[2]; + uint64_t stop_time[2]; + struct stream2_user_data user_data; + struct stream2_image_data_map data; +}; + +struct stream2_end_msg { + enum stream2_msg_type type; + uint64_t series_id; + char* series_unique_id; +}; + +enum stream2_result stream2_parse_msg(const uint8_t* buffer, + const size_t size, + struct stream2_msg** msg_out); +void stream2_free_msg(struct stream2_msg* msg); + +// Gets the element size of a typed array. +enum stream2_result stream2_typed_array_elem_size( + const struct stream2_typed_array* array, + uint64_t* elem_size); + +#if defined(__cplusplus) +} +#endif diff --git a/tests/CBORTest.cpp b/tests/CBORTest.cpp index c38b2def..5126d0fd 100644 --- a/tests/CBORTest.cpp +++ b/tests/CBORTest.cpp @@ -5,6 +5,8 @@ #include "../frame_serialize/JFJochFrameSerializer.h" #include "../frame_serialize/JFJochFrameDeserializer.h" +#include "../frame_serialize/stream2.h" +#include "../compression/JFJochCompressor.h" TEST_CASE("CBORSerialize_Start", "[CBOR]") { JFJochFrameSerializer serializer(8*1024*1024); @@ -128,7 +130,9 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") { .max_receiver_delay = 3456, .efficiency = 0.99, .write_master_file = true, - .end_date = "ccc" + .end_date = "ccc", + .series_unique_id = "bla5", + .series_id = 45676782 }; REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); @@ -146,6 +150,8 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") { REQUIRE(output_message.efficiency == Approx(message.efficiency)); REQUIRE(output_message.end_date == message.end_date); REQUIRE(output_message.write_master_file == message.write_master_file); + REQUIRE(output_message.series_id == message.series_id); + REQUIRE(output_message.series_unique_id == message.series_unique_id); } TEST_CASE("CBORSerialize_Image", "[CBOR]") { @@ -175,7 +181,9 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") { .indexing_result = 1, .bunch_id = UINT64_MAX, .jf_info = UINT32_MAX, - .timestamp = 1ul<<27 | 1ul <<35 + .timestamp = 1ul<<27 | 1ul <<35, + .series_unique_id = "bla2", + .series_id = 4567678 }; REQUIRE_NOTHROW(serializer.SerializeImage(message)); @@ -195,6 +203,8 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") { REQUIRE(image_array.image.size == test.size()); REQUIRE(image_array.indexing_result == message.indexing_result); REQUIRE(image_array.number == 456); + REQUIRE(image_array.series_id == message.series_id); + REQUIRE(image_array.series_unique_id == message.series_unique_id); REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0); REQUIRE(image_array.bunch_id == message.bunch_id); @@ -290,7 +300,7 @@ TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") { REQUIRE(image_array.image.pixel_depth_bytes == 4); REQUIRE(image_array.image.channel == "default"); REQUIRE(image_array.image.size == test.size()); - REQUIRE(image_array.image.pixel_is_signed == false); + REQUIRE(image_array.image.pixel_is_signed == true); REQUIRE(image_array.number == 456); REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0); } @@ -383,6 +393,10 @@ TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") { REQUIRE(image_array.spots[1].intensity == 123); } +inline bool CmpString(const char *str1, const std::string& str2) { + return (strncmp(str1, str2.c_str(), str2.size()) == 0); +} + TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") { JFJochFrameSerializer serializer(8*1024*1024); @@ -438,6 +452,145 @@ TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") { REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message)); auto image = serializer.GetBuffer(); + stream2_msg *msg; + + auto ret = stream2_parse_msg(image.data(), image.size(), &msg); + REQUIRE(ret == STREAM2_OK); + CHECK(msg->type == STREAM2_MSG_START); + auto msg2 = (stream2_start_msg *) msg; + + CHECK(msg2->beam_center_x == message.beam_center_x); + CHECK(msg2->beam_center_y == message.beam_center_y); + CHECK(msg2->image_size_x == message.image_size_x); + CHECK(msg2->image_size_y == message.image_size_y); + CHECK(msg2->series_id == message.series_id); + CHECK(CmpString(msg2->series_unique_id, message.series_unique_id)); + CHECK(CmpString(msg2->detector_description, message.detector_description)); + CHECK(CmpString(msg2->arm_date, message.arm_date)); + CHECK(msg2->number_of_images == message.number_of_images); + CHECK(msg2->frame_time == message.frame_time); + CHECK(msg2->count_time == message.count_time); + + stream2_free_msg(msg); +} + +TEST_CASE("CBORSerialize_End_stream2", "[CBOR]") { + JFJochFrameSerializer serializer(8*1024*1024); + + EndMessage message { + .number_of_images = 57789, + .max_receiver_delay = 3456, + .efficiency = 0.99, + .write_master_file = true, + .end_date = "ccc", + .series_unique_id = "bla5", + .series_id = 45676782 + }; + + REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message)); + + auto image = serializer.GetBuffer(); + stream2_msg *msg; + + auto ret = stream2_parse_msg(image.data(), image.size(), &msg); + REQUIRE(ret == STREAM2_OK); + CHECK(msg->type == STREAM2_MSG_END); + auto msg2 = (stream2_end_msg *) msg; + + CHECK(msg2->series_id == message.series_id); + CHECK(CmpString(msg2->series_unique_id, message.series_unique_id)); + + stream2_free_msg(msg); +} -} \ No newline at end of file +TEST_CASE("CBORSerialize_Image_compressed_stream2", "[CBOR]") { + JFJochFrameSerializer serializer(8*1024*1024); + + std::vector spots; + + std::vector test(512*1024); + for (int i = 0; i < test.size(); i++) + test[i] = (i * 253 + 56) % 256; + std::vector compressed_test(512*1024*4); + + JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4); + + compressor.Compress(compressed_test.data(), test); + CBORImage image { + .data = compressed_test.data(), + .size = 1024 * 512, + .xpixel = 1024, + .ypixel = 512, + .pixel_depth_bytes = 2, + .pixel_is_signed = false, + .algorithm = CompressionAlgorithm::BSHUF_LZ4, + .channel = "default" + }; + + DataMessage message { + .number = 456, + .image = image, + .spots = spots + }; + + REQUIRE_NOTHROW(serializer.SerializeImage(message)); + auto cbor_image = serializer.GetBuffer(); + + stream2_msg *msg; + + auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg); + REQUIRE(ret == STREAM2_OK); + CHECK(msg->type == STREAM2_MSG_IMAGE); + auto msg2 = (stream2_image_msg *) msg; + CHECK(msg2->image_id == message.number); + CHECK(msg2->data.len == 1); + + CHECK(msg2->series_id == message.series_id); + CHECK(CmpString(msg2->series_unique_id, message.series_unique_id)); + stream2_free_msg(msg); +} + +TEST_CASE("CBORSerialize_Image_uncompressed_stream2", "[CBOR]") { + JFJochFrameSerializer serializer(8*1024*1024); + + std::vector spots; + + std::vector test(512*1024); + for (int i = 0; i < test.size(); i++) + test[i] = (i * 253 + 56) % 256; + + CBORImage image { + .data = test.data(), + .size = 1024 * 512, + .xpixel = 1024, + .ypixel = 512, + .pixel_depth_bytes = 1, + .pixel_is_signed = false, + .algorithm = CompressionAlgorithm::NO_COMPRESSION, + .channel = "default" + }; + + DataMessage message { + .number = 456, + .image = image, + .spots = spots + }; + + REQUIRE_NOTHROW(serializer.SerializeImage(message)); + auto cbor_image = serializer.GetBuffer(); + + stream2_msg *msg; + + auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg); + REQUIRE(ret == STREAM2_OK); + CHECK(msg->type == STREAM2_MSG_IMAGE); + auto msg2 = (stream2_image_msg *) msg; + CHECK(msg2->image_id == message.number); + CHECK(msg2->data.len == 1); + + CHECK(msg2->series_id == message.series_id); + CHECK(CmpString(msg2->series_unique_id, message.series_unique_id)); + stream2_free_msg(msg); +} +