mx/Jungfraujoch
0
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases 37 Wiki Activity

CBOR: Adjust for DECTRIS stream2 compatibility:

Browse Source 
* date/time tag
* initial tag self-described CBOR
* series ID
* fixes to typed array header in case of compression
This commit is contained in:
leonarski_f
2023-04-19 11:09:58 +02:00
parent d2df1b3ce3
commit 25359f0d60
11 changed files with 1401 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
compression/JFJochCompressor.cpp
View File

@@ -72,7 +72,6 @@ size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size
if (tmp_space.size() < DefaultBlockSize * elem_size)
tmp_space.resize(DefaultBlockSize * elem_size);
size_t num_full_blocks = nelements / DefaultBlockSize;
size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize;
size_t compressed_size = 12;

4
frame_serialize/CMakeLists.txt
View File

@@ -2,6 +2,7 @@ ADD_LIBRARY(FrameSerialize STATIC
JFJochFrameSerializer.cpp JFJochFrameSerializer.h
JFJochFrameDeserializer.cpp JFJochFrameDeserializer.h
ImageMessage.h
tinycbor/src/cborparser_dup_string.c
tinycbor/src/cborencoder.c
tinycbor/src/cborencoder_close_container_checked.c
tinycbor/src/cborencoder_float.c
@@ -10,4 +11,5 @@ ADD_LIBRARY(FrameSerialize STATIC
tinycbor/src/cborpretty.c
tinycbor/src/cborerrorstrings.c
tinycbor/src/cbor.h
tinycbor/src/tinycbor-version.h CborErr.h StartMessage.h EndMessage.h CborUtil.h)
tinycbor/src/tinycbor-version.h CborErr.h StartMessage.h EndMessage.h CborUtil.h
stream2.h stream2.c)

3
frame_serialize/CborUtil.h
View File

@@ -6,8 +6,7 @@
#include "tinycbor/src/cbor.h"
constexpr const CborTag SelfDescribedCBOR = 55799;
constexpr const CborTag TagMultiDimArray = 40;
constexpr const CborTag TagMultiDimArray = 40;
constexpr const CborTag TagDECTRISCompression = 56500;
constexpr const CborTag TagFloatLE = 0b01010101;

3
frame_serialize/EndMessage.h
View File

@@ -12,6 +12,9 @@ struct EndMessage {
bool write_master_file;
std::string end_date;
std::string series_unique_id;
uint64_t series_id;
};
#endif //JUNGFRAUJOCH_ENDMESSAGE_H

3
frame_serialize/ImageMessage.h
View File

@@ -31,6 +31,9 @@ struct DataMessage {
uint64_t bunch_id;
uint32_t jf_info;
uint64_t timestamp;
std::string series_unique_id;
uint64_t series_id;
};
#endif //JUNGFRAUJOCH_IMAGEMESSAGE_H

32
frame_serialize/JFJochFrameDeserializer.cpp
View File

@@ -65,6 +65,19 @@ inline CborTag GetCBORTag(CborValue &value) {
return tmp;
}
inline std::string GetCBORDateTime(CborValue &value) {
auto tag = GetCBORTag(value);
if (tag == CborDateTimeStringTag)
return GetCBORString(value);
else if (tag == CborUnixTime_tTag) {
time_t t = GetCBORUInt(value);
char buf1[255];
strftime(buf1, sizeof(buf1), "%FT%T", gmtime(&t));
return std::string(buf1) + "Z";
} else
throw JFJochException(JFJochExceptionCategory::CBORError, "Time/date tag error");
}
inline uint64_t GetCBORArrayLen(CborValue &value) {
if (!cbor_value_is_array(&value) )
throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected");
@@ -355,6 +368,10 @@ bool JFJochFrameDeserializer::ProcessImageMessageElement(CborValue &value) {
data_message.number = GetCBORInt(value);
else if (key == "data")
ProcessImageData(value);
else if (key == "series_unique_id")
data_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
data_message.series_id = GetCBORUInt(value);
else if (key == "user_data")
ProcessImageMessageUserDataElement(value);
else
@@ -491,9 +508,9 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) {
start_message.detector_distance = GetCBORFloat(value);
else if (key == "number_of_images")
start_message.number_of_images = GetCBORUInt(value);
else if (key == "xpixel")
else if (key == "image_size_x")
start_message.image_size_x = GetCBORUInt(value);
else if (key == "ypixel")
else if (key == "image_size_y")
start_message.image_size_y = GetCBORUInt(value);
else if (key == "incident_energy")
start_message.incident_energy = GetCBORFloat(value);
@@ -520,7 +537,7 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) {
else if (key == "series_unique_id")
start_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
start_message.series_id = GetCBORInt(value);
start_message.series_id = GetCBORUInt(value);
else if (key == "pixel_mask")
ProcessPixelMaskElement(value);
else if (key == "channels")
@@ -532,7 +549,7 @@ bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) {
else if (key == "pixel_mask_enabled")
start_message.pixel_mask_enabled = GetCBORBool(value);
else if (key == "arm_date")
start_message.arm_date = GetCBORString(value);
start_message.arm_date = GetCBORDateTime(value);
else if (key == "user_data")
ProcessStartMessageUserDataElement(value);
else
@@ -556,6 +573,11 @@ bool JFJochFrameDeserializer::ProcessEndMessageElement(CborValue &value) {
end_message.write_master_file = GetCBORBool(value);
else if (key == "end_date")
end_message.end_date = GetCBORString(value);
else if (key == "series_unique_id")
end_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
end_message.series_id = GetCBORUInt(value);
else
cbor_value_advance(&value);
return true;
@@ -588,7 +610,7 @@ std::unique_lock<std::mutex> ul(m);
CborValue value;
cborErr(cbor_parser_init(buffer.data(), buffer.size(), 0, &parser, &value));
if (GetCBORTag(value) != SelfDescribedCBOR)
if (GetCBORTag(value) != CborSignatureTag)
throw JFJochException(JFJochExceptionCategory::CBORError, "CBOR must start with dedicated tag");

71
frame_serialize/JFJochFrameSerializer.cpp
View File

@@ -19,6 +19,15 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::string &v
cborErr(cbor_encode_text_stringz(&encoder, value.c_str()));
}
inline void CBOR_ENC_DATE(CborEncoder &encoder, const char* key, const std::string &value) {
cborErr(cbor_encode_text_stringz(&encoder, key));
cbor_encode_tag(&encoder, CborDateTimeStringTag);
if (value.empty())
cborErr(cbor_encode_text_stringz(&encoder, "."));
else
cborErr(cbor_encode_text_stringz(&encoder, value.c_str()));
}
inline void CBOR_ENC(CborEncoder &encoder, const char* key, float value) {
cborErr(cbor_encode_text_stringz(&encoder, key));
cborErr(cbor_encode_float(&encoder, value));
@@ -88,28 +97,25 @@ inline void CBOR_ENC_MULTIDIM_TYPED_ARRAY(CborEncoder &encoder, const char* key,
cborErr(cbor_encoder_close_container(&arrayEncoder, &arrayEncoder_2));
CborTag typed_array_tag;
if (algorithm != CompressionAlgorithm::NO_COMPRESSION)
typed_array_tag = TagUnsignedInt8Bit;
else {
if (elem_sign) {
if (elem_size == 4)
typed_array_tag = TagSignedInt32BitLE;
else if (elem_size == 2)
typed_array_tag = TagSignedInt16BitLE;
else if (elem_size == 1)
typed_array_tag = TagSignedInt8Bit;
else
throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported");
} else {
if (elem_size == 4)
typed_array_tag = TagUnsignedInt32BitLE;
else if (elem_size == 2)
typed_array_tag = TagUnsignedInt16BitLE;
else if (elem_size == 1)
typed_array_tag = TagUnsignedInt8Bit;
else
throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported");
}
if (elem_sign) {
if (elem_size == 4)
typed_array_tag = TagSignedInt32BitLE;
else if (elem_size == 2)
typed_array_tag = TagSignedInt16BitLE;
else if (elem_size == 1)
typed_array_tag = TagSignedInt8Bit;
else
throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported");
} else {
if (elem_size == 4)
typed_array_tag = TagUnsignedInt32BitLE;
else if (elem_size == 2)
typed_array_tag = TagUnsignedInt16BitLE;
else if (elem_size == 1)
typed_array_tag = TagUnsignedInt8Bit;
else
throw JFJochException(JFJochExceptionCategory::CBORError, "Array size not supported");
}
cbor_encode_tag(&arrayEncoder, typed_array_tag);
@@ -295,7 +301,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message)
CborEncoder encoder, mapEncoder;
cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0);
cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR ));
cborErr(cbor_encode_tag(&encoder, CborSignatureTag ));
size_t elements = 27;
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, elements));
@@ -306,8 +312,8 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message)
CBOR_ENC(mapEncoder, "beam_center_y", message.beam_center_y);
CBOR_ENC(mapEncoder, "number_of_images", message.number_of_images);
CBOR_ENC(mapEncoder, "xpixel", message.image_size_x);
CBOR_ENC(mapEncoder, "ypixel", message.image_size_y);
CBOR_ENC(mapEncoder, "image_size_x", message.image_size_x);
CBOR_ENC(mapEncoder, "image_size_y", message.image_size_y);
CBOR_ENC(mapEncoder, "incident_energy", message.incident_energy);
CBOR_ENC(mapEncoder, "incident_wavelength", message.incident_wavelength);
@@ -321,7 +327,7 @@ void JFJochFrameSerializer::SerializeSequenceStart(const StartMessage& message)
CBOR_ENC(mapEncoder, "pixel_size_y", message.pixel_size_y);
CBOR_ENC(mapEncoder, "sensor_thickness", message.sensor_thickness);
CBOR_ENC(mapEncoder, "sensor_material", message.sensor_material);
CBOR_ENC(mapEncoder, "arm_date", message.arm_date);
CBOR_ENC_DATE(mapEncoder, "arm_date", message.arm_date);
CBOR_ENC(mapEncoder, "pixel_mask_enabled", message.pixel_mask_enabled);
CBOR_ENC(mapEncoder, "detector_description", message.detector_description);
CBOR_ENC(mapEncoder, "detector_serial_number", message.detector_serial_number);
@@ -345,11 +351,12 @@ void JFJochFrameSerializer::SerializeSequenceEnd(const EndMessage& message) {
CborEncoder encoder, mapEncoder;
cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0);
cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR ));
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 6));
cborErr(cbor_encode_tag(&encoder,CborSignatureTag ));
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 8));
CBOR_ENC(mapEncoder, "type", "end");
CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id);
CBOR_ENC(mapEncoder, "series_id", message.series_id);
CBOR_ENC(mapEncoder, "number_of_images", message.number_of_images);
CBOR_ENC(mapEncoder, "max_receiver_delay", message.max_receiver_delay);
CBOR_ENC(mapEncoder, "receiver_efficiency", message.efficiency);
@@ -367,10 +374,12 @@ void JFJochFrameSerializer::SerializeImage(const DataMessage& message) {
CborEncoder encoder, mapEncoder, userDataMapEncoder;
cbor_encoder_init(&encoder, buffer.data(), buffer.size(), 0);
cborErr(cbor_encode_tag(&encoder, SelfDescribedCBOR ));
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 4));
cborErr(cbor_encode_tag(&encoder, CborSignatureTag ));
cborErr(cbor_encoder_create_map(&encoder, &mapEncoder, 6));
CBOR_ENC(mapEncoder, "type", "image");
CBOR_ENC(mapEncoder, "series_unique_id", message.series_unique_id);
CBOR_ENC(mapEncoder, "series_id", message.series_id);
CBOR_ENC(mapEncoder, "image_id", message.number);
CBOR_ENC(mapEncoder, "data", message.image);

2
frame_serialize/StartMessage.h
View File

@@ -65,7 +65,7 @@ struct StartMessage {
std::string detector_description;
std::string detector_serial_number;
std::string series_unique_id;
int64_t series_id;
uint64_t series_id;
std::map<std::string, GoniometerAxis> goniometer;
float detector_translation[3];

953
frame_serialize/stream2.c Normal file
View File

@@ -0,0 +1,953 @@
// DECTRIS proprietary license
#define __STDC_WANT_IEC_60559_TYPES_EXT__
#include "stream2.h"
#include <assert.h>
#include <float.h>
#if FLT16_MANT_DIG > 0 || __FLT16_MANT_DIG__ > 0
#else
#include <immintrin.h>
#if defined(_MSC_VER)
#include <intrin.h>
#endif
#endif
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include "tinycbor/src/cbor.h"
enum { MAX_KEY_LEN = 64 };
static const CborTag MULTI_DIMENSIONAL_ARRAY_ROW_MAJOR = 40;
static const CborTag DECTRIS_COMPRESSION = 56500;
static uint64_t read_u64_be(const uint8_t* buf) {
return ((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) |
((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) |
((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) |
((uint64_t)buf[6] << 8) | (uint64_t)buf[7];
}
static enum stream2_result CBOR_RESULT(CborError e) {
switch (e) {
case CborNoError:
return STREAM2_OK;
case CborErrorUnknownLength:
case CborErrorDataTooLarge:
return STREAM2_ERROR_PARSE;
case CborErrorOutOfMemory:
return STREAM2_ERROR_OUT_OF_MEMORY;
default:
return STREAM2_ERROR_DECODE;
}
}
static enum stream2_result consume_byte_string_nocopy(const CborValue* it,
const uint8_t** bstr,
size_t* bstr_len,
CborValue* next) {
enum stream2_result r;
assert(cbor_value_is_byte_string(it));
if ((r = CBOR_RESULT(cbor_value_get_string_length(it, bstr_len))))
return r;
const uint8_t* ptr = cbor_value_get_next_byte(it);
assert(*ptr >= 0x40 && *ptr <= 0x5b);
switch (*ptr++) {
case 0x58:
ptr += 1;
break;
case 0x59:
ptr += 2;
break;
case 0x5a:
ptr += 4;
break;
case 0x5b:
ptr += 8;
break;
}
*bstr = ptr;
if (next) {
*next = *it;
if ((r = CBOR_RESULT(cbor_value_advance(next))))
return r;
}
return STREAM2_OK;
}
static enum stream2_result parse_key(CborValue* it, char key[MAX_KEY_LEN]) {
if (!cbor_value_is_text_string(it))
return STREAM2_ERROR_PARSE;
size_t key_len = MAX_KEY_LEN;
CborError e = cbor_value_copy_text_string(it, key, &key_len, it);
if (e == CborErrorOutOfMemory || key_len == MAX_KEY_LEN) {
// The key is longer than any we suppport. Return the empty key which
// should be handled by the caller like other unknown keys.
key[0] = '\0';
return STREAM2_OK;
}
return CBOR_RESULT(e);
}
static enum stream2_result parse_tag(CborValue* it, CborTag* value) {
enum stream2_result r;
if (!cbor_value_is_tag(it))
return STREAM2_ERROR_PARSE;
if ((r = CBOR_RESULT(cbor_value_get_tag(it, value))))
return r;
return CBOR_RESULT(cbor_value_advance_fixed(it));
}
static enum stream2_result parse_bool(CborValue* it, bool* value) {
enum stream2_result r;
if (!cbor_value_is_boolean(it))
return STREAM2_ERROR_PARSE;
if ((r = CBOR_RESULT(cbor_value_get_boolean(it, value))))
return r;
return CBOR_RESULT(cbor_value_advance_fixed(it));
}
static float half_to_float(uint16_t x) {
#if FLT16_MANT_DIG > 0 || __FLT16_MANT_DIG__ > 0
_Float16 f;
memcpy(&f, &x, 2);
return (float)f;
#else
return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)));
#endif
}
static enum stream2_result parse_double(CborValue* it, double* value) {
enum stream2_result r;
if (cbor_value_is_half_float(it)) {
uint16_t h;
if ((r = CBOR_RESULT(cbor_value_get_half_float(it, &h))))
return r;
*value = (double)half_to_float(h);
} else if (cbor_value_is_float(it)) {
float f;
if ((r = CBOR_RESULT(cbor_value_get_float(it, &f))))
return r;
*value = (double)f;
} else if (cbor_value_is_double(it)) {
if ((r = CBOR_RESULT(cbor_value_get_double(it, value))))
return r;
} else {
return STREAM2_ERROR_PARSE;
}
return CBOR_RESULT(cbor_value_advance_fixed(it));
}
static enum stream2_result parse_uint64(CborValue* it, uint64_t* value) {
enum stream2_result r;
if (!cbor_value_is_unsigned_integer(it))
return STREAM2_ERROR_PARSE;
if ((r = CBOR_RESULT(cbor_value_get_uint64(it, value))))
return r;
return CBOR_RESULT(cbor_value_advance_fixed(it));
}
static enum stream2_result parse_text_string(CborValue* it, char** tstr) {
if (!cbor_value_is_text_string(it))
return STREAM2_ERROR_PARSE;
size_t len;
return CBOR_RESULT(cbor_value_dup_text_string(it, tstr, &len, it));
}
static enum stream2_result parse_array_2_uint64(CborValue* it,
uint64_t array[2]) {
enum stream2_result r;
if (!cbor_value_is_array(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len))))
return r;
if (len != 2)
return STREAM2_ERROR_PARSE;
CborValue elt;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_uint64(&elt, &array[i])))
return r;
}
return CBOR_RESULT(cbor_value_leave_container(it, &elt));
}
static enum stream2_result parse_dectris_compression(
CborValue* it,
struct stream2_compression* compression,
const uint8_t** bstr,
size_t* bstr_len) {
enum stream2_result r;
CborTag tag;
if ((r = parse_tag(it, &tag)))
return r;
if (tag != DECTRIS_COMPRESSION)
return STREAM2_ERROR_PARSE;
if (!cbor_value_is_array(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len))))
return r;
if (len != 3)
return STREAM2_ERROR_PARSE;
CborValue elt;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt))))
return r;
if ((r = parse_text_string(&elt, &compression->algorithm)))
return r;
if ((r = parse_uint64(&elt, &compression->elem_size)))
return r;
if (!cbor_value_is_byte_string(&elt))
return STREAM2_ERROR_PARSE;
if ((r = consume_byte_string_nocopy(&elt, bstr, bstr_len, &elt)))
return r;
// https://github.com/dectris/compression/blob/v0.2.3/src/compression.c#L42
if (strcmp(compression->algorithm, "bslz4") == 0 ||
strcmp(compression->algorithm, "lz4") == 0)
{
if (*bstr_len < 12)
return STREAM2_ERROR_DECODE;
compression->orig_size = read_u64_be(*bstr);
} else {
return STREAM2_ERROR_NOT_IMPLEMENTED;
}
return CBOR_RESULT(cbor_value_leave_container(it, &elt));
}
static enum stream2_result parse_bytes(CborValue* it,
struct stream2_bytes* bytes) {
enum stream2_result r;
if (cbor_value_is_tag(it)) {
CborTag tag;
if ((r = CBOR_RESULT(cbor_value_get_tag(it, &tag))))
return r;
if (tag == DECTRIS_COMPRESSION) {
return parse_dectris_compression(it, &bytes->compression,
&bytes->ptr, &bytes->len);
} else {
return STREAM2_ERROR_PARSE;
}
}
if (!cbor_value_is_byte_string(it))
return STREAM2_ERROR_PARSE;
bytes->compression.algorithm = NULL;
bytes->compression.elem_size = 0;
bytes->compression.orig_size = 0;
return consume_byte_string_nocopy(it, &bytes->ptr, &bytes->len, it);
}
// Parses a typed array from [RFC 8746 section 2].
//
// [RFC 8746 section 2]:
// https://www.rfc-editor.org/rfc/rfc8746.html#name-typed-arrays
static enum stream2_result parse_typed_array(CborValue* it,
struct stream2_typed_array* array,
uint64_t* len) {
enum stream2_result r;
if ((r = parse_tag(it, &array->tag)))
return r;
if ((r = parse_bytes(it, &array->data)))
return r;
uint64_t elem_size;
if ((r = stream2_typed_array_elem_size(array, &elem_size)))
return r;
uint64_t size;
if (array->data.compression.algorithm == NULL)
size = array->data.len;
else
size = array->data.compression.orig_size;
if (size % elem_size != 0)
return STREAM2_ERROR_PARSE;
*len = size / elem_size;
return STREAM2_OK;
}
// Parses a multi-dimensional array from [RFC 8746 section 3.1.1].
//
// [RFC 8746 section 3.1.1]:
// https://www.rfc-editor.org/rfc/rfc8746.html#name-row-major-order
static enum stream2_result parse_multidim_array(
CborValue* it,
struct stream2_multidim_array* multidim) {
enum stream2_result r;
CborTag tag;
if ((r = parse_tag(it, &tag)))
return r;
if (tag != MULTI_DIMENSIONAL_ARRAY_ROW_MAJOR)
return STREAM2_ERROR_PARSE;
if (!cbor_value_is_array(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len))))
return r;
if (len != 2)
return STREAM2_ERROR_PARSE;
CborValue elt;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt))))
return r;
if ((r = parse_array_2_uint64(&elt, multidim->dim)))
return r;
uint64_t array_len;
if ((r = parse_typed_array(&elt, &multidim->array, &array_len)))
return r;
if (multidim->dim[0] * multidim->dim[1] != array_len)
return STREAM2_ERROR_PARSE;
return CBOR_RESULT(cbor_value_leave_container(it, &elt));
}
static enum stream2_result parse_goniometer_axis(
CborValue* it,
struct stream2_goniometer_axis* axis) {
enum stream2_result r;
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
while (cbor_value_is_valid(&field)) {
char key[MAX_KEY_LEN];
if ((r = parse_key(&field, key)))
return r;
if ((r = CBOR_RESULT(cbor_value_skip_tag(&field))))
return r;
if (strcmp(key, "increment") == 0) {
if ((r = parse_double(&field, &axis->increment)))
return r;
} else if (strcmp(key, "start") == 0) {
if ((r = parse_double(&field, &axis->start)))
return r;
} else {
if ((r = CBOR_RESULT(cbor_value_advance(&field))))
return r;
}
}
return CBOR_RESULT(cbor_value_leave_container(it, &field));
}
static enum stream2_result parse_goniometer(
CborValue* it,
struct stream2_goniometer* goniometer) {
enum stream2_result r;
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
while (cbor_value_is_valid(&field)) {
char key[MAX_KEY_LEN];
if ((r = parse_key(&field, key)))
return r;
if ((r = CBOR_RESULT(cbor_value_skip_tag(&field))))
return r;
if (strcmp(key, "chi") == 0) {
if ((r = parse_goniometer_axis(&field, &goniometer->chi)))
return r;
} else if (strcmp(key, "kappa") == 0) {
if ((r = parse_goniometer_axis(&field, &goniometer->kappa)))
return r;
} else if (strcmp(key, "omega") == 0) {
if ((r = parse_goniometer_axis(&field, &goniometer->omega)))
return r;
} else if (strcmp(key, "phi") == 0) {
if ((r = parse_goniometer_axis(&field, &goniometer->phi)))
return r;
} else if (strcmp(key, "two_theta") == 0) {
if ((r = parse_goniometer_axis(&field, &goniometer->two_theta)))
return r;
} else {
if ((r = CBOR_RESULT(cbor_value_advance(&field))))
return r;
}
}
return CBOR_RESULT(cbor_value_leave_container(it, &field));
}
static enum stream2_result parse_user_data(
CborValue* it,
struct stream2_user_data* user_data) {
enum stream2_result r;
const uint8_t* ptr = cbor_value_get_next_byte(it);
if ((r = CBOR_RESULT(cbor_value_advance(it))))
return r;
user_data->ptr = ptr;
user_data->len = cbor_value_get_next_byte(it) - ptr;
return STREAM2_OK;
}
static enum stream2_result parse_start_msg(CborValue* it,
struct stream2_msg** msg_out) {
enum stream2_result r;
struct stream2_start_msg* msg = calloc(1, sizeof(struct stream2_start_msg));
*msg_out = (struct stream2_msg*)msg;
if (msg == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->type = STREAM2_MSG_START;
msg->countrate_correction_lookup_table.tag = UINT64_MAX;
while (cbor_value_is_valid(it)) {
char key[MAX_KEY_LEN];
if ((r = parse_key(it, key)))
return r;
// skip any tag for a value, except where verified
if (strcmp(key, "countrate_correction_lookup_table") != 0) {
if ((r = CBOR_RESULT(cbor_value_skip_tag(it))))
return r;
}
if (strcmp(key, "series_id") == 0) {
if ((r = parse_uint64(it, &msg->series_id)))
return r;
} else if (strcmp(key, "series_unique_id") == 0) {
if ((r = parse_text_string(it, &msg->series_unique_id)))
return r;
} else if (strcmp(key, "arm_date") == 0) {
if ((r = parse_text_string(it, &msg->arm_date)))
return r;
} else if (strcmp(key, "beam_center_x") == 0) {
if ((r = parse_double(it, &msg->beam_center_x)))
return r;
} else if (strcmp(key, "beam_center_y") == 0) {
if ((r = parse_double(it, &msg->beam_center_y)))
return r;
} else if (strcmp(key, "channels") == 0) {
if (!cbor_value_is_array(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len))))
return r;
msg->channels.ptr = calloc(len, sizeof(char*));
if (msg->channels.ptr == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->channels.len = len;
CborValue elt;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_text_string(&elt, &msg->channels.ptr[i])))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &elt))))
return r;
} else if (strcmp(key, "count_time") == 0) {
if ((r = parse_double(it, &msg->count_time)))
return r;
} else if (strcmp(key, "countrate_correction_enabled") == 0) {
if ((r = parse_bool(it, &msg->countrate_correction_enabled)))
return r;
} else if (strcmp(key, "countrate_correction_lookup_table") == 0) {
uint64_t len;
if ((r = parse_typed_array(
it, &msg->countrate_correction_lookup_table, &len)))
return r;
} else if (strcmp(key, "detector_description") == 0) {
if ((r = parse_text_string(it, &msg->detector_description)))
return r;
} else if (strcmp(key, "detector_serial_number") == 0) {
if ((r = parse_text_string(it, &msg->detector_serial_number)))
return r;
} else if (strcmp(key, "detector_translation") == 0) {
if (!cbor_value_is_array(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_array_length(it, &len))))
return r;
if (len != 3)
return STREAM2_ERROR_PARSE;
CborValue elt;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &elt))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_double(&elt, &msg->detector_translation[i])))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &elt))))
return r;
} else if (strcmp(key, "flatfield") == 0) {
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len))))
return r;
msg->flatfield.ptr = calloc(len, sizeof(struct stream2_flatfield));
if (msg->flatfield.ptr == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->flatfield.len = len;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_text_string(&field,
&msg->flatfield.ptr[i].channel)))
return r;
if ((r = parse_multidim_array(
&field, &msg->flatfield.ptr[i].flatfield)))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field))))
return r;
} else if (strcmp(key, "flatfield_enabled") == 0) {
if ((r = parse_bool(it, &msg->flatfield_enabled)))
return r;
} else if (strcmp(key, "frame_time") == 0) {
if ((r = parse_double(it, &msg->frame_time)))
return r;
} else if (strcmp(key, "goniometer") == 0) {
if ((r = parse_goniometer(it, &msg->goniometer)))
return r;
} else if (strcmp(key, "image_size_x") == 0) {
if ((r = parse_uint64(it, &msg->image_size_x)))
return r;
} else if (strcmp(key, "image_size_y") == 0) {
if ((r = parse_uint64(it, &msg->image_size_y)))
return r;
} else if (strcmp(key, "incident_energy") == 0) {
if ((r = parse_double(it, &msg->incident_energy)))
return r;
} else if (strcmp(key, "incident_wavelength") == 0) {
if ((r = parse_double(it, &msg->incident_wavelength)))
return r;
} else if (strcmp(key, "number_of_images") == 0) {
if ((r = parse_uint64(it, &msg->number_of_images)))
return r;
} else if (strcmp(key, "pixel_mask") == 0) {
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len))))
return r;
msg->pixel_mask.ptr =
calloc(len, sizeof(struct stream2_pixel_mask));
if (msg->pixel_mask.ptr == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->pixel_mask.len = len;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_text_string(&field,
&msg->pixel_mask.ptr[i].channel)))
return r;
if ((r = parse_multidim_array(
&field, &msg->pixel_mask.ptr[i].pixel_mask)))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field))))
return r;
} else if (strcmp(key, "pixel_mask_enabled") == 0) {
if ((r = parse_bool(it, &msg->pixel_mask_enabled)))
return r;
} else if (strcmp(key, "pixel_size_x") == 0) {
if ((r = parse_double(it, &msg->pixel_size_x)))
return r;
} else if (strcmp(key, "pixel_size_y") == 0) {
if ((r = parse_double(it, &msg->pixel_size_y)))
return r;
} else if (strcmp(key, "saturation_value") == 0) {
if ((r = parse_uint64(it, &msg->saturation_value)))
return r;
} else if (strcmp(key, "sensor_material") == 0) {
if ((r = parse_text_string(it, &msg->sensor_material)))
return r;
} else if (strcmp(key, "sensor_thickness") == 0) {
if ((r = parse_double(it, &msg->sensor_thickness)))
return r;
} else if (strcmp(key, "threshold_energy") == 0) {
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len))))
return r;
msg->threshold_energy.ptr =
calloc(len, sizeof(struct stream2_threshold_energy));
if (msg->threshold_energy.ptr == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->threshold_energy.len = len;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_text_string(
&field, &msg->threshold_energy.ptr[i].channel)))
return r;
if ((r = parse_double(&field,
&msg->threshold_energy.ptr[i].energy)))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field))))
return r;
} else if (strcmp(key, "user_data") == 0) {
if ((r = parse_user_data(it, &msg->user_data)))
return r;
} else if (strcmp(key, "virtual_pixel_interpolation_enabled") == 0) {
if ((r = parse_bool(it, &msg->virtual_pixel_interpolation_enabled)))
return r;
} else {
if ((r = CBOR_RESULT(cbor_value_advance(it))))
return r;
}
}
return STREAM2_OK;
}
static enum stream2_result parse_image_msg(CborValue* it,
struct stream2_msg** msg_out) {
enum stream2_result r;
struct stream2_image_msg* msg = calloc(1, sizeof(struct stream2_image_msg));
*msg_out = (struct stream2_msg*)msg;
if (msg == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->type = STREAM2_MSG_IMAGE;
while (cbor_value_is_valid(it)) {
char key[MAX_KEY_LEN];
if ((r = parse_key(it, key)))
return r;
if ((r = CBOR_RESULT(cbor_value_skip_tag(it))))
return r;
if (strcmp(key, "series_id") == 0) {
if ((r = parse_uint64(it, &msg->series_id)))
return r;
} else if (strcmp(key, "series_unique_id") == 0) {
if ((r = parse_text_string(it, &msg->series_unique_id)))
return r;
} else if (strcmp(key, "image_id") == 0) {
if ((r = parse_uint64(it, &msg->image_id)))
return r;
} else if (strcmp(key, "real_time") == 0) {
if ((r = parse_array_2_uint64(it, msg->real_time)))
return r;
} else if (strcmp(key, "series_date") == 0) {
if ((r = parse_text_string(it, &msg->series_date)))
return r;
} else if (strcmp(key, "start_time") == 0) {
if ((r = parse_array_2_uint64(it, msg->start_time)))
return r;
} else if (strcmp(key, "stop_time") == 0) {
if ((r = parse_array_2_uint64(it, msg->stop_time)))
return r;
} else if (strcmp(key, "user_data") == 0) {
if ((r = parse_user_data(it, &msg->user_data)))
return r;
} else if (strcmp(key, "data") == 0) {
if (!cbor_value_is_map(it))
return STREAM2_ERROR_PARSE;
size_t len;
if ((r = CBOR_RESULT(cbor_value_get_map_length(it, &len))))
return r;
msg->data.ptr = calloc(len, sizeof(struct stream2_image_data));
if (msg->data.ptr == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->data.len = len;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(it, &field))))
return r;
for (size_t i = 0; i < len; i++) {
if ((r = parse_text_string(&field, &msg->data.ptr[i].channel)))
return r;
if ((r = parse_multidim_array(&field, &msg->data.ptr[i].data)))
return r;
}
if ((r = CBOR_RESULT(cbor_value_leave_container(it, &field))))
return r;
} else {
if ((r = CBOR_RESULT(cbor_value_advance(it))))
return r;
}
}
return STREAM2_OK;
}
static enum stream2_result parse_end_msg(CborValue* it,
struct stream2_msg** msg_out) {
enum stream2_result r;
struct stream2_end_msg* msg = calloc(1, sizeof(struct stream2_end_msg));
*msg_out = (struct stream2_msg*)msg;
if (msg == NULL)
return STREAM2_ERROR_OUT_OF_MEMORY;
msg->type = STREAM2_MSG_END;
while (cbor_value_is_valid(it)) {
char key[MAX_KEY_LEN];
if ((r = parse_key(it, key)))
return r;
if ((r = CBOR_RESULT(cbor_value_skip_tag(it))))
return r;
if (strcmp(key, "series_id") == 0) {
if ((r = parse_uint64(it, &msg->series_id)))
return r;
} else if (strcmp(key, "series_unique_id") == 0) {
if ((r = parse_text_string(it, &msg->series_unique_id)))
return r;
} else {
if ((r = CBOR_RESULT(cbor_value_advance(it))))
return r;
}
}
return STREAM2_OK;
}
static enum stream2_result parse_msg_type(CborValue* it,
char type[MAX_KEY_LEN]) {
enum stream2_result r;
char key[MAX_KEY_LEN];
if ((r = parse_key(it, key)))
return r;
if (strcmp(key, "type") != 0)
return STREAM2_ERROR_PARSE;
if ((r = CBOR_RESULT(cbor_value_skip_tag(it))))
return r;
return parse_key(it, type);
}
static enum stream2_result parse_msg(const uint8_t* buffer,
size_t size,
struct stream2_msg** msg_out) {
enum stream2_result r;
// https://www.rfc-editor.org/rfc/rfc8949.html#name-self-described-cbor
const uint8_t MAGIC[3] = {0xd9, 0xd9, 0xf7};
if (size < sizeof(MAGIC) || memcmp(buffer, MAGIC, sizeof(MAGIC)) != 0)
return STREAM2_ERROR_SIGNATURE;
buffer += sizeof(MAGIC);
size -= sizeof(MAGIC);
CborParser parser;
CborValue it;
if ((r = CBOR_RESULT(cbor_parser_init(buffer, size, 0, &parser, &it))))
return r;
if (!cbor_value_is_map(&it))
return STREAM2_ERROR_PARSE;
CborValue field;
if ((r = CBOR_RESULT(cbor_value_enter_container(&it, &field))))
return r;
char type[MAX_KEY_LEN];
if ((r = parse_msg_type(&field, type)))
return r;
if (strcmp(type, "start") == 0) {
if ((r = parse_start_msg(&field, msg_out)))
return r;
} else if (strcmp(type, "image") == 0) {
if ((r = parse_image_msg(&field, msg_out)))
return r;
} else if (strcmp(type, "end") == 0) {
if ((r = parse_end_msg(&field, msg_out)))
return r;
} else {
return STREAM2_ERROR_PARSE;
}
return CBOR_RESULT(cbor_value_leave_container(&it, &field));
}
enum stream2_result stream2_parse_msg(const uint8_t* buffer,
size_t size,
struct stream2_msg** msg_out) {
enum stream2_result r;
*msg_out = NULL;
if ((r = parse_msg(buffer, size, msg_out))) {
if (*msg_out) {
stream2_free_msg(*msg_out);
*msg_out = NULL;
}
return r;
}
return STREAM2_OK;
}
static void free_start_msg(struct stream2_start_msg* msg) {
free(msg->arm_date);
for (size_t i = 0; i < msg->channels.len; i++)
free(msg->channels.ptr[i]);
free(msg->channels.ptr);
free(msg->countrate_correction_lookup_table.data.compression.algorithm);
free(msg->detector_description);
free(msg->detector_serial_number);
for (size_t i = 0; i < msg->flatfield.len; i++) {
free(msg->flatfield.ptr[i].channel);
free(msg->flatfield.ptr[i].flatfield.array.data.compression.algorithm);
}
free(msg->flatfield.ptr);
for (size_t i = 0; i < msg->pixel_mask.len; i++) {
free(msg->pixel_mask.ptr[i].channel);
free(msg->pixel_mask.ptr[i]
.pixel_mask.array.data.compression.algorithm);
}
free(msg->pixel_mask.ptr);
free(msg->sensor_material);
for (size_t i = 0; i < msg->threshold_energy.len; i++)
free(msg->threshold_energy.ptr[i].channel);
free(msg->threshold_energy.ptr);
}
static void free_image_msg(struct stream2_image_msg* msg) {
free(msg->series_date);
for (size_t i = 0; i < msg->data.len; i++) {
free(msg->data.ptr[i].channel);
free(msg->data.ptr[i].data.array.data.compression.algorithm);
}
free(msg->data.ptr);
}
void stream2_free_msg(struct stream2_msg* msg) {
switch (msg->type) {
case STREAM2_MSG_START:
free_start_msg((struct stream2_start_msg*)msg);
break;
case STREAM2_MSG_IMAGE:
free_image_msg((struct stream2_image_msg*)msg);
break;
case STREAM2_MSG_END:
break;
}
free(msg->series_unique_id);
free(msg);
}
enum stream2_result stream2_typed_array_elem_size(
const struct stream2_typed_array* array,
uint64_t* elem_size) {
// https://www.rfc-editor.org/rfc/rfc8746.html#name-types-of-numbers
if (array->tag >= 64 && array->tag <= 87) {
const uint64_t f = (array->tag >> 4) & 1;
const uint64_t ll = array->tag & 3;
*elem_size = 1 << (f + ll);
return STREAM2_OK;
}
return STREAM2_ERROR_NOT_IMPLEMENTED;
}

213
frame_serialize/stream2.h Normal file
View File

@@ -0,0 +1,213 @@
// DECTRIS proprietary license
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#if defined(__cplusplus)
extern "C" {
#endif
enum stream2_result {
STREAM2_OK = 0,
STREAM2_ERROR_OUT_OF_MEMORY,
STREAM2_ERROR_SIGNATURE,
STREAM2_ERROR_DECODE,
STREAM2_ERROR_PARSE,
STREAM2_ERROR_NOT_IMPLEMENTED,
};
// https://github.com/dectris/documentation/blob/main/cbor/dectris-compression-tag.md
struct stream2_compression {
// Name of compression algorithm used.
char* algorithm;
// Element size if required for decompression, reserved otherwise.
// Required by algorithm "bslz4".
uint64_t elem_size;
// Uncompressed size of the data.
uint64_t orig_size;
};
// Represents a byte string, possibly compressed.
struct stream2_bytes {
const uint8_t* ptr;
size_t len;
struct stream2_compression compression;
};
// CBOR tags used with typed arrays.
//
// https://www.rfc-editor.org/rfc/rfc8746.html#tab-tag-values
enum stream2_typed_array_tag {
STREAM2_TYPED_ARRAY_UINT8 = 64,
STREAM2_TYPED_ARRAY_UINT16_LITTLE_ENDIAN = 69,
STREAM2_TYPED_ARRAY_UINT32_LITTLE_ENDIAN = 70,
STREAM2_TYPED_ARRAY_FLOAT32_LITTLE_ENDIAN = 85,
};
// A typed array defined in RFC 8746 section 2.
//
// https://www.rfc-editor.org/rfc/rfc8746.html#name-typed-arrays
struct stream2_typed_array {
// CBOR tag of the typed array.
uint64_t tag;
// Byte representation of the array elements.
struct stream2_bytes data;
};
// A multi-dimensional array defined in RFC 8746 section 3.1.
//
// The array is always a typed array of row-major order.
//
// https://www.rfc-editor.org/rfc/rfc8746.html#name-multi-dimensional-array
// https://www.rfc-editor.org/rfc/rfc8746.html#name-row-major-order
struct stream2_multidim_array {
uint64_t dim[2];
struct stream2_typed_array array;
};
struct stream2_array_text_string {
char** ptr;
size_t len;
};
struct stream2_flatfield {
char* channel;
struct stream2_multidim_array flatfield;
};
struct stream2_flatfield_map {
struct stream2_flatfield* ptr;
size_t len;
};
struct stream2_goniometer_axis {
double increment;
double start;
};
struct stream2_goniometer {
struct stream2_goniometer_axis chi;
struct stream2_goniometer_axis kappa;
struct stream2_goniometer_axis omega;
struct stream2_goniometer_axis phi;
struct stream2_goniometer_axis two_theta;
};
struct stream2_image_data {
char* channel;
struct stream2_multidim_array data;
};
struct stream2_image_data_map {
struct stream2_image_data* ptr;
size_t len;
};
struct stream2_pixel_mask {
char* channel;
struct stream2_multidim_array pixel_mask;
};
struct stream2_pixel_mask_map {
struct stream2_pixel_mask* ptr;
size_t len;
};
struct stream2_threshold_energy {
char* channel;
double energy;
};
struct stream2_threshold_energy_map {
struct stream2_threshold_energy* ptr;
size_t len;
};
struct stream2_user_data {
const uint8_t* ptr;
size_t len;
};
enum stream2_msg_type {
STREAM2_MSG_START,
STREAM2_MSG_IMAGE,
STREAM2_MSG_END,
};
struct stream2_msg {
enum stream2_msg_type type;
uint64_t series_id;
char* series_unique_id;
};
struct stream2_start_msg {
enum stream2_msg_type type;
uint64_t series_id;
char* series_unique_id;
char* arm_date;
double beam_center_x;
double beam_center_y;
struct stream2_array_text_string channels;
double count_time;
bool countrate_correction_enabled;
struct stream2_typed_array countrate_correction_lookup_table;
char* detector_description;
char* detector_serial_number;
double detector_translation[3];
struct stream2_flatfield_map flatfield;
bool flatfield_enabled;
double frame_time;
struct stream2_goniometer goniometer;
uint64_t image_size_x;
uint64_t image_size_y;
double incident_energy;
double incident_wavelength;
uint64_t number_of_images;
struct stream2_pixel_mask_map pixel_mask;
bool pixel_mask_enabled;
double pixel_size_x;
double pixel_size_y;
uint64_t saturation_value;
char* sensor_material;
double sensor_thickness;
struct stream2_threshold_energy_map threshold_energy;
struct stream2_user_data user_data;
bool virtual_pixel_interpolation_enabled;
};
struct stream2_image_msg {
enum stream2_msg_type type;
uint64_t series_id;
char* series_unique_id;
uint64_t image_id;
uint64_t real_time[2];
char* series_date;
uint64_t start_time[2];
uint64_t stop_time[2];
struct stream2_user_data user_data;
struct stream2_image_data_map data;
};
struct stream2_end_msg {
enum stream2_msg_type type;
uint64_t series_id;
char* series_unique_id;
};
enum stream2_result stream2_parse_msg(const uint8_t* buffer,
const size_t size,
struct stream2_msg** msg_out);
void stream2_free_msg(struct stream2_msg* msg);
// Gets the element size of a typed array.
enum stream2_result stream2_typed_array_elem_size(
const struct stream2_typed_array* array,
uint64_t* elem_size);
#if defined(__cplusplus)
}
#endif

161
tests/CBORTest.cpp
View File

@@ -5,6 +5,8 @@
#include "../frame_serialize/JFJochFrameSerializer.h"
#include "../frame_serialize/JFJochFrameDeserializer.h"
#include "../frame_serialize/stream2.h"
#include "../compression/JFJochCompressor.h"
TEST_CASE("CBORSerialize_Start", "[CBOR]") {
JFJochFrameSerializer serializer(8*1024*1024);
@@ -128,7 +130,9 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {
.max_receiver_delay = 3456,
.efficiency = 0.99,
.write_master_file = true,
.end_date = "ccc"
.end_date = "ccc",
.series_unique_id = "bla5",
.series_id = 45676782
};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
@@ -146,6 +150,8 @@ TEST_CASE("CBORSerialize_End", "[CBOR]") {
REQUIRE(output_message.efficiency == Approx(message.efficiency));
REQUIRE(output_message.end_date == message.end_date);
REQUIRE(output_message.write_master_file == message.write_master_file);
REQUIRE(output_message.series_id == message.series_id);
REQUIRE(output_message.series_unique_id == message.series_unique_id);
}
TEST_CASE("CBORSerialize_Image", "[CBOR]") {
@@ -175,7 +181,9 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
.indexing_result = 1,
.bunch_id = UINT64_MAX,
.jf_info = UINT32_MAX,
.timestamp = 1ul<<27 | 1ul <<35
.timestamp = 1ul<<27 | 1ul <<35,
.series_unique_id = "bla2",
.series_id = 4567678
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
@@ -195,6 +203,8 @@ TEST_CASE("CBORSerialize_Image", "[CBOR]") {
REQUIRE(image_array.image.size == test.size());
REQUIRE(image_array.indexing_result == message.indexing_result);
REQUIRE(image_array.number == 456);
REQUIRE(image_array.series_id == message.series_id);
REQUIRE(image_array.series_unique_id == message.series_unique_id);
REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0);
REQUIRE(image_array.bunch_id == message.bunch_id);
@@ -290,7 +300,7 @@ TEST_CASE("CBORSerialize_Image_Compressed", "[CBOR]") {
REQUIRE(image_array.image.pixel_depth_bytes == 4);
REQUIRE(image_array.image.channel == "default");
REQUIRE(image_array.image.size == test.size());
REQUIRE(image_array.image.pixel_is_signed == false);
REQUIRE(image_array.image.pixel_is_signed == true);
REQUIRE(image_array.number == 456);
REQUIRE(memcmp(image_array.image.data, test.data(), test.size()) == 0);
}
@@ -383,6 +393,10 @@ TEST_CASE("CBORSerialize_Image_Spots", "[CBOR]") {
REQUIRE(image_array.spots[1].intensity == 123);
}
inline bool CmpString(const char *str1, const std::string& str2) {
return (strncmp(str1, str2.c_str(), str2.size()) == 0);
}
TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") {
JFJochFrameSerializer serializer(8*1024*1024);
@@ -438,6 +452,145 @@ TEST_CASE("CBORSerialize_Start_stream2", "[CBOR]") {
REQUIRE_NOTHROW(serializer.SerializeSequenceStart(message));
auto image = serializer.GetBuffer();
stream2_msg *msg;
auto ret = stream2_parse_msg(image.data(), image.size(), &msg);
REQUIRE(ret == STREAM2_OK);
CHECK(msg->type == STREAM2_MSG_START);
auto msg2 = (stream2_start_msg *) msg;
CHECK(msg2->beam_center_x == message.beam_center_x);
CHECK(msg2->beam_center_y == message.beam_center_y);
CHECK(msg2->image_size_x == message.image_size_x);
CHECK(msg2->image_size_y == message.image_size_y);
CHECK(msg2->series_id == message.series_id);
CHECK(CmpString(msg2->series_unique_id, message.series_unique_id));
CHECK(CmpString(msg2->detector_description, message.detector_description));
CHECK(CmpString(msg2->arm_date, message.arm_date));
CHECK(msg2->number_of_images == message.number_of_images);
CHECK(msg2->frame_time == message.frame_time);
CHECK(msg2->count_time == message.count_time);
stream2_free_msg(msg);
}
TEST_CASE("CBORSerialize_End_stream2", "[CBOR]") {
JFJochFrameSerializer serializer(8*1024*1024);
EndMessage message {
.number_of_images = 57789,
.max_receiver_delay = 3456,
.efficiency = 0.99,
.write_master_file = true,
.end_date = "ccc",
.series_unique_id = "bla5",
.series_id = 45676782
};
REQUIRE_NOTHROW(serializer.SerializeSequenceEnd(message));
auto image = serializer.GetBuffer();
stream2_msg *msg;
auto ret = stream2_parse_msg(image.data(), image.size(), &msg);
REQUIRE(ret == STREAM2_OK);
CHECK(msg->type == STREAM2_MSG_END);
auto msg2 = (stream2_end_msg *) msg;
CHECK(msg2->series_id == message.series_id);
CHECK(CmpString(msg2->series_unique_id, message.series_unique_id));
stream2_free_msg(msg);
}
}
TEST_CASE("CBORSerialize_Image_compressed_stream2", "[CBOR]") {
JFJochFrameSerializer serializer(8*1024*1024);
std::vector<SpotToSave> spots;
std::vector<uint16_t> test(512*1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
std::vector<uint8_t> compressed_test(512*1024*4);
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
compressor.Compress(compressed_test.data(), test);
CBORImage image {
.data = compressed_test.data(),
.size = 1024 * 512,
.xpixel = 1024,
.ypixel = 512,
.pixel_depth_bytes = 2,
.pixel_is_signed = false,
.algorithm = CompressionAlgorithm::BSHUF_LZ4,
.channel = "default"
};
DataMessage message {
.number = 456,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto cbor_image = serializer.GetBuffer();
stream2_msg *msg;
auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg);
REQUIRE(ret == STREAM2_OK);
CHECK(msg->type == STREAM2_MSG_IMAGE);
auto msg2 = (stream2_image_msg *) msg;
CHECK(msg2->image_id == message.number);
CHECK(msg2->data.len == 1);
CHECK(msg2->series_id == message.series_id);
CHECK(CmpString(msg2->series_unique_id, message.series_unique_id));
stream2_free_msg(msg);
}
TEST_CASE("CBORSerialize_Image_uncompressed_stream2", "[CBOR]") {
JFJochFrameSerializer serializer(8*1024*1024);
std::vector<SpotToSave> spots;
std::vector<uint8_t> test(512*1024);
for (int i = 0; i < test.size(); i++)
test[i] = (i * 253 + 56) % 256;
CBORImage image {
.data = test.data(),
.size = 1024 * 512,
.xpixel = 1024,
.ypixel = 512,
.pixel_depth_bytes = 1,
.pixel_is_signed = false,
.algorithm = CompressionAlgorithm::NO_COMPRESSION,
.channel = "default"
};
DataMessage message {
.number = 456,
.image = image,
.spots = spots
};
REQUIRE_NOTHROW(serializer.SerializeImage(message));
auto cbor_image = serializer.GetBuffer();
stream2_msg *msg;
auto ret = stream2_parse_msg(cbor_image.data(), cbor_image.size(), &msg);
REQUIRE(ret == STREAM2_OK);
CHECK(msg->type == STREAM2_MSG_IMAGE);
auto msg2 = (stream2_image_msg *) msg;
CHECK(msg2->image_id == message.number);
CHECK(msg2->data.len == 1);
CHECK(msg2->series_id == message.series_id);
CHECK(CmpString(msg2->series_unique_id, message.series_unique_id));
stream2_free_msg(msg);
}