// Copyright (2019-2022) Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-or-later #include "JFJochFrameDeserializer.h" #include "tinycbor/src/cbor.h" #include "CborErr.h" #include "CborUtil.h" inline std::string GetCBORString(CborValue &value) { if (!cbor_value_is_text_string(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "String expected"); char buf[256]; size_t len = 255; cborErr(cbor_value_copy_text_string(&value, buf, &len, nullptr)); buf[len+1] = 0; cborErr(cbor_value_advance(&value)); return {buf}; } inline int64_t GetCBORInt(CborValue &value) { if (!cbor_value_is_integer(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Integer expected"); int64_t tmp; cborErr(cbor_value_get_int64(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } inline uint64_t GetCBORUInt(CborValue &value) { if (!cbor_value_is_unsigned_integer(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Unsigned integer expected"); uint64_t tmp; cborErr(cbor_value_get_uint64(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } inline float GetCBORFloat(CborValue &value) { if (!cbor_value_is_float(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Float expected"); float tmp; cborErr(cbor_value_get_float(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } inline bool GetCBORBool(CborValue &value) { if (!cbor_value_is_boolean(&value) ) { throw JFJochException(JFJochExceptionCategory::CBORError, "Bool expected " + std::to_string(cbor_value_get_type(&value))); } bool tmp; cborErr(cbor_value_get_boolean(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } inline CborTag GetCBORTag(CborValue &value) { if (!cbor_value_is_tag(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Tag expected"); CborTag tmp; cborErr(cbor_value_get_tag(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } inline std::string GetCBORDateTime(CborValue &value) { auto tag = GetCBORTag(value); if (tag == CborDateTimeStringTag) return GetCBORString(value); else if (tag == CborUnixTime_tTag) { time_t t = GetCBORUInt(value); char buf1[255]; strftime(buf1, sizeof(buf1), "%FT%T", gmtime(&t)); return std::string(buf1) + "Z"; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Time/date tag error"); } inline uint64_t GetCBORArrayLen(CborValue &value) { if (!cbor_value_is_array(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); size_t array_len; cborErr(cbor_value_get_array_length(&value, &array_len)); return array_len; } inline uint64_t GetCBORMapLen(CborValue &value) { size_t array_len; cborErr(cbor_value_get_map_length(&value, &array_len)); return array_len; } inline std::pair GetRational(CborValue &value) { std::pair ret; if (GetCBORArrayLen(value) != 2) throw JFJochException(JFJochExceptionCategory::CBORError, "Rational number expects array of 2 elements"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); ret.first = GetCBORUInt(array_value); ret.second = GetCBORUInt(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); return ret; } inline void GetCBORFloatArray(CborValue &value, std::vector &v) { if (GetCBORTag(value) != TagFloatLE) throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag"); auto len = GetCBORArrayLen(value); if (len > 0) { CborValue array_value; v.resize(len); cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < len; i++) v[i] = GetCBORFloat(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); } else cbor_value_advance(&value); } inline std::pair GetCBORDimension2DArray(CborValue &value) { std::pair ret; if (GetCBORArrayLen(value) != 2) throw JFJochException(JFJochExceptionCategory::CBORError, "Only 2D arrays allowed"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); ret.second = GetCBORUInt(array_value); ret.first = GetCBORUInt(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); return ret; } inline std::pair GetCBORByteString(CborValue &value) { if (!cbor_value_is_byte_string(&value) ) throw JFJochException(JFJochExceptionCategory::CBORError, "Bool expected"); size_t len; cborErr(cbor_value_get_string_length(&value, &len)); // Code below from: https://github.com/dectris/documentation/blob/main/stream_v2/examples/stream2.c const uint8_t *ptr = cbor_value_get_next_byte(&value); uint8_t val = *ptr; if (val < 0x40) throw JFJochException(JFJochExceptionCategory::CBORError, "Something wrong with byte string tag"); val -= 0x40; if (val < 24) ptr += 1; else if (val == 24) ptr += 1 + 1; else if (val == 25) ptr += 1 + 2; else if (val == 26) ptr += 1 + 4; else if (val == 27) ptr += 1 + 8; else throw JFJochException(JFJochExceptionCategory::CBORError, "Something wrong with byte string tag"); cborErr(cbor_value_advance(&value)); return {ptr, len}; } inline void GetCBORTypedArray(CBORImage &v, CborValue &value) { CborTag tag = GetCBORTag(value); switch (tag) { case TagSignedInt8Bit: v.pixel_is_signed = true; v.pixel_depth_bytes = 1; break; case TagUnsignedInt8Bit: v.pixel_is_signed = false; v.pixel_depth_bytes = 1; break; case TagSignedInt16BitLE: v.pixel_is_signed = true; v.pixel_depth_bytes = 2; break; case TagUnsignedInt16BitLE: v.pixel_is_signed = false; v.pixel_depth_bytes = 2; break; case TagSignedInt32BitLE: v.pixel_is_signed = true; v.pixel_depth_bytes = 4; break; case TagUnsignedInt32BitLE: v.pixel_is_signed = false; v.pixel_depth_bytes = 4; break; default: throw JFJochException(JFJochExceptionCategory::CBORError, "Only int/uint arrays of 1, 2, 4 bytes per element allowed"); } if (cbor_value_is_tag(&value)) { if (GetCBORTag(value) != TagDECTRISCompression) throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported tag"); if (GetCBORArrayLen(value) != 3) throw JFJochException(JFJochExceptionCategory::CBORError, "Expected 3 element array"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); auto algorithm_text = GetCBORString(array_value); if (algorithm_text == "bslz4") v.algorithm = CompressionAlgorithm::BSHUF_LZ4; else if (algorithm_text == "bszstd") v.algorithm = CompressionAlgorithm::BSHUF_ZSTD; else throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported compression algorithm"); v.pixel_depth_bytes = GetCBORUInt(array_value); auto ret = GetCBORByteString(array_value); v.data = ret.first; v.size = ret.second; cborErr(cbor_value_leave_container(&value, &array_value)); } else if (cbor_value_is_byte_string(&value)) { v.algorithm = CompressionAlgorithm::NO_COMPRESSION; auto ret = GetCBORByteString(value); v.data = ret.first; v.size = ret.second; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Byte string or compressed array expected"); } void GetCBORMultidimTypedArray(CBORImage &v, CborValue &value) { if (GetCBORTag(value) != TagMultiDimArray) throw JFJochException(JFJochExceptionCategory::CBORError, "Multidim array expected"); if (GetCBORArrayLen(value) != 2) throw JFJochException(JFJochExceptionCategory::CBORError, "2 element array expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); auto [x,y] = GetCBORDimension2DArray(array_value); v.xpixel = x; v.ypixel = y; GetCBORTypedArray(v, array_value); cborErr(cbor_value_leave_container(&value, &array_value)); } void JFJochFrameDeserializer::GetCBORSpots(CborValue &value) { size_t array_len = GetCBORArrayLen(value); if (array_len % 4 != 0) throw JFJochException(JFJochExceptionCategory::CBORError, "Spot array has to have elements multiple of 4"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < array_len / 4; i++) { SpotToSave s{ .x = GetCBORFloat(array_value), .y = GetCBORFloat(array_value), .intensity = GetCBORFloat(array_value), .indexed = GetCBORBool(array_value) }; data_message.spots.push_back(s); } cborErr(cbor_value_leave_container(&value, &array_value)); } void JFJochFrameDeserializer::DecodeType(CborValue &value) { if (cbor_value_at_end(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Message empty"); auto key = GetCBORString(value); if (key != "type") throw JFJochException(JFJochExceptionCategory::CBORError, "First CBOR entry must by type"); if (cbor_value_at_end(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Message empty"); auto type_str = GetCBORString(value); if (type_str == "start") msg_type = Type::START; else if (type_str == "end") msg_type = Type::END; else if (type_str == "image") msg_type = Type::IMAGE; else throw JFJochException(JFJochExceptionCategory::CBORError, "Unknown message type"); } void JFJochFrameDeserializer::ProcessGoniometerMap(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); start_message.goniometer[key] = ProcessGoniometer(map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } GoniometerAxis JFJochFrameDeserializer::ProcessGoniometer(CborValue &value) { CborValue map_value; GoniometerAxis ret{}; cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "increment") ret.increment = GetCBORFloat(map_value); else if (key == "start") ret.start = GetCBORFloat(map_value); else cbor_value_advance(&value); } cborErr(cbor_value_leave_container(&value, &map_value)); return ret; } void JFJochFrameDeserializer::ProcessDetTranslation(CborValue &value) { if (GetCBORArrayLen(value) != 3) throw JFJochException(JFJochExceptionCategory::CBORError, "Array with 3 floats expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < 3; i++) start_message.detector_translation[i] = GetCBORFloat(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); } void JFJochFrameDeserializer::ProcessImageMessageUserDataElement(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "spots") GetCBORSpots(map_value); else if (key == "rad_int_profile") GetCBORFloatArray(map_value, data_message.rad_int_profile); else if (key == "indexing_result") data_message.indexing_result = GetCBORUInt(map_value); else if (key == "indexing_lattice") GetCBORFloatArray(map_value, data_message.indexing_lattice); else if (key == "jf_info") data_message.jf_info = GetCBORUInt(map_value) & UINT32_MAX; else if (key == "receiver_available_send_buffers") data_message.receiver_available_send_buffers = GetCBORFloat(map_value); else if (key == "receiver_aq_dev_delay") data_message.receiver_aq_dev_delay = GetCBORInt(map_value); else if (key == "storage_cell") data_message.storage_cell = GetCBORUInt(map_value) & UINT32_MAX; else if (key == "bunch_id") data_message.bunch_id = GetCBORUInt(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } bool JFJochFrameDeserializer::ProcessImageMessageElement(CborValue &value) { if (cbor_value_at_end(&value)) return false; else { auto key = GetCBORString(value); if (key == "image_id") data_message.number = GetCBORInt(value); else if (key == "data") ProcessImageData(value); else if (key == "series_unique_id") data_message.series_unique_id = GetCBORString(value); else if (key == "series_id") data_message.series_id = GetCBORUInt(value); else if (key == "real_time") { auto r = GetRational(value); data_message.exptime = r.first; data_message.exptime_base = r.second; } else if (key == "start_time") { auto r = GetRational(value); data_message.timestamp = r.first; data_message.timestamp_base = r.second; } else if (key == "user_data") ProcessImageMessageUserDataElement(value); else cbor_value_advance(&value); return true; } } void JFJochFrameDeserializer::ProcessPixelMaskElement(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); CBORImage cbor_multidim_array; GetCBORMultidimTypedArray(cbor_multidim_array, map_value); if (cbor_multidim_array.size != cbor_multidim_array.xpixel * cbor_multidim_array.ypixel * cbor_multidim_array.pixel_depth_bytes) throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask size mismatch"); if (cbor_multidim_array.pixel_depth_bytes != sizeof(uint32_t)) throw JFJochException(JFJochExceptionCategory::CBORError, "Pixel mask must be 32-bit"); std::vector v(cbor_multidim_array.xpixel * cbor_multidim_array.ypixel); memcpy(v.data(), cbor_multidim_array.data, cbor_multidim_array.size); start_message.pixel_mask[key] = v; } cborErr(cbor_value_leave_container(&value, &map_value)); } void JFJochFrameDeserializer::ProcessRadIntResultElement(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); std::vector val; GetCBORFloatArray(map_value, val); end_message.rad_int_result[key] = val; } cborErr(cbor_value_leave_container(&value, &map_value)); } void JFJochFrameDeserializer::ProcessChannels(CborValue &value) { if (!cbor_value_is_array(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); while (! cbor_value_at_end(&array_value)) { const std::string s = GetCBORString(array_value); start_message.channels.push_back(s); } cborErr(cbor_value_leave_container(&value, &array_value)); } void JFJochFrameDeserializer::ProcessUnitCellElement(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (! cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "a") start_message.unit_cell[0] = GetCBORFloat(map_value); else if (key == "b") start_message.unit_cell[1] = GetCBORFloat(map_value); else if (key == "c") start_message.unit_cell[2] = GetCBORFloat(map_value); else if (key == "alpha") start_message.unit_cell[3] = GetCBORFloat(map_value); else if (key == "beta") start_message.unit_cell[4] = GetCBORFloat(map_value); else if (key == "gamma") start_message.unit_cell[5] = GetCBORFloat(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } void JFJochFrameDeserializer::ProcessStartMessageUserDataElement(CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "space_group_number") start_message.space_group_number = GetCBORUInt(map_value); else if (key == "unit_cell") ProcessUnitCellElement(map_value); else if (key == "sample_name") start_message.sample_name = GetCBORString(map_value); else if (key == "source_name") start_message.source_name = GetCBORString(map_value); else if (key == "source_name_short") start_message.source_name_short = GetCBORString(map_value); else if (key == "instrument_name") start_message.instrument_name = GetCBORString(map_value); else if (key == "instrument_name_short") start_message.instrument_name_short = GetCBORString(map_value); else if (key == "file_prefix") start_message.file_prefix = GetCBORString(map_value); else if (key == "max_spot_count") start_message.max_spot_count = GetCBORUInt(map_value); else if (key == "data_file_count") start_message.data_file_count = GetCBORUInt(map_value); else if (key == "pixel_bit_depth") start_message.pixel_bit_depth = GetCBORUInt(map_value); else if (key == "pixel_signed") start_message.pixel_signed = GetCBORBool(map_value); else if (key == "min_value") start_message.min_value = GetCBORInt(map_value); else if (key == "rad_int_bin_number") start_message.rad_int_bin_number = GetCBORInt(map_value); else if (key == "rad_int_bin_to_q") GetCBORFloatArray(map_value, start_message.rad_int_bin_to_q); else if (key == "rad_int_solid_angle_corr") GetCBORFloatArray(map_value, start_message.rad_int_solid_angle_corr); else if (key == "summation") start_message.summation = GetCBORUInt(map_value); else if (key == "storage_cell_number") start_message.storage_cell_number = GetCBORUInt(map_value); else if (key == "compression_algorithm") { auto tmp = GetCBORString(map_value); if (tmp == "bslz4") start_message.compression_algorithm = CompressionAlgorithm::BSHUF_LZ4; else if (tmp == "bszstd") start_message.compression_algorithm = CompressionAlgorithm::BSHUF_ZSTD; else if (tmp == "none") start_message.compression_algorithm = CompressionAlgorithm::NO_COMPRESSION; else throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported compression"); } else if (key == "compression_block_size") start_message.compression_block_size = GetCBORUInt(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } bool JFJochFrameDeserializer::ProcessStartMessageElement(CborValue &value) { if (cbor_value_at_end(&value)) return false; else { auto key = GetCBORString(value); if (key == "beam_center_x") start_message.beam_center_x = GetCBORFloat(value); else if (key == "beam_center_y") start_message.beam_center_y = GetCBORFloat(value); else if (key == "detector_distance") start_message.detector_distance = GetCBORFloat(value); else if (key == "number_of_images") start_message.number_of_images = GetCBORUInt(value); else if (key == "image_size_x") start_message.image_size_x = GetCBORUInt(value); else if (key == "image_size_y") start_message.image_size_y = GetCBORUInt(value); else if (key == "incident_energy") start_message.incident_energy = GetCBORFloat(value); else if (key == "incident_wavelength") start_message.incident_wavelength = GetCBORFloat(value); else if (key == "frame_time") start_message.frame_time = GetCBORFloat(value); else if (key == "count_time") start_message.count_time = GetCBORFloat(value); else if (key == "saturation_value") start_message.saturation_value = GetCBORInt(value); else if (key == "pixel_size_x") start_message.pixel_size_x = GetCBORFloat(value); else if (key == "pixel_size_y") start_message.pixel_size_y = GetCBORFloat(value); else if (key == "sensor_thickness") start_message.sensor_thickness = GetCBORFloat(value); else if (key == "sensor_material") start_message.sensor_material = GetCBORString(value); else if (key == "detector_description") start_message.detector_description = GetCBORString(value); else if (key == "detector_serial_number") start_message.detector_serial_number = GetCBORString(value); else if (key == "series_unique_id") start_message.series_unique_id = GetCBORString(value); else if (key == "series_id") start_message.series_id = GetCBORUInt(value); else if (key == "pixel_mask") ProcessPixelMaskElement(value); else if (key == "channels") ProcessChannels(value); else if (key == "detector_translation") ProcessDetTranslation(value); else if (key == "goniometer") ProcessGoniometerMap(value); else if (key == "pixel_mask_enabled") start_message.pixel_mask_enabled = GetCBORBool(value); else if (key == "arm_date") start_message.arm_date = GetCBORDateTime(value); else if (key == "user_data") ProcessStartMessageUserDataElement(value); else cbor_value_advance(&value); return true; } } bool JFJochFrameDeserializer::ProcessEndMessageElement(CborValue &value) { if (cbor_value_at_end(&value)) return false; else { auto key = GetCBORString(value); if (key == "number_of_images") end_message.number_of_images = GetCBORUInt(value); else if (key == "max_receiver_delay") end_message.max_receiver_delay = GetCBORUInt(value); else if (key == "receiver_efficiency") end_message.efficiency = GetCBORFloat(value); else if (key == "write_master_file") end_message.write_master_file = GetCBORBool(value); else if (key == "end_date") end_message.end_date = GetCBORString(value); else if (key == "series_unique_id") end_message.series_unique_id = GetCBORString(value); else if (key == "series_id") end_message.series_id = GetCBORUInt(value); else if (key == "rad_int_result") ProcessRadIntResultElement(value); else cbor_value_advance(&value); return true; } } void JFJochFrameDeserializer::ProcessImageData(CborValue &value) { if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Map expected"); if (GetCBORMapLen(value) != 1) throw JFJochException(JFJochExceptionCategory::CBORError, "Single channel images only supported at the moment"); CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); data_message.image.channel = GetCBORString(map_value); GetCBORMultidimTypedArray(data_message.image, map_value); cborErr(cbor_value_leave_container(&value, &map_value)); } void JFJochFrameDeserializer::Process(const std::vector &buffer) { Process(buffer.data(), buffer.size()); } void JFJochFrameDeserializer::Process(const uint8_t *msg, size_t msg_size) { std::unique_lock ul(m); data_message = DataMessage(); CborParser parser; CborValue value; cborErr(cbor_parser_init(msg, msg_size, 0, &parser, &value)); if (GetCBORTag(value) != CborSignatureTag) throw JFJochException(JFJochExceptionCategory::CBORError, "CBOR must start with dedicated tag"); if (cbor_value_is_map(&value)) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); DecodeType(map_value); switch (msg_type) { case Type::IMAGE: while (ProcessImageMessageElement(map_value)); break; case Type::START: start_message = StartMessage{ .data_file_count = 1, .compression_algorithm = CompressionAlgorithm::NO_COMPRESSION, .compression_block_size = 0 }; while (ProcessStartMessageElement(map_value)); break; case Type::END: end_message = EndMessage{}; while (ProcessEndMessageElement(map_value)); break; } cborErr(cbor_value_leave_container(&value, &map_value)); } else throw JFJochException(JFJochExceptionCategory::CBORError, "Serialized frame must be map in top level"); } DataMessage JFJochFrameDeserializer::GetDataMessage() const { std::unique_lock ul(m); return data_message; } JFJochFrameDeserializer::Type JFJochFrameDeserializer::GetType() const { std::unique_lock ul(m); return msg_type; } EndMessage JFJochFrameDeserializer::GetEndMessage() const { std::unique_lock ul(m); return end_message; } StartMessage JFJochFrameDeserializer::GetStartMessage() const { std::unique_lock ul(m); return start_message; }