// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include "CBORStream2Deserializer.h" #include "CborErr.h" #include "CborUtil.h" #include "../compression/JFJochDecompress.h" #include namespace { std::string GetCBORString(CborValue &value) { if (!cbor_value_is_text_string(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "String expected"); std::vector s(16384); size_t len = s.size() - 1; cborErr(cbor_value_copy_text_string(&value, s.data(), &len, nullptr)); s[len + 1] = 0; cborErr(cbor_value_advance(&value)); return {s.data()}; } int64_t GetCBORInt(CborValue &value) { if (!cbor_value_is_integer(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Integer expected"); int64_t tmp; cborErr(cbor_value_get_int64(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } uint64_t GetCBORUInt(CborValue &value) { if (!cbor_value_is_unsigned_integer(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Unsigned integer expected"); uint64_t tmp; cborErr(cbor_value_get_uint64(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } float GetCBORFloat(CborValue &value) { if (cbor_value_is_double(&value)) { double tmp; cborErr(cbor_value_get_double(&value, &tmp)); cborErr(cbor_value_advance(&value)); return static_cast(tmp); } else if (cbor_value_is_float(&value)) { float tmp; cborErr(cbor_value_get_float(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } else if (cbor_value_is_half_float(&value)) { float tmp; cborErr(cbor_value_get_half_float_as_float(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } throw JFJochException(JFJochExceptionCategory::CBORError, "Float expected"); } bool GetCBORBool(CborValue &value) { if (!cbor_value_is_boolean(&value)) { throw JFJochException(JFJochExceptionCategory::CBORError, "Bool expected " + std::to_string(cbor_value_get_type(&value))); } bool tmp; cborErr(cbor_value_get_boolean(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } CborTag GetCBORTag(CborValue &value) { if (!cbor_value_is_tag(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Tag expected"); CborTag tmp; cborErr(cbor_value_get_tag(&value, &tmp)); cborErr(cbor_value_advance(&value)); return tmp; } std::string GetCBORDateTime(CborValue &value) { auto tag = GetCBORTag(value); if (tag == CborDateTimeStringTag) return GetCBORString(value); else if (tag == CborUnixTime_tTag) { time_t t = GetCBORUInt(value); char buf1[255]; strftime(buf1, sizeof(buf1), "%FT%T", gmtime(&t)); return std::string(buf1) + "Z"; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Time/date tag error"); } uint64_t GetCBORArrayLen(CborValue &value) { if (!cbor_value_is_array(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); size_t array_len; cborErr(cbor_value_get_array_length(&value, &array_len)); return array_len; } uint64_t GetCBORMapLen(CborValue &value) { if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Map expected"); size_t map_len; cborErr(cbor_value_get_map_length(&value, &map_len)); return map_len; } Coord GetCoord(CborValue &value) { CborValue array_value; Coord c; if (GetCBORArrayLen(value) != 3) throw JFJochException(JFJochExceptionCategory::CBORError, "Vector of 3 elements expected"); cborErr(cbor_value_enter_container(&value, &array_value)); c.x = GetCBORFloat(array_value); c.y = GetCBORFloat(array_value); c.z = GetCBORFloat(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); return c; } std::pair GetRational(CborValue &value) { std::pair ret; if (GetCBORArrayLen(value) != 2) throw JFJochException(JFJochExceptionCategory::CBORError, "Rational number expects array of 2 elements"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); ret.first = GetCBORUInt(array_value); ret.second = GetCBORUInt(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); return ret; } std::map ProcessThresholdEnergy(CborValue &value) { if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Map expected"); std::map ret; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); auto val = GetCBORFloat(map_value); ret[key] = val; } cborErr(cbor_value_leave_container(&value, &map_value)); return ret; } UnitCell ProcessUnitCellElement(CborValue &value) { UnitCell unit_cell{}; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "a") unit_cell.a = GetCBORFloat(map_value); else if (key == "b") unit_cell.b = GetCBORFloat(map_value); else if (key == "c") unit_cell.c = GetCBORFloat(map_value); else if (key == "alpha") unit_cell.alpha = GetCBORFloat(map_value); else if (key == "beta") unit_cell.beta = GetCBORFloat(map_value); else if (key == "gamma") unit_cell.gamma = GetCBORFloat(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); return unit_cell; } std::vector GetCBORDimensionArray(CborValue &value) { std::vector ret; CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); while (!cbor_value_at_end(&array_value)) ret.push_back(GetCBORUInt(array_value)); cborErr(cbor_value_leave_container(&value, &array_value)); return ret; } std::pair GetCBORByteString(CborValue &value) { if (!cbor_value_is_byte_string(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Bool expected"); size_t len; cborErr(cbor_value_get_string_length(&value, &len)); // Code below from: https://github.com/dectris/documentation/blob/main/stream_v2/examples/stream2.c const uint8_t *ptr = cbor_value_get_next_byte(&value); uint8_t val = *ptr; if (val < 0x40) throw JFJochException(JFJochExceptionCategory::CBORError, "Something wrong with byte string tag"); val -= 0x40; if (val < 24) ptr += 1; else if (val == 24) ptr += 1 + 1; else if (val == 25) ptr += 1 + 2; else if (val == 26) ptr += 1 + 4; else if (val == 27) ptr += 1 + 8; else throw JFJochException(JFJochExceptionCategory::CBORError, "Something wrong with byte string tag"); cborErr(cbor_value_advance(&value)); return {ptr, len}; } void GetCBORFloatArray(CborValue &value, std::vector &v) { if (GetCBORTag(value) != TagFloatLE) throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag"); auto [ptr, len] = GetCBORByteString(value); if (len % sizeof(float)) throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch"); v.resize(len / sizeof(float)); memcpy(v.data(), ptr, len); } void GetCBORStringArray(CborValue &value, std::vector &v) { if (!cbor_value_is_array(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); while (!cbor_value_at_end(&array_value)) { v.emplace_back(GetCBORString(array_value)); } cborErr(cbor_value_leave_container(&value, &array_value)); } void GetCBORUInt64Array(CborValue &value, std::vector &v) { if (GetCBORTag(value) != TagUnsignedInt64BitLE) throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag"); auto [ptr, len] = GetCBORByteString(value); if (len % sizeof(uint64_t)) throw JFJochException(JFJochExceptionCategory::CBORError, "Size mismatch"); v.resize(len / sizeof(uint64_t)); memcpy(v.data(), ptr, len); } CompressedImageMode GetImageMode(CborTag input) { switch (input) { case TagHalfLE: return CompressedImageMode::Float16; case TagFloatLE: return CompressedImageMode::Float32; case TagDoubleLE: return CompressedImageMode::Float64; case TagSignedInt8Bit: return CompressedImageMode::Int8; case TagUnsignedInt8Bit: return CompressedImageMode::Uint8; case TagSignedInt16BitLE: return CompressedImageMode::Int16; case TagUnsignedInt16BitLE: return CompressedImageMode::Uint16; case TagSignedInt32BitLE: return CompressedImageMode::Int32; case TagUnsignedInt32BitLE: return CompressedImageMode::Uint32; default: throw JFJochException(JFJochExceptionCategory::CBORError, "Only int/uint arrays of 1, 2, 4 bytes per element allowed"); } } CompressedImage GetCBORTypedArray(CborValue &value, const std::vector &dim) { CompressedImageMode mode = GetImageMode(GetCBORTag(value)); CompressionAlgorithm algorithm = CompressionAlgorithm::NO_COMPRESSION; size_t xpixel, ypixel; size_t size; const uint8_t *ptr = nullptr; if (cbor_value_is_tag(&value)) { if (GetCBORTag(value) != TagDECTRISCompression) throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported tag"); if (GetCBORArrayLen(value) != 3) throw JFJochException(JFJochExceptionCategory::CBORError, "Expected 3 element array"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); auto algorithm_text = GetCBORString(array_value); if (algorithm_text == "bslz4") algorithm = CompressionAlgorithm::BSHUF_LZ4; else if (algorithm_text == "bszstd") algorithm = CompressionAlgorithm::BSHUF_ZSTD; else throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported compression algorithm"); auto pixel_depth_bytes = GetCBORUInt(array_value); auto ret = GetCBORByteString(array_value); ptr = ret.first; size = ret.second; cborErr(cbor_value_leave_container(&value, &array_value)); } else if (cbor_value_is_byte_string(&value)) { algorithm = CompressionAlgorithm::NO_COMPRESSION; auto ret = GetCBORByteString(value); ptr = ret.first; size = ret.second; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Byte string or compressed array expected"); if (dim.size() == 2) { xpixel = dim[1]; ypixel = dim[0]; } else if (dim.size() == 3 && dim[0] == 3 && mode == CompressedImageMode::Uint8) { mode = CompressedImageMode::RGB; xpixel = dim[2]; ypixel = dim[1]; } else throw JFJochException(JFJochExceptionCategory::CBORError, "Image dimension not supported"); return CompressedImage(ptr, size, xpixel, ypixel, mode, algorithm); } CompressedImage GetCBORMultidimTypedArray(CborValue &value) { if (GetCBORTag(value) != TagMultiDimArray) throw JFJochException(JFJochExceptionCategory::CBORError, "Multidim array expected"); if (GetCBORArrayLen(value) != 2) throw JFJochException(JFJochExceptionCategory::CBORError, "2 element array expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); auto dim = GetCBORDimensionArray(array_value); CompressedImage image = GetCBORTypedArray(array_value, dim); cborErr(cbor_value_leave_container(&value, &array_value)); return image; } void CheckMagicNumber(CborValue &v) { if (GetCBORUInt(v) != user_data_magic_number) throw JFJochException(JFJochExceptionCategory::CBORError, "Inconsistency between Jungfraujoch server and writer"); } CompressedImage ProcessImageData(CborValue &value) { if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Map expected"); if (GetCBORMapLen(value) != 1) throw JFJochException(JFJochExceptionCategory::CBORError, "Single channel images only supported at the moment"); CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); std::string channel = GetCBORString(map_value); CompressedImage image = GetCBORMultidimTypedArray(map_value); image.Channel(channel); cborErr(cbor_value_leave_container(&value, &map_value)); return image; } // Data message SpotToSave GetSpot(CborValue &value) { SpotToSave s{}; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "x") s.x = GetCBORFloat(map_value); else if (key == "y") s.y = GetCBORFloat(map_value); else if (key == "I") s.intensity = GetCBORFloat(map_value); else if (key == "maxc") s.maxc = GetCBORInt(map_value); else if (key == "ice_ring") s.ice_ring = GetCBORBool(map_value); else if (key == "indexed") s.indexed = GetCBORBool(map_value); else if (key == "h") s.h = GetCBORInt(map_value); else if (key == "k") s.k = GetCBORInt(map_value); else if (key == "l") s.l = GetCBORInt(map_value); else if (key == "dist_ewald") s.dist_ewald_sphere = GetCBORFloat(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); return s; } void GetCBORSpots(DataMessage &message, CborValue &value) { size_t array_len = GetCBORArrayLen(value); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < array_len; i++) message.spots.push_back(GetSpot(array_value)); cborErr(cbor_value_leave_container(&value, &array_value)); } Reflection GetReflection(CborValue &value) { Reflection r{}; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "h") r.h = GetCBORInt(map_value); else if (key == "k") r.k = GetCBORInt(map_value); else if (key == "l") r.l = GetCBORInt(map_value); else if (key == "x") r.predicted_x = GetCBORFloat(map_value); else if (key == "y") r.predicted_y = GetCBORFloat(map_value); else if (key == "d") r.d = GetCBORFloat(map_value); else if (key == "I") r.I = GetCBORFloat(map_value); else if (key == "bkg") r.bkg = GetCBORFloat(map_value); else if (key == "sigma") r.sigma = GetCBORFloat(map_value); else if (key == "image") r.image_number = GetCBORFloat(map_value); else if (key == "rp") r.dist_ewald = GetCBORFloat(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); return r; } void GetCBORReflections(DataMessage &message, CborValue &value) { size_t array_len = GetCBORArrayLen(value); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < array_len; i++) message.reflections.push_back(GetReflection(array_value)); cborErr(cbor_value_leave_container(&value, &array_value)); } LatticeMessage GetCBORLatticeMessage(CborValue &value) { LatticeMessage lm{}; lm.centering = 'P'; lm.niggli_class = 0; lm.crystal_system = gemmi::CrystalSystem::Triclinic; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "centering") { auto s = GetCBORString(map_value); char c = s.empty() ? 'P' : s[0]; switch (c) { case 'P': case 'A': case 'B': case 'C': case 'I': case 'F': case 'R': lm.centering = c; break; default: break; } } else if (key == "niggli_class") { lm.niggli_class = GetCBORInt(map_value); } else if (key == "system") { auto sys = GetCBORString(map_value); // map known strings to enum if (sys == "triclinic") lm.crystal_system = gemmi::CrystalSystem::Triclinic; else if (sys == "monoclinic") lm.crystal_system = gemmi::CrystalSystem::Monoclinic; else if (sys == "orthorhombic") lm.crystal_system = gemmi::CrystalSystem::Orthorhombic; else if (sys == "tetragonal") lm.crystal_system = gemmi::CrystalSystem::Tetragonal; else if (sys == "trigonal") lm.crystal_system = gemmi::CrystalSystem::Trigonal; else if (sys == "hexagonal") lm.crystal_system = gemmi::CrystalSystem::Hexagonal; else if (sys == "cubic") lm.crystal_system = gemmi::CrystalSystem::Cubic; else { // unknown string, keep default } } else if (key == "primitive_lattice") { std::vector tmp; GetCBORFloatArray(map_value, tmp); lm.primitive = CrystalLattice(tmp); } else { cbor_value_advance(&map_value); } } cborErr(cbor_value_leave_container(&value, &map_value)); return lm; } XrayFluorescenceSpectrum GetCBORFluorescenceSpectrum(CborValue &value) { std::vector data, energy; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "energy") GetCBORFloatArray(map_value, energy); else if (key == "data") GetCBORFloatArray(map_value, data); } cborErr(cbor_value_leave_container(&value, &map_value)); if (data.size() == energy.size() && data.size() > 0) return {energy, data}; else return {}; } CBORImageType DecodeType(CborValue &value) { if (cbor_value_at_end(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Message empty"); if (GetCBORString(value) != "type") throw JFJochException(JFJochExceptionCategory::CBORError, "First CBOR entry must by type"); if (cbor_value_at_end(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Message empty"); auto type_str = GetCBORString(value); if (type_str == "start") return CBORImageType::START; if (type_str == "end") return CBORImageType::END; if (type_str == "image") return CBORImageType::IMAGE; if (type_str == "calibration") return CBORImageType::CALIBRATION; if (type_str == "metadata") return CBORImageType::METADATA; throw JFJochException(JFJochExceptionCategory::CBORError, "Unknown message type"); } // Data message ROIMessage ProcessROIElement(CborValue &value) { ROIMessage msg{}; // value CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "sum") msg.sum = GetCBORInt(map_value); else if (key == "sum_square") msg.sum_square = GetCBORUInt(map_value); else if (key == "max_count") msg.max_count = GetCBORInt(map_value); else if (key == "pixels") msg.pixels = GetCBORUInt(map_value); else if (key == "x_weighted_sum") msg.x_weighted = GetCBORInt(map_value); else if (key == "y_weighted_sum") msg.y_weighted = GetCBORInt(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); return msg; } void ProcessROIElementMap(DataMessage &message, CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { // key const std::string roi_name = GetCBORString(map_value); // value message.roi[roi_name] = ProcessROIElement(map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } bool ProcessDataMessageElement(DataMessage &message, CborValue &value) { if (cbor_value_at_end(&value)) return false; const auto key = GetCBORString(value); try { if (key == "image_id") message.number = GetCBORInt(value); else if (key == "original_image_id") message.original_number = GetCBORInt(value); else if (key == "data") message.image = ProcessImageData(value); else if (key == "series_unique_id") message.run_name = GetCBORString(value); else if (key == "series_id") message.run_number = GetCBORUInt(value); else if (key == "real_time") { auto r = GetRational(value); message.exptime = r.first; message.exptime_base = r.second; } else if (key == "start_time") { auto r = GetRational(value); message.timestamp = r.first; message.timestamp_base = r.second; } else if (key == "user_data") { const std::string s = GetCBORString(value); try { message.user_data = nlohmann::json::parse(s); } catch (...) { message.user_data = s; } } else if (key == "spots") GetCBORSpots(message, value); else if (key == "reflections") GetCBORReflections(message, value); else if (key == "spot_count") message.spot_count = GetCBORUInt(value); else if (key == "spot_count_low_res") message.spot_count_low_res = GetCBORUInt(value); else if (key == "spot_count_indexed") message.spot_count_indexed = GetCBORUInt(value); else if (key == "spot_count_ice_rings") message.spot_count_ice_rings = GetCBORUInt(value); else if (key == "az_int_profile") GetCBORFloatArray(value, message.az_int_profile); else if (key == "indexing_result") message.indexing_result = GetCBORBool(value); else if (key == "indexing_lattice") { std::vector tmp; GetCBORFloatArray(value, tmp); message.indexing_lattice = CrystalLattice(tmp); } else if (key == "indexing_time") message.indexing_time_s = GetCBORFloat(value); else if (key == "processing_time") message.processing_time_s = GetCBORFloat(value); else if (key == "profile_radius") message.profile_radius = GetCBORFloat(value); else if (key == "b_factor") message.b_factor = GetCBORFloat(value); else if (key == "indexing_unit_cell") message.indexing_unit_cell = ProcessUnitCellElement(value); else if (key == "lattice_type") message.lattice_type = GetCBORLatticeMessage(value); else if (key == "jf_info") message.jf_info = GetCBORUInt(value) & UINT32_MAX; else if (key == "receiver_aq_dev_delay") message.receiver_aq_dev_delay = GetCBORInt(value); else if (key == "receiver_free_send_buf") message.receiver_free_send_buf = GetCBORInt(value); else if (key == "storage_cell") message.storage_cell = GetCBORUInt(value) & UINT32_MAX; else if (key == "xfel_pulse_id") message.xfel_pulse_id = GetCBORUInt(value); else if (key == "xfel_event_code") message.xfel_event_code = GetCBORUInt(value); else if (key == "saturated_pixel_count") message.saturated_pixel_count = GetCBORUInt(value); else if (key == "error_pixel_count") message.error_pixel_count = GetCBORUInt(value); else if (key == "pixel_sum") message.pixel_sum = GetCBORInt(value); else if (key == "strong_pixel_count") message.strong_pixel_count = GetCBORUInt(value); else if (key == "min_viable_pixel_value") message.min_viable_pixel_value = GetCBORInt(value); else if (key == "max_viable_pixel_value") message.max_viable_pixel_value = GetCBORInt(value); else if (key == "resolution_estimate") message.resolution_estimate = GetCBORFloat(value); else if (key == "data_collection_efficiency") message.image_collection_efficiency = GetCBORFloat(value); else if (key == "packets_expected") message.packets_expected = GetCBORUInt(value); else if (key == "packets_received") message.packets_received = GetCBORUInt(value); else if (key == "bkg_estimate") message.bkg_estimate = GetCBORFloat(value); else if (key == "adu_histogram") GetCBORUInt64Array(value, message.adu_histogram); else if (key == "beam_corr_x") message.beam_corr_x = GetCBORFloat(value); else if (key == "beam_corr_y") message.beam_corr_y = GetCBORFloat(value); else if (key == "roi_integrals") ProcessROIElementMap(message, value); else { if (cbor_value_is_tag(&value)) cbor_value_advance(&value); cbor_value_advance(&value); } return true; } catch (const JFJochException &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing image message, key " + key + ":" + e.what()); } } bool ProcessMetadataImagesElement(MetadataMessage &metadata, CborValue &value) { if (cbor_value_at_end(&value)) return false; if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "images@metadata must be map"); CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); DataMessage message{}; while (ProcessDataMessageElement(message, map_value)) {} metadata.images.push_back(std::move(message)); cborErr(cbor_value_leave_container(&value, &map_value)); return true; } bool ProcessMetadataMessageElement(MetadataMessage &metadata, CborValue &value) { if (cbor_value_at_end(&value)) return false; const auto key = GetCBORString(value); try { if (key == "series_unique_id") metadata.run_name = GetCBORString(value); else if (key == "series_id") metadata.run_number = GetCBORUInt(value); else if (key == "images") { if (!cbor_value_is_array(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "images@metadata must be array"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); while (ProcessMetadataImagesElement(metadata, array_value)) {} cborErr(cbor_value_leave_container(&value, &array_value)); } else { if (cbor_value_is_tag(&value)) cbor_value_advance(&value); cbor_value_advance(&value); } return true; } catch (const JFJochException &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing image message, key " + key + ":" + e.what()); } } // Start message void ProcessAxis(CborValue &value, float v[3]) { if (GetCBORArrayLen(value) != 3) throw JFJochException(JFJochExceptionCategory::CBORError, "Array with 3 floats expected"); CborValue array_value; cborErr(cbor_value_enter_container(&value, &array_value)); for (int i = 0; i < 3; i++) v[i] = GetCBORFloat(array_value); cborErr(cbor_value_leave_container(&value, &array_value)); } std::optional ProcessGridScan(CborValue &value) { int64_t n_fast = 0; int64_t n_slow = 0; float step_x_um = 0; float step_y_um = 0; bool snake = false; bool vertical = false; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "n_fast") n_fast = GetCBORInt(map_value); else if (key == "n_slow") n_slow = GetCBORInt(map_value); else if (key == "step_x_axis") step_x_um = GetCBORFloat(map_value) * 1e6f; else if (key == "step_y_axis") step_y_um = GetCBORFloat(map_value) * 1e6f; else if (key == "snake_scan") snake = GetCBORBool(map_value); else if (key == "vertical_scan") vertical = GetCBORBool(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); GridScanSettings grid(n_fast, step_x_um, step_y_um, snake, vertical); grid.ImageNum(n_fast * n_slow); return grid; } std::optional ProcessGoniometer(std::string &name, CborValue &value) { float start = 0; float increment = 1; Coord axis = {1,0,0}; std::optional helical; std::optional screening_wedge; CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); if (key == "increment") increment = GetCBORFloat(map_value); else if (key == "start") start = GetCBORFloat(map_value); else if (key == "axis") axis = GetCoord(map_value); else if (key == "helical_step") helical = GetCoord(map_value); else if (key == "screening_wedge") screening_wedge = GetCBORFloat(map_value); else cbor_value_advance(&map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); if (increment == 0) return {}; else { auto g = GoniometerAxis(name, start, increment, axis, helical); g.ScreeningWedge(screening_wedge); return g; } } void ProcessROIConfig(StartMessage &message, const nlohmann::json &j) { if (!j.is_array()) return; for (const auto &jr: j) { ROIConfig cfg; try { cfg.name = jr["name"]; if (jr["type"] == "box") { cfg.type = ROIConfig::ROIType::Box; cfg.box.xmin = jr["xmin"]; cfg.box.xmax = jr["xmax"]; cfg.box.ymin = jr["ymin"]; cfg.box.ymax = jr["ymax"]; message.rois.emplace_back(std::move(cfg)); } else if (jr["type"] == "circle") { cfg.type = ROIConfig::ROIType::Circle; cfg.circle.r = jr["r"]; cfg.circle.x = jr["x"]; cfg.circle.y = jr["y"]; message.rois.emplace_back(std::move(cfg)); } else if (jr["type"] == "azim") { cfg.type = ROIConfig::ROIType::Azim; cfg.azim.qmin = jr["qmin"]; cfg.azim.qmax = jr["qmax"]; message.rois.emplace_back(std::move(cfg)); } else continue; } catch (...) { // For now ignore things that are incorrect } } } void ProcessGoniometerMap(StartMessage &message, CborValue &value) { CborValue map_value; if (GetCBORMapLen(value) > 1) throw JFJochException(JFJochExceptionCategory::CBORError, "Max one rotation angle allowed"); cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); message.goniometer = ProcessGoniometer(key, map_value); } cborErr(cbor_value_leave_container(&value, &map_value)); } void ProcessPixelMaskElement(StartMessage &message, CborValue &value) { // For calibration and mask - image might be accessed later on - so it is saved in internal storage of the object // It allows the original start message to be lost CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); CompressedImage image = GetCBORMultidimTypedArray(map_value); image.Channel(key); if (image.GetMode() != CompressedImageMode::Uint32) continue; std::vector mask_uncompressed; JFJochDecompress(mask_uncompressed, image.GetCompressionAlgorithm(), image.GetCompressed(), image.GetCompressedSize(), image.GetWidth() * image.GetHeight()); message.pixel_mask[key] = mask_uncompressed; } cborErr(cbor_value_leave_container(&value, &map_value)); } std::optional ProcessHDF5Format(int input) { auto tmp = static_cast(input); switch (tmp) { case FileWriterFormat::DataOnly: case FileWriterFormat::NXmxLegacy: case FileWriterFormat::NXmxVDS: case FileWriterFormat::CBF: case FileWriterFormat::TIFF: case FileWriterFormat::NoFile: return tmp; default: // don't throw exception, just do default return {}; } } void ProcessStartUserData(StartMessage &message, CborValue &value) { try { const std::string s = GetCBORString(value); auto j = nlohmann::json::parse(s); if (!j.is_object()) return; if (j.contains("file_prefix")) message.file_prefix = j["file_prefix"]; if (j.contains("images_per_file")) message.images_per_file = j["images_per_file"]; if (j.contains("user")) message.user_data = j["user"]; if (j.contains("sample_name")) message.sample_name = j["sample_name"]; if (j.contains("source_name")) message.source_name = j["source_name"]; if (j.contains("source_type")) message.source_type = j["source_type"]; if (j.contains("instrument_name")) message.instrument_name = j["instrument_name"]; if (j.contains("total_flux")) message.total_flux = j["total_flux"]; if (j.contains("attenuator_transmission")) message.attenuator_transmission = j["attenuator_transmission"]; if (j.contains("space_group_number")) message.space_group_number = j["space_group_number"]; if (j.contains("roi")) ProcessROIConfig(message, j["roi"]); if (j.contains("gain_file_names")) message.gain_file_names = j["gain_file_names"]; if (j.contains("write_master_file")) message.write_master_file = j["write_master_file"]; if (j.contains("data_reduction_factor_serialmx")) message.data_reduction_factor_serialmx = j["data_reduction_factor_serialmx"]; if (j.contains("experiment_group")) message.experiment_group = j["experiment_group"]; if (j.contains("jfjoch_release")) message.jfjoch_release = j["jfjoch_release"]; if (j.contains("socket_number")) message.socket_number = j["socket_number"]; if (j.contains("writer_notification_zmq_addr")) message.writer_notification_zmq_addr = j["writer_notification_zmq_addr"]; if (j.contains("bit_depth_readout")) message.bit_depth_readout = j["bit_depth_readout"]; if (j.contains("summation_mode")) message.summation_mode = j["summation_mode"]; if (j.contains("overwrite")) message.overwrite = j["overwrite"]; if (j.contains("xfel_pulse_id")) message.overwrite = j["xfel_pulse_id"]; if (j.contains("file_format")) message.file_format = ProcessHDF5Format(j["file_format"]); if (j.contains("poni_rot1")) message.poni_rot1 = j["poni_rot1"]; if (j.contains("poni_rot2")) message.poni_rot2 = j["poni_rot2"]; if (j.contains("poni_rot3")) message.poni_rot3 = j["poni_rot3"]; if (j.contains("detect_ice_rings")) message.detect_ice_rings = j["detect_ice_rings"]; if (j.contains("images_per_trigger")) message.images_per_trigger = j["images_per_trigger"]; if (j.contains("indexing_algorithm")) { if (j["indexing_algorithm"] == "fft") message.indexing_algorithm = IndexingAlgorithmEnum::FFT; else if (j["indexing_algorithm"] == "fftw") message.indexing_algorithm = IndexingAlgorithmEnum::FFTW; else if (j["indexing_algorithm"] == "ffbidx") message.indexing_algorithm = IndexingAlgorithmEnum::FFBIDX; else message.indexing_algorithm = IndexingAlgorithmEnum::None; } if (j.contains("geom_refinement_algorithm")) { if (j["geom_refinement_algorithm"] == "beam_center") message.geom_refinement_algorithm = GeomRefinementAlgorithmEnum::BeamCenter; else message.geom_refinement_algorithm = GeomRefinementAlgorithmEnum::None; } if (j.contains("ring_current_mA")) message.ring_current_mA = j["ring_current_mA"]; if (j.contains("sample_temperature_K")) message.sample_temperature_K = j["sample_temperature_K"]; } catch (const std::exception &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Cannot parse user_data as valid JSON " + std::string(e.what())); } } bool ProcessStartMessageElement(StartMessage &message, CborValue &value) { if (cbor_value_at_end(&value)) return false; const auto key = GetCBORString(value); try { if (key == "magic_number") CheckMagicNumber(value); else if (key == "beam_center_x") message.beam_center_x = GetCBORFloat(value); else if (key == "beam_center_y") message.beam_center_y = GetCBORFloat(value); else if (key == "detector_distance") message.detector_distance = GetCBORFloat(value); else if (key == "number_of_images") message.number_of_images = GetCBORUInt(value); else if (key == "countrate_correction_enabled") message.countrate_correction_enabled = GetCBORBool(value); else if (key == "flatfield_enabled") message.flatfield_enabled = GetCBORBool(value); else if (key == "image_size_x") message.image_size_x = GetCBORUInt(value); else if (key == "image_size_y") message.image_size_y = GetCBORUInt(value); else if (key == "incident_energy") message.incident_energy = GetCBORFloat(value); else if (key == "incident_wavelength") message.incident_wavelength = GetCBORFloat(value); else if (key == "frame_time") message.frame_time = GetCBORFloat(value); else if (key == "count_time") message.count_time = GetCBORFloat(value); else if (key == "saturation_value") message.saturation_value = GetCBORInt(value); else if (key == "error_value") message.error_value = GetCBORInt(value); else if (key == "pixel_size_x") message.pixel_size_x = GetCBORFloat(value); else if (key == "pixel_size_y") message.pixel_size_y = GetCBORFloat(value); else if (key == "sensor_thickness") message.sensor_thickness = GetCBORFloat(value); else if (key == "sensor_material") message.sensor_material = GetCBORString(value); else if (key == "detector_description") message.detector_description = GetCBORString(value); else if (key == "detector_serial_number") message.detector_serial_number = GetCBORString(value); else if (key == "series_unique_id") message.run_name = GetCBORString(value); else if (key == "series_id") message.run_number = GetCBORUInt(value); else if (key == "pixel_mask") ProcessPixelMaskElement(message, value); else if (key == "channels") GetCBORStringArray(value, message.channels); else if (key == "detector_translation") ProcessAxis(value, message.detector_translation); else if (key == "goniometer") ProcessGoniometerMap(message, value); else if (key == "grid_scan") message.grid_scan = ProcessGridScan(value); else if (key == "pixel_mask_enabled") message.pixel_mask_enabled = GetCBORBool(value); else if (key == "jungfrau_conversion_enabled") message.jungfrau_conversion_enabled = GetCBORBool(value); else if (key == "geometry_transformation_enabled") message.geometry_transformation_enabled = GetCBORBool(value); else if (key == "jungfrau_conversion_factor") message.jungfrau_conversion_factor = GetCBORFloat(value); else if (key == "arm_date") message.arm_date = GetCBORDateTime(value); else if (key == "fluorescence") message.fluorescence_spectrum = GetCBORFluorescenceSpectrum(value); else if (key == "user_data") ProcessStartUserData(message, value); else if (key == "unit_cell") message.unit_cell = ProcessUnitCellElement(value); else if (key == "max_spot_count") message.max_spot_count = GetCBORUInt(value); else if (key == "threshold_energy") message.threshold_energy = ProcessThresholdEnergy(value); else if (key == "image_dtype") { auto val = GetCBORString(value); if (val == "uint8") { message.pixel_signed = false; message.bit_depth_image = 8; } else if (val == "int8") { message.pixel_signed = true; message.bit_depth_image = 8; } else if (val == "uint16") { message.pixel_signed = false; message.bit_depth_image = 16; } else if (val == "int16") { message.pixel_signed = true; message.bit_depth_image = 16; } else if (val == "uint32") { message.pixel_signed = false; message.bit_depth_image = 32; } else if (val == "int32") { message.pixel_signed = true; message.bit_depth_image = 32; } } else if (key == "az_int_q_bin_count") message.az_int_q_bin_count = GetCBORInt(value); else if (key == "az_int_phi_bin_count") message.az_int_phi_bin_count = GetCBORInt(value); else if (key == "az_int_bin_to_q") GetCBORFloatArray(value, message.az_int_bin_to_q); else if (key == "az_int_bin_to_two_theta") GetCBORFloatArray(value, message.az_int_bin_to_two_theta); else if (key == "az_int_bin_to_phi") GetCBORFloatArray(value, message.az_int_bin_to_phi); else if (key == "summation") message.summation = GetCBORUInt(value); else if (key == "storage_cell_number") message.storage_cell_number = GetCBORUInt(value); else if (key == "storage_cell_delay") message.storage_cell_delay_ns = GetRational(value).first; else { if (cbor_value_is_tag(&value)) cbor_value_advance(&value); cbor_value_advance(&value); } return true; } catch (const JFJochException &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing start message, key " + key + ":" + e.what()); } } // Calibration bool ProcessCalibration(CompressedImage &message, CborValue &value) { if (cbor_value_at_end(&value)) return false; else { auto key = GetCBORString(value); try { if (key == "data") { message = ProcessImageData(value); } else { if (cbor_value_is_tag(&value)) cbor_value_advance(&value); cbor_value_advance(&value); } return true; } catch (const JFJochException &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing calibration message, key " + key + ":" + e.what()); } } } // EndMessage void ProcessRadIntResultElement(EndMessage &message, CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); std::vector val; GetCBORFloatArray(map_value, val); message.az_int_result[key] = val; } cborErr(cbor_value_leave_container(&value, &map_value)); } void ProcessADUHistogramElement(EndMessage &message, CborValue &value) { CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); while (!cbor_value_at_end(&map_value)) { auto key = GetCBORString(map_value); std::vector val; GetCBORUInt64Array(map_value, val); message.adu_histogram[key] = val; } cborErr(cbor_value_leave_container(&value, &map_value)); } bool ProcessEndMessageElement(EndMessage &message, CborValue &value) { if (cbor_value_at_end(&value)) return false; auto key = GetCBORString(value); try { if (key == "end_date") message.end_date = GetCBORString(value); else if (key == "series_unique_id") message.run_name = GetCBORString(value); else if (key == "series_id") message.run_number = GetCBORUInt(value); else if (key == "max_image_number") message.max_image_number = GetCBORUInt(value); else if (key == "images_collected") message.images_collected_count = GetCBORUInt(value); else if (key == "images_sent_to_write") message.images_sent_to_write_count = GetCBORUInt(value); else if (key == "max_receiver_delay") message.max_receiver_delay = GetCBORUInt(value); else if (key == "data_collection_efficiency") message.efficiency = GetCBORFloat(value); else if (key == "az_int_result") ProcessRadIntResultElement(message, value); else if (key == "adu_histogram") ProcessADUHistogramElement(message, value); else if (key == "adu_histogram_bin_width") message.adu_histogram_bin_width = GetCBORUInt(value); else if (key == "bkg_estimate") message.bkg_estimate = GetCBORFloat(value); else if (key == "indexing_rate") message.indexing_rate = GetCBORFloat(value); else cbor_value_advance(&value); return true; } catch (const JFJochException &e) { throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing end message, key " + key + ":" + e.what()); } } } std::shared_ptr CBORStream2Deserialize(const uint8_t *msg, size_t msg_size) { CborParser parser; CborValue value; cborErr(cbor_parser_init(msg, msg_size, 0, &parser, &value)); if (GetCBORTag(value) != CborSignatureTag) throw JFJochException(JFJochExceptionCategory::CBORError, "CBOR must start with dedicated tag"); if (!cbor_value_is_map(&value)) throw JFJochException(JFJochExceptionCategory::CBORError, "Serialized frame must be map in top level"); auto ret = std::make_shared(); CborValue map_value; cborErr(cbor_value_enter_container(&value, &map_value)); ret->msg_type = DecodeType(map_value); switch (ret->msg_type) { case CBORImageType::IMAGE: ret->data_message = DataMessage(); while (ProcessDataMessageElement(*ret->data_message, map_value)) {} break; case CBORImageType::CALIBRATION: { ret->calibration = CompressedImage(); while (ProcessCalibration(*ret->calibration, map_value)) {} break; } case CBORImageType::START: { ret->start_message = StartMessage(); while (ProcessStartMessageElement(*ret->start_message, map_value)) {} break; } case CBORImageType::END: ret->end_message = EndMessage(); while (ProcessEndMessageElement(*ret->end_message, map_value)) {} break; case CBORImageType::METADATA: ret->metadata = MetadataMessage(); while (ProcessMetadataMessageElement(*ret->metadata, map_value)) {} break; case CBORImageType::NONE: break; } cborErr(cbor_value_leave_container(&value, &map_value)); return ret; } std::shared_ptr CBORStream2Deserialize(const std::vector& msg) { return CBORStream2Deserialize(msg.data(), msg.size()); } std::shared_ptr CBORStream2Deserialize(const std::string& msg) { return CBORStream2Deserialize(reinterpret_cast(msg.data()), msg.size()); }