mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

* Enhancements for XFEL

Browse Source 
* Enhancements for EIGER
* Writer is more flexible and capable of handling DECTRIS data
This commit is contained in:
leonarski_f
2024-03-05 20:41:47 +01:00
parent 71d862b706
commit d315506633
165 changed files with 5440 additions and 2230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

501
frame_serialize/CBORStream2Deserializer.cpp
View File

@@ -4,17 +4,18 @@
#include "tinycbor/src/cbor.h"
#include "CborErr.h"
#include "CborUtil.h"
#include <nlohmann/json.hpp>
inline std::string GetCBORString(CborValue &value) {
if (!cbor_value_is_text_string(&value))
throw JFJochException(JFJochExceptionCategory::CBORError, "String expected");
char buf[256];
size_t len = 255;
cborErr(cbor_value_copy_text_string(&value, buf, &len, nullptr));
buf[len+1] = 0;
std::vector<char> s(16384);
size_t len = s.size() - 1;
cborErr(cbor_value_copy_text_string(&value, s.data(), &len, nullptr));
s[len+1] = 0;
cborErr(cbor_value_advance(&value));
return {buf};
return {s.data()};
}
inline int64_t GetCBORInt(CborValue &value) {
@@ -37,12 +38,23 @@ inline uint64_t GetCBORUInt(CborValue &value) {
}
inline float GetCBORFloat(CborValue &value) {
if (!cbor_value_is_float(&value) )
throw JFJochException(JFJochExceptionCategory::CBORError, "Float expected");
float tmp;
cborErr(cbor_value_get_float(&value, &tmp));
cborErr(cbor_value_advance(&value));
return tmp;
if (cbor_value_is_double(&value)) {
double tmp;
cborErr(cbor_value_get_double(&value, &tmp));
cborErr(cbor_value_advance(&value));
return static_cast<float>(tmp);
} else if (cbor_value_is_float(&value)) {
float tmp;
cborErr(cbor_value_get_float(&value, &tmp));
cborErr(cbor_value_advance(&value));
return tmp;
} else if (cbor_value_is_half_float(&value)) {
float tmp;
cborErr(cbor_value_get_half_float_as_float(&value, &tmp));
cborErr(cbor_value_advance(&value));
return tmp;
}
throw JFJochException(JFJochExceptionCategory::CBORError, "Float expected");
}
inline bool GetCBORBool(CborValue &value) {
@@ -198,6 +210,17 @@ inline void GetCBORFloatArray(CborValue &value, std::vector<float> &v) {
memcpy(v.data(), ptr, len);
}
inline void GetCBORStringArray(CborValue &value, std::vector<std::string> &v) {
if (!cbor_value_is_array(&value))
throw JFJochException(JFJochExceptionCategory::CBORError, "Array expected");
CborValue array_value;
cborErr(cbor_value_enter_container(&value, &array_value));
while (! cbor_value_at_end(&array_value)) {
v.emplace_back(GetCBORString(array_value));
}
cborErr(cbor_value_leave_container(&value, &array_value));
}
inline void GetCBORUInt64Array(CborValue &value, std::vector<uint64_t> &v) {
if (GetCBORTag(value) != TagUnsignedInt64BitLE)
throw JFJochException(JFJochExceptionCategory::CBORError, "Incorrect array type tag");
@@ -382,7 +405,8 @@ void CBORStream2Deserializer::ProcessGoniometerMap(CborValue &value) {
cborErr(cbor_value_enter_container(&value, &map_value));
while (! cbor_value_at_end(&map_value)) {
auto key = GetCBORString(map_value);
start_message.goniometer[key] = ProcessGoniometer(map_value);
if (key == "omega")
ProcessGoniometerOmega(map_value);
}
cborErr(cbor_value_leave_container(&value, &map_value));
}
@@ -398,7 +422,7 @@ void CBORStream2Deserializer::ProcessAxis(CborValue &value, float v[3]) {
cborErr(cbor_value_leave_container(&value, &array_value));
}
GoniometerAxis CBORStream2Deserializer::ProcessGoniometer(CborValue &value) {
void CBORStream2Deserializer::ProcessGoniometerOmega(CborValue &value) {
CborValue map_value;
GoniometerAxis ret{};
@@ -406,16 +430,13 @@ GoniometerAxis CBORStream2Deserializer::ProcessGoniometer(CborValue &value) {
while (! cbor_value_at_end(&map_value)) {
auto key = GetCBORString(map_value);
if (key == "increment")
ret.increment = GetCBORFloat(map_value);
start_message.omega.increment = GetCBORFloat(map_value);
else if (key == "start")
ret.start = GetCBORFloat(map_value);
else if (key == "axis")
ProcessAxis(map_value, ret.axis);
start_message.omega.start = GetCBORFloat(map_value);
else
cbor_value_advance(&value);
}
cborErr(cbor_value_leave_container(&value, &map_value));
return ret;
}
bool CBORStream2Deserializer::ProcessImageMessageElement(CborValue &value) {
@@ -423,68 +444,66 @@ bool CBORStream2Deserializer::ProcessImageMessageElement(CborValue &value) {
return false;
else {
auto key = GetCBORString(value);
if (key == "image_id")
data_message.number = GetCBORInt(value);
else if (key == "data")
ProcessImageData(value);
else if (key == "series_unique_id")
data_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
data_message.series_id = GetCBORUInt(value);
else if (key == "real_time") {
auto r = GetRational(value);
data_message.exptime = r.first;
data_message.exptime_base = r.second;
} else if (key == "start_time") {
auto r = GetRational(value);
data_message.timestamp = r.first;
data_message.timestamp_base = r.second;
} else if (key == "user_data")
data_message.user_data = GetCBORString(value);
else if (key == "spots")
GetCBORSpots(value);
else if (key == "az_int_profile")
GetCBORFloatArray(value, data_message.az_int_profile);
else if (key == "indexing_result")
data_message.indexing_result = GetCBORUInt(value);
else if (key == "indexing_lattice")
GetCBORFloatArray(value, data_message.indexing_lattice);
else if (key == "jf_info")
data_message.jf_info = GetCBORUInt(value) & UINT32_MAX;
else if (key == "receiver_aq_dev_delay")
data_message.receiver_aq_dev_delay = GetCBORInt(value);
else if (key == "storage_cell")
data_message.storage_cell = GetCBORUInt(value) & UINT32_MAX;
else if (key == "xfel_bunch_id")
data_message.xfel_bunch_id = GetCBORUInt(value);
else if (key == "xfel_event_code")
data_message.xfel_event_code = GetCBORUInt(value);
else if (key == "saturated_pixel_count")
data_message.saturated_pixel_count = GetCBORUInt(value);
else if (key == "error_pixel_count")
data_message.error_pixel_count = GetCBORUInt(value);
else if (key == "strong_pixel_count")
data_message.strong_pixel_count = GetCBORUInt(value);
else if (key == "data_collection_efficiency")
data_message.image_collection_efficiency = GetCBORFloat(value);
else if (key == "bkg_estimate")
data_message.bkg_estimate = GetCBORFloat(value);
else if (key == "adu_histogram")
GetCBORUInt64Array(value, data_message.adu_histogram);
else if (key == "beam_center_x")
data_message.beam_center_x = GetCBORFloat(value);
else if (key == "beam_center_y")
data_message.beam_center_y = GetCBORFloat(value);
else if (key == "detector_distance")
data_message.detector_distance = GetCBORFloat(value);
else if (key == "incident_energy")
data_message.incident_energy = GetCBORFloat(value);
else if (key == "roi_sum")
data_message.roi_sum = GetCBORInt(value);
else
cbor_value_advance(&value);
return true;
try {
if (key == "image_id")
data_message.number = GetCBORInt(value);
else if (key == "data")
ProcessImageData(value);
else if (key == "series_unique_id")
data_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
data_message.series_id = GetCBORUInt(value);
else if (key == "real_time") {
auto r = GetRational(value);
data_message.exptime = r.first;
data_message.exptime_base = r.second;
} else if (key == "start_time") {
auto r = GetRational(value);
data_message.timestamp = r.first;
data_message.timestamp_base = r.second;
} else if (key == "user_data")
data_message.user_data = GetCBORString(value);
else if (key == "spots")
GetCBORSpots(value);
else if (key == "az_int_profile")
GetCBORFloatArray(value, data_message.az_int_profile);
else if (key == "indexing_result")
data_message.indexing_result = GetCBORUInt(value);
else if (key == "indexing_lattice")
GetCBORFloatArray(value, data_message.indexing_lattice);
else if (key == "jf_info")
data_message.jf_info = GetCBORUInt(value) & UINT32_MAX;
else if (key == "receiver_aq_dev_delay")
data_message.receiver_aq_dev_delay = GetCBORInt(value);
else if (key == "storage_cell")
data_message.storage_cell = GetCBORUInt(value) & UINT32_MAX;
else if (key == "xfel_pulse_id")
data_message.xfel_pulse_id = GetCBORUInt(value);
else if (key == "xfel_event_code")
data_message.xfel_event_code = GetCBORUInt(value);
else if (key == "saturated_pixel_count")
data_message.saturated_pixel_count = GetCBORUInt(value);
else if (key == "error_pixel_count")
data_message.error_pixel_count = GetCBORUInt(value);
else if (key == "strong_pixel_count")
data_message.strong_pixel_count = GetCBORUInt(value);
else if (key == "data_collection_efficiency")
data_message.image_collection_efficiency = GetCBORFloat(value);
else if (key == "bkg_estimate")
data_message.bkg_estimate = GetCBORFloat(value);
else if (key == "adu_histogram")
GetCBORUInt64Array(value, data_message.adu_histogram);
else if (key == "roi_sum")
data_message.roi_sum = GetCBORInt(value);
else {
if (cbor_value_is_tag(&value))
cbor_value_advance(&value);
cbor_value_advance(&value);
}
return true;
} catch (const JFJochException &e) {
throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing image message, key " + key + ":" + e.what());
}
}
}
@@ -585,123 +604,159 @@ void CBORStream2Deserializer::ProcessUnitCellElement(CborValue &value) {
start_message.unit_cell = unit_cell;
}
void CBORStream2Deserializer::ProcessStartUserData(CborValue &value) {
try {
const std::string s = GetCBORString(value);
auto j = nlohmann::json::parse(s);
if (!j.is_object())
return;
if (j.contains("file_prefix"))
start_message.file_prefix = j["file_prefix"];
if (j.contains("data_file_count"))
start_message.data_file_count = j["data_file_count"];
if (j.contains("user"))
start_message.user_data = j["user"];
if (j.contains("sample_name"))
start_message.sample_name = j["sample_name"];
if (j.contains("source_name"))
start_message.source_name = j["source_name"];
if (j.contains("source_name_short"))
start_message.source_name_short = j["source_name_short"];
if (j.contains("instrument_name"))
start_message.instrument_name = j["instrument_name"];
if (j.contains("instrument_name_short"))
start_message.instrument_name_short = j["instrument_name_short"];
if (j.contains("total_flux"))
start_message.total_flux = j["total_flux"];
if (j.contains("attenuator_transmission"))
start_message.attenuator_transmission = j["attenuator_transmission"];
if (j.contains("omega_rotation_axis") && j["omega_rotation_axis"].is_array() &&
(j["omega_rotation_axis"].size() == 3)) {
for (int i = 0; i < 3; i++)
start_message.omega.axis[i] = j["omega_rotation_axis"][i];
}
if (j.contains("space_group_number"))
start_message.space_group_number = j["space_group_number"];
} catch (const std::exception &e) {
throw JFJochException(JFJochExceptionCategory::CBORError, "Cannot parse user_data as valid JSON " + std::string(e.what()));
}
}
bool CBORStream2Deserializer::ProcessStartMessageElement(CborValue &value) {
if (cbor_value_at_end(&value))
return false;
else {
auto key = GetCBORString(value);
if (key == "beam_center_x")
start_message.beam_center_x = GetCBORFloat(value);
else if (key == "beam_center_y")
start_message.beam_center_y = GetCBORFloat(value);
else if (key == "detector_distance")
start_message.detector_distance = GetCBORFloat(value);
else if (key == "number_of_images")
start_message.number_of_images = GetCBORUInt(value);
else if (key == "image_size_x")
start_message.image_size_x = GetCBORUInt(value);
else if (key == "image_size_y")
start_message.image_size_y = GetCBORUInt(value);
else if (key == "incident_energy")
start_message.incident_energy = GetCBORFloat(value);
else if (key == "incident_wavelength")
start_message.incident_wavelength = GetCBORFloat(value);
else if (key == "frame_time")
start_message.frame_time = GetCBORFloat(value);
else if (key == "count_time")
start_message.count_time = GetCBORFloat(value);
else if (key == "saturation_value")
start_message.saturation_value = GetCBORInt(value);
else if (key == "error_value")
start_message.error_value = GetCBORInt(value);
else if (key == "pixel_size_x")
start_message.pixel_size_x = GetCBORFloat(value);
else if (key == "pixel_size_y")
start_message.pixel_size_y = GetCBORFloat(value);
else if (key == "sensor_thickness")
start_message.sensor_thickness = GetCBORFloat(value);
else if (key == "sensor_material")
start_message.sensor_material = GetCBORString(value);
else if (key == "detector_description")
start_message.detector_description = GetCBORString(value);
else if (key == "detector_serial_number")
start_message.detector_serial_number = GetCBORString(value);
else if (key == "series_unique_id")
start_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
start_message.series_id = GetCBORUInt(value);
else if (key == "pixel_mask")
ProcessPixelMaskElement(value);
else if (key == "channels")
ProcessChannels(value);
else if (key == "detector_translation")
ProcessAxis(value, start_message.detector_translation);
else if (key == "goniometer")
ProcessGoniometerMap(value);
else if (key == "pixel_mask_enabled")
start_message.pixel_mask_enabled = GetCBORBool(value);
else if (key == "arm_date")
start_message.arm_date = GetCBORDateTime(value);
else if (key == "user_data")
start_message.user_data = GetCBORString(value);
else if (key == "space_group_number")
start_message.space_group_number = GetCBORUInt(value);
else if (key == "unit_cell")
ProcessUnitCellElement(value);
else if (key == "sample_name")
start_message.sample_name = GetCBORString(value);
else if (key == "source_name")
start_message.source_name = GetCBORString(value);
else if (key == "source_name_short")
start_message.source_name_short = GetCBORString(value);
else if (key == "instrument_name")
start_message.instrument_name = GetCBORString(value);
else if (key == "instrument_name_short")
start_message.instrument_name_short = GetCBORString(value);
else if (key == "file_prefix")
start_message.file_prefix = GetCBORString(value);
else if (key == "max_spot_count")
start_message.max_spot_count = GetCBORUInt(value);
else if (key == "data_file_count")
start_message.data_file_count = GetCBORUInt(value);
else if (key == "pixel_bit_depth")
start_message.pixel_bit_depth = GetCBORUInt(value);
else if (key == "pixel_signed")
start_message.pixel_signed = GetCBORBool(value);
else if (key == "az_int_bin_number")
start_message.az_int_bin_number = GetCBORInt(value);
else if (key == "az_int_bin_to_q")
GetCBORFloatArray(value, start_message.az_int_bin_to_q);
else if (key == "summation")
start_message.summation = GetCBORUInt(value);
else if (key == "storage_cell_number")
start_message.storage_cell_number = GetCBORUInt(value);
else if (key == "storage_cell_delay")
start_message.storage_cell_delay_ns = GetRational(value).first;
else if (key == "total_flux")
start_message.total_flux = GetCBORFloat(value);
else if (key == "attenuator_transmission")
start_message.attenuator_transmission = GetCBORFloat(value);
else if (key == "roi_sum_area")
start_message.roi_summation_area = GetROIRectangle(value);
else if (key == "compression_algorithm") {
auto tmp = GetCBORString(value);
if (tmp == "bslz4")
start_message.compression_algorithm = CompressionAlgorithm::BSHUF_LZ4;
else if (tmp == "bszstd")
start_message.compression_algorithm = CompressionAlgorithm::BSHUF_ZSTD;
else if (tmp == "none")
start_message.compression_algorithm = CompressionAlgorithm::NO_COMPRESSION;
else
throw JFJochException(JFJochExceptionCategory::CBORError, "Unsupported compression");
} else if (key == "compression_block_size")
start_message.compression_block_size = GetCBORUInt(value);
else if (key == "calibration")
ProcessCalibration(value);
else
cbor_value_advance(&value);
return true;
try {
if (key == "beam_center_x")
start_message.beam_center_x = GetCBORFloat(value);
else if (key == "beam_center_y")
start_message.beam_center_y = GetCBORFloat(value);
else if (key == "detector_distance")
start_message.detector_distance = GetCBORFloat(value);
else if (key == "number_of_images")
start_message.number_of_images = GetCBORUInt(value);
else if (key == "countrate_correction_enabled")
start_message.countrate_correction_enabled = GetCBORBool(value);
else if (key == "flatfield_enabled")
start_message.flatfield_enabled = GetCBORBool(value);
else if (key == "image_size_x")
start_message.image_size_x = GetCBORUInt(value);
else if (key == "image_size_y")
start_message.image_size_y = GetCBORUInt(value);
else if (key == "incident_energy")
start_message.incident_energy = GetCBORFloat(value);
else if (key == "incident_wavelength")
start_message.incident_wavelength = GetCBORFloat(value);
else if (key == "frame_time")
start_message.frame_time = GetCBORFloat(value);
else if (key == "count_time")
start_message.count_time = GetCBORFloat(value);
else if (key == "saturation_value")
start_message.saturation_value = GetCBORInt(value);
else if (key == "error_value")
start_message.error_value = GetCBORInt(value);
else if (key == "pixel_size_x")
start_message.pixel_size_x = GetCBORFloat(value);
else if (key == "pixel_size_y")
start_message.pixel_size_y = GetCBORFloat(value);
else if (key == "sensor_thickness")
start_message.sensor_thickness = GetCBORFloat(value);
else if (key == "sensor_material")
start_message.sensor_material = GetCBORString(value);
else if (key == "detector_description")
start_message.detector_description = GetCBORString(value);
else if (key == "detector_serial_number")
start_message.detector_serial_number = GetCBORString(value);
else if (key == "series_unique_id")
start_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
start_message.series_id = GetCBORUInt(value);
else if (key == "pixel_mask")
ProcessPixelMaskElement(value);
else if (key == "channels")
GetCBORStringArray(value, start_message.channels);
else if (key == "detector_translation")
ProcessAxis(value, start_message.detector_translation);
else if (key == "goniometer")
ProcessGoniometerMap(value);
else if (key == "pixel_mask_enabled")
start_message.pixel_mask_enabled = GetCBORBool(value);
else if (key == "arm_date")
start_message.arm_date = GetCBORDateTime(value);
else if (key == "user_data")
ProcessStartUserData(value);
else if (key == "unit_cell")
ProcessUnitCellElement(value);
else if (key == "max_spot_count")
start_message.max_spot_count = GetCBORUInt(value);
else if (key == "image_dtype") {
auto val = GetCBORString(value);
if (val == "uint8") {
start_message.pixel_signed = false;
start_message.pixel_bit_depth = 8;
} else if (val == "int8") {
start_message.pixel_signed = true;
start_message.pixel_bit_depth = 8;
} else if (val == "uint16") {
start_message.pixel_signed = false;
start_message.pixel_bit_depth = 16;
} else if (val == "int16") {
start_message.pixel_signed = true;
start_message.pixel_bit_depth = 16;
} else if (val == "uint32") {
start_message.pixel_signed = false;
start_message.pixel_bit_depth = 32;
} else if (val == "int32") {
start_message.pixel_signed = true;
start_message.pixel_bit_depth = 32;
}
} else if (key == "az_int_bin_number")
start_message.az_int_bin_number = GetCBORInt(value);
else if (key == "az_int_bin_to_q")
GetCBORFloatArray(value, start_message.az_int_bin_to_q);
else if (key == "summation")
start_message.summation = GetCBORUInt(value);
else if (key == "storage_cell_number")
start_message.storage_cell_number = GetCBORUInt(value);
else if (key == "storage_cell_delay")
start_message.storage_cell_delay_ns = GetRational(value).first;
else if (key == "roi_sum_area")
start_message.roi_summation_area = GetROIRectangle(value);
else if (key == "gain_file_names")
GetCBORStringArray(value, start_message.gain_file_names);
else if (key == "calibration")
ProcessCalibration(value);
else {
if (cbor_value_is_tag(&value))
cbor_value_advance(&value);
cbor_value_advance(&value);
}
return true;
} catch (const JFJochException &e) {
throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing start message, key " + key + ":" + e.what());
}
}
}
@@ -710,33 +765,37 @@ bool CBORStream2Deserializer::ProcessEndMessageElement(CborValue &value) {
return false;
else {
auto key = GetCBORString(value);
if (key == "end_date")
end_message.end_date = GetCBORString(value);
else if (key == "series_unique_id")
end_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
end_message.series_id = GetCBORUInt(value);
else if (key == "max_image_number")
end_message.max_image_number = GetCBORUInt(value);
else if (key == "images_collected")
end_message.images_collected_count = GetCBORUInt(value);
else if (key == "images_sent_to_write")
end_message.images_sent_to_write_count = GetCBORUInt(value);
else if (key == "max_receiver_delay")
end_message.max_receiver_delay = GetCBORUInt(value);
else if (key == "data_collection_efficiency")
end_message.efficiency = GetCBORFloat(value);
else if (key == "write_master_file")
end_message.write_master_file = GetCBORBool(value);
else if (key == "az_int_result")
ProcessRadIntResultElement(value);
else if (key == "adu_histogram")
ProcessADUHistogramElement(value);
else if (key == "adu_histogram_bin_width")
end_message.adu_histogram_bin_width = GetCBORUInt(value);
else
cbor_value_advance(&value);
return true;
try {
if (key == "end_date")
end_message.end_date = GetCBORString(value);
else if (key == "series_unique_id")
end_message.series_unique_id = GetCBORString(value);
else if (key == "series_id")
end_message.series_id = GetCBORUInt(value);
else if (key == "max_image_number")
end_message.max_image_number = GetCBORUInt(value);
else if (key == "images_collected")
end_message.images_collected_count = GetCBORUInt(value);
else if (key == "images_sent_to_write")
end_message.images_sent_to_write_count = GetCBORUInt(value);
else if (key == "max_receiver_delay")
end_message.max_receiver_delay = GetCBORUInt(value);
else if (key == "data_collection_efficiency")
end_message.efficiency = GetCBORFloat(value);
else if (key == "write_master_file")
end_message.write_master_file = GetCBORBool(value);
else if (key == "az_int_result")
ProcessRadIntResultElement(value);
else if (key == "adu_histogram")
ProcessADUHistogramElement(value);
else if (key == "adu_histogram_bin_width")
end_message.adu_histogram_bin_width = GetCBORUInt(value);
else
cbor_value_advance(&value);
return true;
} catch (const JFJochException &e) {
throw JFJochException(JFJochExceptionCategory::CBORError, "Error processing end message, key " + key + ":" + e.what());
}
}
}
@@ -785,9 +844,7 @@ std::unique_lock<std::mutex> ul(m);
break;
case Type::START:
start_message = StartMessage{
.data_file_count = 1,
.compression_algorithm = CompressionAlgorithm::NO_COMPRESSION,
.compression_block_size = 0
.data_file_count = 1
};
while (ProcessStartMessageElement(map_value));
break;