v1.0.0-rc.146 (#56)

This is an UNSTABLE release. The release has significant modifications for data processing - in case of troubles go back to 1.0.0-rc.144.

jfjoch_process: Generate a dedicated file (_process.h5), which can be used as a replacement for the _master.h5 file for a reanalyzed dataset.
jfjoch_process: Improve the performance of scaling and merging, implement on the fly scaling.
jfjoch_writer: All final data analysis results are repopulated in the _master.h5 file.
jfjoch_scale: Dedicated tool for rescaling/merging existing data.
jfjoch_viewer: Fix bugs where pixel labels where displayed on a wrong pixel.

WARNING! Scaling and merging are experimental at the moment, and may not provide reasonable results for the time being.

Reviewed-on: #56

frame_serialize/CBORStream2Serializer.cpp

@@ -153,6 +153,23 @@ inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<fl
     cborErr(cbor_encode_byte_string(&encoder, (uint8_t *) v.data(), v.size() * sizeof(float)));
 }
 
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<uint8_t>& v) {
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encode_tag(&encoder, TagUnsignedInt8Bit));
+    cborErr(cbor_encode_byte_string(&encoder, v.data(), v.size() * sizeof(uint8_t)));
+}
+
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<int32_t>& v) {
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encode_tag(&encoder, TagSignedInt32BitLE));
+    cborErr(cbor_encode_byte_string(&encoder, reinterpret_cast<const uint8_t *>(v.data()), v.size() * sizeof(int32_t)));
+}
+
+inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<int64_t>& v) {
+    cborErr(cbor_encode_text_stringz(&encoder, key));
+    cborErr(cbor_encode_tag(&encoder, TagSignedInt64BitLE));
+    cborErr(cbor_encode_byte_string(&encoder, reinterpret_cast<const uint8_t *>(v.data()), v.size() * sizeof(int64_t)));
+}
 
 inline void CBOR_ENC(CborEncoder &encoder, const char* key, const std::vector<uint64_t>& v) {
     cborErr(cbor_encode_text_stringz(&encoder, key));
@@ -238,6 +255,8 @@ inline void CBOR_ENC(CborEncoder &encoder, const Reflection& r) {
     CBOR_ENC(mapEncoder, "phi", r.delta_phi_deg);
     CBOR_ENC(mapEncoder, "x", r.predicted_x);
     CBOR_ENC(mapEncoder, "y", r.predicted_y);
+    CBOR_ENC(mapEncoder, "obs_x", r.observed_x);
+    CBOR_ENC(mapEncoder, "obs_y", r.observed_y);
     CBOR_ENC(mapEncoder, "d", r.d);
     CBOR_ENC(mapEncoder, "I", r.I);
     CBOR_ENC(mapEncoder, "bkg", r.bkg);
@@ -247,7 +266,7 @@ inline void CBOR_ENC(CborEncoder &encoder, const Reflection& r) {
     CBOR_ENC(mapEncoder, "rlp", r.rlp);
     CBOR_ENC(mapEncoder, "partiality", r.partiality);
     CBOR_ENC(mapEncoder, "zeta", r.zeta);
-
+    CBOR_ENC(mapEncoder, "image_scale_corr", r.image_scale_corr);
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 }
 
@@ -493,6 +512,8 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
     j["gain_file_names"] = message.gain_file_names;
     if (message.write_master_file)
         j["write_master_file"] = message.write_master_file.value();
+    if (message.write_images)
+        j["write_images"] = message.write_images.value();
     if (message.data_reduction_factor_serialmx)
         j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value();
     j["experiment_group"] = message.experiment_group;
@@ -677,7 +698,31 @@ void CBORStream2Serializer::SerializeSequenceEnd(const EndMessage& message) {
     CBOR_ENC(mapEncoder, "rotation_lattice_type", message.rotation_lattice_type);
     if (message.rotation_lattice.has_value())
         CBOR_ENC(mapEncoder, "rotation_lattice", message.rotation_lattice->GetVector());
-    CBOR_ENC(mapEncoder, "image_scale_factor", message.scale_factor);
+
+    CBOR_ENC(mapEncoder, "data_collection_efficiency_image", message.data_collection_efficiency);
+    CBOR_ENC(mapEncoder, "spot_count", message.spot_count);
+    CBOR_ENC(mapEncoder, "spot_count_ice_ring", message.spot_count_ice_ring);
+    CBOR_ENC(mapEncoder, "spot_count_low_res", message.spot_count_low_res);
+    CBOR_ENC(mapEncoder, "spot_count_indexed", message.spot_count_indexed);
+    CBOR_ENC(mapEncoder, "image_indexed", message.image_indexed);
+    CBOR_ENC(mapEncoder, "v_bkg_estimate", message.v_bkg_estimate);
+    CBOR_ENC(mapEncoder, "profile_radius", message.profile_radius);
+    CBOR_ENC(mapEncoder, "mosaicity", message.mosaicity);
+    CBOR_ENC(mapEncoder, "bFactor", message.bFactor);
+    CBOR_ENC(mapEncoder, "resolution_estimate", message.resolution_estimate);
+    CBOR_ENC(mapEncoder, "min_viable_pixel_value", message.min_viable_pixel_value);
+    CBOR_ENC(mapEncoder, "max_viable_pixel_value", message.max_viable_pixel_value);
+    CBOR_ENC(mapEncoder, "saturated_pixel_count", message.saturated_pixel_count);
+    CBOR_ENC(mapEncoder, "error_pixel_count", message.error_pixel_count);
+    CBOR_ENC(mapEncoder, "image_scale_factor", message.image_scale_factor);
+    CBOR_ENC(mapEncoder, "image_scale_cc", message.image_scale_cc);
+    CBOR_ENC(mapEncoder, "image_scale_mosaicity", message.image_scale_mosaicity);
+    CBOR_ENC(mapEncoder, "image_scale_b_factor", message.image_scale_b_factor);
+    CBOR_ENC(mapEncoder, "integrated_reflections", message.integrated_reflections);
+    CBOR_ENC(mapEncoder, "niggli_class", message.niggli_class);
+    CBOR_ENC(mapEncoder, "pixel_sum", message.pixel_sum);
+    CBOR_ENC(mapEncoder, "unit_cell", message.unit_cell);
+
     cborErr(cbor_encoder_close_container(&encoder, &mapEncoder));
 
     curr_size = cbor_encoder_get_buffer_size(&encoder, buffer);
@@ -712,6 +757,8 @@ void CBORStream2Serializer::SerializeImageInternal(CborEncoder &mapEncoder, cons
     CBOR_ENC(mapEncoder, "integration_time", message.integration_time_s);
     CBOR_ENC(mapEncoder, "refinement_time", message.refinement_time_s);
     CBOR_ENC(mapEncoder, "preprocessing_time", message.preprocessing_time_s);
+    CBOR_ENC(mapEncoder, "compression_time", message.compression_time_s);
+    CBOR_ENC(mapEncoder, "image_scale_time", message.image_scale_time_s);
     CBOR_ENC(mapEncoder, "index_analysis_time", message.index_analysis_time_s);
     CBOR_ENC(mapEncoder, "indexing_unit_cell", message.indexing_unit_cell);
     CBOR_ENC(mapEncoder, "xfel_pulse_id", message.xfel_pulse_id);
@@ -739,6 +786,10 @@ void CBORStream2Serializer::SerializeImageInternal(CborEncoder &mapEncoder, cons
     CBOR_ENC(mapEncoder, "roi_integrals", message.roi);
     CBOR_ENC(mapEncoder, "beam_corr_x", message.beam_corr_x);
     CBOR_ENC(mapEncoder, "beam_corr_y", message.beam_corr_y);
+    CBOR_ENC(mapEncoder, "image_scale_factor", message.image_scale_factor);
+    CBOR_ENC(mapEncoder, "image_scale_mosaicity", message.image_scale_mosaicity);
+    CBOR_ENC(mapEncoder, "image_scale_b_factor", message.image_scale_b_factor);
+    CBOR_ENC(mapEncoder, "image_scale_cc", message.image_scale_cc);
     CBOR_ENC(mapEncoder, "user_data", message.user_data.dump());
 
     if (!metadata_only) {