diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index 88935b53..5e99db29 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -139,7 +139,7 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) { data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); data_set->SetExtent({images_per_file, ypixel, xpixel}); for (auto &p: plugins) - p->OpenFile(*data_file, msg); + p->OpenFile(*data_file, msg, images_per_file); } void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { diff --git a/writer/HDF5DataFilePlugin.h b/writer/HDF5DataFilePlugin.h index 7b1108c1..08b3fb2c 100644 --- a/writer/HDF5DataFilePlugin.h +++ b/writer/HDF5DataFilePlugin.h @@ -11,7 +11,7 @@ class HDF5DataFilePlugin { protected: size_t max_image_number = 0; public: - virtual void OpenFile(HDF5File &data_file, const DataMessage& msg) = 0; + virtual void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) = 0; virtual void Write(const DataMessage& msg, uint64_t image_number) = 0; virtual void WriteFinal(HDF5File &data_file) = 0; virtual ~HDF5DataFilePlugin() = default; diff --git a/writer/HDF5DataFilePluginAzInt.cpp b/writer/HDF5DataFilePluginAzInt.cpp index c32159c6..49847188 100644 --- a/writer/HDF5DataFilePluginAzInt.cpp +++ b/writer/HDF5DataFilePluginAzInt.cpp @@ -3,8 +3,6 @@ #include "HDF5DataFilePluginAzInt.h" -#define RESERVE_IMAGES 10000 - HDF5DataFilePluginAzInt::HDF5DataFilePluginAzInt(const StartMessage &message) : az_int_bin_to_q(message.az_int_bin_to_q), az_int_bin_to_two_theta(message.az_int_bin_to_two_theta), @@ -13,7 +11,7 @@ azimuthal_bins(message.az_int_phi_bin_count.value_or(1)), q_bins(message.az_int_q_bin_count.value_or(1)){ } -void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg) { +void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) { if (az_int_bin_to_q.empty() || q_bins <= 0 || azimuthal_bins <= 0) return; @@ -26,7 +24,7 @@ void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &m if (!az_int_bin_to_phi.empty()) data_file.SaveVector("/entry/azint/bin_to_phi", az_int_bin_to_phi, dim); - az_int_image.reserve(RESERVE_IMAGES * azimuthal_bins * q_bins); + az_int_image.reserve(images_per_file * azimuthal_bins * q_bins); az_int_image.resize(msg.number * azimuthal_bins * q_bins); } diff --git a/writer/HDF5DataFilePluginAzInt.h b/writer/HDF5DataFilePluginAzInt.h index 0f3bd407..dc4e3c9a 100644 --- a/writer/HDF5DataFilePluginAzInt.h +++ b/writer/HDF5DataFilePluginAzInt.h @@ -18,7 +18,7 @@ class HDF5DataFilePluginAzInt : public HDF5DataFilePlugin { // std::unique_ptr dataset; public: explicit HDF5DataFilePluginAzInt(const StartMessage &start); - void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; }; diff --git a/writer/HDF5DataFilePluginDetector.cpp b/writer/HDF5DataFilePluginDetector.cpp index ef3f86fc..8d50098a 100644 --- a/writer/HDF5DataFilePluginDetector.cpp +++ b/writer/HDF5DataFilePluginDetector.cpp @@ -3,7 +3,17 @@ #include "HDF5DataFilePluginDetector.h" -void HDF5DataFilePluginDetector::OpenFile(HDF5File &in_data_file, const DataMessage &msg) {} +void HDF5DataFilePluginDetector::OpenFile(HDF5File &in_data_file, const DataMessage &msg, size_t images_per_file) { + jf_info.reserve(images_per_file); + storage_cell.reserve(images_per_file); + receiver_aq_dev_delay.reserve(images_per_file); + receiver_free_buffers.reserve(images_per_file); + efficiency.reserve(images_per_file); + packets_received.reserve(images_per_file); + packets_expected.reserve(images_per_file); + pixel_sum.reserve(images_per_file); + processing_time.reserve(images_per_file); +} void HDF5DataFilePluginDetector::Write(const DataMessage &msg, uint64_t image_number) { if (msg.jf_info.has_value()) diff --git a/writer/HDF5DataFilePluginDetector.h b/writer/HDF5DataFilePluginDetector.h index 90253319..6d530a48 100644 --- a/writer/HDF5DataFilePluginDetector.h +++ b/writer/HDF5DataFilePluginDetector.h @@ -18,7 +18,7 @@ class HDF5DataFilePluginDetector : public HDF5DataFilePlugin { AutoIncrVector pixel_sum; AutoIncrVector processing_time; public: - void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; }; diff --git a/writer/HDF5DataFilePluginImageStats.cpp b/writer/HDF5DataFilePluginImageStats.cpp index 6ac02a71..74c8982e 100644 --- a/writer/HDF5DataFilePluginImageStats.cpp +++ b/writer/HDF5DataFilePluginImageStats.cpp @@ -2,10 +2,12 @@ // SPDX-License-Identifier: GPL-3.0-only #include "HDF5DataFilePluginImageStats.h" -#define RESERVE_IMAGES 1000 -void HDF5DataFilePluginImageStats::OpenFile(HDF5File &data_file, const DataMessage &msg) { - max_value.reserve(RESERVE_IMAGES); +void HDF5DataFilePluginImageStats::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) { + max_value.reserve(images_per_file); + min_value.reserve(images_per_file); + error_pixels.reserve(images_per_file); + saturated_pixels.reserve(images_per_file); } void HDF5DataFilePluginImageStats::Write(const DataMessage &msg, uint64_t image_number) { @@ -23,7 +25,7 @@ void HDF5DataFilePluginImageStats::WriteFinal(HDF5File &data_file) { HDF5Group(data_file, "/entry/image").NXClass("NXcollection"); if (!max_value.empty()) data_file.SaveVector("/entry/image/max_value", max_value.vec()); - if (!max_value.empty()) + if (!min_value.empty()) data_file.SaveVector("/entry/image/min_value", min_value.vec()); if (!error_pixels.empty()) data_file.SaveVector("/entry/image/error_pixels", error_pixels.vec()); diff --git a/writer/HDF5DataFilePluginImageStats.h b/writer/HDF5DataFilePluginImageStats.h index 12593098..0d4dc87e 100644 --- a/writer/HDF5DataFilePluginImageStats.h +++ b/writer/HDF5DataFilePluginImageStats.h @@ -14,12 +14,11 @@ class HDF5DataFilePluginImageStats : public HDF5DataFilePlugin { AutoIncrVector error_pixels; AutoIncrVector saturated_pixels; public: - void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; ~HDF5DataFilePluginImageStats() override = default; }; - #endif //JFJOCH_HDF5DATAFILEPLUGINIMAGESTATS_H diff --git a/writer/HDF5DataFilePluginMX.cpp b/writer/HDF5DataFilePluginMX.cpp index b5e403fd..e70f940b 100644 --- a/writer/HDF5DataFilePluginMX.cpp +++ b/writer/HDF5DataFilePluginMX.cpp @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only #include "HDF5DataFilePluginMX.h" -#define RESERVE_IMAGES 1000 inline std::string to_bravais_code(const std::optional &lm_opt) { if (!lm_opt.has_value()) return ""; @@ -50,38 +49,50 @@ HDF5DataFilePluginMX::HDF5DataFilePluginMX(const StartMessage &msg) : max_spots(msg.max_spot_count), indexing(msg.indexing_algorithm != IndexingAlgorithmEnum::None) { } -void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg) { - bkg_estimate.reserve(RESERVE_IMAGES); +void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) { + bkg_estimate.reserve(images_per_file); if (max_spots == 0) return; - spot_x.reserve(max_spots * RESERVE_IMAGES); - spot_y.reserve(max_spots * RESERVE_IMAGES); - spot_int.reserve(max_spots * RESERVE_IMAGES); - spot_indexed.reserve(max_spots * RESERVE_IMAGES); - spot_ice_ring.reserve(max_spots * RESERVE_IMAGES); - spot_h.reserve(max_spots * RESERVE_IMAGES); - spot_k.reserve(max_spots * RESERVE_IMAGES); - spot_l.reserve(max_spots * RESERVE_IMAGES); - spot_dist_ewald.reserve(max_spots * RESERVE_IMAGES); + spot_x.reserve(max_spots * images_per_file); + spot_y.reserve(max_spots * images_per_file); + spot_int.reserve(max_spots * images_per_file); + spot_indexed.reserve(max_spots * images_per_file); + spot_ice_ring.reserve(max_spots * images_per_file); + spot_h.reserve(max_spots * images_per_file); + spot_k.reserve(max_spots * images_per_file); + spot_l.reserve(max_spots * images_per_file); + spot_dist_ewald.reserve(max_spots * images_per_file); - beam_corr_x.reserve(RESERVE_IMAGES); - beam_corr_y.reserve(RESERVE_IMAGES); + spot_x.reserve(max_spots * images_per_file); + spot_y.reserve(max_spots * images_per_file); + spot_int.reserve(max_spots * images_per_file); + spot_ice_ring.reserve(max_spots * images_per_file); + spot_h.reserve(max_spots * images_per_file); + spot_k.reserve(max_spots * images_per_file); + spot_l.reserve(max_spots * images_per_file); + spot_dist_ewald.reserve(max_spots * images_per_file); + if (indexing) + spot_indexed.reserve(max_spots * images_per_file); + if (indexing) + indexed_lattice.reserve(images_per_file * 9); - npeaks.reserve(RESERVE_IMAGES); - strong_pixel_count.reserve(RESERVE_IMAGES); - indexed.reserve(RESERVE_IMAGES); - profile_radius.reserve(RESERVE_IMAGES); - mosaicity_deg.reserve(RESERVE_IMAGES); - b_factor.reserve(RESERVE_IMAGES); - indexed_lattice.reserve(9 * RESERVE_IMAGES); - resolution_estimate.reserve(RESERVE_IMAGES); + beam_corr_x.reserve(images_per_file); + beam_corr_y.reserve(images_per_file); - spot_count_total.reserve(RESERVE_IMAGES); - spot_count_ice.reserve(RESERVE_IMAGES); - spot_count_indexed.reserve(RESERVE_IMAGES); - spot_count_low_res.reserve(RESERVE_IMAGES); + npeaks.reserve(images_per_file); + strong_pixel_count.reserve(images_per_file); + indexed.reserve(images_per_file); + profile_radius.reserve(images_per_file); + mosaicity_deg.reserve(images_per_file); + b_factor.reserve(images_per_file); + resolution_estimate.reserve(images_per_file); + + spot_count_total.reserve(images_per_file); + spot_count_ice.reserve(images_per_file); + spot_count_indexed.reserve(images_per_file); + spot_count_low_res.reserve(images_per_file); } void HDF5DataFilePluginMX::Write(const DataMessage &msg, uint64_t image_number) { diff --git a/writer/HDF5DataFilePluginMX.h b/writer/HDF5DataFilePluginMX.h index 8e1b3b3c..7298391d 100644 --- a/writer/HDF5DataFilePluginMX.h +++ b/writer/HDF5DataFilePluginMX.h @@ -51,7 +51,7 @@ class HDF5DataFilePluginMX : public HDF5DataFilePlugin { AutoIncrVector bravais_lattice; public: explicit HDF5DataFilePluginMX(const StartMessage& msg); - void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; }; diff --git a/writer/HDF5DataFilePluginROI.cpp b/writer/HDF5DataFilePluginROI.cpp index 8589fb97..d3c58090 100644 --- a/writer/HDF5DataFilePluginROI.cpp +++ b/writer/HDF5DataFilePluginROI.cpp @@ -3,20 +3,31 @@ #include "HDF5DataFilePluginROI.h" -#define RESERVE_IMAGES 1000 +void HDF5DataFilePluginROI::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t in_images_per_file) { + images_per_file = in_images_per_file; -void HDF5DataFilePluginROI::OpenFile(HDF5File &data_file, const DataMessage &msg) {} + for (const auto &r: msg.roi) { + auto &entry = roi_data[r.first]; + entry.max.reserve(images_per_file); + entry.sum.reserve(images_per_file); + entry.sum_sq.reserve(images_per_file); + entry.npixel.reserve(images_per_file); + entry.x.reserve(images_per_file); + entry.y.reserve(images_per_file); + } +} void HDF5DataFilePluginROI::Write(const DataMessage &msg, uint64_t image_number) { for (const auto &r: msg.roi) { if (!roi_data.contains(r.first)) { - roi_data[r.first].max.reserve(RESERVE_IMAGES); - roi_data[r.first].sum.reserve(RESERVE_IMAGES); - roi_data[r.first].sum_sq.reserve(RESERVE_IMAGES); - roi_data[r.first].npixel.reserve(RESERVE_IMAGES); - roi_data[r.first].x.reserve(RESERVE_IMAGES); - roi_data[r.first].y.reserve(RESERVE_IMAGES); + roi_data[r.first].max.reserve(images_per_file); + roi_data[r.first].sum.reserve(images_per_file); + roi_data[r.first].sum_sq.reserve(images_per_file); + roi_data[r.first].npixel.reserve(images_per_file); + roi_data[r.first].x.reserve(images_per_file); + roi_data[r.first].y.reserve(images_per_file); } + roi_data[r.first].max[image_number] = r.second.max_count; roi_data[r.first].sum[image_number] = r.second.sum; roi_data[r.first].npixel[image_number] = r.second.pixels; diff --git a/writer/HDF5DataFilePluginROI.h b/writer/HDF5DataFilePluginROI.h index 98ed9eba..00bbe492 100644 --- a/writer/HDF5DataFilePluginROI.h +++ b/writer/HDF5DataFilePluginROI.h @@ -19,8 +19,9 @@ struct ROIData { class HDF5DataFilePluginROI : public HDF5DataFilePlugin { std::map roi_data; + size_t images_per_file = 1; public: - void OpenFile(HDF5File &data_file, const DataMessage &msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage &msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; }; diff --git a/writer/HDF5DataFilePluginReflection.cpp b/writer/HDF5DataFilePluginReflection.cpp index 03c70a78..7f8780ef 100644 --- a/writer/HDF5DataFilePluginReflection.cpp +++ b/writer/HDF5DataFilePluginReflection.cpp @@ -5,10 +5,9 @@ #include "../include/spdlog/fmt/fmt.h" #include "../common/Reflection.h" -void HDF5DataFilePluginReflection::OpenFile(HDF5File &data_file, const DataMessage &msg) { +void HDF5DataFilePluginReflection::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) { reflection_group = std::make_unique(data_file, "/entry/reflections"); reflection_group->NXClass("NXcollection"); - } void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_number) { @@ -19,6 +18,21 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_ std::vector I, sigma, d, lp; std::vector image, phi, pred_x, pred_y, bkg, partiality, zeta; + h.reserve(msg.reflections.size()); + k.reserve(msg.reflections.size()); + l.reserve(msg.reflections.size()); + I.reserve(msg.reflections.size()); + sigma.reserve(msg.reflections.size()); + d.reserve(msg.reflections.size()); + pred_x.reserve(msg.reflections.size()); + pred_y.reserve(msg.reflections.size()); + bkg.reserve(msg.reflections.size()); + lp.reserve(msg.reflections.size()); + partiality.reserve(msg.reflections.size()); + image.reserve(msg.reflections.size()); + phi.reserve(msg.reflections.size()); + zeta.reserve(msg.reflections.size()); + for (const auto &refl : msg.reflections) { image.emplace_back(refl.image_number); h.emplace_back(refl.h); diff --git a/writer/HDF5DataFilePluginReflection.h b/writer/HDF5DataFilePluginReflection.h index 6600100d..c43aa570 100644 --- a/writer/HDF5DataFilePluginReflection.h +++ b/writer/HDF5DataFilePluginReflection.h @@ -9,7 +9,7 @@ class HDF5DataFilePluginReflection : public HDF5DataFilePlugin { std::unique_ptr reflection_group; public: - void OpenFile(HDF5File &data_file, const DataMessage &msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage &msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; ~HDF5DataFilePluginReflection() override = default; diff --git a/writer/HDF5DataFilePluginXFEL.cpp b/writer/HDF5DataFilePluginXFEL.cpp index c1d1ac2d..f3efc7c1 100644 --- a/writer/HDF5DataFilePluginXFEL.cpp +++ b/writer/HDF5DataFilePluginXFEL.cpp @@ -3,7 +3,13 @@ #include "HDF5DataFilePluginXFEL.h" -void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg) {} +void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) { + if (msg.xfel_pulse_id.has_value() + && msg.xfel_event_code.has_value()) { + pulseid.reserve(images_per_file); + event_code.reserve(images_per_file); + } +} void HDF5DataFilePluginXFEL::Write(const DataMessage &msg, uint64_t image_number) { if (!msg.xfel_pulse_id.has_value() diff --git a/writer/HDF5DataFilePluginXFEL.h b/writer/HDF5DataFilePluginXFEL.h index 5579b959..c26a544d 100644 --- a/writer/HDF5DataFilePluginXFEL.h +++ b/writer/HDF5DataFilePluginXFEL.h @@ -11,7 +11,7 @@ class HDF5DataFilePluginXFEL : public HDF5DataFilePlugin { AutoIncrVector pulseid; AutoIncrVector event_code; public: - void OpenFile(HDF5File &data_file, const DataMessage& msg) override; + void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override; void Write(const DataMessage& msg, uint64_t image_number) override; void WriteFinal(HDF5File &data_file) override; };