jfjoch_writer: Reserve all metadata vectors when opening file

writer/HDF5DataFile.cpp

@@ -139,7 +139,7 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) {
     data_set = std::make_unique<HDF5DataSet>(*data_file, "/entry/data/data", data_type, data_space, dcpl);
     data_set->SetExtent({images_per_file, ypixel, xpixel});
     for (auto &p: plugins)
-        p->OpenFile(*data_file, msg);
+        p->OpenFile(*data_file, msg, images_per_file);
 }
 
 void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) {

writer/HDF5DataFilePlugin.h

@@ -11,7 +11,7 @@ class HDF5DataFilePlugin {
 protected:
     size_t max_image_number = 0;
 public:
-    virtual void OpenFile(HDF5File &data_file, const DataMessage& msg) = 0;
+    virtual void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) = 0;
     virtual void Write(const DataMessage& msg, uint64_t image_number) = 0;
     virtual void WriteFinal(HDF5File &data_file) = 0;
     virtual ~HDF5DataFilePlugin() = default;

writer/HDF5DataFilePluginAzInt.cpp

@@ -3,8 +3,6 @@
 
 #include "HDF5DataFilePluginAzInt.h"
 
-#define RESERVE_IMAGES 10000
-
 HDF5DataFilePluginAzInt::HDF5DataFilePluginAzInt(const StartMessage &message) :
 az_int_bin_to_q(message.az_int_bin_to_q),
 az_int_bin_to_two_theta(message.az_int_bin_to_two_theta),
@@ -13,7 +11,7 @@ azimuthal_bins(message.az_int_phi_bin_count.value_or(1)),
 q_bins(message.az_int_q_bin_count.value_or(1)){
 }
 
-void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg) {
+void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) {
     if (az_int_bin_to_q.empty() || q_bins <= 0 || azimuthal_bins <= 0)
         return;
 
@@ -26,7 +24,7 @@ void HDF5DataFilePluginAzInt::OpenFile(HDF5File &data_file, const DataMessage &m
     if (!az_int_bin_to_phi.empty())
         data_file.SaveVector("/entry/azint/bin_to_phi", az_int_bin_to_phi, dim);
 
-    az_int_image.reserve(RESERVE_IMAGES * azimuthal_bins * q_bins);
+    az_int_image.reserve(images_per_file * azimuthal_bins * q_bins);
     az_int_image.resize(msg.number * azimuthal_bins * q_bins);
 }
 

writer/HDF5DataFilePluginAzInt.h

@@ -18,7 +18,7 @@ class HDF5DataFilePluginAzInt : public HDF5DataFilePlugin {
     // std::unique_ptr<HDF5DataSet> dataset;
 public:
     explicit HDF5DataFilePluginAzInt(const StartMessage &start);
-    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage& msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 };

writer/HDF5DataFilePluginDetector.cpp

@@ -3,7 +3,17 @@
 
 #include "HDF5DataFilePluginDetector.h"
 
-void HDF5DataFilePluginDetector::OpenFile(HDF5File &in_data_file, const DataMessage &msg) {}
+void HDF5DataFilePluginDetector::OpenFile(HDF5File &in_data_file, const DataMessage &msg, size_t images_per_file) {
+    jf_info.reserve(images_per_file);
+    storage_cell.reserve(images_per_file);
+    receiver_aq_dev_delay.reserve(images_per_file);
+    receiver_free_buffers.reserve(images_per_file);
+    efficiency.reserve(images_per_file);
+    packets_received.reserve(images_per_file);
+    packets_expected.reserve(images_per_file);
+    pixel_sum.reserve(images_per_file);
+    processing_time.reserve(images_per_file);
+}
 
 void HDF5DataFilePluginDetector::Write(const DataMessage &msg, uint64_t image_number) {
     if (msg.jf_info.has_value())

writer/HDF5DataFilePluginDetector.h

@@ -18,7 +18,7 @@ class HDF5DataFilePluginDetector : public HDF5DataFilePlugin {
     AutoIncrVector<int64_t> pixel_sum;
     AutoIncrVector<float> processing_time;
 public:
-    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage& msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 };

writer/HDF5DataFilePluginImageStats.cpp

@@ -2,10 +2,12 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 #include "HDF5DataFilePluginImageStats.h"
-#define RESERVE_IMAGES 1000
 
-void HDF5DataFilePluginImageStats::OpenFile(HDF5File &data_file, const DataMessage &msg) {
-    max_value.reserve(RESERVE_IMAGES);
+void HDF5DataFilePluginImageStats::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) {
+    max_value.reserve(images_per_file);
+    min_value.reserve(images_per_file);
+    error_pixels.reserve(images_per_file);
+    saturated_pixels.reserve(images_per_file);
 }
 
 void HDF5DataFilePluginImageStats::Write(const DataMessage &msg, uint64_t image_number) {
@@ -23,7 +25,7 @@ void HDF5DataFilePluginImageStats::WriteFinal(HDF5File &data_file) {
     HDF5Group(data_file, "/entry/image").NXClass("NXcollection");
     if (!max_value.empty())
         data_file.SaveVector("/entry/image/max_value", max_value.vec());
-    if (!max_value.empty())
+    if (!min_value.empty())
         data_file.SaveVector("/entry/image/min_value", min_value.vec());
     if (!error_pixels.empty())
         data_file.SaveVector("/entry/image/error_pixels", error_pixels.vec());

writer/HDF5DataFilePluginImageStats.h

@@ -14,12 +14,11 @@ class HDF5DataFilePluginImageStats : public HDF5DataFilePlugin {
     AutoIncrVector<int64_t> error_pixels;
     AutoIncrVector<int64_t> saturated_pixels;
 public:
-    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage& msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 
     ~HDF5DataFilePluginImageStats() override = default;
 };
 
-
 #endif //JFJOCH_HDF5DATAFILEPLUGINIMAGESTATS_H

writer/HDF5DataFilePluginMX.cpp

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 #include "HDF5DataFilePluginMX.h"
-#define RESERVE_IMAGES 1000
 
 inline std::string to_bravais_code(const std::optional<LatticeMessage> &lm_opt) {
     if (!lm_opt.has_value()) return "";
@@ -50,38 +49,50 @@ HDF5DataFilePluginMX::HDF5DataFilePluginMX(const StartMessage &msg)
     : max_spots(msg.max_spot_count), indexing(msg.indexing_algorithm != IndexingAlgorithmEnum::None) {
 }
 
-void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg) {
-    bkg_estimate.reserve(RESERVE_IMAGES);
+void HDF5DataFilePluginMX::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) {
+    bkg_estimate.reserve(images_per_file);
 
     if (max_spots == 0)
         return;
 
-    spot_x.reserve(max_spots * RESERVE_IMAGES);
-    spot_y.reserve(max_spots * RESERVE_IMAGES);
-    spot_int.reserve(max_spots * RESERVE_IMAGES);
-    spot_indexed.reserve(max_spots * RESERVE_IMAGES);
-    spot_ice_ring.reserve(max_spots * RESERVE_IMAGES);
-    spot_h.reserve(max_spots * RESERVE_IMAGES);
-    spot_k.reserve(max_spots * RESERVE_IMAGES);
-    spot_l.reserve(max_spots * RESERVE_IMAGES);
-    spot_dist_ewald.reserve(max_spots * RESERVE_IMAGES);
+    spot_x.reserve(max_spots * images_per_file);
+    spot_y.reserve(max_spots * images_per_file);
+    spot_int.reserve(max_spots * images_per_file);
+    spot_indexed.reserve(max_spots * images_per_file);
+    spot_ice_ring.reserve(max_spots * images_per_file);
+    spot_h.reserve(max_spots * images_per_file);
+    spot_k.reserve(max_spots * images_per_file);
+    spot_l.reserve(max_spots * images_per_file);
+    spot_dist_ewald.reserve(max_spots * images_per_file);
 
-    beam_corr_x.reserve(RESERVE_IMAGES);
-    beam_corr_y.reserve(RESERVE_IMAGES);
+    spot_x.reserve(max_spots * images_per_file);
+    spot_y.reserve(max_spots * images_per_file);
+    spot_int.reserve(max_spots * images_per_file);
+    spot_ice_ring.reserve(max_spots * images_per_file);
+    spot_h.reserve(max_spots * images_per_file);
+    spot_k.reserve(max_spots * images_per_file);
+    spot_l.reserve(max_spots * images_per_file);
+    spot_dist_ewald.reserve(max_spots * images_per_file);
+    if (indexing)
+        spot_indexed.reserve(max_spots * images_per_file);
+    if (indexing)
+        indexed_lattice.reserve(images_per_file * 9);
 
-    npeaks.reserve(RESERVE_IMAGES);
-    strong_pixel_count.reserve(RESERVE_IMAGES);
-    indexed.reserve(RESERVE_IMAGES);
-    profile_radius.reserve(RESERVE_IMAGES);
-    mosaicity_deg.reserve(RESERVE_IMAGES);
-    b_factor.reserve(RESERVE_IMAGES);
-    indexed_lattice.reserve(9 * RESERVE_IMAGES);
-    resolution_estimate.reserve(RESERVE_IMAGES);
+    beam_corr_x.reserve(images_per_file);
+    beam_corr_y.reserve(images_per_file);
 
-    spot_count_total.reserve(RESERVE_IMAGES);
-    spot_count_ice.reserve(RESERVE_IMAGES);
-    spot_count_indexed.reserve(RESERVE_IMAGES);
-    spot_count_low_res.reserve(RESERVE_IMAGES);
+    npeaks.reserve(images_per_file);
+    strong_pixel_count.reserve(images_per_file);
+    indexed.reserve(images_per_file);
+    profile_radius.reserve(images_per_file);
+    mosaicity_deg.reserve(images_per_file);
+    b_factor.reserve(images_per_file);
+    resolution_estimate.reserve(images_per_file);
+
+    spot_count_total.reserve(images_per_file);
+    spot_count_ice.reserve(images_per_file);
+    spot_count_indexed.reserve(images_per_file);
+    spot_count_low_res.reserve(images_per_file);
 }
 
 void HDF5DataFilePluginMX::Write(const DataMessage &msg, uint64_t image_number) {

writer/HDF5DataFilePluginMX.h

@@ -51,7 +51,7 @@ class HDF5DataFilePluginMX : public HDF5DataFilePlugin {
     AutoIncrVector<std::string> bravais_lattice;
 public:
     explicit HDF5DataFilePluginMX(const StartMessage& msg);
-    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage& msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 };

writer/HDF5DataFilePluginROI.cpp

@@ -3,20 +3,31 @@
 
 #include "HDF5DataFilePluginROI.h"
 
-#define RESERVE_IMAGES 1000
+void HDF5DataFilePluginROI::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t in_images_per_file) {
+    images_per_file = in_images_per_file;
 
-void HDF5DataFilePluginROI::OpenFile(HDF5File &data_file, const DataMessage &msg) {}
+    for (const auto &r: msg.roi) {
+        auto &entry = roi_data[r.first];
+        entry.max.reserve(images_per_file);
+        entry.sum.reserve(images_per_file);
+        entry.sum_sq.reserve(images_per_file);
+        entry.npixel.reserve(images_per_file);
+        entry.x.reserve(images_per_file);
+        entry.y.reserve(images_per_file);
+    }
+}
 
 void HDF5DataFilePluginROI::Write(const DataMessage &msg, uint64_t image_number) {
     for (const auto &r: msg.roi) {
         if (!roi_data.contains(r.first)) {
-            roi_data[r.first].max.reserve(RESERVE_IMAGES);
-            roi_data[r.first].sum.reserve(RESERVE_IMAGES);
-            roi_data[r.first].sum_sq.reserve(RESERVE_IMAGES);
-            roi_data[r.first].npixel.reserve(RESERVE_IMAGES);
-            roi_data[r.first].x.reserve(RESERVE_IMAGES);
-            roi_data[r.first].y.reserve(RESERVE_IMAGES);
+            roi_data[r.first].max.reserve(images_per_file);
+            roi_data[r.first].sum.reserve(images_per_file);
+            roi_data[r.first].sum_sq.reserve(images_per_file);
+            roi_data[r.first].npixel.reserve(images_per_file);
+            roi_data[r.first].x.reserve(images_per_file);
+            roi_data[r.first].y.reserve(images_per_file);
         }
+
         roi_data[r.first].max[image_number] = r.second.max_count;
         roi_data[r.first].sum[image_number] = r.second.sum;
         roi_data[r.first].npixel[image_number] = r.second.pixels;

writer/HDF5DataFilePluginROI.h

@@ -19,8 +19,9 @@ struct ROIData {
 
 class HDF5DataFilePluginROI : public HDF5DataFilePlugin {
     std::map<std::string, ROIData> roi_data;
+    size_t images_per_file = 1;
 public:
-    void OpenFile(HDF5File &data_file, const DataMessage &msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage &msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 };

writer/HDF5DataFilePluginReflection.cpp

@@ -5,10 +5,9 @@
 #include "../include/spdlog/fmt/fmt.h"
 #include "../common/Reflection.h"
 
-void HDF5DataFilePluginReflection::OpenFile(HDF5File &data_file, const DataMessage &msg) {
+void HDF5DataFilePluginReflection::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) {
     reflection_group = std::make_unique<HDF5Group>(data_file, "/entry/reflections");
     reflection_group->NXClass("NXcollection");
-
 }
 
 void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_number) {
@@ -19,6 +18,21 @@ void HDF5DataFilePluginReflection::Write(const DataMessage &msg, uint64_t image_
     std::vector<float> I, sigma, d, lp;
     std::vector<float> image, phi, pred_x, pred_y, bkg, partiality, zeta;
 
+    h.reserve(msg.reflections.size());
+    k.reserve(msg.reflections.size());
+    l.reserve(msg.reflections.size());
+    I.reserve(msg.reflections.size());
+    sigma.reserve(msg.reflections.size());
+    d.reserve(msg.reflections.size());
+    pred_x.reserve(msg.reflections.size());
+    pred_y.reserve(msg.reflections.size());
+    bkg.reserve(msg.reflections.size());
+    lp.reserve(msg.reflections.size());
+    partiality.reserve(msg.reflections.size());
+    image.reserve(msg.reflections.size());
+    phi.reserve(msg.reflections.size());
+    zeta.reserve(msg.reflections.size());
+
     for (const auto &refl : msg.reflections) {
         image.emplace_back(refl.image_number);
         h.emplace_back(refl.h);

writer/HDF5DataFilePluginReflection.h

@@ -9,7 +9,7 @@
 class HDF5DataFilePluginReflection : public HDF5DataFilePlugin {
     std::unique_ptr<HDF5Group> reflection_group;
 public:
-    void OpenFile(HDF5File &data_file, const DataMessage &msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage &msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
     ~HDF5DataFilePluginReflection() override = default;

writer/HDF5DataFilePluginXFEL.cpp

@@ -3,7 +3,13 @@
 
 #include "HDF5DataFilePluginXFEL.h"
 
-void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg) {}
+void HDF5DataFilePluginXFEL::OpenFile(HDF5File &data_file, const DataMessage &msg, size_t images_per_file) {
+    if (msg.xfel_pulse_id.has_value()
+        && msg.xfel_event_code.has_value()) {
+        pulseid.reserve(images_per_file);
+        event_code.reserve(images_per_file);
+    }
+}
 
 void HDF5DataFilePluginXFEL::Write(const DataMessage &msg, uint64_t image_number) {
     if (!msg.xfel_pulse_id.has_value()

writer/HDF5DataFilePluginXFEL.h

@@ -11,7 +11,7 @@ class HDF5DataFilePluginXFEL : public HDF5DataFilePlugin {
     AutoIncrVector<uint64_t> pulseid;
     AutoIncrVector<uint32_t> event_code;
 public:
-    void OpenFile(HDF5File &data_file, const DataMessage& msg) override;
+    void OpenFile(HDF5File &data_file, const DataMessage& msg, size_t images_per_file) override;
     void Write(const DataMessage& msg, uint64_t image_number) override;
     void WriteFinal(HDF5File &data_file) override;
 };