HDF5DataFile: Include File name explicitly + FileWriter: Handle NXmxIntegrated in a smarter way

common/JFJochMessages.h

@@ -251,6 +251,7 @@ struct StartMessage {
     std::optional<float> attenuator_transmission;
 
     std::optional<bool> write_master_file;
+    std::optional<bool> write_images;
 
     nlohmann::json user_data;
 

docs/CBOR.md

@@ -96,6 +96,7 @@ There are minor differences at the moment:
 |                                  |                      | type "azim": qmin, qmax (numbers)                                                                                                 |                           |
 | - gain_file_names                | Array(string)        | Names of JUNGFRAU gain files used for the current detector                                                                        |                           |
 | - write_master_file              | bool                 | With multiple sockets, it selects which socket will provide master file                                                           |                           |
+| - write_images                   | bool                 | Write images in the HDF5 file (if false, will only write metadata)                                                                |                           |
 | - data_reduction_factor_serialmx | uint64               | Data reduction factor for serial MX                                                                                               |                           |
 | - experiment_group               | string               | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number                                                            |                           |
 | - jfjoch_release                 | string               | Jungfraujoch release number                                                                                                       |                           |

frame_serialize/CBORStream2Deserializer.cpp

@@ -1012,6 +1012,8 @@ namespace {
                 ProcessROIConfig(message, j["roi"]);
             if (j.contains("gain_file_names"))
                 message.gain_file_names = j["gain_file_names"];
+            if (j.contains("write_images"))
+                message.write_images = j["write_images"];
             if (j.contains("write_master_file"))
                 message.write_master_file = j["write_master_file"];
             if (j.contains("data_reduction_factor_serialmx"))

frame_serialize/CBORStream2Serializer.cpp

@@ -493,6 +493,8 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key,
     j["gain_file_names"] = message.gain_file_names;
     if (message.write_master_file)
         j["write_master_file"] = message.write_master_file.value();
+    if (message.write_images)
+        j["write_images"] = message.write_images.value();
     if (message.data_reduction_factor_serialmx)
         j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value();
     j["experiment_group"] = message.experiment_group;

writer/FileWriter.cpp

@@ -71,26 +71,37 @@ void FileWriter::WriteHDF5(const DataMessage& msg) {
     if (msg.number < 0)
         throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images");
 
-    const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file;
-    const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file;
-
-    if (closed_files.contains(file_number))
-        return;
-
-    if (files.size() <= file_number)
-        files.resize(file_number + 1);
-
-    if (!files[file_number]) {
-        files[file_number] = std::make_unique<HDF5DataFile>(start_message, file_number, true);
-        if (format == FileWriterFormat::NXmxIntegrated && master_file)
-            files[file_number]->CreateFile(msg, master_file->GetFile());
-    }
-    files[file_number]->Write(msg, image_number);
-
-    if (files[file_number]->GetNumImages() == start_message.images_per_file) {
-        CloseFile(file_number);
+    if (format == FileWriterFormat::NXmxIntegrated && master_file) {
+        if (files.empty() )
+            files.resize(1);
+        if (!files[0]) {
+            files[0] = std::make_unique<HDF5DataFile>(start_message,
+                0,
+            HDF5Metadata::MasterFileName(start_message),
+            true);
+            files[0]->CreateFile(msg, master_file->GetFile());
+        }
+        files[0]->Write(msg, msg.number);
     } else {
-        CloseOldFiles(static_cast<uint64_t>(msg.number));
+        const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file;
+        const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file;
+
+        if (closed_files.contains(file_number))
+            return;
+
+        if (files.size() <= file_number)
+            files.resize(file_number + 1);
+
+        if (!files[file_number])
+            files[file_number] = std::make_unique<HDF5DataFile>(start_message, file_number,
+                                                                HDF5Metadata::DataFileName(start_message, file_number));
+        files[file_number]->Write(msg, image_number);
+
+        if (files[file_number]->GetNumImages() == start_message.images_per_file) {
+            CloseFile(file_number);
+        } else {
+            CloseOldFiles(static_cast<uint64_t>(msg.number));
+        }
     }
 }
 
@@ -230,6 +241,15 @@ void FileWriter::WriteHDF5(const CompressedImage &msg) {
 void FileWriter::WriteHDF5(const EndMessage &msg) {
     if (master_file) {
         std::lock_guard<std::mutex> lock(hdf5_mutex);
+
+        if (format == FileWriterFormat::NXmxIntegrated) {
+            try {
+                CloseFile(0);
+            } catch (...) {
+                throw;
+            }
+        }
+
         master_file->Finalize(msg);
     }
 }

writer/HDF5DataFile.cpp

@@ -20,10 +20,10 @@
 #include "HDF5NXmx.h"
 #include "../common/time_utc.h"
 
-HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number, bool write_images) :
-write_images(write_images){
-    file_number = in_file_number;
-
+HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename) :
+filename(filename),
+file_number(file_number),
+write_images(msg.write_images.value_or(true)) {
     if (msg.overwrite.has_value())
         overwrite = msg.overwrite.value();
 
@@ -31,9 +31,14 @@ write_images(write_images){
     ypixel = 0;
     max_image_number = 0;
     nimages = 0;
-    filename = HDF5Metadata::DataFileName(msg, file_number);
-    image_low = file_number * msg.images_per_file;
-    images_per_file = msg.images_per_file;
+
+    if (msg.file_format == FileWriterFormat::NXmxIntegrated) {
+        image_low = 0;
+        images_per_file = msg.number_of_images;
+    } else {
+        image_low = file_number * msg.images_per_file;
+        images_per_file = msg.images_per_file;
+    }
 
     timestamp.reserve(images_per_file);
     exptime.reserve(images_per_file);

writer/HDF5DataFile.h

@@ -22,7 +22,10 @@ struct HDF5DataFileStatistics {
 };
 
 class HDF5DataFile {
-    std::string filename;
+    const std::string filename;
+    const uint64_t file_number;
+    const bool write_images;
+
     std::string tmp_filename;
 
     std::shared_ptr<HDF5File> data_file = nullptr;
@@ -45,12 +48,10 @@ class HDF5DataFile {
     bool closed = false;
 
     bool overwrite = false;
-    int64_t file_number;
     bool new_file = true;
     bool manage_file = false;
-    const bool write_images;
 public:
-    HDF5DataFile(const StartMessage &msg, uint64_t file_number, bool write_images);
+    HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename);
     ~HDF5DataFile();
     std::optional<HDF5DataFileStatistics> Close();
     void Write(const DataMessage& msg, uint64_t image_number);

writer/HDF5NXmx.cpp

@@ -11,15 +11,13 @@
 #include "../common/time_utc.h"
 #include "gemmi/symmetry.hpp"
 
-namespace {
-    std::string GenFilename(const StartMessage &start) {
-        return fmt::format("{:s}_master.h5", start.file_prefix);
-    }
+std::string HDF5Metadata::MasterFileName(const StartMessage &start) {
+    return fmt::format("{:s}_master.h5", start.file_prefix);
 }
 
 NXmx::NXmx(const StartMessage &start)
         : start_message(start),
-          filename(GenFilename(start)) {
+          filename(HDF5Metadata::MasterFileName(start)) {
     uint64_t tmp_suffix;
     try {
         if (!start.arm_date.empty())

writer/HDF5NXmx.h

@@ -9,6 +9,7 @@
 #include "HDF5Objects.h"
 
 namespace HDF5Metadata {
+    std::string MasterFileName(const StartMessage &msg);
     std::string DataFileName(const StartMessage &msg, int64_t file_number);
 }