From 173198be40c8f31d2a72aabd9388e0615772fec9 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 8 May 2026 12:41:46 +0200 Subject: [PATCH] HDF5DataFile: Include File name explicitly + FileWriter: Handle NXmxIntegrated in a smarter way --- common/JFJochMessages.h | 1 + docs/CBOR.md | 1 + frame_serialize/CBORStream2Deserializer.cpp | 2 + frame_serialize/CBORStream2Serializer.cpp | 2 + writer/FileWriter.cpp | 58 ++++++++++++++------- writer/HDF5DataFile.cpp | 19 ++++--- writer/HDF5DataFile.h | 9 ++-- writer/HDF5NXmx.cpp | 8 ++- writer/HDF5NXmx.h | 1 + 9 files changed, 66 insertions(+), 35 deletions(-) diff --git a/common/JFJochMessages.h b/common/JFJochMessages.h index 259da9ff..e5abba6e 100644 --- a/common/JFJochMessages.h +++ b/common/JFJochMessages.h @@ -251,6 +251,7 @@ struct StartMessage { std::optional attenuator_transmission; std::optional write_master_file; + std::optional write_images; nlohmann::json user_data; diff --git a/docs/CBOR.md b/docs/CBOR.md index bb2b18e2..b8a785a6 100644 --- a/docs/CBOR.md +++ b/docs/CBOR.md @@ -96,6 +96,7 @@ There are minor differences at the moment: | | | type "azim": qmin, qmax (numbers) | | | - gain_file_names | Array(string) | Names of JUNGFRAU gain files used for the current detector | | | - write_master_file | bool | With multiple sockets, it selects which socket will provide master file | | +| - write_images | bool | Write images in the HDF5 file (if false, will only write metadata) | | | - data_reduction_factor_serialmx | uint64 | Data reduction factor for serial MX | | | - experiment_group | string | ID of instrument user, e.g., p-group (SLS/SwissFEL) or proposal number | | | - jfjoch_release | string | Jungfraujoch release number | | diff --git a/frame_serialize/CBORStream2Deserializer.cpp b/frame_serialize/CBORStream2Deserializer.cpp index 5013c556..e60188d4 100644 --- a/frame_serialize/CBORStream2Deserializer.cpp +++ b/frame_serialize/CBORStream2Deserializer.cpp @@ -1012,6 +1012,8 @@ namespace { ProcessROIConfig(message, j["roi"]); if (j.contains("gain_file_names")) message.gain_file_names = j["gain_file_names"]; + if (j.contains("write_images")) + message.write_images = j["write_images"]; if (j.contains("write_master_file")) message.write_master_file = j["write_master_file"]; if (j.contains("data_reduction_factor_serialmx")) diff --git a/frame_serialize/CBORStream2Serializer.cpp b/frame_serialize/CBORStream2Serializer.cpp index f2dc1a87..195e5e05 100644 --- a/frame_serialize/CBORStream2Serializer.cpp +++ b/frame_serialize/CBORStream2Serializer.cpp @@ -493,6 +493,8 @@ inline void CBOR_ENC_START_USER_DATA(CborEncoder& encoder, const char* key, j["gain_file_names"] = message.gain_file_names; if (message.write_master_file) j["write_master_file"] = message.write_master_file.value(); + if (message.write_images) + j["write_images"] = message.write_images.value(); if (message.data_reduction_factor_serialmx) j["data_reduction_factor_serialmx"] = message.data_reduction_factor_serialmx.value(); j["experiment_group"] = message.experiment_group; diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index c57119a7..7537bf76 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -71,26 +71,37 @@ void FileWriter::WriteHDF5(const DataMessage& msg) { if (msg.number < 0) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images"); - const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file; - const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file; - - if (closed_files.contains(file_number)) - return; - - if (files.size() <= file_number) - files.resize(file_number + 1); - - if (!files[file_number]) { - files[file_number] = std::make_unique(start_message, file_number, true); - if (format == FileWriterFormat::NXmxIntegrated && master_file) - files[file_number]->CreateFile(msg, master_file->GetFile()); - } - files[file_number]->Write(msg, image_number); - - if (files[file_number]->GetNumImages() == start_message.images_per_file) { - CloseFile(file_number); + if (format == FileWriterFormat::NXmxIntegrated && master_file) { + if (files.empty() ) + files.resize(1); + if (!files[0]) { + files[0] = std::make_unique(start_message, + 0, + HDF5Metadata::MasterFileName(start_message), + true); + files[0]->CreateFile(msg, master_file->GetFile()); + } + files[0]->Write(msg, msg.number); } else { - CloseOldFiles(static_cast(msg.number)); + const uint64_t file_number = (start_message.images_per_file == 0) ? 0 : msg.number / start_message.images_per_file; + const uint64_t image_number = (start_message.images_per_file == 0) ? msg.number : msg.number % start_message.images_per_file; + + if (closed_files.contains(file_number)) + return; + + if (files.size() <= file_number) + files.resize(file_number + 1); + + if (!files[file_number]) + files[file_number] = std::make_unique(start_message, file_number, + HDF5Metadata::DataFileName(start_message, file_number)); + files[file_number]->Write(msg, image_number); + + if (files[file_number]->GetNumImages() == start_message.images_per_file) { + CloseFile(file_number); + } else { + CloseOldFiles(static_cast(msg.number)); + } } } @@ -230,6 +241,15 @@ void FileWriter::WriteHDF5(const CompressedImage &msg) { void FileWriter::WriteHDF5(const EndMessage &msg) { if (master_file) { std::lock_guard lock(hdf5_mutex); + + if (format == FileWriterFormat::NXmxIntegrated) { + try { + CloseFile(0); + } catch (...) { + throw; + } + } + master_file->Finalize(msg); } } diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index 811b4675..bee8b278 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -20,10 +20,10 @@ #include "HDF5NXmx.h" #include "../common/time_utc.h" -HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number, bool write_images) : -write_images(write_images){ - file_number = in_file_number; - +HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename) : +filename(filename), +file_number(file_number), +write_images(msg.write_images.value_or(true)) { if (msg.overwrite.has_value()) overwrite = msg.overwrite.value(); @@ -31,9 +31,14 @@ write_images(write_images){ ypixel = 0; max_image_number = 0; nimages = 0; - filename = HDF5Metadata::DataFileName(msg, file_number); - image_low = file_number * msg.images_per_file; - images_per_file = msg.images_per_file; + + if (msg.file_format == FileWriterFormat::NXmxIntegrated) { + image_low = 0; + images_per_file = msg.number_of_images; + } else { + image_low = file_number * msg.images_per_file; + images_per_file = msg.images_per_file; + } timestamp.reserve(images_per_file); exptime.reserve(images_per_file); diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index 334208c2..aac77fb0 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -22,7 +22,10 @@ struct HDF5DataFileStatistics { }; class HDF5DataFile { - std::string filename; + const std::string filename; + const uint64_t file_number; + const bool write_images; + std::string tmp_filename; std::shared_ptr data_file = nullptr; @@ -45,12 +48,10 @@ class HDF5DataFile { bool closed = false; bool overwrite = false; - int64_t file_number; bool new_file = true; bool manage_file = false; - const bool write_images; public: - HDF5DataFile(const StartMessage &msg, uint64_t file_number, bool write_images); + HDF5DataFile(const StartMessage &msg, uint64_t file_number, const std::string &filename); ~HDF5DataFile(); std::optional Close(); void Write(const DataMessage& msg, uint64_t image_number); diff --git a/writer/HDF5NXmx.cpp b/writer/HDF5NXmx.cpp index b368808b..29f2a757 100644 --- a/writer/HDF5NXmx.cpp +++ b/writer/HDF5NXmx.cpp @@ -11,15 +11,13 @@ #include "../common/time_utc.h" #include "gemmi/symmetry.hpp" -namespace { - std::string GenFilename(const StartMessage &start) { - return fmt::format("{:s}_master.h5", start.file_prefix); - } +std::string HDF5Metadata::MasterFileName(const StartMessage &start) { + return fmt::format("{:s}_master.h5", start.file_prefix); } NXmx::NXmx(const StartMessage &start) : start_message(start), - filename(GenFilename(start)) { + filename(HDF5Metadata::MasterFileName(start)) { uint64_t tmp_suffix; try { if (!start.arm_date.empty()) diff --git a/writer/HDF5NXmx.h b/writer/HDF5NXmx.h index a02bfe2b..8bc0b36d 100644 --- a/writer/HDF5NXmx.h +++ b/writer/HDF5NXmx.h @@ -9,6 +9,7 @@ #include "HDF5Objects.h" namespace HDF5Metadata { + std::string MasterFileName(const StartMessage &msg); std::string DataFileName(const StartMessage &msg, int64_t file_number); }