From 99e252bc72caa5cf82b3c7fc7f5eda001567cf7e Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Wed, 4 Mar 2026 20:03:24 +0100 Subject: [PATCH] jfjoch_writer: Some (likely small) performance gains --- writer/FileWriter.cpp | 12 ++++++------ writer/FileWriter.h | 4 ++-- writer/HDF5DataFile.cpp | 22 +++++++++++++++++----- writer/HDF5DataFile.h | 2 +- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/writer/FileWriter.cpp b/writer/FileWriter.cpp index b4fb84ba..25ae1c38 100644 --- a/writer/FileWriter.cpp +++ b/writer/FileWriter.cpp @@ -15,6 +15,10 @@ FileWriter::FileWriter(const StartMessage &request) if (start_message.file_format) format = start_message.file_format.value(); + // defailt + if (start_message.images_per_file <= 0) + start_message.images_per_file = default_images_per_file; + CheckPath(start_message.file_prefix); MakeDirectory(start_message.file_prefix); if (start_message.write_master_file && start_message.write_master_file.value()) { @@ -66,12 +70,8 @@ void FileWriter::WriteHDF5(const DataMessage& msg) { if (msg.number < 0) throw JFJochException(JFJochExceptionCategory::ArrayOutOfBounds, "No support for negative images"); - uint64_t file_number = 0; - size_t image_number = msg.number; - if (start_message.images_per_file > 0) { - file_number = msg.number / start_message.images_per_file; - image_number = msg.number % start_message.images_per_file; - } + const uint64_t file_number = msg.number / start_message.images_per_file; + const uint64_t image_number = msg.number % start_message.images_per_file; if (closed_files.contains(file_number)) return; diff --git a/writer/FileWriter.h b/writer/FileWriter.h index 89768abe..30857a22 100644 --- a/writer/FileWriter.h +++ b/writer/FileWriter.h @@ -24,8 +24,8 @@ class FileWriter { std::unique_ptr cbf_writer; std::unordered_set closed_files; - uint64_t close_file_lag_images = 1000; - + constexpr static uint64_t close_file_lag_images = 1000; + constexpr static uint64_t default_images_per_file = 1000; void CreateHDF5MasterFile(const StartMessage& msg); void AddStats(const std::optional& s); void CloseFile(uint64_t file_number); diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index d4772f3e..88935b53 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -31,6 +31,12 @@ HDF5DataFile::HDF5DataFile(const StartMessage &msg, uint64_t in_file_number) { nimages = 0; filename = HDF5Metadata::DataFileName(msg, file_number); image_low = file_number * msg.images_per_file; + images_per_file = msg.images_per_file; + + timestamp.reserve(images_per_file); + exptime.reserve(images_per_file); + number.reserve(images_per_file); + uint64_t tmp_suffix; try { if (!msg.arm_date.empty()) @@ -63,6 +69,7 @@ std::optional HDF5DataFile::Close() { p->WriteFinal(*data_file); if (data_set) { + data_set->SetExtent({max_image_number + 1, ypixel, xpixel}); data_set ->Attr("image_nr_low", (int32_t) (image_low + 1)) .Attr("image_nr_high", (int32_t) (image_low + 1 + max_image_number)); @@ -98,8 +105,6 @@ HDF5DataFile::~HDF5DataFile() { void HDF5DataFile::CreateFile(const DataMessage& msg) { HDF5Dcpl dcpl; - bool pixel_signed; - HDF5DataType data_type(msg.image.GetMode()); xpixel = msg.image.GetWidth(); @@ -107,6 +112,10 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) { dcpl.SetCompression(msg.image.GetCompressionAlgorithm(), JFJochBitShuffleCompressor::DefaultBlockSize); dcpl.SetChunking( {1, ypixel, xpixel}); + + H5Pset_fill_time(dcpl.GetID(), H5D_FILL_TIME_NEVER); + H5Pset_alloc_time(dcpl.GetID(), H5D_ALLOC_TIME_INCR); + switch (msg.image.GetMode()) { case CompressedImageMode::Int8: dcpl.SetFillValue8(INT8_MIN); @@ -128,14 +137,18 @@ void HDF5DataFile::CreateFile(const DataMessage& msg) { HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel}); data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); + data_set->SetExtent({images_per_file, ypixel, xpixel}); for (auto &p: plugins) p->OpenFile(*data_file, msg); } void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { if (closed) - return; - + throw JFJochException(JFJochExceptionCategory::FileWriteError, + "Trying to write to already closed file"); + if (image_number >= images_per_file) + throw JFJochException(JFJochExceptionCategory::FileWriteError, + "Image number out of bounds"); bool new_file = false; if (!data_file) { @@ -145,7 +158,6 @@ void HDF5DataFile::Write(const DataMessage &msg, uint64_t image_number) { if (new_file || (static_cast(image_number) > max_image_number)) { max_image_number = image_number; - data_set->SetExtent({max_image_number+1, ypixel, xpixel}); timestamp.resize(max_image_number + 1); exptime.resize(max_image_number + 1); number.resize(max_image_number + 1); diff --git a/writer/HDF5DataFile.h b/writer/HDF5DataFile.h index 31a485fe..f2fe3bec 100644 --- a/writer/HDF5DataFile.h +++ b/writer/HDF5DataFile.h @@ -29,7 +29,7 @@ class HDF5DataFile { std::unique_ptr data_set = nullptr; std::unique_ptr data_set_image_number = nullptr; std::vector> plugins; - + size_t images_per_file; size_t xpixel; size_t ypixel; size_t max_image_number;