From 831f261be1d5d7bb6ddc2f34eac04c339225743e Mon Sep 17 00:00:00 2001 From: Andrej Babic Date: Fri, 15 May 2020 18:12:46 +0200 Subject: [PATCH] Add logic for setting maximum chunk size --- core-buffer/include/buffer_config.hpp | 3 +++ core-buffer/src/WriterH5Writer.cpp | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core-buffer/include/buffer_config.hpp b/core-buffer/include/buffer_config.hpp index 705b360..a743b2f 100644 --- a/core-buffer/include/buffer_config.hpp +++ b/core-buffer/include/buffer_config.hpp @@ -73,6 +73,9 @@ namespace core_buffer { // Size of buffer between the receiving and writing part. const int WRITER_FASTQUEUE_N_SLOTS = 5; + // How large are metadata chunks in the HDF5. + const size_t WRITER_METADATA_CHUNK_N_IMAGES = 100; + // Number of pulses between each statistics print out. const size_t STATS_MODULO = 100; diff --git a/core-buffer/src/WriterH5Writer.cpp b/core-buffer/src/WriterH5Writer.cpp index eabc5d9..4fda3d2 100644 --- a/core-buffer/src/WriterH5Writer.cpp +++ b/core-buffer/src/WriterH5Writer.cpp @@ -33,7 +33,7 @@ WriterH5Writer::WriterH5Writer( uint compression_prop[] = {MODULE_N_PIXELS, BSHUF_H5_COMPRESS_LZ4}; H5Pset_filter(image_dataset_properties.getId(), BSHUF_H5FILTER, - H5Z_FLAG_MANDATORY, + H5Z_FLAG_OPTIONAL, 2, &(compression_prop[0])); @@ -46,7 +46,13 @@ WriterH5Writer::WriterH5Writer( hsize_t metadata_dataset_dims[] = {n_frames_, 1}; H5::DataSpace metadata_dataspace(2, metadata_dataset_dims); - hsize_t metadata_dataset_chunking[] = {100, 1}; + // Chunk cannot be larger than n_frames. + auto metadata_chunk_size = WRITER_METADATA_CHUNK_N_IMAGES; + if (n_frames < metadata_chunk_size) { + metadata_chunk_size = n_frames; + } + + hsize_t metadata_dataset_chunking[] = {metadata_chunk_size, 1}; H5::DSetCreatPropList metadata_dataset_properties; metadata_dataset_properties.setChunk(2, metadata_dataset_chunking);