From aadba5b34304429ad9eaf57df3cdeb5180426476 Mon Sep 17 00:00:00 2001 From: leonarski_f Date: Fri, 26 Jun 2026 20:35:32 +0200 Subject: [PATCH] Compressor: bump bitshuffle block size 4096 -> 16384 elements On sparse lyso frames the larger block improves compression ratio across all bshuf algorithms (16-bit data): ZSTD 8.58 -> 9.30, LZ4 7.38 -> 7.58, RLE 6.82 -> 6.90. 16384 captures most of the gain available from even larger blocks (ZSTD tops out ~9.55 at 65536) while staying close to the cache sweet spot: the cheap codecs (LZ4, RLE) peak in throughput once a block's working set fits L1d (~4096 elem here), so very large blocks trade real throughput for diminishing ratio - and that penalty is worse on the Xeon Gold/Platinum production hosts (smaller private L2, shared-L3 contention under many parallel compression threads). The block size is stored per-dataset in the bitshuffle HDF5 filter params, so existing readers (XDS/Neggia/Durin/CrystFEL) stay compatible. Move the per-block bitshuffle scratch off the inline member array onto a lazily-sized heap vector, like tmp_space, so the block size no longer bloats every stack-allocated compressor (incl. the transient ones in CBORStream2Serializer). Co-Authored-By: Claude Opus 4.8 --- compression/JFJochCompressor.cpp | 4 +++- compression/JFJochCompressor.h | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compression/JFJochCompressor.cpp b/compression/JFJochCompressor.cpp index 1afa0177..c144abb1 100644 --- a/compression/JFJochCompressor.cpp +++ b/compression/JFJochCompressor.cpp @@ -37,7 +37,7 @@ JFJochBitShuffleCompressor::JFJochBitShuffleCompressor(CompressionAlgorithm in_a size_t JFJochBitShuffleCompressor::CompressBlock(char *dest, const char *source, size_t nelements, size_t elem_size) { // Assert nelements < block_size const char *src_ptr; - int64_t bshuf_ret = bitshuf_encode_block(tmp_space.data(), source, scratch, nelements, elem_size); + int64_t bshuf_ret = bitshuf_encode_block(tmp_space.data(), source, scratch.data(), nelements, elem_size); if (bshuf_ret < 0) throw JFJochException(JFJochExceptionCategory::Compression, "bshuf_trans_bit_elem error"); src_ptr = tmp_space.data(); @@ -100,6 +100,8 @@ int64_t JFJochBitShuffleCompressor::Compress(void *dest, size_t dest_size, const if (tmp_space.size() < DefaultBlockSize * elem_size) tmp_space.resize(DefaultBlockSize * elem_size); + if (scratch.size() < DefaultBlockSize * elem_size) + scratch.resize(DefaultBlockSize * elem_size); size_t num_full_blocks = nelements / DefaultBlockSize; size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize; diff --git a/compression/JFJochCompressor.h b/compression/JFJochCompressor.h index 255a49cb..30bcbcc7 100644 --- a/compression/JFJochCompressor.h +++ b/compression/JFJochCompressor.h @@ -17,10 +17,11 @@ class JFJochBitShuffleCompressor { JFJochZstdCompressor zstd_compressor; CompressionAlgorithm algorithm; std::vector tmp_space; + std::vector scratch; size_t CompressBlock(char *dest, const char * source, size_t nelements, size_t elem_size); public: - constexpr static const size_t DefaultBlockSize = 4096; + constexpr static const size_t DefaultBlockSize = 16384; explicit JFJochBitShuffleCompressor(CompressionAlgorithm algorithm); @@ -35,8 +36,6 @@ public: } std::vector Compress(const void* source, size_t nelements, size_t elem_size); int64_t Compress(void *dest, size_t dest_size, const void* source, size_t nelements, size_t elem_size); -private: - char scratch[DefaultBlockSize * sizeof(uint64_t)]; }; template std::vector bitshuffle(const std::vector &input, size_t block_size) {