mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 67 Wiki Activity

Compressor: scale block size to a per-algorithm byte target

Browse Source 
Replace the fixed-element DefaultBlockSize with a byte target divided by
elem_size to get the block element count, so the per-block working set (and
thus cache behaviour) stays constant across pixel bit depths instead of halving
from 8- to 16- to 32-bit. The target is per-algorithm, following the measured
sweet spots on sparse data: LZ4 wants a small, cache-resident block for
throughput (16 kB), ZSTD/RLE want a large block for ratio (128 kB). The gap is
widest on extreme-sparsity inputs such as the uint32 pixel_mask, where
large-block ZSTD reaches 100-1800x vs ~160x for LZ4.

The block size is read back per-dataset from the bitshuffle stream header
(block_size = header_bytes / elem_size) and the HDF5 filter params, so the
decompressor and external readers (XDS/Neggia/Durin/CrystFEL) need no change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
leonarski_f
2026-06-27 07:56:08 +02:00
co-authored by Claude Opus 4.8
parent aadba5b343
commit 74584c23ac
5 changed files with 51 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compression/JFJochCompressor.cpp
+20 -13
View File
@@ -16,6 +16,13 @@ extern "C" {
void bshuf_write_uint64_BE(void* buf, uint64_t num);
}
// Necessary condition for BlockSize() to be a valid bitshuffle block: with elem_size a power of
// two, a byte target that is a multiple of BSHUF_BLOCKED_MULT keeps the element count a multiple
// of BSHUF_BLOCKED_MULT too.
static_assert(JFJochBitShuffleCompressor::DefaultBlockSizeBytes(CompressionAlgorithm::BSHUF_LZ4) % BSHUF_BLOCKED_MULT == 0
&& JFJochBitShuffleCompressor::DefaultBlockSizeBytes(CompressionAlgorithm::BSHUF_ZSTD) % BSHUF_BLOCKED_MULT == 0,
"block byte target must be a multiple of the bitshuffle block multiple");
// Worst-case size of one compressed block, including its 4-byte length prefix. Mirrors the
// per-block term of MaxCompressedSize(), so a dest sized to MaxCompressedSize() never fails.
static size_t MaxCompressedBlockSize(CompressionAlgorithm algorithm, size_t src_size) {
@@ -82,8 +89,6 @@ int64_t JFJochBitShuffleCompressor::Compress(void *dest, size_t dest_size, const
auto c_dest = (char *) dest;
auto c_source = (char *) source;
static_assert(DefaultBlockSize % BSHUF_BLOCKED_MULT == 0, "Block size must be multiple of 8");
if (algorithm == CompressionAlgorithm::NO_COMPRESSION) {
// Trivial case if no compression - copy content
if (nelements * elem_size > dest_size)
@@ -95,25 +100,27 @@ int64_t JFJochBitShuffleCompressor::Compress(void *dest, size_t dest_size, const
if (dest_size < 12)
return -1;
const size_t block_size = BlockSize(algorithm, elem_size);
bshuf_write_uint64_BE(c_dest, nelements * elem_size);
bshuf_write_uint32_BE(c_dest + 8, DefaultBlockSize * elem_size);
bshuf_write_uint32_BE(c_dest + 8, block_size * elem_size);
if (tmp_space.size() < DefaultBlockSize * elem_size)
tmp_space.resize(DefaultBlockSize * elem_size);
if (scratch.size() < DefaultBlockSize * elem_size)
scratch.resize(DefaultBlockSize * elem_size);
if (tmp_space.size() < block_size * elem_size)
tmp_space.resize(block_size * elem_size);
if (scratch.size() < block_size * elem_size)
scratch.resize(block_size * elem_size);
size_t num_full_blocks = nelements / DefaultBlockSize;
size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize;
size_t num_full_blocks = nelements / block_size;
size_t reminder_size = nelements - num_full_blocks * block_size;
size_t compressed_size = 12;
// Blocks are small relative to the image, so before each one we just check that the
// remaining space still covers that block's worst case, and bail out (-1) if not.
for (int i = 0; i < num_full_blocks; i++) {
if (compressed_size + MaxCompressedBlockSize(algorithm, DefaultBlockSize * elem_size) > dest_size)
if (compressed_size + MaxCompressedBlockSize(algorithm, block_size * elem_size) > dest_size)
return -1;
compressed_size += CompressBlock(c_dest + compressed_size,
c_source + i * DefaultBlockSize * elem_size, DefaultBlockSize, elem_size);
c_source + i * block_size * elem_size, block_size, elem_size);
}
size_t last_block_size = reminder_size - reminder_size % BSHUF_BLOCKED_MULT;
@@ -121,14 +128,14 @@ int64_t JFJochBitShuffleCompressor::Compress(void *dest, size_t dest_size, const
if (compressed_size + MaxCompressedBlockSize(algorithm, last_block_size * elem_size) > dest_size)
return -1;
compressed_size += CompressBlock(c_dest + compressed_size,
c_source + num_full_blocks * DefaultBlockSize * elem_size, last_block_size, elem_size);
c_source + num_full_blocks * block_size * elem_size, last_block_size, elem_size);
}
size_t leftover_bytes = (reminder_size % BSHUF_BLOCKED_MULT) * elem_size;
if (leftover_bytes > 0) {
if (compressed_size + leftover_bytes > dest_size)
return -1;
memcpy(c_dest + compressed_size, c_source + (num_full_blocks * DefaultBlockSize + last_block_size) * elem_size, leftover_bytes);
memcpy(c_dest + compressed_size, c_source + (num_full_blocks * block_size + last_block_size) * elem_size, leftover_bytes);
compressed_size += leftover_bytes;
}
return compressed_size;