diff --git a/compression/CompressionAlgorithmEnum.h b/compression/CompressionAlgorithmEnum.h index e65591403..b8dbded6d 100644 --- a/compression/CompressionAlgorithmEnum.h +++ b/compression/CompressionAlgorithmEnum.h @@ -3,4 +3,8 @@ #pragma once -enum class CompressionAlgorithm {BSHUF_LZ4, BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF, NO_COMPRESSION}; +// LZ4_NO_SHUFFLE is READ-ONLY: HDF5 filter 32004, plain LZ4 with no bit shuffle, written by DECTRIS +// Eiger firmware 1.x. Nothing here produces it. It is last so the values of the others do not move - +// the enum crosses the CBOR stream. +enum class CompressionAlgorithm {BSHUF_LZ4, BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF, NO_COMPRESSION, + LZ4_NO_SHUFFLE}; diff --git a/compression/JFJochDecompress.h b/compression/JFJochDecompress.h index bb6d8d553..73c0518f4 100644 --- a/compression/JFJochDecompress.h +++ b/compression/JFJochDecompress.h @@ -118,6 +118,54 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output, return static_cast(src_ptr - source); } +// Plain LZ4, HDF5 filter 32004, as DECTRIS Eiger firmware 1.x wrote it. The framing is the same as +// bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size IN BYTES, then each +// block prefixed by its 32-bit big-endian compressed size - and the only difference is that no bit +// shuffle was applied, so each block decompresses straight into place. It cannot go through the +// bitshuffle path: that one requires the block to be a multiple of BSHUF_BLOCKED_MULT elements, and +// these files put the WHOLE image in one block (measured: 40666360 bytes, i.e. 10166590 uint32). +inline void JFJochDecompressLZ4Ptr(uint8_t *output, + const uint8_t *source, + size_t source_size, + size_t nelements, + size_t elem_size) { + const size_t expected_total = nelements * elem_size; + + if (source_size < 12) + throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the LZ4 header"); + if (bshuf_read_uint64_BE(const_cast(source)) != expected_total) + throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size"); + + size_t block_bytes = bshuf_read_uint32_BE(source + 8); + if (block_bytes == 0) + block_bytes = expected_total; // some writers leave it zero and mean "one block" + + const uint8_t *src_ptr = source + 12; + const uint8_t *const source_end = source + source_size; + size_t written = 0; + + while (written < expected_total) { + if (source_end - src_ptr < 4) + throw JFJochException(JFJochExceptionCategory::Compression, "Truncated LZ4 block header"); + + const auto compressed_size = static_cast(bshuf_read_uint32_BE(src_ptr)); + src_ptr += 4; + if (compressed_size == 0 || static_cast(source_end - src_ptr) < compressed_size) + throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 block extends past the input buffer"); + + const size_t this_block = std::min(block_bytes, expected_total - written); + const int ret = LZ4_decompress_safe(reinterpret_cast(src_ptr), + reinterpret_cast(output + written), + static_cast(compressed_size), + static_cast(this_block)); + if (ret < 0 || static_cast(ret) != this_block) + throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 decompression error"); + + src_ptr += compressed_size; + written += this_block; + } +} + inline void JFJochDecompressPtr(uint8_t *output, CompressionAlgorithm algorithm, const uint8_t *source, @@ -125,6 +173,11 @@ inline void JFJochDecompressPtr(uint8_t *output, size_t nelements, size_t elem_size, bool use_hperf = true) { + if (algorithm == CompressionAlgorithm::LZ4_NO_SHUFFLE) { + JFJochDecompressLZ4Ptr(output, source, source_size, nelements, elem_size); + return; + } + size_t block_size = 0; if (algorithm != CompressionAlgorithm::NO_COMPRESSION) { // The 12-byte bitshuffle header must be there before it can be read, and before diff --git a/writer/HDF5Objects.cpp b/writer/HDF5Objects.cpp index a234f2175..382cfab5e 100644 --- a/writer/HDF5Objects.cpp +++ b/writer/HDF5Objects.cpp @@ -7,6 +7,11 @@ #include #include "HDF5Objects.h" + +// HDF Group contributed LZ4 filter, the one DECTRIS Eiger firmware 1.x wrote its images with. +// Read-only here: bshuf_h5filter.h defines 32008 for bitshuffle but nothing defines this one. +#define LZ4_H5FILTER 32004 + #include "H5FDpoison_sec2.h" std::mutex hdf5_mutex; @@ -996,9 +1001,20 @@ CompressionAlgorithm HDF5Dcpl::GetCompression() { size_t cd_nelemts = sizeof(cd_values); memset(cd_values, 0, sizeof(cd_values)); - if (H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr) - != BSHUF_H5FILTER) - throw JFJochException(JFJochExceptionCategory::HDF5,"Only Bitshuffle filter supported"); + const auto filter = H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr); + + // Plain LZ4, no bit shuffle. DECTRIS Eiger firmware 1.x wrote images this way, and those files + // are still deposited; read-only, we never write it. Same block framing as bitshuffle - a 64-bit + // total size, a 32-bit block size, then 32-bit-prefixed blocks - which is why only the unshuffle + // step differs downstream. + if (filter == LZ4_H5FILTER) + return CompressionAlgorithm::LZ4_NO_SHUFFLE; + + if (filter != BSHUF_H5FILTER) + throw JFJochException(JFJochExceptionCategory::HDF5, + "Unsupported HDF5 compression filter " + std::to_string(filter) + + " (supported: bitshuffle " + std::to_string(BSHUF_H5FILTER) + + ", LZ4 " + std::to_string(LZ4_H5FILTER) + ")"); if (cd_values[4] == BSHUF_H5_COMPRESS_LZ4) return CompressionAlgorithm::BSHUF_LZ4;