compression: read plain LZ4 (HDF5 filter 32004), and name the filter when one is unsupported

DECTRIS Eiger firmware 1.x compressed its images with the HDF Group's plain LZ4 filter rather than
bitshuffle, and files from that era are still what a repository hands you. Read-only: nothing here
produces it, and the new enumerator goes last so the existing values do not move, the enum being
part of the CBOR stream.

The framing is the same as bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block
size in bytes, then each block prefixed by its 32-bit big-endian compressed size - so only the
unshuffle step differs. Verified on a real chunk: the declared total matched width*height*4 exactly
and the block walk consumed the chunk to the byte. It cannot reuse the bitshuffle path, which
requires the block to be a multiple of BSHUF_BLOCKED_MULT elements: these files put the whole image
in ONE block, which is not.

The unsupported-filter message now names the filter it found and the ones it knows. Before, an
Eiger 1.x file did not report a codec problem at all - it reached spot finding with nothing decoded
and failed as "0 spots from 60 images" then "found no lattice", which reads as a crystallography
failure rather than a format one.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

compression/CompressionAlgorithmEnum.h

@@ -3,4 +3,8 @@
 
 #pragma once
 
-enum class CompressionAlgorithm {BSHUF_LZ4, BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF, NO_COMPRESSION};
+// LZ4_NO_SHUFFLE is READ-ONLY: HDF5 filter 32004, plain LZ4 with no bit shuffle, written by DECTRIS
+// Eiger firmware 1.x. Nothing here produces it. It is last so the values of the others do not move -
+// the enum crosses the CBOR stream.
+enum class CompressionAlgorithm {BSHUF_LZ4, BSHUF_ZSTD, BSHUF_ZSTD_RLE, BSHUF_ZSTD_RLE_HUFF, NO_COMPRESSION,
+                                 LZ4_NO_SHUFFLE};

compression/JFJochDecompress.h

@@ -118,6 +118,54 @@ inline size_t JFJochDecompressHperfPtr(uint8_t *output,
     return static_cast<size_t>(src_ptr - source);
 }
 
+// Plain LZ4, HDF5 filter 32004, as DECTRIS Eiger firmware 1.x wrote it. The framing is the same as
+// bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size IN BYTES, then each
+// block prefixed by its 32-bit big-endian compressed size - and the only difference is that no bit
+// shuffle was applied, so each block decompresses straight into place. It cannot go through the
+// bitshuffle path: that one requires the block to be a multiple of BSHUF_BLOCKED_MULT elements, and
+// these files put the WHOLE image in one block (measured: 40666360 bytes, i.e. 10166590 uint32).
+inline void JFJochDecompressLZ4Ptr(uint8_t *output,
+                                   const uint8_t *source,
+                                   size_t source_size,
+                                   size_t nelements,
+                                   size_t elem_size) {
+    const size_t expected_total = nelements * elem_size;
+
+    if (source_size < 12)
+        throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the LZ4 header");
+    if (bshuf_read_uint64_BE(const_cast<uint8_t *>(source)) != expected_total)
+        throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
+
+    size_t block_bytes = bshuf_read_uint32_BE(source + 8);
+    if (block_bytes == 0)
+        block_bytes = expected_total;   // some writers leave it zero and mean "one block"
+
+    const uint8_t *src_ptr = source + 12;
+    const uint8_t *const source_end = source + source_size;
+    size_t written = 0;
+
+    while (written < expected_total) {
+        if (source_end - src_ptr < 4)
+            throw JFJochException(JFJochExceptionCategory::Compression, "Truncated LZ4 block header");
+
+        const auto compressed_size = static_cast<size_t>(bshuf_read_uint32_BE(src_ptr));
+        src_ptr += 4;
+        if (compressed_size == 0 || static_cast<size_t>(source_end - src_ptr) < compressed_size)
+            throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 block extends past the input buffer");
+
+        const size_t this_block = std::min(block_bytes, expected_total - written);
+        const int ret = LZ4_decompress_safe(reinterpret_cast<const char *>(src_ptr),
+                                            reinterpret_cast<char *>(output + written),
+                                            static_cast<int>(compressed_size),
+                                            static_cast<int>(this_block));
+        if (ret < 0 || static_cast<size_t>(ret) != this_block)
+            throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 decompression error");
+
+        src_ptr += compressed_size;
+        written += this_block;
+    }
+}
+
 inline void JFJochDecompressPtr(uint8_t *output,
                       CompressionAlgorithm algorithm,
                       const uint8_t *source,
@@ -125,6 +173,11 @@ inline void JFJochDecompressPtr(uint8_t *output,
                       size_t nelements,
                       size_t elem_size,
                       bool use_hperf = true) {
+    if (algorithm == CompressionAlgorithm::LZ4_NO_SHUFFLE) {
+        JFJochDecompressLZ4Ptr(output, source, source_size, nelements, elem_size);
+        return;
+    }
+
     size_t block_size = 0;
     if (algorithm != CompressionAlgorithm::NO_COMPRESSION) {
         // The 12-byte bitshuffle header must be there before it can be read, and before

writer/HDF5Objects.cpp

@@ -7,6 +7,11 @@
 
 #include <bitshuffle/bshuf_h5filter.h>
 #include "HDF5Objects.h"
+
+// HDF Group contributed LZ4 filter, the one DECTRIS Eiger firmware 1.x wrote its images with.
+// Read-only here: bshuf_h5filter.h defines 32008 for bitshuffle but nothing defines this one.
+#define LZ4_H5FILTER 32004
+
 #include "H5FDpoison_sec2.h"
 
 std::mutex hdf5_mutex;
@@ -996,9 +1001,20 @@ CompressionAlgorithm HDF5Dcpl::GetCompression() {
     size_t cd_nelemts = sizeof(cd_values);
     memset(cd_values, 0, sizeof(cd_values));
 
-    if (H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr)
-        != BSHUF_H5FILTER)
-        throw JFJochException(JFJochExceptionCategory::HDF5,"Only Bitshuffle filter supported");
+    const auto filter = H5Pget_filter2(id, 0, nullptr, &cd_nelemts, cd_values, 0, nullptr, nullptr);
+
+    // Plain LZ4, no bit shuffle. DECTRIS Eiger firmware 1.x wrote images this way, and those files
+    // are still deposited; read-only, we never write it. Same block framing as bitshuffle - a 64-bit
+    // total size, a 32-bit block size, then 32-bit-prefixed blocks - which is why only the unshuffle
+    // step differs downstream.
+    if (filter == LZ4_H5FILTER)
+        return CompressionAlgorithm::LZ4_NO_SHUFFLE;
+
+    if (filter != BSHUF_H5FILTER)
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Unsupported HDF5 compression filter " + std::to_string(filter)
+                              + " (supported: bitshuffle " + std::to_string(BSHUF_H5FILTER)
+                              + ", LZ4 " + std::to_string(LZ4_H5FILTER) + ")");
 
     if (cd_values[4] == BSHUF_H5_COMPRESS_LZ4)
         return CompressionAlgorithm::BSHUF_LZ4;