leonarski_fandClaude Opus 5
a7615c8778
DECTRIS Eiger firmware 1.x compressed its images with the HDF Group's plain LZ4 filter rather than bitshuffle, and files from that era are still what a repository hands you. Read-only: nothing here produces it, and the new enumerator goes last so the existing values do not move, the enum being part of the CBOR stream. The framing is the same as bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size in bytes, then each block prefixed by its 32-bit big-endian compressed size - so only the unshuffle step differs. Verified on a real chunk: the declared total matched width*height*4 exactly and the block walk consumed the chunk to the byte. It cannot reuse the bitshuffle path, which requires the block to be a multiple of BSHUF_BLOCKED_MULT elements: these files put the whole image in ONE block, which is not. The unsupported-filter message now names the filter it found and the ones it knows. Before, an Eiger 1.x file did not report a codec problem at all - it reached spot finding with nothing decoded and failed as "0 spots from 60 images" then "found no lattice", which reads as a crystallography failure rather than a format one. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
241 lines
12 KiB
C++
241 lines
12 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#pragma once
|
|
|
|
#include <vector>
|
|
#include <cstring>
|
|
|
|
#include <bitshuffle/bitshuffle.h>
|
|
#include <bitshuffle/bitshuffle_internals.h>
|
|
#include <lz4/lz4.h>
|
|
#include <zstd.h>
|
|
|
|
#include "BitShuffleBlock.h"
|
|
#include "../compression/CompressionAlgorithmEnum.h"
|
|
|
|
#include "../common/JFJochException.h"
|
|
#include "../common/CompressedImage.h"
|
|
|
|
extern "C" {
|
|
uint64_t bshuf_read_uint64_BE(const void* buf);
|
|
};
|
|
|
|
inline size_t JFJochDecompressHperfPtr(uint8_t *output,
|
|
CompressionAlgorithm algorithm,
|
|
const uint8_t *source,
|
|
size_t source_size,
|
|
size_t nelements,
|
|
size_t elem_size,
|
|
size_t block_size) {
|
|
if ((algorithm != CompressionAlgorithm::BSHUF_LZ4) &&
|
|
(algorithm != CompressionAlgorithm::BSHUF_ZSTD) &&
|
|
(algorithm != CompressionAlgorithm::BSHUF_ZSTD_RLE) &&
|
|
(algorithm != CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF))
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Algorithm not supported by hperf decompressor");
|
|
|
|
if ((block_size == 0) || ((block_size % BSHUF_BLOCKED_MULT) != 0))
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Invalid block size");
|
|
|
|
std::vector<char> decompressed_block(block_size * elem_size);
|
|
std::vector<char> scratch(block_size * elem_size);
|
|
|
|
const uint8_t *src_ptr = source;
|
|
const uint8_t *const source_end = source + source_size;
|
|
uint8_t *dst_ptr = output;
|
|
|
|
const size_t num_full_blocks = nelements / block_size;
|
|
const size_t reminder_size = nelements - num_full_blocks * block_size;
|
|
const size_t last_block_size = reminder_size - reminder_size % BSHUF_BLOCKED_MULT;
|
|
|
|
auto decode_block = [&](size_t current_nelements) {
|
|
// Both the block length and the block itself come from the stream, which for the CBOR path
|
|
// and for the XDS plugin is data we did not produce.
|
|
if (source_end - src_ptr < 4)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Truncated compressed block header");
|
|
|
|
const auto compressed_size = static_cast<size_t>(bshuf_read_uint32_BE(src_ptr));
|
|
src_ptr += 4;
|
|
|
|
if (static_cast<size_t>(source_end - src_ptr) < compressed_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Compressed block extends past the input buffer");
|
|
|
|
const size_t expected_size = current_nelements * elem_size;
|
|
size_t decompressed_size = 0;
|
|
|
|
switch (algorithm) {
|
|
case CompressionAlgorithm::BSHUF_LZ4: {
|
|
const int ret = LZ4_decompress_safe(reinterpret_cast<const char *>(src_ptr),
|
|
decompressed_block.data(),
|
|
static_cast<int>(compressed_size),
|
|
static_cast<int>(expected_size));
|
|
if (ret < 0 || static_cast<size_t>(ret) != expected_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 decompression error");
|
|
decompressed_size = static_cast<size_t>(ret);
|
|
break;
|
|
}
|
|
case CompressionAlgorithm::BSHUF_ZSTD:
|
|
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
|
|
case CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF: {
|
|
const size_t ret = ZSTD_decompress(decompressed_block.data(),
|
|
expected_size,
|
|
src_ptr,
|
|
compressed_size);
|
|
if (ZSTD_isError(ret) || ret != expected_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "ZSTD decompression error");
|
|
decompressed_size = ret;
|
|
break;
|
|
}
|
|
default:
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Algorithm not supported");
|
|
}
|
|
|
|
if (JFJochBitUnshuffleBlock(reinterpret_cast<char *>(dst_ptr),
|
|
decompressed_block.data(),
|
|
scratch.data(),
|
|
current_nelements,
|
|
elem_size) < 0)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "bitshuffle block decode error");
|
|
|
|
src_ptr += compressed_size;
|
|
dst_ptr += decompressed_size;
|
|
};
|
|
|
|
for (size_t i = 0; i < num_full_blocks; ++i)
|
|
decode_block(block_size);
|
|
|
|
if (last_block_size > 0)
|
|
decode_block(last_block_size);
|
|
|
|
const size_t leftover_bytes = (reminder_size % BSHUF_BLOCKED_MULT) * elem_size;
|
|
if (leftover_bytes > 0) {
|
|
if (static_cast<size_t>(source_end - src_ptr) < leftover_bytes)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Truncated trailing bytes");
|
|
memcpy(dst_ptr, src_ptr, leftover_bytes);
|
|
src_ptr += leftover_bytes;
|
|
}
|
|
|
|
return static_cast<size_t>(src_ptr - source);
|
|
}
|
|
|
|
// Plain LZ4, HDF5 filter 32004, as DECTRIS Eiger firmware 1.x wrote it. The framing is the same as
|
|
// bitshuffle's - a 64-bit big-endian total size, a 32-bit big-endian block size IN BYTES, then each
|
|
// block prefixed by its 32-bit big-endian compressed size - and the only difference is that no bit
|
|
// shuffle was applied, so each block decompresses straight into place. It cannot go through the
|
|
// bitshuffle path: that one requires the block to be a multiple of BSHUF_BLOCKED_MULT elements, and
|
|
// these files put the WHOLE image in one block (measured: 40666360 bytes, i.e. 10166590 uint32).
|
|
inline void JFJochDecompressLZ4Ptr(uint8_t *output,
|
|
const uint8_t *source,
|
|
size_t source_size,
|
|
size_t nelements,
|
|
size_t elem_size) {
|
|
const size_t expected_total = nelements * elem_size;
|
|
|
|
if (source_size < 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the LZ4 header");
|
|
if (bshuf_read_uint64_BE(const_cast<uint8_t *>(source)) != expected_total)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
|
|
|
|
size_t block_bytes = bshuf_read_uint32_BE(source + 8);
|
|
if (block_bytes == 0)
|
|
block_bytes = expected_total; // some writers leave it zero and mean "one block"
|
|
|
|
const uint8_t *src_ptr = source + 12;
|
|
const uint8_t *const source_end = source + source_size;
|
|
size_t written = 0;
|
|
|
|
while (written < expected_total) {
|
|
if (source_end - src_ptr < 4)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Truncated LZ4 block header");
|
|
|
|
const auto compressed_size = static_cast<size_t>(bshuf_read_uint32_BE(src_ptr));
|
|
src_ptr += 4;
|
|
if (compressed_size == 0 || static_cast<size_t>(source_end - src_ptr) < compressed_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 block extends past the input buffer");
|
|
|
|
const size_t this_block = std::min(block_bytes, expected_total - written);
|
|
const int ret = LZ4_decompress_safe(reinterpret_cast<const char *>(src_ptr),
|
|
reinterpret_cast<char *>(output + written),
|
|
static_cast<int>(compressed_size),
|
|
static_cast<int>(this_block));
|
|
if (ret < 0 || static_cast<size_t>(ret) != this_block)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "LZ4 decompression error");
|
|
|
|
src_ptr += compressed_size;
|
|
written += this_block;
|
|
}
|
|
}
|
|
|
|
inline void JFJochDecompressPtr(uint8_t *output,
|
|
CompressionAlgorithm algorithm,
|
|
const uint8_t *source,
|
|
size_t source_size,
|
|
size_t nelements,
|
|
size_t elem_size,
|
|
bool use_hperf = true) {
|
|
if (algorithm == CompressionAlgorithm::LZ4_NO_SHUFFLE) {
|
|
JFJochDecompressLZ4Ptr(output, source, source_size, nelements, elem_size);
|
|
return;
|
|
}
|
|
|
|
size_t block_size = 0;
|
|
if (algorithm != CompressionAlgorithm::NO_COMPRESSION) {
|
|
// The 12-byte bitshuffle header must be there before it can be read, and before
|
|
// source_size - 12 is handed to the decompressors below.
|
|
if (source_size < 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Buffer too short for the bitshuffle header");
|
|
if (bshuf_read_uint64_BE(const_cast<uint8_t *>(source)) != nelements * elem_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
|
|
auto tmp = bshuf_read_uint32_BE(source + 8);
|
|
block_size = tmp / elem_size;
|
|
}
|
|
|
|
switch (algorithm) {
|
|
case CompressionAlgorithm::NO_COMPRESSION:
|
|
if (source_size != nelements * elem_size)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Mismatch in size");
|
|
memcpy(output, source, source_size);
|
|
break;
|
|
case CompressionAlgorithm::BSHUF_LZ4:
|
|
if (use_hperf) {
|
|
if (JFJochDecompressHperfPtr(output, algorithm, source + 12, source_size - 12,
|
|
nelements, elem_size, block_size) != source_size - 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
|
|
} else {
|
|
if (bshuf_decompress_lz4(source + 12, output, nelements,
|
|
elem_size, block_size) != source_size - 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
|
|
}
|
|
break;
|
|
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
|
|
case CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF:
|
|
case CompressionAlgorithm::BSHUF_ZSTD:
|
|
if (use_hperf) {
|
|
if (JFJochDecompressHperfPtr(output, algorithm, source + 12, source_size - 12,
|
|
nelements, elem_size, block_size) != source_size - 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
|
|
} else {
|
|
if (bshuf_decompress_zstd(source + 12, output, nelements,
|
|
elem_size, block_size) != source_size - 12)
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Decompression error");
|
|
}
|
|
break;
|
|
default:
|
|
throw JFJochException(JFJochExceptionCategory::Compression, "Not implemented algorithm");
|
|
}
|
|
}
|
|
|
|
template <class Td, class Ts>
|
|
void JFJochDecompress(std::vector<Td> &output, CompressionAlgorithm algorithm, const Ts *source_v, size_t source_size,
|
|
size_t nelements, bool use_hperf = true) {
|
|
output.resize(nelements);
|
|
JFJochDecompressPtr((uint8_t *) output.data(), algorithm, (uint8_t *) source_v, source_size,
|
|
nelements, sizeof(Td), use_hperf);
|
|
}
|
|
|
|
template <class Td, class Ts>
|
|
void JFJochDecompress(std::vector<Td> &output, CompressionAlgorithm algorithm, const std::vector<Ts> source_v,
|
|
size_t nelements, bool use_hperf = true) {
|
|
JFJochDecompress(output, algorithm, source_v.data(), source_v.size() * sizeof(Ts), nelements, use_hperf);
|
|
}
|