Compressor: make Compress size-aware, drop frames that don't fit
JFJochBitShuffleCompressor::Compress now takes a dest_size and returns a negative value when the compressed output would not fit, instead of writing past the destination buffer. The check is lazy: before each block it verifies the remaining space still covers that block's worst case (mirrored by the new MaxCompressedBlockSize helper, consistent with MaxCompressedSize so a dest sized to MaxCompressedSize never fails). On overflow the dest content is undefined - no rescue. The receiver uses this to compress directly into the writer buffer slot and drop just the oversized frame instead of pre-reserving the full worst-case image size next to the per-image CBOR metadata. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,20 @@ extern "C" {
|
||||
void bshuf_write_uint64_BE(void* buf, uint64_t num);
|
||||
}
|
||||
|
||||
// Worst-case size of one compressed block, including its 4-byte length prefix. Mirrors the
|
||||
// per-block term of MaxCompressedSize(), so a dest sized to MaxCompressedSize() never fails.
|
||||
static size_t MaxCompressedBlockSize(CompressionAlgorithm algorithm, size_t src_size) {
|
||||
switch (algorithm) {
|
||||
case CompressionAlgorithm::BSHUF_LZ4:
|
||||
return LZ4_compressBound(src_size) + 4;
|
||||
case CompressionAlgorithm::BSHUF_ZSTD:
|
||||
case CompressionAlgorithm::BSHUF_ZSTD_RLE:
|
||||
return ZSTD_compressBound(src_size) + 4;
|
||||
default:
|
||||
return src_size + 4;
|
||||
}
|
||||
}
|
||||
|
||||
JFJochBitShuffleCompressor::JFJochBitShuffleCompressor(CompressionAlgorithm in_algorithm) {
|
||||
algorithm = in_algorithm;
|
||||
}
|
||||
@@ -59,12 +73,12 @@ size_t JFJochBitShuffleCompressor::CompressBlock(char *dest, const char *source,
|
||||
|
||||
std::vector<uint8_t> JFJochBitShuffleCompressor::Compress(const void *source, size_t nelements, size_t elem_size) {
|
||||
std::vector<uint8_t> tmp(MaxCompressedSize(algorithm, nelements, elem_size));
|
||||
size_t tmp_size = Compress(tmp.data(), source, nelements, elem_size);
|
||||
size_t tmp_size = Compress(tmp.data(), tmp.size(), source, nelements, elem_size);
|
||||
tmp.resize(tmp_size);
|
||||
return tmp;
|
||||
}
|
||||
|
||||
size_t JFJochBitShuffleCompressor::Compress(void *dest, const void *source, size_t nelements, size_t elem_size) {
|
||||
int64_t JFJochBitShuffleCompressor::Compress(void *dest, size_t dest_size, const void *source, size_t nelements, size_t elem_size) {
|
||||
auto c_dest = (char *) dest;
|
||||
auto c_source = (char *) source;
|
||||
|
||||
@@ -72,10 +86,15 @@ size_t JFJochBitShuffleCompressor::Compress(void *dest, const void *source, size
|
||||
|
||||
if (algorithm == CompressionAlgorithm::NO_COMPRESSION) {
|
||||
// Trivial case if no compression - copy content
|
||||
if (nelements * elem_size > dest_size)
|
||||
return -1;
|
||||
memcpy(dest, source, nelements * elem_size);
|
||||
return nelements * elem_size;
|
||||
}
|
||||
|
||||
if (dest_size < 12)
|
||||
return -1;
|
||||
|
||||
bshuf_write_uint64_BE(c_dest, nelements * elem_size);
|
||||
bshuf_write_uint32_BE(c_dest + 8, DefaultBlockSize * elem_size);
|
||||
|
||||
@@ -86,18 +105,27 @@ size_t JFJochBitShuffleCompressor::Compress(void *dest, const void *source, size
|
||||
size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize;
|
||||
size_t compressed_size = 12;
|
||||
|
||||
|
||||
for (int i = 0; i < num_full_blocks; i++)
|
||||
// Blocks are small relative to the image, so before each one we just check that the
|
||||
// remaining space still covers that block's worst case, and bail out (-1) if not.
|
||||
for (int i = 0; i < num_full_blocks; i++) {
|
||||
if (compressed_size + MaxCompressedBlockSize(algorithm, DefaultBlockSize * elem_size) > dest_size)
|
||||
return -1;
|
||||
compressed_size += CompressBlock(c_dest + compressed_size,
|
||||
c_source + i * DefaultBlockSize * elem_size, DefaultBlockSize, elem_size);
|
||||
}
|
||||
|
||||
size_t last_block_size = reminder_size - reminder_size % BSHUF_BLOCKED_MULT;
|
||||
if (last_block_size > 0)
|
||||
if (last_block_size > 0) {
|
||||
if (compressed_size + MaxCompressedBlockSize(algorithm, last_block_size * elem_size) > dest_size)
|
||||
return -1;
|
||||
compressed_size += CompressBlock(c_dest + compressed_size,
|
||||
c_source + num_full_blocks * DefaultBlockSize * elem_size, last_block_size, elem_size);
|
||||
}
|
||||
|
||||
size_t leftover_bytes = (reminder_size % BSHUF_BLOCKED_MULT) * elem_size;
|
||||
if (leftover_bytes > 0) {
|
||||
if (compressed_size + leftover_bytes > dest_size)
|
||||
return -1;
|
||||
memcpy(c_dest + compressed_size, c_source + (num_full_blocks * DefaultBlockSize + last_block_size) * elem_size, leftover_bytes);
|
||||
compressed_size += leftover_bytes;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user