mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 44 Wiki Activity

Generalized serializer

Browse Source
This commit is contained in:
leonarski_f
2023-12-11 06:49:24 +01:00
parent 75913b83eb
commit f1935526a7
52 changed files with 579 additions and 557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
compression/JFJochCompressor.cpp
View File

@@ -56,7 +56,17 @@ size_t JFJochBitShuffleCompressor::CompressBlock(char *dest, const char *source,
return compressed_size + 4;
}
size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size_t nelements, size_t elem_size) {
std::vector<uint8_t> JFJochBitShuffleCompressor::Compress(const void *source, size_t nelements, size_t elem_size) {
std::vector<uint8_t> tmp(MaxCompressedSize(algorithm, nelements, elem_size));
size_t tmp_size = Compress(tmp.data(), source, nelements, elem_size);
tmp.resize(tmp_size);
return tmp;
}
size_t JFJochBitShuffleCompressor::Compress(void *dest, const void *source, size_t nelements, size_t elem_size) {
auto c_dest = (char *) dest;
auto c_source = (char *) source;
static_assert(DefaultBlockSize % BSHUF_BLOCKED_MULT == 0, "Block size must be multiple of 8");
if (algorithm == CompressionAlgorithm::NO_COMPRESSION) {
@@ -65,8 +75,8 @@ size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size
return nelements * elem_size;
}
bshuf_write_uint64_BE(dest, nelements * elem_size);
bshuf_write_uint32_BE(dest + 8, DefaultBlockSize * elem_size);
bshuf_write_uint64_BE(c_dest, nelements * elem_size);
bshuf_write_uint32_BE(c_dest + 8, DefaultBlockSize * elem_size);
if (tmp_space.size() < DefaultBlockSize * elem_size)
tmp_space.resize(DefaultBlockSize * elem_size);
@@ -75,18 +85,19 @@ size_t JFJochBitShuffleCompressor::Compress(char *dest, const char *source, size
size_t reminder_size = nelements - num_full_blocks * DefaultBlockSize;
size_t compressed_size = 12;
for (int i = 0; i < num_full_blocks; i++)
compressed_size += CompressBlock(dest + compressed_size,
source + i * DefaultBlockSize * elem_size, DefaultBlockSize, elem_size);
compressed_size += CompressBlock(c_dest + compressed_size,
c_source + i * DefaultBlockSize * elem_size, DefaultBlockSize, elem_size);
size_t last_block_size = reminder_size - reminder_size % BSHUF_BLOCKED_MULT;
if (last_block_size > 0)
compressed_size += CompressBlock(dest + compressed_size,
source + num_full_blocks * DefaultBlockSize * elem_size, last_block_size, elem_size);
compressed_size += CompressBlock(c_dest + compressed_size,
c_source + num_full_blocks * DefaultBlockSize * elem_size, last_block_size, elem_size);
size_t leftover_bytes = (reminder_size % BSHUF_BLOCKED_MULT) * elem_size;
if (leftover_bytes > 0) {
memcpy(dest + compressed_size, source + (num_full_blocks * DefaultBlockSize + last_block_size) * elem_size, leftover_bytes);
memcpy(c_dest + compressed_size, c_source + (num_full_blocks * DefaultBlockSize + last_block_size) * elem_size, leftover_bytes);
compressed_size += leftover_bytes;
}
return compressed_size;