// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #pragma once #include #include "../../common/CompressedImage.h" #include "../indexing/CUDAMemHelpers.h" // One bitshuffle block, located by the host scan and consumed by both kernels. struct BSLZ4BlockDesc { uint32_t in_off; // byte offset of the LZ4 payload within the chunk uint32_t in_len; // compressed length uint32_t out_off; // byte offset of this block's output in the image uint32_t nelem; // elements in this block (the last one is usually shorter) }; // What DecodeShuffled() leaves on the device: the LZ4 output, still bitshuffled, plus everything the // un-transpose needs to finish the image. The tail is the handful of elements bitshuffle stores // verbatim; it is already on the device inside the uploaded chunk, so it is handed over as a device // pointer rather than copied again from the host. struct BSLZ4ShuffledImage { const uint8_t *shuffled = nullptr; const BSLZ4BlockDesc *desc = nullptr; int nblocks = 0; uint32_t elem_size = 0; const uint8_t *tail_src = nullptr; uint32_t tail_elems = 0; uint32_t tail_elem0 = 0; // index of the first tail element in the image }; // Decompress a bitshuffle+LZ4 image ON THE DEVICE, so the compressed bytes are what crosses PCIe. // // The idea - upload the compressed chunk and decode it on the GPU rather than decompressing on the // host - is Jon Wright's (ESRF); see https://github.com/jonwright/bslz4decoders and his 2021 HDF5 // User Group talk "Experiences with GPU decompression for bitshuffle + LZ4 data". The kernels here // are our own, but the approach, and the observation that it is worth doing at all, are his. // // Why it pays: a full 18 Mpx uint32 frame is 72 MB decompressed and about 4 MB compressed, and // profiling showed the host-to-device copy owning ~78% of the per-image loop against ~39% for // kernels. Decoding on the device removes both that transfer and the host-side decompression, // whose memory traffic was itself holding the copy engine well below the link rate. // // Only BSHUF_LZ4 is handled. The zstd variants have no device decoder, so Supports() returns false // and the caller decompresses on the host exactly as before. // // The container comes off the network or off disk, so it is not trusted. Everything the host can // check cheaply is checked before any work is queued and throws; what only the kernel can see - a // block that does not decode to its declared length, which is what a corrupt LZ4 payload looks like // - raises a device-side flag that ThrowIfDecodeFailed() reports once the caller has synchronised. // The CPU decoder makes exactly the same checks (LZ4_decompress_safe's length check plus the // consumed-input check in JFJochDecompress.h), so a chunk either decodes identically on both or // fails on both. It is never silently wrong on one and right on the other. class BSLZ4DecoderGPU { std::shared_ptr stream; CudaDevicePtr gpu_compressed; CudaDevicePtr gpu_shuffled; // LZ4 output, still bitshuffled CudaDevicePtr gpu_desc; CudaHostPtr host_desc; // pinned, so the descriptor upload is truly async CudaDevicePtr gpu_status; // set by the kernel when a block decodes short CudaHostPtr host_status; // Bracket the decode so the time it takes can still be reported as decompression, which is what // it is. Both are recorded on the decoder's stream and read after the caller synchronises. CudaEvent decode_start; CudaEvent decode_stop; bool decode_timed = false; size_t max_uncompressed_bytes = 0; size_t compressed_capacity = 0; size_t max_blocks = 0; void EnsureCompressedCapacity(size_t bytes); void EnsureBlockCapacity(size_t nblocks); public: BSLZ4DecoderGPU(size_t max_uncompressed_bytes, std::shared_ptr stream); // True when this image can be decoded on the device. Everything else must go the host route. static bool Supports(const CompressedImage &image); // Locate the blocks, upload the chunk, and run the LZ4 pass. The result is still bitshuffled - // the caller finishes it, either with Decode()'s un-transpose or by fusing the un-transpose into // its own kernel. Work is queued on the decoder's stream and the caller synchronises. BSLZ4ShuffledImage DecodeShuffled(const CompressedImage &image); // Decode into gpu_out, which must hold image.GetUncompressedSize() bytes. The plain raw-bytes // path: DecodeShuffled() plus the un-transpose. Used by the tests and by any caller that wants // the decompressed image rather than a preprocessed one. void Decode(const CompressedImage &image, uint8_t *gpu_out); // Report a block that did not decode to its declared length. MUST be called after the caller has // synchronised the stream; until then the flag has not arrived. Throws on failure. void ThrowIfDecodeFailed(); // Device time spent decoding the last image, in seconds. Valid after the caller synchronises. [[nodiscard]] float GetDecodeTime_s() const; };