From 33eacea54cc138df41f6bd7331cb3ce06c96b827 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 23 Oct 2026 11:24:08 +0200 Subject: [PATCH] XFELReader: a byte-shuffled frame is unshuffled a piece at a time, a whole pixel per step The unshuffle of a deflated + shuffled chunk (a panel-table HDF5 run: 8.4 MB per frame) moved one byte at a time with the pixel size read at run time, and wrote the whole frame before it was converted. It now assembles each 2- or 4-byte pixel from its byte planes in a loop the compiler vectorises, a 128 kB piece at a time, and each piece is converted into the image while it is in the cache, as for an unshuffled chunk. Bit-identical: the FNV pixel hashes of 2000 frames on the five reader configurations are unchanged. Reader alone on the panel-table run, 16 threads, 2000 frames: the unshuffle went from 58% of the reader's cycles to 13%; wall 7.7-8.6 -> 5.1-7.0 s on a loaded machine. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu --- reader/JFJochXFELReader.cpp | 37 ++++++++++++++++++++++++++++++++----- reader/JFJochXFELReader.h | 2 +- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/reader/JFJochXFELReader.cpp b/reader/JFJochXFELReader.cpp index 3235062c9..fdc6d4e0d 100644 --- a/reader/JFJochXFELReader.cpp +++ b/reader/JFJochXFELReader.cpp @@ -68,6 +68,26 @@ int32_t Pick(bool use_b, int32_t a, int32_t b) { return (a & ~all) | (b & all); } +// Pixels are decoded and converted a piece at a time where they can be (ReadStored): a piece fits the core's +// own cache with the window inflate keeps, so it is converted before it leaves. +constexpr size_t PIECE_BYTES = 128 * 1024; + +// The pixels [first, first + count) of a byte-shuffled frame of n pixels of size bytes each (byte b of pixel +// i is shuffled[b * n + i]), into out as stored: 2 or 4 bytes (plain_type), little-endian, as the frames are read. +void Unshuffle(const uint8_t *shuffled, size_t n, size_t size, size_t first, size_t count, uint8_t *out) { + const uint8_t *b0 = shuffled + first, *b1 = b0 + n; + if (size == 2) { + auto *o = reinterpret_cast(out); + for (size_t i = 0; i < count; i++) + o[i] = static_cast(b0[i] | b1[i] << 8); + } else { + const uint8_t *b2 = b1 + n, *b3 = b2 + n; + auto *o = reinterpret_cast(out); + for (size_t i = 0; i < count; i++) + o[i] = b0[i] | b1[i] << 8 | b2[i] << 16 | static_cast(b3[i]) << 24; + } +} + } // namespace JFJochXFELReader::JFJochXFELReader() = default; @@ -903,10 +923,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buff throw JFJochException(JFJochExceptionCategory::InputParameterInvalid, "Cannot inflate " + frame.dataset + " of " + files_[frame.file]); if (layout.shuffled) { + if constexpr (in_pieces) { + // A piece at a time, as for an unshuffled chunk. + thread_local std::vector out(PIECE_BYTES); + const size_t piece_pixels = PIECE_BYTES / size; + for (size_t first = 0; first < n; first += piece_pixels) { + const size_t count = std::min(piece_pixels, n - first); + Unshuffle(shuffled.data(), n, size, first, count, out.data()); + piece(first, out.data(), count); + } + return false; + } stored.resize(n * size); - for (size_t i = 0; i < n; i++) - for (size_t b = 0; b < size; b++) - stored[i * size + b] = shuffled[b * n + i]; + Unshuffle(shuffled.data(), n, size, 0, n, stored.data()); } } else { stored.resize(n * size); @@ -920,8 +949,6 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buff template void JFJochXFELReader::InflateInPieces(const std::vector &compressed, size_t bytes, size_t size, const Frame &frame, Piece piece) const { - // A piece fits the core's own cache with the window inflate keeps, so it is converted before it leaves. - constexpr size_t PIECE_BYTES = 128 * 1024; thread_local std::vector out(PIECE_BYTES); z_stream z{}; if (inflateInit(&z) != Z_OK) diff --git a/reader/JFJochXFELReader.h b/reader/JFJochXFELReader.h index 3a2824486..73c77014b 100644 --- a/reader/JFJochXFELReader.h +++ b/reader/JFJochXFELReader.h @@ -126,7 +126,7 @@ class JFJochXFELReader : public JFJochReader { // The frame's pixels, stored as layout says, as stored (plain_type), or as float, into stored; with // keep_compressed, a chunk decoded by CompressedImage (not deflated) is left as stored in the file, and // true returned. Given piece, the pixels are handed to piece(first pixel, their bytes, pixels) instead: a - // deflated chunk a piece at a time as it is inflated (InflateInPieces), any other at once. + // deflated chunk a piece at a time, as it is inflated (InflateInPieces) or unshuffled, any other at once. template bool ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, bool keep_compressed, Piece piece = nullptr) const;