XFELReader: a byte-shuffled frame is unshuffled a piece at a time, a whole pixel per step

The unshuffle of a deflated + shuffled chunk (a panel-table HDF5 run: 8.4 MB per frame) moved one byte at
a time with the pixel size read at run time, and wrote the whole frame before it was converted. It now
assembles each 2- or 4-byte pixel from its byte planes in a loop the compiler vectorises, a 128 kB piece
at a time, and each piece is converted into the image while it is in the cache, as for an unshuffled
chunk.

Bit-identical: the FNV pixel hashes of 2000 frames on the five reader configurations are unchanged.
Reader alone on the panel-table run, 16 threads, 2000 frames: the unshuffle went from 58% of the
reader's cycles to 13%; wall 7.7-8.6 -> 5.1-7.0 s on a loaded machine.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu

reader/JFJochXFELReader.cpp

@@ -68,6 +68,26 @@ int32_t Pick(bool use_b, int32_t a, int32_t b) {
     return (a & ~all) | (b & all);
 }
 
+// Pixels are decoded and converted a piece at a time where they can be (ReadStored): a piece fits the core's
+// own cache with the window inflate keeps, so it is converted before it leaves.
+constexpr size_t PIECE_BYTES = 128 * 1024;
+
+// The pixels [first, first + count) of a byte-shuffled frame of n pixels of size bytes each (byte b of pixel
+// i is shuffled[b * n + i]), into out as stored: 2 or 4 bytes (plain_type), little-endian, as the frames are read.
+void Unshuffle(const uint8_t *shuffled, size_t n, size_t size, size_t first, size_t count, uint8_t *out) {
+    const uint8_t *b0 = shuffled + first, *b1 = b0 + n;
+    if (size == 2) {
+        auto *o = reinterpret_cast<uint16_t *>(out);
+        for (size_t i = 0; i < count; i++)
+            o[i] = static_cast<uint16_t>(b0[i] | b1[i] << 8);
+    } else {
+        const uint8_t *b2 = b1 + n, *b3 = b2 + n;
+        auto *o = reinterpret_cast<uint32_t *>(out);
+        for (size_t i = 0; i < count; i++)
+            o[i] = b0[i] | b1[i] << 8 | b2[i] << 16 | static_cast<uint32_t>(b3[i]) << 24;
+    }
+}
+
 }  // namespace
 
 JFJochXFELReader::JFJochXFELReader() = default;
@@ -903,10 +923,19 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buff
             throw JFJochException(JFJochExceptionCategory::InputParameterInvalid,
                                   "Cannot inflate " + frame.dataset + " of " + files_[frame.file]);
         if (layout.shuffled) {
+            if constexpr (in_pieces) {
+                // A piece at a time, as for an unshuffled chunk.
+                thread_local std::vector<uint8_t> out(PIECE_BYTES);
+                const size_t piece_pixels = PIECE_BYTES / size;
+                for (size_t first = 0; first < n; first += piece_pixels) {
+                    const size_t count = std::min(piece_pixels, n - first);
+                    Unshuffle(shuffled.data(), n, size, first, count, out.data());
+                    piece(first, out.data(), count);
+                }
+                return false;
+            }
             stored.resize(n * size);
-            for (size_t i = 0; i < n; i++)
-                for (size_t b = 0; b < size; b++)
-                    stored[i * size + b] = shuffled[b * n + i];
+            Unshuffle(shuffled.data(), n, size, 0, n, stored.data());
         }
     } else {
         stored.resize(n * size);
@@ -920,8 +949,6 @@ bool JFJochXFELReader::ReadStored(const Frame &frame, const Layout &layout, Buff
 template <class Piece>
 void JFJochXFELReader::InflateInPieces(const std::vector<uint8_t> &compressed, size_t bytes, size_t size,
                                        const Frame &frame, Piece piece) const {
-    // A piece fits the core's own cache with the window inflate keeps, so it is converted before it leaves.
-    constexpr size_t PIECE_BYTES = 128 * 1024;
     thread_local std::vector<uint8_t> out(PIECE_BYTES);
     z_stream z{};
     if (inflateInit(&z) != Z_OK)

reader/JFJochXFELReader.h

@@ -126,7 +126,7 @@ class JFJochXFELReader : public JFJochReader {
     // The frame's pixels, stored as layout says, as stored (plain_type), or as float, into stored; with
     // keep_compressed, a chunk decoded by CompressedImage (not deflated) is left as stored in the file, and
     // true returned. Given piece, the pixels are handed to piece(first pixel, their bytes, pixels) instead: a
-    // deflated chunk a piece at a time as it is inflated (InflateInPieces), any other at once.
+    // deflated chunk a piece at a time, as it is inflated (InflateInPieces) or unshuffled, any other at once.
     template <class Buffer, class Piece = std::nullptr_t>
     bool ReadStored(const Frame &frame, const Layout &layout, Buffer &stored, bool keep_compressed,
                     Piece piece = nullptr) const;