v1.0.0-rc.156 (#66)

This is an UNSTABLE release. It includes many experimental features, as well as many AI generated fixes. We recommend using rc.152 for production use.

* jfjoch_process: Major rotation (rot3d) data processing overhaul - robust profile-fit integration, Cauchy-loss scaling with optional absorption surface, de-novo indexing and space-group/centering determination fixes, and merging statistics + ISa in the mmCIF output.
* jfjoch_process: Add EXPERIMENTAL ice-ring detection (--detect-ice-rings) that excludes ice reflections from scaling.
* Compression: Add BSHUF_ZSTD_RLE_HUFF, make compression size-aware (drop frames that don't fit rather than aborting), and add the jfjoch_recompress tool.
* jfjoch_viewer: Report "Multiple lattices detected" and grey out "Analyze dataset" on a live connection.
* jfjoch_broker: Write smargon chi/phi goniometer positions to NXmx; read sensor thickness/material from HDF5 metadata.
* CI: Build Windows (CUDA and non-CUDA) installers.Reviewed-on: #66

Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>

tests/ZSTDCompressorTest.cpp

@@ -10,6 +10,7 @@
 #include "../compression/JFJochCompressor.h"
 #include "../compression/JFJochDecompress.h"
 #include "../common/DiffractionExperiment.h"
+#include "../writer/HDF5Objects.h"
 
 TEST_CASE("JFjochZstdCompressor_Raw_Block","[ZSTD]") {
     uint8_t src[128*8];
@@ -240,7 +241,7 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") {
 
     std::vector<char> tmp(x.GetPixelsNum() * sizeof(int32_t) * 4 + 12);
 
-    auto tmp_size = compressor.Compress(tmp.data(), image);
+    auto tmp_size = compressor.Compress(tmp.data(), tmp.size(), image);
     tmp.resize(tmp_size);
 
     std::vector<int32_t> output;
@@ -249,6 +250,24 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_ZSTD","[ZSTD]") {
     REQUIRE(memcmp(image.data(), output.data(), x.GetPixelsNum() * sizeof(int32_t)) == 0);
 }
 
+TEST_CASE("JFJochCompressor_DestTooSmall_Throws","[ZSTD]") {
+    DiffractionExperiment x(DetJF4M());
+    x.Compression(CompressionAlgorithm::BSHUF_ZSTD).BitDepthImage(32).PixelSigned(true);
+
+    std::vector<int32_t> image(x.GetPixelsNum(), 345);
+
+    JFJochBitShuffleCompressor compressor(x.GetCompressionAlgorithm());
+
+    // A destination far too small for the compressed output must throw, not overflow it.
+    std::vector<char> tiny(64);
+    REQUIRE_THROWS_AS(compressor.Compress(tiny.data(), tiny.size(), image),
+                      CompressionBufferTooSmallException);
+
+    // A buffer sized to the worst case never throws.
+    std::vector<char> big(MaxCompressedSize(x.GetCompressionAlgorithm(), x.GetPixelsNum(), sizeof(int32_t)));
+    REQUIRE_NOTHROW(compressor.Compress(big.data(), big.size(), image));
+}
+
 TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") {
     DiffractionExperiment x(DetJF4M());
     x.Compression(CompressionAlgorithm::BSHUF_LZ4).BitDepthImage(32).PixelSigned(true);
@@ -262,7 +281,7 @@ TEST_CASE("JFJochCompressor_JFJochDecompressor_LZ4","[ZSTD]") {
 
     std::vector<char> tmp(x.GetPixelsNum() * sizeof(int32_t) * 4 + 12);
 
-    auto tmp_size = compressor.Compress(tmp.data(), image);
+    auto tmp_size = compressor.Compress(tmp.data(), tmp.size(), image);
     tmp.resize(tmp_size);
 
     std::vector<int32_t> output;
@@ -297,4 +316,94 @@ TEST_CASE("Bitshuffle_ZSTD","[ZSTD]") {
     REQUIRE(bshuf_decompress_zstd(compressed.data(), decompressed.data(), RAW_MODULE_SIZE, 4, 0) == out_size);
 
     REQUIRE(memcmp(image.data(), decompressed.data(), RAW_MODULE_SIZE*sizeof(uint32_t)) == 0);
-}
+}
+// ---- BSHUF_ZSTD_RLE_HUFF (RLE runs + adaptive Huffman literals, standard zstd frame) ----
+// Round-trips through JFJochDecompress (which calls stock ZSTD_decompress per block) and checks
+// the output never exceeds the advertised worst-case size. Run on good and adversarial data to
+// confirm the algorithm always produces valid output, even when the ratio is poor.
+template<class T>
+static void RequireHuffRoundTrip(const std::vector<T> &image) {
+    JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF);
+    auto compressed = compressor.Compress(image);
+
+    REQUIRE(compressed.size() <= (size_t) MaxCompressedSize(CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF,
+                                                            image.size(), sizeof(T)));
+    std::vector<T> output;
+    REQUIRE_NOTHROW(JFJochDecompress(output, CompressionAlgorithm::BSHUF_ZSTD_RLE_HUFF,
+                                     compressed, image.size()));
+    REQUIRE(output.size() == image.size());
+    REQUIRE(memcmp(image.data(), output.data(), image.size() * sizeof(T)) == 0);
+}
+
+TEST_CASE("ZstdHuff_PureZeros", "[ZSTD]") {
+    std::vector<int16_t> image(200000, 0);            // all runs, no literals
+    RequireHuffRoundTrip(image);
+}
+
+TEST_CASE("ZstdHuff_Poisson10", "[ZSTD]") {
+    std::vector<uint16_t> image(200000);
+    std::mt19937 gen(12345);
+    std::poisson_distribution<int> dist(10);
+    for (auto &v : image) v = (uint16_t) dist(gen);
+    RequireHuffRoundTrip(image);
+}
+
+TEST_CASE("ZstdHuff_RandomIncompressible", "[ZSTD]") {
+    std::vector<uint16_t> image(200000);              // Mersenne-Twister noise: ~zero compression
+    std::mt19937 gen(98765);
+    for (auto &v : image) v = (uint16_t) gen();
+    RequireHuffRoundTrip(image);                      // must still be valid and within bound
+}
+
+TEST_CASE("ZstdHuff_MaskLike", "[ZSTD]") {
+    std::vector<uint32_t> image(200000, 0);           // extreme sparsity like a pixel_mask
+    std::mt19937 gen(555);
+    for (auto &v : image) {
+        uint32_t r = gen() % 100;
+        if (r < 5) v = 1;
+        else if (r == 5) v = 0x80000000u;             // module-gap style flag
+    }
+    RequireHuffRoundTrip(image);
+}
+
+TEST_CASE("ZstdHuff_LysoImage", "[ZSTD]") {
+    RegisterHDF5Filter();   // bitshuffle filter, needed to read the compressed benchmark dataset
+    HDF5ReadOnlyFile data("../../tests/test_data/compression_benchmark.h5");
+    HDF5DataSet dataset(data, "/entry/data/data");
+    HDF5DataSpace file_space(dataset);
+    auto dims = file_space.GetDimensions();
+    std::vector<int16_t> image(dims[1] * dims[2]);
+    std::vector<hsize_t> start = {0, 0, 0}, size = {1, dims[1], dims[2]};
+    dataset.ReadVector(image, start, size);
+    RequireHuffRoundTrip(image);
+}
+
+// On-the-record proof that a Compressed_Block with Huffman literals and Number_of_Sequences = 0 is
+// a valid Zstandard block that outputs its literals. Per zstd_compression_format.md: "if
+// (Number_of_Sequences == 0) ... Block's decompressed content is defined solely by the Literals
+// Section content." We build exactly such a block and confirm stock ZSTD_decompress returns it.
+TEST_CASE("ZstdHuff_LiteralsOnly_NbSeqZero", "[ZSTD]") {
+    std::vector<uint8_t> lits(4096);
+    std::mt19937 gen(7);
+    for (auto &b : lits) b = (uint8_t)(gen() % 6 + 1);   // 6-symbol alphabet: Huffman-compressible, no 0x00/0xFF runs
+
+    JFJochZstdHuffCompressor huff;
+    std::vector<uint8_t> frame(lits.size() * 2 + 4096);
+    size_t fsz = huff.Compress(frame.data(), (const uint64_t *) lits.data(), lits.size());
+
+    // Frame = Magic(4) + Frame_Header_Descriptor(1) + Frame_Content_Size(4) = 9 bytes, then one Block.
+    // Block_Header (3 bytes, little-endian): [ Last_Block:1 | Block_Type:2 | Block_Size:21 ].
+    uint32_t bh = frame[9] | (frame[10] << 8) | (frame[11] << 16);
+    bool     last_block = bh & 1;
+    uint32_t block_type = (bh >> 1) & 3;
+    uint32_t block_size = bh >> 3;
+    REQUIRE(last_block);                            // a single block for this all-literal input
+    REQUIRE(block_type == 2);                       // Compressed_Block (Huffman literals path, not Raw/RLE)
+    REQUIRE(frame[12 + block_size - 1] == 0x00);    // its Sequences_Section is Number_of_Sequences = 0
+
+    std::vector<uint8_t> out(lits.size());
+    size_t r = ZSTD_decompress(out.data(), out.size(), frame.data(), fsz);
+    REQUIRE_FALSE(ZSTD_isError(r));
+    REQUIRE(r == lits.size());
+    REQUIRE(memcmp(lits.data(), out.data(), lits.size()) == 0);
+}