v1.0.0-rc.134 (#43)

This is an UNSTABLE release. The release has significant modifications and bug fixes, if things go wrong, it is better to revert to 1.0.0-rc.132. * jfjoch_broker: Add better locking for detector object - should help, when detector initialization takes too long * jfjoch_writer: Enable writing single, integrated HDF5 file with both data and metadata * XDS plugin: Add generation of Jungfraujoch plugin for XDS * CI: Add tests with XDS and DIALS (`xia2.ssx`) Reviewed-on: #43
2026-04-09 13:30:47 +02:00
parent 81bd9a06a1
commit 4a852b4d6b
174 changed files with 1981 additions and 325 deletions
@@ -13,6 +13,64 @@

 using namespace std::literals::chrono_literals;

+TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
+    {
+        HDF5File file("scratch_group_reopen.h5");
+
+        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
+        REQUIRE(file.Exists("/group1"));
+
+        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
+        REQUIRE(file.Exists("/group1"));
+
+        REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
+    }
+
+    remove("scratch_group_reopen.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
+TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
+    const std::string first_value = "abc";
+    const std::string second_value = "a much longer attribute value";
+
+    {
+        HDF5File file("scratch_attr_string_update.h5");
+        REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
+        REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
+    }
+
+    {
+        HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
+        REQUIRE(file.ReadAttrStr("str_attr") == second_value);
+    }
+
+    remove("scratch_attr_string_update.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
+TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
+    const int64_t first_value = -1234567890123LL;
+    const int64_t second_value = 9876543210123LL;
+
+    {
+        HDF5File file("scratch_attr_int64_update.h5");
+        REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
+        REQUIRE(file.ReadAttrInt("int_attr") == first_value);
+
+        REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
+        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
+    }
+
+    {
+        HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
+        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
+    }
+
+    remove("scratch_attr_int64_update.h5");
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+}
+
 TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
    uint16_t tmp_scalar = 16788;
    {
@@ -759,6 +817,274 @@ TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
 }

+TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") {
+    DiffractionExperiment x(DetJF(1));
+
+    x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
+     .FilePrefix("integrated_basic");
+    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
+
+    // NXmxIntegrated forces all images into one file
+    REQUIRE(x.GetImagesPerFile() == x.GetImageNum());
+
+    {
+        RegisterHDF5Filter();
+
+        StartMessage start_message;
+        x.FillMessage(start_message);
+
+        REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated);
+        // images_per_file should equal total images for integrated
+        REQUIRE(start_message.images_per_file == x.GetImageNum());
+
+        EndMessage end_message;
+        end_message.max_image_number = x.GetImageNum();
+
+        FileWriter writer(start_message);
+        std::vector<int16_t> image(x.GetPixelsNum(), 42);
+
+        for (int i = 0; i < x.GetImageNum(); i++) {
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.number = i;
+            REQUIRE_NOTHROW(writer.Write(message));
+        }
+
+        writer.WriteHDF5(end_message);
+        auto stats = writer.Finalize();
+
+        // All images in one file — only one stats entry
+        REQUIRE(stats.size() == 1);
+        REQUIRE(stats[0].total_images == x.GetImageNum());
+    }
+
+    // Single integrated file, no separate master or data files
+    REQUIRE(!std::filesystem::exists("integrated_basic.h5"));
+    REQUIRE(std::filesystem::exists("integrated_basic_master.h5"));
+    REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5"));
+
+    {
+        HDF5ReadOnlyFile file("integrated_basic_master.h5");
+
+        // Data should be directly in the file
+        std::unique_ptr<HDF5DataSet> dataset;
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
+        HDF5DataSpace file_space(*dataset);
+        REQUIRE(file_space.GetNumOfDimensions() == 3);
+        REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum());
+        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
+        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
+
+        // Master metadata should also be present
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/instrument/detector/beam_center_x"));
+        REQUIRE(dataset->ReadScalar<float>() == Catch::Approx(x.GetBeamX_pxl()));
+
+        // No external links (unlike NXmxLegacy)
+        REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data_000001"));
+    }
+
+    // No leftover HDF5 objects
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+    remove("integrated_basic_master.h5");
+}
+
+TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") {
+    DiffractionExperiment x(DetJF(1));
+
+    x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
+     .FilePrefix("integrated_spots");
+    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
+
+    {
+        RegisterHDF5Filter();
+
+        StartMessage start_message;
+        x.FillMessage(start_message);
+
+        EndMessage end_message;
+        end_message.max_image_number = x.GetImageNum();
+
+        FileWriter writer(start_message);
+        std::vector<int16_t> image(x.GetPixelsNum(), 10);
+
+        std::vector<SpotToSave> spots;
+        spots.push_back({10.0f, 20.0f, 100.0f});
+        spots.push_back({30.0f, 40.0f, 200.0f});
+
+        for (int i = 0; i < x.GetImageNum(); i++) {
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.spots = spots;
+            message.number = i;
+            message.image_collection_efficiency = 1.0f;
+            REQUIRE_NOTHROW(writer.Write(message));
+        }
+
+        writer.WriteHDF5(end_message);
+        auto stats = writer.Finalize();
+        REQUIRE(stats.size() == 1);
+    }
+
+    REQUIRE(std::filesystem::exists("integrated_spots_master.h5"));
+    {
+        HDF5ReadOnlyFile file("integrated_spots_master.h5");
+
+        // Detector plugin data should exist in the same file
+        REQUIRE(file.Exists("/entry/detector"));
+
+        // Image data should exist
+        std::unique_ptr<HDF5DataSet> dataset;
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
+    }
+
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+    remove("integrated_spots_master.h5");
+}
+
+TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") {
+    DiffractionExperiment x(DetJF(1));
+
+    x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
+     .FilePrefix("integrated_zero");
+    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
+
+    {
+        RegisterHDF5Filter();
+
+        StartMessage start_message;
+        x.FillMessage(start_message);
+
+        EndMessage end_message;
+        end_message.max_image_number = 0;
+
+        FileWriter writer(start_message);
+        // Write no images — just finalize
+        writer.WriteHDF5(end_message);
+        auto stats = writer.Finalize();
+
+        // No data files created
+        REQUIRE(stats.empty());
+    }
+
+    // Master file should still exist with metadata
+    REQUIRE(std::filesystem::exists("integrated_zero_master.h5"));
+    {
+        HDF5ReadOnlyFile file("integrated_zero_master.h5");
+        REQUIRE(file.Exists("/entry"));
+        // No data dataset since no images written
+        REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
+    }
+
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+    remove("integrated_zero_master.h5");
+}
+
+TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") {
+    DiffractionExperiment x(DetJF(1));
+
+    x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500);
+    x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
+    x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
+     .FilePrefix("integrated_azint");
+    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
+
+    PixelMask pixel_mask(x);
+    AzimuthalIntegration mapping(x, pixel_mask);
+
+    {
+        RegisterHDF5Filter();
+
+        StartMessage start_message;
+        x.FillMessage(start_message);
+        start_message.az_int_bin_to_q = mapping.GetBinToQ();
+        start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
+        start_message.az_int_q_bin_count = mapping.GetQBinCount();
+
+        EndMessage end_message;
+        end_message.max_image_number = x.GetImageNum();
+
+        FileWriter writer(start_message);
+        std::vector<int16_t> image(x.GetPixelsNum(), 5);
+
+        for (int i = 0; i < x.GetImageNum(); i++) {
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), static_cast<float>(i));
+            message.number = i;
+            REQUIRE_NOTHROW(writer.Write(message));
+        }
+
+        writer.WriteHDF5(end_message);
+        auto stats = writer.Finalize();
+        REQUIRE(stats.size() == 1);
+    }
+
+    REQUIRE(std::filesystem::exists("integrated_azint_master.h5"));
+    {
+        HDF5ReadOnlyFile file("integrated_azint_master.h5");
+
+        // Azimuthal integration bin mapping should exist (written by plugin)
+        std::unique_ptr<HDF5DataSet> dataset;
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/bin_to_q"));
+
+        // Per-image azint data should exist
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/image"));
+        HDF5DataSpace space(*dataset);
+        REQUIRE(space.GetNumOfDimensions() == 3);
+        REQUIRE(space.GetDimensions()[0] == x.GetImageNum());
+    }
+
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+    remove("integrated_azint_master.h5");
+}
+
+TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") {
+    // Test that out-of-order image delivery works with NXmxIntegrated
+    DiffractionExperiment x(DetJF(1));
+
+    x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
+     .FilePrefix("integrated_ooo");
+    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
+
+    {
+        RegisterHDF5Filter();
+
+        StartMessage start_message;
+        x.FillMessage(start_message);
+
+        EndMessage end_message;
+        end_message.max_image_number = x.GetImageNum();
+
+        FileWriter writer(start_message);
+        std::vector<int16_t> image(x.GetPixelsNum(), 7);
+
+        // Write images out of order
+        std::vector<int> order = {3, 1, 4, 0, 2};
+        for (int idx : order) {
+            DataMessage message{};
+            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
+            message.number = idx;
+            REQUIRE_NOTHROW(writer.Write(message));
+        }
+
+        writer.WriteHDF5(end_message);
+        auto stats = writer.Finalize();
+        REQUIRE(stats.size() == 1);
+        REQUIRE(stats[0].total_images == 5);
+    }
+
+    REQUIRE(std::filesystem::exists("integrated_ooo_master.h5"));
+    {
+        HDF5ReadOnlyFile file("integrated_ooo_master.h5");
+        std::unique_ptr<HDF5DataSet> dataset;
+        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
+        HDF5DataSpace file_space(*dataset);
+        REQUIRE(file_space.GetDimensions()[0] == 5);
+    }
+
+    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
+    remove("integrated_ooo_master.h5");
+}

 TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));