Jungfraujoch/tests/HDF5WritingTest.cpp

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only

#include <catch2/catch_all.hpp>
#include <iostream>
#include <fstream>
#include <filesystem>

#include "../common/DiffractionExperiment.h"
#include "../writer/HDF5Objects.h"
#include "../writer/FileWriter.h"
#include "../image_pusher/HDF5FilePusher.h"
#include "../compression/JFJochCompressor.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include <nlohmann/json.hpp>

using namespace std::literals::chrono_literals;

TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
    {
        HDF5File file("scratch_group_reopen.h5");

        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
        REQUIRE(file.Exists("/group1"));

        REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
        REQUIRE(file.Exists("/group1"));

        REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
    }

    remove("scratch_group_reopen.h5");
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
    const std::string first_value = "abc";
    const std::string second_value = "a much longer attribute value";

    {
        HDF5File file("scratch_attr_string_update.h5");
        REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
        REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
    }

    {
        HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
        REQUIRE(file.ReadAttrStr("str_attr") == second_value);
    }

    remove("scratch_attr_string_update.h5");
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
    const int64_t first_value = -1234567890123LL;
    const int64_t second_value = 9876543210123LL;

    {
        HDF5File file("scratch_attr_int64_update.h5");
        REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
        REQUIRE(file.ReadAttrInt("int_attr") == first_value);

        REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
    }

    {
        HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
        REQUIRE(file.ReadAttrInt("int_attr") == second_value);
    }

    remove("scratch_attr_int64_update.h5");
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
    uint16_t tmp_scalar = 16788;
    {
        HDF5File file("scratch1.h5");
        file.SaveScalar("scalar", tmp_scalar);
    }
    {
        HDF5ReadOnlyFile file("scratch1.h5");
        HDF5DataSet scalar_dataset(file, "scalar");
        HDF5DataType data_type_scalar(scalar_dataset);
        HDF5DataSpace data_space_scalar(scalar_dataset);
        REQUIRE(data_type_scalar.GetElemSize() == 2);
        REQUIRE(data_space_scalar.GetNumOfDimensions() == 0);
        REQUIRE(scalar_dataset.ReadScalar<uint16_t>() == tmp_scalar);
        REQUIRE(scalar_dataset.ReadScalar<int64_t>() == tmp_scalar);
        std::vector<uint16_t> v;
        REQUIRE_NOTHROW(scalar_dataset.ReadVector(v));
        REQUIRE(v.size() == 1);
        REQUIRE(v[0] == tmp_scalar);
    }
    remove("scratch1.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
    std::string tmp_string = "HDF5Content";

    {
        HDF5File file("scratch2.h5");
        file.SaveScalar("str", tmp_string);
    }
    {
        HDF5ReadOnlyFile file("scratch2.h5");
        HDF5DataSet string_dataset(file, "str");
        HDF5DataType data_type_str(string_dataset);
        HDF5DataSpace data_space_str(string_dataset);
        CHECK(data_type_str.GetElemSize() == tmp_string.size() + 1);
        CHECK(data_space_str.GetNumOfDimensions() == 0);
        CHECK(string_dataset.ReadString() == tmp_string);
    }
    remove("scratch2.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
    std::vector<double> tmp_vector (16384);
    tmp_vector[0] = 599.88;
    tmp_vector[1000] = 800.12;
    tmp_vector[15000] = 3.1415926;

    {
        RegisterHDF5Filter();
        HDF5File file("scratch3.h5");
        file.SaveVector("vec", tmp_vector);
    }

    {
        HDF5ReadOnlyFile file("scratch3.h5");
        HDF5DataSet vector_dataset(file, "vec");
        HDF5DataType data_type_vec(vector_dataset);
        HDF5DataSpace data_space_vec(vector_dataset);
        CHECK(data_type_vec.GetElemSize() == 8);
        CHECK(data_space_vec.GetNumOfDimensions() == 1);
        CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
        std::vector<double> output;
        REQUIRE_NOTHROW(vector_dataset.ReadVector(output));
        CHECK (output[0] == tmp_vector[0]);
        CHECK (output[1000] == tmp_vector[1000]);
        CHECK (output[15000] == tmp_vector[15000]);
        CHECK (output.size() == tmp_vector.size());

        std::vector<double> output2(10);
        REQUIRE_NOTHROW(vector_dataset.ReadVector(output2, {15000},{10}));
        REQUIRE_THROWS(vector_dataset.ReadVector(output2, {15000},{100}));
        CHECK(output2[0] == tmp_vector[15000]);
    }

    remove("scratch3.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_chunking", "[HDF5][Unit]") {
    size_t xpixel = 512;
    size_t ypixel = 256;

    std::vector<int32_t> tmp(xpixel*ypixel, -134);
    {
        RegisterHDF5Filter();
        HDF5File file("scratch4.h5");
        HDF5Dcpl dcpl;
        HDF5DataType data_type(4, true);
        dcpl.SetChunking( {1, ypixel, xpixel});
        HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
        HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
        dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {0, 0, 0});
        dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {2, 0, 0});
    }

    {
        HDF5ReadOnlyFile file("scratch4.h5");
        HDF5DataSet vector_dataset(file, "/data");
        HDF5DataType data_type_vec(vector_dataset);
        HDF5Dcpl dcpl(vector_dataset);
        HDF5DataSpace data_space_vec(vector_dataset);
        CHECK(data_type_vec.GetElemSize() == 4);
        CHECK(data_space_vec.GetNumOfDimensions() == 3);
        CHECK(data_space_vec.GetDimensions()[0] == 3);
        CHECK(data_space_vec.GetDimensions()[1] == ypixel);
        CHECK(data_space_vec.GetDimensions()[2] == xpixel);
        CHECK(dcpl.GetNumOfDimensions() == 3);
        CHECK(dcpl.GetChunking()[0] == 1);
        CHECK(dcpl.GetChunking()[1] == ypixel);
        CHECK(dcpl.GetChunking()[2] == xpixel);
        REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::NO_COMPRESSION);
        std::vector<uint8_t> read_v;
        REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
        REQUIRE(read_v.size() == xpixel * ypixel * sizeof(uint32_t));
        CHECK(memcmp(read_v.data(), tmp.data(), xpixel * ypixel * sizeof(uint32_t)) == 0);
    }

    remove("scratch4.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_chunking_bslz4", "[HDF5][Unit]") {
    size_t xpixel = 512;
    size_t ypixel = 256;
    size_t len = 1234;
    std::vector<uint8_t> tmp(len, 200);
    {
        RegisterHDF5Filter();
        HDF5File file("scratch5.h5");
        HDF5Dcpl dcpl;
        HDF5DataType data_type(4, true);
        dcpl.SetChunking( {1, ypixel, xpixel});
        dcpl.SetCompression(CompressionAlgorithm::BSHUF_LZ4, 0);
        HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
        HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
        dataset.WriteDirectChunk(tmp.data(), 1234, {0, 0, 0});
        dataset.WriteDirectChunk(tmp.data(), 1234, {2, 0, 0});
    }

    {
        HDF5ReadOnlyFile file("scratch5.h5");
        HDF5DataSet vector_dataset(file, "/data");
        HDF5DataType data_type_vec(vector_dataset);
        HDF5Dcpl dcpl(vector_dataset);
        HDF5DataSpace data_space_vec(vector_dataset);
        CHECK(data_type_vec.GetElemSize() == 4);
        CHECK(data_space_vec.GetNumOfDimensions() == 3);
        CHECK(data_space_vec.GetDimensions()[0] == 3);
        CHECK(data_space_vec.GetDimensions()[1] == ypixel);
        CHECK(data_space_vec.GetDimensions()[2] == xpixel);
        CHECK(dcpl.GetNumOfDimensions() == 3);
        CHECK(dcpl.GetChunking()[0] == 1);
        CHECK(dcpl.GetChunking()[1] == ypixel);
        CHECK(dcpl.GetChunking()[2] == xpixel);
        REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::BSHUF_LZ4);
        std::vector<uint8_t> read_v;
        REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
        REQUIRE(read_v == tmp);
    }

    remove("scratch5.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_chunking_ReadToU8", "[HDF5][Unit]") {
    size_t xpixel = 512;
    size_t ypixel = 256;
    size_t len = xpixel*ypixel;
    std::vector<uint32_t> tmp(len, 200);
    {
        RegisterHDF5Filter();
        HDF5File file("scratch5.h5");
        HDF5Dcpl dcpl;
        HDF5DataType data_type(4, true);
        dcpl.SetChunking( {1, ypixel, xpixel});
        dcpl.SetCompression(CompressionAlgorithm::NO_COMPRESSION, 0);
        HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
        HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
        dataset.WriteDirectChunk(tmp.data(), xpixel * ypixel * sizeof(uint32_t), {1, 0, 0});
    }

    {
        HDF5ReadOnlyFile file("scratch5.h5");
        HDF5DataSet vector_dataset(file, "/data");

        std::vector<uint8_t> read_v;
        REQUIRE_NOTHROW(vector_dataset.ReadVectorToU8(read_v, {1,0,0}, {1, ypixel, xpixel}));
        REQUIRE(read_v.size() == xpixel * ypixel *sizeof(uint32_t));
        REQUIRE(memcmp(read_v.data(), tmp.data(), read_v.size()) == 0);
    }

    remove("scratch5.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
    std::string long_string = "ccdcsdcdscsdcsdcs";
    std::vector<std::string> tmp_vector = {"aaaaaaaaa", "b", long_string};

    {
        RegisterHDF5Filter();
        HDF5File file("scratch4.h5");
        REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
    }

    {
        HDF5ReadOnlyFile file("scratch4.h5");
        HDF5DataSet vector_dataset(file, "vec");
        HDF5DataType data_type_vec(vector_dataset);
        HDF5DataSpace data_space_vec(vector_dataset);
        CHECK(data_type_vec.GetElemSize() == long_string.size() + 1);
        CHECK(data_space_vec.GetNumOfDimensions() == 1);
        CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
    }

    remove("scratch4.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Attr", "[HDF5][Unit]") {
    uint16_t tmp_scalar = 16788;

    std::vector<double> vec = {0,1,2,3,4,6788.0};
    std::string sattr = "val";
    double dattr = 456.567567;
    int32_t iattr = 115;

    {
        HDF5File file("scratch32.h5");
        file.SaveScalar("scalar", tmp_scalar)
        ->Attr("int", iattr)
        .Attr("double", dattr)
        .Attr("str", sattr)
        .Attr("vec", vec);
    }
    {
        HDF5ReadOnlyFile file("scratch32.h5");
        HDF5DataSet scalar_dataset(file, "scalar");
        CHECK(scalar_dataset.ReadAttrDouble("double") == dattr);
        CHECK(scalar_dataset.ReadAttrInt("int") == iattr);
        CHECK(scalar_dataset.ReadAttrStr("str") == sattr);
        CHECK(scalar_dataset.ReadAttrVec("vec") == vec);
        REQUIRE_THROWS(scalar_dataset.ReadAttrStr("int"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrStr("double"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrStr("vec"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrVec("int"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrVec("double"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrDouble("vec"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrInt("vec"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrInt("str"));
        REQUIRE_THROWS(scalar_dataset.ReadAttrInt("bla"));
    }
    remove("scratch32.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5LeafDetection", "[HDF5]") {
    {
        RegisterHDF5Filter();
        HDF5File file("scratch5.h5");
        HDF5Group(file, "/group");
        HDF5Group(file, "/group/sub1");
        HDF5Group(file, "/group/sub2");
        file.SaveScalar("/group/scalar", 5.0);
    }
    {
        HDF5ReadOnlyFile file("scratch5.h5");
        auto vec = file.FindLeafs("/group");
        REQUIRE(vec.size() == 3);
        bool found0 = false, found1 = false;
        for (int i = 0; i < vec.size(); i++) {
            if (vec[i] == "sub1")
                found0 = true;
            if (vec[i] == "scalar")
                found1 = true;
        }
        REQUIRE(found0);
        REQUIRE(found1);
    }

    remove("scratch5.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5ExternalLink", "[HDF5][Unit]") {
    uint16_t tmp_scalar = 16788;
    {
        HDF5File file("scratch123.h5");
        file.ExternalLink("../abc.h5", "/zzz/fgh6", "fgh6");
    }
    {
        HDF5ReadOnlyFile file("scratch123.h5");
        REQUIRE(file.GetLinkedFileName("/fgh6") == "abc.h5");
    }
    remove("scratch123.h5");
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());

        x.FilePrefix("test01").ImagesPerTrigger(950);

        StartMessage start_message;
        x.FillMessage(start_message);
        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();
        std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
        master->Finalize(end_message);
        master.reset();

        x.FilePrefix("test02");
        x.FillMessage(start_message);
        master = std::make_unique<NXmx>(start_message);
        master->Finalize(end_message);
        master.reset();
    }
    remove("test01_master.h5");
    remove("test02_master.h5");
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5MasterFile_UserData", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());

        x.FilePrefix("test07").ImagesPerTrigger(950);

        StartMessage start_message;
        x.FillMessage(start_message);
        start_message.user_data["hdf5"]["val1"] = 7;
        start_message.user_data["hdf5"]["val2"] = "str";

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();
        std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
        master->Finalize(end_message);
        master.reset();
    }
    {
        HDF5ReadOnlyFile file("test07_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val1"));
        REQUIRE(dataset->ReadScalar<double>() == 7.0);

        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val2"));
        REQUIRE(dataset->ReadString() == "str");
    }
    remove("test07_master.h5");
    // No leftover HDF5 objects
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5MasterFile_RadInt", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());

        x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
        x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);

        x.FilePrefix("test01_rad_int").ImagesPerTrigger(950);

        PixelMask pixel_mask(x);
        AzimuthalIntegrationMapping mapping(x, pixel_mask);
        AzimuthalIntegrationProfile profile(mapping);

        StartMessage start_message;
        x.FillMessage(start_message);
        start_message.az_int_bin_to_q = mapping.GetBinToQ();
        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();
        end_message.az_int_result["avg1"] = profile.GetResult();
        end_message.az_int_result["avg2"] = profile.GetResult();

        std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
        master->Finalize(end_message);
        master.reset();
    }
    remove("test01_rad_int_master.h5");
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Writer", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());
        std::vector<SpotToSave> spots;

        x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
        .OverwriteExistingFiles(true);

        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        std::vector<uint16_t> image(x.GetPixelsNum());

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image,  x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }

        auto v = file_set.Finalize();
        REQUIRE(v.size() == 3); // 3 files
        REQUIRE(v[0].filename == "test02_1p10_data_000001.h5");
        REQUIRE(v[0].total_images == 2);
        REQUIRE(v[1].filename == "test02_1p10_data_000002.h5");
        REQUIRE(v[1].total_images == 2);
        REQUIRE(v[2].filename == "test02_1p10_data_000003.h5");
        REQUIRE(v[2].total_images == 1);

        REQUIRE(!file_set.GetZMQAddr());
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("test02_1p10_data_000001.h5");
    remove("test02_1p10_data_000002.h5");
    remove("test02_1p10_data_000003.h5");
}

TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());
        x.FrameTime(std::chrono::microseconds(1000), std::chrono::microseconds(100));

        DatasetSettings d;
        d.FilePrefix("run0345_lysozyme_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
                .HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
                .PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
                .SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("lysozyme").RunName("run1");

        x.ImportDatasetSettings(d).OverwriteExistingFiles(true);
        std::vector<SpotToSave> spots;

        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        file_set.SetupFinalizedFileSocket("ipc://#1");
        std::vector<uint16_t> image(x.GetPixelsNum());

        ZMQSocket s(ZMQSocketType::Sub);
        s.Connect("ipc://#1");
        s.SubscribeAll();
        s.ReceiveTimeout(std::chrono::seconds(5));

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image,  x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }
        REQUIRE(file_set.Finalize().size() == 3);

        ZMQMessage msg;
        nlohmann::json j;

        REQUIRE(s.Receive(msg, true));

        j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
        REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000001.h5");
        REQUIRE(j["file_number"] == 1);
        REQUIRE(j["nimages"] == 2);
        REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetIncidentEnergy_keV() * 1000.0));
        REQUIRE(j["space_group_number"] == 96);
        REQUIRE(j["experiment_group"] == "p12345");
        REQUIRE(j["run_number"] == 345);
        REQUIRE(j["run_name"] == "run1");
        REQUIRE(j.contains("user_data"));
        REQUIRE(j["user_data"]["z"] == 567);
        std::cout << j.dump(4) << std::endl;

        REQUIRE(s.Receive(msg, true));
        j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
        REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000002.h5");
        REQUIRE(j["file_number"] == 2);
        REQUIRE(j["nimages"] == 2);
        REQUIRE(j.contains("user_data"));
        REQUIRE(j["user_data"]["z"] == 567);

        REQUIRE(s.Receive(msg, true));
        j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
        REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000003.h5");
        REQUIRE(j["file_number"] == 3);
        REQUIRE(j["nimages"] == 1);
        REQUIRE(j.contains("user_data"));
        REQUIRE(j["user_data"]["z"] == 567);
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);

    remove("test05_data_000001.h5");
    remove("test05_data_000002.h5");
    remove("test05_data_000003.h5");
}

TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());
        std::vector<SpotToSave> spots;

        spots.push_back({10,10,7});
        spots.push_back({20,50,12});
        spots.push_back({1000,500,3});

        x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        std::vector<uint16_t> image(x.GetPixelsNum());

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image,  x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);

    remove("test02_1p10_spots_data_000001.h5");
    remove("test02_1p10_spots_data_000002.h5");
}

TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
    {
        RegisterHDF5Filter();
        DiffractionExperiment x(DetJF4M());
        x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
        x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);

        PixelMask pixel_mask(x);
        AzimuthalIntegrationMapping mapping(x, pixel_mask);

        std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
        std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);

        x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION).OverwriteExistingFiles(true);
        StartMessage start_message;
        x.FillMessage(start_message);
        start_message.az_int_bin_to_q = mapping.GetBinToQ();
        start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
        start_message.az_int_q_bin_count = mapping.GetQBinCount();

        FileWriter file_set(start_message);
        std::vector<uint16_t> image(x.GetPixelsNum());

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), i);
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
    StartMessage message;
    message.file_prefix = "z/x";
    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
    REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
}

TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
    StartMessage message;
    message.file_prefix = "acq";
    message.source_name = "SwissFEL";
    message.detector_serial_number = "JF17T16V01";
    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
    REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
    REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));

    message.detector_serial_number = "";
    REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
}

TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {
    REQUIRE(ExtractFilename("filename_data_000001.h5") == "filename_data_000001.h5");
    REQUIRE(ExtractFilename("dir1/filename_data_000001.h5") == "filename_data_000001.h5");
    REQUIRE(ExtractFilename("dir1/dir2/filename_data_000001.h5") == "filename_data_000001.h5");
}

TEST_CASE("HDF5DataType", "[HDF5]") {
    HDF5DataType type1(1,true);
    REQUIRE(type1.GetElemSize() == 1);

    HDF5DataType type2(2,true);
    REQUIRE(type2.GetElemSize() == 2);

    HDF5DataType type4(4,true);
    REQUIRE(type4.GetElemSize() == 4);

    HDF5DataType type8(8,true);
    REQUIRE(type8.GetElemSize() == 8);

    REQUIRE_THROWS(HDF5DataType(7,true));
}

TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link").SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
    x.OverwriteExistingFiles(true);
    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum() - 2;

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum());
        std::vector<SpotToSave> spots;
        for (int i = 0; i < x.GetImageNum() - 2; i++) {
            for (auto &j: image)
                j = i;
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(writer.Write(message));
        }
        writer.WriteHDF5(end_message);
        writer.Finalize();
    }
    {
        HDF5ReadOnlyFile file("link_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetNumOfDimensions() == 3);
        REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
    }
    {
        HDF5ReadOnlyFile file("link_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000002"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetNumOfDimensions() == 3);
        REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
    }
    {
        HDF5ReadOnlyFile file("link_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000003"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetNumOfDimensions() == 3);
        REQUIRE(file_space.GetDimensions()[0] == 1);
        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
    }
    {
        HDF5ReadOnlyFile file("link_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000004"));
    }

    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_vds");
    x.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true);
    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        REQUIRE(start_message.file_format == FileWriterFormat::NXmxVDS);
        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum() - 2;

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum());
        std::vector<SpotToSave> spots;
        for (int i = 0; i < x.GetImageNum() - 2; i++) {
            for (auto &j: image)
                j = i;
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        writer.Finalize();
    }
    {
        HDF5ReadOnlyFile file("link_vds_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetNumOfDimensions() == 3);
        REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() - 2);
        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
    }
    {
        HDF5ReadOnlyFile file("link_vds_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
    }

    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
     .FilePrefix("integrated_basic");
    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);

    // NXmxIntegrated forces all images into one file
    REQUIRE(x.GetImagesPerFile() == x.GetImageNum());

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated);
        // images_per_file should equal total images for integrated
        REQUIRE(start_message.images_per_file == x.GetImageNum());

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 42);

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.number = i;
            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        auto stats = writer.Finalize();

        // All images in one file — only one stats entry
        REQUIRE(stats.size() == 1);
        REQUIRE(stats[0].total_images == x.GetImageNum());
    }

    // Single integrated file, no separate master or data files
    REQUIRE(!std::filesystem::exists("integrated_basic.h5"));
    REQUIRE(std::filesystem::exists("integrated_basic_master.h5"));
    REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5"));

    {
        HDF5ReadOnlyFile file("integrated_basic_master.h5");

        // Data should be directly in the file
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetNumOfDimensions() == 3);
        REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum());
        REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
        REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());

        // Master metadata should also be present
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/instrument/detector/beam_center_x"));
        REQUIRE(dataset->ReadScalar<float>() == Catch::Approx(x.GetBeamX_pxl()));

        // No external links (unlike NXmxLegacy)
        REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data_000001"));
    }

    // No leftover HDF5 objects
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("integrated_basic_master.h5");
}

TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
     .FilePrefix("integrated_spots");
    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 10);

        std::vector<SpotToSave> spots;
        spots.push_back({10.0f, 20.0f, 100.0f});
        spots.push_back({30.0f, 40.0f, 200.0f});

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;
            message.image_collection_efficiency = 1.0f;
            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        auto stats = writer.Finalize();
        REQUIRE(stats.size() == 1);
    }

    REQUIRE(std::filesystem::exists("integrated_spots_master.h5"));
    {
        HDF5ReadOnlyFile file("integrated_spots_master.h5");

        // Detector plugin data should exist in the same file
        REQUIRE(file.Exists("/entry/detector"));

        // Image data should exist
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
    }

    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("integrated_spots_master.h5");
}

TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
     .FilePrefix("integrated_zero");
    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = 0;

        FileWriter writer(start_message);
        // Write no images — just finalize
        writer.WriteHDF5(end_message);
        auto stats = writer.Finalize();

        // No data files created
        REQUIRE(stats.empty());
    }

    // Master file should still exist with metadata
    REQUIRE(std::filesystem::exists("integrated_zero_master.h5"));
    {
        HDF5ReadOnlyFile file("integrated_zero_master.h5");
        REQUIRE(file.Exists("/entry"));
        // No data dataset since no images written
        REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
    }

    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("integrated_zero_master.h5");
}

TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500);
    x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
    x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
     .FilePrefix("integrated_azint");
    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);

    PixelMask pixel_mask(x);
    AzimuthalIntegrationMapping mapping(x, pixel_mask);

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);
        start_message.az_int_bin_to_q = mapping.GetBinToQ();
        start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
        start_message.az_int_q_bin_count = mapping.GetQBinCount();

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 5);

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), static_cast<float>(i));
            message.number = i;
            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        auto stats = writer.Finalize();
        REQUIRE(stats.size() == 1);
    }

    REQUIRE(std::filesystem::exists("integrated_azint_master.h5"));
    {
        HDF5ReadOnlyFile file("integrated_azint_master.h5");

        // Azimuthal integration bin mapping should exist (written by plugin)
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/bin_to_q"));

        // Per-image azint data should exist
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/image"));
        HDF5DataSpace space(*dataset);
        REQUIRE(space.GetNumOfDimensions() == 3);
        REQUIRE(space.GetDimensions()[0] == x.GetImageNum());
    }

    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("integrated_azint_master.h5");
}

TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") {
    // Test that out-of-order image delivery works with NXmxIntegrated
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
     .FilePrefix("integrated_ooo");
    x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum();

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 7);

        // Write images out of order
        std::vector<int> order = {3, 1, 4, 0, 2};
        for (int idx : order) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.number = idx;
            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        auto stats = writer.Finalize();
        REQUIRE(stats.size() == 1);
        REQUIRE(stats[0].total_images == 5);
    }

    REQUIRE(std::filesystem::exists("integrated_ooo_master.h5"));
    {
        HDF5ReadOnlyFile file("integrated_ooo_master.h5");
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
        HDF5DataSpace file_space(*dataset);
        REQUIRE(file_space.GetDimensions()[0] == 5);
    }

    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
    remove("integrated_ooo_master.h5");
}

TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));

    x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("data_only");
    x.SetFileWriterFormat(FileWriterFormat::DataOnly).OverwriteExistingFiles(true);
    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        REQUIRE(start_message.file_format == FileWriterFormat::DataOnly);
        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum() - 2;

        FileWriter writer(start_message);
        std::vector<int16_t> image(x.GetPixelsNum());
        std::vector<SpotToSave> spots;
        for (int i = 0; i < x.GetImageNum() - 2; i++) {
            for (auto &j: image)
                j = i;
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(writer.Write(message));
        }

        writer.WriteHDF5(end_message);
        writer.Finalize();
    }
    REQUIRE(!std::filesystem::exists("data_only_master.h5"));
    REQUIRE(std::filesystem::exists("data_only_data_000001.h5"));

    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(2));

    std::vector<int16_t> calib_1(x.GetModulesNum() * RAW_MODULE_SIZE);
    std::vector<float> calib_2(x.GetModulesNum() * RAW_MODULE_SIZE);
    for (int i = 0; i < x.GetModulesNum(); i++) {
        calib_1[i] = i * 3 - 1024;
        calib_2[i] = static_cast<float>(i) / 16.0 + 123.25f;
    }

    JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
    std::vector<uint8_t> calib_3 = compressor.Compress(calib_1);

    x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("calib");
    x.OverwriteExistingFiles(true);
    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = x.GetImageNum() - 2;

        FileWriter writer(start_message);

        CompressedImage image_1(calib_1, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
        CompressedImage image_2(calib_2, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
        CompressedImage image_3(calib_3, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES,
            CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4);
        image_1.Channel("calib1");
        image_2.Channel("calib2");
        image_3.Channel("calib3");

        writer.WriteHDF5(image_1);
        writer.WriteHDF5(image_2);
        writer.WriteHDF5(image_3);

        writer.WriteHDF5(end_message);
        writer.Finalize();
    }
    REQUIRE(std::filesystem::exists("calib_master.h5"));
    {
        HDF5ReadOnlyFile file("calib_master.h5");
        {
            std::unique_ptr<HDF5DataSet> dataset;
            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib1"));
            HDF5DataSpace file_space(*dataset);
            REQUIRE(file_space.GetNumOfDimensions() == 2);
            HDF5DataType type(*dataset);
            REQUIRE(type.GetElemSize() == 2);
            REQUIRE(type.IsSigned());
            REQUIRE(type.IsInteger());

            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
            std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
            CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
        }
        {
            std::unique_ptr<HDF5DataSet> dataset;
            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib2"));
            HDF5DataSpace file_space(*dataset);
            REQUIRE(file_space.GetNumOfDimensions() == 2);
            HDF5DataType type(*dataset);
            REQUIRE(type.GetElemSize() == 4);
            REQUIRE(type.IsSigned());
            REQUIRE(!type.IsInteger());

            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
            std::vector<float> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
            CHECK(memcmp(output.data(), calib_2.data(), output.size() * type.GetElemSize()) == 0);
        }
        {
            std::unique_ptr<HDF5DataSet> dataset;
            REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib3"));
            HDF5DataSpace file_space(*dataset);
            REQUIRE(file_space.GetNumOfDimensions() == 2);
            HDF5DataType type(*dataset);
            REQUIRE(type.GetElemSize() == 2);
            REQUIRE(type.IsSigned());
            REQUIRE(type.IsInteger());

            REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
            REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
            std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
            dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
            CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
        }
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}


TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
    DiffractionExperiment x(DetJF(1));
    x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero").OverwriteExistingFiles(true);

    {
        RegisterHDF5Filter();

        StartMessage start_message;
        x.FillMessage(start_message);

        EndMessage end_message;
        end_message.max_image_number = 0;

        std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
        master->Finalize(end_message);
        master.reset();
    }
    {
        std::unique_ptr<HDF5ReadOnlyFile> file;
        std::unique_ptr<HDF5DataSet> dataset;
        REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("link_zero_master.h5"));
        REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(*file,"/entry/data/data_000001"));
    }
    // No leftover HDF5 objects
    REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("FileWriter_CBF_16bit", "[HDF5][Full]") {
    {
        DetectorSetup det = DetJF4M("DET1");
        det.SerialNumber("1");
        DiffractionExperiment x(det);
        std::vector<SpotToSave> spots;

        x.FilePrefix("lyso_cbf_16").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
        x.PixelSigned(true).BitDepthImage(16);
        x.SetFileWriterFormat(FileWriterFormat::CBF);
        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 45);

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }

        auto v = file_set.Finalize();

        REQUIRE(!file_set.GetZMQAddr());
    }
}

TEST_CASE("FileWriter_CBF", "[HDF5][Full]") {
    {
        DetectorSetup det = DetJF4M("DET1");
        det.SerialNumber("1");
        DiffractionExperiment x(det);
        std::vector<SpotToSave> spots;

        x.FilePrefix("lyso_cbf").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
        x.PixelSigned(false).BitDepthImage(32);
        x.SetFileWriterFormat(FileWriterFormat::CBF);
        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        std::vector<uint32_t> image(x.GetPixelsNum());

        for (int i = 0; i < image.size(); i++)
            image[i] = i;

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }

        auto v = file_set.Finalize();

        REQUIRE(!file_set.GetZMQAddr());
    }
}

TEST_CASE("FileWriter_TIFF", "[HDF5][Full]") {
    {
        DetectorSetup det = DetJF4M("DET1");
        det.SerialNumber("1");
        DiffractionExperiment x(det);
        std::vector<SpotToSave> spots;

        x.FilePrefix("lyso_tiff").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
        x.SetFileWriterFormat(FileWriterFormat::TIFF);
        StartMessage start_message;
        x.FillMessage(start_message);

        FileWriter file_set(start_message);
        std::vector<int16_t> image(x.GetPixelsNum(), 45);

        for (int i = 0; i < x.GetImageNum(); i++) {
            DataMessage message{};
            message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
            message.spots = spots;
            message.number = i;

            REQUIRE_NOTHROW(file_set.Write(message));
        }

        auto v = file_set.Finalize();

        REQUIRE(!file_set.GetZMQAddr());
    }
}

TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
    {
        RegisterHDF5Filter();

        HDF5File file("scratch_vds_reverse_contiguous.h5", true);
        HDF5Dcpl dcpl;
        HDF5DataType data_type((int16_t) 0);
        HDF5DataSpace full_space({5, 4, 3});

        {
            HDF5DataSpace source_space({2, 4, 3});
            HDF5DataSpace virtual_space({5, 4, 3});
            virtual_space.SelectHyperslab({0, 0, 0}, {2, 4, 3});
            dcpl.SetVirtual("file_000001.h5", "/entry/data/data", source_space, virtual_space);
        }

        {
            HDF5DataSpace source_space({3, 4, 3});
            HDF5DataSpace virtual_space({5, 4, 3});
            virtual_space.SelectHyperslab({2, 0, 0}, {3, 4, 3});
            dcpl.SetVirtual("file_000002.h5", "/entry/data/data", source_space, virtual_space);
        }

        HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);

        HDF5Dcpl read_dcpl(dataset);
        auto mappings = read_dcpl.GetVirtualMappings();

        REQUIRE(mappings.size() == 2);
        REQUIRE(mappings[0].ContainsVirtualImage(0));
        REQUIRE(mappings[0].ContainsVirtualImage(1));
        REQUIRE(!mappings[0].ContainsVirtualImage(2));
        CHECK(mappings[0].SourceImage(0) == 0);
        CHECK(mappings[0].SourceImage(1) == 1);

        REQUIRE(mappings[1].ContainsVirtualImage(2));
        REQUIRE(mappings[1].ContainsVirtualImage(4));
        CHECK(mappings[1].SourceImage(2) == 0);
        CHECK(mappings[1].SourceImage(3) == 1);
        CHECK(mappings[1].SourceImage(4) == 2);
    }

    remove("scratch_vds_reverse_contiguous.h5");
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

TEST_CASE("HDF5Objects_VDS_reverse_strided", "[HDF5][Unit]") {
    {
        RegisterHDF5Filter();

        HDF5File file("scratch_vds_reverse_strided.h5", true);
        HDF5Dcpl dcpl;
        HDF5DataType data_type((int16_t) 0);
        HDF5DataSpace full_space({6, 4, 3});

        {
            HDF5DataSpace source_space({3, 4, 3});
            HDF5DataSpace virtual_space({6, 4, 3});
            virtual_space.SelectHyperslabWithStride({0, 0, 0}, {3, 4, 3}, {2, 1, 1});
            dcpl.SetVirtual("file_even.h5", "/entry/data/data", source_space, virtual_space);
        }

        {
            HDF5DataSpace source_space({3, 4, 3});
            HDF5DataSpace virtual_space({6, 4, 3});
            virtual_space.SelectHyperslabWithStride({1, 0, 0}, {3, 4, 3}, {2, 1, 1});
            dcpl.SetVirtual("file_odd.h5", "/entry/data/data", source_space, virtual_space);
        }

        HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);

        HDF5Dcpl read_dcpl(dataset);
        auto mappings = read_dcpl.GetVirtualMappings();

        REQUIRE(mappings.size() == 2);

        REQUIRE(mappings[0].ContainsVirtualImage(0));
        REQUIRE(mappings[0].ContainsVirtualImage(2));
        REQUIRE(mappings[0].ContainsVirtualImage(4));
        REQUIRE(!mappings[0].ContainsVirtualImage(1));
        CHECK(mappings[0].SourceImage(0) == 0);
        CHECK(mappings[0].SourceImage(2) == 1);
        CHECK(mappings[0].SourceImage(4) == 2);

        REQUIRE(mappings[1].ContainsVirtualImage(1));
        REQUIRE(mappings[1].ContainsVirtualImage(3));
        REQUIRE(mappings[1].ContainsVirtualImage(5));
        REQUIRE(!mappings[1].ContainsVirtualImage(0));
        CHECK(mappings[1].SourceImage(1) == 0);
        CHECK(mappings[1].SourceImage(3) == 1);
        CHECK(mappings[1].SourceImage(5) == 2);
    }

    remove("scratch_vds_reverse_strided.h5");
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// Overwrite is detected up front for back-channel transports (default) by checking
// the master file only - never the staggered per-writer data files. The ZeroMQ path
// (no back-channel) must opt out and keep writing .tmp files instead.
TEST_CASE("FileWriter_overwrite_detected_at_start", "[HDF5][Overwrite]") {
    RegisterHDF5Filter();

    DiffractionExperiment x(DetJF4M());
    x.FilePrefix("fw_overwrite_start").ImagesPerTrigger(3).ImagesPerFile(2)
     .Compression(CompressionAlgorithm::NO_COMPRESSION)
     .SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(false);

    StartMessage start_message;
    x.FillMessage(start_message);
    REQUIRE(start_message.write_master_file.value_or(false)); // this writer owns the master

    // A stray data file (owned by another, staggered writer) must NOT trip the check -
    // only the master file is inspected. The temporary writer cleans up its own tmp.
    { std::ofstream(HDF5Metadata::DataFileName(start_message, 0)) << "blocker"; }
    REQUIRE_NOTHROW(FileWriter(start_message));
    remove(HDF5Metadata::DataFileName(start_message, 0).c_str());

    // The master file, on the other hand, does collide.
    { std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }

    // Back-channel transport (direct HDF5 / TCP): fail fast in the constructor.
    REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);

    // ZeroMQ transport (no back-channel): must not throw - it will write .tmp and
    // only fail at the final rename. The un-finalized writer cleans up its own tmp.
    REQUIRE_NOTHROW(FileWriter(start_message, /*check_overwrite_at_start=*/false));

    remove(HDF5Metadata::MasterFileName(start_message).c_str());
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}

// If EndDataCollection throws while finalizing (here the master file appears mid-run,
// so the early check can't catch it), the pusher must still tear down its writer so a
// subsequent collection can start instead of dying with "already writing images".
TEST_CASE("HDF5FilePusher_finalize_failure_recovers", "[HDF5FilePusher][Repro]") {
    RegisterHDF5Filter();

    DiffractionExperiment x(DetJF4M());
    x.FilePrefix("pusher_finalize_repro").ImagesPerTrigger(1)
     .Compression(CompressionAlgorithm::NO_COMPRESSION)
     .SetFileWriterFormat(FileWriterFormat::NXmxVDS)
     .OverwriteExistingFiles(false);

    StartMessage start_message;
    x.FillMessage(start_message);
    EndMessage end_message{};

    HDF5FilePusher pusher;

    pusher.StartDataCollection(start_message);
    // Create the conflict after the start-time check has already passed.
    { std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
    REQUIRE_THROWS_AS(pusher.EndDataCollection(end_message), JFJochException);

    // Writer released despite the failure: the next collection starts cleanly
    // instead of dying with "already writing images".
    remove(HDF5Metadata::MasterFileName(start_message).c_str());
    REQUIRE_NOTHROW(pusher.StartDataCollection(start_message));
    REQUIRE_NOTHROW(pusher.EndDataCollection(end_message));

    // The failed finalize intentionally leaves a .tmp behind - sweep the prefix.
    for (const auto &e : std::filesystem::directory_iterator("."))
        if (e.path().filename().string().rfind("pusher_finalize_repro", 0) == 0)
            std::filesystem::remove(e.path());
    REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}