mx/Jungfraujoch
2
1
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases 71 Wiki Activity
Files
1f4f77fe4202f4e89331a1115bf8ecf7a0521d25
Jungfraujoch/tests/HDF5WritingTest.cpp
T
leonarski_fandClaude Opus 5 259b43154e Writer: refuse a mistyped stream, and mark unreadable VDS frames 
Two ways the written files could misdescribe themselves without anyone noticing.

The pixel format is stated twice and the two were never compared: each image
carries its own type as a CBOR tag, which is what the data files are written
with, while the master is typed from the start message. A stream whose header
contradicts its images produced data files of one type under a master declaring
another, and with NXmxVDS, HDF5 then converts silently on every read.
HDF5DataFile::CreateFile now checks the two agree and refuses the run otherwise -
the point where the values first meet, so it covers every path into the writer.

Two test fixtures were relying on exactly that inconsistency. The HDF5 writer
tests wrote uint16 buffers under a JUNGFRAU experiment, which converts to photon
counts by default and so declares int16; they never read the pixels back, so it
went unnoticed. The receiver-lite tests feed frames from
compression_benchmark.h5, which really are signed int16, through a DECTRIS
experiment, which declares unsigned by default - the same class of bug the
pixel_signed propagation fixed on the live path. Both now declare what they send.

Second: a virtual dataset whose source file is absent reads as the fill value,
and HDF5 defaults that to zero, so a data file that was not copied alongside the
master is indistinguishable from frames of genuine zero counts. Measured with
DIALS on a four-file set with one file removed: 25 frames of pure zeros, no error
and no warning. The image VDS is now filled with the error marker instead, which
sits outside underload_value..saturation_value, so a reader masks those frames.
Same measurement after the change: -32768 throughout, which DIALS excludes.

Only the images ask for a fill value; the per-image metadata datasets keep the
default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-22 21:44:04 +02:00

1424 lines
53 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <iostream>
#include <fstream>
#include <filesystem>
#include "../common/DiffractionExperiment.h"
#include "../writer/HDF5Objects.h"
#include "../writer/FileWriter.h"
#include "../image_pusher/HDF5FilePusher.h"
#include "../compression/JFJochCompressor.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include <nlohmann/json.hpp>
using namespace std::literals::chrono_literals;
TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
{
HDF5File file("scratch_group_reopen.h5");
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
}
remove("scratch_group_reopen.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
const std::string first_value = "abc";
const std::string second_value = "a much longer attribute value";
{
HDF5File file("scratch_attr_string_update.h5");
REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
}
{
HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
REQUIRE(file.ReadAttrStr("str_attr") == second_value);
}
remove("scratch_attr_string_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
const int64_t first_value = -1234567890123LL;
const int64_t second_value = 9876543210123LL;
{
HDF5File file("scratch_attr_int64_update.h5");
REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
REQUIRE(file.ReadAttrInt("int_attr") == first_value);
REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
{
HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
remove("scratch_attr_int64_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch1.h5");
file.SaveScalar("scalar", tmp_scalar);
}
{
HDF5ReadOnlyFile file("scratch1.h5");
HDF5DataSet scalar_dataset(file, "scalar");
HDF5DataType data_type_scalar(scalar_dataset);
HDF5DataSpace data_space_scalar(scalar_dataset);
REQUIRE(data_type_scalar.GetElemSize() == 2);
REQUIRE(data_space_scalar.GetNumOfDimensions() == 0);
REQUIRE(scalar_dataset.ReadScalar<uint16_t>() == tmp_scalar);
REQUIRE(scalar_dataset.ReadScalar<int64_t>() == tmp_scalar);
std::vector<uint16_t> v;
REQUIRE_NOTHROW(scalar_dataset.ReadVector(v));
REQUIRE(v.size() == 1);
REQUIRE(v[0] == tmp_scalar);
}
remove("scratch1.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
std::string tmp_string = "HDF5Content";
{
HDF5File file("scratch2.h5");
file.SaveScalar("str", tmp_string);
}
{
HDF5ReadOnlyFile file("scratch2.h5");
HDF5DataSet string_dataset(file, "str");
HDF5DataType data_type_str(string_dataset);
HDF5DataSpace data_space_str(string_dataset);
CHECK(data_type_str.GetElemSize() == tmp_string.size() + 1);
CHECK(data_space_str.GetNumOfDimensions() == 0);
CHECK(string_dataset.ReadString() == tmp_string);
}
remove("scratch2.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
std::vector<double> tmp_vector (16384);
tmp_vector[0] = 599.88;
tmp_vector[1000] = 800.12;
tmp_vector[15000] = 3.1415926;
{
RegisterHDF5Filter();
HDF5File file("scratch3.h5");
file.SaveVector("vec", tmp_vector);
}
{
HDF5ReadOnlyFile file("scratch3.h5");
HDF5DataSet vector_dataset(file, "vec");
HDF5DataType data_type_vec(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 8);
CHECK(data_space_vec.GetNumOfDimensions() == 1);
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
std::vector<double> output;
REQUIRE_NOTHROW(vector_dataset.ReadVector(output));
CHECK (output[0] == tmp_vector[0]);
CHECK (output[1000] == tmp_vector[1000]);
CHECK (output[15000] == tmp_vector[15000]);
CHECK (output.size() == tmp_vector.size());
std::vector<double> output2(10);
REQUIRE_NOTHROW(vector_dataset.ReadVector(output2, {15000},{10}));
REQUIRE_THROWS(vector_dataset.ReadVector(output2, {15000},{100}));
CHECK(output2[0] == tmp_vector[15000]);
}
remove("scratch3.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
std::vector<int32_t> tmp(xpixel*ypixel, -134);
{
RegisterHDF5Filter();
HDF5File file("scratch4.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {0, 0, 0});
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {2, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch4.h5");
HDF5DataSet vector_dataset(file, "/data");
HDF5DataType data_type_vec(vector_dataset);
HDF5Dcpl dcpl(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 4);
CHECK(data_space_vec.GetNumOfDimensions() == 3);
CHECK(data_space_vec.GetDimensions()[0] == 3);
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
CHECK(dcpl.GetNumOfDimensions() == 3);
CHECK(dcpl.GetChunking()[0] == 1);
CHECK(dcpl.GetChunking()[1] == ypixel);
CHECK(dcpl.GetChunking()[2] == xpixel);
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::NO_COMPRESSION);
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
REQUIRE(read_v.size() == xpixel * ypixel * sizeof(uint32_t));
CHECK(memcmp(read_v.data(), tmp.data(), xpixel * ypixel * sizeof(uint32_t)) == 0);
}
remove("scratch4.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking_bslz4", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
size_t len = 1234;
std::vector<uint8_t> tmp(len, 200);
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
dcpl.SetCompression(CompressionAlgorithm::BSHUF_LZ4, 0);
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), 1234, {0, 0, 0});
dataset.WriteDirectChunk(tmp.data(), 1234, {2, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch5.h5");
HDF5DataSet vector_dataset(file, "/data");
HDF5DataType data_type_vec(vector_dataset);
HDF5Dcpl dcpl(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 4);
CHECK(data_space_vec.GetNumOfDimensions() == 3);
CHECK(data_space_vec.GetDimensions()[0] == 3);
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
CHECK(dcpl.GetNumOfDimensions() == 3);
CHECK(dcpl.GetChunking()[0] == 1);
CHECK(dcpl.GetChunking()[1] == ypixel);
CHECK(dcpl.GetChunking()[2] == xpixel);
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
REQUIRE(read_v == tmp);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking_ReadToU8", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
size_t len = xpixel*ypixel;
std::vector<uint32_t> tmp(len, 200);
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
dcpl.SetCompression(CompressionAlgorithm::NO_COMPRESSION, 0);
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), xpixel * ypixel * sizeof(uint32_t), {1, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch5.h5");
HDF5DataSet vector_dataset(file, "/data");
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadVectorToU8(read_v, {1,0,0}, {1, ypixel, xpixel}));
REQUIRE(read_v.size() == xpixel * ypixel *sizeof(uint32_t));
REQUIRE(memcmp(read_v.data(), tmp.data(), read_v.size()) == 0);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
std::string long_string = "ccdcsdcdscsdcsdcs";
std::vector<std::string> tmp_vector = {"aaaaaaaaa", "b", long_string};
{
RegisterHDF5Filter();
HDF5File file("scratch4.h5");
REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
}
{
HDF5ReadOnlyFile file("scratch4.h5");
HDF5DataSet vector_dataset(file, "vec");
HDF5DataType data_type_vec(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == long_string.size() + 1);
CHECK(data_space_vec.GetNumOfDimensions() == 1);
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
}
remove("scratch4.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
std::vector<double> vec = {0,1,2,3,4,6788.0};
std::string sattr = "val";
double dattr = 456.567567;
int32_t iattr = 115;
{
HDF5File file("scratch32.h5");
file.SaveScalar("scalar", tmp_scalar)
->Attr("int", iattr)
.Attr("double", dattr)
.Attr("str", sattr)
.Attr("vec", vec);
}
{
HDF5ReadOnlyFile file("scratch32.h5");
HDF5DataSet scalar_dataset(file, "scalar");
CHECK(scalar_dataset.ReadAttrDouble("double") == dattr);
CHECK(scalar_dataset.ReadAttrInt("int") == iattr);
CHECK(scalar_dataset.ReadAttrStr("str") == sattr);
CHECK(scalar_dataset.ReadAttrVec("vec") == vec);
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("int"));
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("double"));
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("int"));
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("double"));
REQUIRE_THROWS(scalar_dataset.ReadAttrDouble("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("str"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("bla"));
}
remove("scratch32.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5LeafDetection", "[HDF5]") {
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Group(file, "/group");
HDF5Group(file, "/group/sub1");
HDF5Group(file, "/group/sub2");
file.SaveScalar("/group/scalar", 5.0);
}
{
HDF5ReadOnlyFile file("scratch5.h5");
auto vec = file.FindLeafs("/group");
REQUIRE(vec.size() == 3);
bool found0 = false, found1 = false;
for (int i = 0; i < vec.size(); i++) {
if (vec[i] == "sub1")
found0 = true;
if (vec[i] == "scalar")
found1 = true;
}
REQUIRE(found0);
REQUIRE(found1);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5ExternalLink", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch123.h5");
file.ExternalLink("../abc.h5", "/zzz/fgh6", "fgh6");
}
{
HDF5ReadOnlyFile file("scratch123.h5");
REQUIRE(file.GetLinkedFileName("/fgh6") == "abc.h5");
}
remove("scratch123.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("test01").ImagesPerTrigger(950);
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
x.FilePrefix("test02");
x.FillMessage(start_message);
master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
remove("test01_master.h5");
remove("test02_master.h5");
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile_UserData", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("test07").ImagesPerTrigger(950);
StartMessage start_message;
x.FillMessage(start_message);
start_message.user_data["hdf5"]["val1"] = 7;
start_message.user_data["hdf5"]["val2"] = "str";
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
{
HDF5ReadOnlyFile file("test07_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val1"));
REQUIRE(dataset->ReadScalar<double>() == 7.0);
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val2"));
REQUIRE(dataset->ReadString() == "str");
}
remove("test07_master.h5");
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile_RadInt", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
x.FilePrefix("test01_rad_int").ImagesPerTrigger(950);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
AzimuthalIntegrationProfile profile(mapping);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
end_message.az_int_result["avg1"] = profile.GetResult();
end_message.az_int_result["avg2"] = profile.GetResult();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
remove("test01_rad_int_master.h5");
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
std::vector<SpotToSave> spots;
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.OverwriteExistingFiles(true);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
auto v = file_set.Finalize();
REQUIRE(v.size() == 3); // 3 files
REQUIRE(v[0].filename == "test02_1p10_data_000001.h5");
REQUIRE(v[0].total_images == 2);
REQUIRE(v[1].filename == "test02_1p10_data_000002.h5");
REQUIRE(v[1].total_images == 2);
REQUIRE(v[2].filename == "test02_1p10_data_000003.h5");
REQUIRE(v[2].total_images == 1);
REQUIRE(!file_set.GetZMQAddr());
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test02_1p10_data_000001.h5");
remove("test02_1p10_data_000002.h5");
remove("test02_1p10_data_000003.h5");
}
TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FrameTime(std::chrono::microseconds(1000), std::chrono::microseconds(100));
DatasetSettings d;
d.FilePrefix("run0345_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
.PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
.SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("test_sample").RunName("run1");
x.ImportDatasetSettings(d).OverwriteExistingFiles(true);
std::vector<SpotToSave> spots;
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
file_set.SetupFinalizedFileSocket("ipc://#1");
std::vector<int16_t> image(x.GetPixelsNum());
ZMQSocket s(ZMQSocketType::Sub);
s.Connect("ipc://#1");
s.SubscribeAll();
s.ReceiveTimeout(std::chrono::seconds(5));
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
REQUIRE(file_set.Finalize().size() == 3);
ZMQMessage msg;
nlohmann::json j;
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000001.h5");
REQUIRE(j["file_number"] == 1);
REQUIRE(j["nimages"] == 2);
REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetIncidentEnergy_keV() * 1000.0));
REQUIRE(j["space_group_number"] == 96);
REQUIRE(j["experiment_group"] == "p12345");
REQUIRE(j["run_number"] == 345);
REQUIRE(j["run_name"] == "run1");
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
std::cout << j.dump(4) << std::endl;
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000002.h5");
REQUIRE(j["file_number"] == 2);
REQUIRE(j["nimages"] == 2);
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000003.h5");
REQUIRE(j["file_number"] == 3);
REQUIRE(j["nimages"] == 1);
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test05_data_000001.h5");
remove("test05_data_000002.h5");
remove("test05_data_000003.h5");
}
TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
std::vector<SpotToSave> spots;
spots.push_back({10,10,7});
spots.push_back({20,50,12});
spots.push_back({1000,500,3});
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test02_1p10_spots_data_000001.h5");
remove("test02_1p10_spots_data_000002.h5");
}
TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION).OverwriteExistingFiles(true);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
start_message.az_int_q_bin_count = mapping.GetQBinCount();
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), i);
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
StartMessage message;
message.file_prefix = "z/x";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
}
TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
StartMessage message;
message.file_prefix = "acq";
message.source_name = "SwissFEL";
message.detector_serial_number = "JF17T16V01";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
message.detector_serial_number = "";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
}
TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {
REQUIRE(ExtractFilename("filename_data_000001.h5") == "filename_data_000001.h5");
REQUIRE(ExtractFilename("dir1/filename_data_000001.h5") == "filename_data_000001.h5");
REQUIRE(ExtractFilename("dir1/dir2/filename_data_000001.h5") == "filename_data_000001.h5");
}
TEST_CASE("HDF5DataType", "[HDF5]") {
HDF5DataType type1(1,true);
REQUIRE(type1.GetElemSize() == 1);
HDF5DataType type2(2,true);
REQUIRE(type2.GetElemSize() == 2);
HDF5DataType type4(4,true);
REQUIRE(type4.GetElemSize() == 4);
HDF5DataType type8(8,true);
REQUIRE(type8.GetElemSize() == 8);
REQUIRE_THROWS(HDF5DataType(7,true));
}
TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link").SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
x.OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000002"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000003"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == 1);
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000004"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_vds");
x.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::NXmxVDS);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
HDF5ReadOnlyFile file("link_vds_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() - 2);
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_vds_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_basic");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
// NXmxIntegrated forces all images into one file
REQUIRE(x.GetImagesPerFile() == x.GetImageNum());
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated);
// images_per_file should equal total images for integrated
REQUIRE(start_message.images_per_file == x.GetImageNum());
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 42);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
// All images in one file — only one stats entry
REQUIRE(stats.size() == 1);
REQUIRE(stats[0].total_images == x.GetImageNum());
}
// Single integrated file, no separate master or data files
REQUIRE(!std::filesystem::exists("integrated_basic.h5"));
REQUIRE(std::filesystem::exists("integrated_basic_master.h5"));
REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5"));
{
HDF5ReadOnlyFile file("integrated_basic_master.h5");
// Data should be directly in the file
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
// Master metadata should also be present
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/instrument/detector/beam_center_x"));
REQUIRE(dataset->ReadScalar<float>() == Catch::Approx(x.GetBeamX_pxl()));
// No external links (unlike NXmxLegacy)
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_basic_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_spots");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 10);
std::vector<SpotToSave> spots;
spots.push_back({10.0f, 20.0f, 100.0f});
spots.push_back({30.0f, 40.0f, 200.0f});
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
message.image_collection_efficiency = 1.0f;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
}
REQUIRE(std::filesystem::exists("integrated_spots_master.h5"));
{
HDF5ReadOnlyFile file("integrated_spots_master.h5");
// Detector plugin data should exist in the same file
REQUIRE(file.Exists("/entry/detector"));
// Image data should exist
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_spots_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_zero");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = 0;
FileWriter writer(start_message);
// Write no images — just finalize
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
// No data files created
REQUIRE(stats.empty());
}
// Master file should still exist with metadata
REQUIRE(std::filesystem::exists("integrated_zero_master.h5"));
{
HDF5ReadOnlyFile file("integrated_zero_master.h5");
REQUIRE(file.Exists("/entry"));
// No data dataset since no images written
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_zero_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_azint");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
start_message.az_int_q_bin_count = mapping.GetQBinCount();
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 5);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), static_cast<float>(i));
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
}
REQUIRE(std::filesystem::exists("integrated_azint_master.h5"));
{
HDF5ReadOnlyFile file("integrated_azint_master.h5");
// Azimuthal integration bin mapping should exist (written by plugin)
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/bin_to_q"));
// Per-image azint data should exist
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/image"));
HDF5DataSpace space(*dataset);
REQUIRE(space.GetNumOfDimensions() == 3);
REQUIRE(space.GetDimensions()[0] == x.GetImageNum());
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_azint_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") {
// Test that out-of-order image delivery works with NXmxIntegrated
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_ooo");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 7);
// Write images out of order
std::vector<int> order = {3, 1, 4, 0, 2};
for (int idx : order) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = idx;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
REQUIRE(stats[0].total_images == 5);
}
REQUIRE(std::filesystem::exists("integrated_ooo_master.h5"));
{
HDF5ReadOnlyFile file("integrated_ooo_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetDimensions()[0] == 5);
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_ooo_master.h5");
}
TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("data_only");
x.SetFileWriterFormat(FileWriterFormat::DataOnly).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::DataOnly);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
REQUIRE(!std::filesystem::exists("data_only_master.h5"));
REQUIRE(std::filesystem::exists("data_only_data_000001.h5"));
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(2));
std::vector<int16_t> calib_1(x.GetModulesNum() * RAW_MODULE_SIZE);
std::vector<float> calib_2(x.GetModulesNum() * RAW_MODULE_SIZE);
for (int i = 0; i < x.GetModulesNum(); i++) {
calib_1[i] = i * 3 - 1024;
calib_2[i] = static_cast<float>(i) / 16.0 + 123.25f;
}
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> calib_3 = compressor.Compress(calib_1);
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("calib");
x.OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
CompressedImage image_1(calib_1, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
CompressedImage image_2(calib_2, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
CompressedImage image_3(calib_3, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES,
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4);
image_1.Channel("calib1");
image_2.Channel("calib2");
image_3.Channel("calib3");
writer.WriteHDF5(image_1);
writer.WriteHDF5(image_2);
writer.WriteHDF5(image_3);
writer.WriteHDF5(end_message);
writer.Finalize();
}
REQUIRE(std::filesystem::exists("calib_master.h5"));
{
HDF5ReadOnlyFile file("calib_master.h5");
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib1"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 2);
REQUIRE(type.IsSigned());
REQUIRE(type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
}
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib2"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 4);
REQUIRE(type.IsSigned());
REQUIRE(!type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<float> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_2.data(), output.size() * type.GetElemSize()) == 0);
}
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib3"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 2);
REQUIRE(type.IsSigned());
REQUIRE(type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero").OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = 0;
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
{
std::unique_ptr<HDF5ReadOnlyFile> file;
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("link_zero_master.h5"));
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(*file,"/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_contiguous.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({5, 4, 3});
{
HDF5DataSpace source_space({2, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({0, 0, 0}, {2, 4, 3});
dcpl.SetVirtual("file_000001.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({2, 0, 0}, {3, 4, 3});
dcpl.SetVirtual("file_000002.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(1));
REQUIRE(!mappings[0].ContainsVirtualImage(2));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(1) == 1);
REQUIRE(mappings[1].ContainsVirtualImage(2));
REQUIRE(mappings[1].ContainsVirtualImage(4));
CHECK(mappings[1].SourceImage(2) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(4) == 2);
}
remove("scratch_vds_reverse_contiguous.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Objects_VDS_reverse_strided", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_strided.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({6, 4, 3});
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({0, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_even.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({1, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_odd.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(2));
REQUIRE(mappings[0].ContainsVirtualImage(4));
REQUIRE(!mappings[0].ContainsVirtualImage(1));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(2) == 1);
CHECK(mappings[0].SourceImage(4) == 2);
REQUIRE(mappings[1].ContainsVirtualImage(1));
REQUIRE(mappings[1].ContainsVirtualImage(3));
REQUIRE(mappings[1].ContainsVirtualImage(5));
REQUIRE(!mappings[1].ContainsVirtualImage(0));
CHECK(mappings[1].SourceImage(1) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(5) == 2);
}
remove("scratch_vds_reverse_strided.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// Overwrite is detected up front for back-channel transports (default) by checking
// the master file only - never the staggered per-writer data files. The ZeroMQ path
// (no back-channel) must opt out and keep writing .tmp files instead.
TEST_CASE("FileWriter_overwrite_detected_at_start", "[HDF5][Overwrite]") {
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("fw_overwrite_start").ImagesPerTrigger(3).ImagesPerFile(2)
.Compression(CompressionAlgorithm::NO_COMPRESSION)
.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(false);
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.write_master_file.value_or(false)); // this writer owns the master
// A stray data file (owned by another, staggered writer) must NOT trip the check -
// only the master file is inspected. The temporary writer cleans up its own tmp.
{ std::ofstream(HDF5Metadata::DataFileName(start_message, 0)) << "blocker"; }
REQUIRE_NOTHROW(FileWriter(start_message));
remove(HDF5Metadata::DataFileName(start_message, 0).c_str());
// The master file, on the other hand, does collide.
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
// Back-channel transport (direct HDF5 / TCP): fail fast in the constructor.
REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);
// ZeroMQ transport (no back-channel): must not throw - it will write .tmp and
// only fail at the final rename. The un-finalized writer cleans up its own tmp.
REQUIRE_NOTHROW(FileWriter(start_message, /*check_overwrite_at_start=*/false));
remove(HDF5Metadata::MasterFileName(start_message).c_str());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// If EndDataCollection throws while finalizing (here the master file appears mid-run,
// so the early check can't catch it), the pusher must still tear down its writer so a
// subsequent collection can start instead of dying with "already writing images".
TEST_CASE("HDF5FilePusher_finalize_failure_recovers", "[HDF5FilePusher][Repro]") {
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("pusher_finalize_repro").ImagesPerTrigger(1)
.Compression(CompressionAlgorithm::NO_COMPRESSION)
.SetFileWriterFormat(FileWriterFormat::NXmxVDS)
.OverwriteExistingFiles(false);
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message{};
HDF5FilePusher pusher;
pusher.StartDataCollection(start_message);
// Create the conflict after the start-time check has already passed.
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
REQUIRE_THROWS_AS(pusher.EndDataCollection(end_message), JFJochException);
// Writer released despite the failure: the next collection starts cleanly
// instead of dying with "already writing images".
remove(HDF5Metadata::MasterFileName(start_message).c_str());
REQUIRE_NOTHROW(pusher.StartDataCollection(start_message));
REQUIRE_NOTHROW(pusher.EndDataCollection(end_message));
// The failed finalize intentionally leaves a .tmp behind - sweep the prefix.
for (const auto &e : std::filesystem::directory_iterator("."))
if (e.path().filename().string().rfind("pusher_finalize_repro", 0) == 0)
std::filesystem::remove(e.path());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
Reference in New Issue View Git Blame Copy Permalink