mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 84 Wiki Activity
Files
b3039245f97d8d8fb19ca1343b02df7ddfbe6ecb
Jungfraujoch/tests/HDF5WritingTest.cpp
T
leonarski_f 8261b7354e
Build Packages / Create release (push) Successful in 9s
Details
Build Packages / build:portable:macos-arm64:nocuda (push) Successful in 5m8s
Details
Build Packages / build:rugnux:linux-aarch64:cuda (push) Successful in 11m12s
Details
Build Packages / build:portable:linux-x86_64:nocuda (push) Successful in 16m43s
Details
Build Packages / build:portable:linux-x86_64:cuda (push) Successful in 20m56s
Details
Build Packages / build:portable:windows-x86_64:nocuda (push) Successful in 22m4s
Details
Build Packages / build:jfjoch:rocky8:nocuda (push) Successful in 26m57s
Details
Build Packages / HDF5 consumer tests (DIALS, XDS) (push) Successful in 28m24s
Details
Build Packages / build:jfjoch:rocky9:nocuda (push) Successful in 27m58s
Details
Build Packages / build:portable:windows-x86_64:cuda (push) Successful in 29m29s
Details
Build Packages / build:jfjoch:ubuntu2204:nocuda (push) Successful in 18m23s
Details
Build Packages / build:jfjoch:ubuntu2404:nocuda (push) Successful in 19m55s
Details
Build Packages / Generate python client (push) Successful in 32s
Details
Build Packages / Build documentation (push) Successful in 1m32s
Details
Build Packages / build:jfjoch:rocky8:cuda-sls9 (push) Successful in 27m20s
Details
Build Packages / build:jfjoch:ubuntu2204:cuda (push) Successful in 29m27s
Details
Build Packages / build:jfjoch:ubuntu2404:cuda (push) Successful in 32m41s
Details
Build Packages / build:jfjoch:rocky8:cuda (push) Successful in 34m46s
Details
Build Packages / build:jfjoch:rocky9:cuda-sls9 (push) Successful in 37m25s
Details
Build Packages / build:jfjoch:rocky9:cuda (push) Successful in 35m56s
Details
Build Packages / Unit tests (push) Successful in 2h8m44s
Details
Build Packages / Validation Battery (push) Successful in 33m35s
Details
v1.0.0-rc.176 (#86) 
* jfjoch_broker: Optional live Rugnux analysis of DECTRIS rotation collections inside the broker, with merged data, maps and the placed model written beside the master file, the rotation lattice in the scan result, and a "Rugnux live" tab and dashboard row in the web frontend.
* Rugnux: Faster processing, with molecular replacement and model validation spread over several GPUs and run on the Apple GPU on macOS.
* Rugnux: The polarization stated in miniCBF and d*TREK headers is used, twinned crystals' space groups are decided more reliably, and a run that finds no lattice still reports every image.

Reviewed-on: #86
Co-authored-by: Filip Leonarski <filip.leonarski@psi.ch>
2026-10-19 07:03:28 +02:00

1568 lines
60 KiB
C++
Raw Blame History

// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
// SPDX-License-Identifier: GPL-3.0-only
#include <catch2/catch_all.hpp>
#include <iostream>
#include <fstream>
#include <filesystem>
#include <thread>
#include "../common/DiffractionExperiment.h"
#include "../writer/HDF5Objects.h"
#include "../writer/FileWriter.h"
#include "../image_pusher/HDF5FilePusher.h"
#include "../compression/JFJochCompressor.h"
#include "../common/AzimuthalIntegrationProfile.h"
#include <nlohmann/json.hpp>
using namespace std::literals::chrono_literals;
TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
{
HDF5File file("scratch_group_reopen.h5");
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
REQUIRE(file.Exists("/group1"));
REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
}
remove("scratch_group_reopen.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
const std::string first_value = "abc";
const std::string second_value = "a much longer attribute value";
{
HDF5File file("scratch_attr_string_update.h5");
REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
}
{
HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
REQUIRE(file.ReadAttrStr("str_attr") == second_value);
}
remove("scratch_attr_string_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
const int64_t first_value = -1234567890123LL;
const int64_t second_value = 9876543210123LL;
{
HDF5File file("scratch_attr_int64_update.h5");
REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
REQUIRE(file.ReadAttrInt("int_attr") == first_value);
REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
{
HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
}
remove("scratch_attr_int64_update.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch1.h5");
file.SaveScalar("scalar", tmp_scalar);
}
{
HDF5ReadOnlyFile file("scratch1.h5");
HDF5DataSet scalar_dataset(file, "scalar");
HDF5DataType data_type_scalar(scalar_dataset);
HDF5DataSpace data_space_scalar(scalar_dataset);
REQUIRE(data_type_scalar.GetElemSize() == 2);
REQUIRE(data_space_scalar.GetNumOfDimensions() == 0);
REQUIRE(scalar_dataset.ReadScalar<uint16_t>() == tmp_scalar);
REQUIRE(scalar_dataset.ReadScalar<int64_t>() == tmp_scalar);
std::vector<uint16_t> v;
REQUIRE_NOTHROW(scalar_dataset.ReadVector(v));
REQUIRE(v.size() == 1);
REQUIRE(v[0] == tmp_scalar);
}
remove("scratch1.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// NXmx puts no rank on distance, saturation_value or an axis setting, and a writer outside the
// DECTRIS toolchain may store any of them as a length-1 array. That is the same one number.
TEST_CASE("HDF5DataSet_scalar_stored_rank1", "[HDF5][Unit]") {
{
HDF5File file("scratch1b.h5");
file.SaveVector("one", std::vector<double>{85.5});
file.SaveVector("two", std::vector<double>{85.5, 90.0});
}
{
HDF5ReadOnlyFile file("scratch1b.h5");
HDF5DataSet one(file, "one");
CHECK(HDF5DataSpace(one).GetNumOfDimensions() == 1);
CHECK(one.ReadScalar<double>() == 85.5);
REQUIRE_THROWS(HDF5DataSet(file, "two").ReadScalar<double>());
}
remove("scratch1b.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
std::string tmp_string = "HDF5Content";
{
HDF5File file("scratch2.h5");
file.SaveScalar("str", tmp_string);
}
{
HDF5ReadOnlyFile file("scratch2.h5");
HDF5DataSet string_dataset(file, "str");
HDF5DataType data_type_str(string_dataset);
HDF5DataSpace data_space_str(string_dataset);
CHECK(data_type_str.GetElemSize() == tmp_string.size() + 1);
CHECK(data_space_str.GetNumOfDimensions() == 0);
CHECK(string_dataset.ReadString() == tmp_string);
}
remove("scratch2.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_string_stored_rank1", "[HDF5][Unit]") {
// Facilities write a one-element string array where the application definition allows a scalar -
// JUNGFRAU masters from an early beamline deployment store sensor_material and description that
// way - and the file is unreadable if that is refused, because the metadata parse never finishes.
{
HDF5File file("scratch2b.h5");
file.SaveVector("one", std::vector<std::string>{"Si"});
file.SaveVector("two", std::vector<std::string>{"Si", "CdTe"});
}
{
HDF5ReadOnlyFile file("scratch2b.h5");
HDF5DataSet one(file, "one");
CHECK(HDF5DataSpace(one).GetNumOfDimensions() == 1);
CHECK(one.ReadString() == "Si");
REQUIRE_THROWS(HDF5DataSet(file, "two").ReadString());
}
remove("scratch2b.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
std::vector<double> tmp_vector (16384);
tmp_vector[0] = 599.88;
tmp_vector[1000] = 800.12;
tmp_vector[15000] = 3.1415926;
{
RegisterHDF5Filter();
HDF5File file("scratch3.h5");
file.SaveVector("vec", tmp_vector);
}
{
HDF5ReadOnlyFile file("scratch3.h5");
HDF5DataSet vector_dataset(file, "vec");
HDF5DataType data_type_vec(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 8);
CHECK(data_space_vec.GetNumOfDimensions() == 1);
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
std::vector<double> output;
REQUIRE_NOTHROW(vector_dataset.ReadVector(output));
CHECK (output[0] == tmp_vector[0]);
CHECK (output[1000] == tmp_vector[1000]);
CHECK (output[15000] == tmp_vector[15000]);
CHECK (output.size() == tmp_vector.size());
std::vector<double> output2(10);
REQUIRE_NOTHROW(vector_dataset.ReadVector(output2, {15000},{10}));
REQUIRE_THROWS(vector_dataset.ReadVector(output2, {15000},{100}));
CHECK(output2[0] == tmp_vector[15000]);
}
remove("scratch3.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
std::vector<int32_t> tmp(xpixel*ypixel, -134);
{
RegisterHDF5Filter();
HDF5File file("scratch4.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {0, 0, 0});
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {2, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch4.h5");
HDF5DataSet vector_dataset(file, "/data");
HDF5DataType data_type_vec(vector_dataset);
HDF5Dcpl dcpl(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 4);
CHECK(data_space_vec.GetNumOfDimensions() == 3);
CHECK(data_space_vec.GetDimensions()[0] == 3);
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
CHECK(dcpl.GetNumOfDimensions() == 3);
CHECK(dcpl.GetChunking()[0] == 1);
CHECK(dcpl.GetChunking()[1] == ypixel);
CHECK(dcpl.GetChunking()[2] == xpixel);
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::NO_COMPRESSION);
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
REQUIRE(read_v.size() == xpixel * ypixel * sizeof(uint32_t));
CHECK(memcmp(read_v.data(), tmp.data(), xpixel * ypixel * sizeof(uint32_t)) == 0);
}
remove("scratch4.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking_bslz4", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
size_t len = 1234;
std::vector<uint8_t> tmp(len, 200);
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
dcpl.SetCompression(CompressionAlgorithm::BSHUF_LZ4, 0);
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), 1234, {0, 0, 0});
dataset.WriteDirectChunk(tmp.data(), 1234, {2, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch5.h5");
HDF5DataSet vector_dataset(file, "/data");
HDF5DataType data_type_vec(vector_dataset);
HDF5Dcpl dcpl(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == 4);
CHECK(data_space_vec.GetNumOfDimensions() == 3);
CHECK(data_space_vec.GetDimensions()[0] == 3);
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
CHECK(dcpl.GetNumOfDimensions() == 3);
CHECK(dcpl.GetChunking()[0] == 1);
CHECK(dcpl.GetChunking()[1] == ypixel);
CHECK(dcpl.GetChunking()[2] == xpixel);
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
REQUIRE(read_v == tmp);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_chunking_ReadToU8", "[HDF5][Unit]") {
size_t xpixel = 512;
size_t ypixel = 256;
size_t len = xpixel*ypixel;
std::vector<uint32_t> tmp(len, 200);
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Dcpl dcpl;
HDF5DataType data_type(4, true);
dcpl.SetChunking( {1, ypixel, xpixel});
dcpl.SetCompression(CompressionAlgorithm::NO_COMPRESSION, 0);
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
dataset.WriteDirectChunk(tmp.data(), xpixel * ypixel * sizeof(uint32_t), {1, 0, 0});
}
{
HDF5ReadOnlyFile file("scratch5.h5");
HDF5DataSet vector_dataset(file, "/data");
std::vector<uint8_t> read_v;
REQUIRE_NOTHROW(vector_dataset.ReadVectorToU8(read_v, {1,0,0}, {1, ypixel, xpixel}));
REQUIRE(read_v.size() == xpixel * ypixel *sizeof(uint32_t));
REQUIRE(memcmp(read_v.data(), tmp.data(), read_v.size()) == 0);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
std::string long_string = "ccdcsdcdscsdcsdcs";
std::vector<std::string> tmp_vector = {"aaaaaaaaa", "b", long_string};
{
RegisterHDF5Filter();
HDF5File file("scratch4.h5");
REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
}
{
HDF5ReadOnlyFile file("scratch4.h5");
HDF5DataSet vector_dataset(file, "vec");
HDF5DataType data_type_vec(vector_dataset);
HDF5DataSpace data_space_vec(vector_dataset);
CHECK(data_type_vec.GetElemSize() == long_string.size() + 1);
CHECK(data_space_vec.GetNumOfDimensions() == 1);
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
}
remove("scratch4.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Attr", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
std::vector<double> vec = {0,1,2,3,4,6788.0};
std::string sattr = "val";
double dattr = 456.567567;
int32_t iattr = 115;
{
HDF5File file("scratch32.h5");
file.SaveScalar("scalar", tmp_scalar)
->Attr("int", iattr)
.Attr("double", dattr)
.Attr("str", sattr)
.Attr("vec", vec);
}
{
HDF5ReadOnlyFile file("scratch32.h5");
HDF5DataSet scalar_dataset(file, "scalar");
CHECK(scalar_dataset.ReadAttrDouble("double") == dattr);
CHECK(scalar_dataset.ReadAttrInt("int") == iattr);
CHECK(scalar_dataset.ReadAttrStr("str") == sattr);
CHECK(scalar_dataset.ReadAttrVec("vec") == vec);
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("int"));
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("double"));
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("int"));
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("double"));
REQUIRE_THROWS(scalar_dataset.ReadAttrDouble("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("vec"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("str"));
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("bla"));
}
remove("scratch32.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5LeafDetection", "[HDF5]") {
{
RegisterHDF5Filter();
HDF5File file("scratch5.h5");
HDF5Group(file, "/group");
HDF5Group(file, "/group/sub1");
HDF5Group(file, "/group/sub2");
file.SaveScalar("/group/scalar", 5.0);
}
{
HDF5ReadOnlyFile file("scratch5.h5");
auto vec = file.FindLeafs("/group");
REQUIRE(vec.size() == 3);
bool found0 = false, found1 = false;
for (int i = 0; i < vec.size(); i++) {
if (vec[i] == "sub1")
found0 = true;
if (vec[i] == "scalar")
found1 = true;
}
REQUIRE(found0);
REQUIRE(found1);
}
remove("scratch5.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5ExternalLink", "[HDF5][Unit]") {
uint16_t tmp_scalar = 16788;
{
HDF5File file("scratch123.h5");
file.ExternalLink("../abc.h5", "/zzz/fgh6", "fgh6");
}
{
HDF5ReadOnlyFile file("scratch123.h5");
REQUIRE(file.GetLinkedFileName("/fgh6") == "abc.h5");
}
remove("scratch123.h5");
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("test01").ImagesPerTrigger(950);
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
x.FilePrefix("test02");
x.FillMessage(start_message);
master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
remove("test01_master.h5");
remove("test02_master.h5");
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile_UserData", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("test07").ImagesPerTrigger(950);
StartMessage start_message;
x.FillMessage(start_message);
start_message.user_data["hdf5"]["val1"] = 7;
start_message.user_data["hdf5"]["val2"] = "str";
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
{
HDF5ReadOnlyFile file("test07_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val1"));
REQUIRE(dataset->ReadScalar<double>() == 7.0);
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val2"));
REQUIRE(dataset->ReadString() == "str");
}
remove("test07_master.h5");
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5MasterFile_RadInt", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
x.FilePrefix("test01_rad_int").ImagesPerTrigger(950);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
AzimuthalIntegrationProfile profile(mapping);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
end_message.az_int_result["avg1"] = profile.GetResult();
end_message.az_int_result["avg2"] = profile.GetResult();
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
remove("test01_rad_int_master.h5");
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
std::vector<SpotToSave> spots;
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.OverwriteExistingFiles(true);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
auto v = file_set.Finalize();
REQUIRE(v.size() == 3); // 3 files
REQUIRE(v[0].filename == "test02_1p10_data_000001.h5");
REQUIRE(v[0].total_images == 2);
REQUIRE(v[1].filename == "test02_1p10_data_000002.h5");
REQUIRE(v[1].total_images == 2);
REQUIRE(v[2].filename == "test02_1p10_data_000003.h5");
REQUIRE(v[2].total_images == 1);
REQUIRE(!file_set.GetZMQAddr());
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test02_1p10_data_000001.h5");
remove("test02_1p10_data_000002.h5");
remove("test02_1p10_data_000003.h5");
}
TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FrameTime(std::chrono::microseconds(1000), std::chrono::microseconds(100));
DatasetSettings d;
d.FilePrefix("run0345_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
.PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
.SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("test_sample").RunName("run1");
x.ImportDatasetSettings(d).OverwriteExistingFiles(true);
std::vector<SpotToSave> spots;
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
file_set.SetupFinalizedFileSocket("ipc://#1");
std::vector<int16_t> image(x.GetPixelsNum());
ZMQSocket s(ZMQSocketType::Sub);
s.Connect("ipc://#1");
s.SubscribeAll();
s.ReceiveTimeout(std::chrono::seconds(5));
// PUB drops what it sends before the subscription has reached it, and the first file is closed
// (and announced) right after the second image - ensure sub socket is properly connected
std::this_thread::sleep_for(std::chrono::seconds(2));
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
REQUIRE(file_set.Finalize().size() == 3);
ZMQMessage msg;
nlohmann::json j;
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000001.h5");
REQUIRE(j["file_number"] == 1);
REQUIRE(j["nimages"] == 2);
REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetIncidentEnergy_keV() * 1000.0));
REQUIRE(j["space_group_number"] == 96);
REQUIRE(j["experiment_group"] == "p12345");
REQUIRE(j["run_number"] == 345);
REQUIRE(j["run_name"] == "run1");
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
std::cout << j.dump(4) << std::endl;
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000002.h5");
REQUIRE(j["file_number"] == 2);
REQUIRE(j["nimages"] == 2);
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
REQUIRE(s.Receive(msg, true));
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
REQUIRE(j["filename"] == "run0345_acq_data_000003.h5");
REQUIRE(j["file_number"] == 3);
REQUIRE(j["nimages"] == 1);
REQUIRE(j.contains("user_data"));
REQUIRE(j["user_data"]["z"] == 567);
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test05_data_000001.h5");
remove("test05_data_000002.h5");
remove("test05_data_000003.h5");
}
TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
std::vector<SpotToSave> spots;
spots.push_back({10,10,7});
spots.push_back({20,50,12});
spots.push_back({1000,500,3});
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("test02_1p10_spots_data_000001.h5");
remove("test02_1p10_spots_data_000002.h5");
}
TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
{
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION).OverwriteExistingFiles(true);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
start_message.az_int_q_bin_count = mapping.GetQBinCount();
FileWriter file_set(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), i);
message.number = i;
REQUIRE_NOTHROW(file_set.Write(message));
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
StartMessage message;
message.file_prefix = "z/x";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
}
TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
StartMessage message;
message.file_prefix = "acq";
message.source_name = "SwissFEL";
message.detector_serial_number = "JF17T16V01";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
message.detector_serial_number = "";
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
}
TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {
REQUIRE(ExtractFilename("filename_data_000001.h5") == "filename_data_000001.h5");
REQUIRE(ExtractFilename("dir1/filename_data_000001.h5") == "filename_data_000001.h5");
REQUIRE(ExtractFilename("dir1/dir2/filename_data_000001.h5") == "filename_data_000001.h5");
}
TEST_CASE("HDF5DataType", "[HDF5]") {
HDF5DataType type1(1,true);
REQUIRE(type1.GetElemSize() == 1);
HDF5DataType type2(2,true);
REQUIRE(type2.GetElemSize() == 2);
HDF5DataType type4(4,true);
REQUIRE(type4.GetElemSize() == 4);
HDF5DataType type8(8,true);
REQUIRE(type8.GetElemSize() == 8);
REQUIRE_THROWS(HDF5DataType(7,true));
}
TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link").SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
x.OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000002"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000003"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == 1);
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000004"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_vds");
x.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::NXmxVDS);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
HDF5ReadOnlyFile file("link_vds_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() - 2);
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
}
{
HDF5ReadOnlyFile file("link_vds_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_basic");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
// NXmxIntegrated forces all images into one file
REQUIRE(x.GetImagesPerFile() == x.GetImageNum());
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated);
// images_per_file should equal total images for integrated
REQUIRE(start_message.images_per_file == x.GetImageNum());
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 42);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
// All images in one file — only one stats entry
REQUIRE(stats.size() == 1);
REQUIRE(stats[0].total_images == x.GetImageNum());
}
// Single integrated file, no separate master or data files
REQUIRE(!std::filesystem::exists("integrated_basic.h5"));
REQUIRE(std::filesystem::exists("integrated_basic_master.h5"));
REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5"));
{
HDF5ReadOnlyFile file("integrated_basic_master.h5");
// Data should be directly in the file
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 3);
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum());
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
// Master metadata should also be present
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/instrument/detector/beam_center_x"));
REQUIRE(dataset->ReadScalar<float>() == Catch::Approx(x.GetBeamX_pxl()));
// No external links (unlike NXmxLegacy)
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_basic_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_spots");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 10);
std::vector<SpotToSave> spots;
spots.push_back({10.0f, 20.0f, 100.0f});
spots.push_back({30.0f, 40.0f, 200.0f});
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
message.image_collection_efficiency = 1.0f;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
}
REQUIRE(std::filesystem::exists("integrated_spots_master.h5"));
{
HDF5ReadOnlyFile file("integrated_spots_master.h5");
// Detector plugin data should exist in the same file
REQUIRE(file.Exists("/entry/detector"));
// Image data should exist
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_spots_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_zero");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = 0;
FileWriter writer(start_message);
// Write no images — just finalize
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
// No data files created
REQUIRE(stats.empty());
}
// Master file should still exist with metadata
REQUIRE(std::filesystem::exists("integrated_zero_master.h5"));
{
HDF5ReadOnlyFile file("integrated_zero_master.h5");
REQUIRE(file.Exists("/entry"));
// No data dataset since no images written
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_zero_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500);
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_azint");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
PixelMask pixel_mask(x);
AzimuthalIntegrationMapping mapping(x, pixel_mask);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
start_message.az_int_q_bin_count = mapping.GetQBinCount();
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 5);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), static_cast<float>(i));
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
}
REQUIRE(std::filesystem::exists("integrated_azint_master.h5"));
{
HDF5ReadOnlyFile file("integrated_azint_master.h5");
// Azimuthal integration bin mapping should exist (written by plugin)
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/bin_to_q"));
// Per-image azint data should exist
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/image"));
HDF5DataSpace space(*dataset);
REQUIRE(space.GetNumOfDimensions() == 3);
REQUIRE(space.GetDimensions()[0] == x.GetImageNum());
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_azint_master.h5");
}
TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") {
// Test that out-of-order image delivery works with NXmxIntegrated
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix("integrated_ooo");
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 7);
// Write images out of order
std::vector<int> order = {3, 1, 4, 0, 2};
for (int idx : order) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = idx;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
auto stats = writer.Finalize();
REQUIRE(stats.size() == 1);
REQUIRE(stats[0].total_images == 5);
}
REQUIRE(std::filesystem::exists("integrated_ooo_master.h5"));
{
HDF5ReadOnlyFile file("integrated_ooo_master.h5");
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetDimensions()[0] == 5);
}
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
remove("integrated_ooo_master.h5");
}
TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("data_only");
x.SetFileWriterFormat(FileWriterFormat::DataOnly).OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.file_format == FileWriterFormat::DataOnly);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum());
std::vector<SpotToSave> spots;
for (int i = 0; i < x.GetImageNum() - 2; i++) {
for (auto &j: image)
j = i;
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.spots = spots;
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
REQUIRE(!std::filesystem::exists("data_only_master.h5"));
REQUIRE(std::filesystem::exists("data_only_data_000001.h5"));
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(2));
std::vector<int16_t> calib_1(x.GetModulesNum() * RAW_MODULE_SIZE);
std::vector<float> calib_2(x.GetModulesNum() * RAW_MODULE_SIZE);
for (int i = 0; i < x.GetModulesNum(); i++) {
calib_1[i] = i * 3 - 1024;
calib_2[i] = static_cast<float>(i) / 16.0 + 123.25f;
}
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
std::vector<uint8_t> calib_3 = compressor.Compress(calib_1);
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("calib");
x.OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = x.GetImageNum() - 2;
FileWriter writer(start_message);
CompressedImage image_1(calib_1, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
CompressedImage image_2(calib_2, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
CompressedImage image_3(calib_3, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES,
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4);
image_1.Channel("calib1");
image_2.Channel("calib2");
image_3.Channel("calib3");
writer.WriteHDF5(image_1);
writer.WriteHDF5(image_2);
writer.WriteHDF5(image_3);
writer.WriteHDF5(end_message);
writer.Finalize();
}
REQUIRE(std::filesystem::exists("calib_master.h5"));
{
HDF5ReadOnlyFile file("calib_master.h5");
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib1"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 2);
REQUIRE(type.IsSigned());
REQUIRE(type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
}
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib2"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 4);
REQUIRE(type.IsSigned());
REQUIRE(!type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<float> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_2.data(), output.size() * type.GetElemSize()) == 0);
}
{
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib3"));
HDF5DataSpace file_space(*dataset);
REQUIRE(file_space.GetNumOfDimensions() == 2);
HDF5DataType type(*dataset);
REQUIRE(type.GetElemSize() == 2);
REQUIRE(type.IsSigned());
REQUIRE(type.IsInteger());
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
}
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero").OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message;
end_message.max_image_number = 0;
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
master->Finalize(end_message);
master.reset();
}
{
std::unique_ptr<HDF5ReadOnlyFile> file;
std::unique_ptr<HDF5DataSet> dataset;
REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("link_zero_master.h5"));
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(*file,"/entry/data/data_000001"));
}
// No leftover HDF5 objects
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_contiguous.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({5, 4, 3});
{
HDF5DataSpace source_space({2, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({0, 0, 0}, {2, 4, 3});
dcpl.SetVirtual("file_000001.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({2, 0, 0}, {3, 4, 3});
dcpl.SetVirtual("file_000002.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(1));
REQUIRE(!mappings[0].ContainsVirtualImage(2));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(1) == 1);
REQUIRE(mappings[1].ContainsVirtualImage(2));
REQUIRE(mappings[1].ContainsVirtualImage(4));
CHECK(mappings[1].SourceImage(2) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(4) == 2);
}
remove("scratch_vds_reverse_contiguous.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Objects_VDS_reverse_strided", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_strided.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({6, 4, 3});
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({0, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_even.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({1, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_odd.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(2));
REQUIRE(mappings[0].ContainsVirtualImage(4));
REQUIRE(!mappings[0].ContainsVirtualImage(1));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(2) == 1);
CHECK(mappings[0].SourceImage(4) == 2);
REQUIRE(mappings[1].ContainsVirtualImage(1));
REQUIRE(mappings[1].ContainsVirtualImage(3));
REQUIRE(mappings[1].ContainsVirtualImage(5));
REQUIRE(!mappings[1].ContainsVirtualImage(0));
CHECK(mappings[1].SourceImage(1) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(5) == 2);
}
remove("scratch_vds_reverse_strided.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// Overwrite is detected up front for back-channel transports (default): the writer that owns the
// master file checks the master file and every data file the run will write, its staggered
// siblings' included - they share a directory by construction, since the master links them by
// relative name. The ZeroMQ path (no back-channel) must opt out and keep writing .tmp files
// instead.
TEST_CASE("FileWriter_overwrite_detected_at_start", "[HDF5][Overwrite]") {
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("fw_overwrite_start").ImagesPerTrigger(3).ImagesPerFile(2)
.Compression(CompressionAlgorithm::NO_COMPRESSION)
.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(false);
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.write_master_file.value_or(false)); // this writer owns the master
// A data file in the way is refused up front, even one another writer would have written.
// It used to pass here and fail only when the file was renamed into place at the end of the
// run - after the whole dataset had been collected.
{ std::ofstream(HDF5Metadata::DataFileName(start_message, 0)) << "blocker"; }
REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);
// A writer that does not own the master file speaks for none of them: it would otherwise stat
// files its siblings are already creating. The temporary writer cleans up its own tmp.
{
StartMessage not_master = start_message;
not_master.write_master_file = false;
REQUIRE_NOTHROW(FileWriter(not_master));
}
remove(HDF5Metadata::DataFileName(start_message, 0).c_str());
// The master file, on the other hand, does collide.
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
// Back-channel transport (direct HDF5 / TCP): fail fast in the constructor.
REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);
// ZeroMQ transport (no back-channel): must not throw - it will write .tmp and
// only fail at the final rename. The un-finalized writer cleans up its own tmp.
REQUIRE_NOTHROW(FileWriter(start_message, /*check_overwrite_at_start=*/false));
remove(HDF5Metadata::MasterFileName(start_message).c_str());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// If EndDataCollection throws while finalizing (here the master file appears mid-run,
// so the early check can't catch it), the pusher must still tear down its writer so a
// subsequent collection can start instead of dying with "already writing images".
TEST_CASE("HDF5FilePusher_finalize_failure_recovers", "[HDF5FilePusher][Repro]") {
RegisterHDF5Filter();
DiffractionExperiment x(DetJF4M());
x.FilePrefix("pusher_finalize_repro").ImagesPerTrigger(1)
.Compression(CompressionAlgorithm::NO_COMPRESSION)
.SetFileWriterFormat(FileWriterFormat::NXmxVDS)
.OverwriteExistingFiles(false);
StartMessage start_message;
x.FillMessage(start_message);
EndMessage end_message{};
HDF5FilePusher pusher;
pusher.StartDataCollection(start_message);
// Create the conflict after the start-time check has already passed.
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
REQUIRE_THROWS_AS(pusher.EndDataCollection(end_message), JFJochException);
// Writer released despite the failure: the next collection starts cleanly
// instead of dying with "already writing images".
remove(HDF5Metadata::MasterFileName(start_message).c_str());
REQUIRE_NOTHROW(pusher.StartDataCollection(start_message));
REQUIRE_NOTHROW(pusher.EndDataCollection(end_message));
// The failed finalize intentionally leaves a .tmp behind - sweep the prefix.
for (const auto &e : std::filesystem::directory_iterator("."))
if (e.path().filename().string().rfind("pusher_finalize_repro", 0) == 0)
std::filesystem::remove(e.path());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// A link that is written in the file and points at something not there is NOT an existing dataset.
// Every DECTRIS Eiger master links saturation_value, pixel_mask, bit_depth_readout and
// serial_number into a companion <prefix>_meta.h5, and that file is routinely not kept when a
// dataset is archived or deposited. Asking only whether the LINK exists then answers yes and the
// read that follows throws, which turns every optional-field guard in the reader into a hard
// failure - measured on a deposited Eiger 16M set that could not be opened at all.
TEST_CASE("HDF5Objects_dangling_external_link_does_not_exist", "[HDF5][Unit]") {
const std::string fname = "test_dangling_link.h5";
remove(fname.c_str());
{
HDF5File file(fname);
HDF5Group group(file, "/entry");
group.SaveScalar("present", static_cast<int64_t>(7));
// ...and a link into a file that does not exist, exactly as an orphaned Eiger master has.
REQUIRE(H5Lcreate_external("no_such_meta.h5", "/_dectris/whatever",
group.GetID(), "absent", H5P_DEFAULT, H5P_DEFAULT) >= 0);
}
{
HDF5ReadOnlyFile file(fname);
CHECK(file.Exists("/entry/present"));
CHECK(file.GetInt("/entry/present") == 7);
// The link is there, the object is not.
CHECK(H5Lexists(file.GetID(), "/entry/absent", H5P_DEFAULT) > 0);
CHECK_FALSE(file.Exists("/entry/absent"));
// ...so an optional read of it falls back instead of throwing.
CHECK(file.GetString("/entry/absent", "fallback") == "fallback");
}
remove(fname.c_str());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
// The direct beam - where the undeflected beam lands - is not the PONI stored as beam_center_x/y:
// on a tilted detector the two are D*tan(rot)/pixel apart, which is what a downstream program (XDS,
// say) needs and cannot read out of beam_center_x/y. Written into detectorSpecific for all three
// NXmx layouts, so check the value in all three and check it is really the tilted point.
TEST_CASE("HDF5Writer_DirectBeamInDetectorSpecific", "[HDF5][Full]") {
const std::array formats{FileWriterFormat::NXmxLegacy, FileWriterFormat::NXmxVDS,
FileWriterFormat::NXmxIntegrated};
for (const auto format : formats) {
const std::string prefix = "direct_beam_" + std::to_string(static_cast<int>(format));
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(2).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
.FilePrefix(prefix);
x.SetFileWriterFormat(format).OverwriteExistingFiles(true);
x.BeamX_pxl(500).BeamY_pxl(400).DetectorDistance_mm(150)
.PoniRot1_rad(0.031f).PoniRot2_rad(-0.047f).PoniRot3_rad(0.019f);
const auto expected = x.GetDiffractionGeometry().GetDirectBeam_pxl();
// The tilt must actually move the point, or the test would pass on a writer that stored the PONI.
REQUIRE(std::abs(expected.first - x.GetBeamX_pxl()) > 1.0f);
REQUIRE(std::abs(expected.second - x.GetBeamY_pxl()) > 1.0f);
{
RegisterHDF5Filter();
StartMessage start_message;
x.FillMessage(start_message);
REQUIRE(start_message.direct_beam_x);
REQUIRE(start_message.direct_beam_y);
CHECK(start_message.direct_beam_x.value() == Catch::Approx(expected.first));
CHECK(start_message.direct_beam_y.value() == Catch::Approx(expected.second));
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
FileWriter writer(start_message);
std::vector<int16_t> image(x.GetPixelsNum(), 42);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = i;
REQUIRE_NOTHROW(writer.Write(message));
}
writer.WriteHDF5(end_message);
writer.Finalize();
}
{
HDF5ReadOnlyFile file(prefix + "_master.h5");
CHECK(file.GetFloat("/entry/instrument/detector/beam_center_x") == Catch::Approx(500));
CHECK(file.GetFloat("/entry/instrument/detector/detectorSpecific/direct_beam_x")
== Catch::Approx(expected.first));
CHECK(file.GetFloat("/entry/instrument/detector/detectorSpecific/direct_beam_y")
== Catch::Approx(expected.second));
}
remove((prefix + "_master.h5").c_str());
remove((prefix + "_data_000001.h5").c_str());
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
}
Reference in new issue View Git Blame Copy Permalink