leonarski_f
6ed4ea541e
Every DECTRIS Eiger master links saturation_value, pixel_mask, bit_depth_readout and serial_number into a companion <prefix>_meta.h5, and that file is routinely not kept when a dataset is archived or deposited. The existence test asked only whether the LINK was written, which it is, so every optional-field guard in the reader answered yes and the read that followed threw. A deposited Eiger 16M set could not be opened at all, over values the reader was perfectly prepared to do without. Exists() now asks the second question too - whether the object the link names can be reached - so an orphaned link reads as absent and the fallbacks behind it do their job. The saturation value is then allowed to be missing outright, because on such a file it is: neither the NXmx name nor the DECTRIS one is readable, and there is no third place to look. Left unset, GetSaturationLimit() falls back to the container's own overflow. That is the safe direction - it can only fail to call a pixel saturated, where too LOW a value drops the whole reflection and silently removes the strongest data - and the run says out loud that nothing will be called saturated. The set that could not be opened now processes to 2.17 A against a deposited 2.40 A, in the deposited space group, with a cell agreeing to 0.08%. Output is byte-identical on datasets that already opened. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EFEJG6WBQv8th4UJFNe53N (cherry picked from commit 0637979f6d95b446406ab70d1f3982841d195b36)
1485 lines
56 KiB
C++
1485 lines
56 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
#include <iostream>
|
|
#include <fstream>
|
|
#include <filesystem>
|
|
|
|
#include "../common/DiffractionExperiment.h"
|
|
#include "../writer/HDF5Objects.h"
|
|
#include "../writer/FileWriter.h"
|
|
#include "../image_pusher/HDF5FilePusher.h"
|
|
#include "../compression/JFJochCompressor.h"
|
|
#include "../common/AzimuthalIntegrationProfile.h"
|
|
#include <nlohmann/json.hpp>
|
|
|
|
using namespace std::literals::chrono_literals;
|
|
|
|
TEST_CASE("HDF5Group_create_reopen_and_fail", "[HDF5][Unit]") {
|
|
{
|
|
HDF5File file("scratch_group_reopen.h5");
|
|
|
|
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
|
|
REQUIRE(file.Exists("/group1"));
|
|
|
|
REQUIRE_NOTHROW(HDF5Group(file, "/group1"));
|
|
REQUIRE(file.Exists("/group1"));
|
|
|
|
REQUIRE_THROWS(HDF5Group(file, "/missing_parent/group2"));
|
|
}
|
|
|
|
remove("scratch_group_reopen.h5");
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Attr_string_update", "[HDF5][Unit]") {
|
|
const std::string first_value = "abc";
|
|
const std::string second_value = "a much longer attribute value";
|
|
|
|
{
|
|
HDF5File file("scratch_attr_string_update.h5");
|
|
REQUIRE_NOTHROW(file.Attr("str_attr", first_value));
|
|
REQUIRE_NOTHROW(file.Attr("str_attr", second_value));
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch_attr_string_update.h5");
|
|
REQUIRE(file.ReadAttrStr("str_attr") == second_value);
|
|
}
|
|
|
|
remove("scratch_attr_string_update.h5");
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Attr_int64_update", "[HDF5][Unit]") {
|
|
const int64_t first_value = -1234567890123LL;
|
|
const int64_t second_value = 9876543210123LL;
|
|
|
|
{
|
|
HDF5File file("scratch_attr_int64_update.h5");
|
|
REQUIRE_NOTHROW(file.Attr("int_attr", first_value));
|
|
REQUIRE(file.ReadAttrInt("int_attr") == first_value);
|
|
|
|
REQUIRE_NOTHROW(file.Attr("int_attr", second_value));
|
|
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch_attr_int64_update.h5");
|
|
REQUIRE(file.ReadAttrInt("int_attr") == second_value);
|
|
}
|
|
|
|
remove("scratch_attr_int64_update.h5");
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
|
|
uint16_t tmp_scalar = 16788;
|
|
{
|
|
HDF5File file("scratch1.h5");
|
|
file.SaveScalar("scalar", tmp_scalar);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch1.h5");
|
|
HDF5DataSet scalar_dataset(file, "scalar");
|
|
HDF5DataType data_type_scalar(scalar_dataset);
|
|
HDF5DataSpace data_space_scalar(scalar_dataset);
|
|
REQUIRE(data_type_scalar.GetElemSize() == 2);
|
|
REQUIRE(data_space_scalar.GetNumOfDimensions() == 0);
|
|
REQUIRE(scalar_dataset.ReadScalar<uint16_t>() == tmp_scalar);
|
|
REQUIRE(scalar_dataset.ReadScalar<int64_t>() == tmp_scalar);
|
|
std::vector<uint16_t> v;
|
|
REQUIRE_NOTHROW(scalar_dataset.ReadVector(v));
|
|
REQUIRE(v.size() == 1);
|
|
REQUIRE(v[0] == tmp_scalar);
|
|
}
|
|
remove("scratch1.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
// NXmx puts no rank on distance, saturation_value or an axis setting, and a writer outside the
|
|
// DECTRIS toolchain may store any of them as a length-1 array. That is the same one number.
|
|
TEST_CASE("HDF5DataSet_scalar_stored_rank1", "[HDF5][Unit]") {
|
|
{
|
|
HDF5File file("scratch1b.h5");
|
|
file.SaveVector("one", std::vector<double>{85.5});
|
|
file.SaveVector("two", std::vector<double>{85.5, 90.0});
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch1b.h5");
|
|
HDF5DataSet one(file, "one");
|
|
CHECK(HDF5DataSpace(one).GetNumOfDimensions() == 1);
|
|
CHECK(one.ReadScalar<double>() == 85.5);
|
|
REQUIRE_THROWS(HDF5DataSet(file, "two").ReadScalar<double>());
|
|
}
|
|
remove("scratch1b.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
|
|
std::string tmp_string = "HDF5Content";
|
|
|
|
{
|
|
HDF5File file("scratch2.h5");
|
|
file.SaveScalar("str", tmp_string);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch2.h5");
|
|
HDF5DataSet string_dataset(file, "str");
|
|
HDF5DataType data_type_str(string_dataset);
|
|
HDF5DataSpace data_space_str(string_dataset);
|
|
CHECK(data_type_str.GetElemSize() == tmp_string.size() + 1);
|
|
CHECK(data_space_str.GetNumOfDimensions() == 0);
|
|
CHECK(string_dataset.ReadString() == tmp_string);
|
|
}
|
|
remove("scratch2.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
|
|
std::vector<double> tmp_vector (16384);
|
|
tmp_vector[0] = 599.88;
|
|
tmp_vector[1000] = 800.12;
|
|
tmp_vector[15000] = 3.1415926;
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch3.h5");
|
|
file.SaveVector("vec", tmp_vector);
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch3.h5");
|
|
HDF5DataSet vector_dataset(file, "vec");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 8);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 1);
|
|
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
|
|
std::vector<double> output;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVector(output));
|
|
CHECK (output[0] == tmp_vector[0]);
|
|
CHECK (output[1000] == tmp_vector[1000]);
|
|
CHECK (output[15000] == tmp_vector[15000]);
|
|
CHECK (output.size() == tmp_vector.size());
|
|
|
|
std::vector<double> output2(10);
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVector(output2, {15000},{10}));
|
|
REQUIRE_THROWS(vector_dataset.ReadVector(output2, {15000},{100}));
|
|
CHECK(output2[0] == tmp_vector[15000]);
|
|
}
|
|
|
|
remove("scratch3.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
|
|
std::vector<int32_t> tmp(xpixel*ypixel, -134);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch4.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {0, 0, 0});
|
|
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {2, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch4.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5Dcpl dcpl(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 4);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 3);
|
|
CHECK(data_space_vec.GetDimensions()[0] == 3);
|
|
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
|
|
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
|
|
CHECK(dcpl.GetNumOfDimensions() == 3);
|
|
CHECK(dcpl.GetChunking()[0] == 1);
|
|
CHECK(dcpl.GetChunking()[1] == ypixel);
|
|
CHECK(dcpl.GetChunking()[2] == xpixel);
|
|
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::NO_COMPRESSION);
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
|
|
REQUIRE(read_v.size() == xpixel * ypixel * sizeof(uint32_t));
|
|
CHECK(memcmp(read_v.data(), tmp.data(), xpixel * ypixel * sizeof(uint32_t)) == 0);
|
|
}
|
|
|
|
remove("scratch4.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking_bslz4", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
size_t len = 1234;
|
|
std::vector<uint8_t> tmp(len, 200);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
dcpl.SetCompression(CompressionAlgorithm::BSHUF_LZ4, 0);
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), 1234, {0, 0, 0});
|
|
dataset.WriteDirectChunk(tmp.data(), 1234, {2, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5Dcpl dcpl(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 4);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 3);
|
|
CHECK(data_space_vec.GetDimensions()[0] == 3);
|
|
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
|
|
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
|
|
CHECK(dcpl.GetNumOfDimensions() == 3);
|
|
CHECK(dcpl.GetChunking()[0] == 1);
|
|
CHECK(dcpl.GetChunking()[1] == ypixel);
|
|
CHECK(dcpl.GetChunking()[2] == xpixel);
|
|
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::BSHUF_LZ4);
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
|
|
REQUIRE(read_v == tmp);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking_ReadToU8", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
size_t len = xpixel*ypixel;
|
|
std::vector<uint32_t> tmp(len, 200);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
dcpl.SetCompression(CompressionAlgorithm::NO_COMPRESSION, 0);
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), xpixel * ypixel * sizeof(uint32_t), {1, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVectorToU8(read_v, {1,0,0}, {1, ypixel, xpixel}));
|
|
REQUIRE(read_v.size() == xpixel * ypixel *sizeof(uint32_t));
|
|
REQUIRE(memcmp(read_v.data(), tmp.data(), read_v.size()) == 0);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
|
|
std::string long_string = "ccdcsdcdscsdcsdcs";
|
|
std::vector<std::string> tmp_vector = {"aaaaaaaaa", "b", long_string};
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch4.h5");
|
|
REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch4.h5");
|
|
HDF5DataSet vector_dataset(file, "vec");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == long_string.size() + 1);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 1);
|
|
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
|
|
}
|
|
|
|
remove("scratch4.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Attr", "[HDF5][Unit]") {
|
|
uint16_t tmp_scalar = 16788;
|
|
|
|
std::vector<double> vec = {0,1,2,3,4,6788.0};
|
|
std::string sattr = "val";
|
|
double dattr = 456.567567;
|
|
int32_t iattr = 115;
|
|
|
|
{
|
|
HDF5File file("scratch32.h5");
|
|
file.SaveScalar("scalar", tmp_scalar)
|
|
->Attr("int", iattr)
|
|
.Attr("double", dattr)
|
|
.Attr("str", sattr)
|
|
.Attr("vec", vec);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch32.h5");
|
|
HDF5DataSet scalar_dataset(file, "scalar");
|
|
CHECK(scalar_dataset.ReadAttrDouble("double") == dattr);
|
|
CHECK(scalar_dataset.ReadAttrInt("int") == iattr);
|
|
CHECK(scalar_dataset.ReadAttrStr("str") == sattr);
|
|
CHECK(scalar_dataset.ReadAttrVec("vec") == vec);
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("int"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("double"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrStr("vec"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("int"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrVec("double"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrDouble("vec"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("vec"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("str"));
|
|
REQUIRE_THROWS(scalar_dataset.ReadAttrInt("bla"));
|
|
}
|
|
remove("scratch32.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5LeafDetection", "[HDF5]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Group(file, "/group");
|
|
HDF5Group(file, "/group/sub1");
|
|
HDF5Group(file, "/group/sub2");
|
|
file.SaveScalar("/group/scalar", 5.0);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
auto vec = file.FindLeafs("/group");
|
|
REQUIRE(vec.size() == 3);
|
|
bool found0 = false, found1 = false;
|
|
for (int i = 0; i < vec.size(); i++) {
|
|
if (vec[i] == "sub1")
|
|
found0 = true;
|
|
if (vec[i] == "scalar")
|
|
found1 = true;
|
|
}
|
|
REQUIRE(found0);
|
|
REQUIRE(found1);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5ExternalLink", "[HDF5][Unit]") {
|
|
uint16_t tmp_scalar = 16788;
|
|
{
|
|
HDF5File file("scratch123.h5");
|
|
file.ExternalLink("../abc.h5", "/zzz/fgh6", "fgh6");
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch123.h5");
|
|
REQUIRE(file.GetLinkedFileName("/fgh6") == "abc.h5");
|
|
}
|
|
remove("scratch123.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.FilePrefix("test01").ImagesPerTrigger(950);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
|
|
x.FilePrefix("test02");
|
|
x.FillMessage(start_message);
|
|
master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
remove("test01_master.h5");
|
|
remove("test02_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile_UserData", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.FilePrefix("test07").ImagesPerTrigger(950);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.user_data["hdf5"]["val1"] = 7;
|
|
start_message.user_data["hdf5"]["val2"] = "str";
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("test07_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val1"));
|
|
REQUIRE(dataset->ReadScalar<double>() == 7.0);
|
|
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val2"));
|
|
REQUIRE(dataset->ReadString() == "str");
|
|
}
|
|
remove("test07_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile_RadInt", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
|
|
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
|
|
|
|
x.FilePrefix("test01_rad_int").ImagesPerTrigger(950);
|
|
|
|
PixelMask pixel_mask(x);
|
|
AzimuthalIntegrationMapping mapping(x, pixel_mask);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
end_message.az_int_result["avg1"] = profile.GetResult();
|
|
end_message.az_int_result["avg2"] = profile.GetResult();
|
|
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
remove("test01_rad_int_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
std::vector<SpotToSave> spots;
|
|
|
|
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.OverwriteExistingFiles(true);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
|
|
auto v = file_set.Finalize();
|
|
REQUIRE(v.size() == 3); // 3 files
|
|
REQUIRE(v[0].filename == "test02_1p10_data_000001.h5");
|
|
REQUIRE(v[0].total_images == 2);
|
|
REQUIRE(v[1].filename == "test02_1p10_data_000002.h5");
|
|
REQUIRE(v[1].total_images == 2);
|
|
REQUIRE(v[2].filename == "test02_1p10_data_000003.h5");
|
|
REQUIRE(v[2].total_images == 1);
|
|
|
|
REQUIRE(!file_set.GetZMQAddr());
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("test02_1p10_data_000001.h5");
|
|
remove("test02_1p10_data_000002.h5");
|
|
remove("test02_1p10_data_000003.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.FrameTime(std::chrono::microseconds(1000), std::chrono::microseconds(100));
|
|
|
|
DatasetSettings d;
|
|
d.FilePrefix("run0345_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
|
|
.PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
|
|
.SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("test_sample").RunName("run1");
|
|
|
|
x.ImportDatasetSettings(d).OverwriteExistingFiles(true);
|
|
std::vector<SpotToSave> spots;
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
file_set.SetupFinalizedFileSocket("ipc://#1");
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
|
|
ZMQSocket s(ZMQSocketType::Sub);
|
|
s.Connect("ipc://#1");
|
|
s.SubscribeAll();
|
|
s.ReceiveTimeout(std::chrono::seconds(5));
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
REQUIRE(file_set.Finalize().size() == 3);
|
|
|
|
ZMQMessage msg;
|
|
nlohmann::json j;
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_acq_data_000001.h5");
|
|
REQUIRE(j["file_number"] == 1);
|
|
REQUIRE(j["nimages"] == 2);
|
|
REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetIncidentEnergy_keV() * 1000.0));
|
|
REQUIRE(j["space_group_number"] == 96);
|
|
REQUIRE(j["experiment_group"] == "p12345");
|
|
REQUIRE(j["run_number"] == 345);
|
|
REQUIRE(j["run_name"] == "run1");
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
std::cout << j.dump(4) << std::endl;
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_acq_data_000002.h5");
|
|
REQUIRE(j["file_number"] == 2);
|
|
REQUIRE(j["nimages"] == 2);
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_acq_data_000003.h5");
|
|
REQUIRE(j["file_number"] == 3);
|
|
REQUIRE(j["nimages"] == 1);
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
|
|
remove("test05_data_000001.h5");
|
|
remove("test05_data_000002.h5");
|
|
remove("test05_data_000003.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
std::vector<SpotToSave> spots;
|
|
|
|
spots.push_back({10,10,7});
|
|
spots.push_back({20,50,12});
|
|
spots.push_back({1000,500,3});
|
|
|
|
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
|
|
remove("test02_1p10_spots_data_000001.h5");
|
|
remove("test02_1p10_spots_data_000002.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
|
|
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
|
|
|
|
PixelMask pixel_mask(x);
|
|
AzimuthalIntegrationMapping mapping(x, pixel_mask);
|
|
|
|
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
|
|
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
|
|
|
|
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION).OverwriteExistingFiles(true);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
|
|
start_message.az_int_q_bin_count = mapping.GetQBinCount();
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), i);
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
|
|
StartMessage message;
|
|
message.file_prefix = "z/x";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
|
|
}
|
|
|
|
TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
|
|
StartMessage message;
|
|
message.file_prefix = "acq";
|
|
message.source_name = "SwissFEL";
|
|
message.detector_serial_number = "JF17T16V01";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
|
|
|
|
message.detector_serial_number = "";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {
|
|
REQUIRE(ExtractFilename("filename_data_000001.h5") == "filename_data_000001.h5");
|
|
REQUIRE(ExtractFilename("dir1/filename_data_000001.h5") == "filename_data_000001.h5");
|
|
REQUIRE(ExtractFilename("dir1/dir2/filename_data_000001.h5") == "filename_data_000001.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5DataType", "[HDF5]") {
|
|
HDF5DataType type1(1,true);
|
|
REQUIRE(type1.GetElemSize() == 1);
|
|
|
|
HDF5DataType type2(2,true);
|
|
REQUIRE(type2.GetElemSize() == 2);
|
|
|
|
HDF5DataType type4(4,true);
|
|
REQUIRE(type4.GetElemSize() == 4);
|
|
|
|
HDF5DataType type8(8,true);
|
|
REQUIRE(type8.GetElemSize() == 8);
|
|
|
|
REQUIRE_THROWS(HDF5DataType(7,true));
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link").SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
|
|
x.OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000002"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000003"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == 1);
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000004"));
|
|
}
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_vds");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
REQUIRE(start_message.file_format == FileWriterFormat::NXmxVDS);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_vds_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() - 2);
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_vds_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
|
|
}
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NXmxIntegrated_Basic", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.FilePrefix("integrated_basic");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
|
|
|
|
// NXmxIntegrated forces all images into one file
|
|
REQUIRE(x.GetImagesPerFile() == x.GetImageNum());
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
REQUIRE(start_message.file_format == FileWriterFormat::NXmxIntegrated);
|
|
// images_per_file should equal total images for integrated
|
|
REQUIRE(start_message.images_per_file == x.GetImageNum());
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 42);
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.number = i;
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
auto stats = writer.Finalize();
|
|
|
|
// All images in one file — only one stats entry
|
|
REQUIRE(stats.size() == 1);
|
|
REQUIRE(stats[0].total_images == x.GetImageNum());
|
|
}
|
|
|
|
// Single integrated file, no separate master or data files
|
|
REQUIRE(!std::filesystem::exists("integrated_basic.h5"));
|
|
REQUIRE(std::filesystem::exists("integrated_basic_master.h5"));
|
|
REQUIRE(!std::filesystem::exists("integrated_basic_data_000001.h5"));
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("integrated_basic_master.h5");
|
|
|
|
// Data should be directly in the file
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum());
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
|
|
// Master metadata should also be present
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/instrument/detector/beam_center_x"));
|
|
REQUIRE(dataset->ReadScalar<float>() == Catch::Approx(x.GetBeamX_pxl()));
|
|
|
|
// No external links (unlike NXmxLegacy)
|
|
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data_000001"));
|
|
}
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("integrated_basic_master.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NXmxIntegrated_WithSpots", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.FilePrefix("integrated_spots");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 10);
|
|
|
|
std::vector<SpotToSave> spots;
|
|
spots.push_back({10.0f, 20.0f, 100.0f});
|
|
spots.push_back({30.0f, 40.0f, 200.0f});
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
message.image_collection_efficiency = 1.0f;
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
auto stats = writer.Finalize();
|
|
REQUIRE(stats.size() == 1);
|
|
}
|
|
|
|
REQUIRE(std::filesystem::exists("integrated_spots_master.h5"));
|
|
{
|
|
HDF5ReadOnlyFile file("integrated_spots_master.h5");
|
|
|
|
// Detector plugin data should exist in the same file
|
|
REQUIRE(file.Exists("/entry/detector"));
|
|
|
|
// Image data should exist
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
|
|
}
|
|
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("integrated_spots_master.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NXmxIntegrated_ZeroImages", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.FilePrefix("integrated_zero");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = 0;
|
|
|
|
FileWriter writer(start_message);
|
|
// Write no images — just finalize
|
|
writer.WriteHDF5(end_message);
|
|
auto stats = writer.Finalize();
|
|
|
|
// No data files created
|
|
REQUIRE(stats.empty());
|
|
}
|
|
|
|
// Master file should still exist with metadata
|
|
REQUIRE(std::filesystem::exists("integrated_zero_master.h5"));
|
|
{
|
|
HDF5ReadOnlyFile file("integrated_zero_master.h5");
|
|
REQUIRE(file.Exists("/entry"));
|
|
// No data dataset since no images written
|
|
REQUIRE_THROWS(std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
|
|
}
|
|
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("integrated_zero_master.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NXmxIntegrated_AzInt", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.DetectorDistance_mm(50).BeamX_pxl(500).BeamY_pxl(500);
|
|
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
|
|
x.ImagesPerTrigger(3).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.FilePrefix("integrated_azint");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
|
|
|
|
PixelMask pixel_mask(x);
|
|
AzimuthalIntegrationMapping mapping(x, pixel_mask);
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
|
|
start_message.az_int_q_bin_count = mapping.GetQBinCount();
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 5);
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), static_cast<float>(i));
|
|
message.number = i;
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
auto stats = writer.Finalize();
|
|
REQUIRE(stats.size() == 1);
|
|
}
|
|
|
|
REQUIRE(std::filesystem::exists("integrated_azint_master.h5"));
|
|
{
|
|
HDF5ReadOnlyFile file("integrated_azint_master.h5");
|
|
|
|
// Azimuthal integration bin mapping should exist (written by plugin)
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/bin_to_q"));
|
|
|
|
// Per-image azint data should exist
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/azint/image"));
|
|
HDF5DataSpace space(*dataset);
|
|
REQUIRE(space.GetNumOfDimensions() == 3);
|
|
REQUIRE(space.GetDimensions()[0] == x.GetImageNum());
|
|
}
|
|
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("integrated_azint_master.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NXmxIntegrated_OutOfOrder", "[HDF5][Full]") {
|
|
// Test that out-of-order image delivery works with NXmxIntegrated
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(5).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.FilePrefix("integrated_ooo");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).OverwriteExistingFiles(true);
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 7);
|
|
|
|
// Write images out of order
|
|
std::vector<int> order = {3, 1, 4, 0, 2};
|
|
for (int idx : order) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.number = idx;
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
auto stats = writer.Finalize();
|
|
REQUIRE(stats.size() == 1);
|
|
REQUIRE(stats[0].total_images == 5);
|
|
}
|
|
|
|
REQUIRE(std::filesystem::exists("integrated_ooo_master.h5"));
|
|
{
|
|
HDF5ReadOnlyFile file("integrated_ooo_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file, "/entry/data/data"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetDimensions()[0] == 5);
|
|
}
|
|
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("integrated_ooo_master.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("data_only");
|
|
x.SetFileWriterFormat(FileWriterFormat::DataOnly).OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
REQUIRE(start_message.file_format == FileWriterFormat::DataOnly);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
REQUIRE(!std::filesystem::exists("data_only_master.h5"));
|
|
REQUIRE(std::filesystem::exists("data_only_data_000001.h5"));
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(2));
|
|
|
|
std::vector<int16_t> calib_1(x.GetModulesNum() * RAW_MODULE_SIZE);
|
|
std::vector<float> calib_2(x.GetModulesNum() * RAW_MODULE_SIZE);
|
|
for (int i = 0; i < x.GetModulesNum(); i++) {
|
|
calib_1[i] = i * 3 - 1024;
|
|
calib_2[i] = static_cast<float>(i) / 16.0 + 123.25f;
|
|
}
|
|
|
|
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_LZ4);
|
|
std::vector<uint8_t> calib_3 = compressor.Compress(calib_1);
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("calib");
|
|
x.OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
|
|
CompressedImage image_1(calib_1, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
|
|
CompressedImage image_2(calib_2, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES);
|
|
CompressedImage image_3(calib_3, RAW_MODULE_COLS, x.GetModulesNum() * RAW_MODULE_LINES,
|
|
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_LZ4);
|
|
image_1.Channel("calib1");
|
|
image_2.Channel("calib2");
|
|
image_3.Channel("calib3");
|
|
|
|
writer.WriteHDF5(image_1);
|
|
writer.WriteHDF5(image_2);
|
|
writer.WriteHDF5(image_3);
|
|
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
REQUIRE(std::filesystem::exists("calib_master.h5"));
|
|
{
|
|
HDF5ReadOnlyFile file("calib_master.h5");
|
|
{
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib1"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 2);
|
|
HDF5DataType type(*dataset);
|
|
REQUIRE(type.GetElemSize() == 2);
|
|
REQUIRE(type.IsSigned());
|
|
REQUIRE(type.IsInteger());
|
|
|
|
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
|
|
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
|
|
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
|
|
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
|
|
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
|
|
}
|
|
{
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib2"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 2);
|
|
HDF5DataType type(*dataset);
|
|
REQUIRE(type.GetElemSize() == 4);
|
|
REQUIRE(type.IsSigned());
|
|
REQUIRE(!type.IsInteger());
|
|
|
|
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
|
|
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
|
|
std::vector<float> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
|
|
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
|
|
CHECK(memcmp(output.data(), calib_2.data(), output.size() * type.GetElemSize()) == 0);
|
|
}
|
|
{
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/instrument/detector/calibration/calib3"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 2);
|
|
HDF5DataType type(*dataset);
|
|
REQUIRE(type.GetElemSize() == 2);
|
|
REQUIRE(type.IsSigned());
|
|
REQUIRE(type.IsInteger());
|
|
|
|
REQUIRE(file_space.GetDimensions()[0] == RAW_MODULE_COLS);
|
|
REQUIRE(file_space.GetDimensions()[1] == RAW_MODULE_LINES * x.GetModulesNum());
|
|
std::vector<int16_t> output(file_space.GetDimensions()[0] * file_space.GetDimensions()[1]);
|
|
dataset->ReadVector(output, {0,0}, file_space.GetDimensions());
|
|
CHECK(memcmp(output.data(), calib_1.data(), output.size() * type.GetElemSize()) == 0);
|
|
}
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
|
|
TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero").OverwriteExistingFiles(true);
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = 0;
|
|
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
{
|
|
std::unique_ptr<HDF5ReadOnlyFile> file;
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("link_zero_master.h5"));
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(*file,"/entry/data/data_000001"));
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
HDF5File file("scratch_vds_reverse_contiguous.h5", true);
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type((int16_t) 0);
|
|
HDF5DataSpace full_space({5, 4, 3});
|
|
|
|
{
|
|
HDF5DataSpace source_space({2, 4, 3});
|
|
HDF5DataSpace virtual_space({5, 4, 3});
|
|
virtual_space.SelectHyperslab({0, 0, 0}, {2, 4, 3});
|
|
dcpl.SetVirtual("file_000001.h5", "/entry/data/data", source_space, virtual_space);
|
|
}
|
|
|
|
{
|
|
HDF5DataSpace source_space({3, 4, 3});
|
|
HDF5DataSpace virtual_space({5, 4, 3});
|
|
virtual_space.SelectHyperslab({2, 0, 0}, {3, 4, 3});
|
|
dcpl.SetVirtual("file_000002.h5", "/entry/data/data", source_space, virtual_space);
|
|
}
|
|
|
|
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
|
|
|
|
HDF5Dcpl read_dcpl(dataset);
|
|
auto mappings = read_dcpl.GetVirtualMappings();
|
|
|
|
REQUIRE(mappings.size() == 2);
|
|
REQUIRE(mappings[0].ContainsVirtualImage(0));
|
|
REQUIRE(mappings[0].ContainsVirtualImage(1));
|
|
REQUIRE(!mappings[0].ContainsVirtualImage(2));
|
|
CHECK(mappings[0].SourceImage(0) == 0);
|
|
CHECK(mappings[0].SourceImage(1) == 1);
|
|
|
|
REQUIRE(mappings[1].ContainsVirtualImage(2));
|
|
REQUIRE(mappings[1].ContainsVirtualImage(4));
|
|
CHECK(mappings[1].SourceImage(2) == 0);
|
|
CHECK(mappings[1].SourceImage(3) == 1);
|
|
CHECK(mappings[1].SourceImage(4) == 2);
|
|
}
|
|
|
|
remove("scratch_vds_reverse_contiguous.h5");
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Objects_VDS_reverse_strided", "[HDF5][Unit]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
HDF5File file("scratch_vds_reverse_strided.h5", true);
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type((int16_t) 0);
|
|
HDF5DataSpace full_space({6, 4, 3});
|
|
|
|
{
|
|
HDF5DataSpace source_space({3, 4, 3});
|
|
HDF5DataSpace virtual_space({6, 4, 3});
|
|
virtual_space.SelectHyperslabWithStride({0, 0, 0}, {3, 4, 3}, {2, 1, 1});
|
|
dcpl.SetVirtual("file_even.h5", "/entry/data/data", source_space, virtual_space);
|
|
}
|
|
|
|
{
|
|
HDF5DataSpace source_space({3, 4, 3});
|
|
HDF5DataSpace virtual_space({6, 4, 3});
|
|
virtual_space.SelectHyperslabWithStride({1, 0, 0}, {3, 4, 3}, {2, 1, 1});
|
|
dcpl.SetVirtual("file_odd.h5", "/entry/data/data", source_space, virtual_space);
|
|
}
|
|
|
|
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
|
|
|
|
HDF5Dcpl read_dcpl(dataset);
|
|
auto mappings = read_dcpl.GetVirtualMappings();
|
|
|
|
REQUIRE(mappings.size() == 2);
|
|
|
|
REQUIRE(mappings[0].ContainsVirtualImage(0));
|
|
REQUIRE(mappings[0].ContainsVirtualImage(2));
|
|
REQUIRE(mappings[0].ContainsVirtualImage(4));
|
|
REQUIRE(!mappings[0].ContainsVirtualImage(1));
|
|
CHECK(mappings[0].SourceImage(0) == 0);
|
|
CHECK(mappings[0].SourceImage(2) == 1);
|
|
CHECK(mappings[0].SourceImage(4) == 2);
|
|
|
|
REQUIRE(mappings[1].ContainsVirtualImage(1));
|
|
REQUIRE(mappings[1].ContainsVirtualImage(3));
|
|
REQUIRE(mappings[1].ContainsVirtualImage(5));
|
|
REQUIRE(!mappings[1].ContainsVirtualImage(0));
|
|
CHECK(mappings[1].SourceImage(1) == 0);
|
|
CHECK(mappings[1].SourceImage(3) == 1);
|
|
CHECK(mappings[1].SourceImage(5) == 2);
|
|
}
|
|
|
|
remove("scratch_vds_reverse_strided.h5");
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
// Overwrite is detected up front for back-channel transports (default): the writer that owns the
|
|
// master file checks the master file and every data file the run will write, its staggered
|
|
// siblings' included - they share a directory by construction, since the master links them by
|
|
// relative name. The ZeroMQ path (no back-channel) must opt out and keep writing .tmp files
|
|
// instead.
|
|
TEST_CASE("FileWriter_overwrite_detected_at_start", "[HDF5][Overwrite]") {
|
|
RegisterHDF5Filter();
|
|
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.FilePrefix("fw_overwrite_start").ImagesPerTrigger(3).ImagesPerFile(2)
|
|
.Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(false);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
REQUIRE(start_message.write_master_file.value_or(false)); // this writer owns the master
|
|
|
|
// A data file in the way is refused up front, even one another writer would have written.
|
|
// It used to pass here and fail only when the file was renamed into place at the end of the
|
|
// run - after the whole dataset had been collected.
|
|
{ std::ofstream(HDF5Metadata::DataFileName(start_message, 0)) << "blocker"; }
|
|
REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);
|
|
|
|
// A writer that does not own the master file speaks for none of them: it would otherwise stat
|
|
// files its siblings are already creating. The temporary writer cleans up its own tmp.
|
|
{
|
|
StartMessage not_master = start_message;
|
|
not_master.write_master_file = false;
|
|
REQUIRE_NOTHROW(FileWriter(not_master));
|
|
}
|
|
remove(HDF5Metadata::DataFileName(start_message, 0).c_str());
|
|
|
|
// The master file, on the other hand, does collide.
|
|
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
|
|
|
|
// Back-channel transport (direct HDF5 / TCP): fail fast in the constructor.
|
|
REQUIRE_THROWS_AS(FileWriter(start_message), JFJochException);
|
|
|
|
// ZeroMQ transport (no back-channel): must not throw - it will write .tmp and
|
|
// only fail at the final rename. The un-finalized writer cleans up its own tmp.
|
|
REQUIRE_NOTHROW(FileWriter(start_message, /*check_overwrite_at_start=*/false));
|
|
|
|
remove(HDF5Metadata::MasterFileName(start_message).c_str());
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
// If EndDataCollection throws while finalizing (here the master file appears mid-run,
|
|
// so the early check can't catch it), the pusher must still tear down its writer so a
|
|
// subsequent collection can start instead of dying with "already writing images".
|
|
TEST_CASE("HDF5FilePusher_finalize_failure_recovers", "[HDF5FilePusher][Repro]") {
|
|
RegisterHDF5Filter();
|
|
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.FilePrefix("pusher_finalize_repro").ImagesPerTrigger(1)
|
|
.Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.SetFileWriterFormat(FileWriterFormat::NXmxVDS)
|
|
.OverwriteExistingFiles(false);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
EndMessage end_message{};
|
|
|
|
HDF5FilePusher pusher;
|
|
|
|
pusher.StartDataCollection(start_message);
|
|
// Create the conflict after the start-time check has already passed.
|
|
{ std::ofstream(HDF5Metadata::MasterFileName(start_message)) << "blocker"; }
|
|
REQUIRE_THROWS_AS(pusher.EndDataCollection(end_message), JFJochException);
|
|
|
|
// Writer released despite the failure: the next collection starts cleanly
|
|
// instead of dying with "already writing images".
|
|
remove(HDF5Metadata::MasterFileName(start_message).c_str());
|
|
REQUIRE_NOTHROW(pusher.StartDataCollection(start_message));
|
|
REQUIRE_NOTHROW(pusher.EndDataCollection(end_message));
|
|
|
|
// The failed finalize intentionally leaves a .tmp behind - sweep the prefix.
|
|
for (const auto &e : std::filesystem::directory_iterator("."))
|
|
if (e.path().filename().string().rfind("pusher_finalize_repro", 0) == 0)
|
|
std::filesystem::remove(e.path());
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
// A link that is written in the file and points at something not there is NOT an existing dataset.
|
|
// Every DECTRIS Eiger master links saturation_value, pixel_mask, bit_depth_readout and
|
|
// serial_number into a companion <prefix>_meta.h5, and that file is routinely not kept when a
|
|
// dataset is archived or deposited. Asking only whether the LINK exists then answers yes and the
|
|
// read that follows throws, which turns every optional-field guard in the reader into a hard
|
|
// failure - measured on a deposited Eiger 16M set that could not be opened at all.
|
|
TEST_CASE("HDF5Objects_dangling_external_link_does_not_exist", "[HDF5][Unit]") {
|
|
const std::string fname = "test_dangling_link.h5";
|
|
remove(fname.c_str());
|
|
{
|
|
HDF5File file(fname);
|
|
HDF5Group group(file, "/entry");
|
|
group.SaveScalar("present", static_cast<int64_t>(7));
|
|
// ...and a link into a file that does not exist, exactly as an orphaned Eiger master has.
|
|
REQUIRE(H5Lcreate_external("no_such_meta.h5", "/_dectris/whatever",
|
|
group.GetID(), "absent", H5P_DEFAULT, H5P_DEFAULT) >= 0);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file(fname);
|
|
CHECK(file.Exists("/entry/present"));
|
|
CHECK(file.GetInt("/entry/present") == 7);
|
|
// The link is there, the object is not.
|
|
CHECK(H5Lexists(file.GetID(), "/entry/absent", H5P_DEFAULT) > 0);
|
|
CHECK_FALSE(file.Exists("/entry/absent"));
|
|
// ...so an optional read of it falls back instead of throwing.
|
|
CHECK(file.GetString("/entry/absent", "fallback") == "fallback");
|
|
}
|
|
remove(fname.c_str());
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|