887 lines
32 KiB
C++
887 lines
32 KiB
C++
// SPDX-FileCopyrightText: 2024 Filip Leonarski, Paul Scherrer Institute <filip.leonarski@psi.ch>
|
|
// SPDX-License-Identifier: GPL-3.0-only
|
|
|
|
#include <catch2/catch_all.hpp>
|
|
#include <iostream>
|
|
|
|
#include "../common/DiffractionExperiment.h"
|
|
#include "../writer/HDF5Objects.h"
|
|
#include "../writer/FileWriter.h"
|
|
#include "../compression/JFJochCompressor.h"
|
|
#include "../common/AzimuthalIntegrationProfile.h"
|
|
#include <nlohmann/json.hpp>
|
|
|
|
using namespace std::literals::chrono_literals;
|
|
|
|
TEST_CASE("HDF5DataSet_scalar", "[HDF5][Unit]") {
|
|
uint16_t tmp_scalar = 16788;
|
|
{
|
|
HDF5File file("scratch1.h5");
|
|
file.SaveScalar("scalar", tmp_scalar);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch1.h5");
|
|
HDF5DataSet scalar_dataset(file, "scalar");
|
|
HDF5DataType data_type_scalar(scalar_dataset);
|
|
HDF5DataSpace data_space_scalar(scalar_dataset);
|
|
REQUIRE(data_type_scalar.GetElemSize() == 2);
|
|
REQUIRE(data_space_scalar.GetNumOfDimensions() == 0);
|
|
REQUIRE(scalar_dataset.ReadScalar<uint16_t>() == tmp_scalar);
|
|
REQUIRE(scalar_dataset.ReadScalar<int64_t>() == tmp_scalar);
|
|
std::vector<uint16_t> v;
|
|
REQUIRE_NOTHROW(scalar_dataset.ReadVector(v));
|
|
REQUIRE(v.size() == 1);
|
|
REQUIRE(v[0] == tmp_scalar);
|
|
}
|
|
remove("scratch1.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_string", "[HDF5][Unit]") {
|
|
std::string tmp_string = "HDF5Content";
|
|
|
|
{
|
|
HDF5File file("scratch2.h5");
|
|
file.SaveScalar("str", tmp_string);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch2.h5");
|
|
HDF5DataSet string_dataset(file, "str");
|
|
HDF5DataType data_type_str(string_dataset);
|
|
HDF5DataSpace data_space_str(string_dataset);
|
|
CHECK(data_type_str.GetElemSize() == tmp_string.size() + 1);
|
|
CHECK(data_space_str.GetNumOfDimensions() == 0);
|
|
CHECK(string_dataset.ReadString() == tmp_string);
|
|
}
|
|
remove("scratch2.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_vector", "[HDF5][Unit]") {
|
|
std::vector<double> tmp_vector (16384);
|
|
tmp_vector[0] = 599.88;
|
|
tmp_vector[1000] = 800.12;
|
|
tmp_vector[15000] = 3.1415926;
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch3.h5");
|
|
file.SaveVector("vec", tmp_vector);
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch3.h5");
|
|
HDF5DataSet vector_dataset(file, "vec");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 8);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 1);
|
|
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
|
|
std::vector<double> output;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVector(output));
|
|
CHECK (output[0] == tmp_vector[0]);
|
|
CHECK (output[1000] == tmp_vector[1000]);
|
|
CHECK (output[15000] == tmp_vector[15000]);
|
|
CHECK (output.size() == tmp_vector.size());
|
|
|
|
std::vector<double> output2(10);
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVector(output2, {15000},{10}));
|
|
REQUIRE_THROWS(vector_dataset.ReadVector(output2, {15000},{100}));
|
|
CHECK(output2[0] == tmp_vector[15000]);
|
|
}
|
|
|
|
remove("scratch3.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
|
|
std::vector<int32_t> tmp(xpixel*ypixel, -134);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch4.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {0, 0, 0});
|
|
dataset.WriteDirectChunk(tmp.data(), tmp.size() * sizeof(int32_t), {2, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch4.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5Dcpl dcpl(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 4);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 3);
|
|
CHECK(data_space_vec.GetDimensions()[0] == 3);
|
|
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
|
|
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
|
|
CHECK(dcpl.GetNumOfDimensions() == 3);
|
|
CHECK(dcpl.GetChunking()[0] == 1);
|
|
CHECK(dcpl.GetChunking()[1] == ypixel);
|
|
CHECK(dcpl.GetChunking()[2] == xpixel);
|
|
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::NO_COMPRESSION);
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
|
|
REQUIRE(read_v.size() == xpixel * ypixel * sizeof(uint32_t));
|
|
CHECK(memcmp(read_v.data(), tmp.data(), xpixel * ypixel * sizeof(uint32_t)) == 0);
|
|
}
|
|
|
|
remove("scratch4.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking_bslz4", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
size_t len = 1234;
|
|
std::vector<uint8_t> tmp(len, 200);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
dcpl.SetCompression(CompressionAlgorithm::BSHUF_LZ4, 0);
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), 1234, {0, 0, 0});
|
|
dataset.WriteDirectChunk(tmp.data(), 1234, {2, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5Dcpl dcpl(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == 4);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 3);
|
|
CHECK(data_space_vec.GetDimensions()[0] == 3);
|
|
CHECK(data_space_vec.GetDimensions()[1] == ypixel);
|
|
CHECK(data_space_vec.GetDimensions()[2] == xpixel);
|
|
CHECK(dcpl.GetNumOfDimensions() == 3);
|
|
CHECK(dcpl.GetChunking()[0] == 1);
|
|
CHECK(dcpl.GetChunking()[1] == ypixel);
|
|
CHECK(dcpl.GetChunking()[2] == xpixel);
|
|
REQUIRE(dcpl.GetCompression() == CompressionAlgorithm::BSHUF_LZ4);
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadDirectChunk(read_v, {2, 0, 0}));
|
|
REQUIRE(read_v == tmp);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_chunking_ReadToU8", "[HDF5][Unit]") {
|
|
size_t xpixel = 512;
|
|
size_t ypixel = 256;
|
|
size_t len = xpixel*ypixel;
|
|
std::vector<uint32_t> tmp(len, 200);
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Dcpl dcpl;
|
|
HDF5DataType data_type(4, true);
|
|
dcpl.SetChunking( {1, ypixel, xpixel});
|
|
dcpl.SetCompression(CompressionAlgorithm::NO_COMPRESSION, 0);
|
|
HDF5DataSpace data_space({3, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel});
|
|
HDF5DataSet dataset(file, "/data", data_type, data_space, dcpl);
|
|
dataset.WriteDirectChunk(tmp.data(), xpixel * ypixel * sizeof(uint32_t), {1, 0, 0});
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
HDF5DataSet vector_dataset(file, "/data");
|
|
|
|
std::vector<uint8_t> read_v;
|
|
REQUIRE_NOTHROW(vector_dataset.ReadVectorToU8(read_v, {1,0,0}, {1, ypixel, xpixel}));
|
|
REQUIRE(read_v.size() == xpixel * ypixel *sizeof(uint32_t));
|
|
REQUIRE(memcmp(read_v.data(), tmp.data(), read_v.size()) == 0);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5DataSet_vector_string", "[HDF5][Unit]") {
|
|
std::string long_string = "ccdcsdcdscsdcsdcs";
|
|
std::vector<std::string> tmp_vector = {"aaaaaaaaa", "b", long_string};
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch4.h5");
|
|
REQUIRE_NOTHROW(file.SaveVector("vec", tmp_vector));
|
|
}
|
|
|
|
{
|
|
HDF5ReadOnlyFile file("scratch4.h5");
|
|
HDF5DataSet vector_dataset(file, "vec");
|
|
HDF5DataType data_type_vec(vector_dataset);
|
|
HDF5DataSpace data_space_vec(vector_dataset);
|
|
CHECK(data_type_vec.GetElemSize() == long_string.size() + 1);
|
|
CHECK(data_space_vec.GetNumOfDimensions() == 1);
|
|
CHECK(data_space_vec.GetDimensions()[0] == tmp_vector.size());
|
|
}
|
|
|
|
remove("scratch4.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5LeafDetection", "[HDF5]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
HDF5File file("scratch5.h5");
|
|
HDF5Group(file, "/group");
|
|
HDF5Group(file, "/group/sub1");
|
|
HDF5Group(file, "/group/sub2");
|
|
file.SaveScalar("/group/scalar", 5.0);
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch5.h5");
|
|
auto vec = file.FindLeafs("/group");
|
|
REQUIRE(vec.size() == 3);
|
|
bool found0 = false, found1 = false;
|
|
for (int i = 0; i < vec.size(); i++) {
|
|
if (vec[i] == "sub1")
|
|
found0 = true;
|
|
if (vec[i] == "scalar")
|
|
found1 = true;
|
|
}
|
|
REQUIRE(found0);
|
|
REQUIRE(found1);
|
|
}
|
|
|
|
remove("scratch5.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5ExternalLink", "[HDF5][Unit]") {
|
|
uint16_t tmp_scalar = 16788;
|
|
{
|
|
HDF5File file("scratch123.h5");
|
|
file.ExternalLink("../abc.h5", "/zzz/fgh6", "fgh6");
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("scratch123.h5");
|
|
REQUIRE(file.GetLinkedFileName("/fgh6") == "abc.h5");
|
|
}
|
|
remove("scratch123.h5");
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.FilePrefix("test01").ImagesPerTrigger(950);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
|
|
x.FilePrefix("test02");
|
|
x.FillMessage(start_message);
|
|
master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
remove("test01_master.h5");
|
|
remove("test02_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile_UserData", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.FilePrefix("test07").ImagesPerTrigger(950);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.user_data["hdf5"]["val1"] = 7;
|
|
start_message.user_data["hdf5"]["val2"] = "str";
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("test07_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val1"));
|
|
REQUIRE(dataset->ReadScalar<double>() == 7.0);
|
|
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/user/val2"));
|
|
REQUIRE(dataset->ReadString() == "str");
|
|
}
|
|
remove("test07_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5MasterFile_RadInt", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
|
|
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
|
|
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
|
|
|
|
x.FilePrefix("test01_rad_int").ImagesPerTrigger(950);
|
|
|
|
PixelMask pixel_mask(x);
|
|
AzimuthalIntegration mapping(x, pixel_mask);
|
|
AzimuthalIntegrationProfile profile(mapping);
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum();
|
|
end_message.az_int_result["avg1"] = profile.GetResult();
|
|
end_message.az_int_result["avg2"] = profile.GetResult();
|
|
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
remove("test01_rad_int_master.h5");
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
std::vector<SpotToSave> spots;
|
|
|
|
x.FilePrefix("test02_1p10").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<uint16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
|
|
auto v = file_set.Finalize();
|
|
REQUIRE(v.size() == 3); // 3 files
|
|
REQUIRE(v[0].filename == "test02_1p10_data_000001.h5");
|
|
REQUIRE(v[0].total_images == 2);
|
|
REQUIRE(v[1].filename == "test02_1p10_data_000002.h5");
|
|
REQUIRE(v[1].total_images == 2);
|
|
REQUIRE(v[2].filename == "test02_1p10_data_000003.h5");
|
|
REQUIRE(v[2].total_images == 1);
|
|
|
|
REQUIRE(!file_set.GetZMQAddr());
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
remove("test02_1p10_data_000001.h5");
|
|
remove("test02_1p10_data_000002.h5");
|
|
remove("test02_1p10_data_000003.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.FrameTime(std::chrono::microseconds(1000), std::chrono::microseconds(100));
|
|
|
|
DatasetSettings d;
|
|
d.FilePrefix("run0345_lysozyme_acq").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION)
|
|
.HeaderAppendix(R"({"z":567})"_json).DetectorDistance_mm(155).BeamX_pxl(1606.62).BeamY_pxl(1669.59)
|
|
.PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
|
|
.SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("lysozyme").RunName("run1");
|
|
|
|
x.ImportDatasetSettings(d);
|
|
std::vector<SpotToSave> spots;
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
file_set.SetupFinalizedFileSocket("ipc://#1");
|
|
std::vector<uint16_t> image(x.GetPixelsNum());
|
|
|
|
ZMQSocket s(ZMQSocketType::Sub);
|
|
s.Connect("ipc://#1");
|
|
s.SubscribeAll();
|
|
s.ReceiveTimeout(std::chrono::seconds(5));
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
REQUIRE(file_set.Finalize().size() == 3);
|
|
|
|
ZMQMessage msg;
|
|
nlohmann::json j;
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000001.h5");
|
|
REQUIRE(j["file_number"] == 1);
|
|
REQUIRE(j["nimages"] == 2);
|
|
REQUIRE(j["incident_energy_eV"] == Catch::Approx(x.GetIncidentEnergy_keV() * 1000.0));
|
|
REQUIRE(j["space_group_number"] == 96);
|
|
REQUIRE(j["experiment_group"] == "p12345");
|
|
REQUIRE(j["run_number"] == 345);
|
|
REQUIRE(j["run_name"] == "run1");
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
std::cout << j.dump(4) << std::endl;
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000002.h5");
|
|
REQUIRE(j["file_number"] == 2);
|
|
REQUIRE(j["nimages"] == 2);
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
|
|
REQUIRE(s.Receive(msg, true));
|
|
j = nlohmann::json::parse(std::string((char *) msg.data(), msg.size()));
|
|
REQUIRE(j["filename"] == "run0345_lysozyme_acq_data_000003.h5");
|
|
REQUIRE(j["file_number"] == 3);
|
|
REQUIRE(j["nimages"] == 1);
|
|
REQUIRE(j.contains("user_data"));
|
|
REQUIRE(j["user_data"]["z"] == 567);
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
|
|
remove("test05_data_000001.h5");
|
|
remove("test05_data_000002.h5");
|
|
remove("test05_data_000003.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Spots", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
std::vector<SpotToSave> spots;
|
|
|
|
spots.push_back({10,10,7});
|
|
spots.push_back({20,50,12});
|
|
spots.push_back({1000,500,3});
|
|
|
|
x.FilePrefix("test02_1p10_spots").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<uint16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
|
|
remove("test02_1p10_spots_data_000001.h5");
|
|
remove("test02_1p10_spots_data_000002.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
|
|
{
|
|
RegisterHDF5Filter();
|
|
DiffractionExperiment x(DetJF4M());
|
|
x.DetectorDistance_mm(50).BeamX_pxl(1000).BeamY_pxl(1000);
|
|
x.QSpacingForAzimInt_recipA(0.1).QRangeForAzimInt_recipA(0.1, 4.0);
|
|
|
|
PixelMask pixel_mask(x);
|
|
AzimuthalIntegration mapping(x, pixel_mask);
|
|
|
|
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
|
|
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
|
|
|
|
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
start_message.az_int_bin_to_q = mapping.GetBinToQ();
|
|
start_message.az_int_phi_bin_count = mapping.GetAzimuthalBinCount();
|
|
start_message.az_int_q_bin_count = mapping.GetQBinCount();
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<uint16_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.az_int_profile = std::vector<float>(mapping.GetBinNumber(), i);
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5NXmx_DataFileName", "[HDF5]") {
|
|
StartMessage message;
|
|
message.file_prefix = "z/x";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "z/x_data_000035.h5");
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "z/x_data_000001.h5");
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 1000000));
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
|
|
}
|
|
|
|
TEST_CASE("HDF5NXmx_DataFileName_SwissFEL", "[HDF5]") {
|
|
StartMessage message;
|
|
message.file_prefix = "acq";
|
|
message.source_name = "SwissFEL";
|
|
message.detector_serial_number = "JF17T16V01";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF17T16V01.h5");
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 0) == "acq0001.JF17T16V01.h5");
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, 10000));
|
|
REQUIRE_THROWS(HDF5Metadata::DataFileName(message, -1));
|
|
|
|
message.detector_serial_number = "";
|
|
REQUIRE(HDF5Metadata::DataFileName(message, 34) == "acq0035.JF.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5Objects_ExtractFilename", "[HDF5]") {
|
|
REQUIRE(ExtractFilename("filename_data_000001.h5") == "filename_data_000001.h5");
|
|
REQUIRE(ExtractFilename("dir1/filename_data_000001.h5") == "filename_data_000001.h5");
|
|
REQUIRE(ExtractFilename("dir1/dir2/filename_data_000001.h5") == "filename_data_000001.h5");
|
|
}
|
|
|
|
TEST_CASE("HDF5DataType", "[HDF5]") {
|
|
HDF5DataType type1(1,true);
|
|
REQUIRE(type1.GetElemSize() == 1);
|
|
|
|
HDF5DataType type2(2,true);
|
|
REQUIRE(type2.GetElemSize() == 2);
|
|
|
|
HDF5DataType type4(4,true);
|
|
REQUIRE(type4.GetElemSize() == 4);
|
|
|
|
HDF5DataType type8(8,true);
|
|
REQUIRE(type8.GetElemSize() == 8);
|
|
|
|
REQUIRE_THROWS(HDF5DataType(7,true));
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link");
|
|
x.OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000002"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImagesPerFile());
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000003"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == 1);
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000004"));
|
|
}
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("HDF5Writer_Link_VDS", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_vds");
|
|
x.SetFileWriterFormat(FileWriterFormat::NXmxVDS).OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
REQUIRE(start_message.file_format == FileWriterFormat::NXmxVDS);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_vds_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data"));
|
|
HDF5DataSpace file_space(*dataset);
|
|
REQUIRE(file_space.GetNumOfDimensions() == 3);
|
|
REQUIRE(file_space.GetDimensions()[0] == x.GetImageNum() - 2);
|
|
REQUIRE(file_space.GetDimensions()[1] == x.GetYPixelsNum());
|
|
REQUIRE(file_space.GetDimensions()[2] == x.GetXPixelsNum());
|
|
}
|
|
{
|
|
HDF5ReadOnlyFile file("link_vds_master.h5");
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(file,"/entry/data/data_000001"));
|
|
}
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
|
|
TEST_CASE("HDF5Writer_NoMasterFile", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
|
|
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("data_only");
|
|
x.SetFileWriterFormat(FileWriterFormat::DataOnly).OverwriteExistingFiles(true);
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
REQUIRE(start_message.file_format == FileWriterFormat::DataOnly);
|
|
EndMessage end_message;
|
|
end_message.max_image_number = x.GetImageNum() - 2;
|
|
|
|
FileWriter writer(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum());
|
|
std::vector<SpotToSave> spots;
|
|
for (int i = 0; i < x.GetImageNum() - 2; i++) {
|
|
for (auto &j: image)
|
|
j = i;
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(writer.Write(message));
|
|
}
|
|
|
|
writer.WriteHDF5(end_message);
|
|
writer.Finalize();
|
|
}
|
|
REQUIRE(!std::filesystem::exists("data_only_master.h5"));
|
|
REQUIRE(std::filesystem::exists("data_only_data_000001.h5"));
|
|
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
|
|
TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
|
|
DiffractionExperiment x(DetJF(1));
|
|
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero");
|
|
|
|
{
|
|
RegisterHDF5Filter();
|
|
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
EndMessage end_message;
|
|
end_message.max_image_number = 0;
|
|
|
|
std::unique_ptr<NXmx> master = std::make_unique<NXmx>(start_message);
|
|
master->Finalize(end_message);
|
|
master.reset();
|
|
}
|
|
{
|
|
std::unique_ptr<HDF5ReadOnlyFile> file;
|
|
std::unique_ptr<HDF5DataSet> dataset;
|
|
REQUIRE_NOTHROW(file = std::make_unique<HDF5ReadOnlyFile>("link_zero_master.h5"));
|
|
REQUIRE_THROWS(dataset = std::make_unique<HDF5DataSet>(*file,"/entry/data/data_000001"));
|
|
}
|
|
// No leftover HDF5 objects
|
|
REQUIRE (H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
|
|
}
|
|
|
|
TEST_CASE("FileWriter_CBF_16bit", "[HDF5][Full]") {
|
|
{
|
|
DetectorSetup det = DetJF4M("DET1");
|
|
det.SerialNumber("1");
|
|
DiffractionExperiment x(det);
|
|
std::vector<SpotToSave> spots;
|
|
|
|
x.FilePrefix("lyso_cbf_16").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
x.PixelSigned(true).BitDepthImage(16);
|
|
x.SetFileWriterFormat(FileWriterFormat::CBF);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 45);
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
|
|
auto v = file_set.Finalize();
|
|
|
|
REQUIRE(!file_set.GetZMQAddr());
|
|
}
|
|
}
|
|
|
|
TEST_CASE("FileWriter_CBF", "[HDF5][Full]") {
|
|
{
|
|
DetectorSetup det = DetJF4M("DET1");
|
|
det.SerialNumber("1");
|
|
DiffractionExperiment x(det);
|
|
std::vector<SpotToSave> spots;
|
|
|
|
x.FilePrefix("lyso_cbf").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
x.PixelSigned(false).BitDepthImage(32);
|
|
x.SetFileWriterFormat(FileWriterFormat::CBF);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<uint32_t> image(x.GetPixelsNum());
|
|
|
|
for (int i = 0; i < image.size(); i++)
|
|
image[i] = i;
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
|
|
auto v = file_set.Finalize();
|
|
|
|
REQUIRE(!file_set.GetZMQAddr());
|
|
}
|
|
}
|
|
|
|
TEST_CASE("FileWriter_TIFF", "[HDF5][Full]") {
|
|
{
|
|
DetectorSetup det = DetJF4M("DET1");
|
|
det.SerialNumber("1");
|
|
DiffractionExperiment x(det);
|
|
std::vector<SpotToSave> spots;
|
|
|
|
x.FilePrefix("lyso_tiff").ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION);
|
|
x.SetFileWriterFormat(FileWriterFormat::TIFF);
|
|
StartMessage start_message;
|
|
x.FillMessage(start_message);
|
|
|
|
FileWriter file_set(start_message);
|
|
std::vector<int16_t> image(x.GetPixelsNum(), 45);
|
|
|
|
for (int i = 0; i < x.GetImageNum(); i++) {
|
|
DataMessage message{};
|
|
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
|
|
message.spots = spots;
|
|
message.number = i;
|
|
|
|
REQUIRE_NOTHROW(file_set.Write(message));
|
|
}
|
|
|
|
auto v = file_set.Finalize();
|
|
|
|
REQUIRE(!file_set.GetZMQAddr());
|
|
}
|
|
}
|