mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

v1.0.0-rc.145 (#55)
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 16m26s
Details
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 14m26s
Details
Build Packages / build:rpm (rocky8) (push) Successful in 17m23s
Details
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 17m32s
Details
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 18m16s
Details
Build Packages / build:rpm (rocky9) (push) Successful in 12m45s
Details
Build Packages / build:rpm (ubuntu2404) (push) Successful in 12m58s
Details
Build Packages / XDS test (durin plugin) (push) Successful in 11m22s
Details
Build Packages / DIALS test (push) Successful in 14m28s
Details
Build Packages / Generate python client (push) Successful in 1m1s
Details
Build Packages / Build documentation (push) Successful in 2m40s
Details
Build Packages / Create release (push) Has been skipped
Details
Build Packages / XDS test (neggia plugin) (push) Successful in 10m52s
Details
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 15m2s
Details
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 17m25s
Details
Build Packages / build:rpm (ubuntu2204) (push) Successful in 11m49s
Details
Build Packages / XDS test (JFJoch plugin) (push) Successful in 11m34s
Details
Build Packages / Unit tests (push) Successful in 44m51s
Details

Browse Source 
This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.
* XDS plugin: Improve performance of VDS reading.
* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.
* jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

Reviewed-on: #55
This commit was merged in pull request #55.
This commit is contained in:
leonarski_f
2026-05-06 21:50:02 +02:00
parent 7d34e8a049
commit caef26873e
152 changed files with 1995 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/HDF5WritingTest.cpp
+101 -4
View File
@@ -514,7 +514,7 @@ TEST_CASE("HDF5Writer_Socket", "[HDF5][Full]") {
.PhotonEnergy_keV(12.07).SetUnitCell(UnitCell{.a = 97, .b = 97, .c = 38, .alpha= 90, .beta = 90, .gamma = 90})
.SpaceGroupNumber(96).RunNumber(345).ExperimentGroup("p12345").SampleName("lysozyme").RunName("run1");
x.ImportDatasetSettings(d);
x.ImportDatasetSettings(d).OverwriteExistingFiles(true);
std::vector<SpotToSave> spots;
StartMessage start_message;
@@ -627,7 +627,7 @@ TEST_CASE("HDF5Writer_Rad_Int_Profile", "[HDF5][Full]") {
std::vector<float> rad_int_profile(mapping.GetBinNumber(), 4.0);
std::vector<float> rad_int_avg(mapping.GetBinNumber(), 0.33);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION);
x.FilePrefix("test02_1p10_rad_int").ImagesPerTrigger(5).ImagesPerFile(3).Compression(CompressionAlgorithm::NO_COMPRESSION).OverwriteExistingFiles(true);
StartMessage start_message;
x.FillMessage(start_message);
start_message.az_int_bin_to_q = mapping.GetBinToQ();
@@ -698,7 +698,7 @@ TEST_CASE("HDF5DataType", "[HDF5]") {
TEST_CASE("HDF5Writer_Link", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link");
x.ImagesPerTrigger(7).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link").SetFileWriterFormat(FileWriterFormat::NXmxLegacy);
x.OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
@@ -1225,7 +1225,7 @@ TEST_CASE("HDF5Writer_Calibration", "[HDF5][Full]") {
TEST_CASE("HDF5Writer_Link_zero_images", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero");
x.ImagesPerTrigger(5).ImagesPerFile(2).Compression(CompressionAlgorithm::NO_COMPRESSION).FilePrefix("link_zero").OverwriteExistingFiles(true);
{
RegisterHDF5Filter();
@@ -1344,3 +1344,100 @@ TEST_CASE("FileWriter_TIFF", "[HDF5][Full]") {
REQUIRE(!file_set.GetZMQAddr());
}
}
TEST_CASE("HDF5Objects_VDS_reverse_contiguous", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_contiguous.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({5, 4, 3});
{
HDF5DataSpace source_space({2, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({0, 0, 0}, {2, 4, 3});
dcpl.SetVirtual("file_000001.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({5, 4, 3});
virtual_space.SelectHyperslab({2, 0, 0}, {3, 4, 3});
dcpl.SetVirtual("file_000002.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(1));
REQUIRE(!mappings[0].ContainsVirtualImage(2));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(1) == 1);
REQUIRE(mappings[1].ContainsVirtualImage(2));
REQUIRE(mappings[1].ContainsVirtualImage(4));
CHECK(mappings[1].SourceImage(2) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(4) == 2);
}
remove("scratch_vds_reverse_contiguous.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("HDF5Objects_VDS_reverse_strided", "[HDF5][Unit]") {
{
RegisterHDF5Filter();
HDF5File file("scratch_vds_reverse_strided.h5", true);
HDF5Dcpl dcpl;
HDF5DataType data_type((int16_t) 0);
HDF5DataSpace full_space({6, 4, 3});
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({0, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_even.h5", "/entry/data/data", source_space, virtual_space);
}
{
HDF5DataSpace source_space({3, 4, 3});
HDF5DataSpace virtual_space({6, 4, 3});
virtual_space.SelectHyperslabWithStride({1, 0, 0}, {3, 4, 3}, {2, 1, 1});
dcpl.SetVirtual("file_odd.h5", "/entry/data/data", source_space, virtual_space);
}
HDF5DataSet dataset(file, "/data", data_type, full_space, dcpl);
HDF5Dcpl read_dcpl(dataset);
auto mappings = read_dcpl.GetVirtualMappings();
REQUIRE(mappings.size() == 2);
REQUIRE(mappings[0].ContainsVirtualImage(0));
REQUIRE(mappings[0].ContainsVirtualImage(2));
REQUIRE(mappings[0].ContainsVirtualImage(4));
REQUIRE(!mappings[0].ContainsVirtualImage(1));
CHECK(mappings[0].SourceImage(0) == 0);
CHECK(mappings[0].SourceImage(2) == 1);
CHECK(mappings[0].SourceImage(4) == 2);
REQUIRE(mappings[1].ContainsVirtualImage(1));
REQUIRE(mappings[1].ContainsVirtualImage(3));
REQUIRE(mappings[1].ContainsVirtualImage(5));
REQUIRE(!mappings[1].ContainsVirtualImage(0));
CHECK(mappings[1].SourceImage(1) == 0);
CHECK(mappings[1].SourceImage(3) == 1);
CHECK(mappings[1].SourceImage(5) == 2);
}
remove("scratch_vds_reverse_strided.h5");
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}