mx/Jungfraujoch
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 53 Wiki Activity

v1.0.0-rc.145 (#55)
Build Packages / build:rpm (ubuntu2204_nocuda) (push) Successful in 16m26s
Details
Build Packages / build:rpm (rocky8_nocuda) (push) Successful in 14m26s
Details
Build Packages / build:rpm (rocky8) (push) Successful in 17m23s
Details
Build Packages / build:rpm (rocky8_sls9) (push) Successful in 17m32s
Details
Build Packages / build:rpm (rocky9_sls9) (push) Successful in 18m16s
Details
Build Packages / build:rpm (rocky9) (push) Successful in 12m45s
Details
Build Packages / build:rpm (ubuntu2404) (push) Successful in 12m58s
Details
Build Packages / XDS test (durin plugin) (push) Successful in 11m22s
Details
Build Packages / DIALS test (push) Successful in 14m28s
Details
Build Packages / Generate python client (push) Successful in 1m1s
Details
Build Packages / Build documentation (push) Successful in 2m40s
Details
Build Packages / Create release (push) Has been skipped
Details
Build Packages / XDS test (neggia plugin) (push) Successful in 10m52s
Details
Build Packages / build:rpm (ubuntu2404_nocuda) (push) Successful in 15m2s
Details
Build Packages / build:rpm (rocky9_nocuda) (push) Successful in 17m25s
Details
Build Packages / build:rpm (ubuntu2204) (push) Successful in 11m49s
Details
Build Packages / XDS test (JFJoch plugin) (push) Successful in 11m34s
Details
Build Packages / Unit tests (push) Successful in 44m51s
Details

Browse Source 
This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.
* XDS plugin: Improve performance of VDS reading.
* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.
* jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

Reviewed-on: #55
This commit was merged in pull request #55.
This commit is contained in:
leonarski_f
2026-05-06 21:50:02 +02:00
parent 7d34e8a049
commit caef26873e
152 changed files with 1995 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/JFJochReaderTest.cpp
+120 -4
View File
@@ -273,13 +273,13 @@ TEST_CASE("JFJochReader_Goniometer", "[HDF5][Full]") {
DataMessage message{};
for (int i = 0; i < 5; i++) {
message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum());
message.number = 0;
message.number = i;
REQUIRE_NOTHROW(file_set.WriteHDF5(message));
}
EndMessage end_message;
end_message.max_image_number = 4;
end_message.max_image_number = 5;
file_set.WriteHDF5(end_message);
file_set.Finalize();
@@ -337,7 +337,7 @@ TEST_CASE("JFJochReader_GridScan", "[HDF5][Full]") {
}
EndMessage end_message;
end_message.max_image_number = 4;
end_message.max_image_number = 5;
file_set.WriteHDF5(end_message);
file_set.Finalize();
@@ -1497,7 +1497,7 @@ TEST_CASE("JFJochReader_NXmxIntegrated", "[HDF5][Full]") {
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("JFJochReader_GetRawImage", "[HDF5][Full]") {
TEST_CASE("JFJochReader_GetRawImage_NXmxLegacy", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true);
@@ -1554,4 +1554,120 @@ TEST_CASE("JFJochReader_GetRawImage", "[HDF5][Full]") {
remove("test_read_raw_image_data_000002.h5");
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("JFJochReader_GetRawImage_VDS", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true);
x.BitDepthImage(16).ImagesPerFile(2).SetFileWriterFormat(FileWriterFormat::NXmxVDS).PixelSigned(true)
.IndexingAlgorithm(IndexingAlgorithmEnum::FFT);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < image.size(); i++)
image[i] = static_cast<int16_t>((i * 7 + 33) % UINT16_MAX);
RegisterHDF5Filter();
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_ZSTD);
auto compressed_image = compressor.Compress(image);
{
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(compressed_image, x.GetXPixelsNum(), x.GetYPixelsNum(),
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_ZSTD);
message.number = i;
REQUIRE_NOTHROW(file_set.WriteHDF5(message));
}
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
file_set.WriteHDF5(end_message);
file_set.Finalize();
}
{
JFJochHDF5Reader reader;
REQUIRE_NOTHROW(reader.ReadFile("test_read_raw_image_master.h5"));
auto dataset = reader.GetDataset();
CHECK(dataset->experiment.GetImageNum() == 4);
std::shared_ptr<JFJochReaderRawImage> reader_image;
for (int i = 0; i < 4; i++) {
REQUIRE_NOTHROW(reader_image = reader.GetRawImage(i));
CHECK(reader_image->image.GetMode() == CompressedImageMode::Int16);
CHECK(reader_image->image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_ZSTD);
CHECK(reader_image->image.GetWidth() == x.GetXPixelsNum());
CHECK(reader_image->image.GetHeight() == x.GetYPixelsNum());
CHECK(reader_image->image.GetCompressedSize() == compressed_image.size());
CHECK(reader_image->image.GetCompressed() == reader_image->image_buffer.data());
REQUIRE(reader_image->image_buffer.size() == compressed_image.size());
CHECK(memcmp(reader_image->image_buffer.data(), compressed_image.data(), compressed_image.size()) == 0);
}
}
remove("test_read_raw_image_master.h5");
remove("test_read_raw_image_data_000001.h5");
remove("test_read_raw_image_data_000002.h5");
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}
TEST_CASE("JFJochReader_GetRawImage_Integrated", "[HDF5][Full]") {
DiffractionExperiment x(DetJF(1));
x.FilePrefix("test_read_raw_image").ImagesPerTrigger(4).OverwriteExistingFiles(true);
x.BitDepthImage(16).ImagesPerFile(2).SetFileWriterFormat(FileWriterFormat::NXmxIntegrated).PixelSigned(true)
.IndexingAlgorithm(IndexingAlgorithmEnum::FFT);
x.Compression(CompressionAlgorithm::BSHUF_ZSTD);
std::vector<int16_t> image(x.GetPixelsNum());
for (int i = 0; i < image.size(); i++)
image[i] = static_cast<int16_t>((i * 7 + 33) % UINT16_MAX);
RegisterHDF5Filter();
JFJochBitShuffleCompressor compressor(CompressionAlgorithm::BSHUF_ZSTD);
auto compressed_image = compressor.Compress(image);
{
StartMessage start_message;
x.FillMessage(start_message);
FileWriter file_set(start_message);
for (int i = 0; i < x.GetImageNum(); i++) {
DataMessage message{};
message.image = CompressedImage(compressed_image, x.GetXPixelsNum(), x.GetYPixelsNum(),
CompressedImageMode::Int16, CompressionAlgorithm::BSHUF_ZSTD);
message.number = i;
REQUIRE_NOTHROW(file_set.WriteHDF5(message));
}
EndMessage end_message;
end_message.max_image_number = x.GetImageNum();
file_set.WriteHDF5(end_message);
file_set.Finalize();
}
{
JFJochHDF5Reader reader;
REQUIRE_NOTHROW(reader.ReadFile("test_read_raw_image_master.h5"));
auto dataset = reader.GetDataset();
CHECK(dataset->experiment.GetImageNum() == 4);
std::shared_ptr<JFJochReaderRawImage> reader_image;
for (int i = 0; i < 4; i++) {
REQUIRE_NOTHROW(reader_image = reader.GetRawImage(i));
CHECK(reader_image->image.GetMode() == CompressedImageMode::Int16);
CHECK(reader_image->image.GetCompressionAlgorithm() == CompressionAlgorithm::BSHUF_ZSTD);
CHECK(reader_image->image.GetWidth() == x.GetXPixelsNum());
CHECK(reader_image->image.GetHeight() == x.GetYPixelsNum());
CHECK(reader_image->image.GetCompressedSize() == compressed_image.size());
CHECK(reader_image->image.GetCompressed() == reader_image->image_buffer.data());
REQUIRE(reader_image->image_buffer.size() == compressed_image.size());
CHECK(memcmp(reader_image->image_buffer.data(), compressed_image.data(), compressed_image.size()) == 0);
}
}
remove("test_read_raw_image_master.h5");
// No leftover HDF5 objects
REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0);
}