diff --git a/.gitea/workflows/build_and_test.yml b/.gitea/workflows/build_and_test.yml index dc12ee49..b9036ba0 100644 --- a/.gitea/workflows/build_and_test.yml +++ b/.gitea/workflows/build_and_test.yml @@ -170,13 +170,6 @@ jobs: cd build cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. ninja -j16 jfjoch_hdf5_test - - name: Run XDS with Durin and single HDF5 format - shell: bash - run: | - cd tests/xds_durin - ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S - /opt/xds/xds_par - test -f IDXREF.LP - name: Run XDS with Durin and legacy HDF5 format shell: bash run: | @@ -184,6 +177,41 @@ jobs: ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 /opt/xds/xds_par test -f IDXREF.LP + xds-test: + name: XDS test (JFJoch plugin) + runs-on: jfjoch_rocky9 + steps: + - uses: actions/checkout@v4 + - name: Build processing test + shell: bash + run: | + mkdir -p build + cd build + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release .. + ninja -j16 jfjoch_hdf5_test jfjoch_xds_plugin + - name: Run XDS with legacy HDF5 format + shell: bash + run: | + cd tests/xds + ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 + /opt/xds/xds_par + test -f IDXREF.LP + - name: Run XDS with VDS HDF5 format + shell: bash + run: | + cd tests/xds + rm *.h5 + ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -f10 -V + /opt/xds/xds_par + test -f IDXREF.LP + - name: Run XDS with single HDF5 format + shell: bash + run: | + cd tests/xds + rm *.h5 + ../../build/tools/jfjoch_hdf5_test ../../tests/test_data/compression_benchmark.h5 -n25 -S + /opt/xds/xds_par + test -f IDXREF.LP xds-neggia-test: name: XDS test (neggia plugin) runs-on: jfjoch_rocky9 diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bfa64dd..e2fbf219 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,6 +155,7 @@ ADD_SUBDIRECTORY(detector_control) ADD_SUBDIRECTORY(image_puller) ADD_SUBDIRECTORY(preview) ADD_SUBDIRECTORY(symmetry) +ADD_SUBDIRECTORY(xds-plugin) IF (JFJOCH_WRITER_ONLY) MESSAGE(STATUS "Compiling HDF5 writer only") diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 257422b7..a69422da 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -26,11 +26,12 @@ MESSAGE(STATUS "Jungfraujoch version: ${PACKAGE_VERSION}") CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/GitInfo.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp" @ONLY) +ADD_LIBRARY(JFJochVersion STATIC + ${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp GitInfo.h) ADD_LIBRARY(JFJochLogger STATIC Logger.cpp Logger.h - ${CMAKE_CURRENT_BINARY_DIR}/GitInfo.cpp GitInfo.h ) -TARGET_LINK_LIBRARIES(JFJochLogger PUBLIC spdlog::spdlog) +TARGET_LINK_LIBRARIES(JFJochLogger PUBLIC spdlog::spdlog JFJochVersion) ADD_LIBRARY(JFJochZMQ STATIC ZMQWrappers.cpp ZMQWrappers.h) diff --git a/compression/CMakeLists.txt b/compression/CMakeLists.txt index 144a61fc..aacc889e 100644 --- a/compression/CMakeLists.txt +++ b/compression/CMakeLists.txt @@ -13,8 +13,7 @@ ADD_LIBRARY(Compression STATIC JFJochDecompress.h MaxCompressedSize.cpp MaxCompressedSize.h) - - +set_target_properties(Compression PROPERTIES POSITION_INDEPENDENT_CODE ON) TARGET_COMPILE_DEFINITIONS(Compression PUBLIC -DZSTD_SUPPORT -DUSE_ZSTD) TARGET_LINK_LIBRARIES(Compression libzstd_static) TARGET_INCLUDE_DIRECTORIES(Compression PUBLIC . zstd/lib) diff --git a/reader/CMakeLists.txt b/reader/CMakeLists.txt index 430fc722..15d65b73 100644 --- a/reader/CMakeLists.txt +++ b/reader/CMakeLists.txt @@ -10,5 +10,6 @@ ADD_LIBRARY(JFJochReader STATIC JFJochHttpReader.h ) -TARGET_LINK_LIBRARIES(JFJochReader JFJochImageAnalysis JFJochAPI JFJochCommon JFJochZMQ JFJochLogger JFJochHDF5Wrappers CBORStream2FrameSerialize +TARGET_LINK_LIBRARIES(JFJochReader JFJochImageAnalysis JFJochAPI JFJochCommon JFJochZMQ JFJochLogger + JFJochHDF5Wrappers CBORStream2FrameSerialize httplib::httplib ${CMAKE_DL_LIBS}) diff --git a/reader/JFJochHDF5Reader.cpp b/reader/JFJochHDF5Reader.cpp index b3979215..ebf1c542 100644 --- a/reader/JFJochHDF5Reader.cpp +++ b/reader/JFJochHDF5Reader.cpp @@ -119,15 +119,11 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) { if (master_file->Exists("/entry/data/data")) format = FileWriterFormat::NXmxVDS; else if (master_file->Exists("/entry/data/data_000001")) { - if (master_file->IsExternalLink("/entry/data/data_000001")) - format = FileWriterFormat::NXmxLegacy; - else - format = FileWriterFormat::NXmxIntegrated; + format = FileWriterFormat::NXmxLegacy; } - if (format == FileWriterFormat::NXmxVDS || format == FileWriterFormat::NXmxIntegrated) { - auto dim = GetDimension(*master_file, - (format == FileWriterFormat::NXmxIntegrated) ? "/entry/data/data_000001" : "/entry/data/data"); + if (format == FileWriterFormat::NXmxVDS ) { + auto dim = GetDimension(*master_file, "/entry/data/data"); number_of_images = dim[0]; image_size_y = dim[1]; image_size_x = dim[2]; @@ -465,7 +461,7 @@ uint64_t JFJochHDF5Reader::GetNumberOfImages() const { CompressedImage JFJochHDF5Reader::LoadImageDataset(std::vector &tmp, HDF5Object &file, hsize_t number) { std::vector start = {static_cast(number), 0, 0}; - const std::string dataset_name = (format == FileWriterFormat::NXmxIntegrated) ? "/entry/data/data_000001" : "/entry/data/data"; + const std::string dataset_name = "/entry/data/data"; HDF5DataSet dataset(file, dataset_name); HDF5DataSpace dataspace(dataset); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dd755f7a..d5083ddb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -66,7 +66,9 @@ ADD_EXECUTABLE(jfjoch_test TCPImagePusherTest.cpp BraggIntegrate2DTest.cpp SearchSpaceGroupTest.cpp + XDSPluginTest.cpp ) -target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview) +target_link_libraries(jfjoch_test Catch2WithMain JFJochBroker JFJochReceiver JFJochReader JFJochWriter JFJochImageAnalysis JFJochCommon JFJochHLSSimulation JFJochPreview + jfjoch_xds_plugin) target_include_directories(jfjoch_test PRIVATE .) diff --git a/tests/XDSPluginTest.cpp b/tests/XDSPluginTest.cpp new file mode 100644 index 00000000..926893bf --- /dev/null +++ b/tests/XDSPluginTest.cpp @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include "../common/DiffractionExperiment.h" +#include "../writer/FileWriter.h" +#include "../xds-plugin/plugin.h" + +namespace { + void CleanupFiles(const std::string& prefix, int file_count) { + remove((prefix + "_master.h5").c_str()); + for (int i = 1; i <= file_count; ++i) { + char data_file[256]; + snprintf(data_file, sizeof(data_file), "%s_data_%06d.h5", prefix.c_str(), i); + remove(data_file); + } + } + + void ClosePlugin() { + int error_flag = 0; + plugin_close(&error_flag); + REQUIRE(error_flag == 0); + } +} + +TEST_CASE("XDSPlugin_GetData_VDS", "[HDF5][XDS][Plugin]") { + DiffractionExperiment x(DetJF(1)); + + x.FilePrefix("test_xds_vds").ImagesPerTrigger(3).ImagesPerFile(1).OverwriteExistingFiles(true); + x.SetFileWriterFormat(FileWriterFormat::NXmxVDS); + x.BitDepthImage(16).PixelSigned(false); + x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) + .IncidentEnergy_keV(WVL_1A_IN_KEV) + .FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10)); + x.Compression(CompressionAlgorithm::NO_COMPRESSION); + + RegisterHDF5Filter(); + + std::vector pixel_mask(x.GetPixelsNum(), 0); + std::vector image(x.GetPixelsNum(), 7); + image[0] = UINT16_MAX; + image[1] = 123; + + { + StartMessage start_message; + x.FillMessage(start_message); + start_message.pixel_mask["default"] = pixel_mask; + + FileWriter file_set(start_message); + + for (int i = 0; i < 3; ++i) { + DataMessage message{}; + image[5678] = static_cast(200 + i); + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = i; + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + } + + EndMessage end_message; + end_message.max_image_number = 3; + file_set.WriteHDF5(end_message); + file_set.Finalize(); + } + + { + int info[1024] = {}; + int error_flag = 0; + + REQUIRE_NOTHROW(plugin_open("test_xds_vds_master.h5", info, &error_flag)); + REQUIRE(error_flag == 0); + + int nx = 0, ny = 0, nbytes = 0, number_of_frames = 0; + float qx = 0.0f, qy = 0.0f; + + REQUIRE_NOTHROW(plugin_get_header(&nx, &ny, &nbytes, &qx, &qy, &number_of_frames, info, &error_flag)); + REQUIRE(error_flag == 0); + + CHECK(nx == static_cast(x.GetXPixelsNum())); + CHECK(ny == static_cast(x.GetYPixelsNum())); + CHECK(nbytes == 2); + CHECK(number_of_frames == 3); + + std::vector data(static_cast(nx) * static_cast(ny), 0); + + int frame_number = 2; + REQUIRE_NOTHROW(plugin_get_data(&frame_number, &nx, &ny, data.data(), info, &error_flag)); + REQUIRE(error_flag == 0); + + CHECK(data[0] == -1); + CHECK(data[1] == 123); + CHECK(data[5678] == 201); + + ClosePlugin(); + } + + CleanupFiles("test_xds_vds", 3); + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} + +TEST_CASE("XDSPlugin_GetData_Legacy", "[HDF5][XDS][Plugin]") { + DiffractionExperiment x(DetJF(1)); + + x.FilePrefix("test_xds_legacy").ImagesPerTrigger(2).ImagesPerFile(1).OverwriteExistingFiles(true); + x.SetFileWriterFormat(FileWriterFormat::NXmxLegacy); + x.BitDepthImage(16).PixelSigned(true); + x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) + .IncidentEnergy_keV(WVL_1A_IN_KEV) + .FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10)); + x.Compression(CompressionAlgorithm::NO_COMPRESSION); + + RegisterHDF5Filter(); + + std::vector pixel_mask(x.GetPixelsNum(), 0); + std::vector image(x.GetPixelsNum(), 0); + image[0] = INT16_MAX; + image[1] = INT16_MIN; + image[2] = 456; + + { + StartMessage start_message; + x.FillMessage(start_message); + start_message.pixel_mask["default"] = pixel_mask; + + FileWriter file_set(start_message); + + for (int i = 0; i < 2; ++i) { + DataMessage message{}; + image[5678] = static_cast(10 + i); + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = i; + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + } + + EndMessage end_message; + end_message.max_image_number = 2; + file_set.WriteHDF5(end_message); + file_set.Finalize(); + } + + { + int info[1024] = {}; + int error_flag = 0; + + REQUIRE_NOTHROW(plugin_open("test_xds_legacy_master.h5", info, &error_flag)); + REQUIRE(error_flag == 0); + + int nx = 0, ny = 0, nbytes = 0, number_of_frames = 0; + float qx = 0.0f, qy = 0.0f; + + REQUIRE_NOTHROW(plugin_get_header(&nx, &ny, &nbytes, &qx, &qy, &number_of_frames, info, &error_flag)); + REQUIRE(error_flag == 0); + REQUIRE(number_of_frames == 2); + + std::vector data(static_cast(nx) * static_cast(ny), 0); + + int frame_number = 1; + REQUIRE_NOTHROW(plugin_get_data(&frame_number, &nx, &ny, data.data(), info, &error_flag)); + REQUIRE(error_flag == 0); + + CHECK(data[0] == -1); + CHECK(data[1] == -1); + CHECK(data[2] == 456); + CHECK(data[5678] == 10); + + frame_number = 2; + REQUIRE_NOTHROW(plugin_get_data(&frame_number, &nx, &ny, data.data(), info, &error_flag)); + REQUIRE(error_flag == 0); + CHECK(data[5678] == 11); + + ClosePlugin(); + } + + CleanupFiles("test_xds_legacy", 2); + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} + +TEST_CASE("XDSPlugin_GetData_Integrated", "[HDF5][XDS][Plugin]") { + DiffractionExperiment x(DetJF(1)); + + x.FilePrefix("test_xds_integrated").ImagesPerTrigger(3).OverwriteExistingFiles(true); + x.SetFileWriterFormat(FileWriterFormat::NXmxIntegrated); + x.BitDepthImage(16).PixelSigned(false); + x.BeamX_pxl(100).BeamY_pxl(200).DetectorDistance_mm(150) + .IncidentEnergy_keV(WVL_1A_IN_KEV) + .FrameTime(std::chrono::microseconds(500), std::chrono::microseconds(10)); + x.Compression(CompressionAlgorithm::NO_COMPRESSION); + + RegisterHDF5Filter(); + + AzimuthalIntegrationSettings azint_settings; + azint_settings.AzimuthalBinCount(4); + x.ImportAzimuthalIntegrationSettings(azint_settings); + + std::vector image(x.GetPixelsNum(), 0); + image[0] = UINT16_MAX; + image[1] = 321; + image[5678] = 777; + + { + StartMessage start_message; + x.FillMessage(start_message); + start_message.pixel_mask["default"] = std::vector(x.GetPixelsNum(), 0); + + FileWriter file_set(start_message); + + for (int i = 0; i < 3; ++i) { + DataMessage message{}; + image[5678] = static_cast(777 + i); + message.image = CompressedImage(image, x.GetXPixelsNum(), x.GetYPixelsNum()); + message.number = i; + message.az_int_profile = std::vector(azint_settings.GetBinCount(), static_cast(10 + i)); + REQUIRE_NOTHROW(file_set.WriteHDF5(message)); + } + + EndMessage end_message; + end_message.max_image_number = 3; + file_set.WriteHDF5(end_message); + file_set.Finalize(); + } + + { + int info[1024] = {}; + int error_flag = 0; + + REQUIRE_NOTHROW(plugin_open("test_xds_integrated_master.h5", info, &error_flag)); + REQUIRE(error_flag == 0); + + int nx = 0, ny = 0, nbytes = 0, number_of_frames = 0; + float qx = 0.0f, qy = 0.0f; + + REQUIRE_NOTHROW(plugin_get_header(&nx, &ny, &nbytes, &qx, &qy, &number_of_frames, info, &error_flag)); + REQUIRE(error_flag == 0); + + CHECK(nx == static_cast(x.GetXPixelsNum())); + CHECK(ny == static_cast(x.GetYPixelsNum())); + CHECK(qx == Catch::Approx(0.075)); + CHECK(qy == Catch::Approx(0.075)); + CHECK(nbytes == 2); + CHECK(number_of_frames == 3); + + std::vector data(static_cast(nx) * static_cast(ny), 0); + + int frame_number = 3; + REQUIRE_NOTHROW(plugin_get_data(&frame_number, &nx, &ny, data.data(), info, &error_flag)); + REQUIRE(error_flag == 0); + + CHECK(data[0] == -1); + CHECK(data[1] == 321); + CHECK(data[5678] == 779); + + ClosePlugin(); + } + + remove("test_xds_integrated_master.h5"); + REQUIRE(H5Fget_obj_count(H5F_OBJ_ALL, H5F_OBJ_ALL) == 0); +} \ No newline at end of file diff --git a/tests/xds/XDS.INP b/tests/xds/XDS.INP new file mode 100644 index 00000000..b35f1150 --- /dev/null +++ b/tests/xds/XDS.INP @@ -0,0 +1,73 @@ + +!Settings for EIGER16M @ X06SA SLS +!job control +JOB=XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT + +MAXIMUM_NUMBER_OF_JOBS=1 +MAXIMUM_NUMBER_OF_PROCESSORS=1 + +! for this experiment: +ORGX= 1097 +ORGY= 1130 +DETECTOR_DISTANCE= 75 +OSCILLATION_RANGE= 0.088 +X-RAY_WAVELENGTH= 1.0 +NAME_TEMPLATE_OF_DATA_FRAMES=writing_test_??????.h5 +DATA_RANGE= 1 25 +!DATA_RANGE_FIXED_SCALE_FACTOR= 1900 6000 1.0 +SPOT_RANGE= 1 25 + +!BACKGROUND_RANGE= 1 11 +!REFERENCE_DATA_SET= + +SPACE_GROUP_NUMBER=96 +UNIT_CELL_CONSTANTS= 78.95 78.95 36.91 90.000 90.000 90.000 +!REIDX= 0 0 -1 0 0 -1 0 0 -1 0 0 0 + +REFINE(IDXREF)=BEAM AXIS ORIENTATION POSITION !CELL +REFINE(INTEGRATE)=POSITION BEAM ORIENTATION CELL AXIS +REFINE(CORRECT)=POSITION BEAM ORIENTATION CELL AXIS + +FRIEDEL'S_LAW=FALSE +STRICT_ABSORPTION_CORRECTION=FALSE ! but read Tips_and_Tricks in XDSwiki + +! parameters with changes wrt default values: +TRUSTED_REGION=0.00 1.21 +VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS=4000. 30000. +!MINIMUM_ZETA=0.05 +CORRECTIONS=DECAY MODULATION ABSORP !default value +INCLUDE_RESOLUTION_RANGE=50 0.0 + +! parameters specifically for this detector and beamline: +DETECTOR=EIGER NX=2068 NY=2164 QX=0.075 QY=0.075 !EIGER 16M +! Defines path to the Dectris neggia library (to use H5ToXds comment this line out) +LIB=../../build/xds-plugin/libjfjoch-xds-plugin.so + +MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=30000 +DIRECTION_OF_DETECTOR_X-AXIS=1 0 0 +DIRECTION_OF_DETECTOR_Y-AXIS=0 1 0 +INCIDENT_BEAM_DIRECTION=0 0 1 +ROTATION_AXIS=1 0 0 +FRACTION_OF_POLARIZATION=0.99 +POLARIZATION_PLANE_NORMAL=0 1 0 +SENSOR_THICKNESS=0.32 +!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE EIGER 16M DETECTOR + ONE PIXEL ON EACH SIDE +! UNTRUSTED_RECTANGLE= 0 4151 513 553 +! UNTRUSTED_RECTANGLE= 0 4151 1064 1104 +! UNTRUSTED_RECTANGLE= 0 4151 1615 1655 +! UNTRUSTED_RECTANGLE= 0 4151 2166 2206 +! UNTRUSTED_RECTANGLE= 0 4151 2717 2757 +! UNTRUSTED_RECTANGLE= 0 4151 3268 3308 +! UNTRUSTED_RECTANGLE= 0 4151 3819 3859 +!EXCLUSION OF VERTICAL DEAD AREAS OF THE EIGER 16M DETECTOR + ONE PIXEL ON EACH SIDE +! UNTRUSTED_RECTANGLE= 1029 1042 0 4372 +! UNTRUSTED_RECTANGLE= 2069 2082 0 4372 +! UNTRUSTED_RECTANGLE= 3109 3122 0 4372 + +UNTRUSTED_RECTANGLE=1036 2017 1082 1637 +NUMBER_OF_PROFILE_GRID_POINTS_ALONG_ALPHA/BETA=21 !used by: INTEGRATE +!MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT= +MINIMUM_FRACTION_OF_INDEXED_SPOTS=0.2 +SEPMIN=4 CLUSTER_RADIUS=2 +SIGNAL_PIXEL=6 + diff --git a/writer/CMakeLists.txt b/writer/CMakeLists.txt index 8b060291..9237fbfe 100644 --- a/writer/CMakeLists.txt +++ b/writer/CMakeLists.txt @@ -1,4 +1,6 @@ ADD_LIBRARY(JFJochHDF5Wrappers STATIC HDF5Objects.cpp HDF5Objects.h ../compression/bitshuffle/bshuf_h5filter.c) +set_target_properties(JFJochHDF5Wrappers PROPERTIES POSITION_INDEPENDENT_CODE ON) + TARGET_LINK_LIBRARIES(JFJochHDF5Wrappers Compression hdf5-static) ADD_LIBRARY(JFJochWriter STATIC diff --git a/writer/HDF5DataFile.cpp b/writer/HDF5DataFile.cpp index 7e6a66d2..85a727b4 100644 --- a/writer/HDF5DataFile.cpp +++ b/writer/HDF5DataFile.cpp @@ -135,10 +135,8 @@ void HDF5DataFile::CreateFile(const DataMessage& msg, std::shared_ptr HDF5Group(*data_file, "/entry").NXClass("NXentry"); HDF5Group(*data_file, "/entry/data").NXClass("NXdata"); - const std::string dataset_name = integrated ? "/entry/data/data_000001" : "/entry/data/data"; - HDF5DataSpace data_space({1, ypixel, xpixel}, {H5S_UNLIMITED, ypixel, xpixel}); - data_set = std::make_unique(*data_file, dataset_name, data_type, data_space, dcpl); + data_set = std::make_unique(*data_file, "/entry/data/data", data_type, data_space, dcpl); data_set->SetExtent({images_per_file, ypixel, xpixel}); for (auto &p: plugins) p->OpenFile(*data_file, msg, images_per_file); diff --git a/xds-plugin/CMakeLists.txt b/xds-plugin/CMakeLists.txt new file mode 100644 index 00000000..b9977310 --- /dev/null +++ b/xds-plugin/CMakeLists.txt @@ -0,0 +1,4 @@ +ADD_LIBRARY(jfjoch_xds_plugin SHARED plugin.cpp plugin.h) + +TARGET_LINK_LIBRARIES(jfjoch_xds_plugin JFJochHDF5Wrappers JFJochVersion Compression hdf5-static) +INSTALL(TARGETS jfjoch_xds_plugin DESTINATION ${CMAKE_INSTALL_LIBDIR}/jfjoch COMPONENT viewer) diff --git a/xds-plugin/plugin.cpp b/xds-plugin/plugin.cpp new file mode 100644 index 00000000..f066374d --- /dev/null +++ b/xds-plugin/plugin.cpp @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Based on Durin plugin code from Diamond Light Source Ltd. and Global Phasing (BSD-3 license) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../writer/HDF5Objects.h" + +#include "plugin.h" + +#include "JFJochDecompress.h" +#include "../common/GitInfo.h" +#include "../common/JFJochMessages.h" + +namespace { + class PluginError : public std::runtime_error { + int err_code; + public: + PluginError(int code, const std::string &message) : std::runtime_error(message), err_code(code) {} + }; + + std::shared_mutex plugin_mutex; + std::unique_ptr hdf5_file; + FileWriterFormat format = FileWriterFormat::NoFile; + uint64_t images_per_file = 0; + size_t image_size_x, image_size_y; + float pixel_size_x, pixel_size_y; + size_t total_image_number; + uint32_t pixel_byte_depth = 0; + bool pixel_signed = false; + + std::vector one_byte_mask; + + template + void ConvertAndMaskTyped(const std::vector &in_8bit, + int marker_value, + int *out) { + auto in = reinterpret_cast(in_8bit.data()); + size_t size = in_8bit.size() / sizeof(T); + + for (size_t i = 0; i < size; ++i) { + if (!one_byte_mask.empty() && (one_byte_mask.at(i) == 1)) + out[i] = -1; + else if (!one_byte_mask.empty() && (one_byte_mask.at(i) == 2)) + out[i] = -2; + else if (marker_value != 0 && in[i] == static_cast(marker_value)) + out[i] = -1; + else if (std::is_signed_v && in[i] < 0) + out[i] = -1; + else if (in[i] > INT32_MAX) + out[i] = INT32_MAX; + else + out[i] = static_cast(in[i]); + } + } + + void LoadDataset() { + std::string dataset_name; + if (hdf5_file->Exists("/entry/data/data")) { + format = FileWriterFormat::NXmxVDS; + dataset_name = "/entry/data/data"; + } else if (hdf5_file->Exists("/entry/data/data_000001")) { + format = FileWriterFormat::NXmxLegacy; + dataset_name = "/entry/data/data_000001"; + } else { + throw PluginError(-1, "Could not locate detector dataset"); + } + + HDF5DataSet data(*hdf5_file, dataset_name); + HDF5DataSpace dataspace(data); + HDF5DataType datatype(data); + + auto dim = dataspace.GetDimensions(); + if (dim.size() != 3) + throw PluginError(-1, "Wrong dimension of /entry/data/data"); + image_size_x = dim[2]; + image_size_y = dim[1]; + images_per_file = dim[0]; + total_image_number = dim[0]; + + if (format == FileWriterFormat::NXmxLegacy) { + int dataset = 2; + while (dataset < 100000) { + char name[255]; + snprintf(name, sizeof(name), "/entry/data/data_%06d", dataset); + + if (!hdf5_file->Exists(name)) + break; + auto leg_dims = hdf5_file->GetDimension(name); + if (leg_dims.size() != 3) + throw PluginError(-1, "Wrong dimension of " + std::string(name)); + if (leg_dims[2] != image_size_x) + throw PluginError(-1, "Image size of " + std::string(name) + " does not match"); + if (leg_dims[1] != image_size_y) + throw PluginError(-1, "Image size of " + std::string(name) + " does not match"); + total_image_number += leg_dims[0]; + dataset++; + } + } + + if (!datatype.IsInteger()) + throw PluginError(-1, "Data type of /entry/data/data is not integer"); + pixel_signed = datatype.IsSigned(); + pixel_byte_depth = datatype.GetElemSize(); + + pixel_size_x = hdf5_file->GetFloat("/entry/instrument/detector/x_pixel_size"); + pixel_size_y = hdf5_file->GetFloat("/entry/instrument/detector/y_pixel_size"); + + one_byte_mask = std::vector(image_size_x * image_size_y, 0); + + auto mask_tmp = hdf5_file->ReadVector( + "/entry/instrument/detector/pixel_mask", + {0, 0}, + {image_size_y, image_size_x} + ); + + for (int i = 0; i < mask_tmp.size(); i++) { + if (mask_tmp[i] & (1 << 30)) + one_byte_mask[i] = 2; + else if (mask_tmp[i] & 0xFFFF) + one_byte_mask[i] = 1; + } + } + + void ConvertToIntAndMask(const std::vector &in_8bit, int *out_buffer) { + switch (pixel_byte_depth) { + case 1: + if (pixel_signed) + return ConvertAndMaskTyped(in_8bit, INT8_MAX, out_buffer); + else + return ConvertAndMaskTyped(in_8bit, UINT8_MAX, out_buffer); + case 2: + if (pixel_signed) + return ConvertAndMaskTyped(in_8bit, INT16_MAX, out_buffer); + else + return ConvertAndMaskTyped(in_8bit, UINT16_MAX, out_buffer); + case 4: + if (pixel_signed) + return ConvertAndMaskTyped(in_8bit, INT32_MAX, out_buffer); + else + return ConvertAndMaskTyped(in_8bit, UINT32_MAX, out_buffer); + default: + throw PluginError(-1, "Unsupported conversion to int"); + } + } + + void FillInfoArray(int info[1024]) { + info[0] = 0x01; + info[1] = VERSION_MAJOR; + info[2] = VERSION_MINOR; + info[3] = VERSION_PATCH; + info[4] = VERSION_TIMESTAMP; + info[5] = 0; + info[6] = -1; + } +} // namespace + +extern "C" { +void plugin_open(const char *filename, int info[1024], int *error_flag) { + std::unique_lock sl(plugin_mutex); + + std::cout << "Jungfraujoch XDS plugin" << std::endl; + std::cout << "Version " << jfjoch_version() << std::endl; + try { + FillInfoArray(info); + + RegisterHDF5Filter(); + hdf5_file = std::make_unique(filename); + LoadDataset(); + *error_flag = 0; + } catch (std::exception &e) { + std::cerr << e.what() << std::endl; + *error_flag = -1; + hdf5_file.reset(); + total_image_number = 0; + } +} + +void plugin_get_header(int *nx, int *ny, int *nbytes, float *qx, float *qy, + int *number_of_frames, int info[1024], int *error_flag) { + std::shared_lock sl(plugin_mutex); + + try { + FillInfoArray(info); + + if (!hdf5_file) + throw PluginError(-1, "HDF5 file not open"); + + *nx = image_size_x; + *ny = image_size_y; + *nbytes = pixel_byte_depth; + *number_of_frames = total_image_number; + *qx = pixel_size_x * 1e3; // mm + *qy = pixel_size_y * 1e3; // mm + *error_flag = 0; + } catch (std::exception &e) { + std::cerr << e.what() << std::endl; + *error_flag = -1; + } +} + +void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, + int info[1024], int *error_flag) { + std::shared_lock sl(plugin_mutex); + + try { + if (!hdf5_file) + throw PluginError(-1, "HDF5 file not open"); + + FillInfoArray(info); + + if (*frame_number <= 0 || *frame_number > total_image_number) + throw PluginError(-1, "Frame number out of range"); + + std::string dataset_name; + hsize_t image_id; + + if (format == FileWriterFormat::NXmxLegacy) { + char str[256]; + size_t dataset_index = (*frame_number - 1) / images_per_file + 1; + snprintf(str, sizeof(str), "/entry/data/data_%06ld", dataset_index); + dataset_name = std::string(str); + image_id = (*frame_number - 1) % images_per_file; + } else { + dataset_name = "/entry/data/data"; + image_id = *frame_number - 1; + } + + HDF5DataSet dataset(*hdf5_file, dataset_name); + HDF5Dcpl dcpl(dataset); + + std::vector tmp; + std::vector start = {image_id, 0, 0}; + + CompressionAlgorithm algorithm = CompressionAlgorithm::NO_COMPRESSION; + auto chunk_size = dcpl.GetChunking(); + + if ((chunk_size.size() == 3) + && (chunk_size[0] == 1) + && (chunk_size[1] == image_size_y) + && (chunk_size[2] == image_size_x)) { + dataset.ReadDirectChunk(tmp, start); + algorithm = dcpl.GetCompression(); + } else { + dataset.ReadVectorToU8(tmp, start, {1, image_size_y, image_size_x}); + algorithm = CompressionAlgorithm::NO_COMPRESSION; + } + + if (algorithm != CompressionAlgorithm::NO_COMPRESSION) { + std::vector decompressed_image(image_size_x * image_size_y * pixel_byte_depth); + JFJochDecompressPtr(decompressed_image.data(), + algorithm, + tmp.data(), + tmp.size(), + image_size_x * image_size_y, + pixel_byte_depth); + ConvertToIntAndMask(decompressed_image, data_array); + } else { + ConvertToIntAndMask(tmp, data_array); + } + *error_flag = 0; + } catch (std::exception &e) { + std::cerr << e.what() << std::endl; + *error_flag = -1; + } +} + +void plugin_close(int *error_flag) { + std::unique_lock sl(plugin_mutex); + try { + hdf5_file.reset(); + one_byte_mask.clear(); + total_image_number = 0; + format = FileWriterFormat::NoFile; + images_per_file = 0; + image_size_x = 0; + image_size_y = 0; + pixel_size_x = 0; + pixel_size_y = 0; + *error_flag = 0; + } catch (std::exception &e) { + std::cerr << e.what() << std::endl; + *error_flag = -1; + } +} +} /* extern "C" */ \ No newline at end of file diff --git a/xds-plugin/plugin.h b/xds-plugin/plugin.h new file mode 100644 index 00000000..938159a0 --- /dev/null +++ b/xds-plugin/plugin.h @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2025 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// Based on Durin plugin code from Diamond Light Source Ltd. and Global Phasing (BSD-3 license) + +/* + * External library interface for XDS. + * Ref: https://wiki.uni-konstanz.de/xds/index.php/LIB + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#define VERSION_MAJOR 0 +#define VERSION_MINOR 0 +#define VERSION_PATCH 0 +#define VERSION_TIMESTAMP -1 /* good enough for Dectris apparantely */ + +void plugin_open(const char *filename, int info[1024], int *error_flag); + +void plugin_get_header(int *nx, int *ny, int *nbytes, float *qx, float *qy, + int *number_of_frames, int info[1024], int *error_flag); + +void plugin_get_data(int *frame_number, int *nx, int *ny, int *data_array, + int info[1024], int *error_flag); + +void plugin_close(int *error_flag); + +#ifdef __cplusplus +} /* extern "C" */ +#endif