v1.0.0-rc.145 (#55)

This is an UNSTABLE release. The release has significant modifications for HDF5 writing logic - in case of troubles go back to 1.0.0-rc.144.

* **Default HDF5 writing mode is with VDS, not soft-links** - this improves DIALS compatibility and makes format more future-proof, NXmx legacy format might be phased-out in the future.
* XDS plugin: Improve performance of VDS reading.
* jfjoch_writer: Significant improvement on how file systems I/O are handled through a dedicated pass-through VFD.
* jfjoch_writer: Clean-up of HDF5 routines to better handle issues.

Reviewed-on: #55

reader/JFJochHDF5Reader.cpp

@@ -62,6 +62,37 @@ std::vector<hsize_t> GetDimension(HDF5Object& object, const std::string& path) {
     return dim;
 }
 
+std::vector<HDF5VirtualDatasetMapping> ReadVDSImageMappings(HDF5Object &file,
+                                                            const std::string &dataset_name) {
+    HDF5DataSet dataset(file, dataset_name);
+    HDF5Dcpl dcpl(dataset);
+    auto mappings = dcpl.GetVirtualMappings();
+
+    if (mappings.empty())
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              dataset_name + " is not a virtual dataset");
+
+    for (const auto &mapping: mappings) {
+        if (mapping.dataset.empty())
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "VDS mapping has empty source dataset name");
+        if (mapping.virtual_start.size() != 3)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Only 3D image VDS mappings are supported");
+    }
+
+    return mappings;
+}
+
+std::string ResolveRelativeToMaster(const std::string &directory,
+                                    const std::string &filename) {
+    std::filesystem::path path(filename);
+    if (path.is_absolute() || directory.empty())
+        return filename;
+
+    return (std::filesystem::path(directory) / path).string();
+}
+
 template <class T>
 void JFJochHDF5Reader::ReadVector(std::vector<T> &v,
                                   HDF5Object &file,
@@ -104,6 +135,10 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) {
         master_file = std::make_shared<HDF5ReadOnlyFile>(filename);
         dataset->experiment = default_experiment;
 
+        std::filesystem::path master_path(filename);
+        master_file_directory = master_path.parent_path().string();
+        vds_data_mappings.clear();
+
         dataset->arm_date = master_file->GetString("/entry/start_time");
 
         dataset->experiment.FilePrefix(dataset_name(filename));
@@ -115,19 +150,19 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) {
         size_t image_size_x = 0;
         size_t image_size_y = 0;
 
-        if (master_file->Exists("/entry/data/data"))
-            format = FileWriterFormat::NXmxVDS;
-        else if (master_file->Exists("/entry/data/data_000001")) {
-            format = FileWriterFormat::NXmxLegacy;
-        }
+        if (master_file->Exists("/entry/data/data")) {
+            HDF5DataSet data_dataset(*master_file, "/entry/data/data");
+            HDF5Dcpl dcpl(data_dataset);
+            data_layout = dcpl.GetLayout();
 
-        if (format == FileWriterFormat::NXmxVDS ) {
             auto dim = GetDimension(*master_file, "/entry/data/data");
             number_of_images = dim[0];
             image_size_y = dim[1];
             image_size_x = dim[2];
 
             images_per_file = number_of_images;
+            if (data_layout == HDF5DataSetLayout::VIRTUAL)
+                vds_data_mappings = ReadVDSImageMappings(*master_file, "/entry/data/data");
 
             if (master_file->Exists("/entry/instrument/detector/detectorSpecific/data_collection_efficiency_image"))
             dataset->efficiency = master_file->ReadVector<float>(
@@ -166,7 +201,12 @@ void JFJochHDF5Reader::ReadFile(const std::string& filename) {
             }
             if (master_file->Exists("/entry/image"))
                 dataset->max_value = master_file->ReadOptVector<int64_t>("/entry/image/max_value");
-        } else if (format == FileWriterFormat::NXmxLegacy) {
+
+            format = FileWriterFormat::NXmxVDS;
+        } else if (master_file->Exists("/entry/data/data_000001")) {
+            format = FileWriterFormat::NXmxLegacy;
+            data_layout = HDF5DataSetLayout::CONTIGUOUS;
+
             legacy_format_files.clear();
 
             image_size_x = master_file->GetInt("/entry/instrument/detector/detectorSpecific/x_pixels_in_detector");
@@ -509,6 +549,22 @@ std::pair<std::shared_ptr<HDF5ReadOnlyFile>, uint32_t> JFJochHDF5Reader::GetImag
         uint32_t file_id = image_number / images_per_file;
         image_id = image_number % images_per_file;
         data_file = std::make_shared<HDF5ReadOnlyFile>(legacy_format_files.at(file_id));
+    } else if (format == FileWriterFormat::NXmxVDS && data_layout == HDF5DataSetLayout::VIRTUAL) {
+        const auto image = static_cast<hsize_t>(image_number);
+
+        for (const auto &mapping: vds_data_mappings) {
+            if (!mapping.ContainsVirtualImage(image))
+                continue;
+
+            image_id = static_cast<uint32_t>(mapping.SourceImage(image));
+            data_file = std::make_shared<HDF5ReadOnlyFile>(
+                    ResolveRelativeToMaster(master_file_directory, mapping.filename)
+            );
+            return {std::move(data_file), image_id};
+        }
+
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Image not covered by /entry/data/data VDS mappings");
     } else {
         image_id = image_number;
         data_file = master_file;
@@ -774,6 +830,8 @@ void JFJochHDF5Reader::Close() {
     master_file = {};
     number_of_images = 0;
     legacy_format_files.clear();
+    vds_data_mappings.clear();
+    master_file_directory.clear();
     SetStartMessage({});
 }