HDF5: Use explicit mapping for VDS

writer/HDF5Objects.cpp

@@ -231,23 +231,26 @@ bool HDF5DataType::IsFloat() const {
 HDF5Dcpl::HDF5Dcpl() : HDF5Id() {
     id = H5Pcreate(H5P_DATASET_CREATE);
     ndim = 0;
+    layout = HDF5DataSetLayout::CONTIGUOUS;
 }
 
 HDF5Dcpl::HDF5Dcpl(const HDF5DataSet &data_set) : HDF5Id() {
     id = H5Dget_create_plist(data_set.GetID());
 
     // Check if chunking is enabled
-    H5D_layout_t layout = H5Pget_layout(id);
-    if (layout != H5D_CHUNKED) {
-        ndim = 0;
-    } else {
+    H5D_layout_t h5_layout = H5Pget_layout(id);
+    if (h5_layout == H5D_VIRTUAL)
+        layout = HDF5DataSetLayout::VIRTUAL;
+    else if (h5_layout == H5D_CHUNKED) {
+        layout = HDF5DataSetLayout::CHUNKED;
         ndim = H5Pget_chunk(id, 0, nullptr);
         if (ndim <= 0) {
             H5Pclose(id);
             throw JFJochException(JFJochExceptionCategory::HDF5,
                                   "Error getting number of chunk dimensions");
         }
-    }
+    } else
+        layout = HDF5DataSetLayout::CONTIGUOUS;
 }
 
 HDF5Dcpl::~HDF5Dcpl() {
@@ -255,6 +258,7 @@ HDF5Dcpl::~HDF5Dcpl() {
 }
 
 void HDF5Dcpl::SetChunking(const std::vector<hsize_t> &dims) {
+    layout = HDF5DataSetLayout::CHUNKED;
     if ((dims.empty()) || dims[0] == 0)
         throw JFJochException(JFJochExceptionCategory::HDF5, "Value dimension cannot be 0");
     ndim = dims.size();
@@ -298,11 +302,16 @@ void HDF5Dcpl::SetFillValueU16(uint16_t val) {
 }
 
 void HDF5Dcpl::SetVirtual(const std::string &path, const std::string &dataset, const HDF5DataSpace &src_dataspace, const HDF5DataSpace &dest_dataspace) {
+    layout = HDF5DataSetLayout::VIRTUAL;
     std::string filename = ExtractFilename(path);
     if (H5Pset_virtual(id, dest_dataspace.GetID(), filename.c_str(), dataset.c_str(), src_dataspace.GetID()) < 0)
         throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot set virtual mapping");
 }
 
+HDF5DataSetLayout HDF5Dcpl::GetLayout() const {
+    return layout;
+}
+
 HDF5Fapl::HDF5Fapl() : HDF5Id() {
     id = H5Pcreate(H5P_FILE_ACCESS);
 }
@@ -995,3 +1004,208 @@ std::string HDF5Object::GetLinkedFileName(const std::string& name) const {
 
     return s;
 }
+
+
+namespace {
+    std::vector<hsize_t> GetDataspaceDimensions(hid_t dataspace_id) {
+        const int ndims = H5Sget_simple_extent_ndims(dataspace_id);
+        if (ndims < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot read dataspace dimensions");
+
+        std::vector<hsize_t> dims(ndims);
+        if (ndims > 0 && H5Sget_simple_extent_dims(dataspace_id, dims.data(), nullptr) < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5, "Cannot read dataspace dimensions");
+
+        return dims;
+    }
+
+    void GetRegularSelection(hid_t dataspace_id,
+                             std::vector<hsize_t> &start,
+                             std::vector<hsize_t> &stride,
+                             std::vector<hsize_t> &count,
+                             std::vector<hsize_t> &block) {
+        const auto dims = GetDataspaceDimensions(dataspace_id);
+
+        const H5S_sel_type selection_type = H5Sget_select_type(dataspace_id);
+        if (selection_type == H5S_SEL_ALL) {
+            start.clear();
+            stride.clear();
+            count.clear();
+            block.clear();
+            return;
+        }
+
+        start.assign(dims.size(), 0);
+        stride.assign(dims.size(), 1);
+        count.assign(dims.size(), 1);
+        block = dims;
+
+        if (selection_type != H5S_SEL_HYPERSLABS)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Only regular hyperslab VDS selections are supported");
+
+        if (H5Sis_regular_hyperslab(dataspace_id) <= 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Only regular hyperslab VDS selections are supported");
+
+        if (H5Sget_regular_hyperslab(dataspace_id,
+                                     start.data(),
+                                     stride.data(),
+                                     count.data(),
+                                     block.data()) < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot decode VDS hyperslab selection");
+    }
+
+    bool ContainsInRegularHyperslab(hsize_t value,
+                                    hsize_t start,
+                                    hsize_t stride,
+                                    hsize_t count,
+                                    hsize_t block) {
+        for (hsize_t i = 0; i < count; i++) {
+            const hsize_t block_start = start + i * stride;
+            if ((value >= block_start) && (value < block_start + block))
+                return true;
+        }
+        return false;
+    }
+
+    hsize_t IndexInRegularHyperslab(hsize_t value,
+                                    hsize_t start,
+                                    hsize_t stride,
+                                    hsize_t count,
+                                    hsize_t block) {
+        for (hsize_t i = 0; i < count; i++) {
+            const hsize_t block_start = start + i * stride;
+            if ((value >= block_start) && (value < block_start + block))
+                return i * block + (value - block_start);
+        }
+
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Image is not part of VDS hyperslab");
+    }
+
+    hsize_t ValueFromRegularHyperslabIndex(hsize_t index,
+                                           hsize_t start,
+                                           hsize_t stride,
+                                           hsize_t count,
+                                           hsize_t block) {
+        if (index >= count * block)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Source image is outside of VDS source hyperslab");
+
+        const hsize_t block_number = index / block;
+        const hsize_t in_block = index % block;
+        return start + block_number * stride + in_block;
+    }
+}
+
+bool HDF5VirtualDatasetMapping::ContainsVirtualImage(hsize_t image_number) const {
+    if (virtual_start.empty() || virtual_stride.empty() || virtual_count.empty() || virtual_block.empty())
+        return false;
+
+    return ContainsInRegularHyperslab(image_number,
+                                      virtual_start[0],
+                                      virtual_stride[0],
+                                      virtual_count[0],
+                                      virtual_block[0]);
+}
+
+hsize_t HDF5VirtualDatasetMapping::SourceImage(hsize_t image_number) const {
+    if (!ContainsVirtualImage(image_number))
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Image is outside of VDS mapping");
+
+    const hsize_t source_index = IndexInRegularHyperslab(image_number,
+                                                        virtual_start[0],
+                                                        virtual_stride[0],
+                                                        virtual_count[0],
+                                                        virtual_block[0]);
+
+    if (source_start.empty() || source_stride.empty() || source_count.empty() || source_block.empty())
+        return source_index;
+
+    return ValueFromRegularHyperslabIndex(source_index,
+                                          source_start[0],
+                                          source_stride[0],
+                                          source_count[0],
+                                          source_block[0]);
+}
+
+
+std::vector<HDF5VirtualDatasetMapping> HDF5Dcpl::GetVirtualMappings() const {
+    size_t mapping_count = 0;
+
+    if (H5Pget_virtual_count(id, &mapping_count) < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Cannot get number of VDS mappings");
+    if (mapping_count < 0)
+        throw JFJochException(JFJochExceptionCategory::HDF5,
+                              "Cannot get number of VDS mappings");
+
+    std::vector<HDF5VirtualDatasetMapping> ret;
+    ret.reserve(static_cast<size_t>(mapping_count));
+
+    for (size_t i = 0; i < static_cast<size_t>(mapping_count); i++) {
+        HDF5VirtualDatasetMapping mapping;
+
+        const ssize_t filename_size = H5Pget_virtual_filename(id, i, nullptr, 0);
+        if (filename_size < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS source filename size");
+
+        std::vector<char> filename(filename_size + 1, '\0');
+        if (H5Pget_virtual_filename(id, i, filename.data(), filename.size()) < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS source filename");
+        mapping.filename = filename.data();
+
+        const ssize_t dataset_size = H5Pget_virtual_dsetname(id, i, nullptr, 0);
+        if (dataset_size < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS source dataset size");
+
+        std::vector<char> dataset(dataset_size + 1, '\0');
+        if (H5Pget_virtual_dsetname(id, i, dataset.data(), dataset.size()) < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS source dataset");
+        mapping.dataset = dataset.data();
+
+        const hid_t virtual_space = H5Pget_virtual_vspace(id, i);
+        if (virtual_space < 0)
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS virtual dataspace");
+
+        const hid_t source_space = H5Pget_virtual_srcspace(id, i);
+        if (source_space < 0) {
+            H5Sclose(virtual_space);
+            throw JFJochException(JFJochExceptionCategory::HDF5,
+                                  "Cannot get VDS source dataspace");
+        }
+
+        try {
+            GetRegularSelection(virtual_space,
+                                mapping.virtual_start,
+                                mapping.virtual_stride,
+                                mapping.virtual_count,
+                                mapping.virtual_block);
+
+            GetRegularSelection(source_space,
+                                mapping.source_start,
+                                mapping.source_stride,
+                                mapping.source_count,
+                                mapping.source_block);
+        } catch (...) {
+            H5Sclose(source_space);
+            H5Sclose(virtual_space);
+            throw;
+        }
+
+        H5Sclose(source_space);
+        H5Sclose(virtual_space);
+
+        ret.emplace_back(std::move(mapping));
+    }
+
+    return ret;
+}