From 74b117f8c65719eca016f0cf5fa635ff591b7515 Mon Sep 17 00:00:00 2001 From: Florez Ospina Juan Felipe Date: Tue, 15 Aug 2023 08:46:17 +0200 Subject: [PATCH] Renamed h52table.m and table2h5.m as read_hdf5_as_table.m and save_table_as_hdf5.m. These reflect the scripts' functionality. --- import/read_hdf5_as_table.m | 85 +++++++++++++++++++++++++++++ import/save_table_as_hdf5.m | 105 ++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 import/read_hdf5_as_table.m create mode 100644 import/save_table_as_hdf5.m diff --git a/import/read_hdf5_as_table.m b/import/read_hdf5_as_table.m new file mode 100644 index 0000000..7cb239f --- /dev/null +++ b/import/read_hdf5_as_table.m @@ -0,0 +1,85 @@ +function output_table = load_hdf5_as_table(filename,exclude_var_set) +% receives a h5 file written with function table2h5. +% reduced_var_set (cell of character_vectors), not usable yet. + + +% Last update: 14.08.2023 +% By: Juan F. Florez Ospina +% Email: juan.florez-ospina@psi.ch + +if ~iscellstr(exclude_var_set) + disp("exclude_var_set needs to be a cell of character vectors, e.g, {'var1','var2'} to be excluded.") + disp("To proceed, exclude_var_set was set to {}. So vars will be excluded.") + exclude_var_set = {}; +end + +INFO = h5info(filename); + +num_rows = length(INFO.Groups); + +num_attr1 = length(INFO.Groups(1).Datasets); +num_attr2 = length(INFO.Groups(1).Attributes); +num_cols = num_attr1 + num_attr2; +sz = [num_rows,num_cols]; + +% Extract group level attributes, they align for all +% groups/measurements/rows of the table in filename .h5 +group_columns = {INFO.Groups(1).Attributes.Name}; + +dataset_columns = {}; +for j = 1 : num_attr1 + dataset_columns{j} = INFO.Groups(1).Datasets(j).Attributes.Value; +end + +%TODO: Implement feature to exclude varibales/columns as well as rows at +%reading time. This requires propagating modifications +% % % Select group and dataset columns to be loaded from hdf5 file, which are +% % % those not in the exclude_var_set. +% % if ~isempty(exclude_var_set) +% % for j = 1 : length(exclude_var_set) +% % % keep all vars except exclude_var_set{j} +% % var_filter = ~strcmp(exclude_var_set{j},group_columns); +% % group_columns = group_columns(var_filter); +% % if sum(var_filter) > 0 +% % exclude_var_set(j) = []; +% % end +% % end +% % +% % for j = 1 : length(exclude_var_set) +% % % keep all vars except exclude_var_set{j} +% % var_filter = ~strcmp(exclude_var_set{j},dataset_columns); +% % dataset_columns = dataset_columns(var_filter); +% % if sum(var_filter) > 0 +% % exclude_var_set(j) = []; +% % end +% % end +% % end + + +% Construct cell array with table data, where each row denotes a group, and +% the columns are either datasets or attributes in the given group. +tmp = {}; +for i = 1 : num_rows + disp(['Begin construction of row:',INFO.Groups(i).Name]) + + for j = 1 : num_cols + if j <= num_attr1 + ds_name = [INFO.Groups(i).Name '/' INFO.Groups(i).Datasets(j).Name]; + %disp(ds_name) + tmp{i,j} = h5read(filename,ds_name); + end + if j > num_attr2 + disp(INFO.Groups(i).Attributes(j-num_attr2).Name) + tmp{i,j} = h5readatt(filename,INFO.Groups(i).Name,INFO.Groups(i).Attributes(j-num_attr2).Name); + + end + + end + disp(['Ends construction of row:',INFO.Groups(i).Name]) + +end +% TODO: how can we keep the original ordering of the columns if required by +% Thorsten. +varNames = [dataset_columns,group_columns]; +output_table = array2table(tmp, 'VariableNames', varNames); + diff --git a/import/save_table_as_hdf5.m b/import/save_table_as_hdf5.m new file mode 100644 index 0000000..f95cab5 --- /dev/null +++ b/import/save_table_as_hdf5.m @@ -0,0 +1,105 @@ +function success = save_table_as_hdf5(input_table, filename) + +% This function takes input_table (of table type), structured with +% Thorsten's FileTable format, and stores it as .h5 file with name +% filename. Each row is treated as a group, where columns can be attributes +% of the group or datasets inside the group. + +% Last update: 11.08.2023 +% By: Juan F. Florez Ospina +% Email: juan.florez-ospina@psi.ch + +if istable(input_table) + column_names = input_table.Properties.VariableNames; +else + success = 0; + return +end + +% Divide input_table into tables with numeric and cellstr variables/columns +numeric_table = input_table(:,vartype('numeric')); % group attributes of a timestamped measurement +cellstr_table = input_table(:,vartype('cellstr')); % group datasets of a timestamped measurement + +numeric_columns = numeric_table.Properties.VariableNames; +str_columns = cellstr_table.Properties.VariableNames; + +% Based on homogeneous columns/cell arrays, we identify the remaining columns +% with possibly heterogeneous entries. Ideally, these should store numeric +% n-dimensional arrays/datasets. + +rest_columns = {}; +cntr = 1; +for i = 1 : length(input_table.Properties.VariableNames) + variable = input_table.Properties.VariableNames{i}; + + is_str = sum(strcmp(variable,str_columns))>0; + is_num = sum(strcmp(variable,numeric_columns))>0; + if ~(is_num || is_str) + rest_columns{cntr} = variable; + disp(variable) + cntr = cntr + 1; + end +end + +% Create an hdf5 file, where rows are seen as measurements/groups with +% attributes given by numeric and str variables and datasets given by the +% remaining variables as long as they are numeric n-dimensional arrays. + +cell_table = input_table(:,rest_columns); +[rows, cols] = size(input_table); + +for i = 1 : rows + group_name = strcat("/MEAS",string(i),"/"); + for j = 1 : length(rest_columns) + dataset_name = strcat("DS",string(j)); + + % Some table entries do not provide direct access to the object + % of interest, but may contain either {object} or {{object}}. + % The following loop solves that problem. TODO: is this generic enough? + + data = cell_table{i,rest_columns{j}}{1};%{1}; + while iscell(data) + data = data{1}; + end + % when there is no data, we still need to provide a valid size + if ~isempty(data) + sz = size(data); + else + sz = [1,1]; + end + %%%% + % Group's dataset: Create dataset of size sz in a certain group + h5create(filename, strcat(group_name,dataset_name),sz); + % Dataset's attribute: Add attribute with dataset_name equal to column_name + attr = "column_name"; + attr_value = rest_columns{j}; + h5writeatt(filename,strcat(group_name,dataset_name),attr,attr_value) + % Assign 'data' to the dataset. 'data' has to be of admissible type. + try + h5write(filename,strcat(group_name,dataset_name),data) + catch exception + disp(exception) + %h5write(filename,strcat(group_name,dataset_name),cell_table{i,rest_columns{j}}) + end + end + + % Group's attribute: Assign group attributes + for attr_idx = 1 : max([length(numeric_columns),length(str_columns)]) + + if attr_idx <= length(numeric_columns) + attr = numeric_columns{attr_idx}; + attr_value = numeric_table{i,numeric_columns{attr_idx}}; + h5writeatt(filename,group_name,attr,attr_value) + end + + if attr_idx <= length(str_columns) + attr = str_columns{attr_idx}; + attr_value = cellstr_table{i,str_columns{attr_idx}}; + h5writeatt(filename,group_name,attr,attr_value) + end + + end + %h5create(filename,"/g2/DS2",[20 Inf],"Chunksize",[5 5]); +end + +success = 1;