nappMATLAB-toolbox/import/nappExportH5.m

function nappExportH5(T, h5FileName)
%NAPPEXPORTH5 Export a napp* data table -- plus its campaign/column
%   metadata and the original raw data files -- into one self-contained
%   HDF5 file.
%
%   NAPPEXPORTH5(T, h5FileName) writes:
%
%     /                              root attributes: campaign metadata
%                                    (from T.Properties.CustomProperties)
%     /columnMetadata/<name>/description, /unit
%                                    one entry per documented column
%     /rows/<date>_<name>/           one group per table row, e.g.
%                                    /rows/20160404_PHOENIX0183183.ibw:
%         name                        string dataset (also forces the
%                                     group to exist before any attribute
%                                     is written to it)
%         <small scalar columns>     attributes (dates, temperatures,
%                                     pressures, sample/region names, ...)
%         <array columns>            datasets (image, imageRoh,
%                                     scientaEkin_eV, spectrum_*, ...)
%         rawFile                    uint8 bytes of the original file,
%                                     read from fullfile(folder,name) --
%                                     only written if that file is still
%                                     found on disk
%
%   Run this while the original SES/NEXAFS/Photos files are still at the
%   folder/name recorded in T -- raw-file embedding can't recover files
%   that have since moved or been archived elsewhere.
%
%   IMPLEMENTATION NOTE: this uses MATLAB's low-level HDF5 interface
%   (H5F/H5G/H5D/H5A/H5T/H5S/H5P), holding ONE file handle open for the
%   entire export, rather than the top-level convenience functions
%   (h5create/h5write/h5writeatt). An earlier version used the top-level
%   functions and reproducibly hit "Too many open files" plus
%   "H5FD_truncate driver truncate request failed" errors on tables of a
%   few hundred rows: each top-level call reopens the whole file
%   internally, and under repeated heavy use some of those reopen/close
%   cycles failed to release their file handle, leaking one descriptor
%   per row until the OS limit was hit. Holding a single handle open (the
%   same pattern already used in this toolbox's own splitH5Scans) avoids
%   the repeated open/close cycle entirely.
%
%   The file is still built on local disk (in a temp location) and only
%   moved to h5FileName once fully written, so a failed run leaves your
%   previous, valid h5FileName untouched.

SMALL_LIMIT = 10;  % numel() at/under this -> attribute; above -> dataset

if isfile(h5FileName)
    warning('nappExportH5:fileMayBeOpen', ...
        ['If "%s" is currently open in HDFView or another program, close it now -- ' ...
         'the final step of this export will fail if the file is locked.'], h5FileName);
end

workFileName = [tempname '.h5'];   % always a local path, regardless of
                                    % where h5FileName itself points

fid = H5F.create(workFileName, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');

try
    % Link creation property list that auto-creates missing intermediate
    % groups (e.g. writing '/rows/X/name' creates both '/rows' and
    % '/rows/X' automatically) -- the low-level equivalent of what
    % h5create did for us for free.
    lcpl = H5P.create('H5P_LINK_CREATE');
    H5P.set_create_intermediate_group(lcpl, true);
    acpl = H5P.create('H5P_ATTRIBUTE_CREATE');

    writeStringDataset(fid, lcpl, '/fileFormatVersion', '1.0');

    %% Campaign metadata -> root attributes
    campaignFields = {'campaignName','projectName','contact','sourceDirectory','startDate','endDate','dataLevel'};
    for f = campaignFields
        fname = f{1};
        if isprop(T.Properties.CustomProperties, fname)
            val = T.Properties.CustomProperties.(fname);
            if isnumeric(val)
                writeNumericAttribute(fid, '/', acpl, fname, val);
            else
                writeStringAttribute(fid, '/', acpl, fname, char(string(val)));
            end
        end
    end

    %% Column documentation -> one small group per documented column
    varNames     = T.Properties.VariableNames;
    descriptions = T.Properties.VariableDescriptions;
    units        = T.Properties.VariableUnits;
    for k = 1:numel(varNames)
        hasDesc = ~isempty(descriptions) && ~isempty(descriptions{k});
        hasUnit = ~isempty(units) && ~isempty(units{k});
        if hasDesc || hasUnit
            base = sprintf('/columnMetadata/%s', varNames{k});
            descStr = ''; unitStr = '';
            if hasDesc, descStr = descriptions{k}; end
            if hasUnit, unitStr = units{k};        end
            writeStringDataset(fid, lcpl, [base '/description'], descStr);
            writeStringDataset(fid, lcpl, [base '/unit'], unitStr);
        end
    end

    %% One group per row, named after each file's last-modified date + filename
    nRows     = height(T);
    usedNames = containers.Map('KeyType','char','ValueType','double');

    for i = 1:nRows
        dateStr  = datestr(T.lastModifiedDatenum(i), 'yyyymmddHHMM');
        baseName = sprintf('%s_%s', dateStr, T.name{i});
        if isKey(usedNames, baseName)
            usedNames(baseName) = usedNames(baseName) + 1;
            rowName = sprintf('%s_dup%d', baseName, usedNames(baseName));
        else
            usedNames(baseName) = 0;
            rowName = baseName;
        end
        rowGroup = sprintf('/rows/%s', rowName);

        % Guaranteed first write: forces the group to exist before any
        % attribute below targets it.
        writeStringDataset(fid, lcpl, [rowGroup '/name'], T.name{i});

        % Explicit row index -- HDF5's group listing order is not
        % guaranteed to match table row order, so this is the reliable
        % way to recover which table row a given /rows/... group came
        % from, regardless of how it's listed or iterated later.
        writeNumericAttribute(fid, rowGroup, acpl, 'rowIndex', i);

        for k = 1:numel(varNames)
            vname = varNames{k};
            if strcmp(vname,'name'), continue, end   % already written above

            val = T.(vname)(i,:);
            if iscell(val) && isscalar(val)
                val = val{1};
            end
            if isempty(val) || (isnumeric(val) && all(isnan(val(:))))
                continue
            end

            try
                if ischar(val) || (isstring(val) && isscalar(val))
                    writeStringAttribute(fid, rowGroup, acpl, vname, char(val));
                elseif (isnumeric(val) || islogical(val)) && numel(val) <= SMALL_LIMIT
                    writeNumericAttribute(fid, rowGroup, acpl, vname, val);
                elseif isnumeric(val)
                    writeNumericDataset(fid, lcpl, [rowGroup '/' vname], val);
                end
                % anything else (categorical, datetime, unexpected types) is
                % silently skipped rather than failing the whole export
            catch ME
                fprintf('  row %d, column %s: skipped (%s)\n', i, vname, ME.message);
            end
        end

        % Embed the original raw file, if it's still where FileList says it is
        srcPath = fullfile(T.folder{i}, T.name{i});
        if isfile(srcPath)
            rfid  = fopen(srcPath, 'r');
            bytes = fread(rfid, Inf, '*uint8');
            fclose(rfid);
            try
                writeUint8Dataset(fid, lcpl, [rowGroup '/rawFile'], bytes);
            catch ME
                fprintf('  row %d: raw file embed skipped (%s)\n', i, ME.message);
            end
        end
    end

    H5P.close(lcpl);
    H5P.close(acpl);
catch ME
    H5F.close(fid);   % always release the file handle, even on error
    rethrow(ME);
end

H5F.close(fid);

%% Move the finished file into place, replacing any previous version
try
    if isfile(h5FileName)
        delete(h5FileName);
    end
    movefile(workFileName, h5FileName);
catch ME
    warning('nappExportH5:replaceFailed', ...
        ['Could not replace "%s" -- it is likely still open in another program ' ...
         '(e.g. HDFView). Close it, then move the finished file manually from:\n  %s\n' ...
         'Original error: %s'], h5FileName, workFileName, ME.message);
end
end


%% ------------------------------------------------------------------
%  Low-level HDF5 write helpers (all assume fid is already open)
%  ------------------------------------------------------------------

function writeStringDataset(fid, lcpl, path, valueStr)
strType = H5T.copy('H5T_C_S1');
H5T.set_size(strType, 'H5T_VARIABLE');
space = H5S.create_simple(1, 1, []);
did = H5D.create(fid, path, strType, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT');
H5D.write(did, strType, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {char(valueStr)});
H5D.close(did);
H5S.close(space);
H5T.close(strType);
end

function writeNumericDataset(fid, lcpl, path, data)
data = double(data);
typeId = H5T.copy('H5T_NATIVE_DOUBLE');
if isvector(data) || isscalar(data)
    space = H5S.create_simple(1, numel(data), []);
    did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT');
    H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data(:));
else
    h5dims = fliplr(size(data));   % HDF5 is C-order, MATLAB is Fortran-order
    space = H5S.create_simple(numel(h5dims), h5dims, []);
    did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT');
    H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data);
end
H5D.close(did);
H5S.close(space);
H5T.close(typeId);
end

function writeUint8Dataset(fid, lcpl, path, bytes)
typeId = H5T.copy('H5T_NATIVE_UINT8');
space = H5S.create_simple(1, numel(bytes), []);
did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT');
H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', bytes);
H5D.close(did);
H5S.close(space);
H5T.close(typeId);
end

function writeStringAttribute(fid, groupPath, acpl, name, valueStr)
gid = H5G.open(fid, groupPath);
strType = H5T.copy('H5T_C_S1');
H5T.set_size(strType, 'H5T_VARIABLE');
space = H5S.create('H5S_SCALAR');
aid = H5A.create(gid, name, strType, space, acpl);
H5A.write(aid, strType, {char(valueStr)});
H5A.close(aid);
H5S.close(space);
H5T.close(strType);
H5G.close(gid);
end

function writeNumericAttribute(fid, groupPath, acpl, name, data)
gid = H5G.open(fid, groupPath);
data = double(data);
typeId = H5T.copy('H5T_NATIVE_DOUBLE');
if isscalar(data)
    space = H5S.create('H5S_SCALAR');
else
    space = H5S.create_simple(1, numel(data), []);
end
aid = H5A.create(gid, name, typeId, space, acpl);
if isscalar(data)
    H5A.write(aid, 'H5ML_DEFAULT', data);
else
    H5A.write(aid, 'H5ML_DEFAULT', data(:));
end
H5A.close(aid);
H5S.close(space);
H5T.close(typeId);
H5G.close(gid);
end