nappMATLAB-toolbox/openBIS/callOpenbisSync.m

%% Call nappSyncToOpenbis.py from MATLAB -- real runs, full FileList
% For quick connectivity/logic testing on a small slice, use testpybis.m
% instead. This file is for actually syncing your data.
%
% Prerequisite: pyenv points at a Python with pybis installed.
% PAT/token auth is disabled on this openBIS instance (confirmed) --
% username/password is the only working auth method for now.

%% Step 1: point at the venv (always needed fresh after a MATLAB restart)
pyenv('Version', '/Users/bartels-rausch/matlab-pybis-venv/bin/python3')

%% Step 2: import the sync module
py.sys.path().append('/Users/bartels-rausch');
py.importlib.import_module('nappSyncToOpenbis');

%% Step 3: connection details
openbisUrl   = "https://openbis-psi.labnotebook.ch";
spaceCode    = "5505_BEAMTIMES";
collectionId = [];     % <-- fill in if you want these under a specific
                        %     Experiment/Collection, e.g. '/5505_BEAMTIMES/PROJECT/COLLECTION'
objectType   = "LAC_EXPERIMENTAL_STEP_BEAMTIME";
parentIdentifier   = "/5505_BEAMTIMES/ENTRY783";
verifyCertificates = false;

username = "bartels-rausch_admin";
password = input('openBIS password: ', 's');
clc

%% Row filter -- work on a subset of FileList, or leave empty for everything
% Any valid MATLAB row index/logical works, e.g.:
%   rowFilter = 1:20;                                  % first 20 rows
%   rowFilter = [5 12 47];                              % specific rows
%   rowFilter = FileList.lastModifiedDatenum > datenum('01-May-2016');  % date-based
rowFilter = find(startsWith(FileList.name,"A"));   % <-- leave empty ([]) to process the whole table

if isempty(rowFilter)
    tmpWorkTable = FileList;
else
    tmpWorkTable = FileList(rowFilter,:);
end
fprintf('Processing %d of %d rows\n', height(tmpWorkTable), height(FileList));

%% Generate plot images for each XPS/NEXAFS row (same plots as the HTML
% report), saved to a temp folder just for this openBIS upload -- deleted
% at the end regardless of success or failure. Photo rows are skipped here
% since the original photo already uploads as its own raw-data file.
%
% This can be slow for a large selection (two plots per XPS/NEXAFS row).
THUMB_WIDTH_PX = 600;   % target width for photo/video thumbnails (height
                         % scales to preserve aspect ratio) -- adjust here
                         % if you want a different size

tmpPngDir = tempname;
mkdir(tmpPngDir);

photoExts = {'.bmp','.jpg','.jpeg','.png'};
videoExts = {'.mov','.avi','.mp4','.wmv'};

nRows = height(tmpWorkTable);
resultImagePaths = cell(nRows,1);   % {} for rows with no plot, else {path1,path2}

for ii = 1:nRows
    tmpRow  = tmpWorkTable(ii,:);
    [~,~,tmpExt] = fileparts(tmpRow.name{1});
    tmpExt  = lower(tmpExt);
    tmpSrcPath = fullfile(tmpRow.folder{1}, tmpRow.name{1});

    if ismember(tmpExt, {'.ibw','.h5'})
        try
            hdlImage = nappPlot(tmpRow,"mode","OneEach","figureMode","screen","tableVariable","image");
            if strcmp(tmpExt, '.h5')
                hdlSpectrum = nappPlot(tmpRow,"mode","OneEach","figureMode","screen", ...
                    "tableVariable","spectrum_counts","energyScale","xRay");
            else
                hdlSpectrum = nappPlot(tmpRow,"mode","OneEach","figureMode","screen", ...
                    "tableVariable","spectrum_countsPerSecond");
            end

            imgPath  = fullfile(tmpPngDir, sprintf('%04d_image.png', ii));
            specPath = fullfile(tmpPngDir, sprintf('%04d_spectrum.png', ii));
            exportgraphics(hdlImage, imgPath);
            exportgraphics(hdlSpectrum, specPath);
            resultImagePaths{ii} = {imgPath, specPath};

            close(hdlImage);
            close(hdlSpectrum);
        catch ME
            fprintf('  row %d: could not generate plot images (%s)\n', ii, ME.message);
        end

    elseif ismember(tmpExt, photoExts)
        try
            img = imread(tmpSrcPath);
            imgSmall = imresize(img, [NaN THUMB_WIDTH_PX]);
            thumbPath = fullfile(tmpPngDir, sprintf('%04d_thumb.png', ii));
            imwrite(imgSmall, thumbPath, 'png');
            resultImagePaths{ii} = {thumbPath};
        catch ME
            fprintf('  row %d: could not generate photo thumbnail (%s)\n', ii, ME.message);
        end

    elseif ismember(tmpExt, videoExts)
        try
            vidObj = VideoReader(tmpSrcPath);   %#ok<TNMLP>
            frame = readFrame(vidObj);          % first frame as a stand-in thumbnail
            frameSmall = imresize(frame, [NaN THUMB_WIDTH_PX]);
            thumbPath = fullfile(tmpPngDir, sprintf('%04d_thumb.png', ii));
            imwrite(frameSmall, thumbPath, 'png');
            resultImagePaths{ii} = {thumbPath};
        catch ME
            fprintf('  row %d: could not generate video thumbnail (%s)\n', ii, ME.message);
        end
    end
    close all hidden
end

%% Select the columns needed: direct mappings + everything the derived
% fields (file number, category, photon/KE energy ranges) need to compute from
metaCols = {'name','folder','creationDate','lastModifiedDatenum','sample', ...
            'sampleTemp_dC','cellPressure_mbar','iceTemp_dC','comment','regionName', ...
            'scientaLensMode','scientaAcquisitionMode','scientaDwellTime_ms', ...
            'scientaPassEnergy_eV','scientaSequenceIterations','scientaRegionIterations', ...
            'scientaEkinStep_eV','xRayEkin_eV','xRayEkinRange_eV','scientaEkinRange_eV', ...
            'smplX_mm','smplY_mm','smplZ_mm'};
metaCols = intersect(metaCols, tmpWorkTable.Properties.VariableNames, 'stable');

tmpSyncTable = tmpWorkTable(:, metaCols);
tmpSyncTable.resultImages = resultImagePaths;

rowsJson = jsonencode(table2struct(tmpSyncTable));

%% Campaign-level properties, applied once to every row (never overwriting
% a value already present on an existing entry)
campaignProps = struct( ...
    'campaignLocation', tmpWorkTable.Properties.CustomProperties.campaignLocation, ...
    'team',             tmpWorkTable.Properties.CustomProperties.team, ...
    'projectName',      tmpWorkTable.Properties.CustomProperties.projectName);
campaignPropsJson = jsonencode(campaignProps);

%% Sync the selected rows
try
    resultJson = py.nappSyncToOpenbis.sync_rows(rowsJson, openbisUrl, spaceCode, ...
        pyargs('collection', collectionId, 'object_type', objectType, ...
               'username', username, 'password', password, ...
               'campaign_props_json', campaignPropsJson, 'upload_files', true, ...
               'parent_identifier', parentIdentifier, ...
               'verify_certificates', verifyCertificates));
catch ME
    rmdir(tmpPngDir, 's');
    rethrow(ME);
end
rmdir(tmpPngDir, 's');

results = struct2table(jsondecode(char(resultJson)));
disp(results)

fprintf('Created: %d, Updated: %d, Errors: %d\n', ...
    sum(strcmp(results.action,'created')), ...
    sum(strcmp(results.action,'updated')), ...
    sum(contains(results.action,'error')));
if any(contains(results.action,'error'))
    disp(results(contains(results.action,'error'), :))
end