From 2ec52b7ee7c2e82411f3117276b9c0cd1ff31437 Mon Sep 17 00:00:00 2001 From: ThorBarRa Date: Mon, 31 Aug 2026 14:31:39 +0200 Subject: [PATCH] Add openBIS sync pipeline; fix nappFileList/nappGenerateSpectra/nappPlot bugs; add importData.m ground script beamtime/nappFileList.m - Fix h5 LensMode: translate numeric codes to descriptive text (e.g. 1 -> T_HiPPHAXPES) to match what .ibw files already store, applied across all three h5read fallback branches, not just the oldest-format one beamtime/nappGenerateSpectra.m - Bug fixes beamtime/nappPlot.m - Bug fixes import/importData.m (new) - Ground script for one beamtime: import (nappFileList, nappGenerateSpectra), sample/spot and photon-energy assignment per row range, campaign metadata (CustomProperties) and column descriptions/units set inline, tidy-up, and export to .csv/.mat/.h5 (via nappExportH5). Leaves FileList in the workspace for PublishReport.m and callOpenbisSync.m to run against afterward. import/nappExportH5.m (new) - Export FileList + campaign metadata + raw source files to a self-contained HDF5 archive - Low-level HDF5 API (H5F/H5G/H5D/H5A/H5T/H5S/H5P) to fix H5FD_truncate/"too many open files" errors from the top-level h5create/h5write functions each reopening the file internally - Explicit rowIndex attribute per row group, since HDF5 group listing order isn't guaranteed to match table row order openBIS/ (new) - nappSyncToOpenbis.py, callOpenbisSync.m, testpybis.m, README.md - openBIS sync pipeline: create-or-update per row matched by $name, correct lowercase property codes, timestamp formatting, parent linking, raw file upload, plot/photo/video thumbnail embedding, config/engine split for maintainability -- see README.md for full details --- beamtime/nappFileList.m | 397 ++++++------------- beamtime/nappGenerateSpectra.m | 70 ++-- beamtime/nappPlot.m | 7 +- import/nappExportH5.m | 268 +++++++++++++ openBIS/README.md | 157 ++++++++ openBIS/callOpenbisSync.m | 163 ++++++++ openBIS/importData.m | 161 ++++++++ openBIS/nappSyncToOpenbis.py | 684 +++++++++++++++++++++++++++++++++ openBIS/testpybis.m | 46 +++ 9 files changed, 1637 insertions(+), 316 deletions(-) create mode 100644 import/nappExportH5.m create mode 100644 openBIS/README.md create mode 100644 openBIS/callOpenbisSync.m create mode 100644 openBIS/importData.m create mode 100644 openBIS/nappSyncToOpenbis.py create mode 100644 openBIS/testpybis.m diff --git a/beamtime/nappFileList.m b/beamtime/nappFileList.m index ef5abb0..e2bc882 100755 --- a/beamtime/nappFileList.m +++ b/beamtime/nappFileList.m @@ -1,4 +1,4 @@ -function ListFiles = nappFileList(dataPath,varargin) +function ListFiles = nappFileList(dataPath, opts) %NAPPFILELIST Import spectroscopy data (XPS, NEXAFS) from a beam time folder into data table. % Photos, analyser and sample settings such as pass energy, lens mode, temperature, % position, and pressure are automatically extracted and added to the table. @@ -6,8 +6,13 @@ function ListFiles = nappFileList(dataPath,varargin) % SYNTAX % ------ % ListFiles = NAPPFILELIST(dataPath) -% ListFiles = NAPPFILELIST(dataPath, DataTable) -% ListFiles = NAPPFILELIST(dataPath, DataTable, 'Name', Value) +% ListFiles = NAPPFILELIST(dataPath, 'Name', Value, ...) +% +% Note: as of March 2026, DataTable moved from a positional argument to +% the name-value argument 'DataTable' (see below) to remove ambiguity +% with the other name-value arguments. Existing calls of the form +% NAPPFILELIST(dataPath, DataTable, ...) must be updated to +% NAPPFILELIST(dataPath, 'DataTable', DataTable, ...). % % DESCRIPTION % ----------- @@ -32,15 +37,13 @@ function ListFiles = nappFileList(dataPath,varargin) % Note: The folder structure must follow the strict naming convention % described in the "Folder Structure" section below. % -% OPTIONAL POSITIONAL INPUT -% ------------------------- -% DataTable -% Existing data table to append imported data to (table object). -% If provided, new files are merged with existing table. -% If not provided, a new table is created from scratch. -% % NAME-VALUE PAIR ARGUMENTS % ------------------------- +% 'DataTable' +% Existing data table to append imported data to (table object). +% If provided, new files are merged with existing table. +% If not provided (default), a new table is created from scratch. +% % 'mode' % Import mode (char or string, default: 'local'). % Supported values: @@ -59,6 +62,13 @@ function ListFiles = nappFileList(dataPath,varargin) % 'on' - Import LabView logs from Pressure subfolder % 'off' - Skip LabView log import (this is much,much faster) % +% 'verbose' +% Show progress banners in the command window (logical, default: true). +% Set to false to suppress routine status output (e.g. when calling +% from publish() or a script you don't want to clutter with progress +% messages). Warnings and per-file problem messages are always shown +% regardless of this setting. +% % OUTPUT ARGUMENTS % ---------------- % ListFiles @@ -115,22 +125,22 @@ function ListFiles = nappFileList(dataPath,varargin) % EXAMPLE FOLDER LAYOUT % --------------------- % 2025/November_NappIss/ -%  SES/ [XPS data] -%   0001_C1s_525eV.ibw -%   0002_O1s_850eV.ibw -%   0003_Survey_850eV.ibw -%   ... -%  NEXAFS/ [NEXAFS maps] -%   0010_O_Kedge.h5 -%   0011_N_Kedge.h5 -%   ... -%  Photos/ [Sample photos] -%   A0001-20251123_102407.bmp -%   A0002-20251123_102408.bmp -%   ... -%  Pressure/ [Environmental logs] -%  LabView_T_RH_20251119.txt -%  logfile-23-11-2025-12h46min.txt +% SES/ [XPS data] +% 0001_C1s_525eV.ibw +% 0002_O1s_850eV.ibw +% 0003_Survey_850eV.ibw +% ... +% NEXAFS/ [NEXAFS maps] +% 0010_O_Kedge.h5 +% 0011_N_Kedge.h5 +% ... +% Photos/ [Sample photos] +% A0001-20251123_102407.bmp +% A0002-20251123_102408.bmp +% ... +% Pressure/ [Environmental logs] +% LabView_T_RH_20251119.txt +% logfile-23-11-2025-12h46min.txt % % FILE NAMING CONVENTIONS % ----------------------- @@ -191,10 +201,10 @@ function ListFiles = nappFileList(dataPath,varargin) % FileList = nappFileList('2025/November_NappIss', 'mode', 'local'); % % % Example 2: Add new data to existing table -% FileList = nappFileList('2025/November_NappIss', FileList, 'mode', 'local'); +% FileList = nappFileList('2025/November_NappIss', 'DataTable', FileList, 'mode', 'local'); % % % Example 3: Import only new files not yet in table -% FileList = nappFileList('2025/November_NappIss', FileList, ... +% FileList = nappFileList('2025/November_NappIss', 'DataTable', FileList, ... % 'mode', 'localNew'); % % % Example 4: Import from network drive with file browser @@ -205,6 +215,9 @@ function ListFiles = nappFileList(dataPath,varargin) % FileList = nappFileList('2025/November_NappIss', ... % 'mode', 'local', 'importLabView', 'off'); % +% % Example 6: Suppress routine progress output (e.g. inside publish()) +% FileList = nappFileList('2025/November_NappIss', 'verbose', false); +% % DEPENDENCIES % ============ % - IBWread function (Igor Pro format reader for MATLAB) @@ -228,6 +241,7 @@ function ListFiles = nappFileList(dataPath,varargin) % MATLAB VERSION REQUIREMENTS % =========================== % - MATLAB 2018a or higher (for '**' notation in dir patterns) +% - MATLAB 2021a or higher (for the arguments-block argument handling used here) % % HISTORY & UPDATES % ================= @@ -239,6 +253,11 @@ function ListFiles = nappFileList(dataPath,varargin) % - March 2022: Strict data structure required for local and remote sources % - November 2025: Added support for PShell h5 file structure (NEXAFS) % - November 2025: Added creation date extraction for Mac (NEXAFS) +% - March 2026: Converted argument handling from inputParser/varargin to +% an arguments block for autocomplete support; DataTable moved from a +% positional argument to the name-value argument 'DataTable'; added +% 'verbose' name-value argument (default true) to suppress routine +% progress output. % % See also: TABLE, IBWread, h5read, h5info, DIR, inputParser % @@ -259,22 +278,17 @@ function ListFiles = nappFileList(dataPath,varargin) % * November 2025, added creation date to mac for NEXAFS. % * March 2026, added movie import from photos folder, added jpg as photo % file +% * March 2026, converted to arguments block (autocomplete); DataTable is +% now a name-value argument; added 'verbose' name-value argument. %% Input -% Define conditions and defaults of input -defaultMode = 'local'; -expectedMode = {'local','nappDrive','dataDrive','localNew','nappDrivePick','localPick','dataPick'}; -defaultImportLabView = 'on'; -expectedImportLabView = {'on','off'}; -% Parse input -p = inputParser; -addRequired(p,'dataPath'); -addOptional(p,'DataTable',@istable); -addParameter(p,'mode',defaultMode,... - @(x) any(validatestring(x,expectedMode))); -addParameter(p,'importLabView',defaultImportLabView,... - @(x) any(validatestring(x,expectedImportLabView))); -parse(p,dataPath,varargin{:}); +arguments + dataPath (1,:) char + opts.DataTable table = table() + opts.mode (1,:) char {mustBeMember(opts.mode,{'local','nappDrive','dataDrive','localNew','nappDrivePick','localPick','dataPick'})} = 'local' + opts.importLabView (1,:) char {mustBeMember(opts.importLabView,{'on','off'})} = 'on' + opts.verbose (1,1) logical = true +end %% Data Paths % % Mac and Windows use different notations to address network drives. For @@ -283,21 +297,21 @@ parse(p,dataPath,varargin{:}); % The letters might be different for the same drive on different computer. -if contains(p.Results.mode,'local') +if contains(opts.mode,'local') % Data server address if ismac server = '/Users/bartels-rausch/Documents/Forschung_Einsam/PSI_SLS/'; elseif ispc server = userpath; end -elseif contains(p.Results.mode,'nappDrive') +elseif contains(opts.mode,'nappDrive') % Data server address if ismac server = '/Volumes/'; elseif ispc server = '\\fs03\'; end -elseif contains(p.Results.mode,'data') +elseif contains(opts.mode,'data') % Data server address if ismac server = '/Volumes/'; @@ -305,7 +319,7 @@ elseif contains(p.Results.mode,'data') server = '\\fs101\'; end end -if contains(p.Results.mode,'data') +if contains(opts.mode,'data') dataPath = fullfile(server,'5505','Data','beamtimes',dataPath); else dataPath = fullfile(server,'NAPP','SC Data',dataPath); @@ -362,7 +376,7 @@ end % % THE ** notation needs a recent version of Matlab, 2014 does not work. -if endsWith(p.Results.mode, 'Pick') +if endsWith(opts.mode, 'Pick') % --- PICK MODE --- [filePick, pathPick] = uigetfile({'*.ibw;*.xy;*.h5'}, ... 'Select One or More Files', dataPath, 'MultiSelect', 'on'); @@ -442,7 +456,7 @@ else % % OUTPUT: % For each multi-scan file, creates separate output files: - % Example: 0002_..._O_Kedge.h5  0002_..._O_Kedge_scan_1.h5 + % Example: 0002_..._O_Kedge.h5 0002_..._O_Kedge_scan_1.h5 % 0002_..._O_Kedge_scan_2.h5 % ========================================================================= % Loop through each file in the NEXAFS file list @@ -499,9 +513,11 @@ else TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.h5']); TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok - disp('*************************************************') - disp('* Voila: Splitting NEXAFS files done *') - disp('*************************************************') + if opts.verbose + disp('*************************************************') + disp('* Voila: Splitting NEXAFS files done *') + disp('*************************************************') + end % --- Date of First File Check --- % Since TmpFileList{1} now safely contains all .ibw and .xy files, @@ -866,9 +882,11 @@ if ~isempty(TmpFileList{1}) TmpFileList{1} = ProcessedXPS; end -disp('*************************************************') -disp('* Voila: Done reading & writing XPS data! *') -disp('*************************************************') +if opts.verbose + disp('*************************************************') + disp('* Voila: Done reading & writing XPS data! *') + disp('*************************************************') +end %% Read and write NEXAFS meta data % % The h5 file is much nicer organised, so we do not need string operations @@ -1106,7 +1124,17 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element TmpFileList{2}(ii).scientaLensMode = tmp{1}; catch tmp = h5read([TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name],'/entry/instrument/NDAttributes/LensMode'); - TmpFileList{2}(ii).scientaLensMode = num2str(tmp(1)); + % Translate numeric lens mode codes (old NDAttributes + % format) to the same descriptive names .ibw files already + % store as text. Only code 1 is confirmed so far -- add + % more elseif branches here as other codes are encountered; + % anything unmapped still falls back to the raw number + % rather than guessing. + if tmp(1) == 1 + TmpFileList{2}(ii).scientaLensMode = 'T_HiPPHAXPES'; + else + TmpFileList{2}(ii).scientaLensMode = num2str(tmp(1)); + end end end @@ -1194,9 +1222,11 @@ end -disp('*************************************************') -disp('* Voila: Done reading and writing NEXAFS data! *') -disp('*************************************************') +if opts.verbose + disp('*************************************************') + disp('* Voila: Done reading and writing NEXAFS data! *') + disp('*************************************************') +end %% Create Picture List % % The file structure needs same number of fields as SES and NEXAFS data. @@ -1254,7 +1284,7 @@ if ~isempty(tmpDeletedNames) end % Actually remove the rows from the table ListFiles(tmpEmptyImgLogic, :) = []; -clearvars -except ListFiles server dataPath* p +clearvars -except ListFiles server dataPath* opts %% Sort by date @@ -1263,12 +1293,14 @@ ListFiles = sortrows(ListFiles,6); % Then, remove un-needed columns ListFiles = removevars(ListFiles,{'bytes','isdir'}); -disp('*************************************************') -disp('* Voila: Done creating data list! *') -disp('*************************************************') +if opts.verbose + disp('*************************************************') + disp('* Voila: Done creating data list! *') + disp('*************************************************') +end %% Import LabView data % -if matches(p.Results.importLabView,'on') +if matches(opts.importLabView,'on') if isfolder(dataPath_LabView) % verify that this folder exists. try % Get list of all files in that folder @@ -1311,9 +1343,11 @@ if matches(p.Results.importLabView,'on') disp('Problem reading LabView file') end end - disp('*************************************************') - disp('* Voila: Done reading LabView data! *') - disp('*************************************************') + if opts.verbose + disp('*************************************************') + disp('* Voila: Done reading LabView data! *') + disp('*************************************************') + end clearvars TmpFileList @@ -1389,9 +1423,11 @@ if matches(p.Results.importLabView,'on') end end - disp('*************************************************') - disp('* Voila: Done writing LabView data! *') - disp('*************************************************') + if opts.verbose + disp('*************************************************') + disp('* Voila: Done writing LabView data! *') + disp('*************************************************') + end end %% Make table nicer % @@ -1443,44 +1479,44 @@ end % add the imported data to previous list (if passed to function upon call) -if istable(p.Results.DataTable) % do this only if previous list was input to function call +if ~isempty(opts.DataTable) % do this only if previous list was input to function call % find table variables that are missing in ListFiles - existColumn = strcmp('sample',p.Results.DataTable.Properties.VariableNames); + existColumn = strcmp('sample',opts.DataTable.Properties.VariableNames); if ~any(existColumn) - p.Results.DataTable.sample = cellstr(repmat(' ',size(p.Results.DataTable,1),1)); + opts.DataTable.sample = cellstr(repmat(' ',size(opts.DataTable,1),1)); end - tmpMissing = ~ismember(p.Results.DataTable.Properties.VariableNames,ListFiles.Properties.VariableNames); + tmpMissing = ~ismember(opts.DataTable.Properties.VariableNames,ListFiles.Properties.VariableNames); % find all table variables that are cells - tmpCell = cellfun(@(x) iscell(p.Results.DataTable.(x)), p.Results.DataTable.Properties.VariableNames); + tmpCell = cellfun(@(x) iscell(opts.DataTable.(x)), opts.DataTable.Properties.VariableNames); % find all table variables that are numbers - tmpNumber = cellfun(@(x) isnumeric(p.Results.DataTable.(x)), p.Results.DataTable.Properties.VariableNames); + tmpNumber = cellfun(@(x) isnumeric(opts.DataTable.(x)), opts.DataTable.Properties.VariableNames); % table variabes that hold cells and are missing - tmpCellVariables = p.Results.DataTable.Properties.VariableNames(tmpMissing & tmpCell); + tmpCellVariables = opts.DataTable.Properties.VariableNames(tmpMissing & tmpCell); % table variables that hold numbers and are missing - tmpNumberVariables = p.Results.DataTable.Properties.VariableNames(tmpMissing & tmpNumber); + tmpNumberVariables = opts.DataTable.Properties.VariableNames(tmpMissing & tmpNumber); % add empty table variables for missing cell and for number variables if ~isempty(tmpCellVariables) - for i = 1: size(p.Results.DataTable(:,tmpMissing & tmpCell),2) + for i = 1: size(opts.DataTable(:,tmpMissing & tmpCell),2) ListFiles = addvars(ListFiles,cell(size(ListFiles,1),1),'NewVariableNames',tmpCellVariables{i}); end end if ~isempty(tmpNumberVariables) - for i = 1: size(p.Results.DataTable(:,tmpMissing & tmpNumber),2) + for i = 1: size(opts.DataTable(:,tmpMissing & tmpNumber),2) ListFiles = addvars(ListFiles,nan(size(ListFiles,1),1),'NewVariableNames',tmpNumberVariables{i}); end end % % % sort the tables, if they have not the same order - % tmpCompareTables = cellfun(@isequal, ListFiles.Properties.VariableNames', p.Results.DataTable.Properties.VariableNames'); + % tmpCompareTables = cellfun(@isequal, ListFiles.Properties.VariableNames', opts.DataTable.Properties.VariableNames'); % % if ~all(tmpCompareTables) - % indexDifferenzes = find(ismember(p.Results.DataTable.Properties.VariableNames',ListFiles.Properties.VariableNames(~tmpCompareTables)','rows')); + % indexDifferenzes = find(ismember(opts.DataTable.Properties.VariableNames',ListFiles.Properties.VariableNames(~tmpCompareTables)','rows')); % nameListFiles =ListFiles.Properties.VariableNames(~tmpCompareTables); % % for ii = 1:size(indexDifferenzes,1) - % tmpIndx = find(ismember(p.Results.DataTable.Properties.VariableNames',nameListFiles{ii},'rows')); - % tmpPreviousVarName = p.Results.DataTable.Properties.VariableNames{tmpIndx-1}; + % tmpIndx = find(ismember(opts.DataTable.Properties.VariableNames',nameListFiles{ii},'rows')); + % tmpPreviousVarName = opts.DataTable.Properties.VariableNames{tmpIndx-1}; % ListFiles = movevars(ListFiles,nameListFiles{ii},'After',tmpPreviousVarName); % fprintf("%s comes after %s \n", nameListFiles{ii},tmpPreviousVarName) % end @@ -1490,8 +1526,8 @@ if istable(p.Results.DataTable) % do this only if previous list was input to fun % merge tables (append imported data to previous table) - % ListFiles = [p.Results.DataTable; ListFiles]; - ListFiles = outerjoinRowToTable(p.Results.DataTable, ListFiles); + % ListFiles = [opts.DataTable; ListFiles]; + ListFiles = outerjoinRowToTable(opts.DataTable, ListFiles); % Then, we sort the table based on the time (datenum) ListFiles = sortrows(ListFiles,'lastModifiedDatenum'); % check fromat @@ -1518,194 +1554,13 @@ end -disp('*************************************************') -disp('* Voila: nappFileList.m finished ! *') -disp('*************************************************') - - +if opts.verbose + disp('*************************************************') + disp('* Voila: nappFileList.m finished ! *') + disp('*************************************************') end -% function LabView = LabView_importData(fullFileName,varargin) -% % -% % LabView_importData reads data log ASCII files stored by LabView into -% % a Matlab time series. -% % -% % LabView = LabView_importData(A) imports file A -% % and stores its content as time series in a structure called "LabView". -% % -% % LabView = LabView_importData(A,LabView) imports file A -% % and appends its content as time series to the LabView structure. -% % -% % -% % Example -% % LabView = LabView_importData('exmpl.dat',LabView) -% % -% % Input -% % * Napp_chamber_file: Complete and absolute path to the Napp chamber log file. -% % * LabView: name of output structure. Always call this'LabView' -% % -% % Output -% % * LabView: time series, the name starting with "ts" -% % followed by date and time the recording started. -% % -% % See also LABVIEW_COMBINE, LABVIEW_PLOT, LABVIEW_CUTDATA -% % -% %% Data Structure -% % -% % Data structure before 4.3.2019 -% % col 1: Date -% % col 2: Time -% % col 3: Reservoir "Pressure 1" - Unit "1 water" - mbar -> ts.Data:2 -% % col 4: Reservoir "Pressure 2" - Unit "2 other" - mbar -> ts.Data:3 -% % col 5: Baratron 1 - cell - 1mbar max - mbar -> ts.Data:4 -% % col 6: Baratron 2 - cell - 10mbar max - mbar -> ts.Data:8 -% % col 7: Baratron 3 - cell - 0.05mbar max - mbar -> ts.Data:9 -% % col 8: Baratron 4 - - mbar -% % col 9: Temp.Ice Sample - °C -> ts.Data:1 -% % col 10: Temp.Heated Sample -% % col 11: none -% % col 12: Temp. Reservoir "Cooler 1" - °C -> ts.Data:5 -% % col 13: Temp. Reservoir "Cooler 2" - °C -> ts.Data:6 -% % col 14: Flow He(cryoholder) - l/min -> ts.Data:7 -% % col 15: none -% % col 16: none -% % -% % Data structure after 4.3.2019 -% % col 1: Date -% % col 2: Time -% % col 3: Reservoir "Pressure 1" - Unit "1 water" - mbar -> ts.Data:2 -% % col 4: Reservoir "Pressure 2" - Unit "2 other" - mbar -> ts.Data:3 -% % col 5: Baratron 1 - cell - 1mbar max - mbar -> ts.Data:4 -% % col 6: Baratron 2 - cell - 10mbar max - mbar -> ts.Data:8 -% % col 7: Baratron 3 - cell - 0.05mbar max - mbar -> ts.Data:9 -% % col 8: Baratron 4 - - mbar -% % col 9: Temp.Ice Sample - °C -> ts.Data:1 -% % col 10: Temp.Heated Sample -% % col 11: Temp. Reservoir "Cooler 1" - °C -> ts.Data:5 -% % col 12: Temp. Reservoir "Cooler 2" - °C -> ts.Data:6 -% % col 13: Flow He(cryoholder) - l/min -> ts.Data:7 -% % col 14: Flow gas 1 -% % col 15: Pressure chamber - mbar -% % col 16: x-position of cryoholder - mm -> ts.Data:10 -% % col 17: y-position of cryoholder - mm -> ts.Data:11 -% % col 18: z-position of cryoholder - mm -> ts.Data:12 -% % col 19: nothing -% % col 20: Temp.Sealing - °C -> ts.Data:13 -% -% -% %% -% -% % Create an InputParser object -% p = inputParser; -% % Add inputs to the scheme. -% defaultAppendToStructure = 0; -% addRequired(p,'fullFileName',@isfile); -% addOptional(p,'appendToStructure',defaultAppendToStructure,@isstruct); -% % Parse the inputs. -% parse(p,fullFileName,varargin{:}) -% -% %% -% % get filedate -% [~,name,~] = fileparts(p.Results.fullFileName); -% tmp = strsplit(name,'_PC'); -% filedate = datenum(tmp{1},'yyyy-mm-dd_HH.MM.SS'); -% -% clearvars name tmp -% -% %% -% % parse structure to append to -% -% if isstruct(p.Results.appendToStructure) -% LabView = p.Results.appendToStructure; -% end -% %% -% if datenum(filedate) <= datenum('04-Mar-2019 13:30:00') -% -% fid = fopen(p.Results.fullFileName); -% data = textscan(fid, ... -% '%s %s %f %f %f %f %f %f %f %f %f %f %f %f %f %f',... -% 'HeaderLines',1); % data file has 2 text columns, followed by 14 numeric columns -% fclose(fid); -% % define time series -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]) ... -% = timeseries([data{9} data{3} data{4} data{5} data{12} ... -% data{13} data{14} data{6} data{7}] , ... -% datestr(datenum([char(data{1}), char(data{2})],... -% 'dd.mm.yyyyHH:MM:SS'),0)); -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]).Name ... -% = 'Readout Napp Chamber raw signal'; -% clear fid ans formatOut nans t x -% -% elseif datenum(filedate) >= datenum('15-Sep-2021 01:00:00') && datenum(filedate) <= datenum('01-Apr-2023 01:00:00') -% % during this time, date and time were in american format. -% fid = fopen(p.Results.fullFileName); -% data = textscan(fid, ... -% '%s %s %s %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f',... -% 'HeaderLines',1); % -% -% fclose(fid); -% % the date formating in the dat file is really messed up. Try to make -% % it consitent with MATLAB. The problem is that MATLAB does not -% % understand 1 digit month and hours. -% % -% % -% % Split the date in column 1 into year,month,day -% newDateSplit = cellfun(@(x) split(char(x),'/'),data{1},'UniformOutput', false); -% % Split the time in column 2 into hours, minutes, seconds -% newTimeSplit = cellfun(@(x) split(char(x),':'),data{2},'UniformOutput', false); -% % combine both with the am/pm string in column 3 to a new datestring -% customDateStr = cellfun(@(x,y,z) [datestr(datenum([str2double(x{3}),str2double(x{1}),str2double(x{2}),str2double(y{1}),str2double(y{2}),str2double(y{3})])), ' ', char(z)], newDateSplit,newTimeSplit,data{3}, 'uniformOutput',false) ; -% -% % define time series -% LabView.(['ts_' datestr(customDateStr{1},'YYYYmmdd_HHMM')]) ... -% = timeseries([data{10} data{4} data{5} data{6} data{12} ... -% data{13} data{14} data{7} data{8} data{17} data{18} data{19} data{21}], ... -% customDateStr); -% LabView.(['ts_' datestr(customDateStr{1},'YYYYmmdd_HHMM')]).Name ... -% = 'Readout Napp Chamber raw signal'; -% clearvars fid ans formatOut nans t x new* custom* -% -% -% elseif datenum(filedate) >= datenum('01-Apr-2023 02:00:00') -% -% fid = fopen(p.Results.fullFileName); -% data = textscan(fid, ... -% '%s %s %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f',... -% 'HeaderLines',1); -% % data file has 2 text columns, followed by 18 numeric columns (THE AM -% % PM IS MISSING and some changes to the LabView file. Confirm with Luca....) -% fclose(fid); -% % define time series -% -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]) ... -% = timeseries([data{9} data{3} data{4} data{5} data{11} ... -% data{12} data{20} data{6} data{7} data{15} data{16} data{17} data{19}] , ... -% datestr(datenum([char(data{1}), char(data{2})],... -% 'dd.mm.yyyyHH:MM:SS'),0)); -% -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]).Name ... -% = 'Readout Napp Chamber raw signal'; -% clear fid ans formatOut nans t x -% -% else -% fid = fopen(p.Results.fullFileName); -% data = textscan(fid, ... -% '%s %s %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f %f',... -% 'HeaderLines',1); % data file has 2 text columns, followed by 14 numeric columns -% fclose(fid); -% % define time series -% -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]) ... -% = timeseries([data{9} data{3} data{4} data{5} data{11} ... -% data{12} data{13} data{6} data{7} data{16} data{17} data{18} data{20}] , ... -% datestr(datenum([char(data{1}), char(data{2})],... -% 'dd.mm.yyyyHH:MM:SS'),0)); -% -% LabView.(['ts_' datestr(datenum([char(data{1}{1}), char(data{2}{1})],'dd.mm.yyyyHH:MM:SS'),'YYYYmmdd_HHMM')]).Name ... -% = 'Readout Napp Chamber raw signal'; -% clear fid ans formatOut nans t x -% end -% end +end function data_structure = LabView_combine(data_structure) % LabView_combine can be used to merge time series into one. diff --git a/beamtime/nappGenerateSpectra.m b/beamtime/nappGenerateSpectra.m index a92105e..f859de7 100644 --- a/beamtime/nappGenerateSpectra.m +++ b/beamtime/nappGenerateSpectra.m @@ -83,19 +83,21 @@ function DataTable = nappGenerateSpectra(DataTable,varargin) % this stores the spectrum obtained by averaging over the chosen % iteration range. % -% logGenerateMode -% Cell array of character vectors specifying the mode used for -% each processed row. +% logNappGenerateSpectraMode +% String array specifying the mode used for each processed row. % -% logGenerateRange +% logNappGenerateSpectraRange % Numeric matrix with two columns. For '.h5' entries this stores % [eKinStart eKinEnd]. For '.ibw' entries this stores % [iterationStart iterationEnd]. % -% logGenerateDate -% Cell array of character vectors containing the date and time -% when the spectra were generated. +% logNappGenerateSpectraDate +% String array containing the date and time when the spectra +% were generated for each processed row. % +% logNappGenerateSpectraCaller +% String array containing the file path of the script or +% function that called nappGenerateSpectra, for provenance.% % Notes % ----- % - The function distinguishes between '.h5' and '.ibw' files based on @@ -133,17 +135,21 @@ existColumn = strcmp('spectrum_counts',p.Results.DataTable.Properties.VariableNa if ~any(existColumn) DataTable.spectrum_counts = cell(size(p.Results.DataTable,1),1); end -existColumn = strcmp('logGenerateRange',p.Results.DataTable.Properties.VariableNames); +existColumn = strcmp('logNappGenerateSpectraRange',p.Results.DataTable.Properties.VariableNames); if ~any(existColumn) - DataTable.logGenerateRange = NaN(size(p.Results.DataTable,1),2); + DataTable.logNappGenerateSpectraRange = NaN(size(p.Results.DataTable,1),2); end -existColumn = strcmp('logGenerateDate',p.Results.DataTable.Properties.VariableNames); +existColumn = strcmp('logNappGenerateSpectraMode',p.Results.DataTable.Properties.VariableNames); if ~any(existColumn) - DataTable.logGenerateDate = cellstr(repmat(' ',size(p.Results.DataTable,1),1)); + DataTable.logNappGenerateSpectraMode = repmat("",size(p.Results.DataTable,1),1); end -existColumn = strcmp('logGenerateMode',p.Results.DataTable.Properties.VariableNames); +existColumn = strcmp('logNappGenerateSpectraDate',p.Results.DataTable.Properties.VariableNames); if ~any(existColumn) - DataTable.logGenerateDate = cellstr(repmat(' ',size(p.Results.DataTable,1),1)); + DataTable.logNappGenerateSpectraDate = repmat("",size(p.Results.DataTable,1),1); +end +existColumn = strcmp('logNappGenerateSpectraCaller',p.Results.DataTable.Properties.VariableNames); +if ~any(existColumn) + DataTable.logNappGenerateSpectraCaller = repmat("",size(p.Results.DataTable,1),1); end % check input (all elements of input variable need to have non-zero size) @@ -205,41 +211,19 @@ end % write results to table DataTable.spectrum_counts(p.Results.tableSelection) = tmp_3; DataTable.spectrum_countsPerSecond(p.Results.tableSelection) = tmp_6; -% write log to table - DataTable.logGenerateMode(p.Results.tableSelection,:) = cellstr(repmat(p.Results.mode,size(p.Results.DataTable(p.Results.tableSelection,:),1),1)); - DataTable.logGenerateRange(p.Results.tableSelection,:) = [tmp_4 tmp_5]; - % --- Define variables for the Universal Logging Block --- - numRowsTable = height(DataTable); +% write log to table processIdx = find(p.Results.tableSelection); - % ------------------------------------------------------------------- - - % 1. Dynamically get this function's name - thisFuncName = mfilename; - colDate = sprintf('log%sDate', thisFuncName); - colCaller = sprintf('log%sCaller', thisFuncName); - - - % 2. Probe for column existence, create empty string arrays if missing - if ~ismember(colDate, DataTable.Properties.VariableNames) - DataTable.(colDate) = repmat("", numRowsTable, 1); - end - if ~ismember(colCaller, DataTable.Properties.VariableNames) - DataTable.(colCaller) = repmat("", numRowsTable, 1); - end - - % 3. Get current time as a string - currentTimeStr = string(datetime('now', 'Format', 'yyyy-MM-dd HH:mm:ss')); - - % 4. Determine the calling script using the call stack + DataTable.logNappGenerateSpectraRange(processIdx,:) = [tmp_4 tmp_5]; + DataTable.logNappGenerateSpectraMode(processIdx) = string(p.Results.mode); + DataTable.logNappGenerateSpectraDate(processIdx) = string(datetime('now', 'Format', 'yyyy-MM-dd HH:mm:ss')); + + % Determine the calling script using the call stack stackInfo = dbstack; if length(stackInfo) > 1 callerScript = string(stackInfo(2).file); % The script that called this function else callerScript = " "; % Called directly from the command window end - - % 5. Write the log data only to the rows being processed - DataTable.(colDate)(processIdx) = currentTimeStr; - DataTable.(colCaller)(processIdx) = callerScript; -end \ No newline at end of file + DataTable.logNappGenerateSpectraCaller(processIdx) = callerScript; +end diff --git a/beamtime/nappPlot.m b/beamtime/nappPlot.m index c2ce676..ac7231e 100644 --- a/beamtime/nappPlot.m +++ b/beamtime/nappPlot.m @@ -641,8 +641,11 @@ if contains(p.Results.tableVariable,'spectrum') p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}); hplot.LineWidth = lw; hXLabel = xlabel('Photon Energy eV'); - xlim([p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,1) p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,2)]); - + try + xlim([p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,1) p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,2)]); + catch + xlim([p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,2) p.Results.DataTable(p.Results.tableSelection,:).xRayEkinRange_eV(ii,1)]); + end end if contains(p.Results.tableVariable,'Norm') hYLabel = ylabel('Counts [arb. unit]'); diff --git a/import/nappExportH5.m b/import/nappExportH5.m new file mode 100644 index 0000000..bac3636 --- /dev/null +++ b/import/nappExportH5.m @@ -0,0 +1,268 @@ +function nappExportH5(T, h5FileName) +%NAPPEXPORTH5 Export a napp* data table -- plus its campaign/column +% metadata and the original raw data files -- into one self-contained +% HDF5 file. +% +% NAPPEXPORTH5(T, h5FileName) writes: +% +% / root attributes: campaign metadata +% (from T.Properties.CustomProperties) +% /columnMetadata//description, /unit +% one entry per documented column +% /rows/_/ one group per table row, e.g. +% /rows/20160404_PHOENIX0183183.ibw: +% name string dataset (also forces the +% group to exist before any attribute +% is written to it) +% attributes (dates, temperatures, +% pressures, sample/region names, ...) +% datasets (image, imageRoh, +% scientaEkin_eV, spectrum_*, ...) +% rawFile uint8 bytes of the original file, +% read from fullfile(folder,name) -- +% only written if that file is still +% found on disk +% +% Run this while the original SES/NEXAFS/Photos files are still at the +% folder/name recorded in T -- raw-file embedding can't recover files +% that have since moved or been archived elsewhere. +% +% IMPLEMENTATION NOTE: this uses MATLAB's low-level HDF5 interface +% (H5F/H5G/H5D/H5A/H5T/H5S/H5P), holding ONE file handle open for the +% entire export, rather than the top-level convenience functions +% (h5create/h5write/h5writeatt). An earlier version used the top-level +% functions and reproducibly hit "Too many open files" plus +% "H5FD_truncate driver truncate request failed" errors on tables of a +% few hundred rows: each top-level call reopens the whole file +% internally, and under repeated heavy use some of those reopen/close +% cycles failed to release their file handle, leaking one descriptor +% per row until the OS limit was hit. Holding a single handle open (the +% same pattern already used in this toolbox's own splitH5Scans) avoids +% the repeated open/close cycle entirely. +% +% The file is still built on local disk (in a temp location) and only +% moved to h5FileName once fully written, so a failed run leaves your +% previous, valid h5FileName untouched. + +SMALL_LIMIT = 10; % numel() at/under this -> attribute; above -> dataset + +if isfile(h5FileName) + warning('nappExportH5:fileMayBeOpen', ... + ['If "%s" is currently open in HDFView or another program, close it now -- ' ... + 'the final step of this export will fail if the file is locked.'], h5FileName); +end + +workFileName = [tempname '.h5']; % always a local path, regardless of + % where h5FileName itself points + +fid = H5F.create(workFileName, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT'); + +try + % Link creation property list that auto-creates missing intermediate + % groups (e.g. writing '/rows/X/name' creates both '/rows' and + % '/rows/X' automatically) -- the low-level equivalent of what + % h5create did for us for free. + lcpl = H5P.create('H5P_LINK_CREATE'); + H5P.set_create_intermediate_group(lcpl, true); + acpl = H5P.create('H5P_ATTRIBUTE_CREATE'); + + writeStringDataset(fid, lcpl, '/fileFormatVersion', '1.0'); + + %% Campaign metadata -> root attributes + campaignFields = {'campaignName','projectName','contact','sourceDirectory','startDate','endDate','dataLevel'}; + for f = campaignFields + fname = f{1}; + if isprop(T.Properties.CustomProperties, fname) + val = T.Properties.CustomProperties.(fname); + if isnumeric(val) + writeNumericAttribute(fid, '/', acpl, fname, val); + else + writeStringAttribute(fid, '/', acpl, fname, char(string(val))); + end + end + end + + %% Column documentation -> one small group per documented column + varNames = T.Properties.VariableNames; + descriptions = T.Properties.VariableDescriptions; + units = T.Properties.VariableUnits; + for k = 1:numel(varNames) + hasDesc = ~isempty(descriptions) && ~isempty(descriptions{k}); + hasUnit = ~isempty(units) && ~isempty(units{k}); + if hasDesc || hasUnit + base = sprintf('/columnMetadata/%s', varNames{k}); + descStr = ''; unitStr = ''; + if hasDesc, descStr = descriptions{k}; end + if hasUnit, unitStr = units{k}; end + writeStringDataset(fid, lcpl, [base '/description'], descStr); + writeStringDataset(fid, lcpl, [base '/unit'], unitStr); + end + end + + %% One group per row, named after each file's last-modified date + filename + nRows = height(T); + usedNames = containers.Map('KeyType','char','ValueType','double'); + + for i = 1:nRows + dateStr = datestr(T.lastModifiedDatenum(i), 'yyyymmddHHMM'); + baseName = sprintf('%s_%s', dateStr, T.name{i}); + if isKey(usedNames, baseName) + usedNames(baseName) = usedNames(baseName) + 1; + rowName = sprintf('%s_dup%d', baseName, usedNames(baseName)); + else + usedNames(baseName) = 0; + rowName = baseName; + end + rowGroup = sprintf('/rows/%s', rowName); + + % Guaranteed first write: forces the group to exist before any + % attribute below targets it. + writeStringDataset(fid, lcpl, [rowGroup '/name'], T.name{i}); + + % Explicit row index -- HDF5's group listing order is not + % guaranteed to match table row order, so this is the reliable + % way to recover which table row a given /rows/... group came + % from, regardless of how it's listed or iterated later. + writeNumericAttribute(fid, rowGroup, acpl, 'rowIndex', i); + + for k = 1:numel(varNames) + vname = varNames{k}; + if strcmp(vname,'name'), continue, end % already written above + + val = T.(vname)(i,:); + if iscell(val) && isscalar(val) + val = val{1}; + end + if isempty(val) || (isnumeric(val) && all(isnan(val(:)))) + continue + end + + try + if ischar(val) || (isstring(val) && isscalar(val)) + writeStringAttribute(fid, rowGroup, acpl, vname, char(val)); + elseif (isnumeric(val) || islogical(val)) && numel(val) <= SMALL_LIMIT + writeNumericAttribute(fid, rowGroup, acpl, vname, val); + elseif isnumeric(val) + writeNumericDataset(fid, lcpl, [rowGroup '/' vname], val); + end + % anything else (categorical, datetime, unexpected types) is + % silently skipped rather than failing the whole export + catch ME + fprintf(' row %d, column %s: skipped (%s)\n', i, vname, ME.message); + end + end + + % Embed the original raw file, if it's still where FileList says it is + srcPath = fullfile(T.folder{i}, T.name{i}); + if isfile(srcPath) + rfid = fopen(srcPath, 'r'); + bytes = fread(rfid, Inf, '*uint8'); + fclose(rfid); + try + writeUint8Dataset(fid, lcpl, [rowGroup '/rawFile'], bytes); + catch ME + fprintf(' row %d: raw file embed skipped (%s)\n', i, ME.message); + end + end + end + + H5P.close(lcpl); + H5P.close(acpl); +catch ME + H5F.close(fid); % always release the file handle, even on error + rethrow(ME); +end + +H5F.close(fid); + +%% Move the finished file into place, replacing any previous version +try + if isfile(h5FileName) + delete(h5FileName); + end + movefile(workFileName, h5FileName); +catch ME + warning('nappExportH5:replaceFailed', ... + ['Could not replace "%s" -- it is likely still open in another program ' ... + '(e.g. HDFView). Close it, then move the finished file manually from:\n %s\n' ... + 'Original error: %s'], h5FileName, workFileName, ME.message); +end +end + + +%% ------------------------------------------------------------------ +% Low-level HDF5 write helpers (all assume fid is already open) +% ------------------------------------------------------------------ + +function writeStringDataset(fid, lcpl, path, valueStr) +strType = H5T.copy('H5T_C_S1'); +H5T.set_size(strType, 'H5T_VARIABLE'); +space = H5S.create_simple(1, 1, []); +did = H5D.create(fid, path, strType, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT'); +H5D.write(did, strType, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {char(valueStr)}); +H5D.close(did); +H5S.close(space); +H5T.close(strType); +end + +function writeNumericDataset(fid, lcpl, path, data) +data = double(data); +typeId = H5T.copy('H5T_NATIVE_DOUBLE'); +if isvector(data) || isscalar(data) + space = H5S.create_simple(1, numel(data), []); + did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT'); + H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data(:)); +else + h5dims = fliplr(size(data)); % HDF5 is C-order, MATLAB is Fortran-order + space = H5S.create_simple(numel(h5dims), h5dims, []); + did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT'); + H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data); +end +H5D.close(did); +H5S.close(space); +H5T.close(typeId); +end + +function writeUint8Dataset(fid, lcpl, path, bytes) +typeId = H5T.copy('H5T_NATIVE_UINT8'); +space = H5S.create_simple(1, numel(bytes), []); +did = H5D.create(fid, path, typeId, space, lcpl, 'H5P_DEFAULT', 'H5P_DEFAULT'); +H5D.write(did, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', bytes); +H5D.close(did); +H5S.close(space); +H5T.close(typeId); +end + +function writeStringAttribute(fid, groupPath, acpl, name, valueStr) +gid = H5G.open(fid, groupPath); +strType = H5T.copy('H5T_C_S1'); +H5T.set_size(strType, 'H5T_VARIABLE'); +space = H5S.create('H5S_SCALAR'); +aid = H5A.create(gid, name, strType, space, acpl); +H5A.write(aid, strType, {char(valueStr)}); +H5A.close(aid); +H5S.close(space); +H5T.close(strType); +H5G.close(gid); +end + +function writeNumericAttribute(fid, groupPath, acpl, name, data) +gid = H5G.open(fid, groupPath); +data = double(data); +typeId = H5T.copy('H5T_NATIVE_DOUBLE'); +if isscalar(data) + space = H5S.create('H5S_SCALAR'); +else + space = H5S.create_simple(1, numel(data), []); +end +aid = H5A.create(gid, name, typeId, space, acpl); +if isscalar(data) + H5A.write(aid, 'H5ML_DEFAULT', data); +else + H5A.write(aid, 'H5ML_DEFAULT', data(:)); +end +H5A.close(aid); +H5S.close(space); +H5T.close(typeId); +H5G.close(gid); +end diff --git a/openBIS/README.md b/openBIS/README.md new file mode 100644 index 0000000..88680b5 --- /dev/null +++ b/openBIS/README.md @@ -0,0 +1,157 @@ +# nappMATLAB → openBIS Sync + +Syncs beamtime metadata from a MATLAB `FileList` table into openBIS as +ELN-LIMS entries — including raw file attachments and embedded plot/photo/ +video thumbnails. + +**Requires the MATLAB Napp Toolbox** (`nappFileList`, `nappGenerateSpectra`, +`nappPlot`, etc.) to already be set up and to have produced a populated +`FileList` table, with campaign metadata set via +`FileList.Properties.CustomProperties`. This repo picks up from there — it +doesn't cover importing raw beamtime files, generating spectra, HDF5 export, +or the HTML report; those are generic toolbox functionality, not specific +to openBIS. + +## Files + +| File | Purpose | +|---|---| +| `nappSyncToOpenbis.py` | Does the actual openBIS sync via [pybis](https://pypi.org/project/PyBIS/): matching/creating/updating objects, setting properties, linking a parent, uploading raw files, embedding thumbnails. Called from MATLAB — never run standalone. | +| `callOpenbisSync.m` | The script you actually run: connects, optionally filters rows, generates plot/photo/video thumbnails, and calls into `nappSyncToOpenbis.py` to sync. | +| `testpybis.m` | Sandbox for testing connection/logic changes on a small row slice before running `callOpenbisSync.m` on real data. Also holds one-off utilities (e.g. adding a vocabulary term). | + +`callOpenbisSync.m` and `testpybis.m` both depend on `nappSyncToOpenbis.py` +being importable — they're two different entry points into the same engine, +not alternatives to each other. + +## Requirements + +- MATLAB (R2025b or later; **R2025b supports Python 3.9–3.12 only** — Python + 3.13 needs R2026a or later. Check + [MathWorks' compatibility table](https://www.mathworks.com/support/requirements/python-compatibility.html) + before choosing a Python version.) +- A Python environment **separate from your system/Xcode Python**, with + [`pybis`](https://pypi.org/project/PyBIS/) installed. A dedicated venv is + strongly recommended — installing packages into a Homebrew-managed Python + directly will hit macOS's "externally managed environment" guard (PEP 668). +- An openBIS instance with: + - A Space, and (optionally) a parent object representing the beamtime as a + whole, that per-file entries link to as a parent. + - An object type matching (or adapted from) the property mappings in + `nappSyncToOpenbis.py`'s configuration section (see below). + - A `RAW_DATA`-type dataset type for attaching original files. + +### openBIS object type setup + +This code does not create an object type for you — an admin needs to +define one (e.g. an "Experimental Step" type) in openBIS first, with a +property for every entry referenced in `nappSyncToOpenbis.py`'s +configuration section (`DIRECT_MAP`, `NUMERIC_MAP`, `TIMESTAMP_MAP`, +`TEXT_JOIN_MAP`, `CONSTANT_MAP`, `CAMPAIGN_MAP`, plus `FILENUMBER_PROPERTY`, +`CATEGORY_PROPERTY`, `PHOTON_ENERGY_PROPERTY`, `KE_RANGE_PROPERTY`, +`RESULTS_PROPERTY`), and each property's **data type must match** what the +script expects to write: `REAL`/`INTEGER` for anything in `NUMERIC_MAP` +(per its `"float"`/`"int"` tag), `TIMESTAMP` for `TIMESTAMP_MAP` entries, +`CONTROLLEDVOCABULARY` (with matching terms, e.g. `XPS`/`NEXAFS`/`PHOTO`) +for the category property, and `VARCHAR`/`MULTILINE_VARCHAR` for +everything else. A mismatch here isn't silent — openBIS returns a clear +validation error naming the property and the type it expected — but it's +still much faster to get the schema right up front than to debug it +property-by-property. `get_property_assignments()` (already in +`nappSyncToOpenbis.py`) is a read-only way to dump an *existing* type's +actual current structure, useful as ground truth when setting up a new one +or checking one for drift. + +## Setup + +```bash +python3 -m venv ~/matlab-pybis-venv +source ~/matlab-pybis-venv/bin/activate +pip install --upgrade pip +pip install pybis +``` + +In MATLAB, every session: + +```matlab +pyenv('Version', '/path/to/matlab-pybis-venv/bin/python3') +py.sys.path().append('/path/to/folder/containing/nappSyncToOpenbis.py'); +py.importlib.import_module('nappSyncToOpenbis'); +``` + +`pyenv` and the Python module import don't persist across a MATLAB restart — +both need re-running at the start of every session. + +### Authentication + +Personal Access Tokens (PATs) can be created and listed via the API, but +**PAT-based authentication may be disabled at the server-config level** +independent of that — this is a system-admin setting +([confirmed by openBIS developers](https://community.openbis.ch/t/personal-access-tokens-disabled/134)), +not something fixable from a script. If PATs fail with "user is not logged +in" despite a valid, unexpired token, ask your openBIS system admin whether +PAT authentication is enabled for your instance. Username/password +(`o.login(...)`) is unaffected by this and works regardless. + +## Usage + +With a populated `FileList` already in your MATLAB workspace (via the Napp +Toolbox): fill in connection details at the top of `callOpenbisSync.m` +(URL, space, object type, parent identifier), set `rowFilter` if you only +want a subset, and run it. Test changes on a small slice via `testpybis.m` +first, rather than editing `nappSyncToOpenbis.py` and running the full +sync straight away. + +## Known limitations + +- Video thumbnails always use the **first frame** of the file — not + necessarily the most representative one. Change the frame read in + `callOpenbisSync.m`'s thumbnail loop if that matters for your data. +- Re-running the sync on a row that already has embedded images in its + Results field will append the images again (no de-duplication). +- `VideoReader`'s format support depends on codecs available on the machine + running MATLAB. + +## pybis gotchas this code works around + +Found by reading pybis 1.37.5's own source directly rather than guessing — +worth checking whether these still apply if you upgrade pybis: + +- **Property codes are lowercase at the wire level**, even though the + admin UI and xlsx schema exports display them uppercase. +- **`get_parents()` is buggy**: it populates the internal parent cache as a + flat list of plain identifier strings, a different shape than + `add_parents()`/`set_parents()` produce. Merely *calling* it corrupts + state that `save()` later can't handle — use `add_parents()` directly, + without a preceding `get_parents()` check. +- **Iterating a `Things` collection can be buggy** (e.g. from `get_terms()`) + — it internally calls a single-item fetch method missing a required + argument. Read `.df` (the underlying DataFrame) directly instead. +- **`set_token()`/`is_session_active()` both call the `isSessionActive` RPC**, + which can reject a token that's genuinely valid per other endpoints (see + Authentication above). Writing directly to `o.__dict__["token"]` bypasses + the check without affecting anything else. +- Passing `collection=None` explicitly (rather than omitting the kwarg) + causes a malformed `"experimentId": {}` in the create request, which the + server rejects. Only include `collection` in the kwargs dict when it has + a real value. +- Values crossing from MATLAB via `pyargs(...)` (not through + `jsonencode`/`json.loads`) don't always arrive as genuine native Python + types — a bare string can end up wrapped in a way that breaks pybis's own + internal type checks, and MATLAB's `[]` can cross as a NumPy array whose + truthiness is ambiguous. Anything simple crossing this way gets coerced + explicitly (see `_clean_str_or_none` in `nappSyncToOpenbis.py`). +- `jsonencode(table2struct(T))` for a **single-row** table produces a JSON + object, not a one-element array (MATLAB's `table2struct` gives a scalar + struct, not a 1-element struct array, for one row). `sync_rows` normalizes + this on the way in, but it's worth knowing if you ever build the JSON + payload a different way. + +## Adapting for a different openBIS object type + +`nappSyncToOpenbis.py` is split into a **CONFIGURATION** section (plain +dicts: column → property mappings, fixed values, text-join rules) and an +**ENGINE** section (generic logic that reads that config). To point this at +a different object type, replace the configuration section only — the +engine, including all the pybis workarounds above, should keep working +unmodified. diff --git a/openBIS/callOpenbisSync.m b/openBIS/callOpenbisSync.m new file mode 100644 index 0000000..23e0461 --- /dev/null +++ b/openBIS/callOpenbisSync.m @@ -0,0 +1,163 @@ +%% Call nappSyncToOpenbis.py from MATLAB -- real runs, full FileList +% For quick connectivity/logic testing on a small slice, use testpybis.m +% instead. This file is for actually syncing your data. +% +% Prerequisite: pyenv points at a Python with pybis installed. +% PAT/token auth is disabled on this openBIS instance (confirmed) -- +% username/password is the only working auth method for now. + +%% Step 1: point at the venv (always needed fresh after a MATLAB restart) +pyenv('Version', '/Users/bartels-rausch/matlab-pybis-venv/bin/python3') + +%% Step 2: import the sync module +py.sys.path().append('/Users/bartels-rausch'); +py.importlib.import_module('nappSyncToOpenbis'); + +%% Step 3: connection details +openbisUrl = "https://openbis-psi.labnotebook.ch"; +spaceCode = "5505_BEAMTIMES"; +collectionId = []; % <-- fill in if you want these under a specific + % Experiment/Collection, e.g. '/5505_BEAMTIMES/PROJECT/COLLECTION' +objectType = "LAC_EXPERIMENTAL_STEP_BEAMTIME"; +parentIdentifier = "/5505_BEAMTIMES/ENTRY783"; +verifyCertificates = false; + +username = "bartels-rausch_admin"; +password = input('openBIS password: ', 's'); +clc + +%% Row filter -- work on a subset of FileList, or leave empty for everything +% Any valid MATLAB row index/logical works, e.g.: +% rowFilter = 1:20; % first 20 rows +% rowFilter = [5 12 47]; % specific rows +% rowFilter = FileList.lastModifiedDatenum > datenum('01-May-2016'); % date-based +rowFilter = find(startsWith(FileList.name,"A")); % <-- leave empty ([]) to process the whole table + +if isempty(rowFilter) + tmpWorkTable = FileList; +else + tmpWorkTable = FileList(rowFilter,:); +end +fprintf('Processing %d of %d rows\n', height(tmpWorkTable), height(FileList)); + +%% Generate plot images for each XPS/NEXAFS row (same plots as the HTML +% report), saved to a temp folder just for this openBIS upload -- deleted +% at the end regardless of success or failure. Photo rows are skipped here +% since the original photo already uploads as its own raw-data file. +% +% This can be slow for a large selection (two plots per XPS/NEXAFS row). +THUMB_WIDTH_PX = 600; % target width for photo/video thumbnails (height + % scales to preserve aspect ratio) -- adjust here + % if you want a different size + +tmpPngDir = tempname; +mkdir(tmpPngDir); + +photoExts = {'.bmp','.jpg','.jpeg','.png'}; +videoExts = {'.mov','.avi','.mp4','.wmv'}; + +nRows = height(tmpWorkTable); +resultImagePaths = cell(nRows,1); % {} for rows with no plot, else {path1,path2} + +for ii = 1:nRows + tmpRow = tmpWorkTable(ii,:); + [~,~,tmpExt] = fileparts(tmpRow.name{1}); + tmpExt = lower(tmpExt); + tmpSrcPath = fullfile(tmpRow.folder{1}, tmpRow.name{1}); + + if ismember(tmpExt, {'.ibw','.h5'}) + try + hdlImage = nappPlot(tmpRow,"mode","OneEach","figureMode","screen","tableVariable","image"); + if strcmp(tmpExt, '.h5') + hdlSpectrum = nappPlot(tmpRow,"mode","OneEach","figureMode","screen", ... + "tableVariable","spectrum_counts","energyScale","xRay"); + else + hdlSpectrum = nappPlot(tmpRow,"mode","OneEach","figureMode","screen", ... + "tableVariable","spectrum_countsPerSecond"); + end + + imgPath = fullfile(tmpPngDir, sprintf('%04d_image.png', ii)); + specPath = fullfile(tmpPngDir, sprintf('%04d_spectrum.png', ii)); + exportgraphics(hdlImage, imgPath); + exportgraphics(hdlSpectrum, specPath); + resultImagePaths{ii} = {imgPath, specPath}; + + close(hdlImage); + close(hdlSpectrum); + catch ME + fprintf(' row %d: could not generate plot images (%s)\n', ii, ME.message); + end + + elseif ismember(tmpExt, photoExts) + try + img = imread(tmpSrcPath); + imgSmall = imresize(img, [NaN THUMB_WIDTH_PX]); + thumbPath = fullfile(tmpPngDir, sprintf('%04d_thumb.png', ii)); + imwrite(imgSmall, thumbPath, 'png'); + resultImagePaths{ii} = {thumbPath}; + catch ME + fprintf(' row %d: could not generate photo thumbnail (%s)\n', ii, ME.message); + end + + elseif ismember(tmpExt, videoExts) + try + vidObj = VideoReader(tmpSrcPath); %#ok + frame = readFrame(vidObj); % first frame as a stand-in thumbnail + frameSmall = imresize(frame, [NaN THUMB_WIDTH_PX]); + thumbPath = fullfile(tmpPngDir, sprintf('%04d_thumb.png', ii)); + imwrite(frameSmall, thumbPath, 'png'); + resultImagePaths{ii} = {thumbPath}; + catch ME + fprintf(' row %d: could not generate video thumbnail (%s)\n', ii, ME.message); + end + end + close all hidden +end + +%% Select the columns needed: direct mappings + everything the derived +% fields (file number, category, photon/KE energy ranges) need to compute from +metaCols = {'name','folder','creationDate','lastModifiedDatenum','sample', ... + 'sampleTemp_dC','cellPressure_mbar','iceTemp_dC','comment','regionName', ... + 'scientaLensMode','scientaAcquisitionMode','scientaDwellTime_ms', ... + 'scientaPassEnergy_eV','scientaSequenceIterations','scientaRegionIterations', ... + 'scientaEkinStep_eV','xRayEkin_eV','xRayEkinRange_eV','scientaEkinRange_eV', ... + 'smplX_mm','smplY_mm','smplZ_mm'}; +metaCols = intersect(metaCols, tmpWorkTable.Properties.VariableNames, 'stable'); + +tmpSyncTable = tmpWorkTable(:, metaCols); +tmpSyncTable.resultImages = resultImagePaths; + +rowsJson = jsonencode(table2struct(tmpSyncTable)); + +%% Campaign-level properties, applied once to every row (never overwriting +% a value already present on an existing entry) +campaignProps = struct( ... + 'campaignLocation', tmpWorkTable.Properties.CustomProperties.campaignLocation, ... + 'team', tmpWorkTable.Properties.CustomProperties.team, ... + 'projectName', tmpWorkTable.Properties.CustomProperties.projectName); +campaignPropsJson = jsonencode(campaignProps); + +%% Sync the selected rows +try + resultJson = py.nappSyncToOpenbis.sync_rows(rowsJson, openbisUrl, spaceCode, ... + pyargs('collection', collectionId, 'object_type', objectType, ... + 'username', username, 'password', password, ... + 'campaign_props_json', campaignPropsJson, 'upload_files', true, ... + 'parent_identifier', parentIdentifier, ... + 'verify_certificates', verifyCertificates)); +catch ME + rmdir(tmpPngDir, 's'); + rethrow(ME); +end +rmdir(tmpPngDir, 's'); + +results = struct2table(jsondecode(char(resultJson))); +disp(results) + +fprintf('Created: %d, Updated: %d, Errors: %d\n', ... + sum(strcmp(results.action,'created')), ... + sum(strcmp(results.action,'updated')), ... + sum(contains(results.action,'error'))); +if any(contains(results.action,'error')) + disp(results(contains(results.action,'error'), :)) +end diff --git a/openBIS/importData.m b/openBIS/importData.m new file mode 100644 index 0000000..e9235cf --- /dev/null +++ b/openBIS/importData.m @@ -0,0 +1,161 @@ +%% BeamTime2016PhoenixIceNapp + +% add ../Functions to path (works on Windows and macOS) +funcDir = fullfile(fileparts(mfilename('fullpath')), '..', 'Functions'); +funcDir = char(java.io.File(funcDir).getCanonicalPath); % resolves .. and symlinks cross-platform +if isfolder(funcDir) + addpath(funcDir); +else + warning('Functions directory not found: %s', funcDir); +end + + +clearvars +close all +clc +%% Import and Generate Spectra +% @DEP_IN: BeamTime2016PhoenixIceNapp +FileList = nappFileList('BeamTime2016PhoenixIceNapp','mode','local'); +FileList = nappGenerateSpectra(FileList); + + +%% Add photon energy and sample type + + +idxSpotChange1 = find(matches(FileList.name,"PHOENIX0207207.ibw")); +idxSpotChange2 = find(matches(FileList.name,"PHOENIX0225225.ibw")); +idxSpotChange3 = find(matches(FileList.name,"PHOENIX0229229.ibw")); +idxSpotChange4 = find(matches(FileList.name,"PHOENIX0235235.ibw")); +idxSpotChange5 = find(matches(FileList.name,"PHOENIX0235235.ibw")); +idxSpotChange6 = find(matches(FileList.name,"PHOENIX0238238.ibw")); + +isIBW = endsWith(FileList.name,'.ibw'); +k = find(isIBW); +for ii = 1:length(k) + FileList(k(ii),:).xRayEkin_eV{:} = 2200; +end + +for ii = 1:idxSpotChange1 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#1 (0/0/6)'; +end +for ii = idxSpotChange1+1:idxSpotChange2 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#2 (0/0.6/6)'; +end +for ii = idxSpotChange2+1:idxSpotChange3 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#3 (-1/0.6/6.6)'; +end +for ii = idxSpotChange3+1:idxSpotChange4 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#4 (0/0.6/6.1)'; +end +for ii = idxSpotChange4+1:idxSpotChange5 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#5 (0.5/1.7/5.8)'; +end +for ii = idxSpotChange5+1:idxSpotChange6 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#6 (0/0.9/6)'; +end +for ii = idxSpotChange6+1:308 + FileList(ii,:).sample{:} = 'NaCl; H2O gas, Spot#6 (-1/0.4/6.4)'; +end +%% Add meta-data (keywords) +propNames = ["campaignName","campaignLocation","campaignInstrument","projectName","contact","sourceDirectory","startDate","endDate","dataLevel","publicationDoi", "funding", "contactORCID", "publicationDataDoi", "rawDataDoi", "team"]; +try + FileList = addprop(FileList, propNames, repmat("table",1,numel(propNames))); +catch + % properties already exist from a previous run -- fine, just update values below +end +FileList.Properties.CustomProperties.campaignName = "Beamtime Phoenix Spring 2016"; +FileList.Properties.CustomProperties.campaignLocation = "SLS/Phoenix"; +FileList.Properties.CustomProperties.campaignInstrument = "NappEndstation"; +FileList.Properties.CustomProperties.projectName = "NappIce"; +FileList.Properties.CustomProperties.contact = "thorsten.bartels-rausch@psi.ch"; +FileList.Properties.CustomProperties.contactORCID = "0000-0002-7548-2572"; +FileList.Properties.CustomProperties.team = "Thorsten Bartels-Rausch, Xiangrui Kong, Fabrizio Orlando, Luca Artiglia, Astrid Waldner,Thomas Huthwelker,Markus Ammann"; +FileList.Properties.CustomProperties.sourceDirectory = FileList.folder{1}; +FileList.Properties.CustomProperties.startDate = FileList(1,:).lastModifiedDatestr{:}; +FileList.Properties.CustomProperties.endDate = FileList(end,:).lastModifiedDatestr{:}; +FileList.Properties.CustomProperties.dataLevel = 0; +FileList.Properties.CustomProperties.publicationDoi = "10.5194/tc-15-2001-2021"; +FileList.Properties.CustomProperties.publicationDataDoi = "10.16904/envidat.164"; +FileList.Properties.CustomProperties.rawDataDoi = " "; +FileList.Properties.CustomProperties.funding = "SNF 149629"; + + +% Adds human-readable column descriptions and units to the standard table +% properties Properties.VariableDescriptions / Properties.VariableUnits + +meta.name = struct('desc','File name of the imported data or photo file.', 'unit',''); +meta.folder = struct('desc','Full path to the folder containing the file.', 'unit',''); +meta.creationDate = struct('desc','Acquisition timestamp: from instrument metadata for XPS/NEXAFS files, or OS file creation time for photos. Stored as a datevec [Y M D H MI S].', 'unit',''); +meta.lastModifiedDatenum = struct('desc','OS file modification time, as a MATLAB datenum.', 'unit',''); +meta.lastModifiedDatestr = struct('desc','OS file modification time, as a formatted date string.', 'unit',''); +meta.image = struct('desc','2D spectral map (kinetic energy vs. iteration/scan), normalized to counts per second.', 'unit','counts/s'); +meta.imageRoh = struct('desc','Raw copy of the 2D spectral map.', 'unit','counts/s'); +meta.xRayEkin_eV = struct('desc','X-ray excitation (photon) energy: the scan axis for NEXAFS, or the single excitation energy used for an XPS region.', 'unit','eV'); +meta.xRayEkinRange_eV = struct('desc','[min max] range of the X-ray excitation energy scan.', 'unit','eV'); +meta.scientaPassEnergy_eV = struct('desc','Analyzer pass energy setting.', 'unit','eV'); +meta.scientaDwellTime_ms = struct('desc','Aquisition (step) time per data point.', 'unit','ms'); +meta.regionName = struct('desc','Name of the measured region, e.g. an XPS core level (''C1s''), a survey scan, or a NEXAFS edge.', 'unit',''); +meta.scientaAcquisitionMode = struct('desc','Analyzer acquisition mode: ''Swept'' (scanning) or ''Fixed'' (fixed energy window).', 'unit',''); +meta.scientaEkinRange_eV = struct('desc','[min max] kinetic energy range of the analyzer scan.', 'unit','eV'); +meta.scientaEkinStep_eV = struct('desc','Kinetic energy step size between data points.', 'unit','eV'); +meta.scientaEkin_eV = struct('desc','The kinetic energy axis of the spectrum.', 'unit','eV'); +meta.scientaEkinRoh_eV = struct('desc','Raw copy of the kinetic energy axis of the spectrum, kept alongside scientaEkin_eV in case later processing modifies the data.', 'unit','eV'); +meta.beamlineInt = struct('desc','Beamline (incident photon) intensity, I0, used to normalize NEXAFS signal.', 'unit','a.u.'); +meta.scientaLensMode = struct('desc','Analyzer lens transmission mode.', 'unit',''); +meta.scientaAnalyzerSlit = struct('desc','Analyzer entrance slit setting.', 'unit',''); +meta.scientaWorkfunction_eV = struct('desc','Analyzer work function used in the kinetic-to-binding-energy conversion.', 'unit','eV'); +meta.scientaRegionIterations = struct('desc','Number of repeated scans within one region measurement, averaged together during import.', 'unit',''); +meta.scientaSequenceIterations = struct('desc','Number of sequence repeats.', 'unit',''); +meta.importDate = struct('desc','Timestamp of when this row was generated by nappFileList.', 'unit',''); +meta.analysisDir = struct('desc','Working directory (or calling script''s folder) at the time of import.', 'unit',''); +meta.sample = struct('desc','Sample name or description, from the SpecsLab comment field (.xy files) or set later from the lab journal.', 'unit',''); +meta.sampleTemp_dC = struct('desc','Sample (cryo-holder) temperature, from the LabView log.', 'unit','degC'); +meta.cellPressure_mbar = struct('desc','Sample cell pressure, from the LabView Baratron log.', 'unit','mbar'); +meta.iceTemp_dC = struct('desc','Ice equilibrium temperature, derived from cellPressure_mbar via the Marti & Mauersberger (1993) vapour-pressure parameterization.', 'unit','degC'); +meta.smplX_mm = struct('desc','Sample holder X position.', 'unit','mm'); +meta.smplY_mm = struct('desc','Sample holder Y position.', 'unit','mm'); +meta.smplZ_mm = struct('desc','Sample holder Z position.', 'unit','mm'); +meta.sealingTemp = struct('desc','Sealing temperature, from the LabView log.', 'unit','degC'); +meta.originalFileName = struct('desc','Original file name before a multi-region Specs .xy file was split into one row per region.', 'unit',''); +% added by generate spectra +meta.spectrum_countsPerSecond = struct('desc','1D spectrum for .ibw/.xy rows in ''mean''/''trapz'' mode: the mean (or trapz) of the image data -- already counts/s, since nappFileList normalizes by dwell time before storing image. Not populated for .h5/NEXAFS rows (their result goes to spectrum_counts instead).', 'unit','counts/s'); +meta.spectrum_counts = struct('desc','1D spectrum: for .h5/NEXAFS rows, the sum or trapezoidal integral of the image over the selected kinetic-energy range (not time-normalized), for any mode; for .ibw/.xy rows, only populated when mode=''sum'' (sum over the selected iteration range).', 'unit','counts'); +meta.logNappGenerateSpectraRange = struct('desc','[start end] range used to generate the spectrum: kinetic-energy limits for .h5 rows, or iteration indices for .ibw/.xy rows.', 'unit','eV (h5) or index (ibw/xy)'); +meta.logNappGenerateSpectraDate = struct('desc','Logs when the spectrum was generated', 'unit',''); +meta.logNappGenerateSpectraMode = struct('desc','Reduction mode used to generate the spectrum for this row (''mean'', ''sum'', ''trapz'', or ''AugerMeitnerMean'').', 'unit',''); +meta.logNappGenerateSpectraCaller = struct('desc','File path of the script or function that called nappGenerateSpectra, for provenance.', 'unit',''); + +varNames = FileList.Properties.VariableNames; +descriptions = repmat({''}, 1, numel(varNames)); +units = repmat({''}, 1, numel(varNames)); +for k = 1:numel(varNames) + if isfield(meta, varNames{k}) + descriptions{k} = meta.(varNames{k}).desc; + units{k} = meta.(varNames{k}).unit; + end +end +FileList.Properties.VariableDescriptions = descriptions; +FileList.Properties.VariableUnits = units; + + +%% Tidy up +FileList = removevars(FileList,{'smplX_mm','smplY_mm','smplZ_mm','scientaAnalyzerSlit','scientaWorkfunction_eV','sealingTemp'}); +FileList.comment = repmat({''}, height(FileList), 1); +clearvars -except FileList +%% Export and Save +ExportList = removevars(FileList, {'image', 'scientaEkin_eV', 'beamlineInt', 'imageRoh', 'scientaEkinRoh_eV', 'xRayEkin_eV', 'spectrum_countsPerSecond', 'spectrum_counts' }); +% @DEP_OUT: Apr2016_DataTable.csv +writetable(ExportList,fullfile([datestr(ExportList.lastModifiedDatenum(1),'mmmYYYY') '_DataList.csv']),'Delimiter',','); + +% write to command line +whos('FileList'); % basic size/type info +disp('--- Table Properties ---') +disp(FileList.Properties) % show table properties (Description, UserData, VariableDescriptions, etc.) +disp('--- Summary of Table ---') +summary(FileList) % concise summary of variables and types +%% +clearvars -except FileList +% @DEP_OUT: Apr2016_DataTable.mat +save("Apr2016_DataTable.mat") +% @DEP_OUT: Apr2016_DataTable.h5 +nappExportH5(FileList, "Apr2016_DataTable.h5"); \ No newline at end of file diff --git a/openBIS/nappSyncToOpenbis.py b/openBIS/nappSyncToOpenbis.py new file mode 100644 index 0000000..27ab265 --- /dev/null +++ b/openBIS/nappSyncToOpenbis.py @@ -0,0 +1,684 @@ +""" +nappSyncToOpenbis.py + +Sync napp* beamtime metadata rows into openBIS objects: update an existing +object if one already exists for a given file name, otherwise create a new +one. One object per table row. Fields already filled in openBIS (e.g. by +hand during the beamtime) are never overwritten by automated values. + +Called from MATLAB via: + py.importlib.import_module('nappSyncToOpenbis') + resultJson = py.nappSyncToOpenbis.sync_rows(rowsJson, url, space, ...) + +============================================================================ +HOW THIS FILE IS ORGANIZED +============================================================================ +Two sections: + + CONFIGURATION -- everything you're likely to want to change: which + FileList column goes to which openBIS property, fixed values, how text + fields get combined, etc. All plain data (dicts/lists/strings) -- no + logic. This is what you edit for: + - a new column -> property mapping + - changing which column(s) feed a text field (e.g. Notes, Sample Notes) + - a new fixed/constant value + - adapting this file for a DIFFERENT openBIS object type entirely: + replace this whole section, leave the engine below alone. + + ENGINE -- generic code that reads the configuration above and does the + actual work (connecting, matching existing objects, uploading files, + embedding images, etc.). You shouldn't need to touch this for ordinary + changes -- only if you need a genuinely new KIND of derivation that + doesn't fit the existing patterns (direct copy, numeric conversion, + timestamp, text-join, category-dependent source). + +Several things below are workarounds for confirmed pybis (v1.37.5) quirks, +found by reading pybis's own source directly rather than guessing -- see +the comments at each one for what was actually wrong and how it was +confirmed. These live in the engine, not the config, since they're not +something a schema change would ever need to touch. +""" + +import json +import re +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo +from pandas import DataFrame +from pybis import Openbis + +LOCAL_TZ = ZoneInfo("Europe/Zurich") # <-- change if a future beamtime runs elsewhere + + +# ============================================================================ +# CONFIGURATION -- edit this section for ordinary changes. See file docstring. +# ============================================================================ + +OBJECT_TYPE = "LAC_EXPERIMENTAL_STEP_BEAMTIME" +MEASUREMENT_TYPE_VOCABULARY = "LAC_EXPERIMENTALSTEPTYPE_BEALINE" +RAW_DATASET_TYPE = "RAW_DATA" +RESULTS_PROPERTY = "default_experiment.experimental_results" +FILENUMBER_PROPERTY = "filenumber" +CATEGORY_PROPERTY = "lac_steptype" +KE_RANGE_PROPERTY = "ke_range_center" +KE_RANGE_SOURCE_COLUMN = "scientaEkinRange_eV" + +# --- File category, derived from filename extension -- drives lac_steptype +# and which photon-energy source column is used below. Add new extensions +# / categories here if a future beamtime uses different file types. +CATEGORY_BY_EXTENSION = { + "ibw": "XPS", + "h5": "NEXAFS", + "bmp": "PHOTO", "jpg": "PHOTO", "jpeg": "PHOTO", "png": "PHOTO", + "wmv": "PHOTO", "mov": "PHOTO", +} + +# --- Direct column -> property mappings ------------------------------------ +# Simple 1:1 copies, no conversion needed (text properties, or numeric +# properties where the JSON value's native type already matches -- e.g. +# ke_step is REAL and a plain float is exactly right as-is). +# Property codes are lowercase -- confirmed openBIS wants that at the wire +# level regardless of what the admin UI/xlsx export displays. +DIRECT_MAP = { + "sample": "sample_name", + "regionName": "region", + "scientaLensMode": "lens_mode", + "scientaAcquisitionMode": "acq_mode", + "scientaEkinStep_eV": "ke_step", +} + +# --- Numeric columns needing a type conversion ----------------------------- +# value = (openBIS property code, "int" or "float"). +# "int" -> rounded to a genuine Python int (for INTEGER-typed properties; +# MATLAB/JSON numbers arrive as floats, e.g. 57.0, even for +# whole-number fields). +# "float" -> passed through as-is (REAL-typed properties -- JSON already +# gives a plain float, nothing to convert). +# Note: sequence_iterations_real / region_iterations_real are misleadingly +# named -- their CODEs say "_real" but their actual openBIS data type is +# INTEGER, confirmed from the schema export. Named per the actual type +# needed here, not per the (inconsistent) code name. +NUMERIC_MAP = { + "sampleTemp_dC": ("temp_reading", "float"), + "iceTemp_dC": ("temp_calibrated", "float"), + "cellPressure_mbar": ("cell_pressure_real", "float"), + "scientaDwellTime_ms": ("dwell_time_integer", "int"), + "scientaPassEnergy_eV": ("passenergy_integer", "int"), + "scientaSequenceIterations": ("sequence_iterations_real", "int"), + "scientaRegionIterations": ("region_iterations_real", "int"), + "smplX_mm": ("position_x_real", "float"), + "smplY_mm": ("position_y_real", "float"), + "smplZ_mm": ("position_z_real", "float"), +} + +# --- Timestamp columns ------------------------------------------------------ +# value = (openBIS property code, MATLAB source format). +# "datevec" = a 6-element [Y M D H MI S] list (e.g. creationDate) +# "datenum" = a MATLAB serial datenum (e.g. lastModifiedDatenum) +TIMESTAMP_MAP = { + "creationDate": ("start_date", "datevec"), + "lastModifiedDatenum": ("end_date", "datenum"), +} + +# --- Text fields built by joining one or more columns with newlines -------- +# key = target property code, value = list of source columns, in order. +# A column that's empty/missing for a given row is simply skipped; if NONE +# of a field's source columns have anything, the property is left unset +# entirely (never written as a blank string). +# This is exactly where to make a change like "comment should go to +# Notes, not Sample Notes" -- edit the list(s) below, nothing else. +TEXT_JOIN_MAP = { + "notes": ["comment"], +} + +# --- Photon energy: which column to use depends on the file category ------- +# XPS uses a single fixed excitation energy; NEXAFS genuinely scans a +# range. "scalar" reads one number (unwrapping a 1-element list if needed); +# "range" expects a 2-element [min, max] list. +PHOTON_ENERGY_PROPERTY = "photon_energy" +PHOTON_ENERGY_SOURCE_BY_CATEGORY = { + "XPS": ("xRayEkin_eV", "scalar"), + "NEXAFS": ("xRayEkinRange_eV", "range"), +} + +# --- Fixed/constant values, written once per sync (not derived from any +# row's own data) ------------------------------------------------------ +# Note: these are VOCABULARY TERM values, not property codes -- a +# different namespace, genuinely uppercase per the LAC_ENDSTATIONS +# vocabulary definition (confirmed from the schema export). +CONSTANT_MAP = { + "endstation_name": "SOLID-GAS", # <-- change if a different campaign/instrument uses this script +} + +# --- Campaign-level values, from FileList.Properties.CustomProperties, +# same value applied to every row this run ------------------------------ +CAMPAIGN_MAP = { + "beamline": "campaignLocation", + "team": "team", + "researchproject": "projectName", +} + +# Deliberately not represented anywhere in this openBIS object type +# (confirmed OK to leave unmapped): contact, contactORCID, funding, +# publicationDoi, publicationDataDoi, rawDataDoi, dataLevel, sourceDirectory, +# campaignName. + + +# ============================================================================ +# ENGINE -- generic logic reading the configuration above. See file docstring. +# ============================================================================ + +def _connect(url, token=None, username=None, password=None, verify_certificates=True): + o = Openbis(url, verify_certificates=verify_certificates) + if token: + # NOT o.set_token(token) -- that calls is_token_valid(), which hits + # openBIS's isSessionActive RPC. Confirmed (by reading pybis + # 1.37.5's source) that set_token()/the token property setter/ + # is_session_active() all route through that one same check, and + # confirmed separately that PAT authentication is disabled at the + # server level on this instance (community.openbis.ch/t/134) -- + # tokens are creatable/listable but rejected on real use. Writing + # directly to __dict__ bypasses the property setter (and that + # check) without affecting anything else -- every other pybis + # method reads self.token via the plain getter, no validation on + # read. Kept for whenever PAT auth gets enabled server-side. + o.__dict__["token"] = token + elif username and password: + o.login(username, password) + else: + raise ValueError("Provide either a token, or both username and password.") + return o + + +def _clean_str_or_none(value): + """Coerce a value that may have crossed the MATLAB<->Python boundary + (via pyargs, not JSON) into a clean native str, or None if it + represents 'nothing'. Deliberately avoids `if value:` -- MATLAB's [] + can arrive as a NumPy array, and NumPy raises its own error ("ambiguous + truth value") for exactly that kind of implicit boolean check, rather + than just being falsy the way None/'' are.""" + if value is None: + return None + if hasattr(value, "size"): # NumPy array or similar + return None if value.size == 0 else str(value) + text = str(value).strip() + return text if text else None + + +def _to_clean_int(value): + """Round a MATLAB-crossed numeric value (arrives as a JSON float, e.g. + 57.0) to a genuine Python int for an INTEGER-typed property. Returns + None if it can't be cleanly interpreted as a number.""" + if value is None: + return None + try: + return int(round(float(value))) + except (ValueError, TypeError): + return None + + +def _matlab_datenum_to_str(datenum): + """MATLAB datenum (days since 0-Jan-0000) -> 'YYYY-MM-DD HH:MM:SS +ZZZZ', + matching the format openBIS itself displays for these TIMESTAMP + properties (confirmed from a real exported entry). The -366 offset is + the standard MATLAB<->Python datenum conversion constant (Python's + ordinal day 1 = 1-Jan-0001). Localized to LOCAL_TZ so the UTC offset is + correct across the CET/CEST boundary.""" + if datenum is None: + return None + try: + days = int(datenum) + frac = datenum - days + dt = datetime.fromordinal(days - 366) + timedelta(days=frac) + return dt.replace(tzinfo=LOCAL_TZ).strftime("%Y-%m-%d %H:%M:%S %z") + except (ValueError, TypeError, OverflowError): + return None + + +def _matlab_datevec_to_str(datevec): + """[Y, M, D, H, MI, S] -> 'YYYY-MM-DD HH:MM:SS +ZZZZ' (see above).""" + if not isinstance(datevec, list) or len(datevec) < 6: + return None + try: + y, mo, d, h, mi, s = datevec[:6] + dt = datetime(int(y), int(mo), int(d), int(h), int(mi), int(s)) + return dt.replace(tzinfo=LOCAL_TZ).strftime("%Y-%m-%d %H:%M:%S %z") + except (ValueError, TypeError): + return None + + +def _file_number(name): + """Doubled-digit convention: 'PHOENIX0183183_1.ibw' -> 183 (the leading + digit run '0183183' splits into '0183' + '183', the same number with + and without a leading zero). For names that don't match that pattern + but have a later, short digit run (e.g. 'PHOENIX04042016_001.h5', where + the leading run is a date, not a real index), that trailing run is used + instead. Returns None if no clean number can be extracted.""" + if not name: + return None + digit_runs = re.findall(r"\d+", name) + if not digit_runs: + return None + + first = digit_runs[0] + n = len(first) + for split_at in range(-(-n // 2), n): # ceil(n/2) .. n-1 + a, b = first[:split_at], first[split_at:] + if a and b and int(a) == int(b): + try: + return int(a) + except ValueError: + return None + + if len(digit_runs) > 1 and len(digit_runs[-1]) <= 4: + try: + return int(digit_runs[-1]) + except ValueError: + pass + + try: + return int(first) + except ValueError: + return None # e.g. a date-based name too long/ambiguous to use as-is + + +def _category_for_name(name): + if not name or "." not in name: + return None + ext = name.lower().rsplit(".", 1)[-1] + return CATEGORY_BY_EXTENSION.get(ext) + + +def _row_derived_props(row, category): + """Everything that isn't a straight column copy: timestamps, file + number, category, photon/KE energy (which may depend on category), + and the text-join fields. Driven entirely by the CONFIGURATION section + above -- add a new derivation *kind* here only if none of the existing + patterns (direct/numeric/timestamp/text-join/category-dependent) fit.""" + props = {} + + for col, (prop, fmt) in TIMESTAMP_MAP.items(): + val = row.get(col) + formatted = _matlab_datevec_to_str(val) if fmt == "datevec" else _matlab_datenum_to_str(val) + if formatted: + props[prop] = formatted + + fnum = _file_number(row.get("name")) + if fnum is not None: + props[FILENUMBER_PROPERTY] = fnum + + if category is not None: + props[CATEGORY_PROPERTY] = category + + if category in PHOTON_ENERGY_SOURCE_BY_CATEGORY: + col, kind = PHOTON_ENERGY_SOURCE_BY_CATEGORY[category] + val = row.get(col) + if kind == "scalar": + if isinstance(val, list): + val = val[0] if len(val) > 0 else None + if isinstance(val, (int, float)): + props[PHOTON_ENERGY_PROPERTY] = f"{val:.4g} eV" + elif kind == "range": + if isinstance(val, list) and len(val) == 2 and all(v is not None for v in val): + props[PHOTON_ENERGY_PROPERTY] = f"{val[0]:.4g} - {val[1]:.4g} eV" + + ke_range = row.get(KE_RANGE_SOURCE_COLUMN) + if isinstance(ke_range, list) and len(ke_range) == 2 and all(v is not None for v in ke_range): + props[KE_RANGE_PROPERTY] = f"{ke_range[0]:.4g} - {ke_range[1]:.4g} eV" + + for prop, cols in TEXT_JOIN_MAP.items(): + parts = [str(row[c]) for c in cols if row.get(c)] + if parts: + props[prop] = "\n".join(parts) + + return props + + +def attach_result_image(o, obj, image_path, url, verify_certificates=True): + """Uploads image_path to openBIS's ELN file service and appends it to + RESULTS_PROPERTY, alongside any existing content there rather than + replacing it. + Confirmed working pattern from openBIS support: community.openbis.ch/t/391 + """ + import requests + + upload_url = f"{url}/openbis/openbis/file-service/eln-lims?type=Files&sessionID={o.token}" + with open(image_path, "rb") as f: + r = requests.post(upload_url, files={"file": f}, verify=verify_certificates) + r.raise_for_status() + file_url = json.loads(r.text)["url"] + + existing = _current_prop_safe(obj, RESULTS_PROPERTY) + figure_html = f'
' + obj.props[RESULTS_PROPERTY] = (existing or "") + figure_html + + +def _current_prop_safe(obj, prop_code): + try: + return obj.props[prop_code] + except Exception: + return None + + +def add_photo_vocabulary_term(url, token=None, username=None, password=None, + verify_certificates=True): + """One-time (or idempotent) setup: adds a PHOTO term to + MEASUREMENT_TYPE_VOCABULARY, so photo rows can set CATEGORY_PROPERTY. + Safe to call more than once -- checks for the term first. + Run this deliberately, not as part of routine syncing. + """ + o = _connect(url, token=token, username=username, password=password, + verify_certificates=verify_certificates) + try: + existing_terms = o.get_terms(vocabulary=MEASUREMENT_TYPE_VOCABULARY) + # Read the underlying DataFrame directly rather than iterating + # existing_terms -- iterating a Things collection internally calls + # a single-item fetch (get_term) that's missing a required arg in + # this pybis version (confirmed via a real traceback: TypeError: + # get_term() missing 'vocabularyCode'), the same class of bug as + # get_parents() below. .df sidesteps that entirely. + if existing_terms is not None and len(existing_terms) > 0: + df = existing_terms.df + code_col = "code" if "code" in df.columns else df.columns[0] + existing_codes = list(df[code_col]) + else: + existing_codes = [] + if "PHOTO" in existing_codes: + return json.dumps({"action": "already_exists"}) + + term = o.new_term( + code="PHOTO", + vocabularyCode=MEASUREMENT_TYPE_VOCABULARY, + label="Photo", + description="Camera photo of the sample or setup", + ) + term.save() + return json.dumps({"action": "created"}) + finally: + o.logout() + + +def get_property_assignments(url, object_type=None, token=None, username=None, + password=None, verify_certificates=True): + """Read-only: dumps the complete current property structure of an + object type -- code, data type, section, order, mandatory -- exactly + as openBIS has it right now. Use this as ground truth before designing + a new/modified object type, rather than working from memory of an old + export. Makes no changes.""" + o = _connect(url, token=token, username=username, password=password, + verify_certificates=verify_certificates) + object_type = object_type or OBJECT_TYPE + try: + sample_type = o.get_object_type(object_type) + pas = sample_type.get_property_assignments() + df = pas.df if hasattr(pas, "df") else DataFrame(pas) + cols = [c for c in ["code", "dataType", "section", "ordinal", "mandatory"] if c in df.columns] + df = df[cols].sort_values("ordinal") if "ordinal" in df.columns else df[cols] + return df.to_json(orient="records") + finally: + o.logout() + + +def property_usage_report(url, object_type=None, token=None, username=None, + password=None, verify_certificates=True): + """Read-only report: for every property on the given object type, + counts how many EXISTING entries have a non-empty value for it. Run + this BEFORE removing or changing a property in openBIS's schema, so + you know how much real data would actually be affected rather than + guessing. Makes no changes -- purely a search + count. + + The property list is read live via get_property_assignments() rather + than hardcoded, so this works unmodified for any object type -- part + of what makes this file reusable as a template. + + Returns a JSON string: {"total_entries": N, "property_fill_counts": + {"temp_reading": 12, "cell_pressure_real": 12, ...}} -- a property + absent from the dict (or at 0) currently has no data anywhere, so + removing it is safe; anything with a high count is exactly what you'd + lose.""" + o = _connect(url, token=token, username=username, password=password, + verify_certificates=verify_certificates) + object_type = object_type or OBJECT_TYPE + try: + sample_type = o.get_object_type(object_type) + pas = sample_type.get_property_assignments() + pa_df = pas.df if hasattr(pas, "df") else DataFrame(pas) + all_props = list(pa_df["code"]) if "code" in pa_df.columns else [] + + df = o.get_objects(type=object_type, props=all_props) + total = len(df) + counts = {} + for col in all_props: + if col not in df.columns: + continue + non_empty = df[col].apply( + lambda v: v is not None and v == v and v != "" # v==v excludes NaN + ).sum() + if non_empty > 0: + counts[col] = int(non_empty) + return json.dumps({"total_entries": int(total), "property_fill_counts": counts}) + finally: + o.logout() + + +def sync_rows(rows_json, url, space, collection=None, object_type=None, + campaign_props_json=None, parent_identifier=None, + upload_files=True, token=None, username=None, password=None, + verify_certificates=True): + """ + rows_json -- JSON string: array of objects, one per table row + (built in MATLAB via jsonencode(table2struct(T))). Each + row must include 'name' and 'folder' if upload_files=True, + and 'resultImages' (a list of local PNG paths, or an + empty list) if you want plots embedded in the Results + field. Columns absent from a given table (e.g. a + beamtime without smplX_mm/Y/Z) are simply skipped -- + nothing needs to exist in every row. + url -- openBIS instance URL + space -- openBIS space code to create/find objects in + collection -- openBIS experiment/collection identifier, + e.g. '/SPACE/PROJECT/COLLECTION' (optional) + object_type -- openBIS object type code (falls back to OBJECT_TYPE) + campaign_props_json -- JSON string: {"campaignLocation": "...", "team": + "...", "projectName": "..."} -- same value applied to + every row this run (falls back to skipping campaign + properties if not provided) + parent_identifier -- identifier (e.g. '/SPACE/BEAMTIME_CODE') of a + parent object representing the beamtime as a whole. + Applied to every row's object via add_parents() (see the + comment at that call for why not set_parents()/ + get_parents()). + upload_files -- if True, attaches the original raw file (from + fullfile(folder,name)) as a RAW_DATASET_TYPE dataset on + each object + token / username+password -- authentication; provide one or the other + + Update behaviour: an existing object is matched by its $name property + (not Code, which openBIS auto-generates for this type). Any property + that already has a non-empty value on an existing object is left + untouched -- automated sync only fills in blanks, never overwrites + something entered by hand (0/0.0 counts as a real value, not empty). + A property that can't be set (e.g. a vocabulary term that doesn't + exist yet) is skipped and reported in that row's message, without + preventing the rest of the row from saving. + + Returns a JSON string: one summary entry per row -- + [{"name": "...", "action": "created"|"updated"|"error", "message": "..."}] + """ + rows = json.loads(rows_json) + object_type = _clean_str_or_none(object_type) or OBJECT_TYPE + campaign_props = json.loads(campaign_props_json) if campaign_props_json else {} + + url = _clean_str_or_none(url) + space = _clean_str_or_none(space) + collection = _clean_str_or_none(collection) + parent_identifier = _clean_str_or_none(parent_identifier) + + o = _connect(url, token=token, username=username, password=password, + verify_certificates=verify_certificates) + results = [] + + # Resolve the parent identifier to an actual object once, up front, + # rather than handing pybis a bare string to resolve internally on + # every row. Confirmed via a real traceback that passing the string + # directly works fine at object CREATION time but breaks UPDATE + # (pybis/attribute.py's internal string->object resolution ends up + # leaving a raw string where save() later expects a fully-resolved + # identifier dict, causing "string indices must be integers, not + # 'str'"). Passing an already-resolved object sidesteps that path. + parent_obj = None + parent_warning = None + if parent_identifier: + try: + parent_obj = o.get_object(parent_identifier) + except Exception as e: + parent_warning = f"could not resolve parent_identifier '{parent_identifier}': {e}" + + def _is_empty(value): + """Deliberately not `not value` -- that would treat a genuine 0 or + 0.0 (a perfectly valid REAL/INTEGER reading, e.g. a position or + pressure) as 'empty' and overwrite it. Only None and '' count.""" + return value is None or value == "" + + def _current_value(obj, prop_code): + try: + return obj.props[prop_code] + except Exception: + return None + + def _set_if_empty(obj, prop_code, value, warnings): + """Sets a property only if currently empty. Any failure (e.g. an + unrecognized property code, or a vocabulary term that doesn't + exist yet) is recorded as a warning rather than raised, so it + can't prevent the rest of the row's properties from saving. + prop_code is force-lowercased -- confirmed (via a real openBIS + error listing every valid code) that property codes are lowercase + at the wire level even though the admin UI/xlsx export displays + them uppercase.""" + prop_code = prop_code.lower() + if _is_empty(value): + return + try: + if _is_empty(_current_value(obj, prop_code)): + obj.props[prop_code] = value + except Exception as e: + warnings.append(f"{prop_code}: {e}") + + try: + for row in rows: + name = row.get("name", "") + warnings = [] + if parent_warning: + warnings.append(parent_warning) + try: + category = _category_for_name(name) + + existing = o.get_objects(type=object_type, space=space, **{"$name": name}) + if len(existing) > 0: + obj = existing[0] + action = "updated" + else: + new_kwargs = dict(type=object_type, space=space, props={"$name": name}) + if collection: + # Only included when truthy: explicitly passing + # collection=None causes pybis to emit a malformed + # empty "experimentId": {} in the request (missing + # its required @type discriminator), which the + # server then rejects outright -- confirmed via a + # real server error. Omitting the key entirely + # avoids that path. + new_kwargs["collection"] = collection + if parent_obj: + new_kwargs["parents"] = [parent_obj] + obj = o.new_object(**new_kwargs) + action = "created" + + if parent_obj and action == "updated": + try: + # add_parents() (not set_parents(), and NOT + # preceded by a get_parents() check): get_parents() + # has a confirmed bug where it populates the + # internal _parents cache as a flat list of plain + # identifier strings (via a DataFrame + # .values.flatten()) -- a different shape than + # add_parents()/set_parents() produce. Merely + # calling it corrupts state that obj.save() later + # can't handle ("string indices must be integers, + # not 'str'"), even on rows that never touch + # parents at all this run. add_parents() sidesteps + # that path entirely, and unlike set_parents() it + # only adds our parent rather than replacing the + # whole list -- so it can't remove a different + # parent someone added by hand. + obj.add_parents([parent_obj]) + except Exception as e: + warnings.append(f"parent link: {e}") + + for col, prop in DIRECT_MAP.items(): + if col in row: + _set_if_empty(obj, prop, row[col], warnings) + + for col, (prop, kind) in NUMERIC_MAP.items(): + if col in row: + value = row[col] + if kind == "int": + value = _to_clean_int(value) + _set_if_empty(obj, prop, value, warnings) + + for prop, value in _row_derived_props(row, category).items(): + _set_if_empty(obj, prop, value, warnings) + + for prop, campaign_key in CAMPAIGN_MAP.items(): + _set_if_empty(obj, prop, campaign_props.get(campaign_key), warnings) + + for prop, value in CONSTANT_MAP.items(): + _set_if_empty(obj, prop, value, warnings) + + for img_path in (row.get("resultImages") or []): + try: + attach_result_image(o, obj, img_path, url, verify_certificates=verify_certificates) + except Exception as e: + warnings.append(f"result image ({img_path}): {e}") + + obj.save() + + if upload_files and row.get("folder"): + src_path = f"{row['folder']}/{name}" + try: + ds = o.new_dataset(type=RAW_DATASET_TYPE, sample=obj, files=[src_path]) + ds.save() + except Exception as e: + warnings.append(f"file upload: {e}") + + results.append({"name": name, "action": action, "message": "; ".join(warnings)}) + except Exception as e: + results.append({"name": name, "action": "error", "message": str(e)}) + finally: + o.logout() + + return json.dumps(results) + + +def check_connection(url, token=None, username=None, password=None, + verify_certificates=True): + """Quick standalone connectivity/auth check -- call this before sync_rows + on a full table, to confirm login/token works. + + Deliberately does NOT use is_session_active()/isSessionActive: that RPC + has been observed rejecting valid PATs on this openBIS instance even + though the same token works fine for real operations (get_spaces here, + and get_objects/new_object/etc. in sync_rows). Space listing is used + instead as a cheap, harmless, real call that actually proves the + token/login works end to end.""" + o = _connect(url, token=token, username=username, password=password, + verify_certificates=verify_certificates) + try: + spaces = o.get_spaces() + working = spaces is not None + except Exception as e: + return json.dumps({"connection_works": False, "message": str(e)}) + finally: + try: + o.logout() + except Exception: + pass # harmless either way; not the check we care about here + return json.dumps({"connection_works": working, "message": ""}) diff --git a/openBIS/testpybis.m b/openBIS/testpybis.m new file mode 100644 index 0000000..1619286 --- /dev/null +++ b/openBIS/testpybis.m @@ -0,0 +1,46 @@ +%% 1. Point at the venv (always needed fresh after restart) +pyenv('Version', '/Users/bartels-rausch/matlab-pybis-venv/bin/python3') + +%% 2. Import (first time this session -- guaranteed to read from disk) +py.sys.path().append('/Users/bartels-rausch'); +py.importlib.import_module('nappSyncToOpenbis'); + +%% 3. Your connection details +openbisUrl = "https://openbis-psi.labnotebook.ch"; +spaceCode = "5505_BEAMTIMES"; +collectionId = []; +objectType = "LAC_EXPERIMENTAL_STEP_BEAMTIME"; +parentIdentifier = "/5505_BEAMTIMES/ENTRY783"; +verifyCertificates = false; +username = "bartels-rausch_admin"; +password = input('openBIS password: ', 's'); +clc +%% 4. Rebuild the test data (also fresh, in case anything upstream changed) +testRows = FileList(11:12,:); +metaCols = {'name','folder','creationDate','lastModifiedDatenum','sample', ... + 'sampleTemp_dC','cellPressure_mbar','iceTemp_dC','comment','regionName', ... + 'scientaLensMode','scientaAcquisitionMode','scientaDwellTime_ms', ... + 'scientaPassEnergy_eV','scientaSequenceIterations','scientaRegionIterations', ... + 'scientaEkinStep_eV','xRayEkin_eV','xRayEkinRange_eV','scientaEkinRange_eV', ... + 'smplX_mm','smplY_mm','smplZ_mm'}; +metaCols = intersect(metaCols, testRows.Properties.VariableNames, 'stable'); +testTable = testRows(:, metaCols); +testTable.resultImages = repmat({{}}, height(testTable), 1); +rowsJsonTest = jsonencode(table2struct(testTable)); + +campaignProps = struct( ... + 'campaignLocation', FileList.Properties.CustomProperties.campaignLocation, ... + 'team', FileList.Properties.CustomProperties.team, ... + 'projectName', FileList.Properties.CustomProperties.projectName); +campaignPropsJson = jsonencode(campaignProps); + +%% 5. The actual test +resultJson = py.nappSyncToOpenbis.sync_rows(rowsJsonTest, openbisUrl, spaceCode, ... + pyargs('collection', collectionId, 'object_type', objectType, ... + 'username', username, 'password', password, ... + 'campaign_props_json', campaignPropsJson, 'upload_files', false, ... + 'parent_identifier', parentIdentifier, ... + 'verify_certificates', verifyCertificates)); + +results = struct2table(jsondecode(char(resultJson))); +disp(results.message{1}) \ No newline at end of file