From 00d34110db24c82d0ecfd0eb96b92a1af2a36cc6 Mon Sep 17 00:00:00 2001 From: ThorBarRa Date: Thu, 12 Mar 2026 10:08:47 +0100 Subject: [PATCH] MAJOR UPDATE: Add support for Specs .xy files from offline XPS (Soleil/Luca) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both Scienta (.ibw) and SpecsLab (.xy) raw files capture multidimensional XPS data but use different terminology to structure it. Scienta uses "Region Iterations" and "Sequence Iterations". SpecsLab organizes data by "Groups" or "Regions" (different elemental energy sweeps), "Scans" (time-resolved repetitions), and "Curves". Scienta Region Iterations are automatically averaged to improve the signal-to-noise ratio, while Scienta Sequence Iterations and SpecsLab Scans are preserved to keep the time-resolved data. For SpecsLab files specifically, distinct Regions are split into separate table rows. Ultimately, the preserved time-resolved data is saved in the table's image variable as a 2D matrix ([Energy bins × Sequence iterations/Scans]). File type handling: Added .xy as a recognized XPS format. Currently, the script differentiates data types strictly by file extension (NEXAFS: .h5, XPS: .ibw and .xy). This extension-based differentiation needs to be improved in future updates. Metadata parsing: Implemented global header scanning for .xy files to correctly extract instrument parameters (Acquisition Mode, Lens Mode, Slit, Workfunction) located before the first region block. Timezone alignment: Added timestamp conversion for .xy files from UTC to local time (Europe/Zurich) to ensure synchronization with LabView logs. Region separation: Modified .xy parsing to append a _RegX suffix to filenames. This correctly splits single .xy files that contain multiple elements or energy sweeps into distinct table rows so they can be processed individually. Import efficiency: Added an originalFileName variable. The script now cross-references the input DataTable and skips files that have already been imported to reduce execution time on re-runs. Testing status: Processing of .xy data has currently only been tested and debugged with nappGenerateSpectra and nappPlot. Compatibility with other functions in the suite remains untested. --- beamtime/nappFileList.m | 1068 ++++++++++++++++---------------- beamtime/nappGenerateSpectra.m | 4 +- beamtime/nappPlot.m | 20 +- beamtime/nappScale.m | 2 +- 4 files changed, 553 insertions(+), 541 deletions(-) diff --git a/beamtime/nappFileList.m b/beamtime/nappFileList.m index fa52d82..929764a 100755 --- a/beamtime/nappFileList.m +++ b/beamtime/nappFileList.m @@ -255,7 +255,7 @@ function ListFiles = nappFileList(dataPath,varargin) % * April 2021, X-ray kinetic energy and images are now made unique % * March 2022, input now using varargin % * March 2022, strict data structure required for local and remote sources -% * November 2025, PShell h5 files for NEXAFS have a new structure; +% * November 2025, PShell h5 files for NEXAFS have a new structure; % * November 2025, added creation date to mac for NEXAFS. % * March 2026, added movie import from photos folder, added jpg as photo % file @@ -362,196 +362,149 @@ end % % THE ** notation needs a recent version of Matlab, 2014 does not work. -if endsWith(p.Results.mode,'Pick') - % select files (Xps or Nexafs, ignoring photos) to be imported from data directory - % (data can be taken from any directory; files need to be in same directory) - [filePick,pathPick] = uigetfile({'*.ibw;*.h5';'*.xy'},... - 'Select One or More Files', dataPath,... - 'MultiSelect', 'on'); - % make lists for Nexafs and for XPS - if ischar(filePick) - filePick = {filePick}; - end - filePickXps = filePick(endsWith(filePick,'.ibw')); - filePickNexafs = filePick(endsWith(filePick,'.h5')); - filePickXY = filePick(endsWith(filePick,'.xy')); +if endsWith(p.Results.mode, 'Pick') + % --- PICK MODE --- + [filePick, pathPick] = uigetfile({'*.ibw;*.xy;*.h5'}, ... + 'Select One or More Files', dataPath, 'MultiSelect', 'on'); - if isempty(filePickXps) - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickXps,2) - TmpFileList{1}(ii,1) = dir([pathPick filePickXps{ii}]); + if ischar(filePick), filePick = {filePick}; end % Force cell array + + % Initialize empty dir structs + TmpFileList{1} = struct('name',{},'folder',{},'date',{},'bytes',{},'isdir',{},'datenum',{}); + TmpFileList{2} = TmpFileList{1}; + TmpFileList{3} = TmpFileList{1}; % Photos always empty in pick mode + + % Populate XPS and NEXAFS + if ~isequal(filePick, 0) % If user didn't cancel + for ii = 1:length(filePick) + if endsWith(filePick{ii}, {'.ibw', '.xy'}, 'IgnoreCase', true) + TmpFileList{1} = [TmpFileList{1}; dir(fullfile(pathPick, filePick{ii}))]; + elseif endsWith(filePick{ii}, '.h5', 'IgnoreCase', true) + TmpFileList{2} = [TmpFileList{2}; dir(fullfile(pathPick, filePick{ii}))]; + end end end - if isempty(filePickNexafs) - TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickNexafs,2) - TmpFileList{2}(ii,1) = dir([pathPick filePickNexafs{ii}]); - end - end - if isempty(filePickXY) - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickXps,2) - TmpFileList{1}(ii,1) = dir([pathPick filePickXps{ii}]); - end - end - % make a empty file list for photos, to be consitent - TmpFileList{3} = dir([dataPath_Photo filesep '**' filesep '*.trallalla']); + else - % SES files in IGOR format or SpecsLab in .xy format: list all files, then get rid of system files - if contains(p.Results.mode,'data') - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.xy']); - TmpFileList{1} = TmpFileList{1}(find(~cellfun(@isempty,regexp({TmpFileList{1}.name},'^[a-zA-Z0-9]\w*')))); %#ok - else - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.ibw']); - TmpFileList{1} = TmpFileList{1}(find(~cellfun(@isempty,regexp({TmpFileList{1}.name},'^[a-zA-Z0-9]\w*')))); %#ok + % --- FOLDER SEARCH MODE --- + + % 1. XPS (Both .ibw and .xy) + tmpIbw = dir(fullfile(dataPath_XPS, '**', '*.ibw')); + tmpIbw = tmpIbw(find(~cellfun(@isempty,regexp({tmpIbw .name},'^[a-zA-Z0-9]\w*')))); %#ok + tmpXy = dir(fullfile(dataPath_XPS, '**', '*.xy')); + tmpXy = tmpXy(find(~cellfun(@isempty,regexp({tmpXy .name},'^[a-zA-Z0-9]\w*')))); %#ok + TmpFileList{1} = [tmpIbw; tmpXy]; % Stack them together + + % 2. NEXAFS + TmpFileList{2} = dir(fullfile(dataPath_NEXAFS, '**', '*.h5')); + TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok + + + % 3. Photos + photoExts = {'*.bmp', '*.jpg', '*.JPG', '*.mov', '*.avi', '*.mp4', '*.wmv'}; + TmpFileList{3} = []; + for i = 1:length(photoExts) + TmpFileList{3} = [TmpFileList{3}; dir(fullfile(dataPath_Photo, '**', photoExts{i}))]; end + TmpFileList{3} = TmpFileList{3}(find(~cellfun(@isempty,regexp({TmpFileList{3}.name},'^[a-zA-Z0-9]\w*')))); %#ok + + % Clean up system/hidden files for all lists + for i = 1:3 + if ~isempty(TmpFileList{i}) + % Keep only files that do NOT start with '.' or '~' + validIdx = ~startsWith({TmpFileList{i}.name}, {'.', '~'}); + TmpFileList{i} = TmpFileList{i}(validIdx); + end + end + + % ========================================================================= + % SPLIT HDF5 FILES WITH MULTIPLE DATASETS INTO INDIVIDUAL FILES + % ========================================================================= + % + % BACKGROUND: + % HDF5 files from the beamline can contain multiple experimental scans + % (e.g., scan_1, scan_2, scan_3) stored as separate groups within a + % single file. However, the NEXAFS and XPS data processing pipelines + % are designed to work with one scan per file. + % + % PURPOSE: + % This script identifies HDF5 files that contain multiple scans and + % splits each scan into a separate file with a descriptive name. + % Each output file contains only one scan (renamed to scan_1) plus + % the logs and metadata groups. + % + % PROCESS: + % 1. Loop through all NEXAFS HDF5 files in the file list + % 2. Check if each file contains more than 3 groups (indicating multiple scans) + % 3. If it does, split it using the splitH5Scans function + % 4. The original file is deleted after splitting + % + % OUTPUT: + % For each multi-scan file, creates separate output files: + % Example: 0002_..._O_Kedge.h5  0002_..._O_Kedge_scan_1.h5 + % 0002_..._O_Kedge_scan_2.h5 + % ========================================================================= + % Loop through each file in the NEXAFS file list + % TmpFileList{2} contains the full paths to all NEXAFS HDF5 files + % ii is the loop counter (1, 2, 3, ... up to the total number of files) + for ii = 1:size(TmpFileList{2}, 1) + + % Get the filename of the current file being processed + current_file = [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]; + + % Read information about the structure of the HDF5 file + % This tells us what groups (datasets) are stored in the file + fileinfo = h5info(current_file); + + % Extract the names of all groups at the root level + % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} + groupnames = {fileinfo.Groups.Name}; + + % Check if this file has more than 3 groups + % Files with more than 3 groups likely contain multiple scans + % (the extra groups beyond scans are typically '/logs' and '/__DATA_TYPES__') + if numel(groupnames) > 3 + + % This file contains multiple scans, so split it + % The splitH5Scans function will: + % - Create separate files for each scan + % - Rename each scan to '/scan_1' within its new file + % - Delete the original multi-scan file when done + splitH5Scans(current_file); + + end + + end + + % ========================================================================= + % NOTE: After this loop completes, all NEXAFS files in data folder + % will contain exactly one scan per file, ready for processing + % ========================================================================= % EPICS files in h5 format list all files, then get rid of system files TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.h5']); TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok - % Photos list all files, then get rid of system files - TmpFileList{3} = [ - dir([dataPath_Photo filesep '**' filesep '*.bmp']) - dir([dataPath_Photo filesep '**' filesep '*.jpg']) - dir([dataPath_Photo filesep '**' filesep '*.JPG']) - dir([dataPath_Photo filesep '**' filesep '*.mov']) - dir([dataPath_Photo filesep '**' filesep '*.avi']) - dir([dataPath_Photo filesep '**' filesep '*.mp4']) - dir([dataPath_Photo filesep '**' filesep '*.wmv']) - ]; - TmpFileList{3} = TmpFileList{3}(find(~cellfun(@isempty,regexp({TmpFileList{3}.name},'^[a-zA-Z0-9]\w*')))); %#ok - % The second line of code filters all files that start with Numbers or - % letters. All system files starting with a . or ~ are discarded. -% ========================================================================= -% SPLIT HDF5 FILES WITH MULTIPLE DATASETS INTO INDIVIDUAL FILES -% ========================================================================= -% -% BACKGROUND: -% HDF5 files from the beamline can contain multiple experimental scans -% (e.g., scan_1, scan_2, scan_3) stored as separate groups within a -% single file. However, the NEXAFS and XPS data processing pipelines -% are designed to work with one scan per file. -% -% PURPOSE: -% This script identifies HDF5 files that contain multiple scans and -% splits each scan into a separate file with a descriptive name. -% Each output file contains only one scan (renamed to scan_1) plus -% the logs and metadata groups. -% -% PROCESS: -% 1. Loop through all NEXAFS HDF5 files in the file list -% 2. Check if each file contains more than 3 groups (indicating multiple scans) -% 3. If it does, split it using the splitH5Scans function -% 4. The original file is deleted after splitting -% -% OUTPUT: -% For each multi-scan file, creates separate output files: -% Example: 0002_..._O_Kedge.h5  0002_..._O_Kedge_scan_1.h5 -% 0002_..._O_Kedge_scan_2.h5 -% ========================================================================= -% Loop through each file in the NEXAFS file list -% TmpFileList{2} contains the full paths to all NEXAFS HDF5 files -% ii is the loop counter (1, 2, 3, ... up to the total number of files) -for ii = 1:size(TmpFileList{2}, 1) - - % Get the filename of the current file being processed - current_file = [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]; - - % Read information about the structure of the HDF5 file - % This tells us what groups (datasets) are stored in the file - fileinfo = h5info(current_file); - - % Extract the names of all groups at the root level - % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} - groupnames = {fileinfo.Groups.Name}; - - % Check if this file has more than 3 groups - % Files with more than 3 groups likely contain multiple scans - % (the extra groups beyond scans are typically '/logs' and '/__DATA_TYPES__') - if numel(groupnames) > 3 - - % This file contains multiple scans, so split it - % The splitH5Scans function will: - % - Create separate files for each scan - % - Rename each scan to '/scan_1' within its new file - % - Delete the original multi-scan file when done - splitH5Scans(current_file); - + disp('*************************************************') + disp('* Voila: Splitting NEXAFS files done *') + disp('*************************************************') + + % --- Date of First File Check --- + % Since TmpFileList{1} now safely contains all .ibw and .xy files, + % we don't need to run dir() again! We just check the struct directly. + if isempty(TmpFileList{1}) + warning('There appears to be no XPS file (.ibw or .xy) in the folder. Add at least one file first.'); + return; end - -end - -% ========================================================================= -% NOTE: After this loop completes, all NEXAFS files in data folder -% will contain exactly one scan per file, ready for processing -% ========================================================================= - -% EPICS files in h5 format list all files, then get rid of system files - TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.h5']); - TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok - -disp('*************************************************') -disp('* Voila: Splitting NEXAFS files done *') -disp('*************************************************') - - % only process new files - if (istable(p.Results.DataTable) & matches(p.Results.mode,'localNew')) % do this only if previous list was input to function call - tmpXpsFiles = endsWith(p.Results.DataTable.name,'.ibw'); % get Scienta XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{1}(:).name},p.Results.DataTable.name(tmpXpsFiles,:)); % find those files that are yet imported - TmpFileList{1} = TmpFileList{1}(tmpMissingFiles); - - tmpXpsFiles = endsWith(p.Results.DataTable.name,'.xy'); % get Specs XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{1}(:).name},p.Results.DataTable.name(tmpXpsFiles,:)); % find those files that are yet imported - TmpFileList{1} = TmpFileList{1}(tmpMissingFiles); - - tmpNexafsFiles = endsWith(p.Results.DataTable.name,'.h5'); % get XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{2}(:).name},p.Results.DataTable.name(tmpNexafsFiles,:)); % find those files that are yet imported - TmpFileList{2} = TmpFileList{2}(tmpMissingFiles); - - tmpPhotoFiles = endsWith(p.Results.DataTable.name,'.bmp'); % get XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{3}(:).name},p.Results.DataTable.name(tmpPhotoFiles,:)); % find those files that are yet imported - TmpFileList{3} = TmpFileList{3}(tmpMissingFiles); + try + filedate_first = TmpFileList{1}(1).datenum; + catch + warning('There appears to be something wrong with the XPS file metadata.'); end -end - - - -% get date of first .ibw or .xy file in folder . -if contains(p.Results.mode,'data') - tmp.fileinfo = dir([dataPath_XPS '/*.xy']); -else - tmp.fileinfo = dir([dataPath_XPS '/*.ibw']); -end -if isempty(tmp.fileinfo) - warning('There appears to be no file in the SES folder. Add at least one file first.') - disp('There appears to be no file in the SES folder. Add at least one file first.') - disp('I quit now') - return -end - -try - if contains(p.Results.mode,'data') - tmp.fileinfo = dir([dataPath_XPS '/*.xy']); - filedate_first = tmp.fileinfo(1).datenum; - clearvars name tmp - else - tmp.fileinfo = dir([dataPath_XPS '/*.ibw']); - filedate_first = tmp.fileinfo(1).datenum; - clearvars name tmp - end -catch - warning('There appears to be something wrong with the SES or Specs file.') - disp('There appears to be something wrong with the SES or Specs file.') end - - %% Import XPS data to get SES settings % % Uses the IgorPro importer for Matlab: @@ -564,250 +517,317 @@ end % and try to resolve it. I suggest to rename these original data files that % are damaged, or not complete (because measurement was aborted) to % "ignore_filename". +%% 2. Import XPS data +ProcessedXPS = []; % This will hold our final list of structs -if contains(p.Results.mode,'data') % for Specs xy data %TODO this is based on file location, needs work!!!! - for ii = 1:size(TmpFileList{1},1) % cycle through each XPS file - +if ~isempty(TmpFileList{1}) + for ii = 1:length(TmpFileList{1}) + current_file_info = TmpFileList{1}(ii); + current_path = fullfile(current_file_info.folder, current_file_info.name); + % ================================================================= + % 1. INITIALIZE MASTER TEMPLATE (Strict Definition) + % ================================================================= + % Every variable MUST be declared here so the struct sizes match perfectly + + current_file_info.creationDate = [NaN NaN NaN NaN NaN NaN]; + current_file_info.lastModifiedDatenum = current_file_info.datenum; + current_file_info.lastModifiedDatestr = current_file_info.date; + current_file_info.image = {[]}; + current_file_info.imageRoh = {[]}; + current_file_info.xRayEkin_eV = NaN; + current_file_info.xRayEkinRange_eV = [NaN NaN]; + current_file_info.scientaPassEnergy_eV = NaN; + current_file_info.scientaDwellTime_ms = NaN; + current_file_info.regionName = ''; + current_file_info.scientaAcquisitionMode = ''; + current_file_info.scientaEkinRange_eV = [NaN NaN]; + current_file_info.scientaEkinStep_eV = NaN; + current_file_info.scientaEkin_eV = {[]}; + current_file_info.scientaEkinRoh_eV = {[]}; + current_file_info.beamlineInt = {[]}; + current_file_info.scientaLensMode = ''; + current_file_info.scientaAnalyzerSlit = ''; + current_file_info.scientaWorkfunction_eV = NaN; + current_file_info.scientaRegionIterations = NaN; + current_file_info.scientaSequenceIterations = NaN; + current_file_info.importDate = datestr(now); try - %data - [fid,~] = fopen([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name],'rt'); - tmpSpecsData = textscan(fid,'%f%f','Delimiter', '\b', 'CommentStyle','#'); % this reads the data; comment lines are ignored - fclose(fid); - % comments - opts = delimitedTextImportOptions("NumVariables", 1); - % Specify range and delimiter - opts.DataLines = [1, 45]; - opts.Delimiter = "\t"; - % Specify column names and types - opts.VariableNames = "metaData"; - opts.VariableTypes = "char"; - % Specify file level properties - opts.ExtraColumnsRule = "ignore"; - opts.EmptyLineRule = "skip"; - opts.ConsecutiveDelimitersRule = "join"; - % Specify variable properties - opts = setvaropts(opts, "metaData", "WhitespaceRule", "preserve"); - opts = setvaropts(opts, "metaData", "EmptyFieldRule", "auto"); - tmpSpecsComments = readtable([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name],opts); - - - catch - warning('Problem reading the Specs file') - disp('Problem reading the Specs file') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) + current_file_info.analysisDir = file.mFile; + catch + current_file_info.analysisDir = pwd; end - - - - try - %% Convert Specs meta data into variables - - % data - tmpEndFirstSpectrum = find(diff(tmpSpecsData{1})<0,1); % FIXME: imports only first spectrum - % counts - PE.y = tmpSpecsData{2}(1:tmpEndFirstSpectrum); - % kinetic energy - PE.x = tmpSpecsData{1}(1:tmpEndFirstSpectrum); - % RayEnergy - PE.xRayEkin_eV{:} = str2double(extractAfter(tmpSpecsComments(29,:).metaData{:},"# Excitation Energy:")); - PE.xRayEkinRange_eV =[ PE.xRayEkin_eV{1} PE.xRayEkin_eV{end}]; - % Start date/time of measurement - PE.creationDate = extractBetween(char(tmpSpecsComments(20,:).metaData{:}),"# Acquisition Date:","UTC"); - PE.creationDate = datetime(PE.creationDate,'InputFormat','MM/dd/yy HH:mm:ss','PivotYear',2000,'TimeZone','UTC'); - % Pass energy - PE.passEnergy = str2double(extractAfter(tmpSpecsComments(31,:).metaData{:},"# Pass Energy:")); - % Dwell time (or step time) - PE.dwellTime = str2double(extractAfter(tmpSpecsComments(28,:).metaData{:},"# Dwell Time: ")); - % Energy step - tmpEnergyStep = diff(PE.x); - PE.scientaEkinStep = tmpEnergyStep(1); - if range(tmpEnergyStep) ~= 0 - disp("Ups something wrong with the energy steps") - end - % Region name - PE.regionName = strtrim(extractAfter(tmpSpecsComments(18,:).metaData{:},"# Region:")); - % Acquisition Mode - PE.acquisitionMode = strtrim(extractAfter(tmpSpecsComments(25,:).metaData{:},"# Scan Mode:")); - PE.regionIterations = str2double(extractAfter(tmpSpecsComments(39,:).metaData{:},"# Number of Scans:")); - % counts/s - PE.y = PE.y./PE.regionIterations; - % Pass energy - PE.passEnergy = str2double(extractAfter(tmpSpecsComments(31,:).metaData{:},"# Pass Energy:")); - catch - warning('Problem reading the PE variable') - disp('Problem reading the PE variable') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - - %% Write SES meta data variables into XPS FileList structure - - try - TmpFileList{1}(ii).creationDate = datevec(PE.creationDate); - TmpFileList{1}(ii).image = {NaN}; - TmpFileList{1}(ii).spectrum_counts = NaN; - TmpFileList{1}(ii).spectrum_countsPerSecond = PE.y; - TmpFileList{1}(ii).xRayEkin_eV = PE.xRayEkin_eV; - TmpFileList{1}(ii).xRayEkinRange_eV = PE.xRayEkinRange_eV; - % SES pass energy - TmpFileList{1}(ii).scientaPassEnergy_eV = PE.passEnergy; % in eV - % SES dwell time - TmpFileList{1}(ii).scientaDwellTime_ms = PE.dwellTime; % in ms - % SES region Name - TmpFileList{1}(ii).regionName = PE.regionName; - TmpFileList{1}(ii).scientaAcquisitionMode = PE.acquisitionMode; - % SES kinetic energy window - TmpFileList{1}(ii).scientaEkinRange_eV = [PE.x(1) PE.x(end)]; - TmpFileList{1}(ii).scientaEkinStep_eV = PE.scientaEkinStep; - TmpFileList{1}(ii).scientaEkin_eV = PE.x; - % Beamline Intensity (I0 during NEXAFS) - TmpFileList{1}(ii).beamlineInt = cell(1); - % SES lens mode - TmpFileList{1}(ii).scientaLensMode = {' '}; - TmpFileList{1}(ii).scientaRegionIterations = PE.regionIterations; - TmpFileList{1}(ii).scientaSequenceIterations = NaN; - TmpFileList{1}(ii).imageRoh = {NaN}; - TmpFileList{1}(ii).spectrum_countsPerSecondRoh = PE.y; - TmpFileList{1}(ii).importDate = datestr(now); + current_file_info.sample = ''; + % ================================================================= + % --- Case A: Scienta (.ibw) Files --- + % ================================================================= + if endsWith(current_file_info.name, '.ibw', 'IgnoreCase', true) try - TmpFileList{1}(ii).analysisDir = file.mFile; - catch - TmpFileList{1}(ii).analysisDir = pwd; + PE = IBWread(current_path); + fclose('all'); + tmp = regexp(PE.WaveNotes, '\r', 'split'); + + try + dateStr = [char(regexp(tmp{29},'((?<=Date=).*)','match')) ' ' char(regexp(tmp{30},'((?<=Time=).*)','match'))]; + parsedDate = datevec(dateStr); + if size(parsedDate, 2) == 6 + current_file_info.creationDate = parsedDate; + current_file_info.lastModifiedDatenum = datenum(parsedDate); + current_file_info.lastModifiedDatestr = dateStr; + end + catch + end + + current_file_info.scientaPassEnergy_eV = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); + current_file_info.scientaDwellTime_ms = str2double(char(regexp(tmp{15},'((?<=Step Time=).*)','match'))); + current_file_info.scientaEkinStep_eV = str2double(char(regexp(tmp{14},'((?<=Energy Step=).*)','match'))); + current_file_info.regionName = char(regexp(tmp{3},'((?<=Region Name=).*)','match')); + current_file_info.scientaAcquisitionMode = char(regexp(tmp{9},'((?<=Acquisition Mode=).*)','match')); + current_file_info.scientaLensMode = char(regexp(tmp{4},'((?<=Lens Mode=).*)','match')); + + PE.x = PE.x0(1) + (0:size(PE.y,1)-1)*PE.dx(1); + current_file_info.scientaEkin_eV = {PE.x}; + current_file_info.scientaEkinRange_eV = [PE.x(1) PE.x(end)]; + + if PE.Ndim == 3 + current_file_info.scientaRegionIterations = size(PE.y,2); + current_file_info.scientaSequenceIterations = size(PE.y,3); + PE.y = squeeze(mean(PE.y,2)); + else + current_file_info.scientaRegionIterations = 1; + current_file_info.scientaSequenceIterations = size(PE.y,2); + end + + PE.y = PE.y .* 1000 ./ current_file_info.scientaDwellTime_ms; + current_file_info.image = {PE.y}; + current_file_info.imageRoh = {PE.y}; + + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok + catch ME + fprintf('Warning: Failed to parse %s (%s)\n', current_file_info.name, ME.message); + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok + end + + % ================================================================= + % --- Case B: Specs (.xy) Files --- + % ================================================================= + elseif endsWith(current_file_info.name, '.xy', 'IgnoreCase', true) + try + [cDateOS, mDateOS] = getOSDates(current_path); + try, osDateVec = datevec(datetime(cDateOS, 'InputFormat', 'dd-MMM-yyyy HH:mm:ss')); + catch, try osDateVec = datevec(mDateOS); catch, osDateVec = [NaN NaN NaN NaN NaN NaN]; end, end + if isempty(osDateVec) || size(osDateVec, 1) == 0, osDateVec = [NaN NaN NaN NaN NaN NaN]; end + + fileLines = readlines(current_path); + regionStarts = find(startsWith(fileLines, '# Region:')); + + % --- 1. PARSE GLOBAL HEADER (Before First Region) --- + % SpecsLab moved many parameters to the top of the file in newer versions + globalScanMode = ''; + globalLensMode = ''; + globalAnalyzerSlit = ''; + globalWorkFunction = NaN; + globalExEnergy = NaN; + + if ~isempty(regionStarts) + globalHeader = fileLines(1:regionStarts(1)-1); + for L = 1:length(globalHeader) + lineStr = strtrim(globalHeader(L)); + lineStr = regexprep(lineStr, '^#\s+', '# '); % Normalize spaces after # + + if startsWith(lineStr, '# Scan Mode:') + globalScanMode = strtrim(extractAfter(lineStr, '# Scan Mode:')); + elseif startsWith(lineStr, '# Analyzer Lens Voltage:') + globalLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens Voltage:')); + elseif startsWith(lineStr, '# Analyzer Lens:') + globalLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens:')); + elseif startsWith(lineStr, '# Analyzer Slit:') + globalAnalyzerSlit = strtrim(extractAfter(lineStr, '# Analyzer Slit:')); + elseif startsWith(lineStr, '# Work Function:') + globalWorkFunction = sscanf(strtrim(extractAfter(lineStr, '# Work Function:')), '%f'); + elseif startsWith(lineStr, '# Eff. Workfunction:') + globalWorkFunction = sscanf(strtrim(extractAfter(lineStr, '# Eff. Workfunction:')), '%f'); + elseif startsWith(lineStr, '# Excitation Energy:') + globalExEnergy = sscanf(strtrim(extractAfter(lineStr, '# Excitation Energy:')), '%f'); + end + end + end + + for r = 1:length(regionStarts) + regionEntry = current_file_info; + startIdx = regionStarts(r); + if r < length(regionStarts), endIdx = regionStarts(r+1)-1; else, endIdx = length(fileLines); end + regionBlock = fileLines(startIdx:endIdx); + + % ===================================================== + % 2. Extract Metadata + % ===================================================== + % Initialize defaults + rName = 'UnknownRegion'; + exEnergyStr = num2str(globalExEnergy); + specID = r; + regionEntry.xRayEkin_eV = globalExEnergy; + regionEntry.scientaPassEnergy_eV = NaN; + regionEntry.scientaDwellTime_ms = NaN; + + % Inherit defaults from Global Header + regionEntry.scientaAcquisitionMode = globalScanMode; + regionEntry.scientaLensMode = globalLensMode; + regionEntry.scientaAnalyzerSlit = globalAnalyzerSlit; + regionEntry.scientaWorkfunction_eV = globalWorkFunction; + regionEntry.creationDate = osDateVec; + + % Scan the header lines directly + for L = 1:min(150, length(regionBlock)) % Metadata is always at the top + lineStr = strtrim(regionBlock(L)); + if isempty(lineStr) || ~startsWith(lineStr, '#'), continue; end + + lineStr = regexprep(lineStr, '^#\s+', '# '); % Normalize spaces after # + + if startsWith(lineStr, '# Region:') + rName = strtrim(extractAfter(lineStr, '# Region:')); + elseif startsWith(lineStr, '# Excitation Energy:') + regionEntry.xRayEkin_eV = sscanf(strtrim(extractAfter(lineStr, '# Excitation Energy:')), '%f'); + exEnergyStr = num2str(regionEntry.xRayEkin_eV); + elseif startsWith(lineStr, '# Spectrum ID:') + specID = sscanf(strtrim(extractAfter(lineStr, '# Spectrum ID:')), '%f'); + elseif startsWith(lineStr, '# Acquisition Date:') + rawDateStr = strtrim(extractAfter(lineStr, '# Acquisition Date:')); + dateStr = strtrim(regexprep(rawDateStr, 'UTC$', '')); + try + dtUTC = datetime(dateStr, 'InputFormat', 'MM/dd/yy HH:mm:ss', 'PivotYear', 2000, 'TimeZone', 'UTC'); + dtLocal = datetime(dtUTC, 'TimeZone', 'Europe/Zurich'); + regionEntry.lastModifiedDatenum = datenum(dtLocal); + regionEntry.lastModifiedDatestr = datestr(dtLocal); + catch + regionEntry.lastModifiedDatestr = rawDateStr; + end + elseif startsWith(lineStr, '# Pass Energy:') + regionEntry.scientaPassEnergy_eV = str2double(extractAfter(lineStr, '# Pass Energy:')); + elseif startsWith(lineStr, '# Dwell Time:') + regionEntry.scientaDwellTime_ms = str2double(extractAfter(lineStr, '# Dwell Time:')) * 1000; + elseif startsWith(lineStr, '# Scan Mode:') + regionEntry.scientaAcquisitionMode = strtrim(extractAfter(lineStr, '# Scan Mode:')); + elseif startsWith(lineStr, '# Analyzer Lens Voltage:') + regionEntry.scientaLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens Voltage:')); + elseif startsWith(lineStr, '# Analyzer Lens:') + regionEntry.scientaLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens:')); + elseif startsWith(lineStr, '# Analyzer Slit:') + regionEntry.scientaAnalyzerSlit = strtrim(extractAfter(lineStr, '# Analyzer Slit:')); + elseif startsWith(lineStr, '# Work Function:') + regionEntry.scientaWorkfunction_eV = sscanf(strtrim(extractAfter(lineStr, '# Work Function:')), '%f'); + elseif startsWith(lineStr, '# Eff. Workfunction:') + regionEntry.scientaWorkfunction_eV = sscanf(strtrim(extractAfter(lineStr, '# Eff. Workfunction:')), '%f'); + end + end + + % --- Format Name & Region --- + regionEntry.regionName = rName; + cleanRName = strrep(rName, ' ', '_'); + + % Write the original OS file name + regionEntry.originalFileName = current_file_info.name; + + % Prevent identical filenames by appending _Reg1, _Reg2 if multiple regions exist + if length(regionStarts) > 1 + regionEntry.name = sprintf('%04d_%s_Reg%d_%s.xy', specID, cleanRName, r, exEnergyStr); + else + regionEntry.name = sprintf('%04d_%s_%s.xy', specID, cleanRName, exEnergyStr); + end + + + + % 2. Extract Data + hIdx = find(startsWith(strtrim(regionBlock), '# Cycle:')); + tempD = []; enAxis = []; + + if ~isempty(hIdx) + for d = 1:length(hIdx) + cScn = 1; cCrv = 1; + + headerStr = regionBlock(hIdx(d)); + curveMatch = regexp(headerStr, 'Curve:\s*(\d+)', 'tokens', 'once'); + if ~isempty(curveMatch), cCrv = str2double(curveMatch{1}) + 1; end + + scanMatch = regexp(headerStr, 'Scan:\s*(\d+)', 'tokens', 'once'); + if ~isempty(scanMatch), cScn = str2double(scanMatch{1}) + 1; end + + row = hIdx(d) + 1; + while row <= length(regionBlock) && (startsWith(strtrim(regionBlock(row)), '#') || strlength(strtrim(regionBlock(row))) < 2), row = row + 1; end + dataS = row; + while row <= length(regionBlock) && ~startsWith(strtrim(regionBlock(row)), '#') && strlength(strtrim(regionBlock(row))) > 1, row = row + 1; end + + if dataS < row + numCols = length(sscanf(regionBlock(dataS), '%f')); + raw = sscanf(join(regionBlock(dataS:row-1), ' '), '%f'); + if ~isempty(raw) && numCols >= 2 + m = reshape(raw, numCols, [])'; + if isempty(enAxis), enAxis = m(:,1); end + tempD(1:size(m,1), cCrv, cScn) = m(:,2); + end + end + end + else + row = 1; + while row <= length(regionBlock) && (startsWith(strtrim(regionBlock(row)), '#') || strlength(strtrim(regionBlock(row))) < 2), row = row + 1; end + dataS = row; + while row <= length(regionBlock) && ~startsWith(strtrim(regionBlock(row)), '#') && strlength(strtrim(regionBlock(row))) > 1, row = row + 1; end + + if dataS < row + numCols = length(sscanf(regionBlock(dataS), '%f')); + raw = sscanf(join(regionBlock(dataS:row-1), ' '), '%f'); + if ~isempty(raw) && numCols >= 2 + m = reshape(raw, numCols, [])'; + enAxis = m(:,1); + tempD(1:size(m,1), 1, 1) = m(:,2); + end + end + end + + % 3. Map Dimensions + if ~isempty(tempD) + image2D = squeeze(mean(tempD, 2, 'omitnan')); + if isvector(image2D), image2D = image2D(:); end + else + image2D = []; + end + + regionEntry.image = {image2D}; + regionEntry.imageRoh = {image2D}; + + if ~isnan(regionEntry.xRayEkin_eV) + regionEntry.xRayEkinRange_eV = [regionEntry.xRayEkin_eV regionEntry.xRayEkin_eV]; + end + + if ~isempty(enAxis) + enAxis = reshape(enAxis, 1, []); + regionEntry.scientaEkinRange_eV = [enAxis(1) enAxis(end)]; + regionEntry.scientaEkinStep_eV = enAxis(2) - enAxis(1); + regionEntry.scientaEkin_eV = {enAxis}; + regionEntry.scientaEkinRoh_eV = {enAxis}; + end + + regionEntry.scientaRegionIterations = size(tempD, 3); + + if isempty(ProcessedXPS) + ProcessedXPS = regionEntry; + else + ProcessedXPS = [ProcessedXPS; regionEntry]; %#ok + end + end + catch ME + fprintf('Warning: Failed to parse %s (%s)\n', current_file_info.name, ME.message); + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok end - catch - warning('Problem with a XPS data file for XPS list') - disp('Problem with a XPS data file for XPS list') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - fprintf('Number %.0f: %s \n',ii,datestr(TmpFileList{1}(ii).datenum)) end end -else % for Scienta igor data - for ii = 1:size(TmpFileList{1},1) % cycle through each XPS file - - - try - PE = IBWread([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]); % read XPS file - catch - warning('Problem reading the Igor file') - disp('Problem reading the Igor file') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - fclose('all'); - - try - %% Convert SES meta data into variables - % - % Upon import, all meta data are written into _WaveNotes_ variable. - % Here, I select the meta data for each setting and write those - % into individual variables that are then added to the list. - - % Split the WaveNotes text string into individual strings at each line break. - % (This is a work-around. Did not manage to select values with regexp - % function only.) - tmp = regexp(PE.WaveNotes, '\r', 'split'); - % Then, read out the values into variables using a combination of string operations: - - % Start date/time of measurement - PE.creationDate = datevec([char(regexp(tmp{29},'((?<=Date=).*)','match')) ... - ' ' char(regexp(tmp{30},'((?<=Time=).*)','match'))]); - % Pass energy - PE.passEnergy = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); - % Dwell time (or step time) - PE.dwellTime = str2double(char(regexp(tmp{15},'((?<=Step Time=).*)','match'))); - % Energy step - PE.scientaEkinStep = str2double(char(regexp(tmp{14},'((?<=Energy Step=).*)','match'))); - % Region name - PE.regionName = char(regexp(tmp{3},'((?<=Region Name=).*)','match')); - % Acquisition Mode - PE.acquisitionMode = char(regexp(tmp{9},'((?<=Acquisition Mode=).*)','match')); - - % kinetic energy - for n = 1:size(PE.y,1) - PE.x(n)= PE.x0(1) + (n-1)*PE.dx(1); - end - % lens mode - PE.lensMode = char(regexp(tmp{4},'((?<=Lens Mode=).*)','match')); - % define iterations - % - % There is a BUG in this code. The SES file apparently has the same - % structure for 1 seq. iteration and several reg. iterations and - % several seq. iterations and 1 region iteration. - if PE.Ndim == 3 - PE.regionIterations = size(PE.y,2); - PE.sequenceIterations = size(PE.y,3); - elseif PE.Ndim == 2 - PE.regionIterations = 1; - PE.sequenceIterations = size(PE.y,2); - else - PE.regionIterations = 1; - PE.sequenceIterations = size(PE.y,2); - end - - % average region iterations - if PE.Ndim == 3 - PE.y = squeeze(mean(PE.y,2)); - elseif PE.Ndim == 2 - % PE.y = squeeze(mean(PE.y,2)); - - end - % normalise data to 1ms - PE.y = PE.y .* 1000 ./ PE.dwellTime; - % Pass energy - PE.passEnergy = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); - catch - warning('Problem reading the PE variable') - disp('Problem reading the PE variable') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - - %% Write SES meta data variables into XPS FileList structure - - try - TmpFileList{1}(ii).creationDate = PE.creationDate; - TmpFileList{1}(ii).image = PE.y; - TmpFileList{1}(ii).spectrum_counts = NaN; - TmpFileList{1}(ii).spectrum_countsPerSecond = NaN; - TmpFileList{1}(ii).xRayEkin_eV = NaN; - TmpFileList{1}(ii).xRayEkinRange_eV = [NaN NaN]; - % SES pass energy - TmpFileList{1}(ii).scientaPassEnergy_eV = PE.passEnergy; % in eV - % SES dwell time - TmpFileList{1}(ii).scientaDwellTime_ms = PE.dwellTime; % in ms - % SES region Name - TmpFileList{1}(ii).regionName = PE.regionName; - TmpFileList{1}(ii).scientaAcquisitionMode = PE.acquisitionMode; - % SES kinetic energy window - TmpFileList{1}(ii).scientaEkinRange_eV = [PE.x(1) PE.x(end)]; - TmpFileList{1}(ii).scientaEkinStep_eV = PE.scientaEkinStep; - TmpFileList{1}(ii).scientaEkin_eV = PE.x; - % Beamline Intensity (I0 during NEXAFS) - TmpFileList{1}(ii).beamlineInt = cell(1); - % SES lens mode - TmpFileList{1}(ii).scientaLensMode = PE.lensMode; - TmpFileList{1}(ii).scientaRegionIterations = PE.regionIterations; - TmpFileList{1}(ii).scientaSequenceIterations = PE.sequenceIterations; - TmpFileList{1}(ii).imageRoh = PE.y; - TmpFileList{1}(ii).spectrum_countsPerSecondRoh = NaN; - TmpFileList{1}(ii).importDate = datestr(now); - try - TmpFileList{1}(ii).analysisDir = file.mFile; - catch - TmpFileList{1}(ii).analysisDir = pwd; - end - catch - warning('Problem with a XPS data file for XPS list') - disp('Problem with a XPS data file for XPS list') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - fprintf('Number %.0f: %s \n',ii,datestr(TmpFileList{1}(ii).datenum)) - end - end + TmpFileList{1} = ProcessedXPS; end + disp('*************************************************') disp('* Voila: Done reading & writing XPS data! *') disp('*************************************************') - - %% Read and write NEXAFS meta data % % The h5 file is much nicer organised, so we do not need string operations @@ -817,15 +837,12 @@ disp('*************************************************') % if datenum(filedate_first) >= datenum('31-Dec-2017 23:59:00') %('31-Dec-2019 23:59:00') for ii = 1:size(TmpFileList{2},1) % do this for each list element - %if ~isempty(h5read([TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name],'/scan 1/energy')) - % creation date of file - if ismac - tmpCommand = sprintf('stat -f "%%SB" "%s"', [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]); - [~,creation] = system(tmpCommand); - TmpFileList{2}(ii).creationDate = datevec(creation); - else - TmpFileList{2}(ii).creationDate = datevec(NaN); - end + % Get OS Dates + tmpCurrentFile = fullfile(TmpFileList{2}(ii).folder, TmpFileList{2}(ii).name); + [cDateOS, ~] = getOSDates(tmpCurrentFile); + TmpFileList{2}(ii).creationDate = datevec(cDateOS); + + try % read data %if datenum(filedate_first) <= datenum('21-Nov-2020 00:01:00') % Date not exaclty certain. @@ -843,14 +860,9 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end end end - % 21.8.20 changed from spectrum to scienta spectrum, later - % switched back to spectrum. - % Spectrum - TmpFileList{2}(ii).spectrum_counts = NaN; - TmpFileList{2}(ii).spectrum_countsPerSecond = NaN; - + % import Photon Energy (cols) try TmpXrayEkin = h5read([TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name],'/scan_1/energy'); % X-ray excitation energy; Photon energy in eV @@ -866,9 +878,7 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end - if isempty(TmpXrayEkin) - TmpFileList{2}(ii).xRayEkinRange_eV = [NaN NaN]; - else + if ~isempty(TmpXrayEkin) TmpFileList{2}(ii).xRayEkinRange_eV = [TmpXrayEkin(1) TmpXrayEkin(end)]; end @@ -960,7 +970,6 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element TmpFileList{2}(ii).scientaEkinRange_eV = [double(tmp(1))-TmpFileList{2}(ii).scientaPassEnergy_eV/20 double(tmp(1))+TmpFileList{2}(ii).scientaPassEnergy_eV/20]; - TmpFileList{2}(ii).scientaEkinStep_eV = NaN; TmpFileList{2}(ii).scientaEkin_eV = linspace(TmpFileList{2}(ii).scientaEkinRange_eV(:,1)... ,TmpFileList{2}(ii).scientaEkinRange_eV(:,2),size(TmpFileList{2}(ii).image,1))'; else @@ -1054,7 +1063,7 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end TmpFileList{2}(ii).imageRoh = TmpFileList{2}(ii).image; - TmpFileList{2}(ii).spectrum_countsPerSecondRoh = TmpFileList{2}(ii).spectrum_countsPerSecond; + % TmpFileList{2}(ii).spectrum_countsPerSecondRoh = TmpFileList{2}(ii).spectrum_countsPerSecond; % log TmpFileList{2}(ii).importDate = datestr(now); try @@ -1123,62 +1132,58 @@ disp('*************************************************') for ii = 1:size(TmpFileList{3},1) % do this for each picture in the list - - - TmpFileList{3}(ii).creationDate = datevec(NaN); - TmpFileList{3}(ii).image = NaN; - TmpFileList{3}(ii).spectrum_counts = NaN; - TmpFileList{3}(ii).spectrum_countsPerSecond = NaN; - TmpFileList{3}(ii).xRayEkin_eV = NaN; - TmpFileList{3}(ii).xRayEkinRange_eV = [NaN NaN]; - TmpFileList{3}(ii).scientaPassEnergy_eV = NaN; - TmpFileList{3}(ii).scientaDwellTime_ms = NaN; - TmpFileList{3}(ii).regionName = ' '; - TmpFileList{3}(ii).scientaAcquisitionMode = ' '; - TmpFileList{3}(ii).scientaEkinRange_eV = [NaN NaN]; - TmpFileList{3}(ii).scientaEkinStep_eV = NaN; - TmpFileList{3}(ii).scientaEkin_eV = NaN; - TmpFileList{3}(ii).beamlineInt = cell(1); - TmpFileList{3}(ii).scientaLensMode = ' '; - TmpFileList{3}(ii).scientaRegionIterations = NaN; - TmpFileList{3}(ii).scientaSequenceIterations = NaN; - TmpFileList{3}(ii).imageRoh = {NaN}; - TmpFileList{3}(ii).spectrum_countsPerSecondRoh = NaN; - TmpFileList{3}(ii).importDate = ' '; - TmpFileList{3}(ii).analysisDir = ' '; - - + % Get OS Dates + tmpCurrentFile = fullfile(TmpFileList{3}(ii).folder, TmpFileList{3}(ii).name); + [cDateOS, ~] = getOSDates(tmpCurrentFile); + TmpFileList{3}(ii).creationDate = datevec(cDateOS); end %% Merge lists into one % -% get index of non-empty structures -tf =~cellfun(@isempty,TmpFileList); +%% Merge lists into one +ListFiles = table(); +for i = 1:length(TmpFileList) + if ~isempty(TmpFileList{i}) -% merge the non-empty list structures into one structure -ListFiles = cat(1,TmpFileList{tf}); + % Convert struct to table + tmpTable = struct2table(TmpFileList{i}, 'AsArray', true); -% remove lines with empty images -for ii = 1:size(ListFiles,1) - tmpLogic(ii) = isempty(ListFiles(ii).image); - noncells = tmpLogic; + % Ensure the 'name' column is a cell-string for consistency + if iscell(tmpTable.name) == 0 + tmpTable.name = cellstr(tmpTable.name); + end + + % Merge using helper + if isempty(ListFiles) + ListFiles = tmpTable; + else + ListFiles = outerjoinRowToTable(ListFiles, tmpTable); + end + end end -ListFiles(noncells) = []; + +% Remove rows where the 'image' column is empty +% not sure why this code is here, probably a workaround for earlier errors. + +% Identify rows where the 'image' column is empty +tmpEmptyImgLogic = cellfun(@isempty, ListFiles.image); +% Extract the names of those files for the warning +tmpDeletedNames = ListFiles.name(tmpEmptyImgLogic); +% If any were found, display the warning +if ~isempty(tmpDeletedNames) + warning('The following files were removed because they contain no image data:'); + fprintf(' - %s\n', tmpDeletedNames{:}); % Vectorized print of all names +end +% Actually remove the rows from the table +ListFiles(tmpEmptyImgLogic, :) = []; clearvars -except ListFiles server dataPath* p - -%% - - - %% Sort by date -% First, we need to convert into a table, because structures can not be sorted: -ListFiles = struct2table(ListFiles,'AsArray',true); -% Then, we sort the table based on the time (6th column, datenum) +% sort the table based on the time (6th column, datenum) ListFiles = sortrows(ListFiles,6); % Then, remove un-needed columns ListFiles = removevars(ListFiles,{'bytes','isdir'}); @@ -1307,14 +1312,12 @@ end %% Make table nicer % -if ~iscell(ListFiles.spectrum_countsPerSecond) - ListFiles.spectrum_countsPerSecond = cell(size(ListFiles,1),1); -end -if ~iscell(ListFiles.spectrum_counts) - ListFiles.spectrum_counts = cell(size(ListFiles,1),1); -end + + + if ~iscell(ListFiles.xRayEkin_eV) - ListFiles.xRayEkin_eV = cell(size(ListFiles,1),1); + % Use num2cell to wrap the existing numbers in cells. + ListFiles.xRayEkin_eV = num2cell(ListFiles.xRayEkin_eV); end if ~iscell(ListFiles.scientaEkin_eV) for ii = 1:size(ListFiles,1) @@ -1325,11 +1328,18 @@ end -% rename date fields and delete those fields not needed -[ListFiles.lastModifiedDatestr] = ListFiles.date; -[ListFiles.lastModifiedDatenum] = ListFiles.datenum; +% ------------------------------------------------------------------------- +% MAP DATES & CLEANUP +% ------------------------------------------------------------------------- + +% Delete the raw OS fields and the temporary uppercase fields to avoid duplicates ListFiles = removevars(ListFiles, {'date','datenum'}); +% Move Dates upfront +ListFiles = movevars(ListFiles,'lastModifiedDatestr','After','creationDate'); +ListFiles = movevars(ListFiles,'lastModifiedDatenum','After','creationDate'); +% ------------------------------------------------------------------------- + % Copy Ekin Data existColumn = strcmp('scientaEkinRoh_eV',ListFiles.Properties.VariableNames); if any(existColumn) @@ -1346,43 +1356,8 @@ if ~iscell(ListFiles.scientaEkinRoh_eV) end ListFiles.scientaEkinRoh_eV = tmp; end -% work around for table variables with fitting results having the wrong -% size: -% first delete the old entries -existColumn = strcmp('peakPosition',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakPosition'); -end -existColumn = strcmp('peakArea',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakArea'); -end -existColumn = strcmp('peakFWHM',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakFWHM'); -end -% then re-create them with right size -% 11/22 change to 5 entries from 4. -existColumn = strcmp('peakPosition',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakPosition= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('peakArea',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakArea= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('peakFWHM',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakFWHM= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('sample',ListFiles.Properties.VariableNames); -if ~any(existColumn) - ListFiles.sample = cellstr(repmat(' ',size(ListFiles,1),1)); -end + % add the imported data to previous list (if passed to function upon call) if istable(p.Results.DataTable) % do this only if previous list was input to function call @@ -1439,7 +1414,7 @@ end %% Export -ExportList = removevars(ListFiles, {'image', 'spectrum_counts', 'spectrum_countsPerSecond', 'scientaEkin_eV', 'beamlineInt', 'imageRoh', 'scientaEkinRoh_eV', 'xRayEkin_eV'}); +ExportList = removevars(ListFiles, {'image', 'scientaEkin_eV', 'beamlineInt', 'imageRoh', 'scientaEkinRoh_eV', 'xRayEkin_eV'}); % TODO check windows formatting try % write a ;-separated list for export to excel. @@ -1723,10 +1698,10 @@ function splitH5Scans(infile) fileinfo = h5info(infile); % Get the original file's modification date before we start - % This will be copied to all new files to preserve the original timestamp - orig_file_info = dir(infile); - orig_mod_datetime = orig_file_info.date; % Get as datetime object - +% This will be copied to all new files to preserve the original timestamp +orig_file_info = dir(infile); +orig_mod_datetime = orig_file_info.date; % Get as datetime object + % Extract the names of all groups at the root level of the file % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} @@ -1816,25 +1791,60 @@ if numel(groupnames) > 3 % Close the HDF5 file to save all changes H5F.close(fid); - % Copy the modification date from the original file to the new file - % Convert MATLAB datenum to timestamp format: YYYYMMDDhhmm.ss - timestamp_str = string(datetime(orig_mod_datetime), 'yyyyMMddHHmm.ss'); - if ismac - touch_cmd = sprintf('touch -t %s "%s"', timestamp_str, outname); - system(touch_cmd); - end + % Copy the modification date from the original file to the new file + % Convert MATLAB datenum to timestamp format: YYYYMMDDhhmm.ss + timestamp_str = string(datetime(orig_mod_datetime), 'yyyyMMddHHmm.ss'); + if ismac + touch_cmd = sprintf('touch -t %s "%s"', timestamp_str, outname); + system(touch_cmd); + end % Print confirmation message showing which file was created - % fprintf('Created: %s\n', outname); + % fprintf('Created: %s\n', outname); end % After all split files are successfully created, delete the original file delete(infile); - % fprintf('Deleted original file: %s\n', infile); + % fprintf('Deleted original file: %s\n', infile); else % If the file has 3 or fewer groups, it doesn't need splitting - % disp('File has three or fewer groups. No processing required.'); + % disp('File has three or fewer groups. No processing required.'); end end + + +function [creationDateOS, lastModifiedDate] = getOSDates(fullPath) +% GETOSDATES Extracts the true OS-level creation and modification dates + +% Get standard modification date +d = dir(fullPath); +if isempty(d) + creationDateOS = ''; + lastModifiedDate = ''; + return; +end +lastModifiedDate = d.date; + +% Get Creation Date based on Operating System +if ispc + % Windows: Use .NET framework for accurate Creation Time + try + fileObj = System.IO.FileInfo(fullPath); + creationDateOS = char(fileObj.CreationTime.ToString('dd-MMM-yyyy HH:mm:ss')); + catch + creationDateOS = d.date; % Fallback + end +elseif ismac || isunix + % macOS/Linux: Use the stat command to get the "Birth" time + [status, cmdOut] = system(['stat -f "%SB" -t "%d-%b-%Y %H:%M:%S" "', fullPath, '"']); + if status == 0 + creationDateOS = strtrim(cmdOut); + else + creationDateOS = d.date; % Fallback + end +else + creationDateOS = d.date; % Fallback for unknown OS +end +end \ No newline at end of file diff --git a/beamtime/nappGenerateSpectra.m b/beamtime/nappGenerateSpectra.m index 1d4984d..f09b1bb 100644 --- a/beamtime/nappGenerateSpectra.m +++ b/beamtime/nappGenerateSpectra.m @@ -109,7 +109,7 @@ defaultMode = 'mean'; expectedMode = {'mean','sum','trapz','AugerMeitnerMean'}; defaultEkinStart = 0; defaultEkinEnd = 0; -defaultTableSelection = true(size(DataTable,1)); +defaultTableSelection = true(size(DataTable,1),1); % Parse input p = inputParser; @@ -182,7 +182,7 @@ for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) % log tmp_4(ii,1) = p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(tmp_1); tmp_5(ii,1) = p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(tmp_2); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') % then do Igorfiles + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') % then do Igorfiles or xy files if p.Results.iterationRange(1) == 0 startRange = 1; endRange = size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2); diff --git a/beamtime/nappPlot.m b/beamtime/nappPlot.m index 7b0468e..c2ce676 100644 --- a/beamtime/nappPlot.m +++ b/beamtime/nappPlot.m @@ -428,7 +428,7 @@ if matches(p.Results.tableVariable,'image') && ~matches(p.Results.mode,'AugerMei 'XColor' , [0 0 0], ... 'YColor' , [0 0 0], ... 'LineWidth' , alw ); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') if contains(p.Results.energyScale,'kinetic') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); @@ -441,7 +441,9 @@ if matches(p.Results.tableVariable,'image') && ~matches(p.Results.mode,'AugerMei 'Parent',gca,'CDataMapping','scaled'); axis(gca,'ij'); set(gca,'Layer','top'); - ylim([1 size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2)]); + if size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2) >= 2 + ylim([1 size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2)]); + end xlim([p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(1) p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(end)]); hXLabel = xlabel('Electron energy [eV]'); hYLabel = ylabel('Sequence iteration'); @@ -536,7 +538,7 @@ if matches(p.Results.tableVariable,'imageRoh') 'XColor' , [0 0 0], ... 'YColor' , [0 0 0], ... 'LineWidth' , alw ); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') if contains(p.Results.energyScale,'kinetic') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); @@ -620,21 +622,21 @@ if contains(p.Results.tableVariable,'spectrum') % Create axes hold(gca,'on'); - if (contains(p.Results.energyScale,'kinetic') & (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) + if (contains(p.Results.energyScale,'kinetic') && (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}'); hplot.LineWidth = lw; hXLabel = xlabel('Electron kinetic energy [eV]'); xlim([p.Results.DataTable(p.Results.tableSelection,:).scientaEkinRange_eV(ii,1) p.Results.DataTable(p.Results.tableSelection,:).scientaEkinRange_eV(ii,2)]); - elseif (contains(p.Results.energyScale,'binding') & (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw')| endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) + elseif (contains(p.Results.energyScale,'binding') && (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw')|| endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}'); hplot.LineWidth = lw; hXLabel= xlabel('Binding energy [eV]'); xlim([min(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii}) max(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii})]); - elseif (contains(p.Results.energyScale,'xRay') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'h5')) + elseif (contains(p.Results.energyScale,'xRay') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'h5')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).xRayEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}); hplot.LineWidth = lw; @@ -699,7 +701,7 @@ if contains(p.Results.tableVariable,'spectrum') % axes('Parent',gcf,'XDir','reverse') hold(gca,'on'); for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) + if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}','DisplayName',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'Color',tmp_color(ii,:)); hplot.LineWidth = lw; @@ -711,7 +713,7 @@ if contains(p.Results.tableVariable,'spectrum') hold(gca,'on'); for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) + if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}','DisplayName',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'Color',tmp_color(ii,:)); hplot.LineWidth = lw; @@ -776,7 +778,7 @@ if contains(p.Results.mode,'iterations') % generate colormap for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); if size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2) >= 7 diff --git a/beamtime/nappScale.m b/beamtime/nappScale.m index b96dc2b..e313769 100644 --- a/beamtime/nappScale.m +++ b/beamtime/nappScale.m @@ -200,7 +200,7 @@ end % repeat this for all columns in input. for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') % get name and date of input file and replace old region with new % region. This will be the files where the peak area is taken from. tmpName = replace(p.Results.DataTable(p.Results.tableSelection,:).name{ii}, ...