diff --git a/beamtime/nappFileList.m b/beamtime/nappFileList.m index fa52d82..929764a 100755 --- a/beamtime/nappFileList.m +++ b/beamtime/nappFileList.m @@ -255,7 +255,7 @@ function ListFiles = nappFileList(dataPath,varargin) % * April 2021, X-ray kinetic energy and images are now made unique % * March 2022, input now using varargin % * March 2022, strict data structure required for local and remote sources -% * November 2025, PShell h5 files for NEXAFS have a new structure; +% * November 2025, PShell h5 files for NEXAFS have a new structure; % * November 2025, added creation date to mac for NEXAFS. % * March 2026, added movie import from photos folder, added jpg as photo % file @@ -362,196 +362,149 @@ end % % THE ** notation needs a recent version of Matlab, 2014 does not work. -if endsWith(p.Results.mode,'Pick') - % select files (Xps or Nexafs, ignoring photos) to be imported from data directory - % (data can be taken from any directory; files need to be in same directory) - [filePick,pathPick] = uigetfile({'*.ibw;*.h5';'*.xy'},... - 'Select One or More Files', dataPath,... - 'MultiSelect', 'on'); - % make lists for Nexafs and for XPS - if ischar(filePick) - filePick = {filePick}; - end - filePickXps = filePick(endsWith(filePick,'.ibw')); - filePickNexafs = filePick(endsWith(filePick,'.h5')); - filePickXY = filePick(endsWith(filePick,'.xy')); +if endsWith(p.Results.mode, 'Pick') + % --- PICK MODE --- + [filePick, pathPick] = uigetfile({'*.ibw;*.xy;*.h5'}, ... + 'Select One or More Files', dataPath, 'MultiSelect', 'on'); - if isempty(filePickXps) - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickXps,2) - TmpFileList{1}(ii,1) = dir([pathPick filePickXps{ii}]); + if ischar(filePick), filePick = {filePick}; end % Force cell array + + % Initialize empty dir structs + TmpFileList{1} = struct('name',{},'folder',{},'date',{},'bytes',{},'isdir',{},'datenum',{}); + TmpFileList{2} = TmpFileList{1}; + TmpFileList{3} = TmpFileList{1}; % Photos always empty in pick mode + + % Populate XPS and NEXAFS + if ~isequal(filePick, 0) % If user didn't cancel + for ii = 1:length(filePick) + if endsWith(filePick{ii}, {'.ibw', '.xy'}, 'IgnoreCase', true) + TmpFileList{1} = [TmpFileList{1}; dir(fullfile(pathPick, filePick{ii}))]; + elseif endsWith(filePick{ii}, '.h5', 'IgnoreCase', true) + TmpFileList{2} = [TmpFileList{2}; dir(fullfile(pathPick, filePick{ii}))]; + end end end - if isempty(filePickNexafs) - TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickNexafs,2) - TmpFileList{2}(ii,1) = dir([pathPick filePickNexafs{ii}]); - end - end - if isempty(filePickXY) - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.trallalla']); - else - for ii = 1:size(filePickXps,2) - TmpFileList{1}(ii,1) = dir([pathPick filePickXps{ii}]); - end - end - % make a empty file list for photos, to be consitent - TmpFileList{3} = dir([dataPath_Photo filesep '**' filesep '*.trallalla']); + else - % SES files in IGOR format or SpecsLab in .xy format: list all files, then get rid of system files - if contains(p.Results.mode,'data') - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.xy']); - TmpFileList{1} = TmpFileList{1}(find(~cellfun(@isempty,regexp({TmpFileList{1}.name},'^[a-zA-Z0-9]\w*')))); %#ok - else - TmpFileList{1} = dir([dataPath_XPS filesep '**' filesep '*.ibw']); - TmpFileList{1} = TmpFileList{1}(find(~cellfun(@isempty,regexp({TmpFileList{1}.name},'^[a-zA-Z0-9]\w*')))); %#ok + % --- FOLDER SEARCH MODE --- + + % 1. XPS (Both .ibw and .xy) + tmpIbw = dir(fullfile(dataPath_XPS, '**', '*.ibw')); + tmpIbw = tmpIbw(find(~cellfun(@isempty,regexp({tmpIbw .name},'^[a-zA-Z0-9]\w*')))); %#ok + tmpXy = dir(fullfile(dataPath_XPS, '**', '*.xy')); + tmpXy = tmpXy(find(~cellfun(@isempty,regexp({tmpXy .name},'^[a-zA-Z0-9]\w*')))); %#ok + TmpFileList{1} = [tmpIbw; tmpXy]; % Stack them together + + % 2. NEXAFS + TmpFileList{2} = dir(fullfile(dataPath_NEXAFS, '**', '*.h5')); + TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok + + + % 3. Photos + photoExts = {'*.bmp', '*.jpg', '*.JPG', '*.mov', '*.avi', '*.mp4', '*.wmv'}; + TmpFileList{3} = []; + for i = 1:length(photoExts) + TmpFileList{3} = [TmpFileList{3}; dir(fullfile(dataPath_Photo, '**', photoExts{i}))]; end + TmpFileList{3} = TmpFileList{3}(find(~cellfun(@isempty,regexp({TmpFileList{3}.name},'^[a-zA-Z0-9]\w*')))); %#ok + + % Clean up system/hidden files for all lists + for i = 1:3 + if ~isempty(TmpFileList{i}) + % Keep only files that do NOT start with '.' or '~' + validIdx = ~startsWith({TmpFileList{i}.name}, {'.', '~'}); + TmpFileList{i} = TmpFileList{i}(validIdx); + end + end + + % ========================================================================= + % SPLIT HDF5 FILES WITH MULTIPLE DATASETS INTO INDIVIDUAL FILES + % ========================================================================= + % + % BACKGROUND: + % HDF5 files from the beamline can contain multiple experimental scans + % (e.g., scan_1, scan_2, scan_3) stored as separate groups within a + % single file. However, the NEXAFS and XPS data processing pipelines + % are designed to work with one scan per file. + % + % PURPOSE: + % This script identifies HDF5 files that contain multiple scans and + % splits each scan into a separate file with a descriptive name. + % Each output file contains only one scan (renamed to scan_1) plus + % the logs and metadata groups. + % + % PROCESS: + % 1. Loop through all NEXAFS HDF5 files in the file list + % 2. Check if each file contains more than 3 groups (indicating multiple scans) + % 3. If it does, split it using the splitH5Scans function + % 4. The original file is deleted after splitting + % + % OUTPUT: + % For each multi-scan file, creates separate output files: + % Example: 0002_..._O_Kedge.h5  0002_..._O_Kedge_scan_1.h5 + % 0002_..._O_Kedge_scan_2.h5 + % ========================================================================= + % Loop through each file in the NEXAFS file list + % TmpFileList{2} contains the full paths to all NEXAFS HDF5 files + % ii is the loop counter (1, 2, 3, ... up to the total number of files) + for ii = 1:size(TmpFileList{2}, 1) + + % Get the filename of the current file being processed + current_file = [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]; + + % Read information about the structure of the HDF5 file + % This tells us what groups (datasets) are stored in the file + fileinfo = h5info(current_file); + + % Extract the names of all groups at the root level + % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} + groupnames = {fileinfo.Groups.Name}; + + % Check if this file has more than 3 groups + % Files with more than 3 groups likely contain multiple scans + % (the extra groups beyond scans are typically '/logs' and '/__DATA_TYPES__') + if numel(groupnames) > 3 + + % This file contains multiple scans, so split it + % The splitH5Scans function will: + % - Create separate files for each scan + % - Rename each scan to '/scan_1' within its new file + % - Delete the original multi-scan file when done + splitH5Scans(current_file); + + end + + end + + % ========================================================================= + % NOTE: After this loop completes, all NEXAFS files in data folder + % will contain exactly one scan per file, ready for processing + % ========================================================================= % EPICS files in h5 format list all files, then get rid of system files TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.h5']); TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok - % Photos list all files, then get rid of system files - TmpFileList{3} = [ - dir([dataPath_Photo filesep '**' filesep '*.bmp']) - dir([dataPath_Photo filesep '**' filesep '*.jpg']) - dir([dataPath_Photo filesep '**' filesep '*.JPG']) - dir([dataPath_Photo filesep '**' filesep '*.mov']) - dir([dataPath_Photo filesep '**' filesep '*.avi']) - dir([dataPath_Photo filesep '**' filesep '*.mp4']) - dir([dataPath_Photo filesep '**' filesep '*.wmv']) - ]; - TmpFileList{3} = TmpFileList{3}(find(~cellfun(@isempty,regexp({TmpFileList{3}.name},'^[a-zA-Z0-9]\w*')))); %#ok - % The second line of code filters all files that start with Numbers or - % letters. All system files starting with a . or ~ are discarded. -% ========================================================================= -% SPLIT HDF5 FILES WITH MULTIPLE DATASETS INTO INDIVIDUAL FILES -% ========================================================================= -% -% BACKGROUND: -% HDF5 files from the beamline can contain multiple experimental scans -% (e.g., scan_1, scan_2, scan_3) stored as separate groups within a -% single file. However, the NEXAFS and XPS data processing pipelines -% are designed to work with one scan per file. -% -% PURPOSE: -% This script identifies HDF5 files that contain multiple scans and -% splits each scan into a separate file with a descriptive name. -% Each output file contains only one scan (renamed to scan_1) plus -% the logs and metadata groups. -% -% PROCESS: -% 1. Loop through all NEXAFS HDF5 files in the file list -% 2. Check if each file contains more than 3 groups (indicating multiple scans) -% 3. If it does, split it using the splitH5Scans function -% 4. The original file is deleted after splitting -% -% OUTPUT: -% For each multi-scan file, creates separate output files: -% Example: 0002_..._O_Kedge.h5  0002_..._O_Kedge_scan_1.h5 -% 0002_..._O_Kedge_scan_2.h5 -% ========================================================================= -% Loop through each file in the NEXAFS file list -% TmpFileList{2} contains the full paths to all NEXAFS HDF5 files -% ii is the loop counter (1, 2, 3, ... up to the total number of files) -for ii = 1:size(TmpFileList{2}, 1) - - % Get the filename of the current file being processed - current_file = [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]; - - % Read information about the structure of the HDF5 file - % This tells us what groups (datasets) are stored in the file - fileinfo = h5info(current_file); - - % Extract the names of all groups at the root level - % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} - groupnames = {fileinfo.Groups.Name}; - - % Check if this file has more than 3 groups - % Files with more than 3 groups likely contain multiple scans - % (the extra groups beyond scans are typically '/logs' and '/__DATA_TYPES__') - if numel(groupnames) > 3 - - % This file contains multiple scans, so split it - % The splitH5Scans function will: - % - Create separate files for each scan - % - Rename each scan to '/scan_1' within its new file - % - Delete the original multi-scan file when done - splitH5Scans(current_file); - + disp('*************************************************') + disp('* Voila: Splitting NEXAFS files done *') + disp('*************************************************') + + % --- Date of First File Check --- + % Since TmpFileList{1} now safely contains all .ibw and .xy files, + % we don't need to run dir() again! We just check the struct directly. + if isempty(TmpFileList{1}) + warning('There appears to be no XPS file (.ibw or .xy) in the folder. Add at least one file first.'); + return; end - -end - -% ========================================================================= -% NOTE: After this loop completes, all NEXAFS files in data folder -% will contain exactly one scan per file, ready for processing -% ========================================================================= - -% EPICS files in h5 format list all files, then get rid of system files - TmpFileList{2} = dir([dataPath_NEXAFS filesep '**' filesep '*.h5']); - TmpFileList{2} = TmpFileList{2}(find(~cellfun(@isempty,regexp({TmpFileList{2}.name},'^[a-zA-Z0-9]\w*')))); %#ok - -disp('*************************************************') -disp('* Voila: Splitting NEXAFS files done *') -disp('*************************************************') - - % only process new files - if (istable(p.Results.DataTable) & matches(p.Results.mode,'localNew')) % do this only if previous list was input to function call - tmpXpsFiles = endsWith(p.Results.DataTable.name,'.ibw'); % get Scienta XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{1}(:).name},p.Results.DataTable.name(tmpXpsFiles,:)); % find those files that are yet imported - TmpFileList{1} = TmpFileList{1}(tmpMissingFiles); - - tmpXpsFiles = endsWith(p.Results.DataTable.name,'.xy'); % get Specs XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{1}(:).name},p.Results.DataTable.name(tmpXpsFiles,:)); % find those files that are yet imported - TmpFileList{1} = TmpFileList{1}(tmpMissingFiles); - - tmpNexafsFiles = endsWith(p.Results.DataTable.name,'.h5'); % get XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{2}(:).name},p.Results.DataTable.name(tmpNexafsFiles,:)); % find those files that are yet imported - TmpFileList{2} = TmpFileList{2}(tmpMissingFiles); - - tmpPhotoFiles = endsWith(p.Results.DataTable.name,'.bmp'); % get XPS files in current data table - tmpMissingFiles = ~ismember({TmpFileList{3}(:).name},p.Results.DataTable.name(tmpPhotoFiles,:)); % find those files that are yet imported - TmpFileList{3} = TmpFileList{3}(tmpMissingFiles); + try + filedate_first = TmpFileList{1}(1).datenum; + catch + warning('There appears to be something wrong with the XPS file metadata.'); end -end - - - -% get date of first .ibw or .xy file in folder . -if contains(p.Results.mode,'data') - tmp.fileinfo = dir([dataPath_XPS '/*.xy']); -else - tmp.fileinfo = dir([dataPath_XPS '/*.ibw']); -end -if isempty(tmp.fileinfo) - warning('There appears to be no file in the SES folder. Add at least one file first.') - disp('There appears to be no file in the SES folder. Add at least one file first.') - disp('I quit now') - return -end - -try - if contains(p.Results.mode,'data') - tmp.fileinfo = dir([dataPath_XPS '/*.xy']); - filedate_first = tmp.fileinfo(1).datenum; - clearvars name tmp - else - tmp.fileinfo = dir([dataPath_XPS '/*.ibw']); - filedate_first = tmp.fileinfo(1).datenum; - clearvars name tmp - end -catch - warning('There appears to be something wrong with the SES or Specs file.') - disp('There appears to be something wrong with the SES or Specs file.') end - - %% Import XPS data to get SES settings % % Uses the IgorPro importer for Matlab: @@ -564,250 +517,317 @@ end % and try to resolve it. I suggest to rename these original data files that % are damaged, or not complete (because measurement was aborted) to % "ignore_filename". +%% 2. Import XPS data +ProcessedXPS = []; % This will hold our final list of structs -if contains(p.Results.mode,'data') % for Specs xy data %TODO this is based on file location, needs work!!!! - for ii = 1:size(TmpFileList{1},1) % cycle through each XPS file - +if ~isempty(TmpFileList{1}) + for ii = 1:length(TmpFileList{1}) + current_file_info = TmpFileList{1}(ii); + current_path = fullfile(current_file_info.folder, current_file_info.name); + % ================================================================= + % 1. INITIALIZE MASTER TEMPLATE (Strict Definition) + % ================================================================= + % Every variable MUST be declared here so the struct sizes match perfectly + + current_file_info.creationDate = [NaN NaN NaN NaN NaN NaN]; + current_file_info.lastModifiedDatenum = current_file_info.datenum; + current_file_info.lastModifiedDatestr = current_file_info.date; + current_file_info.image = {[]}; + current_file_info.imageRoh = {[]}; + current_file_info.xRayEkin_eV = NaN; + current_file_info.xRayEkinRange_eV = [NaN NaN]; + current_file_info.scientaPassEnergy_eV = NaN; + current_file_info.scientaDwellTime_ms = NaN; + current_file_info.regionName = ''; + current_file_info.scientaAcquisitionMode = ''; + current_file_info.scientaEkinRange_eV = [NaN NaN]; + current_file_info.scientaEkinStep_eV = NaN; + current_file_info.scientaEkin_eV = {[]}; + current_file_info.scientaEkinRoh_eV = {[]}; + current_file_info.beamlineInt = {[]}; + current_file_info.scientaLensMode = ''; + current_file_info.scientaAnalyzerSlit = ''; + current_file_info.scientaWorkfunction_eV = NaN; + current_file_info.scientaRegionIterations = NaN; + current_file_info.scientaSequenceIterations = NaN; + current_file_info.importDate = datestr(now); try - %data - [fid,~] = fopen([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name],'rt'); - tmpSpecsData = textscan(fid,'%f%f','Delimiter', '\b', 'CommentStyle','#'); % this reads the data; comment lines are ignored - fclose(fid); - % comments - opts = delimitedTextImportOptions("NumVariables", 1); - % Specify range and delimiter - opts.DataLines = [1, 45]; - opts.Delimiter = "\t"; - % Specify column names and types - opts.VariableNames = "metaData"; - opts.VariableTypes = "char"; - % Specify file level properties - opts.ExtraColumnsRule = "ignore"; - opts.EmptyLineRule = "skip"; - opts.ConsecutiveDelimitersRule = "join"; - % Specify variable properties - opts = setvaropts(opts, "metaData", "WhitespaceRule", "preserve"); - opts = setvaropts(opts, "metaData", "EmptyFieldRule", "auto"); - tmpSpecsComments = readtable([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name],opts); - - - catch - warning('Problem reading the Specs file') - disp('Problem reading the Specs file') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) + current_file_info.analysisDir = file.mFile; + catch + current_file_info.analysisDir = pwd; end - - - - try - %% Convert Specs meta data into variables - - % data - tmpEndFirstSpectrum = find(diff(tmpSpecsData{1})<0,1); % FIXME: imports only first spectrum - % counts - PE.y = tmpSpecsData{2}(1:tmpEndFirstSpectrum); - % kinetic energy - PE.x = tmpSpecsData{1}(1:tmpEndFirstSpectrum); - % RayEnergy - PE.xRayEkin_eV{:} = str2double(extractAfter(tmpSpecsComments(29,:).metaData{:},"# Excitation Energy:")); - PE.xRayEkinRange_eV =[ PE.xRayEkin_eV{1} PE.xRayEkin_eV{end}]; - % Start date/time of measurement - PE.creationDate = extractBetween(char(tmpSpecsComments(20,:).metaData{:}),"# Acquisition Date:","UTC"); - PE.creationDate = datetime(PE.creationDate,'InputFormat','MM/dd/yy HH:mm:ss','PivotYear',2000,'TimeZone','UTC'); - % Pass energy - PE.passEnergy = str2double(extractAfter(tmpSpecsComments(31,:).metaData{:},"# Pass Energy:")); - % Dwell time (or step time) - PE.dwellTime = str2double(extractAfter(tmpSpecsComments(28,:).metaData{:},"# Dwell Time: ")); - % Energy step - tmpEnergyStep = diff(PE.x); - PE.scientaEkinStep = tmpEnergyStep(1); - if range(tmpEnergyStep) ~= 0 - disp("Ups something wrong with the energy steps") - end - % Region name - PE.regionName = strtrim(extractAfter(tmpSpecsComments(18,:).metaData{:},"# Region:")); - % Acquisition Mode - PE.acquisitionMode = strtrim(extractAfter(tmpSpecsComments(25,:).metaData{:},"# Scan Mode:")); - PE.regionIterations = str2double(extractAfter(tmpSpecsComments(39,:).metaData{:},"# Number of Scans:")); - % counts/s - PE.y = PE.y./PE.regionIterations; - % Pass energy - PE.passEnergy = str2double(extractAfter(tmpSpecsComments(31,:).metaData{:},"# Pass Energy:")); - catch - warning('Problem reading the PE variable') - disp('Problem reading the PE variable') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - - %% Write SES meta data variables into XPS FileList structure - - try - TmpFileList{1}(ii).creationDate = datevec(PE.creationDate); - TmpFileList{1}(ii).image = {NaN}; - TmpFileList{1}(ii).spectrum_counts = NaN; - TmpFileList{1}(ii).spectrum_countsPerSecond = PE.y; - TmpFileList{1}(ii).xRayEkin_eV = PE.xRayEkin_eV; - TmpFileList{1}(ii).xRayEkinRange_eV = PE.xRayEkinRange_eV; - % SES pass energy - TmpFileList{1}(ii).scientaPassEnergy_eV = PE.passEnergy; % in eV - % SES dwell time - TmpFileList{1}(ii).scientaDwellTime_ms = PE.dwellTime; % in ms - % SES region Name - TmpFileList{1}(ii).regionName = PE.regionName; - TmpFileList{1}(ii).scientaAcquisitionMode = PE.acquisitionMode; - % SES kinetic energy window - TmpFileList{1}(ii).scientaEkinRange_eV = [PE.x(1) PE.x(end)]; - TmpFileList{1}(ii).scientaEkinStep_eV = PE.scientaEkinStep; - TmpFileList{1}(ii).scientaEkin_eV = PE.x; - % Beamline Intensity (I0 during NEXAFS) - TmpFileList{1}(ii).beamlineInt = cell(1); - % SES lens mode - TmpFileList{1}(ii).scientaLensMode = {' '}; - TmpFileList{1}(ii).scientaRegionIterations = PE.regionIterations; - TmpFileList{1}(ii).scientaSequenceIterations = NaN; - TmpFileList{1}(ii).imageRoh = {NaN}; - TmpFileList{1}(ii).spectrum_countsPerSecondRoh = PE.y; - TmpFileList{1}(ii).importDate = datestr(now); + current_file_info.sample = ''; + % ================================================================= + % --- Case A: Scienta (.ibw) Files --- + % ================================================================= + if endsWith(current_file_info.name, '.ibw', 'IgnoreCase', true) try - TmpFileList{1}(ii).analysisDir = file.mFile; - catch - TmpFileList{1}(ii).analysisDir = pwd; + PE = IBWread(current_path); + fclose('all'); + tmp = regexp(PE.WaveNotes, '\r', 'split'); + + try + dateStr = [char(regexp(tmp{29},'((?<=Date=).*)','match')) ' ' char(regexp(tmp{30},'((?<=Time=).*)','match'))]; + parsedDate = datevec(dateStr); + if size(parsedDate, 2) == 6 + current_file_info.creationDate = parsedDate; + current_file_info.lastModifiedDatenum = datenum(parsedDate); + current_file_info.lastModifiedDatestr = dateStr; + end + catch + end + + current_file_info.scientaPassEnergy_eV = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); + current_file_info.scientaDwellTime_ms = str2double(char(regexp(tmp{15},'((?<=Step Time=).*)','match'))); + current_file_info.scientaEkinStep_eV = str2double(char(regexp(tmp{14},'((?<=Energy Step=).*)','match'))); + current_file_info.regionName = char(regexp(tmp{3},'((?<=Region Name=).*)','match')); + current_file_info.scientaAcquisitionMode = char(regexp(tmp{9},'((?<=Acquisition Mode=).*)','match')); + current_file_info.scientaLensMode = char(regexp(tmp{4},'((?<=Lens Mode=).*)','match')); + + PE.x = PE.x0(1) + (0:size(PE.y,1)-1)*PE.dx(1); + current_file_info.scientaEkin_eV = {PE.x}; + current_file_info.scientaEkinRange_eV = [PE.x(1) PE.x(end)]; + + if PE.Ndim == 3 + current_file_info.scientaRegionIterations = size(PE.y,2); + current_file_info.scientaSequenceIterations = size(PE.y,3); + PE.y = squeeze(mean(PE.y,2)); + else + current_file_info.scientaRegionIterations = 1; + current_file_info.scientaSequenceIterations = size(PE.y,2); + end + + PE.y = PE.y .* 1000 ./ current_file_info.scientaDwellTime_ms; + current_file_info.image = {PE.y}; + current_file_info.imageRoh = {PE.y}; + + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok + catch ME + fprintf('Warning: Failed to parse %s (%s)\n', current_file_info.name, ME.message); + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok + end + + % ================================================================= + % --- Case B: Specs (.xy) Files --- + % ================================================================= + elseif endsWith(current_file_info.name, '.xy', 'IgnoreCase', true) + try + [cDateOS, mDateOS] = getOSDates(current_path); + try, osDateVec = datevec(datetime(cDateOS, 'InputFormat', 'dd-MMM-yyyy HH:mm:ss')); + catch, try osDateVec = datevec(mDateOS); catch, osDateVec = [NaN NaN NaN NaN NaN NaN]; end, end + if isempty(osDateVec) || size(osDateVec, 1) == 0, osDateVec = [NaN NaN NaN NaN NaN NaN]; end + + fileLines = readlines(current_path); + regionStarts = find(startsWith(fileLines, '# Region:')); + + % --- 1. PARSE GLOBAL HEADER (Before First Region) --- + % SpecsLab moved many parameters to the top of the file in newer versions + globalScanMode = ''; + globalLensMode = ''; + globalAnalyzerSlit = ''; + globalWorkFunction = NaN; + globalExEnergy = NaN; + + if ~isempty(regionStarts) + globalHeader = fileLines(1:regionStarts(1)-1); + for L = 1:length(globalHeader) + lineStr = strtrim(globalHeader(L)); + lineStr = regexprep(lineStr, '^#\s+', '# '); % Normalize spaces after # + + if startsWith(lineStr, '# Scan Mode:') + globalScanMode = strtrim(extractAfter(lineStr, '# Scan Mode:')); + elseif startsWith(lineStr, '# Analyzer Lens Voltage:') + globalLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens Voltage:')); + elseif startsWith(lineStr, '# Analyzer Lens:') + globalLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens:')); + elseif startsWith(lineStr, '# Analyzer Slit:') + globalAnalyzerSlit = strtrim(extractAfter(lineStr, '# Analyzer Slit:')); + elseif startsWith(lineStr, '# Work Function:') + globalWorkFunction = sscanf(strtrim(extractAfter(lineStr, '# Work Function:')), '%f'); + elseif startsWith(lineStr, '# Eff. Workfunction:') + globalWorkFunction = sscanf(strtrim(extractAfter(lineStr, '# Eff. Workfunction:')), '%f'); + elseif startsWith(lineStr, '# Excitation Energy:') + globalExEnergy = sscanf(strtrim(extractAfter(lineStr, '# Excitation Energy:')), '%f'); + end + end + end + + for r = 1:length(regionStarts) + regionEntry = current_file_info; + startIdx = regionStarts(r); + if r < length(regionStarts), endIdx = regionStarts(r+1)-1; else, endIdx = length(fileLines); end + regionBlock = fileLines(startIdx:endIdx); + + % ===================================================== + % 2. Extract Metadata + % ===================================================== + % Initialize defaults + rName = 'UnknownRegion'; + exEnergyStr = num2str(globalExEnergy); + specID = r; + regionEntry.xRayEkin_eV = globalExEnergy; + regionEntry.scientaPassEnergy_eV = NaN; + regionEntry.scientaDwellTime_ms = NaN; + + % Inherit defaults from Global Header + regionEntry.scientaAcquisitionMode = globalScanMode; + regionEntry.scientaLensMode = globalLensMode; + regionEntry.scientaAnalyzerSlit = globalAnalyzerSlit; + regionEntry.scientaWorkfunction_eV = globalWorkFunction; + regionEntry.creationDate = osDateVec; + + % Scan the header lines directly + for L = 1:min(150, length(regionBlock)) % Metadata is always at the top + lineStr = strtrim(regionBlock(L)); + if isempty(lineStr) || ~startsWith(lineStr, '#'), continue; end + + lineStr = regexprep(lineStr, '^#\s+', '# '); % Normalize spaces after # + + if startsWith(lineStr, '# Region:') + rName = strtrim(extractAfter(lineStr, '# Region:')); + elseif startsWith(lineStr, '# Excitation Energy:') + regionEntry.xRayEkin_eV = sscanf(strtrim(extractAfter(lineStr, '# Excitation Energy:')), '%f'); + exEnergyStr = num2str(regionEntry.xRayEkin_eV); + elseif startsWith(lineStr, '# Spectrum ID:') + specID = sscanf(strtrim(extractAfter(lineStr, '# Spectrum ID:')), '%f'); + elseif startsWith(lineStr, '# Acquisition Date:') + rawDateStr = strtrim(extractAfter(lineStr, '# Acquisition Date:')); + dateStr = strtrim(regexprep(rawDateStr, 'UTC$', '')); + try + dtUTC = datetime(dateStr, 'InputFormat', 'MM/dd/yy HH:mm:ss', 'PivotYear', 2000, 'TimeZone', 'UTC'); + dtLocal = datetime(dtUTC, 'TimeZone', 'Europe/Zurich'); + regionEntry.lastModifiedDatenum = datenum(dtLocal); + regionEntry.lastModifiedDatestr = datestr(dtLocal); + catch + regionEntry.lastModifiedDatestr = rawDateStr; + end + elseif startsWith(lineStr, '# Pass Energy:') + regionEntry.scientaPassEnergy_eV = str2double(extractAfter(lineStr, '# Pass Energy:')); + elseif startsWith(lineStr, '# Dwell Time:') + regionEntry.scientaDwellTime_ms = str2double(extractAfter(lineStr, '# Dwell Time:')) * 1000; + elseif startsWith(lineStr, '# Scan Mode:') + regionEntry.scientaAcquisitionMode = strtrim(extractAfter(lineStr, '# Scan Mode:')); + elseif startsWith(lineStr, '# Analyzer Lens Voltage:') + regionEntry.scientaLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens Voltage:')); + elseif startsWith(lineStr, '# Analyzer Lens:') + regionEntry.scientaLensMode = strtrim(extractAfter(lineStr, '# Analyzer Lens:')); + elseif startsWith(lineStr, '# Analyzer Slit:') + regionEntry.scientaAnalyzerSlit = strtrim(extractAfter(lineStr, '# Analyzer Slit:')); + elseif startsWith(lineStr, '# Work Function:') + regionEntry.scientaWorkfunction_eV = sscanf(strtrim(extractAfter(lineStr, '# Work Function:')), '%f'); + elseif startsWith(lineStr, '# Eff. Workfunction:') + regionEntry.scientaWorkfunction_eV = sscanf(strtrim(extractAfter(lineStr, '# Eff. Workfunction:')), '%f'); + end + end + + % --- Format Name & Region --- + regionEntry.regionName = rName; + cleanRName = strrep(rName, ' ', '_'); + + % Write the original OS file name + regionEntry.originalFileName = current_file_info.name; + + % Prevent identical filenames by appending _Reg1, _Reg2 if multiple regions exist + if length(regionStarts) > 1 + regionEntry.name = sprintf('%04d_%s_Reg%d_%s.xy', specID, cleanRName, r, exEnergyStr); + else + regionEntry.name = sprintf('%04d_%s_%s.xy', specID, cleanRName, exEnergyStr); + end + + + + % 2. Extract Data + hIdx = find(startsWith(strtrim(regionBlock), '# Cycle:')); + tempD = []; enAxis = []; + + if ~isempty(hIdx) + for d = 1:length(hIdx) + cScn = 1; cCrv = 1; + + headerStr = regionBlock(hIdx(d)); + curveMatch = regexp(headerStr, 'Curve:\s*(\d+)', 'tokens', 'once'); + if ~isempty(curveMatch), cCrv = str2double(curveMatch{1}) + 1; end + + scanMatch = regexp(headerStr, 'Scan:\s*(\d+)', 'tokens', 'once'); + if ~isempty(scanMatch), cScn = str2double(scanMatch{1}) + 1; end + + row = hIdx(d) + 1; + while row <= length(regionBlock) && (startsWith(strtrim(regionBlock(row)), '#') || strlength(strtrim(regionBlock(row))) < 2), row = row + 1; end + dataS = row; + while row <= length(regionBlock) && ~startsWith(strtrim(regionBlock(row)), '#') && strlength(strtrim(regionBlock(row))) > 1, row = row + 1; end + + if dataS < row + numCols = length(sscanf(regionBlock(dataS), '%f')); + raw = sscanf(join(regionBlock(dataS:row-1), ' '), '%f'); + if ~isempty(raw) && numCols >= 2 + m = reshape(raw, numCols, [])'; + if isempty(enAxis), enAxis = m(:,1); end + tempD(1:size(m,1), cCrv, cScn) = m(:,2); + end + end + end + else + row = 1; + while row <= length(regionBlock) && (startsWith(strtrim(regionBlock(row)), '#') || strlength(strtrim(regionBlock(row))) < 2), row = row + 1; end + dataS = row; + while row <= length(regionBlock) && ~startsWith(strtrim(regionBlock(row)), '#') && strlength(strtrim(regionBlock(row))) > 1, row = row + 1; end + + if dataS < row + numCols = length(sscanf(regionBlock(dataS), '%f')); + raw = sscanf(join(regionBlock(dataS:row-1), ' '), '%f'); + if ~isempty(raw) && numCols >= 2 + m = reshape(raw, numCols, [])'; + enAxis = m(:,1); + tempD(1:size(m,1), 1, 1) = m(:,2); + end + end + end + + % 3. Map Dimensions + if ~isempty(tempD) + image2D = squeeze(mean(tempD, 2, 'omitnan')); + if isvector(image2D), image2D = image2D(:); end + else + image2D = []; + end + + regionEntry.image = {image2D}; + regionEntry.imageRoh = {image2D}; + + if ~isnan(regionEntry.xRayEkin_eV) + regionEntry.xRayEkinRange_eV = [regionEntry.xRayEkin_eV regionEntry.xRayEkin_eV]; + end + + if ~isempty(enAxis) + enAxis = reshape(enAxis, 1, []); + regionEntry.scientaEkinRange_eV = [enAxis(1) enAxis(end)]; + regionEntry.scientaEkinStep_eV = enAxis(2) - enAxis(1); + regionEntry.scientaEkin_eV = {enAxis}; + regionEntry.scientaEkinRoh_eV = {enAxis}; + end + + regionEntry.scientaRegionIterations = size(tempD, 3); + + if isempty(ProcessedXPS) + ProcessedXPS = regionEntry; + else + ProcessedXPS = [ProcessedXPS; regionEntry]; %#ok + end + end + catch ME + fprintf('Warning: Failed to parse %s (%s)\n', current_file_info.name, ME.message); + if isempty(ProcessedXPS), ProcessedXPS = current_file_info; else, ProcessedXPS = [ProcessedXPS; current_file_info]; end %#ok end - catch - warning('Problem with a XPS data file for XPS list') - disp('Problem with a XPS data file for XPS list') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - fprintf('Number %.0f: %s \n',ii,datestr(TmpFileList{1}(ii).datenum)) end end -else % for Scienta igor data - for ii = 1:size(TmpFileList{1},1) % cycle through each XPS file - - - try - PE = IBWread([TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]); % read XPS file - catch - warning('Problem reading the Igor file') - disp('Problem reading the Igor file') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - fclose('all'); - - try - %% Convert SES meta data into variables - % - % Upon import, all meta data are written into _WaveNotes_ variable. - % Here, I select the meta data for each setting and write those - % into individual variables that are then added to the list. - - % Split the WaveNotes text string into individual strings at each line break. - % (This is a work-around. Did not manage to select values with regexp - % function only.) - tmp = regexp(PE.WaveNotes, '\r', 'split'); - % Then, read out the values into variables using a combination of string operations: - - % Start date/time of measurement - PE.creationDate = datevec([char(regexp(tmp{29},'((?<=Date=).*)','match')) ... - ' ' char(regexp(tmp{30},'((?<=Time=).*)','match'))]); - % Pass energy - PE.passEnergy = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); - % Dwell time (or step time) - PE.dwellTime = str2double(char(regexp(tmp{15},'((?<=Step Time=).*)','match'))); - % Energy step - PE.scientaEkinStep = str2double(char(regexp(tmp{14},'((?<=Energy Step=).*)','match'))); - % Region name - PE.regionName = char(regexp(tmp{3},'((?<=Region Name=).*)','match')); - % Acquisition Mode - PE.acquisitionMode = char(regexp(tmp{9},'((?<=Acquisition Mode=).*)','match')); - - % kinetic energy - for n = 1:size(PE.y,1) - PE.x(n)= PE.x0(1) + (n-1)*PE.dx(1); - end - % lens mode - PE.lensMode = char(regexp(tmp{4},'((?<=Lens Mode=).*)','match')); - % define iterations - % - % There is a BUG in this code. The SES file apparently has the same - % structure for 1 seq. iteration and several reg. iterations and - % several seq. iterations and 1 region iteration. - if PE.Ndim == 3 - PE.regionIterations = size(PE.y,2); - PE.sequenceIterations = size(PE.y,3); - elseif PE.Ndim == 2 - PE.regionIterations = 1; - PE.sequenceIterations = size(PE.y,2); - else - PE.regionIterations = 1; - PE.sequenceIterations = size(PE.y,2); - end - - % average region iterations - if PE.Ndim == 3 - PE.y = squeeze(mean(PE.y,2)); - elseif PE.Ndim == 2 - % PE.y = squeeze(mean(PE.y,2)); - - end - % normalise data to 1ms - PE.y = PE.y .* 1000 ./ PE.dwellTime; - % Pass energy - PE.passEnergy = str2double(char(regexp(tmp{5},'((?<=Pass Energy=).*)','match'))); - catch - warning('Problem reading the PE variable') - disp('Problem reading the PE variable') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - end - - %% Write SES meta data variables into XPS FileList structure - - try - TmpFileList{1}(ii).creationDate = PE.creationDate; - TmpFileList{1}(ii).image = PE.y; - TmpFileList{1}(ii).spectrum_counts = NaN; - TmpFileList{1}(ii).spectrum_countsPerSecond = NaN; - TmpFileList{1}(ii).xRayEkin_eV = NaN; - TmpFileList{1}(ii).xRayEkinRange_eV = [NaN NaN]; - % SES pass energy - TmpFileList{1}(ii).scientaPassEnergy_eV = PE.passEnergy; % in eV - % SES dwell time - TmpFileList{1}(ii).scientaDwellTime_ms = PE.dwellTime; % in ms - % SES region Name - TmpFileList{1}(ii).regionName = PE.regionName; - TmpFileList{1}(ii).scientaAcquisitionMode = PE.acquisitionMode; - % SES kinetic energy window - TmpFileList{1}(ii).scientaEkinRange_eV = [PE.x(1) PE.x(end)]; - TmpFileList{1}(ii).scientaEkinStep_eV = PE.scientaEkinStep; - TmpFileList{1}(ii).scientaEkin_eV = PE.x; - % Beamline Intensity (I0 during NEXAFS) - TmpFileList{1}(ii).beamlineInt = cell(1); - % SES lens mode - TmpFileList{1}(ii).scientaLensMode = PE.lensMode; - TmpFileList{1}(ii).scientaRegionIterations = PE.regionIterations; - TmpFileList{1}(ii).scientaSequenceIterations = PE.sequenceIterations; - TmpFileList{1}(ii).imageRoh = PE.y; - TmpFileList{1}(ii).spectrum_countsPerSecondRoh = NaN; - TmpFileList{1}(ii).importDate = datestr(now); - try - TmpFileList{1}(ii).analysisDir = file.mFile; - catch - TmpFileList{1}(ii).analysisDir = pwd; - end - catch - warning('Problem with a XPS data file for XPS list') - disp('Problem with a XPS data file for XPS list') - fprintf('Number %.0f: %s \n',ii,[TmpFileList{1}(ii).folder filesep TmpFileList{1}(ii).name]) - fprintf('Number %.0f: %s \n',ii,datestr(TmpFileList{1}(ii).datenum)) - end - end + TmpFileList{1} = ProcessedXPS; end + disp('*************************************************') disp('* Voila: Done reading & writing XPS data! *') disp('*************************************************') - - %% Read and write NEXAFS meta data % % The h5 file is much nicer organised, so we do not need string operations @@ -817,15 +837,12 @@ disp('*************************************************') % if datenum(filedate_first) >= datenum('31-Dec-2017 23:59:00') %('31-Dec-2019 23:59:00') for ii = 1:size(TmpFileList{2},1) % do this for each list element - %if ~isempty(h5read([TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name],'/scan 1/energy')) - % creation date of file - if ismac - tmpCommand = sprintf('stat -f "%%SB" "%s"', [TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name]); - [~,creation] = system(tmpCommand); - TmpFileList{2}(ii).creationDate = datevec(creation); - else - TmpFileList{2}(ii).creationDate = datevec(NaN); - end + % Get OS Dates + tmpCurrentFile = fullfile(TmpFileList{2}(ii).folder, TmpFileList{2}(ii).name); + [cDateOS, ~] = getOSDates(tmpCurrentFile); + TmpFileList{2}(ii).creationDate = datevec(cDateOS); + + try % read data %if datenum(filedate_first) <= datenum('21-Nov-2020 00:01:00') % Date not exaclty certain. @@ -843,14 +860,9 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end end end - % 21.8.20 changed from spectrum to scienta spectrum, later - % switched back to spectrum. - % Spectrum - TmpFileList{2}(ii).spectrum_counts = NaN; - TmpFileList{2}(ii).spectrum_countsPerSecond = NaN; - + % import Photon Energy (cols) try TmpXrayEkin = h5read([TmpFileList{2}(ii).folder filesep TmpFileList{2}(ii).name],'/scan_1/energy'); % X-ray excitation energy; Photon energy in eV @@ -866,9 +878,7 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end - if isempty(TmpXrayEkin) - TmpFileList{2}(ii).xRayEkinRange_eV = [NaN NaN]; - else + if ~isempty(TmpXrayEkin) TmpFileList{2}(ii).xRayEkinRange_eV = [TmpXrayEkin(1) TmpXrayEkin(end)]; end @@ -960,7 +970,6 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element TmpFileList{2}(ii).scientaEkinRange_eV = [double(tmp(1))-TmpFileList{2}(ii).scientaPassEnergy_eV/20 double(tmp(1))+TmpFileList{2}(ii).scientaPassEnergy_eV/20]; - TmpFileList{2}(ii).scientaEkinStep_eV = NaN; TmpFileList{2}(ii).scientaEkin_eV = linspace(TmpFileList{2}(ii).scientaEkinRange_eV(:,1)... ,TmpFileList{2}(ii).scientaEkinRange_eV(:,2),size(TmpFileList{2}(ii).image,1))'; else @@ -1054,7 +1063,7 @@ for ii = 1:size(TmpFileList{2},1) % do this for each list element end TmpFileList{2}(ii).imageRoh = TmpFileList{2}(ii).image; - TmpFileList{2}(ii).spectrum_countsPerSecondRoh = TmpFileList{2}(ii).spectrum_countsPerSecond; + % TmpFileList{2}(ii).spectrum_countsPerSecondRoh = TmpFileList{2}(ii).spectrum_countsPerSecond; % log TmpFileList{2}(ii).importDate = datestr(now); try @@ -1123,62 +1132,58 @@ disp('*************************************************') for ii = 1:size(TmpFileList{3},1) % do this for each picture in the list - - - TmpFileList{3}(ii).creationDate = datevec(NaN); - TmpFileList{3}(ii).image = NaN; - TmpFileList{3}(ii).spectrum_counts = NaN; - TmpFileList{3}(ii).spectrum_countsPerSecond = NaN; - TmpFileList{3}(ii).xRayEkin_eV = NaN; - TmpFileList{3}(ii).xRayEkinRange_eV = [NaN NaN]; - TmpFileList{3}(ii).scientaPassEnergy_eV = NaN; - TmpFileList{3}(ii).scientaDwellTime_ms = NaN; - TmpFileList{3}(ii).regionName = ' '; - TmpFileList{3}(ii).scientaAcquisitionMode = ' '; - TmpFileList{3}(ii).scientaEkinRange_eV = [NaN NaN]; - TmpFileList{3}(ii).scientaEkinStep_eV = NaN; - TmpFileList{3}(ii).scientaEkin_eV = NaN; - TmpFileList{3}(ii).beamlineInt = cell(1); - TmpFileList{3}(ii).scientaLensMode = ' '; - TmpFileList{3}(ii).scientaRegionIterations = NaN; - TmpFileList{3}(ii).scientaSequenceIterations = NaN; - TmpFileList{3}(ii).imageRoh = {NaN}; - TmpFileList{3}(ii).spectrum_countsPerSecondRoh = NaN; - TmpFileList{3}(ii).importDate = ' '; - TmpFileList{3}(ii).analysisDir = ' '; - - + % Get OS Dates + tmpCurrentFile = fullfile(TmpFileList{3}(ii).folder, TmpFileList{3}(ii).name); + [cDateOS, ~] = getOSDates(tmpCurrentFile); + TmpFileList{3}(ii).creationDate = datevec(cDateOS); end %% Merge lists into one % -% get index of non-empty structures -tf =~cellfun(@isempty,TmpFileList); +%% Merge lists into one +ListFiles = table(); +for i = 1:length(TmpFileList) + if ~isempty(TmpFileList{i}) -% merge the non-empty list structures into one structure -ListFiles = cat(1,TmpFileList{tf}); + % Convert struct to table + tmpTable = struct2table(TmpFileList{i}, 'AsArray', true); -% remove lines with empty images -for ii = 1:size(ListFiles,1) - tmpLogic(ii) = isempty(ListFiles(ii).image); - noncells = tmpLogic; + % Ensure the 'name' column is a cell-string for consistency + if iscell(tmpTable.name) == 0 + tmpTable.name = cellstr(tmpTable.name); + end + + % Merge using helper + if isempty(ListFiles) + ListFiles = tmpTable; + else + ListFiles = outerjoinRowToTable(ListFiles, tmpTable); + end + end end -ListFiles(noncells) = []; + +% Remove rows where the 'image' column is empty +% not sure why this code is here, probably a workaround for earlier errors. + +% Identify rows where the 'image' column is empty +tmpEmptyImgLogic = cellfun(@isempty, ListFiles.image); +% Extract the names of those files for the warning +tmpDeletedNames = ListFiles.name(tmpEmptyImgLogic); +% If any were found, display the warning +if ~isempty(tmpDeletedNames) + warning('The following files were removed because they contain no image data:'); + fprintf(' - %s\n', tmpDeletedNames{:}); % Vectorized print of all names +end +% Actually remove the rows from the table +ListFiles(tmpEmptyImgLogic, :) = []; clearvars -except ListFiles server dataPath* p - -%% - - - %% Sort by date -% First, we need to convert into a table, because structures can not be sorted: -ListFiles = struct2table(ListFiles,'AsArray',true); -% Then, we sort the table based on the time (6th column, datenum) +% sort the table based on the time (6th column, datenum) ListFiles = sortrows(ListFiles,6); % Then, remove un-needed columns ListFiles = removevars(ListFiles,{'bytes','isdir'}); @@ -1307,14 +1312,12 @@ end %% Make table nicer % -if ~iscell(ListFiles.spectrum_countsPerSecond) - ListFiles.spectrum_countsPerSecond = cell(size(ListFiles,1),1); -end -if ~iscell(ListFiles.spectrum_counts) - ListFiles.spectrum_counts = cell(size(ListFiles,1),1); -end + + + if ~iscell(ListFiles.xRayEkin_eV) - ListFiles.xRayEkin_eV = cell(size(ListFiles,1),1); + % Use num2cell to wrap the existing numbers in cells. + ListFiles.xRayEkin_eV = num2cell(ListFiles.xRayEkin_eV); end if ~iscell(ListFiles.scientaEkin_eV) for ii = 1:size(ListFiles,1) @@ -1325,11 +1328,18 @@ end -% rename date fields and delete those fields not needed -[ListFiles.lastModifiedDatestr] = ListFiles.date; -[ListFiles.lastModifiedDatenum] = ListFiles.datenum; +% ------------------------------------------------------------------------- +% MAP DATES & CLEANUP +% ------------------------------------------------------------------------- + +% Delete the raw OS fields and the temporary uppercase fields to avoid duplicates ListFiles = removevars(ListFiles, {'date','datenum'}); +% Move Dates upfront +ListFiles = movevars(ListFiles,'lastModifiedDatestr','After','creationDate'); +ListFiles = movevars(ListFiles,'lastModifiedDatenum','After','creationDate'); +% ------------------------------------------------------------------------- + % Copy Ekin Data existColumn = strcmp('scientaEkinRoh_eV',ListFiles.Properties.VariableNames); if any(existColumn) @@ -1346,43 +1356,8 @@ if ~iscell(ListFiles.scientaEkinRoh_eV) end ListFiles.scientaEkinRoh_eV = tmp; end -% work around for table variables with fitting results having the wrong -% size: -% first delete the old entries -existColumn = strcmp('peakPosition',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakPosition'); -end -existColumn = strcmp('peakArea',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakArea'); -end -existColumn = strcmp('peakFWHM',ListFiles.Properties.VariableNames); -if any(existColumn) - ListFiles = removevars(ListFiles,'peakFWHM'); -end -% then re-create them with right size -% 11/22 change to 5 entries from 4. -existColumn = strcmp('peakPosition',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakPosition= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('peakArea',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakArea= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('peakFWHM',ListFiles.Properties.VariableNames); -if ~any(existColumn) - % save current ekin data before they are modified - ListFiles.peakFWHM= NaN(size(ListFiles,1),5); -end -existColumn = strcmp('sample',ListFiles.Properties.VariableNames); -if ~any(existColumn) - ListFiles.sample = cellstr(repmat(' ',size(ListFiles,1),1)); -end + % add the imported data to previous list (if passed to function upon call) if istable(p.Results.DataTable) % do this only if previous list was input to function call @@ -1439,7 +1414,7 @@ end %% Export -ExportList = removevars(ListFiles, {'image', 'spectrum_counts', 'spectrum_countsPerSecond', 'scientaEkin_eV', 'beamlineInt', 'imageRoh', 'scientaEkinRoh_eV', 'xRayEkin_eV'}); +ExportList = removevars(ListFiles, {'image', 'scientaEkin_eV', 'beamlineInt', 'imageRoh', 'scientaEkinRoh_eV', 'xRayEkin_eV'}); % TODO check windows formatting try % write a ;-separated list for export to excel. @@ -1723,10 +1698,10 @@ function splitH5Scans(infile) fileinfo = h5info(infile); % Get the original file's modification date before we start - % This will be copied to all new files to preserve the original timestamp - orig_file_info = dir(infile); - orig_mod_datetime = orig_file_info.date; % Get as datetime object - +% This will be copied to all new files to preserve the original timestamp +orig_file_info = dir(infile); +orig_mod_datetime = orig_file_info.date; % Get as datetime object + % Extract the names of all groups at the root level of the file % Example: {'/scan_1', '/scan_2', '/logs', '/__DATA_TYPES__'} @@ -1816,25 +1791,60 @@ if numel(groupnames) > 3 % Close the HDF5 file to save all changes H5F.close(fid); - % Copy the modification date from the original file to the new file - % Convert MATLAB datenum to timestamp format: YYYYMMDDhhmm.ss - timestamp_str = string(datetime(orig_mod_datetime), 'yyyyMMddHHmm.ss'); - if ismac - touch_cmd = sprintf('touch -t %s "%s"', timestamp_str, outname); - system(touch_cmd); - end + % Copy the modification date from the original file to the new file + % Convert MATLAB datenum to timestamp format: YYYYMMDDhhmm.ss + timestamp_str = string(datetime(orig_mod_datetime), 'yyyyMMddHHmm.ss'); + if ismac + touch_cmd = sprintf('touch -t %s "%s"', timestamp_str, outname); + system(touch_cmd); + end % Print confirmation message showing which file was created - % fprintf('Created: %s\n', outname); + % fprintf('Created: %s\n', outname); end % After all split files are successfully created, delete the original file delete(infile); - % fprintf('Deleted original file: %s\n', infile); + % fprintf('Deleted original file: %s\n', infile); else % If the file has 3 or fewer groups, it doesn't need splitting - % disp('File has three or fewer groups. No processing required.'); + % disp('File has three or fewer groups. No processing required.'); end end + + +function [creationDateOS, lastModifiedDate] = getOSDates(fullPath) +% GETOSDATES Extracts the true OS-level creation and modification dates + +% Get standard modification date +d = dir(fullPath); +if isempty(d) + creationDateOS = ''; + lastModifiedDate = ''; + return; +end +lastModifiedDate = d.date; + +% Get Creation Date based on Operating System +if ispc + % Windows: Use .NET framework for accurate Creation Time + try + fileObj = System.IO.FileInfo(fullPath); + creationDateOS = char(fileObj.CreationTime.ToString('dd-MMM-yyyy HH:mm:ss')); + catch + creationDateOS = d.date; % Fallback + end +elseif ismac || isunix + % macOS/Linux: Use the stat command to get the "Birth" time + [status, cmdOut] = system(['stat -f "%SB" -t "%d-%b-%Y %H:%M:%S" "', fullPath, '"']); + if status == 0 + creationDateOS = strtrim(cmdOut); + else + creationDateOS = d.date; % Fallback + end +else + creationDateOS = d.date; % Fallback for unknown OS +end +end \ No newline at end of file diff --git a/beamtime/nappGenerateSpectra.m b/beamtime/nappGenerateSpectra.m index 1d4984d..f09b1bb 100644 --- a/beamtime/nappGenerateSpectra.m +++ b/beamtime/nappGenerateSpectra.m @@ -109,7 +109,7 @@ defaultMode = 'mean'; expectedMode = {'mean','sum','trapz','AugerMeitnerMean'}; defaultEkinStart = 0; defaultEkinEnd = 0; -defaultTableSelection = true(size(DataTable,1)); +defaultTableSelection = true(size(DataTable,1),1); % Parse input p = inputParser; @@ -182,7 +182,7 @@ for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) % log tmp_4(ii,1) = p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(tmp_1); tmp_5(ii,1) = p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(tmp_2); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') % then do Igorfiles + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') % then do Igorfiles or xy files if p.Results.iterationRange(1) == 0 startRange = 1; endRange = size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2); diff --git a/beamtime/nappPlot.m b/beamtime/nappPlot.m index 7b0468e..c2ce676 100644 --- a/beamtime/nappPlot.m +++ b/beamtime/nappPlot.m @@ -428,7 +428,7 @@ if matches(p.Results.tableVariable,'image') && ~matches(p.Results.mode,'AugerMei 'XColor' , [0 0 0], ... 'YColor' , [0 0 0], ... 'LineWidth' , alw ); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') if contains(p.Results.energyScale,'kinetic') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); @@ -441,7 +441,9 @@ if matches(p.Results.tableVariable,'image') && ~matches(p.Results.mode,'AugerMei 'Parent',gca,'CDataMapping','scaled'); axis(gca,'ij'); set(gca,'Layer','top'); - ylim([1 size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2)]); + if size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2) >= 2 + ylim([1 size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2)]); + end xlim([p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(1) p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii}(end)]); hXLabel = xlabel('Electron energy [eV]'); hYLabel = ylabel('Sequence iteration'); @@ -536,7 +538,7 @@ if matches(p.Results.tableVariable,'imageRoh') 'XColor' , [0 0 0], ... 'YColor' , [0 0 0], ... 'LineWidth' , alw ); - elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + elseif endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') if contains(p.Results.energyScale,'kinetic') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); @@ -620,21 +622,21 @@ if contains(p.Results.tableVariable,'spectrum') % Create axes hold(gca,'on'); - if (contains(p.Results.energyScale,'kinetic') & (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) + if (contains(p.Results.energyScale,'kinetic') && (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}'); hplot.LineWidth = lw; hXLabel = xlabel('Electron kinetic energy [eV]'); xlim([p.Results.DataTable(p.Results.tableSelection,:).scientaEkinRange_eV(ii,1) p.Results.DataTable(p.Results.tableSelection,:).scientaEkinRange_eV(ii,2)]); - elseif (contains(p.Results.energyScale,'binding') & (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw')| endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) + elseif (contains(p.Results.energyScale,'binding') && (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw')|| endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy'))) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}'); hplot.LineWidth = lw; hXLabel= xlabel('Binding energy [eV]'); xlim([min(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii}) max(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii})]); - elseif (contains(p.Results.energyScale,'xRay') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'h5')) + elseif (contains(p.Results.energyScale,'xRay') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'h5')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).xRayEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}); hplot.LineWidth = lw; @@ -699,7 +701,7 @@ if contains(p.Results.tableVariable,'spectrum') % axes('Parent',gcf,'XDir','reverse') hold(gca,'on'); for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) + if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).scientaEkin_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}','DisplayName',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'Color',tmp_color(ii,:)); hplot.LineWidth = lw; @@ -711,7 +713,7 @@ if contains(p.Results.tableVariable,'spectrum') hold(gca,'on'); for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') | endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) + if (endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy')) hplot = plot(p.Results.DataTable(p.Results.tableSelection,:).bindingEnergy_eV{ii},... p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii}','DisplayName',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'Color',tmp_color(ii,:)); hplot.LineWidth = lw; @@ -776,7 +778,7 @@ if contains(p.Results.mode,'iterations') % generate colormap for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') hdl = figure('Name',p.Results.DataTable(p.Results.tableSelection,:).name{ii},'units', unt,'Position',pos); if size(p.Results.DataTable(p.Results.tableSelection,:).(p.Results.tableVariable){ii},2) >= 7 diff --git a/beamtime/nappScale.m b/beamtime/nappScale.m index b96dc2b..e313769 100644 --- a/beamtime/nappScale.m +++ b/beamtime/nappScale.m @@ -200,7 +200,7 @@ end % repeat this for all columns in input. for ii = 1:size(p.Results.DataTable(p.Results.tableSelection,:),1) - if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') + if endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'ibw') || endsWith(p.Results.DataTable(p.Results.tableSelection,:).name{ii},'xy') % get name and date of input file and replace old region with new % region. This will be the files where the peak area is taken from. tmpName = replace(p.Results.DataTable(p.Results.tableSelection,:).name{ii}, ...