# PEARL Procedures: pearl-pshell-import.ipf Source File

pearl-pshell-import.ipf
Go to the documentation of this file.
#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3 // Use modern global access method and strict wave access.
#pragma IgorVersion = 8.00
#pragma ModuleName = PearlPShellImport
#pragma version = 2.1
#if IgorVersion() < 9.00
#include <HDF5 Browser>
#endif
#include "pearl-compat"
#include "pearl-gui-tools"
#include "pearl-area-import"
// copyright (c) 2013-22 Paul Scherrer Institut
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http:///www.apache.org/licenses/LICENSE-2.0
strconstant kEnergyDimLabel = "energy"
strconstant kAngleDimLabel = "angle"
strconstant kScanDimLabel = "scan"
strconstant kDataDimLabel = "data"
strconstant kPreviewDatasets = "ImageEnergyDistribution;ScientaSpectrum;ScientaImage;Counts;SampleCurrent;"
strconstant kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChannelEnd;ScientaSliceBegin;ScientaSliceEnd;Eph;"
strconstant kEssentialDiagnostics = "ManipulatorX;ManipulatorY;ManipulatorZ;ManipulatorTheta;ManipulatorTilt;ManipulatorPhi;MonoEnergy;"
strconstant kTransposedDatasets = "ScientaImage;"
constant kDSCPositioners = 0x0001
constant kDSCDetectors = 0x0002
constant kDSCScientaScaling = 0x0004
constant kDSCPreview = 0x0008
constant kDSCEssentialDiags = 0x0010
constant kDSCAttrs = 0x0020
constant kDSCDiags = 0x0040
constant kDSCSnaps = 0x0080
constant kDSCMeta = 0x0100
constant kDSCMonitors = 0x0200
constant kDSCRegions = 0x0400
constant kDSCOther = 0x8000
constant kDSCAll = 0xffff
// ====== main import functions ======
function /df psh5_load(path_name, file_name, scans, regions, datasets, [classes, max_rank, reduction_func, reduction_params, dest_df])
string path_name
string file_name
string scans
string regions
string datasets
variable classes
variable max_rank
string reduction_func
string reduction_params
dfref dest_df
dfref save_df = GetDataFolderDFR()
variable timerRefNum = startMSTimer
if (ParamIsDefault(classes) || (classes == 0))
classes = kDSCPositioners | kDSCDetectors | kDSCScientaScaling | kDSCEssentialDiags
endif
variable essential_classes = kDSCPositioners | kDSCScientaScaling | kDSCEssentialDiags
if (ParamIsDefault(dest_df) || !DataFolderRefStatus(dest_df))
dest_df = psh5_open_file(path_name, file_name)
else
dest_df = psh5_open_file(path_name, file_name, dest_df=dest_df)
endif
if (ParamIsDefault(reduction_func))
reduction_func = ""
endif
if (ParamIsDefault(reduction_params))
reduction_params = ""
endif
if (DataFolderRefStatus(dest_df))
setdatafolder dest_df
psh5_load_general_group(dest_df)
// datasets contained in file
svar /sdfr=dest_df file_datasets = s_datasets
// datasets contained in file up to allowed rank
string ranked_datasets = ""
if (ParamIsDefault(max_rank))
ranked_datasets = file_datasets
else
svar /sdfr=dest_df file_ranks = s_datasets_ranks
ranked_datasets = psh5_filter_datasets_rank(file_datasets, file_ranks, 0, max_rank)
endif
string matching_datasets = ""
string matching_essentials = ""
string scan_datasets = ""
string region_datasets = ""
string free_datasets = ""
string selected_datasets = ""
string essential_datasets = ""
variable i_item
variable n_items
string item
// select datasets belonging to selected scans
n_items = ItemsInList(scans, ";")
for (i_item = 0; i_item < n_items; i_item += 1)
item = StringFromList(i_item, scans, ";")
if (cmpstr(item[0,3], "scan") == 0)
item = "/" + item
endif
item = ReplaceString("//", item + "/*", "/")
matching_datasets = psh5_match_datasets(ranked_datasets, item)
scan_datasets = scan_datasets + matching_datasets
endfor
// select datasets belonging to selected regions
n_items = ItemsInList(regions, ";")
for (i_item = 0; i_item < n_items; i_item += 1)
item = StringFromList(i_item, regions, ";")
if (cmpstr(item[0,3], "scan") == 0)
item = "/" + item
endif
item = ReplaceString("//", item + "/*", "/")
matching_datasets = psh5_match_datasets(ranked_datasets, item)
region_datasets = region_datasets + matching_datasets
endfor
// free select datasets
n_items = ItemsInList(datasets, ";")
for (i_item = 0; i_item < n_items; i_item += 1)
item = StringFromList(i_item, datasets, ";")
if (cmpstr(item[0,3], "scan") == 0)
item = "/" + item
endif
matching_datasets = psh5_match_datasets(ranked_datasets, item)
free_datasets = free_datasets + matching_datasets
endfor
selected_datasets = scan_datasets + region_datasets + free_datasets
string filtered_datasets = ""
string diag_datasets = ""
string selected_scans = psh5_extract_scan_paths(selected_datasets)
variable i_scan
variable n_scans = ItemsInList(selected_scans)
string scan
string selected_regions = psh5_extract_region_paths(selected_datasets)
variable i_region
variable n_regions = ItemsInList(selected_regions)
string region
string positioners
string detectors
// datasets directly under one of the selected regions
region_datasets = ""
for (i_region = 0; i_region < n_regions; i_region += 1)
region = StringFromList(i_region, selected_regions, ";")
region_datasets = region_datasets + GrepList(file_datasets, "(?i)^" + region + "[[:alpha:]]+$")
endfor
// filter selected datasets by class and add essential dependencies
// each scan may have specific positioners and detectors
for (i_scan = 0; i_scan < n_scans; i_scan += 1)
scan = StringFromList(i_scan, selected_scans, ";")
// potentially interesting diagnostics of current scan and selected regions
diag_datasets = psh5_match_datasets(file_datasets, scan + "*")
diag_datasets = psh5_match_dataset_classes(diag_datasets, kDSCAttrs | kDSCDiags | kDSCSnaps)
diag_datasets = diag_datasets + GrepList(file_datasets, "(?i)^" + scan + "[[:alpha:]]+$")
diag_datasets = diag_datasets + psh5_match_datasets(region_datasets, scan + "*")
// explicit positioners and detectors set by pshell
setdatafolder dest_df
dfref scan_df = psh5_create_folders(scan)
setdatafolder scan_df
psh5_load_scan_meta(dest_df, scan)
wave /t /z ScanWritables
wave /t /z ScanReadables
if (WaveExists(ScanWritables))
positioners = twave2list(ScanWritables, ";")
else
positioners = ""
endif
if (WaveExists(ScanReadables))
detectors = twave2list(ScanReadables, ";")
else
detectors = ""
endif
// filtering by classes
matching_datasets = psh5_match_dataset_classes(selected_datasets, classes, positioners=positioners, detectors=detectors)
// add essential diags
if (strlen(matching_datasets) > 1)
essential_datasets = psh5_match_dataset_classes(diag_datasets, essential_classes, positioners=positioners, detectors=detectors)
endif
// scaling datasets before detectors because data reduction needs the scales
filtered_datasets = essential_datasets + filtered_datasets + matching_datasets
endfor
// load the datasets
setdatafolder dest_df
string /g s_loaded_datasets = ""
s_loaded_datasets = psh5_load_datasets(dest_df, filtered_datasets, reduction_func=reduction_func, reduction_params=reduction_params)
// apply scaling by scan
for (i_scan = 0; i_scan < n_scans; i_scan += 1)
scan = StringFromList(i_scan, selected_scans, ";")
ps_scale_datasets(psh5_dataset_to_folder(dest_df, scan))
endfor
psh5_close_file(dest_df)
// performance reporting
if (timerRefNum >= 0)
setdatafolder dest_df
variable /g psh5_perf_secs
psh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
endif
endif
setdatafolder save_df
return dest_df
end
function /df psh5_preview(path_name, file_name, [dest_df, preview_datasets])
string path_name
string file_name
dfref dest_df
string preview_datasets
dfref save_df = GetDataFolderDFR()
if (ParamIsDefault(dest_df))
dest_df = psh5_open_file(path_name, file_name)
else
dest_df = psh5_open_file(path_name, file_name, dest_df=dest_df)
endif
if (ParamIsDefault(preview_datasets))
preview_datasets = kPreviewDatasets
endif
variable essential_classes = kDSCPositioners | kDSCScientaScaling
if (DataFolderRefStatus(dest_df))
setdatafolder dest_df
psh5_load_general_group(dest_df)
// select dataset based on preference
svar /sdfr=dest_df file_datasets = s_datasets
svar /sdfr=dest_df file_datasets_ranks = s_datasets_ranks
string selected_datasets = ""
string essential_datasets = ""
string scan_datasets = ""
string filtered_datasets = ""
variable nds = ItemsInList(preview_datasets, ";")
variable ids
string ds
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(ids, preview_datasets, ";")
selected_datasets = psh5_filter_datasets_rank(file_datasets, file_datasets_ranks, 1, 2)
selected_datasets = psh5_match_datasets(selected_datasets, "/scan*" + ds)
if (strlen(selected_datasets) > 1)
selected_datasets = StringFromList(0, selected_datasets, ";")
break
endif
endfor
// add essential dependencies
if (strlen(selected_datasets) > 1)
string selected_scans = psh5_extract_scan_paths(selected_datasets)
string scan
string positioners
scan = StringFromList(0, selected_scans, ";")
scan_datasets = psh5_match_datasets(file_datasets, scan + "*")
psh5_load_scan_meta(dest_df, scan)
wave /t /z ScanWritables
if (WaveExists(ScanWritables))
positioners = twave2list(ScanWritables, ";")
else
positioners = ""
endif
essential_datasets = psh5_match_dataset_classes(scan_datasets, essential_classes, positioners=positioners)
filtered_datasets = essential_datasets + selected_datasets
// load the datasets
psh5_load_datasets(dest_df, filtered_datasets, create_folders=0)
ps_scale_datasets(dest_df)
string /g s_preview_dataset = StringFromList(0, selected_datasets, ";")
string /g s_preview_wave = StringFromList(ItemsInList(s_preview_dataset, "/") - 1, s_preview_dataset, "/")
endif
psh5_close_file(dest_df)
endif
setdatafolder save_df
return dest_df
end
function /df psh5_open_file(path_name, file_name, [dest_df])
string path_name
string file_name
dfref dest_df
dfref save_df = GetDataFolderDFR()
variable fid
HDF5OpenFile /P=$path_name /R fid as file_name
if (v_flag == 0)
if (!ParamIsDefault(dest_df))
setdatafolder dest_df
else
string dest_name = ad_suggest_foldername(s_filename, sourcename="psh")
setdatafolder root:
newdatafolder /s /o $("root:" + dest_name)
endif
dfref file_df = GetDataFolderDFR()
variable /g file_id = fid
string /g s_filepath
string /g s_scanpaths
string /g s_datasets
string datatypes
string ranks
string dimensions
s_filepath = s_path + s_filename
s_scanpaths = psh5_list_scans(file_id)
s_datasets = psh5_list_all_datasets(file_id)
[datatypes, ranks, dimensions] = psh5_list_dataset_info(file_id, s_datasets)
string /g s_datasets_datatypes = datatypes
string /g s_datasets_ranks = ranks
string /g s_datasets_dimensions = dimensions
else
dfref file_df = $""
endif
setdatafolder save_df
return file_df
end
function psh5_close_file(file_df)
dfref file_df
if (DataFolderRefStatus(file_df))
nvar /sdfr=file_df /z file_id
if (nvar_Exists(file_id))
HDF5CloseFile /z file_id
file_id = 0
KillVariables /z file_id
else
dfref parent_df = $(GetDataFolder(1, file_df) + ":")
if (DataFolderRefStatus(parent_df))
psh5_close_file(parent_df)
endif
endif
endif
end
// === datasets and paths ===
static function /s twave2list(wt, sep)
wave /t wt
string sep
string list = ""
variable n = numpnts(wt)
variable i
for (i = 0; i < n; i += 1)
list = AddListItem(wt[i], list, sep, inf)
endfor
return list
end
static function /s wave2list(w, format, sep)
wave w
string format
string sep
string list = ""
variable n = numpnts(w)
variable i
string s
for (i = 0; i < n; i += 1)
sprintf s, format, w[i]
list = AddListItem(s, list, sep, inf)
endfor
return list
end
function /s psh5_list_scans(file_id)
variable file_id
HDF5ListGroup /F /TYPE=1 file_id, "/"
variable ig
variable ng = ItemsInList(S_HDF5ListGroup, ";")
string sg
string scans = ""
for (ig = 0; ig < ng; ig += 1)
sg = StringFromList(ig, S_HDF5ListGroup, ";")
if (cmpstr(sg[1,4], "scan") == 0)
scans = AddListItem(sg, scans, ";", inf)
endif
endfor
return scans
end
function /s psh5_list_all_datasets(file_id)
variable file_id
HDF5ListGroup /F /R /TYPE=2 /Z file_id, "/"
if (!v_flag)
return S_HDF5ListGroup
else
return ""
endif
end
function [string datatypes, string ranks, string dimensions] psh5_list_dataset_info(variable file_id, string datasets)
variable nds = ItemsInList(datasets, ";")
variable ids
string sds
STRUCT HDF5DataInfo di
InitHDF5DataInfo(di)
variable err
variable idim
string sdims
datatypes = ""
ranks = ""
dimensions = ""
for (ids = 0; ids < nds; ids += 1)
sds = StringFromList(ids, datasets, ";")
err = HDF5DatasetInfo(file_id, sds, 0, di)
if (err == 0)
switch (di.datatype_class)
case H5T_INTEGER:
datatypes = AddListItem("i", datatypes, ";", ids)
break
case H5T_FLOAT:
datatypes = AddListItem("f", datatypes, ";", ids)
break
case H5T_STRING:
datatypes = AddListItem("s", datatypes, ";", ids)
break
default:
datatypes = AddListItem("?", datatypes, ";", ids)
break
endswitch
ranks = AddListItem(num2str(di.ndims), ranks, ";", ids)
sdims = ""
for (idim = 0; idim < di.ndims; idim += 1)
sdims = AddListItem(num2str(di.dims[idim]), sdims, ",", idim)
endfor
if (strlen(sdims) > 1)
sdims = sdims[0, strlen(sdims)-2]
endif
dimensions = AddListItem(sdims, dimensions, ";", ids)
endif
endfor
end
function /s psh5_match_datasets(datasets, match)
string datasets
string match
string result = ""
string spaceless_datasets = ReplaceString(" ", datasets, "")
string spaceless_match = ReplaceString(" ", match, "")
string sep = ";"
variable seplen = strlen(sep)
variable nds = ItemsInList(spaceless_datasets, sep)
variable ids
string ds
variable offset = 0
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(0, spaceless_datasets, sep, offset)
offset += strlen(ds) + seplen
if (StringMatch(ds, spaceless_match))
ds = StringFromList(ids, datasets, sep, 0)
result = AddListItem(ds, result, sep, inf)
endif
endfor
return result
end
function /s psh5_filter_datasets_rank(datasets, ranks, min_rank, max_rank)
string datasets
string ranks
variable min_rank
variable max_rank
string result = ""
string sep = ";"
variable seplen = strlen(sep)
variable nds = ItemsInList(datasets, sep)
variable ids
string ds
variable offset = 0
variable rank
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(0, datasets, sep, offset)
offset += strlen(ds) + seplen
rank = str2num(StringFromList(ids, ranks, sep))
if ((rank >= min_rank) && (rank <= max_rank))
result = AddListItem(ds, result, sep, inf)
endif
endfor
return result
end
static function /s unique_strings(list)
string list
string result = ""
string sep = ";"
variable seplen = strlen(sep)
variable nn = ItemsInList(list, sep)
variable ii
string item
variable offset = 0
make /n=(nn) /t /free wt_in
for (ii = 0; ii < nn; ii += 1)
item = StringFromList(0, list, sep, offset)
offset += strlen(item) + seplen
wt_in[ii] = item
endfor
FindDuplicates /Z /FREE /RT=wt_out wt_in
return twave2list(wt_out, ";")
end
function /s psh5_extract_scan_paths(datasets)
string datasets
string result = ""
string sep = ";"
variable seplen = strlen(sep)
variable nds = ItemsInList(datasets, sep)
variable ids
string ds
string scan
string item
variable offset = 0
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(0, datasets, sep, offset)
offset += strlen(ds) + seplen
if (cmpstr(ds[0], "/") != 0)
ds = "/" + ds
endif
scan = StringFromList(1, ds, "/")
if (StringMatch(scan, "scan*"))
item = "/" + scan + "/"
else
item = ""
endif
if ((strlen(item) > 0) && (WhichListItem(item, result, ";", 0, 0) < 0))
result = AddListItem(item, result, ";", inf)
endif
endfor
return result
end
function /s psh5_extract_region_paths(datasets)
string datasets
string result = ""
string sep = ";"
variable seplen = strlen(sep)
variable nds = ItemsInList(datasets, sep)
variable ids
string ds
string scan
string region
string item
variable offset = 0
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(0, datasets, sep, offset)
offset += strlen(ds) + seplen
if (cmpstr(ds[0], "/") != 0)
ds = "/" + ds
endif
scan = StringFromList(1, ds, "/")
region = StringFromList(2, ds, "/")
if (StringMatch(scan, "scan*") && StringMatch(region, "region*"))
item = "/" + scan + "/" + region + "/"
else
item = ""
endif
if ((strlen(item) > 0) && (WhichListItem(item, result, ";", 0, 0) < 0))
result = AddListItem(item, result, ";", inf)
endif
endfor
return result
end
function /s psh5_match_dataset_classes(datasets, classes, [positioners, detectors])
string datasets
variable classes
string positioners
string detectors
if (ParamIsDefault(positioners))
positioners = ""
endif
if (ParamIsDefault(detectors) || (strlen(detectors) == 0))
detectors = kPreviewDatasets
endif
string result = ""
string sep = ";"
variable seplen = strlen(sep)
variable nds = ItemsInList(datasets, sep)
variable ids
variable offset = 0
string ds
variable nparts
string ds_parent
string ds_name
string ds_scan_rel
variable ds_class
for (ids = 0; ids < nds; ids += 1)
ds = StringFromList(0, datasets, sep, offset)
offset += strlen(ds) + seplen
nparts = ItemsInList(ds, "/")
ds_parent = StringFromList(nparts - 2, ds, "/")
ds_name = StringFromList(nparts - 1, ds, "/")
if (cmpstr(ds[0,4], "/scan") == 0)
ds_scan_rel = RemoveListItem(0, ds[1, strlen(ds) - 1], "/")
else
ds_scan_rel = ds
endif
ds_class = 0
if (strlen(ds_parent) > 0)
ds_class = ds_class | (cmpstr(ds_parent, "attr") == 0 ? kDSCAttrs : 0)
ds_class = ds_class | (cmpstr(ds_parent, "attrs") == 0 ? kDSCAttrs : 0)
ds_class = ds_class | (cmpstr(ds_parent, "diags") == 0 ? kDSCDiags : 0)
ds_class = ds_class | (cmpstr(ds_parent, "snaps") == 0 ? kDSCSnaps : 0)
ds_class = ds_class | (cmpstr(ds_parent, "meta") == 0 ? kDSCMeta : 0)
ds_class = ds_class | (cmpstr(ds_parent, "monitors") == 0 ? kDSCMonitors : 0)
ds_class = ds_class | (cmpstr(ds_parent[0,5], "region") == 0 ? kDSCRegions : 0)
endif
if (strlen(ds_name) > 0)
ds_class = ds_class | (WhichListItem(ds_scan_rel, positioners, sep, 0, 0) >= 0 ? kDSCPositioners : 0)
ds_class = ds_class | (WhichListItem(ds_scan_rel, detectors, sep, 0, 0) >= 0 ? kDSCDetectors : 0)
ds_class = ds_class | (WhichListItem(ds_name, kPreviewDatasets, sep, 0, 0) >= 0 ? kDSCPreview : 0)
ds_class = ds_class | (WhichListItem(ds_name, kScientaScalingDatasets, sep, 0, 0) >= 0 ? kDSCScientaScaling : 0)
ds_class = ds_class | (WhichListItem(ds_name, kEssentialDiagnostics, sep, 0, 0) >= 0 ? kDSCEssentialDiags : 0)
endif
if (ds_class == 0)
ds_class = kDSCOther
endif
if (ds_class & classes)
result = AddListItem(ds, result, sep, inf)
endif
endfor
return result
end
function /df psh5_create_folders(datasetpath)
string datasetpath
if (cmpstr(datasetpath[0], "/") == 0)
datasetpath = datasetpath[1, strlen(datasetpath)-1]
endif
if (cmpstr(datasetpath[strlen(datasetpath)-1], "/") == 0)
datasetpath += "dummy"
endif
variable nfolders = ItemsInList(datasetpath, "/") - 1
variable ifolder
string folder
string inc_path = "/"
for (ifolder = 0; ifolder < nfolders; ifolder += 1)
folder = StringFromList(ifolder, datasetpath, "/")
folder = ps_fix_folder_name(folder)
NewDataFolder /o/s $folder
inc_path += folder
inc_path += "/"
string /g s_hdf5_group = inc_path
endfor
return GetDataFolderDFR()
end
function /df psh5_dataset_to_folder(parent_df, datasetpath)
dfref parent_df
string datasetpath
if (cmpstr(datasetpath[0], "/") == 0)
datasetpath = datasetpath[1, strlen(datasetpath)-1]
endif
if (cmpstr(datasetpath[strlen(datasetpath)-1], "/") == 0)
datasetpath += "dummy"
endif
variable nfolders = ItemsInList(datasetpath, "/") - 1
variable ifolder
string folder
string inc_path = ""
for (ifolder = 0; ifolder < nfolders; ifolder += 1)
folder = StringFromList(ifolder, datasetpath, "/")
folder = ps_fix_folder_name(folder)
if (ifolder)
inc_path += ":"
endif
inc_path += folder
endfor
dfref out_df = parent_df:$inc_path
return out_df
end
function /s ps_fix_folder_name(group_name)
string group_name
string folder_name
folder_name = ReplaceString(" ", group_name, "")
if (cmpstr(folder_name, "attrs") == 0)
folder_name = "attr"
endif
folder_name = PearlCleanupName(folder_name)
return folder_name
end
// ====== import functions ======
function /s psh5_load_datasets(file_df, datasets, [create_folders, reduction_func, reduction_params])
dfref file_df
string datasets
variable create_folders
string reduction_func
string reduction_params
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
if (ParamIsDefault(create_folders))
create_folders = 1
endif
if (ParamIsDefault(reduction_func))
reduction_func = ""
endif
if (ParamIsDefault(reduction_params))
reduction_params = ""
endif
dfref save_df = GetdataFolderDFR()
datasets = unique_strings(datasets)
variable nds = ItemsInList(datasets, ";")
variable ids
string ds
string loaded_datasets = ""
string loaded_waves = ""
for (ids = 0; ids < nds; ids += 1)
SetDataFolder file_df
ds = StringFromList(ids, datasets, ";")
loaded_waves = psh5_load_dataset(file_df, ds, create_folders=create_folders, reduction_func=reduction_func, reduction_params=reduction_params)
if (strlen(loaded_waves) > 1)
loaded_datasets = loaded_datasets + ds + ";"
endif
endfor
setdatafolder save_df
return loaded_datasets
end
function /s psh5_load_dataset(file_df, datasetpath, [create_folders, reduction_func, reduction_params])
dfref file_df
string datasetpath
variable create_folders
string reduction_func
string reduction_params
dfref base_df = GetDataFolderDFR()
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
if (ParamIsDefault(create_folders))
create_folders = 1
endif
if (create_folders)
psh5_create_folders(datasetpath)
endif
if (ParamIsDefault(reduction_func))
reduction_func = ""
endif
if (ParamIsDefault(reduction_params))
reduction_params = ""
endif
STRUCT HDF5DataInfo di // Defined in HDF5 Browser.ipf.
InitHDF5DataInfo(di)
variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
if (err != 0)
// error accessing data
return ""
endif
variable numeric = 0
variable compound = 0
switch (di.datatype_class)
case H5T_INTEGER:
case H5T_FLOAT:
numeric = 1
break
case H5T_STRING:
numeric = 0
break
case H5T_COMPOUND:
compound = 1
break
case H5T_TIME:
case H5T_BITFIELD:
case H5T_OPAQUE:
case H5T_REFERENCE:
case H5T_ENUM:
case H5T_VLEN:
case H5T_ARRAY:
default:
// unsupported data type
return ""
endswitch
string wave_names = ""
if (di.ndims <= 2)
string datasetname = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
variable transpose = WhichListItem(datasetname, kTransposedDatasets) >= 0
HDF5LoadData /O /Q /Z /TRAN=(transpose) file_id, datasetpath
wave_names = S_waveNames
elseif (numeric)
if (exists(reduction_func) == 6)
wave_names = psh5_load_dataset_reduced(file_df, datasetpath, $reduction_func, reduction_params, create_folders=0)
else
wave_names = psh5_load_dataset_slabs(file_df, datasetpath, create_folders=0)
endif
endif
variable nw = ItemsInList(wave_names, ";")
variable iw
string sw
string loaded_waves = ""
for (iw = 0; iw < nw; iw += 1)
sw = StringFromList(iw, wave_names)
wave /z w = $sw
if (WaveExists(w))
loaded_waves = loaded_waves + sw + ";"
ps_set_dimlabels(w)
psh5_load_dataset_meta(file_df, datasetpath, w)
endif
endfor
setdatafolder base_df
return loaded_waves
end
function /s psh5_load_dataset_slabs(file_df, datasetpath, [create_folders, progress])
dfref file_df
string datasetpath
variable create_folders
variable progress
if (ParamIsDefault(create_folders))
create_folders = 1
endif
if (ParamIsDefault(progress))
progress = 1
endif
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
variable result = 0
string datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
if (create_folders)
psh5_create_folders(datasetpath)
endif
STRUCT HDF5DataInfo di // Defined in HDF5 Browser.ipf.
InitHDF5DataInfo(di)
variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
if (err != 0)
print "error accessing dataset", datasetpath
return ""
endif
if (di.ndims < 2)
print "error: rank of dataset < 2", datasetpath
return ""
elseif (di.ndims < 3)
progress = 0
endif
if ((di.datatype_class != H5T_INTEGER) && (di.datatype_class != H5T_FLOAT))
print "error: unsupported datatype", datasetpath
return ""
endif
variable idx, idy, idz, idt, izt
variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
if (transpose)
idx = 1
idy = 0
else
idx = 0
idy = 1
endif
idz = 2
idt = 3
variable nx, ny, nz, nt, nzt
nx = di.dims[idx]
ny = di.dims[idy]
nz = di.dims[idz]
nt = di.dims[idt]
make /n=(nx,ny,nz,nt) /o $datawavename
wave data = $datawavename
nz = max(nz, 1)
nt = max(nt, 1)
nzt = nz * nt
izt = 0
if (progress)
display_progress_panel("HDF5 Import", "Loading " + datasetpath + "...", nzt)
endif
// load data image by image
HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
wave slab
slab[][%Start] = 0
slab[][%Stride] = 1
slab[][%Count] = 1
slab[][%Block] = 1
slab[idx][%Block] = nx
slab[idy][%Block] = ny
variable iz, it
for (iz = 0; iz < nz; iz += 1)
for (it = 0; it < nt; it += 1)
slab[idz][%Start] = iz
slab[idt][%Start] = it
HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata file_id, datasetpath
wave slabdata // 2D, 3D, or 4D with singletons
if (transpose)
data[][][iz][it] = slabdata[q][p][0][0]
else
data[][][iz][it] = slabdata[p][q][0][0]
endif
// progress window
izt += 1
if (progress)
if (update_progress_panel(izt))
result = -4 // user abort
break
endif
endif
endfor
if (result < 0)
break
endif
endfor
if (progress)
kill_progress_panel()
endif
killwaves /z slab, slabdata
if (!result)
ps_set_dimlabels(data)
return NameOfWave(data) + ";"
else
killwaves /z data
return ""
endif
end
// ====== data reduction ======
function /s psh5_load_dataset_reduced(file_df, datasetpath, reduction_func, reduction_params, [create_folders, progress, nthreads])
dfref file_df
string datasetpath
funcref adh5_default_reduction reduction_func
string reduction_params
variable create_folders
variable progress
variable nthreads
if (ParamIsDefault(create_folders))
create_folders = 1
endif
if (ParamIsDefault(progress))
progress = 1
endif
if (ParamIsDefault(nthreads))
nthreads = -1
endif
dfref base_df = GetDataFolderDFR()
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
variable result = 0
string datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
string wavenames = ""
if (create_folders)
psh5_create_folders(datasetpath)
endif
STRUCT HDF5DataInfo di // Defined in HDF5 Browser.ipf.
InitHDF5DataInfo(di)
variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
if (err != 0)
print "error accessing detector/data"
result = -1
return wavenames
endif
if (di.ndims < 2)
print "error: rank of dataset < 2"
result = -2
return wavenames
elseif (di.ndims < 3)
progress = 0
endif
variable idx, idy, idz, idt
variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
if (transpose)
idx = 1
idy = 0
else
idx = 0
idy = 1
endif
idz = 2
idt = 3
variable nx, ny, nz, nt, nzt
nx = di.dims[idx]
ny = di.dims[idy]
nz = di.dims[idz]
nt = di.dims[idt]
// adjust singleton dimensions
nz = max(nz, 1)
nt = max(nt, 1)
nzt = nz * nt
// load data image by image
HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
wave slab
slab[][%Start] = 0
slab[][%Stride] = 1
slab[][%Count] = 1
slab[][%Block] = 1
slab[idx][%Block] = nx
slab[idy][%Block] = ny
// set up multi threading
if (nthreads < 0)
nthreads = ThreadProcessorCount
endif
if (nthreads > 0)
variable threadGroupID = ThreadGroupCreate(nthreads)
variable ithread
for (ithread = 0; ithread < nthreads; ithread += 1)
ThreadStart threadGroupID, ithread, reduce_slab_worker(reduction_func)
endfor
else
make /n=(nzt) /df /free processing_folders
endif
if (progress)
display_progress_panel("PShell Import", "Reducing " + datasetpath + "...", nzt)
endif
// create a template wave with the correct scales and labels
make /n=(nx,ny) /d /o $datawavename
wave template = $datawavename
ps_set_dimlabels2(template, datawavename)
ps_scale_dataset(template)
variable iz, it, izt
variable n_sent = 0
variable n_recvd = 0
variable tmo = 0
string dfname
dfref dfr
variable iw, nw
string sw
make /n=0 /free /wave result_waves
iz = 0
it = 0
do
// fill the processing queue up to a maximum number of folders
if (n_sent < max(1, nthreads) * 10 + n_recvd)
if (iz < nz)
if (it < nt)
// load a slab into a temporary folder
slab[idz][%Start] = iz
slab[idt][%Start] = it
dfname = "processing_" + num2str(n_sent)
NewDataFolder /s $dfname
HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata file_id, datasetpath
duplicate template, image
variable /g r_index = iz
variable /g s_index = it
string /g func_param = reduction_params
if (nthreads > 0)
// send to thread group
WaveClear image
ThreadGroupPutDF threadGroupID, :
else
// process immediately in single-thread mode
processing_folders[n_sent] = GetDataFolderDFR()
make /n=1/d profile1, profile2
wave slabdata
wave /wave reduced_waves = reduce_slab_image(slabdata, image, reduction_func, func_param)
variable /g func_result = numpnts(reduced_waves)
adh5_get_result_waves(reduced_waves, "redw_", 0)
WaveClear slabdata, image, reduced_waves
setdatafolder ::
endif
iz += 1
n_sent += 1
tmo = 0
else
iz += 1
it = 0
endif
endif
else
// throttle the loop if processing is slow
tmo = min(100, tmo + 10)
endif
// receive a slab from the processing queue
if (n_recvd < nzt)
if (nthreads > 0)
dfr = ThreadGroupGetDFR(threadGroupID, tmo)
else
dfr = processing_folders[n_recvd]
processing_folders[n_recvd] = $""
endif
if (DatafolderRefStatus(dfr) != 0)
// access results folder
nvar rr = dfr:r_index
nvar ss = dfr:s_index
nvar func_result = dfr:func_result
if (func_result < 1)
print "error during data reduction."
result = -3
break
endif
// initialize result waves just once
if (numpnts(result_waves) == 0)
redimension /n=(func_result) result_waves
for (iw = 0; iw < func_result; iw += 1)
sw = "redw_" + num2str(iw)
wave profile = dfr:$sw
sw = "ReducedData" + num2str(iw+1)
make /n=(dimsize(profile, 0), nz, nt) /d /o $sw
wave data = $sw
setdimlabel 0, -1, $getdimlabel(profile, 0, -1), data
setdimlabel 1, -1, $kScanDimLabel, data
note data, note(profile)
ps_scale_dataset(data)
setscale /p x dimoffset(profile, 0), dimdelta(profile, 0), waveunits(profile, 0), data
setscale d 0, 0, waveunits(profile, -1), data
result_waves[iw] = data
endfor
endif
// copy results
for (iw = 0; iw < func_result; iw += 1)
sw = "redw_" + num2str(iw)
wave profile = dfr:$sw
wave data = result_waves[iw]
data[][rr][ss] = profile[p]
endfor
n_recvd += 1
KillDataFolder /Z dfr
endif
else
// processing complete
break
endif
// update progress window
if (progress)
if (update_progress_panel(n_recvd))
print "user abort"
result = -4
break
endif
endif
while ((n_recvd < nzt) && (result == 0))
// clean up
killwaves /z slab, slabdata, template
if (nthreads > 0)
variable tstatus = ThreadGroupRelease(threadGroupID)
if (tstatus == -2)
print "error: thread did not terminate properly."
result = -5
endif
endif
// finalize results
nw = numpnts(result_waves)
wavenames = ""
for (iw = 0; iw < nw; iw += 1)
wave /z data = result_waves[iw]
if (WaveExists(data))
if (nz == 1)
redimension /n=(-1, 0, 0) data
elseif (nt == 1)
redimension /n=(-1, nz, 0) data
endif
wavenames += nameofwave(data) + ";"
endif
endfor
if (progress)
kill_progress_panel()
endif
setdatafolder base_df
return wavenames
end
threadsafe static function reduce_slab_worker(reduction_func)
funcref adh5_default_reduction reduction_func
do
// wait for job from main thread
do
dfref dfr = ThreadGroupGetDFR(0, 1000)
if (DataFolderRefStatus(dfr) == 0)
if (GetRTError(2))
return 0 // no more jobs
endif
else
break
endif
while (1)
// get input data
wave slabdata = dfr:slabdata
wave image = dfr:image
svar func_param = dfr:func_param
nvar rr = dfr:r_index
nvar ss = dfr:s_index
// do the work
newdatafolder /s out_df
variable /g r_index = rr
variable /g s_index = ss
wave /wave reduced_waves = reduce_slab_image(slabdata, image, reduction_func, func_param)
variable /g func_result = numpnts(reduced_waves)
// send output to queue and clean up
adh5_get_result_waves(reduced_waves, "redw_", 0)
WaveClear slabdata, image, reduced_waves
ThreadGroupPutDF 0, :
KillDataFolder dfr
while (1)
return 0
end
threadsafe static function /wave reduce_slab_image(slabdata, image, reduction_func, reduction_params)
wave slabdata
wave image
funcref adh5_default_reduction reduction_func
string reduction_params
image = slabdata[q][p][0][0]
return reduction_func(image, reduction_params)
end
// ====== meta and auxiliary data ======
function /s psh5_load_general_group(file_df)
dfref file_df
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
string obj_names = "authors;pgroup;proposal;proposer;sample;"
variable nn = ItemsInList(obj_names, ";")
variable ii
string name
for (ii = 0; ii < nn; ii += 1)
name = StringFromList(ii, obj_names, ";")
psh_load_general_string(file_df, name)
endfor
return obj_names
end
function /s psh_load_general_string(file_df, name)
dfref file_df
string name
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
string path = "/general/" + name
HDF5LoadData /O /Q /Z /N=wt_load_general /TYPE=1 file_id, path
string values = ""
if (!v_flag)
wave /t wt_load_general
variable nn = numpnts(wt_load_general)
variable ii
for (ii = 0; ii < nn; ii += 1)
values = AddListItem(wt_load_general[ii], values, ",", inf)
endfor
killwaves /z wt_load_general
if (strlen(values) >= 1)
values = values[0,strlen(values)-2]
endif
endif
string /g $name = values
return values
end
function psh5_load_dataset_meta(file_df, datasetpath, datawave)
dfref file_df
string datasetpath
wave datawave
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
dfref save_df = GetDataFolderDFR()
SetDataFolder NewFreeDataFolder()
string wnote
HDF5LoadData /O /Q /Z /A="Writable Dimension" /N=WriteDim file_id, datasetpath
if (!v_flag)
wave WriteDim
// scan dimension starts at 1
sprintf wnote, "ScanDimension=%u", WriteDim[0]
Note datawave, wnote
endif
HDF5LoadData /O /Q /Z /A="Writable Index" /N=WriteIndex file_id, datasetpath
if (!v_flag)
wave WriteIndex
sprintf wnote, "WriteableIndex=%u", WriteIndex[0]
Note datawave, wnote
endif
HDF5LoadData /O /Q /Z /A="Readable Index" /N=ReadIndex file_id, datasetpath
if (!v_flag)
wave ReadIndex
sprintf wnote, "ReadableIndex=%u", ReadIndex[0]
Note datawave, wnote
endif
setdatafolder save_df
return 0
end
function /s psh5_load_scan_meta(file_df, scanpath)
// todo: convert to variables/strings
dfref file_df
string scanpath
string wavenames = ""
if (!DataFolderRefStatus(file_df))
dfref file_df = GetDataFolderDFR()
endif
nvar /sdfr=file_df file_id
HDF5LoadData /O /Q /Z /A="Dimensions" /N=ScanDimensions /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
else
make /n=1 /o ScanDimensions
ScanDimensions = 0
wavenames = AddListItem("ScanDimensions", wavenames, ";", inf)
endif
HDF5LoadData /O /Q /Z /A="Readables" /N=ScanReadables /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
else
make /n=1 /o /t ScanReadables
ScanReadables[0] = "ScientaSpectrum"
wavenames = AddListItem("ScanReadables", wavenames, ";", inf)
endif
HDF5LoadData /O /Q /Z /A="Writables" /N=ScanWritables /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
else
// OTF script
HDF5LoadData /O /Q /Z /A="PlotDomain" /N=ScanWritables /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
endif
endif
HDF5LoadData /O /Q /Z /A="Steps" /N=ScanSteps /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
endif
wavenames = ReplaceString(";;", wavenames, ";")
// additional attributes from XPSSpectrum.py
HDF5LoadData /O /Q /Z /A="Iterations" /N=ScanIterations /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
endif
HDF5LoadData /O /Q /Z /A="Step Size" /N=ScanStepSize /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
endif
HDF5LoadData /O /Q /Z /A="Step Time" /N=ScanStepTime /TYPE=1 file_id, scanpath
if (!v_flag)
wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
endif
return wavenames
end
// ====== dimension scaling ======
function ps_set_dimlabels(data)
wave data
ps_set_dimlabels2(data, NameOfWave(data))
end
function ps_set_dimlabels2(data, name)
wave data
string name
variable dummy
try
// intrinsic dimensions
strswitch(name)
case "ScientaImage":
setdimlabel 0, -1, $kEnergyDimLabel, data
setdimlabel 1, -1, $kAngleDimLabel, data
if (WaveDims(data) >= 3)
setdimlabel 2, -1, $kScanDimLabel, data
endif
AbortOnRTE
break
case "ImageAngleDistribution":
case "ScientaAngleDistribution":
if (WaveDims(data) >= 2)
setdimlabel 0, -1, $kScanDimLabel, data
setdimlabel 1, -1, $kAngleDimLabel, data
else
setdimlabel 0, -1, $kAngleDimLabel, data
endif
AbortOnRTE
break
case "ScientaSpectrum":
case "ImageEnergyDistribution":
case "ScientaEnergyDistribution":
if (WaveDims(data) >= 2)
setdimlabel 0, -1, $kScanDimLabel, data
setdimlabel 1, -1, $kEnergyDimLabel, data
else
setdimlabel 0, -1, $kEnergyDimLabel, data
endif
AbortOnRTE
break
default:
if (WaveDims(data) == 1)
setdimlabel 0, -1, $kScanDimLabel, data
AbortOnRTE
else
return 1
endif
endswitch
catch
dummy = GetRTError(1)
return 2
endtry
return 0
end
function /df ps_find_scan_folder(data_df)
dfref data_df
wave /z /t /sdfr=data_df ScanWritables=::ScanWritables
if (WaveExists(ScanWritables))
string sdf = GetDataFolder(1, data_df)
dfref parent_df = $(sdf + ":")
return parent_df
else
return data_df
endif
end
function /df ps_find_attr_folder(scan_df)
dfref scan_df
dfref diags_df = data_df:diags
dfref attrs_df = scan_df:attrs
dfref attr_df = scan_df:attr
if (DataFolderRefStatus(diags_df))
return diags_df
elseif (DataFolderRefStatus(attrs_df))
return attrs_df
elseif (DataFolderRefStatus(attr_df))
return attr_df
else
return $""
endif
end
function /wave ps_find_scale_wave(name, df1, df2, df3)
string name
dfref df1
dfref df2
dfref df3
variable idf
variable ndf=3
make /n=(ndf) /df /free dfs
dfs[0] = {df1, df2, df3}
for (idf = 0; idf < ndf; idf += 1)
if (DataFolderRefStatus(dfs[idf]))
wave /SDFR=dfs[idf] /Z w = $name
if (WaveExists(w))
return w
endif
endif
endfor
return $""
end
function ps_detect_scale(data_df, ax, lo, hi, un)
dfref data_df
wave /t ax
wave lo
wave hi
wave /t un
dfref scan_df = ps_find_scan_folder(data_df)
dfref attr_df = ps_find_attr_folder(scan_df)
redimension /n=4 lo, hi, un, ax
setdimlabel 0, 0, $kEnergyDimLabel, lo, hi, un, ax
setdimlabel 0, 1, $kAngleDimLabel, lo, hi, un, ax
setdimlabel 0, 2, $kScanDimLabel, lo, hi, un, ax
setdimlabel 0, 3, $kDataDimLabel, lo, hi, un, ax
// default values
lo[%$kEnergyDimLabel] = 0
hi[%$kEnergyDimLabel] = 1
un[%$kEnergyDimLabel] = "eV"
ax[%$kEnergyDimLabel] = "Ekin"
lo[%$kAngleDimLabel] = -1
hi[%$kAngleDimLabel] = 1
un[%$kAngleDimLabel] = "arb."
un[%$kAngleDimLabel] = "slice"
lo[%$kScanDimLabel] = 0
hi[%$kScanDimLabel] = 1
un[%$kScanDimLabel] = "arb."
ax[%$kScanDimLabel] = "scan"
lo[%$kDataDimLabel] = 0
hi[%$kDataDimLabel] = 0
un[%$kDataDimLabel] = "arb."
ax[%$kDataDimLabel] = "value"
wave /T /Z LensMode = ps_find_scale_wave("LensMode", data_df, scan_df, attr_df)
wave /Z ChannelBegin = ps_find_scale_wave("ScientaChannelBegin", data_df, scan_df, attr_df)
wave /Z ChannelEnd = ps_find_scale_wave("ScientaChannelEnd", data_df, scan_df, attr_df)
wave /Z SliceBegin = ps_find_scale_wave("ScientaSliceBegin", data_df, scan_df, attr_df)
wave /Z SliceEnd = ps_find_scale_wave("ScientaSliceEnd", data_df, scan_df, attr_df)
wave /Z ScientaChannels = ps_find_scale_wave("ScientaChannels", data_df, scan_df, attr_df)
// lens mode can give more detail
if (waveexists(LensMode) && (numpnts(LensMode) >= 1))
strswitch(LensMode[0])
case "Angular45":
lo[%$kAngleDimLabel] = -45/2
hi[%$kAngleDimLabel] = +45/2
un[%$kAngleDimLabel] = "Â°"
ax[%$kAngleDimLabel] = "angle"
break
case "Angular60":
lo[%$kAngleDimLabel] = -60/2
hi[%$kAngleDimLabel] = +60/2
un[%$kAngleDimLabel] = "Â°"
ax[%$kAngleDimLabel] = "angle"
break
case "Transmission":
un[%$kAngleDimLabel] = "arb."
ax[%$kAngleDimLabel] = "offset"
break
endswitch
endif
// best option if scales are explicit in separate waves
if (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1))
lo[%$kEnergyDimLabel] = ChannelBegin[0]
hi[%$kEnergyDimLabel] = ChannelEnd[0]
elseif (waveexists(ScientaChannels) && (numpnts(ScientaChannels) >= 1))
lo[%$kEnergyDimLabel] = ScientaChannels[0]
hi[%$kEnergyDimLabel] = ScientaChannels[numpnts(ScientaChannels)-1]
endif
if (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1))
lo[%$kAngleDimLabel] = SliceBegin[0]
hi[%$kAngleDimLabel] = SliceEnd[0]
endif
wave /z /t /SDFR=scan_df ScanWritables
if (WaveExists(ScanWritables))
wave /z /SDFR=scan_df scanner = $ScanWritables[0]
if (!WaveExists(scanner))
wave /z /SDFR=attr_df scanner = $ScanWritables[0]
endif
if (WaveExists(scanner) && (numpnts(scanner) >= 1))
lo[%$kScanDimLabel] = scanner[0]
hi[%$kScanDimLabel] = scanner[numpnts(scanner)-1]
ax[%$kScanDimLabel] = NameOfWave(scanner)
strswitch(NameOfWave(scanner))
case "Eph":
ax[%$kScanDimLabel] = "photon energy"
un[%$kScanDimLabel] = "eV"
break
case "ManipulatorX":
case "ManipulatorY":
case "ManipulatorZ":
case "FocusYTrans":
case "FocusZTrans":
case "RefocusYTrans":
case "RefocusZTrans":
case "ExitSlitY":
un[%$kScanDimLabel] = "mm"
break
case "ExitSlit":
un[%$kScanDimLabel] = "Âµm"
break
case "ManipulatorTheta":
case "ManipulatorTilt":
case "ManipulatorPhi":
un[%$kScanDimLabel] = "Â°"
break
case "FocusXRot":
case "FocusYRot":
case "FocusZRot":
case "RefocusXRot":
case "RefocusYRot":
case "RefocusZRot":
un[%$kScanDimLabel] = "mrad"
break
endswitch
endif
endif
end
function ps_scale_dataset_2(data, ax, lo, hi, un)
wave data
wave /t ax
wave lo
wave hi
wave /t un
string snote = note(data)
string sdim
sdim = GetDimLabel(data, 0, -1)
if (strlen(sdim))
setscale /i x lo[%$sdim], hi[%$sdim], un[%$sdim], data
snote = ReplaceStringByKey("AxisLabelX", snote, ax[%$sdim], "=", "\r")
endif
sdim = GetDimLabel(data, 1, -1)
if (strlen(sdim))
setscale /i y lo[%$sdim], hi[%$sdim], un[%$sdim], data
snote = ReplaceStringByKey("AxisLabelY", snote, ax[%$sdim], "=", "\r")
endif
sdim = GetDimLabel(data, 2, -1)
if (strlen(sdim))
setscale /i z lo[%$sdim], hi[%$sdim], un[%$sdim], data
snote = ReplaceStringByKey("AxisLabelZ", snote, ax[%$sdim], "=", "\r")
endif
string data_unit = un[%$kDataDimLabel]
string data_label = ax[%$kDataDimLabel]
string s
variable def = (cmpstr(data_unit, "arb.") == 0) && (cmpstr(data_label, "value") == 0)
if (def)
s = StringByKey("AxisLabelD", snote, "=", "\r")
if (strlen(s) > 0)
data_label = s
def = 0
endif
s = StringByKey("AxisUnitD", snote, "=", "\r")
if (strlen(s) > 0)
data_unit = s
def = 0
endif
endif
if (def)
strswitch(NameOfWave(data))
case "ScientaImage":
case "ImageAngleDistribution":
case "ScientaAngleDistribution":
case "ScientaSpectrum":
case "ImageEnergyDistribution":
case "ScientaEnergyDistribution":
data_unit = "arb."
data_label = "intensity"
def = 0
break
case "SampleCurrent":
case "RefCurrent":
case "AuxCurrent":
data_unit = "A"
data_label = "current"
def = 0
break
case "MachineCurrent":
data_unit = "mA"
data_label = "current"
def = 0
break
endswitch
endif
setscale d 0, 0, data_unit, data
snote = ReplaceStringByKey("AxisLabelD", snote, data_label, "=", "\r")
snote = ReplaceStringByKey("AxisUnitD", snote, data_unit, "=", "\r")
snote = ReplaceStringByKey("Dataset", snote, NameOfWave(data), "=", "\r")
note /k data, snote
end
function ps_scale_datasets(scan_df)
dfref scan_df
make /n=3 /free lo, hi
make /n=3 /t /free ax, un
wave /t /z /SDFR=scan_df ScanReadables
if (WaveExists(ScanReadables))
variable isr
variable nsr = numpnts(ScanReadables)
variable nel
string ssr
string sds
for (isr = 0; isr < nsr; isr += 1)
ssr = ScanReadables[isr]
dfref data_df = psh5_dataset_to_folder(scan_df, ssr)
if (!DataFolderRefStatus(data_df))
dfref data_df = scan_df
endif
nel = ItemsInList(ssr, "/")
sds = StringFromList(nel - 1, ssr, "/")
wave /z /sdfr=data_df wsr=$sds
if (WaveExists(wsr))
ps_detect_scale(data_df, ax, lo, hi, un)
ps_scale_dataset_2(wsr, ax, lo, hi, un)
endif
endfor
endif
end
function ps_scale_dataset(data)
wave data
dfref save_df = GetDataFolderDFR()
dfref data_df = GetWavesDataFolderDFR(data)
setdatafolder data_df
make /n=3 /free lo, hi
make /n=3 /t /free ax, un
ps_detect_scale(data_df, ax, lo, hi, un)
ps_scale_dataset_2(data, ax, lo, hi, un)
setdatafolder save_df
end
// ====== miscellaneous functions ======
function /s kill_matching_waves(dfr, pattern, recurse, [killed])
DFREF dfr
string pattern
variable recurse
string killed
if (ParamIsDefault(killed))
killed = ""
endif
string s
string r
variable index = 0
do
Wave/Z w = WaveRefIndexedDFR(dfr, index)
if (!WaveExists(w))
break
endif
s = NameOfWave(w)
if (stringmatch(s, pattern))
killwaves /z w
killed = AddListItem(s, killed, ";", Inf)
endif
index += 1
while(1)
if (recurse)
Variable numChildDataFolders = CountObjectsDFR(dfr, 4)
Variable i
for(i=0; i<numChildDataFolders; i+=1)
String childDFName = GetIndexedObjNameDFR(dfr, 4, i)
DFREF childDFR = dfr:$childDFName
killed = kill_matching_waves(childDFR, pattern, 1, killed=killed)
endfor
endif
return killed
End
psh5_load_dataset
string psh5_load_dataset(dfref file_df, string datasetpath, variable create_folders=defaultValue, string reduction_func=defaultValue, string reduction_params=defaultValue)
load a dataset from an open PShell HDF5 file.
Definition: pearl-pshell-import.ipf:1176
ps_fix_folder_name
string ps_fix_folder_name(string group_name)
convert HDF5 group name to data folder name and fix compatibility issues
Definition: pearl-pshell-import.ipf:1065
reduce_slab_worker
static threadsafe variable reduce_slab_worker(funcref reduction_func)
Definition: pearl-pshell-import.ipf:1737
psh5_filter_datasets_rank
string psh5_filter_datasets_rank(string datasets, string ranks, variable min_rank, variable max_rank)
filter datasets by rank
Definition: pearl-pshell-import.ipf:728
ps_find_scale_wave
wave ps_find_scale_wave(string name, dfref df1, dfref df2, dfref df3)
find a wave in scan and attr data folders
Definition: pearl-pshell-import.ipf:2161
kDSCEssentialDiags
const variable kDSCEssentialDiags
Definition: pearl-pshell-import.ipf:78
psh5_create_folders
dfr psh5_create_folders(string datasetpath)
create all data folders along a dataset path
Definition: pearl-pshell-import.ipf:990
ps_set_dimlabels
variable ps_set_dimlabels(wave data)
set dimension labels according to the axis type
Definition: pearl-pshell-import.ipf:2031
ps_set_dimlabels2
variable ps_set_dimlabels2(wave data, string name)
set dimension labels according to the axis type
Definition: pearl-pshell-import.ipf:2050
psh5_load_general_group
string psh5_load_general_group(dfref file_df)
load organizational metadata from the general group.
Definition: pearl-pshell-import.ipf:1803
psh_load_general_string
string psh_load_general_string(dfref file_df, string name)
load a string from the general group.
Definition: pearl-pshell-import.ipf:1838
kill_progress_panel
variable kill_progress_panel()
Definition: pearl-gui-tools.ipf:51
psh5_load_dataset_slabs
string psh5_load_dataset_slabs(dfref file_df, string datasetpath, variable create_folders=defaultValue, variable progress=defaultValue)
load a dataset slab-wise from the open PShell HDF5 file.
Definition: pearl-pshell-import.ipf:1289
display_progress_panel
variable display_progress_panel(string title, string message, variable progress_max)
Definition: pearl-gui-tools.ipf:19
update_progress_panel
variable update_progress_panel(variable progress, string message=defaultValue, variable progress_max=defaultValue)
Definition: pearl-gui-tools.ipf:33
kScientaScalingDatasets
const string kScientaScalingDatasets
List of datasets that must be loaded to determine the axis scaling of a Scienta image.
Definition: pearl-pshell-import.ipf:66
kDSCMonitors
const variable kDSCMonitors
Definition: pearl-pshell-import.ipf:83
psh5_load_dataset_reduced
string psh5_load_dataset_reduced(dfref file_df, string datasetpath, funcref reduction_func, string reduction_params, variable create_folders=defaultValue, variable progress=defaultValue, variable nthreads=defaultValue)
load a dataset with reduced dimensionality
Definition: pearl-pshell-import.ipf:1472
kDSCRegions
const variable kDSCRegions
Definition: pearl-pshell-import.ipf:84
kDSCPositioners
const variable kDSCPositioners
Definition: pearl-pshell-import.ipf:74
adh5_get_result_waves
threadsafe variable adh5_get_result_waves(wave results, string result_prefix, variable start_index)
copy waves from wave reference wave into current data folder
Definition: pearl-area-import.ipf:1463
kDSCPreview
const variable kDSCPreview
Definition: pearl-pshell-import.ipf:77
psh5_load_scan_meta
string psh5_load_scan_meta(dfref file_df, string scanpath)
load metadata of a PShell scan group.
Definition: pearl-pshell-import.ipf:1954
twave2list
static string twave2list(wave wt, string sep)
convert text wave to list.
Definition: pearl-pshell-import.ipf:532
kDSCScientaScaling
const variable kDSCScientaScaling
Definition: pearl-pshell-import.ipf:76
psh5_load_dataset_meta
variable psh5_load_dataset_meta(dfref file_df, string datasetpath, wave datawave)
load metadata of a PShell dataset.
Definition: pearl-pshell-import.ipf:1887
ps_scale_dataset
variable ps_scale_dataset(wave data)
set the dimension scales of a loaded PShell Scienta dataset according to attributes.
Definition: pearl-pshell-import.ipf:2528
psh5_extract_region_paths
string psh5_extract_region_paths(string datasets)
trim dataset paths to the scan/region part
Definition: pearl-pshell-import.ipf:853
kAngleDimLabel
const string kAngleDimLabel
Dimension label for the angle dispersive dimension of multi-dimensional datasets.
Definition: pearl-pshell-import.ipf:53
ps_scale_dataset_2
variable ps_scale_dataset_2(wave data, wave ax, wave lo, wave hi, wave un)
set the dimension scales of a dataset.
Definition: pearl-pshell-import.ipf:2389
kTransposedDatasets
const string kTransposedDatasets
List of datasets that must be transposed upon loading.
Definition: pearl-pshell-import.ipf:72
kDSCSnaps
const variable kDSCSnaps
Definition: pearl-pshell-import.ipf:81
psh5_dataset_to_folder
dfr psh5_dataset_to_folder(dfref parent_df, string datasetpath)
map dataset path to datafolder path
Definition: pearl-pshell-import.ipf:1034
ad_suggest_foldername
string ad_suggest_foldername(string filename, variable ignoredate=defaultValue, string sourcename=defaultValue, variable unique=defaultValue)
generate the name of a data folder based on a file name.
Definition: pearl-area-import.ipf:110
kDSCMeta
const variable kDSCMeta
Definition: pearl-pshell-import.ipf:82
kDSCDiags
const variable kDSCDiags
Definition: pearl-pshell-import.ipf:80
kPreviewDatasets
const string kPreviewDatasets
List of preferred datasets to load for preview.
Definition: pearl-pshell-import.ipf:63
kDSCOther
const variable kDSCOther
Definition: pearl-pshell-import.ipf:85
ps_find_attr_folder
dfr ps_find_attr_folder(dfref scan_df)
find the attributes data folder
Definition: pearl-pshell-import.ipf:2134
kDSCAttrs
const variable kDSCAttrs
Definition: pearl-pshell-import.ipf:79
psh5_list_all_datasets
string psh5_list_all_datasets(variable file_id)
list all datasets in a file
Definition: pearl-pshell-import.ipf:605
psh5_load_datasets
string psh5_load_datasets(dfref file_df, string datasets, variable create_folders=defaultValue, string reduction_func=defaultValue, string reduction_params=defaultValue)
load multiple datasets from open file
Definition: pearl-pshell-import.ipf:1098
kEssentialDiagnostics
const string kEssentialDiagnostics
List of diagnostic datasets that are normally loaded with a scan.
Definition: pearl-pshell-import.ipf:69
kDataDimLabel
const string kDataDimLabel
Dimension label for the data dimension.
Definition: pearl-pshell-import.ipf:60
wave2list
static string wave2list(wave w, string format, string sep)
convert numeric wave to list.
Definition: pearl-pshell-import.ipf:549
adh5_default_reduction
threadsafe wave adh5_default_reduction(wave source, string *param)
function prototype for adh5_load_reduced_detector
Definition: pearl-area-import.ipf:1115
kDSCDetectors
const variable kDSCDetectors
Definition: pearl-pshell-import.ipf:75
psh5_preview
dfr psh5_preview(string path_name, string file_name, dfref dest_df=defaultValue, string preview_datasets=defaultValue)
load preview
Definition: pearl-pshell-import.ipf:345
psh5_close_file
variable psh5_close_file(dfref file_df)
close a HDF5 file opened by psh5_open_file.
Definition: pearl-pshell-import.ipf:509
psh5_list_scans
string psh5_list_scans(variable file_id)
list scan groups of a PShell data file.
Definition: pearl-pshell-import.ipf:574
kill_matching_waves
string kill_matching_waves(dfref dfr, string pattern, variable recurse, string killed=defaultValue)
kill any waves matching a pattern in the experiment
Definition: pearl-pshell-import.ipf:2554
PearlCleanupName
string PearlCleanupName(string name)
Definition: pearl-compat.ipf:48
unique_strings
static string unique_strings(string list)
remove duplicate items from list
Definition: pearl-pshell-import.ipf:764
ps_scale_datasets
variable ps_scale_datasets(dfref scan_df)
set the dimension scales of loaded PShell Scienta datasets according to attributes.
Definition: pearl-pshell-import.ipf:2483
psh5_load
dfr psh5_load(string path_name, string file_name, string scans, string regions, string datasets, variable classes=defaultValue, variable max_rank=defaultValue, string reduction_func=defaultValue, string reduction_params=defaultValue, dfref dest_df=defaultValue)
main data loading function
Definition: pearl-pshell-import.ipf:158
kScanDimLabel
const string kScanDimLabel
Dimension label for the scan dimension of multi-dimensional datasets.
Definition: pearl-pshell-import.ipf:56
psh5_match_dataset_classes
string psh5_match_dataset_classes(string datasets, variable classes, string positioners=defaultValue, string detectors=defaultValue)
filter a list of datasets by classification
Definition: pearl-pshell-import.ipf:899
ps_detect_scale
variable ps_detect_scale(dfref data_df, wave ax, wave lo, wave hi, wave un)
detect the dimension scales from attributes.
Definition: pearl-pshell-import.ipf:2225
psh5_list_dataset_info
variable[string datatypes, string ranks, string dimensions] psh5_list_dataset_info(string variable, string file_id, variable string, sds datasets)
list data types and dimensions of datasets
Definition: pearl-pshell-import.ipf:631
kEnergyDimLabel
const string kEnergyDimLabel
Dimension label for the energy dispersive dimension of multi-dimensional datasets.
Definition: pearl-pshell-import.ipf:50
psh5_extract_scan_paths
string psh5_extract_scan_paths(string datasets)
trim dataset paths to the scan part
Definition: pearl-pshell-import.ipf:803
psh5_open_file
dfr psh5_open_file(string path_name, string file_name, dfref dest_df=defaultValue)
open a HDF5 file created by the PShell data acquisition program and prepare the data folder.
Definition: pearl-pshell-import.ipf:450
reduce_slab_image
static threadsafe wave reduce_slab_image(wave slabdata, wave image, funcref reduction_func, string reduction_params)
Definition: pearl-pshell-import.ipf:1776
ps_find_scan_folder
dfr ps_find_scan_folder(dfref data_df)
find the scan folder of current data
Definition: pearl-pshell-import.ipf:2111
kDSCAll
const variable kDSCAll
Definition: pearl-pshell-import.ipf:86