pearl-pshell-import.ipf

Go to the documentation of this file.

#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3     // Use modern global access method and strict wave access.
#pragma IgorVersion = 8.00
#pragma ModuleName = PearlPShellImport
#pragma version = 2.2
#if IgorVersion() < 9.00
#include <HDF5 Browser>
#endif
#include "pearl-compat"
#include "pearl-gui-tools"
#include "pearl-area-import"
 
// copyright (c) 2013-25 Paul Scherrer Institut
//
// Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//   http:///www.apache.org/licenses/LICENSE-2.0
 
 
 
strconstant kEnergyDimLabel = "energy"
 
strconstant kAngleDimLabel = "angle"
 
strconstant kScanDimLabel = "scan"
 
strconstant kDataDimLabel = "data"
 
strconstant kPreviewDatasets = "ImageEnergyDistribution;ScientaSpectrum;ScientaImage;Counts;SampleCurrent;"
 
strconstant kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChannelEnd;ScientaSliceBegin;ScientaSliceEnd;Eph;ScientaHighEnergy;ScientaHighThetaX;ScientaLowEnergy;ScientaLowThetaX;"
 
strconstant kEssentialDiagnostics = "ManipulatorX;ManipulatorY;ManipulatorZ;ManipulatorTheta;ManipulatorTilt;ManipulatorPhi;MonoEnergy;"
 
strconstant kTransposedDatasets = "ScientaImage;"
 
constant kDSCPositioners = 0x0001
constant kDSCDetectors = 0x0002
constant kDSCScientaScaling = 0x0004
constant kDSCPreview = 0x0008
constant kDSCEssentialDiags = 0x0010
constant kDSCAttrs = 0x0020
constant kDSCDiags = 0x0040
constant kDSCSnaps = 0x0080
constant kDSCMeta = 0x0100
constant kDSCMonitors = 0x0200
constant kDSCRegions = 0x0400
constant kDSCOther = 0x8000
constant kDSCAll = 0xffff
 
 
// ====== main import functions ======
 
function /df psh5_load(path_name, file_name, scans, regions, datasets, [classes, max_rank, reduction_func, reduction_params, dest_df])
    string path_name
    string file_name
    string scans
    string regions
    string datasets
    variable classes
    variable max_rank
    string reduction_func
    string reduction_params
    dfref dest_df
    
    dfref save_df = GetDataFolderDFR()
    variable timerRefNum = startMSTimer
    
    if (ParamIsDefault(classes) || (classes == 0))
        classes = kDSCPositioners | kDSCDetectors | kDSCScientaScaling | kDSCEssentialDiags
    endif
    variable essential_classes = kDSCPositioners | kDSCScientaScaling | kDSCEssentialDiags
    
    if (ParamIsDefault(dest_df) || !DataFolderRefStatus(dest_df))
        dest_df = psh5_open_file(path_name, file_name)
    else
        dest_df = psh5_open_file(path_name, file_name, dest_df=dest_df)
    endif
    if (ParamIsDefault(reduction_func))
        reduction_func = ""
    endif
    if (ParamIsDefault(reduction_params))
        reduction_params = ""
    endif
    
    if (DataFolderRefStatus(dest_df))
        setdatafolder dest_df
        psh5_load_general_group(dest_df)
 
        // datasets contained in file
        svar /sdfr=dest_df file_datasets = s_datasets
        
        // datasets contained in file up to allowed rank
        string ranked_datasets = ""
        if (ParamIsDefault(max_rank))
            ranked_datasets = file_datasets
        else
            svar /sdfr=dest_df file_ranks = s_datasets_ranks
            ranked_datasets = psh5_filter_datasets_rank(file_datasets, file_ranks, 0, max_rank)
        endif
 
        string matching_datasets = ""
        string matching_essentials = ""
        string scan_datasets = ""
        string region_datasets = ""
        string free_datasets = ""
        string selected_datasets = ""
        string essential_datasets = ""
 
        variable i_item
        variable n_items
        string item
 
        // select datasets belonging to selected scans
        n_items = ItemsInList(scans, ";")
        for (i_item = 0; i_item < n_items; i_item += 1)
            item = StringFromList(i_item, scans, ";")
            if (cmpstr(item[0,3], "scan") == 0)
                item = "/" + item
            endif
            item = ReplaceString("//", item + "/*", "/")
            matching_datasets = psh5_match_datasets(ranked_datasets, item)
            scan_datasets = scan_datasets + matching_datasets
        endfor
 
        // select datasets belonging to selected regions
        n_items = ItemsInList(regions, ";")
        for (i_item = 0; i_item < n_items; i_item += 1)
            item = StringFromList(i_item, regions, ";")
            if (cmpstr(item[0,3], "scan") == 0)
                item = "/" + item
            endif
            item = ReplaceString("//", item + "/*", "/")
            matching_datasets = psh5_match_datasets(ranked_datasets, item)
            region_datasets = region_datasets + matching_datasets
        endfor
 
        // free select datasets
        n_items = ItemsInList(datasets, ";")
        for (i_item = 0; i_item < n_items; i_item += 1)
            item = StringFromList(i_item, datasets, ";")
            if (cmpstr(item[0,3], "scan") == 0)
                item = "/" + item
            endif
            matching_datasets = psh5_match_datasets(ranked_datasets, item)
            free_datasets = free_datasets + matching_datasets
        endfor
 
        selected_datasets = scan_datasets + region_datasets + free_datasets
        
        string filtered_datasets = ""
        string diag_datasets = ""
        string selected_scans = psh5_extract_scan_paths(selected_datasets)
        variable i_scan
        variable n_scans = ItemsInList(selected_scans)
        string scan
        string selected_regions = psh5_extract_region_paths(selected_datasets)
        variable i_region
        variable n_regions = ItemsInList(selected_regions)
        string region
        string positioners
        string detectors
 
        // datasets directly under one of the selected regions
        region_datasets = ""
        for (i_region = 0; i_region < n_regions; i_region += 1)
            region = StringFromList(i_region, selected_regions, ";")
            region_datasets = region_datasets + GrepList(file_datasets, "(?i)^" + region + "[[:alpha:]]+$")
        endfor
        
        // filter selected datasets by class and add essential dependencies
        // each scan may have specific positioners and detectors
        for (i_scan = 0; i_scan < n_scans; i_scan += 1)
            scan = StringFromList(i_scan, selected_scans, ";")
            
            // potentially interesting diagnostics of current scan and selected regions
            diag_datasets = psh5_match_datasets(file_datasets, scan + "*")
            diag_datasets = psh5_match_dataset_classes(diag_datasets, kDSCAttrs | kDSCDiags | kDSCSnaps)
            diag_datasets = diag_datasets + GrepList(file_datasets, "(?i)^" + scan + "[[:alpha:]]+$")
            diag_datasets = diag_datasets + psh5_match_datasets(region_datasets, scan + "*")
 
            // explicit positioners and detectors set by pshell
            setdatafolder dest_df
            dfref scan_df = psh5_create_folders(scan)
            setdatafolder scan_df
            psh5_load_scan_meta(dest_df, scan)
            wave /t /z ScanWritables
            wave /t /z ScanReadables
            if (WaveExists(ScanWritables))
                positioners = twave2list(ScanWritables, ";")
            else
                positioners = ""
            endif
            if (WaveExists(ScanReadables))
                detectors = twave2list(ScanReadables, ";")
            else
                detectors = ""
            endif
            
            // filtering by classes
            matching_datasets = psh5_match_dataset_classes(selected_datasets, classes, positioners=positioners, detectors=detectors)
            
            // add essential diags
            if (strlen(matching_datasets) > 1)
                essential_datasets = psh5_match_dataset_classes(diag_datasets, essential_classes, positioners=positioners, detectors=detectors)
            endif
            
            // scaling datasets before detectors because data reduction needs the scales
            filtered_datasets = essential_datasets + filtered_datasets + matching_datasets
        endfor
        
        // load the datasets
        setdatafolder dest_df
        string /g s_loaded_datasets = ""
        s_loaded_datasets = psh5_load_datasets(dest_df, filtered_datasets, reduction_func=reduction_func, reduction_params=reduction_params)
 
        // apply scaling by scan
        for (i_scan = 0; i_scan < n_scans; i_scan += 1)
            scan = StringFromList(i_scan, selected_scans, ";")
            ps_scale_datasets(psh5_dataset_to_folder(dest_df, scan))
        endfor
    
        psh5_close_file(dest_df)
 
        // performance reporting
        if (timerRefNum >= 0)
            setdatafolder dest_df
            variable /g psh5_perf_secs
            psh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
        endif
    endif
    
    setdatafolder save_df
    return dest_df
end
 
function /df psh5_preview(path_name, file_name, [dest_df, preview_datasets])
    string path_name
    string file_name
    dfref dest_df
    string preview_datasets
 
    dfref save_df = GetDataFolderDFR()
 
    if (ParamIsDefault(dest_df))
        dest_df = psh5_open_file(path_name, file_name)
    else
        dest_df = psh5_open_file(path_name, file_name, dest_df=dest_df)
    endif
    if (ParamIsDefault(preview_datasets))
        preview_datasets = kPreviewDatasets
    endif
    variable essential_classes = kDSCPositioners | kDSCScientaScaling
    
    if (DataFolderRefStatus(dest_df))
        setdatafolder dest_df
        psh5_load_general_group(dest_df)
 
        // select dataset based on preference
        svar /sdfr=dest_df file_datasets = s_datasets
        svar /sdfr=dest_df file_datasets_ranks = s_datasets_ranks
        string selected_datasets = ""
        string essential_datasets = ""
        string scan_datasets = ""
        string filtered_datasets = ""
        
        variable nds = ItemsInList(preview_datasets, ";")
        variable ids
        string ds
        for (ids = 0; ids < nds; ids += 1)
            ds = StringFromList(ids, preview_datasets, ";")
            selected_datasets = psh5_filter_datasets_rank(file_datasets, file_datasets_ranks, 1, 2)
            selected_datasets = psh5_match_datasets(selected_datasets, "/scan*" + ds)
            if (strlen(selected_datasets) > 1)
                selected_datasets = StringFromList(0, selected_datasets, ";")
                break
            endif
        endfor
 
        // add essential dependencies
        if (strlen(selected_datasets) > 1)
            string selected_scans = psh5_extract_scan_paths(selected_datasets)
            string scan
            string positioners
        
            scan = StringFromList(0, selected_scans, ";")
            scan_datasets = psh5_match_datasets(file_datasets, scan + "*")
 
            psh5_load_scan_meta(dest_df, scan)
            wave /t /z ScanWritables
            if (WaveExists(ScanWritables))
                positioners = twave2list(ScanWritables, ";")
            else
                positioners = ""
            endif
 
            essential_datasets = psh5_match_dataset_classes(scan_datasets, essential_classes, positioners=positioners)
            filtered_datasets = essential_datasets + selected_datasets
 
            // load the datasets
            psh5_load_datasets(dest_df, filtered_datasets, create_folders=0)
            ps_scale_datasets(dest_df)
            string /g s_preview_dataset = StringFromList(0, selected_datasets, ";")
            string /g s_preview_wave = StringFromList(ItemsInList(s_preview_dataset, "/") - 1, s_preview_dataset, "/")
        endif
        
        psh5_close_file(dest_df)
    endif
 
    setdatafolder save_df
    return dest_df
end
 
function /df psh5_open_file(path_name, file_name, [dest_df])
    string path_name
    string file_name
    dfref dest_df
 
    dfref save_df = GetDataFolderDFR()
    
    variable fid
    HDF5OpenFile /P=$path_name /R fid as file_name
    if (v_flag == 0)
        if (!ParamIsDefault(dest_df))
            setdatafolder dest_df
        else
            string dest_name = ad_suggest_foldername(s_filename, sourcename="psh")
            setdatafolder root:
            newdatafolder /s /o $("root:" + dest_name)
        endif
        dfref file_df = GetDataFolderDFR()
 
        variable /g file_id = fid
        string /g s_filepath
        string /g s_scanpaths
        string /g s_datasets
        string datatypes
        string ranks
        string dimensions
        s_filepath = s_path + s_filename
        s_scanpaths = psh5_list_scans(file_id)
        s_datasets = psh5_list_all_datasets(file_id)
        [datatypes, ranks, dimensions] = psh5_list_dataset_info(file_id, s_datasets)
        string /g s_datasets_datatypes = datatypes
        string /g s_datasets_ranks = ranks
        string /g s_datasets_dimensions = dimensions
    else
        dfref file_df = $""
    endif
    
    setdatafolder save_df
    return file_df
end
 
function psh5_close_file(file_df)
    dfref file_df
 
    if (DataFolderRefStatus(file_df))
        nvar /sdfr=file_df /z file_id
        if (nvar_Exists(file_id))
            HDF5CloseFile /z file_id
            file_id = 0
            KillVariables /z file_id
        else
            dfref parent_df = $(GetDataFolder(1, file_df) + ":")
            if (DataFolderRefStatus(parent_df))
                psh5_close_file(parent_df)
            endif
        endif
    endif
end
 
// === datasets and paths ===
 
static function /s twave2list(wt, sep)
    wave /t wt
    string sep
    
    string list = ""
    variable n = numpnts(wt)
    variable i
    for (i = 0; i < n; i += 1)
        list = AddListItem(wt[i], list, sep, inf)
    endfor
    
    return list
end
 
static function /s wave2list(w, format, sep)
    wave w
    string format
    string sep
    
    string list = ""
    variable n = numpnts(w)
    variable i
    string s
    for (i = 0; i < n; i += 1)
        sprintf s, format, w[i]
        list = AddListItem(s, list, sep, inf)
    endfor
    
    return list
end
 
function /s psh5_list_scans(file_id)
    variable file_id
  
    HDF5ListGroup /F /TYPE=1 file_id, "/"
        
    variable ig
    variable ng = ItemsInList(S_HDF5ListGroup, ";")
    string sg
    string scans = ""
        
    for (ig = 0; ig < ng; ig += 1)
        sg = StringFromList(ig, S_HDF5ListGroup, ";")
        if (cmpstr(sg[1,4], "scan") == 0)
            scans = AddListItem(sg, scans, ";", inf)
        endif
    endfor
    
    return scans
end
 
function /s psh5_list_all_datasets(file_id)
    variable file_id
    
    HDF5ListGroup /F /R /TYPE=2 /Z file_id, "/"
    if (!v_flag)
        return S_HDF5ListGroup
    else
        return ""
    endif
end
 
function [string datatypes, string ranks, string dimensions] psh5_list_dataset_info(variable file_id, string datasets)
    variable nds = ItemsInList(datasets, ";")
    variable ids
    string sds
    STRUCT HDF5DataInfo di
    InitHDF5DataInfo(di)
    variable err
    variable idim
    string sdims
    datatypes = ""
    ranks = ""
    dimensions = "" 
 
    for (ids = 0; ids < nds; ids += 1)
        sds = StringFromList(ids, datasets, ";")
        err = HDF5DatasetInfo(file_id, sds, 0, di)
        if (err == 0)
            switch (di.datatype_class)
                case H5T_INTEGER:
                    datatypes = AddListItem("i", datatypes, ";", ids)
                    break
                case H5T_FLOAT:
                    datatypes = AddListItem("f", datatypes, ";", ids)
                    break
                case H5T_STRING:
                    datatypes = AddListItem("s", datatypes, ";", ids)
                    break
                default:
                    datatypes = AddListItem("?", datatypes, ";", ids)
                    break
            endswitch
 
            ranks = AddListItem(num2str(di.ndims), ranks, ";", ids)
 
            sdims = ""
            for (idim = 0; idim < di.ndims; idim += 1)
                sdims = AddListItem(num2str(di.dims[idim]), sdims, ",", idim)
            endfor
            if (strlen(sdims) > 1)
                sdims = sdims[0, strlen(sdims)-2]
            endif
            dimensions = AddListItem(sdims, dimensions, ";", ids)
        endif
    endfor
end
 
function /s psh5_match_datasets(datasets, match)
    string datasets
    string match
    
    string result = ""
 
    string spaceless_datasets = ReplaceString(" ", datasets, "")
    string spaceless_match = ReplaceString(" ", match, "")
 
    string sep = ";"
    variable seplen = strlen(sep)
    variable nds = ItemsInList(spaceless_datasets, sep)
    variable ids
    string ds
    variable offset = 0
    
    for (ids = 0; ids < nds; ids += 1)
        ds = StringFromList(0, spaceless_datasets, sep, offset)
        offset += strlen(ds) + seplen
        if (StringMatch(ds, spaceless_match))
            ds = StringFromList(ids, datasets, sep, 0)
            result = AddListItem(ds, result, sep, inf)
        endif
    endfor
    
    return result
end
 
function /s psh5_filter_datasets_rank(datasets, ranks, min_rank, max_rank)
    string datasets
    string ranks
    variable min_rank
    variable max_rank
    
    string result = ""
    string sep = ";"
    variable seplen = strlen(sep)
    variable nds = ItemsInList(datasets, sep)
    variable ids
    string ds
    variable offset = 0
    variable rank
    
    for (ids = 0; ids < nds; ids += 1)
        ds = StringFromList(0, datasets, sep, offset)
        offset += strlen(ds) + seplen
        rank = str2num(StringFromList(ids, ranks, sep))
        if ((rank >= min_rank) && (rank <= max_rank))
            result = AddListItem(ds, result, sep, inf)
        endif
    endfor
    
    return result
end
 
static function /s unique_strings(list)
    string list
    
    string result = ""
 
    string sep = ";"
    variable seplen = strlen(sep)
    variable nn = ItemsInList(list, sep)
    variable ii
    string item
    variable offset = 0
 
    make /n=(nn) /t /free wt_in
    for (ii = 0; ii < nn; ii += 1)
        item = StringFromList(0, list, sep, offset)
        offset += strlen(item) + seplen
        wt_in[ii] = item
    endfor
    
    FindDuplicates /Z /FREE /RT=wt_out wt_in
    
    return twave2list(wt_out, ";")
end
 
function /s psh5_extract_scan_paths(datasets)
    string datasets
 
    string result = ""
    string sep = ";"
    variable seplen = strlen(sep)
    variable nds = ItemsInList(datasets, sep)
    variable ids
    string ds
    string scan
    string item
    variable offset = 0
 
    for (ids = 0; ids < nds; ids += 1)
        ds = StringFromList(0, datasets, sep, offset)
        offset += strlen(ds) + seplen
        if (cmpstr(ds[0], "/") != 0)
            ds = "/" + ds
        endif
 
        scan = StringFromList(1, ds, "/")
        if (StringMatch(scan, "scan*"))
            item = "/" + scan + "/"
        else
            item = ""
        endif
 
        if ((strlen(item) > 0) && (WhichListItem(item, result, ";", 0, 0) < 0))
            result = AddListItem(item, result, ";", inf)
        endif
    endfor
    
    return result
end
 
function /s psh5_extract_region_paths(datasets)
    string datasets
 
    string result = ""
    string sep = ";"
    variable seplen = strlen(sep)
    variable nds = ItemsInList(datasets, sep)
    variable ids
    string ds
    string scan
    string region
    string item
    variable offset = 0
 
    for (ids = 0; ids < nds; ids += 1)
        ds = StringFromList(0, datasets, sep, offset)
        offset += strlen(ds) + seplen
        if (cmpstr(ds[0], "/") != 0)
            ds = "/" + ds
        endif
 
        scan = StringFromList(1, ds, "/")
        region = StringFromList(2, ds, "/")
        if (StringMatch(scan, "scan*") && StringMatch(region, "region*"))
            item = "/" + scan + "/" + region + "/"
        else
            item = ""
        endif
 
        if ((strlen(item) > 0) && (WhichListItem(item, result, ";", 0, 0) < 0))
            result = AddListItem(item, result, ";", inf)
        endif
    endfor
    
    return result
end
 
function /s psh5_match_dataset_classes(datasets, classes, [positioners, detectors])
    string datasets
    variable classes
    string positioners
    string detectors
    
    if (ParamIsDefault(positioners))
        positioners = ""
    endif
    if (ParamIsDefault(detectors) || (strlen(detectors) == 0))
        detectors = kPreviewDatasets
    endif
    
    string result = ""
    string sep = ";"
    variable seplen = strlen(sep)
    variable nds = ItemsInList(datasets, sep)
    variable ids
    variable offset = 0
    string ds
    variable nparts
    string ds_parent
    string ds_name
    string ds_scan_rel
    variable ds_class
 
    for (ids = 0; ids < nds; ids += 1)
        ds = StringFromList(0, datasets, sep, offset)
        offset += strlen(ds) + seplen
        nparts = ItemsInList(ds, "/")
        ds_parent = StringFromList(nparts - 2, ds, "/")
        ds_name = StringFromList(nparts - 1, ds, "/")
        if (cmpstr(ds[0,4], "/scan") == 0)
            ds_scan_rel = RemoveListItem(0, ds[1, strlen(ds) - 1], "/")
        else
            ds_scan_rel = ds
        endif
        ds_class = 0
 
        if (strlen(ds_parent) > 0)
            ds_class = ds_class | (cmpstr(ds_parent, "attr") == 0 ? kDSCAttrs : 0)
            ds_class = ds_class | (cmpstr(ds_parent, "attrs") == 0 ? kDSCAttrs : 0)
            ds_class = ds_class | (cmpstr(ds_parent, "diags") == 0 ? kDSCDiags : 0)
            ds_class = ds_class | (cmpstr(ds_parent, "snaps") == 0 ? kDSCSnaps : 0)
            ds_class = ds_class | (cmpstr(ds_parent, "meta") == 0 ? kDSCMeta : 0)
            ds_class = ds_class | (cmpstr(ds_parent, "monitors") == 0 ? kDSCMonitors : 0)
            ds_class = ds_class | (cmpstr(ds_parent[0,5], "region") == 0 ? kDSCRegions : 0)
        endif
        
        if (strlen(ds_name) > 0)
            ds_class = ds_class | (WhichListItem(ds_scan_rel, positioners, sep, 0, 0) >= 0 ? kDSCPositioners : 0)
            ds_class = ds_class | (WhichListItem(ds_scan_rel, detectors, sep, 0, 0) >= 0 ? kDSCDetectors : 0)
            ds_class = ds_class | (WhichListItem(ds_name, kPreviewDatasets, sep, 0, 0) >= 0 ? kDSCPreview : 0)
            ds_class = ds_class | (WhichListItem(ds_name, kScientaScalingDatasets, sep, 0, 0) >= 0 ? kDSCScientaScaling : 0)
            ds_class = ds_class | (WhichListItem(ds_name, kEssentialDiagnostics, sep, 0, 0) >= 0 ? kDSCEssentialDiags : 0)
        endif
 
        if (ds_class == 0)
            ds_class = kDSCOther
        endif
 
        if (ds_class & classes)
            result = AddListItem(ds, result, sep, inf)
        endif
    endfor
    
    return result
end
 
function /df psh5_create_folders(datasetpath)
    string datasetpath
 
    if (cmpstr(datasetpath[0], "/") == 0)
        datasetpath = datasetpath[1, strlen(datasetpath)-1]
    endif
    if (cmpstr(datasetpath[strlen(datasetpath)-1], "/") == 0)
        datasetpath += "dummy"
    endif
    
    variable nfolders = ItemsInList(datasetpath, "/") - 1
    variable ifolder
    string folder
    string inc_path = "/"
    
    for (ifolder = 0; ifolder < nfolders; ifolder += 1)
        folder = StringFromList(ifolder, datasetpath, "/")
        folder = ps_fix_folder_name(folder)
        NewDataFolder /o/s $folder
        inc_path += folder
        inc_path += "/"
        string /g s_hdf5_group = inc_path
    endfor
    
    return GetDataFolderDFR()
end
 
function /df psh5_dataset_to_folder(parent_df, datasetpath)
    dfref parent_df
    string datasetpath
 
    if (cmpstr(datasetpath[0], "/") == 0)
        datasetpath = datasetpath[1, strlen(datasetpath)-1]
    endif
    if (cmpstr(datasetpath[strlen(datasetpath)-1], "/") == 0)
        datasetpath += "dummy"
    endif
    
    variable nfolders = ItemsInList(datasetpath, "/") - 1
    variable ifolder
    string folder
    string inc_path = ""
    
    for (ifolder = 0; ifolder < nfolders; ifolder += 1)
        folder = StringFromList(ifolder, datasetpath, "/")
        folder = ps_fix_folder_name(folder)
        if (ifolder)
            inc_path += ":"
        endif
        inc_path += folder
    endfor
    
    dfref out_df = parent_df:$inc_path
    return out_df
end
 
function /s ps_fix_folder_name(group_name)
    string group_name
    string folder_name
 
    folder_name = ReplaceString(" ", group_name, "")
    if (cmpstr(folder_name, "attrs") == 0)
        folder_name = "attr"
    endif
    folder_name = PearlCleanupName(folder_name)
    
    return folder_name
end
 
// ====== import functions ======
 
function /s psh5_load_datasets(file_df, datasets, [create_folders, reduction_func, reduction_params])
    dfref file_df
    string datasets
    variable create_folders
    string reduction_func
    string reduction_params
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    if (ParamIsDefault(create_folders))
        create_folders = 1
    endif
    if (ParamIsDefault(reduction_func))
        reduction_func = ""
    endif
    if (ParamIsDefault(reduction_params))
        reduction_params = ""
    endif
 
    dfref save_df = GetdataFolderDFR()
    
    datasets = unique_strings(datasets)
    variable nds = ItemsInList(datasets, ";")
    variable ids
    string ds
    string loaded_datasets = ""
    string loaded_waves = ""
    
    for (ids = 0; ids < nds; ids += 1)
        SetDataFolder file_df
        ds = StringFromList(ids, datasets, ";")
        loaded_waves = psh5_load_dataset(file_df, ds, create_folders=create_folders, reduction_func=reduction_func, reduction_params=reduction_params)
        if (strlen(loaded_waves) > 1)
            loaded_datasets = loaded_datasets + ds + ";"
        endif
    endfor
    
    setdatafolder save_df
    return loaded_datasets
end
 
function /s psh5_load_dataset(file_df, datasetpath, [create_folders, reduction_func, reduction_params])
    dfref file_df
    string datasetpath
    variable create_folders
    string reduction_func
    string reduction_params
    
    dfref base_df = GetDataFolderDFR()
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
 
    if (ParamIsDefault(create_folders))
        create_folders = 1
    endif
    if (create_folders)
        psh5_create_folders(datasetpath)
    endif
    if (ParamIsDefault(reduction_func))
        reduction_func = ""
    endif
    if (ParamIsDefault(reduction_params))
        reduction_params = ""
    endif
 
    STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
    if (err != 0)
        // error accessing data
        return ""
    endif
    
    variable numeric = 0
    variable compound = 0
    
    switch (di.datatype_class)
        case H5T_INTEGER:
        case H5T_FLOAT:
            numeric = 1
            break
        case H5T_STRING:
            numeric = 0
            break
        case H5T_COMPOUND:
            compound = 1
            break
        case H5T_TIME:
        case H5T_BITFIELD:
        case H5T_OPAQUE:
        case H5T_REFERENCE:
        case H5T_ENUM:
        case H5T_VLEN:
        case H5T_ARRAY:
        default:
            // unsupported data type
            return ""
    endswitch
    
    string wave_names = ""
    if (di.ndims <= 2)
        string datasetname = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
        variable transpose = WhichListItem(datasetname, kTransposedDatasets) >= 0
        HDF5LoadData /O /Q /Z /TRAN=(transpose) file_id, datasetpath
        wave_names = S_waveNames
    elseif (numeric)
        if (exists(reduction_func) == 6)
            wave_names = psh5_load_dataset_reduced(file_df, datasetpath, $reduction_func, reduction_params, create_folders=0)
        else
            wave_names = psh5_load_dataset_slabs(file_df, datasetpath, create_folders=0)
        endif
    endif
 
    variable nw = ItemsInList(wave_names, ";")
    variable iw
    string sw
    string loaded_waves = ""
    for (iw = 0; iw < nw; iw += 1)
        sw = StringFromList(iw, wave_names)
        wave /z w = $sw
        if (WaveExists(w))
            loaded_waves = loaded_waves + sw + ";"
            ps_set_dimlabels(w)
            psh5_load_dataset_meta(file_df, datasetpath, w)
        endif
    endfor
    
    setdatafolder base_df
    return loaded_waves
end
 
function /s psh5_load_dataset_slabs(file_df, datasetpath, [create_folders, progress])
    dfref file_df
    string datasetpath
    variable create_folders
    variable progress
 
    if (ParamIsDefault(create_folders))
        create_folders = 1
    endif
    if (ParamIsDefault(progress))
        progress = 1
    endif
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
    
    variable result = 0
    string datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    
    if (create_folders)
        psh5_create_folders(datasetpath)
    endif
        
    STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
    if (err != 0)
        print "error accessing dataset", datasetpath
        return ""
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2", datasetpath
        return ""
    elseif (di.ndims < 3)
        progress = 0
    endif
    if ((di.datatype_class != H5T_INTEGER) && (di.datatype_class != H5T_FLOAT))
        print "error: unsupported datatype", datasetpath
        return ""
    endif
    
    variable idx, idy, idz, idt, izt
    variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3
    
    variable nx, ny, nz, nt, nzt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = di.dims[idz]
    nt = di.dims[idt]
    make /n=(nx,ny,nz,nt) /o $datawavename
    wave data = $datawavename
 
    nz = max(nz, 1)
    nt = max(nt, 1)
    nzt = nz * nt
    izt = 0
    if (progress)
        display_progress_panel("HDF5 Import", "Loading " + datasetpath + "...", nzt)
    endif
    
    // load data image by image
    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    variable iz, it
    for (iz = 0; iz < nz; iz += 1)
        for (it = 0; it < nt; it += 1)
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata file_id, datasetpath
            wave slabdata // 2D, 3D, or 4D with singletons
            if (transpose)
                data[][][iz][it] = slabdata[q][p][0][0]
            else
                data[][][iz][it] = slabdata[p][q][0][0]
            endif
            
            // progress window
            izt += 1
            if (progress)
                if (update_progress_panel(izt))
                    result = -4 // user abort
                    break
                endif
            endif
        endfor
        if (result < 0)
            break
        endif
    endfor
    
    if (progress)
        kill_progress_panel()
    endif
    
    killwaves /z slab, slabdata
    if (!result)
        ps_set_dimlabels(data)
        return NameOfWave(data) + ";"
    else
        killwaves /z data
        return ""
    endif
end
 
// ====== data reduction ======
 
function /s psh5_load_dataset_reduced(file_df, datasetpath, reduction_func, reduction_params, [create_folders, progress, nthreads])
    dfref file_df
    string datasetpath
    funcref adh5_default_reduction reduction_func
    string reduction_params
    variable create_folders
    variable progress
    variable nthreads
 
    if (ParamIsDefault(create_folders))
        create_folders = 1
    endif
    if (ParamIsDefault(progress))
        progress = 1
    endif
    if (ParamIsDefault(nthreads))
        nthreads = -1
    endif
 
    dfref base_df = GetDataFolderDFR()
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
 
    variable result = 0
    string datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    string wavenames = ""
    
    if (create_folders)
        psh5_create_folders(datasetpath)
    endif
 
    STRUCT HDF5DataInfo di  // Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(file_id, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        result = -1
        return wavenames
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        result = -2
        return wavenames
    elseif (di.ndims < 3)
        progress = 0
    endif
    
    variable idx, idy, idz, idt
    variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3
    
    variable nx, ny, nz, nt, nzt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = di.dims[idz]
    nt = di.dims[idt]
    // adjust singleton dimensions
    nz = max(nz, 1)
    nt = max(nt, 1)
    nzt = nz * nt
    
    // load data image by image
    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    // set up multi threading
    if (nthreads < 0)
        nthreads = ThreadProcessorCount
    endif
    if (nthreads > 0)
        variable threadGroupID = ThreadGroupCreate(nthreads)
        variable ithread
        for (ithread = 0; ithread < nthreads; ithread += 1)
            ThreadStart threadGroupID, ithread, reduce_slab_worker(reduction_func)
        endfor
    else
        make /n=(nzt) /df /free processing_folders
    endif
    
    if (progress)
        display_progress_panel("PShell Import", "Reducing " + datasetpath + "...", nzt)
    endif
 
    // create a template wave with the correct scales and labels
    make /n=(nx,ny) /d /o $datawavename
    wave template = $datawavename
    ps_set_dimlabels2(template, datawavename)
    ps_scale_dataset(template)
 
    variable iz, it, izt
    variable n_sent = 0
    variable n_recvd = 0
    variable tmo = 0
    string dfname
    dfref dfr
    variable iw, nw
    string sw
    make /n=0 /free /wave result_waves
 
    iz = 0
    it = 0
    
    do
        // fill the processing queue up to a maximum number of folders
        if (n_sent < max(1, nthreads) * 10 + n_recvd)
            if (iz < nz)
                if (it < nt)
                    // load a slab into a temporary folder
                    slab[idz][%Start] = iz
                    slab[idt][%Start] = it
                    dfname = "processing_" + num2str(n_sent)
                    NewDataFolder /s $dfname
                    HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata file_id, datasetpath
            
                    duplicate template, image
                    variable /g r_index = iz
                    variable /g s_index = it
                    string /g func_param = reduction_params
 
                    if (nthreads > 0)
                        // send to thread group
                        WaveClear image
                        ThreadGroupPutDF threadGroupID, :
                    else
                        // process immediately in single-thread mode
                        processing_folders[n_sent] = GetDataFolderDFR()
                        make /n=1/d profile1, profile2
                        wave slabdata
                        wave /wave reduced_waves = reduce_slab_image(slabdata, image, reduction_func, func_param)
                        variable /g func_result = numpnts(reduced_waves)
                        adh5_get_result_waves(reduced_waves, "redw_", 0)
                        WaveClear slabdata, image, reduced_waves
                        setdatafolder ::
                    endif
 
                    iz += 1
                    n_sent += 1
                    tmo = 0
                else
                    iz += 1
                    it = 0
                endif
            endif
        else
            // throttle the loop if processing is slow
            tmo = min(100, tmo + 10)
        endif
 
        // receive a slab from the processing queue
        if (n_recvd < nzt)
            if (nthreads > 0)
                dfr = ThreadGroupGetDFR(threadGroupID, tmo)
            else
                dfr = processing_folders[n_recvd]
                processing_folders[n_recvd] = $""
            endif
 
            if (DatafolderRefStatus(dfr) != 0)
                // access results folder
                nvar rr = dfr:r_index
                nvar ss = dfr:s_index
                nvar func_result = dfr:func_result
 
                if (func_result < 1)
                    print "error during data reduction."
                    result = -3
                    break
                endif
 
                // initialize result waves just once
                if (numpnts(result_waves) == 0)
                    redimension /n=(func_result) result_waves
                    for (iw = 0; iw < func_result; iw += 1)
                        sw = "redw_" + num2str(iw)
                        wave profile = dfr:$sw
                        sw = "ReducedData" + num2str(iw+1)
                        make /n=(dimsize(profile, 0), nz, nt) /d /o $sw
                        wave data = $sw
                        setdimlabel 0, -1, $getdimlabel(profile, 0, -1), data
                        setdimlabel 1, -1, $kScanDimLabel, data
                        note data, note(profile)
                        ps_scale_dataset(data)
                        setscale /p x dimoffset(profile, 0), dimdelta(profile, 0), waveunits(profile, 0), data
                        setscale d 0, 0, waveunits(profile, -1), data
                        result_waves[iw] = data
                    endfor
                endif
 
                // copy results
                for (iw = 0; iw < func_result; iw += 1)
                    sw = "redw_" + num2str(iw)
                    wave profile = dfr:$sw
                    wave data = result_waves[iw]
                    data[][rr][ss] = profile[p]
                endfor
                
                n_recvd += 1
                KillDataFolder /Z dfr
            endif
        else
            // processing complete
            break
        endif
 
        // update progress window
        if (progress)
            if (update_progress_panel(n_recvd))
                print "user abort"
                result = -4
                break
            endif
        endif
    while ((n_recvd < nzt) && (result == 0))
 
    // clean up 
    killwaves /z slab, slabdata, template
 
    if (nthreads > 0)
        variable tstatus = ThreadGroupRelease(threadGroupID)
        if (tstatus == -2)
            print "error: thread did not terminate properly."
            result = -5
        endif
    endif
    
    // finalize results
    nw = numpnts(result_waves)
    wavenames = ""
    for (iw = 0; iw < nw; iw += 1)
        wave /z data = result_waves[iw]
        if (WaveExists(data))
            if (nz == 1)
                redimension /n=(-1, 0, 0) data
            elseif (nt == 1)
                redimension /n=(-1, nz, 0) data
            endif
            wavenames += nameofwave(data) + ";"
        endif
    endfor
 
    if (progress)
        kill_progress_panel()
    endif
    
    setdatafolder base_df
    return wavenames
end
 
 
threadsafe static function reduce_slab_worker(reduction_func)
    funcref adh5_default_reduction reduction_func
    do
        // wait for job from main thread
        do
            dfref dfr = ThreadGroupGetDFR(0, 1000)
            if (DataFolderRefStatus(dfr) == 0)
                if (GetRTError(2))
                    return 0 // no more jobs
                endif
            else
                break
            endif
        while (1)
 
        // get input data
        wave slabdata = dfr:slabdata
        wave image = dfr:image
        svar func_param = dfr:func_param
        nvar rr = dfr:r_index
        nvar ss = dfr:s_index
        
        // do the work
        newdatafolder /s out_df
        variable /g r_index = rr
        variable /g s_index = ss
        wave /wave reduced_waves = reduce_slab_image(slabdata, image, reduction_func, func_param)
        variable /g func_result = numpnts(reduced_waves)
 
        // send output to queue and clean up
        adh5_get_result_waves(reduced_waves, "redw_", 0)
        WaveClear slabdata, image, reduced_waves
        ThreadGroupPutDF 0, :
        KillDataFolder dfr
    while (1)
    
    return 0
end
 
threadsafe static function /wave reduce_slab_image(slabdata, image, reduction_func, reduction_params)
    wave slabdata
    wave image
    funcref adh5_default_reduction reduction_func
    string reduction_params
    
    image = slabdata[q][p][0][0]
    
    return reduction_func(image, reduction_params)
end
 
// ====== meta and auxiliary data ======
 
function /s psh5_load_general_group(file_df)
    dfref file_df
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
 
    string obj_names = "authors;pgroup;proposal;proposer;sample;"
    variable nn = ItemsInList(obj_names, ";")
    variable ii
    string name
 
    for (ii = 0; ii < nn; ii += 1)
        name = StringFromList(ii, obj_names, ";")
        psh_load_general_string(file_df, name)
    endfor
 
    return obj_names
end
 
function /s psh_load_general_string(file_df, name)
    dfref file_df
    string name
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
    
    string path = "/general/" + name
    HDF5LoadData /O /Q /Z /N=wt_load_general /TYPE=1 file_id, path
    string values = ""
    if (!v_flag)
        wave /t wt_load_general
        variable nn = numpnts(wt_load_general)
        variable ii
        for (ii = 0; ii < nn; ii += 1)
            values = AddListItem(wt_load_general[ii], values, ",", inf)
        endfor
        killwaves /z wt_load_general
        if (strlen(values) >= 1)
            values = values[0,strlen(values)-2]
        endif
    endif
    string /g $name = values
    return values
end
 
function psh5_load_dataset_meta(file_df, datasetpath, datawave)
    dfref file_df
    string datasetpath
    wave datawave
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
 
    dfref save_df = GetDataFolderDFR()
    SetDataFolder NewFreeDataFolder()
 
    string wnote
    
    HDF5LoadData /O /Q /Z /A="Writable Dimension" /N=WriteDim file_id, datasetpath
    if (!v_flag)
        wave WriteDim
        // scan dimension starts at 1
        sprintf wnote, "ScanDimension=%u", WriteDim[0]
        Note datawave, wnote
    endif
    
    HDF5LoadData /O /Q /Z /A="Writable Index" /N=WriteIndex file_id, datasetpath
    if (!v_flag)
        wave WriteIndex
        sprintf wnote, "WriteableIndex=%u", WriteIndex[0]
        Note datawave, wnote
    endif
    
    HDF5LoadData /O /Q /Z /A="Readable Index" /N=ReadIndex file_id, datasetpath
    if (!v_flag)
        wave ReadIndex
        sprintf wnote, "ReadableIndex=%u", ReadIndex[0]
        Note datawave, wnote
    endif
 
    setdatafolder save_df   
    return 0
end
 
function /s psh5_load_scan_meta(file_df, scanpath)
    // todo: convert to variables/strings
    dfref file_df
    string scanpath
    string wavenames = ""
 
    if (!DataFolderRefStatus(file_df))
        dfref file_df = GetDataFolderDFR()
    endif
    nvar /sdfr=file_df file_id
 
    HDF5LoadData /O /Q /Z /A="Dimensions" /N=ScanDimensions /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    else
        make /n=1 /o ScanDimensions
        ScanDimensions = 0
        wavenames = AddListItem("ScanDimensions", wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Readables" /N=ScanReadables /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    else
        make /n=1 /o /t ScanReadables
        ScanReadables[0] = "ScientaSpectrum"
        wavenames = AddListItem("ScanReadables", wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Writables" /N=ScanWritables /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    else
        // OTF script
        HDF5LoadData /O /Q /Z /A="PlotDomain" /N=ScanWritables /TYPE=1 file_id, scanpath
        if (!v_flag)
            wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
        endif
    endif
    HDF5LoadData /O /Q /Z /A="Steps" /N=ScanSteps /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
    wavenames = ReplaceString(";;", wavenames, ";")
 
    // additional attributes from XPSSpectrum.py
    HDF5LoadData /O /Q /Z /A="Iterations" /N=ScanIterations /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Step Size" /N=ScanStepSize /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Step Time" /N=ScanStepTime /TYPE=1 file_id, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
 
    return wavenames
end
 
// ====== dimension scaling ======
 
function ps_set_dimlabels(data)
    wave data
    
    ps_set_dimlabels2(data, NameOfWave(data))
end
 
function ps_set_dimlabels2(data, name)
    wave data
    string name
    
    variable dummy
    try 
        // intrinsic dimensions
        strswitch(name)
            case "ScientaImage":
                setdimlabel 0, -1, $kEnergyDimLabel, data
                setdimlabel 1, -1, $kAngleDimLabel, data
                if (WaveDims(data) >= 3)
                    setdimlabel 2, -1, $kScanDimLabel, data
                endif
                AbortOnRTE
                break
            case "ImageAngleDistribution":
            case "ScientaAngleDistribution":
                if (WaveDims(data) >= 2)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    setdimlabel 1, -1, $kAngleDimLabel, data
                else
                    setdimlabel 0, -1, $kAngleDimLabel, data
                endif
                AbortOnRTE
                break
            case "ScientaSpectrum":
            case "ImageEnergyDistribution":
            case "ScientaEnergyDistribution":
                if (WaveDims(data) >= 2)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    setdimlabel 1, -1, $kEnergyDimLabel, data
                else
                    setdimlabel 0, -1, $kEnergyDimLabel, data
                endif
                AbortOnRTE
                break
            default:
                if (WaveDims(data) == 1)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    AbortOnRTE
                else
                    return 1
                endif
        endswitch
    catch
        dummy = GetRTError(1)
        return 2
    endtry
    return 0
end
 
function /df ps_find_scan_folder(data_df)
    dfref data_df
    
    wave /z /t /sdfr=data_df ScanWritables=::ScanWritables
    if (WaveExists(ScanWritables))
        string sdf = GetDataFolder(1, data_df)
        dfref parent_df = $(sdf + ":")
        return parent_df
    else
        return data_df
    endif
end
 
function /df ps_find_attr_folder(scan_df)
    dfref scan_df
    
    dfref diags_df = data_df:diags
    dfref attrs_df = scan_df:attrs
    dfref attr_df = scan_df:attr
    if (DataFolderRefStatus(diags_df))
        return diags_df
    elseif (DataFolderRefStatus(attrs_df))
        return attrs_df
    elseif (DataFolderRefStatus(attr_df))
        return attr_df
    else
        return $""
    endif
end
 
function /wave ps_find_scale_wave(name, df1, df2, df3)
    string name
    dfref df1
    dfref df2
    dfref df3
    
    variable idf
    variable ndf=3
    make /n=(ndf) /df /free dfs
    dfs[0] = {df1, df2, df3}
    for (idf = 0; idf < ndf; idf += 1)
        if (DataFolderRefStatus(dfs[idf]))
            wave /SDFR=dfs[idf] /Z w = $name
            if (WaveExists(w))
                return w
            endif
        endif
    endfor
 
    return $""
end
 
function ps_detect_scale(data_df, ax, lo, hi, un)
    dfref data_df
    wave /t ax
    wave lo
    wave hi
    wave /t un
 
    dfref scan_df = ps_find_scan_folder(data_df)
    dfref attr_df = ps_find_attr_folder(scan_df)
    
    redimension /n=4 lo, hi, un, ax
    setdimlabel 0, 0, $kEnergyDimLabel, lo, hi, un, ax
    setdimlabel 0, 1, $kAngleDimLabel, lo, hi, un, ax
    setdimlabel 0, 2, $kScanDimLabel, lo, hi, un, ax
    setdimlabel 0, 3, $kDataDimLabel, lo, hi, un, ax
 
    // default values
    lo[%$kEnergyDimLabel] = 0
    hi[%$kEnergyDimLabel] = 1
    un[%$kEnergyDimLabel] = "eV"
    ax[%$kEnergyDimLabel] = "Ekin"
 
    lo[%$kAngleDimLabel] = -1
    hi[%$kAngleDimLabel] = 1
    un[%$kAngleDimLabel] = "arb."
    un[%$kAngleDimLabel] = "slice"
 
    lo[%$kScanDimLabel] = 0
    hi[%$kScanDimLabel] = 1
    un[%$kScanDimLabel] = "arb."
    ax[%$kScanDimLabel] = "scan"
    
    lo[%$kDataDimLabel] = 0
    hi[%$kDataDimLabel] = 0
    un[%$kDataDimLabel] = "arb."
    ax[%$kDataDimLabel] = "value"
    
    wave /T /Z LensMode = ps_find_scale_wave("LensMode", data_df, scan_df, attr_df)
    wave /Z LowEnergy = ps_find_scale_wave("ScientaLowEnergy", data_df, scan_df, attr_df)
    wave /Z HighEnergy = ps_find_scale_wave("ScientaHighEnergy", data_df, scan_df, attr_df)
    wave /Z LowThetaX = ps_find_scale_wave("ScientaLowThetaX", data_df, scan_df, attr_df)
    wave /Z HighThetaX = ps_find_scale_wave("ScientaHighThetaX", data_df, scan_df, attr_df)
    wave /Z ChannelBegin = ps_find_scale_wave("ScientaChannelBegin", data_df, scan_df, attr_df)
    wave /Z ChannelEnd = ps_find_scale_wave("ScientaChannelEnd", data_df, scan_df, attr_df)
    wave /Z SliceBegin = ps_find_scale_wave("ScientaSliceBegin", data_df, scan_df, attr_df)
    wave /Z SliceEnd = ps_find_scale_wave("ScientaSliceEnd", data_df, scan_df, attr_df)
    wave /Z ScientaChannels = ps_find_scale_wave("ScientaChannels", data_df, scan_df, attr_df)
    
    // lens mode can give more detail
    if (waveexists(LensMode) && (numpnts(LensMode) >= 1))
        if (stringmatch(LensMode[0], "*Transmission*"))
            un[%$kAngleDimLabel] = "mm"
            ax[%$kAngleDimLabel] = "position"
        else
            un[%$kAngleDimLabel] = "°"
            ax[%$kAngleDimLabel] = "angle"
        endif
    endif
    
    // best option if scales are explicit in separate waves
    if (waveexists(LowEnergy) && waveexists(HighEnergy) && (numpnts(LowEnergy) >= 1) && (numpnts(HighEnergy) >= 1))
        lo[%$kEnergyDimLabel] = LowEnergy[0]
        hi[%$kEnergyDimLabel] = HighEnergy[0]
    elseif (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1))
        lo[%$kEnergyDimLabel] = ChannelBegin[0]
        hi[%$kEnergyDimLabel] = ChannelEnd[0]
    elseif (waveexists(ScientaChannels) && (numpnts(ScientaChannels) >= 1))
        lo[%$kEnergyDimLabel] = ScientaChannels[0]
        hi[%$kEnergyDimLabel] = ScientaChannels[numpnts(ScientaChannels)-1]
    endif
    if (waveexists(LowThetaX) && waveexists(HighThetaX) && (numpnts(LowThetaX) >= 1) && (numpnts(HighThetaX) >= 1))
        lo[%$kAngleDimLabel] = LowThetaX[0]
        hi[%$kAngleDimLabel] = HighThetaX[0]
    elseif (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1))
        lo[%$kAngleDimLabel] = SliceBegin[0]
        hi[%$kAngleDimLabel] = SliceEnd[0]
    endif
    
    wave /z /t /SDFR=scan_df ScanWritables
    if (WaveExists(ScanWritables))
        wave /z /SDFR=scan_df scanner = $ScanWritables[0]
        if (!WaveExists(scanner))
            wave /z /SDFR=attr_df scanner = $ScanWritables[0]
        endif
        if (WaveExists(scanner) && (numpnts(scanner) >= 1))
            lo[%$kScanDimLabel] = scanner[0]
            hi[%$kScanDimLabel] = scanner[numpnts(scanner)-1]
            ax[%$kScanDimLabel] = NameOfWave(scanner)
            strswitch(NameOfWave(scanner))
                case "Eph":
                    ax[%$kScanDimLabel] = "photon energy"
                    un[%$kScanDimLabel] = "eV"
                    break
                case "ManipulatorX":
                case "ManipulatorY":
                case "ManipulatorZ":
                case "FocusYTrans":
                case "FocusZTrans":
                case "RefocusYTrans":
                case "RefocusZTrans":
                case "ExitSlitY":
                    un[%$kScanDimLabel] = "mm"
                    break
                case "ExitSlit":
                    un[%$kScanDimLabel] = "µm"
                    break
                case "ManipulatorTheta":
                case "ManipulatorTilt":
                case "ManipulatorPhi":
                    un[%$kScanDimLabel] = "°"
                    break
                case "FocusXRot":
                case "FocusYRot":
                case "FocusZRot":
                case "RefocusXRot":
                case "RefocusYRot":
                case "RefocusZRot":
                    un[%$kScanDimLabel] = "mrad"
                    break
            endswitch
        endif
    endif
end
 
function ps_scale_dataset_2(data, ax, lo, hi, un)
    wave data
    wave /t ax
    wave lo
    wave hi
    wave /t un
    
    string snote = note(data)
    string sdim
    sdim = GetDimLabel(data, 0, -1)
    if (strlen(sdim))
        setscale /i x lo[%$sdim], hi[%$sdim], un[%$sdim], data
        snote = ReplaceStringByKey("AxisLabelX", snote, ax[%$sdim], "=", "\r")
    endif
 
    sdim = GetDimLabel(data, 1, -1)
    if (strlen(sdim))
        setscale /i y lo[%$sdim], hi[%$sdim], un[%$sdim], data
        snote = ReplaceStringByKey("AxisLabelY", snote, ax[%$sdim], "=", "\r")
    endif
    
    sdim = GetDimLabel(data, 2, -1)
    if (strlen(sdim))
        setscale /i z lo[%$sdim], hi[%$sdim], un[%$sdim], data
        snote = ReplaceStringByKey("AxisLabelZ", snote, ax[%$sdim], "=", "\r")
    endif
 
    string data_unit = un[%$kDataDimLabel]
    string data_label = ax[%$kDataDimLabel]
    string s
    variable def = (cmpstr(data_unit, "arb.") == 0) && (cmpstr(data_label, "value") == 0)
    
    if (def)
        s = StringByKey("AxisLabelD", snote, "=", "\r")
        if (strlen(s) > 0)
            data_label = s
            def = 0
        endif
        s = StringByKey("AxisUnitD", snote, "=", "\r")
        if (strlen(s) > 0)
            data_unit = s
            def = 0
        endif
    endif
    
    if (def)
        strswitch(NameOfWave(data))
            case "ScientaImage":
            case "ImageAngleDistribution":
            case "ScientaAngleDistribution":
            case "ScientaSpectrum":
            case "ImageEnergyDistribution":
            case "ScientaEnergyDistribution":
                data_unit = "arb."
                data_label = "intensity"
                def = 0
                break
            case "SampleCurrent":
            case "RefCurrent":
            case "AuxCurrent":
                data_unit = "A"
                data_label = "current"
                def = 0
                break
            case "MachineCurrent":
                data_unit = "mA"
                data_label = "current"
                def = 0
                break
        endswitch
    endif
    
    setscale d 0, 0, data_unit, data
    snote = ReplaceStringByKey("AxisLabelD", snote, data_label, "=", "\r")
    snote = ReplaceStringByKey("AxisUnitD", snote, data_unit, "=", "\r")
    snote = ReplaceStringByKey("Dataset", snote, NameOfWave(data), "=", "\r")
    note /k data, snote
end
 
function ps_scale_datasets(scan_df)
    dfref scan_df
    
    make /n=3 /free lo, hi
    make /n=3 /t /free ax, un
    wave /t /z /SDFR=scan_df ScanReadables
    if (WaveExists(ScanReadables))
        variable isr
        variable nsr = numpnts(ScanReadables)
        variable nel
        string ssr
        string sds
        for (isr = 0; isr < nsr; isr += 1)
            ssr = ScanReadables[isr]
            dfref data_df = psh5_dataset_to_folder(scan_df, ssr)
            if (!DataFolderRefStatus(data_df))
                dfref data_df = scan_df
            endif
            nel = ItemsInList(ssr, "/")
            sds = StringFromList(nel - 1, ssr, "/")
            wave /z /sdfr=data_df wsr=$sds
            if (WaveExists(wsr))
                ps_detect_scale(data_df, ax, lo, hi, un)
                ps_scale_dataset_2(wsr, ax, lo, hi, un)
            endif
        endfor
    endif
end
 
function ps_scale_dataset(data)
    wave data
 
    dfref save_df = GetDataFolderDFR()
    dfref data_df = GetWavesDataFolderDFR(data)
    
    setdatafolder data_df
    make /n=3 /free lo, hi
    make /n=3 /t /free ax, un
    ps_detect_scale(data_df, ax, lo, hi, un)
    ps_scale_dataset_2(data, ax, lo, hi, un)
    setdatafolder save_df
end
 
 
// ====== miscellaneous functions ======
 
function /s kill_matching_waves(dfr, pattern, recurse, [killed])
    DFREF dfr
    string pattern
    variable recurse
    string killed
    
    if (ParamIsDefault(killed))
        killed = ""
    endif
    
    string s
    string r
    variable index = 0
    do
        Wave/Z w = WaveRefIndexedDFR(dfr, index)
        if (!WaveExists(w))
            break
        endif
 
        s = NameOfWave(w)
        if (stringmatch(s, pattern))
            killwaves /z w
            killed = AddListItem(s, killed, ";", Inf)
        endif
 
        index += 1
    while(1)
    
    if (recurse)
        Variable numChildDataFolders = CountObjectsDFR(dfr, 4)
        Variable i
        for(i=0; i<numChildDataFolders; i+=1)
            String childDFName = GetIndexedObjNameDFR(dfr, 4, i)
            DFREF childDFR = dfr:$childDFName
            killed = kill_matching_waves(childDFR, pattern, 1, killed=killed)
        endfor
    endif
    
    return killed
End