pearl-pshell-import.ipf

Go to the documentation of this file.

#pragma rtGlobals=3// Use modern global access method and strict wave access.
#pragma IgorVersion = 6.36
#pragma ModuleName = PearlPShellImport
#pragma version = 1.03
#include <HDF5 Browser>
#include "pearl-gui-tools"
#include "pearl-area-import"

// copyright (c) 2013-16 Paul Scherrer Institut
//
// Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//   http:///www.apache.org/licenses/LICENSE-2.0



const string kEnergyDimLabel = "energy";

const string kAngleDimLabel = "angle";

const string kScanDimLabel = "scan";

const string kDataDimLabel = "data";

const string kPreviewDatasets = "ScientaImage;ScientaSpectrum;ImageAngleDistribution;ImageEnergyDistribution;Counts;SampleCurrent;";

const string kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChannelEnd;ScientaSliceBegin;ScientaSliceEnd;";

const string kTransposedDatasets = "ScientaImage;";

const variable kDetectorSensitivity = 4;

variable psh5_open_file(string ANickName, string APathName, string AFileName){
    string ANickName
    string APathName
    string AFileName
    
    setdatafolder root:
    newdatafolder /s /o $("root:" + ANickName)
    dfref fileDF = GetDataFolderDFR()
  
    variable fileID
    HDF5OpenFile /P=$APathName /R fileID as AFileName
    if (v_flag == 0)
        string /g s_filepath
        string /g s_scanpaths
        s_filepath = s_path + s_filename
        s_scanpaths = psh5_list_scans(fileID)
    else
        fileID = 0
    endif
    
    return fileID
};

variable psh5_close_file(variable fileID){
    variable fileID

    HDF5CloseFile fileID
};

string psh5_load_complete(string ANickName, string APathName, string AFileName, variable load_data = defaultValue, variable load_attr = defaultValue){
    string ANickName
    string APathName
    string AFileName
    variable load_data
    variable load_attr
    
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif
  
    dfref saveDF = GetDataFolderDFR()
  
    // performance monitoring
    variable timerRefNum
    variable /g psh5_perf_secs
    timerRefNum = startMSTimer
    
    variable fileID = psh5_open_file(ANickName, APathName, AFileName)
    if (fileID)
        dfref fileDF = GetDataFolderDFR()
        svar s_filepath
        svar s_scanpaths
        AFileName = s_filepath
        print "loading " + s_filepath + "\r"

        variable ig
        variable ng = ItemsInList(s_scanpaths, ";")
        string sg
        string folder
        
        for (ig = 0; ig < ng; ig += 1)
            sg = StringFromList(ig, s_scanpaths, ";")
            folder = ReplaceString("/", sg, "")
            folder = ReplaceString(" ", folder, "")
            folder = CleanupName(folder, 0)
            setdatafolder fileDF
            newdatafolder /s /o $folder
            psh5_load_scan_complete(fileID, sg, load_data=load_data, load_attr=load_attr)
        endfor
        
        psh5_close_file(fileID)
    else
        AFileName = ""
    endif

    psh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    
    setdatafolder saveDF
    return AFileName
};

string psh5_load_preview(string APathName, string AFileName, variable load_data = defaultValue, variable load_attr = defaultValue, string pref_scans = defaultValue, string pref_datasets = defaultValue){
    string APathName
    string AFileName
    variable load_data
    variable load_attr
    string pref_scans
    string pref_datasets
        
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif
    if (ParamIsDefault(pref_scans))
        pref_scans = "*scan1*;"
    endif
    if (ParamIsDefault(pref_datasets))
        pref_datasets = ""
    endif
    
    dfref saveDF = GetDataFolderDFR()

    variable fileID
    string scanpaths = ""
    string dataname = ""
    
    // performance monitoring
    variable timerRefNum
    variable /g adh5_perf_secs
    timerRefNum = startMSTimer
    
    HDF5OpenFile /P=$APathName /R /Z fileID as AFileName
    if (v_flag == 0)
        AFileName = s_path + s_filename
        dfref fileDF = GetDataFolderDFR()

        scanpaths = psh5_list_scans(fileID)
        variable ng = ItemsInList(scanpaths)
        variable ig
        string sg
        variable np = ItemsInList(pref_scans)
        variable ip
        string sp
        variable found = 0
        if (ng > 0)
            for (ip = 0; ip < np; ip += 1)
                for (ig = 0; ig < ng; ig += 1)
                    sg = StringFromList(ig, scanpaths)
                    sp = StringFromList(ip, pref_scans)
                    if (StringMatch(sg, sp))
                        found = 1
                        break
                    endif
                endfor
                if (found)
                    break
                endif
            endfor
            if (!found)
                ig = 0
            endif
            sg = StringFromList(ig, scanpaths)
            
            if (load_attr)
                setdatafolder fileDF
                newdatafolder /o/s attr
                killwaves /a/z
                psh5_load_scan_attrs(fileID, sg)
            endif
            
            setdatafolder fileDF
            dataname = psh5_load_scan_preview(fileID, sg, set_scale=load_attr, pref_datasets=pref_datasets)
        else
            print "no scans found in file " + AFileName
        endif

        HDF5CloseFile fileID
    endif

    if (timerRefNum >= 0)
        adh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    endif
    
    setdatafolder saveDF
    return dataname
};

string psh5_load_scan_complete(variable fileID, string scanpath, variable load_data = defaultValue, variable load_attr = defaultValue){
    variable fileID
    string scanpath
    variable load_data
    variable load_attr
    
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif
  
    dfref saveDF = GetDataFolderDFR()
  
    dfref dataDF = GetDataFolderDFR()
    string wavenames
    string attrnames
    psh5_load_scan_meta(fileID, scanpath)
    if (load_attr)
        newdatafolder /s /o attr
        attrnames = psh5_load_scan_attrs(fileID, scanpath)
    endif
    if (load_data)
        setdatafolder dataDF
        wavenames = psh5_load_scan_data(fileID, scanpath)
    endif
    if (load_data && load_attr)
        setdatafolder dataDF
        ps_scale_datasets()
    endif

    setdatafolder saveDF
    return wavenames
};

string psh5_list_scans(variable fileID){
    variable fileID
  
    HDF5ListGroup /F /TYPE=1 fileID, "/"
        
    variable ig
    variable ng = ItemsInList(S_HDF5ListGroup, ";")
    string sg
    string scans = ""
        
    for (ig = 0; ig < ng; ig += 1)
        sg = StringFromList(ig, S_HDF5ListGroup, ";")
        if (cmpstr(sg[1,4], "scan") == 0)
            scans = AddListItem(sg, scans, ";", inf)
        endif
    endfor
    
    return scans
};

string psh5_list_scan_datasets(variable fileID, string scanpath, variable include_regions = defaultValue){
    variable fileID
    string scanpath
    variable include_regions
    
    if (ParamIsDefault(include_regions))
        include_regions = 0
    endif
    string result
    
    HDF5ListGroup /TYPE=2 /Z fileID, scanpath
    result = S_HDF5ListGroup
    
    if (include_regions)
        HDF5ListGroup /R /TYPE=2 /Z fileID, scanpath
        variable n = ItemsInList(S_HDF5ListGroup)
        variable i
        string ds
        string region_datasets
        for (i = 0; i < n; i += 1)
            ds = StringFromList(i, S_HDF5ListGroup)
            if (StringMatch(ds, "region*/*"))
                //region_datasets = psh5_list_scan_datasets(fileID, ReplaceString("//", scanpath + "/" + region, "/"), include_regions=0)
                result = AddListItem(ds, result, ";", inf)
            endif
        endfor
    endif
    
    return result
};

string psh5_list_scan_regions(variable fileID, string scanpath){
    variable fileID
    string scanpath
    
    HDF5ListGroup /TYPE=1 /Z fileID, scanpath
    variable n = ItemsInList(S_HDF5ListGroup)
    variable i
    string result = ""
    string s
    for (i = 0; i < n; i += 1)
        s = StringFromList(i, S_HDF5ListGroup)
        if (StringMatch(s, "region*"))
            result = AddListItem(s, result, ";", inf)
        endif
    endfor
    
    return result
};

string psh5_load_scan_data(variable fileID, string scanpath){
    variable fileID
    string scanpath
    
    string datasets = psh5_list_scan_datasets(fileID, scanpath, include_regions=1)
    variable nds = ItemsInList(datasets)
    variable ids
    string sds
    string sw
    string wavenames = ""
    for (ids = 0; ids < nds; ids += 1)
        sds = StringFromList(ids, datasets)
        sw = psh5_load_dataset(fileID, scanpath, sds, set_scale=0)
        wavenames = AddListItem(sw, wavenames, ";", inf)
    endfor
    
    return wavenames
};

string psh5_load_scan_attrs(variable fileID, string scanpath, variable attr_sets = defaultValue){
    variable fileID
    string scanpath
    variable attr_sets

    if (ParamIsDefault(attr_sets))
        attr_sets = 1
    endif

    string attr_path = ReplaceString("//", scanpath + "/attrs", "/")
    string attr_list = ""
    if (attr_sets & 1)
        HDF5ListGroup /TYPE=2 /Z fileID, attr_path
        if (!v_flag)
            attr_list = S_HDF5ListGroup
        endif
    endif
    
    variable ids
    variable nds
    string sds

    if (attr_sets & 2)
        nds = ItemsInList(kScientaScalingDatasets, ";")
        for (ids = 0; ids < nds; ids += 1)
            sds = StringFromList(ids, kScientaScalingDatasets)
            if (WhichListItem(sds, attr_list) < 0)
                attr_list = AddListItem(sds, attr_list, ";", inf)
            endif
        endfor
    endif

    nds = ItemsInList(attr_list, ";")
    string wavenames = ""
    for (ids = 0; ids < nds; ids += 1)
        sds = StringFromList(ids, attr_list, ";")
        HDF5LoadData /O /Q /Z fileID, attr_path + "/" + sds
        if (!v_flag)
            wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
        endif
    endfor
    wavenames = ReplaceString(";;", wavenames, ";")
    
    return wavenames
};

string psh5_load_scan_meta(variable fileID, string scanpath){
    variable fileID
    string scanpath
    string wavenames = ""

    HDF5LoadData /O /Q /Z /A="Dimensions" /N=ScanDimensions /TYPE=1 fileID, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    else
        make /n=1 /o ScanDimensions
        ScanDimensions = 0
        wavenames = AddListItem("ScanDimensions", wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Readables" /N=ScanReadables /TYPE=1 fileID, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    else
        make /n=1 /o /t ScanReadables
        ScanReadables[0] = "ScientaSpectrum"
        wavenames = AddListItem("ScanReadables", wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Writables" /N=ScanWritables /TYPE=1 fileID, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
    HDF5LoadData /O /Q /Z /A="Steps" /N=ScanSteps /TYPE=1 fileID, scanpath
    if (!v_flag)
        wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
    endif
    wavenames = ReplaceString(";;", wavenames, ";")

    return wavenames
};

string psh5_load_dataset(variable fileID, string scanpath, string datasetname, variable set_scale = defaultValue){
    variable fileID
    string scanpath
    string datasetname
    variable set_scale

    if (ParamIsDefault(set_scale))
        set_scale = 1
    endif
    
    dfref base_df = GetDataFolderDFR()
    
    string datasetpath
    datasetpath = scanpath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    
    string regionname
    string regionpath
    if (ItemsInList(datasetname, "/") >= 2)
        regionname = StringFromList(0, datasetname, "/")
        regionpath = ReplaceString("//", scanpath + "/" + regionname, "/")
        datasetname = RemoveListItem(0, datasetname, "/")
        NewDataFolder /o/s $regionname
    else
        regionname = ""
        regionpath = scanpath
    endif
    
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return ""
    endif
    
    string dataname
    if (di.ndims < 2)
        HDF5LoadData /O /Q /Z fileID, datasetpath
        dataname = StringFromList(0, S_waveNames)
    else
        dataname = psh5_load_dataset_slabs(fileID, regionpath, datasetname)
    endif
    
    wave /z data = $dataname
    if (waveexists(data))
        psh5_load_dataset_meta(fileID, regionpath, datasetname, data)
        ps_set_dimlabels(data)
        if (set_scale)
            ps_scale_dataset(data)
        endif
    else
        dataname = ""
    endif
    
    setdatafolder base_df
    return dataname
};

static string select_dataset(string file_datasets, string pref_datasets){
    string file_datasets
    string pref_datasets

    variable index
    variable nds = ItemsInList(file_datasets)
    variable ids
    string sds = ""
    string mds = ""
    variable np = ItemsInList(pref_datasets)
    variable ip
    string sp
    variable found = 0
    if (nds > 0)
        for (ip = 0; ip < np; ip += 1)
            for (ids = 0; ids < nds; ids += 1)
                sds = StringFromList(ids, file_datasets)
                index = ItemsInList(sds, "/") - 1
                mds = StringFromList(index, sds, "/")
                sp = StringFromList(ip, pref_datasets)
                if (StringMatch(mds, sp))
                    found = 1
                    break
                endif
            endfor
            if (found)
                break
            endif
        endfor
        if (!found)
            ids = 0
            sds = StringFromList(ids, file_datasets)
        endif
    endif
    
    return sds
};

string psh5_load_scan_preview(variable fileID, string scanpath, variable set_scale = defaultValue, string pref_datasets = defaultValue){
    variable fileID
    string scanpath
    variable set_scale
    string pref_datasets

    if (ParamIsDefault(set_scale))
        set_scale = 1
    endif
    if (ParamIsDefault(pref_datasets) || (strlen(pref_datasets) == 0))
        pref_datasets = kPreviewDatasets
    endif

    dfref saveDF = GetDataFolderDFR()
    dfref dataDF = saveDF
    
    string datasets = psh5_list_scan_datasets(fileID, scanpath, include_regions=1)
    string datasetname = select_dataset(datasets, pref_datasets)
    string datasetpath
    datasetpath = scanpath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return ""
    endif
    
    string dataname
    if (di.ndims < 2)
        HDF5LoadData /O /Q /Z fileID, datasetpath
        dataname = StringFromList(0, S_waveNames)
        wave /z data = $dataname
        if (waveexists(data))
            ps_set_dimlabels(data)
        endif
    else
        variable dim2start = 0
        variable dim2count = 1
        variable dim3start = 0
        variable dim3count = 1
        if (di.ndims >= 3)
            dim2start = floor(di.dims[2] / 2)
            dim2count = 1
        endif
        if (di.ndims >= 4)
            dim3start = floor(di.dims[3] / 2)
            dim3count = 1
        endif
        
        dataname = psh5_load_dataset_slab(fileID, scanpath, datasetname, dim2start, dim2count, dim3start, dim3count)
    endif
    
    wave /z data = $dataname
    if (waveexists(data))
        if (set_scale)
            setdatafolder dataDF
            string positioners
            string positioner
            string positionerpath
            positioners = psh5_load_scan_meta(fileID, scanpath)
            wave /t /z ScanWritables
            if (waveexists(ScanWritables) && (numpnts(ScanWritables) >= 1))
                positioner = ScanWritables[0]
                if (strlen(positioner) > 0)
                    positionerpath = scanpath + "/" + positioner
                    positionerpath = ReplaceString("//", positionerpath, "/")
                    HDF5LoadData /O /Q /Z fileID, positionerpath
                endif
            endif
            
            setdatafolder dataDF
            newdatafolder /o/s attr
            psh5_load_scan_attrs(fileID, scanpath, attr_sets=2)
            setdatafolder dataDF
            ps_scale_dataset(data)
        endif
    else
        dataname = ""
    endif

    return dataname
};

string psh5_load_scan_section(variable fileID, string scanpath, variable dim, variable set_scale = defaultValue, string pref_datasets = defaultValue){
    variable fileID
    string scanpath
    variable dim
    variable set_scale
    string pref_datasets

    // select first dimension (future argument)
    // 0 = first dimension is x axis (energy of scienta image)
    dim = 0
    
    if (ParamIsDefault(set_scale))
        set_scale = 1
    endif
    if (ParamIsDefault(pref_datasets) || (strlen(pref_datasets) == 0))
        pref_datasets = kPreviewDatasets
    endif

    dfref saveDF = GetDataFolderDFR()
    dfref dataDF = saveDF
    
    string datasets = psh5_list_scan_datasets(fileID, scanpath)
    string datasetname = select_dataset(datasets, pref_datasets)
    string datasetpath
    datasetpath = scanpath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    string dataname = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    string destname = dataname[0,29] + num2str(dim)
    
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return ""
    else if (di.ndims != 3)
        print "error: rank of dataset != 3"
        return ""
    endif
    
    variable idx, idy, idz, idt
    variable transpose = WhichListItem(dataname, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3

    variable nx, ny, nz
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = di.dims[idz]

    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    
    if (dim == 0)
        slab[idy][%Start] = floor(ny / 2)
        slab[idx][%Block] = nx
        make /n=(nx,nz) /o $destname
    else
        slab[idx][%Start] = floor(nx / 2)
        slab[idy][%Block] = ny
        make /n=(ny,nz) /o $destname
    endif
    slab[idz][%Block] = nz
    wave data = $destname
    data = 0

    HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetpath
    if (!v_flag)
        wave slabdata
        if (transpose)
            data += slabdata[0][p][q][0]
        else
            data += slabdata[p][0][q][0]
        endif
    endif
    killwaves /z slab, slabdata
    
    if (set_scale)
        make /n=(1,1,1) /free dummy
        ps_set_dimlabels2(dummy, dataname)
        setdimlabel 0, -1, $GetDimLabel(dummy, dim, -1), data
        setdimlabel 1, -1, $kScanDimLabel, data
        
        setdatafolder dataDF
        string positioners
        string positioner
        string positionerpath
        positioners = psh5_load_scan_meta(fileID, scanpath)
        wave /t /z ScanWritables
        if (waveexists(ScanWritables) && (numpnts(ScanWritables) >= 1))
            positioner = ScanWritables[0]
            if (strlen(positioner) > 0)
                positionerpath = scanpath + "/" + positioner
                positionerpath = ReplaceString("//", positionerpath, "/")
                HDF5LoadData /O /Q /Z fileID, positionerpath
            endif
        endif
        
        setdatafolder dataDF
        newdatafolder /o/s attr
        killwaves /a/z
        psh5_load_scan_attrs(fileID, scanpath, attr_sets=2)
        setdatafolder dataDF
        ps_scale_dataset(data)
    endif

    return destname
};

variable psh5_load_dataset_meta(variable fileID, string datapath, string datasetname, wave datawave){
    variable fileID
    string datapath
    string datasetname
    wave datawave

    dfref saveDF = GetDataFolderDFR()
    SetDataFolder NewFreeDataFolder()

    string datasetpath = datapath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    string wnote
    
    HDF5LoadData /O /Q /Z /A="Writable Dimension" /N=WriteDim fileID, datasetpath
    if (!v_flag)
        wave WriteDim
        // scan dimension starts at 1
        sprintf wnote, "ScanDimension=%u", WriteDim[0]
        Note datawave, wnote
    endif
    
    HDF5LoadData /O /Q /Z /A="Writable Index" /N=WriteIndex fileID, datasetpath
    if (!v_flag)
        wave WriteIndex
        sprintf wnote, "WriteableIndex=%u", WriteIndex[0]
        Note datawave, wnote
    endif
    
    HDF5LoadData /O /Q /Z /A="Readable Index" /N=ReadIndex fileID, datasetpath
    if (!v_flag)
        wave ReadIndex
        sprintf wnote, "ReadableIndex=%u", ReadIndex[0]
        Note datawave, wnote
    endif

    setdatafolder saveDF
    return 0
};

string psh5_load_dataset_slabs(variable fileID, string datapath, string datasetname, variable progress = defaultValue){
    variable fileID
    string datapath
    string datasetname
    variable progress
    
    if (ParamIsDefault(progress))
        progress = 1
    endif

    variable result = 0
    string datasetpath
    string datawavename
    datasetpath = datapath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return ""
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        return ""
    else if (di.ndims < 3)
        progress = 0
    endif
    
    variable idx, idy, idz, idt, izt
    variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3
    
    variable nx, ny, nz, nt, nzt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = di.dims[idz]
    nt = di.dims[idt]
    make /n=(nx,ny,nz,nt) /o $datawavename
    wave data = $datawavename

    nz = max(nz, 1)
    nt = max(nt, 1)
    nzt = nz * nt
    izt = 0
    if (progress)
        display_progress_panel("HDF5 Import", "Loading data...", nzt)
    endif
    
    // load data image by image
    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    variable iz, it
    for (iz = 0; iz < nz; iz += 1)
        for (it = 0; it < nt; it += 1)
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetpath
            wave slabdata// 2D, 3D, or 4D with singletons
            if (transpose)
                data[][][iz][it] = slabdata[q][p][0][0]
            else
                data[][][iz][it] = slabdata[p][q][0][0]
            endif
            
            // progress window
            izt += 1
            if (progress)
                if (update_progress_panel(izt))
                    result = -4// user abort
                    break
                endif
            endif
        endfor
        if (result < 0)
            break
        endif
    endfor
    
    if (progress)
        kill_progress_panel()
    endif
    
    killwaves /z slab, slabdata
    if (!result)
        ps_set_dimlabels(data)
        return datawavename
    else
        killwaves /z data
        return ""
    endif
};

string psh5_load_dataset_slab(variable fileID, string datapath, string datasetname, variable dim2start, variable dim2count, variable dim3start, variable dim3count){
    variable fileID
    string datapath
    string datasetname
    variable dim2start
    variable dim2count
    variable dim3start
    variable dim3count
    
    string datasetpath
    string datawavename
    datasetpath = datapath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    
    STRUCT HDF5DataInfo di
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return ""
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        return ""
    endif
    
    variable idx, idy, idz, idt
    variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3

    variable nx, ny
    nx = di.dims[idx]
    ny = di.dims[idy]
    make /n=(nx,ny) /o $datawavename
    wave data = $datawavename
    data = 0

    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    variable iz, it
    variable navg = 0
    variable dim2end = dim2start + dim2count - 1
    variable dim3end = dim3start + dim3count - 1
    for (iz = dim2start; iz <= dim2end; iz += 1)
        for (it = dim3start; it <= dim3end; it += 1)
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetpath
            if (!v_flag)
                wave slabdata
                if (transpose)
                    data += slabdata[q][p][0][0]
                else
                    data += slabdata[p][q][0][0]
                endif
                navg += 1
            endif
        endfor
    endfor
    if (navg)
        data /= navg
    endif
    
    killwaves /z slab, slabdata
    ps_set_dimlabels(data)
    return datawavename
};

variable ps_set_dimlabels(wave data){
    wave data
    
    ps_set_dimlabels2(data, NameOfWave(data))
};

variable ps_set_dimlabels2(wave data, string name){
    wave data
    string name
    
    variable dummy
    try
        // intrinsic dimensions
        strswitch(name)
            case "ScientaImage":
                setdimlabel 0, -1, $kEnergyDimLabel, data
                setdimlabel 1, -1, $kAngleDimLabel, data
                if (WaveDims(data) >= 3)
                    setdimlabel 2, -1, $kScanDimLabel, data
                endif
                AbortOnRTE
                break
            case "ImageAngleDistribution":
            case "ScientaAngleDistribution":
                if (WaveDims(data) >= 2)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    setdimlabel 1, -1, $kAngleDimLabel, data
                else
                    setdimlabel 0, -1, $kAngleDimLabel, data
                endif
                AbortOnRTE
                break
            case "ScientaSpectrum":
            case "ImageEnergyDistribution":
            case "ScientaEnergyDistribution":
                if (WaveDims(data) >= 2)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    setdimlabel 1, -1, $kEnergyDimLabel, data
                else
                    setdimlabel 0, -1, $kEnergyDimLabel, data
                endif
                AbortOnRTE
                break
            default:
                if (WaveDims(data) == 1)
                    setdimlabel 0, -1, $kScanDimLabel, data
                    AbortOnRTE
                else
                    return 1
                endif
        endswitch
    catch
        dummy = GetRTError(1)
        return 2
    endtry
    return 0
};

static dfr find_scan_folder(dfref dataDF){
    dfref dataDF
    
    dfref attrDF = dataDF:attr
    if (!DataFolderRefStatus(attrDF))
        string df = GetDataFolder(1, dataDF) + ":"
        dfref scanDF = $df
    else
        dfref scanDF = dataDF
    endif
    return scanDF
};

static dfr find_attr_folder(dfref dataDF){
    dfref dataDF
    
    dfref attrDF = dataDF:attr
    if (!DataFolderRefStatus(attrDF))
        string df = GetDataFolder(1, dataDF) + ":"
        dfref scanDF = $df
        dfref attrDF = scanDF:attr
    endif
    return attrDF
};

variable ps_scale_datasets(){
    dfref scanDF = GetDataFolderDFR()
    dfref attrDF = find_attr_folder(scanDF)
    
    make /n=3 /free lo, hi
    make /n=3 /t /free ax, un
    wave /t /z /SDFR=scanDF ScanReadables
    if (WaveExists(ScanReadables))
        variable isr
        variable nsr = numpnts(ScanReadables)
        string ssr
        string sdf
        for (isr = 0; isr < nsr; isr += 1)
            setdatafolder scanDF
            ssr = ScanReadables[isr]
            if (ItemsInList(ssr, "/") >= 2)
                sdf = StringFromList(0, ssr, "/")
                ssr = RemoveListItem(0, ssr, "/")
                setdatafolder $sdf
            endif
            wave /z wsr=$ssr
            if (WaveExists(wsr))
                ps_detect_scale(ax, lo, hi, un)
                ps_scale_dataset_2(wsr, ax, lo, hi, un)
            endif
        endfor
    endif
    setdatafolder scanDF
};

variable ps_scale_dataset(wave data){
    wave data

    dfref saveDF = GetDataFolderDFR()
    dfref dataDF = GetWavesDataFolderDFR(data)
    
    setdatafolder dataDF
    make /n=3 /free lo, hi
    make /n=3 /t /free ax, un
    ps_detect_scale(ax, lo, hi, un)
    ps_scale_dataset_2(data, ax, lo, hi, un)
    setdatafolder saveDF
};

static wave find_scale_wave(string name, dfref dataDF, dfref scanDF, dfref attrDF){
    string name
    dfref dataDF
    dfref scanDF
    dfref attrDF
    
    wave /SDFR=dataDF /Z w = $name
    if (!WaveExists(w))
        wave /SDFR=scanDF /Z w = $name
        if (!WaveExists(w))
            wave /SDFR=attrDF /Z w = $name
        endif
    endif
    return w
};

variable ps_detect_scale(wave ax, wave lo, wave hi, wave un){
    wave /t ax
    wave lo
    wave hi
    wave /t un

    dfref dataDF = GetDataFolderDFR()
    dfref scanDF = find_scan_folder(dataDF)
    dfref attrDF = find_attr_folder(dataDF)
    
    redimension /n=4 lo, hi, un, ax
    setdimlabel 0, 0, $kEnergyDimLabel, lo, hi, un, ax
    setdimlabel 0, 1, $kAngleDimLabel, lo, hi, un, ax
    setdimlabel 0, 2, $kScanDimLabel, lo, hi, un, ax
    setdimlabel 0, 3, $kDataDimLabel, lo, hi, un, ax

    // default values
    lo[%$kEnergyDimLabel] = 0
    hi[%$kEnergyDimLabel] = 1
    un[%$kEnergyDimLabel] = "eV"
    ax[%$kEnergyDimLabel] = "Ekin"

    lo[%$kAngleDimLabel] = -1
    hi[%$kAngleDimLabel] = 1
    un[%$kAngleDimLabel] = "arb."
    un[%$kAngleDimLabel] = "slice"

    lo[%$kScanDimLabel] = 0
    hi[%$kScanDimLabel] = 1
    un[%$kScanDimLabel] = "arb."
    ax[%$kScanDimLabel] = "scan"
    
    lo[%$kDataDimLabel] = 0
    hi[%$kDataDimLabel] = 0
    un[%$kDataDimLabel] = "arb."
    ax[%$kDataDimLabel] = "value"
    
    wave /SDFR=attrDF /T /Z LensMode
    wave /Z ChannelBegin = find_scale_wave("ScientaChannelBegin", dataDF, scanDF, attrDF)
    wave /Z ChannelEnd = find_scale_wave("ScientaChannelEnd", dataDF, scanDF, attrDF)
    wave /Z SliceBegin = find_scale_wave("ScientaSliceBegin", dataDF, scanDF, attrDF)
    wave /Z SliceEnd = find_scale_wave("ScientaSliceEnd", dataDF, scanDF, attrDF)

    // lens mode can give more detail
    if (waveexists(LensMode) && (numpnts(LensMode) >= 1))
        strswitch(LensMode[0])
            case "Angular45":
                lo[%$kAngleDimLabel] = -45/2
                hi[%$kAngleDimLabel] = +45/2
                un[%$kAngleDimLabel] = ""
                ax[%$kAngleDimLabel] = "angle"
                break
            case "Angular60":
                lo[%$kAngleDimLabel] = -60/2
                hi[%$kAngleDimLabel] = +60/2
                un[%$kAngleDimLabel] = ""
                ax[%$kAngleDimLabel] = "angle"
                break
            case "Transmission":
                un[%$kAngleDimLabel] = "arb."
                ax[%$kAngleDimLabel] = "offset"
                break
        endswitch
    endif
    
    // best option if scales are explicit in separate waves
    if (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1))
        lo[%$kEnergyDimLabel] = ChannelBegin[0]
        hi[%$kEnergyDimLabel] = ChannelEnd[0]
    endif
    if (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1))
        lo[%$kAngleDimLabel] = SliceBegin[0]
        hi[%$kAngleDimLabel] = SliceEnd[0]
    endif
    
    wave /z /t /SDFR=scanDF ScanWritables
    if (WaveExists(ScanWritables))
        wave /z /SDFR=scanDF scanner = $ScanWritables[0]
        if (!WaveExists(scanner))
            wave /z /SDFR=attrDF scanner = $ScanWritables[0]
        endif
        if (WaveExists(scanner) && (numpnts(scanner) >= 1))
            lo[%$kScanDimLabel] = scanner[0]
            hi[%$kScanDimLabel] = scanner[numpnts(scanner)-1]
            ax[%$kScanDimLabel] = NameOfWave(scanner)
            strswitch(NameOfWave(scanner))
                case "Eph":
                    ax[%$kScanDimLabel] = "photon energy"
                    un[%$kScanDimLabel] = "eV"
                    break
                case "ManipulatorX":
                case "ManipulatorY":
                case "ManipulatorZ":
                case "FocusYTrans":
                case "FocusZTrans":
                case "RefocusYTrans":
                case "RefocusZTrans":
                case "ExitSlitY":
                    un[%$kScanDimLabel] = "mm"
                    break
                case "ExitSlit":
                    un[%$kScanDimLabel] = "m"
                    break
                case "ManipulatorTheta":
                case "ManipulatorTilt":
                case "ManipulatorPhi":
                    un[%$kScanDimLabel] = ""
                    break
                case "FocusXRot":
                case "FocusYRot":
                case "FocusZRot":
                case "RefocusXRot":
                case "RefocusYRot":
                case "RefocusZRot":
                    un[%$kScanDimLabel] = "mrad"
                    break
            endswitch
        endif
    endif
};

variable ps_scale_dataset_2(wave data, wave ax, wave lo, wave hi, wave un){
    wave data
    wave /t ax
    wave lo
    wave hi
    wave /t un
    
    string sdim
    sdim = GetDimLabel(data, 0, -1)
    if (strlen(sdim))
        setscale /i x lo[%$sdim], hi[%$sdim], un[%$sdim], data
        Note data, "AxisLabelX=" + ax[%$sdim]
    endif

    sdim = GetDimLabel(data, 1, -1)
    if (strlen(sdim))
        setscale /i y lo[%$sdim], hi[%$sdim], un[%$sdim], data
        Note data, "AxisLabelY=" + ax[%$sdim]
    endif
    
    sdim = GetDimLabel(data, 2, -1)
    if (strlen(sdim))
        setscale /i z lo[%$sdim], hi[%$sdim], un[%$sdim], data
        Note data, "AxisLabelZ=" + ax[%$sdim]
    endif
    
    string data_unit = un[%$kDataDimLabel]
    string data_label = ax[%$kDataDimLabel]
    if (cmpstr(data_unit, "arb.") == 0)
        strswitch(NameOfWave(data))
            case "ScientaImage":
            case "ImageAngleDistribution":
            case "ScientaAngleDistribution":
            case "ScientaSpectrum":
            case "ImageEnergyDistribution":
            case "ScientaEnergyDistribution":
                data *= kDetectorSensitivity
                data_unit = "counts"
                data_label = "intensity"
                break
            case "SampleCurrent":
            case "RefCurrent":
            case "AuxCurrent":
                data_unit = "A"
                data_label = "current"
                break
            case "MachineCurrent":
                data_unit = "mA"
                data_label = "current"
                break
        endswitch
    endif
    setscale d 0, 0, data_unit, data
    Note data, "AxisLabelD=" + data_label
    Note data, "Dataset=" + NameOfWave(data)
};

string psh5_load_reduced(string ANickName, string APathName, string AFileName, funcref reduction_func, string reduction_param, variable progress = defaultValue){
    string ANickName
    string APathName
    string AFileName
    funcref adh5_default_reduction reduction_func
    string reduction_param
    variable progress
        
    if (ParamIsDefault(progress))
        progress = 1
    endif
  
    dfref saveDF = GetDataFolderDFR()

    // performance monitoring
    variable timerRefNum
    variable /g psh5_perf_secs
    timerRefNum = startMSTimer

    variable fileID = psh5_open_file(ANickName, APathName, AFileName)
    string wavenames = ""
    if (fileID)
        dfref fileDF = GetDataFolderDFR()
        svar s_filepath
        svar s_scanpaths
        AFileName = s_filepath
        print "loading " + s_filepath + "\r"
        
        variable ig = 0
        variable ng = ItemsInList(s_scanpaths)
        string scanpath
        string folder
        string positioners
        string positioner
        string positionerpath
        
        scanpath = StringFromList(ig, s_scanpaths)
        folder = ReplaceString("/", scanpath, "")
        folder = ReplaceString(" ", folder, "")
        folder = CleanupName(folder, 0)
        setdatafolder fileDF
        newdatafolder /s /o $folder
        dfref dataDF = GetDataFolderDFR()
        positioners = psh5_load_scan_meta(fileID, scanpath)
        newdatafolder /s /o attr
        killwaves /a/z
        psh5_load_scan_attrs(fileID, scanpath)
        setdatafolder dataDF
        wave /t /z ScanWritables
        if (waveexists(ScanWritables) && (numpnts(ScanWritables) >= 1))
            positioner = ScanWritables[0]
            if (strlen(positioner) > 0)
                positionerpath = scanpath + "/" + positioner
                positionerpath = ReplaceString("//", positionerpath, "/")
                HDF5LoadData /O /Q /Z fileID, positionerpath
            endif
        endif
        
        setdatafolder dataDF
        string datasets = psh5_list_scan_datasets(fileID, scanpath, include_regions=1)
        string dataset = select_dataset(datasets, "ScientaImage")
        wavenames = psh5_load_dataset_reduced(fileID, scanpath, dataset, reduction_func, reduction_param, progress=progress)

        psh5_close_file(fileID)
    endif

    if (timerRefNum >= 0)
        psh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    endif
    
    setdatafolder saveDF
    return wavenames
};


string psh5_load_dataset_reduced(variable fileID, string scanpath, string datasetname, funcref reduction_func, string reduction_param, variable progress = defaultValue, variable nthreads = defaultValue){
    variable fileID
    string scanpath
    string datasetname
    funcref adh5_default_reduction reduction_func
    string reduction_param
    variable progress
    variable nthreads
        
    if (ParamIsDefault(progress))
        progress = 1
    endif
    if (ParamIsDefault(nthreads))
        nthreads = -1
    endif
    
    dfref base_df = GetDataFolderDFR()
    variable result = 0
    string datasetpath
    string datawavename
    string wavenames = ""

    datasetpath = scanpath + "/" + datasetname
    datasetpath = ReplaceString("//", datasetpath, "/")
    datawavename = StringFromList(ItemsInList(datasetpath, "/") - 1, datasetpath, "/")
    
    string regionname
    string regionpath
    if (ItemsInList(datasetname, "/") >= 2)
        regionname = StringFromList(0, datasetname, "/")
        regionpath = ReplaceString("//", scanpath + "/" + regionname, "/")
        datasetname = RemoveListItem(0, datasetname, "/")
        NewDataFolder /o/s $regionname
    else
        regionname = ""
        regionpath = scanpath
    endif

    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetpath, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        result = -1
        return wavenames
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        result = -2
        return wavenames
    else if (di.ndims < 3)
        progress = 0
    endif
    
    variable idx, idy, idz, idt
    variable transpose = WhichListItem(datawavename, kTransposedDatasets) >= 0
    if (transpose)
        idx = 1
        idy = 0
    else
        idx = 0
        idy = 1
    endif
    idz = 2
    idt = 3
    
    variable nx, ny, nz, nt, nzt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = di.dims[idz]
    nt = di.dims[idt]
    // adjust singleton dimensions
    nz = max(nz, 1)
    nt = max(nt, 1)
    nzt = nz * nt
    
    // load data image by image
    HDF5MakeHyperslabWave("slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    // set up multi threading
    if (nthreads < 0)
        nthreads = ThreadProcessorCount
    endif
    if (nthreads > 0)
        variable threadGroupID = ThreadGroupCreate(nthreads)
        variable ithread
        for (ithread = 0; ithread < nthreads; ithread += 1)
            ThreadStart threadGroupID, ithread, reduce_slab_worker(reduction_func)
        endfor
    else
        make /n=(nzt) /df /free processing_folders
    endif
    
    if (progress)
        display_progress_panel("HDF5 Import", "Loading data (step 1 of 2)...", nzt)
    endif

    // create a template wave with the correct scales and labels
    make /n=(nx,ny) /d /o $datawavename
    wave template = $datawavename
    ps_set_dimlabels2(template, datawavename)
    ps_scale_dataset(template)

    variable iz, it, izt
    string dfname
    izt = 0
    for (iz = 0; iz < nz; iz += 1)
        for (it = 0; it < nt; it += 1)
            // load hyperslab
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            dfname = "processing_" + num2str(izt)
            newdatafolder /s $dfname
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetpath
            
            // send to processing queue
            duplicate template, image
            variable /g r_index = iz
            variable /g s_index = it
            string /g func_param = reduction_param

            if (nthreads > 0)
                WaveClear image
                ThreadGroupPutDF threadGroupID, :
            else
                processing_folders[izt] = GetDataFolderDFR()
                make /n=1/d profile1, profile2
                wave slabdata
                variable /g func_result
                func_result = reduce_slab_image(slabdata, image, profile1, profile2, reduction_func, func_param)
                WaveClear slabdata, image, profile1, profile2
                setdatafolder ::
            endif
            
            izt += 1
            // progress window
            if (progress)
                if (update_progress_panel(izt))
                    print "user abort"
                    result = -4
                    break
                endif
            endif
        endfor
    endfor
    
    killwaves /z slab, slabdata, template
    if (progress)
        update_progress_panel(0, message="Processing data (step 2 of 2)...")
    endif

    dfref dfr
    for (izt = 0; (izt < nzt) && (result == 0); izt += 1)
        if (nthreads > 0)
            do
                if (progress)
                    if (update_progress_panel(izt))
                        print "user abort"
                        result = -4
                        break
                    endif
                endif
                dfr = ThreadGroupGetDFR(threadGroupID, 1000)
                if (DatafolderRefStatus(dfr) != 0)
                    break
                endif
            while (1)
        else
            if (progress)
                if (update_progress_panel(izt))
                    print "user abort"
                    result = -4
                    break
                endif
            endif
            dfr = processing_folders[izt]
        endif
        
        if (result != 0)
            break
        endif
        
        nvar rr = dfr:r_index
        nvar ss = dfr:s_index
        nvar func_result = dfr:func_result
        wave profile1 = dfr:profile1
        wave profile2 = dfr:profile2

        if (func_result == 0)
            if (izt == 0)
                make /n=(dimsize(profile1, 0), nz, nt) /d /o ReducedData1
                make /n=(dimsize(profile2, 0), nz, nt) /d /o ReducedData2
                setdimlabel 0, -1, $getdimlabel(profile1, 0, -1), ReducedData1
                setdimlabel 0, -1, $getdimlabel(profile2, 0, -1), ReducedData2
                setdimlabel 1, -1, $kScanDimLabel, ReducedData1
                setdimlabel 1, -1, $kScanDimLabel, ReducedData2
                ps_scale_dataset(ReducedData1)
                ps_scale_dataset(ReducedData2)
                setscale /p x dimoffset(profile1, 0), dimdelta(profile1, 0), waveunits(profile1, 0), ReducedData1
                setscale /p x dimoffset(profile2, 0), dimdelta(profile2, 0), waveunits(profile2, 0), ReducedData2
                setscale d 0, 0, waveunits(profile1, -1), ReducedData1
                setscale d 0, 0, waveunits(profile2, -1), ReducedData2
            endif
            ReducedData1[][rr][ss] = profile1[p]
            ReducedData2[][rr][ss] = profile2[p]
        else
            print "error during data reduction."
            result = -3
            break
        endif
    endfor
    
    if (nthreads > 0)
        variable tstatus = ThreadGroupRelease(threadGroupID)
        if (tstatus == -2)
            print "error: thread did not terminate properly."
            result = -5
        endif
    else
        for (izt = 0; izt < nzt; izt += 1)
            KillDataFolder /Z processing_folders[izt]
        endfor
    endif
    
    if (result == 0)
        if (nz == 1)
            redimension /n=(-1, 0, 0) ReducedData1
            redimension /n=(-1, 0, 0) ReducedData2
        else if (nt == 1)
            redimension /n=(-1, nz, 0) ReducedData1
            redimension /n=(-1, nz, 0) ReducedData2
        endif
        wavenames = "ReducedData1;ReducedData2;"
    endif
    if (progress)
        kill_progress_panel()
    endif
    
    setdatafolder base_df
    return wavenames
};

threadsafe static variable reduce_slab_worker(funcref reduction_func){
    funcref adh5_default_reduction reduction_func
    do
        // wait for job from main thread
        do
            dfref dfr = ThreadGroupGetDFR(0, 1000)
            if (DataFolderRefStatus(dfr) == 0)
                if (GetRTError(2))
                    return 0// no more jobs
                endif
            else
                break
            endif
        while (1)

        // get input data
        wave slabdata = dfr:slabdata
        wave image = dfr:image
        svar func_param = dfr:func_param
        nvar rr = dfr:r_index
        nvar ss = dfr:s_index
        
        // do the work
        newdatafolder /s outDF
        make /n=1/d profile1, profile2
        variable /g r_index = rr
        variable /g s_index = ss
        variable /g func_result
        func_result = reduce_slab_image(slabdata, image, profile1, profile2, reduction_func, func_param)

        // send output to queue and clean up
        WaveClear slabdata, image, profile1, profile2
        ThreadGroupPutDF 0, :
        KillDataFolder dfr
    while (1)
    
    return 0
};

threadsafe static variable reduce_slab_image(wave slabdata, wave image, wave profile1, wave profile2, funcref reduction_func, string reduction_param){
    wave slabdata
    wave image
    wave profile1
    wave profile2
    funcref adh5_default_reduction reduction_func
    string reduction_param
    
    // the multiplication by detector sensitivity assumes that we are loading a ScientaImage.
    image = slabdata[q][p][0][0] * kDetectorSensitivity

    return reduction_func(image, profile1, profile2, reduction_param)
};

string psh5_load_info(string APathName, string AFileName){
    string APathName
    string AFileName
        
    dfref saveDF = GetDataFolderDFR()
    dfref fileDF = NewFreeDataFolder()
    setdatafolder fileDF

    variable fileID
    string filepath
    string scanpaths
    variable nscans
    variable iscan
    string scanpath
    string info = ""
      
    HDF5OpenFile /P=$APathName /R fileID as AFileName
    if (v_flag == 0)
        filepath = s_path + s_filename
        scanpaths = psh5_list_scans(fileID)
        nscans = ItemsInList(scanpaths)
        for (iscan = 0; iscan < nscans; iscan += 1)
            scanpath = StringFromList(iscan, scanpaths)
            info = info + scanpath + "\r"
            info = info + psh5_load_scan_info(fileID, scanpath)
        endfor
        HDF5CloseFile fileID
    endif
    
    setdatafolder saveDF
    return info
};

string psh5_load_scan_info(variable fileID, string scanpath){
    variable fileID
    string scanpath
        
    string info = ""
    string positions = ""
    string positioners = ""
    string readables = ""
    string detectors = ""
    string regions = ""

    psh5_load_scan_meta(fileID, scanpath)

    wave /z ScanDimensions
    wave /t /z ScanWritables
    wave /t /z ScanReadables
    wave /z ScanSteps

    if (WaveExists(ScanSteps) && (numpnts(ScanSteps) >= 1))
        ScanSteps += 1
        positions = "positions = (" + wave2list(ScanSteps, "%u", ",") + ")"
        info = AddListItem(positions, info, "\r", inf)
    endif
    if (WaveExists(ScanWritables) && (numpnts(ScanWritables) >= 1))
        positioners = "positioners = " + twave2list(ScanWritables, ",")
        info = AddListItem(positioners, info, "\r", inf)
    endif

    variable i, m, n
    string s
    if (WaveExists(ScanReadables) && (numpnts(ScanReadables) >= 1))
        readables = twave2list(ScanReadables, ",")
        n = ItemsInList(readables, ",")
        for (i = 0; i < n; i += 1)
            s = StringFromList(i, readables, ",")
            m = ItemsInList(s, "/")
            if (m > 1)
                s = StringFromList(m - 1, s, "/")
            endif
            if (WhichListItem(s, detectors, ",") < 0)
                detectors = AddListItem(s, detectors, ",", inf)
            endif
        endfor
        detectors = "detectors = " + detectors
        info = AddListItem(detectors, info, "\r", inf)
    endif

    regions = psh5_list_scan_regions(fileID, scanpath)
    if (strlen(regions) > 0)
        regions = "regions = " + regions
        info = AddListItem(regions, info, "\r", inf)
    endif
    
    return info
};

static string twave2list(wave wt, string sep){
    wave /t wt
    string sep
    
    string list = ""
    variable n = numpnts(wt)
    variable i
    for (i = 0; i < n; i += 1)
        list = AddListItem(wt[i], list, sep, inf)
    endfor
    
    return list
};

static string wave2list(wave w, string format, string sep){
    wave w
    string format
    string sep
    
    string list = ""
    variable n = numpnts(w)
    variable i
    string s
    for (i = 0; i < n; i += 1)
        sprintf s, format, w[i]
        list = AddListItem(s, list, sep, inf)
    endfor
    
    return list
};