pearl-area-import.ipf

Go to the documentation of this file.

#pragma rtGlobals=3// Use modern global access method and strict wave access.
#pragma IgorVersion = 6.2
#pragma ModuleName = PearlAreaImport
#pragma version = 1.06
#include <HDF5 Browser>
#include "pearl-gui-tools"

// copyright (c) 2013-16 Paul Scherrer Institut
//
// Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//   http:///www.apache.org/licenses/LICENSE-2.0



static variable BeforeFileOpenHook(variable refNum, string fileName, string path, string type, string creator, variable kind){
    variable refNum, kind
    string fileName, path, type, creator
    
    variable handledOpen = 0

    //PathInfo $path
    //string FilePath = s_path + filename
    string NickName = CleanupName(ParseFilePath(3, FileName, ":", 0, 0), 0)
    string FileExt = LowerStr(ParseFilePath(4, FileName, ":", 0, 0))
    string result = ""
  
    // override nickname with custom setting
    svar /z cnn = gsCustomNickName
    if (svar_exists(cnn))
        if (exists("gvNickNameIndex") != 2)
            variable/g gvNickNameIndex = 1
        endif
        nvar nni = gvNickNameIndex
        NickName = cnn + num2str(nni)
        nni += 1
    endif

    if (stringmatch(FileExt, "h5") == 1)
        result = adh5_load_complete(NickName, path, FileName)
    endif
  
    string/g s_latest_datafile = result
    string/g s_latest_nickname = nickname
  
    handledOpen = strlen(result) > 0
    if (handledOpen)
        close refnum
    endif
    
    return handledOpen// 1 tells Igor not to open the file
};

string ad_suggest_foldername(string filename, variable ignoredate = defaultValue, string sourcename = defaultValue, variable unique = defaultValue){
    string filename
    variable ignoredate
    string sourcename
    variable unique
    
    if (ParamIsDefault(ignoredate))
        ignoredate = 0
    endif
    if (ParamIsDefault(unique))
        unique = 0
    endif
    
    string basename = ParseFilePath(3, filename, ":", 0, 0)
    string extension = ParseFilePath(4, filename, ":", 0, 0)
    string nickname
    
    string autosource
    if (strsearch(basename, "scienta", 0, 2) >= 0)
        autosource = "sci"
    else if (strsearch(basename, "pshell", 0, 2) >= 0)
        autosource = "psh"
    else if (strsearch(basename, "OP-SL", 0, 2) >= 0)
        autosource = "sl"
    else if (strsearch(basename, "ES-PS", 0, 2) >= 0)
        autosource = "es"
    else
        autosource = "xy"
    endif
    if (ParamIsDefault(sourcename))
        sourcename = autosource
    endif
    
    variable nparts = ItemsInList(basename, "-")
    if (nparts >= 3)
        string datepart = StringFromList(1, basename, "-")
        variable l_datepart = strlen(datepart)
        if (l_datepart == 8)
            datepart = datepart[l_datepart-6, l_datepart-1]
        endif
        string indexpart = StringFromList(2, basename, "-")
        if (ignoredate)
            sprintf nickname, "%s_%s", sourcename, indexpart
        else
            sprintf nickname, "%s_%s_%s", sourcename, datepart, indexpart
        endif
    else
        nickname = CleanupName(basename, 0)
    endif
    
    if (unique && CheckName(nickname, 11))
        nickname = UniqueName(nickname + "_", 11, 0)
    endif
    
    return nickname
};

variable ad_load_dialog(string APathName){
    string APathName
    
    variable refNum
    string message = "Select data files"
    string filepaths
    string filefilters = "Area Detector HDF5 Files (*.h5):.h5;"
    filefilters += "All Files:.*;"

    PathInfo /S $APathName
    Open /D /R /F=filefilters /M=message /MULT=1 refNum
    filepaths = S_fileName

    dfref saveDF = GetDataFolderDFR()
    setdatafolder root:

    if (strlen(filepaths) > 0)
        variable nfiles = ItemsInList(filepaths, "\r")
        variable ifile
        for(ifile = 0; ifile < nfiles; ifile += 1)
            String path = StringFromList(ifile, filepaths, "\r")
            string nickname = ad_suggest_foldername(path)
            adh5_load_complete(nickname, "", path)
        endfor
    endif

    setdatafolder saveDF
};

string adh5_load_complete(string ANickName, string APathName, string AFileName, variable load_data = defaultValue, variable load_attr = defaultValue){
    string ANickName
    string APathName
    string AFileName
    variable load_data
    variable load_attr
        
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif
  
    dfref saveDF = GetDataFolderDFR()
    setdatafolder root:
    newdatafolder /s/o $("root:" + ANickName)
  
    // open file
    variable fileID
    string instrumentpath = "/entry/instrument/"
    string detectorpath = instrumentpath + "detector/"
    string attributespath = instrumentpath + "NDAttributes/"
    string datasetname
    string datawavename
    
    // performance monitoring
    variable timerRefNum
    variable /g adh5_perf_secs
    timerRefNum = startMSTimer
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5OpenFile")
    HDF5OpenFile /P=$APathName/R fileID as AFileName
    if (v_flag == 0)
        AFileName = s_path + s_filename
        print "loading " + s_filename + "\r"
        
        if (load_data)
            adh5_load_detector_slabs(fileID, detectorpath)
        endif
        if (load_attr)
            newdatafolder /o/s attr
            adh5_loadattr_all(fileID, attributespath)
            setdatafolder ::
        endif

        wave /z data
        if (waveexists(data))
            //adh5_redim(data) // not to be used with adh5_load_detector_slabs
            adh5_scale(data)
        endif

        HDF5CloseFile fileID
    else
        AFileName = ""
    endif
    #else
    Abort "HDF5 XOP not loaded."
    #endif

    if (timerRefNum >= 0)
        adh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    endif
    
    setdatafolder saveDF
    return AFileName
};

string adh5_load_reduced(string ANickName, string APathName, string AFileName, funcref reduction_func, string reduction_param, variable load_data = defaultValue, variable load_attr = defaultValue, variable progress = defaultValue){
    string ANickName
    string APathName
    string AFileName

    funcref adh5_default_reduction reduction_func
    string reduction_param

    variable load_data
    variable load_attr
    variable progress
        
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif
    if (ParamIsDefault(progress))
        progress = 1
    endif
  
    dfref saveDF = GetDataFolderDFR()
    setdatafolder root:
    newdatafolder /s/o $("root:" + ANickName)
  
    // open file
    variable fileID
    string instrumentpath = "/entry/instrument/"
    string detectorpath = instrumentpath + "detector/"
    string attributespath = instrumentpath + "NDAttributes/"
    string datasetname
    string datawavename
    
    // performance monitoring
    variable timerRefNum
    variable /g adh5_perf_secs
    timerRefNum = startMSTimer
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5OpenFile")
    HDF5OpenFile /P=$APathName/R fileID as AFileName
    if (v_flag == 0)
        AFileName = s_path + s_filename
        print "loading " + s_filename + "\r"
        
        if (load_attr)
            newdatafolder /o/s attr
            adh5_loadattr_all(fileID, attributespath)
            setdatafolder ::
        endif
        if (load_data)
            adh5_load_reduced_detector(fileID, detectorpath, reduction_func, reduction_param, progress=progress)
        endif

        HDF5CloseFile fileID
    else
        AFileName = ""
    endif
    #else
    Abort "HDF5 XOP not loaded."
    #endif

    if (timerRefNum >= 0)
        adh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    endif
    
    setdatafolder saveDF
    return AFileName
};

string adh5_load_preview(string ANickName, string APathName, string AFileName, variable load_data = defaultValue, variable load_attr = defaultValue){
    string ANickName
    string APathName
    string AFileName
    variable load_data
    variable load_attr
        
    if (ParamIsDefault(load_data))
        load_data = 1
    endif
    if (ParamIsDefault(load_attr))
        load_attr = 1
    endif

    dfref saveDF = GetDataFolderDFR()
    setdatafolder root:
    newdatafolder /o/s pearl_area
    newdatafolder /o/s preview

    // open file
    variable fileID
    string instrumentpath = "/entry/instrument/"
    string detectorpath = instrumentpath + "detector/"
    string attributespath = instrumentpath + "NDAttributes/"
    string datasetname
    string datawavename
    
    // performance monitoring
    variable timerRefNum
    variable /g adh5_perf_secs
    timerRefNum = startMSTimer
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5OpenFile")
    HDF5OpenFile /P=$APathName/R/Z fileID as AFileName
    if (v_flag == 0)
        AFileName = s_path + s_filename
        
        // detector data
        datasetname = detectorpath + "data"
        STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
        InitHDF5DataInfo(di)
        variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
        if (err != 0)
            print "error accessing detector/data"
            return ""
        endif
        if (di.ndims < 2)
            print "error: rank of dataset < 2"
            return ""
        endif

        variable dim2start = 0, dim2count = 1, dim3start = 0, dim3count = 1
        if (di.ndims >= 3)
            dim2start = floor(di.dims[di.ndims - 3] / 2)
            dim2count = 1
        endif
        if (di.ndims >= 4)
            dim3start = floor(di.dims[di.ndims - 4] / 2)
            dim3count = 1
        endif
        
        if (load_data)
            adh5_load_detector_image(fileID, detectorpath, dim2start, dim2count, dim3start, dim3count)
            wave /z data
            string destpath = GetDataFolder(1, saveDF) + ANickName
            if (waveexists(data))
                duplicate /o data, $destpath
                wave /z data = $destpath
            endif
        endif
        
        if (load_attr)
            setdatafolder saveDF
            newdatafolder /o/s attr
            killwaves /a/z
            adh5_loadattr_all(fileID, attributespath)
            setdatafolder ::
            if (waveexists(data))
                adh5_scale(data)
            endif
        endif

        HDF5CloseFile fileID
    else
        print "error opening file " + AFileName
        AFileName = ""
    endif
    #else
    Abort "HDF5 XOP not loaded."
    #endif

    if (timerRefNum >= 0)
        adh5_perf_secs = stopMSTimer(timerRefNum) / 1e6
    endif
    
    setdatafolder saveDF
    return AFileName
};

string adh5_load_info(string APathName, string AFileName){
    string APathName
    string AFileName
        
    dfref saveDF = GetDataFolderDFR()

    // open file
    variable fileID
    string instrumentpath = "/entry/instrument/"
    string detectorpath = instrumentpath + "detector/"
    string attributespath = instrumentpath + "NDAttributes/"
    string datasetname
    string datawavename
    
    string s_info = ""
    string s
    
    variable idim
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5OpenFile")
    HDF5OpenFile /P=$APathName/R/Z fileID as AFileName
    if (v_flag == 0)
        AFileName = s_path + s_filename
        
        // detector data
        datasetname = detectorpath + "data"
        STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
        InitHDF5DataInfo(di)
        variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
        if (err != 0)
            print "error accessing detector/data"
            return ""
        endif

        for (idim = 0; idim < di.ndims; idim += 1)
            sprintf s, "dim %u: %u points", idim, di.dims[idim]
            if (strlen(s_info) > 0)
                s_info = s_info + "\r" + s
            else
                s_info = s
            endif
        endfor
        
        dfref df = NewFreeDataFolder()
        setdatafolder df
        adh5_loadattr_all(fileID, attributespath)

        for (idim = 1; idim < 5; idim += 1)
            sprintf s, "Scan%uActive", idim
            wave /z w = $s
            if (waveexists(w) && (numpnts(w) > 0) && (w[0] > 0))
                sprintf s, "Scan%uPositioner1", idim
                wave /t wt = $s
                sprintf s, "scan %u: %s", idim, wt[0]
                if (strlen(s_info) > 0)
                    s_info = s_info + "\r" + s
                else
                    s_info = s
                endif
            endif
        endfor
        
        HDF5CloseFile fileID
    else
        print "error opening file " + AFileName
        AFileName = ""
    endif
    #else
    Abort "HDF5 XOP not loaded."
    #endif

    setdatafolder saveDF
    return s_info
};

variable adh5_load_detector(variable fileID, string detectorpath){
    variable fileID
    string detectorpath

    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5LoadData")
    string datasetname
    string datawavename

    // detector data
    datasetname = detectorpath + "data"
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return -1
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        return -2
    endif
    
    HDF5LoadData /O /Q /Z fileID, datasetname
    wave data
    
    #else
    Abort "HDF5 XOP not loaded."
    #endif
};

variable adh5_redim(wave data){
    wave data
    
    duplicate /free data, tempdata
    variable nd = wavedims(tempdata)
    variable nx = dimsize(tempdata, nd - 1)
    variable ny = dimsize(tempdata, nd - 2)
    variable nz = dimsize(tempdata, nd - 3)
    variable nt = dimsize(tempdata, nd - 4)

    switch (nd)
        case 2:
            if (nx <= 1)
                redimension /n=(ny) data
                setdimlabel 0, -1, AD_Dim1, data
                data = tempdata[p][0]
            else if (ny <= 1)
                redimension /n=(nx) data
                setdimlabel 0, -1, AD_Dim0, data
                data = tempdata[0][p]
            else
                redimension /n=(nx,ny) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_Dim1, data
                data = tempdata[q][p]
            endif
            break
        case 3:
            if (nx <= 1)
                redimension /n=(ny,nz) data
                setdimlabel 0, -1, AD_Dim1, data
                setdimlabel 1, -1, AD_DimN, data
                multithread data = tempdata[q][p][0]
            else if (ny <= 1)
                redimension /n=(nx,nz) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_DimN, data
                multithread data = tempdata[q][0][p]
            else if (nz <= 1)
                redimension /n=(nx,ny) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_Dim1, data
                multithread data = tempdata[0][q][p]
            else
                redimension /n=(nx,ny,nz) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_Dim1, data
                setdimlabel 2, -1, AD_DimN, data
                multithread data = tempdata[r][q][p]
            endif
            break
        case 4:
            if (nz <= 1)
                // singleton "frame number" dimension
                redimension /n=(nx,ny,nt) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_Dim1, data
                setdimlabel 2, -1, AD_DimX, data
                multithread data = tempdata[r][0][q][p]
            else
                redimension /n=(nx,ny,nz,nt) data
                setdimlabel 0, -1, AD_Dim0, data
                setdimlabel 1, -1, AD_Dim1, data
                setdimlabel 2, -1, AD_DimN, data
                setdimlabel 3, -1, AD_DimX, data
                multithread data = tempdata[s][r][q][p]
            endif
            break
    endswitch
};

static dfr GetAttrDataFolderDFR(wave data){
    wave data
    
    dfref dataDF = GetWavesDataFolderDFR(data)
    dfref attrDF = dataDF:attr
    if (DataFolderRefStatus(attrDF) == 0)
        attrDF = dataDF
    endif
    
    return attrDF
};

variable adh5_scale(wave data, string source = defaultValue){
    wave data
    string source

    dfref saveDF = GetDataFolderDFR()
    dfref dataDF = GetWavesDataFolderDFR(data)
    dfref attrDF = GetAttrDataFolderDFR(data)

    if (ParamIsDefault(source))
        // is the source a Scienta analyser?
        wave /SDFR=attrDF /Z AcquisitionMode
        wave /SDFR=attrDF /T /Z Manufacturer
        source = "unknown"
        if (waveexists(Manufacturer) && (numpnts(Manufacturer) >= 1))
            strswitch(Manufacturer[0])
                case "VG Scienta":
                    source = "scienta"
                    break
                case "Prosilica":
                    source = "prosilica"
                    break
            endswitch
        else if (waveexists(AcquisitionMode) && (numpnts(AcquisitionMode) >= 1))
            if (stringmatch(note(AcquisitionMode), "*SCIENTA*"))
                source = "scienta"
            endif
        endif
    endif
    
    strswitch(source)
        case "prosilica":
            // pixel scale - nothing to do
            break
        case "scienta":
            adh5_scale_scienta(data)
            break
    endswitch

    setdatafolder saveDF
};

variable adh5_load_detector_slabs(variable fileID, string detectorpath, variable progress = defaultValue){
    variable fileID
    string detectorpath
    variable progress
    
    if (ParamIsDefault(progress))
        progress = 1
    endif
    variable result = 0

    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5LoadData")
    string datasetname
    string datawavename

    // detector data
    datasetname = detectorpath + "data"
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return -1
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        return -2
    endif
    
    // nx and nz are the image dimensions
    variable idx, idy, idz, idt, izt
    idx = di.ndims - 1
    idy = di.ndims - 2
    idz = -1
    idt = -1
    
    variable nx, ny, nz, nt, nzt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = 1
    nt = 1
    
    make /n=(nx,ny,nz,nt) /o data
    string dim_labels = "AD_Dim0;AD_Dim1;AD_DimN;AD_DimX;AD_DimY"
    string dim_label
    dim_label = StringFromList(0, dim_labels, ";")
    setdimlabel 0, -1, $dim_label, data
    dim_labels = RemoveFromList(dim_label, dim_labels, ";")
    dim_label = StringFromList(0, dim_labels, ";")
    setdimlabel 1, -1, $dim_label, data
    dim_labels = RemoveFromList(dim_label, dim_labels, ";")

    // find additional dimensions, ignore singletons
    variable id
    for (id = idy - 1; (id >= 0) && (nz == 1); id -= 1)
        if (di.dims[id] > 1)
            idz = id
            nz = di.dims[id]
            dim_label = StringFromList(0, dim_labels, ";")
            setdimlabel 2, -1, $dim_label, data
        endif
        dim_labels = RemoveListItem(0, dim_labels, ";")
    endfor
    for (id = idz - 1; (id >= 0) && (nt == 1); id -= 1)
        if (di.dims[id] > 1)
            idt = id
            nt = di.dims[id]
            dim_label = StringFromList(0, dim_labels, ";")
            setdimlabel 3, -1, $dim_label, data
        endif
        dim_labels = RemoveListItem(0, dim_labels, ";")
    endfor
    redimension /n=(nx,ny,nz,nt) data

    // default values if dimensions are not present in dataset
    if (idz < 0)
        idz = idx + 1
        idt = idz + 1
    else if (idt < 0)
        idt = idx + 1
    endif
    
    nzt = nz * nt
    izt = 0
    if (progress)
        display_progress_panel("HDF5 Import", "Loading data...", nzt)
    endif
    
    // load data image by image
    HDF5MakeHyperslabWave(GetDataFolder(1) + "slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    variable iz, it
    for (iz = 0; iz < nz; iz += 1)
        for (it = 0; it < nt; it += 1)
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetname
            wave slabdata// 2D, 3D, or 4D with singletons
            switch (WaveDims(slabdata))
                case 2:
                    data[][][iz][it] = slabdata[q][p]
                    break
                case 3:
                    data[][][iz][it] = slabdata[0][q][p]
                    break
                case 4:
                    data[][][iz][it] = slabdata[0][0][q][p]
                    break
            endswitch
            // progress window
            izt += 1
            if (progress)
                if (update_progress_panel(izt))
                    result = -4// user abort
                    break
                endif
            endif
        endfor
        if (result < 0)
            break
        endif
    endfor
    
    if (nz == 1)
        redimension /n=(nx,ny) data
    else if (nt == 1)
        redimension /n=(nx,ny,nz) data
    endif
    
    if (progress)
        kill_progress_panel()
    endif
    #else
    Abort "HDF5 XOP not loaded."
    #endif
    
    return result
};

variable adh5_load_detector_image(variable fileID, string detectorpath, variable dim2start, variable dim2count, variable dim3start, variable dim3count){
    variable fileID
    string detectorpath
    variable dim2start
    variable dim2count
    variable dim3start
    variable dim3count
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5LoadData")
    string datasetname
    string datawavename

    // detector data
    datasetname = detectorpath + "data"
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return -1
    endif
    if (di.ndims < 1)
        print "error: rank of dataset < 1"
        return -2
    endif
    
    // nx and nz are the image dimensions
    variable idx, idy, idz, idt
    idx = di.ndims - 1
    idy = di.ndims >= 2 ? di.ndims - 2 : 1
    idz = di.ndims >= 3 ? di.ndims - 3 : 2
    idt = di.ndims >= 4 ? di.ndims - 4 : 3
    
    variable nx, ny
    nx = di.dims[idx]
    ny = di.ndims >= 2 ? di.dims[idy] : 1
    
    variable dim2end = dim2start + dim2count - 1
    variable dim3end = dim3start + dim3count - 1
    
    // the slab wave is at least 4-dimensional
    // it will also load lower-dimensional datasets
    HDF5MakeHyperslabWave(GetDataFolder(1) + "slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    make /n=(nx,ny)/o/d data
    data = 0
    variable iz, it
    variable navg = 0
    for (iz = dim2start; iz <= dim2end; iz += 1)
        for (it = dim3start; it <= dim3end; it += 1)
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetname
            wave slabdata// 2D, 3D, or 4D with singletons
            switch (WaveDims(slabdata))
                case 1:
                    data += slabdata[p]
                    navg += 1
                    break
                case 2:
                    data += slabdata[q][p]
                    navg += 1
                    break
                case 3:
                    data += slabdata[0][q][p]
                    navg += 1
                    break
                case 4:
                    data += slabdata[0][0][q][p]
                    navg += 1
                    break
            endswitch
        endfor
    endfor
    data /= navg
    setdimlabel 0, -1, AD_Dim0, data
    setdimlabel 1, -1, AD_Dim1, data
    
    #else
    Abort "HDF5 XOP not loaded."
    #endif
};

string adh5_list_reduction_funcs(){
    string all_funcs = FunctionList("*", ";", "KIND:6,NPARAMS:4,VALTYPE:1")
    string result = ""
    
    variable ii
    variable nn = ItemsInList(all_funcs, ";")
    
    string funcname
    string info
    variable nparams
    variable accept
    
    for (ii = 0; ii < nn; ii += 1)
        funcname = StringFromList(ii, all_funcs, ";")
        info = FunctionInfo(funcname)
        accept = (NumberByKey("RETURNTYPE", info, ":", ";") == 0x0004)
        accept = accept && (cmpstr(StringByKey("THREADSAFE", info, ":", ";"), "yes") == 0)
        accept = accept && (NumberByKey("N_PARAMS", info, ":", ";") == 4)
        accept = accept && (NumberByKey("N_OPT_PARAMS", info, ":", ";") == 0)
        if (accept)
            // 3 numeric waves and one pass-by-reference string
            accept = accept && (NumberByKey("PARAM_0_TYPE", info, ":", ";") == 0x4002)
            accept = accept && (NumberByKey("PARAM_1_TYPE", info, ":", ";") == 0x4002)
            accept = accept && (NumberByKey("PARAM_2_TYPE", info, ":", ";") == 0x4002)
            accept = accept && (NumberByKey("PARAM_3_TYPE", info, ":", ";") == 0x3000)
        endif
        if (accept)
            result = AddListItem(funcname, result, ";")
        endif
    endfor
    
    result = SortList(result, ";", 4)
    return result
};

// the resulting wave must have the same size as either dimension of the source image.
threadsafe variable adh5_default_reduction(wave source, wave dest1, wave dest2, string* param){
    wave source
    wave dest1, dest2
    string &param
    
    // demo code
    // integrate along the dimensions   
    adh5_setup_profile(source, dest1, 0)
    ad_profile_x_w(source, 0, -1, dest1)
    adh5_setup_profile(source, dest2, 1)
    ad_profile_y_w(source, 0, -1, dest2)
    
    return 0
};

threadsafe variable adh5_setup_profile(wave image, wave profile, variable dim){
    wave image// prototype
    wave profile// destination wave
    variable dim// which dimension to keep: 0 = X, 1 = Y
    
    redimension /n=(dimsize(image, dim)) profile
    setscale /p x dimoffset(image, dim), dimdelta(image, dim), waveunits(image, dim), profile
    setscale d 0, 0, waveunits(image, -1), profile
    setdimlabel 0, -1, $getdimlabel(image, dim, -1), profile
};

string adh5_test_reduction_func(wave source, wave dest1, wave dest2, funcref reduction_func, string reduction_param){
    wave source
    wave dest1
    wave dest2
    funcref adh5_default_reduction reduction_func
    string reduction_param
    
    reduction_func(source, dest1, dest2, reduction_param)
    
    return reduction_param
};

variable adh5_load_reduced_detector(variable fileID, string detectorpath, funcref reduction_func, string reduction_param, variable progress = defaultValue, variable nthreads = defaultValue){
    variable fileID
    string detectorpath
    funcref adh5_default_reduction reduction_func
    string reduction_param
    variable progress
    variable nthreads
        
    if (ParamIsDefault(progress))
        progress = 1
    endif
    if (ParamIsDefault(nthreads))
        nthreads = -1
    endif
    variable result = 0
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5LoadData")
    string datasetname
    string datawavename
    
    // detector data
    datasetname = detectorpath + "data"
    STRUCT HDF5DataInfo di// Defined in HDF5 Browser.ipf.
    InitHDF5DataInfo(di)
    variable err = HDF5DatasetInfo(fileID, datasetname, 0, di)
    if (err != 0)
        print "error accessing detector/data"
        return -1
    endif
    if (di.ndims < 2)
        print "error: rank of dataset < 2"
        return -2
    endif
    
    // nx and nz are the image dimensions
    variable idx, idy, idz, idt
    idx = di.ndims - 1
    idy = di.ndims - 2
    idz = -1
    idt = -1
    
    variable nx, ny, nz, nt
    nx = di.dims[idx]
    ny = di.dims[idy]
    nz = 1
    nt = 1
    
    // find additional dimensions, ignore singletons
    variable id
    for (id = idy - 1; (id >= 0) && (nz == 1); id -= 1)
        if (di.dims[id] > 1)
            idz = id
            nz = di.dims[id]
        endif
    endfor
    for (id = idz - 1; (id >= 0) && (nt == 1); id -= 1)
        if (di.dims[id] > 1)
            idt = id
            nt = di.dims[id]
        endif
    endfor
    // default values if dimensions are not present in dataset
    if (idz < 0)
        idz = idx + 1
        idt = idz + 1
    else if (idt < 0)
        idt = idx + 1
    endif
    variable nzt = nz * nt
    variable izt
    
    // load data image by image
    HDF5MakeHyperslabWave(GetDataFolder(1) + "slab", max(di.ndims, 4))
    wave slab
    slab[][%Start] = 0
    slab[][%Stride] = 1
    slab[][%Count] = 1
    slab[][%Block] = 1
    slab[idx][%Block] = nx
    slab[idy][%Block] = ny
    
    // set up multi threading
    if (nthreads < 0)
        nthreads = ThreadProcessorCount
    endif
    if (nthreads > 0)
        variable threadGroupID = ThreadGroupCreate(nthreads)
        variable ithread
        for (ithread = 0; ithread < nthreads; ithread += 1)
            ThreadStart threadGroupID, ithread, reduce_slab_worker(reduction_func)
        endfor
    else
        make /n=(nzt) /df /free processing_folders
    endif
    
    if (progress)
        display_progress_panel("HDF5 Import", "Loading data (step 1 of 2)...", nzt)
    endif
    
    make /n=(nx,ny)/d image_template
    setdimlabel 0, -1, AD_Dim0, image_template
    setdimlabel 1, -1, AD_Dim1, image_template
    adh5_scale(image_template)

    variable iz, it
    string dfname
    izt = 0
    for (iz = 0; iz < nz; iz += 1)
        for (it = 0; it < nt; it += 1)
            // load hyperslab
            slab[idz][%Start] = iz
            slab[idt][%Start] = it
            dfname = "processing_" + num2str(izt)
            newdatafolder /s $dfname
            HDF5LoadData /O /Q /Z /SLAB=slab /N=slabdata fileID, datasetname

            // send to processing queue
            duplicate image_template, image
            variable /g r_index = iz
            variable /g s_index = it
            string /g func_param = reduction_param

            if (nthreads > 0)
                WaveClear image
                ThreadGroupPutDF threadGroupID, :
            else
                processing_folders[izt] = GetDataFolderDFR()
                make /n=1/d profile1, profile2
                wave slabdata
                variable /g func_result
                func_result = reduce_slab_image(slabdata, image, profile1, profile2, reduction_func, func_param)
                WaveClear slabdata, image, profile1, profile2
                setdatafolder ::
            endif
            
            izt += 1
            // progress window
            if (progress)
                if (update_progress_panel(izt))
                    result = -4// user abort
                    break
                endif
            endif
        endfor
    endfor
    
    killwaves /z slab, image_template
    if (progress)
        update_progress_panel(0, message="Processing data (step 2 of 2)...")
    endif

    dfref dfr
    for (izt = 0; (izt < nzt) && (result == 0); izt += 1)
        if (nthreads > 0)
            do
                dfr = ThreadGroupGetDFR(threadGroupID, 1000)
                if (DatafolderRefStatus(dfr) != 0)
                    break
                endif
                if (progress)
                    if (update_progress_panel(izt))
                        result = -4// user abort
                        break
                    endif
                endif
            while (1)
        else
            dfr = processing_folders[izt]
            if (progress)
                if (update_progress_panel(izt))
                    result = -4// user abort
                    break
                endif
            endif
        endif
        
        if (result != 0)
            break
        endif
        
        nvar rr = dfr:r_index
        nvar ss = dfr:s_index
        nvar func_result = dfr:func_result
        wave profile1 = dfr:profile1
        wave profile2 = dfr:profile2

        if (func_result == 0)
            if (izt == 0)
                make /n=(dimsize(profile1, 0), nz, nt)/d/o data1
                make /n=(dimsize(profile2, 0), nz, nt)/d/o data2
                setdimlabel 0, -1, $getdimlabel(profile1, 0, -1), data1
                setdimlabel 0, -1, $getdimlabel(profile2, 0, -1), data2
                setscale /p x dimoffset(profile1, 0), dimdelta(profile1, 0), waveunits(profile1, 0), data1
                setscale /p x dimoffset(profile2, 0), dimdelta(profile2, 0), waveunits(profile2, 0), data2
                setscale d 0, 0, waveunits(profile1, -1), data1
                setscale d 0, 0, waveunits(profile2, -1), data2
            endif
            data1[][rr][ss] = profile1[p]
            data2[][rr][ss] = profile2[p]
        else
            result = -3// dimension reduction error
            break
        endif
    endfor
    
    if (nthreads > 0)
        variable tstatus = ThreadGroupRelease(threadGroupID)
        if (tstatus == -2)
            result = -5// thread did not terminate properly
        endif
    else
        for (izt = 0; izt < nzt; izt += 1)
            KillDataFolder /Z processing_folders[izt]
        endfor
    endif
    
    if (result == 0)
        if (nz == 1)
            redimension /n=(dimsize(data1,0)) data1
            redimension /n=(dimsize(data2,0)) data2
        else if (nt == 1)
            redimension /n=(dimsize(data1,0),nz) data1
            redimension /n=(dimsize(data2,0),nz) data2
            setdimlabel 1, -1, AD_DimN, data1
            setdimlabel 1, -1, AD_DimN, data2
        else
            setdimlabel 1, -1, AD_DimN, data1
            setdimlabel 1, -1, AD_DimN, data2
            setdimlabel 2, -1, AD_DimX, data1
            setdimlabel 2, -1, AD_DimX, data2
        endif
    endif
    if (progress)
        kill_progress_panel()
    endif
    
    #else
    Abort "HDF5 XOP not loaded."
    #endif
    return result
};

threadsafe static variable reduce_slab_worker(funcref reduction_func){
    funcref adh5_default_reduction reduction_func
    do
        // wait for job from main thread
        do
            dfref dfr = ThreadGroupGetDFR(0, 1000)
            if (DataFolderRefStatus(dfr) == 0)
                if (GetRTError(2))
                    return 0// no more jobs
                endif
            else
                break
            endif
        while (1)

        // get input data
        wave slabdata = dfr:slabdata
        wave image = dfr:image
        svar func_param = dfr:func_param
        nvar rr = dfr:r_index
        nvar ss = dfr:s_index
        
        // do the work
        newdatafolder /s outDF
        make /n=1/d profile1, profile2
        variable /g r_index = rr
        variable /g s_index = ss
        variable /g func_result
        func_result = reduce_slab_image(slabdata, image, profile1, profile2, reduction_func, func_param)

        // send output to queue and clean up
        WaveClear slabdata, image, profile1, profile2
        ThreadGroupPutDF 0, :
        KillDataFolder dfr
    while (1)
    
    return 0
};

threadsafe static variable reduce_slab_image(wave slabdata, wave image, wave profile1, wave profile2, funcref reduction_func, string reduction_param){
    wave slabdata
    wave image
    wave profile1
    wave profile2
    funcref adh5_default_reduction reduction_func
    string reduction_param
    
    switch (WaveDims(slabdata))
        case 2:
            image = slabdata[q][p]
            break
        case 3:
            image = slabdata[0][q][p]
            break
        case 4:
            image = slabdata[0][0][q][p]
            break
    endswitch

    return reduction_func(image, profile1, profile2, reduction_param)
};

variable adh5_loadattr_all(variable fileID, string attributespath){
    variable fileID
    string attributespath

    string datasetname
    string datawavename
    
    // avoid compilation error if HDF5 XOP has not been loaded
    #if Exists("HDF5LoadData")

    // datasets in NDAttributes group
    HDF5ListGroup /F /TYPE=2 fileID, attributespath
    string h5datasets = S_HDF5ListGroup
    HDF5ListAttributes /TYPE=1 /Z fileID, attributespath
    string h5attributes = S_HDF5ListAttributes
    
    variable nds = ItemsInList(h5datasets, ";")
    variable na = ItemsInList(h5attributes, ";")
    variable ids
    variable idest = 0
    variable n_attr
    string s_attr
    string s_source

    make /n=(nds+na) /t /o IN, ID, IV, IU
    
    for (ids = 0; ids < nds; ids += 1)
        datasetname = StringFromList(ids, h5datasets, ";")
        HDF5LoadData /O/Q fileID, datasetname
        if (v_flag == 0)
            datawavename = StringFromList(0, s_wavenames)
        else
            datawavename = ""
        endif
        HDF5LoadData /A="source"/O/Q/TYPE=2 fileID, datasetname
        if (v_flag == 0)
            wave /t source
            s_source = source[0]
        else
            s_source = ""
        endif
        read_attribute_info(datawavename, s_source, idest)
    endfor
    
    // attributes of NDAttributes group
    if (v_flag == 0)
        nds = ItemsInList(h5attributes, ";")
    else
        nds = 0
    endif
    for (ids = 0; ids < nds; ids += 1)
        datasetname = StringFromList(ids, h5attributes, ";")
        HDF5LoadData /A=datasetname/O/Q/TYPE=1 fileID, attributespath
        if (v_flag == 0)
            datawavename = StringFromList(0, s_wavenames)
            read_attribute_info(datawavename, "", idest)// we don't get the source of these attributes
        endif
    endfor
    
    redimension /n=(idest) IN, ID, IV, IU
    sort {IN, ID}, IN, ID, IV, IU

    killwaves /z source
    #else
    Abort "HDF5 XOP not loaded."
    #endif
        
};

static variable read_attribute_info(string datawavename, string source, variable* idest){
    string datawavename// name of the attribute wave in the current folder.
        // can be text or numeric.
    string source
        // source identifier (EPICS name) of the attribute.
    variable &idest
        // destination index in IN, ID, IV, IU where the results are written.
        // the variable is incremented if data was written, otherwise it is left unchanged.
        // make sure IN, ID, IV, IU have at least idest + 1 elements.
    
    wave /t IN
    wave /t ID
    wave /t IV
    wave /t IU
    
    variable n_attr
    string s_attr
    
    if (exists(datawavename) == 1)
        if (strlen(source) > 0)
            Note $datawavename, "PV=" + source
        endif
        switch(WaveType($datawavename, 1))
            case 1:// numeric
                wave w_attr = $datawavename
                n_attr = numpnts(w_attr)
                sprintf s_attr, "%.12g", w_attr[0]
                break
            case 2:// text
                wave /t wt_attr = $datawavename
                n_attr = numpnts(wt_attr)
                s_attr = wt_attr[0]
                break
            default:// unknown
                n_attr = 0
        endswitch
        if (n_attr == 1)
            IN[idest] = source
            ID[idest] = datawavename
            IV[idest] = s_attr
            IU[idest] = ""// we don't get the units
            idest += 1
        endif
    endif
};

variable adh5_scale_scienta(wave data){
    wave data
    
    dfref saveDF = GetDataFolderDFR()
    
    dfref dataDF = GetWavesDataFolderDFR(data)
    dfref attrDF = GetAttrDataFolderDFR(data)
    
    wave /SDFR=attrDF LensMode
    wave /SDFR=attrDF /Z ChannelBegin, ChannelEnd
    wave /SDFR=attrDF /Z SliceBegin, SliceEnd
    
    variable EDim, ADim
    variable ELow, EHigh, ALow, AHigh
    string EUnit, AUnit
    
    // which dimension is angle and which one is energy?
    strswitch(GetDimLabel(data, 0, -1))
        case "AD_Dim0":
            EDim = 0
            break
        case "AD_Dim1":
            EDim = 1
            break
        default:
            EDim = -1
    endswitch
    strswitch(GetDimLabel(data, 1, -1))
        case "AD_Dim0":
            ADim = 0
            break
        case "AD_Dim1":
            ADim = 1
            break
        default:
            ADim = -1
    endswitch
    
    // defaults (point scaling)
    if (EDim >= 0)
        ELow = dimoffset(data, EDim)
        EHigh = dimoffset(data, EDim) + dimdelta(data, EDim) * (dimsize(data, EDim) - 1)
        EUnit = "eV"
    endif
    if (ADim >= 0)
        ALow = dimoffset(data, ADim)
        AHigh = dimoffset(data, ADim) + dimdelta(data, ADim) * (dimsize(data, ADim) - 1)
        AUnit = "arb."
    endif
    
    // lens mode can give more detail
    if (waveexists(LensMode) && (numpnts(LensMode) >= 1))
        switch(LensMode[0])
            case 1:// Angular45
                ALow = -45/2
                AHigh = +45/2
                AUnit = "°"
                break
            case 2:// Angular60
                ALow = -60/2
                AHigh = +60/2
                AUnit = "°"
                break
        endswitch
    endif
    
    // best option if scales are explicit in separate waves
    if (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1))
        ELow = ChannelBegin[0]
        EHigh = ChannelEnd[0]
    endif
    if (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1))
        ALow = SliceBegin[0]
        AHigh = SliceEnd[0]
    endif

    // apply new scales
    switch(EDim)
        case 0:
            setscale /i x ELow, EHigh, EUnit, data
            break
        case 1:
            setscale /i y ELow, EHigh, EUnit, data
            break
    endswitch
    switch(ADim)
        case 0:
            setscale /i x ALow, AHigh, AUnit, data
            break
        case 1:
            setscale /i y ALow, AHigh, AUnit, data
            break
    endswitch

    setscale d 0, 0, "arb.", data
    
    setdatafolder saveDF
};

variable adh5_scale_scan(wave data){
    wave data

    dfref saveDF = GetDataFolderDFR()
    
    dfref dataDF = GetWavesDataFolderDFR(data)
    wave /SDFR=dataDF AcquisitionMode, DetectorMode, EnergyMode

    wave /SDFR=dataDF /z Scan1Active, Scan2Active
    wave /SDFR=dataDF /t /z Scan1Positioner1, Scan1Readback1
    wave /SDFR=dataDF /t /z Scan1Positioner2, Scan1Readback2
    wave /SDFR=dataDF /t /z Scan2Positioner1, Scan2Readback1
    wave /SDFR=dataDF /t /z Scan2Positioner2, Scan2Readback2

    // TODO : search the data folder for positioner waves,
    // i.e. waves with the PV name corresponding to Scan1Positioner1 in their wave note.
    wave /z zscale
    
    strswitch(GetDimLabel(data, 0, -1))
        case "AD_DimN":
            setscale /i x zscale[0], zscale[numpnts(zscale)-1], "", data
            break
    endswitch
    strswitch(GetDimLabel(data, 1, -1))
        case "AD_DimN":
            setscale /i y zscale[0], zscale[numpnts(zscale)-1], "", data
            break
    endswitch
    strswitch(GetDimLabel(data, 2, -1))
        case "AD_DimN":
            setscale /i z zscale[0], zscale[numpnts(zscale)-1], "", data
            break
    endswitch
    
    setdatafolder saveDF
};