pearl-matrix-import.ipf

Go to the documentation of this file.

#pragma rtGlobals=3
#pragma version = 1.00
#pragma IgorVersion = 6.36
#pragma ModuleName = PearlMatrixImport

// author: matthias.muntwiler@psi.ch
// Copyright (c) 2016 Paul Scherrer Institut

// Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0



static const string package_name = "pearl_matrix_import";
static const string package_path = "root:packages:pearl_matrix_import:";

static const string ks_filematch_mtrx = "*_mtrx";

static variable init_package(){
    
    dfref savedf = getdatafolderdfr()
    
    setdatafolder root:
    newdatafolder /o/s packages
    newdatafolder /o/s $package_name
    
    variable /g loglevel = 3
    string /g dataFilePath = ""
    string /g resultFilePath = ""
    variable /g runCycle = 0
    variable /g scanCycle = 0
    string /g channelName = ""
    variable /g brickletID = 0
    variable /g V_MatrixFileReaderOverwrite = 1
    variable /g V_MatrixFileReaderFolder = 0
    variable /g V_MatrixFileReaderDouble = 0

    setdatafolder savedf
    return 0
};

static variable check_package_folder(){
    dfref df_pack = $(package_path)
    if (DataFolderRefStatus(df_pack))
        svar /sdfr=df_pack /z resultFilePath
        if (!svar_exists(resultFilePath))
            init_package()
        endif
    else
        init_package()
    endif
};

static variable AfterFileOpenHook(variable refNum, string file, string pathName, string type, string creator, variable kind){
    Variable refNum,kind
    String file,pathName,type,creator
    if( (kind >= 1) && (kind <= 2))
        init_package()
        //load_prefs()
    endif
    return 0
};

static variable BeforeFileOpenHook(variable refNum, string fileName, string path, string type, string creator, variable kind){
    Variable refNum,kind
    String fileName,path,type,creator

    Variable handledOpen = 0
    if (StringMatch(fileName, ks_filematch_mtrx))
        setdatafolder root:
        newdatafolder /o /s matrix
        mtrx_load_preview("matrix", path, fileName)
        handledOpen = 1
    endif
    return handledOpen
};

string matrix_format_elog_message(wave metadata){
    wave /t metadata

    string key
    string value
    variable nkeys = dimsize(metadata, 0)
    variable ikey
    string message_keys
    message_keys = "resultFileName;sampleName;channelName;"
    message_keys += "XYScanner.Points.value;XYScanner.Raster_Time.value;XYScanner.Raster_Time.unit;XYScanner.Width.value;XYScanner.Width.unit;XYScanner.Height.value;XYScanner.Height.unit;"
    message_keys += "GapVoltageControl.Voltage.value;GapVoltageControl.Voltage.unit;"
    message_keys += "Regulator.Loop_Gain_1_I.value;Regulator.Loop_Gain_1_I.unit;Regulator.Setpoint_1.value;Regulator.Setpoint_1.unit;"
    message_keys += "Spectroscopy.Device_1_Start.value;Spectroscopy.Device_1_Start.unit;Spectroscopy.Spectroscopy_Mode.value;"
    string message
    
    message_keys = ""
    for (ikey = 0; ikey < nkeys; ikey += 1)
        key = metadata[ikey][0]
        value = metadata[ikey][1]
        if (WhichListItem(key, message_keys) >= 0)
            message += key + " = " + value + "\r"
        endif
    endfor
};

variable matrix_preview_2d(wave data, wave metadata){
    wave data
    wave /t metadata
    
    Display
    AppendImage data
    ModifyImage data ctab= {*,*,Mud,0}
    ModifyGraph margin(left)=30,margin(bottom)=30,margin(top)=5,margin(right)=5,height={Plan,1,left,bottom}
    ModifyGraph mirror=2
    ModifyGraph nticks=3
    ModifyGraph axThick=0.5
    ModifyGraph btLen=4

};

static wave preview_matrix_file(string filename){
    string filename

    dfref saveDF = GetDataFolderDFR()
    setdatafolder $package_path
    svar s_preview_file
    svar s_preview_source
    mtrx_load_preview("preview_image", "pearl_explorer_filepath", filename)
    s_preview_file = filename
    s_preview_source = ""
    wave /z preview_image
    
    svar /z s_file_info
    if (! svar_exists(s_file_info))
        string /g s_file_info
    endif
    if (strlen(s_preview_file) > 0)
        s_file_info = mtrx_load_info("pearl_explorer_filepath", filename)
    else
        s_file_info = ""
    endif
    
    setdatafolder saveDF
    return preview_image
};

struct errorCode{;
    int32    SUCCESS;
    int32    UNKNOWN_ERROR;
    int32    ALREADY_FILE_OPEN;
    int32    EMPTY_RESULTFILE;
    int32    FILE_NOT_READABLE;
    int32    NO_NEW_BRICKLETS;
    int32    WRONG_PARAMETER;
    int32    INTERNAL_ERROR_CONVERTING_DATA;
    int32    NO_FILE_OPEN;
    int32    INVALID_RANGE;
    int32    WAVE_EXIST;
};

static variable initStruct(errorCode* errorCode){
    Struct errorCode &errorCode

    errorCode.SUCCESS =0
    errorCode.UNKNOWN_ERROR=10001
    errorCode.ALREADY_FILE_OPEN=10002
    errorCode.EMPTY_RESULTFILE=10004
    errorCode.FILE_NOT_READABLE=10008
    errorCode.NO_NEW_BRICKLETS=10016
    errorCode.WRONG_PARAMETER=10032
    errorCode.INTERNAL_ERROR_CONVERTING_DATA=10064
    errorCode.NO_FILE_OPEN=10128
    errorCode.INVALID_RANGE=10256
    errorCode.WAVE_EXIST=10512
    
};

variable mtrx_load_all(){

    struct errorCode errorCode
    initStruct(errorCode)

#if exists("MFR_OpenResultFile")
    MFR_OpenResultFile
    if(V_flag != errorCode.SUCCESS)
        MFR_GetXOPErrorMessage
        return -1
    endif

    MFR_GetBrickletData
    if(V_flag != errorCode.SUCCESS)
        MFR_GetXOPErrorMessage
        return -1
    endif

    MFR_GetBrickletMetaData
    if(V_flag != errorCode.SUCCESS)
        MFR_GetXOPErrorMessage
        return -1
    endif

    return 0
#else
    return -1
#endif
};

variable mtrx_parse_filename(string fileName, string* resultFile, variable* runCycle, variable* scanCycle, string* channel){
    string fileName
    string &resultFile
    variable &runCycle
    variable &scanCycle
    string &channel
    
    variable fileType = 0
    resultFile = ""
    channel = ""
    runCycle = 0
    scanCycle = 0

    string regexp = ""
    string index1 = ""
    string index2 = ""
    string extension = ""
    
    if (StringMatch(fileName, "*.mtrx"))
        regexp = "(.+)_([[:digit:]]+)\.(.+)"
        SplitString /E=regexp fileName, resultFile, index1, extension
        fileType = 0
    else
        regexp = "(.+)--([[:digit:]]+)_([[:digit:]]+)\.((.+)_mtrx)"
        SplitString /E=regexp fileName, resultFile, index1, index2, extension, channel
        fileType = 1
        runCycle = str2num(index1)
        scanCycle = str2num(index2)
    endif
    
    return fileType
};

string mtrx_split_filename(string fileName, string* prefix, string* datepart, string* timepart){
    string fileName
    string &prefix
    string &datepart
    string &timepart
    
    string regexp
    regexp = "([[:alpha:][:digit:]]+)[-_]([[:alpha:][:digit:]]+)[-_]([[:alpha:][:digit:]]+)[-_].+"
    SplitString /E=regexp fileName, prefix, datepart, timepart
    return datepart
};

dfr mtrx_create_folder(string fileName, dfref df_base = defaultValue){
    string fileName
    dfref df_base
    
    if (ParamIsDefault(df_base))
        df_base = GetDataFolderDFR()
    endif

    string prefix
    string datepart
    string timepart
    string folderName
    
    mtrx_split_filename(fileName, prefix, datepart, timepart)
    folderName = "mtrx_" + datepart + "_" + timepart
    folderName = CleanupName(folderName, 0)
    
    dfref df_save = GetDataFolderDFR()
    setdatafolder root:
    newdatafolder /o /s $foldername
    dfref df = GetDataFolderDFR()

    setdatafolder df_save
    return df
};

dfr mtrx_get_cycle_folder(dfref df_base = defaultValue, variable runCycle = defaultValue, variable scanCycle = defaultValue){
    dfref df_base
    variable runCycle
    variable scanCycle

    dfref df_save = GetDataFolderDFR()
    dfref df_pack = $(package_path)
    if (ParamIsDefault(df_base))
        df_base = GetDataFolderDFR()
    endif
    if (ParamIsDefault(runCycle))
        nvar /sdfr=df_pack defRunCycle = runCycle
        runCycle = defRunCycle
    endif
    if (ParamIsDefault(scanCycle))
        nvar /sdfr=df_pack defScanCycle = scanCycle
        scanCycle = defScanCycle
    endif

    string dfname
    if ((runCycle >= 1) && (scanCycle >= 1))
        sprintf dfname, "r%us%u", runCycle, scanCycle
        setdatafolder df_base
        dfref df = $dfname
        if (DataFolderRefStatus(df) == 0)
            newdatafolder $dfname
            dfref df = $dfname
        endif
    else
        dfref df = df_base
    endif
    
    setdatafolder df_save
    return df
};

variable mtrx_file_brickletID(string resultFile, variable runCycle, variable scanCycle, string channel){
    string resultFile
    variable runCycle
    variable scanCycle
    string channel

    dfref df_overview = NewFreeDataFolder()
    variable link = 1
    variable id = -1
    variable idx = 0
    
    string resultFileName
#if exists("MFR_OpenResultFile")
    struct errorCode errorCode
    initStruct(errorCode)
    do
        sprintf resultFileName, "%s_%04u.mtrx", resultFile, link
        MFR_OpenResultFile /K resultFileName
        if(V_flag != errorCode.SUCCESS)
            return -1
        endif
        MFR_CreateOverviewTable /DEST=df_overview
        // dimension labels are: brickletID, scanCycleCount, runCycleCount, sequenceID, dimension, channelName
        wave /t /sdfr=df_overview overviewTable
        make /n=(dimsize(overviewTable, 0)) /i /u /free runcycles, scancycles, ids, match
        make /n=(dimsize(overviewTable, 0)) /t /free channels
        ids = str2num(overviewtable[p][%brickletID])
        if (runcycle > 0)
            runcycles = str2num(overviewtable[p][%runCycleCount])
        else
            runcycles = runcycle
        endif
        if (scancycle > 0)
            scancycles = str2num(overviewtable[p][%scanCycleCount])
        else
            scancycles = scancycle
        endif
        if (strlen(channel) > 0)
            channels = overviewTable[p][%channelName]
        else
            channels = channel
        endif
        Extract /FREE ids, match_ids, (scancycles == scanCycle) && (runcycles == runCycle) && (cmpstr(channels, channel) == 0)
        if (numpnts(match_ids) > 0)
            id = match_ids[0]
        else
            link += 1
        endif
    while (id < 0)
#endif
    return id
};

variable mtrx_open_file(string pathName, string fileNameOrPath){
    string pathName
    string fileNameOrPath

    check_package_folder()
    dfref df_save = GetDataFolderDFR()
    dfref df_pack = $(package_path)
    svar /sdfr=df_pack dataFilePath
    svar /sdfr=df_pack resultFilePath
    nvar /sdfr=df_pack runCycle
    nvar /sdfr=df_pack scanCycle
    svar /sdfr=df_pack channelName
    nvar /sdfr=df_pack brickletID

    string loc_resultFileName
    string loc_resultFilePath
    variable loc_runCycle
    variable loc_scanCycle
    string loc_channelName

    // make sure we have a valid and complete file path
    GetFileFolderInfo /P=$pathName /Q /Z=2 fileNameOrPath
    string filePath
    if ((v_flag == 0) && (v_isFile))
        filePath = s_path
    else
        return -1
    endif

    // get base file name
    string fileName
    string fileDir
    string baseFileName
    variable fileType
    fileName = ParseFilePath(0, filePath, ":", 1, 0)
    fileDir = ParseFilePath(1, filePath, ":", 1, 0)
    fileType = mtrx_parse_filename(fileName, baseFileName, loc_runCycle, loc_scanCycle, loc_channelName)
    
    variable link = 1
    variable id = -1
    variable result = -1
    variable using_cache = 0
    
    struct errorCode errorCode
    initStruct(errorCode)
    do
        sprintf loc_resultFileName, "%s_%04u.mtrx", baseFileName, link
        loc_resultFilePath = fileDir + loc_resultFileName
#if exists("MFR_OpenResultFile")
        if ((strlen(resultFilePath) == 0) || (cmpstr(loc_resultFilePath, resultFilePath) != 0))
            MFR_OpenResultFile /K loc_resultFilePath
            if(V_flag != errorCode.SUCCESS)
                MFR_GetXOPErrorMessage
                result = -1
                break
            endif
            resultFilePath = loc_resultFilePath
            if (fileType == 1)
                dataFilePath = filePath
            else
                dataFilePath = ""
            endif
            runCycle = 0
            scanCycle = 0
            channelName = ""
            brickletID = 0
            MFR_CreateOverviewTable /DEST=df_pack
            if(V_flag != errorCode.SUCCESS)
                MFR_GetXOPErrorMessage
                result = -1
                break
            endif
            using_cache = 0
        else
            using_cache = 1
        endif
#else
        print "matrixfilereader.xop not installed"
        result = -2
        break
#endif
        // dimension labels are: brickletID, scanCycleCount, runCycleCount, sequenceID, dimension, channelName
        wave /t /sdfr=df_pack overviewTable
        make /n=(dimsize(overviewTable, 0)) /i /u /o df_pack:runCycles, df_pack:scanCycles, df_pack:ids
        make /n=(dimsize(overviewTable, 0)) /t /o df_pack:channels
        wave /sdfr=df_pack ids, runCycles, scanCycles
        wave /t /sdfr=df_pack channels
        ids = str2num(overviewtable[p][%brickletID])
        runCycles = str2num(overviewtable[p][%runCycleCount])
        scanCycles = str2num(overviewtable[p][%scanCycleCount])
        channels = overviewTable[p][%channelName]
        result = fileType

        // if a data file is opened, make sure we found the right result file
        if ((loc_runCycle > 0) && (loc_scanCycle > 0))
            Extract /FREE ids, match_ids, (runCycles == loc_runCycle) && (scanCycles == loc_scanCycle) && (cmpstr(channels, loc_channelName) == 0)
            if (numpnts(match_ids) > 0)
                id = match_ids[0]
                runCycle = loc_runCycle
                scanCycle = loc_scanCycle
                channelName = loc_channelName
                brickletID = id
                break
            else if (using_cache)
                resultFilePath = ""
            else
                link += 1
            endif
        else
            break
        endif
    while (id < 0)
    
    return result
};

string mtrx_load_preview(string destName, string pathName, string fileName, string traces = defaultValue){
    string destName
    string pathName
    string fileName
    string traces
        
    if (ParamIsDefault(traces))
        traces = "*Up;*Down;*ReUp;*ReDown;"
    endif
    
    dfref df_save = GetDataFolderDFR()

    string datanames = ""
    string datapaths = ""
    variable filestatus = mtrx_open_file(pathName, fileName)
    if (filestatus != 1)
        return ""
    endif

    dfref df_pack = $(package_path)
    svar /sdfr=df_pack dataFilePath
    svar /sdfr=df_pack resultFilePath
    nvar /sdfr=df_pack runCycle
    nvar /sdfr=df_pack scanCycle
    svar /sdfr=df_pack channelName
    nvar /sdfr=df_pack brickletID

#if exists("MFR_OpenResultFile")
    dfref df_data = df_save
    variable /g df_data:V_MatrixFileReaderOverwrite = 1
    variable /g df_data:V_MatrixFileReaderFolder = 0
    variable /g df_data:V_MatrixFileReaderDouble = 0
    struct errorCode errorCode
    initStruct(errorCode)
    MFR_GetBrickletData /N=destName /R=(brickletID) /S=2 /DEST=df_data
    if(V_flag == errorCode.SUCCESS)
        datanames = S_waveNames
        variable i
        variable n = ItemsInList(datanames)
        string s
        s = StringFromList(0, datanames)
        wave data = $s
        mtrx_scale_dataset(data)
        if (WaveDims(data) == 2)
            subtract_line_bg(data)
        endif
        datapaths = AddListItem(GetWavesDataFolder(data, 4), datapaths, ";", inf)
        for (i = 1; i < n; i += 1)
            s = StringFromList(i, datanames)
            killwaves /z $s
        endfor
    else
        MFR_GetXOPErrorMessage
    endif
    //MFR_GetBrickletMetaData /N=ANickName /R=(brickletID) // /DEST=dfref
#else
    print "matrixfilereader.xop not installed"
#endif
    
    setdatafolder df_save
    return datapaths
};

string mtrx_load_file(string pathName, string fileName, string traces = defaultValue){
    string pathName
    string fileName
    string traces
        
    if (ParamIsDefault(traces))
        traces = "*Up;*Down;*ReUp;*ReDown;"
    endif
    
    dfref df_save = GetDataFolderDFR()

    string datanames = ""
    string datapaths = ""
    variable filestatus = mtrx_open_file(pathName, fileName)
    if (filestatus != 1)
        return ""
    endif

    dfref df_pack = $(package_path)
    svar /sdfr=df_pack dataFilePath
    svar /sdfr=df_pack resultFilePath
    nvar /sdfr=df_pack runCycle
    nvar /sdfr=df_pack scanCycle
    svar /sdfr=df_pack channelName
    nvar /sdfr=df_pack brickletID

#if exists("MFR_OpenResultFile")
    string resultFileName = ParseFilePath(0, resultFilePath, ":", 1, 0)
    dfref df_result = mtrx_create_folder(resultFileName, df_base=df_save)
    dfref df_data = mtrx_get_cycle_folder(df_base = df_result)
    variable /g df_data:V_MatrixFileReaderOverwrite = 1
    variable /g df_data:V_MatrixFileReaderFolder = 0
    variable /g df_data:V_MatrixFileReaderDouble = 0
    
    struct errorCode errorCode
    initStruct(errorCode)
    string name
    name = CleanupName(channelName, 0)
    MFR_GetBrickletData /N=name /R=(brickletID) /DEST=df_data
    if(V_flag == errorCode.SUCCESS)
        datanames = S_waveNames
        variable i
        variable n = ItemsInList(datanames)
        string s
        for (i = 0; i < n; i += 1)
            s = StringFromList(i, datanames)
            wave /sdfr=df_data data = $s
            mtrx_scale_dataset(data)
            datapaths = AddListItem(GetWavesDataFolder(data, 4), datapaths, ";", inf)
        endfor
    else
        MFR_GetXOPErrorMessage
    endif
    //MFR_GetBrickletMetaData /N=ANickName /R=(brickletID) // /DEST=dfref
#else
    print "matrixfilereader.xop not installed"
#endif
    
    setdatafolder df_save
    return datapaths
};

variable mtrx_scale_dataset(wave data){
    wave data

    dfref df_pack = $(package_path)
    nvar /sdfr=df_pack runCycle
    nvar /sdfr=df_pack scanCycle
    svar /sdfr=df_pack channelName
    nvar /sdfr=df_pack brickletID

    string scanDir = StringFromList(2, NameOfWave(data), "_")
    if (WaveDims(data) == 2)
        Note data, "AxisLabelX=X"
        Note data, "AxisLabelY=Y"
    endif
    Note data, "AxisLabelD=" + channelName
    string title
    sprintf title, "%u-%u %s %s", runCycle, scanCycle, channelName, scanDir
    Note data, "Dataset=" + title
};

string mtrx_load_info(string APathName, string AFileName){
    string APathName
    string AFileName
        
    dfref saveDF = GetDataFolderDFR()
    dfref fileDF = NewFreeDataFolder()
    setdatafolder fileDF

    variable fileID
    string filepath
    string scanpaths
    variable nscans
    variable iscan
    string scanpath
    string info = ""
      
    
    setdatafolder saveDF
    return info
};

variable subtract_line_bg(wave img){
    wave img
    
    variable nx = dimsize(img, 0)
    variable ny = dimsize(img, 1)
    variable iy
    make /n=(nx) /free line, fit
    for (iy = 0; iy < ny; iy += 1)
        line = img[p][iy]
        if (numtype(sum(line)) == 0)
            CurveFit /N /Q /NTHR=0 line line /D=fit
            img[][iy] = line[p] - fit[p]
        endif
    endfor
};