From ea838b395ffadb01fbc511f866217e7230b035c4 Mon Sep 17 00:00:00 2001 From: matthias muntwiler Date: Mon, 22 Dec 2025 17:04:08 +0100 Subject: [PATCH] code changes for release 3.1.0: SciLog, DA30-PShell data files --- .gitea/workflows/deploy-pages.yaml | 46 + README.md | 8 +- doc/config.dox | 2 +- pearl/pearl-area-display.ipf | 10 +- pearl/pearl-arpes.ipf | 3 +- pearl/pearl-data-explorer.ipf | 311 +++-- pearl/pearl-elog.ipf | 72 +- pearl/pearl-menu.ipf | 6 +- pearl/pearl-pmsco-import.ipf | 29 +- pearl/pearl-pshell-import.ipf | 49 +- pearl/pearl-scilog.ipf | 1739 ++++++++++++++++++++++++++++ pearl/scilog-ingest.py | 331 ++++++ 12 files changed, 2437 insertions(+), 169 deletions(-) create mode 100644 .gitea/workflows/deploy-pages.yaml create mode 100644 pearl/pearl-scilog.ipf create mode 100644 pearl/scilog-ingest.py diff --git a/.gitea/workflows/deploy-pages.yaml b/.gitea/workflows/deploy-pages.yaml new file mode 100644 index 0000000..f5163ed --- /dev/null +++ b/.gitea/workflows/deploy-pages.yaml @@ -0,0 +1,46 @@ +name: build and deploy documentation + +on: + push: + branches: + - distro + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + container: + image: gitea.psi.ch/pearl/docs + credentials: + username: ${{ gitea.actor }} + password: ${{ secrets.package_token }} + + steps: + - name: checkout + working-directory: /app + run: | + git clone --branch master --single-branch https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git + + - name: build + working-directory: /app/igor-procs/doc + run: | + REVISION=$(git describe --always --tags --dirty --long || date +"%F %T %z") + export REVISION + doxygen config.dox + mv html/ /app/ + + - name: configure git + working-directory: /app/igor-procs + run: | + git config --global user.name "Gitea Actions" + git config --global user.email "actions@gitea.local" + + - name: push to gitea-pages + working-directory: /app/igor-procs + run: | + git checkout --orphan gitea-pages + git reset --hard + cp -r /app/html/* . + git add . + git commit -m "Deploy documentation to gitea" + git push -f https://${{ secrets.REPO_TOKEN }}@gitea.psi.ch/${{ github.repository }}.git gitea-pages + diff --git a/README.md b/README.md index 330f2a9..707f648 100644 --- a/README.md +++ b/README.md @@ -36,12 +36,18 @@ Matthias Muntwiler, Copyright --------- -Copyright 2009-2022 by [Paul Scherrer Institut](http://www.psi.ch) +Copyright 2009-2025 by [Paul Scherrer Institut](http://www.psi.ch) Release Notes ============= +## rev-distro-3.1.0 + +- Ingestor to SciLog electronic logbook +- Support for PShell files from DA30/DFS30 analyser +- Fix Gizmo window in Igor 8 and higher + ## rev-distro-3.0.0 - New panel and procedure interface for PShell data file import. diff --git a/doc/config.dox b/doc/config.dox index c68c68b..8723be7 100644 --- a/doc/config.dox +++ b/doc/config.dox @@ -1581,7 +1581,7 @@ EXTRA_SEARCH_MAPPINGS = # If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. # The default value is: YES. -GENERATE_LATEX = YES +GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of diff --git a/pearl/pearl-area-display.ipf b/pearl/pearl-area-display.ipf index 36fb96a..7eb767a 100644 --- a/pearl/pearl-area-display.ipf +++ b/pearl/pearl-area-display.ipf @@ -1,8 +1,8 @@ #pragma TextEncoding = "UTF-8" #pragma rtGlobals=3 // Use modern global access method and strict wave access. -#pragma IgorVersion = 6.2 +#pragma IgorVersion = 7.0 #pragma ModuleName = PearlAreaDisplay -#pragma version = 1.04 +#pragma version = 1.05 #include "pearl-compat" #include "pearl-area-profiles" @@ -1161,7 +1161,7 @@ function /s ad_display_brick(data) string /g gizmo_graphname = graphname_from_dfref(datadf, "giz_") svar graphname = gizmo_graphname - if ((strlen(graphname) > 0) && (wintype(graphname) == 13)) + if ((strlen(graphname) > 0) && (wintype(graphname) == 17)) setdatafolder savedf return graphname // gizmo window exists endif @@ -1380,7 +1380,7 @@ function ad_brick_slicer(data) z_slice_pos = dimoffset(data, 2) + dimsize(data, 2) * dimdelta(data, 2) / 2 svar /z /sdfr=viewdf gizmo_graphname - if (svar_exists(gizmo_graphname) && (strlen(gizmo_graphname) > 0) && (wintype(gizmo_graphname) == 13)) + if (svar_exists(gizmo_graphname) && (strlen(gizmo_graphname) > 0) && (wintype(gizmo_graphname) == 17)) ad_gizmo_set_plane(data, 0, x_slice_pos) ad_gizmo_set_plane(data, 1, y_slice_pos) ad_gizmo_set_plane(data, 2, z_slice_pos) @@ -1495,7 +1495,7 @@ function ad_gizmo_set_plane(brick, dim, value) return -1 // requested value out of range endif - if (svar_exists(graphname) && (strlen(graphname) > 0) && (wintype(graphname) == 13)) + if (svar_exists(graphname) && (strlen(graphname) > 0) && (wintype(graphname) == 17)) string axes = "xyz" string obj = "surface_" + axes[dim] + "mid" string cmd diff --git a/pearl/pearl-arpes.ipf b/pearl/pearl-arpes.ipf index 0185cdb..263e228 100644 --- a/pearl/pearl-arpes.ipf +++ b/pearl/pearl-arpes.ipf @@ -2,7 +2,7 @@ #pragma rtGlobals=3 // Use modern global access method and strict wave access. #pragma IgorVersion = 6.1 #pragma ModuleName = PearlArpes -#pragma version = 1.05 +#pragma version = 1.06 #include "pearl-area-display" // 2D and 3D data visualization #include "pearl-area-profiles" // data processing for multi-dimensional datasets #include "pearl-area-import" // import data files generated by area detector software @@ -14,6 +14,7 @@ #include "pearl-anglescan-tracker" // live preview of hemispherical angle scan #include "pearl-scienta-preprocess" // pre-processing functions for Scienta detector images #include "pearl-elog" +#include "pearl-scilog" #if exists("pvOpen") #include "pearl-area-live" // live view of area detector #include "pearl-epics" // EPICS access under Igor diff --git a/pearl/pearl-data-explorer.ipf b/pearl/pearl-data-explorer.ipf index 69ff26a..7fa7ffe 100644 --- a/pearl/pearl-data-explorer.ipf +++ b/pearl/pearl-data-explorer.ipf @@ -2,7 +2,7 @@ #pragma rtGlobals=3 // Use modern global access method and strict wave access. #pragma IgorVersion = 6.36 #pragma ModuleName = PearlDataExplorer -#pragma version = 2.1 +#pragma version = 2.2 #include , version >= 1.14 #include "pearl-area-import" #include "pearl-area-profiles" @@ -10,7 +10,7 @@ #include "pearl-compat" #include "pearl-pshell-import" -// copyright (c) 2013-22 Paul Scherrer Institut +// copyright (c) 2013-25 Paul Scherrer Institut // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -543,87 +543,6 @@ static function notebook_add_attributes(notebook_name, attr_filter, attr_names, endfor end -/// send general metadata to ELOG panel - if available -/// -/// the following metatdata are sent. -/// they must be present as strings in the specified data folder: -/// -/// | ELOG parameter | global string | function argument | -/// | --- | --- | --- | -/// | file | s_filepath | filename | -/// | graph attachment | | graphname | -/// | author | authors | | -/// | p-group | pgroup | | -/// | project | proposal | | -/// | sample | sample | | -/// -/// @param file_df data folder that contains the metadata. -/// -/// @param filename override file path read from s_filepath global string variable. -/// if neither is declared, the file name is reset to empty field. -/// -/// @param graphname select this graph window for attaching. -/// default: do not change the selection. -/// -static function set_elog_attributes(file_df, [filename, graphname]) - dfref file_df - string filename - string graphname - - if (ParamIsDefault(filename)) - svar /sdfr=file_df /z loaded_file=s_filepath - if (svar_Exists(loaded_file)) - filename = loaded_file - else - filename = "" - endif - endif - - if (ParamIsDefault(graphname)) - graphname = "" - endif - - string cmd - - if (exists("PearlElog#set_panel_attributes") == 6) - sprintf cmd, "PearlElog#set_panel_attributes(\"\", \"File=%s\")", ParseFilePath(0, filename, ":", 1, 0) - execute /Q/Z cmd - if ((strlen(graphname) > 0) && (WinType(graphname) == 1)) - sprintf cmd, "PearlElog#set_panel_graphs(\"\", \"%s\")", graphname - execute /Q/Z cmd - endif - svar /sdfr=file_df /z authors - if (svar_Exists(authors)) - if (strlen(authors)>=1) - sprintf cmd, "PearlElog#set_panel_attributes(\"\", \"author=%s\")", authors - execute /Q/Z cmd - endif - endif - svar /sdfr=file_df /z pgroup - if (svar_Exists(pgroup)) - if (strlen(pgroup)>=1) - sprintf cmd, "PearlElog#set_panel_attributes(\"\", \"p-group=%s\")", pgroup - execute /Q/Z cmd - endif - endif - svar /sdfr=file_df /z proposal - if (svar_Exists(proposal)) - if (strlen(proposal)>=1) - sprintf cmd, "PearlElog#set_panel_attributes(\"\", \"project=%s\")", proposal - execute /Q/Z cmd - endif - endif - svar /sdfr=file_df /z proposer - svar /sdfr=file_df /z sample - if (svar_Exists(sample)) - if (strlen(sample)>=1) - sprintf cmd, "PearlElog#set_panel_attributes(\"\", \"sample=%s\")", sample - execute /Q/Z cmd - endif - endif - endif -end - // ====== preview ====== static function preview_file(filename) @@ -1475,6 +1394,12 @@ end // ====== panel ====== +/// macro to create the data explorer panel +/// +/// note: after editing the panel using the graphical editor, +/// remove all automatically generated assignments from the lb_contents listbox. +/// only the pos, size and keySelectCol properties should remain. +/// Window PearlDataExplorer() : Panel PauseUpdate; Silent 1 // building window... NewPanel /K=1 /W=(510,45,1190,539) as "PEARL Data Explorer" @@ -1486,6 +1411,7 @@ Window PearlDataExplorer() : Panel TitleBox tb_filepath,variable= root:packages:pearl_explorer:s_short_filepath,fixedSize=1 Button b_browse_filepath,pos={303.00,24.00},size={20.00,20.00},proc=PearlDataExplorer#bp_browse_filepath,title="..." Button b_browse_filepath,fColor=(65280,48896,32768) + Button b_browse_filepath,help={"select the file system folder that contains the data files"} GroupBox gb_prefs,pos={8.00,351.00},size={65.00,131.00},title="prefs" GroupBox gb_prefs,help={"explorer package preferences"} Button b_save_prefs,pos={21.00,394.00},size={38.00,17.00},proc=PearlDataExplorer#bp_save_prefs,title="save" @@ -1500,18 +1426,27 @@ Window PearlDataExplorer() : Panel ListBox lb_files,selWave=root:packages:pearl_explorer:wSelectedFiles,mode= 4 Button b_update_filelist,pos={246.00,315.00},size={76.00,22.00},proc=PearlDataExplorer#bp_update_filelist,title="update list" Button b_update_filelist,fColor=(65280,48896,32768) - CheckBox cb_file_preview,pos={78.00,318.00},size={59.00,14.00},title="preview" + Button b_update_filelist,help={"reload the file list"} + CheckBox cb_file_preview,pos={94.00,375.00},size={59.00,14.00},title="preview" CheckBox cb_file_preview,help={"enable/disable automatic preview window when selecting a data file"} CheckBox cb_file_preview,value= 0 + CheckBox cb_file_elog,pos={94.00,395.00},size={82.00,14.00},title="ELOG" + CheckBox cb_file_elog,help={"enable/disable sending metadata to ELOG panel when selecting a data file (does not submit to ELOG)"} + CheckBox cb_file_elog,value= 0 + Button b_attr_notebook,pos={94.00,415.00},size={64.00,22.00},disable=2,proc=PearlDataExplorer#bp_attr_notebook,title="notebook" + Button b_attr_notebook,help={"show a summary of attributes in a notebook window"} + Button b_attr_notebook,fColor=(65280,48896,32768) Button b_file_prev,pos={20.00,314.00},size={22.00,22.00},proc=PearlDataExplorer#bp_file_prev,title="\\W646" - Button b_file_prev,help={"previous file"},fColor=(65280,48896,32768) + Button b_file_prev,fColor=(65280,48896,32768) + Button b_file_prev,help={"select previous file from list"} Button b_file_next,pos={44.00,314.00},size={22.00,22.00},proc=PearlDataExplorer#bp_file_next,title="\\W649" - Button b_file_next,help={"next file"},fColor=(65280,48896,32768) + Button b_file_next,fColor=(65280,48896,32768) + Button b_file_next,help={"select next file from list"} Button b_goto_dataset,pos={355.00,315.00},size={64.00,22.00},disable=2,proc=PearlDataExplorer#bp_goto_dataset,title="goto DF" - Button b_goto_dataset,help={"change the current data folder ot where the selected dataset could be located"} + Button b_goto_dataset,help={"change the current data folder to find the selected dataset (if loaded)"} Button b_goto_dataset,fColor=(65280,48896,32768) Button b_display_dataset,pos={423.00,315.00},size={64.00,22.00},disable=2,proc=PearlDataExplorer#bp_display_dataset,title="display" - Button b_display_dataset,help={"display the selected dataset in its own window"} + Button b_display_dataset,help={"display the selected dataset"} Button b_display_dataset,fColor=(65280,48896,32768) Button b_load_complete,pos={355.00,451.00},size={92.00,22.00},disable=2,proc=PearlDataExplorer#bp_load_options,title="all data" Button b_load_complete,help={"load all datasets of the selected file."} @@ -1520,35 +1455,29 @@ Window PearlDataExplorer() : Panel TitleBox tb_selected_file,pos={360.00,28.00},size={309.00,22.00},frame=0 TitleBox tb_selected_file,variable= root:packages:pearl_explorer:s_selected_file,fixedSize=1 GroupBox gb_contents,pos={346.00,55.00},size={327.00,294.00},title="datasets" - Button b_attr_notebook,pos={97.00,375.00},size={64.00,22.00},disable=2,proc=PearlDataExplorer#bp_attr_notebook,title="notebook" - Button b_attr_notebook,help={"show a summary of attributes in a notebook window"} - Button b_attr_notebook,fColor=(65280,48896,32768) ListBox lb_contents,pos={355.00,84.00},size={305.00,222.00} ListBox lb_contents,keySelectCol= 1 GroupBox gb_selected_file,pos={346.00,4.00},size={328.00,48.00},title="selected file" Button b_load_region,pos={355.00,426.00},size={92.00,22.00},disable=2,proc=PearlDataExplorer#bp_load_options,title="region" - Button b_load_region,help={"load the selected region"} + Button b_load_region,help={"load significant datasets and metadata from the selected region"} Button b_load_region,userdata= "mode:load_region;",fColor=(65280,48896,32768) PopupMenu popup_reduction,pos={366.00,391.00},size={200.00,17.00},bodyWidth=200,proc=PearlDataExplorer#pmp_reduction_func - PopupMenu popup_reduction,help={"data reduction of 3d ScientaImage. note: the list may contain unsuitable functions. check the code or documentation!"} + PopupMenu popup_reduction,help={"data reduction function for 3d ScientaImage. note: the list may contain unsuitable functions. check the code or documentation!"} PopupMenu popup_reduction,mode=1,popvalue="None",value= #"PearlDataExplorer#pm_reduction_values()" GroupBox group_import,pos={346.00,351.00},size={326.00,131.00},title="import" Button b_load_scan,pos={450.00,426.00},size={94.00,22.00},disable=2,proc=PearlDataExplorer#bp_load_options,title="scan" - Button b_load_scan,help={"load the selected scan"},userdata= "mode:load_scan;" + Button b_load_scan,help={"load significant datasets and metadata from the selected scan"},userdata= "mode:load_scan;" Button b_load_scan,fColor=(65280,48896,32768) Button b_load_diags,pos={450.00,451.00},size={94.00,22.00},disable=2,proc=PearlDataExplorer#bp_load_options,title="diagnostics" - Button b_load_diags,help={"load diagnostics of selected scans"},userdata= "mode:load_diags;" - Button b_load_diags,fColor=(65280,48896,32768) + Button b_load_diags,help={"load diagnostic datasets of the selected scans"} + Button b_load_diags,userdata= "mode:load_diags;",fColor=(65280,48896,32768) Button b_load_dataset,pos={547.00,426.00},size={101.00,22.00},disable=2,proc=PearlDataExplorer#bp_load_options,title="dataset" - Button b_load_dataset,help={"load the selected datasets"} + Button b_load_dataset,help={"load the selected dataset(s) and significant metadata"} Button b_load_dataset,userdata= "mode:load_dataset;",fColor=(65280,48896,32768) Button b_reduction_params,pos={571.00,390.00},size={71.00,19.00},disable=2,proc=PearlDataExplorer#bp_reduction_params,title="set params" Button b_reduction_params,help={"set data reduction parameters"} Button b_reduction_params,fColor=(65280,48896,32768) GroupBox g_fileinfo,pos={85.00,351.00},size={251.00,131.00},title="file info" - Button b_elog,pos={97.00,401.00},size={64.00,22.00},disable=2,proc=PearlDataExplorer#bp_elog,title="ELOG" - Button b_elog,help={"send file metadata to ELOG panel (does not submit to ELOG)"} - Button b_elog,fColor=(65280,48896,32768) ToolsGrid grid=(0,28.35,5) EndMacro @@ -1591,8 +1520,6 @@ static function update_controls() dis = file_selected && scan_selected ? 0 : 2 Button b_attr_notebook win=PearlDataExplorer,disable=dis - dis = file_selected && (strlen(WinList("*ElogPanel*", ";", "WIN:64")) > 1) ? 0 : 2 - Button b_elog win=PearlDataExplorer,disable=dis dis = scan_selected ? 0 : 2 Button b_load_scan win=PearlDataExplorer,disable=dis dis = region_selected ? 0 : 2 @@ -1782,24 +1709,34 @@ End /// /// - load metadata /// - load preview if requested +/// - send to elog panel if requested /// /// @param file name of selected file +/// /// @param do_preview enable/disable loading of preview data /// non-zero: load preview, /// zero: don't load preview /// -static function selected_file(file, do_preview) +/// @param do_elog enable/disable sending metadata to elog panel +/// non-zero: send, +/// zero: don't send +/// +static function selected_file(file, do_preview, do_elog) string file variable do_preview + variable do_elog dfref save_df = GetDataFolderDFR() setdatafolder $package_path svar s_selected_file s_selected_file = file - get_file_info(file) - if (do_preview) + variable fi = get_file_info(file) + if (fi == 0 && do_preview != 0) preview_file(file) endif + if (fi == 0 && do_elog != 0) + send_to_elog() + endif update_controls() setdatafolder save_df @@ -1825,7 +1762,10 @@ static function bp_file_next(ba) : ButtonControl if (v_value >= 0) variable ifile = v_value ControlInfo /W=PearlDataExplorer cb_file_preview - selected_file(wtFiles[ifile], v_value) + variable do_preview = v_value + ControlInfo /W=PearlDataExplorer cb_file_elog + variable do_elog = v_value + selected_file(wtFiles[ifile], do_preview, do_elog) endif update_controls() break @@ -1856,7 +1796,10 @@ static function bp_file_prev(ba) : ButtonControl if (v_value >= 0) variable ifile = v_value ControlInfo /W=PearlDataExplorer cb_file_preview - selected_file(wtFiles[ifile], v_value) + variable do_preview = v_value + ControlInfo /W=PearlDataExplorer cb_file_elog + variable do_elog = v_value + selected_file(wtFiles[ifile], do_preview, do_elog) endif update_controls() break @@ -1887,9 +1830,12 @@ static function lbp_filelist(lba) : ListBoxControl if (selWave[row]) if (sum(wSelectedFiles) == 1) ControlInfo /W=PearlDataExplorer cb_file_preview - selected_file(listWave[row], v_value) + variable do_preview = v_value + ControlInfo /W=PearlDataExplorer cb_file_elog + variable do_elog = v_value + selected_file(listWave[row], do_preview, do_elog) else - selected_file(listWave[row], 0) + selected_file(listWave[row], 0, 0) endif endif update_controls() @@ -2285,6 +2231,24 @@ static function bp_display_dataset(ba) : ButtonControl return 0 End + +/// ******* +static function /s get_default_elog_module() + string modules = "PearlSciLog;PearlElog" + + variable imod + variable nmod = ItemsInList(modules, ";") + string smod + for (imod = 0; imod < nmod; imod += 1) + smod = StringFromList(imod, modules, ";") + if (exists(smod + "#set_panel_attributes") == 6) + return smod + endif + endfor + return "" +end + + /// send file metadata to the ELOG panel /// /// metadate is looked up in the following locations: @@ -2292,6 +2256,9 @@ End /// 2. file info folder inside package folder /// 3. package folder if it contains preview data from the selected file (???) /// +/// the data is sent to the first ElogPanel in the window list. +/// call open_pearl_elog() to ensure a new default panel. +/// static function send_to_elog() dfref save_df = GetDataFolderDFR() @@ -2338,27 +2305,123 @@ static function send_to_elog() graphname = "" endif + string module = get_default_elog_module() + funcref PearlDataExplorer_proto_get_panel_name f_get_panel_name = $(module + "#get_default_panel_name") + string panel = f_get_panel_name() + if (result == 0) - set_elog_attributes(data_df, filename=s_selected_file, graphname=graphname) - string windowname - windowname = StringFromList(0, WinList("*ElogPanel*", ";", "WIN:64"), ";") - DoWindow /F $windowname + set_elog_attributes(module, panel, data_df, filename=s_selected_file, graphname=graphname) endif setdatafolder save_df end -static function bp_elog(ba) : ButtonControl - STRUCT WMButtonAction &ba +function /s PearlDataExplorer_proto_get_panel_name() + return "" +end - switch( ba.eventCode ) - case 2: // mouse up - send_to_elog() - break - case -1: // control being killed - break - endswitch +function /s PearlDataExplorer_proto_set_panel_attributes(windowname, attributes, [clear]) + string windowname + string attributes + variable clear + return "" +end - return 0 -End +function /s PearlDataExplorer_proto_set_panel_graphs(windowname, graphs) + string windowname + string graphs + return "" +end +/// send general metadata to ELOG panel - if available +/// +/// the function works with any electronic logbook that has the same interfaces as pearl-elog.ipf. +/// the set_panel_attributes and set_panel_graphs functions are required. +/// +/// the following metatdata are sent. +/// they must be present as strings in the specified data folder: +/// +/// | ELOG parameter | global string | function argument | +/// | --- | --- | --- | +/// | file | s_filepath | filename | +/// | graph attachment | | graphname | +/// | author | authors | | +/// | p-group | pgroup | | +/// | project | proposal | | +/// | sample | sample | | +/// +/// @param elog_module Igor module name of the electronic logbook, PearlElog or PearlSciLog. +/// +/// @param panel_name Window name of the logbook panel +/// +/// @param file_df data folder that contains the metadata. +/// +/// @param filename override file path read from s_filepath global string variable. +/// if neither is declared, the file name is reset to empty field. +/// +/// @param graphname select this graph window for attaching. +/// default: do not change the selection. +/// +static function set_elog_attributes(elog_module, panel_name, file_df, [filename, graphname]) + string elog_module + string panel_name + dfref file_df + string filename + string graphname + + if (ParamIsDefault(filename)) + svar /sdfr=file_df /z loaded_file=s_filepath + if (svar_Exists(loaded_file)) + filename = loaded_file + else + filename = "" + endif + endif + + if (ParamIsDefault(graphname)) + graphname = "" + endif + + string cmd + string attrib = "" + + if (exists(elog_module + "#set_panel_attributes") == 6) + funcref PearlDataExplorer_proto_set_panel_attributes f_set_attributes = $(elog_module + "#set_panel_attributes") + funcref PearlDataExplorer_proto_set_panel_graphs f_set_graphs = $(elog_module + "#set_panel_graphs") + + attrib = ReplaceStringByKey("file", attrib, ParseFilePath(0, filename, ":", 1, 0), ":", ";") + svar /sdfr=file_df /z authors + if (svar_Exists(authors)) + if (strlen(authors)>=1) + attrib = ReplaceStringByKey("author", attrib, authors, ":", ";") + endif + endif + svar /sdfr=file_df /z pgroup + if (svar_Exists(pgroup)) + if (strlen(pgroup)>=1) + attrib = ReplaceStringByKey("p-group", attrib, pgroup, ":", ";") + endif + endif + svar /sdfr=file_df /z proposal + if (svar_Exists(proposal)) + if (strlen(proposal)>=1) + attrib = ReplaceStringByKey("project", attrib, proposal, ":", ";") + endif + endif + svar /sdfr=file_df /z proposer + svar /sdfr=file_df /z sample + if (svar_Exists(sample)) + if (strlen(sample)>=1) + attrib = ReplaceStringByKey("sample", attrib, sample, ":", ";") + endif + endif + + if (strlen(attrib)>=3) + f_set_attributes(panel_name, attrib) + endif + + if ((strlen(graphname) > 0) && (WinType(graphname) == 1)) + f_set_graphs(panel_name, graphname) + endif + endif +end diff --git a/pearl/pearl-elog.ipf b/pearl/pearl-elog.ipf index e31d0bf..bdae468 100644 --- a/pearl/pearl-elog.ipf +++ b/pearl/pearl-elog.ipf @@ -1,11 +1,11 @@ #pragma TextEncoding = "UTF-8" #pragma rtGlobals=3 // Use modern global access method and strict wave access. -#pragma version = 1.50 +#pragma version = 2.2 #pragma IgorVersion = 6.36 #pragma ModuleName = PearlElog // author: matthias.muntwiler@psi.ch -// Copyright (c) 2013-20 Paul Scherrer Institut +// Copyright (c) 2013-25 Paul Scherrer Institut // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -145,8 +145,7 @@ static function AfterFileOpenHook(refNum,file,pathName,type,creator,kind) Variable refNum,kind String file,pathName,type,creator if( (kind >= 1) && (kind <= 2)) - init_package(clean=1) - load_prefs() + clear_package_data() endif return 0 end @@ -206,6 +205,30 @@ static function /df get_elog_df(name, category) endif end +/// delete all package data +/// +/// also kills any ELOG panels +/// +static function clear_package_data() + dfref savedf = getdatafolderdfr() + dfref df_root = get_elog_df("", kdfRoot) + + if (DataFolderRefStatus(df_root) == 1) + string wins = WinList("*ElogPanel*", ";", "WIN:64") + variable iwin + variable nwin = ItemsInList(wins, ";") + string swin + for (iwin = 0; iwin < nwin; iwin += 1) + swin = StringFromList(iwin, wins, ";") + KillWindow /Z $swin + endfor + KillDataFolder /Z df_root + endif + + setdatafolder savedf + return 0 +end + /// initialize the package data folder. /// /// the data folder is initialized with a default, local configuration without any logbooks. @@ -1406,6 +1429,7 @@ static function elog_panel_hook(s) svar /sdfr=df_volatile url url = format_url(logbook) update_attach_items(logbook) + update_buttons(s.winName, logbook) endif break case 6: // resize @@ -1427,6 +1451,32 @@ static function elog_panel_hook(s) return hookResult // 0 if nothing done, else 1 end +static function update_buttons(win_name, logbook) + string win_name + string logbook + dfref df = get_elog_df(logbook, kdfVolatile) + //string win_name = StringFromList(0, WinList(logbook + "ElogPanel*", ";", "WIN:64"), ";") + + variable logged = 0 + if (strlen(logbook) > 0) + svar /z /sdfr=df g_username=username + svar /z /sdfr=df g_password=password + if (svar_exists(g_username) && svar_exists(g_password)) + logged = strlen(g_username) > 0 && strlen(g_password) > 0 + endif + endif + + if (strlen(win_name) > 0) + if (logged) + Button b_login, win=$win_name, disable=3 + Button b_logout, win=$win_name, disable=0 + else + Button b_login, win=$win_name, disable=0 + Button b_logout, win=$win_name, disable=3 + endif + endif +end + static constant kAttachColSel = 0 static constant kAttachColTitle = 1 static constant kAttachColName = 2 @@ -1756,7 +1806,7 @@ end /// @param windowname window name of the ELOG panel /// if empty, use default name "PearlElogPanel" /// -/// @return list of attributes to in the format "key1=value1;key2=value2". +/// @return list of attributes to in the format `"key1:value1;key2:value2"`. /// static function /s get_panel_attributes(windowname) string windowname @@ -1781,11 +1831,11 @@ static function /s get_panel_attributes(windowname) ControlInfo /w=$windowname $control switch(v_flag) case 2: // checkbox - attributes = ReplaceNumberByKey(attribute, attributes, v_value, "=", ";") + attributes = ReplaceNumberByKey(attribute, attributes, v_value, ":", ";") break case 3: // popupmenu case 5: // setvariable - attributes = ReplaceStringByKey(attribute, attributes, s_value, "=", ";") + attributes = ReplaceStringByKey(attribute, attributes, s_value, ":", ";") break endswitch endif @@ -1799,7 +1849,7 @@ end /// @param windowname window name of the ELOG panel /// if empty, use default name "PearlElogPanel" /// -/// @param attributes list of attributes to set (format "key1=value1;key2=value2") +/// @param attributes list of attributes to set (format "key1:value1;key2:value2") /// /// @param clear what to do if a key is missing in attributes? /// @arg 0 (default) leave the field unchanged @@ -1839,12 +1889,12 @@ static function /s set_panel_attributes(windowname, attributes, [clear]) control = StringFromList(ico, controls, ";") attribute = GetUserData(windowname, control, "attribute") if (strlen(attribute)) - value = StringByKey(attribute, attributes, "=", ";") + value = StringByKey(attribute, attributes, ":", ";") if (strlen(value) || clear) ControlInfo /w=$windowname $control switch(v_flag) case 2: // checkbox - numval = NumberByKey(attribute, attributes, "=", ";") + numval = NumberByKey(attribute, attributes, ":", ";") if ((numtype(numval) != 0) && clear) numval = 0 endif @@ -1853,7 +1903,7 @@ static function /s set_panel_attributes(windowname, attributes, [clear]) endif break case 3: // popupmenu - options_path = persistent_path + StringByKey(attribute, options, "=", ";") + options_path = persistent_path + StringByKey(attribute, options, ":", ";") svar values = $options_path numval = WhichListItem(value, values, ";") + 1 if (numval >= 1) diff --git a/pearl/pearl-menu.ipf b/pearl/pearl-menu.ipf index 998e5e5..a31d283 100644 --- a/pearl/pearl-menu.ipf +++ b/pearl/pearl-menu.ipf @@ -1,13 +1,13 @@ #pragma TextEncoding = "UTF-8" #pragma rtGlobals=1 // Use modern global access method. #pragma ModuleName = PearlMenu -#pragma version = 1.02 +#pragma version = 1.03 // main menu for PEARL data acquisition and analysis packages // $Id$ // author: matthias.muntwiler@psi.ch -// Copyright (c) 2013-14 Paul Scherrer Institut +// Copyright (c) 2013-25 Paul Scherrer Institut // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,6 +65,8 @@ menu "PEARL" submenu "Services" PearlMenuEnableFunc("pearl_elog") + "Open ELOG Panel", /Q, pearl_elog("") help = {"Open an ELOG panel to send entries to an ELOG logbook"} + PearlMenuEnableFunc("pearl_scilog") + "Open SciLog Panel", /Q, pearl_scilog("") + help = {"Open a panel to send entries to a SciLog logbook"} end submenu "Sample Preparation" diff --git a/pearl/pearl-pmsco-import.ipf b/pearl/pearl-pmsco-import.ipf index 92d9f67..398623e 100644 --- a/pearl/pearl-pmsco-import.ipf +++ b/pearl/pearl-pmsco-import.ipf @@ -3,7 +3,7 @@ #pragma IgorVersion = 6.2 #pragma ModuleName = PearlPmscoImport -// copyright (c) 2018 Paul Scherrer Institut +// copyright (c) 2018-25 Paul Scherrer Institut // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -326,3 +326,30 @@ function /s pmsco_load_xyz(pathname, filename) wave /t at at = unpadstring(at, 32) end + +function pmsco_load_alm(pathname, filename, wname) + string pathname + string filename + string wname + + variable fid + if (strlen(pathname) > 0) + HDF5OpenFile /P=pathname /R /Z fid as filename + else + HDF5OpenFile /R /Z fid as filename + endif + + if (v_flag == 0) + HDF5LoadData /N=$wname /O /Q /Z fid, "alm" + HDF5CloseFile fid + + wave alm_r = $(wname + "_r") + wave alm_i = $(wname + "_i") + make /c /n=(dimsize(alm_r, 0), dimsize(alm_r, 1)) /o $wname + wave /c alm = $wname + alm = cmplx(alm_r, alm_i) + variable lmax = (dimsize(alm, 0) - 1) * 2 + setscale /i x 0, lmax, "l", alm, alm_r, alm_i + setscale /i y -lmax, lmax, "m", alm, alm_r, alm_i + endif +end diff --git a/pearl/pearl-pshell-import.ipf b/pearl/pearl-pshell-import.ipf index 68fa39e..965bfbd 100644 --- a/pearl/pearl-pshell-import.ipf +++ b/pearl/pearl-pshell-import.ipf @@ -2,7 +2,7 @@ #pragma rtGlobals=3 // Use modern global access method and strict wave access. #pragma IgorVersion = 8.00 #pragma ModuleName = PearlPShellImport -#pragma version = 2.1 +#pragma version = 2.2 #if IgorVersion() < 9.00 #include #endif @@ -10,7 +10,7 @@ #include "pearl-gui-tools" #include "pearl-area-import" -// copyright (c) 2013-22 Paul Scherrer Institut +// copyright (c) 2013-25 Paul Scherrer Institut // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ strconstant kDataDimLabel = "data" strconstant kPreviewDatasets = "ImageEnergyDistribution;ScientaSpectrum;ScientaImage;Counts;SampleCurrent;" /// List of datasets that must be loaded to determine the axis scaling of a Scienta image -strconstant kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChannelEnd;ScientaSliceBegin;ScientaSliceEnd;Eph;" +strconstant kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChannelEnd;ScientaSliceBegin;ScientaSliceEnd;Eph;ScientaHighEnergy;ScientaHighThetaX;ScientaLowEnergy;ScientaLowThetaX;" /// List of diagnostic datasets that are normally loaded with a scan strconstant kEssentialDiagnostics = "ManipulatorX;ManipulatorY;ManipulatorZ;ManipulatorTheta;ManipulatorTilt;ManipulatorPhi;MonoEnergy;" @@ -2202,6 +2202,10 @@ end /// where the first folder in the list takes precedence. /// it may fall back to more or less reasonable default values if no data is not found. /// @arg `LensMode` +/// @arg `ScientaLowEnergy` +/// @arg `ScientaHighEnergy` +/// @arg `ScientaLowThetaX` +/// @arg `ScientaHighThetaX` /// @arg `ScientaChannelBegin` /// @arg `ScientaChannelEnd` /// @arg `ScientaSliceBegin` @@ -2260,6 +2264,10 @@ function ps_detect_scale(data_df, ax, lo, hi, un) ax[%$kDataDimLabel] = "value" wave /T /Z LensMode = ps_find_scale_wave("LensMode", data_df, scan_df, attr_df) + wave /Z LowEnergy = ps_find_scale_wave("ScientaLowEnergy", data_df, scan_df, attr_df) + wave /Z HighEnergy = ps_find_scale_wave("ScientaHighEnergy", data_df, scan_df, attr_df) + wave /Z LowThetaX = ps_find_scale_wave("ScientaLowThetaX", data_df, scan_df, attr_df) + wave /Z HighThetaX = ps_find_scale_wave("ScientaHighThetaX", data_df, scan_df, attr_df) wave /Z ChannelBegin = ps_find_scale_wave("ScientaChannelBegin", data_df, scan_df, attr_df) wave /Z ChannelEnd = ps_find_scale_wave("ScientaChannelEnd", data_df, scan_df, attr_df) wave /Z SliceBegin = ps_find_scale_wave("ScientaSliceBegin", data_df, scan_df, attr_df) @@ -2268,35 +2276,30 @@ function ps_detect_scale(data_df, ax, lo, hi, un) // lens mode can give more detail if (waveexists(LensMode) && (numpnts(LensMode) >= 1)) - strswitch(LensMode[0]) - case "Angular45": - lo[%$kAngleDimLabel] = -45/2 - hi[%$kAngleDimLabel] = +45/2 - un[%$kAngleDimLabel] = "°" - ax[%$kAngleDimLabel] = "angle" - break - case "Angular60": - lo[%$kAngleDimLabel] = -60/2 - hi[%$kAngleDimLabel] = +60/2 - un[%$kAngleDimLabel] = "°" - ax[%$kAngleDimLabel] = "angle" - break - case "Transmission": - un[%$kAngleDimLabel] = "arb." - ax[%$kAngleDimLabel] = "offset" - break - endswitch + if (stringmatch(LensMode[0], "*Transmission*")) + un[%$kAngleDimLabel] = "mm" + ax[%$kAngleDimLabel] = "position" + else + un[%$kAngleDimLabel] = "°" + ax[%$kAngleDimLabel] = "angle" + endif endif // best option if scales are explicit in separate waves - if (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1)) + if (waveexists(LowEnergy) && waveexists(HighEnergy) && (numpnts(LowEnergy) >= 1) && (numpnts(HighEnergy) >= 1)) + lo[%$kEnergyDimLabel] = LowEnergy[0] + hi[%$kEnergyDimLabel] = HighEnergy[0] + elseif (waveexists(ChannelBegin) && waveexists(ChannelEnd) && (numpnts(ChannelBegin) >= 1) && (numpnts(ChannelEnd) >= 1)) lo[%$kEnergyDimLabel] = ChannelBegin[0] hi[%$kEnergyDimLabel] = ChannelEnd[0] elseif (waveexists(ScientaChannels) && (numpnts(ScientaChannels) >= 1)) lo[%$kEnergyDimLabel] = ScientaChannels[0] hi[%$kEnergyDimLabel] = ScientaChannels[numpnts(ScientaChannels)-1] endif - if (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1)) + if (waveexists(LowThetaX) && waveexists(HighThetaX) && (numpnts(LowThetaX) >= 1) && (numpnts(HighThetaX) >= 1)) + lo[%$kAngleDimLabel] = LowThetaX[0] + hi[%$kAngleDimLabel] = HighThetaX[0] + elseif (waveexists(SliceBegin) && waveexists(SliceEnd) && (numpnts(SliceBegin) >= 1) && (numpnts(SliceEnd) >= 1)) lo[%$kAngleDimLabel] = SliceBegin[0] hi[%$kAngleDimLabel] = SliceEnd[0] endif diff --git a/pearl/pearl-scilog.ipf b/pearl/pearl-scilog.ipf new file mode 100644 index 0000000..541a690 --- /dev/null +++ b/pearl/pearl-scilog.ipf @@ -0,0 +1,1739 @@ +#pragma TextEncoding = "UTF-8" +#pragma rtGlobals=3 // Use modern global access method and strict wave access. +#pragma version = 2.2 +#pragma IgorVersion = 8.0 +#pragma ModuleName = PearlSciLog + +// author: matthias.muntwiler@psi.ch +// Copyright (c) 2025 Paul Scherrer Institut + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 + +/// @file +/// @brief Interface for writing SciLog entries with Igor graphs as attachment. +/// @ingroup ArpesPackage +/// +/// +/// The functions in this module support the following SciLog features: +/// - Submit new entries. +/// - Add text message. +/// - Add tags. Drop-down lists with common tags are provided. +/// - Attach Igor graphs. +/// - Not specific to the configuration at PEARL. +/// PEARL code is concentrated in the scilog_init_pearl_templates() function. +/// - The configuration of the server and logbooks +/// as well as the most recently used attributes are persisted in the preference file. +/// +/// Setup: +/// 1. If there is no data folder `root:packages:pearl_scilog` yet: +/// Call `PearlScilog#init_package()`. +/// 2. The administrator of the SciLog server creates logbook templates +/// according to the configuration of the logbooks. +/// The templates are written in Igor code. +/// For PEARL, this is done in the scilog_init_pearl_templates() function. +/// 3. Install uv (https://docs.astral.sh/uv/getting-started/installation/). +/// Alternatively: Setup an environment containing Python and the pillow and scilog packages from PyPI. +/// 4. Adjust the root:packages:pearl_scilog:persistent:python_activate and python_command global strings. +/// By default, they are set for uv. In this case you can skip this step. +/// +/// The first string must contain the command to activate the environment in a batch file, +/// for example: `call c:\programdata\miniforge3\scripts\activate.bat scilog`. +/// The second must be an executable that can be followed by a python script and arguments, +/// for example: `python.exe` or `uv run --script`. +/// 5. Save the user name and password in a credentials file. +/// +/// Usage: +/// 1. The user opens logbooks via the _Open SciLog panel_ menu item or calling pearl_scilog(). +/// Before first use, select a template and enter a name for the logbook. +/// The new logbook is written to the preference file, +/// and can afterwards be opened directly. +/// The name of the logbook is not connected to the name in SciLog. +/// 2. Enter the name of the SciLog logbook in the logbook edit field. +/// 3. Enter the p-group of the SciLog logbook in the pgroup edit field. +/// 4. Edit the message, choose tags and attachments as necessary, and submit to SciLog. +/// Each of the fields on the left, adds a tag. +/// Be sure to use memorable tags (for searching the logbook) and avoid spaces and commas. +/// +/// +/// @author matthias muntwiler, matthias.muntwiler@psi.ch +/// +/// @copyright 2025 Paul Scherrer Institut @n +/// Licensed under the Apache License, Version 2.0 (the "License"); @n +/// you may not use this file except in compliance with the License. @n +/// You may obtain a copy of the License at +/// http://www.apache.org/licenses/LICENSE-2.0 + +/// @namespace PearlSciLog +/// @brief interface for writing SciLog entries with Igor graphs as attachment. +/// +/// PearlSciLog is declared in @ref pearl-scilog.ipf. + +static strconstant package_name = "pearl_scilog" +static strconstant package_path = "root:packages:pearl_scilog:" + +static constant kdfRoot = 0 +static constant kdfVolatile = 1 +static constant kdfPersistent = 2 +static constant kdfTemplates = 3 + +/// main function to initialize and open a SciLog panel. +/// +/// this function takes care of all necessary initialization, configuration, and preferences. +/// if a panel exists, it will be moved to the front. +/// +/// @param logbook name of the logbook +/// if empty, the user is prompted to select or create a logbook by scilog_prompt_logbook(). +/// +/// +function pearl_scilog(logbook) + string logbook + + if (init_package() == 0) + load_prefs() + string templates = list_logbooks(templates=1) + if (ItemsInList(templates) < 1) + scilog_init_pearl_templates() + endif + endif + + if (strlen(logbook) > 0) + dfref df = get_scilog_df(logbook, kdfPersistent) + if (DataFolderRefStatus(df) != 1) + print "Configuration of logbook", logbook, "not found. Please create a new one." + logbook = "" + endif + endif + + if (strlen(logbook) == 0) + logbook = scilog_prompt_logbook() + endif + + string win_name = CleanupName(logbook, 0) + "SciLogPanel" + if (strlen(logbook) > 0) + string wins = WinList(win_name + "*", ";", "WIN:64") + if (strlen(wins) > 0) + win_name = StringFromList(0, wins, ";") + DoWindow /F $win_name + else + win_name = PearlSciLogPanel(logbook, win_name) + STRUCT WMWinHookStruct s + s.eventCode = 0 + s.winName = win_name + scilog_panel_hook(s) + endif + endif +end + +/// save preferences and recent values before Igor opens a new experiment. +static function IgorBeforeNewHook(igorApplicationNameStr) + string igorApplicationNameStr + save_prefs() + cleanup_temp_files() + return 0 +end + +/// save preferences and recent values before Igor quits. +static function IgorQuitHook(igorApplicationNameStr) + string igorApplicationNameStr + save_prefs() + cleanup_temp_files() + return 0 +end + +/// initialize the package and reload preferences after an experiment is loaded. +static function AfterFileOpenHook(refNum,file,pathName,type,creator,kind) + Variable refNum,kind + String file,pathName,type,creator + if( (kind >= 1) && (kind <= 2)) + clear_package_data() + endif + return 0 +end + +/// get the package, logbook, or template datafolder. +/// +/// @param name name of logbook or template, or empty string for respective parent folder. +/// +/// @param category parameter category: +/// @arg kdfRoot package root +/// @arg kdfVolatile volatile +/// @arg kdfPersistent persistent +/// @arg kdfTemplates template +/// +/// @returns data folder reference +/// +static function /df get_scilog_df(name, category) + string name + variable category + + dfref df_package = $package_path + dfref df_persistent = df_package:persistent + dfref df_volatile = df_package:volatile + + switch(category) + case kdfRoot: + dfref df_parent = df_package + break + case kdfPersistent: + dfref df_parent = df_persistent + break + case kdfTemplates: + dfref df_parent = df_persistent:templates + break + case kdfVolatile: + dfref df_parent = df_volatile + break + default: + Abort "get_scilog_df: undefined data folder category." + endswitch + + if ((strlen(name) > 0) && (category >= 1)) + if (category == kdfTemplates) + dfref df_logbooks = df_parent + else + dfref df_logbooks = df_parent:logbooks + endif + dfref df_logbook = df_logbooks:$name + return df_logbook + else + return df_parent + endif +end + +/// delete all package data +/// +/// also kills any SciLog panels +/// +static function clear_package_data() + dfref savedf = getdatafolderdfr() + dfref df_root = get_scilog_df("", kdfRoot) + + if (DataFolderRefStatus(df_root) == 1) + string wins = WinList("*SciLogPanel*", ";", "WIN:64") + variable iwin + variable nwin = ItemsInList(wins, ";") + string swin + for (iwin = 0; iwin < nwin; iwin += 1) + swin = StringFromList(iwin, wins, ";") + KillWindow /Z $swin + endfor + KillDataFolder /Z df_root + endif + + setdatafolder savedf + return 0 +end + +/// initialize the package data folder. +/// +/// the data folder is initialized with a default, local configuration without any logbooks. +/// the server configuration should be set in the preferences. +/// +/// @param clean decides what to do if the package configuration exists. +/// @arg 0 (default) keep existing configuration. +/// @arg 1 overwrite existing configuration. +/// +static function init_package([clean]) + variable clean + + if (ParamIsDefault(clean)) + clean = 0 + endif + + dfref savedf = getdatafolderdfr() + dfref df_root = get_scilog_df("", kdfRoot) + if ((clean == 0) && (DataFolderRefStatus(df_root) == 1)) + return 1 + endif + + setdatafolder root: + newdatafolder /o/s packages + newdatafolder /o/s $package_name + dfref df_package_root = getdatafolderdfr() + newdatafolder /o/s volatile + dfref df_volatile = getdatafolderdfr() + newdatafolder /o logbooks + setdatafolder df_package_root + newdatafolder /o/s persistent + dfref df_persistent = getdatafolderdfr() + newdatafolder /o logbooks + newdatafolder /o templates + + // common configuration + setdatafolder df_persistent + // batch commands to activate the python environment and to run python. + // adjust the global strings manually after initialization! + string /g python_activate = "echo calling uv" + string /g python_command = "uv run --script" + variable /g loglevel = 4 + + setdatafolder savedf + return 0 +end + +/// setup PEARL template logbooks. +/// +/// template logbooks for PEARL. +/// +/// @remark this function is specific to the setup at PEARL. +/// +function scilog_init_pearl_templates() + dfref savedf = getdatafolderdfr() + + dfref df_root = get_scilog_df("", kdfRoot) + dfref df_persistent = get_scilog_df("", kdfPersistent) + dfref df_templates = get_scilog_df("", kdfTemplates) + + // Experiments template + setdatafolder df_templates + newdatafolder /o/s Experiments + + // attributes (persistent) + // available attributes + string /g attributes = "author;pgroup;logbook;project;sample;file;system;source;task;method;technical;program;revision;machine;job;tags;valid;" + // controls corresponding to attributes + // prefix determines the control type: sv_ = setvariable (string), pm_ = popup menu, cb = check box + string /g controls = "author:sv_author;pgroup:sv_pgroup;logbook:sv_logbook;project:sv_project;sample:sv_sample;file:sv_file;system:pm_system;source:pm_source;task:pm_task;method:pm_method;technical:pm_technical;program:sv_program;revision:sv_revision;machine:sv_machine;job:sv_job;tags:sv_tags;valid:cb_valid;" + // attributes with fixed options, value item declares the options string + string /g options = "source=sources;task=tasks;method=methods;technical=technicals;system=systems;" + // attributes which must be defined + string /g required_attributes = "author;pgroup;logbook;" + // help strings + string /g helps = "author:e-mail address of author (required);" + helps += "pgroup:p-group (required);" + helps += "logbook:title of existing scilog logbook (required);" + helps += "project:proposal number or (short) project name (optional), avoid spaces and commas;" + helps += "sample:short sample name (optional), avoid spaces and commas;" + helps += "file:name of original data file (optional), avoid spaces and commas, do not include path;" + helps += "system:instrument part, choose Other to suppress the tag;" + helps += "source:data source, choose Other to suppress the tag;" + helps += "task:purpose of the measurement, choose Other to suppress the tag;" + helps += "method:measurement method or technique, choose Other to suppress the tag;" + helps += "technical:technical procedure, choose Other to suppress the tag;" + helps += "tags:other comma-separated tags (optional), avoid spaces;" + helps += "program:program code for calculations (optional), avoid spaces and commas;" + helps += "revision:code revision for calculations, e.g. git hash or tag (optional), avoid spaces and commas;" + helps += "machine:computing host name for calculations (optional), avoid spaces and commas;" + helps += "job:calculation job name (optional), avoid spaces and commas;" + helps += "valid:uncheck if data is unusable, incomplete or obviously wrong;" + + // option lists + string /g sources = "PShell;Scienta;SScan;Prosilica;OTF;EPICS;LEED;QMS;Matrix;Nanonis;Igor Pro;Other" + string /g tasks = "Measurement;Optimization;Analysis;Sample Preparation;Sample Storage;Comment;Development;Maintenance;Test;Other" + string /g methods = "XPS;XPD;UPS;XAS;XMCD;PED;ARPES;STM;STS;LEED;AES;QMS;MBE;Sputter/Anneal;Test;Other" + string /g technicals = "Installation;Repair;Maintenance;Test;Commissioning;Bakeout;Incident;Cool-down;Warm-up;Storage;Other" + string /g systems = "Vacuum;Control System;BL;XA;XP;SA;SP;T;LL;Monochromator;Carving;Scienta;STM;PC-Scienta;PC-Matrix;PC-Console;PC-Console-Win;PC-XP;EPS;LAC;Desiccator;Other" + + setdatafolder savedf + return 0 +end + +/// initialize volatile variables. +/// +/// create and initialize all volatile variables for the configured notebooks. +/// values of existing variables are not changed. +/// +/// this function must be called after new logbooks have been configured, +/// specifically by scilog_create_logbook() and load_prefs(). +/// +static function init_volatile_vars() + dfref savedf = GetDataFolderDFR() + + dfref df_volatile_root = get_scilog_df("", kdfVolatile) + dfref df_volatile_parent = df_volatile_root:logbooks + + string logbooks = list_logbooks() + string logbook + variable nlb = ItemsInList(logbooks) + variable ilb + + SetDataFolder df_volatile_root + if (exists("temp_graph_files") != 2) + string /g temp_graph_files = "" + endif + + for (ilb = 0; ilb < nlb; ilb += 1) + logbook = StringFromList(ilb, logbooks) + + SetDataFolder df_volatile_parent + if (DataFolderExists(logbook)) + SetDataFolder $logbook + else + NewDataFolder /o/s $logbook + endif + + if (exists("att_list") != 1) + make /n=(0,3) /t /o attach_list + make /n=(0,3) /i /o attach_sel + endif + if (exists("url") != 2) + string /g url = "" + endif + endfor + + SetDataFolder savedf + return 0 +end + +/// @var persistent:loglevel +/// @brief filter history messages by log level. +/// +/// messages are printed to the history only if their level exceeds this setting. +/// this is a global variable in the "persistent" folder of the package configuration. +/// +/// @arg 0 none. do not print any messages. no messages have this level. +/// @arg 1 critical. severe error with possible data corruption. +/// @arg 2 error. a function did not complete successfully. +/// @arg 3 warning. everything worked fine, but some attention of the user is required. +/// @arg 4 info. status message which may be useful to the normal user. +/// @arg 5 debug. status message which may be useful to the developer. +/// + +/// create a new logbook. +/// +/// create a new empty logbook or duplicate from a template. +/// +/// @param name name of the new logbook. +/// if the logbook exists, the existing logbook folder is killed +/// and replaced by a new one. +/// this may fail if a window is still open. +/// +/// @param template name of the template. +/// if empty string, a new empty logbook is created. +/// +function scilog_create_logbook(name, [script_path, template]) + string name + string script_path + string template + + if (ParamIsDefault(script_path)) + script_path = "" + endif + if (ParamIsDefault(template)) + template = "" + endif + + dfref savedf = getdatafolderdfr() + dfref df_root = get_scilog_df("", kdfRoot) + dfref df_persistent_root = get_scilog_df("", kdfPersistent) + dfref df_persistent_parent = df_persistent_root:logbooks + dfref df_volatile_root = get_scilog_df("", kdfVolatile) + dfref df_volatile_parent = df_volatile_root:logbooks + + setdatafolder df_persistent_parent + if (CheckName(name, 11) != 0) + setdatafolder savedf + Abort "invalid logbook name" + return -1 + endif + + if (strlen(template) > 0) + dfref df_template = get_scilog_df(template, kdfTemplates) + dfref df_existing = get_scilog_df(name, kdfPersistent) + if (DataFolderRefStatus(df_existing)) + KillDataFolder /Z df_existing + endif + DuplicateDataFolder df_template, df_persistent_parent:$name + else + NewDataFolder /o/s df_persistent_parent:$name + + // SciLog logbook name + string /g logbook = name + // SciLog script path + string /g scilog_path = script_path + // attributes (persistent) + // available attributes + string /g attributes = "" + // controls corresponding to attributes + // prefix determines the control type: sv_ = setvariable (string), pm_ = popup menu, cb = check box + string /g controls = "" + // attributes with fixed options, value item declares the options string + string /g options = "" + // attributes which must be defined + string /g required_attributes = "" + endif + + // usage data (persistent) + setdatafolder get_scilog_df(name, kdfPersistent) + string /g recent = "" + string /g recent_message = "" + + init_volatile_vars() + + setdatafolder savedf + return 0 +end + +/// save persistent package data to the preferences file. +/// +/// saves everything under the persistent folder of the package. +/// +static function save_prefs() + dfref saveDF = GetDataFolderDFR() + + dfref df = get_scilog_df("", kdfPersistent) + if (DataFolderRefStatus(df) == 1) + string fullPath = SpecialDirPath("Packages", 0, 0, 0) + fullPath += package_name + NewPath/O/C/Q tempPackagePrefsPath, fullPath + fullPath += ":preferences.pxp" + SetDataFolder df + SaveData /O /Q /R fullPath + KillPath/Z tempPackagePrefsPath + endif + + SetDataFolder saveDF +end + +/// load persistent package data from the preferences file. +/// +/// the preferences file is an Igor packed experiment file in a special preferences folder +static function load_prefs() + dfref saveDF = GetDataFolderDFR() + + variable result = -1 + init_package() + setdatafolder get_scilog_df("", kdfPersistent) + + string fullPath = SpecialDirPath("Packages", 0, 0, 0) + fullPath += package_name + + GetFileFolderInfo /Q /Z fullPath + if (V_Flag == 0) // Disk directory exists? + fullPath += ":preferences.pxp" + GetFileFolderInfo /Q /Z fullPath + if (V_Flag == 0) // Preference file exist? + LoadData /O /R /Q fullPath + init_volatile_vars() + result = 0 + endif + endif + + SetDataFolder saveDF + return result +end + +/// get a list of configured logbooks or templates. +/// +/// this is list of data folder names under persistent:logbooks (or persistent:templates). +/// the function does not check whether the folders contain valid data. +/// +/// @param templates select whether logbooks (0, default) or templates (1) are returned. +/// +/// @return semicolon-separated list of logbooks +/// +static function /s list_logbooks([templates]) + variable templates + + if (ParamIsDefault(templates)) + templates = 0 + endif + + dfref df_persistent = get_scilog_df("", kdfPersistent) + if (templates) + dfref df_logbooks = df_persistent:templates + else + dfref df_logbooks = df_persistent:logbooks + endif + string logbooks = "" + + variable nlb = CountObjectsDFR(df_logbooks, 4) + variable ilb + string slb + for (ilb = 0; ilb < nlb; ilb += 1) + slb = GetIndexedObjNameDFR(df_logbooks, 4, ilb) + if (strlen(slb) > 0) + logbooks = AddListItem(slb, logbooks) + endif + endfor + + return SortList(logbooks, ";", 16) +end + +/// validate attributes +/// +/// @returns 0 if all required attributes are present and enumerated items are correct. +/// non-zero if a violation is detected. +/// -1: invalid author +/// -2: invalid p-group +/// -3: empty message +/// -4: missing required attribute +/// +function scilog_validate_attributes(logbook, attributes, message) + + string logbook // name of the logbook (as in igor folder name) + string attributes // key=value list of attributes, semicolon separated + string message + + dfref df_logbook = get_scilog_df(logbook, kdfPersistent) + svar /sdfr=df_logbook required_attributes + variable result = 0 + + variable ii + variable nn = ItemsInList(required_attributes, ";") + string att, val + for (ii=0; ii < nn; ii+=1) + att = StringFromList(ii, required_attributes, ";") + val = StringByKey(att, attributes, ":", ";") + if (strlen(val) == 0) + print "SciLog: Missing required attribute", att + result = -4 + endif + endfor + + if (strlen(message) == 0) + print "SciLog: Empty message" + result = -3 + endif + + if (!GrepString(StringByKey("pgroup", attributes, ":", ";"), "p[1-9][0-9]{4}")) + print "SciLog: Invalid p-group" + result = -2 + endif + + if (StringMatch(StringByKey("author", attributes, ":", ";"), "!*@*.*")) + print "SciLog: Author must be an e-mail address" + result = -1 + endif + + return result +end + +/// create a new entry in SciLog +/// +/// this is the main function to create a new entry in a logbook. +/// +/// @param logbook name of the target logbook. +/// +/// @param attributes attributes list from get_panel_attributes. +/// +/// @param message plain text part of the entry. +/// +/// @param graphs names of graph windows to be added as attachments, semicolon separated. +/// +/// @return ID number of the new entry (> 0), or error code (< 0). +/// @arg -1: failed to save temporary message file. +/// @arg -2: invalid/missing command line. +/// @arg -3: invalid/missing attributes. +/// @arg -4: scilog returned error +/// +function scilog_create_entry(logbook, attributes, message, [graphs]) + string logbook + string attributes + string message + string graphs + + if (ParamIsDefault(graphs)) + graphs = "" + endif + + dfref savedf = getdatafolderdfr() + dfref df_general = get_scilog_df("", kdfPersistent) + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + + variable result = 0 + nvar /sdfr=df_general loglevel + + if (strlen(message) == 0) + return -1 + endif + if (strlen(attributes) == 0) + return -3 + endif + + message = scilog_format_message(logbook, attributes, message, graphs=graphs) + string graph_files = prepare_graph_attachments(graphs) + string attrs_file = create_attrs_file(logbook, attributes, graph_files) + string message_file = create_message_file(message) + string cmd_file_path = create_cmd_file(logbook, attrs_file, message_file) + string log_file_path = ReplaceString(".bat", cmd_file_path, ".log") + ExecuteScriptText cmd_file_path + string error = parse_result(log_file_path) + if (strlen(error) == 0) + print "Message sent successfully." + result = 0 + else + print "Error sending message:" + print error + result = -4 + endif + + setdatafolder savedf + return result +end + +/// format the message with special attributes +/// +/// adds a first line with important attributes: file and sample if present. +/// +/// @param logbook name of the target logbook. +/// +/// @param attributes attributes list from get_panel_attributes. +/// +/// @param message plain text part of the entry. +/// +/// @param graphs names of graph windows to be added as attachments, semicolon separated. +/// +/// @return formatted message +/// +function /s scilog_format_message(logbook, attributes, message, [graphs]) + string logbook + string attributes + string message + string graphs + + if (ParamIsDefault(graphs)) + graphs = "" + endif + + string file = StringByKey("file", attributes, ":", ";") + string sample = StringByKey("sample", attributes, ":", ";") + + string result = "" + string sep = "" + if (strlen(file) > 0) + result += "File: " + file + "\r\n" + sep = "\r\n" + endif + if (strlen(sample) > 0) + result += "Sample: " + sample + "\r\n" + sep = "\r\n" + endif + + result += sep + result += message + + setdatafolder savedf + return result +end + +/// create a batch file for the ingestion command +/// +/// the file contains commands to: +/// 1. activate the python environment (from the python_activate global string). +/// 2. call scilog-ingest.py with python. +/// +/// the file is written to the user's temporary directory and can be executed by ExecuteScriptText. +/// stderr and stdout of the python shell are captured in the scilog.log file. +/// +/// @param logbook name of the target logbook +/// +static function /s create_cmd_file(logbook, attrs_file, message_file) + string logbook + string attrs_file + string message_file + + dfref df_general = get_scilog_df("", kdfPersistent) + svar /sdfr=df_general python_activate + svar /sdfr=df_general python_command + + string path = SpecialDirPath("Temporary", 0, 1, 0) + string log_file = path + "scilog.log" + + String ipf_path_igor = ParseFilePath(1, FunctionPath(""), ":", 1, 0) + String ipf_path_win = ParseFilePath(5, ipf_path_igor, "\\", 0, 0) + String bat_text + + bat_text = "cd \"" + ipf_path_win + "\"\r\n" + bat_text += python_activate + "\r\n" + bat_text += python_command + " scilog-ingest.py \"" + attrs_file + "\" \"" + message_file + "\"" + bat_text += " > \"" + log_file + "\" 2>&1\r\n" + + string filename + variable len = strlen(path) + if (numtype(len) == 0) + filename = "scilog.bat" + path += filename + variable f1 + Open/z f1 as path + fprintf f1, bat_text + Close f1 + else + filename = "" + endif + + return path +end + +/// prepare screenshots of graph windows for attachments +/// +/// prepares the attachment files from Igor graph windows. +/// +/// @param graphs names of graph windows to be added as attachments, semicolon separated +/// +static function /s prepare_graph_attachments(graphs) + string graphs // names of graph windows to be added as attachments, semicolon separated + + string cmd = "" + variable ngraphs = ItemsInList(graphs, ";") + variable igraph + string sgraph + string graph_path + for (igraph = 0; igraph < ngraphs; igraph += 1) + sgraph = StringFromList(igraph, graphs, ";") + graph_path = create_graph_file(sgraph, igraph) + if (strlen(graph_path) > 0) + cmd += graph_path + ";" + endif + endfor + + return cmd +end + +static function /s get_timestamp(sep) + string sep + Variable now = DateTime + string dat = ReplaceString("-", Secs2Date(DateTime, -2), "") + string tim = ReplaceString(":", Secs2Time(DateTime, 3), "") + return dat + sep + tim +end + +/// save the message to a temporary text file +/// +/// the file is saved to the Temporary directory returned by igor's SpecialDirPath function +/// under the file name "scilog_temp_message.txt". +/// the function returns the file path +/// +/// @param message text message to save to the file. +/// @return (string) path of the created file. +/// empty string if unsuccessful. +/// +/// +static function /s create_message_file(message) + string message + + string path = SpecialDirPath("Temporary", 0, 1, 0) + variable len = strlen(path) + string filename + + if (numtype(len) == 0) + filename = "scilog_temp_message.txt" + path += filename + variable f1 + Open f1 as path + fprintf f1, message + Close f1 + else + filename = "" + endif + + return path +end + +/// save a graph to a temporary graphics file +/// +/// the file is saved to the Temporary directory returned by igor's SpecialDirPath function. +/// the file name contains a time stamp and the specified file index to make it unique. +/// the function returns the name of the file (excluding path!) +/// +/// the full path is added to the temp_graph_files global list. +/// a hook function will delete the files listed there when igor quits. +/// +/// @param graphname object name of the graph to save. +/// @param fileindex incrememtal index of the file within one submission. +/// the file name is made unique by a time stamp and this file index. +/// submissions within the same second must have a unique file index. +/// @return (string) path of the created file. +/// empty string if unsuccessful. +/// +static function /s create_graph_file(graphname, fileindex) + string graphname + variable fileindex + + dfref df_volatile_root = get_scilog_df("", kdfVolatile) + svar /sdfr=df_volatile_root temp_graph_files + + string path = SpecialDirPath("Temporary", 0, 1, 0) + string ts = get_timestamp("_") + variable len = strlen(path) + string filename + + if (numtype(len) == 0) + filename = "scilog_" + ts + "_" + num2str(fileindex) + ".png" + path += filename + SavePICT /B=72 /E=-5 /M /O /W=(0,0,8,6) /WIN=$graphname /Z as path + if (v_flag == 0) + temp_graph_files = AddListItem(path, temp_graph_files, ";", inf) + else + filename = "" + endif + else + filename = "" + endif + + return path +end + +/// write the attributes to a file. +/// +/// +/// @param attributes attributes list from get_panel_attributes. + +static function /s create_attrs_file(logbook, attributes, attachments) + string logbook + string attributes + string attachments + + dfref df_general = get_scilog_df("", kdfPersistent) + nvar /sdfr=df_general loglevel + + variable i_, n_, j_, m_ + string s_ + string k_, v_ + + string work_path = SpecialDirPath("Temporary", 0, 1, 0) + variable len = strlen(work_path) + if (numtype(len) == 0) + string cmdx = "" + string cmd_path = work_path + "scilog_temp_attr.dat" + + variable f1 + Open f1 as cmd_path + + n_ = ItemsInList(attributes, ";") + for (i_=0; i_ 0) && (cmpstr(v_, "Other") != 0)) + cmdx = "tag:" + v_ + "\r\n" + else + cmdx = "" + endif + + if (strlen(v_) && strlen(cmdx)) + fprintf f1, cmdx + endif + endfor + + n_ = ItemsInList(attachments, ";") + for (i_=0; i_= 5) + print output + endif + if (success) + return "" + elseif (strlen(error) == 0) + return output + else + return error + endif +end + +/// prompt to open or create a logbook +/// +function /s scilog_prompt_logbook() + string logbooks = list_logbooks(templates=0) + logbooks = AddListItem("(new)", logbooks) + string templates = list_logbooks(templates=1) + templates = AddListItem("(none)", templates) + + string logbook = StringFromList(0, logbooks) + string template = StringFromList(0, logbooks) + string name = "" + + prompt logbook, "logbook", popup logbooks + prompt template, "template", popup templates + prompt name, "new logbook name" + + doprompt "select logbook", logbook, template, name + if (!v_flag) + if (cmpstr(logbook, "(new)") == 0) + scilog_create_logbook(name, template=template) + logbook = name + endif + else + logbook = "" + endif + return logbook +end + +/// open a new panel for submitting data to SciLog. +/// +/// this function creates only the panel but not the necessary data folders. +/// call init_package() and load_prefs() once before creating panels. +/// +/// @param logbook name of the target logbook +/// @param win_name desired window name. igor may modify it to make it unique. +/// @return actual window name +/// +function /s PearlSciLogPanel(logbook, win_name) + string logbook + string win_name + + dfref savedf = getdatafolderdfr() + dfref df_general = get_scilog_df("", kdfPersistent) + dfref df_persistent = get_scilog_df(logbook, kdfPersistent) + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + + string win_title = "SciLog " + logbook + + NewPanel /K=1 /N=$win_name /W=(600,200,1200,700) as win_title + win_name = s_name + ModifyPanel /w=$win_name cbRGB=(52224,52224,65280) + + svar /sdfr=df_persistent attributes + svar /sdfr=df_persistent controls + svar /sdfr=df_persistent options + svar /sdfr=df_persistent helps + wave /t /sdfr=df_volatile attach_list + wave /sdfr=df_volatile attach_sel + svar /sdfr=df_volatile url + + variable iattr + variable nattr = ItemsInList(attributes, ";") + string s_attr + string s_control + string s_option + string s_help + string persistent_path = GetDataFolder(1, df_persistent) + string volatile_path = GetDataFolder(1, df_volatile) + string options_path + string variable_path + variable ypos = 2 + variable height = 0 + + for (iattr = 0; iattr < nattr; iattr += 1) + s_attr = StringFromList(iattr, attributes, ";") + s_control = StringByKey(s_attr, controls, ":", ";") + s_help = StringByKey(s_attr, helps, ":", ";") + strswitch(s_control[0,1]) + case "sv": + SetVariable $s_control, win=$win_name, pos={0,ypos}, size={300,16}, bodyWidth=230 + SetVariable $s_control, win=$win_name, title=s_attr, value= _STR:"" + SetVariable $s_control, win=$win_name, userdata(attribute)=s_attr + if (strlen(s_help)) + SetVariable $s_control, win=$win_name, help={s_help} + endif + ypos += 18 + break + case "pm": + options_path = persistent_path + StringByKey(s_attr, options, "=", ";") + PopupMenu $s_control, win=$win_name, pos={0,ypos}, size={300,21}, bodyWidth=230 + PopupMenu $s_control, win=$win_name, title=s_attr + PopupMenu $s_control, win=$win_name, mode=1, popvalue="Other", value= #options_path + PopupMenu $s_control, win=$win_name, userdata(attribute)=s_attr + if (strlen(s_help)) + PopupMenu $s_control, win=$win_name, help={s_help} + endif + ypos += 23 + break + case "cb": + CheckBox $s_control, win=$win_name, pos={70,ypos}, size={300,14} + CheckBox $s_control, win=$win_name, title=s_attr, value= 1 + CheckBox $s_control, win=$win_name, userdata(attribute)=s_attr + if (strlen(s_help)) + CheckBox $s_control, win=$win_name, help={s_help} + endif + ypos += 17 + break + endswitch + endfor + ypos = max(ypos, 80) + + TitleBox t_attach, win=$win_name, pos={308,5}, size={70,14}, title="Attachments", frame=0 + TitleBox t_attach, help={"Plain text message. Each line is converted into a paragraph in HTML. To use HTML formatting, start with tag."} + height = ypos - 21 - 4 + ListBox lb_attach, win=$win_name, pos={308,21}, size={264,height} + ListBox lb_attach, win=$win_name, listWave=attach_list + ListBox lb_attach, win=$win_name, mode=1, selWave=attach_sel, selRow=-1 + ListBox lb_attach, win=$win_name, widths={20,160,80} + ListBox lb_attach, win=$win_name, help={"Choose graphs to attach to the message."} + + Button b_attach_top, win=$win_name, pos={420,2}, size={40,18}, title="top" + Button b_attach_top, win=$win_name, fcolor=(56576,60928,47872) + Button b_attach_top, win=$win_name, proc=PearlSciLog#bp_attach_top + Button b_attach_top, win=$win_name, help={"Select top graph for attachment."} + Button b_attach_all, win=$win_name, pos={460,2}, size={40,18}, title="all" + Button b_attach_all, win=$win_name, fcolor=(56576,60928,47872) + Button b_attach_all, win=$win_name, proc=PearlSciLog#bp_attach_allnone + Button b_attach_all, win=$win_name, help={"Select all graphs for attachment."} + Button b_attach_none, win=$win_name, pos={500,2}, size={40,18}, title="none" + Button b_attach_none, win=$win_name, fcolor=(56576,60928,47872) + Button b_attach_none, win=$win_name, proc=PearlSciLog#bp_attach_allnone + Button b_attach_none, win=$win_name, help={"Deselect all attachments."} + Button b_save_graphs, win=$win_name, pos={540,2}, size={40,18}, title="save" + Button b_save_graphs, win=$win_name, fcolor=(56576,60928,47872) + Button b_save_graphs, win=$win_name, proc=PearlSciLog#bp_save_graphs + Button b_save_graphs, win=$win_name, help={"Save selected graphs as PNG bitmap files."} + Button b_attach_up, win=$win_name, pos={576,20}, size={20,20}, title="\\W517" + Button b_attach_up, win=$win_name, fcolor=(56576,60928,47872) + Button b_attach_up, win=$win_name, proc=PearlSciLog#bp_attach_updown + Button b_attach_up, win=$win_name, help={"Move selected graph up."} + Button b_attach_dw, win=$win_name, pos={576,40}, size={20,20}, title="\\W523" + Button b_attach_dw, win=$win_name, fcolor=(56576,60928,47872) + Button b_attach_dw, win=$win_name, proc=PearlSciLog#bp_attach_updown + Button b_attach_dw, win=$win_name, help={"Move selected graph down."} + + ypos += 246-160 + Button b_submit,win=$win_name, pos={70,ypos},size={46,20},proc=PearlSciLog#bp_submit,title="Submit" + Button b_submit,win=$win_name, help={"Submit form data to SciLog (new entry)."} + Button b_submit,win=$win_name, fcolor=(56576,60928,47872) + Button b_clear,win=$win_name, pos={120,ypos},size={46,20},proc=PearlSciLog#bp_clear,title="Clear" + Button b_clear,win=$win_name, help={"Clear the form fields"} + Button b_clear,win=$win_name, fcolor=(56576,60928,47872) + + ypos += 272-246 + + ypos += 270-272 + + SetWindow $win_name, hook(scilogPanelHook)=PearlSciLog#scilog_panel_hook + SetWindow $win_name, userdata(logbook)=logbook + + ypos += 160-270 + TitleBox t_message,win=$win_name, pos={10,ypos},size={58,16},fixedSize=1,frame=0,anchor=RT,title="Message" + DefineGuide UGH0={FT,ypos},UGV0={FL,70},UGH1={FB,-52},UGV1={FR,-2} + NewNotebook /F=0 /N=Message /OPTS=3 /W=(115,404,345,341)/FG=(UGV0,UGH0,UGV1,UGH1) /HOST=# + Notebook kwTopWin, defaultTab=20, statusWidth=0, autoSave=0 + Notebook kwTopWin fSize=10, fStyle=0, textRGB=(0,0,0) + RenameWindow #,Message + string nb_name = win_name + "#Message" + SetActiveSubwindow ## + + // restore recently used attributes and message + svar /z /sdfr=df_persistent recent + if (svar_exists(recent) && (strlen(recent) > 0)) + set_panel_attributes(win_name, recent) + endif + svar /z /sdfr=df_persistent recent_message + if (svar_exists(recent_message) && (strlen(recent_message) > 0)) + set_panel_message(win_name, recent_message) + endif + Notebook $nb_name selection={startOfFile,startOfFile}, findText={"",1} + + setdatafolder savedf + return win_name +end + +static function scilog_panel_hook(s) + STRUCT WMWinHookStruct &s + + Variable hookResult = 0 + + switch(s.eventCode) + case 0: // activate + string logbook = GetUserData(s.winName, "", "logbook") + if (strlen(logbook) > 0) + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + update_attach_items(logbook) + endif + break + case 6: // resize + // move bottom-aligned controls when the window is resized + variable b_top = s.winRect.bottom - 20 + Button b_submit,pos={70,b_top} + Button b_clear,pos={120,b_top} + break + endswitch + + return hookResult // 0 if nothing done, else 1 +end + +static constant kAttachColSel = 0 +static constant kAttachColTitle = 1 +static constant kAttachColName = 2 + +/// update the list of attachments +static function update_attach_items(logbook) + string logbook + + dfref savedf = getdatafolderdfr() + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + wave /t /sdfr=df_volatile attach_list + wave /sdfr=df_volatile attach_sel + + if (!waveexists(attach_list)) + return -1 + endif + string names = WinList("*", ";", "WIN:1;VISIBLE:1") + names = SortList(names, ";", 16) + + // remove closed graphs + variable i + variable k + variable n = DimSize(attach_list, 0) + string s + for (i = n-1; i >= 0; i -= 1) + s = attach_list[i][kAttachColName] + if (WhichListItem(s, names) < 0) + DeletePoints /M=0 i, 1, attach_list, attach_sel + endif + endfor + + // add new graphs + n = ItemsInList(names) + for (i = 0; i < n; i += 1) + s = StringFromList(i, names) + FindValue /text=s /txop=4 /z attach_list + if (v_value < 0) + k = DimSize(attach_list, 0) + Redimension /n=(k+1,3) attach_list, attach_sel + //InsertPoints /M=0 k, 1, attach_list, attach_sel + attach_list[k][kAttachColSel] = "" + attach_list[k][kAttachColTitle] = "" + attach_list[k][kAttachColName] = s + attach_sel[k][kAttachColSel] = 32 + attach_sel[k][kAttachColTitle] = 0 + attach_sel[k][kAttachColName] = 0 + endif + endfor + + // update titles + n = DimSize(attach_list, 0) + for (i = n-1; i >= 0; i -= 1) + s = attach_list[i][kAttachColName] + getwindow /z $s, wtitle + if (v_flag == 0) + attach_list[i][kAttachColTitle] = s_value + else + attach_list[i][kAttachColTitle] = s + endif + endfor + + setdatafolder savedf + return 0 +end + +/// move an attachment item in the list of attachments +static function move_attach_item(logbook, item, distance) + string logbook + variable item + variable distance + + dfref savedf = getdatafolderdfr() + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + wave /t /sdfr=df_volatile attach_list + wave /sdfr=df_volatile attach_sel + variable n = DimSize(attach_list, 0) + variable dest = item + distance + + if ((item >= 0) && (item < n) && (dest >= 0) && (dest < n)) + string name = attach_list[item][kAttachColName] + variable sel = attach_sel[item][kAttachColSel] + DeletePoints /M=0 item, 1, attach_list, attach_sel + InsertPoints /M=0 dest, 1, attach_list, attach_sel + attach_list[dest][kAttachColName] = name + update_attach_items(logbook) + attach_sel[dest][kAttachColSel] = sel + endif +end + +/// button procedure for the attachment up and down buttons +static function bp_attach_updown(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + string logbook = GetUserData(ba.win, "", "logbook") + ControlInfo /w=$ba.win lb_attach + variable row = v_value + dfref df = $s_datafolder + wave /t /sdfr=df attach_list = $s_value + if (cmpstr(ba.ctrlName, "b_attach_up") == 0) + // up button + if (row >= 1) + move_attach_item(logbook, row, -1) + ListBox lb_attach, win=$ba.win, selRow=(row-1) + endif + else + // down button + if (row < DimSize(attach_list, 0) - 1) + move_attach_item(logbook, row, +1) + ListBox lb_attach, win=$ba.win, selRow=(row+1) + endif + endif + break + case -1: // control being killed + break + endswitch + + return 0 +end + +/// button procedure for the Submit and Reply buttons +static function bp_submit(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + string logbook = GetUserData(ba.win, "", "logbook") + string attributes + string message + string graphs + attributes = get_panel_attributes(ba.win) + message = get_panel_message(ba.win) + graphs = get_panel_graphs(ba.win) + + if ((scilog_validate_attributes(logbook, attributes, message) == 0)) + variable result + result = scilog_create_entry(logbook, attributes, message, graphs=graphs) + if (result == 0) + dfref df = get_scilog_df(logbook, kdfPersistent) + svar /sdfr=df recent + recent = attributes + svar /sdfr=df recent_message + recent_message = message + else + abort "Submission failed. Error code " + num2str(result) + "." + endif + else + abort "Submission failed due to missing/invalid attribute." + endif + break + case -1: // control being killed + break + endswitch + + return 0 +end + +/// select top graph window for attachment +static function bp_attach_top(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + string graphs = WinName(0, 1, 1) + set_panel_graphs(ba.win, graphs) + break + case -1: // control being killed + break + endswitch + + return 0 +end + +/// select/deselect all graph windows for attachment +static function bp_attach_allnone(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + string logbook = GetUserData(ba.win, "", "logbook") + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + wave /sdfr=df_volatile attach_sel + if (cmpstr(ba.ctrlName, "b_attach_all") == 0) + attach_sel[][kAttachColSel] = attach_sel[p][kAttachColSel] | 16 + else + attach_sel[][kAttachColSel] = attach_sel[p][kAttachColSel] & ~16 + endif + break + case -1: // control being killed + break + endswitch + + return 0 +end + +static function bp_save_graphs(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + string logbook = GetUserData(ba.win, "", "logbook") + string graphs = get_panel_graphs(ba.win) + variable ngraphs = ItemsInList(graphs, ";") + + variable igraph + string sgraph + string graph_path + for (igraph = 0; igraph < ngraphs; igraph += 1) + sgraph = StringFromList(igraph, graphs, ";") + graph_path = create_graph_file(sgraph, igraph) + if (strlen(graph_path) > 0) + print graph_path + endif + endfor + + break + case -1: // control being killed + break + endswitch + + return 0 +end + +static function bp_clear(ba) : ButtonControl + STRUCT WMButtonAction &ba + + switch( ba.eventCode ) + case 2: // mouse up + set_panel_attributes(ba.win, "", clear=1) + set_panel_message(ba.win, "") + set_panel_graphs(ba.win, "") + break + case -1: // control being killed + break + endswitch + + return 0 +end + +static function /s get_default_panel_name() + string windowname + windowname = StringFromList(0, WinList("*SciLogPanel*", ";", "WIN:64"), ";") + return windowname +end + +/// get a list of attributes from the fields of the SciLog panel. +/// +/// @param windowname window name of the SciLog panel +/// if empty, use default name "PearlSciLogPanel" +/// +/// @return list of attributes in the format "key1:value1;key2:value2". +/// +static function /s get_panel_attributes(windowname) + string windowname + + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + if (strlen(windowname) == 0) + return "" + endif + + string controls = ControlNameList(windowname, ";") + string attributes = "" + string control + string attribute + variable ico + variable nco = ItemsInList(controls, ";") + for (ico = 0; ico < nco; ico += 1) + control = StringFromList(ico, controls, ";") + attribute = GetUserData(windowname, control, "attribute") + if (strlen(attribute) > 0) + ControlInfo /w=$windowname $control + switch(v_flag) + case 2: // checkbox + attributes = ReplaceNumberByKey(attribute, attributes, v_value, ":", ";") + break + case 3: // popupmenu + case 5: // setvariable + attributes = ReplaceStringByKey(attribute, attributes, s_value, ":", ";") + break + endswitch + endif + endfor + + return attributes +end + +/// set the fields of the SciLog panel +/// +/// @param windowname window name of the SciLog panel +/// if empty, use default name "PearlSciLogPanel" +/// +/// @param attributes list of attributes to set (format "key1:value1;key2:value2") +/// +/// @param clear what to do if a key is missing in attributes? +/// @arg 0 (default) leave the field unchanged +/// @arg 1 clear the field +/// +static function /s set_panel_attributes(windowname, attributes, [clear]) + string windowname + string attributes + variable clear + + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + if (strlen(windowname) == 0) + return "" + endif + if (ParamIsDefault(clear)) + clear = 0 + endif + + string path + + string logbook = GetUserData(windowname, "", "logbook") + dfref df_persistent = get_scilog_df(logbook, kdfPersistent) + string persistent_path = GetDataFolder(1, df_persistent) + svar /sdfr=df_persistent options + string options_path + + string controls = ControlNameList(windowname, ";") + string control + string attribute + string value + variable numval + variable ico + variable nco = ItemsInList(controls, ";") + for (ico = 0; ico < nco; ico += 1) + control = StringFromList(ico, controls, ";") + attribute = GetUserData(windowname, control, "attribute") + if (strlen(attribute)) + value = StringByKey(attribute, attributes, ":", ";") + if (strlen(value) || clear) + ControlInfo /w=$windowname $control + switch(v_flag) + case 2: // checkbox + numval = NumberByKey(attribute, attributes, ":", ";") + if ((numtype(numval) != 0) && clear) + numval = 0 + endif + if (numtype(numval) == 0) + CheckBox $control, value=numval, win=$windowname + endif + break + case 3: // popupmenu + options_path = persistent_path + StringByKey(attribute, options, ":", ";") + svar values = $options_path + numval = WhichListItem(value, values, ";") + 1 + if (numval >= 1) + PopupMenu $control, mode=numval, win=$windowname + endif + break + case 5: // setvariable + SetVariable /z $control, value= _STR:value, win=$windowname + break + endswitch + endif + endif + endfor + + return attributes +end + +/// get the message field of the SciLog panel +/// +/// @param windowname window name of the SciLog panel +/// if empty, use default name "PearlSciLogPanel" +/// +/// @return message text +/// +static function /s get_panel_message(windowname) + string windowname + + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + if (strlen(windowname) == 0) + return "" + endif + + string nb = windowname + "#Message" + notebook $nb selection={startOfFile, endOfFile} + getselection notebook, $nb, 2 + + return s_selection +end + +/// set the message field of the SciLog panel +/// +/// @param windowname window name of the SciLog panel +/// if empty, use default name "PearlSciLogPanel" +/// +/// @param message message text that can be passed to the @c Notebook operation. +/// +/// @return original message (unchanged) +/// +static function /s set_panel_message(windowname, message) + string windowname + string message + + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + + string nb = windowname + "#Message" + notebook $nb selection={startOfFile, endOfFile},text=message + + return message +end + +/// get the names of the graphs selected for attachment +/// +/// @param windowname panel window name +/// @returns a semicolon-separated list, +/// or the empty string if the selection is not valid. +/// +static function /s get_panel_graphs(windowname) + string windowname // panel window name + + dfref savedf = getdatafolderdfr() + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + if (strlen(windowname) == 0) + return "" + endif + + string logbook = GetUserData(windowname, "", "logbook") + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + wave /t /sdfr=df_volatile attach_list + wave /sdfr=df_volatile attach_sel + string graphs = "" + string windows = "" + string graphname + + variable n = DimSize(attach_sel, 0) + variable i + for (i = 0; i < n; i += 1) + if (attach_sel[i][kAttachColSel] & 16) + graphname = attach_list[i][kAttachColName] + windows = WinList(graphname, ";", "WIN:1") + if (ItemsInList(windows) == 1) + graphs = AddListItem(graphname, graphs, ";", inf) + endif + endif + endfor + + return graphs +end + +/// update selection of graphs for attachment +/// +/// @param windowname panel window name. looks for default panel if empty. +/// +/// @param graphs semicolon-separated list of names of graph windows to select for attachment. +/// +static function /s set_panel_graphs(windowname, graphs) + string windowname + string graphs + + if (strlen(windowname) == 0) + windowname = get_default_panel_name() + endif + if (strlen(windowname) == 0) + return "" + endif + + string logbook = GetUserData(windowname, "", "logbook") + update_attach_items(logbook) + dfref df_volatile = get_scilog_df(logbook, kdfVolatile) + wave /t /sdfr=df_volatile attach_list + wave /sdfr=df_volatile attach_sel + + variable n = DimSize(attach_sel, 0) + variable i + string graphname + for (i = 0; i < n; i += 1) + graphname = attach_list[i][kAttachColName] + if (WhichListItem(graphname, graphs)>= 0) + attach_sel[i][kAttachColSel] = 48 + else + attach_sel[i][kAttachColSel] = 32 + endif + endfor +end diff --git a/pearl/scilog-ingest.py b/pearl/scilog-ingest.py new file mode 100644 index 0000000..f4b15dd --- /dev/null +++ b/pearl/scilog-ingest.py @@ -0,0 +1,331 @@ +#!/usr/bin/env -S uv run --script +# +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "pillow", +# "scilog", +# ] +# /// + + +import argparse +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from PIL import Image, UnidentifiedImageError + +from scilog import SciLog, LogbookMessage + + +def get_image_size(file_path): + """ + + Args: + file_path: + + Returns: + + Raises: + OSError: If the file does not exist. + PIL.Image.UnidentifiedImageError: If the image is not recognized. + """ + + with Image.open(file_path) as img: + return img.size + + +re_single_nbsp = re.compile(r"(? str: + """ + Replace multiple spaces with the corresponding number of   entities. + + Args: + s: string + + Returns: string + + """ + def nbsp(mo): + return " " * len(mo.group()) + + return re_rep_pre_spc.sub(nbsp, s) + + +class ScilogIngestor: + def __init__(self): + super().__init__() + self._author: str = "" + self._pgroup: str = "" + self._logbook: str = "" + self._message: str = "" + self._tags: List[str] = [] + self._attachments: List[str] = [] + self._url: str = "" + self._user: str = "" + self._password: str = "" + self._location: str = "" + self._scilog: Optional[SciLog] = None + + def prepare(self): + pass + + @staticmethod + def _adjust_image_size(file_info): + try: + filepath = file_info['filepath'] + except KeyError: + return + try: + w, h = get_image_size(filepath) + except (OSError, UnidentifiedImageError): + return + else: + # limit image size + while w > 400: + w //= 2 + file_info['style'] = {'width': f'{w}px', 'height': ''} + + def convert_plain(self, mesg: str) -> str: + """ + convert plain-text entry to HTML + + - Convert plain text message to HTML paragraph with formatting. + - Replace line feeds by
tags. + - Replace tabs and spaces. + + Args: + mesg: + + Returns: string + """ + + mesg = mesg.replace("&", "&").replace("<", "<").replace(">", ">") + mesg = mesg.replace("\n", "
") + mesg = mesg.expandtabs(8) + mesg = replace_pre_spaces(mesg) + mesg = "

" + mesg + "

" + + return mesg + + @staticmethod + def is_pgroup(value: str) -> str: + value = value.lower() + if len(value) == 6 and value[0] in {"e", "p"} and int(value[1:]): + return "p" + value[1:] + else: + return "" + + @property + def scilog(self) -> SciLog: + """ + Return a SciLog instance. + + Returns: SciLog + """ + + if self._scilog is None: + options = {"username": self._user, + "password": self._password} + url = self._url + self._scilog = SciLog(url, options=options) + + return self._scilog + + def ingest_message(self): + """ + ingest message into scilog + + this method ingests the current message into scilog. + + """ + + log = self.scilog + + try: + logbook_name = self._logbook + except KeyError: + lb_filter = {"ownerGroup": self._pgroup, "deleted": False} + else: + lb_filter = {"ownerGroup": self._pgroup, "name": logbook_name, "deleted": False} + + logbooks = log.get_logbooks(where=lb_filter, limit=10) + try: + logbook = logbooks[0] + except IndexError: + raise ValueError(f"no logbook found for {lb_filter}") + else: + if len(logbooks) > 1: + raise ValueError(f"multiple logbooks found for {lb_filter}") + else: + log.select_logbook(logbook) + + msg = LogbookMessage() + if self._message[0] == '<': + msg.add_text(self._message) + else: + for line in self._message.split('\n'): + msg.add_text(line) + for att in self._attachments: + msg.add_file(att) + file_info = msg._content.files[-1] + self._adjust_image_size(file_info) + if self._tags: + msg.add_tag(self._tags) + log.send_logbook_message(msg) + + def load_attributes_from_file(self, attributes_file): + """ + Load attributes, tags and attachment paths from a file. + + Each line of the file declares a key:value pair. + The keys are: + - author: (required) Value is the e-mail address of the author. + - pgroup: (required) Value is the p-group of the logbook. + - logbook: (required) Value is the name of the logbook. + - location: (not used) Value is the name of the location. Currently not used. + - tag: (optional) Value is a tag to be added to the snippet. Can occur multiple times. + Duplicates are ignored. Spaces and commas are removed. + - attachment: (optional) Value is the path of an attachment. Can occur multiple times. + Duplicates are ignored. + + :param attributes_file: + :return: + """ + + unique_tags = set() + unique_attachments = set() + + with open(attributes_file, "rt", encoding="utf8") as f: + for line in f.readlines(): + try: + key, val = line.split(":", 1) + key = key.strip() + val = val.strip() + except ValueError: + continue + + if key == "tag": + val = val.replace(" ", "") + val = val.replace(",", "") + if val not in unique_tags: + self._tags.append(val) + unique_tags.add(val) + elif key == "attachment": + if val not in unique_attachments: + self._attachments.append(val) + unique_attachments.add(val) + elif key == "author": + self._author = val + elif key == "pgroup": + self._pgroup = val + elif key == "location": + self._location = val + elif key == "logbook": + self._logbook = val + + def load_message_from_file(self, message_file): + """ + Load message from file + + If the file starts with a `<`, the content is assumed to be HTML. + Otherwise, it is assumed to be plain text and will be tagged and escaped. + + :param message_file: + :return: + """ + + with open(message_file, "rt", encoding="utf8") as f: + self._message = "\n".join(f.readlines()) + + def load_credentials_from_file(self, credentials_file): + """ + Load the credentials from a file. + + The file must contain three lines, each in the form key:value. + The keys are `url`, `user`, and `password`. + The URL must start with `https://` and end with `/api/v1`. + + :param credentials_file: path of the credentials file. + If None, it is read from `.scilog.cred` the home directory. + Make sure to protect the credentials from unauthorized access! + :return: None + """ + + if credentials_file is None: + credentials_file = Path.home() / ".scilog.cred" + if not Path(credentials_file).exists(): + raise ValueError("Missing credentials") + with open(credentials_file, "rt", encoding="utf8") as f: + for line in f.readlines(): + try: + key, val = line.split(":", 1) + key = key.strip() + val = val.strip() + except ValueError: + continue + + if key == "user": + self._user = val + elif key == "password": + self._password = val + elif key == "url": + self._url = val + + def validate(self): + """ + Perform a number of basic validity checks on the attributes. + + :return: + :raise ValueError if a problem is found. + """ + + if not re.match(r"[^@]+@[^@]+\.[^@]+", self._author): + raise ValueError("Invalid email address.") + if not self._user or not self._password: + raise ValueError("Invalid credentials.") + if not self.is_pgroup(self._pgroup): + raise ValueError("Invalid pgroup.") + if not self._logbook: + raise ValueError("Empty logbook name.") + if not self._message: + raise ValueError("Empty message.") + if not re.match(r"https://[a-zA-Z0-9]+\.[a-zA-Z0-9.]+(:[0-9]+)?/api/v1", self._url): + raise ValueError("Invalid URL.") + + def run(self): + self.validate() + self.ingest_message() + + +def main(attributes_file, message_file, credentials_file): + ingestor = ScilogIngestor() + ingestor.load_attributes_from_file(attributes_file) + ingestor.load_message_from_file(message_file) + ingestor.load_credentials_from_file(credentials_file) + ingestor.run() + print("Message successfully transmitted") + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Simple file interface to ingest an entry into a SciLog logbook", + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("-c", "--credentials", default=None, help="credentials file, UTF-8 encoded") + parser.add_argument("attributes", help="attributes file, UTF-8 encoded") + parser.add_argument("message", help="message file, UTF-8 encoded") + + clargs = parser.parse_args() + + return clargs + + +if __name__ == "__main__": + clargs = parse_args() + main(clargs.attributes, clargs.message, clargs.credentials)