code changes for release 3.0.0: new PShell import

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2009-2019 Paul Scherrer Institut
+   Copyright 2009-2022 Paul Scherrer Institut
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,7 +1,7 @@
 Introduction
 ============
 
-PEARL Procedures is a suite of Igor Pro procedures developed for data acquisition and data processing at the PEARL beamline at the Swiss Light Source.
+PEARL Procedures is a suite of Igor Pro procedures developed for data acquisition and data processing at the PEARL beamline at the Swiss Light Source. PEARL Procedures requires Igor Pro 8 or newer.
 
 
 Installation
@@ -11,17 +11,14 @@ PEARL Procedures should be installed according to the regular Igor Pro guideline
 
 - Make a `pearl-procs` directory in your private or shared `User Procedures` folder, and copy the PEARL Procedures distribution there.
 - Create shortcuts of the `pearl-arpes.ipf` and `pearl-menu.ipf` files, and move them to the `Igor Procedures` folder next to your `User Procedures` folder.
-- Find the `HDF5.XOP` (`HDF5-64.xop` for Igor 7 64-bit) extension in the `Igor Pro Folder` under `More Extensions/File Loaders` (`More Extensions (64-bit)/File Loaders`), create a shortcut, and move the shortcut to the `Igor Extensions` folder next to your `User Procedures` folder.
+
+Igor Pro 9 imports the HDF5 library by default. For earlier versions:
+
+- Find the `HDF5.XOP` (`HDF5-64.xop` for 64-bit) extension in the `Igor Pro Folder` under `More Extensions/File Loaders` (`More Extensions (64-bit)/File Loaders`), create a shortcut, and move the shortcut to the `Igor Extensions` folder next to your `User Procedures` folder.
 - Find the `HDF5 Help.ihf` next to `HDF5.XOP`, create a shortcut, and move the shortcut to the `Igor Help Files` folder next to your `User Procedures` folder.
 
-PEARL Procedures are tested on Igor 8.04, 64-bit.
+PEARL Procedures are tested on Igor Pro 8.04, 64-bit.
-Please make sure to use the latest release version.
+Please make sure to use the latest release version of Igor Pro.
-
-While most of the code remains compatible with Igor 6.37, it is not tested and not supported.
-Importing recent PShell data files may requires Igor 8 due to changes in the HDF5 library.
-Igor 7 contains some bugs which affect PEARL Procedures and should not be used.
-
-As long as no Igor 8 specific features are used (long object names), the produced experiment files remain compatible with Igor 6.
 
 
 License
@@ -39,12 +36,18 @@ Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
 Copyright
 ---------
 
-Copyright 2009-2021 by [Paul Scherrer Institut](http://www.psi.ch)
+Copyright 2009-2022 by [Paul Scherrer Institut](http://www.psi.ch)
 
 
 Release Notes
 =============
 
+## rev-distro-3.0.0
+
+- New panel and procedure interface for PShell data file import.
+- Support for latest PShell file structure.
+- Igor Pro 8.04 or later is required.
+
 ## rev-distro-2.2.0
 
 - Updates, bugfixes and performance improvements in angle scan processing.

doc/src/mainpage.dox

@@ -9,11 +9,18 @@ PEARL Procedures is a suite of Igor Pro procedures developed for data acquisitio
 
 \section sec_install Installation
 
+PEARL Procedures are tested on Igor Pro 8.04, 64-bit.
+Compatibility with earlier versions of Igor has been dropped.
+Please make sure to use the latest release version of Igor Pro.
+
 PEARL Procedures should be installed according to the regular Igor Pro guidelines. Please read the Igor help `About Igor Pro User Files` for details.
 
 - Make a `pearl-procs` directory in your private or shared `User Procedures` folder, and copy the PEARL Procedures distribution there.
 - Create shortcuts of the `pearl-arpes.ipf` and `pearl-menu.ipf` files, and move them to the `Igor Procedures` folder next to your `User Procedures` folder.
-- Find the `HDF5.XOP` extension in the `Igor Pro Folder` under `More Extensions/File Loaders`, create a shortcut, and move the shortcut to the `Igor Extensions` folder next to your `User Procedures` folder.
+
+Igor Pro 9 imports the HDF5 library by default. For earlier versions:
+
+- Find the `HDF5.XOP` (`HDF5-64.xop` for 64-bit) extension in the `Igor Pro Folder` under `More Extensions/File Loaders` (`More Extensions (64-bit)/File Loaders`), create a shortcut, and move the shortcut to the `Igor Extensions` folder next to your `User Procedures` folder.
 - Find the `HDF5 Help.ihf` next to `HDF5.XOP`, create a shortcut, and move the shortcut to the `Igor Help Files` folder next to your `User Procedures` folder.
 
 
@@ -25,6 +32,6 @@ Please read and respect the respective license agreements.
 
 \author    Matthias Muntwiler, <mailto:matthias.muntwiler@psi.ch>
 \version   This documentation is compiled from version $(REVISION).
-\copyright 2009-2016 by [Paul Scherrer Institut](http://www.psi.ch)
+\copyright 2009-2022 by [Paul Scherrer Institut](http://www.psi.ch)
 \copyright Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
 */

mm/mm-physconst.ipf

@@ -1,3 +1,4 @@
+#pragma TextEncoding = "UTF-8"
 #pragma rtGlobals=1		// Use modern global access method.
 #pragma version = 1.05
 

pearl/pearl-area-import.ipf

@@ -1,12 +1,15 @@
-#pragma TextEncoding = "Windows-1252"
+#pragma TextEncoding = "UTF-8"
 #pragma rtGlobals=3		// Use modern global access method and strict wave access.
-#pragma IgorVersion = 6.2
+#pragma IgorVersion = 6.36
 #pragma ModuleName = PearlAreaImport
+#pragma version = 1.13
+#if IgorVersion() < 9.00
 #include <HDF5 Browser>
+#endif
 #include "pearl-compat"
 #include "pearl-gui-tools"
 
-// copyright (c) 2013-20 Paul Scherrer Institut
+// copyright (c) 2013-21 Paul Scherrer Institut
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 //   you may not use this file except in compliance with the License.
@@ -1195,6 +1198,7 @@ end
 ///							a numeric index is appended to distinguish the results.
 ///							the index starts at 1. existing waves are overwritten.
 ///
+/// @return	result code: 0 for success, < 0 for error
 ///
 function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefix, [progress, nthreads])
 	wave source
@@ -1211,7 +1215,10 @@ function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefi
 	if (ParamIsDefault(nthreads))
 		nthreads = -1
 	endif
+
+	dfref base_df = GetDataFolderDFR()
 	variable result = 0
+	string wavenames = ""
 	
 	// nx and nz are the image dimensions
 	variable nx, ny, nz, nt
@@ -1219,10 +1226,9 @@ function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefi
 	ny = dimsize(source, 1)
 	nz = dimsize(source, 2)
 	// force 4th dimension to singleton (ad_extract_slab handles 3 dimensions only)
-	nt = 0
+	nt = 1
 	
 	variable nzt = max(nz, 1) * max(nt, 1)
-	variable izt
 	
 	// set up multi threading
 	if (nthreads < 0)
@@ -1239,34 +1245,44 @@ function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefi
 	endif
 	
 	if (progress)
-		display_progress_panel("data reduction", "extracting data (step 1 of 2)...", nzt)
+		display_progress_panel("Reduction", "Processing data...", nzt)
 	endif
 	
 	variable iz, it
+	variable n_sent = 0
+	variable n_recvd = 0
+	variable tmo = 0
 	string dfname
+	dfref dfr
 	variable iw, nw
 	string sw
 	make /n=0 /free /wave result_waves
 
-	izt = 0
+	iz = 0
-	for (iz = 0; iz < max(nz, 1); iz += 1)
+	it = 0
-		for (it = 0; it < max(nt, 1); it += 1)
+	
-			dfname = "processing_" + num2str(izt)
+	do
-			newdatafolder /s $dfname
+		// fill the processing queue up to a maximum number of folders
+		if (n_sent < max(1, nthreads) * 10 + n_recvd)
+			if (iz < nz)
+				if (it < nt)
+					// load a slab into a temporary folder
+					dfname = "processing_" + num2str(n_sent)
+					NewDataFolder /s $dfname
 					ad_extract_slab(source, nan, nan, nan, nan, iz, iz, "image", pscale=1)
 					wave image
-
-			// send to processing queue
 					variable /g r_index = iz
 					variable /g s_index = it
 					string /g func_param = reduction_param
 
 					if (nthreads > 0)
+						// send to thread group
 						WaveClear image
 						ThreadGroupPutDF threadGroupID, :
 					else
-				processing_folders[izt] = GetDataFolderDFR()
+						// process immediately in single-thread mode
-				string param = reduction_param
+						processing_folders[n_sent] = GetDataFolderDFR()
+						string param = func_param
 						wave /wave reduced_waves = reduction_func(image, param)
 						variable /g func_result = numpnts(reduced_waves)
 						adh5_get_result_waves(reduced_waves, "redw_", 0)
@@ -1274,65 +1290,47 @@ function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefi
 						setdatafolder ::
 					endif
 
-			izt += 1
+					iz += 1
-			// progress window
+					n_sent += 1
-			if (progress)
+					tmo = 0
-				if (update_progress_panel(izt))
-					result = -4 // user abort
-					break
-				endif
-			endif
-		endfor
-	endfor
-	
-	if (progress)
-		update_progress_panel(0, message="processing data (step 2 of 2)...")
-	endif
-
-	dfref dfr
-	for (izt = 0; (izt < nzt) && (result == 0); izt += 1)
-		if (nthreads > 0)
-			do
-				dfr = ThreadGroupGetDFR(threadGroupID, 1000)
-				if (DatafolderRefStatus(dfr) != 0)
-					break
-				endif
-				if (progress)
-					if (update_progress_panel(izt))
-						result = -4 // user abort
-						break
-					endif
-				endif
-			while (1)
 				else
-			dfr = processing_folders[izt]
+					iz += 1
-			if (progress)
+					it = 0
-				if (update_progress_panel(izt))
-					result = -4 // user abort
-					break
 				endif
 			endif
+		else
+			// throttle the loop if processing is slow
+			tmo = min(100, tmo + 10)
 		endif
 
-		if (result != 0)
+		// receive a slab from the processing queue
-			break
+		if (n_recvd < nzt)
+			if (nthreads > 0)
+				dfr = ThreadGroupGetDFR(threadGroupID, tmo)
+			else
+				dfr = processing_folders[n_recvd]
+				processing_folders[n_recvd] = $""
 			endif
 
+			if (DatafolderRefStatus(dfr) != 0)
+				// access results folder
 				nvar rr = dfr:r_index
 				nvar ss = dfr:s_index
 				nvar func_result = dfr:func_result
 
 				if (func_result < 1)
-			result = -3 // dimension reduction error
+					print "error during data reduction."
+					result = -3
 					break
 				endif
 
+				// initialize result waves just once
 				if (numpnts(result_waves) == 0)
 					redimension /n=(func_result) result_waves
 					for (iw = 0; iw < func_result; iw += 1)
 						sw = "redw_" + num2str(iw)
 						wave profile = dfr:$sw
-				sw = result_prefix + num2str(iw+1)
+						sw = "ReducedData" + num2str(iw+1)
 						make /n=(dimsize(profile, 0), nz, nt) /d /o $sw
 						wave data = $sw
 						setdimlabel 0, -1, $getdimlabel(profile, 0, -1), data
@@ -1340,32 +1338,66 @@ function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefi
 						setscale /p y dimoffset(source, 2), dimdelta(source, 2), waveunits(source, 2), data
 						setscale /p z dimoffset(source, 3), dimdelta(source, 3), waveunits(source, 3), data
 						setscale d 0, 0, waveunits(profile, -1), data
+						note data, note(profile)
 						result_waves[iw] = data
 					endfor
 				endif
+
+				// copy results
 				for (iw = 0; iw < func_result; iw += 1)
 					sw = "redw_" + num2str(iw)
 					wave profile = dfr:$sw
 					wave data = result_waves[iw]
 					data[][rr][ss] = profile[p]
 				endfor
-	endfor
 				
+				n_recvd += 1
+				KillDataFolder /Z dfr
+			endif
+		else
+			// processing complete
+			break
+		endif
+
+		// update progress window
+		if (progress)
+			if (update_progress_panel(n_recvd))
+				print "user abort"
+				result = -4
+				break
+			endif
+		endif
+	while ((n_recvd < nzt) && (result == 0))
+
+	// clean up	
 	if (nthreads > 0)
 		variable tstatus = ThreadGroupRelease(threadGroupID)
 		if (tstatus == -2)
-			result = -5 // thread did not terminate properly
+			print "error: thread did not terminate properly."
+			result = -5
+		endif
+	endif
+	
+	// finalize results
+	nw = numpnts(result_waves)
+	wavenames = ""
+	for (iw = 0; iw < nw; iw += 1)
+		wave /z data = result_waves[iw]
+		if (WaveExists(data))
+			if (nz == 1)
+				redimension /n=(-1, 0, 0) data
+			elseif (nt == 1)
+				redimension /n=(-1, nz, 0) data
+			endif
+			wavenames += nameofwave(data) + ";"
 		endif
-	else
-		for (izt = 0; izt < nzt; izt += 1)
-			KillDataFolder /Z processing_folders[izt]
 	endfor
-	endif
 
 	if (progress)
 		kill_progress_panel()
 	endif
 	
+	setdatafolder base_df
 	return result
 end
 
@@ -1979,12 +2011,12 @@ function adh5_scale_scienta(data)
 			case 1: // Angular45
 				ALow = -45/2
 				AHigh = +45/2
-				AUnit = "°"
+				AUnit = "°"
 				break
 			case 2: // Angular60
 				ALow = -60/2
 				AHigh = +60/2
-				AUnit = "°"
+				AUnit = "°"
 				break
 		endswitch
 	endif

pearl/pearl-elog.ipf

@@ -273,14 +273,14 @@ function elog_init_pearl_templates()
 	
 	// attributes (persistent)
 	// available attributes
-	string /g attributes = "author;project;pgroup;sample;source;task;technique;file;valid;"
+	string /g attributes = "author;project;p-group;sample;source;task;technique;file;valid;"
 	// controls corresponding to attributes
 	// prefix determines the control type: sv_ = setvariable (string), pm_ = popup menu, cb = check box
 	string /g controls = "sv_author;sv_project;sv_pgroup;sv_sample;pm_source;pm_task;pm_technique;sv_file;cb_valid;"
 	// attributes with fixed options, value item declares the options string
 	string /g options = "source=sources;task=tasks;technique=techniques"
 	// attributes which must be defined
-	string /g required_attributes = "author;project;pgroup;sample;source;task;technique;valid"
+	string /g required_attributes = "author;project;sample;source;task;technique;valid"
 
 	// option lists
 	string /g sources = "Manual Entry;PShell;Scienta Data;SScan Data;Prosilica Data;OTF Data;Beamline Status;LEED Data;QMS Data;Matrix Data;Igor Pro;Other"
@@ -293,7 +293,7 @@ function elog_init_pearl_templates()
 	
 	// attributes (persistent)
 	// available attributes
-	string /g attributes = "author;project;pgroup;sample;program;revision;machine;job;experiment;source path;result path;valid"
+	string /g attributes = "author;project;p-group;sample;program;revision;machine;job;experiment;source path;result path;valid"
 	// controls corresponding to attributes
 	// prefix determines the control type: sv_ = setvariable (string), pm_ = popup menu, cb = check box
 	string /g controls = "sv_author;sv_project;sv_pgroup;sv_sample;pm_program;sv_revision;pm_machine;sv_job;sv_experiment;sv_sourcepath;sv_resultpath;cb_valid"

pearl/pearl-scienta-preprocess.ipf

@@ -853,6 +853,148 @@ threadsafe function /wave gauss4_reduction(source, param)
 	return result_waves
 end
 
+threadsafe function /wave gauss6_reduction(source, param)
+	wave source
+	string &param
+		
+	variable nx = dimsize(source, 0)
+	variable ny = dimsize(source, 1)
+	
+	// read parameters
+	variable rngl = NumberByKey("rngl", param, "=", ";")
+	variable rngh = NumberByKey("rngh", param, "=", ";")
+	variable pos1 = NumberByKey("pos1", param, "=", ";")
+	variable wid1 = NumberByKey("wid1", param, "=", ";")
+	variable pos2 = NumberByKey("pos2", param, "=", ";")
+	variable wid2 = NumberByKey("wid2", param, "=", ";")
+	variable pos3 = NumberByKey("pos3", param, "=", ";")
+	variable wid3 = NumberByKey("wid3", param, "=", ";")
+	variable pos4 = NumberByKey("pos4", param, "=", ";")
+	variable wid4 = NumberByKey("wid4", param, "=", ";")
+	variable pos5 = NumberByKey("pos5", param, "=", ";")
+	variable wid5 = NumberByKey("wid5", param, "=", ";")
+	variable pos6 = NumberByKey("pos6", param, "=", ";")
+	variable wid6 = NumberByKey("wid6", param, "=", ";")
+	variable npeaks = NumberByKey("npeaks", param, "=", ";")
+	variable ybox = NumberByKey("ybox", param, "=", ";")
+	
+	// prepare curve fit
+	variable ipk
+	make /free xprof
+	adh5_setup_profile(source, xprof, 0)
+	duplicate /free xprof, xprof_sig
+	variable pl = max(x2pnt(xprof, rngl), 0)
+	variable ph = min(x2pnt(xprof, rngh), numpnts(xprof) - 1)
+	
+	make /free /n=(npeaks) peak_coef
+	peak_coef = p * 3 + 2
+	variable n_coef = npeaks * 3 + 2
+	make /free /d /n=(n_coef) w_coef, W_sigma
+	w_coef[0] = {0, 0, 1, pos1, wid1, 1, pos2, wid2, 1, pos3, wid3, 1, pos4, wid4, 1, pos5, wid5, 1, pos6, wid6}
+	redimension /n=(n_coef)  w_coef, w_sigma
+
+	// text constraints cannot be used in threadsafe functions.
+	// the following matrix-vector forumlation is equivalent to:
+	// make /free /T /N=6 constraints
+	// constraints[0] = {"K2 >= 0", "K5 >= 0", "K8 >= 0", "K11 >= 0", "K1 <= 0", "K0 => 0"}
+	make /free /n=(npeaks + 2, numpnts(w_coef)) cmat
+	make /free /n=(npeaks + 2) cvec
+	cmat = 0
+	cmat[0][0] = -1
+	cmat[1][1] = 1
+	cvec = 0
+
+	string hold = "00"
+	for (ipk=0; ipk < npeaks; ipk += 1)
+		hold += "011"
+		cmat[2 + ipk][2 + ipk*3] = -1
+	endfor
+
+	// prepare output
+	make /free /n=(npeaks * 2) /wave result_waves
+	string s_note
+	for (ipk = 0; ipk < npeaks; ipk += 1)
+		make /free /n=0 pk_int
+		adh5_setup_profile(source, pk_int, 1)
+		pk_int = nan
+		sprintf s_note, "AxisLabelD=peak %u integral", ipk+1
+		Note pk_int, s_note
+		sprintf s_note, "KineticEnergy=%.3f", w_coef[3 + ipk * 3]
+		Note pk_int, s_note
+		result_waves[ipk] = pk_int
+		
+		make /free /n=0 pk_sig
+		adh5_setup_profile(source, pk_sig, 1)
+		pk_sig = nan
+		sprintf s_note, "AxisLabelD=peak %u sigma", ipk+1
+		Note pk_sig, s_note
+		sprintf s_note, "KineticEnergy=%.3f", w_coef[3 + ipk * 3]
+		Note pk_sig, s_note
+		result_waves[ipk + npeaks] = pk_sig
+		
+		waveclear pk_int, pk_sig
+	endfor
+
+	// loop over angle scale
+	variable p0 = 0
+	variable p1 = dimsize(source, 1) - 1
+	variable pp
+	variable wmin
+	variable wmax
+	if (ybox > 1)
+		p0 += ceil((ybox - 1) / 2)
+		p1 -= ceil((ybox - 1) / 2)
+	endif
+	variable V_FitNumIters
+	variable V_FitError
+	
+	for (pp = p0; pp <= p1; pp += 1)
+		// box average
+		xprof = source[p][pp]
+		if (ybox > 1)
+			xprof += source[p][pp-1] + source[p][pp+1]
+		endif
+		xprof_sig = max(sqrt(xprof), 1)
+		xprof /= ybox
+		xprof_sig /= ybox
+		
+		// generate guess
+		wmin = wavemin(xprof)
+		wmax = wavemax(xprof)
+		w_coef[0] = wmin
+		w_coef[1] = 0
+		for (ipk=0; ipk < npeaks; ipk += 1)
+			w_coef[2 + ipk*3] = wmax - wmin
+		endfor
+		
+		V_FitError = 0
+		FuncFit /H=hold /Q /NTHR=1 /N /W=2 MultiGaussLinBG_AO w_coef xprof[pl,ph] /C={cmat, cvec} /I=1 /W=xprof_sig[pl,ph]
+		wave w_sigma
+	
+		// retrieve results, leave them at nan if the fit did not converge
+		if (V_FitNumIters < 40)
+			for (ipk = 0; ipk < npeaks; ipk += 1)
+				wave val = result_waves[ipk]
+				wave sig = result_waves[ipk + npeaks]
+				val[pp] = max(w_coef[peak_coef[ipk]], 0)
+				sig[pp] = max(w_sigma[peak_coef[ipk]], 0)
+			endfor
+		endif
+	endfor
+
+	// calculate integral
+	for (ipk = 0; ipk < npeaks; ipk += 1)
+		wave val = result_waves[ipk]
+		wave sig = result_waves[ipk + npeaks]
+		val *= w_coef[peak_coef[ipk] + 2] * sqrt(pi)
+		sig *= w_coef[peak_coef[ipk] + 2] * sqrt(pi)
+	endfor
+	
+	return result_waves
+end
+
+
+
 
 /// find peak positions for the gauss-fit reduction function
 ///