igor 8 compatibility

README.md

@@ -14,9 +14,10 @@ PEARL Procedures should be installed according to the regular Igor Pro guideline
 - Find the `HDF5.XOP` (`HDF5-64.xop` for Igor 7 64-bit) extension in the `Igor Pro Folder` under `More Extensions/File Loaders` (`More Extensions (64-bit)/File Loaders`), create a shortcut, and move the shortcut to the `Igor Extensions` folder next to your `User Procedures` folder.
 - Find the `HDF5 Help.ihf` next to `HDF5.XOP`, create a shortcut, and move the shortcut to the `Igor Help Files` folder next to your `User Procedures` folder.
 
-PEARL Procedures has been tested under Igor Pro version 6.37 (32-bit). Older versions prior to 6.36 are not be compatible. Please update to the latest Igor Pro 6 version before reporting any problems.
-
-PEARL Procedures compiles under Igor 7.00. Some features, in particular 3D graphics, may not work properly.
+PEARL Procedures are compatible with Igor Pro versions 6-8, 32-bit and 64-bit.
+They are not compatible with Igor 6.36 and older.
+As long as no Igor 8 specific features are used, the produced experiment files remain compatible with Igor 6.
+Some features, in particular 3D graphics, may not work properly under Igor 7 and newer.
 
 
 License
@@ -40,6 +41,11 @@ Copyright 2009-2019 by [Paul Scherrer Institut](http://www.psi.ch)
 Release Notes
 =============
 
+## rev-distro-2.1.0
+
+- Check compatibility of major features with Igor 8.
+- pshell-import does not apply a detector sensitivity scaling any more. The returned intensities have arbitrary units.
+
 ## rev-distro-2.0.3
 
 - The interpolate_hemi_scan function now requires a projection argument unless stereographic projection is desired.

pearl/pearl-area-display.ipf

@@ -2,6 +2,7 @@
 #pragma IgorVersion = 6.2
 #pragma ModuleName = PearlAreaDisplay
 #pragma version = 1.04
+#include "pearl-compat"
 
 /// @file
 /// @brief visualization tools for 2D and 3D data.
@@ -63,7 +64,7 @@ static function /s graphname_from_dfref(df, prefix)
 	name = ReplaceString("root:", name, "")
 	name = name[0, strlen(name) - 2]
 	name = ReplaceString(" ", name, "")
-	name = CleanupName(prefix + name, 0)
+	name = PearlCleanupName(prefix + name)
 	if (CheckName(name, 6))
 		name = UniqueName(name, 6, 0)
 	endif
@@ -179,7 +180,7 @@ function /s ad_display_profiles(image, [filter])
 	dfref imagedf = GetWavesDataFolderDFR(image)
 	string s_imagedf = GetDataFolder(1, imagedf)
 	setdatafolder imagedf
-	string s_viewdf = CleanupName("view_" + NameOfWave(image), 0)
+	string s_viewdf = PearlCleanupName("view_" + NameOfWave(image))
 	newdatafolder /o/s $s_viewdf
 	dfref viewdf = GetDataFolderDFR()
 	s_viewdf = GetDataFolder(1, viewdf)
@@ -566,7 +567,7 @@ static function /df make_view_folder(source)
 	dfref imagedf = GetWavesDataFolderDFR(source)
 	string s_imagedf = GetDataFolder(1, imagedf)
 	setdatafolder imagedf
-	string s_viewdf = CleanupName("view_" + NameOfWave(source), 0)
+	string s_viewdf = PearlCleanupName("view_" + NameOfWave(source))
 	newdatafolder /o/s $s_viewdf
 	dfref viewdf = GetDataFolderDFR()
 
@@ -590,7 +591,7 @@ static function /df get_view_folder(source)
 	dfref imagedf = GetWavesDataFolderDFR(source)
 	dfref viewdf
 	setdatafolder imagedf
-	string s_viewdf = CleanupName("view_" + NameOfWave(source), 0)
+	string s_viewdf = PearlCleanupName("view_" + NameOfWave(source))
 	if (DataFolderExists(s_viewdf))
 		setdatafolder $s_viewdf
 		viewdf = GetDataFolderDFR()
@@ -1362,7 +1363,7 @@ function /s ad_display_slice(data)
 	if (exists("slice_graphname") != 2)
 		string /g slice_graphname = ""
 	endif
-	string /g slice_wavename = CleanupName("slice_" + NameOfWave(data), 0)
+	string /g slice_wavename = PearlCleanupName("slice_" + NameOfWave(data))
 	svar graphname = slice_graphname
 	svar slicename = slice_wavename
 

pearl/pearl-area-import.ipf

@@ -2,6 +2,7 @@
 #pragma IgorVersion = 6.2
 #pragma ModuleName = PearlAreaImport
 #include <HDF5 Browser>
+#include "pearl-compat"
 #include "pearl-gui-tools"
 
 // copyright (c) 2013-18 Paul Scherrer Institut
@@ -46,7 +47,7 @@ static function BeforeFileOpenHook(refNum,fileName,path,type,creator,kind)
 
 	//PathInfo $path
 	//string FilePath = s_path + filename
-	string NickName = CleanupName(ParseFilePath(3, FileName, ":", 0, 0), 0)
+	string NickName = PearlCleanupName(ParseFilePath(3, FileName, ":", 0, 0))
 	string FileExt = LowerStr(ParseFilePath(4, FileName, ":", 0, 0))
 	string result = ""
   
@@ -149,7 +150,7 @@ function /s ad_suggest_foldername(filename, [ignoredate,sourcename,unique])
 			sprintf nickname, "%s_%s_%s", sourcename, datepart, indexpart
 		endif
 	else
-		nickname = CleanupName(basename, 0)
+		nickname = PearlCleanupName(basename)
 	endif
 	
 	if (unique && CheckName(nickname, 11))
@@ -1175,6 +1176,243 @@ function /s adh5_test_reduction_func(source, reduction_func, reduction_param, re
 	return reduction_param
 end
 
+/// reduce a three-dimensional intensity distribution
+///
+/// this function reduces a three-dimensional intensity distribution
+/// to a two-dimensional intensity map.
+/// the given reduction function is applied once on each Z section.
+///
+/// @param	source			source wave.
+///							three-dimensional intensity distribution (image).
+///							the scales are carried over to the result waves.
+///
+/// @param	reduction_func	name of the reduction function to apply to the source data.
+///
+/// @param	reduction_param	string with reduction parameters as required by the specific reduction function.
+///
+/// @param	result_prefix		name prefix of result waves.
+///							a numeric index is appended to distinguish the results.
+///							the index starts at 1. existing waves are overwritten.
+///
+///
+function adh5_reduce_brick(source, reduction_func, reduction_param, result_prefix, [progress, nthreads])
+	wave source
+	funcref adh5_default_reduction reduction_func
+	string reduction_param
+	string result_prefix
+	
+	variable progress
+	variable nthreads
+		
+	if (ParamIsDefault(progress))
+		progress = 1
+	endif
+	if (ParamIsDefault(nthreads))
+		nthreads = -1
+	endif
+	variable result = 0
+	
+	// nx and nz are the image dimensions
+	variable nx, ny, nz, nt
+	nx = dimsize(source, 0)
+	ny = dimsize(source, 1)
+	nz = dimsize(source, 2)
+	// force 4th dimension to singleton (ad_extract_slab handles 3 dimensions only)
+	nt = 0
+	
+	variable nzt = max(nz, 1) * max(nt, 1)
+	variable izt
+	
+	// set up multi threading
+	if (nthreads < 0)
+		nthreads = ThreadProcessorCount
+	endif
+	if (nthreads > 0)
+		variable threadGroupID = ThreadGroupCreate(nthreads)
+		variable ithread
+		for (ithread = 0; ithread < nthreads; ithread += 1)
+			ThreadStart threadGroupID, ithread, reduce_slab_worker(reduction_func)
+		endfor
+	else
+		make /n=(nzt) /df /free processing_folders
+	endif
+	
+	if (progress)
+		display_progress_panel("data reduction", "extracting data (step 1 of 2)...", nzt)
+	endif
+	
+	variable iz, it
+	string dfname
+	variable iw, nw
+	string sw
+	make /n=0 /free /wave result_waves
+	
+	izt = 0
+	for (iz = 0; iz < max(nz, 1); iz += 1)
+		for (it = 0; it < max(nt, 1); it += 1)
+			dfname = "processing_" + num2str(izt)
+			newdatafolder /s $dfname
+			ad_extract_slab(source, nan, nan, nan, nan, iz, iz, "image", pscale=1)
+			wave image
+
+			// send to processing queue
+			variable /g r_index = iz
+			variable /g s_index = it
+			string /g func_param = reduction_param
+
+			if (nthreads > 0)
+				WaveClear image
+				ThreadGroupPutDF threadGroupID, :
+			else
+				processing_folders[izt] = GetDataFolderDFR()
+				string param = reduction_param
+				wave /wave reduced_waves = reduction_func(image, param)
+				variable /g func_result = numpnts(reduced_waves)
+				adh5_get_result_waves(reduced_waves, "redw_", 0)
+				WaveClear image, reduced_waves
+				setdatafolder ::
+			endif
+			
+			izt += 1
+			// progress window
+			if (progress)
+				if (update_progress_panel(izt))
+					result = -4 // user abort
+					break
+				endif
+			endif
+		endfor
+	endfor
+	
+	if (progress)
+		update_progress_panel(0, message="processing data (step 2 of 2)...")
+	endif
+
+	dfref dfr
+	for (izt = 0; (izt < nzt) && (result == 0); izt += 1)
+		if (nthreads > 0)
+			do
+				dfr = ThreadGroupGetDFR(threadGroupID, 1000)
+				if (DatafolderRefStatus(dfr) != 0)
+					break
+				endif
+				if (progress)
+					if (update_progress_panel(izt))
+						result = -4 // user abort
+						break
+					endif
+				endif
+			while (1)
+		else
+			dfr = processing_folders[izt]
+			if (progress)
+				if (update_progress_panel(izt))
+					result = -4 // user abort
+					break
+				endif
+			endif
+		endif
+		
+		if (result != 0)
+			break
+		endif
+		
+		nvar rr = dfr:r_index
+		nvar ss = dfr:s_index
+		nvar func_result = dfr:func_result
+
+		if (func_result < 1)
+			result = -3 // dimension reduction error
+			break
+		endif
+
+		if (numpnts(result_waves) == 0)
+			redimension /n=(func_result) result_waves
+			for (iw = 0; iw < func_result; iw += 1)
+				sw = "redw_" + num2str(iw)
+				wave profile = dfr:$sw
+				sw = "ReducedData" + num2str(iw+1)
+				make /n=(dimsize(profile, 0), nz, nt) /d /o $sw
+				wave data = $sw
+				setdimlabel 0, -1, $getdimlabel(profile, 0, -1), data
+				setscale /p x dimoffset(profile, 0), dimdelta(profile, 0), waveunits(profile, 0), data
+				setscale /p y dimoffset(source, 2), dimdelta(source, 2), waveunits(source, 2), data
+				setscale /p z dimoffset(source, 3), dimdelta(source, 3), waveunits(source, 3), data
+				setscale d 0, 0, waveunits(profile, -1), data
+				result_waves[iw] = data
+			endfor
+		endif
+		for (iw = 0; iw < func_result; iw += 1)
+			sw = "redw_" + num2str(iw)
+			wave profile = dfr:$sw
+			wave data = result_waves[iw]
+			data[][rr][ss] = profile[p]
+		endfor
+	endfor
+	
+	if (nthreads > 0)
+		variable tstatus = ThreadGroupRelease(threadGroupID)
+		if (tstatus == -2)
+			result = -5 // thread did not terminate properly
+		endif
+	else
+		for (izt = 0; izt < nzt; izt += 1)
+			KillDataFolder /Z processing_folders[izt]
+		endfor
+	endif
+	
+	if (progress)
+		kill_progress_panel()
+	endif
+	
+	return result
+end
+
+/// thread worker for adh5_reduce_brick
+///
+/// this function polls job data folders from thread group 0
+/// and calls the reduction function on their contents.
+/// the result waves have prefix "redw_" and are saved in the job folder.
+///
+threadsafe static function reduce_brick_worker(reduction_func)
+	funcref adh5_default_reduction reduction_func
+	do
+		// wait for job from main thread
+		do
+			dfref dfr = ThreadGroupGetDFR(0, 1000)
+			if (DataFolderRefStatus(dfr) == 0)
+				if (GetRTError(2))
+					return 0 // no more jobs
+				endif
+			else
+				break
+			endif
+		while (1)
+
+		// get input data
+		wave image = dfr:image
+		svar func_param = dfr:func_param
+		nvar rr = dfr:r_index
+		nvar ss = dfr:s_index
+		
+		// do the work
+		newdatafolder /s outDF
+		variable /g r_index = rr
+		variable /g s_index = ss
+		string param = func_param
+		wave /wave reduced_waves = reduction_func(image, param)
+		variable /g func_result = numpnts(reduced_waves)
+
+		// send output to queue and clean up
+		adh5_get_result_waves(reduced_waves, "redw_", 0)
+		WaveClear image, reduced_waves
+		ThreadGroupPutDF 0, :
+		KillDataFolder dfr
+	while (1)
+	
+	return 0
+end
+
 /// copy waves from wave reference wave into current data folder
 ///
 /// this function copies waves that are referenced in a wave reference wave into the current data folder.

pearl/pearl-arpes.ipf

@@ -6,6 +6,7 @@
 #include "pearl-area-profiles" // data processing for multi-dimensional datasets
 #include "pearl-area-import" // import data files generated by area detector software
 #include "pearl-pshell-import"
+#include "pearl-compat" // compatibility with igor 6
 #include "pearl-data-explorer" // preview and import panel for PEARL data
 #include "pearl-anglescan-process" // angle scan (XPD) processing functions
 #include "pearl-anglescan-panel" // panel interface to angle scan processing
@@ -16,18 +17,9 @@
 #include "pearl-area-live" // live view of area detector
 #include "pearl-epics" // EPICS access under Igor
 #include "pearl-arpes-scans" // run ARPES scans under Igor
+#include "pearl-sample-tracker" // live tracking and adjustment of sample position
 #endif
 
-// $Id$
-//
-// author: matthias.muntwiler@psi.ch
-// Copyright (c) 2012-15 Paul Scherrer Institut
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-//   you may not use this file except in compliance with the License.
-//   You may obtain a copy of the License at
-//   http://www.apache.org/licenses/LICENSE-2.0
-
 /// @file
 /// @brief data acquisition and analysis package for ARPES at PEARL.
 ///
@@ -77,6 +69,7 @@
 /// * pearl-area-live.ipf
 /// * pearl-epics.ipf
 /// * pearl-arpes-scans.ipf
+/// * pearl-sample-tracker.ipf
 ///
 
 /// initializes package data once when the procedure is first loaded

pearl/pearl-compat.ipf

@@ -0,0 +1,57 @@
+#pragma rtGlobals=3		// Use modern global access method and strict wave access.
+#pragma IgorVersion = 6.1
+#pragma ModuleName = PearlCompat
+#pragma version = 1.01
+
+// copyright (c) 2019 Paul Scherrer Institut
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+//   you may not use this file except in compliance with the License.
+//   You may obtain a copy of the License at
+//   http:///www.apache.org/licenses/LICENSE-2.0
+
+/// @file
+/// @brief compatibility procedures for igor 8
+/// @ingroup ArpesPackage
+///
+///
+/// the compatibility procedures ensure that igor experiments
+/// created with the PEARL procedures under igor 8
+/// can be opened with earlier igor versions (>= 6.34).
+///
+/// the following possible issues are addressed:
+///
+/// @arg length of object names
+
+/// @namespace PearlCompat
+/// @brief compatibility procedures for igor 8
+///
+/// PearlCompat is declared in @ref pearl-compat.ipf.
+
+
+// Compatible CleanupName function
+//
+// Igor 8's CleanupName may return long object names (> 31 characters).
+// This breaks compatibility with earlier Igor versions.
+// Experiments that include waves, folders, windows etc. with long names
+// cannot be loaded with an earlier version.
+//
+// This is a drop-in replacement function for CleanupName.
+// In addition to the behaviour of CleanupName,
+// this replacement ensures that names are limited to 31 characters.
+//
+// @param	name	object name to clean up
+//
+// @return		(str) clean object name
+//
+function /s PearlCleanupName(name)
+	string name
+	
+#if IgorVersion() >= 8.00
+	// note: this function is not threadsafe
+	return CleanupName(name, 0, 31)
+#else
+	return CleanupName(name, 0)
+#endif
+
+end

pearl/pearl-data-explorer.ipf

@@ -5,6 +5,7 @@
 #include "pearl-area-import"
 #include "pearl-area-profiles"
 #include "pearl-area-display"
+#include "pearl-compat"
 #include "pearl-pshell-import"
 #if exists("MFR_OpenResultFile")
 #include "pearl-matrix-import"
@@ -826,7 +827,7 @@ static function attributes_notebook(attr_names, attr_values, title)
 	setdatafolder $package_path
 	wave /t/z attr_filter, attr_filter_summary
 
-	string name = CleanupName("nb_" + title[0,28], 0)
+	string name = PearlCleanupName("nb_" + title[0,27])
 	if (WinType(name) == 5)
 		Notebook $name selection={startOfFile, endOfFile}	
 		Notebook $name text=""
@@ -1302,7 +1303,7 @@ function /s itx_suggest_foldername(filename, [ignoredate,sourcename,unique])
 			sprintf nickname, "%s_%s_%s", sourcename, datepart, indexpart
 		endif
 	else
-		nickname = CleanupName(basename, 0)
+		nickname = PearlCleanupName(basename)
 	endif
 	
 	if (unique && CheckName(nickname, 11))

pearl/pearl-pshell-import.ipf

@@ -2,6 +2,7 @@
 #pragma IgorVersion = 6.36
 #pragma ModuleName = PearlPShellImport
 #include <HDF5 Browser>
+#include "pearl-compat"
 #include "pearl-gui-tools"
 #include "pearl-area-import"
 
@@ -83,7 +84,7 @@ strconstant kScientaScalingDatasets = "LensMode;ScientaChannelBegin;ScientaChann
 strconstant kTransposedDatasets = "ScientaImage;"
 
 /// multiply scienta detector intensity by this value to get actual counts.
-constant kDetectorSensitivity = 4
+constant kDetectorSensitivity = 1
 
 /// open a HDF5 file created by the PShell data acquisition program and prepare the data folder.
 ///
@@ -204,7 +205,7 @@ function /s psh5_load_complete(ANickName, APathName, AFileName, [load_data, load
 			sg = StringFromList(ig, s_scanpaths, ";")
 			folder = ReplaceString("/", sg, "")
 			folder = ReplaceString(" ", folder, "")
-			folder = CleanupName(folder, 0)
+			folder = PearlCleanupName(folder)
 			setdatafolder fileDF
 			newdatafolder /s /o $folder
 			psh5_load_scan_complete(fileID, sg, load_data=load_data, load_attr=load_attr)
@@ -606,6 +607,9 @@ end
 /// - Writables (ScanWritables)
 /// - Readables (ScanReadables)
 /// - Steps (ScanSteps)
+/// - Iterations (ScanIterations) - if present (XPSSpectrum script)
+/// - Step Size (ScanStepSize) - if present (XPSSpectrum script)
+/// - Step Time (ScanStepTime) - if present (XPSSpectrum script)
 ///
 /// if they are missing in the file, `ScanDimensions` and `ScanReadables` are set to default values
 /// assuming the file contains a single spectrum.
@@ -649,6 +653,20 @@ function /s psh5_load_scan_meta(fileID, scanpath)
 	endif
 	wavenames = ReplaceString(";;", wavenames, ";")
 
+	// additional attributes from XPSSpectrum.py
+	HDF5LoadData /O /Q /Z /A="Iterations" /N=ScanIterations /TYPE=1 fileID, scanpath
+	if (!v_flag)
+		wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
+	endif
+	HDF5LoadData /O /Q /Z /A="Step Size" /N=ScanStepSize /TYPE=1 fileID, scanpath
+	if (!v_flag)
+		wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
+	endif
+	HDF5LoadData /O /Q /Z /A="Step Time" /N=ScanStepTime /TYPE=1 fileID, scanpath
+	if (!v_flag)
+		wavenames = AddListItem(s_wavenames, wavenames, ";", inf)
+	endif
+
 	return wavenames
 end
 
@@ -1933,7 +1951,7 @@ function /s psh5_load_reduced(ANickName, APathName, AFileName, reduction_func, r
 		scanpath = StringFromList(ig, s_scanpaths)
 		folder = ReplaceString("/", scanpath, "")
 		folder = ReplaceString(" ", folder, "")
-		folder = CleanupName(folder, 0)
+		folder = PearlCleanupName(folder)
 		setdatafolder fileDF
 		newdatafolder /s /o $folder
 		dfref dataDF = GetDataFolderDFR()