update public distribution

based on internal repository c9a2ac8 2019-01-03 16:04:57 +0100
tagged rev-master-2.0.0

pmsco/data.py

@@ -1,21 +1,29 @@
 """
 @package pmsco.data
-import, export, evaluation of msc data
+import, export, evaluation of msc data.
+
+this module provides common functions for loading/saving and manipulating PED scan data sets.
 
 @author Matthias Muntwiler
 
-@copyright (c) 2015 by Paul Scherrer Institut @n
+@copyright (c) 2015-17 by Paul Scherrer Institut @n
 Licensed under the Apache License, Version 2.0 (the "License"); @n
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
   http://www.apache.org/licenses/LICENSE-2.0
 """
 
-import os
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
 import logging
 import numpy as np
+import os
 import scipy.optimize as so
-import loess.loess as loess
+
+from pmsco.compat import open
+import pmsco.loess.loess as loess
 
 logger = logging.getLogger(__name__)
 
@@ -149,7 +157,7 @@ def load_edac_pd(filename, int_column=-1, energy=0.0, theta=0.0, phi=0.0, fixed_
     data[i]['i'] = selected intensity column
     @endverbatim
     """
-    with open(filename, 'r') as f:
+    with open(filename, "r") as f:
         header1 = f.readline().strip()
         header2 = f.readline().strip()
     if not header1 == '--- scan PD':
@@ -259,19 +267,38 @@ def load_data(filename, dtype=None):
         DTYPE_EI, DTYPE_ETPI, DTYPE_ETPIS, DTYPE_ETPAI, or DTYPE_ETPAIS.
         by default, the function uses the extension to determine the data type.
         the actual type can be read from the dtype attribute of the returned array.
+        if the extension is missing, DTYPE_EI is assumed.
         
     @return one-dimensional numpy structured ndarray with data
+
+    @raise IOError if the file cannot be read.
+
+    @raise IndexError if the number of columns is lower than expected based on the dtype or extension.
     """
     if not dtype:
         (root, ext) = os.path.splitext(filename)
-        datatype = ext[1:].upper()
-        dtype = DTYPES[datatype]
+        ext_type = ext[1:].upper()
+        try:
+            dtype = DTYPES[ext_type]
+        except KeyError:
+            dtype = DTYPE_EI
 
     data = np.loadtxt(filename, dtype=dtype)
     sort_data(data)
     return data
 
 
+def format_extension(data):
+    """
+    format the file extension based on the contents of an array.
+
+    @param data ETPI-like structured numpy.ndarray.
+
+    @return: file extension string including the leading period.
+    """
+    return "." + "".join(data.dtype.names)
+
+
 def save_data(filename, data):
     """
     save column data (ETPI, and the like) to a text file.
@@ -331,20 +358,24 @@ def restructure_data(data, dtype=DTYPE_ETPAIS, defaults=None):
         undefined fields are initialized to zero.
         if the parameter is unspecified, all fields are initialized to zero.
 
-    @return: re-structured numpy array
+    @return: re-structured numpy array or
+        @c data if the new and original data types are the same.
     """
-    new_data = np.zeros(data.shape, dtype=dtype)
-    fields = [dt[0] for dt in dtype if dt[0] in data.dtype.names]
+    if data.dtype == dtype:
+        return data
+    else:
+        new_data = np.zeros(data.shape, dtype=dtype)
+        fields = [dt[0] for dt in dtype if dt[0] in data.dtype.names]
 
-    if defaults is not None:
-        for field, value in defaults.iteritems():
-            if field in new_data.dtype.names:
-                new_data[field] = value
+        if defaults is not None:
+            for field, value in defaults.items():
+                if field in new_data.dtype.names:
+                    new_data[field] = value
 
-    for field in fields:
-        new_data[field] = data[field]
+        for field in fields:
+            new_data[field] = data[field]
 
-    return new_data
+        return new_data
 
 
 def common_dtype(scans):
@@ -584,7 +615,7 @@ def calc_modfunc_mean(data):
     return modf
 
 
-def calc_modfunc_loess(data):
+def calc_modfunc_loess(data, smth=0.4):
     """
     calculate the modulation function using LOESS (locally weighted regression) smoothing.
 
@@ -609,9 +640,11 @@ def calc_modfunc_loess(data):
         the modulation function is calculated for the finite-valued scan points.
         NaNs are ignored and do not affect the finite values.
 
-    @return copy of the data array with the modulation function in the 'i' column.
+    @param smth: size of the smoothing window relative to the size of the scan.
+        reasonable values are between 0.2 and 0.5.
+        the default value 0.4 has been found to work in many cases.
 
-    @todo is a fixed smoothing factor of 0.5 okay?
+    @return copy of the data array with the modulation function in the 'i' column.
     """
     sel = np.isfinite(data['i'])
     _data = data[sel]
@@ -626,7 +659,7 @@ def calc_modfunc_loess(data):
         factors = [_data[axis] for axis in scan_mode]
         lo.set_x(np.hstack(tuple(factors)))
         lo.set_y(_data['i'])
-        lo.model.span = 0.5
+        lo.model.span = smth
         loess.loess(lo)
 
         modf['i'][sel] = lo.get_fitted_residuals() / lo.get_fitted_values()