495 lines
16 KiB
Python
495 lines
16 KiB
Python
# *****************************************************************************
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it under
|
|
# the terms of the GNU General Public License as published by the Free Software
|
|
# Foundation; either version 2 of the License, or (at your option) any later
|
|
# version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
|
# details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along with
|
|
# this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#
|
|
# Module authors:
|
|
# Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
|
|
# Markus Zolliker <markus.zolliker@psi.ch>
|
|
# Alexander Zaft <a.zaft@fz-juelich.de>
|
|
#
|
|
# *****************************************************************************
|
|
"""Define helpers"""
|
|
|
|
import importlib
|
|
import linecache
|
|
import re
|
|
import socket
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from configparser import ConfigParser
|
|
from os import environ, path
|
|
from pathlib import Path
|
|
|
|
|
|
SECoP_DEFAULT_PORT = 10767
|
|
|
|
|
|
class GeneralConfig:
|
|
"""generalConfig holds server configuration items
|
|
|
|
generalConfig.init is to be called before starting the server.
|
|
Accessing generalConfig.<key> raises an error, when generalConfig.init is
|
|
not yet called, except when a default for <key> is set.
|
|
For tests and for imports from client code, a module may access generalConfig
|
|
without calling generalConfig.init before. For this, it should call
|
|
generalConfig.set_default on import to define defaults for the needed keys.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._config = None
|
|
self.defaults = {} #: default values. may be set before or after :meth:`init`
|
|
|
|
def init(self, configfile=None):
|
|
"""init default server configuration
|
|
|
|
:param configfile: if present, keys and values from the [FRAPPY] section are read
|
|
|
|
The following locations are searched for the generalConfig.cfg file.
|
|
- command line argument
|
|
- environment variable FRAPPY_CONFIG_FILE
|
|
- git location (../cfg)
|
|
- local location (cwd)
|
|
- global location (/etc/frappy)
|
|
|
|
if a configfile is given, the values from the FRAPPY section are
|
|
overriding above defaults
|
|
|
|
finally, the env. variables FRAPPY_PIDDIR, FRAPPY_LOGDIR and FRAPPY_CONFDIR
|
|
are overriding these values when given
|
|
"""
|
|
|
|
configfile = self._get_file_location(configfile)
|
|
|
|
cfg = {}
|
|
mandatory = 'piddir', 'logdir', 'confdir'
|
|
repodir = Path(__file__).parents[2].expanduser().resolve()
|
|
if configfile:
|
|
parser = ConfigParser()
|
|
parser.optionxform = str
|
|
parser.read([configfile])
|
|
# only the FRAPPY section is relevant, other sections might be used by others
|
|
for key, value in parser['FRAPPY'].items():
|
|
if value.startswith('./'):
|
|
cfg[key] = (repodir / value).absolute()
|
|
else:
|
|
# expand ~ to username, also in path lists separated with ':'
|
|
cfg[key] = ':'.join(path.expanduser(v) for v in value.split(':'))
|
|
if cfg.get('confdir') is None:
|
|
cfg['confdir'] = configfile.parent
|
|
# environment variables will overwrite the config file
|
|
missing_keys = []
|
|
for key in mandatory:
|
|
env = environ.get(f'FRAPPY_{key.upper()}') or cfg.get(key)
|
|
if env is None:
|
|
if self.defaults.get(key) is None:
|
|
missing_keys.append(key)
|
|
else:
|
|
if not isinstance(env, Path):
|
|
if key == 'confdir':
|
|
env = [Path(v) for v in env.split(':')]
|
|
else:
|
|
env = Path(env)
|
|
cfg[key] = env
|
|
if missing_keys:
|
|
if configfile:
|
|
raise KeyError(f"missing value for {' and '.join(missing_keys)} in {configfile}")
|
|
|
|
if len(missing_keys) < 3:
|
|
# user specified at least one env variable already
|
|
missing = ' (missing %s)' % ', '.join('FRAPPY_%s' % k.upper() for k in missing_keys)
|
|
else:
|
|
missing = ''
|
|
raise FileNotFoundError(
|
|
'Could not determine config file location for the general frappy config. '
|
|
f'Provide a config file or all required environment variables{missing}. '
|
|
'For more information, see frappy-server --help.'
|
|
)
|
|
if 'confdir' in cfg and isinstance(cfg['confdir'], Path):
|
|
cfg['confdir'] = [cfg['confdir']]
|
|
# this is not customizable
|
|
cfg['basedir'] = repodir
|
|
self._config = cfg
|
|
|
|
def _get_file_location(self, configfile):
|
|
"""Determining the defaultConfig.cfg location as documented in init()"""
|
|
# given as command line arg
|
|
if configfile and Path(configfile).exists():
|
|
return configfile
|
|
# if not given as argument, check different sources
|
|
# env variable
|
|
fromenv = environ.get('FRAPPY_CONFIG_FILE')
|
|
if fromenv and Path(fromenv).exists():
|
|
return fromenv
|
|
# from ../cfg (there if running from checkout)
|
|
repodir = Path(__file__).parents[2].expanduser().resolve()
|
|
if (repodir / 'cfg' / 'generalConfig.cfg').exists():
|
|
return repodir / 'cfg' / 'generalConfig.cfg'
|
|
localfile = Path.cwd() / 'generalConfig.cfg'
|
|
if localfile.exists():
|
|
return localfile
|
|
# TODO: leave this hardcoded?
|
|
globalfile = Path('/etc/frappy/generalConfig.cfg')
|
|
if globalfile.exists():
|
|
return globalfile
|
|
return None
|
|
|
|
def __getitem__(self, key):
|
|
"""access for keys known to exist
|
|
|
|
:param key: the key (raises an error when key is not available)
|
|
:return: the value
|
|
"""
|
|
try:
|
|
return self._config[key]
|
|
except KeyError:
|
|
return self.defaults[key]
|
|
except TypeError:
|
|
if key in self.defaults:
|
|
# accept retrieving defaults before init
|
|
# e.g. 'lazy_number_validation' in frappy.datatypes
|
|
return self.defaults[key]
|
|
raise TypeError('generalConfig.init() has to be called first') from None
|
|
|
|
def get(self, key, default=None):
|
|
"""access for keys not known to exist"""
|
|
try:
|
|
return self[key]
|
|
except KeyError:
|
|
return default
|
|
|
|
def getint(self, key, default=None):
|
|
"""access and convert to int"""
|
|
try:
|
|
return int(self[key])
|
|
except KeyError:
|
|
return default
|
|
|
|
def __getattr__(self, key):
|
|
"""goodie: use generalConfig.<key> instead of generalConfig.get('<key>')"""
|
|
return self.get(key)
|
|
|
|
@property
|
|
def initialized(self):
|
|
return bool(self._config)
|
|
|
|
def update_defaults(self, **updates):
|
|
"""Set a default value, when there is not already one for each dict entry."""
|
|
for key, value in updates.items():
|
|
self.set_default(key, value)
|
|
|
|
def set_default(self, key, value):
|
|
"""set a default value, in case not set already"""
|
|
if key not in self.defaults:
|
|
self.defaults[key] = value
|
|
|
|
def testinit(self, **kwds):
|
|
"""for test purposes"""
|
|
self._config = kwds
|
|
|
|
|
|
generalConfig = GeneralConfig()
|
|
|
|
|
|
class lazy_property:
|
|
"""A property that calculates its value only once."""
|
|
|
|
def __init__(self, func):
|
|
self._func = func
|
|
self.__name__ = func.__name__
|
|
self.__doc__ = func.__doc__
|
|
|
|
def __get__(self, obj, obj_class):
|
|
if obj is None:
|
|
return self
|
|
obj.__dict__[self.__name__] = self._func(obj)
|
|
return obj.__dict__[self.__name__]
|
|
|
|
|
|
class attrdict(dict):
|
|
"""a normal dict, providing access also via attributes"""
|
|
|
|
def __getattr__(self, key):
|
|
return self[key]
|
|
|
|
def __setattr__(self, key, value):
|
|
self[key] = value
|
|
|
|
|
|
def clamp(_min, value, _max):
|
|
"""return the median of 3 values,
|
|
|
|
i.e. value if min <= value <= max, else min or max depending on which side
|
|
value lies outside the [min..max] interval. This works even when min > max!
|
|
"""
|
|
# return median, i.e. clamp the value between min and max
|
|
return sorted([_min, value, _max])[1]
|
|
|
|
|
|
def get_class(spec):
|
|
"""loads an object given by string in dotted notation (as python would do)
|
|
|
|
import the specified module and get the specified item from it
|
|
examples: 'frappy_demo.lakeshore.TemperatureSensor', 'frappy.modules.Readable.Status'
|
|
|
|
:param spec: a dot-separated list of module names followed by the name of
|
|
a class (or any object) and optionally names of attributes
|
|
:return: the object
|
|
"""
|
|
for maxsplit in range(1, len(spec)):
|
|
# len(spec) is high enough for all cases
|
|
module, *attrs = spec.rsplit('.', maxsplit)
|
|
try:
|
|
obj = importlib.import_module(module)
|
|
break
|
|
except ImportError:
|
|
if '.' in module:
|
|
continue
|
|
raise
|
|
for na, attr in enumerate(attrs):
|
|
try:
|
|
obj = getattr(obj, attr)
|
|
except AttributeError:
|
|
print(na, attrs)
|
|
raise AttributeError(f'{".".join(attrs[:na+1])!r} not found in {module!r}') from None
|
|
return obj
|
|
|
|
|
|
def mkthread(func, *args, **kwds):
|
|
t = threading.Thread(
|
|
name=f'{func.__module__}:{func.__name__}',
|
|
target=func,
|
|
args=args,
|
|
kwargs=kwds)
|
|
t.daemon = True
|
|
t.start()
|
|
return t
|
|
|
|
|
|
def formatExtendedFrame(frame):
|
|
ret = []
|
|
for key, value in frame.f_locals.items():
|
|
try:
|
|
valstr = repr(value)[:256]
|
|
except Exception:
|
|
valstr = '<cannot be displayed>'
|
|
ret.append(' %-20s = %s\n' % (key, valstr))
|
|
ret.append('\n')
|
|
return ret
|
|
|
|
|
|
def formatExtendedTraceback(exc_info=None):
|
|
if exc_info is None:
|
|
etype, value, tb = sys.exc_info()
|
|
else:
|
|
etype, value, tb = exc_info
|
|
ret = ['Traceback (most recent call last):\n']
|
|
while tb is not None:
|
|
frame = tb.tb_frame
|
|
filename = frame.f_code.co_filename
|
|
item = f' File "{filename}", line {tb.tb_lineno}, in {frame.f_code.co_name}\n'
|
|
linecache.checkcache(filename)
|
|
line = linecache.getline(filename, tb.tb_lineno, frame.f_globals)
|
|
if line:
|
|
item = item + f' {line.strip()}\n'
|
|
ret.append(item)
|
|
if filename not in ('<script>', '<string>'):
|
|
ret += formatExtendedFrame(tb.tb_frame)
|
|
tb = tb.tb_next
|
|
ret += traceback.format_exception_only(etype, value)
|
|
return ''.join(ret).rstrip('\n')
|
|
|
|
|
|
def formatExtendedStack(level=1):
|
|
f = sys._getframe(level)
|
|
ret = ['Stack trace (most recent call last):\n\n']
|
|
while f is not None:
|
|
lineno = f.f_lineno
|
|
co = f.f_code
|
|
filename = co.co_filename
|
|
name = co.co_name
|
|
item = f' File "{filename}", line {lineno}, in {name}\n'
|
|
linecache.checkcache(filename)
|
|
line = linecache.getline(filename, lineno, f.f_globals)
|
|
if line:
|
|
item = item + f' {line.strip()}\n'
|
|
ret.insert(1, item)
|
|
if filename != '<script>':
|
|
ret[2:2] = formatExtendedFrame(f)
|
|
f = f.f_back
|
|
return ''.join(ret).rstrip('\n')
|
|
|
|
|
|
def formatException(cut=0, exc_info=None, verbose=False):
|
|
"""Format an exception with traceback, but leave out the first `cut`
|
|
number of frames.
|
|
"""
|
|
if verbose:
|
|
return formatExtendedTraceback(exc_info)
|
|
if exc_info is None:
|
|
typ, val, tb = sys.exc_info()
|
|
else:
|
|
typ, val, tb = exc_info
|
|
res = ['Traceback (most recent call last):\n']
|
|
tbres = traceback.format_tb(tb, sys.maxsize)
|
|
res += tbres[cut:]
|
|
res += traceback.format_exception_only(typ, val)
|
|
return ''.join(res)
|
|
|
|
|
|
HOSTNAMEPART = re.compile(r'^(?!-)[a-z0-9-]{1,63}(?<!-)$', re.IGNORECASE)
|
|
|
|
|
|
def validate_hostname(host):
|
|
"""checks if the rules for valid hostnames are adhered to"""
|
|
if len(host) > 255:
|
|
return False
|
|
for part in host.split('.'):
|
|
if not HOSTNAMEPART.match(part):
|
|
return False
|
|
return True
|
|
|
|
|
|
def validate_ipv4(addr):
|
|
"""check if v4 address is valid."""
|
|
try:
|
|
socket.inet_aton(addr)
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def validate_ipv6(addr):
|
|
"""check if v6 address is valid."""
|
|
try:
|
|
socket.inet_pton(socket.AF_INET6, addr)
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def parse_ipv6_host_and_port(addr, defaultport=SECoP_DEFAULT_PORT):
|
|
""" Parses IPv6 addresses with optional port. See parse_host_port for valid formats"""
|
|
if ']' in addr:
|
|
host, port = addr.rsplit(':', 1)
|
|
return host[1:-1], int(port)
|
|
if '.' in addr:
|
|
host, port = addr.rsplit('.', 1)
|
|
return host, int(port)
|
|
return addr, defaultport
|
|
|
|
|
|
def parse_host_port(host, defaultport=SECoP_DEFAULT_PORT):
|
|
"""Parses hostnames and IP (4/6) addressses.
|
|
|
|
The accepted formats are:
|
|
- a standard hostname
|
|
- base IPv6 or 4 addresses
|
|
- 'hostname:port'
|
|
- IPv4 addresses in the form of 'IPv4:port'
|
|
- IPv6 addresses in the forms '[IPv6]:port' or 'IPv6.port'
|
|
"""
|
|
colons = host.count(':')
|
|
if colons == 0: # hostname/ipv4 without port
|
|
port = defaultport
|
|
elif colons == 1: # hostname or ipv4 with port
|
|
host, port = host.split(':')
|
|
port = int(port)
|
|
else: # ipv6
|
|
host, port = parse_ipv6_host_and_port(host, defaultport)
|
|
if (validate_ipv4(host) or validate_hostname(host) or validate_ipv6(host)) \
|
|
and 0 < port < 65536:
|
|
return host, port
|
|
raise ValueError(f'invalid host {host!r} or port {port}')
|
|
|
|
|
|
# keep a reference to socket to avoid (interpreter) shut-down problems
|
|
def closeSocket(sock, socket=socket): # pylint: disable=redefined-outer-name
|
|
"""Do our best to close a socket."""
|
|
if sock is None:
|
|
return
|
|
try:
|
|
sock.shutdown(socket.SHUT_RDWR)
|
|
except socket.error:
|
|
pass
|
|
try:
|
|
sock.close()
|
|
except socket.error:
|
|
pass
|
|
|
|
|
|
def getfqdn(name=''):
|
|
"""Get fully qualified hostname."""
|
|
return socket.getfqdn(name)
|
|
|
|
|
|
def formatStatusBits(sword, labels, start=0):
|
|
"""Return a list of labels according to bit state in `sword` starting
|
|
with bit `start` and the first label in `labels`.
|
|
"""
|
|
result = []
|
|
for i, lbl in enumerate(labels, start):
|
|
if sword & (1 << i) and lbl:
|
|
result.append(lbl)
|
|
return result
|
|
|
|
|
|
class UniqueObject:
|
|
def __init__(self, name):
|
|
self.name = name
|
|
|
|
def __repr__(self):
|
|
return self.name
|
|
|
|
|
|
def merge_status(*args):
|
|
"""merge status
|
|
|
|
for combining stati of different mixins
|
|
- the status with biggest code wins
|
|
- texts matching maximal code are joined with ', '
|
|
- if texts already contain ', ', it is considered as composed by
|
|
individual texts and duplication is avoided. when commas are used
|
|
for other purposes, the behaviour might be surprising
|
|
"""
|
|
maxcode = max(a[0] for a in args)
|
|
merged = [a[1] for a in args if a[0] == maxcode and a[1]]
|
|
# use dict instead of set for preserving order
|
|
merged = {m: True for mm in merged for m in mm.split(', ')}
|
|
return maxcode, ', '.join(merged)
|
|
|
|
|
|
class _Raiser:
|
|
def __init__(self, modname):
|
|
self.modname = modname
|
|
|
|
def __getattr__(self, name):
|
|
# Just retry the import, it will give the most useful exception.
|
|
__import__(self.modname)
|
|
|
|
def __bool__(self):
|
|
return False
|
|
|
|
|
|
def delayed_import(modname):
|
|
"""Import a module, and return an object that raises a delayed exception
|
|
on access if it failed.
|
|
"""
|
|
try:
|
|
module = __import__(modname, None, None, ['*'])
|
|
except Exception:
|
|
return _Raiser(modname)
|
|
return module
|