introduce general config file

+ redesign general config
+ remove obsolete secop/paths.py

Change-Id: Ice08ec37c54b1a6e2e2e6e29fdaaf0bd2dd725dc
Reviewed-on: https://forge.frm2.tum.de/review/c/sine2020/secop/playground/+/27362
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

bin/secop-server

@@ -32,7 +32,7 @@ import mlzlog
 # Add import path for inplace usage
 sys.path.insert(0, path.abspath(path.join(path.dirname(__file__), '..')))
 
-from secop.lib import getGeneralConfig
+from secop.lib import generalConfig
 from secop.server import Server
 
 
@@ -60,15 +60,21 @@ def parseArgv(argv):
     parser.add_argument('-c',
                         '--cfgfiles',
                         action='store',
-                        help="comma separated list of cfg files\n"
+                        help="comma separated list of cfg files,\n"
-                        "defaults to <name_of_the_instance>\n"
+                        "defaults to <name_of_the_instance>.\n"
-                        "cfgfiles given without '.cfg' extension are searched in the configuration directory,"
+                        "cfgfiles given without '.cfg' extension are searched in the configuration directory, "
                         "else they are treated as path names",
                         default=None)
+    parser.add_argument('-g',
+                        '--gencfg',
+                        action='store',
+                        help="full path of general config file,\n"
+                        "defaults to env. variable FRAPPY_CONFIG_FILE\n",
+                        default=None)
     parser.add_argument('-t',
                         '--test',
                         action='store_true',
-                        help='Check cfg files only',
+                        help='check cfg files only',
                         default=False)
     return parser.parse_args(argv)
 
@@ -80,7 +86,8 @@ def main(argv=None):
     args = parseArgv(argv[1:])
 
     loglevel = 'debug' if args.verbose else ('error' if args.quiet else 'info')
-    mlzlog.initLogging('secop', loglevel, getGeneralConfig()['logdir'])
+    generalConfig.init(args.gencfg)
+    mlzlog.initLogging('frappy', loglevel, generalConfig.logdir)
 
     srv = Server(args.name, mlzlog.log, cfgfiles=args.cfgfiles, interface=args.port, testonly=args.test)
 

cfg/generalConfig.cfg

@@ -0,0 +1,5 @@
+[FRAPPY]
+# general config for running in git repo
+logdir = ./log
+piddir = ./pid
+confdir = ./cfg

secop/gui/cfg_editor/utils.py

@@ -29,7 +29,7 @@ from secop.modules import Module
 from secop.params import Parameter
 from secop.properties import Property
 from secop.protocol.interface.tcp import TCPServer
-from secop.server import getGeneralConfig
+from secop.server import generalConfig
 
 uipath = path.dirname(__file__)
 
@@ -106,7 +106,7 @@ def get_file_paths(widget, open_file=True):
 
 def get_modules():
     modules = {}
-    base_path = getGeneralConfig()['basedir']
+    base_path = generalConfig.basedir
     # pylint: disable=too-many-nested-blocks
     for dirname in listdir(base_path):
         if dirname.startswith('secop_'):
@@ -156,7 +156,7 @@ def get_interface_class_from_name(name):
 def get_interfaces():
     # TODO class must be found out like for modules
     interfaces = []
-    interface_path = path.join(getGeneralConfig()['basedir'], 'secop',
+    interface_path = path.join(generalConfig.basedir, 'secop',
                                'protocol', 'interface')
     for filename in listdir(interface_path):
         if path.isfile(path.join(interface_path, filename)) and \

secop/lib/__init__.py

@@ -27,41 +27,81 @@ import socket
 import sys
 import threading
 import traceback
+from configparser import ConfigParser
 from os import environ, path
 
-repodir = path.abspath(path.join(path.dirname(__file__), '..', '..'))
-
-if path.splitext(sys.executable)[1] == ".exe" and not path.basename(sys.executable).startswith('python'):
-    CONFIG = {
-        'piddir': './',
-        'logdir': './log',
-        'confdir': './',
-    }
-elif not path.exists(path.join(repodir, '.git')):
-    CONFIG = {
-        'piddir': '/var/run/secop',
-        'logdir': '/var/log',
-        'confdir': '/etc/secop',
-    }
-else:
-    CONFIG = {
-        'piddir': path.join(repodir, 'pid'),
-        'logdir': path.join(repodir, 'log'),
-        'confdir': path.join(repodir, 'cfg'),
-    }
-# overwrite with env variables SECOP_LOGDIR, SECOP_PIDDIR, SECOP_CONFDIR, if present
-for dirname in CONFIG:
-    CONFIG[dirname] = environ.get('SECOP_%s' % dirname.upper(), CONFIG[dirname])
-
-# this is not customizable
-CONFIG['basedir'] = repodir
-
-# TODO: if ever more general options are need, we should think about a general config file
-
 
 unset_value = object()
 
 
+class GeneralConfig:
+    def __init__(self):
+        self._config = None
+
+    def init(self, configfile=None):
+        cfg = {}
+        mandatory = 'piddir', 'logdir', 'confdir'
+        repodir = path.abspath(path.join(path.dirname(__file__), '..', '..'))
+        # create default paths
+        if path.splitext(sys.executable)[1] == ".exe" and not path.basename(sys.executable).startswith('python'):
+            # special MS windows environment
+            cfg.update(piddir='./', logdir='./log', confdir='./')
+        elif path.exists(path.join(repodir, '.git')):
+            # running from git repo
+            cfg['confdir'] = path.join(repodir, 'cfg')
+            # take logdir and piddir from <repodir>/cfg/generalConfig.cfg
+        else:
+            # running on installed system (typically with systemd)
+            cfg.update(piddir='/var/run/frappy', logdir='/var/log', confdir='/etc/frappy')
+        if configfile is None:
+            configfile = environ.get('FRAPPY_CONFIG_FILE',
+                                     path.join(cfg['confdir'], 'generalConfig.cfg'))
+        if configfile and path.exists(configfile):
+            parser = ConfigParser()
+            parser.optionxform = str
+            parser.read([configfile])
+            # mandatory in a general config file:
+            cfg['logdir'] = cfg['piddir'] = None
+            cfg['confdir'] = path.dirname(configfile)
+            # only the FRAPPY section is relevant, other sections might be used by others
+            for key, value in parser['FRAPPY'].items():
+                if value.startswith('./'):
+                    cfg[key] = path.abspath(path.join(repodir, value))
+                else:
+                    # expand ~ to username, also in path lists separated with ':'
+                    cfg[key] = ':'.join(path.expanduser(v) for v in value.split(':'))
+        else:
+            for key in mandatory:
+                cfg[key] = environ.get('FRAPPY_%s' % key.upper(), cfg[key])
+        missing_keys = [key for key in mandatory if cfg[key] is None]
+        if missing_keys:
+            if path.exists(configfile):
+                raise KeyError('missing value for %s in %s' % (' and '.join(missing_keys), configfile))
+            raise FileNotFoundError(configfile)
+        # this is not customizable
+        cfg['basedir'] = repodir
+        self._config = cfg
+
+    def __getitem__(self, key):
+        try:
+            return self._config[key]
+        except TypeError:
+            raise TypeError('generalConfig.init() has to be called first') from None
+
+    def get(self, key, default=None):
+        try:
+            return self.__getitem__(key)
+        except KeyError:
+            return default
+
+    def __getattr__(self, key):
+        """goodie: use generalConfig.<key> instead of generalConfig.get('<key>')"""
+        return self.get(key)
+
+
+generalConfig = GeneralConfig()
+
+
 class lazy_property:
     """A property that calculates its value only once."""
 
@@ -252,10 +292,6 @@ def getfqdn(name=''):
     return socket.getfqdn(name)
 
 
-def getGeneralConfig():
-    return CONFIG
-
-
 def formatStatusBits(sword, labels, start=0):
     """Return a list of labels according to bit state in `sword` starting
     with bit `start` and the first label in `labels`.

secop/paths.py

@@ -1,32 +0,0 @@
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-#
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Pathes. how to find what and where..."""
-
-
-import sys
-from os import path
-
-basepath = path.abspath(path.join(sys.path[0], '..'))
-etc_path = path.join(basepath, 'etc')
-pid_path = path.join(basepath, 'pid')
-log_path = path.join(basepath, 'log')
-sys.path[0] = path.join(basepath, 'src')

secop/persistent.py

@@ -55,7 +55,7 @@ class MyClass(PersistentMixin, ...):
 import os
 import json
 
-from secop.lib import getGeneralConfig
+from secop.lib import generalConfig
 from secop.datatypes import EnumType
 from secop.params import Parameter, Property, Command
 from secop.modules import HasAccessibles
@@ -69,7 +69,7 @@ class PersistentParam(Parameter):
 class PersistentMixin(HasAccessibles):
     def __init__(self, *args, **kwds):
         super().__init__(*args, **kwds)
-        persistentdir = os.path.join(getGeneralConfig()['logdir'], 'persistent')
+        persistentdir = os.path.join(generalConfig.logdir, 'persistent')
         os.makedirs(persistentdir, exist_ok=True)
         self.persistentFile = os.path.join(persistentdir, '%s.%s.json' % (self.DISPATCHER.equipment_id, self.name))
         self.initData = {}

secop/server.py

@@ -33,7 +33,7 @@ import traceback
 from collections import OrderedDict
 
 from secop.errors import ConfigError, SECoPError
-from secop.lib import formatException, get_class, getGeneralConfig
+from secop.lib import formatException, get_class, generalConfig
 from secop.modules import Attached
 from secop.params import PREDEFINED_ACCESSIBLES
 
@@ -89,7 +89,6 @@ class Server:
         ...
         """
         self._testonly = testonly
-        cfg = getGeneralConfig()
 
         self.log = parent_logger.getChild(name, True)
         if not cfgfiles:
@@ -114,22 +113,21 @@ class Server:
         if ambiguous_sections:
             self.log.warning('ambiguous sections in %s: %r' % (cfgfiles, tuple(ambiguous_sections)))
         self._cfgfiles = cfgfiles
-        self._pidfile = os.path.join(cfg['piddir'], name + '.pid')
+        self._pidfile = os.path.join(generalConfig.piddir, name + '.pid')
 
     def loadCfgFile(self, cfgfile):
         if not cfgfile.endswith('.cfg'):
             cfgfile += '.cfg'
-        cfg = getGeneralConfig()
         if os.sep in cfgfile:  # specified as full path
             filename = cfgfile if os.path.exists(cfgfile) else None
         else:
-            for filename in [os.path.join(d, cfgfile) for d in cfg['confdir'].split(os.pathsep)]:
+            for filename in [os.path.join(d, cfgfile) for d in generalConfig.confdir.split(os.pathsep)]:
                 if os.path.exists(filename):
                     break
             else:
                 filename = None
         if filename is None:
-            raise ConfigError("Couldn't find cfg file %r in %s" % (cfgfile, cfg['confdir']))
+            raise ConfigError("Couldn't find cfg file %r in %s" % (cfgfile, generalConfig.confdir))
         self.log.debug('Parse config file %s ...' % filename)
         result = OrderedDict()
         parser = configparser.ConfigParser()