support for multiple secop servers

- server port can be given as cmd line argument
- multiple cfg files may be merged on one server

needed for the way how frappy is planned to be used at PSI

+ add --test option in bin/secop-server

Change-Id: I1e77f65891b15a70b191cbac8168e69715ace3dc
Reviewed-on: https://forge.frm2.tum.de/review/c/sine2020/secop/playground/+/22947
Tested-by: JenkinsCodeReview <bjoern_pedersen@frm2.tum.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

bin/secop-server

@@ -46,14 +46,31 @@ def parseArgv(argv):
                           action='store_true', default=False)
     parser.add_argument("name",
                         type=str,
-                        help="Name of the instance.\n"
+                        help="Name of the instance.\n",)
-                        " Uses etc/name.cfg for configuration\n",)
     parser.add_argument('-d',
                         '--daemonize',
                         action='store_true',
                         help='Run as daemon',
                         default=False)
-    return parser.parse_args()
+    parser.add_argument('-p',
+                        '--port',
+                        action='store',
+                        help='server port or uri',
+                        default=None)
+    parser.add_argument('-c',
+                        '--cfgfiles',
+                        action='store',
+                        help="comma separated list of cfg files\n"
+                        "defaults to <name_of_the_instance>\n"
+                        "cfgfiles given without '.cfg' extension are searched in the configuration directory,"
+                        "else they are treated as path names",
+                        default=None)
+    parser.add_argument('-t',
+                        '--test',
+                        action='store_true',
+                        help='Check cfg files only',
+                        default=False)
+    return parser.parse_args(argv)
 
 
 def main(argv=None):
@@ -65,7 +82,7 @@ def main(argv=None):
     loglevel = 'debug' if args.verbose else ('error' if args.quiet else 'info')
     mlzlog.initLogging('secop', loglevel, getGeneralConfig()['logdir'])
 
-    srv = Server(args.name, mlzlog.log)
+    srv = Server(args.name, mlzlog.log, cfgfiles=args.cfgfiles, interface=args.port, testonly=args.test)
 
     if args.daemonize:
         srv.start()

secop/server.py

@@ -29,7 +29,6 @@ import time
 import threading
 import configparser
 from collections import OrderedDict
-
 try:
     from daemon import DaemonContext
     try:
@@ -53,9 +52,9 @@ except ImportError:
 class Server:
     # list allowed section prefixes
     # if mapped dict does not exist -> section need a 'class' option
-    # otherwise a 'type' option is evaluatet and the class from the mapping dict used
+    # otherwise a 'type' option is evaluated and the class from the mapping dict used
     #
-    # IMPORTANT: keep he order! (node MUST be first, as the others are referencing it!)
+    # IMPORTANT: keep the order! (node MUST be first, as the others are referencing it!)
     CFGSECTIONS = [
         # section_prefix, default type, mapping of selectable classes
         ('node', 'std', {'std': "protocol.dispatcher.Dispatcher",
@@ -65,27 +64,71 @@ class Server:
     ]
     _restart = True
 
-    def __init__(self, name, parent_logger=None):
+    def __init__(self, name, parent_logger=None, cfgfiles=None, interface=None, testonly=False):
+        """initialize server
+
+        the configuration is taken either from <name>.cfg or from cfgfiles
+        if cfgfiles is given, also the serverport has to be given.
+        interface is either an uri or a bare serverport number (with tcp as default)
+        """
+        self._testonly = testonly
         cfg = getGeneralConfig()
 
-        # also handle absolut paths
+        self.log = parent_logger.getChild(name, True)
-        if os.path.abspath(name) == name and os.path.exists(name) and \
+        configuration = {k: OrderedDict() for k, _, _ in self.CFGSECTIONS}
-                name.endswith('.cfg'):
+        if interface:
-            self._cfgfile = name
+            try:
-            self._pidfile = os.path.join(cfg['piddir'],
+                typ, interface = str(interface).split('://', 1)
-                                         name[:-4].replace(os.path.sep, '_') + '.pid')
+            except ValueError:
-            name = os.path.basename(name[:-4])
+                typ = 'tcp'
+            try:
+                host, port = interface.split(':', 1)
+            except ValueError:
+                host, port = '0.0.0.0', interface
+            options = {'type': typ, 'bindto': host, 'bindport': port}
+            configuration['interface %s' % options['type']] = options
+        if not cfgfiles:
+            cfgfiles = name
+        for cfgfile in cfgfiles.split(','):
+            if cfgfile.endswith('.cfg') and os.path.exists(cfgfile):
+                filename = cfgfile
             else:
-            self._cfgfile = os.path.join(cfg['confdir'], name + '.cfg')
+                filename = os.path.join(cfg['confdir'], cfgfile + '.cfg')
+            self.mergeCfgFile(configuration, filename)
+        if len(configuration['node']) > 1:
+            description = ['merged node\n']
+            for section, opt in configuration['node']:
+                description.append("--- %s:\n%s\n" % (section[5:], opt['description']))
+            configuration['node'] = {cfgfiles: {'description': '\n'.join(description)}}
+        self._configuration = configuration
+        self._cfgfile = cfgfiles  # used for reference in error messages only
         self._pidfile = os.path.join(cfg['piddir'], name + '.pid')
 
-        self._name = name
+    def mergeCfgFile(self, configuration, filename):
-
+        self.log.debug('Parse config file %s ...' % filename)
-        self.log = parent_logger.getChild(name, True)
+        parser = configparser.ConfigParser()
-
+        parser.optionxform = str
-        self._dispatcher = None
+        if not parser.read([filename]):
-        self._interface = None
+            self.log.error("Couldn't read cfg file %r!" % filename)
-        self._restart_event = threading.Event()
+            raise ConfigError("Couldn't read cfg file %r" % filename)
+        for section, options in parser.items():
+            try:
+                kind, name = section.split(' ', 1)
+                kind = kind.lower()
+                cfgdict = configuration[kind]
+            except (ValueError, KeyError):
+                if section != 'DEFAULT':
+                    self.log.warning('skip unknown section %s' % section)
+                continue
+            opt = dict(options)
+            if name in cfgdict:
+                if kind == 'interface':
+                    opt = dict(type='tcp', bindto='0.0.0.0')
+                    opt.update(options)
+                if opt != cfgdict[name]:
+                    self.log.warning('omit conflicting section %r in %s' % (section, filename))
+            else:
+                cfgdict[name] = dict(options)
 
     def start(self):
         if not DaemonContext:
@@ -110,6 +153,8 @@ class Server:
                 if systemd:
                     systemd.daemon.notify("STATUS=initializing")
                 self._processCfg()
+                if self._testonly:
+                    return
             except Exception:
                 print(formatException(verbose=True))
                 raise
@@ -135,22 +180,11 @@ class Server:
     def _processCfg(self):
         self.log.debug('Parse config file %s ...' % self._cfgfile)
 
-        parser = configparser.ConfigParser()
-        parser.optionxform = str
-
-        if not parser.read([self._cfgfile]):
-            self.log.error('Couldn\'t read cfg file !')
-            raise ConfigError('Couldn\'t read cfg file %r' % self._cfgfile)
-
         for kind, default_type, classmapping in self.CFGSECTIONS:
-            kinds = '%ss' % kind
             objs = OrderedDict()
-            self.__dict__[kinds] = objs
+            self.__dict__['%ss' % kind] = objs
-            for section in parser.sections():
+            for name, options in self._configuration[kind].items():
-                prefix = '%s ' % kind
+                opts = dict(options)
-                if section.lower().startswith(prefix):
-                    name = section[len(prefix):]
-                    opts = dict(item for item in parser.items(section))
                 if 'class' in opts:
                     cls = opts.pop('class')
                 else: