From 64a3bf534b958f1c18a8d024b84ed3c4e5d8803b Mon Sep 17 00:00:00 2001 From: Markus Zolliker Date: Fri, 1 May 2020 15:02:41 +0200 Subject: [PATCH] change cfg file format config file format change: The section names no longer contain a space, the are either bare module names or 'NODE' or 'INTERFACE' (capitalized in order to distingish from module names). The present code still accepts the old form. Moving to the 'toml' format was considered too, but this needs some more investigations. The necessary code changes would be limited to the method Server.loadCfgFile. Change-Id: I6020058c9dcc4c1cbf38f5b9e8f67e9aad670183 Reviewed-on: https://forge.frm2.tum.de/review/c/sine2020/secop/playground/+/23031 Reviewed-by: Enrico Faulhaber Reviewed-by: Markus Zolliker Tested-by: JenkinsCodeReview --- cfg/ppms.cfg | 45 +++---- secop/protocol/dispatcher.py | 2 +- secop/protocol/interface/tcp.py | 37 ++---- secop/server.py | 228 ++++++++++++++++---------------- 4 files changed, 145 insertions(+), 167 deletions(-) diff --git a/cfg/ppms.cfg b/cfg/ppms.cfg index 0316d92..8f8a927 100644 --- a/cfg/ppms.cfg +++ b/cfg/ppms.cfg @@ -1,123 +1,122 @@ -[node PPMS.psi.ch] +[NODE] +id = PPMS.psi.ch description = PPMS at PSI -[interface tcp] -type = tcp -bindto = 0.0.0.0 -bindport = 5000 +[INTERFACE] +uri = tcp://5000 -[module tt] +[tt] class = secop_psi.ppms.Temp description = main temperature iodev = ppms -[module mf] +[mf] class = secop_psi.ppms.Field target.min = -9 target.max = 9 .description = magnetic field .iodev = ppms -[module pos] +[pos] class = secop_psi.ppms.Position .description = sample rotator .iodev = ppms -[module lev] +[lev] class = secop_psi.ppms.Level .description = helium level .iodev = ppms -[module chamber] +[chamber] class = secop_psi.ppms.Chamber .description = chamber state .iodev = ppms -[module r1] +[r1] class = secop_psi.ppms.BridgeChannel .description = resistivity channel 1 .no = 1 value.unit = Ohm .iodev = ppms -[module r2] +[r2] class = secop_psi.ppms.BridgeChannel .description = resistivity channel 2 .no = 2 value.unit = Ohm .iodev = ppms -[module r3] +[r3] class = secop_psi.ppms.BridgeChannel .description = resistivity channel 3 .no = 3 value.unit = Ohm .iodev = ppms -[module r4] +[r4] class = secop_psi.ppms.BridgeChannel .description = resistivity channel 4 .no = 4 value.unit = Ohm .iodev = ppms -[module i1] +[i1] class = secop_psi.ppms.Channel .description = current channel 1 .no = 1 value.unit = uA .iodev = ppms -[module i2] +[i2] class = secop_psi.ppms.Channel .description = current channel 2 .no = 2 value.unit = uA .iodev = ppms -[module i3] +[i3] class = secop_psi.ppms.Channel .description = current channel 3 .no = 3 value.unit = uA .iodev = ppms -[module i4] +[i4] class = secop_psi.ppms.Channel .description = current channel 4 .no = 4 value.unit = uA .iodev = ppms -[module v1] +[v1] class = secop_psi.ppms.DriverChannel .description = voltage channel 1 .no = 1 value.unit = V .iodev = ppms -[module v2] +[v2] class = secop_psi.ppms.DriverChannel .description = voltage channel 2 .no = 2 value.unit = V .iodev = ppms -[module tv] +[tv] class = secop_psi.ppms.UserChannel .description = VTI temperature enabled = 1 value.unit = K .iodev = ppms -[module ts] +[ts] class = secop_psi.ppms.UserChannel .description = sample temperature enabled = 1 value.unit = K .iodev = ppms -[module ppms] +[ppms] class = secop_psi.ppms.Main .description = the main and poller module .class_id = QD.MULTIVU.PPMS.1 diff --git a/secop/protocol/dispatcher.py b/secop/protocol/dispatcher.py index 4a0b9b3..cd2360e 100644 --- a/secop/protocol/dispatcher.py +++ b/secop/protocol/dispatcher.py @@ -64,7 +64,7 @@ class Dispatcher: def __init__(self, name, logger, options, srv): # to avoid errors, we want to eat all options here - self.equipment_id = name + self.equipment_id = options.pop('id', name) self.nodeprops = {} for k in list(options): self.nodeprops[k] = options.pop(k) diff --git a/secop/protocol/interface/tcp.py b/secop/protocol/interface/tcp.py index 6e9463b..3f1a3fa 100644 --- a/secop/protocol/interface/tcp.py +++ b/secop/protocol/interface/tcp.py @@ -26,11 +26,11 @@ import socket import collections import socketserver -from secop.datatypes import StringType, IntRange, BoolType +from secop.datatypes import StringType, BoolType from secop.errors import SECoPError from secop.lib import formatException, \ formatExtendedStack, formatExtendedTraceback -from secop.properties import HasProperties, Property +from secop.properties import Property from secop.protocol.interface import decode_msg, encode_msg_frame, get_msg from secop.protocol.messages import ERRORPREFIX, \ HELPREPLY, HELPREQUEST, HelpMessage @@ -187,42 +187,27 @@ class TCPRequestHandler(socketserver.BaseRequestHandler): self.request.close() -class TCPServer(HasProperties, socketserver.ThreadingTCPServer): +class TCPServer(socketserver.ThreadingTCPServer): daemon_threads = True allow_reuse_address = True - properties = { - 'bindto': Property('hostname or ip address for binding', StringType(), - default='localhost:%d' % DEF_PORT, export=False), - 'bindport': Property('port number to bind', IntRange(1, 65535), - default=DEF_PORT, export=False), + # for cfg-editor + configurables = { + 'uri': Property('hostname or ip address for binding', StringType(), + default='tcp://%d' % DEF_PORT, export=False), 'detailed_errors': Property('Flag to enable detailed Errorreporting.', BoolType(), default=False, export=False), } - # XXX: create configurables from Metaclass! - configurables = properties - def __init__(self, name, logger, options, srv): # pylint: disable=super-init-not-called self.dispatcher = srv.dispatcher self.name = name self.log = logger - # do not call HasProperties.__init__, as this will supercall ThreadingTCPServer - self.initProperties() - bindto = options.pop('bindto', 'localhost') - bindport = int(options.pop('bindport', DEF_PORT)) - detailed_errors = options.pop('detailed_errors', False) - if ':' in bindto: - bindto, _port = bindto.rsplit(':') - bindport = int(_port) - - self.setProperty('bindto', bindto) - self.setProperty('bindport', bindport) - self.setProperty('detailed_errors', detailed_errors) - self.checkProperties() + port = int(options.pop('uri').split('://', 1)[-1]) + self.detailed_errors = options.pop('detailed_errors', False) self.allow_reuse_address = True - self.log.info("TCPServer %s binding to %s:%d" % (name, self.bindto, self.bindport)) + self.log.info("TCPServer %s binding to port %d" % (name, port)) socketserver.ThreadingTCPServer.__init__( - self, (self.bindto, self.bindport), TCPRequestHandler, bind_and_activate=True) + self, ('0.0.0.0', port), TCPRequestHandler, bind_and_activate=True) self.log.info("TCPServer initiated") diff --git a/secop/server.py b/secop/server.py index 37665c5..47384ed 100644 --- a/secop/server.py +++ b/secop/server.py @@ -39,7 +39,7 @@ except ImportError: DaemonContext = None from secop.errors import ConfigError -from secop.lib import formatException, get_class, getGeneralConfig, mkthread +from secop.lib import formatException, get_class, getGeneralConfig from secop.modules import Attached try: @@ -48,87 +48,106 @@ except ImportError: systemd = None - class Server: - # list allowed section prefixes - # if mapped dict does not exist -> section need a 'class' option - # otherwise a 'type' option is evaluated and the class from the mapping dict used - # - # IMPORTANT: keep the order! (node MUST be first, as the others are referencing it!) - CFGSECTIONS = [ - # section_prefix, default type, mapping of selectable classes - ('node', 'std', {'std': "protocol.dispatcher.Dispatcher", - 'router': 'protocol.router.Router'}), - ('module', None, None), - ('interface', "tcp", {"tcp": "protocol.interface.tcp.TCPServer"}), - ] + INTERFACES = { + 'tcp': 'protocol.interface.tcp.TCPServer', + } _restart = True - def __init__(self, name, parent_logger=None, cfgfiles=None, interface=None, testonly=False): + def __init__(self, name, parent_logger, cfgfiles=None, interface=None, testonly=False): """initialize server - the configuration is taken either from .cfg or from cfgfiles - if cfgfiles is given, also the serverport has to be given. - interface is either an uri or a bare serverport number (with tcp as default) + Arguments: + - name: the node name + - parent_logger: the logger to inherit from + - cfgfiles: if not given, defaults to name + may be a comma separated list of cfg files + items ending with .cfg are taken as paths, else .cfg is appended and + files are looked up in the config path retrieved from the general config + - interface: an uri of the from tcp:// or a bare port number for tcp + if not given, the interface is taken from the config file. In case of + multiple cfg files, the interface is taken from the first cfg file + - testonly: test mode. tries to build all modules, but the server is not started + + Format of cfg file (for now, both forms are accepted): + old form: new form: + + [node ] [NODE] + description= id= + description= + + [interface tcp] [INTERFACE] + bindport=10769 uri=tcp://10769 + bindto=0.0.0.0 + + [module temp] [temp] + ramp=12 ramp=12 + ... """ self._testonly = testonly cfg = getGeneralConfig() self.log = parent_logger.getChild(name, True) - configuration = {k: OrderedDict() for k, _, _ in self.CFGSECTIONS} - if interface: - try: - typ, interface = str(interface).split('://', 1) - except ValueError: - typ = 'tcp' - try: - host, port = interface.split(':', 1) - except ValueError: - host, port = '0.0.0.0', interface - options = {'type': typ, 'bindto': host, 'bindport': port} - configuration['interface %s' % options['type']] = options if not cfgfiles: cfgfiles = name + merged_cfg = OrderedDict() + ambiguous_sections = set() for cfgfile in cfgfiles.split(','): if cfgfile.endswith('.cfg') and os.path.exists(cfgfile): filename = cfgfile else: filename = os.path.join(cfg['confdir'], cfgfile + '.cfg') - self.mergeCfgFile(configuration, filename) - if len(configuration['node']) > 1: - description = ['merged node\n'] - for section, opt in configuration['node']: - description.append("--- %s:\n%s\n" % (section[5:], opt['description'])) - configuration['node'] = {cfgfiles: {'description': '\n'.join(description)}} - self._configuration = configuration - self._cfgfile = cfgfiles # used for reference in error messages only + cfgdict = self.loadCfgFile(filename) + ambiguous_sections |= set(merged_cfg) & set(cfgdict) + merged_cfg.update(cfgdict) + self.node_cfg = merged_cfg.pop('NODE') + self.interface_cfg = merged_cfg.pop('INTERFACE') + self.module_cfg = merged_cfg + if interface: + ambiguous_sections.discard('interface') + ambiguous_sections.discard('node') + self.node_cfg['name'] = name + self.node_cfg['id'] = cfgfiles + self.interface_cfg['uri'] = str(interface) + if ambiguous_sections: + self.log.warning('ambiguous sections in %s: %r' % (cfgfiles, tuple(ambiguous_sections))) + self._cfgfiles = cfgfiles self._pidfile = os.path.join(cfg['piddir'], name + '.pid') - def mergeCfgFile(self, configuration, filename): + def loadCfgFile(self, filename): self.log.debug('Parse config file %s ...' % filename) + result = OrderedDict() parser = configparser.ConfigParser() parser.optionxform = str if not parser.read([filename]): - self.log.error("Couldn't read cfg file %r!" % filename) raise ConfigError("Couldn't read cfg file %r" % filename) for section, options in parser.items(): - try: - kind, name = section.split(' ', 1) - kind = kind.lower() - cfgdict = configuration[kind] - except (ValueError, KeyError): - if section != 'DEFAULT': - self.log.warning('skip unknown section %s' % section) + if section == 'DEFAULT': continue - opt = dict(options) - if name in cfgdict: - if kind == 'interface': - opt = dict(type='tcp', bindto='0.0.0.0') - opt.update(options) - if opt != cfgdict[name]: - self.log.warning('omit conflicting section %r in %s' % (section, filename)) - else: - cfgdict[name] = dict(options) + opts = {} + for k, v in options.items(): + # is the following really needed? - ConfigParser supports multiple lines! + while '\n.\n' in v: + v = v.replace('\n.\n', '\n\n') + try: + opts[k] = ast.literal_eval(v) + except Exception: + opts[k] = v + # convert old form + name, _, arg = section.partition(' ') + if arg: + if name == 'node': + name = 'NODE' + opts['id'] = arg + elif name == 'interface': + name = 'INTERFACE' + if 'bindport' in opts: + opts.pop('bindto', None) + opts['uri'] = '%s://%s' % (opts.pop('type', arg), opts.pop('bindport')) + elif name == 'module': + name = arg + result[name] = opts + return result def start(self): if not DaemonContext: @@ -146,6 +165,10 @@ class Server: files_preserve=self.log.getLogfileStreams()): self.run() + def unknown_options(self, cls, options): + raise ConfigError("%s class don't know how to handle option(s): %s" % + (cls.__name__, ', '.join(options))) + def run(self): while self._restart: self._restart = False @@ -159,73 +182,44 @@ class Server: print(formatException(verbose=True)) raise - self.log.info('startup done, handling transport messages') - threads = [] - for ifname, ifobj in self.interfaces.items(): - self.log.debug('starting thread for interface %r' % ifname) - threads.append((ifname, mkthread(ifobj.serve_forever))) - if systemd: - systemd.daemon.notify("READY=1\nSTATUS=accepting requests") - for ifname, t in threads: - t.join() - self.log.debug('thread for %r died' % ifname) + opts = dict(self.interface_cfg) + scheme, _, _ = opts['uri'].rpartition('://') + scheme = scheme or 'tcp' + cls = get_class(self.INTERFACES[scheme]) + with cls(scheme, self.log.getChild(scheme), opts, self) as self.interface: + if opts: + self.unknown_options(cls, opts) + self.log.info('startup done, handling transport messages') + if systemd: + systemd.daemon.notify("READY=1\nSTATUS=accepting requests") + self.interface.serve_forever() + self.interface.server_close() + if self._restart: + self.restart_hook() + self.log.info('restart') + else: + self.log.info('shut down') def restart(self): if not self._restart: self._restart = True - for ifobj in self.interfaces.values(): - ifobj.shutdown() - ifobj.server_close() + self.interface.shutdown() def _processCfg(self): - self.log.debug('Parse config file %s ...' % self._cfgfile) - - for kind, default_type, classmapping in self.CFGSECTIONS: - objs = OrderedDict() - self.__dict__['%ss' % kind] = objs - for name, options in self._configuration[kind].items(): - opts = dict(options) - if 'class' in opts: - cls = opts.pop('class') - else: - if not classmapping: - self.log.error('%s %s needs a class option!' % (kind.title(), name)) - raise ConfigError('cfgfile %r: %s %s needs a class option!' % - (self._cfgfile, kind.title(), name)) - type_ = opts.pop('type', default_type) - cls = classmapping.get(type_, None) - if not cls: - self.log.error('%s %s needs a type option (select one of %s)!' % - (kind.title(), name, ', '.join(repr(r) for r in classmapping))) - raise ConfigError('cfgfile %r: %s %s needs a type option (select one of %s)!' % - (self._cfgfile, kind.title(), name, ', '.join(repr(r) for r in classmapping))) - # MAGIC: transform \n.\n into \n\n which are normally stripped - # by the ini parser - for k in opts: - v = opts[k] - while '\n.\n' in v: - v = v.replace('\n.\n', '\n\n') - try: - opts[k] = ast.literal_eval(v) - except Exception: - opts[k] = v - - # try to import the class, raise if this fails - self.log.debug('Creating %s %s ...' % (kind.title(), name)) - # cls.__init__ should pop all used args from options! - logname = 'dispatcher' if kind == 'node' else '%s_%s' % (kind, name.lower()) - obj = get_class(cls)(name, self.log.getChild(logname), opts, self) - if opts: - raise ConfigError('%s %s: class %s: don\'t know how to handle option(s): %s' % - (kind, name, cls, ', '.join(opts))) - - # all went well so far - objs[name] = obj - - # following line is the reason for 'node' beeing the first entry in CFGSECTIONS - if len(self.nodes) != 1: - raise ConfigError('cfgfile %r: needs exactly one node section!' % self._cfgfile) - self.dispatcher, = tuple(self.nodes.values()) + opts = dict(self.node_cfg) + cls = get_class(opts.pop('class', 'protocol.dispatcher.Dispatcher')) + self.dispatcher = cls(opts.pop('name', self._cfgfiles), self.log.getChild('dispatcher'), opts, self) + if opts: + self.unknown_options(cls, opts) + self.modules = OrderedDict() + for modname, options in self.module_cfg.items(): + opts = dict(options) + cls = get_class(opts.pop('class')) + modobj = cls(modname, self.log.getChild(modname), opts, self) + # all used args should be popped from opts! + if opts: + self.unknown_options(cls, opts) + self.modules[modname] = modobj poll_table = dict() # all objs created, now start them up and interconnect