server: add option to dynamically create devices

add module which scans a connection and registers new devices depending
on the answer.
* change module initialization to demand-based
* move code from server to dispatcher
- remove intermediate step in Attached __get__

TODO:
  factor out dispatcher (regards to playground)
  discuss factoring out of module creation code from server AND
  dispatcher

Change-Id: I7af959b99a84c291c526aac067a4e2bf3cd741d4
Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/31470
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
Reviewed-by: Georg Brandl <g.brandl@fz-juelich.de>
Reviewed-by: Alexander Zaft <a.zaft@fz-juelich.de>

cfg/test_cfg.py

@@ -10,6 +10,22 @@ The needed fields are Equipment id (1st argument), description (this)
      'tcp://10768',
 )
 
+Mod('attachtest',
+    'frappy_demo.test.WithAtt',
+    'test attached',
+    att = 'LN2',
+)
+
+Mod('pinata',
+    'frappy_demo.test.Pin',
+    'scan test',
+)
+
+Mod('recursive',
+    'frappy_demo.test.RecPin',
+    'scan test',
+)
+
 Mod('LN2',
     'frappy_demo.test.LN2',
     'random value between 0..100%',

frappy/dynamic.py

@@ -0,0 +1,41 @@
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+# Alexander Zaft <a.zaft@fz-juelich.de>
+#
+# *****************************************************************************
+
+from .core import Module
+
+class Pinata(Module):
+    """Base class for scanning conections and adding modules accordingly.
+
+    Like a piñata. You poke it, and modules fall out.
+
+    To use it, subclass it for your connection type and override the function
+    'scanModules'. For each module you want to register, you should yield the
+    modules name and its config options.
+    The connection will then be scanned during server startup.
+    """
+    export = False
+
+    # POKE
+    def scanModules(self):
+        """yield (modname, options) for each module the Pinata should create.
+        Options has to include keys for class and the config for the module.
+        """
+        raise NotImplementedError

frappy/modules.py

@@ -329,7 +329,6 @@ class Module(HasAccessibles):
 
     # reference to the dispatcher (used for sending async updates)
     DISPATCHER = None
-    attachedModules = None
     pollInfo = None
     triggerPoll = None  # trigger event for polls. used on io modules and modules without io
 
@@ -347,7 +346,9 @@ class Module(HasAccessibles):
         self.accessLock = threading.RLock()  # for read_* / write_* methods
         self.updateLock = threading.RLock()  # for announceUpdate
         self.polledModules = []  # modules polled by thread started in self.startModules
+        self.attachedModules = {}
         errors = []
+        self._isinitialized = False
 
         # handle module properties
         # 1) make local copies of properties
@@ -932,10 +933,12 @@ class Attached(Property):
     def __get__(self, obj, owner):
         if obj is None:
             return self
-        if obj.attachedModules is None:
+        if self.name not in obj.attachedModules:
-            # return the name of the module (called from Server on startup)
+            modobj = obj.DISPATCHER.get_module(super().__get__(obj, owner))
-            return super().__get__(obj, owner)
+            if not isinstance(modobj, self.basecls):
-        # return the module (called after startup)
+                raise ConfigError(f'attached module {self.name}={modobj.name!r} '\
+                    f'must inherit from {self.basecls.__qualname__!r}')
+            obj.attachedModules[self.name] = modobj
         return obj.attachedModules.get(self.name)  # return None if not given
 
     def copy(self):

frappy/protocol/dispatcher.py

@@ -39,16 +39,18 @@ Interface to the modules:
 """
 
 import threading
+import traceback
 from collections import OrderedDict
 from time import time as currenttime
 
 from frappy.errors import NoSuchCommandError, NoSuchModuleError, \
-    NoSuchParameterError, ProtocolError, ReadOnlyError
+    NoSuchParameterError, ProtocolError, ReadOnlyError, ConfigError
 from frappy.params import Parameter
 from frappy.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \
     DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \
     HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \
     LOGGING_REPLY, LOG_EVENT
+from frappy.lib import get_class
 
 
 def make_update(modulename, pobj):
@@ -84,6 +86,13 @@ class Dispatcher:
         self.name = name
         self.restart = srv.restart
         self.shutdown = srv.shutdown
+        # handle to server
+        self.srv = srv
+        # set of modules that failed creation
+        self.failed_modules = set()
+        # list of errors that occured during initialization
+        self.errors = []
+        self.traceback_counter = 0
 
     def broadcast_event(self, msg, reallyall=False):
         """broadcasts a msg to all active connections
@@ -147,11 +156,92 @@ class Dispatcher:
             self._export.append(modulename)
 
     def get_module(self, modulename):
+        """ Returns a fully initialized module. Or None, if something went
+        wrong during instatiating/initializing the module."""
+        modobj = self.get_module_instance(modulename)
+        if modobj is None:
+            return None
+        if modobj._isinitialized:
+            return modobj
+
+        # also call earlyInit on the modules
+        self.log.info('initializing module %r', modulename)  # TODO: change to debug
+        try:
+            modobj.earlyInit()
+            if not modobj.earlyInitDone:
+                self.errors.append(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call')
+            modobj.initModule()
+            if not modobj.initModuleDone:
+                self.errors.append(f'{modobj.initModule.__qualname__} was not called, probably missing super call')
+        except Exception as e:
+            if self.traceback_counter == 0:
+                self.log.exception(traceback.format_exc())
+            self.traceback_counter += 1
+            self.errors.append(f'error initializing {modulename}: {e!r}')
+        modobj._isinitialized = True
+        self.log.info('initialized module %r', modulename)  # TODO: change to debug
+        return modobj
+
+    def get_module_instance(self, modulename):
+        """ Returns the module in its current initialization state or creates a
+        new uninitialized modle to return.
+
+        When creating a new module, srv.module_config is accessed to get the
+        modules configuration.
+        """
         if modulename in self._modules:
             return self._modules[modulename]
         if modulename in list(self._modules.values()):
+            # it's actually already the module object
             return modulename
-        raise NoSuchModuleError(f'Module {modulename!r} does not exist on this SEC-Node!')
+        # create module from srv.module_cfg, store and return
+        self.log.info('registering module %r', modulename)
+
+        opts = self.srv.module_cfg.get(modulename, None)
+        if opts is None:
+            raise NoSuchModuleError(f'Module {modulename!r} does not exist on this SEC-Node!')
+        pymodule = None
+        try:  # pylint: disable=no-else-return
+            classname = opts.pop('cls')
+            if isinstance(classname, str):
+                pymodule = classname.rpartition('.')[0]
+                if pymodule in self.failed_modules:
+                    # creation has failed already once, do not try again
+                    return None
+                cls = get_class(classname)
+            else:
+                pymodule = classname.__module__
+                if pymodule in self.failed_modules:
+                    # creation has failed already once, do not try again
+                    return None
+                cls = classname
+        except Exception as e:
+            if str(e) == 'no such class':
+                self.errors.append(f'{classname} not found')
+            else:
+                self.failed_modules.add(pymodule)
+                if self.traceback_counter == 0:
+                    self.log.exception(traceback.format_exc())
+                self.traceback_counter += 1
+                self.errors.append(f'error importing {classname}')
+            return None
+        else:
+            try:
+                modobj = cls(modulename, self.log.getChild(modulename), opts, self.srv)
+            except ConfigError as e:
+                self.errors.append(f'error creating module {modulename}:')
+                for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]:
+                    self.errors.append('  ' + errtxt)
+                modobj = None
+            except Exception as e:
+                if self.traceback_counter == 0:
+                    self.log.exception(traceback.format_exc())
+                self.traceback_counter += 1
+                self.errors.append(f'error creating {modulename}')
+                modobj = None
+        self.register_module(modobj, modulename, modobj.export)
+        self.srv.modules[modulename] = modobj # IS HERE THE CORRECT PLACE?
+        return modobj
 
     def remove_module(self, modulename_or_obj):
         moduleobj = self.get_module(modulename_or_obj)
@@ -183,6 +273,7 @@ class Dispatcher:
 
     def get_descriptive_data(self, specifier):
         """returns a python object which upon serialisation results in the descriptive data"""
+        specifier = specifier or ''
         modules = {}
         result = {'modules': modules}
         for modulename in self._export:
@@ -194,7 +285,7 @@ class Dispatcher:
             mod_desc.update(module.exportProperties())
             mod_desc.pop('export', False)
             modules[modulename] = mod_desc
-        modname, _, pname = (specifier or '').partition(':')
+        modname, _, pname = specifier.partition(':')
         if modname in modules:  # extension to SECoP standard: description of a single module
             result = modules[modname]
             if pname in result['accessibles']:  # extension to SECoP standard: description of a single accessible

frappy/server.py

@@ -25,15 +25,15 @@
 
 import os
 import sys
-import traceback
 from collections import OrderedDict
 
-from frappy.errors import ConfigError, SECoPError
+from frappy.config import load_config
-from frappy.lib import formatException, get_class, generalConfig
+from frappy.errors import ConfigError
+from frappy.dynamic import Pinata
+from frappy.lib import formatException, generalConfig, get_class, mkthread
 from frappy.lib.multievent import MultiEvent
 from frappy.params import PREDEFINED_ACCESSIBLES
 from frappy.modules import Attached
-from frappy.config import load_config
 
 try:
     from daemon import DaemonContext
@@ -174,85 +174,57 @@ class Server:
         self.interface.shutdown()
 
     def _processCfg(self):
+        """Processes the module configuration.
+
+        All modules specified in the config file and read recursively from
+        Pinata class Modules are instantiated, initialized and started by the
+        end of this function.
+        If there are errors that occur, they will be collected and emitted
+        together in the end.
+        """
         errors = []
         opts = dict(self.node_cfg)
         cls = get_class(opts.pop('cls'))
-        self.dispatcher = cls(opts.pop('name', self._cfgfiles), self.log.getChild('dispatcher'), opts, self)
+        self.dispatcher = cls(opts.pop('name', self._cfgfiles),
+                              self.log.getChild('dispatcher'), opts, self)
 
         if opts:
-            errors.append(self.unknown_options(cls, opts))
+            self.dispatcher.errors.append(self.unknown_options(cls, opts))
         self.modules = OrderedDict()
-        failure_traceback = None  # traceback for the first error
-        failed = set()  # python modules failed to load
-        self.lastError = None
-        for modname, options in self.module_cfg.items():
-            opts = dict(options)
-            pymodule = None
-            try:
-                classname = opts.pop('cls')
-                pymodule = classname.rpartition('.')[0]
-                if pymodule in failed:
-                    continue
-                cls = get_class(classname)
-            except Exception as e:
-                if str(e) == 'no such class':
-                    errors.append(f'{classname} not found')
-                else:
-                    failed.add(pymodule)
-                    if failure_traceback is None:
-                        failure_traceback = traceback.format_exc()
-                    errors.append(f'error importing {classname}')
-            else:
-                try:
-                    modobj = cls(modname, self.log.getChild(modname), opts, self)
-                    self.modules[modname] = modobj
-                except ConfigError as e:
-                    errors.append(f'error creating module {modname}:')
-                    for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]:
-                        errors.append('  ' + errtxt)
-                except Exception:
-                    if failure_traceback is None:
-                        failure_traceback = traceback.format_exc()
-                    errors.append(f'error creating {modname}')
 
-        missing_super = set()
+        # create and initialize modules
-        # all objs created, now start them up and interconnect
+        todos = list(self.module_cfg.items())
-        for modname, modobj in self.modules.items():
+        while todos:
-            self.log.info('registering module %r', modname)
+            modname, options = todos.pop(0)
-            self.dispatcher.register_module(modobj, modname, modobj.export)
+            if modname in self.modules:
-            # also call earlyInit on the modules
+                # already created by Dispatcher (via Attached)
-            modobj.earlyInit()
+                continue
-            if not modobj.earlyInitDone:
+            # For Pinata modules: we need to access this in Dispatcher.get_module
-                missing_super.add(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call')
+            self.module_cfg[modname] = dict(options)
+            modobj = self.dispatcher.get_module_instance(modname) # lazy
+            if modobj is None:
+                self.log.debug('Module %s returned None', modname)
+                continue
+            self.modules[modname] = modobj
+            if isinstance(modobj, Pinata):
+                # scan for dynamic devices
+                pinata = self.dispatcher.get_module(modname)
+                pinata_modules = list(pinata.scanModules())
+                for name, _cfg in pinata_modules:
+                    if name in self.module_cfg:
+                        self.log.error('Module %s, from pinata %s, already'
+                                       ' exists in config file!', name, modname)
+                self.log.info('Pinata %s found %d modules', modname, len(pinata_modules))
+                todos.extend(pinata_modules)
 
-        # handle attached modules
+        # initialize all modules by getting them with Dispatcher.get_module,
-        for modname, modobj in self.modules.items():
+        # which is done in the get_descriptive data
-            attached_modules = {}
+        # TODO: caching, to not make this extra work
-            for propname, propobj in modobj.propertyDict.items():
+        self.dispatcher.get_descriptive_data('')
-                if isinstance(propobj, Attached):
+        # =========== All modules are initialized ===========
-                    try:
-                        attname = getattr(modobj, propname)
-                        if attname:  # attached module specified in cfg file
-                            attobj = self.dispatcher.get_module(attname)
-                            if isinstance(attobj, propobj.basecls):
-                                attached_modules[propname] = attobj
-                            else:
-                                errors.append(f'attached module {propname}={attname!r} '\
-                                              f'must inherit from {propobj.basecls.__qualname__!r}')
-                    except SECoPError as e:
-                        errors.append(f'module {modname}, attached {propname}: {str(e)}')
-            modobj.attachedModules = attached_modules
 
-        # call init on each module after registering all
+        # all errors from initialization process
-        for modname, modobj in self.modules.items():
+        errors = self.dispatcher.errors
-            try:
-                modobj.initModule()
-                if not modobj.initModuleDone:
-                    missing_super.add(f'{modobj.initModule.__qualname__} was not called, probably missing super call')
-            except Exception as e:
-                if failure_traceback is None:
-                    failure_traceback = traceback.format_exc()
-                errors.append(f'error initializing {modname}: {e!r}')
 
         if not self._testonly:
             start_events = MultiEvent(default_timeout=30)
@@ -261,8 +233,7 @@ class Server:
                 start_events.name = f'module {modname}'
                 modobj.startModule(start_events)
                 if not modobj.startModuleDone:
-                    missing_super.add(f'{modobj.startModule.__qualname__} was not called, probably missing super call')
+                    errors.append(f'{modobj.startModule.__qualname__} was not called, probably missing super call')
-            errors.extend(missing_super)
 
         if errors:
             for errtxt in errors:
@@ -271,8 +242,6 @@ class Server:
             # print a list of config errors to stderr
             sys.stderr.write('\n'.join(errors))
             sys.stderr.write('\n')
-            if failure_traceback:
-                sys.stderr.write(failure_traceback)
             sys.exit(1)
 
         if self._testonly:

frappy/version.py

@@ -70,12 +70,11 @@ def get_version(abbrev=4):
         if git_version != release_version:
             write_release_version(git_version)
         return git_version
-    elif release_version:
+    if release_version:
         return release_version
-    else:
+    raise ValueError('Cannot find a version number - make sure that '
-        raise ValueError('Cannot find a version number - make sure that '
+                     'git is installed or a RELEASE-VERSION file is '
-                         'git is installed or a RELEASE-VERSION file is '
+                     'present!')
-                         'present!')
 
 
 if __name__ == "__main__":

frappy_demo/test.py

@@ -24,10 +24,33 @@
 import random
 
 from frappy.datatypes import FloatRange, StringType, ValueType, TupleOf, StructOf, ArrayOf
-from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module
+from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module, Attached
 from frappy.params import Command
+from frappy.dynamic import Pinata
 from frappy.errors import RangeError
 
+class Pin(Pinata):
+    def scanModules(self):
+        yield ('pin_a', {'cls': LN2, 'description':'hi'})
+        yield ('pin_b', {'cls': LN2, 'description':'hi'})
+
+class RecPin(Pinata):
+    def scanModules(self):
+        yield ('rec_a', {'cls': RecPinInner, 'description':'hi'})
+        yield ('rec_b', {'cls': RecPinInner, 'description':'hi'})#, 'idx':'_2'})
+
+class RecPinInner(Pinata):
+    idx = Property('', StringType(), default='')
+    def scanModules(self):
+        yield ('pin_pin_a' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']})
+        yield ('pin_pin_b' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']})
+
+
+class WithAtt(Readable):
+    att = Attached()
+
+    def read_value(self):
+        return self.att.read_value()
 
 class LN2(Readable):
     """Just a readable.

test/test_attach.py

@@ -75,6 +75,4 @@ def test_attach():
     assert m.propertyValues['att'] == 'a'
     srv.dispatcher.register_module(a, 'a')
     srv.dispatcher.register_module(m, 'm')
-    assert m.att == 'a'
-    m.attachedModules = {'att': a}
     assert m.att == a