From 7904f243cb8996e69196b900bd5183b0fdeb7f3c Mon Sep 17 00:00:00 2001 From: Alexander Zaft Date: Wed, 28 Jun 2023 08:55:41 +0200 Subject: [PATCH] server: add option to dynamically create devices add module which scans a connection and registers new devices depending on the answer. * change module initialization to demand-based * move code from server to dispatcher - remove intermediate step in Attached __get__ TODO: factor out dispatcher (regards to playground) discuss factoring out of module creation code from server AND dispatcher Change-Id: I7af959b99a84c291c526aac067a4e2bf3cd741d4 Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/31470 Tested-by: Jenkins Automated Tests Reviewed-by: Enrico Faulhaber Reviewed-by: Georg Brandl Reviewed-by: Alexander Zaft --- cfg/test_cfg.py | 16 +++++ frappy/dynamic.py | 41 +++++++++++ frappy/modules.py | 13 ++-- frappy/protocol/dispatcher.py | 97 +++++++++++++++++++++++++- frappy/server.py | 125 +++++++++++++--------------------- frappy/version.py | 9 ++- frappy_demo/test.py | 25 ++++++- test/test_attach.py | 2 - 8 files changed, 234 insertions(+), 94 deletions(-) create mode 100644 frappy/dynamic.py diff --git a/cfg/test_cfg.py b/cfg/test_cfg.py index 83cea44..dff379a 100644 --- a/cfg/test_cfg.py +++ b/cfg/test_cfg.py @@ -10,6 +10,22 @@ The needed fields are Equipment id (1st argument), description (this) 'tcp://10768', ) +Mod('attachtest', + 'frappy_demo.test.WithAtt', + 'test attached', + att = 'LN2', +) + +Mod('pinata', + 'frappy_demo.test.Pin', + 'scan test', +) + +Mod('recursive', + 'frappy_demo.test.RecPin', + 'scan test', +) + Mod('LN2', 'frappy_demo.test.LN2', 'random value between 0..100%', diff --git a/frappy/dynamic.py b/frappy/dynamic.py new file mode 100644 index 0000000..46b54f5 --- /dev/null +++ b/frappy/dynamic.py @@ -0,0 +1,41 @@ +# ***************************************************************************** +# +# This program is free software; you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Module authors: +# Alexander Zaft +# +# ***************************************************************************** + +from .core import Module + +class Pinata(Module): + """Base class for scanning conections and adding modules accordingly. + + Like a piñata. You poke it, and modules fall out. + + To use it, subclass it for your connection type and override the function + 'scanModules'. For each module you want to register, you should yield the + modules name and its config options. + The connection will then be scanned during server startup. + """ + export = False + + # POKE + def scanModules(self): + """yield (modname, options) for each module the Pinata should create. + Options has to include keys for class and the config for the module. + """ + raise NotImplementedError diff --git a/frappy/modules.py b/frappy/modules.py index 9f7a03e..bf3a661 100644 --- a/frappy/modules.py +++ b/frappy/modules.py @@ -329,7 +329,6 @@ class Module(HasAccessibles): # reference to the dispatcher (used for sending async updates) DISPATCHER = None - attachedModules = None pollInfo = None triggerPoll = None # trigger event for polls. used on io modules and modules without io @@ -347,7 +346,9 @@ class Module(HasAccessibles): self.accessLock = threading.RLock() # for read_* / write_* methods self.updateLock = threading.RLock() # for announceUpdate self.polledModules = [] # modules polled by thread started in self.startModules + self.attachedModules = {} errors = [] + self._isinitialized = False # handle module properties # 1) make local copies of properties @@ -932,10 +933,12 @@ class Attached(Property): def __get__(self, obj, owner): if obj is None: return self - if obj.attachedModules is None: - # return the name of the module (called from Server on startup) - return super().__get__(obj, owner) - # return the module (called after startup) + if self.name not in obj.attachedModules: + modobj = obj.DISPATCHER.get_module(super().__get__(obj, owner)) + if not isinstance(modobj, self.basecls): + raise ConfigError(f'attached module {self.name}={modobj.name!r} '\ + f'must inherit from {self.basecls.__qualname__!r}') + obj.attachedModules[self.name] = modobj return obj.attachedModules.get(self.name) # return None if not given def copy(self): diff --git a/frappy/protocol/dispatcher.py b/frappy/protocol/dispatcher.py index 53671e3..b7858fe 100644 --- a/frappy/protocol/dispatcher.py +++ b/frappy/protocol/dispatcher.py @@ -39,16 +39,18 @@ Interface to the modules: """ import threading +import traceback from collections import OrderedDict from time import time as currenttime from frappy.errors import NoSuchCommandError, NoSuchModuleError, \ - NoSuchParameterError, ProtocolError, ReadOnlyError + NoSuchParameterError, ProtocolError, ReadOnlyError, ConfigError from frappy.params import Parameter from frappy.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \ DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \ HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \ LOGGING_REPLY, LOG_EVENT +from frappy.lib import get_class def make_update(modulename, pobj): @@ -84,6 +86,13 @@ class Dispatcher: self.name = name self.restart = srv.restart self.shutdown = srv.shutdown + # handle to server + self.srv = srv + # set of modules that failed creation + self.failed_modules = set() + # list of errors that occured during initialization + self.errors = [] + self.traceback_counter = 0 def broadcast_event(self, msg, reallyall=False): """broadcasts a msg to all active connections @@ -147,11 +156,92 @@ class Dispatcher: self._export.append(modulename) def get_module(self, modulename): + """ Returns a fully initialized module. Or None, if something went + wrong during instatiating/initializing the module.""" + modobj = self.get_module_instance(modulename) + if modobj is None: + return None + if modobj._isinitialized: + return modobj + + # also call earlyInit on the modules + self.log.info('initializing module %r', modulename) # TODO: change to debug + try: + modobj.earlyInit() + if not modobj.earlyInitDone: + self.errors.append(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call') + modobj.initModule() + if not modobj.initModuleDone: + self.errors.append(f'{modobj.initModule.__qualname__} was not called, probably missing super call') + except Exception as e: + if self.traceback_counter == 0: + self.log.exception(traceback.format_exc()) + self.traceback_counter += 1 + self.errors.append(f'error initializing {modulename}: {e!r}') + modobj._isinitialized = True + self.log.info('initialized module %r', modulename) # TODO: change to debug + return modobj + + def get_module_instance(self, modulename): + """ Returns the module in its current initialization state or creates a + new uninitialized modle to return. + + When creating a new module, srv.module_config is accessed to get the + modules configuration. + """ if modulename in self._modules: return self._modules[modulename] if modulename in list(self._modules.values()): + # it's actually already the module object return modulename - raise NoSuchModuleError(f'Module {modulename!r} does not exist on this SEC-Node!') + # create module from srv.module_cfg, store and return + self.log.info('registering module %r', modulename) + + opts = self.srv.module_cfg.get(modulename, None) + if opts is None: + raise NoSuchModuleError(f'Module {modulename!r} does not exist on this SEC-Node!') + pymodule = None + try: # pylint: disable=no-else-return + classname = opts.pop('cls') + if isinstance(classname, str): + pymodule = classname.rpartition('.')[0] + if pymodule in self.failed_modules: + # creation has failed already once, do not try again + return None + cls = get_class(classname) + else: + pymodule = classname.__module__ + if pymodule in self.failed_modules: + # creation has failed already once, do not try again + return None + cls = classname + except Exception as e: + if str(e) == 'no such class': + self.errors.append(f'{classname} not found') + else: + self.failed_modules.add(pymodule) + if self.traceback_counter == 0: + self.log.exception(traceback.format_exc()) + self.traceback_counter += 1 + self.errors.append(f'error importing {classname}') + return None + else: + try: + modobj = cls(modulename, self.log.getChild(modulename), opts, self.srv) + except ConfigError as e: + self.errors.append(f'error creating module {modulename}:') + for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]: + self.errors.append(' ' + errtxt) + modobj = None + except Exception as e: + if self.traceback_counter == 0: + self.log.exception(traceback.format_exc()) + self.traceback_counter += 1 + self.errors.append(f'error creating {modulename}') + modobj = None + self.register_module(modobj, modulename, modobj.export) + self.srv.modules[modulename] = modobj # IS HERE THE CORRECT PLACE? + return modobj def remove_module(self, modulename_or_obj): moduleobj = self.get_module(modulename_or_obj) @@ -183,6 +273,7 @@ class Dispatcher: def get_descriptive_data(self, specifier): """returns a python object which upon serialisation results in the descriptive data""" + specifier = specifier or '' modules = {} result = {'modules': modules} for modulename in self._export: @@ -194,7 +285,7 @@ class Dispatcher: mod_desc.update(module.exportProperties()) mod_desc.pop('export', False) modules[modulename] = mod_desc - modname, _, pname = (specifier or '').partition(':') + modname, _, pname = specifier.partition(':') if modname in modules: # extension to SECoP standard: description of a single module result = modules[modname] if pname in result['accessibles']: # extension to SECoP standard: description of a single accessible diff --git a/frappy/server.py b/frappy/server.py index e846da0..0cec1f6 100644 --- a/frappy/server.py +++ b/frappy/server.py @@ -25,15 +25,15 @@ import os import sys -import traceback from collections import OrderedDict -from frappy.errors import ConfigError, SECoPError -from frappy.lib import formatException, get_class, generalConfig +from frappy.config import load_config +from frappy.errors import ConfigError +from frappy.dynamic import Pinata +from frappy.lib import formatException, generalConfig, get_class, mkthread from frappy.lib.multievent import MultiEvent from frappy.params import PREDEFINED_ACCESSIBLES from frappy.modules import Attached -from frappy.config import load_config try: from daemon import DaemonContext @@ -174,85 +174,57 @@ class Server: self.interface.shutdown() def _processCfg(self): + """Processes the module configuration. + + All modules specified in the config file and read recursively from + Pinata class Modules are instantiated, initialized and started by the + end of this function. + If there are errors that occur, they will be collected and emitted + together in the end. + """ errors = [] opts = dict(self.node_cfg) cls = get_class(opts.pop('cls')) - self.dispatcher = cls(opts.pop('name', self._cfgfiles), self.log.getChild('dispatcher'), opts, self) + self.dispatcher = cls(opts.pop('name', self._cfgfiles), + self.log.getChild('dispatcher'), opts, self) if opts: - errors.append(self.unknown_options(cls, opts)) + self.dispatcher.errors.append(self.unknown_options(cls, opts)) self.modules = OrderedDict() - failure_traceback = None # traceback for the first error - failed = set() # python modules failed to load - self.lastError = None - for modname, options in self.module_cfg.items(): - opts = dict(options) - pymodule = None - try: - classname = opts.pop('cls') - pymodule = classname.rpartition('.')[0] - if pymodule in failed: - continue - cls = get_class(classname) - except Exception as e: - if str(e) == 'no such class': - errors.append(f'{classname} not found') - else: - failed.add(pymodule) - if failure_traceback is None: - failure_traceback = traceback.format_exc() - errors.append(f'error importing {classname}') - else: - try: - modobj = cls(modname, self.log.getChild(modname), opts, self) - self.modules[modname] = modobj - except ConfigError as e: - errors.append(f'error creating module {modname}:') - for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]: - errors.append(' ' + errtxt) - except Exception: - if failure_traceback is None: - failure_traceback = traceback.format_exc() - errors.append(f'error creating {modname}') - missing_super = set() - # all objs created, now start them up and interconnect - for modname, modobj in self.modules.items(): - self.log.info('registering module %r', modname) - self.dispatcher.register_module(modobj, modname, modobj.export) - # also call earlyInit on the modules - modobj.earlyInit() - if not modobj.earlyInitDone: - missing_super.add(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call') + # create and initialize modules + todos = list(self.module_cfg.items()) + while todos: + modname, options = todos.pop(0) + if modname in self.modules: + # already created by Dispatcher (via Attached) + continue + # For Pinata modules: we need to access this in Dispatcher.get_module + self.module_cfg[modname] = dict(options) + modobj = self.dispatcher.get_module_instance(modname) # lazy + if modobj is None: + self.log.debug('Module %s returned None', modname) + continue + self.modules[modname] = modobj + if isinstance(modobj, Pinata): + # scan for dynamic devices + pinata = self.dispatcher.get_module(modname) + pinata_modules = list(pinata.scanModules()) + for name, _cfg in pinata_modules: + if name in self.module_cfg: + self.log.error('Module %s, from pinata %s, already' + ' exists in config file!', name, modname) + self.log.info('Pinata %s found %d modules', modname, len(pinata_modules)) + todos.extend(pinata_modules) - # handle attached modules - for modname, modobj in self.modules.items(): - attached_modules = {} - for propname, propobj in modobj.propertyDict.items(): - if isinstance(propobj, Attached): - try: - attname = getattr(modobj, propname) - if attname: # attached module specified in cfg file - attobj = self.dispatcher.get_module(attname) - if isinstance(attobj, propobj.basecls): - attached_modules[propname] = attobj - else: - errors.append(f'attached module {propname}={attname!r} '\ - f'must inherit from {propobj.basecls.__qualname__!r}') - except SECoPError as e: - errors.append(f'module {modname}, attached {propname}: {str(e)}') - modobj.attachedModules = attached_modules + # initialize all modules by getting them with Dispatcher.get_module, + # which is done in the get_descriptive data + # TODO: caching, to not make this extra work + self.dispatcher.get_descriptive_data('') + # =========== All modules are initialized =========== - # call init on each module after registering all - for modname, modobj in self.modules.items(): - try: - modobj.initModule() - if not modobj.initModuleDone: - missing_super.add(f'{modobj.initModule.__qualname__} was not called, probably missing super call') - except Exception as e: - if failure_traceback is None: - failure_traceback = traceback.format_exc() - errors.append(f'error initializing {modname}: {e!r}') + # all errors from initialization process + errors = self.dispatcher.errors if not self._testonly: start_events = MultiEvent(default_timeout=30) @@ -261,8 +233,7 @@ class Server: start_events.name = f'module {modname}' modobj.startModule(start_events) if not modobj.startModuleDone: - missing_super.add(f'{modobj.startModule.__qualname__} was not called, probably missing super call') - errors.extend(missing_super) + errors.append(f'{modobj.startModule.__qualname__} was not called, probably missing super call') if errors: for errtxt in errors: @@ -271,8 +242,6 @@ class Server: # print a list of config errors to stderr sys.stderr.write('\n'.join(errors)) sys.stderr.write('\n') - if failure_traceback: - sys.stderr.write(failure_traceback) sys.exit(1) if self._testonly: diff --git a/frappy/version.py b/frappy/version.py index 3b87306..3fc3880 100644 --- a/frappy/version.py +++ b/frappy/version.py @@ -70,12 +70,11 @@ def get_version(abbrev=4): if git_version != release_version: write_release_version(git_version) return git_version - elif release_version: + if release_version: return release_version - else: - raise ValueError('Cannot find a version number - make sure that ' - 'git is installed or a RELEASE-VERSION file is ' - 'present!') + raise ValueError('Cannot find a version number - make sure that ' + 'git is installed or a RELEASE-VERSION file is ' + 'present!') if __name__ == "__main__": diff --git a/frappy_demo/test.py b/frappy_demo/test.py index b9ecac2..9f494a8 100644 --- a/frappy_demo/test.py +++ b/frappy_demo/test.py @@ -24,10 +24,33 @@ import random from frappy.datatypes import FloatRange, StringType, ValueType, TupleOf, StructOf, ArrayOf -from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module +from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module, Attached from frappy.params import Command +from frappy.dynamic import Pinata from frappy.errors import RangeError +class Pin(Pinata): + def scanModules(self): + yield ('pin_a', {'cls': LN2, 'description':'hi'}) + yield ('pin_b', {'cls': LN2, 'description':'hi'}) + +class RecPin(Pinata): + def scanModules(self): + yield ('rec_a', {'cls': RecPinInner, 'description':'hi'}) + yield ('rec_b', {'cls': RecPinInner, 'description':'hi'})#, 'idx':'_2'}) + +class RecPinInner(Pinata): + idx = Property('', StringType(), default='') + def scanModules(self): + yield ('pin_pin_a' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']}) + yield ('pin_pin_b' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']}) + + +class WithAtt(Readable): + att = Attached() + + def read_value(self): + return self.att.read_value() class LN2(Readable): """Just a readable. diff --git a/test/test_attach.py b/test/test_attach.py index ebb6a47..6b6c251 100644 --- a/test/test_attach.py +++ b/test/test_attach.py @@ -75,6 +75,4 @@ def test_attach(): assert m.propertyValues['att'] == 'a' srv.dispatcher.register_module(a, 'a') srv.dispatcher.register_module(m, 'm') - assert m.att == 'a' - m.attachedModules = {'att': a} assert m.att == a