linse/frappy
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

server: add option to dynamically create devices

Browse Source 
add module which scans a connection and registers new devices depending
on the answer.
* change module initialization to demand-based
* move code from server to dispatcher
- remove intermediate step in Attached __get__

TODO:
  factor out dispatcher (regards to playground)
  discuss factoring out of module creation code from server AND
  dispatcher

Change-Id: I7af959b99a84c291c526aac067a4e2bf3cd741d4
Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/31470
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
Reviewed-by: Georg Brandl <g.brandl@fz-juelich.de>
Reviewed-by: Alexander Zaft <a.zaft@fz-juelich.de>
This commit is contained in:
Alexander Zaft 2023-06-28 08:55:41 +02:00 committed by Markus Zolliker
parent a2fed8df03
commit 7904f243cb
8 changed files with 234 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cfg/test_cfg.py
View File

@ -10,6 +10,22 @@ The needed fields are Equipment id (1st argument), description (this)
'tcp://10768',
)
Mod('attachtest',
'frappy_demo.test.WithAtt',
'test attached',
att = 'LN2',
)
Mod('pinata',
'frappy_demo.test.Pin',
'scan test',
)
Mod('recursive',
'frappy_demo.test.RecPin',
'scan test',
)
Mod('LN2',
'frappy_demo.test.LN2',
'random value between 0..100%',

41
frappy/dynamic.py Normal file
View File

@ -0,0 +1,41 @@
# *****************************************************************************
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Module authors:
# Alexander Zaft <a.zaft@fz-juelich.de>
#
# *****************************************************************************
from .core import Module
class Pinata(Module):
"""Base class for scanning conections and adding modules accordingly.
Like a piñata. You poke it, and modules fall out.
To use it, subclass it for your connection type and override the function
'scanModules'. For each module you want to register, you should yield the
modules name and its config options.
The connection will then be scanned during server startup.
"""
export = False
# POKE
def scanModules(self):
"""yield (modname, options) for each module the Pinata should create.
Options has to include keys for class and the config for the module.
"""
raise NotImplementedError

13
frappy/modules.py
View File

@ -329,7 +329,6 @@ class Module(HasAccessibles):
# reference to the dispatcher (used for sending async updates)
DISPATCHER = None
attachedModules = None
pollInfo = None
triggerPoll = None # trigger event for polls. used on io modules and modules without io
@ -347,7 +346,9 @@ class Module(HasAccessibles):
self.accessLock = threading.RLock() # for read_* / write_* methods
self.updateLock = threading.RLock() # for announceUpdate
self.polledModules = [] # modules polled by thread started in self.startModules
self.attachedModules = {}
errors = []
self._isinitialized = False
# handle module properties
# 1) make local copies of properties
@ -932,10 +933,12 @@ class Attached(Property):
def __get__(self, obj, owner):
if obj is None:
return self
if obj.attachedModules is None:
# return the name of the module (called from Server on startup)
return super().__get__(obj, owner)
# return the module (called after startup)
if self.name not in obj.attachedModules:
modobj = obj.DISPATCHER.get_module(super().__get__(obj, owner))
if not isinstance(modobj, self.basecls):
raise ConfigError(f'attached module {self.name}={modobj.name!r} '\
f'must inherit from {self.basecls.__qualname__!r}')
obj.attachedModules[self.name] = modobj
return obj.attachedModules.get(self.name) # return None if not given
def copy(self):

95
frappy/protocol/dispatcher.py
View File

@ -39,16 +39,18 @@ Interface to the modules:
"""
import threading
import traceback
from collections import OrderedDict
from time import time as currenttime
from frappy.errors import NoSuchCommandError, NoSuchModuleError, \
NoSuchParameterError, ProtocolError, ReadOnlyError
NoSuchParameterError, ProtocolError, ReadOnlyError, ConfigError
from frappy.params import Parameter
from frappy.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \
DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \
HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \
LOGGING_REPLY, LOG_EVENT
from frappy.lib import get_class
def make_update(modulename, pobj):
@ -84,6 +86,13 @@ class Dispatcher:
self.name = name
self.restart = srv.restart
self.shutdown = srv.shutdown
# handle to server
self.srv = srv
# set of modules that failed creation
self.failed_modules = set()
# list of errors that occured during initialization
self.errors = []
self.traceback_counter = 0
def broadcast_event(self, msg, reallyall=False):
"""broadcasts a msg to all active connections
@ -147,11 +156,92 @@ class Dispatcher:
self._export.append(modulename)
def get_module(self, modulename):
""" Returns a fully initialized module. Or None, if something went
wrong during instatiating/initializing the module."""
modobj = self.get_module_instance(modulename)
if modobj is None:
return None
if modobj._isinitialized:
return modobj
# also call earlyInit on the modules
self.log.info('initializing module %r', modulename) # TODO: change to debug
try:
modobj.earlyInit()
if not modobj.earlyInitDone:
self.errors.append(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call')
modobj.initModule()
if not modobj.initModuleDone:
self.errors.append(f'{modobj.initModule.__qualname__} was not called, probably missing super call')
except Exception as e:
if self.traceback_counter == 0:
self.log.exception(traceback.format_exc())
self.traceback_counter += 1
self.errors.append(f'error initializing {modulename}: {e!r}')
modobj._isinitialized = True
self.log.info('initialized module %r', modulename) # TODO: change to debug
return modobj
def get_module_instance(self, modulename):
""" Returns the module in its current initialization state or creates a
new uninitialized modle to return.
When creating a new module, srv.module_config is accessed to get the
modules configuration.
"""
if modulename in self._modules:
return self._modules[modulename]
if modulename in list(self._modules.values()):
# it's actually already the module object
return modulename
# create module from srv.module_cfg, store and return
self.log.info('registering module %r', modulename)
opts = self.srv.module_cfg.get(modulename, None)
if opts is None:
raise NoSuchModuleError(f'Module {modulename!r} does not exist on this SEC-Node!')
pymodule = None
try: # pylint: disable=no-else-return
classname = opts.pop('cls')
if isinstance(classname, str):
pymodule = classname.rpartition('.')[0]
if pymodule in self.failed_modules:
# creation has failed already once, do not try again
return None
cls = get_class(classname)
else:
pymodule = classname.__module__
if pymodule in self.failed_modules:
# creation has failed already once, do not try again
return None
cls = classname
except Exception as e:
if str(e) == 'no such class':
self.errors.append(f'{classname} not found')
else:
self.failed_modules.add(pymodule)
if self.traceback_counter == 0:
self.log.exception(traceback.format_exc())
self.traceback_counter += 1
self.errors.append(f'error importing {classname}')
return None
else:
try:
modobj = cls(modulename, self.log.getChild(modulename), opts, self.srv)
except ConfigError as e:
self.errors.append(f'error creating module {modulename}:')
for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]:
self.errors.append(' ' + errtxt)
modobj = None
except Exception as e:
if self.traceback_counter == 0:
self.log.exception(traceback.format_exc())
self.traceback_counter += 1
self.errors.append(f'error creating {modulename}')
modobj = None
self.register_module(modobj, modulename, modobj.export)
self.srv.modules[modulename] = modobj # IS HERE THE CORRECT PLACE?
return modobj
def remove_module(self, modulename_or_obj):
moduleobj = self.get_module(modulename_or_obj)
@ -183,6 +273,7 @@ class Dispatcher:
def get_descriptive_data(self, specifier):
"""returns a python object which upon serialisation results in the descriptive data"""
specifier = specifier or ''
modules = {}
result = {'modules': modules}
for modulename in self._export:
@ -194,7 +285,7 @@ class Dispatcher:
mod_desc.update(module.exportProperties())
mod_desc.pop('export', False)
modules[modulename] = mod_desc
modname, _, pname = (specifier or '').partition(':')
modname, _, pname = specifier.partition(':')
if modname in modules: # extension to SECoP standard: description of a single module
result = modules[modname]
if pname in result['accessibles']: # extension to SECoP standard: description of a single accessible

123
frappy/server.py
View File

@ -25,15 +25,15 @@
import os
import sys
import traceback
from collections import OrderedDict
from frappy.errors import ConfigError, SECoPError
from frappy.lib import formatException, get_class, generalConfig
from frappy.config import load_config
from frappy.errors import ConfigError
from frappy.dynamic import Pinata
from frappy.lib import formatException, generalConfig, get_class, mkthread
from frappy.lib.multievent import MultiEvent
from frappy.params import PREDEFINED_ACCESSIBLES
from frappy.modules import Attached
from frappy.config import load_config
try:
from daemon import DaemonContext
@ -174,85 +174,57 @@ class Server:
self.interface.shutdown()
def _processCfg(self):
"""Processes the module configuration.
All modules specified in the config file and read recursively from
Pinata class Modules are instantiated, initialized and started by the
end of this function.
If there are errors that occur, they will be collected and emitted
together in the end.
"""
errors = []
opts = dict(self.node_cfg)
cls = get_class(opts.pop('cls'))
self.dispatcher = cls(opts.pop('name', self._cfgfiles), self.log.getChild('dispatcher'), opts, self)
self.dispatcher = cls(opts.pop('name', self._cfgfiles),
self.log.getChild('dispatcher'), opts, self)
if opts:
errors.append(self.unknown_options(cls, opts))
self.dispatcher.errors.append(self.unknown_options(cls, opts))
self.modules = OrderedDict()
failure_traceback = None # traceback for the first error
failed = set() # python modules failed to load
self.lastError = None
for modname, options in self.module_cfg.items():
opts = dict(options)
pymodule = None
try:
classname = opts.pop('cls')
pymodule = classname.rpartition('.')[0]
if pymodule in failed:
# create and initialize modules
todos = list(self.module_cfg.items())
while todos:
modname, options = todos.pop(0)
if modname in self.modules:
# already created by Dispatcher (via Attached)
continue
# For Pinata modules: we need to access this in Dispatcher.get_module
self.module_cfg[modname] = dict(options)
modobj = self.dispatcher.get_module_instance(modname) # lazy
if modobj is None:
self.log.debug('Module %s returned None', modname)
continue
cls = get_class(classname)
except Exception as e:
if str(e) == 'no such class':
errors.append(f'{classname} not found')
else:
failed.add(pymodule)
if failure_traceback is None:
failure_traceback = traceback.format_exc()
errors.append(f'error importing {classname}')
else:
try:
modobj = cls(modname, self.log.getChild(modname), opts, self)
self.modules[modname] = modobj
except ConfigError as e:
errors.append(f'error creating module {modname}:')
for errtxt in e.args[0] if isinstance(e.args[0], list) else [e.args[0]]:
errors.append(' ' + errtxt)
except Exception:
if failure_traceback is None:
failure_traceback = traceback.format_exc()
errors.append(f'error creating {modname}')
if isinstance(modobj, Pinata):
# scan for dynamic devices
pinata = self.dispatcher.get_module(modname)
pinata_modules = list(pinata.scanModules())
for name, _cfg in pinata_modules:
if name in self.module_cfg:
self.log.error('Module %s, from pinata %s, already'
' exists in config file!', name, modname)
self.log.info('Pinata %s found %d modules', modname, len(pinata_modules))
todos.extend(pinata_modules)
missing_super = set()
# all objs created, now start them up and interconnect
for modname, modobj in self.modules.items():
self.log.info('registering module %r', modname)
self.dispatcher.register_module(modobj, modname, modobj.export)
# also call earlyInit on the modules
modobj.earlyInit()
if not modobj.earlyInitDone:
missing_super.add(f'{modobj.earlyInit.__qualname__} was not called, probably missing super call')
# initialize all modules by getting them with Dispatcher.get_module,
# which is done in the get_descriptive data
# TODO: caching, to not make this extra work
self.dispatcher.get_descriptive_data('')
# =========== All modules are initialized ===========
# handle attached modules
for modname, modobj in self.modules.items():
attached_modules = {}
for propname, propobj in modobj.propertyDict.items():
if isinstance(propobj, Attached):
try:
attname = getattr(modobj, propname)
if attname: # attached module specified in cfg file
attobj = self.dispatcher.get_module(attname)
if isinstance(attobj, propobj.basecls):
attached_modules[propname] = attobj
else:
errors.append(f'attached module {propname}={attname!r} '\
f'must inherit from {propobj.basecls.__qualname__!r}')
except SECoPError as e:
errors.append(f'module {modname}, attached {propname}: {str(e)}')
modobj.attachedModules = attached_modules
# call init on each module after registering all
for modname, modobj in self.modules.items():
try:
modobj.initModule()
if not modobj.initModuleDone:
missing_super.add(f'{modobj.initModule.__qualname__} was not called, probably missing super call')
except Exception as e:
if failure_traceback is None:
failure_traceback = traceback.format_exc()
errors.append(f'error initializing {modname}: {e!r}')
# all errors from initialization process
errors = self.dispatcher.errors
if not self._testonly:
start_events = MultiEvent(default_timeout=30)
@ -261,8 +233,7 @@ class Server:
start_events.name = f'module {modname}'
modobj.startModule(start_events)
if not modobj.startModuleDone:
missing_super.add(f'{modobj.startModule.__qualname__} was not called, probably missing super call')
errors.extend(missing_super)
errors.append(f'{modobj.startModule.__qualname__} was not called, probably missing super call')
if errors:
for errtxt in errors:
@ -271,8 +242,6 @@ class Server:
# print a list of config errors to stderr
sys.stderr.write('\n'.join(errors))
sys.stderr.write('\n')
if failure_traceback:
sys.stderr.write(failure_traceback)
sys.exit(1)
if self._testonly:

3
frappy/version.py
View File

@ -70,9 +70,8 @@ def get_version(abbrev=4):
if git_version != release_version:
write_release_version(git_version)
return git_version
elif release_version:
if release_version:
return release_version
else:
raise ValueError('Cannot find a version number - make sure that '
'git is installed or a RELEASE-VERSION file is '
'present!')

25
frappy_demo/test.py
View File

@ -24,10 +24,33 @@
import random
from frappy.datatypes import FloatRange, StringType, ValueType, TupleOf, StructOf, ArrayOf
from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module
from frappy.modules import Communicator, Drivable, Parameter, Property, Readable, Module, Attached
from frappy.params import Command
from frappy.dynamic import Pinata
from frappy.errors import RangeError
class Pin(Pinata):
def scanModules(self):
yield ('pin_a', {'cls': LN2, 'description':'hi'})
yield ('pin_b', {'cls': LN2, 'description':'hi'})
class RecPin(Pinata):
def scanModules(self):
yield ('rec_a', {'cls': RecPinInner, 'description':'hi'})
yield ('rec_b', {'cls': RecPinInner, 'description':'hi'})#, 'idx':'_2'})
class RecPinInner(Pinata):
idx = Property('', StringType(), default='')
def scanModules(self):
yield ('pin_pin_a' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']})
yield ('pin_pin_b' + self.idx, {'cls': Mapped, 'description':'recursive!', 'choices':['A', 'B']})
class WithAtt(Readable):
att = Attached()
def read_value(self):
return self.att.read_value()
class LN2(Readable):
"""Just a readable.

2
test/test_attach.py
View File

@ -75,6 +75,4 @@ def test_attach():
assert m.propertyValues['att'] == 'a'
srv.dispatcher.register_module(a, 'a')
srv.dispatcher.register_module(m, 'm')
assert m.att == 'a'
m.attachedModules = {'att': a}
assert m.att == a