simplify configuration of IO modules

As the communicator class needed for a module can be specified,
in the configuration we do not need to specifiy it explicitly.

A new configurator function IO() is introduced for this, defining
names and uri only.

- update also configuration reference and a tutorial example
- update get_class function to accept attributes of classes like
  'frappy_demo.lakshore.TemperatureSensor.ioClass' and import from
  modules other than frappy... like 'test.test_iocfg.Mod'.
- add ioClass to the example class for the temperature controller
  tutorial

Change-Id: I3115371d612f14024e43bc6d38b642e1d27b314d
Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/38071
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

.pylintrc

@@ -86,7 +86,7 @@ dummy-variables-rgx=_|dummy
 
 # List of additional names supposed to be defined in builtins. Remember that
 # you should avoid to define new builtins when possible.
-additional-builtins=Node,Mod,Param,Command,Group
+additional-builtins=Node,Mod,Param,Command,Group,IO
 
 
 [BASIC]

cfg/ls336_cfg.py

@@ -6,11 +6,7 @@ lakeshore_uri = environ.get('LS_URI', 'tcp://<host>:7777')
 Node('example_cryo.psi.ch',  # a globally unique identification
      'this is an example cryostat for the Frappy tutorial',  # describes the node
       interface='tcp://10767')  # you might choose any port number > 1024
-Mod('io',  # the name of the module
+IO('io', lakeshore_uri)  # the communicator (its class will be detected automatically)
-    'frappy_demo.lakeshore.LakeshoreIO',  # the class used for communication
-    'communication to main controller',  # a description
-    uri=lakeshore_uri,  # the serial connection
-    )
 Mod('T',
     'frappy_demo.lakeshore.TemperatureLoop',
     'Sample Temperature',

doc/source/configuration.inc

@@ -3,10 +3,14 @@ Configuration File
 
 .. _node configuration:
 
-:Node(equipment_id, description, interface, \*\*kwds):
+:Node:
 
     Specify the SEC-node properties.
 
+    .. code::
+
+        Node(equipment_id, description, interface, **kwds):
+
     The arguments are SECoP node properties and additional internal node configurations
 
     :Parameters:
@@ -18,9 +22,14 @@ Configuration File
 
 .. _mod configuration:
 
-:Mod(name, cls, description, \*\*kwds):
+:Mod:
 
     Create a SECoP module.
+
+    .. code::
+
+        Mod(name, cls, description, **kwds)
+
     Keyworded argument matching a parameter name are used to configure
     the initial value of a parameter. For configuring the parameter properties
     the value must be an instance of **Param**, using the keyworded arguments
@@ -37,22 +46,60 @@ Configuration File
 
 .. _param configuration:
 
-:Param(value=<undef>, \*\*kwds):
+:Param:
 
     Configure a parameter
 
+    .. code::
+
+        Param(value=<undef>, **kwds):
+
     :Parameters:
 
         - **value** - if given, the initial value of the parameter
         - **kwds** - parameter or datatype SECoP properties (see :class:`frappy.param.Parameter`
           and :class:`frappy.datatypes.Datatypes`)
 
+.. _io configuration:
+
+:IO:
+
+    Configure IO modules (communicators)
+
+    .. code::
+
+        IO(<io name>, <uri>, ...)
+
+
+    It is recommended that the class of the needed IO is specified as class
+    attribute ioClass on the modules class. In this case, for the configuration
+    of the IO modules only their name and URI is needed, for example:
+
+    .. code::
+
+        IO('io_T', 'tcp://192.168.1.1:7777', export=False)
+        IO('io_C', 'serial:///dev/tty_USB0&baudrate=9600', export=False)
+
+        Mod('T_sample', 'frappy_psi.lakeshore.TemperatureSensor', 'the sample T',
+            io='io_T', channel='C')
+        Mod('T_main', 'frappy_psi.lakeshore.TemperatureLoop', 'the main T',
+            io='io_T', channel='A')
+        Mod('C_sample', 'frappy_psi.ah2700.Capacitance', 'the sample capacitance',
+            io='io_C')
+
+    The ``export=False`` argument tells Frappy to hide both communicators.
+
+
 .. _command configuration:
 
-:Command(\*\*kwds):
+:Command:
 
     Configure a command
 
+    .. code::
+
+        Command(**kwds)
+
     :Parameters:
 
         - **kwds** - command SECoP properties (see :class:`frappy.param.Commands`)

doc/source/reference.rst

@@ -140,4 +140,4 @@ Exception classes
 .. automodule:: frappy.errors
     :members:
 
-.. include:: configuration.rst
+.. include:: configuration.inc

doc/source/tutorial_t_control.rst

@@ -99,16 +99,11 @@ We choose the name *example_cryo* and create therefore a configuration file
     Node('example_cryo.psi.ch',  # a globally unique identification
          'this is an example cryostat for the Frappy tutorial',  # describes the node
           interface='tcp://10767')  # you might choose any port number > 1024
-    Mod('io',  # the name of the module
+    IO('io', 'serial://COM6:?baudrate=57600+parity=odd+bytesize=7')
-        'frappy_psi.lakeshore.LakeshoreIO',  # the class used for communication
-        'communication to main controller',  # a description
-        # the serial connection, including serial settings (see frappy.io.IOBase):
-        uri='serial://COM6:?baudrate=57600+parity=odd+bytesize=7',
-       )
     Mod('T',
         'frappy_psi.lakeshore.TemperatureSensor',
         'Sample Temperature',
-        io='io',  # refers to above defined module 'io'
+        io='io',  # refers to above defined io module called 'io'
         channel='A',  # the channel on the LakeShore for this module
         value=Param(max=470),  # alter the maximum expected T
        )
@@ -120,8 +115,8 @@ Usually the only important value in the server address is the TCP port under whi
 server will be accessible. Currently only the tcp scheme is supported.
 
 Then for each module a :ref:`Mod <mod configuration>` section follows.
-We have to create the ``io`` module for communication first, with
+But first we have to create the ``io`` module for communication.
-the ``uri`` as its most important argument.
+For this we use an :ref:`IO <io configuration>` section.
 In case of a serial connection the prefix is ``serial://``. On a Windows machine, the full
 uri is something like ``serial://COM6:?baudrate=9600`` on a linux system it might be
 ``serial:///dev/ttyUSB0?baudrate=9600``. In case of a LAN connection, the uri should

frappy/config.py

@@ -114,14 +114,18 @@ class Collector:
         self.modules = {}
         self.warnings = []
 
-    def add(self, *args, **kwds):
+    def add(self, name, cls, description, **kwds):
-        mod = Mod(*args, **kwds)
+        mod = Mod(name, cls, description, **kwds)
         name = mod.pop('name')
         if name in self.modules:
             self.warnings.append(f'duplicate module {name} overrides previous')
         self.modules[name] = mod
         return mod
 
+    def add_io(self, name, uri, **kwds):
+        mod = Mod(name, cls='<auto>', description='', uri=uri, **kwds)
+        self.modules[mod.pop('name')] = mod
+
     def override(self, name, **kwds):
         """override properties/parameters of previously configured modules
 
@@ -180,12 +184,37 @@ class Include:
         exec(compile(filename.read_bytes(), filename, 'exec'), self.namespace)
 
 
-def process_file(filename, log):
+def fix_io_modules(cfgdict, log):
+    node = cfgdict.pop('node')
+    io_modules = {k: [] for k, v in cfgdict.items() if v.get('cls') == '<auto>'}
+    for modname, modcfg in cfgdict.items():
+        ioname = modcfg.get('io', {}).get('value')
+        if ioname:
+            iocfg = cfgdict.get(ioname)
+            if iocfg:
+                referring_modules = io_modules.get(ioname)
+                if referring_modules is not None:
+                    iocfg['cls'] = f"{modcfg['cls']}.ioClass"
+                    referring_modules.append(modname)
+
+    # fix description:
+    for ioname, referring_modules in io_modules.items():
+        if referring_modules:
+            if not cfgdict[ioname]['description']:
+                cfgdict[ioname]['description'] = f"communicator for {', '.join(k for k in referring_modules)}"
+        else:
+            log.warning('remove unused io module %r', ioname)
+            cfgdict.pop(ioname)
+    cfgdict['node'] = node
+
+
+def process_file(filename, log, config_text=None):
+    if config_text is None:
         config_text = filename.read_bytes()
     node = NodeCollector()
     mods = Collector()
     ns = {'Node': node.add, 'Mod': mods.add, 'Param': Param, 'Command': Param, 'Group': Group,
-          'override': mods.override, 'overrideNode': node.override}
+          'override': mods.override, 'overrideNode': node.override, 'IO': mods.add_io}
     ns['include'] = Include(ns, log)
     # pylint: disable=exec-used
     exec(compile(config_text, filename, 'exec'), ns)
@@ -236,6 +265,7 @@ def load_config(cfgfiles, log):
         filename = to_config_path(str(cfgfile), log)
         log.debug('Parsing config file %s...', filename)
         cfg = process_file(filename, log)
+        fix_io_modules(cfg, log)
         if config:
             config.merge_modules(cfg)
         else:

frappy/lib/__init__.py

@@ -234,22 +234,37 @@ def clamp(_min, value, _max):
     i.e. value if min <= value <= max, else min or max depending on which side
     value lies outside the [min..max] interval. This works even when min > max!
     """
-    # return median, i.e. clamp the the value between min and max
+    # return median, i.e. clamp the value between min and max
     return sorted([_min, value, _max])[1]
 
 
 def get_class(spec):
-    """loads a class given by string in dotted notation (as python would do)"""
+    """loads an object given by string in dotted notation (as python would do)
-    modname, classname = spec.rsplit('.', 1)
+
-    if modname.startswith('frappy'):
+    import the specified module and get the specified item from it
-        module = importlib.import_module(modname)
+    examples: 'frappy_demo.lakeshore.TemperatureSensor', 'frappy.modules.Readable.Status'
-    else:
+
-        # rarely needed by now....
+    :param spec: a dot-separated list of module names followed by the name of
-        module = importlib.import_module('frappy.' + modname)
+       a class (or any object) and optionally names of attributes
+    :return: the object
+    """
+    for maxsplit in range(1, len(spec)):
+        # len(spec) is high enough for all cases
+        module, *attrs = spec.rsplit('.', maxsplit)
         try:
-        return getattr(module, classname)
+            obj = importlib.import_module(module)
+            break
+        except ImportError:
+            if '.' in module:
+                continue
+            raise
+    for na, attr in enumerate(attrs):
+        try:
+            obj = getattr(obj, attr)
         except AttributeError:
-        raise AttributeError('no such class') from None
+            print(na, attrs)
+            raise AttributeError(f'{".".join(attrs[:na+1])!r} not found in {module!r}') from None
+    return obj
 
 
 def mkthread(func, *args, **kwds):

frappy_demo/lakeshore.py

@@ -36,6 +36,7 @@ class LakeshoreIO(StringIO):
 
 class TemperatureSensor(HasIO, Readable):
     """a temperature sensor (generic for different models)"""
+    ioClass = LakeshoreIO
     # internal property to configure the channel
     channel = Property('the Lakeshore channel', datatype=StringType())
     # 0, 1500 is the allowed range by the LakeShore controller
@@ -66,6 +67,7 @@ class TemperatureSensor(HasIO, Readable):
 
 
 class TemperatureLoop(TemperatureSensor, Drivable):
+    ioClass = LakeshoreIO
     # lakeshore loop number to be used for this module
     loop = Property('lakeshore loop', IntRange(1, 2), default=1)
     target = Parameter(datatype=FloatRange(unit='K', min=0, max=1500))

test/test_iocfg.py

@@ -0,0 +1,68 @@
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Markus Zolliker <markus.zolliker@psi.ch>
+#
+# *****************************************************************************
+
+from logging import Logger
+import pytest
+from frappy.core import StringIO, Module, HasIO
+from frappy.config import process_file, fix_io_modules, Param
+
+
+class Mod(HasIO, Module):
+    ioClass = StringIO
+
+
+CONFIG = """
+IO('io_a', 'tcp://test.psi.ch:7777', visibility='w--')
+IO('io_b', 'tcp://test2.psi.ch:8080')
+
+Mod('mod1', 'test.test_iocfg.Mod', '',
+    io='io_a',
+    )
+
+Mod('mod2', 'test.test_iocfg.Mod', '',
+    io='io_b',
+    )
+
+Mod('mod3', 'test.test_iocfg.Mod', '',
+    io='io_b',
+    )
+"""
+
+
+@pytest.mark.parametrize('mod, ioname, iocfg', [
+    ('mod1', 'io_a', {
+        'cls': 'test.test_iocfg.Mod.ioClass',
+        'description': 'communicator for mod1',
+        'uri': Param('tcp://test.psi.ch:7777'),
+        'visibility': Param('w--')
+    },),
+    ('mod2', 'io_b', {
+        'cls': 'test.test_iocfg.Mod.ioClass',
+        'description': 'communicator for mod2, mod3',
+        'uri': Param('tcp://test2.psi.ch:8080'),
+    }),
+])
+def test_process_file(mod, ioname, iocfg):
+    log = Logger('dummy')
+    config = process_file('<test>',log, CONFIG)
+    fix_io_modules(config, log)
+    assert config[mod]['io'] == {'value': ioname}
+    assert config[ioname] == iocfg