improve Py2/3 compat

Change-Id: I1dfdcb88a492401851d5157c734cd708496bf004
Reviewed-on: https://forge.frm2.tum.de/review/17734
Tested-by: JenkinsCodeReview <bjoern_pedersen@frm2.tum.de>
Reviewed-by: Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>

etc/sim_mlz_stressihtf2_v2.cfg

@@ -28,12 +28,12 @@ ramp.default=60
 ramp.unit=K/min
 abslimits.datatype=["tuple",[["double"],["double"]]]
 abslimits.default=[0,2000]
-abslimits.description=currently active absolute limits for the setpoint. depend on the regulationmode parameter (both/stick->0..600, tube->0..300K).
+abslimits.description=currently active absolute limits for the setpoint.
 abslimits.unit='degC'
 abslimits.readonly=True
 userlimits.datatype=["tuple",[["double"],["double"]]]
 userlimits.default=[0,300]
-userlimits.description=current user set limits for the setpoint. must be inside abslimits.
+userlimits.description=current user set limits for the setpoint. must be inside or coincide with abslimits.
 userlimits.unit='degC'
 
 [module T_sample]

secop/__init__.py

@@ -18,6 +18,7 @@
 #
 # Module authors:
 #   Alexander Lenz <alexander.lenz@frm2.tum.de>
+#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
 #
 # *****************************************************************************
 
@@ -26,5 +27,4 @@ try:
     sip.setapi('QString', 2)
     sip.setapi('QVariant', 2)
 except ImportError:
-    print('can not import sip, the gui may not work as expected')
+    pass
-

secop/client/__init__.py

@@ -26,6 +26,19 @@
 from __future__ import print_function
 
 import code
+from os import path
+import socket
+import threading
+from collections import deque
+
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+import mlzlog
+from secop.protocol.interface.tcp import decode_msg, get_msg, encode_msg_frame
+from secop.protocol.messages import EVENTREPLY, DESCRIPTIONREQUEST, Message
 
 
 class NameSpace(dict):
@@ -49,14 +62,9 @@ class NameSpace(dict):
         dict.__delitem__(self, name)
 
 
-try:
-    import ConfigParser
-except ImportError:
-    import configparser as ConfigParser
-
 
 def getClientOpts(cfgfile):
-    parser = ConfigParser.SafeConfigParser()
+    parser = configparser.SafeConfigParser()
     if not parser.read([cfgfile + '.cfg']):
         print("Error reading cfg file %r" % cfgfile)
         return {}
@@ -65,9 +73,6 @@ def getClientOpts(cfgfile):
     return dict(item for item in parser.items('client'))
 
 
-from os import path
-
-
 class ClientConsole(object):
 
     def __init__(self, cfgname, basepath):
@@ -93,23 +98,10 @@ class ClientConsole(object):
             help(arg)
 
 
-import socket
-import threading
-from collections import deque
-
-import mlzlog
-
-from secop.protocol.encoding import ENCODERS
-from secop.protocol.framing import FRAMERS
-from secop.protocol.messages import EventMessage, DescribeRequest
-
-
 class TCPConnection(object):
 
-    def __init__(self, connect, port, encoding, framing, **kwds):
+    def __init__(self, connect, port, **kwds):
         self.log = mlzlog.log.getChild('connection', False)
-        self.encoder = ENCODERS[encoding]()
-        self.framer = FRAMERS[framing]()
         self.connection = socket.create_connection((connect, port), 3)
         self.queue = deque()
         self._rcvdata = ''
@@ -120,8 +112,7 @@ class TCPConnection(object):
 
     def send(self, msg):
         self.log.debug("Sending msg %r" % msg)
-        frame = self.encoder.encode(msg)
+        data = encode_msg_frame(*msg.serialize())
-        data = self.framer.encode(frame)
         self.log.debug("raw data: %r" % data)
         self.connection.sendall(data)
 
@@ -133,20 +124,29 @@ class TCPConnection(object):
                 self.log.exception("Exception in RCV thread: %r" % e)
 
     def thread_step(self):
+        data = b''
         while True:
-            data = self.connection.recv(1024)
+            newdata = self.connection.recv(1024)
-            self.log.debug("RCV: got raw data %r" % data)
+            self.log.debug("RCV: got raw data %r" % newdata)
-            if data:
+            data = data + newdata
-                frames = self.framer.decode(data)
+            while True:
-                self.log.debug("RCV: frames %r" % frames)
+                origin, data = get_msg(data)
-                for frame in frames:
+                if origin is None:
-                    msgs = self.encoder.decode(frame)
+                    break  # no more messages to process
-                    self.log.debug("RCV: msgs %r" % msgs)
+                if not origin:  # empty string
-                    for msg in msgs:
+                    continue  # ???
-                        self.handle(msg)
+                msg = decode_msg(origin)
+                # construct msgObj from msg
+                try:
+                    msgObj = Message(*msg)
+                    msgObj.origin = origin.decode('latin-1')
+                    self.handle(msgObj)
+                except Exception:
+                    # ??? what to do here?
+                    pass
 
     def handle(self, msg):
-        if isinstance(msg, EventMessage):
+        if msg.action == EVENTREPLY:
             self.log.info("got Async: %r" % msg)
             for cb in self.callbacks:
                 try:
@@ -188,7 +188,7 @@ class Client(object):
         # XXX: further notification-callbacks needed ???
 
     def populateNamespace(self, namespace):
-        self.connection.send(DescribeRequest())
+        self.connection.send(Message(DESCRIPTIONREQUEST))
         #        reply = self.connection.read()
         #        self.log.info("found modules %r" % reply)
         # create proxies, populate cache....

secop/client/baseclient.py

@@ -32,11 +32,12 @@ from collections import OrderedDict
 import time
 import serial
 
-# Py2/3
 try:
-    import Queue
+    # py3
+    import queue
 except ImportError:
-    import queue as Queue
+    # py2
+    import Queue as queue
 
 import mlzlog
 
@@ -61,7 +62,7 @@ class TCPConnection(object):
         self.connect()
 
     def connect(self):
-        self._readbuffer = Queue.Queue(100)
+        self._readbuffer = queue.Queue(100)
         io = socket.create_connection((self._host, self._port))
         io.setblocking(False)
         io.settimeout(0.3)
@@ -75,7 +76,7 @@ class TCPConnection(object):
             data = u''
             while True:
                 try:
-                    newdata = u''
+                    newdata = b''
                     dlist = [self._io.fileno()]
                     rlist, wlist, xlist = select(dlist, dlist, dlist, 1)
                     if dlist[0] in rlist + wlist:
@@ -92,14 +93,14 @@ class TCPConnection(object):
                     for cb, arg in self.callbacks:
                         cb(arg)
                     return
-                data += newdata
+                data += newdata.decode('latin-1')
                 while '\n' in data:
                     line, data = data.split('\n', 1)
                     try:
                         self._readbuffer.put(line.strip('\r'),
                                              block=True,
                                              timeout=1)
-                    except Queue.Full:
+                    except queue.Full:
                         self.log.debug('rcv queue full! dropping line: %r' %
                                        line)
         finally:
@@ -111,7 +112,7 @@ class TCPConnection(object):
         while i:
             try:
                 return self._readbuffer.get(block=True, timeout=1)
-            except Queue.Empty:
+            except queue.Empty:
                 continue
             if not block:
                 i -= 1
@@ -120,7 +121,7 @@ class TCPConnection(object):
         return not self._readbuffer.empty()
 
     def write(self, data):
-        self._io.sendall(data)
+        self._io.sendall(data.encode('latin-1'))
 
     def writeline(self, line):
         self.write(line + '\n')
@@ -370,7 +371,7 @@ class Client(object):
         try:
             describing_data = self._decode_substruct(
                 ['modules'], describing_data)
-            for modname, module in describing_data['modules'].items():
+            for modname, module in list(describing_data['modules'].items()):
                 describing_data['modules'][modname] = self._decode_substruct(
                     ['parameters', 'commands'], module)
 
@@ -383,14 +384,12 @@ class Client(object):
 #            pprint.pprint(r(describing_data))
 
             for module, moduleData in self.describing_data['modules'].items():
-                for parameter, parameterData in moduleData[
+                for parameter, parameterData in moduleData['parameters'].items():
-                        'parameters'].items():
                     datatype = get_datatype(parameterData['datatype'])
                     self.describing_data['modules'][module]['parameters'] \
                         [parameter]['datatype'] = datatype
-                for _cmdname, cmdData in moduleData[
+                for _cmdname, cmdData in moduleData['commands'].items():
-                        'commands'].items():
+                    cmdData['arguments'] = list(map(get_datatype, cmdData['arguments']))
-                    cmdData['arguments'] = map(get_datatype, cmdData['arguments'])
                     cmdData['resulttype'] = get_datatype(cmdData['resulttype'])
         except Exception as _exc:
             print(formatException(verbose=True))
@@ -544,10 +543,10 @@ class Client(object):
 
     @property
     def modules(self):
-        return self.describing_data['modules'].keys()
+        return list(self.describing_data['modules'].keys())
 
     def getParameters(self, module):
-        return self.describing_data['modules'][module]['parameters'].keys()
+        return list(self.describing_data['modules'][module]['parameters'].keys())
 
     def getModuleProperties(self, module):
         return self.describing_data['modules'][module]['properties']

secop/datatypes.py

@@ -21,6 +21,13 @@
 # *****************************************************************************
 """Define validated data types."""
 
+try:
+    # py2
+    unicode(u'')
+except NameError:
+    # py3
+    unicode = str  # pylint: disable=redefined-builtin
+
 from base64 import b64encode, b64decode
 
 from .errors import ProgrammingError, ParsingError
@@ -30,19 +37,19 @@ Parser = Parser()
 
 # Only export these classes for 'from secop.datatypes import *'
 __all__ = [
-    "DataType",
+    u'DataType',
-    "FloatRange", "IntRange",
+    u'FloatRange', u'IntRange',
-    "BoolType", "EnumType",
+    u'BoolType', u'EnumType',
-    "BLOBType", "StringType",
+    u'BLOBType', u'StringType',
-    "TupleOf", "ArrayOf", "StructOf",
+    u'TupleOf', u'ArrayOf', u'StructOf',
-    "Command",
+    u'Command',
 ]
 
 # base class for all DataTypes
 
 
 class DataType(object):
-    as_json = ['undefined']
+    as_json = [u'undefined']
     IS_COMMAND = False
 
     def validate(self, value):
@@ -80,39 +87,39 @@ class FloatRange(DataType):
         self.min = None if minval is None else float(minval)
         self.max = None if maxval is None else float(maxval)
         # note: as we may compare to Inf all comparisons would be false
-        if (self.min or float('-inf')) <= (self.max or float('+inf')):
+        if (self.min or float(u'-inf')) <= (self.max or float(u'+inf')):
             if minval is None and maxval is None:
-                self.as_json = ['double']
+                self.as_json = [u'double']
             else:
-                self.as_json = ['double', minval, maxval]
+                self.as_json = [u'double', minval, maxval]
         else:
-            raise ValueError('Max must be larger then min!')
+            raise ValueError(u'Max must be larger then min!')
 
     def validate(self, value):
         try:
             value = float(value)
         except:
-            raise ValueError('Can not validate %r to float' % value)
+            raise ValueError(u'Can not validate %r to float' % value)
         if self.min is not None and value < self.min:
-            raise ValueError('%r should not be less then %s' %
+            raise ValueError(u'%r should not be less then %s' %
                              (value, self.min))
         if self.max is not None and value > self.max:
-            raise ValueError('%r should not be greater than %s' %
+            raise ValueError(u'%r should not be greater than %s' %
                              (value, self.max))
         if None in (self.min, self.max):
             return value
         if self.min <= value <= self.max:
             return value
-        raise ValueError('%r should be an float between %.3f and %.3f' %
+        raise ValueError(u'%r should be an float between %.3f and %.3f' %
                          (value, self.min, self.max))
 
     def __repr__(self):
         if self.max is not None:
-            return "FloatRange(%r, %r)" % (
+            return u'FloatRange(%r, %r)' % (
-                float('-inf') if self.min is None else self.min, self.max)
+                float(u'-inf') if self.min is None else self.min, self.max)
         if self.min is not None:
-            return "FloatRange(%r)" % self.min
+            return u'FloatRange(%r)' % self.min
-        return "FloatRange()"
+        return u'FloatRange()'
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -134,31 +141,31 @@ class IntRange(DataType):
         self.min = int(minval) if minval is not None else minval
         self.max = int(maxval) if maxval is not None else maxval
         if self.min is not None and self.max is not None and self.min > self.max:
-            raise ValueError('Max must be larger then min!')
+            raise ValueError(u'Max must be larger then min!')
         if self.min is None and self.max is None:
-            self.as_json = ['int']
+            self.as_json = [u'int']
         else:
-            self.as_json = ['int', self.min, self.max]
+            self.as_json = [u'int', self.min, self.max]
 
     def validate(self, value):
         try:
             value = int(value)
             if self.min is not None and value < self.min:
-                raise ValueError('%r should be an int between %d and %d' %
+                raise ValueError(u'%r should be an int between %d and %d' %
                                  (value, self.min, self.max or 0))
             if self.max is not None and value > self.max:
-                raise ValueError('%r should be an int between %d and %d' %
+                raise ValueError(u'%r should be an int between %d and %d' %
                                  (value, self.min or 0, self.max))
             return value
         except:
-            raise ValueError('Can not validate %r to int' % value)
+            raise ValueError(u'Can not validate %r to int' % value)
 
     def __repr__(self):
         if self.max is not None:
-            return "IntRange(%d, %d)" % (self.min, self.max)
+            return u'IntRange(%d, %d)' % (self.min, self.max)
         if self.min is not None:
-            return "IntRange(%d)" % self.min
+            return u'IntRange(%d)' % self.min
-        return "IntRange()"
+        return u'IntRange()'
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -174,37 +181,37 @@ class IntRange(DataType):
 
 
 class EnumType(DataType):
-    as_json = ['enum']
+    as_json = [u'enum']
 
     def __init__(self, *args, **kwds):
         # enum keys are ints! remember mapping from intvalue to 'name'
-        self.entries = {}
+        self.entries = {}  # maps ints to strings
         num = 0
         for arg in args:
-            if not isinstance(arg, str):
+            if not isinstance(arg, (str, unicode)):
-                raise ValueError('EnumType entries MUST be strings!')
+                raise ValueError(u'EnumType entries MUST be strings!')
             self.entries[num] = arg
             num += 1
-        for k, v in kwds.items():
+        for k, v in list(kwds.items()):
             v = int(v)
             if v in self.entries:
                 raise ValueError(
-                    'keyword argument %r=%d is already assigned %r' %
+                    u'keyword argument %r=%d is already assigned %r' %
                     (k, v, self.entries[v]))
-            self.entries[v] = k
+            self.entries[v] = unicode(k)
 #        if len(self.entries) == 0:
 #            raise ValueError('Empty enums ae not allowed!')
         # also keep a mapping from name strings to numbers
-        self.reversed = {}
+        self.reversed = {}  # maps Strings to ints
         for k, v in self.entries.items():
             if v in self.reversed:
-                raise ValueError('Mapping for %r=%r is not Unique!' % (v, k))
+                raise ValueError(u'Mapping for %r=%r is not Unique!' % (v, k))
             self.reversed[v] = k
-        self.as_json = ['enum', self.reversed.copy()]
+        self.as_json = [u'enum', self.reversed.copy()]
 
     def __repr__(self):
-        return "EnumType(%s)" % ', '.join(
+        return u'EnumType(%s)' % u', '.join(
-            ['%s=%d' % (v, k) for k, v in self.entries.items()])
+            [u'%s=%d' % (v, k) for k, v in list(self.entries.items())])
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -212,8 +219,8 @@ class EnumType(DataType):
             return self.reversed[value]
         if int(value) in self.entries:
             return int(value)
-        raise ValueError('%r is not one of %s' %
+        raise ValueError(u'%r is not one of %s' %
-            (str(value), ', '.join(self.reversed.keys())))
+            (unicode(value), u', '.join(list(self.reversed.keys()))))
 
     def import_value(self, value):
         """returns a python object from serialisation"""
@@ -223,11 +230,11 @@ class EnumType(DataType):
     def validate(self, value):
         """return the validated (internal) value or raise"""
         if value in self.reversed:
-            return value
+            return self.reversed[value]
         if int(value) in self.entries:
-            return self.entries[int(value)]
+            return int(value)
-        raise ValueError('%r is not one of %s' %
+        raise ValueError(u'%r is not one of %s' %
-                         (str(value), ', '.join(map(str, self.entries.keys()))))
+                         (unicode(value), u', '.join(map(unicode, self.entries))))
 
     def from_string(self, text):
         value = text
@@ -240,36 +247,36 @@ class BLOBType(DataType):
     def __init__(self, maxsize=None, minsize=0):
 
         if maxsize is None:
-            raise ValueError('BLOBType needs a maximum number of Bytes count!')
+            raise ValueError(u'BLOBType needs a maximum number of Bytes count!')
         minsize, maxsize = min(minsize, maxsize), max(minsize, maxsize)
         self.minsize = minsize
         self.maxsize = maxsize
         if minsize < 0:
-            raise ValueError('sizes must be bigger than or equal to 0!')
+            raise ValueError(u'sizes must be bigger than or equal to 0!')
         if minsize:
-            self.as_json = ['blob', maxsize, minsize]
+            self.as_json = [u'blob', maxsize, minsize]
         else:
-            self.as_json = ['blob', maxsize]
+            self.as_json = [u'blob', maxsize]
 
     def __repr__(self):
         if self.minsize:
-            return 'BLOB(%s, %s)' % (
+            return u'BLOB(%s, %s)' % (
-                str(self.minsize) if self.minsize else 'unspecified',
+                unicode(self.minsize) if self.minsize else u'unspecified',
-                str(self.maxsize) if self.maxsize else 'unspecified')
+                unicode(self.maxsize) if self.maxsize else u'unspecified')
-        return 'BLOB(%s)' % (str(self.minsize) if self.minsize else 'unspecified')
+        return u'BLOB(%s)' % (unicode(self.minsize) if self.minsize else u'unspecified')
 
     def validate(self, value):
         """return the validated (internal) value or raise"""
-        if type(value) not in [str, unicode]:
+        if type(value) not in [unicode, str]:
-            raise ValueError('%r has the wrong type!' % value)
+            raise ValueError(u'%r has the wrong type!' % value)
         size = len(value)
         if size < self.minsize:
             raise ValueError(
-                '%r must be at least %d bytes long!' % (value, self.minsize))
+                u'%r must be at least %d bytes long!' % (value, self.minsize))
         if self.maxsize is not None:
             if size > self.maxsize:
                 raise ValueError(
-                    '%r must be at most %d bytes long!' % (value, self.maxsize))
+                    u'%r must be at most %d bytes long!' % (value, self.maxsize))
         return value
 
     def export_value(self, value):
@@ -287,76 +294,76 @@ class BLOBType(DataType):
 
 
 class StringType(DataType):
-    as_json = ['string']
+    as_json = [u'string']
     minsize = None
     maxsize = None
 
     def __init__(self, maxsize=255, minsize=0):
         if maxsize is None:
-            raise ValueError('StringType needs a maximum bytes count!')
+            raise ValueError(u'StringType needs a maximum bytes count!')
         minsize, maxsize = min(minsize, maxsize), max(minsize, maxsize)
 
         if minsize < 0:
-            raise ValueError('sizes must be >= 0')
+            raise ValueError(u'sizes must be >= 0')
         if minsize:
-            self.as_json = ['string', maxsize, minsize]
+            self.as_json = [u'string', maxsize, minsize]
         else:
-            self.as_json = ['string', maxsize]
+            self.as_json = [u'string', maxsize]
         self.minsize = minsize
         self.maxsize = maxsize
 
     def __repr__(self):
         if self.minsize:
-            return 'StringType(%s, %s)' % (
+            return u'StringType(%s, %s)' % (
-                str(self.minsize) or 'unspecified', str(self.maxsize) or 'unspecified')
+                unicode(self.minsize) or u'unspecified', unicode(self.maxsize) or u'unspecified')
-        return 'StringType(%s)' % str(self.maxsize)
+        return u'StringType(%s)' % unicode(self.maxsize)
 
     def validate(self, value):
         """return the validated (internal) value or raise"""
-        if type(value) not in [str, unicode]:
+        if type(value) not in (unicode, str):
-            raise ValueError('%r has the wrong type!' % value)
+            raise ValueError(u'%r has the wrong type!' % value)
         size = len(value)
         if size < self.minsize:
             raise ValueError(
-                '%r must be at least %d bytes long!' % (value, self.minsize))
+                u'%r must be at least %d bytes long!' % (value, self.minsize))
         if self.maxsize is not None:
             if size > self.maxsize:
                 raise ValueError(
-                    '%r must be at most %d bytes long!' % (value, self.maxsize))
+                    u'%r must be at most %d bytes long!' % (value, self.maxsize))
-        if '\0' in value:
+        if u'\0' in value:
             raise ValueError(
-                'Strings are not allowed to embed a \\0! Use a Blob instead!')
+                u'Strings are not allowed to embed a \\0! Use a Blob instead!')
         return value
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
-        return '%s' % value
+        return u'%s' % value
 
     def import_value(self, value):
         """returns a python object from serialisation"""
         # XXX: do we keep it as unicode str, or convert it to something else? (UTF-8 maybe?)
-        return str(value)
+        return unicode(value)
 
     def from_string(self, text):
-        value = str(text)
+        value = unicode(text)
         return self.validate(value)
 
 # Bool is a special enum
 
 
 class BoolType(DataType):
-    as_json = ['bool']
+    as_json = [u'bool']
 
     def __repr__(self):
-        return 'BoolType()'
+        return u'BoolType()'
 
     def validate(self, value):
         """return the validated (internal) value or raise"""
-        if value in [0, '0', 'False', 'false', 'no', 'off', False]:
+        if value in [0, u'0', u'False', u'false', u'no', u'off', False]:
             return False
-        if value in [1, '1', 'True', 'true', 'yes', 'on', True]:
+        if value in [1, u'1', u'True', u'true', u'yes', u'on', True]:
             return True
-        raise ValueError('%r is not a boolean value!' % value)
+        raise ValueError(u'%r is not a boolean value!' % value)
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -382,26 +389,26 @@ class ArrayOf(DataType):
         self.subtype = subtype
         if not isinstance(subtype, DataType):
             raise ValueError(
-                'ArrayOf only works with DataType objs as first argument!')
+                u'ArrayOf only works with DataType objs as first argument!')
 
         if maxsize is None:
-            raise ValueError('ArrayOf needs a maximum size')
+            raise ValueError(u'ArrayOf needs a maximum size')
         minsize, maxsize = min(minsize, maxsize), max(minsize, maxsize)
         if minsize < 0:
-            raise ValueError('sizes must be > 0')
+            raise ValueError(u'sizes must be > 0')
         if maxsize < 1:
-            raise ValueError('Maximum size must be >= 1!')
+            raise ValueError(u'Maximum size must be >= 1!')
         # if only one arg is given, it is maxsize!
         if minsize:
-            self.as_json = ['array', subtype.as_json, maxsize, minsize]
+            self.as_json = [u'array', subtype.as_json, maxsize, minsize]
         else:
-            self.as_json = ['array', subtype.as_json, maxsize]
+            self.as_json = [u'array', subtype.as_json, maxsize]
         self.minsize = minsize
         self.maxsize = maxsize
 
     def __repr__(self):
-        return 'ArrayOf(%s, %s, %s)' % (
+        return u'ArrayOf(%s, %s, %s)' % (
-            repr(self.subtype), self.minsize or 'unspecified', self.maxsize or 'unspecified')
+            repr(self.subtype), self.minsize or u'unspecified', self.maxsize or u'unspecified')
 
     def validate(self, value):
         """validate a external representation to an internal one"""
@@ -409,15 +416,15 @@ class ArrayOf(DataType):
             # check number of elements
             if self.minsize is not None and len(value) < self.minsize:
                 raise ValueError(
-                    'Array too small, needs at least %d elements!' %
+                    u'Array too small, needs at least %d elements!' %
                     self.minsize)
             if self.maxsize is not None and len(value) > self.maxsize:
                 raise ValueError(
-                    'Array too big, holds at most %d elements!' % self.minsize)
+                    u'Array too big, holds at most %d elements!' % self.minsize)
             # apply subtype valiation to all elements and return as list
             return [self.subtype.validate(elem) for elem in value]
         raise ValueError(
-            'Can not convert %s to ArrayOf DataType!' % repr(value))
+            u'Can not convert %s to ArrayOf DataType!' % repr(value))
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -430,7 +437,7 @@ class ArrayOf(DataType):
     def from_string(self, text):
         value, rem = Parser.parse(text)
         if rem:
-            raise ParsingError('trailing garbage: %r' % rem)
+            raise ParsingError(u'trailing garbage: %r' % rem)
         return self.validate(value)
 
 
@@ -438,16 +445,16 @@ class TupleOf(DataType):
 
     def __init__(self, *subtypes):
         if not subtypes:
-            raise ValueError('Empty tuples are not allowed!')
+            raise ValueError(u'Empty tuples are not allowed!')
         for subtype in subtypes:
             if not isinstance(subtype, DataType):
                 raise ValueError(
-                    'TupleOf only works with DataType objs as arguments!')
+                    u'TupleOf only works with DataType objs as arguments!')
         self.subtypes = subtypes
-        self.as_json = ['tuple', [subtype.as_json for subtype in subtypes]]
+        self.as_json = [u'tuple', [subtype.as_json for subtype in subtypes]]
 
     def __repr__(self):
-        return 'TupleOf(%s)' % ', '.join([repr(st) for st in self.subtypes])
+        return u'TupleOf(%s)' % u', '.join([repr(st) for st in self.subtypes])
 
     def validate(self, value):
         """return the validated value or raise"""
@@ -455,13 +462,13 @@ class TupleOf(DataType):
         try:
             if len(value) != len(self.subtypes):
                 raise ValueError(
-                    'Illegal number of Arguments! Need %d arguments.' %
+                    u'Illegal number of Arguments! Need %d arguments.' %
                         (len(self.subtypes)))
             # validate elements and return as list
             return [sub.validate(elem)
                     for sub, elem in zip(self.subtypes, value)]
         except Exception as exc:
-            raise ValueError('Can not validate:', str(exc))
+            raise ValueError(u'Can not validate:', unicode(exc))
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -474,7 +481,7 @@ class TupleOf(DataType):
     def from_string(self, text):
         value, rem = Parser.parse(text)
         if rem:
-            raise ParsingError('trailing garbage: %r' % rem)
+            raise ParsingError(u'trailing garbage: %r' % rem)
         return self.validate(value)
 
 
@@ -482,57 +489,57 @@ class StructOf(DataType):
 
     def __init__(self, **named_subtypes):
         if not named_subtypes:
-            raise ValueError('Empty structs are not allowed!')
+            raise ValueError(u'Empty structs are not allowed!')
-        for name, subtype in named_subtypes.items():
+        for name, subtype in list(named_subtypes.items()):
             if not isinstance(subtype, DataType):
                 raise ProgrammingError(
-                    'StructOf only works with named DataType objs as keyworded arguments!')
+                    u'StructOf only works with named DataType objs as keyworded arguments!')
-            if not isinstance(name, (str, unicode)):
+            if not isinstance(name, (unicode, str)):
                 raise ProgrammingError(
-                    'StructOf only works with named DataType objs as keyworded arguments!')
+                    u'StructOf only works with named DataType objs as keyworded arguments!')
         self.named_subtypes = named_subtypes
-        self.as_json = ['struct', dict((n, s.as_json)
+        self.as_json = [u'struct', dict((n, s.as_json)
-                                       for n, s in named_subtypes.items())]
+                                       for n, s in list(named_subtypes.items()))]
 
     def __repr__(self):
-        return 'StructOf(%s)' % ', '.join(
+        return u'StructOf(%s)' % u', '.join(
-            ['%s=%s' % (n, repr(st)) for n, st in self.named_subtypes.iteritems()])
+            [u'%s=%s' % (n, repr(st)) for n, st in list(self.named_subtypes.items())])
 
     def validate(self, value):
         """return the validated value or raise"""
         try:
-            if len(value.keys()) != len(self.named_subtypes.keys()):
+            if len(list(value.keys())) != len(list(self.named_subtypes.keys())):
                 raise ValueError(
-                    'Illegal number of Arguments! Need %d arguments.' %
+                    u'Illegal number of Arguments! Need %d arguments.' %
-                        len(self.named_subtypes.keys()))
+                        len(list(self.named_subtypes.keys())))
             # validate elements and return as dict
-            return dict((str(k), self.named_subtypes[k].validate(v))
+            return dict((unicode(k), self.named_subtypes[k].validate(v))
-                        for k, v in value.items())
+                        for k, v in list(value.items()))
         except Exception as exc:
-            raise ValueError('Can not validate %s: %s' % (repr(value), str(exc)))
+            raise ValueError(u'Can not validate %s: %s' % (repr(value), unicode(exc)))
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
-        if len(value.keys()) != len(self.named_subtypes.keys()):
+        if len(list(value.keys())) != len(list(self.named_subtypes.keys())):
             raise ValueError(
-                'Illegal number of Arguments! Need %d arguments.' % len(
+                u'Illegal number of Arguments! Need %d arguments.' % len(
-                    self.namd_subtypes.keys()))
+                    list(self.namd_subtypes.keys())))
-        return dict((str(k), self.named_subtypes[k].export_value(v))
+        return dict((unicode(k), self.named_subtypes[k].export_value(v))
-                    for k, v in value.items())
+                    for k, v in list(value.items()))
 
     def import_value(self, value):
         """returns a python object from serialisation"""
-        if len(value.keys()) != len(self.named_subtypes.keys()):
+        if len(list(value.keys())) != len(list(self.named_subtypes.keys())):
             raise ValueError(
-                'Illegal number of Arguments! Need %d arguments.' % len(
+                u'Illegal number of Arguments! Need %d arguments.' % len(
-                    self.namd_subtypes.keys()))
+                    list(self.namd_subtypes.keys())))
-        return dict((str(k), self.named_subtypes[k].import_value(v))
+        return dict((unicode(k), self.named_subtypes[k].import_value(v))
-                    for k, v in value.items())
+                    for k, v in list(value.items()))
 
     def from_string(self, text):
         value, rem = Parser.parse(text)
         if rem:
-            raise ParsingError('trailing garbage: %r' % rem)
+            raise ParsingError(u'trailing garbage: %r' % rem)
         return self.validate(dict(value))
 
 
@@ -543,50 +550,50 @@ class Command(DataType):
     def __init__(self, argtypes=tuple(), resulttype=None):
         for arg in argtypes:
             if not isinstance(arg, DataType):
-                raise ValueError('Command: Argument types must be DataTypes!')
+                raise ValueError(u'Command: Argument types must be DataTypes!')
         if resulttype is not None:
             if not isinstance(resulttype, DataType):
-                raise ValueError('Command: result type must be DataTypes!')
+                raise ValueError(u'Command: result type must be DataTypes!')
         self.argtypes = argtypes
         self.resulttype = resulttype
 
         if resulttype is not None:
-            self.as_json = ['command',
+            self.as_json = [u'command',
                             [t.as_json for t in argtypes],
                             resulttype.as_json]
         else:
-            self.as_json = ['command',
+            self.as_json = [u'command',
                             [t.as_json for t in argtypes],
                             None]  # XXX: or NoneType ???
 
     def __repr__(self):
-        argstr = ', '.join(repr(arg) for arg in self.argtypes)
+        argstr = u', '.join(repr(arg) for arg in self.argtypes)
         if self.resulttype is None:
-            return 'Command(%s)' % argstr
+            return u'Command(%s)' % argstr
-        return 'Command(%s)->%s' % (argstr, repr(self.resulttype))
+        return u'Command(%s)->%s' % (argstr, repr(self.resulttype))
 
     def validate(self, value):
         """return the validated arguments value or raise"""
         try:
             if len(value) != len(self.argtypes):
                 raise ValueError(
-                    'Illegal number of Arguments! Need %d arguments.' %
+                    u'Illegal number of Arguments! Need %d arguments.' %
                         len(self.argtypes))
             # validate elements and return
             return [t.validate(v) for t, v in zip(self.argtypes, value)]
         except Exception as exc:
-            raise ValueError('Can not validate %s: %s' % (repr(value), str(exc)))
+            raise ValueError(u'Can not validate %s: %s' % (repr(value), unicode(exc)))
 
     def export_value(self, value):
-        raise ProgrammingError('values of type command can not be transported!')
+        raise ProgrammingError(u'values of type command can not be transported!')
 
     def import_value(self, value):
-        raise ProgrammingError('values of type command can not be transported!')
+        raise ProgrammingError(u'values of type command can not be transported!')
 
     def from_string(self, text):
         value, rem = Parser.parse(text)
         if rem:
-            raise ParsingError('trailing garbage: %r' % rem)
+            raise ParsingError(u'trailing garbage: %r' % rem)
         return self.validate(value)
 
 
@@ -601,7 +608,7 @@ DATATYPES = dict(
     tuple=lambda subtypes: TupleOf(*map(get_datatype, subtypes)),
     enum=lambda kwds: EnumType(**kwds),
     struct=lambda named_subtypes: StructOf(
-        **dict((n, get_datatype(t)) for n, t in named_subtypes.items())),
+        **dict((n, get_datatype(t)) for n, t in list(named_subtypes.items()))),
     command=Command,
 )
 
@@ -616,12 +623,12 @@ def get_datatype(json):
         return json
     if not isinstance(json, list):
         raise ValueError(
-            'Can not interpret datatype %r, it should be a list!' % json)
+            u'Can not interpret datatype %r, it should be a list!' % json)
     if len(json) < 1:
-        raise ValueError('can not validate %r' % json)
+        raise ValueError(u'can not validate %r' % json)
     base = json[0]
     if base in DATATYPES:
-        if base in ('enum', 'struct'):
+        if base in (u'enum', u'struct'):
             if len(json) > 1:
                 args = json[1:]
             else:
@@ -631,5 +638,5 @@ def get_datatype(json):
         try:
             return DATATYPES[base](*args)
         except (TypeError, AttributeError):
-            raise ValueError('Invalid datatype descriptor in %r' % json)
+            raise ValueError(u'Invalid datatype descriptor in %r' % json)
-    raise ValueError('can not convert %r to datatype' % json)
+    raise ValueError(u'can not convert %r to datatype' % json)

secop/errors.py

@@ -1,7 +1,5 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
-#
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
 # Foundation; either version 2 of the License, or (at your option) any later

secop/gui/modulectrl.py

@@ -24,10 +24,17 @@
 
 from __future__ import print_function
 
+try:
+    # py2
+    unicode(u'')
+except NameError:
+    # py3
+    unicode = str  # pylint: disable=redefined-builtin
+
 from secop.gui.util import loadUi
 from secop.gui.params import ParameterView
 
-from secop.datatypes import *  # pylint: disable=unused-wildcard-import,wildcard-import
+#from secop.datatypes import ...
 
 from secop.gui.qt import QDialog, QLabel, QCheckBox, QWidget, QMessageBox, \
     QPushButton, QSizePolicy
@@ -93,7 +100,7 @@ class ParameterGroup(QWidget):
         self._row = 0
         self._widgets = []
 
-        self.paramGroupBox.setTitle('Group: ' + str(groupname))
+        self.paramGroupBox.setTitle('Group: ' + unicode(groupname))
         self.paramGroupBox.toggled.connect(self.on_toggle_clicked)
         self.paramGroupBox.setChecked(False)
 
@@ -275,7 +282,7 @@ class ModuleCtrl(QWidget):
             label.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Preferred)
 
             # make 'display' label
-            view = QLabel(str(props[prop]))
+            view = QLabel(unicode(props[prop]))
             view.setFont(self.font())
             view.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred)
             view.setWordWrap(True)
@@ -351,9 +358,9 @@ class ModuleCtrl(QWidget):
         try:
             self._node.setParameter(module, parameter, target)
         except Exception as e:
-            QMessageBox.warning(self.parent(), 'Operation failed', str(e))
+            QMessageBox.warning(self.parent(), 'Operation failed', unicode(e))
 
     def _updateValue(self, module, parameter, value):
         if module != self._module:
             return
-        self._paramWidgets[parameter][1].updateValue(str(value[0]))
+        self._paramWidgets[parameter][1].updateValue(unicode(value[0]))

secop/gui/nodectrl.py

@@ -21,6 +21,14 @@
 #
 # *****************************************************************************
 
+from __future__ import print_function, division
+
+try:
+    # py2
+    unicode(u'')
+except NameError:
+    unicode = str  # pylint: disable=redefined-builtin
+
 import pprint
 import json
 from time import sleep
@@ -121,9 +129,9 @@ class NodeCtrl(QWidget):
 
     def _getLogWidth(self):
         fontMetrics = QFontMetrics(QFont('Monospace'))
-        # calculate max avail characters by using an a (which is possible
+        # calculate max avail characters by using an m (which is possible
         # due to monospace)
-        result = self.logTextBrowser.width() / fontMetrics.width('a')
+        result = self.logTextBrowser.width() / fontMetrics.width('m')
         return result
 
     def _init_modules_tab(self):
@@ -277,6 +285,7 @@ class DrivableWidget(ReadableWidget):
         if self._is_enum:
             # EnumType: disable Linedit
             self.targetLineEdit.setHidden(True)
+            self.cmdPushButton.setHidden(True)
         else:
             # normal types: disable Combobox
             self.targetComboBox.setHidden(True)

secop/gui/params/__init__.py

@@ -21,11 +21,17 @@
 #
 # *****************************************************************************
 
+try:
+    # py2
+    unicode(u'')
+except NameError:
+    unicode = str  # pylint: disable=redefined-builtin
+
 from secop.gui.qt import QWidget, QLabel, QPushButton as QButton, QLineEdit, \
     QMessageBox, QCheckBox, QSizePolicy, Qt, pyqtSignal, pyqtSlot
 
 from secop.gui.util import loadUi
-from secop.datatypes import *  # pylint: disable=wildcard-import
+from secop.datatypes import EnumType
 
 
 class ParameterWidget(QWidget):
@@ -107,9 +113,8 @@ class EnumParameterWidget(GenericParameterWidget):
 
     @pyqtSlot()
     def on_setPushButton_clicked(self):
-        enumval, enumname = self._map[self.setComboBox.currentIndex()]
+        _enumval, enumname = self._map[self.setComboBox.currentIndex()]
         self.setRequested.emit(self._module, self._paramcmd, enumname)
-        self.setRequested.emit(self._module, self._paramcmd, str(enumval))
 
     def updateValue(self, valuestr):
         try:

secop/gui/paramview.py

@@ -21,10 +21,16 @@
 #
 # *****************************************************************************
 
+try:
+    # py2
+    unicode(u'')
+except NameError:
+    # py3
+    unicode = str  # pylint: disable=redefined-builtin
+
 from secop.gui.qt import QWidget, QLabel, QSizePolicy
 
 from secop.gui.util import loadUi
-#from secop.datatypes import get_datatype
 
 
 class ParameterView(QWidget):

secop/gui/qt.py

@@ -1,3 +1,26 @@
+#  -*- coding: utf-8 -*-
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
+# *****************************************************************************
+"""Import needed stuff from PyQt4/PyQt5"""
+
 # pylint: disable=unused-import
 from __future__ import print_function
 

secop/gui/valuewidgets.py

@@ -23,7 +23,8 @@
 
 from __future__ import print_function
 
-from secop.datatypes import *  # pylint: disable=unused-wildcard-import,wildcard-import
+from secop.datatypes import FloatRange, IntRange, StringType, BLOBType, \
+    EnumType, BoolType, TupleOf, StructOf, ArrayOf
 
 from secop.gui.qt import QDialog, QLabel, QLineEdit,\
     QGroupBox, QSpinBox, QDoubleSpinBox, QComboBox, QCheckBox, \
@@ -112,8 +113,8 @@ class FloatWidget(QDoubleSpinBox):
         self.datatype = datatype
         if readonly:
             self.setEnabled(False)
-        self.setMaximum(datatype.max)
+        self.setMaximum(datatype.max or 1e6)  # XXX!
-        self.setMinimum(datatype.min)
+        self.setMinimum(datatype.min or 0)  # XXX!
         self.setDecimals(12)
 
     def get_value(self):
@@ -141,7 +142,7 @@ class TupleWidget(QFrame):
         self.update()
 
     def get_value(self):
-        return [v.validate(w.get_value()) for w,v in zip(self.subwidgets, self.datatypes)]
+        return [v.validate(w.get_value()) for w, v in zip(self.subwidgets, self.datatypes)]
 
     def set_value(self, value):
         for w, _ in zip(self.subwidgets, value):
@@ -225,7 +226,7 @@ class msg(QDialog):
         row = 0
 
         self.gridLayout.addWidget(QLabel('struct'), row, 0)
-        dt = StructOf(i=IntRange(0,10), f=FloatRange(), b=BoolType())
+        dt = StructOf(i=IntRange(0, 10), f=FloatRange(), b=BoolType())
         w = StructWidget(dt)
         self.gridLayout.addWidget(w, row, 1)
         row+=1

secop/lib/__init__.py

@@ -109,7 +109,7 @@ def mkthread(func, *args, **kwds):
 
 def formatExtendedFrame(frame):
     ret = []
-    for key, value in frame.f_locals.iteritems():
+    for key, value in frame.f_locals.items():
         try:
             valstr = repr(value)[:256]
         except Exception:
@@ -217,7 +217,8 @@ def tcpSocket(host, defaultport, timeout=None):
     return s
 
 
-def closeSocket(sock, socket=socket):
+# keep a reference to socket to avoid (interpreter) shut-down problems
+def closeSocket(sock, socket=socket):  # pylint: disable=redefined-outer-name
     """Do our best to close a socket."""
     if sock is None:
         return

secop/lib/parsing.py

@@ -44,14 +44,12 @@ class LocalTimezone(tzinfo):
     def utcoffset(self, dt):
         if self._isdst(dt):
             return self.DSTOFFSET
-        else:
+        return self.STDOFFSET
-            return self.STDOFFSET
 
     def dst(self, dt):
         if self._isdst(dt):
             return self.DSTDIFF
-        else:
+        return self.ZERO
-            return self.ZERO
 
     def tzname(self, dt):
         return time.tzname[self._isdst(dt)]
@@ -81,7 +79,7 @@ def format_time(timestamp=None):
 
 class Timezone(tzinfo):
 
-    def __init__(self, offset, name='unknown timezone'):
+    def __init__(self, offset, name='unknown timezone'): # pylint: disable=W0231
         self.offset = offset
         self.name = name
 
@@ -114,7 +112,7 @@ def _parse_isostring(isostring):
             kw['microsecond'] = kw['microsecond'].ljust(6, '0')
         _tzinfo = kw.pop('tzinfo')
         if _tzinfo == 'Z':
-            _tzinfo = timezone.utc
+            _tzinfo = timezone.utc  # pylint: disable=E0602
         elif _tzinfo is not None:
             offset_mins = int(_tzinfo[-2:]) if len(_tzinfo) > 3 else 0
             offset_hours = int(_tzinfo[1:3])
@@ -304,9 +302,9 @@ class ArgsParser(object):
         if self.peek() == '-':
             self.get()
             number = self.parse_pos_int()
-            if number is None:
+            if number is not None:
-                return number
+                return -number  # pylint: disable=invalid-unary-operand-type
-            return -number
+            return None
         return self.parse_pos_int()
 
     def parse_pos_int(self):
@@ -314,7 +312,7 @@ class ArgsParser(object):
         number = 0
         if self.peek() not in self.DIGITS_CHARS:
             return None
-        while (self.peek() in self.DIGITS_CHARS):
+        while self.peek() in self.DIGITS_CHARS:
             number = number * 10 + int(self.get())
         self.skip()
         return number

secop/lib/sequence.py

@@ -73,7 +73,7 @@ class SequencerMixin(object):
         self._seq_fault_on_error = fault_on_error
         self._seq_fault_on_stop = fault_on_stop
         self._seq_stopflag = False
-        self._seq_phase = ''
+        self._seq_phase = u''
         self._seq_error = None
         self._seq_stopped = None
 
@@ -115,7 +115,7 @@ class SequencerMixin(object):
         the default is to only go into ALARM.
         """
         if self.seq_is_alive():
-            raise IsBusyError('move sequence already in progress')
+            raise IsBusyError(u'move sequence already in progress')
 
         self._seq_stopflag = False
         self._seq_error = self._seq_stopped = None
@@ -128,7 +128,7 @@ class SequencerMixin(object):
 
     def read_status(self, maxage=0):
         if self.seq_is_alive():
-            return status.BUSY, 'moving: ' + self._seq_phase
+            return status.BUSY, u'moving: ' + self._seq_phase
         elif self._seq_error:
             if self._seq_fault_on_error:
                 return status.ERROR, self._seq_error
@@ -137,9 +137,9 @@ class SequencerMixin(object):
             if self._seq_fault_on_stop:
                 return status.ERROR, self._seq_stopped
             return status.WARN, self._seq_stopped
-        if hasattr(self, 'read_hw_status'):
+        if hasattr(self, u'read_hw_status'):
             return self.read_hw_status(maxage)
-        return status.OK, ''
+        return status.OK, u''
 
     def do_stop(self):
         if self.seq_is_alive():
@@ -149,7 +149,7 @@ class SequencerMixin(object):
         try:
             self._seq_thread_inner(seq, store_init)
         except Exception as e:
-            self.log.exception('unhandled error in sequence thread: %s', e)
+            self.log.exception(u'unhandled error in sequence thread: %s', e)
             self._seq_error = str(e)
         finally:
             self._seq_thread = None
@@ -158,11 +158,11 @@ class SequencerMixin(object):
     def _seq_thread_inner(self, seq, store_init):
         store = Namespace()
         store.__dict__.update(store_init)
-        self.log.debug('sequence: starting, values %s', store_init)
+        self.log.debug(u'sequence: starting, values %s', store_init)
 
         for step in seq:
             self._seq_phase = step.desc
-            self.log.debug('sequence: entering phase: %s', step.desc)
+            self.log.debug(u'sequence: entering phase: %s', step.desc)
             try:
                 i = 0
                 while True:
@@ -170,10 +170,10 @@ class SequencerMixin(object):
                     result = step.func(store, *step.args)
                     if self._seq_stopflag:
                         if result:
-                            self._seq_stopped = 'stopped while %s' % step.desc
+                            self._seq_stopped = u'stopped while %s' % step.desc
                         else:
-                            self._seq_stopped = 'stopped after %s' % step.desc
+                            self._seq_stopped = u'stopped after %s' % step.desc
-                        cleanup_func = step.kwds.get('cleanup', None)
+                        cleanup_func = step.kwds.get(u'cleanup', None)
                         if callable(cleanup_func):
                             try:
                                 cleanup_func(store, result, *step.args)
@@ -187,6 +187,6 @@ class SequencerMixin(object):
                     i += 1
             except Exception as e:
                 self.log.exception(
-                    'error in sequence step %r: %s', step.desc, e)
+                    u'error in sequence step %r: %s', step.desc, e)
-                self._seq_error = 'during %s: %s' % (step.desc, e)
+                self._seq_error = u'during %s: %s' % (step.desc, e)
                 break

secop/modules.py

@@ -30,6 +30,8 @@ import time
 import types
 import inspect
 
+import six  # for py2/3 compat
+
 from secop.lib import formatExtendedStack, mkthread
 from secop.lib.parsing import format_time
 from secop.errors import ConfigError, ProgrammingError
@@ -123,7 +125,7 @@ class Override(object):
 
     def apply(self, paramobj):
         if isinstance(paramobj, Param):
-            for k, v in self.kwds.iteritems():
+            for k, v in self.kwds.items():
                 if hasattr(paramobj, k):
                     setattr(paramobj, k, v)
                     return paramobj
@@ -184,9 +186,9 @@ class ModuleMeta(type):
         newparams = getattr(newtype, 'parameters')
         for base in reversed(bases):
             overrides = getattr(base, 'overrides', {})
-            for n, o in overrides.iteritems():
+            for n, o in overrides.items():
                 newparams[n] = o.apply(newparams[n].copy())
-        for n, o in attrs.get('overrides', {}).iteritems():
+        for n, o in attrs.get('overrides', {}).items():
             newparams[n] = o.apply(newparams[n].copy())
 
         # check validity of Param entries
@@ -291,9 +293,9 @@ class ModuleMeta(type):
 # if you want to 'update from the hardware', call self.read_<pname>
 # the return value of this method will be used as the new cached value and
 # be returned.
+@six.add_metaclass(ModuleMeta)
 class Module(object):
     """Basic Module, doesn't do much"""
-    __metaclass__ = ModuleMeta
     # static properties, definitions in derived classes should overwrite earlier ones.
     # how to configure some stuff which makes sense to take from configfile???
     properties = {
@@ -318,13 +320,13 @@ class Module(object):
         self.name = devname
         # make local copies of parameter
         params = {}
-        for k, v in self.parameters.items()[:]:
+        for k, v in list(self.parameters.items()):
             params[k] = v.copy()
 
         self.parameters = params
         # make local copies of properties
         props = {}
-        for k, v in self.properties.items()[:]:
+        for k, v in list(self.properties.items()):
             props[k] = v
 
         self.properties = props
@@ -332,7 +334,7 @@ class Module(object):
         # check and apply properties specified in cfgdict
         # moduleproperties are to be specified as
         # '.<propertyname>=<propertyvalue>'
-        for k, v in cfgdict.items():
+        for k, v in list(cfgdict.items()):  # keep list() as dict may change during iter
             if k[0] == '.':
                 if k[1:] in self.properties:
                     self.properties[k[1:]] = v
@@ -347,13 +349,13 @@ class Module(object):
         #self.properties['interface'] = self.properties['interfaces'][0]
 
         # remove unset (default) module properties
-        for k, v in self.properties.items():
+        for k, v in list(self.properties.items()):  # keep list() as dict may change during iter
             if v is None:
                 del self.properties[k]
 
         # check and apply parameter_properties
         # specified as '<paramname>.<propertyname> = <propertyvalue>'
-        for k, v in cfgdict.items()[:]:
+        for k, v in list(cfgdict.items()):  # keep list() as dict may change during iter
             if '.' in k[1:]:
                 paramname, propname = k.split('.', 1)
                 if paramname in self.parameters:
@@ -370,8 +372,8 @@ class Module(object):
             if k not in self.parameters:
                 raise ConfigError(
                     'Module %s:config Parameter %r '
-                    'not unterstood! (use on of %r)' %
+                    'not unterstood! (use one of %s)' %
-                    (self.name, k, self.parameters.keys()))
+                    (self.name, k, ', '.join(self.parameters)))
 
         # complain if a Param entry has no default value and
         # is not specified in cfgdict
@@ -464,7 +466,7 @@ class Readable(Module):
             fastpoll = stat[0] == status.BUSY
 #        if fastpoll:
 #            self.log.info('fastpoll!')
-        for pname, pobj in self.parameters.iteritems():
+        for pname, pobj in self.parameters.items():
             if not pobj.poll:
                 continue
             if pname == 'status':
@@ -520,7 +522,7 @@ class Communicator(Module):
     """
 
     commands = {
-        "communicate" : Command("provides the simplest mean to communication",
+        "communicate": Command("provides the simplest mean to communication",
                             arguments=[StringType()],
                             result=StringType()
                            ),

secop/parse.py

@@ -36,6 +36,7 @@ text -> string
 
 further convertions are done by the validator of the datatype....
 """
+from __future__ import print_function
 
 from collections import OrderedDict
 
@@ -160,7 +161,7 @@ class Parser(object):
         return self.parse_string(orgtext)
 
     def parse(self, orgtext):
-        print "parsing %r" % orgtext
+        print("parsing %r" % orgtext)
         res, rem = self.parse_sub(orgtext)
         if rem and rem[0] in u',;':
             return self.parse_sub(u'[%s]' % orgtext)

secop/protocol/dispatcher.py

@@ -36,24 +36,28 @@ Interface to the modules:
  - remove_module(modulename_or_obj): removes the module (during shutdown)
 
 """
+from __future__ import print_function
 
-import time
+from time import time as currenttime
 import threading
 
-from secop.protocol.messages import Value, CommandReply, HelpMessage, \
+from secop.protocol.messages import Message, EVENTREPLY, IDENTREQUEST
-    DeactivateReply, IdentifyReply, DescribeReply, HeartbeatReply, \
-    ActivateReply, WriteReply
 from secop.protocol.errors import SECOPError, NoSuchModuleError, \
     NoSuchCommandError, NoSuchParamError, BadValueError, ReadonlyError
-from secop.lib.parsing import format_time
 from secop.lib import formatExtendedStack, formatException
 
+try:
+    unicode('a')
+except NameError:
+    # no unicode on py3
+    unicode = str  # pylint: disable=redefined-builtin
+
 
 class Dispatcher(object):
 
     def __init__(self, logger, options):
         # to avoid errors, we want to eat all options here
-        self.equipment_id = options['equipment_id']
+        self.equipment_id = options[u'equipment_id']
         self.nodeopts = {}
         for k in list(options):
             self.nodeopts[k] = options.pop(k)
@@ -71,80 +75,43 @@ class Dispatcher(object):
         self._subscriptions = {}
         self._lock = threading.RLock()
 
-    def handle_request(self, conn, msg):
-        """handles incoming request
-
-        will call 'queue.request(data)' on conn to send reply before returning
-        """
-        self.log.debug('Dispatcher: handling msg: %r' % msg)
-        # play thread safe !
-        with self._lock:
-            reply = None
-            # generate reply (coded and framed)
-            msgname = msg.__class__.__name__
-            if msgname.endswith('Request'):
-                msgname = msgname[:-len('Request')]
-            if msgname.endswith('Message'):
-                msgname = msgname[:-len('Message')]
-            self.log.debug('Looking for handle_%s' % msgname)
-            handler = getattr(self, 'handle_%s' % msgname, None)
-            if handler:
-                try:
-                    reply = handler(conn, msg)
-                except SECOPError as err:
-                    self.log.exception(err)
-                    reply = msg.get_error(
-                        errorclass=err.__class__.__name__,
-                        errorinfo=[repr(err), str(msg)])
-                except (ValueError, TypeError) as err:
-                    self.log.exception(err)
-                    reply = msg.get_error(
-                        errorclass='BadValue',
-                        errorinfo=[repr(err), str(msg)])
-                except Exception as err:
-                    self.log.exception(err)
-                    reply = msg.get_error(
-                        errorclass='InternalError', errorinfo=[
-                            formatException(), str(msg), formatExtendedStack()])
-            else:
-                self.log.debug('Can not handle msg %r' % msg)
-                reply = self.unhandled(conn, msg)
-            if reply:
-                conn.queue_reply(reply)
-
     def broadcast_event(self, msg, reallyall=False):
-        """broadcasts a msg to all active connections"""
+        """broadcasts a msg to all active connections
+
+        used from the dispatcher"""
         if reallyall:
             listeners = self._connections
         else:
-            if getattr(msg, 'command', None) is None:
+            if getattr(msg, u'command', None) is None:
-                eventname = '%s:%s' % (msg.module, msg.parameter
+                eventname = u'%s:%s' % (msg.module, msg.parameter
-                                       if msg.parameter else 'value')
+                                       if msg.parameter else u'value')
             else:
-                eventname = '%s:%s()' % (msg.module, msg.command)
+                eventname = u'%s:%s()' % (msg.module, msg.command)
-            listeners = self._subscriptions.get(eventname, [])
+            listeners = self._subscriptions.get(eventname, set()).copy()
-            listeners += list(self._active_connections)
+            listeners.update(self._subscriptions.get(msg.module, set()))
+            listeners.update(self._active_connections)
         for conn in listeners:
             conn.queue_async_reply(msg)
 
     def announce_update(self, moduleobj, pname, pobj):
         """called by modules param setters to notify subscribers of new values
         """
-        msg = Value(
+        msg = Message(EVENTREPLY, module=moduleobj.name, parameter=pname)
-            moduleobj.name,
+        msg.set_result(pobj.export_value(), dict(t=pobj.timestamp))
-            parameter=pname,
-            value=pobj.export_value(),
-            t=pobj.timestamp)
         self.broadcast_event(msg)
 
-    def subscribe(self, conn, modulename, pname='value'):
+    def subscribe(self, conn, modulename, pname=u'value'):
-        eventname = '%s:%s' % (modulename, pname)
+        eventname = modulename
+        if pname:
+            eventname = u'%s:%s' % (modulename, pname)
         self._subscriptions.setdefault(eventname, set()).add(conn)
 
-    def unsubscribe(self, conn, modulename, pname='value'):
+    def unsubscribe(self, conn, modulename, pname=u'value'):
-        eventname = '%s:%s' % (modulename, pname)
+        eventname = modulename
+        if pname:
+            eventname = u'%s:%s' % (modulename, pname)
         if eventname in self._subscriptions:
-            self._subscriptions.remove(conn)
+            self._subscriptions.setdefault(eventname, set()).discard(conn)
 
     def add_connection(self, conn):
         """registers new connection"""
@@ -154,17 +121,11 @@ class Dispatcher(object):
         """removes now longer functional connection"""
         if conn in self._connections:
             self._connections.remove(conn)
-        for _evt, conns in self._subscriptions.items():
+        for _evt, conns in list(self._subscriptions.items()):
             conns.discard(conn)
 
-    def activate_connection(self, conn):
-        self._active_connections.add(conn)
-
-    def deactivate_connection(self, conn):
-        self._active_connections.discard(conn)
-
     def register_module(self, moduleobj, modulename, export=True):
-        self.log.debug('registering module %r as %s (export=%r)' %
+        self.log.debug(u'registering module %r as %s (export=%r)' %
                        (moduleobj, modulename, export))
         self._modules[modulename] = moduleobj
         if export:
@@ -173,9 +134,9 @@ class Dispatcher(object):
     def get_module(self, modulename):
         if modulename in self._modules:
             return self._modules[modulename]
-        elif modulename in self._modules.values():
+        elif modulename in list(self._modules.values()):
             return modulename
-        raise NoSuchModuleError(module=str(modulename))
+        raise NoSuchModuleError(module=unicode(modulename))
 
     def remove_module(self, modulename_or_obj):
         moduleobj = self.get_module(modulename_or_obj) or modulename_or_obj
@@ -190,75 +151,55 @@ class Dispatcher(object):
         return self._export[:]
 
     def list_module_params(self, modulename, only_static=False):
-        self.log.debug('list_module_params(%r)' % modulename)
+        self.log.debug(u'list_module_params(%r)' % modulename)
         if modulename in self._export:
             # omit export=False params!
             res = {}
-            for paramname, param in self.get_module(modulename).parameters.items():
+            for paramname, param in list(self.get_module(modulename).parameters.items()):
                 if param.export:
                     res[paramname] = param.as_dict(only_static)
-            self.log.debug('list params for module %s -> %r' %
+            self.log.debug(u'list params for module %s -> %r' %
                            (modulename, res))
             return res
-        self.log.debug('-> module is not to be exported!')
+        self.log.debug(u'-> module is not to be exported!')
         return {}
 
     def list_module_cmds(self, modulename):
-        self.log.debug('list_module_cmds(%r)' % modulename)
+        self.log.debug(u'list_module_cmds(%r)' % modulename)
         if modulename in self._export:
             # omit export=False params!
             res = {}
-            for cmdname, cmdobj in self.get_module(modulename).commands.items():
+            for cmdname, cmdobj in list(self.get_module(modulename).commands.items()):
                 res[cmdname] = cmdobj.as_dict()
-            self.log.debug('list cmds for module %s -> %r' % (modulename, res))
+            self.log.debug(u'list cmds for module %s -> %r' % (modulename, res))
             return res
-        self.log.debug('-> module is not to be exported!')
+        self.log.debug(u'-> module is not to be exported!')
         return {}
 
     def get_descriptive_data(self):
         """returns a python object which upon serialisation results in the descriptive data"""
         # XXX: be lazy and cache this?
         # format: {[{[{[, specific entries first
-        result = {'modules': []}
+        result = {u'modules': []}
         for modulename in self._export:
             module = self.get_module(modulename)
             # some of these need rework !
-            mod_desc = {'parameters': [], 'commands': []}
+            mod_desc = {u'parameters': [], u'commands': []}
-            for pname, param in self.list_module_params(
+            for pname, param in list(self.list_module_params(
-                    modulename, only_static=True).items():
+                    modulename, only_static=True).items()):
-                mod_desc['parameters'].extend([pname, param])
+                mod_desc[u'parameters'].extend([pname, param])
-            for cname, cmd in self.list_module_cmds(modulename).items():
+            for cname, cmd in list(self.list_module_cmds(modulename).items()):
-                mod_desc['commands'].extend([cname, cmd])
+                mod_desc[u'commands'].extend([cname, cmd])
-            for propname, prop in module.properties.items():
+            for propname, prop in list(module.properties.items()):
                 mod_desc[propname] = prop
-            result['modules'].extend([modulename, mod_desc])
+            result[u'modules'].extend([modulename, mod_desc])
-        result['equipment_id'] = self.equipment_id
+        result[u'equipment_id'] = self.equipment_id
-        result['firmware'] = 'The SECoP playground'
+        result[u'firmware'] = u'The SECoP playground'
-        result['version'] = "2017.07"
+        result[u'version'] = u'2017.07'
         result.update(self.nodeopts)
         # XXX: what else?
         return result
 
-    def get_descriptive_data_old(self):
-        # XXX: be lazy and cache this?
-        result = {'modules': {}}
-        for modulename in self._export:
-            module = self.get_module(modulename)
-            # some of these need rework !
-            dd = {
-                'parameters': self.list_module_params(
-                    modulename,
-                    only_static=True),
-                'commands': self.list_module_cmds(modulename),
-                'properties': module.properties,
-            }
-            result['modules'][modulename] = dd
-        result['equipment_id'] = self.equipment_id
-        result['firmware'] = 'The SECoP playground'
-        result['version'] = "2017.01"
-        # XXX: what else?
-        return result
-
     def _execute_command(self, modulename, command, arguments=None):
         if arguments is None:
             arguments = []
@@ -274,19 +215,13 @@ class Dispatcher(object):
             raise BadValueError(
                 module=modulename,
                 command=command,
-                reason='Wrong number of arguments!')
+                reason=u'Wrong number of arguments!')
 
         # now call func and wrap result as value
         # note: exceptions are handled in handle_request, not here!
-        func = getattr(moduleobj, 'do_' + command)
+        func = getattr(moduleobj, u'do_' + command)
         res = func(*arguments)
-        res = CommandReply(
+        return res, dict(t=currenttime())
-            module=modulename,
-            command=command,
-            result=res,
-            qualifiers=dict(t=time.time()))
-        # res = Value(modulename, command=command, value=func(*arguments), t=time.time())
-        return res
 
     def _setParamValue(self, modulename, pname, value):
         moduleobj = self.get_module(modulename)
@@ -299,19 +234,15 @@ class Dispatcher(object):
         if pobj.readonly:
             raise ReadonlyError(module=modulename, parameter=pname)
 
-        writefunc = getattr(moduleobj, 'write_%s' % pname, None)
+        writefunc = getattr(moduleobj, u'write_%s' % pname, None)
         # note: exceptions are handled in handle_request, not here!
         if writefunc:
             value = writefunc(value)
         else:
             setattr(moduleobj, pname, value)
         if pobj.timestamp:
-            return WriteReply(
+            return pobj.export_value(), dict(t=pobj.timestamp)
-                module=modulename,
+        return pobj.export_value(), {}
-                parameter=pname,
-                value=[pobj.value, dict(t=format_time(pobj.timestamp))])
-        return WriteReply(
-            module=modulename, parameter=pname, value=[pobj.value, {}])
 
     def _getParamValue(self, modulename, pname):
         moduleobj = self.get_module(modulename)
@@ -322,121 +253,181 @@ class Dispatcher(object):
         if pobj is None:
             raise NoSuchParamError(module=modulename, parameter=pname)
 
-        readfunc = getattr(moduleobj, 'read_%s' % pname, None)
+        readfunc = getattr(moduleobj, u'read_%s' % pname, None)
         if readfunc:
             # should also update the pobj (via the setter from the metaclass)
             # note: exceptions are handled in handle_request, not here!
             readfunc()
         if pobj.timestamp:
-            res = Value(
+            return pobj.export_value(), dict(t=pobj.timestamp)
-                modulename,
+        return pobj.export_value(), {}
-                parameter=pname,
+
-                value=pobj.export_value(),
+    #
-                t=pobj.timestamp)
+    # api to be called from the 'interface'
-        else:
+    # any method above has no idea about 'messages', this is handled here
-            res = Value(modulename, parameter=pname, value=pobj.export_value())
+    #
-        return res
+    def handle_request(self, conn, msg):
+        """handles incoming request
+
+        will call 'queue.request(data)' on conn to send reply before returning
+        """
+        self.log.debug(u'Dispatcher: handling msg: %r' % msg)
+        # if there was an error in the frontend, bounce the resulting
+        # error msgObj directly back to the client
+        if msg.errorclass:
+            return msg
+
+        # play thread safe !
+        with self._lock:
+            if msg.action == IDENTREQUEST:
+                self.log.debug(u'Looking for handle_ident')
+                handler = self.handle_ident
+            else:
+                self.log.debug(u'Looking for handle_%s' % msg.action)
+                handler = getattr(self, u'handle_%s' % msg.action, None)
+            if handler:
+                try:
+                    reply = handler(conn, msg)
+                    if reply:
+                        conn.queue_reply(reply)
+                    return None
+                except SECOPError as err:
+                    self.log.exception(err)
+                    msg.set_error(err.name, unicode(err), {})#u'traceback': formatException(),
+                                                       #u'extended_stack':formatExtendedStack()})
+                    return msg
+                except (ValueError, TypeError) as err:
+                    self.log.exception(err)
+                    msg.set_error(u'BadValue', unicode(err), {u'traceback': formatException()})
+                    print(u'--------------------')
+                    print(formatExtendedStack())
+                    print(u'====================')
+                    return msg
+                except Exception as err:
+                    self.log.exception(err)
+                    msg.set_error(u'InternalError', unicode(err), {u'traceback': formatException()})
+                    print(u'--------------------')
+                    print(formatExtendedStack())
+                    print(u'====================')
+                    return msg
+            else:
+                self.log.error(u'Can not handle msg %r' % msg)
+                msg.set_error(u'Protocol', u'unhandled msg', {})
+                return msg
 
     # now the (defined) handlers for the different requests
-    def handle_Help(self, conn, msg):
+    def handle_help(self, conn, msg):
-        return HelpMessage()
+        msg.mkreply()
+        return msg
 
-    def handle_Identify(self, conn, msg):
+    def handle_ident(self, conn, msg):
-        return IdentifyReply(version_string='currently,is,ignored,here')
+        msg.mkreply()
+        return msg
 
-    def handle_Describe(self, conn, msg):
+    def handle_describe(self, conn, msg):
         # XXX:collect descriptive data
-        return DescribeReply(
+        msg.setvalue(u'specifier', u'.')
-            equipment_id=self.equipment_id,
+        msg.setvalue(u'data', self.get_descriptive_data())
-            description=self.get_descriptive_data())
+        msg.mkreply()
+        return msg
 
-    def handle_Poll(self, conn, msg):
+    def handle_read(self, conn, msg):
         # XXX: trigger polling and force sending event
-        res = self._getParamValue(msg.module, msg.parameter or 'value')
+        if not msg.parameter:
-        # self.broadcast_event(res)
+            msg.parameter = u'value'
-        if conn in self._active_connections:
+        msg.set_result(*self._getParamValue(msg.module, msg.parameter))
-            return None  # already send to myself
-        return res  # send reply to inactive conns
 
-    def handle_Write(self, conn, msg):
+        #if conn in self._active_connections:
-        # notify all by sending WriteReply
+        #    return None  # already send to myself
-        #msg1 = WriteReply(**msg.as_dict())
+        #if conn in self._subscriptions.get(msg.module, set()):
-        # self.broadcast_event(msg1)
+        #    return None  # already send to myself
+        msg.mkreply()
+        return msg  # send reply to inactive conns
+
+    def handle_change(self, conn, msg):
         # try to actually write  XXX: should this be done asyncron? we could
         # just return the reply in that case
-        if msg.parameter:
+        if not msg.parameter:
-            res = self._setParamValue(msg.module, msg.parameter, msg.value)
+            msg.parameter = u'target'
-        else:
+        msg.set_result(*self._setParamValue(msg.module, msg.parameter, msg.data))
-            # first check if module has a target
-            if 'target' not in self.get_module(msg.module).parameters:
-                raise ReadonlyError(module=msg.module, parameter=None)
-            res = self._setParamValue(msg.module, 'target', msg.value)
-            res.parameter = 'target'
-        # self.broadcast_event(res)
-        # if conn in self._active_connections:
-        #    return None  # already send to myself
-        return res
 
-    def handle_Command(self, conn, msg):
+        #if conn in self._active_connections:
-        # notify all by sending CommandReply
+        #    return None  # already send to myself
-        #msg1 = CommandReply(**msg.as_dict())
+        #if conn in self._subscriptions.get(msg.module, set()):
-        # self.broadcast_event(msg1)
+        #    return None  # already send to myself
+        msg.mkreply()
+        return msg  # send reply to inactive conns
+
+    def handle_do(self, conn, msg):
         # XXX: should this be done asyncron? we could just return the reply in
         # that case
-
+        if not msg.args:
+            msg.args = []
         # try to actually execute command
-        res = self._execute_command(msg.module, msg.command, msg.arguments)
+        msg.set_result(*self._execute_command(msg.module, msg.command, msg.args))
-        # self.broadcast_event(res)
+
-        # if conn in self._active_connections:
+        #if conn in self._active_connections:
         #    return None  # already send to myself
-        return res  # send reply to inactive conns
+        #if conn in self._subscriptions.get(msg.module, set()):
+        #    return None  # already send to myself
+        msg.mkreply()
+        return msg  # send reply to inactive conns
 
-    def handle_Heartbeat(self, conn, msg):
+    def handle_ping(self, conn, msg):
-        return HeartbeatReply(**msg.as_dict())
+        msg.setvalue(u'data', {u't':currenttime()})
+        msg.mkreply()
+        return msg
 
-    def handle_Activate(self, conn, msg):
+    def handle_activate(self, conn, msg):
-        self.activate_connection(conn)
+        if msg.module:
-        # easy approach: poll all values...
+            if msg.module not in self._modules:
-        for modulename, moduleobj in self._modules.items():
+                raise NoSuchModuleError()
+            # activate only ONE module
+            self.subscribe(conn, msg.specifier, u'')
+            modules = [msg.specifier]
+        else:
+            # activate all modules
+            self._active_connections.add(conn)
+            modules = self._modules
+
+        # for initial update poll all values...
+        for modulename in modules:
+            moduleobj = self._modules.get(modulename, None)
+            if moduleobj is None:
+                self.log.error(u'activate: can not lookup module %r, skipping it' % modulename)
+                continue
             for pname, pobj in moduleobj.parameters.items():
-                if not pobj.export:
+                if not pobj.export:  # XXX: handle export_as cases!
                     continue
                 # WARNING: THIS READS ALL parameters FROM HW!
                 # XXX: should we send the cached values instead? (pbj.value)
                 # also: ignore errors here.
                 try:
                     res = self._getParamValue(modulename, pname)
+                    if res[0] == Ellipsis:  # means we do not have a value at all so skip this
+                        self.log.error(
+                                u'activate: got no value for %s:%s!' %
+                                (modulename, pname))
+                    #else:
+                        #rm = Message(EVENTREPLY, u'%s:%s' % (modulename, pname))
+                        #rm.set_result(*res)
+                        #self.broadcast_event(rm)
                 except SECOPError as e:
-                    self.log.error('decide what to do here!')
+                    self.log.error(u'decide what to do here! (ignore error and skip update)')
                     self.log.exception(e)
-                    res = Value(
+        msg.mkreply()
-                        module=modulename,
+        conn.queue_async_reply(msg)  # should be sent AFTER all the ^^initial updates
-                        parameter=pname,
-                        value=pobj.export_value(),
-                        t=pobj.timestamp,
-                        unit=pobj.unit)
-                if res.value != Ellipsis:  # means we do not have a value at all so skip this
-                    self.broadcast_event(res)
-                else:
-                    self.log.error(
-                        'activate: got no value for %s:%s!' %
-                        modulename, pname)
-        conn.queue_async_reply(ActivateReply(**msg.as_dict()))
         return None
 
-    def handle_Deactivate(self, conn, msg):
+    def handle_deactivate(self, conn, msg):
-        self.deactivate_connection(conn)
+        if msg.specifier:
-        conn.queue_async_reply(DeactivateReply(**msg.as_dict()))
+            self.unsubscribe(conn, msg.specifier, u'')
-        return None
+        else:
-
+            self._active_connections.discard(conn)
-    def handle_Error(self, conn, msg):
+            # XXX: also check all entries in self._subscriptions?
+        msg.mkreply()
         return msg
 
-    def unhandled(self, conn, msg):
+    def handle_error(self, conn, msg):
-        """handler for unhandled Messages
+        # is already an error-reply (came from interface frontend) -> just send it back
-
+        return msg
-        (no handle_<messagename> method was defined)
-        """
-        self.log.error('IGN: got unhandled request %s' % msg)
-        return msg.get_error(
-            errorclass="InternalError", errorinfo="Unhandled Request")

secop/protocol/encoding.py

@@ -1,332 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Messages"""
-
-# implement as class as they may need some internal 'state' later on
-# (think compressors)
-
-from __future__ import print_function
-
-
-# Base class
-class MessageEncoder(object):
-    """en/decode a single Messageobject"""
-
-    def encode(self, msg):
-        """encodes the given message object into a frame"""
-        raise NotImplementedError
-
-    def decode(self, encoded):
-        """decodes the given frame to a message object"""
-        raise NotImplementedError
-
-
-import re
-import json
-
-#from secop.lib.parsing import format_time
-from secop.protocol.messages import Value, IdentifyRequest, IdentifyReply, \
-    DescribeRequest, DescribeReply, ActivateRequest, ActivateReply, \
-    DeactivateRequest, DeactivateReply, CommandRequest, CommandReply, \
-    WriteRequest, WriteReply, PollRequest, HeartbeatRequest, HeartbeatReply, \
-    ErrorMessage, HelpMessage
-#from secop.protocol.errors import ProtocolError
-
-
-# each message is like <messagetype> [ \space <messageargs> [ \space
-# <json> ]] \lf
-
-# note: the regex allow <> for spec for testing only!
-SECOP_RE = re.compile(
-    r"""^(?P<msgtype>[\*\?\w]+)(?:\s(?P<spec>[\w:<>]+)(?:\s(?P<json>.*))?)?$""",
-    re.X)
-
-#"""
-# messagetypes:
-IDENTREQUEST = '*IDN?'  # literal
-# literal! first part is fixed!
-#IDENTREPLY = 'SECoP, SECoPTCP, V2016-11-30, rc1'
-#IDENTREPLY = 'SINE2020&ISSE,SECoP,V2016-11-30,rc1'
-IDENTREPLY = 'SINE2020&ISSE,SECoP,V2017-01-25,rc1'
-DESCRIPTIONSREQUEST = 'describe'  # literal
-DESCRIPTIONREPLY = 'describing'  # +<id> +json
-ENABLEEVENTSREQUEST = 'activate'  # literal
-ENABLEEVENTSREPLY = 'active'  # literal, is end-of-initial-data-transfer
-DISABLEEVENTSREQUEST = 'deactivate'  # literal
-DISABLEEVENTSREPLY = 'inactive'  # literal
-COMMANDREQUEST = 'do'  # +module:command +json args (if needed)
-# +module:command +json args (if needed) # send after the command finished !
-COMMANDREPLY = 'done'
-# +module[:parameter] +json_value -> NO direct reply, calls TRIGGER internally!
-WRITEREQUEST = 'change'
-# +module[:parameter] +json_value # send with the read back value
-WRITEREPLY = 'changed'
-# +module[:parameter] -> NO direct reply, calls TRIGGER internally!
-TRIGGERREQUEST = 'read'
-EVENT = 'update'  # +module[:parameter] +json_value (value, qualifiers_as_dict)
-HEARTBEATREQUEST = 'ping'  # +nonce_without_space
-HEARTBEATREPLY = 'pong'  # +nonce_without_space
-ERRORREPLY = 'error'  # +errorclass +json_extended_info
-HELPREQUEST = 'help'  # literal
-HELPREPLY = 'helping'  # +line number +json_text
-ERRORCLASSES = [
-    'NoSuchModule',
-    'NoSuchParameter',
-    'NoSuchCommand',
-    'CommandFailed',
-    'ReadOnly',
-    'BadValue',
-    'CommunicationFailed',
-    'IsBusy',
-    'IsError',
-    'ProtocolError',
-    'InternalError',
-    'CommandRunning',
-    'Disabled',
-]
-
-# note: above strings need to be unique in the sense, that none is/or
-# starts with another
-
-
-def encode_cmd_result(msgobj):
-    q = msgobj.qualifiers.copy()
-    if 't' in q:
-        q['t'] = str(q['t'])
-    return msgobj.result, q
-
-
-def encode_value_data(vobj):
-    q = vobj.qualifiers.copy()
-    if 't' in q:
-        q['t'] = str(q['t'])
-    return vobj.value, q
-
-
-def encode_error_msg(emsg):
-    # note: result is JSON-ified....
-    return [
-        emsg.origin, dict((k, getattr(emsg, k)) for k in emsg.ARGS
-                          if k != 'origin')
-    ]
-
-
-class SECoPEncoder(MessageEncoder):
-    # map of msg to msgtype string as defined above.
-    ENCODEMAP = {
-        IdentifyRequest: (IDENTREQUEST, ),
-        IdentifyReply: (IDENTREPLY, ),
-        DescribeRequest: (DESCRIPTIONSREQUEST, ),
-        DescribeReply: (
-            DESCRIPTIONREPLY,
-            'equipment_id',
-            'description', ),
-        ActivateRequest: (ENABLEEVENTSREQUEST, ),
-        ActivateReply: (ENABLEEVENTSREPLY, ),
-        DeactivateRequest: (DISABLEEVENTSREQUEST, ),
-        DeactivateReply: (DISABLEEVENTSREPLY, ),
-        CommandRequest: (
-            COMMANDREQUEST,
-            lambda msg: "%s:%s" % (msg.module, msg.command),
-            'arguments', ),
-        CommandReply: (
-            COMMANDREPLY,
-            lambda msg: "%s:%s" % (msg.module, msg.command),
-            encode_cmd_result, ),
-        WriteRequest: (
-            WRITEREQUEST,
-            lambda msg: "%s:%s" % (
-                msg.module, msg.parameter) if msg.parameter else msg.module,
-            'value', ),
-        WriteReply: (
-            WRITEREPLY,
-            lambda msg: "%s:%s" % (
-                msg.module, msg.parameter) if msg.parameter else msg.module,
-            'value', ),
-        PollRequest: (
-            TRIGGERREQUEST,
-            lambda msg: "%s:%s" % (
-                msg.module, msg.parameter) if msg.parameter else msg.module,
-        ),
-        HeartbeatRequest: (
-            HEARTBEATREQUEST,
-            'nonce', ),
-        HeartbeatReply: (
-            HEARTBEATREPLY,
-            'nonce', ),
-        HelpMessage: (HELPREQUEST, ),
-        ErrorMessage: (
-            ERRORREPLY,
-            "errorclass",
-            encode_error_msg, ),
-        Value: (
-            EVENT,
-            lambda msg: "%s:%s" % (msg.module, msg.parameter or (
-                msg.command + '()')) if msg.parameter or msg.command else msg.module,
-            encode_value_data, ),
-    }
-    DECODEMAP = {
-        IDENTREQUEST: lambda spec, data: IdentifyRequest(),
-        # handled specially, listed here for completeness
-        # IDENTREPLY: lambda spec, data: IdentifyReply(encoded),
-        DESCRIPTIONSREQUEST: lambda spec, data: DescribeRequest(),
-        DESCRIPTIONREPLY: lambda spec, data: DescribeReply(equipment_id=spec[0], description=data),
-        ENABLEEVENTSREQUEST: lambda spec, data: ActivateRequest(),
-        ENABLEEVENTSREPLY: lambda spec, data: ActivateReply(),
-        DISABLEEVENTSREQUEST: lambda spec, data: DeactivateRequest(),
-        DISABLEEVENTSREPLY: lambda spec, data: DeactivateReply(),
-        COMMANDREQUEST: lambda spec, data: CommandRequest(module=spec[0], command=spec[1], arguments=data),
-        COMMANDREPLY: lambda spec, data: CommandReply(module=spec[0], command=spec[1], result=data),
-        WRITEREQUEST: lambda spec, data: WriteRequest(module=spec[0], parameter=spec[1], value=data),
-        WRITEREPLY: lambda spec, data: WriteReply(module=spec[0], parameter=spec[1], value=data),
-        TRIGGERREQUEST: lambda spec, data: PollRequest(module=spec[0], parameter=spec[1]),
-        HEARTBEATREQUEST: lambda spec, data: HeartbeatRequest(nonce=spec[0]),
-        HEARTBEATREPLY: lambda spec, data: HeartbeatReply(nonce=spec[0]),
-        HELPREQUEST: lambda spec, data: HelpMessage(),
-        #        HELPREPLY: lambda spec, data:None,  # ignore this
-        ERRORREPLY: lambda spec, data: ErrorMessage(errorclass=spec[0], errorinfo=data),
-        EVENT: lambda spec, data: Value(module=spec[0], parameter=spec[1], value=data[0],
-                                        qualifiers=data[1] if len(data) > 1 else {}),
-    }
-
-    def __init__(self, *args, **kwds):
-        MessageEncoder.__init__(self, *args, **kwds)
-        # self.tests()
-
-    def encode(self, msg):
-        """msg object -> transport layer message"""
-        # fun for Humans
-        if isinstance(msg, HelpMessage):
-            text = """Try one of the following:
-            '%s' to query protocol version
-            '%s' to read the description
-            '%s <module>[:<parameter>]' to request reading a value
-            '%s <module>[:<parameter>] value' to request changing a value
-            '%s <module>[:<command>()]' to execute a command
-            '%s <nonce>' to request a heartbeat response
-            '%s' to activate async updates
-            '%s' to deactivate updates
-            """ % (IDENTREQUEST, DESCRIPTIONSREQUEST, TRIGGERREQUEST,
-                   WRITEREQUEST, COMMANDREQUEST, HEARTBEATREQUEST,
-                   ENABLEEVENTSREQUEST, DISABLEEVENTSREQUEST)
-            return '\n'.join('%s %d %s' % (HELPREPLY, i + 1, l.strip())
-                             for i, l in enumerate(text.split('\n')[:-1]))
-        if isinstance(msg, HeartbeatRequest):
-            if msg.nonce:
-                return 'ping %s' % msg.nonce
-            return 'ping'
-        if isinstance(msg, HeartbeatReply):
-            if msg.nonce:
-                return 'pong %s' % msg.nonce
-            return 'pong'
-        for msgcls, parts in self.ENCODEMAP.items():
-            if isinstance(msg, msgcls):
-                # resolve lambdas
-                parts = [parts[0]] + [
-                    p(msg) if callable(p) else getattr(msg, p)
-                    for p in parts[1:]
-                ]
-                if len(parts) > 1:
-                    parts[1] = str(parts[1])
-                if len(parts) == 3:
-                    parts[2] = json.dumps(parts[2])
-                return ' '.join(parts)
-
-    def decode(self, encoded):
-        # first check beginning
-        match = SECOP_RE.match(encoded)
-        if not match:
-            print(repr(encoded), repr(IDENTREPLY))
-            if encoded == IDENTREPLY:  # XXX:better just check the first 2 parts...
-                return IdentifyReply(version_string=encoded)
-
-            return HelpMessage()
-#            return ErrorMessage(errorclass='Protocol',
-#                                errorinfo='Regex did not match!',
-#                                is_request=True)
-        msgtype, msgspec, data = match.groups()
-        if msgspec is None and data:
-            return ErrorMessage(
-                errorclass='Internal',
-                errorinfo='Regex matched json, but not spec!',
-                is_request=True,
-                origin=encoded)
-
-        if msgtype in self.DECODEMAP:
-            if msgspec and ':' in msgspec:
-                msgspec = msgspec.split(':', 1)
-            else:
-                msgspec = (msgspec, None)
-            if data:
-                try:
-                    data = json.loads(data)
-                except ValueError as err:
-                    return ErrorMessage(
-                        errorclass='BadValue',
-                        errorinfo=[repr(err), str(encoded)],
-                        origin=encoded)
-            msg = self.DECODEMAP[msgtype](msgspec, data)
-            msg.setvalue("origin", encoded)
-            return msg
-        return ErrorMessage(
-            errorclass='Protocol',
-            errorinfo='%r: No Such Messagetype defined!' % encoded,
-            is_request=True,
-            origin=encoded)
-
-    def tests(self):
-        print("---- Testing encoding  -----")
-        for msgclass in sorted(self.ENCODEMAP):
-            print(msgclass)
-            e = self.encode(
-                msgclass(
-                    module='<module>',
-                    parameter='<paramname>',
-                    value=2.718,
-                    equipment_id='<id>',
-                    description='descriptive data',
-                    command='<cmd>',
-                    arguments='<arguments>',
-                    nonce='<nonce>',
-                    errorclass='InternalError',
-                    errorinfo='nix'))
-            print(e)
-            print(self.decode(e))
-            print()
-        print("---- Testing decoding  -----")
-        for msgtype, _ in sorted(self.DECODEMAP.items()):
-            msg = '%s a:b 3' % msgtype
-            if msgtype == EVENT:
-                msg = '%s a:b [3,{"t":193868}]' % msgtype
-            print(msg)
-            d = self.decode(msg)
-            print(d)
-            print(self.encode(d))
-            print()
-        print("---- Testing done -----")
-
-
-ENCODERS = {
-    'secop': SECoPEncoder,
-}
-
-__ALL__ = ['ENCODERS']

secop/protocol/errors.py

@@ -25,13 +25,14 @@
 class SECOPError(RuntimeError):
 
     def __init__(self, *args, **kwds):
+        RuntimeError.__init__(self)
         self.args = args
-        for k, v in kwds.items():
+        for k, v in list(kwds.items()):
             setattr(self, k, v)
 
     def __repr__(self):
         args = ', '.join(map(repr, self.args))
-        kwds = ', '.join(['%s=%r' % i for i in self.__dict__.items()])
+        kwds = ', '.join(['%s=%r' % i for i in list(self.__dict__.items())])
         res = []
         if args:
             res.append(args)
@@ -45,14 +46,13 @@ class SECOPError(RuntimeError):
 
 
 class InternalError(SECOPError):
-    pass
+    name = 'InternalError'
 
 
 class ProtocolError(SECOPError):
-    pass
+    name = 'SyntaxError'
 
 
-# XXX: unifiy NoSuch...Error ?
 class NoSuchModuleError(SECOPError):
     pass
 
@@ -77,17 +77,42 @@ class CommandFailedError(SECOPError):
     pass
 
 
-class InvalidParamValueError(SECOPError):
+class CommandRunningError(SECOPError):
     pass
 
 
+class CommunicationFailedError(SECOPError):
+    pass
+
+
+class IsBusyError(SECOPError):
+    pass
+
+
+class IsErrorError(SECOPError):
+    pass
+
+
+class DisabledError(SECOPError):
+    pass
+
+
+
 EXCEPTIONS = dict(
-    Internal=InternalError,
-    Protocol=ProtocolError,
     NoSuchModule=NoSuchModuleError,
     NoSuchParam=NoSuchParamError,
     NoSuchCommand=NoSuchCommandError,
-    BadValue=BadValueError,
-    Readonly=ReadonlyError,
     CommandFailed=CommandFailedError,
-    InvalidParam=InvalidParamValueError, )
+    CommandRunning=CommandRunningError,
+    Readonly=ReadonlyError,
+    BadValue=BadValueError,
+    CommunicationFailed=CommunicationFailedError,
+    IsBusy=IsBusyError,
+    IsError=IsErrorError,
+    Disabled=DisabledError,
+    SyntaxError=ProtocolError,
+    InternalError=InternalError,
+# internal short versions (candidates for spec)
+    Protocol=ProtocolError,
+    Internal=InternalError,
+)

secop/protocol/framing/__init__.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Frames"""
-
-
-# Base class
-class Framer(object):
-    """Frames and unframes an encoded message
-
-    also transforms the encoded message to the 'wire-format' (and vise-versa)
-
-    note: not all MessageEncoders can use all Framers,
-          but the intention is to have this for as many as possible.
-    """
-
-    def encode(self, *frames):
-        """return the wire-data for the given messageframes"""
-        raise NotImplemented
-
-    def decode(self, data):
-        """return a list of messageframes found in data"""
-        raise NotImplemented
-
-    def reset(self):
-        """resets the de/encoding stage (clears internal information)"""
-        raise NotImplemented
-
-# now some Implementations
-from .null import NullFramer
-from .eol import EOLFramer
-from .rle import RLEFramer
-from .demo import DemoFramer
-
-FRAMERS = {
-    'null': NullFramer,
-    'eol': EOLFramer,
-    'rle': RLEFramer,
-    'demo': DemoFramer,
-}
-
-__ALL__ = ['FRAMERS']

secop/protocol/framing/demo.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Frames"""
-
-from secop.protocol.framing import Framer
-
-
-class DemoFramer(Framer):
-    """Text based message framer
-
-    frmes are delimited by '\n'
-    messages are delimited by '\n\n'
-    '\r' is ignored
-    """
-
-    def __init__(self):
-        self.data = b''
-        self.decoded = []
-
-    def encode(self, frames):
-        """add transport layer encapsulation/framing of messages"""
-        if isinstance(frames, (tuple, list)):
-            return b'\n'.join(frames) + b'\n\n'
-        return b'%s\n\n' % frames
-
-    def decode(self, data):
-        """remove transport layer encapsulation/framing of messages
-
-        returns a list of messageframes which got decoded from data!
-        """
-        self.data += data
-        res = []
-        while b'\n' in self.data:
-            frame, self.data = self.data.split(b'\n', 1)
-            if frame.endswith('\r'):
-                frame = frame[:-1]
-            if self.data.startswith('\r'):
-                self.data = self.data[1:]
-            res.append(frame)
-        return res
-
-    def decode2(self, data):
-        """remove transport layer encapsulation/framing of messages
-
-        returns a _list_ of messageframes which got decoded from data!
-        """
-        self.data += data.replace(b'\r', '')
-        while b'\n' in self.data:
-            frame, self.data = self.data.split(b'\n', 1)
-            if frame:
-                # not an empty line -> belongs to this set of messages
-                self.decoded.append(frame)
-            else:
-                # empty line -> our set of messages is finished decoding
-                res = self.decoded
-                self.decoded = []
-                return res
-        return None
-
-    def reset(self):
-        self.data = b''
-        self.decoded = []

secop/protocol/framing/eol.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Frames"""
-
-from secop.protocol.framing import Framer
-
-
-class EOLFramer(Framer):
-    """Text based message framer
-
-    messages are delimited by '\r\n'
-    upon reception the end of a message is detected by '\r\n','\n' or '\n\r'
-    """
-    data = b''
-
-    def encode(self, *frames):
-        """add transport layer encapsulation/framing of messages"""
-        return b'%s\r\n' % b'\r\n'.join(frames)
-
-    def decode(self, data):
-        """remove transport layer encapsulation/framing of messages
-
-        returns a list of messageframes which got decoded from data!
-        """
-        self.data += data
-        res = []
-        while b'\n' in self.data:
-            frame, self.data = self.data.split(b'\n', 1)
-            if frame.endswith('\r'):
-                frame = frame[:-1]
-            if self.data.startswith('\r'):
-                self.data = self.data[1:]
-            res.append(frame)
-        return res
-
-    def reset(self):
-        self.data = b''

secop/protocol/framing/null.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Frames"""
-
-from secop.protocol.framing import Framer
-
-
-class NullFramer(Framer):
-    """do-nothing-framer
-
-    assumes messages are framed by themselfs or the interface does it already.
-    """
-
-    def encode(self, *frames):
-        """add transport layer encapsulation/framing of messages"""
-        return ''.join(frames)
-
-    def decode(self, data):
-        """remove transport layer encapsulation/framing of messages"""
-        return [data]

secop/protocol/framing/rle.py

@@ -1,71 +0,0 @@
-#!/usr/bin/env python
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Encoding/decoding Frames"""
-
-from secop.protocol.framing import Framer
-
-
-class RLEFramer(Framer):
-    data = b''
-    frames_to_go = 0
-
-    def encode(self, *frames):
-        """add transport layer encapsulation/framing of messages"""
-        # format is 'number of frames:[framelengt:frme]*N'
-        frdata = ['%d:%s' % (len(frame), frame) for frame in frames]
-        return b'%d:' + b''.join(frdata)
-
-    def decode(self, data):
-        """remove transport layer encapsulation/framing of messages
-
-        returns a list of messageframes which got decoded from data!
-        """
-        self.data += data
-        res = []
-        while self.data:
-            if frames_to_go == 0:
-                if ':' in self.data:
-                    # scan for and decode 'number of frames'
-                    frnum, self.data = self.data.split(':', 1)
-                    try:
-                        self.frames_to_go = int(frnum)
-                    except ValueError:
-                        # can not recover, complain!
-                        raise FramingError('invalid start of message found!')
-                else:
-                    # not enough data to decode number of frames,
-                    # return what we have
-                    return res
-            while self.frames_to_go:
-                # there are still some (partial) frames stuck inside self.data
-                frlen, self.data = self.data.split(':', 1)
-                if len(self.data) >= frlen:
-                    res.append(self.data[:frlen])
-                    self.data = self.data[frlen:]
-                    self.frames_to_go -= 1
-                else:
-                    # not enough data for this frame, return what we have
-                    return res
-
-    def reset(self):
-        self.data = b''
-        self.frames_to_go = 0

secop/protocol/interface/__init__.py

@@ -20,8 +20,9 @@
 #
 # *****************************************************************************
 """provide server interfaces to be used by clients"""
+from __future__ import absolute_import
 
-from tcp import TCPServer
+from .tcp import TCPServer
 
 INTERFACES = {'tcp': TCPServer, }
 

secop/protocol/interface/tcp.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under
@@ -20,27 +19,81 @@
 #
 # *****************************************************************************
 """provides tcp interface to the SECoP Server"""
+from __future__ import print_function
 
-import os
 import socket
 import collections
-import SocketServer
+
+try:
+    import socketserver  # py3
+except ImportError:
+    import SocketServer as socketserver  # py2
+
+from secop.lib import formatExtendedStack, formatException
+from secop.protocol.messages import HELPREPLY, Message, HelpMessage
+
 
 DEF_PORT = 10767
 MAX_MESSAGE_SIZE = 1024
 
-from secop.protocol.encoding import ENCODERS
+EOL = b'\n'
-from secop.protocol.framing import FRAMERS
+CR = b'\r'
-from secop.protocol.messages import HelpMessage
+SPACE = b' '
 
 
-class TCPRequestHandler(SocketServer.BaseRequestHandler):
+def encode_msg_frame(action, specifier=None, data=None):
+    """ encode a msg_tripel into an msg_frame, ready to be sent
+
+    action (and optional specifier) are unicode strings,
+    data may be an json-yfied python object"""
+    action = action.encode('utf-8')
+    if specifier is None:
+        # implicit: data is None
+        return b''.join((action, EOL))
+    specifier = specifier.encode('utf-8')
+    if data:
+        data = data.encode('utf-8')
+        return b''.join((action, SPACE, specifier, SPACE, data, EOL))
+    return b''.join((action, SPACE, specifier, EOL))
+
+
+def get_msg(_bytes):
+    """try to deframe the next msg in (binary) input
+    always return a tupel (msg, remaining_input)
+    msg may also be None
+    """
+    if EOL not in _bytes:
+        return None, _bytes
+    return _bytes.split(EOL, 1)
+
+
+def decode_msg(msg):
+    """decode the (binary) msg into a (unicode) msg_tripel"""
+    # check for leading/trailing CR and remove it
+    if msg and msg[0] == CR:
+        msg = msg[1:]
+    if msg and msg[-1] == CR:
+        msg = msg[:-1]
+
+    res = msg.split(b' ', 2)
+    action = res[0].decode('utf-8')
+    if len(res) == 1:
+        return action, None, None
+    specifier = res[1].decode('utf-8')
+    if len(res) == 2:
+        return action, specifier, None
+    data = res[2].decode('utf-8')
+    return action, specifier, data
+
+
+class TCPRequestHandler(socketserver.BaseRequestHandler):
 
     def setup(self):
         self.log = self.server.log
+        # Queue of msgObjects to send
         self._queue = collections.deque(maxlen=100)
-        self.framing = self.server.framingCLS()
+#        self.framing = self.server.framingCLS()
-        self.encoding = self.server.encodingCLS()
+#        self.encoding = self.server.encodingCLS()
 
     def handle(self):
         """handle a new tcp-connection"""
@@ -49,6 +102,7 @@ class TCPRequestHandler(SocketServer.BaseRequestHandler):
         clientaddr = self.client_address
         serverobj = self.server
         self.log.info("handling new connection from %s:%d" % clientaddr)
+        data = b''
 
         # notify dispatcher of us
         serverobj.dispatcher.add_connection(self)
@@ -57,14 +111,16 @@ class TCPRequestHandler(SocketServer.BaseRequestHandler):
         #        mysocket.setblocking(False)
         # start serving
         while True:
-            # send replys fist, then listen for requests, timing out after 0.1s
+            # send replys first, then listen for requests, timing out after 0.1s
             while self._queue:
                 # put message into encoder to get frame(s)
                 # put frame(s) into framer to get bytestring
                 # send bytestring
                 outmsg = self._queue.popleft()
-                outframes = self.encoding.encode(outmsg)
+                #outmsg.mkreply()
-                outdata = self.framing.encode(outframes)
+                outdata = encode_msg_frame(*outmsg.serialize())
+#                outframes = self.encoding.encode(outmsg)
+#                outdata = self.framing.encode(outframes)
                 try:
                     mysocket.sendall(outdata)
                 except Exception:
@@ -72,9 +128,12 @@ class TCPRequestHandler(SocketServer.BaseRequestHandler):
 
             # XXX: improve: use polling/select here?
             try:
-                data = mysocket.recv(MAX_MESSAGE_SIZE)
+                data = data + mysocket.recv(MAX_MESSAGE_SIZE)
-            except (socket.timeout, socket.error) as e:
+            except socket.timeout as e:
                 continue
+            except socket.error as e:
+                self.log.exception(e)
+                return
             # XXX: should use select instead of busy polling
             if not data:
                 continue
@@ -82,24 +141,49 @@ class TCPRequestHandler(SocketServer.BaseRequestHandler):
             # put frames into (de-) coder and if a message appear,
             # call dispatcher.handle_request(self, message)
             # dispatcher will queue the reply before returning
-            frames = self.framing.decode(data)
+            while True:
-            if frames is not None:
+                origin, data = get_msg(data)
-                if not frames:  # empty list
+                if origin is None:
-                    self.queue_reply(HelpMessage(MSGTYPE=reply))
+                    break  # no more messages to process
-                for frame in frames:
+                if not origin:  # empty string -> send help message
-                    reply = None
+                    for idx, line in enumerate(HelpMessage.splitlines()):
-                    msg = self.encoding.decode(frame)
+                        msg = Message(HELPREPLY, specifier='%d' % idx)
-                    if msg:
+                        msg.data = line
-                        serverobj.dispatcher.handle_request(self, msg)
+                        self.queue_async_reply(msg)
+                    continue
+                msg = decode_msg(origin)
+                # construct msgObj from msg
+                try:
+                    msgObj = Message(*msg)
+                    msgObj.origin = origin.decode('latin-1')
+                    msgObj = serverobj.dispatcher.handle_request(self, msgObj)
+                except Exception as err:
+                    # create Error Obj instead
+                    msgObj.set_error(u'Internal', str(err), {'exception': formatException(),
+                                                          'traceback':formatExtendedStack()})
+                    print('--------------------')
+                    print(formatException())
+                    print('--------------------')
+                    print(formatExtendedStack())
+                    print('====================')
+
+                if msgObj:
+                    self.queue_reply(msgObj)
 
     def queue_async_reply(self, data):
         """called by dispatcher for async data units"""
-        self._queue.append(data)
+        if data:
+            self._queue.append(data)
+        else:
+            self.log.error('should asynq_queue %s' % data)
 
     def queue_reply(self, data):
         """called by dispatcher to queue (sync) replies"""
         # sync replies go first!
-        self._queue.appendleft(data)
+        if data:
+            self._queue.appendleft(data)
+        else:
+            self.log.error('should queue %s' % data)
 
     def finish(self):
         """called when handle() terminates, i.e. the socket closed"""
@@ -115,7 +199,7 @@ class TCPRequestHandler(SocketServer.BaseRequestHandler):
             self.request.close()
 
 
-class TCPServer(SocketServer.ThreadingTCPServer):
+class TCPServer(socketserver.ThreadingTCPServer):
     daemon_threads = True
     allow_reuse_address = True
 
@@ -129,12 +213,14 @@ class TCPServer(SocketServer.ThreadingTCPServer):
             portnum = int(_port)
         # tcp is a byte stream, so we need Framers (to get frames)
         # and encoders (to en/decode messages from frames)
-        self.framingCLS = FRAMERS[interfaceopts.pop('framing', 'none')]
+        interfaceopts.pop('framing')  # HACK
-        self.encodingCLS = ENCODERS[interfaceopts.pop('encoding', 'pickle')]
+        interfaceopts.pop('encoding')  # HACK
+
+#        self.framingCLS = FRAMERS[interfaceopts.pop('framing', 'none')]
+#        self.encodingCLS = ENCODERS[interfaceopts.pop('encoding', 'pickle')]
         self.log.info("TCPServer binding to %s:%d" % (bindto, portnum))
-        self.log.debug("TCPServer using framing=%s" % self.framingCLS.__name__)
+#        self.log.debug("TCPServer using framing=%s" % self.framingCLS.__name__)
-        self.log.debug("TCPServer using encoding=%s" %
+#        self.log.debug("TCPServer using encoding=%s" % self.encodingCLS.__name__)
-                       self.encodingCLS.__name__)
+        socketserver.ThreadingTCPServer.__init__(
-        SocketServer.ThreadingTCPServer.__init__(
             self, (bindto, portnum), TCPRequestHandler, bind_and_activate=True)
         self.log.info("TCPServer initiated")

secop/protocol/messages.py

@@ -20,178 +20,185 @@
 #
 # *****************************************************************************
 """Define SECoP Messages"""
+from __future__ import print_function
+
+import json
+from secop.protocol.errors import EXCEPTIONS
+
+# allowed actions:
+
+IDENTREQUEST = u'*IDN?'  # literal
+# literal! first part is fixed!
+IDENTREPLY = u'SINE2020&ISSE,SECoP,V2018-02-13,rc2'
+
+DESCRIPTIONREQUEST = u'describe'  # literal
+DESCRIPTIONREPLY = u'describing'  # +<id> +json
+
+ENABLEEVENTSREQUEST = u'activate'  # literal + optional spec
+ENABLEEVENTSREPLY = u'active'  # literal + optional spec, is end-of-initial-data-transfer
+
+DISABLEEVENTSREQUEST = u'deactivate'  # literal + optional spec
+DISABLEEVENTSREPLY = u'inactive'  # literal + optional spec
+
+COMMANDREQUEST = u'do'  # +module:command +json args (if needed)
+# +module:command +json args (if needed) # send after the command finished !
+COMMANDREPLY = u'done'
+
+# +module[:parameter] +json_value -> NO direct reply, calls POLL internally
+WRITEREQUEST = u'change'
+# +module[:parameter] +json_value # send with the read back value
+WRITEREPLY = u'changed'
+
+# +module[:parameter] -> NO direct reply, calls POLL internally!
+POLLREQUEST = u'read'
+EVENTREPLY = u'update'  # +module[:parameter] +json_value (value, qualifiers_as_dict)
+
+HEARTBEATREQUEST = u'ping'  # +nonce_without_space
+HEARTBEATREPLY = u'pong'  # +nonce_without_space
+
+ERRORREPLY = u'error'  # +errorclass +json_extended_info
+
+HELPREQUEST = u'help'  # literal
+HELPREPLY = u'helping'  # +line number +json_text
+
+# helper mapping to find the REPLY for a REQUEST
+REQUEST2REPLY = {
+    IDENTREQUEST:         IDENTREPLY,
+    DESCRIPTIONREQUEST:   DESCRIPTIONREPLY,
+    ENABLEEVENTSREQUEST:  ENABLEEVENTSREPLY,
+    DISABLEEVENTSREQUEST: DISABLEEVENTSREPLY,
+    COMMANDREQUEST:       COMMANDREPLY,
+    WRITEREQUEST:         WRITEREPLY,
+    POLLREQUEST:          EVENTREPLY,
+    HEARTBEATREQUEST:     HEARTBEATREPLY,
+    HELPREQUEST:          HELPREPLY,
+}
+
 
 
 class Message(object):
     """base class for messages"""
-    is_request = False
+    origin = u'<unknown source>'
-    is_reply = False
+    action = u'<unknown message type>'
-    is_error = False
+    specifier = None
-    qualifiers = {}
+    data = None
-    origin = "<unknown source>"
 
-    def __init__(self, **kwds):
+    # cooked versions
-        self.ARGS = set()
+    module = None
+    parameter = None
+    command = None
+    args = None
+
+    # if set, these are used for generating the reply
+    qualifiers = None  # will be rectified to dict() in __init__
+    value = None  # also the result of a command
+
+    # if set, these are used for generating the error msg
+    errorclass = ''  # -> specifier
+    errordescription = ''  # -> data[1] (data[0] is origin)
+    errorinfo = {}  # -> data[2]
+
+    def __init__(self, action, specifier=None, data=None, **kwds):
+        self.qualifiers = {}
+        self.action = action
+        if data:
+            data = json.loads(data)
+        if specifier:
+            self.module = specifier
+            self.specifier = specifier
+            if ':' in specifier:
+                self.module, p = specifier.split(':',1)
+                if action in (COMMANDREQUEST, COMMANDREPLY):
+                    self.command = p
+                    # XXX: extract args?
+                    self.args = data
+                else:
+                    self.parameter = p
+                    if data is not None:
+                        self.data = data
+            elif data is not None:
+                self.data = data
+        # record extra values
+        self.__arguments = set()
         for k, v in kwds.items():
             self.setvalue(k, v)
 
     def setvalue(self, key, value):
         setattr(self, key, value)
-        self.ARGS.add(key)
+        self.__arguments.add(key)
+
+    def setqualifier(self, key, value):
+        self.qualifiers[key] = value
 
     def __repr__(self):
-        return self.__class__.__name__ + '(' + \
+        return u'Message(%r' % self.action + \
-            ', '.join('%s=%s' % (k, repr(getattr(self, k)))
+            u', '.join('%s=%s' % (k, repr(getattr(self, k)))
-                      for k in sorted(self.ARGS)) + ')'
+                      for k in sorted(self.__arguments)) + u')'
 
-    def as_dict(self):
+    def serialize(self):
-        """returns set parameters as dict"""
+        """return <action>,<specifier>,<jsonyfied_data> triple"""
-        return dict(map(lambda k: (k, getattr(self, k)), self.ARGS))
+        if self.errorclass:
+            for k in self.__arguments:
+                if k in (u'origin', u'errorclass', u'errorinfo', u'errordescription'):
+                    if k in self.errorinfo:
+                        del self.errorinfo[k]
+                    continue
+                self.errorinfo[k] = getattr(self, k)
+            data = [self.origin, self.errordescription, self.errorinfo]
+            print(repr(data))
+            return ERRORREPLY, self.errorclass, json.dumps(data)
+        elif self.value or self.qualifiers:
+            data = [self.value, self.qualifiers]
+        else:
+            data = self.data
+
+        try:
+            data = json.dumps(data) if data else u''
+        except TypeError:
+            print('Can not serialze: %s' % repr(data))
+            data = u'none'
+
+        if self.specifier:
+            specifier = self.specifier
+        else:
+            specifier = self.module
+            if self.parameter:
+                specifier = u'%s:%s' %(self.module, self.parameter)
+            if self.command:
+                specifier = u'%s:%s' %(self.module, self.command)
+        return self.action, specifier, data
+
+    def mkreply(self):
+        self.action = REQUEST2REPLY.get(self.action, self.action)
+
+    def set_error(self, errorclass, errordescription, errorinfo):
+        if errorclass not in EXCEPTIONS:
+            errordescription = '%s is not an official errorclass!\n%s' % (errorclass, errordescription)
+            errorclass = u'Internal'
+        # used to mark thes as an error message
+        # XXX: check errorclass for allowed values !
+        self.setvalue(u'errorclass', errorclass)  # a str
+        self.setvalue(u'errordescription', errordescription)   # a str
+        self.setvalue(u'errorinfo', errorinfo)  # a dict
+        self.action = ERRORREPLY
+
+    def set_result(self, value, qualifiers):
+        # used to mark thes as an result reply message
+        self.setvalue(u'value', value)
+        self.qualifiers.update(qualifiers)
+        self.__arguments.add(u'qualifier')
 
 
-class Value(object):
 
-    def __init__(self,
+HelpMessage = u"""Try one of the following:
-                 module,
+            '%s' to query protocol version
-                 parameter=None,
+            '%s' to read the description
-                 command=None,
+            '%s <module>[:<parameter>]' to request reading a value
-                 value=Ellipsis,
+            '%s <module>[:<parameter>] value' to request changing a value
-                 **qualifiers):
+            '%s <module>[:<command>]' to execute a command
-        self.module = module
+            '%s <nonce>' to request a heartbeat response
-        self.parameter = parameter
+            '%s' to activate async updates
-        self.command = command
+            '%s' to deactivate updates
-        self.value = value
+            """ % (IDENTREQUEST, DESCRIPTIONREQUEST, POLLREQUEST,
-        self.qualifiers = qualifiers
+                   WRITEREQUEST, COMMANDREQUEST, HEARTBEATREQUEST,
-        self.msgtype = 'update'  # 'changed' or 'done'
+                   ENABLEEVENTSREQUEST, DISABLEEVENTSREQUEST)
-
-    def __repr__(self):
-        devspec = self.module
-        if self.parameter:
-            devspec = '%s:%s' % (devspec, self.parameter)
-        elif self.command:
-            devspec = '%s:%s()' % (devspec, self.command)
-        return '%s:Value(%s)' % (devspec, ', '.join([repr(self.value)] + [
-            '%s=%s' % (k, format_time(v) if k == "timestamp" else repr(v))
-            for k, v in self.qualifiers.items()
-        ]))
-
-
-class Request(Message):
-    is_request = True
-
-    def get_reply(self):
-        """returns a Reply object prefilled with the attributes from this request."""
-        m = Message()
-        m.is_request = False
-        m.is_reply = True
-        m.is_error = False
-        m.qualifiers = self.qualifiers
-        m.origin = self.origin
-        for k in self.ARGS:
-            m.setvalue(k, self.__dict__[k])
-        return m
-
-    def get_error(self, errorclass, errorinfo):
-        """returns a Reply object prefilled with the attributes from this request."""
-        m = ErrorMessage()
-        m.qualifiers = self.qualifiers
-        m.origin = self.origin
-        for k in self.ARGS:
-            m.setvalue(k, self.__dict__[k])
-        m.setvalue("errorclass", errorclass[:-5]
-                   if errorclass.endswith('rror') else errorclass)
-        m.setvalue("errorinfo", errorinfo)
-        return m
-
-
-class IdentifyRequest(Request):
-    pass
-
-
-class IdentifyReply(Message):
-    is_reply = True
-    version_string = None
-
-
-class DescribeRequest(Request):
-    pass
-
-
-class DescribeReply(Message):
-    is_reply = True
-    equipment_id = None
-    description = None
-
-
-class ActivateRequest(Request):
-    pass
-
-
-class ActivateReply(Message):
-    is_reply = True
-
-
-class DeactivateRequest(Request):
-    pass
-
-
-class DeactivateReply(Message):
-    is_reply = True
-
-
-class CommandRequest(Request):
-    command = ''
-    arguments = []
-
-
-class CommandReply(Message):
-    is_reply = True
-    command = ''
-    result = None
-
-
-class WriteRequest(Request):
-    module = None
-    parameter = None
-    value = None
-
-
-class WriteReply(Message):
-    is_reply = True
-    module = None
-    parameter = None
-    value = None
-
-
-class PollRequest(Request):
-    is_request = True
-    module = None
-    parameter = None
-
-
-class HeartbeatRequest(Request):
-    nonce = 'alive'
-
-
-class HeartbeatReply(Message):
-    is_reply = True
-    nonce = 'undefined'
-
-
-class EventMessage(Message):
-    # use Value directly for Replies !
-    is_reply = True
-    module = None
-    parameter = None
-    command = None
-    value = None  # Value object ! (includes qualifiers!)
-
-
-class ErrorMessage(Message):
-    is_error = True
-    errorclass = 'InternalError'
-    errorinfo = None
-
-
-class HelpMessage(Request):
-    is_reply = True  # !sic!

secop/protocol/messages_old.py

@@ -1,235 +0,0 @@
-#  -*- coding: utf-8 -*-
-# *****************************************************************************
-#
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free Software
-# Foundation; either version 2 of the License, or (at your option) any later
-# version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-# Module authors:
-#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
-#
-# *****************************************************************************
-"""Define SECoP Messages"""
-
-# Request Types
-REQUEST = 'request'
-REPLY = 'reply'
-ERROR = 'error'
-
-# Message types ('actions') hint: fetch is list+read
-LIST = 'list'
-READ = 'read'
-WRITE = 'write'
-COMMAND = 'command'
-POLL = 'poll'
-SUBSCRIBE = 'subscribe'
-UNSUBSCRIBE = 'unsubscribe'
-TRIGGER = 'trigger'
-EVENT = 'event'
-ERROR = 'error'
-HELP = 'help'
-
-# base class for messages
-
-
-class Message(object):
-
-    MSGTYPE = 'Undefined'
-    devs = None
-    pars = None
-    props = None
-    result = None
-    error = None
-    ARGS = None
-    errortype = None
-
-    def __init__(self, **kwds):
-        self.devs = []
-        self.pars = []
-        self.props = []
-        self.result = []
-        self.ARGS = set()
-        for k, v in kwds.items():
-            self.setvalue(k, v)
-
-    def setvalue(self, key, value):
-        setattr(self, key, value)
-        self.ARGS.add(key)
-
-    @property
-    def NAME(self):
-        # generate sensible name
-        r = 'Message'
-        if self.props:
-            r = 'Property' if self.props != ['*'] else 'Properties'
-        elif self.pars:
-            r = 'Parameter' if self.pars != ['*'] else 'Parameters'
-        elif self.devs:
-            r = 'Module' if self.devs != ['*'] else 'Modules'
-
-        t = ''
-        if self.MSGTYPE in [
-                LIST, READ, WRITE, COMMAND, POLL, SUBSCRIBE, UNSUBSCRIBE, HELP
-        ]:
-            t = 'Request' if not self.result else 'Reply'
-
-        if self.errortype is None:
-            return self.MSGTYPE.title() + r + t
-        else:
-            return self.errortype + 'Error'
-
-    def __repr__(self):
-        return self.NAME + '(' + \
-            ', '.join('%s=%r' % (k, getattr(self, k))
-                      for k in self.ARGS if getattr(self, k) is not None) + ')'
-
-
-class Value(object):
-
-    def __init__(self, value=Ellipsis, qualifiers=None, **kwds):
-        self.dev = ''
-        self.param = ''
-        self.prop = ''
-        self.value = value
-        self.qualifiers = qualifiers or dict()
-        self.__dict__.update(kwds)
-
-    def __repr__(self):
-        devspec = self.dev
-        if self.param:
-            devspec = '%s:%s' % (devspec, self.param)
-        if self.prop:
-            devspec = '%s:%s' % (devspec, self.prop)
-        return '%s:Value(%s)' % (
-            devspec,
-            ', '.join([repr(self.value)] +
-                      ['%s=%r' % (k, v) for k, v in self.qualifiers.items()]))
-
-
-class ListMessage(Message):
-    MSGTYPE = LIST
-
-
-class ReadMessage(Message):
-    MSGTYPE = READ  # return cached value
-
-
-class WriteMessage(Message):
-    MSGTYPE = WRITE  # write value to some spec
-    target = None  # actually float or string
-
-
-class CommandMessage(Message):
-    MSGTYPE = COMMAND
-    cmd = ''  # always string
-    args = []
-    result = []
-
-
-class PollMessage(Message):
-    MSGTYPE = POLL  # read HW and return hw_value
-
-
-class SubscribeMessage(Message):
-    MSGTYPE = SUBSCRIBE
-
-
-class UnsubscribeMessage(Message):
-    MSGTYPE = UNSUBSCRIBE
-
-
-class TriggerMessage(Message):
-    MSGTYPE = TRIGGER
-
-
-class EventMessage(Message):
-    MSGTYPE = EVENT
-
-
-class ErrorMessage(Message):
-    MSGTYPE = ERROR
-    errorstring = 'an unhandled error occured'
-    errortype = 'UnknownError'
-
-
-class HelpMessage(Message):
-    MSGTYPE = HELP
-
-
-class NoSuchModuleError(ErrorMessage):
-
-    def __init__(self, *devs):
-        ErrorMessage.__init__(
-            self,
-            devs=devs,
-            errorstring="Module %r does not exist" % devs[0],
-            errortype='NoSuchModule')
-
-
-class NoSuchParamError(ErrorMessage):
-
-    def __init__(self, dev, *params):
-        ErrorMessage.__init__(
-            self,
-            devs=(dev, ),
-            params=params,
-            errorstring="Module %r has no parameter %r" % (dev, params[0]),
-            errortype='NoSuchParam')
-
-
-class ParamReadonlyError(ErrorMessage):
-
-    def __init__(self, dev, *params):
-        ErrorMessage.__init__(
-            self,
-            devs=(dev, ),
-            params=params,
-            errorstring="Module %r, parameter %r is not writeable!" %
-            (dev, params[0]),
-            errortype='ParamReadOnly')
-
-
-class InvalidParamValueError(ErrorMessage):
-
-    def __init__(self, dev, param, value, e):
-        ErrorMessage.__init__(
-            self,
-            devs=(dev, ),
-            params=params,
-            values=(value),
-            errorstring=str(e),
-            errortype='InvalidParamValueError')
-
-
-class InternalError(ErrorMessage):
-
-    def __init__(self, err, **kwds):
-        ErrorMessage.__init__(
-            self, errorstring=str(err), errortype='InternalError', **kwds)
-
-
-MESSAGE = dict((cls.MSGTYPE, cls)
-               for cls in [
-                   HelpMessage, ErrorMessage, EventMessage, TriggerMessage,
-                   UnsubscribeMessage, SubscribeMessage, PollMessage,
-                   CommandMessage, WriteMessage, ReadMessage, ListMessage
-])
-
-if __name__ == '__main__':
-    print("Minimal testing of messages....")
-    m = Message(MSGTYPE='test', a=1, b=2, c='x')
-    print m
-    print ReadMessage(devs=['a'], result=[Value(12.3)])
-
-    print "OK"
-    print

secop/protocol/status.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under

secop/server.py

@@ -21,11 +21,17 @@
 #
 # *****************************************************************************
 """Define helpers"""
+from __future__ import print_function
+
 import os
 import ast
 import time
 import threading
-import ConfigParser
+
+try:
+    import configparser  # py3
+except ImportError:
+    import ConfigParser as configparser  # py2
 
 from daemon import DaemonContext
 
@@ -42,14 +48,14 @@ from secop.errors import ConfigError
 
 class Server(object):
 
-    def __init__(self, name, parentLogger=None):
+    def __init__(self, name, parent_logger=None):
         self._name = name
 
-        self.log = parentLogger.getChild(name, True)
+        self.log = parent_logger.getChild(name, True)
 
         cfg = getGeneralConfig()
-        self._pidfile = os.path.join(cfg['piddir'], name + '.pid')
+        self._pidfile = os.path.join(cfg[u'piddir'], name + u'.pid')
-        self._cfgfile = os.path.join(cfg['confdir'], name + '.cfg')
+        self._cfgfile = os.path.join(cfg[u'confdir'], name + u'.cfg')
 
         self._dispatcher = None
         self._interface = None
@@ -61,7 +67,7 @@ class Server(object):
         pidfile = pidlockfile.TimeoutPIDLockFile(self._pidfile)
 
         if pidfile.is_locked():
-            self.log.error('Pidfile already exists. Exiting')
+            self.log.error(u'Pidfile already exists. Exiting')
 
         with DaemonContext(
                 pidfile=pidfile,
@@ -72,13 +78,13 @@ class Server(object):
         try:
             self._processCfg()
         except Exception:
-            print formatException(verbose=True)
+            print(formatException(verbose=True))
             raise
 
-        self.log.info('startup done, handling transport messages')
+        self.log.info(u'startup done, handling transport messages')
         self._threads = set()
         for _if in self._interfaces:
-            self.log.debug('starting thread for interface %r' % _if)
+            self.log.debug(u'starting thread for interface %r' % _if)
             t = threading.Thread(target=_if.serve_forever)
             t.daemon = True
             t.start()
@@ -87,20 +93,20 @@ class Server(object):
             time.sleep(1)
             for t in self._threads:
                 if not t.is_alive():
-                    self.log.debug('thread %r died (%d still running)' %
+                    self.log.debug(u'thread %r died (%d still running)' %
                                    (t, len(self._threads)))
                     t.join()
                     self._threads.discard(t)
 
     def _processCfg(self):
-        self.log.debug('Parse config file %s ...' % self._cfgfile)
+        self.log.debug(u'Parse config file %s ...' % self._cfgfile)
 
-        parser = ConfigParser.SafeConfigParser()
+        parser = configparser.SafeConfigParser()
         parser.optionxform = str
 
         if not parser.read([self._cfgfile]):
-            self.log.error("Couldn't read cfg file !")
+            self.log.error(u'Couldn\'t read cfg file !')
-            raise ConfigError("Couldn't read cfg file %r" % self._cfgfile)
+            raise ConfigError(u'Couldn\'t read cfg file %r' % self._cfgfile)
 
         self._interfaces = []
 
@@ -109,63 +115,63 @@ class Server(object):
         equipment_id = None
         nodeopts = []
         for section in parser.sections():
-            if section.lower().startswith('module '):
+            if section.lower().startswith(u'module '):
                 # module section
                 # omit leading 'module ' string
-                devname = section[len('module '):]
+                devname = section[len(u'module '):]
                 devopts = dict(item for item in parser.items(section))
-                if 'class' not in devopts:
+                if u'class' not in devopts:
-                    self.log.error('Module %s needs a class option!')
+                    self.log.error(u'Module %s needs a class option!')
                     raise ConfigError(
-                        'cfgfile %r: Module %s needs a class option!' %
+                        u'cfgfile %r: Module %s needs a class option!' %
                         (self._cfgfile, devname))
                 # MAGIC: transform \n.\n into \n\n which are normally stripped
                 # by the ini parser
                 for k in devopts:
                     v = devopts[k]
-                    while '\n.\n' in v:
+                    while u'\n.\n' in v:
-                        v = v.replace('\n.\n', '\n\n')
+                        v = v.replace(u'\n.\n', u'\n\n')
                     devopts[k] = v
                 # try to import the class, raise if this fails
-                devopts['class'] = get_class(devopts['class'])
+                devopts[u'class'] = get_class(devopts[u'class'])
                 # all went well so far
                 moduleopts.append([devname, devopts])
-            if section.lower().startswith('interface '):
+            if section.lower().startswith(u'interface '):
                 # interface section
                 # omit leading 'interface ' string
-                ifname = section[len('interface '):]
+                ifname = section[len(u'interface '):]
                 ifopts = dict(item for item in parser.items(section))
-                if 'interface' not in ifopts:
+                if u'interface' not in ifopts:
-                    self.log.error('Interface %s needs an interface option!')
+                    self.log.error(u'Interface %s needs an interface option!')
                     raise ConfigError(
-                        'cfgfile %r: Interface %s needs an interface option!' %
+                        u'cfgfile %r: Interface %s needs an interface option!' %
                         (self._cfgfile, ifname))
                 # all went well so far
                 interfaceopts.append([ifname, ifopts])
-            if section.lower().startswith('equipment ') or section.lower().startswith('node '):
+            if section.lower().startswith(u'equipment ') or section.lower().startswith(u'node '):
                 if equipment_id is not None:
-                    raise ConfigError('cfgfile %r: only one [node <id>] section allowed, found another [%s]!' % (
+                    raise ConfigError(u'cfgfile %r: only one [node <id>] section allowed, found another [%s]!' % (
                         self._cfgfile, section))
                 # equipment/node settings
-                equipment_id = section.split(' ', 1)[1].replace(' ', '_')
+                equipment_id = section.split(u' ', 1)[1].replace(u' ', u'_')
                 nodeopts = dict(item for item in parser.items(section))
-                nodeopts['equipment_id'] = equipment_id
+                nodeopts[u'equipment_id'] = equipment_id
-                nodeopts['id'] = equipment_id
+                nodeopts[u'id'] = equipment_id
                 # MAGIC: transform \n.\n into \n\n which are normally stripped
                 # by the ini parser
                 for k in nodeopts:
                     v = nodeopts[k]
-                    while '\n.\n' in v:
+                    while u'\n.\n' in v:
-                        v = v.replace('\n.\n', '\n\n')
+                        v = v.replace(u'\n.\n', u'\n\n')
                     nodeopts[k] = v
 
         if equipment_id is None:
-            self.log.error('Need a [node <id>] section, none found!')
+            self.log.error(u'Need a [node <id>] section, none found!')
             raise ConfigError(
-                'cfgfile %r: need an [node <id>] option!' % (self._cfgfile))
+                u'cfgfile %r: need an [node <id>] option!' % (self._cfgfile))
 
         self._dispatcher = self._buildObject(
-            'Dispatcher', Dispatcher, nodeopts)
+            u'Dispatcher', Dispatcher, nodeopts)
         self._processInterfaceOptions(interfaceopts)
         self._processModuleOptions(moduleopts)
 
@@ -173,13 +179,13 @@ class Server(object):
         # check modules opts by creating them
         devs = []
         for devname, devopts in moduleopts:
-            devclass = devopts.pop('class')
+            devclass = devopts.pop(u'class')
             # create module
-            self.log.debug('Creating Module %r' % devname)
+            self.log.debug(u'Creating Module %r' % devname)
-            export = devopts.pop('export', '1')
+            export = devopts.pop(u'export', u'1')
-            export = export.lower() in ('1', 'on', 'true', 'yes')
+            export = export.lower() in (u'1', u'on', u'true', u'yes')
-            if 'default' in devopts:
+            if u'default' in devopts:
-                devopts['value'] = devopts.pop('default')
+                devopts[u'value'] = devopts.pop(u'default')
             # strip '"
             for k, v in devopts.items():
                 try:
@@ -192,13 +198,13 @@ class Server(object):
 
         # connect modules with dispatcher
         for devname, devobj, export in devs:
-            self.log.info('registering module %r' % devname)
+            self.log.info(u'registering module %r' % devname)
             self._dispatcher.register_module(devobj, devname, export)
             # also call init on the modules
             devobj.init()
         # call a possibly empty postinit on each module after registering all
         for _devname, devobj, _export in devs:
-            postinit = getattr(devobj, 'postinit', None)
+            postinit = getattr(devobj, u'postinit', None)
             if postinit:
                 postinit()
 
@@ -206,17 +212,17 @@ class Server(object):
         # eval interfaces
         self._interfaces = []
         for ifname, ifopts in interfaceopts:
-            ifclass = ifopts.pop('interface')
+            ifclass = ifopts.pop(u'interface')
             ifclass = INTERFACES[ifclass]
             interface = self._buildObject(ifname, ifclass, ifopts,
                                           self._dispatcher)
             self._interfaces.append(interface)
 
     def _buildObject(self, name, cls, options, *args):
-        self.log.debug('Creating %s ...' % name)
+        self.log.debug(u'Creating %s ...' % name)
         # cls.__init__ should pop all used args from options!
         obj = cls(self.log.getChild(name.lower()), options, *args)
         if options:
-            raise ConfigError('%s: don\'t know how to handle option(s): %s' %
+            raise ConfigError(u'%s: don\'t know how to handle option(s): %s' %
-                              (cls.__name__, ', '.join(options.keys())))
+                              (cls.__name__, u', '.join(options)))
         return obj

secop/simulation.py

@@ -40,7 +40,7 @@ class SimBase(object):
         if '.extra_params' in cfgdict:
             extra_params = cfgdict.pop('.extra_params')
             # make a copy of self.parameter
-            self.parameters = dict((k,v.copy()) for k,v in self.parameters.items())
+            self.parameters = dict((k, v.copy()) for k, v in self.parameters.items())
             for k in extra_params.split(','):
                 k = k.strip()
                 self.parameters[k] = Param('extra_param: %s' % k.strip(),

secop_demo/cryo.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under
@@ -17,6 +16,7 @@
 #
 # Module authors:
 #   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
 # *****************************************************************************
 """playing implementation of a (simple) simulated cryostat"""
 
@@ -277,11 +277,11 @@ class Cryostat(CryoBase):
 
                 error = self.setpoint - newregulation
                 # use a simple filter to smooth delta a little
-                delta = (delta + regulation - newregulation) / 2.
+                delta = (delta + regulation - newregulation) * 0.5
 
-                kp = self.p / 10.  # LakeShore P = 10*k_p
+                kp = self.p * 0.1             # LakeShore P = 10*k_p
                 ki = kp * abs(self.i) / 500.  # LakeShore I = 500/T_i
-                kd = kp * abs(self.d) / 2.  # LakeShore D = 2*T_d
+                kd = kp * abs(self.d) / 2.    # LakeShore D = 2*T_d
                 _P = kp * error
                 _I += ki * error * h
                 _D = kd * delta / h
@@ -301,7 +301,7 @@ class Cryostat(CryoBase):
 
                 # damp oscillations due to D switching signs
                 if _D * lastD < -0.2:
-                    v = (v + heater) / 2.
+                    v = (v + heater) * 0.5
                 # clamp new heater power to 0..100%
                 heater = clamp(v, 0., 100.)
                 lastD = _D
@@ -357,10 +357,10 @@ class Cryostat(CryoBase):
                 self.status = status.BUSY, 'unstable'
             elif self.setpoint == self.target:
                 self.status = status.OK, 'at target'
-                damper -= (damper - 1) / 10.  # max value for damper is 11
+                damper -= (damper - 1) * 0.1  # max value for damper is 11
             else:
                 self.status = status.BUSY, 'ramping setpoint'
-            damper -= (damper - 1) / 20.
+            damper -= (damper - 1) * 0.05
             self.regulationtemp = round(regulation, 3)
             self.sampletemp = round(sample, 3)
             self.heaterpower = round(heater * self.maxpower * 0.01, 3)

secop_demo/modules.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under
@@ -17,6 +16,7 @@
 #
 # Module authors:
 #   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
 # *****************************************************************************
 """testing devices"""
 
@@ -312,6 +312,6 @@ class DatatypesTest(Readable):
 
 class ArrayTest(Readable):
     parameters = {
-        "x" : Param('value', datatype=ArrayOf(FloatRange(),100000,100000),
+        "x": Param('value', datatype=ArrayOf(FloatRange(), 100000, 100000),
                         default = 100000 * [0]),
     }

secop_demo/test.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under
@@ -17,6 +16,7 @@
 #
 # Module authors:
 #   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
 # *****************************************************************************
 """testing devices"""
 

secop_ess/epics.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 #  -*- coding: utf-8 -*-
 # *****************************************************************************
 # This program is free software; you can redistribute it and/or modify it under
@@ -18,8 +17,11 @@
 # Module authors:
 #   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
 #   Erik Dahlbäck <erik.dahlback@esss.se>
+#
 # *****************************************************************************
 
+from __future__ import absolute_import
+
 from secop.datatypes import EnumType, FloatRange, StringType
 from secop.modules import Readable, Drivable, Param
 from secop.protocol import status

secop_mlz/amagnet.py

@@ -20,8 +20,7 @@
 #
 # *****************************************************************************
 
-"""
+"""Supporting classes for FRM2 magnets, currently only Garfield (amagnet).
-Supporting classes for FRM2 magnets, currently only Garfield (amagnet).
 """
 
 # partially borrowed from nicos
@@ -127,7 +126,7 @@ class GarfieldMagnet(SequencerMixin, Drivable):
                 maxfield = tryfield
             # if interval is so small, that any error within is acceptable:
             if maxfield - minfield < 1e-4:
-                ratio = (field - minfield) / (maxfield - minfield)
+                ratio = (field - minfield) / float(maxfield - minfield)
                 trycurr = (maxcurr - mincurr) * ratio + mincurr
                 self.log.debug('current for %g T is %g A', field, trycurr)
                 return trycurr  # interpolated
@@ -184,7 +183,7 @@ class GarfieldMagnet(SequencerMixin, Drivable):
 
     def write_ramp(self, newramp):
         # This is an approximation!
-        self._currentsource.ramp = newramp / self.calibration[0]
+        self._currentsource.ramp = float(newramp) / self.calibration[0]
 
     def _get_field_polarity(self):
         sign = int(self._polswitch.read_value())

secop_mlz/entangle.py

@@ -37,7 +37,7 @@ import PyTango
 
 from secop.lib import lazy_property
 from secop.protocol import status
-
+#from secop.parse import Parser
 from secop.datatypes import IntRange, FloatRange, StringType, TupleOf, \
     ArrayOf, EnumType
 from secop.errors import ConfigError, ProgrammingError, CommunicationError, \

test/test_datatypes.py

@@ -107,9 +107,9 @@ def test_EnumType():
     with pytest.raises(TypeError):
         dt.validate([19, 'X'])
 
-    assert dt.validate('a') == 'a'
+    assert dt.validate('a') == 3
-    assert dt.validate('stuff') == 'stuff'
+    assert dt.validate('stuff') == 1
-    assert dt.validate(1) == 'stuff'
+    assert dt.validate(1) == 1
     with pytest.raises(ValueError):
         dt.validate(2)