rework property handling

+ DataType validators are shifted to __call__
+ as_json is moved to export_datatape()
+ new HasProperties Base Mixin for Modules/DataTypes
+ accessibles can be accessed via iterator of a module
+ properties are properly 'derived' and checked, are set with .setPropertyValue

remember: parameters only have properties, so use getPropertyValue()

Change-Id: Iae0273f971aacb00fe6bf05e6a4d24a6d1be881a
Reviewed-on: https://forge.frm2.tum.de/review/20635
Tested-by: JenkinsCodeReview <bjoern_pedersen@frm2.tum.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

secop/basic_validators.py

@@ -146,3 +146,10 @@ def TupleProperty(*checkers):
             return tuple(c(v) for c, v in zip(checkers, values))
         raise ValueError(u'Value needs %d elements!' % len(checkers))
     return TupleChecker
+
+def ListOfProperty(checker):
+    if not callable(checker):
+        raise ProgrammingError(u'ListOfProperty needs a basic validator as Argument!')
+    def ListOfChecker(values):
+        return [checker(v) for v in values]
+    return ListOfChecker

secop/datatypes.py

@@ -21,11 +21,13 @@
 # *****************************************************************************
 """Define validated data types."""
 
+# pylint: disable=abstract-method
+
 from __future__ import division, print_function
 
 from base64 import b64decode, b64encode
 
-from secop.errors import ProgrammingError, ProtocolError
+from secop.errors import ProgrammingError, ProtocolError, BadValueError
 from secop.lib.enum import Enum
 from secop.parse import Parser
 
@@ -37,8 +39,6 @@ except NameError:
     unicode = str  # pylint: disable=redefined-builtin
 
 
-Parser = Parser()
-
 # Only export these classes for 'from secop.datatypes import *'
 __all__ = [
     u'DataType',
@@ -53,15 +53,16 @@ __all__ = [
 DEFAULT_MIN_INT = -16777216
 DEFAULT_MAX_INT = 16777216
 
+Parser = Parser()
+
 # base class for all DataTypes
-
-
 class DataType(object):
     IS_COMMAND = False
     unit = u''
     fmtstr = u'%r'
+    default = None
 
-    def validate(self, value):
+    def __call__(self, value):
         """check if given value (a python obj) is valid for this datatype
 
         returns the value or raises an appropriate exception"""
@@ -115,6 +116,7 @@ class FloatRange(DataType):
 
     def __init__(self, minval=None, maxval=None, unit=None, fmtstr=None,
                        absolute_resolution=None, relative_resolution=None,):
+        self.default = 0 if minval <= 0 <= maxval else minval
         self._defaults = {}
         self.setprop('min', minval, float(u'-inf'), float)
         self.setprop('max', maxval, float(u'+inf'), float)
@@ -125,27 +127,27 @@ class FloatRange(DataType):
 
         # check values
         if self.min > self.max:
-            raise ValueError(u'max must be larger then min!')
+            raise BadValueError(u'max must be larger then min!')
         if '%' not in self.fmtstr:
-            raise ValueError(u'Invalid fmtstr!')
+            raise BadValueError(u'Invalid fmtstr!')
         if self.absolute_resolution < 0:
-            raise ValueError(u'absolute_resolution MUST be >=0')
+            raise BadValueError(u'absolute_resolution MUST be >=0')
         if self.relative_resolution < 0:
-            raise ValueError(u'relative_resolution MUST be >=0')
+            raise BadValueError(u'relative_resolution MUST be >=0')
 
     def export_datatype(self):
         return [u'double', {k: getattr(self, k) for k, v in self._defaults.items()
                             if v != getattr(self, k)}]
 
-    def validate(self, value):
+    def __call__(self, value):
         try:
             value = float(value)
         except Exception:
-            raise ValueError(u'Can not validate %r to float' % value)
+            raise BadValueError(u'Can not __call__ %r to float' % value)
         prec = max(abs(value * self.relative_resolution), self.absolute_resolution)
         if self.min - prec <= value <= self.max + prec:
             return min(max(value, self.min), self.max)
-        raise ValueError(u'%g should be a float between %g and %g' %
+        raise BadValueError(u'%g should be a float between %g and %g' %
                          (value, self.min, self.max))
 
     def __repr__(self):
@@ -162,7 +164,7 @@ class FloatRange(DataType):
 
     def from_string(self, text):
         value = float(text)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         if unit is None:
@@ -178,26 +180,27 @@ class IntRange(DataType):
     def __init__(self, minval=None, maxval=None):
         self.min = DEFAULT_MIN_INT if minval is None else int(minval)
         self.max = DEFAULT_MAX_INT if maxval is None else int(maxval)
+        self.default = 0 if minval <= 0 <= maxval else minval
 
         # check values
         if self.min > self.max:
-            raise ValueError(u'Max must be larger then min!')
+            raise BadValueError(u'Max must be larger then min!')
 
     def export_datatype(self):
         return [u'int', {"min": self.min, "max": self.max}]
 
-    def validate(self, value):
+    def __call__(self, value):
         try:
             value = int(value)
             if value < self.min:
-                raise ValueError(u'%r should be an int between %d and %d' %
+                raise BadValueError(u'%r should be an int between %d and %d' %
                                  (value, self.min, self.max or 0))
             if value > self.max:
-                raise ValueError(u'%r should be an int between %d and %d' %
+                raise BadValueError(u'%r should be an int between %d and %d' %
                                  (value, self.min or 0, self.max))
             return value
         except Exception:
-            raise ValueError(u'Can not validate %r to int' % value)
+            raise BadValueError(u'Can not convert %r to int' % value)
 
     def __repr__(self):
         return u'IntRange(%d, %d)' % (self.min, self.max)
@@ -212,7 +215,7 @@ class IntRange(DataType):
 
     def from_string(self, text):
         value = int(text)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         return u'%d' % value
@@ -226,10 +229,11 @@ class ScaledInteger(DataType):
 
     def __init__(self, scale, minval=None, maxval=None, unit=None, fmtstr=None,
                        absolute_resolution=None, relative_resolution=None,):
+        self.default = 0 if minval <= 0 <= maxval else minval
         self._defaults = {}
         self.scale = float(scale)
         if not self.scale > 0:
-            raise ValueError(u'Scale MUST be positive!')
+            raise BadValueError(u'Scale MUST be positive!')
         self.setprop('unit', unit, u'', unicode)
         self.setprop('fmtstr', fmtstr, u'%g', unicode)
         self.setprop('absolute_resolution', absolute_resolution, self.scale, float)
@@ -240,13 +244,13 @@ class ScaledInteger(DataType):
 
         # check values
         if self.min > self.max:
-            raise ValueError(u'Max must be larger then min!')
+            raise BadValueError(u'Max must be larger then min!')
         if '%' not in self.fmtstr:
-            raise ValueError(u'Invalid fmtstr!')
+            raise BadValueError(u'Invalid fmtstr!')
         if self.absolute_resolution < 0:
-            raise ValueError(u'absolute_resolution MUST be >=0')
+            raise BadValueError(u'absolute_resolution MUST be >=0')
         if self.relative_resolution < 0:
-            raise ValueError(u'relative_resolution MUST be >=0')
+            raise BadValueError(u'relative_resolution MUST be >=0')
         # Remark: Datatype.copy() will round min, max to a multiple of self.scale
         # this should be o.k.
 
@@ -258,17 +262,17 @@ class ScaledInteger(DataType):
         info['max'] = int((self.max + self.scale * 0.5) // self.scale)
         return [u'scaled', info]
 
-    def validate(self, value):
+    def __call__(self, value):
         try:
             value = float(value)
         except Exception:
-            raise ValueError(u'Can not validate %r to float' % value)
+            raise BadValueError(u'Can not convert %r to float' % value)
         prec = max(self.scale, abs(value * self.relative_resolution),
                    self.absolute_resolution)
         if self.min - prec <= value <= self.max + prec:
             value = min(max(value, self.min), self.max)
         else:
-            raise ValueError(u'%g should be a float between %g and %g' %
+            raise BadValueError(u'%g should be a float between %g and %g' %
                              (value, self.min, self.max))
         intval = int((value + self.scale * 0.5) // self.scale)
         value = float(intval * self.scale)
@@ -293,7 +297,7 @@ class ScaledInteger(DataType):
 
     def from_string(self, text):
         value = float(text)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         if unit is None:
@@ -324,21 +328,21 @@ class EnumType(DataType):
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
-        return int(self.validate(value))
+        return int(self(value))
 
     def import_value(self, value):
         """returns a python object from serialisation"""
-        return self.validate(value)
+        return self(value)
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated (internal) value or raise"""
         try:
             return self._enum[value]
         except KeyError:
-            raise ValueError(u'%r is not a member of enum %r' % (value, self._enum))
+            raise BadValueError(u'%r is not a member of enum %r' % (value, self._enum))
 
     def from_string(self, text):
-        return self.validate(text)
+        return self(text)
 
     def format_value(self, value, unit=None):
         return u'%s<%s>' % (self._enum[value].name, self._enum[value].value)
@@ -356,9 +360,10 @@ class BLOBType(DataType):
         self.minsize = int(minsize)
         self.maxsize = int(maxsize)
         if self.minsize < 0:
-            raise ValueError(u'sizes must be bigger than or equal to 0!')
+            raise BadValueError(u'sizes must be bigger than or equal to 0!')
         elif self.minsize > self.maxsize:
-            raise ValueError(u'maxsize must be bigger than or equal to minsize!')
+            raise BadValueError(u'maxsize must be bigger than or equal to minsize!')
+        self.default = b'\0' * self.minsize
 
     def export_datatype(self):
         return [u'blob', dict(min=self.minsize, max=self.maxsize)]
@@ -366,16 +371,16 @@ class BLOBType(DataType):
     def __repr__(self):
         return u'BLOB(%d, %d)' % (self.minsize, self.maxsize)
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated (internal) value or raise"""
         if type(value) not in [unicode, str]:
-            raise ValueError(u'%r has the wrong type!' % value)
+            raise BadValueError(u'%r has the wrong type!' % value)
         size = len(value)
         if size < self.minsize:
-            raise ValueError(
+            raise BadValueError(
                 u'%r must be at least %d bytes long!' % (value, self.minsize))
         if size > self.maxsize:
-            raise ValueError(
+            raise BadValueError(
                 u'%r must be at most %d bytes long!' % (value, self.maxsize))
         return value
 
@@ -390,7 +395,7 @@ class BLOBType(DataType):
     def from_string(self, text):
         value = text
         # XXX:
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         return repr(value)
@@ -402,13 +407,14 @@ class StringType(DataType):
 
     def __init__(self, minsize=0, maxsize=None):
         if maxsize is None:
-            maxsize = minsize or 255
+            maxsize = minsize or 255*256
         self.minsize = int(minsize)
         self.maxsize = int(maxsize)
         if self.minsize < 0:
-            raise ValueError(u'sizes must be bigger than or equal to 0!')
+            raise BadValueError(u'sizes must be bigger than or equal to 0!')
         elif self.minsize > self.maxsize:
-            raise ValueError(u'maxsize must be bigger than or equal to minsize!')
+            raise BadValueError(u'maxsize must be bigger than or equal to minsize!')
+        self.default = u' ' * self.minsize
 
     def export_datatype(self):
         return [u'string', dict(min=self.minsize, max=self.maxsize)]
@@ -416,19 +422,19 @@ class StringType(DataType):
     def __repr__(self):
         return u'StringType(%d, %d)' % (self.minsize, self.maxsize)
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated (internal) value or raise"""
         if type(value) not in (unicode, str):
-            raise ValueError(u'%r has the wrong type!' % value)
+            raise BadValueError(u'%r has the wrong type!' % value)
         size = len(value)
         if size < self.minsize:
-            raise ValueError(
+            raise BadValueError(
                 u'%r must be at least %d bytes long!' % (value, self.minsize))
         if size > self.maxsize:
-            raise ValueError(
+            raise BadValueError(
                 u'%r must be at most %d bytes long!' % (value, self.maxsize))
         if u'\0' in value:
-            raise ValueError(
+            raise BadValueError(
                 u'Strings are not allowed to embed a \\0! Use a Blob instead!')
         return value
 
@@ -443,7 +449,7 @@ class StringType(DataType):
 
     def from_string(self, text):
         value = unicode(text)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         return repr(value)
@@ -451,6 +457,7 @@ class StringType(DataType):
 
 # Bool is a special enum
 class BoolType(DataType):
+    default = False
 
     def export_datatype(self):
         return [u'bool', {}]
@@ -458,25 +465,25 @@ class BoolType(DataType):
     def __repr__(self):
         return u'BoolType()'
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated (internal) value or raise"""
         if value in [0, u'0', u'False', u'false', u'no', u'off', False]:
             return False
         if value in [1, u'1', u'True', u'true', u'yes', u'on', True]:
             return True
-        raise ValueError(u'%r is not a boolean value!' % value)
+        raise BadValueError(u'%r is not a boolean value!' % value)
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
-        return True if self.validate(value) else False
+        return True if self(value) else False
 
     def import_value(self, value):
         """returns a python object from serialisation"""
-        return self.validate(value)
+        return self(value)
 
     def from_string(self, text):
         value = text
-        return self.validate(value)
+        return self(value)
 
 
     def format_value(self, value, unit=None):
@@ -492,25 +499,27 @@ class ArrayOf(DataType):
     maxsize = None
     members = None
 
-    def __init__(self, members, minsize=0, maxsize=None, unit=u''):
+    def __init__(self, members, minsize=0, maxsize=None, unit=None):
         if not isinstance(members, DataType):
-            raise ValueError(
+            raise BadValueError(
                 u'ArrayOf only works with a DataType as first argument!')
         # one argument -> exactly that size
-        # argument default to 10
+        # argument default to 100
         if maxsize is None:
-            maxsize = minsize or 10
+            maxsize = minsize or 100
         self.members = members
-        self.unit = unit
+        if unit:
+            self.members.unit = unit
 
         self.minsize = int(minsize)
         self.maxsize = int(maxsize)
         if self.minsize < 0:
-            raise ValueError(u'sizes must be > 0')
+            raise BadValueError(u'sizes must be > 0')
         elif self.maxsize < 1:
-            raise ValueError(u'Maximum size must be >= 1!')
+            raise BadValueError(u'Maximum size must be >= 1!')
         elif self.minsize > self.maxsize:
-            raise ValueError(u'maxsize must be bigger than or equal to minsize!')
+            raise BadValueError(u'maxsize must be bigger than or equal to minsize!')
+        self.default = [members.default] * self.minsize
 
     def export_datatype(self):
         return [u'array', dict(min=self.minsize, max=self.maxsize,
@@ -520,20 +529,20 @@ class ArrayOf(DataType):
         return u'ArrayOf(%s, %s, %s)' % (
             repr(self.members), self.minsize, self.maxsize)
 
-    def validate(self, value):
-        """validate a external representation to an internal one"""
+    def __call__(self, value):
+        """validate an external representation to an internal one"""
         if isinstance(value, (tuple, list)):
             # check number of elements
             if self.minsize is not None and len(value) < self.minsize:
-                raise ValueError(
+                raise BadValueError(
                     u'Array too small, needs at least %d elements!' %
                     self.minsize)
             if self.maxsize is not None and len(value) > self.maxsize:
-                raise ValueError(
+                raise BadValueError(
                     u'Array too big, holds at most %d elements!' % self.minsize)
             # apply subtype valiation to all elements and return as list
-            return [self.members.validate(elem) for elem in value]
-        raise ValueError(
+            return [self.members(elem) for elem in value]
+        raise BadValueError(
             u'Can not convert %s to ArrayOf DataType!' % repr(value))
 
     def export_value(self, value):
@@ -548,7 +557,7 @@ class ArrayOf(DataType):
         value, rem = Parser.parse(text)
         if rem:
             raise ProtocolError(u'trailing garbage: %r' % rem)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         if unit is None:
@@ -563,12 +572,13 @@ class TupleOf(DataType):
 
     def __init__(self, *members):
         if not members:
-            raise ValueError(u'Empty tuples are not allowed!')
+            raise BadValueError(u'Empty tuples are not allowed!')
         for subtype in members:
             if not isinstance(subtype, DataType):
-                raise ValueError(
+                raise BadValueError(
                     u'TupleOf only works with DataType objs as arguments!')
         self.members = members
+        self.default = tuple(el.default for el in members)
 
     def export_datatype(self):
         return [u'tuple', dict(members=[subtype.export_datatype() for subtype in self.members])]
@@ -576,19 +586,19 @@ class TupleOf(DataType):
     def __repr__(self):
         return u'TupleOf(%s)' % u', '.join([repr(st) for st in self.members])
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated value or raise"""
         # keep the ordering!
         try:
             if len(value) != len(self.members):
-                raise ValueError(
+                raise BadValueError(
                     u'Illegal number of Arguments! Need %d arguments.' %
                         (len(self.members)))
             # validate elements and return as list
-            return [sub.validate(elem)
+            return [sub(elem)
                     for sub, elem in zip(self.members, value)]
         except Exception as exc:
-            raise ValueError(u'Can not validate:', unicode(exc))
+            raise BadValueError(u'Can not validate:', unicode(exc))
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
@@ -602,7 +612,7 @@ class TupleOf(DataType):
         value, rem = Parser.parse(text)
         if rem:
             raise ProtocolError(u'trailing garbage: %r' % rem)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         return u'(%s)' % (', '.join([sub.format_value(elem)
@@ -614,7 +624,7 @@ class StructOf(DataType):
     def __init__(self, optional=None, **members):
         self.members = members
         if not members:
-            raise ValueError(u'Empty structs are not allowed!')
+            raise BadValueError(u'Empty structs are not allowed!')
         self.optional = list(optional or [])
         for name, subtype in list(members.items()):
             if not isinstance(subtype, DataType):
@@ -627,35 +637,37 @@ class StructOf(DataType):
             if name not in members:
                 raise ProgrammingError(
                     u'Only members of StructOf may be declared as optional!')
+        self.default = dict((k,el.default) for k, el in members.items())
 
+    def export_datatype(self):
+        res = [u'struct', dict(members=dict((n, s.export_datatype())
+                                       for n, s in list(self.members.items())))]
+        if self.optional:
+            res[1]['optional'] = self.optional
+        return res
 
     def __repr__(self):
         return u'StructOf(%s)' % u', '.join(
             [u'%s=%s' % (n, repr(st)) for n, st in list(self.members.items())])
 
-    def export_datatype(self):
-        return [u'struct', dict(members=dict((n, s.export_datatype())
-                                             for n, s in list(self.members.items())),
-                                optional=self.optional)]
-
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated value or raise"""
         try:
             # XXX: handle optional elements !!!
             if len(list(value.keys())) != len(list(self.members.keys())):
-                raise ValueError(
+                raise BadValueError(
                     u'Illegal number of Arguments! Need %d arguments.' %
                         len(list(self.members.keys())))
             # validate elements and return as dict
-            return dict((unicode(k), self.members[k].validate(v))
+            return dict((unicode(k), self.members[k](v))
                         for k, v in list(value.items()))
         except Exception as exc:
-            raise ValueError(u'Can not validate %s: %s' % (repr(value), unicode(exc)))
+            raise BadValueError(u'Can not validate %s: %s' % (repr(value), unicode(exc)))
 
     def export_value(self, value):
         """returns a python object fit for serialisation"""
         if len(list(value.keys())) != len(list(self.members.keys())):
-            raise ValueError(
+            raise BadValueError(
                 u'Illegal number of Arguments! Need %d arguments.' % len(
                     list(self.members.keys())))
         return dict((unicode(k), self.members[k].export_value(v))
@@ -664,7 +676,7 @@ class StructOf(DataType):
     def import_value(self, value):
         """returns a python object from serialisation"""
         if len(list(value.keys())) != len(list(self.members.keys())):
-            raise ValueError(
+            raise BadValueError(
                 u'Illegal number of Arguments! Need %d arguments.' % len(
                     list(self.members.keys())))
         return dict((unicode(k), self.members[k].import_value(v))
@@ -674,7 +686,7 @@ class StructOf(DataType):
         value, rem = Parser.parse(text)
         if rem:
             raise ProtocolError(u'trailing garbage: %r' % rem)
-        return self.validate(dict(value))
+        return self(dict(value))
 
     def format_value(self, value, unit=None):
         return u'{%s}' % (', '.join(['%s=%s' % (k, self.members[k].format_value(v)) for k, v in sorted(value.items())]))
@@ -682,36 +694,36 @@ class StructOf(DataType):
 
 class CommandType(DataType):
     IS_COMMAND = True
-    argtype = None
-    resulttype = None
+    argument = None
+    result = None
 
     def __init__(self, argument=None, result=None):
         if argument is not None:
             if not isinstance(argument, DataType):
-                raise ValueError(u'CommandType: Argument type must be a DataType!')
+                raise BadValueError(u'CommandType: Argument type must be a DataType!')
         if result is not None:
             if not isinstance(result, DataType):
-                raise ValueError(u'CommandType: Result type must be a DataType!')
-        self.argtype = argument
-        self.resulttype = result
+                raise BadValueError(u'CommandType: Result type must be a DataType!')
+        self.argument = argument
+        self.result = result
 
     def export_datatype(self):
-        info = {}
-        if self.argtype:
-            info['argument'] = self.argtype.export_datatype()
-        if self.resulttype:
-            info['result'] = self.argtype.export_datatype()
-        return [u'command', info]
+        a, r = self.argument, self.result
+        if a is not None:
+            a = a.export_datatype()
+        if r is not None:
+            r = r.export_datatype()
+        return [u'command', dict(argument=a, result=r)]
 
     def __repr__(self):
-        argstr = repr(self.argtype) if self.argtype else ''
-        if self.resulttype is None:
+        argstr = repr(self.argument) if self.argument else ''
+        if self.result is None:
             return u'CommandType(%s)' % argstr
-        return u'CommandType(%s)->%s' % (argstr, repr(self.resulttype))
+        return u'CommandType(%s)->%s' % (argstr, repr(self.result))
 
-    def validate(self, value):
+    def __call__(self, value):
         """return the validated argument value or raise"""
-        return self.argtype.validate(value)
+        return self.argument(value)
 
     def export_value(self, value):
         raise ProgrammingError(u'values of type command can not be transported!')
@@ -723,22 +735,105 @@ class CommandType(DataType):
         value, rem = Parser.parse(text)
         if rem:
             raise ProtocolError(u'trailing garbage: %r' % rem)
-        return self.validate(value)
+        return self(value)
 
     def format_value(self, value, unit=None):
         # actually I have no idea what to do here!
         raise NotImplementedError
 
 
+# internally used datatypes (i.e. only for programming the SEC-node
+class DataTypeType(DataType):
+    def __call__(self, value):
+        """check if given value (a python obj) is a valid datatype
+
+        returns the value or raises an appropriate exception"""
+        if isinstance(value, DataType):
+            return value
+        raise ProgrammingError(u'%r should be a DataType!' % value)
+
+    def export_value(self, value):
+        """if needed, reformat value for transport"""
+        return value.export_datatype()
+
+    def import_value(self, value):
+        """opposite of export_value, reformat from transport to internal repr
+
+        note: for importing from gui/configfile/commandline use :meth:`from_string`
+        instead.
+        """
+        raise NotImplementedError
+
+
+class ValueType(DataType):
+    """validates any python value"""
+    def __call__(self, value):
+        """check if given value (a python obj) is valid for this datatype
+
+        returns the value or raises an appropriate exception"""
+        return value
+
+    def export_value(self, value):
+        """if needed, reformat value for transport"""
+        return value
+
+    def import_value(self, value):
+        """opposite of export_value, reformat from transport to internal repr
+
+        note: for importing from gui/configfile/commandline use :meth:`from_string`
+        instead.
+        """
+        raise NotImplementedError
+
+class NoneOr(DataType):
+    """validates a None or smth. else"""
+    default = None
+
+    def __init__(self, other):
+        self.other = other
+
+    def __call__(self, value):
+        return None if value is None else self.other(value)
+
+    def export_value(self, value):
+        if value is None:
+            return None
+        return self.other.export_value(value)
+
+
+class OrType(DataType):
+    def __init__(self, *types):
+        self.types = types
+        self.default = self.types[0].default
+
+    def __call__(self, value):
+        for t in self.types:
+            try:
+                return t(value)
+            except Exception:
+                pass
+        raise BadValueError(u"Invalid Value, must conform to one of %s" % (', '.join((str(t) for t in self.types))))
+
+
+Int8   = IntRange(-(1 << 7),  (1 << 7) - 1)
+Int16  = IntRange(-(1 << 15), (1 << 15) - 1)
+Int32  = IntRange(-(1 << 31), (1 << 31) - 1)
+Int64  = IntRange(-(1 << 63), (1 << 63) - 1)
+UInt8  = IntRange(0, (1 << 8) - 1)
+UInt16 = IntRange(0, (1 << 16) - 1)
+UInt32 = IntRange(0, (1 << 32) - 1)
+UInt64 = IntRange(0, (1 << 64) - 1)
+
+
 # Goodie: Convenience Datatypes for Programming
 class LimitsType(StructOf):
     def __init__(self, _min=None, _max=None):
         StructOf.__init__(self, min=FloatRange(_min,_max), max=FloatRange(_min, _max))
 
-    def validate(self, value):
-        limits = StructOf.validate(self, value)
-        if limits.max < limits.min:
-            raise ValueError(u'Maximum Value %s must be greater than minimum value %s!' % (limits['max'], limits['min']))
+    def __call__(self, value):
+        limits = StructOf.__call__(self, value)
+        if limits['max'] < limits['min']:
+            raise BadValueError(u'Maximum Value %s must be greater than minimum value %s!' % (limits['max'], limits['min']))
         return limits
 
 
@@ -747,6 +842,7 @@ class Status(TupleOf):
     def __init__(self, enum):
         TupleOf.__init__(self, EnumType(enum), StringType())
         self.enum = enum
+
     def __getattr__(self, key):
         enum = TupleOf.__getattr__(self, 'enum')
         if hasattr(enum, key):
@@ -780,14 +876,14 @@ def get_datatype(json):
     if json is None:
         return json
     if not isinstance(json, list):
-        raise ValueError(
+        raise BadValueError(
             u'Can not interpret datatype %r, it should be a list!' % json)
     if len(json) != 2:
-        raise ValueError(u'Can not interpret datatype %r, it should be a list of 2 elements!' % json)
+        raise BadValueError(u'Can not interpret datatype %r, it should be a list of 2 elements!' % json)
     base, args = json
     if base in DATATYPES:
         try:
             return DATATYPES[base](**args)
         except (TypeError, AttributeError):
-            raise ValueError(u'Invalid datatype descriptor in %r' % json)
-    raise ValueError(u'can not convert %r to datatype: unknown descriptor!' % json)
+            raise BadValueError(u'Invalid datatype descriptor in %r' % json)
+    raise BadValueError(u'can not convert %r to datatype: unknown descriptor!' % json)

secop/errors.py

@@ -83,7 +83,7 @@ class ReadOnlyError(SECoPError):
     pass
 
 
-class BadValueError(SECoPError):
+class BadValueError(ValueError, SECoPError):
     pass
 
 

secop/gui/modulectrl.py

@@ -131,8 +131,8 @@ class CommandButton(QPushButton):
         super(CommandButton, self).__init__(parent)
 
         self._cmdname = cmdname
-        self._argintype = cmdinfo['datatype'].argtype   # single datatype
-        self.resulttype = cmdinfo['datatype'].resulttype
+        self._argintype = cmdinfo['datatype'].argument   # single datatype
+        self.result = cmdinfo['datatype'].result
         self._cb = cb  # callback function for exection
 
         self.setText(cmdname)

secop/gui/params/__init__.py

@@ -137,7 +137,7 @@ class GenericCmdWidget(ParameterWidget):
         loadUi(self, 'cmdbuttons.ui')
 
         self.cmdLineEdit.setText('')
-        self.cmdLineEdit.setEnabled(self.datatype.argtype is not None)
+        self.cmdLineEdit.setEnabled(self.datatype.argument is not None)
         self.cmdLineEdit.returnPressed.connect(
             self.on_cmdPushButton_clicked)
 

secop/gui/valuewidgets.py

@@ -40,7 +40,7 @@ class StringWidget(QLineEdit):
 
     def get_value(self):
         res = self.text()
-        return self.datatype.validate(res)
+        return self.datatype(res)
 
     def set_value(self, value):
         self.setText(value)
@@ -136,7 +136,7 @@ class TupleWidget(QFrame):
         self.update()
 
     def get_value(self):
-        return [v.validate(w.get_value()) for w, v in zip(self.subwidgets, self.datatypes)]
+        return [v(w.get_value()) for w, v in zip(self.subwidgets, self.datatypes)]
 
     def set_value(self, value):
         for w, _ in zip(self.subwidgets, value):
@@ -166,14 +166,14 @@ class StructWidget(QGroupBox):
         res = {}
         for name, entry in self.subwidgets.items():
             w, dt = entry
-            res[name] = dt.validate(w.get_value())
+            res[name] = dt(w.get_value())
         return res
 
     def set_value(self, value):
         for k, v in value.items():
             entry = self.subwidgets[k]
             w, dt = entry
-            w.set_value(dt.validate(v))
+            w.set_value(dt(v))
 
 
 class ArrayWidget(QGroupBox):
@@ -190,7 +190,7 @@ class ArrayWidget(QGroupBox):
         self.setLayout(self.layout)
 
     def get_value(self):
-        return [self.datatype.validate(w.get_value()) for w in self.subwidgets]
+        return [self.datatype(w.get_value()) for w in self.subwidgets]
 
     def set_value(self, values):
         for w, v in zip(self.subwidgets, values):

secop/lib/enum.py

@@ -289,7 +289,7 @@ class Enum(dict):
         return self[key]
 
     def __setattr__(self, key, value):
-        if self.name:
+        if self.name and key != 'name':
             raise TypeError('Enum %r can not be changed!' % self.name)
         super(Enum, self).__setattr__(key, value)
 

secop/lib/metaclass.py

@@ -0,0 +1,44 @@
+#  -*- coding: utf-8 -*-
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
+# *****************************************************************************
+"""Define metaclass helper"""
+
+from __future__ import division, print_function
+
+try:
+    # pylint: disable=unused-import
+    from six import add_metaclass # for py2/3 compat
+except ImportError:
+    # copied from six v1.10.0
+    def add_metaclass(metaclass):
+        """Class decorator for creating a class with a metaclass."""
+        def wrapper(cls):
+            orig_vars = cls.__dict__.copy()
+            slots = orig_vars.get('__slots__')
+            if slots is not None:
+                if isinstance(slots, str):
+                    slots = [slots]
+                for slots_var in slots:
+                    orig_vars.pop(slots_var)
+            orig_vars.pop('__dict__', None)
+            orig_vars.pop('__weakref__', None)
+            return metaclass(cls.__name__, cls.__bases__, orig_vars)
+        return wrapper

secop/metaclass.py

@@ -26,38 +26,18 @@ from __future__ import division, print_function
 import time
 from collections import OrderedDict
 
-from secop.datatypes import EnumType
 from secop.errors import ProgrammingError
 from secop.params import Command, Override, Parameter
-
-try:
-    # pylint: disable=unused-import
-    from six import add_metaclass # for py2/3 compat
-except ImportError:
-    # copied from six v1.10.0
-    def add_metaclass(metaclass):
-        """Class decorator for creating a class with a metaclass."""
-        def wrapper(cls):
-            orig_vars = cls.__dict__.copy()
-            slots = orig_vars.get('__slots__')
-            if slots is not None:
-                if isinstance(slots, str):
-                    slots = [slots]
-                for slots_var in slots:
-                    orig_vars.pop(slots_var)
-            orig_vars.pop('__dict__', None)
-            orig_vars.pop('__weakref__', None)
-            return metaclass(cls.__name__, cls.__bases__, orig_vars)
-        return wrapper
+from secop.datatypes import EnumType
+from secop.properties import PropertyMeta
 
 
-
-EVENT_ONLY_ON_CHANGED_VALUES = True
+EVENT_ONLY_ON_CHANGED_VALUES = False
 
 
 # warning: MAGIC!
 
-class ModuleMeta(type):
+class ModuleMeta(PropertyMeta):
     """Metaclass
 
     joining the class's properties, parameters and commands dicts with
@@ -75,12 +55,7 @@ class ModuleMeta(type):
         if '__constructed__' in attrs:
             return newtype
 
-        # merge properties from all sub-classes
-        newentry = {}
-        for base in reversed(bases):
-            newentry.update(getattr(base, "properties", {}))
-        newentry.update(attrs.get("properties", {}))
-        newtype.properties = newentry
+        newtype = PropertyMeta.__join_properties__(newtype, name, bases, attrs)
 
         # merge accessibles from all sub-classes, treat overrides
         # for now, allow to use also the old syntax (parameters/commands dict)
@@ -118,7 +93,7 @@ class ModuleMeta(type):
 
         # Correct naming of EnumTypes
         for k, v in accessibles.items():
-            if isinstance(v.datatype, EnumType) and not v.datatype._enum.name:
+            if isinstance(v, Parameter) and isinstance(v.datatype, EnumType):
                 v.datatype._enum.name = k
 
         # newtype.accessibles will be used in 2 places only:
@@ -172,18 +147,12 @@ class ModuleMeta(type):
                 def wrapped_wfunc(self, value, pname=pname, wfunc=wfunc):
                     self.log.debug("wfunc(%s): set %r" % (pname, value))
                     pobj = self.accessibles[pname]
-                    value = pobj.datatype.validate(value)
+                    value = pobj.datatype(value)
                     if wfunc:
                         self.log.debug('calling %r(%r)' % (wfunc, value))
-                        try:
-                            returned_value = wfunc(self, value)
-                        except Exception as e:
-                            self.DISPATCHER.announce_update_error(self, pname, pobj, e)
-                            raise e
+                        returned_value = wfunc(self, value)
                         if returned_value is not None:
                             value = returned_value
-                    # XXX: use setattr or direct manipulation
-                    # of self.accessibles[pname]?
                     setattr(self, pname, value)
                     return value
 
@@ -198,7 +167,7 @@ class ModuleMeta(type):
 
             def setter(self, value, pname=pname):
                 pobj = self.accessibles[pname]
-                value = pobj.datatype.validate(value)
+                value = pobj.datatype(value)
                 pobj.timestamp = time.time()
                 if (not EVENT_ONLY_ON_CHANGED_VALUES) or (value != pobj.value):
                     pobj.value = value
@@ -218,3 +187,18 @@ class ModuleMeta(type):
 
         attrs['__constructed__'] = True
         return newtype
+
+    @property
+    def configurables(cls):
+        # note: this ends up as an property of the Module class (not on the instance)!
+
+        # list of tuples (cfg-file key, Property/Parameter)
+        res = []
+        # collect info about properties
+        for pn, pv in cls.properties.items():
+            res.append((u'%s' % pn, pv,))
+        # collect info about parameters and their properties
+        for param, pobj in cls.accessibles.items():
+            for pn, pv in pobj.__class__.properties.items():
+                res.append((u'%s.%s' % (param,pn), pv))
+        return res

secop/modules.py

@@ -27,14 +27,17 @@ import sys
 import time
 from collections import OrderedDict
 
-from secop.datatypes import EnumType, FloatRange, \
-    StringType, TupleOf, get_datatype
+from secop.datatypes import EnumType, FloatRange, BoolType, IntRange, \
+    StringType, TupleOf, get_datatype, ArrayOf
 from secop.errors import ConfigError, ProgrammingError
 from secop.lib import formatException, \
     formatExtendedStack, mkthread, unset_value
 from secop.lib.enum import Enum
-from secop.metaclass import ModuleMeta, add_metaclass
-from secop.params import PREDEFINED_ACCESSIBLES, Command, Override, Parameter
+from secop.lib.metaclass import add_metaclass
+from secop.metaclass import ModuleMeta
+from secop.params import PREDEFINED_ACCESSIBLES, Command, Override, Parameter, Parameters, Commands
+from secop.properties import HasProperties, Property
+
 
 # XXX: connect with 'protocol'-Modules.
 # Idea: every Module defined herein is also a 'protocol'-Module,
@@ -43,7 +46,7 @@ from secop.params import PREDEFINED_ACCESSIBLES, Command, Override, Parameter
 
 
 @add_metaclass(ModuleMeta)
-class Module(object):
+class Module(HasProperties):
     """Basic Module
 
     ALL secop Modules derive from this
@@ -61,16 +64,17 @@ class Module(object):
     # static properties, definitions in derived classes should overwrite earlier ones.
     # note: properties don't change after startup and are usually filled
     #       with data from a cfg file...
-    # note: so far all properties are STRINGS
-    # note: only the properties defined here are allowed to be set in the cfg file
+    # note: only the properties predefined here are allowed to be set in the cfg file
+    # note: the names map to a [datatype, value] list, value comes from the cfg file,
+    #       datatype is fixed!
     properties = {
-        'export': True, # should be exported remotely?
-        'group': None,  # some Modules may be grouped together
-        'description': "Short description of this Module class and its functionality.",
-
-        'meaning': None,  # XXX: ???
-        'priority': None,  # XXX: ???
-        'visibility': None,  # XXX: ????
+        'export':          Property(BoolType(), default=True, export=False),
+        'group':           Property(StringType(), default='', extname='group'),
+        'description':     Property(StringType(), extname='description', mandatory=True),
+        'meaning':         Property(TupleOf(StringType(),IntRange(0,50)), default=('',0), extname='meaning'),
+        'visibility':      Property(EnumType('visibility', user=1, advanced=2, expert=3), default=1, extname='visibility'),
+        'implementation':  Property(StringType(), extname='implementation'),
+        'interface_class': Property(ArrayOf(StringType()), extname='interface_class'),
         # what else?
     }
 
@@ -89,32 +93,26 @@ class Module(object):
 
         # handle module properties
         # 1) make local copies of properties
-        # XXX: self.properties = self.properties.copy() ???
-        props = {}
-        for k, v in list(self.properties.items()):
-            props[k] = v
-        self.properties = props
+        super(Module, self).__init__()
 
         # 2) check and apply properties specified in cfgdict
         #    specified as '.<propertyname> = <propertyvalue>'
         for k, v in list(cfgdict.items()):  # keep list() as dict may change during iter
             if k[0] == '.':
-                if k[1:] in self.properties:
-                    self.properties[k[1:]] = cfgdict.pop(k)
-                elif k[1] == '_':
-                    self.properties[k[1:]] = cfgdict.pop(k)
+                if k[1:] in self.__class__.properties:
+                    self.setProperty(k[1:], cfgdict.pop(k))
                 else:
                     raise ConfigError('Module %r has no property %r' %
                                       (self.name, k[1:]))
-        # 3) remove unset (default) module properties
-        for k, v in list(self.properties.items()):  # keep list() as dict may change during iter
-            if v is None:
-                del self.properties[k]
 
         # 4) set automatic properties
         mycls = self.__class__
         myclassname = '%s.%s' % (mycls.__module__, mycls.__name__)
-        self.properties['_implementation'] = myclassname
+        self.properties['implementation'] = myclassname
+        # list of all 'secop' modules
+        self.properties['interface_class'] = [
+            b.__name__ for b in mycls.__mro__ if b.__module__.startswith('secop.modules')]
+        # list of only the 'highest' secop module class
         self.properties['interface_class'] = [[
             b.__name__ for b in mycls.__mro__ if b.__module__.startswith('secop.modules')][0]]
 
@@ -136,20 +134,20 @@ class Module(object):
                     predefined_obj = PREDEFINED_ACCESSIBLES.get(aname, None)
                     if predefined_obj:
                         if isinstance(aobj, predefined_obj):
-                            aobj.export = aname
+                            aobj.setProperty('export', aname)
                         else:
                             raise ProgrammingError("can not use '%s' as name of a %s" %
                                   (aname, aobj.__class__.__name__))
                     else: # create custom parameter
-                        aobj.export = '_' + aname
+                        aobj.setProperty('export', '_' + aname)
                 accessiblename2attr[aobj.export] = aname
             accessibles[aname] = aobj
         # do not re-use self.accessibles as this is the same for all instances
         self.accessibles = accessibles
         self.accessiblename2attr = accessiblename2attr
         # provide properties to 'filter' out the parameters/commands
-        self.parameters = dict((k,v) for k,v in accessibles.items() if isinstance(v, Parameter))
-        self.commands = dict((k,v) for k,v in accessibles.items() if isinstance(v, Command))
+        self.parameters = Parameters((k,v) for k,v in accessibles.items() if isinstance(v, Parameter))
+        self.commands = Commands((k,v) for k,v in accessibles.items() if isinstance(v, Command))
 
         # 2) check and apply parameter_properties
         #    specified as '<paramname>.<propertyname> = <propertyvalue>'
@@ -160,9 +158,9 @@ class Module(object):
                 # paramobj might also be a command (not sure if this is needed)
                 if paramobj:
                     if propname == 'datatype':
-                        paramobj.datatype = get_datatype(cfgdict.pop(k))
-                    elif hasattr(paramobj, propname):
-                        setattr(paramobj, propname, cfgdict.pop(k))
+                        paramobj.setProperty('datatype', get_datatype(cfgdict.pop(k)))
+                    elif propname in paramobj.__class__.properties:
+                        paramobj.setProperty(propname, cfgdict.pop(k))
                     else:
                         raise ConfigError('Module %s: Parameter %r has no property %r!' %
                                           (self.name, paramname, propname))
@@ -194,8 +192,8 @@ class Module(object):
             # apply datatype, complain if type does not fit
             datatype = self.parameters[k].datatype
             try:
-                v = datatype.validate(v)
-                self.parameters[k].default = v
+                v = datatype(v)
+                self.parameters[k].value = v
             except (ValueError, TypeError):
                 self.log.exception(formatExtendedStack())
                 raise
@@ -212,6 +210,18 @@ class Module(object):
             if '$' in v.unit:
                 v.unit = v.unit.replace('$', self.parameters['value'].unit)
 
+        # 6) check complete configuration of * properties
+        self.checkProperties()
+        for p in self.parameters.values():
+            p.checkProperties()
+
+    # helper cfg-editor
+    def __iter__(self):
+        return self.accessibles.__iter__()
+
+    def __getitem__(self, item):
+        return self.accessibles.__getitem__(item)
+
     def isBusy(self):
         '''helper function for treating substates of BUSY correctly'''
         # defined even for non drivable (used for dynamic polling)
@@ -247,10 +257,9 @@ class Readable(Module):
     Status = Enum('Status',
                   IDLE = 100,
                   WARN = 200,
-                  UNSTABLE = 250,
+                  UNSTABLE = 270,
                   ERROR = 400,
-                  DISABLED = 500,
-                  UNKNOWN = 0,
+                  DISABLED = 0,
                  )
     parameters = {
         'value':        Parameter('current value of the Module', readonly=True,

secop/params.py

@@ -23,10 +23,17 @@
 
 from __future__ import division, print_function
 
-from secop.datatypes import CommandType, DataType
-from secop.errors import ProgrammingError
-from secop.lib import unset_value
+from collections import OrderedDict
 
+from secop.datatypes import CommandType, DataType, StringType, BoolType, EnumType, DataTypeType, ValueType, OrType
+from secop.errors import ProgrammingError
+from secop.properties import HasProperties, Property
+
+try:
+    unicode
+except NameError:
+    # pylint: disable=redefined-builtin
+    unicode = str  # py3 compat
 
 class CountedObj(object):
     ctr = [0]
@@ -36,41 +43,34 @@ class CountedObj(object):
         self.ctr = cl[0]
 
 
-class Accessible(CountedObj):
-    '''abstract base class for Parameter and Command'''
+class Accessible(HasProperties, CountedObj):
+    '''base class for Parameter and Command'''
+
+    properties = {}
+
+    def __init__(self, **kwds):
+        super(Accessible, self).__init__()
+        self.properties.update(kwds)
 
     def __repr__(self):
-        return '%s_%d(%s)' % (self.__class__.__name__, self.ctr, ', '.join(
-            ['%s=%r' % (k, v) for k, v in sorted(self.__dict__.items())]))
+        return u'%s_%d(%s)' % (self.__class__.__name__, self.ctr, ',\n\t'.join(
+            [u'%s=%r' % (k, self.properties.get(k, v.default)) for k, v in sorted(self.__class__.properties.items())]))
 
     def copy(self):
-        '''return a copy of ourselfs'''
-        props = self.__dict__.copy()
+        # return a copy of ourselfs
+        props = dict(self.properties, ctr=self.ctr)
         return type(self)(**props)
 
-    def exported_properties(self):
-        res = dict(datatype=self.datatype.export_datatype())
-        for key, value in self.__dict__.items():
-            if key in self.valid_properties:
-                res[self.valid_properties[key]] = value
-        return res
-
-    @classmethod
-    def add_property(cls, *args, **kwds):
-        '''add custom properties
-
-        args: custom properties, exported with leading underscore
-        kwds: special cases, where exported name differs from internal
-
-        intention: to be called in secop_<facility>/__init__.py for
-        facility specific properties
-        '''
-        for name in args:
-            kwds[name] = '_' + name
-        for name, external in kwds.items():
-            if name in cls.valid_properties and name != cls.valid_properties[name]:
-                raise ProgrammingError('can not overrride property name %s' % name)
-            cls.valid_properties[name] = external
+    def for_export(self):
+        # used for serialisation only
+        # some specials:
+        # - datatype needs a special serialisation
+        # - readonly is mandatory for serialisation, but not for declaration in classes
+        r =  self.exportProperties()
+        if isinstance(self, Parameter):
+            if 'readonly' not in r:
+                r['readonly'] = self.__class__.properties['readonly'].default
+        return r
 
 
 class Parameter(Accessible):
@@ -90,80 +90,69 @@ class Parameter(Accessible):
     note: Drivable (and derived classes) poll with 10 fold frequency if module is busy....
     """
 
-    # unit and datatype are not listed (handled separately)
-    valid_properties = dict()
-    for prop in ('description', 'readonly', 'group', 'visibility', 'constant'):
-        valid_properties[prop] = prop
+    properties = {
+        u'description': Property(StringType(), extname=u'description', mandatory=True),
+        u'datatype':    Property(DataTypeType(), extname=u'datatype', mandatory=True),
+        u'unit':        Property(StringType(), extname=u'unit', default=''),  # goodie, should be on the datatype!
+        u'readonly':    Property(BoolType(), extname=u'readonly', default=True),
+        u'group':       Property(StringType(), extname=u'group', default=''),
+        u'visibility':  Property(EnumType(u'visibility', user=1, advanced=2, expert=3),
+                                 extname=u'visibility', default=1),
+        u'constant':    Property(ValueType(), extname=u'constant', default=None),
+        u'default':     Property(ValueType(), export=False, default=None, mandatory=False),
+        u'export':      Property(OrType(BoolType(), StringType()), export=False, default=True),
+        u'poll':        Property(ValueType(), export=False, default=True),  # check default value!
+        u'optional':    Property(BoolType(), export=False, default=False),
+    }
+
+    value = None
+    timestamp = None
+    def __init__(self, description, datatype, ctr=None, **kwds):
+
+        if ctr is not None:
+            self.ctr = ctr
 
-    def __init__(self,
-                 description,
-                 datatype=None,
-                 default=unset_value,
-                 readonly=True,
-                 export=True,
-                 poll=False,
-                 unit=u'',
-                 constant=None,
-                 value=None, # swallow
-                 timestamp=None, # swallow
-                 optional=False,
-                 ctr=None,
-                 **kwds):
-        super(Parameter, self).__init__()
         if not isinstance(datatype, DataType):
             if issubclass(datatype, DataType):
                 # goodie: make an instance from a class (forgotten ()???)
                 datatype = datatype()
             else:
                 raise ValueError(
-                    'datatype MUST be derived from class DataType!')
-        self.description = description
-        self.datatype = datatype
-        self.default = default
-        self.readonly = readonly if constant is None else True
-        self.export = export
-        self.optional = optional
-        self.constant = constant
+                    u'datatype MUST be derived from class DataType!')
+
+        kwds[u'description'] = description
+        kwds[u'datatype'] = datatype
+        super(Parameter, self).__init__(**kwds)
 
         # note: auto-converts True/False to 1/0 which yield the expected
         # behaviour...
-        self.poll = int(poll)
-        for key in kwds:
-            if key not in self.valid_properties:
-                raise ProgrammingError('%s is not a valid parameter property' % key)
-        if constant is not None:
+        self.properties[u'poll'] = int(self.poll)
+
+        if self.constant is not None:
+            self.properties[u'readonly'] = True
             # The value of the `constant` property should be the
             # serialised version of the constant, or unset
-            constant = self.datatype.validate(constant)
-            self.constant = self.datatype.export_value(constant)
-        # helper. unit should be set on the datatype, not on the parameter!
-        if unit:
-            self.datatype.unit = unit
-        self.__dict__.update(kwds)
+            constant = self.datatype(kwds[u'constant'])
+            self.properties[u'constant'] = self.datatype.export_value(constant)
+
+        # helper: unit should be set on the datatype, not on the parameter!
+        if self.unit:
+            self.datatype.unit = self.unit
+            self.properties[u'unit'] = ''
+
         # internal caching: value and timestamp of last change...
-        self.value = default
+        self.value = self.default
         self.timestamp = 0
-        if ctr is not None:
-            self.ctr = ctr
-
-    def copy(self):
-        '''return a copy of ourselfs'''
-        result = Accessible.copy(self)
-        result.datatype = result.datatype.copy()
-        return result
-
-    def for_export(self):
-        # used for serialisation only
-        res = self.exported_properties()
-        return res
 
     def export_value(self):
         return self.datatype.export_value(self.value)
 
+    # helpers...
     def _get_unit_(self):
         return self.datatype.unit
 
     def _set_unit_(self, unit):
+        print(u'DeprecationWarning: setting unit on the parameter is going to be removed')
         self.datatype.unit = unit
 
     unit = property(_get_unit_, _set_unit_)
@@ -171,6 +160,43 @@ class Parameter(Accessible):
     del _set_unit_
 
 
+class UnusedClass(object):
+    # do not derive anything from this!
+    pass
+
+class Parameters(OrderedDict):
+    """class storage for Parameters"""
+    def __init__(self, *args, **kwds):
+        self.exported = {}  # only for lookups!
+        super(Parameters, self).__init__(*args, **kwds)
+
+    def __setitem__(self, key, value):
+        if value.export:
+            if isinstance(value, PREDEFINED_ACCESSIBLES.get(key, UnusedClass)):
+                value.properties[u'export'] = key
+            else:
+                value.properties[u'export'] = '_' + key
+                self.exported[value.export] = key
+        super(Parameters, self).__setitem__(key, value)
+
+    def __getitem__(self, item):
+        if item in self.exported:
+            return self[self.exported[item]]
+        return super(Parameters, self).__getitem__(item)
+
+
+class ParamValue(object):
+    __slots__ = ['value', 'timestamp']
+    def __init__(self, value, timestamp=0):
+        self.value = value
+        self.timestamp = timestamp
+
+
+class Commands(Parameters):
+    """class storage for Commands"""
+    pass
+
+
 class Override(CountedObj):
     """Stores the overrides to be applied to a Parameter
 
@@ -183,37 +209,31 @@ class Override(CountedObj):
         self.reorder = reorder
         # allow to override description without keyword
         if description:
-            self.kwds['description'] = description
+            self.kwds[u'description'] = description
         # for now, do not use the Override ctr
         # self.kwds['ctr'] = self.ctr
 
     def __repr__(self):
-        return '%s_%d(%s)' % (self.__class__.__name__, self.ctr, ', '.join(
-            ['%s=%r' % (k, v) for k, v in sorted(self.kwds.items())]))
+        return u'%s_%d(%s)' % (self.__class__.__name__, self.ctr, ', '.join(
+            [u'%s=%r' % (k, v) for k, v in sorted(self.kwds.items())]))
 
     def apply(self, obj):
         if isinstance(obj, Accessible):
-            props = obj.__dict__.copy()
-            for key in self.kwds:
-                if key == 'unit':
-                    # XXX: HACK!
-                    continue
-                if key not in props and key not in type(obj).valid_properties:
-                    raise ProgrammingError( "%s is not a valid %s property" %
-                                           (key, type(obj).__name__))
+            props = obj.properties.copy()
             if isinstance(obj, Parameter):
                 if u'constant' in self.kwds:
-                    constant = obj.datatype.validate(self.kwds.pop(u'constant'))
+                    constant = obj.datatype(self.kwds.pop(u'constant'))
                     self.kwds[u'constant'] = obj.datatype.export_value(constant)
                     self.kwds[u'readonly'] = True
             props.update(self.kwds)
 
             if self.reorder:
-                props['ctr'] = self.ctr
+                #props['ctr'] = self.ctr
+                return type(obj)(ctr=self.ctr, **props)
             return type(obj)(**props)
         else:
             raise ProgrammingError(
-                "Overrides can only be applied to Accessibles, %r is none!" %
+                u"Overrides can only be applied to Accessibles, %r is none!" %
                 obj)
 
 
@@ -221,39 +241,45 @@ class Command(Accessible):
     """storage for Commands settings (description + call signature...)
     """
     # datatype is not listed (handled separately)
-    valid_properties = dict()
-    for prop in ('description', 'group', 'visibility'):
-        valid_properties[prop] = prop
+    properties = {
+        u'description': Property(StringType(), extname=u'description', export=True, mandatory=True),
+        u'group':       Property(StringType(), extname=u'group', export=True, default=''),
+        u'visibility':  Property(EnumType(u'visibility', user=1, advanced=2, expert=3),
+                                 extname=u'visibility', export=True, default=1),
+        u'export':      Property(OrType(BoolType(), StringType()), export=False, default=True),
+        u'optional':    Property(BoolType(), export=False, default=False, settable=False),
+        u'datatype': Property(DataTypeType(), extname=u'datatype', mandatory=True),
+    }
 
-    def __init__(self,
-                 description,
-                 argument=None,
-                 result=None,
-                 export=True,
-                 optional=False,
-                 datatype=None, # swallow datatype argument on copy
-                 ctr=None,
-                 **kwds):
-        super(Command, self).__init__()
-        # descriptive text for humans
-        self.description = description
-        # datatypes for argument/result
-        self.argument = argument
-        self.result = result
-        self.datatype = CommandType(argument, result)
-        # whether implementation is optional
-        self.optional = optional
-        self.export = export
-        for key in kwds:
-            if key not in self.valid_properties:
-                raise ProgrammingError('%s is not a valid command property' % key)
-        self.__dict__.update(kwds)
+    def __init__(self, description, argument=None, result=None, ctr=None, **kwds):
+        kwds[u'description'] = description
+        kwds[u'datatype'] = CommandType(argument, result)
+        super(Command, self).__init__(**kwds)
         if ctr is not None:
             self.ctr = ctr
 
+    @property
+    def argument(self):
+        return self.datatype.argument
+
+    @property
+    def result(self):
+        return self.datatype.result
+
     def for_export(self):
         # used for serialisation only
-        return self.exported_properties()
+        # some specials:
+        # - datatype needs a special serialisation
+        # - readonly is mandatory for serialisation, but not for declaration in classes
+        r =  self.exportProperties()
+#        if isinstance(self, Parameter):
+#            if u'readonly' not in r:
+#                r[u'readonly'] = self.__class__.properties[u'readonly'].default
+#        if u'datatype' in r:
+#            _d = r[u'datatype']
+#            print(formatExtendedStack())  # for debug
+        return r
+
 
 # list of predefined accessibles with their type
 PREDEFINED_ACCESSIBLES = dict(

secop/properties.py

@@ -0,0 +1,158 @@
+#  -*- coding: utf-8 -*-
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>
+#
+# *****************************************************************************
+"""Define validated data types."""
+
+from __future__ import division, print_function
+
+from collections import OrderedDict
+
+from secop.datatypes import ValueType, DataType
+from secop.errors import ProgrammingError, ConfigError
+from secop.lib.metaclass import add_metaclass
+
+
+
+# storage for 'properties of a property'
+class Property(object):
+    '''base class holding info about a property
+
+    properties are only sent to the ECS if export is True, or an extname is set
+    if mandatory is True, they MUST have avalue in the cfg file assigned to them.
+    otherwise, this is optional in which case the default value is applied.
+    All values MUST pass the datatype.
+    '''
+    # note: this is inteded to be used on base classes.
+    #       the VALUES of the properties are on the instances!
+    def __init__(self, datatype, default=None, extname='', export=False, mandatory=False, settable=True):
+        if not callable(datatype):
+            raise ValueError(u'datatype MUST be a valid DataType or a basic_validator')
+        self.default = datatype.default if default is None else datatype(default)
+        self.datatype = datatype
+        self.extname = unicode(extname)
+        self.export = export or bool(extname)
+        self.mandatory = mandatory or (default is None and not isinstance(datatype, ValueType))
+        self.settable = settable or mandatory  # settable means settable from the cfg file
+
+    def __repr__(self):
+        return u'Property(%s, default=%r, extname=%r, export=%r, mandatory=%r)' % (
+            self.datatype, self.default, self.extname, self.export, self.mandatory)
+
+
+class Properties(OrderedDict):
+    """a collection of `Property` objects
+
+    checks values upon assignment.
+    You can either assign a Property object, or a value
+    (which must pass the validator of the already existing Property)
+    """
+    def __setitem__(self, key, value):
+        if not isinstance(value, Property):
+            raise ProgrammingError(u'setting property %r on classes is not supported!' % key)
+        # make sure, extname is valid if export is True
+        if not value.extname and value.export:
+            value.extname = u'_%s' % key  # generate custom kex
+        elif value.extname and not value.export:
+            value.export = True
+        OrderedDict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        raise ProgrammingError(u'deleting Properties is not supported!')
+
+
+class PropertyMeta(type):
+    """Metaclass for HasProperties
+
+    joining the class's properties with those of base classes.
+    """
+
+    def __new__(mcs, name, bases, attrs):
+        newtype = type.__new__(mcs, name, bases, attrs)
+        if '__constructed__' in attrs:
+            return newtype
+
+        newtype = mcs.__join_properties__(newtype, name, bases, attrs)
+
+        attrs['__constructed__'] = True
+        return newtype
+
+    @classmethod
+    def __join_properties__(mcs, newtype, name, bases, attrs):
+        # merge properties from all sub-classes
+        properties = Properties()
+        for base in reversed(bases):
+            properties.update(getattr(base, "properties", {}))
+        # update with properties from new class
+        properties.update(attrs.get('properties', {}))
+        newtype.properties = properties
+
+        # generate getters
+        for k in properties:
+            def getter(self, pname=k):
+                val = self.__class__.properties[pname].default
+                return self.properties.get(pname, val)
+            if k in attrs:
+                if not isinstance(attrs[k], property):
+                    raise ProgrammingError(u'Name collision with property %r' % k)
+            setattr(newtype, k, property(getter))
+        return newtype
+
+
+@add_metaclass(PropertyMeta)
+class HasProperties(object):
+    properties = {}
+
+    def __init__(self, *args):
+        super(HasProperties, self).__init__()
+        # store property values in the instance, keep descriptors on the class
+        self.properties = {}
+        # pre-init with properties default value (if any)
+        for pn, po in self.__class__.properties.items():
+            if not po.mandatory:
+                self.properties[pn] = po.default
+
+    def checkProperties(self):
+        for pn, po in self.__class__.properties.items():
+            if po.export and po.mandatory:
+                if pn not in self.properties:
+                    name = getattr(self, 'name', repr(self))
+                    raise ConfigError('Property %r of %r needs a value of type %r!' % (pn, name, po.datatype))
+                # apply validator (which may complain further)
+                self.properties[pn] = po.datatype(self.properties[pn])
+        if 'min' in self.properties and 'max' in self.properties:
+            if self.min > self.max:
+                raise ConfigError('min and max of %r need to fulfil min <= max! (is %r>%r)' % (self, self.min, self.max))
+
+    def exportProperties(self):
+        # export properties which have
+        # export=True and
+        # mandatory=True or non_default=True
+        res = {}
+        for pn, po in self.__class__.properties.items():
+            val = self.properties.get(pn, None)
+            if po.export and (po.mandatory or val != po.default):
+                if isinstance(po.datatype, DataType):
+                    val = po.datatype.export_value(val)
+                res[po.extname] = val
+        return res
+
+    def setProperty(self, key, value):
+        self.properties[key] = self.__class__.properties[key].datatype(value)

secop/protocol/dispatcher.py

@@ -181,7 +181,7 @@ class Dispatcher(object):
                            (modulename, res))
             return res
         self.log.debug(u'-> module is not to be exported!')
-        return {}
+        return []
 
     def get_descriptive_data(self):
         """returns a python object which upon serialisation results in the descriptive data"""
@@ -194,36 +194,43 @@ class Dispatcher(object):
                 continue
             # some of these need rework !
             mod_desc = {u'accessibles': self.export_accessibles(modulename)}
-            for propname, prop in list(module.properties.items()):
-                if propname == 'export':
-                    continue
-                mod_desc[propname] = prop
+            mod_desc.update(module.exportProperties())
+            mod_desc.pop('export', False)
             result[u'modules'].append([modulename, mod_desc])
         result[u'equipment_id'] = self.equipment_id
         result[u'firmware'] = u'FRAPPY - The Python Framework for SECoP'
-        result[u'version'] = u'2019.03'
+        result[u'version'] = u'2019.05'
         result.update(self.nodeprops)
         return result
 
-    def _execute_command(self, modulename, command, argument=None):
+    def _execute_command(self, modulename, exportedname, argument=None):
         moduleobj = self.get_module(modulename)
         if moduleobj is None:
             raise NoSuchModuleError(u'Module does not exist on this SEC-Node!')
 
-        cmdspec = moduleobj.accessibles.get(command, None)
-        if cmdspec is None:
-            raise NoSuchCommandError(u'Module has no such command!')
-        if argument is None and cmdspec.datatype.argtype is not None:
+        cmdname = moduleobj.commands.exported.get(exportedname, None)
+        if cmdname is None:
+            raise NoSuchCommandError(u'Module has no command %r on this SEC-Node!' % exportedname)
+        cmdspec = moduleobj.commands[cmdname]
+        if argument is None and cmdspec.datatype.argument is not None:
             raise BadValueError(u'Command needs an argument!')
 
-        if argument is not None and cmdspec.datatype.argtype is None:
+        if argument is not None and cmdspec.datatype.argument is None:
             raise BadValueError(u'Command takes no argument!')
 
-        # now call func and wrap result as value
+        if cmdspec.datatype.argument:
+            # validate!
+            argument = cmdspec.datatype(argument)
+
+        # now call func
         # note: exceptions are handled in handle_request, not here!
-        func = getattr(moduleobj, u'do_' + command)
+        func = getattr(moduleobj, u'do_' + cmdname)
         res = func(argument) if argument else func()
-        # XXX: pipe through cmdspec.datatype.result ?
+
+        # pipe through cmdspec.datatype.result
+        if cmdspec.datatype.result:
+            res = cmdspec.datatype.result(res)
+
         return res, dict(t=currenttime())
 
     def _setParameterValue(self, modulename, exportedname, value):
@@ -231,45 +238,46 @@ class Dispatcher(object):
         if moduleobj is None:
             raise NoSuchModuleError(u'Module does not exist on this SEC-Node!')
 
-        pname = moduleobj.accessiblename2attr.get(exportedname, None)
-        pobj = moduleobj.accessibles.get(pname, None)
-        if pobj is None or not isinstance(pobj, Parameter):
-            raise NoSuchParameterError(u'Module has no such parameter on this SEC-Node!')
+        pname = moduleobj.parameters.exported.get(exportedname, None)
+        if pname is None:
+            raise NoSuchParameterError(u'Module has no parameter %r on this SEC-Node!' % exportedname)
+        pobj = moduleobj.parameters[pname]
         if pobj.constant is not None:
             raise ReadOnlyError(u'This parameter is constant and can not be accessed remotely.')
         if pobj.readonly:
             raise ReadOnlyError(u'This parameter can not be changed remotely.')
 
+        # validate!
+        value = pobj.datatype(value)
         writefunc = getattr(moduleobj, u'write_%s' % pname, None)
         # note: exceptions are handled in handle_request, not here!
         if writefunc:
-            value = writefunc(value)
+            # return value is ignored here, as it is automatically set on the pobj and broadcast
+            writefunc(value)
         else:
             setattr(moduleobj, pname, value)
-        if pobj.timestamp:
-            return pobj.export_value(), dict(t=pobj.timestamp)
-        return pobj.export_value(), {}
+        return pobj.export_value(), dict(t=pobj.timestamp) if pobj.timestamp else {}
 
     def _getParameterValue(self, modulename, exportedname):
         moduleobj = self.get_module(modulename)
         if moduleobj is None:
             raise NoSuchModuleError(u'Module does not exist on this SEC-Node!')
 
-        pname = moduleobj.accessiblename2attr.get(exportedname, None)
-        pobj = moduleobj.accessibles.get(pname, None)
-        if pobj is None or not isinstance(pobj, Parameter):
-            raise NoSuchParameterError(u'Module has no such parameter on this SEC-Node!')
+        pname = moduleobj.parameters.exported.get(exportedname, None)
+        if pname is None:
+            raise NoSuchParameterError(u'Module has no parameter %r on this SEC-Node!' % exportedname)
+        pobj = moduleobj.parameters[pname]
         if pobj.constant is not None:
-            raise ReadOnlyError(u'This parameter is constant and can not be accessed remotely.')
+            # really needed? we could just construct a readreply instead....
+            #raise ReadOnlyError(u'This parameter is constant and can not be accessed remotely.')
+            return pobj.datatype.export_value(pobj.constant)
 
         readfunc = getattr(moduleobj, u'read_%s' % pname, None)
         if readfunc:
             # should also update the pobj (via the setter from the metaclass)
             # note: exceptions are handled in handle_request, not here!
             readfunc()
-        if pobj.timestamp:
-            return pobj.export_value(), dict(t=pobj.timestamp)
-        return pobj.export_value(), {}
+        return pobj.export_value(), dict(t=pobj.timestamp) if pobj.timestamp else {}
 
     #
     # api to be called from the 'interface'

secop/protocol/interface/tcp.py

@@ -141,10 +141,6 @@ class TCPRequestHandler(socketserver.BaseRequestHandler):
                         result = (ERRORPREFIX + msg[0], msg[1], [err.name, str(err),
                                                                  {'exception': formatException(),
                                                                   'traceback': formatExtendedStack()}])
-                    except ValueError as err:
-                        result = (ERRORPREFIX + msg[0], msg[1], [u"BadValue", str(err),
-                                                                 {'exception': formatException(),
-                                                                  'traceback': formatExtendedStack()}])
                     except Exception as err:
                         # create Error Obj instead
                         result = (ERRORPREFIX + msg[0], msg[1], ['InternalError', str(err),