From 16772609f4825098300b1c2c0b67cc97e043da0b Mon Sep 17 00:00:00 2001 From: Enrico Faulhaber Date: Wed, 27 Mar 2019 10:49:03 +0100 Subject: [PATCH] datatypes: implement hints (unit, fmtstr, *_precision) on types where these can be useful. Also fix some issues with ScaledInteger. Change-Id: I9d456c4f237da3a37730c3e451e9fb59307ed982 Reviewed-on: https://forge.frm2.tum.de/review/20240 Tested-by: JenkinsCodeReview Reviewed-by: Enrico Faulhaber --- secop/datatypes.py | 225 ++++++++++++++++++++++++++++------------- test/test_datatypes.py | 76 ++++++++++---- 2 files changed, 215 insertions(+), 86 deletions(-) diff --git a/secop/datatypes.py b/secop/datatypes.py index 3b3a540..c22f2ed 100644 --- a/secop/datatypes.py +++ b/secop/datatypes.py @@ -49,6 +49,10 @@ __all__ = [ u'CommandType', ] +# *DEFAULT* limits for IntRange/ScaledIntegers transport serialisation +DEFAULT_MIN_INT = -16777216 +DEFAULT_MAX_INT = 16777216 + # base class for all DataTypes @@ -87,18 +91,46 @@ class DataType(object): class FloatRange(DataType): """Restricted float type""" - def __init__(self, minval=None, maxval=None): - self.min = None if minval is None else float(minval) - self.max = None if maxval is None else float(maxval) - # note: as we may compare to Inf all comparisons would be false - if (self.min or float(u'-inf')) <= (self.max or float(u'+inf')): - self.as_json = [u'double', dict()] - if self.min: - self.as_json[1]['min'] = self.min - if self.max: - self.as_json[1]['max'] = self.max - else: + unit = u'' + fmtstr = u'%f' + def __init__(self, minval=None, maxval=None, unit=u'', fmtstr=u'', + absolute_precision=None, relative_precision=None,): + # store hints + self.hints = {} + self.unit = unicode(unit) + self.fmtstr = unicode(fmtstr or u'%f') + self.abs_prec = float(absolute_precision or 0.0) + self.rel_prec = float(relative_precision or 1.2e-7) + + # store values for the validator + self.min = float(u'-inf') if minval is None else float(minval) + self.max = float(u'+inf') if maxval is None else float(maxval) + + # check values + if self.min > self.max: raise ValueError(u'Max must be larger then min!') + if '%' not in self.fmtstr: + raise ValueError(u'Invalid fmtstr!') + if self.abs_prec < 0: + raise ValueError(u'absolute_precision MUST be >=0') + if self.rel_prec < 0: + raise ValueError(u'relative_precision MUST be >=0') + + info = {} + if self.min != float(u'-inf'): + info[u'min'] = self.min + if self.max != float(u'inf'): + info[u'max'] = self.max + if unit: + self.hints[u'unit'] = self.unit + if fmtstr: + self.hints[u'fmtstr'] = self.fmtstr + if absolute_precision is not None: + self.hints[u'absolute_precision'] = self.abs_prec + if relative_precision is not None: + self.hints[u'relative_precision'] = self.rel_prec + info.update(self.hints) + self.as_json = [u'double', info] def validate(self, value): try: @@ -119,11 +151,12 @@ class FloatRange(DataType): (value, self.min, self.max)) def __repr__(self): - if self.max is not None: - return u'FloatRange(%r, %r)' % ( - float(u'-inf') if self.min is None else self.min, - float(u'inf') if self.max is None else self.max) - return u'FloatRange()' + items = [] if self.max or self.min is None else \ + [u'-inf' if self.min == float(u'-inf') else self.fmtstr % self.min, + u'inf' if self.max == float(u'inf') else self.fmtstr % self.max] + for k,v in self.hints.items(): + items.append(u'%s=%r' % (k,v)) + return u'FloatRange(%s)' % (', '.join(items)) def export_value(self, value): """returns a python object fit for serialisation""" @@ -138,25 +171,43 @@ class FloatRange(DataType): return self.validate(value) + class IntRange(DataType): """Restricted int type""" - def __init__(self, minval=None, maxval=None): - self.min = -16777216 if minval is None else int(minval) - self.max = 16777216 if maxval is None else int(maxval) + unit = u'' + fmtstr = u'%f' + def __init__(self, minval=None, maxval=None, fmtstr=u'%d', unit=u''): + self.hints = {} + self.fmtstr = unicode(fmtstr) + self.unit = unicode(unit) + self.min = DEFAULT_MIN_INT if minval is None else int(minval) + self.max = DEFAULT_MAX_INT if maxval is None else int(maxval) + + # check values if self.min > self.max: raise ValueError(u'Max must be larger then min!') - if None in (self.min, self.max): - raise ValueError(u'Limits can not be None!') - self.as_json = [u'int', {'min':self.min, 'max':self.max}] + if '%' not in self.fmtstr: + raise ValueError(u'Invalid fmtstr!') + + info = {} + self.hints = {} + info[u'min'] = self.min + info[u'max'] = self.max + if unit: + self.hints[u'unit'] = self.unit + if fmtstr != u'%d': + self.hints[u'fmtstr'] = self.fmtstr + info.update(self.hints) + self.as_json = [u'int', info] def validate(self, value): try: value = int(value) - if self.min is not None and value < self.min: + if value < self.min: raise ValueError(u'%r should be an int between %d and %d' % (value, self.min, self.max or 0)) - if self.max is not None and value > self.max: + if value > self.max: raise ValueError(u'%r should be an int between %d and %d' % (value, self.min or 0, self.max)) return value @@ -164,9 +215,10 @@ class IntRange(DataType): raise ValueError(u'Can not validate %r to int' % value) def __repr__(self): - if self.max is not None: - return u'IntRange(%d, %d)' % (self.min, self.max) - return u'IntRange()' + items = [u"%d, %d" % (self.min, self.max)] + for k, v in self.hints.items(): + items.append(u'%s=%r' % (k, v)) + return u'IntRange(%s)' % (u', '.join(items)) def export_value(self, value): """returns a python object fit for serialisation""" @@ -187,36 +239,75 @@ class ScaledInteger(DataType): note: limits are for the scaled value (i.e. the internal value) the scale is only used for calculating to/from transport serialisation""" - def __init__(self, scale, minval=-16777216, maxval=16777216): - self.min = int(minval) - self.max = int(maxval) + unit = u'' + fmtstr = u'%f' + def __init__(self, scale, minval=None, maxval=None, unit=u'', fmtstr=u'', + absolute_precision=None, relative_precision=None,): self.scale = float(scale) - if self.min > self.max: - raise ValueError(u'Max must be larger then min!') if not self.scale > 0: raise ValueError(u'Scale MUST be positive!') - self.as_json = [u'scaled', dict(min=int(round(minval/scale)), max=int(round(maxval/scale)), scale=scale)] + + # store hints + self.hints = {} + self.unit = unicode(unit) + self.fmtstr = unicode(fmtstr or u'%f') + self.abs_prec = float(absolute_precision or self.scale) + self.rel_prec = float(relative_precision or 0) + + self.min = DEFAULT_MIN_INT * self.scale if minval is None else float(minval) + self.max = DEFAULT_MAX_INT * self.scale if maxval is None else float(maxval) + + # check values + if self.min > self.max: + raise ValueError(u'Max must be larger then min!') + if '%' not in self.fmtstr: + raise ValueError(u'Invalid fmtstr!') + if self.abs_prec < 0: + raise ValueError(u'absolute_precision MUST be >=0') + if self.rel_prec < 0: + raise ValueError(u'relative_precision MUST be >=0') + + info = {} + self.hints = {} + info[u'min'] = int(self.min // self.scale) + info[u'max'] = int((self.max + self.scale * 0.5) // self.scale) + info[u'scale'] = self.scale + if unit: + self.hints[u'unit'] = self.unit + if fmtstr: + self.hints[u'fmtstr'] = self.fmtstr + if absolute_precision is not None: + self.hints[u'absolute_precision'] = self.abs_prec + if relative_precision is not None: + self.hints[u'relative_precision'] = self.rel_prec + info.update(self.hints) + self.as_json = [u'scaled', info] def validate(self, value): try: - value = int(value) - if value < self.min: - raise ValueError(u'%r should be an int between %d and %d' % - (value, self.min, self.max)) - if value > self.max: - raise ValueError(u'%r should be an int between %d and %d' % - (value, self.min, self.max)) - return value + value = float(value) except Exception: - raise ValueError(u'Can not validate %r to int' % value) + raise ValueError(u'Can not validate %r to float' % value) + if value < self.min: + raise ValueError(u'%r should be a float between %d and %d' % + (value, self.min, self.max)) + if value > self.max: + raise ValueError(u'%r should be a float between %d and %d' % + (value, self.min, self.max)) + intval = int((value + self.scale * 0.5) // self.scale) + value = float(intval * self.scale) + return value # return 'actual' value (which is more discrete than a float) def __repr__(self): - return u'ScaledInteger(%f, %d, %d)' % (self.scale, self.min, self.max) + items = [self.fmtstr % self.scale, self.fmtstr % self.min, self.fmtstr % self.max] + for k,v in self.hints.items(): + items.append(u'%s=%r' % (k,v)) + return u'ScaledInteger(%s)' % (', '.join(items)) def export_value(self, value): """returns a python object fit for serialisation""" - # XXX: rounds toward even !!! (i.e. 12.5 -> 12, 13.5 -> 14) - return round(value / self.scale) + # note: round behaves different in Py2 vs. Py3, so use floor division + return int((value + self.scale * 0.5) // self.scale) def import_value(self, value): """returns a python object from serialisation""" @@ -228,16 +319,17 @@ class ScaledInteger(DataType): class EnumType(DataType): + def __init__(self, enum_or_name='', **kwds): - if 'members' in kwds: + if u'members' in kwds: kwds = dict(kwds) - kwds.update(kwds['members']) - kwds.pop('members') + kwds.update(kwds[u'members']) + kwds.pop(u'members') self._enum = Enum(enum_or_name, **kwds) @property def as_json(self): - return [u'enum'] + [{"members":dict((m.name, m.value) for m in self._enum.members)}] + return [u'enum'] + [{u"members":dict((m.name, m.value) for m in self._enum.members)}] def __repr__(self): return u"EnumType(%r, %s)" % (self._enum.name, ', '.join(u'%s=%d' %(m.name, m.value) for m in self._enum.members)) @@ -279,7 +371,7 @@ class BLOBType(DataType): self.as_json = [u'blob', dict(min=minsize, max=maxsize)] def __repr__(self): - return u'BLOB(%s, %s)' % (unicode(self.minsize), unicode(self.maxsize)) + return u'BLOB(%d, %d)' % (self.minsize, self.maxsize) def validate(self, value): """return the validated (internal) value or raise""" @@ -289,10 +381,9 @@ class BLOBType(DataType): if size < self.minsize: raise ValueError( u'%r must be at least %d bytes long!' % (value, self.minsize)) - if self.maxsize is not None: - if size > self.maxsize: - raise ValueError( - u'%r must be at most %d bytes long!' % (value, self.maxsize)) + if size > self.maxsize: + raise ValueError( + u'%r must be at most %d bytes long!' % (value, self.maxsize)) return value def export_value(self, value): @@ -326,7 +417,7 @@ class StringType(DataType): self.as_json = [u'string', dict(min=minsize, max=maxsize)] def __repr__(self): - return u'StringType(%s, %s)' % (unicode(self.minsize), unicode(self.maxsize)) + return u'StringType(%d, %d)' % (self.minsize, self.maxsize) def validate(self, value): """return the validated (internal) value or raise""" @@ -336,10 +427,9 @@ class StringType(DataType): if size < self.minsize: raise ValueError( u'%r must be at least %d bytes long!' % (value, self.minsize)) - if self.maxsize is not None: - if size > self.maxsize: - raise ValueError( - u'%r must be at most %d bytes long!' % (value, self.maxsize)) + if size > self.maxsize: + raise ValueError( + u'%r must be at most %d bytes long!' % (value, self.maxsize)) if u'\0' in value: raise ValueError( u'Strings are not allowed to embed a \\0! Use a Blob instead!') @@ -358,13 +448,12 @@ class StringType(DataType): value = unicode(text) return self.validate(value) + # Bool is a special enum - - class BoolType(DataType): def __init__(self): - self.as_json = [u'bool', dict()] + self.as_json = [u'bool', {}] def __repr__(self): return u'BoolType()' @@ -400,13 +489,13 @@ class ArrayOf(DataType): subtype = None def __init__(self, members, minsize=0, maxsize=None): + if not isinstance(members, DataType): + raise ValueError( + u'ArrayOf only works with a DataType as first argument!') # one argument -> exactly that size # argument default to 10 if maxsize is None: maxsize = minsize or 10 - if not isinstance(members, DataType): - raise ValueError( - u'ArrayOf only works with a DataType as first argument!') self.subtype = members self.minsize = int(minsize) @@ -636,9 +725,9 @@ class Status(TupleOf): # argumentnames to lambda from spec! DATATYPES = dict( bool =BoolType, - int =lambda min, max: IntRange(minval=min,maxval=max), - scaled =lambda scale, min, max: ScaledInteger(scale=scale,minval=min*scale,maxval=max*scale), - double =lambda min=None, max=None: FloatRange(minval=min, maxval=max), + int =lambda min, max, **kwds: IntRange(minval=min, maxval=max, **kwds), + scaled =lambda scale, min, max, **kwds: ScaledInteger(scale=scale, minval=min*scale, maxval=max*scale, **kwds), + double =lambda min=None, max=None, **kwds: FloatRange(minval=min, maxval=max, **kwds), blob =lambda min=0, max=None: BLOBType(minsize=min, maxsize=max), string =lambda min=0, max=None: StringType(minsize=min, maxsize=max), array =lambda min, max, members: ArrayOf(get_datatype(members), minsize=min, maxsize=max), diff --git a/test/test_datatypes.py b/test/test_datatypes.py index e942425..97a79bb 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -64,6 +64,13 @@ def test_FloatRange(): dt = FloatRange() assert dt.as_json == ['double', {}] + dt = FloatRange(unit='X', fmtstr='%r', absolute_precision=1, + relative_precision=0.1) + assert dt.as_json == ['double', {'unit':'X', 'fmtstr':'%r', + 'absolute_precision':1, + 'relative_precision':0.1}] + assert dt.validate(4) == 4 + def test_IntRange(): dt = IntRange(-3, 3) @@ -86,6 +93,10 @@ def test_IntRange(): assert dt.as_json[0] == 'int' assert dt.as_json[1]['min'] < 0 < dt.as_json[1]['max'] + dt = IntRange(unit='X', fmtstr='%r') + assert dt.as_json == ['int', {'fmtstr': '%r', 'max': 16777216, + 'min': -16777216, 'unit': 'X'}] + def test_ScaledInteger(): dt = ScaledInteger(0.01, -3, 3) @@ -113,6 +124,14 @@ def test_ScaledInteger(): assert dt.export_value(2.71819) == int(272) assert dt.import_value(272) == 2.72 + dt = ScaledInteger(0.003, 0, 1, unit='X', fmtstr='%r', + absolute_precision=1, relative_precision=0.1) + assert dt.as_json == ['scaled', {'scale':0.003,'min':0,'max':333, + 'unit':'X', 'fmtstr':'%r', + 'absolute_precision':1, + 'relative_precision':0.1}] + assert dt.validate(0.4) == 0.399 + def test_EnumType(): # test constructor catching illegal arguments @@ -235,10 +254,12 @@ def test_ArrayOf(): with pytest.raises(ValueError): ArrayOf(-3, IntRange(-10,10)) dt = ArrayOf(IntRange(-10, 10), 5) - assert dt.as_json == ['array', {'min':5, 'max':5, 'members':['int', {'min':-10, 'max':10}]}] + assert dt.as_json == ['array', {'min':5, 'max':5, + 'members':['int', {'min':-10, 'max':10}]}] dt = ArrayOf(IntRange(-10, 10), 1, 3) - assert dt.as_json == ['array', {'min':1, 'max':3, 'members':['int', {'min':-10, 'max':10}]}] + assert dt.as_json == ['array', {'min':1, 'max':3, + 'members':['int', {'min':-10, 'max':10}]}] with pytest.raises(ValueError): dt.validate(9) with pytest.raises(ValueError): @@ -256,7 +277,8 @@ def test_TupleOf(): TupleOf(2) dt = TupleOf(IntRange(-10, 10), BoolType()) - assert dt.as_json == ['tuple', {'members':[['int', {'min':-10, 'max':10}], ['bool', {}]]}] + assert dt.as_json == ['tuple', {'members':[['int', {'min':-10, 'max':10}], + ['bool', {}]]}] with pytest.raises(ValueError): dt.validate(9) @@ -276,9 +298,12 @@ def test_StructOf(): with pytest.raises(ProgrammingError): StructOf(IntRange=1) - dt = StructOf(a_string=StringType(0, 55), an_int=IntRange(0, 999), optional=['an_int']) - assert dt.as_json == [u'struct', {'members':{u'a_string': [u'string', {'min':0, 'max':55}], - u'an_int': [u'int', {'min':0, 'max':999}],}, + dt = StructOf(a_string=StringType(0, 55), an_int=IntRange(0, 999), + optional=['an_int']) + assert dt.as_json == [u'struct', {'members':{u'a_string': + [u'string', {'min':0, 'max':55}], + u'an_int': + [u'int', {'min':0, 'max':999}],}, 'optional':['an_int'], }] @@ -291,8 +316,10 @@ def test_StructOf(): assert dt.validate(dict(a_string='XXX', an_int=8)) == {'a_string': 'XXX', 'an_int': 8} - assert dt.export_value({'an_int': 13, 'a_string': 'WFEC'}) == {'a_string': 'WFEC', 'an_int': 13} - assert dt.import_value({'an_int': 13, 'a_string': 'WFEC'}) == {'a_string': 'WFEC', 'an_int': 13} + assert dt.export_value({'an_int': 13, 'a_string': 'WFEC'}) == { + 'a_string': 'WFEC', 'an_int': 13} + assert dt.import_value({'an_int': 13, 'a_string': 'WFEC'}) == { + 'a_string': 'WFEC', 'an_int': 13} def test_get_datatype(): @@ -329,7 +356,8 @@ def test_get_datatype(): assert isinstance(get_datatype(['double', {}]), FloatRange) assert isinstance(get_datatype(['double', {'min':-2.718}]), FloatRange) assert isinstance(get_datatype(['double', {'max':3.14}]), FloatRange) - assert isinstance(get_datatype(['double', {'min':-9.9, 'max':11.1}]), FloatRange) + assert isinstance(get_datatype(['double', {'min':-9.9, 'max':11.1}]), + FloatRange) with pytest.raises(ValueError): get_datatype(['double']) @@ -342,7 +370,9 @@ def test_get_datatype(): get_datatype(['scaled', {'scale':0.01,'min':-2.718}]) with pytest.raises(ValueError): get_datatype(['scaled', {'scale':0.02,'max':3.14}]) - assert isinstance(get_datatype(['scaled', {'scale':0.03,'min':-99, 'max':111}]), ScaledInteger) + assert isinstance(get_datatype(['scaled', {'scale':0.03, + 'min':-99, + 'max':111}]), ScaledInteger) dt = ScaledInteger(scale=0.03, minval=0, maxval=9.9) assert dt.as_json == ['scaled', {'max':330, 'min':0, 'scale':0.03}] @@ -396,13 +426,18 @@ def test_get_datatype(): get_datatype(['array', 1]) with pytest.raises(ValueError): get_datatype(['array', [1], 2, 3]) - assert isinstance(get_datatype(['array', {'min':1, 'max':1, 'members':['blob', {'max':1}]}]), ArrayOf) - assert isinstance(get_datatype(['array', {'min':1, 'max':1, 'members':['blob', {'max':1}]}]).subtype, BLOBType) + assert isinstance(get_datatype(['array', {'min':1, 'max':1, + 'members':['blob', {'max':1}]}] + ), ArrayOf) + assert isinstance(get_datatype(['array', {'min':1, 'max':1, + 'members':['blob', {'max':1}]}] + ).subtype, BLOBType) with pytest.raises(ValueError): get_datatype(['array', {'members':['blob', {'max':1}], 'min':-10}]) with pytest.raises(ValueError): - get_datatype(['array', {'members':['blob', {'max':1}], 'min':10, 'max':1}]) + get_datatype(['array', {'members':['blob', {'max':1}], + 'min':10, 'max':1}]) with pytest.raises(ValueError): get_datatype(['array', ['blob', 1], 10, -10]) @@ -412,15 +447,18 @@ def test_get_datatype(): get_datatype(['tuple', 1]) with pytest.raises(ValueError): get_datatype(['tuple', [1], 2, 3]) - assert isinstance(get_datatype(['tuple', {'members':[['blob', {'max':1}]]}]), TupleOf) - assert isinstance(get_datatype(['tuple', {'members':[['blob', {'max':1}]]}]).subtypes[0], BLOBType) + assert isinstance(get_datatype(['tuple', {'members':[['blob', + {'max':1}]]}]), TupleOf) + assert isinstance(get_datatype(['tuple', {'members':[['blob', + {'max':1}]]}]).subtypes[0], BLOBType) with pytest.raises(ValueError): get_datatype(['tuple', {}]) with pytest.raises(ValueError): get_datatype(['tuple', 10, -10]) - assert isinstance(get_datatype(['tuple', {'members':[['blob', {'max':1}], ['bool',{}]]}]), TupleOf) + assert isinstance(get_datatype(['tuple', {'members':[['blob', {'max':1}], + ['bool',{}]]}]), TupleOf) with pytest.raises(ValueError): get_datatype(['struct']) @@ -428,8 +466,10 @@ def test_get_datatype(): get_datatype(['struct', 1]) with pytest.raises(ValueError): get_datatype(['struct', [1], 2, 3]) - assert isinstance(get_datatype(['struct', {'members':{'name': ['blob', {'max':1}]}}]), StructOf) - assert isinstance(get_datatype(['struct', {'members':{'name': ['blob', {'max':1}]}}]).named_subtypes['name'], BLOBType) + assert isinstance(get_datatype(['struct', {'members': + {'name': ['blob', {'max':1}]}}]), StructOf) + assert isinstance(get_datatype(['struct', {'members': + {'name': ['blob', {'max':1}]}}]).named_subtypes['name'], BLOBType) with pytest.raises(ValueError): get_datatype(['struct', {}])