fix bug in ArrayOf - redesign optional struct handling
Optional structs are a pain to implement :-( The code got a bug at a place, even when optional structs are not used. So I decided to separate filling the missing keys in optional structs from validating them. Claude.AI was used to help to design this including tests. Change-Id: I7fafe45f25b94aebdd92e4d26dd049818634d07a Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/40663 Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de> Reviewed-by: Andrea <andrea.plank@psi.ch> Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch> Tested-by: Andrea <andrea.plank@psi.ch>
This commit is contained in:
1 parent
4e48aec2ba
commit
b68a9bf8b4
3 files changed
+171
-19
No files matched your search
+13
-18
@@ -67,12 +67,11 @@ class SimpleDataType(HasProperties):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
"""convert value to datatype and check for limits
|
||||
|
||||
:param value: the value to be converted
|
||||
:param previous: previous value (used for optional struct members)
|
||||
"""
|
||||
:param value: the value to be validated and converted
|
||||
"""
|
||||
# default: no limits to check
|
||||
return self(value)
|
||||
|
||||
@@ -272,7 +271,7 @@ class FloatRange(HasUnit, DataType):
|
||||
# map +/-infty to +/-max possible number
|
||||
return clamp(-sys.float_info.max, value, sys.float_info.max)
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
# convert
|
||||
value = self(value)
|
||||
# check the limits
|
||||
@@ -349,7 +348,7 @@ class IntRange(DataType):
|
||||
raise WrongTypeError(f'{value} should be an int')
|
||||
return value
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
# convert
|
||||
value = self(value)
|
||||
# check the limits
|
||||
@@ -461,7 +460,7 @@ class ScaledInteger(HasUnit, DataType):
|
||||
intval = int(round(value / self.scale))
|
||||
return float(intval * self.scale) # return 'actual' value (which is more discrete than a float)
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
# convert
|
||||
result = self(value)
|
||||
if self.min - self.scale < value < self.max + self.scale:
|
||||
@@ -841,11 +840,9 @@ class ArrayOf(DataType):
|
||||
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
|
||||
raise errcls(f'can not convert some array elements: {e!r}') from e
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
self.check_type(value)
|
||||
try:
|
||||
if previous:
|
||||
return tuple(self.members.validate(v, p) for v, p in zip(value, previous))
|
||||
return tuple(self.members.validate(v) for v in value)
|
||||
except Exception as e:
|
||||
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
|
||||
@@ -931,12 +928,10 @@ class TupleOf(DataType):
|
||||
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
|
||||
raise errcls(f'can not convert some tuple elements: {e!r}') from e
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
self.check_type(value)
|
||||
try:
|
||||
if previous is None:
|
||||
return tuple(sub.validate(elem) for sub, elem in zip(self.members, value))
|
||||
return tuple(sub.validate(v, p) for sub, v, p in zip(self.members, value, previous))
|
||||
return tuple(sub.validate(elem) for sub, elem in zip(self.members, value))
|
||||
except Exception as e:
|
||||
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
|
||||
raise errcls(f'some tuple elements are invalid: {e!r}') from e
|
||||
@@ -1025,10 +1020,10 @@ class StructOf(DataType):
|
||||
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
|
||||
raise errcls('can not convert struct element %s' % key) from e
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
self.check_type(value, True)
|
||||
try:
|
||||
result = dict(previous or {})
|
||||
result = {}
|
||||
for key, val in value.items():
|
||||
if val is not None: # goodie: allow None instead of missing key
|
||||
result[key] = self.members[key].validate(val)
|
||||
@@ -1266,9 +1261,9 @@ class LimitsType(TupleOf):
|
||||
def __init__(self, member):
|
||||
super().__init__(member, member)
|
||||
|
||||
def validate(self, value, previous=None):
|
||||
def validate(self, value):
|
||||
"""accepts an ordered tuple of numeric member types"""
|
||||
limits = TupleOf.validate(self, value, previous)
|
||||
limits = TupleOf.validate(self, value)
|
||||
if limits[1] < limits[0]:
|
||||
raise RangeError(f'maximum value {limits[1]} must be greater than '
|
||||
f'minimum value {limits[0]}')
|
||||
|
||||
@@ -54,6 +54,31 @@ def make_update(modulename, pobj):
|
||||
{'t': pobj.timestamp} if pobj.timestamp else {}])
|
||||
|
||||
|
||||
def fill_from_previous(value, previous, datatype=None):
|
||||
"""helper function to implement the specialities of the optional struct
|
||||
|
||||
:param value: the value
|
||||
:param previous: take missing values of optional structs from here
|
||||
:param datatype: if given, check result for datatype
|
||||
:return: a filled and checked return value
|
||||
"""
|
||||
if isinstance(value, dict) and isinstance(previous, dict):
|
||||
result = {}
|
||||
for key, prev in previous.items():
|
||||
if key in value:
|
||||
result[key] = fill_from_previous(value[key], prev)
|
||||
else:
|
||||
result[key] = prev
|
||||
elif isinstance(value, tuple) and isinstance(previous, tuple):
|
||||
first = tuple(fill_from_previous(v, d) for v, d in zip(value, previous))
|
||||
result = first + value[len(previous):]
|
||||
else:
|
||||
return value
|
||||
if datatype:
|
||||
return datatype(result)
|
||||
return result
|
||||
|
||||
|
||||
class Dispatcher:
|
||||
def __init__(self, name, logger, options, srv):
|
||||
|
||||
@@ -162,7 +187,9 @@ class Dispatcher:
|
||||
# convert transported value to internal value
|
||||
value = pobj.datatype.import_value(value)
|
||||
# verify range
|
||||
value = pobj.datatype.validate(value, previous=pobj.value)
|
||||
value = pobj.datatype.validate(value)
|
||||
# handle optional structs
|
||||
value = fill_from_previous(value, pobj.value, pobj.datatype)
|
||||
# note: exceptions are handled in handle_request, not here!
|
||||
getattr(moduleobj, 'write_' + pname)(value)
|
||||
# return value is ignored here, as already handled
|
||||
|
||||
@@ -30,6 +30,9 @@ from frappy.datatypes import ArrayOf, BLOBType, BoolType, CommandType, \
|
||||
StructOf, TextType, TupleOf, ValueType, get_datatype
|
||||
from frappy.errors import BadValueError, RangeError, WrongTypeError
|
||||
from frappy.lib import generalConfig
|
||||
from frappy.protocol.dispatcher import fill_from_previous
|
||||
|
||||
|
||||
|
||||
|
||||
def copytest(dt):
|
||||
@@ -713,3 +716,130 @@ def test_value_type_rejecting(validator, value):
|
||||
assert t(value) == value
|
||||
with pytest.raises(ConfigError):
|
||||
tv(value)
|
||||
|
||||
|
||||
# === optional struct stuff ====================================================
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def S(**kwargs):
|
||||
"""Shorthand: StructOf with all-optional members for fill tests."""
|
||||
dt = StructOf(**kwargs)
|
||||
dt.optional = list(kwargs.keys())
|
||||
return dt
|
||||
|
||||
|
||||
# ── fill_from_previous() with Frappy types ──────────────────────────────────────────────────
|
||||
|
||||
class TestFillWithStructOf:
|
||||
def test_missing_optional_restored(self):
|
||||
dt = S(x=IntRange(), y=IntRange())
|
||||
validated = dt.validate(dict(x=5)) # y absent → not in result
|
||||
previous = dt(dict(x=1, y=2)) # full previous value
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result['x'] == 5 # new value kept
|
||||
assert result['y'] == 2 # restored from previous
|
||||
|
||||
def test_all_keys_present_nothing_filled(self):
|
||||
dt = S(x=IntRange(), y=IntRange())
|
||||
validated = dt.validate(dict(x=5, y=9))
|
||||
previous = dt(dict(x=1, y=2))
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result == {'x': 5, 'y': 9}
|
||||
|
||||
|
||||
class TestFillWithArrayOf:
|
||||
def test_array_of_structs_missing_keys_filled(self):
|
||||
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
|
||||
validated = dt.validate((dict(x=10), dict(x=20)))
|
||||
previous = (dict(x=1, y=100), dict(x=2, y=200))
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result[0]['y'] == 100
|
||||
assert result[1]['y'] == 200
|
||||
|
||||
def test_array_longer_than_previous(self):
|
||||
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
|
||||
validated = dt.validate((dict(x=1), dict(x=2), dict(x=3)))
|
||||
previous = (dict(x=0, y=9), dict(x=0, y=8))
|
||||
result = fill_from_previous(validated, previous)
|
||||
# first two items get y filled; third has no previous → unchanged
|
||||
assert result[0]['y'] == 9
|
||||
assert result[1]['y'] == 8
|
||||
assert 'y' not in result[2]
|
||||
|
||||
def test_scalar_array_unchanged(self):
|
||||
dt = ArrayOf(IntRange())
|
||||
validated = dt.validate((3, 1, 4, 1, 5))
|
||||
previous = (0, 0, 0)
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result == (3, 1, 4, 1, 5)
|
||||
|
||||
|
||||
class TestFillWithTupleOf:
|
||||
def test_tuple_of_structs_miss_keys_filled(self):
|
||||
dt = TupleOf(S(a=IntRange(), b=IntRange()), S(c=FloatRange(), d=FloatRange()))
|
||||
validated = dt.validate((dict(a=1), dict(c=3.0)))
|
||||
previous = (dict(a=0, b=99), dict(c=0.0, d=7.5))
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result[0]['b'] == 99
|
||||
assert result[1]['d'] == 7.5
|
||||
|
||||
def test_tuple_of_scalars_unchanged(self):
|
||||
dt = TupleOf(IntRange(), FloatRange())
|
||||
validated = dt.validate((42, 3.14))
|
||||
previous = (0, 0.0)
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result == (42, 3.14)
|
||||
|
||||
|
||||
class TestFillWithNestedStructs:
|
||||
def test_nested_struct_miss_inner_key_filled(self):
|
||||
inner = S(p=IntRange(), q=IntRange())
|
||||
outer = S(name=StringType(), cfg=inner)
|
||||
validated = outer.validate(dict(
|
||||
name='dev1',
|
||||
cfg=dict(p=5), # q missing
|
||||
))
|
||||
previous = outer(dict(name='dev0', cfg=dict(p=0, q=42)))
|
||||
result = fill_from_previous(validated, previous)
|
||||
assert result['cfg']['q'] == 42
|
||||
assert result['cfg']['p'] == 5
|
||||
assert result['name'] == 'dev1'
|
||||
|
||||
|
||||
# ── dispatcher pattern (integration) ─────────────────────────────────────────
|
||||
|
||||
class TestDispatcherPattern:
|
||||
"""Mirrors exactly what dispatcher.py does after the refactor."""
|
||||
|
||||
def test_full_dispatcher_flow(self):
|
||||
dt = S(x=IntRange(), y=IntRange(), z=IntRange())
|
||||
pobj_value = dt(dict(x=1, y=2, z=3)) # simulate pobj.value
|
||||
|
||||
# Incoming partial update (only x changed)
|
||||
incoming = dict(x=99)
|
||||
validated = dt.validate(incoming) # pobj.datatype.validate(value)
|
||||
result = fill_from_previous(validated, pobj_value, dt) # fill_from_previous(value, pobj.value)
|
||||
|
||||
assert result['x'] == 99 # updated
|
||||
assert result['y'] == 2 # kept from previous
|
||||
assert result['z'] == 3 # kept from previous
|
||||
|
||||
def test_none_value_treated_as_missing(self):
|
||||
"""The existing None-means-missing goodie still works via validate."""
|
||||
dt = S(x=IntRange(), y=IntRange())
|
||||
pobj_value = dt(dict(x=1, y=2))
|
||||
|
||||
incoming = dict(x=5, y=None) # None → omitted by validate
|
||||
validated = dt.validate(incoming)
|
||||
result = fill_from_previous(validated, pobj_value, dt)
|
||||
|
||||
assert result['x'] == 5
|
||||
assert result['y'] == 2 # None was dropped by validate, fill restores it
|
||||
|
||||
def test_struct_item_beyond_default_raises(self):
|
||||
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
|
||||
pobj_value = (dict(x=0, y=9),) # only covers index 0
|
||||
incoming = dt.validate((dict(x=1), dict(x=2), dict(x=3)))
|
||||
with pytest.raises(WrongTypeError): # raised by dt(filled)
|
||||
fill_from_previous(incoming, pobj_value, dt)
|
||||
Reference in new issue
Block a user