fix bug in ArrayOf - redesign optional struct handling

Optional structs are a pain to implement :-(
The code got a bug at a place, even when optional structs are
not used. So I decided to separate filling the missing keys
in optional structs from validating them.

Claude.AI was used to help to design this including tests.

Change-Id: I7fafe45f25b94aebdd92e4d26dd049818634d07a
Reviewed-on: https://forge.frm2.tum.de/review/c/secop/frappy/+/40663
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Andrea <andrea.plank@psi.ch>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>
Tested-by: Andrea <andrea.plank@psi.ch>
This commit is contained in:
zolliker committed 2026-11-10 16:24:34 +01:00
1 parent 4e48aec2ba
commit b68a9bf8b4
3 files changed
+171 -19

No files matched your search

+13 -18
View File
@@ -67,12 +67,11 @@ class SimpleDataType(HasProperties):
"""
raise NotImplementedError
def validate(self, value, previous=None):
def validate(self, value):
"""convert value to datatype and check for limits
:param value: the value to be converted
:param previous: previous value (used for optional struct members)
"""
:param value: the value to be validated and converted
"""
# default: no limits to check
return self(value)
@@ -272,7 +271,7 @@ class FloatRange(HasUnit, DataType):
# map +/-infty to +/-max possible number
return clamp(-sys.float_info.max, value, sys.float_info.max)
def validate(self, value, previous=None):
def validate(self, value):
# convert
value = self(value)
# check the limits
@@ -349,7 +348,7 @@ class IntRange(DataType):
raise WrongTypeError(f'{value} should be an int')
return value
def validate(self, value, previous=None):
def validate(self, value):
# convert
value = self(value)
# check the limits
@@ -461,7 +460,7 @@ class ScaledInteger(HasUnit, DataType):
intval = int(round(value / self.scale))
return float(intval * self.scale) # return 'actual' value (which is more discrete than a float)
def validate(self, value, previous=None):
def validate(self, value):
# convert
result = self(value)
if self.min - self.scale < value < self.max + self.scale:
@@ -841,11 +840,9 @@ class ArrayOf(DataType):
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
raise errcls(f'can not convert some array elements: {e!r}') from e
def validate(self, value, previous=None):
def validate(self, value):
self.check_type(value)
try:
if previous:
return tuple(self.members.validate(v, p) for v, p in zip(value, previous))
return tuple(self.members.validate(v) for v in value)
except Exception as e:
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
@@ -931,12 +928,10 @@ class TupleOf(DataType):
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
raise errcls(f'can not convert some tuple elements: {e!r}') from e
def validate(self, value, previous=None):
def validate(self, value):
self.check_type(value)
try:
if previous is None:
return tuple(sub.validate(elem) for sub, elem in zip(self.members, value))
return tuple(sub.validate(v, p) for sub, v, p in zip(self.members, value, previous))
return tuple(sub.validate(elem) for sub, elem in zip(self.members, value))
except Exception as e:
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
raise errcls(f'some tuple elements are invalid: {e!r}') from e
@@ -1025,10 +1020,10 @@ class StructOf(DataType):
errcls = RangeError if isinstance(e, RangeError) else WrongTypeError
raise errcls('can not convert struct element %s' % key) from e
def validate(self, value, previous=None):
def validate(self, value):
self.check_type(value, True)
try:
result = dict(previous or {})
result = {}
for key, val in value.items():
if val is not None: # goodie: allow None instead of missing key
result[key] = self.members[key].validate(val)
@@ -1266,9 +1261,9 @@ class LimitsType(TupleOf):
def __init__(self, member):
super().__init__(member, member)
def validate(self, value, previous=None):
def validate(self, value):
"""accepts an ordered tuple of numeric member types"""
limits = TupleOf.validate(self, value, previous)
limits = TupleOf.validate(self, value)
if limits[1] < limits[0]:
raise RangeError(f'maximum value {limits[1]} must be greater than '
f'minimum value {limits[0]}')
+28 -1
View File
@@ -54,6 +54,31 @@ def make_update(modulename, pobj):
{'t': pobj.timestamp} if pobj.timestamp else {}])
def fill_from_previous(value, previous, datatype=None):
"""helper function to implement the specialities of the optional struct
:param value: the value
:param previous: take missing values of optional structs from here
:param datatype: if given, check result for datatype
:return: a filled and checked return value
"""
if isinstance(value, dict) and isinstance(previous, dict):
result = {}
for key, prev in previous.items():
if key in value:
result[key] = fill_from_previous(value[key], prev)
else:
result[key] = prev
elif isinstance(value, tuple) and isinstance(previous, tuple):
first = tuple(fill_from_previous(v, d) for v, d in zip(value, previous))
result = first + value[len(previous):]
else:
return value
if datatype:
return datatype(result)
return result
class Dispatcher:
def __init__(self, name, logger, options, srv):
@@ -162,7 +187,9 @@ class Dispatcher:
# convert transported value to internal value
value = pobj.datatype.import_value(value)
# verify range
value = pobj.datatype.validate(value, previous=pobj.value)
value = pobj.datatype.validate(value)
# handle optional structs
value = fill_from_previous(value, pobj.value, pobj.datatype)
# note: exceptions are handled in handle_request, not here!
getattr(moduleobj, 'write_' + pname)(value)
# return value is ignored here, as already handled
+130
View File
@@ -30,6 +30,9 @@ from frappy.datatypes import ArrayOf, BLOBType, BoolType, CommandType, \
StructOf, TextType, TupleOf, ValueType, get_datatype
from frappy.errors import BadValueError, RangeError, WrongTypeError
from frappy.lib import generalConfig
from frappy.protocol.dispatcher import fill_from_previous
def copytest(dt):
@@ -713,3 +716,130 @@ def test_value_type_rejecting(validator, value):
assert t(value) == value
with pytest.raises(ConfigError):
tv(value)
# === optional struct stuff ====================================================
# ── helpers ───────────────────────────────────────────────────────────────────
def S(**kwargs):
"""Shorthand: StructOf with all-optional members for fill tests."""
dt = StructOf(**kwargs)
dt.optional = list(kwargs.keys())
return dt
# ── fill_from_previous() with Frappy types ──────────────────────────────────────────────────
class TestFillWithStructOf:
def test_missing_optional_restored(self):
dt = S(x=IntRange(), y=IntRange())
validated = dt.validate(dict(x=5)) # y absent → not in result
previous = dt(dict(x=1, y=2)) # full previous value
result = fill_from_previous(validated, previous)
assert result['x'] == 5 # new value kept
assert result['y'] == 2 # restored from previous
def test_all_keys_present_nothing_filled(self):
dt = S(x=IntRange(), y=IntRange())
validated = dt.validate(dict(x=5, y=9))
previous = dt(dict(x=1, y=2))
result = fill_from_previous(validated, previous)
assert result == {'x': 5, 'y': 9}
class TestFillWithArrayOf:
def test_array_of_structs_missing_keys_filled(self):
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
validated = dt.validate((dict(x=10), dict(x=20)))
previous = (dict(x=1, y=100), dict(x=2, y=200))
result = fill_from_previous(validated, previous)
assert result[0]['y'] == 100
assert result[1]['y'] == 200
def test_array_longer_than_previous(self):
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
validated = dt.validate((dict(x=1), dict(x=2), dict(x=3)))
previous = (dict(x=0, y=9), dict(x=0, y=8))
result = fill_from_previous(validated, previous)
# first two items get y filled; third has no previous → unchanged
assert result[0]['y'] == 9
assert result[1]['y'] == 8
assert 'y' not in result[2]
def test_scalar_array_unchanged(self):
dt = ArrayOf(IntRange())
validated = dt.validate((3, 1, 4, 1, 5))
previous = (0, 0, 0)
result = fill_from_previous(validated, previous)
assert result == (3, 1, 4, 1, 5)
class TestFillWithTupleOf:
def test_tuple_of_structs_miss_keys_filled(self):
dt = TupleOf(S(a=IntRange(), b=IntRange()), S(c=FloatRange(), d=FloatRange()))
validated = dt.validate((dict(a=1), dict(c=3.0)))
previous = (dict(a=0, b=99), dict(c=0.0, d=7.5))
result = fill_from_previous(validated, previous)
assert result[0]['b'] == 99
assert result[1]['d'] == 7.5
def test_tuple_of_scalars_unchanged(self):
dt = TupleOf(IntRange(), FloatRange())
validated = dt.validate((42, 3.14))
previous = (0, 0.0)
result = fill_from_previous(validated, previous)
assert result == (42, 3.14)
class TestFillWithNestedStructs:
def test_nested_struct_miss_inner_key_filled(self):
inner = S(p=IntRange(), q=IntRange())
outer = S(name=StringType(), cfg=inner)
validated = outer.validate(dict(
name='dev1',
cfg=dict(p=5), # q missing
))
previous = outer(dict(name='dev0', cfg=dict(p=0, q=42)))
result = fill_from_previous(validated, previous)
assert result['cfg']['q'] == 42
assert result['cfg']['p'] == 5
assert result['name'] == 'dev1'
# ── dispatcher pattern (integration) ─────────────────────────────────────────
class TestDispatcherPattern:
"""Mirrors exactly what dispatcher.py does after the refactor."""
def test_full_dispatcher_flow(self):
dt = S(x=IntRange(), y=IntRange(), z=IntRange())
pobj_value = dt(dict(x=1, y=2, z=3)) # simulate pobj.value
# Incoming partial update (only x changed)
incoming = dict(x=99)
validated = dt.validate(incoming) # pobj.datatype.validate(value)
result = fill_from_previous(validated, pobj_value, dt) # fill_from_previous(value, pobj.value)
assert result['x'] == 99 # updated
assert result['y'] == 2 # kept from previous
assert result['z'] == 3 # kept from previous
def test_none_value_treated_as_missing(self):
"""The existing None-means-missing goodie still works via validate."""
dt = S(x=IntRange(), y=IntRange())
pobj_value = dt(dict(x=1, y=2))
incoming = dict(x=5, y=None) # None → omitted by validate
validated = dt.validate(incoming)
result = fill_from_previous(validated, pobj_value, dt)
assert result['x'] == 5
assert result['y'] == 2 # None was dropped by validate, fill restores it
def test_struct_item_beyond_default_raises(self):
dt = ArrayOf(S(x=IntRange(), y=IntRange()))
pobj_value = (dict(x=0, y=9),) # only covers index 0
incoming = dt.validate((dict(x=1), dict(x=2), dict(x=3)))
with pytest.raises(WrongTypeError): # raised by dt(filled)
fill_from_previous(incoming, pobj_value, dt)