linse/frappy
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[WIP] work on history writer

Browse Source 
Change-Id: If8c42091c734fb8c7b386c06429f1b21a7e169ec
This commit is contained in:
zolliker 2021-12-03 16:31:52 +01:00
parent 6e73420d0f
commit c523e8f84e
6 changed files with 258 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
secop/params.py
View File

@ -27,7 +27,7 @@ import inspect
from secop.datatypes import BoolType, CommandType, DataType, \
DataTypeType, EnumType, IntRange, NoneOr, OrType, \
StringType, StructOf, TextType, TupleOf, ValueType
StringType, StructOf, TextType, TupleOf, ValueType, ArrayOf
from secop.errors import BadValueError, ProgrammingError
from secop.properties import HasProperties, Property
@ -163,6 +163,52 @@ class Parameter(Accessible):
default None: write if given in config''', NoneOr(BoolType()),
export=False, default=None, settable=False)
history_category = Property(
'''[custom] category for history
major: should be shown by default in a history chart, default for value and target
minor: to be shown optionally in a history chart, default for other parameters
no: history is not saved. default for TextType and ArrayOf
category is ignored (forced to no) for BlobType
For structured types, the category may be a comma separated list, overwriting the
default for the first or all curves.
If it does not contain a comma, it applies for all curves
''',
NoneOr(StringType()), export=True, default=None, settable=False)
history_label = Property(
'''[custom] label for history
default: <modname>:<parname> or <modname> for main value
For structured types, the label may be a comma separated list, overwriting the
default for the first or all curves.
If it does not contain a comma, it applies for the first curve only.
''',
NoneOr(StringType()), export=True, default=None, settable=False)
history_group = Property(
'''[custom] group for history
default: unit
For structured types, the group may be a comma separated list, overwriting the
default for the first or all curves. If it does not contain a comma, it is
applies for all curves.
''',
NoneOr(StringType()), export=True, default=None, settable=False)
history_stepped = Property(
'''[custom] stepped curve
Whether a curve has to be drawn stepped or connected.
default: True when readonly=False, else False
Applicable to FloatRange and ScaledInteger only, other types are stepped by definition.
For structured types, stepped may be a list, overwriting the default for the
first or all curves. If not, applieas to all curves.
''',
OrType(BoolType(), ArrayOf(BoolType())), export=True, default=False, settable=False)
# used on the instance copy only
value = None

13
secop/protocol/dispatcher.py
View File

@ -47,7 +47,8 @@ from secop.errors import NoSuchCommandError, NoSuchModuleError, \
from secop.params import Parameter
from secop.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \
DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \
HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY
HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \
ERRORCLOSED
def make_update(modulename, pobj):
@ -297,6 +298,7 @@ class Dispatcher:
self.log.error('should have been handled in the interface!')
def handle__ident(self, conn, specifier, data):
self._active_connections.discard(conn)
return (IDENTREPLY, None, None)
def handle_describe(self, conn, specifier, data):
@ -372,3 +374,12 @@ class Dispatcher:
self._active_connections.discard(conn)
# XXX: also check all entries in self._subscriptions?
return (DISABLEEVENTSREPLY, None, None)
def close(self):
for conn in self._connections:
try:
# - may be used for the 'closed' message in serial interface
# - is used in frappy history for indicating the close time
conn.close_message((ERRORCLOSED, None, None))
except AttributeError:
pass

2
secop/protocol/messages.py
View File

@ -62,6 +62,8 @@ HEARTBEATREPLY = 'pong' # +nonce_without_space
ERRORPREFIX = 'error_' # + specifier + json_extended_info(error_report)
ERRORCLOSED = 'error_closed'
HELPREQUEST = 'help' # literal
HELPREPLY = 'helping' # +line number +json_text

12
secop/server.py
View File

@ -35,7 +35,6 @@ from collections import OrderedDict
from secop.errors import ConfigError, SECoPError
from secop.lib import formatException, get_class, getGeneralConfig
from secop.modules import Attached
from secop.params import PREDEFINED_ACCESSIBLES
try:
from daemon import DaemonContext
@ -207,7 +206,11 @@ class Server:
self.log.info('startup done, handling transport messages')
if systemd:
systemd.daemon.notify("READY=1\nSTATUS=accepting requests")
try:
self.interface.serve_forever()
except KeyboardInterrupt as e:
self._restart = False
self.dispatcher.close()
self.interface.server_close()
if self._restart:
self.restart_hook()
@ -329,11 +332,8 @@ class Server:
self.log.info('all modules and pollers started')
history_path = os.environ.get('FRAPPY_HISTORY')
if history_path:
from secop_psi.historywriter import FrappyHistoryWriter # pylint: disable=import-outside-toplevel
writer = FrappyHistoryWriter(history_path, PREDEFINED_ACCESSIBLES.keys(), self.dispatcher)
# treat writer as a connection
self.dispatcher.add_connection(writer)
writer.init(self.dispatcher.handle_describe(writer, None, None))
from secop_psi.historywriter import add_writer # pylint: disable=import-outside-toplevel
add_writer(history_path, self)
# TODO: if ever somebody wants to implement an other history writer:
# - a general config file /etc/secp/secop.conf or <frappy repo>/etc/secop.conf
# might be introduced, which contains the log, pid and cfg directory path and

269
secop_psi/historywriter.py
View File

@ -19,127 +19,214 @@
# *****************************************************************************
import time
import frappyhistory # pylint: disable=import-error
from secop.datatypes import get_datatype, IntRange, FloatRange, ScaledInteger,\
EnumType, BoolType, StringType, TupleOf, StructOf
try:
import frappyhistory # pylint: disable=import-error
except ImportError:
pass # do not complain when used for tests
from secop.lib import clamp, formatExtendedTraceback
from secop.datatypes import IntRange, FloatRange, ScaledInteger,\
EnumType, BoolType, StringType, TupleOf, StructOf, ArrayOf, TextType
def make_cvt_list(dt, tail=''):
def make_cvt_list(dt, tail):
"""create conversion list
list of tuple (<conversion function>, <tail>, <curve options>)
tail is a postfix to be appended in case of tuples and structs
"""
if isinstance(dt, (EnumType, IntRange, BoolType)):
return[(int, tail, dict(type='NUM'))]
if isinstance(dt, (IntRange, BoolType)):
return [(int, {'key': tail})]
if isinstance(dt, EnumType):
return [(int, {'key': tail, 'enum': dt.export_datatype()['members']})]
if isinstance(dt, (FloatRange, ScaledInteger)):
return [(dt.import_value, tail, dict(type='NUM', unit=dt.unit, period=5) if dt.unit else {})]
opts = {'key': tail}
if dt.unit:
opts['group'] = dt.unit
opts['stepped'] = True
return [(dt.import_value, opts)]
if isinstance(dt, StringType):
return [(lambda x: x, tail, dict(type='STR'))]
opts = {'key': tail, 'kind': 'STR'}
if isinstance(dt, TextType):
opts['category'] = 'no'
return [(lambda x: x, opts)]
if isinstance(dt, TupleOf):
items = enumerate(dt.members)
elif isinstance(dt, StructOf):
items = dt.members.items()
else:
return [] # ArrayType, BlobType and TextType are ignored: too much data, probably not used
result = []
for subkey, elmtype in items:
for fun, tail_, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)):
def conv(value, key=subkey, func=fun):
try:
return value[key]
except KeyError: # can not use value.get() because value might be a list
return None
result.append((conv, tail_, opts))
for index, elmtype in enumerate(dt.members):
for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, index)):
def conv(value, key=index, func=fun):
return func(value[key])
result.append((conv, opts))
return result
if isinstance(dt, ArrayOf):
result = []
for index in range(dt.maxlen):
for fun, opts in make_cvt_list(dt.members, '%s.%s' % (tail, index)):
opts['category'] = 'no'
def conv(value, key=index, func=fun):
return func(value[key])
result.append((conv, opts))
return result
if isinstance(dt, StructOf):
result = []
for subkey, elmtype in dt.members.items():
for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)):
def conv(value, key=subkey, func=fun):
return func(value.get(key)) # None for missing struct key, should not be needed
result.append((conv, opts))
return result
return [] # other types (BlobType) are ignored: too much data, probably not used
class FrappyHistoryWriter(frappyhistory.FrappyWriter):
"""extend writer to be used as an internal frappy connection
class FrappyAbstractHistoryWriter:
"""abstract writer
API of frappyhistory.FrappyWriter:
:meth:`put_def`(key, opts):
define or overwrite a new curve named <key> with options from dict <opts>
options:
- type:
'NUM' (any number) or 'STR' (text)
remark: tuples and structs create multiple curves
- period:
the typical 'lifetime' of a value.
The intention is, that points in a chart may be connected by a straight line
when the distance is lower than twice this value. If not, the line should be
drawn horizontally from the last point to a point <period> before the next value.
For example a setpoint should have period 0, which will lead to a stepped
line, whereas for a measured value like a temperature, period should be
equal to the poll interval. In order to make full use of this,
we would need some additional parameter property.
- show: True/False, whether this curve should be shown or not by default in
a summary chart
- label: a label for the curve in the chart
:meth:`put`(timestamp, key, value)
timestamp: the timestamp. must not decrease!
key: the curve name
value: the value to be stored, converted to a string. '' indicates an undefined value
self.cache is a dict <key> of <value as string>, containing the last used value
doc only
"""
def __init__(self, directory, predefined_names, dispatcher):
super().__init__(directory)
self.predefined_names = predefined_names
def put_def(self, key, kind='NUM', category='minor', **opts):
"""define or overwrite a new curve named <key> with options from dict <opts>
:param key: the key for the curve
:param kind: 'NUM' (default) for numeric values, 'STR' for strings
:param category: 'major' or 'minor': importance of curve
:param opts: a dict containing some of the following options
- label: a label for the curve in the chart
- group: grouping of the curves in charts (often equal to unit)
- stepped: lines in charts should be drawn as stepped line. Only applicable when kind='NUM'
True by default.
"""
raise NotImplementedError()
def put(self, timestamp, key, value):
"""add a data point
:param timestamp: the timestamp. must not decrease!
:param key: the curve name
:param value: the value to be stored (number or string), None indicates un undefined value
"""
raise NotImplementedError()
def get(self, key):
"""get from cache
:param key: the curve name
:returns: the last stored value or None
"""
raise NotImplementedError()
def close(self, timestamp):
"""close the writer
:param timestamp:
indicate to the writer that all values are getting undefined after <timestamp>
"""
raise NotImplementedError()
class FrappyHistoryHandler:
def __init__(self, writer, modules, dispatcher, logger=None):
self.writer = writer
self.log = logger
self.cvt_lists = {} # dict <mod:param> of <conversion list>
self.activated = False
self.dispatcher = dispatcher
self._init_time = None
def init(self, msg):
"""initialize from the 'describing' message"""
action, _, description = msg
assert action == 'describing'
self._init_time = time.time()
for modname, moddesc in description['modules'].items():
for pname, pdesc in moddesc['accessibles'].items():
ident = key = modname + ':' + pname
if pname.startswith('_') and pname[1:] not in self.predefined_names:
key = modname + ':' + pname[1:]
dt = get_datatype(pdesc['datainfo'])
self._last_time = self._init_time
for modname, modobj in modules.items():
for pname, pobj in modobj.parameters.items():
ident = '%s:%s' % (modname, pobj.export)
key = '%s:%s' % (modname, pname)
dt = pobj.datatype
cvt_list = make_cvt_list(dt, key)
for _, hkey, opts in cvt_list:
# create default opts
if pobj.history_category:
values = pobj.history_category.split(',')
else:
values = [None]
if len(values) == 1:
values *= len(cvt_list)
for cat, (_, opts) in zip(values, cvt_list):
if cat is None:
cat = opts.get('category', 'major' if pname in ('value', 'target') else 'minor')
opts['category'] = cat
if pname == 'value':
opts['period'] = opts.get('period', 0)
opts['show'] = True
opts['label'] = modname
elif pname == 'target':
opts['period'] = 0
opts['label'] = modname + '_target'
opts['show'] = True
self.put_def(hkey, opts)
for _, opts in cvt_list:
opts['key'] = opts['key'].replace(':value', '')
if pname == 'status':
# default labels '<modname>:status' and '<modname>:status_text'
for lbl, (_, opts) in zip([key, key + '_text'], cvt_list):
opts['label'] = lbl
# overwrite opts based on history_* properties
if pobj.history_label:
for lbl, (_, opts) in zip(','.split(pobj.history_label), cvt_list):
opts['label'] = lbl
if pobj.history_group:
values = pobj.history_group.split(',')
if len(values) == 1:
values *= len(cvt_list)
for grp, (_, opts) in zip(values, cvt_list):
opts['group'] = grp
if pobj.history_stepped:
values = pobj.history_stepped
elif pobj.readonly:
values = False
if not isinstance(values, tuple):
values = [values] * len(cvt_list)
for stp, (_, opts) in zip(values, cvt_list):
if not stp and 'stepped' in opts: # only on floats
opts['stepped'] = False
cvt_list = [(key, opts) for key, opts in cvt_list if opts.get('category') != 'no']
for _, opts in cvt_list:
if opts.get('stepped'):
opts.pop('stepped', None)
writer.put_def(**opts)
if cvt_list:
self.cvt_lists[ident] = cvt_list
# self.put(self._init_time, 'STR', 'vars', ' '.join(vars))
self.dispatcher.handle_activate(self, None, None)
self._init_time = None
def close_message(self, msg):
self.writer.close(time.time())
def send_reply(self, msg):
try:
action, ident, value = msg
if not action.endswith('update'):
print('unknown async message %r' % msg)
assert action.endswith('update')
cvt_list = self.cvt_lists.get(ident)
if not cvt_list:
return
now = self._init_time or time.time() # on initialisation, use the same timestamp for all
if self._init_time:
t = self._init_time # on initialisation, use the same timestamp for all
else:
t = value[1].get('t')
if t:
# make sure time stamp is not decreasing, as a potentially decreasing
# value might bring the reader software into trouble
t = clamp(self._last_time, t, time.time())
else:
t = time.time()
self._last_time = t
if action == 'update':
for fun, key, _ in self.cvt_lists[ident]:
for fun, opts in cvt_list:
# we only look at the value, qualifiers are ignored for now
# we do not use the timestamp here, as a potentially decreasing value might
# bring the reader software into trouble
self.put(now, key, str(fun(value[0])))
# print('UPDATE', key, value, t)
self.writer.put(t, opts['key'], fun(value[0]))
else: # error_update
for _, key, _ in self.cvt_lists[ident]:
old = self.cache.get(key)
if old is None:
return # ignore if this key is not yet used
self.put(now, key, '')
for _, opts in cvt_list:
self.writer.put(t, opts['key'], None)
except Exception as e:
self.log.error('FrappyHistoryHandler.send_reply: %r with msg %r: ', repr(e), msg)
print(formatExtendedTraceback())
def add_writer(history_path, srv):
# treat handler as a connection
logger = srv.log.getChild('history')
srv.dispatcher.add_connection(FrappyHistoryHandler(
frappyhistory.Writer(history_path, logger), srv.modules, srv.dispatcher, logger))

2
secop_psi/ppmssim.py
View File

@ -187,7 +187,7 @@ class PpmsSim:
self.i1 = self.t % 10.0
self.r2 = 1000 / self.t
self.i2 = math.log(self.t)
self.level.value = 100 - (self.time - self.start) * 0.01 % 100
self.level.value = round(100 - (self.time - self.start) * 0.01 % 100, 1)
# print('PROGRESS T=%.7g B=%.7g x=%.7g' % (self.t, self.mf, self.pos))
def getdat(self, mask):