[WIP] work on history writer

Change-Id: If8c42091c734fb8c7b386c06429f1b21a7e169ec

secop/params.py

@@ -27,7 +27,7 @@ import inspect
 
 from secop.datatypes import BoolType, CommandType, DataType, \
     DataTypeType, EnumType, IntRange, NoneOr, OrType, \
-    StringType, StructOf, TextType, TupleOf, ValueType
+    StringType, StructOf, TextType, TupleOf, ValueType, ArrayOf
 from secop.errors import BadValueError, ProgrammingError
 from secop.properties import HasProperties, Property
 
@@ -163,6 +163,52 @@ class Parameter(Accessible):
 
         default None: write if given in config''', NoneOr(BoolType()),
         export=False, default=None, settable=False)
+    history_category = Property(
+        '''[custom] category for history
+
+        major: should be shown by default in a history chart, default for value and target
+        minor: to be shown optionally in a history chart, default for other parameters
+        no: history is not saved. default for TextType and ArrayOf
+        
+        category is ignored (forced to no) for BlobType
+        
+        For structured types, the category may be a comma separated list, overwriting the
+        default for the first or all curves.
+        If it does not contain a comma, it applies for all curves
+        ''',
+        NoneOr(StringType()), export=True, default=None, settable=False)
+    history_label = Property(
+        '''[custom] label for history
+
+        default: <modname>:<parname> or <modname> for main value
+        
+        For structured types, the label may be a comma separated list, overwriting the
+        default for the first or all curves.
+        If it does not contain a comma, it applies for the first curve only.
+        ''',
+        NoneOr(StringType()), export=True, default=None, settable=False)
+    history_group = Property(
+        '''[custom] group for history
+
+        default: unit
+
+        For structured types, the group may be a comma separated list, overwriting the
+        default for the first or all curves. If it does not contain a comma, it is
+        applies for all curves.
+        ''',
+        NoneOr(StringType()), export=True, default=None, settable=False)
+    history_stepped = Property(
+        '''[custom] stepped curve
+
+        Whether a curve has to be drawn stepped or connected.
+        default: True when readonly=False, else False
+        
+        Applicable to FloatRange and ScaledInteger only, other types are stepped by definition.
+        
+        For structured types, stepped may be a list, overwriting the default for the
+        first or all curves. If not, applieas to all curves.
+        ''',
+        OrType(BoolType(), ArrayOf(BoolType())), export=True, default=False, settable=False)
 
     # used on the instance copy only
     value = None

secop/protocol/dispatcher.py

@@ -47,7 +47,8 @@ from secop.errors import NoSuchCommandError, NoSuchModuleError, \
 from secop.params import Parameter
 from secop.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \
     DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \
-    HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY
+    HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \
+    ERRORCLOSED
 
 
 def make_update(modulename, pobj):
@@ -297,6 +298,7 @@ class Dispatcher:
         self.log.error('should have been handled in the interface!')
 
     def handle__ident(self, conn, specifier, data):
+        self._active_connections.discard(conn)
         return (IDENTREPLY, None, None)
 
     def handle_describe(self, conn, specifier, data):
@@ -372,3 +374,12 @@ class Dispatcher:
             self._active_connections.discard(conn)
             # XXX: also check all entries in self._subscriptions?
         return (DISABLEEVENTSREPLY, None, None)
+
+    def close(self):
+        for conn in self._connections:
+            try:
+                # - may be used for the 'closed' message in serial interface
+                # - is used in frappy history for indicating the close time
+                conn.close_message((ERRORCLOSED, None, None))
+            except AttributeError:
+                pass

secop/protocol/messages.py

@@ -62,6 +62,8 @@ HEARTBEATREPLY = 'pong'  # +nonce_without_space
 
 ERRORPREFIX = 'error_'  # + specifier + json_extended_info(error_report)
 
+ERRORCLOSED = 'error_closed'
+
 HELPREQUEST = 'help'  # literal
 HELPREPLY = 'helping'  # +line number +json_text
 

secop/server.py

@@ -35,7 +35,6 @@ from collections import OrderedDict
 from secop.errors import ConfigError, SECoPError
 from secop.lib import formatException, get_class, getGeneralConfig
 from secop.modules import Attached
-from secop.params import PREDEFINED_ACCESSIBLES
 
 try:
     from daemon import DaemonContext
@@ -207,7 +206,11 @@ class Server:
                 self.log.info('startup done, handling transport messages')
                 if systemd:
                     systemd.daemon.notify("READY=1\nSTATUS=accepting requests")
-                self.interface.serve_forever()
+                try:
+                    self.interface.serve_forever()
+                except KeyboardInterrupt as e:
+                    self._restart = False
+                self.dispatcher.close()
                 self.interface.server_close()
             if self._restart:
                 self.restart_hook()
@@ -329,11 +332,8 @@ class Server:
         self.log.info('all modules and pollers started')
         history_path = os.environ.get('FRAPPY_HISTORY')
         if history_path:
-            from secop_psi.historywriter import FrappyHistoryWriter  # pylint: disable=import-outside-toplevel
+            from secop_psi.historywriter import add_writer  # pylint: disable=import-outside-toplevel
-            writer = FrappyHistoryWriter(history_path, PREDEFINED_ACCESSIBLES.keys(), self.dispatcher)
+            add_writer(history_path, self)
-            # treat writer as a connection
-            self.dispatcher.add_connection(writer)
-            writer.init(self.dispatcher.handle_describe(writer, None, None))
         # TODO: if ever somebody wants to implement an other history writer:
         # - a general config file /etc/secp/secop.conf or <frappy repo>/etc/secop.conf
         #   might be introduced, which contains the log, pid and cfg directory path and

secop_psi/historywriter.py

@@ -19,127 +19,214 @@
 # *****************************************************************************
 
 import time
-import frappyhistory  # pylint: disable=import-error
+try:
-from secop.datatypes import get_datatype, IntRange, FloatRange, ScaledInteger,\
+    import frappyhistory  # pylint: disable=import-error
-    EnumType, BoolType, StringType, TupleOf, StructOf
+except ImportError:
+    pass  # do not complain when used for tests
+from secop.lib import clamp, formatExtendedTraceback
+from secop.datatypes import IntRange, FloatRange, ScaledInteger,\
+    EnumType, BoolType, StringType, TupleOf, StructOf, ArrayOf, TextType
 
 
-def make_cvt_list(dt, tail=''):
+def make_cvt_list(dt, tail):
     """create conversion list
 
     list of tuple (<conversion function>, <tail>, <curve options>)
     tail is a postfix to be appended in case of tuples and structs
     """
-    if isinstance(dt, (EnumType, IntRange, BoolType)):
+    if isinstance(dt, (IntRange, BoolType)):
-        return[(int, tail, dict(type='NUM'))]
+        return [(int, {'key': tail})]
+    if isinstance(dt, EnumType):
+        return [(int, {'key': tail, 'enum': dt.export_datatype()['members']})]
     if isinstance(dt, (FloatRange, ScaledInteger)):
-        return [(dt.import_value, tail, dict(type='NUM', unit=dt.unit, period=5) if dt.unit else {})]
+        opts = {'key': tail}
+        if dt.unit:
+            opts['group'] = dt.unit
+        opts['stepped'] = True
+        return [(dt.import_value, opts)]
     if isinstance(dt, StringType):
-        return [(lambda x: x, tail, dict(type='STR'))]
+        opts = {'key': tail, 'kind': 'STR'}
+        if isinstance(dt, TextType):
+            opts['category'] = 'no'
+        return [(lambda x: x, opts)]
     if isinstance(dt, TupleOf):
-        items = enumerate(dt.members)
+        result = []
-    elif isinstance(dt, StructOf):
+        for index, elmtype in enumerate(dt.members):
-        items = dt.members.items()
+            for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, index)):
-    else:
+                def conv(value, key=index, func=fun):
-        return []   # ArrayType, BlobType and TextType are ignored: too much data, probably not used
+                    return func(value[key])
-    result = []
+                result.append((conv, opts))
-    for subkey, elmtype in items:
+        return result
-        for fun,  tail_, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)):
+    if isinstance(dt, ArrayOf):
-            def conv(value, key=subkey, func=fun):
+        result = []
-                try:
+        for index in range(dt.maxlen):
-                    return value[key]
+            for fun, opts in make_cvt_list(dt.members, '%s.%s' % (tail, index)):
-                except KeyError:  # can not use value.get() because value might be a list
+                opts['category'] = 'no'
-                    return None
+
-            result.append((conv, tail_, opts))
+                def conv(value, key=index, func=fun):
-    return result
+                    return func(value[key])
+                result.append((conv, opts))
+        return result
+    if isinstance(dt, StructOf):
+        result = []
+        for subkey, elmtype in dt.members.items():
+            for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)):
+                def conv(value, key=subkey, func=fun):
+                    return func(value.get(key))  # None for missing struct key, should not be needed
+                result.append((conv, opts))
+        return result
+    return []   # other types (BlobType) are ignored: too much data, probably not used
 
 
-class FrappyHistoryWriter(frappyhistory.FrappyWriter):
+class FrappyAbstractHistoryWriter:
-    """extend writer to be used as an internal frappy connection
+    """abstract writer
 
-    API of frappyhistory.FrappyWriter:
+    doc only
-
-    :meth:`put_def`(key, opts):
-
-       define or overwrite a new curve named <key> with options from dict <opts>
-       options:
-
-          - type:
-              'NUM' (any number) or 'STR' (text)
-              remark: tuples and structs create multiple curves
-          - period:
-              the typical 'lifetime' of a value.
-              The intention is, that points in a chart may be connected by a straight line
-              when the distance is lower than twice this value. If not, the line should be
-              drawn horizontally from the last point to a point <period> before the next value.
-              For example a setpoint should have period 0, which will lead to a stepped
-              line, whereas for a measured value like a temperature, period should be
-              equal to the poll interval. In order to make full use of this,
-              we would need some additional parameter property.
-          - show: True/False, whether this curve should be shown or not by default in
-              a summary chart
-          - label: a label for the curve in the chart
-
-    :meth:`put`(timestamp, key, value)
-
-        timestamp: the timestamp. must not decrease!
-        key: the curve name
-        value: the value to be stored, converted to a string. '' indicates an undefined value
-
-    self.cache is a dict <key> of <value as string>, containing the last used value
     """
-    def __init__(self, directory, predefined_names, dispatcher):
+
-        super().__init__(directory)
+    def put_def(self, key, kind='NUM', category='minor', **opts):
-        self.predefined_names = predefined_names
+        """define or overwrite a new curve named <key> with options from dict <opts>
+
+        :param key: the key for the curve
+        :param kind: 'NUM' (default) for numeric values, 'STR' for strings
+        :param category: 'major' or 'minor': importance of curve
+        :param opts: a dict containing some of the following options
+
+          - label: a label for the curve in the chart
+          - group: grouping of the curves in charts (often equal to unit)
+          - stepped: lines in charts should be drawn as stepped line. Only applicable when kind='NUM'
+                     True by default.
+        """
+        raise NotImplementedError()
+
+    def put(self, timestamp, key, value):
+        """add a data point
+
+        :param timestamp:  the timestamp. must not decrease!
+        :param key:  the curve name
+        :param value:  the value to be stored (number or string), None indicates un undefined value
+        """
+        raise NotImplementedError()
+
+    def get(self, key):
+        """get from cache
+
+        :param key:  the curve name
+        :returns:  the last stored value or None
+        """
+        raise NotImplementedError()
+
+    def close(self, timestamp):
+        """close the writer
+
+        :param timestamp:
+        indicate to the writer that all values are getting undefined after <timestamp>
+        """
+        raise NotImplementedError()
+
+
+class FrappyHistoryHandler:
+    def __init__(self, writer, modules, dispatcher, logger=None):
+        self.writer = writer
+        self.log = logger
         self.cvt_lists = {}  # dict <mod:param> of <conversion list>
-        self.activated = False
         self.dispatcher = dispatcher
-        self._init_time = None
-
-    def init(self, msg):
-        """initialize from the 'describing' message"""
-        action, _, description = msg
-        assert action == 'describing'
         self._init_time = time.time()
-
+        self._last_time = self._init_time
-        for modname, moddesc in description['modules'].items():
+        for modname, modobj in modules.items():
-            for pname, pdesc in moddesc['accessibles'].items():
+            for pname, pobj in modobj.parameters.items():
-                ident = key = modname + ':' + pname
+                ident = '%s:%s' % (modname, pobj.export)
-                if pname.startswith('_') and pname[1:] not in self.predefined_names:
+                key = '%s:%s' % (modname, pname)
-                    key = modname + ':' + pname[1:]
+                dt = pobj.datatype
-                dt = get_datatype(pdesc['datainfo'])
                 cvt_list = make_cvt_list(dt, key)
-                for _, hkey, opts in cvt_list:
+
-                    if pname == 'value':
+                # create default opts
-                        opts['period'] = opts.get('period', 0)
+                if pobj.history_category:
-                        opts['show'] = True
+                    values = pobj.history_category.split(',')
-                        opts['label'] = modname
+                else:
-                    elif pname == 'target':
+                    values = [None]
-                        opts['period'] = 0
+                if len(values) == 1:
-                        opts['label'] = modname + '_target'
+                    values *= len(cvt_list)
-                        opts['show'] = True
+                for cat, (_, opts) in zip(values, cvt_list):
-                    self.put_def(hkey, opts)
+                    if cat is None:
-                self.cvt_lists[ident] = cvt_list
+                        cat = opts.get('category', 'major' if pname in ('value', 'target') else 'minor')
-        # self.put(self._init_time, 'STR', 'vars', ' '.join(vars))
+                    opts['category'] = cat
+                if pname == 'value':
+                    for _, opts in cvt_list:
+                        opts['key'] = opts['key'].replace(':value', '')
+                if pname == 'status':
+                    # default labels '<modname>:status' and '<modname>:status_text'
+                    for lbl, (_, opts) in zip([key, key + '_text'], cvt_list):
+                        opts['label'] = lbl
+                # overwrite opts based on history_* properties
+                if pobj.history_label:
+                    for lbl, (_, opts) in zip(','.split(pobj.history_label), cvt_list):
+                        opts['label'] = lbl
+                if pobj.history_group:
+                    values = pobj.history_group.split(',')
+                    if len(values) == 1:
+                        values *= len(cvt_list)
+                    for grp, (_, opts) in zip(values, cvt_list):
+                        opts['group'] = grp
+                if pobj.history_stepped:
+                    values = pobj.history_stepped
+                elif pobj.readonly:
+                    values = False
+                if not isinstance(values, tuple):
+                    values = [values] * len(cvt_list)
+                for stp, (_, opts) in zip(values, cvt_list):
+                    if not stp and 'stepped' in opts:  # only on floats
+                        opts['stepped'] = False
+                cvt_list = [(key, opts) for key, opts in cvt_list if opts.get('category') != 'no']
+                for _, opts in cvt_list:
+                    if opts.get('stepped'):
+                        opts.pop('stepped', None)
+                    writer.put_def(**opts)
+
+                if cvt_list:
+                    self.cvt_lists[ident] = cvt_list
+
         self.dispatcher.handle_activate(self, None, None)
         self._init_time = None
 
-    def send_reply(self, msg):
+    def close_message(self, msg):
-        action, ident, value = msg
+        self.writer.close(time.time())
-        if not action.endswith('update'):
-            print('unknown async message %r' % msg)
-            return
-        now = self._init_time or time.time()  # on initialisation, use the same timestamp for all
-        if action == 'update':
-            for fun, key, _ in self.cvt_lists[ident]:
-                # we only look at the value, qualifiers are ignored for now
-                # we do not use the timestamp here, as a potentially decreasing value might
-                # bring the reader software into trouble
-                self.put(now, key, str(fun(value[0])))
 
-        else:  # error_update
+    def send_reply(self, msg):
-            for _, key, _ in self.cvt_lists[ident]:
+        try:
-                old = self.cache.get(key)
+            action, ident, value = msg
-                if old is None:
+            assert action.endswith('update')
-                    return  # ignore if this key is not yet used
+            cvt_list = self.cvt_lists.get(ident)
-                self.put(now, key, '')
+            if not cvt_list:
+                return
+            if self._init_time:
+                t = self._init_time  # on initialisation, use the same timestamp for all
+            else:
+                t = value[1].get('t')
+                if t:
+                    # make sure time stamp is not decreasing, as a potentially decreasing
+                    # value might bring the reader software into trouble
+                    t = clamp(self._last_time, t, time.time())
+                else:
+                    t = time.time()
+            self._last_time = t
+            if action == 'update':
+                for fun, opts in cvt_list:
+                    # we only look at the value, qualifiers are ignored for now
+                    # we do not use the timestamp here, as a potentially decreasing value might
+                    # bring the reader software into trouble
+                    # print('UPDATE', key, value, t)
+                    self.writer.put(t, opts['key'], fun(value[0]))
+            else:  # error_update
+                for _, opts in cvt_list:
+                    self.writer.put(t, opts['key'], None)
+        except Exception as e:
+            self.log.error('FrappyHistoryHandler.send_reply: %r with msg %r: ', repr(e), msg)
+            print(formatExtendedTraceback())
+
+
+def add_writer(history_path, srv):
+    # treat handler as a connection
+    logger = srv.log.getChild('history')
+    srv.dispatcher.add_connection(FrappyHistoryHandler(
+        frappyhistory.Writer(history_path, logger), srv.modules, srv.dispatcher, logger))

secop_psi/ppmssim.py

@@ -187,7 +187,7 @@ class PpmsSim:
         self.i1 = self.t % 10.0
         self.r2 = 1000 / self.t
         self.i2 = math.log(self.t)
-        self.level.value = 100 - (self.time - self.start) * 0.01 % 100
+        self.level.value = round(100 - (self.time - self.start) * 0.01 % 100, 1)
         # print('PROGRESS T=%.7g B=%.7g x=%.7g' % (self.t, self.mf, self.pos))
 
     def getdat(self, mask):