diff --git a/secop/params.py b/secop/params.py index 9add5a8..519c63e 100644 --- a/secop/params.py +++ b/secop/params.py @@ -27,7 +27,7 @@ import inspect from secop.datatypes import BoolType, CommandType, DataType, \ DataTypeType, EnumType, IntRange, NoneOr, OrType, \ - StringType, StructOf, TextType, TupleOf, ValueType + StringType, StructOf, TextType, TupleOf, ValueType, ArrayOf from secop.errors import BadValueError, ProgrammingError from secop.properties import HasProperties, Property @@ -163,6 +163,52 @@ class Parameter(Accessible): default None: write if given in config''', NoneOr(BoolType()), export=False, default=None, settable=False) + history_category = Property( + '''[custom] category for history + + major: should be shown by default in a history chart, default for value and target + minor: to be shown optionally in a history chart, default for other parameters + no: history is not saved. default for TextType and ArrayOf + + category is ignored (forced to no) for BlobType + + For structured types, the category may be a comma separated list, overwriting the + default for the first or all curves. + If it does not contain a comma, it applies for all curves + ''', + NoneOr(StringType()), export=True, default=None, settable=False) + history_label = Property( + '''[custom] label for history + + default: : or for main value + + For structured types, the label may be a comma separated list, overwriting the + default for the first or all curves. + If it does not contain a comma, it applies for the first curve only. + ''', + NoneOr(StringType()), export=True, default=None, settable=False) + history_group = Property( + '''[custom] group for history + + default: unit + + For structured types, the group may be a comma separated list, overwriting the + default for the first or all curves. If it does not contain a comma, it is + applies for all curves. + ''', + NoneOr(StringType()), export=True, default=None, settable=False) + history_stepped = Property( + '''[custom] stepped curve + + Whether a curve has to be drawn stepped or connected. + default: True when readonly=False, else False + + Applicable to FloatRange and ScaledInteger only, other types are stepped by definition. + + For structured types, stepped may be a list, overwriting the default for the + first or all curves. If not, applieas to all curves. + ''', + OrType(BoolType(), ArrayOf(BoolType())), export=True, default=False, settable=False) # used on the instance copy only value = None diff --git a/secop/protocol/dispatcher.py b/secop/protocol/dispatcher.py index 5075b4b..0f0b889 100644 --- a/secop/protocol/dispatcher.py +++ b/secop/protocol/dispatcher.py @@ -47,7 +47,8 @@ from secop.errors import NoSuchCommandError, NoSuchModuleError, \ from secop.params import Parameter from secop.protocol.messages import COMMANDREPLY, DESCRIPTIONREPLY, \ DISABLEEVENTSREPLY, ENABLEEVENTSREPLY, ERRORPREFIX, EVENTREPLY, \ - HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY + HEARTBEATREPLY, IDENTREPLY, IDENTREQUEST, READREPLY, WRITEREPLY, \ + ERRORCLOSED def make_update(modulename, pobj): @@ -297,6 +298,7 @@ class Dispatcher: self.log.error('should have been handled in the interface!') def handle__ident(self, conn, specifier, data): + self._active_connections.discard(conn) return (IDENTREPLY, None, None) def handle_describe(self, conn, specifier, data): @@ -372,3 +374,12 @@ class Dispatcher: self._active_connections.discard(conn) # XXX: also check all entries in self._subscriptions? return (DISABLEEVENTSREPLY, None, None) + + def close(self): + for conn in self._connections: + try: + # - may be used for the 'closed' message in serial interface + # - is used in frappy history for indicating the close time + conn.close_message((ERRORCLOSED, None, None)) + except AttributeError: + pass diff --git a/secop/protocol/messages.py b/secop/protocol/messages.py index 706b6cf..b6d08f4 100644 --- a/secop/protocol/messages.py +++ b/secop/protocol/messages.py @@ -62,6 +62,8 @@ HEARTBEATREPLY = 'pong' # +nonce_without_space ERRORPREFIX = 'error_' # + specifier + json_extended_info(error_report) +ERRORCLOSED = 'error_closed' + HELPREQUEST = 'help' # literal HELPREPLY = 'helping' # +line number +json_text diff --git a/secop/server.py b/secop/server.py index 833dc14..8f49de5 100644 --- a/secop/server.py +++ b/secop/server.py @@ -35,7 +35,6 @@ from collections import OrderedDict from secop.errors import ConfigError, SECoPError from secop.lib import formatException, get_class, getGeneralConfig from secop.modules import Attached -from secop.params import PREDEFINED_ACCESSIBLES try: from daemon import DaemonContext @@ -207,7 +206,11 @@ class Server: self.log.info('startup done, handling transport messages') if systemd: systemd.daemon.notify("READY=1\nSTATUS=accepting requests") - self.interface.serve_forever() + try: + self.interface.serve_forever() + except KeyboardInterrupt as e: + self._restart = False + self.dispatcher.close() self.interface.server_close() if self._restart: self.restart_hook() @@ -329,11 +332,8 @@ class Server: self.log.info('all modules and pollers started') history_path = os.environ.get('FRAPPY_HISTORY') if history_path: - from secop_psi.historywriter import FrappyHistoryWriter # pylint: disable=import-outside-toplevel - writer = FrappyHistoryWriter(history_path, PREDEFINED_ACCESSIBLES.keys(), self.dispatcher) - # treat writer as a connection - self.dispatcher.add_connection(writer) - writer.init(self.dispatcher.handle_describe(writer, None, None)) + from secop_psi.historywriter import add_writer # pylint: disable=import-outside-toplevel + add_writer(history_path, self) # TODO: if ever somebody wants to implement an other history writer: # - a general config file /etc/secp/secop.conf or /etc/secop.conf # might be introduced, which contains the log, pid and cfg directory path and diff --git a/secop_psi/historywriter.py b/secop_psi/historywriter.py index f4b0545..7f21c31 100644 --- a/secop_psi/historywriter.py +++ b/secop_psi/historywriter.py @@ -19,127 +19,214 @@ # ***************************************************************************** import time -import frappyhistory # pylint: disable=import-error -from secop.datatypes import get_datatype, IntRange, FloatRange, ScaledInteger,\ - EnumType, BoolType, StringType, TupleOf, StructOf +try: + import frappyhistory # pylint: disable=import-error +except ImportError: + pass # do not complain when used for tests +from secop.lib import clamp, formatExtendedTraceback +from secop.datatypes import IntRange, FloatRange, ScaledInteger,\ + EnumType, BoolType, StringType, TupleOf, StructOf, ArrayOf, TextType -def make_cvt_list(dt, tail=''): +def make_cvt_list(dt, tail): """create conversion list list of tuple (, , ) tail is a postfix to be appended in case of tuples and structs """ - if isinstance(dt, (EnumType, IntRange, BoolType)): - return[(int, tail, dict(type='NUM'))] + if isinstance(dt, (IntRange, BoolType)): + return [(int, {'key': tail})] + if isinstance(dt, EnumType): + return [(int, {'key': tail, 'enum': dt.export_datatype()['members']})] if isinstance(dt, (FloatRange, ScaledInteger)): - return [(dt.import_value, tail, dict(type='NUM', unit=dt.unit, period=5) if dt.unit else {})] + opts = {'key': tail} + if dt.unit: + opts['group'] = dt.unit + opts['stepped'] = True + return [(dt.import_value, opts)] if isinstance(dt, StringType): - return [(lambda x: x, tail, dict(type='STR'))] + opts = {'key': tail, 'kind': 'STR'} + if isinstance(dt, TextType): + opts['category'] = 'no' + return [(lambda x: x, opts)] if isinstance(dt, TupleOf): - items = enumerate(dt.members) - elif isinstance(dt, StructOf): - items = dt.members.items() - else: - return [] # ArrayType, BlobType and TextType are ignored: too much data, probably not used - result = [] - for subkey, elmtype in items: - for fun, tail_, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)): - def conv(value, key=subkey, func=fun): - try: - return value[key] - except KeyError: # can not use value.get() because value might be a list - return None - result.append((conv, tail_, opts)) - return result + result = [] + for index, elmtype in enumerate(dt.members): + for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, index)): + def conv(value, key=index, func=fun): + return func(value[key]) + result.append((conv, opts)) + return result + if isinstance(dt, ArrayOf): + result = [] + for index in range(dt.maxlen): + for fun, opts in make_cvt_list(dt.members, '%s.%s' % (tail, index)): + opts['category'] = 'no' + + def conv(value, key=index, func=fun): + return func(value[key]) + result.append((conv, opts)) + return result + if isinstance(dt, StructOf): + result = [] + for subkey, elmtype in dt.members.items(): + for fun, opts in make_cvt_list(elmtype, '%s.%s' % (tail, subkey)): + def conv(value, key=subkey, func=fun): + return func(value.get(key)) # None for missing struct key, should not be needed + result.append((conv, opts)) + return result + return [] # other types (BlobType) are ignored: too much data, probably not used -class FrappyHistoryWriter(frappyhistory.FrappyWriter): - """extend writer to be used as an internal frappy connection +class FrappyAbstractHistoryWriter: + """abstract writer - API of frappyhistory.FrappyWriter: - - :meth:`put_def`(key, opts): - - define or overwrite a new curve named with options from dict - options: - - - type: - 'NUM' (any number) or 'STR' (text) - remark: tuples and structs create multiple curves - - period: - the typical 'lifetime' of a value. - The intention is, that points in a chart may be connected by a straight line - when the distance is lower than twice this value. If not, the line should be - drawn horizontally from the last point to a point before the next value. - For example a setpoint should have period 0, which will lead to a stepped - line, whereas for a measured value like a temperature, period should be - equal to the poll interval. In order to make full use of this, - we would need some additional parameter property. - - show: True/False, whether this curve should be shown or not by default in - a summary chart - - label: a label for the curve in the chart - - :meth:`put`(timestamp, key, value) - - timestamp: the timestamp. must not decrease! - key: the curve name - value: the value to be stored, converted to a string. '' indicates an undefined value - - self.cache is a dict of , containing the last used value + doc only """ - def __init__(self, directory, predefined_names, dispatcher): - super().__init__(directory) - self.predefined_names = predefined_names + + def put_def(self, key, kind='NUM', category='minor', **opts): + """define or overwrite a new curve named with options from dict + + :param key: the key for the curve + :param kind: 'NUM' (default) for numeric values, 'STR' for strings + :param category: 'major' or 'minor': importance of curve + :param opts: a dict containing some of the following options + + - label: a label for the curve in the chart + - group: grouping of the curves in charts (often equal to unit) + - stepped: lines in charts should be drawn as stepped line. Only applicable when kind='NUM' + True by default. + """ + raise NotImplementedError() + + def put(self, timestamp, key, value): + """add a data point + + :param timestamp: the timestamp. must not decrease! + :param key: the curve name + :param value: the value to be stored (number or string), None indicates un undefined value + """ + raise NotImplementedError() + + def get(self, key): + """get from cache + + :param key: the curve name + :returns: the last stored value or None + """ + raise NotImplementedError() + + def close(self, timestamp): + """close the writer + + :param timestamp: + indicate to the writer that all values are getting undefined after + """ + raise NotImplementedError() + + +class FrappyHistoryHandler: + def __init__(self, writer, modules, dispatcher, logger=None): + self.writer = writer + self.log = logger self.cvt_lists = {} # dict of - self.activated = False self.dispatcher = dispatcher - self._init_time = None - - def init(self, msg): - """initialize from the 'describing' message""" - action, _, description = msg - assert action == 'describing' self._init_time = time.time() - - for modname, moddesc in description['modules'].items(): - for pname, pdesc in moddesc['accessibles'].items(): - ident = key = modname + ':' + pname - if pname.startswith('_') and pname[1:] not in self.predefined_names: - key = modname + ':' + pname[1:] - dt = get_datatype(pdesc['datainfo']) + self._last_time = self._init_time + for modname, modobj in modules.items(): + for pname, pobj in modobj.parameters.items(): + ident = '%s:%s' % (modname, pobj.export) + key = '%s:%s' % (modname, pname) + dt = pobj.datatype cvt_list = make_cvt_list(dt, key) - for _, hkey, opts in cvt_list: - if pname == 'value': - opts['period'] = opts.get('period', 0) - opts['show'] = True - opts['label'] = modname - elif pname == 'target': - opts['period'] = 0 - opts['label'] = modname + '_target' - opts['show'] = True - self.put_def(hkey, opts) - self.cvt_lists[ident] = cvt_list - # self.put(self._init_time, 'STR', 'vars', ' '.join(vars)) + + # create default opts + if pobj.history_category: + values = pobj.history_category.split(',') + else: + values = [None] + if len(values) == 1: + values *= len(cvt_list) + for cat, (_, opts) in zip(values, cvt_list): + if cat is None: + cat = opts.get('category', 'major' if pname in ('value', 'target') else 'minor') + opts['category'] = cat + if pname == 'value': + for _, opts in cvt_list: + opts['key'] = opts['key'].replace(':value', '') + if pname == 'status': + # default labels ':status' and ':status_text' + for lbl, (_, opts) in zip([key, key + '_text'], cvt_list): + opts['label'] = lbl + # overwrite opts based on history_* properties + if pobj.history_label: + for lbl, (_, opts) in zip(','.split(pobj.history_label), cvt_list): + opts['label'] = lbl + if pobj.history_group: + values = pobj.history_group.split(',') + if len(values) == 1: + values *= len(cvt_list) + for grp, (_, opts) in zip(values, cvt_list): + opts['group'] = grp + if pobj.history_stepped: + values = pobj.history_stepped + elif pobj.readonly: + values = False + if not isinstance(values, tuple): + values = [values] * len(cvt_list) + for stp, (_, opts) in zip(values, cvt_list): + if not stp and 'stepped' in opts: # only on floats + opts['stepped'] = False + cvt_list = [(key, opts) for key, opts in cvt_list if opts.get('category') != 'no'] + for _, opts in cvt_list: + if opts.get('stepped'): + opts.pop('stepped', None) + writer.put_def(**opts) + + if cvt_list: + self.cvt_lists[ident] = cvt_list + self.dispatcher.handle_activate(self, None, None) self._init_time = None - def send_reply(self, msg): - action, ident, value = msg - if not action.endswith('update'): - print('unknown async message %r' % msg) - return - now = self._init_time or time.time() # on initialisation, use the same timestamp for all - if action == 'update': - for fun, key, _ in self.cvt_lists[ident]: - # we only look at the value, qualifiers are ignored for now - # we do not use the timestamp here, as a potentially decreasing value might - # bring the reader software into trouble - self.put(now, key, str(fun(value[0]))) + def close_message(self, msg): + self.writer.close(time.time()) - else: # error_update - for _, key, _ in self.cvt_lists[ident]: - old = self.cache.get(key) - if old is None: - return # ignore if this key is not yet used - self.put(now, key, '') + def send_reply(self, msg): + try: + action, ident, value = msg + assert action.endswith('update') + cvt_list = self.cvt_lists.get(ident) + if not cvt_list: + return + if self._init_time: + t = self._init_time # on initialisation, use the same timestamp for all + else: + t = value[1].get('t') + if t: + # make sure time stamp is not decreasing, as a potentially decreasing + # value might bring the reader software into trouble + t = clamp(self._last_time, t, time.time()) + else: + t = time.time() + self._last_time = t + if action == 'update': + for fun, opts in cvt_list: + # we only look at the value, qualifiers are ignored for now + # we do not use the timestamp here, as a potentially decreasing value might + # bring the reader software into trouble + # print('UPDATE', key, value, t) + self.writer.put(t, opts['key'], fun(value[0])) + else: # error_update + for _, opts in cvt_list: + self.writer.put(t, opts['key'], None) + except Exception as e: + self.log.error('FrappyHistoryHandler.send_reply: %r with msg %r: ', repr(e), msg) + print(formatExtendedTraceback()) + + +def add_writer(history_path, srv): + # treat handler as a connection + logger = srv.log.getChild('history') + srv.dispatcher.add_connection(FrappyHistoryHandler( + frappyhistory.Writer(history_path, logger), srv.modules, srv.dispatcher, logger)) diff --git a/secop_psi/ppmssim.py b/secop_psi/ppmssim.py index 5ade05f..fd4686c 100644 --- a/secop_psi/ppmssim.py +++ b/secop_psi/ppmssim.py @@ -187,7 +187,7 @@ class PpmsSim: self.i1 = self.t % 10.0 self.r2 = 1000 / self.t self.i2 = math.log(self.t) - self.level.value = 100 - (self.time - self.start) * 0.01 % 100 + self.level.value = round(100 - (self.time - self.start) * 0.01 % 100, 1) # print('PROGRESS T=%.7g B=%.7g x=%.7g' % (self.t, self.mf, self.pos)) def getdat(self, mask):