major improvements and rework

- add stream / instrument availability data
- events contain event kind for dispatching db methods

streams.py

@@ -56,7 +56,7 @@ class Stream(Base):
         self.encoding = encoding
         self.timeout = timeout
         self.socket = None
-        self.cache = {}
+        self.cache = {}  # dict <key> of event
         self.errors = {}
         self.start_time = time.time()
         self.next_hour = (self.start_time // 3600 + 1) * 3600
@@ -187,6 +187,11 @@ class Stream(Base):
                 break
 
     def event_generator(self):
+        """a generator returning events
+
+        events are (<kind>, <value>, <key>, <tags>, <timestamp>)
+        kind is one of 'error', 'value', 'stream'
+        """
         raise NotImplementedError
 
     def get_tags(self, key):
@@ -196,9 +201,8 @@ class Stream(Base):
     def finish_events(self, events, end_time):
         for key in list(self.cache):
             self.cache.pop(key)
-            dbkey = '.'.join(key)
-            events.append((True, None, end_time, dbkey, 'float', self.tags))
-            events.append((False, 'END', end_time, dbkey, 'error', self.tags))
+            events.append(('value', None, key, self.tags, end_time))
+            events.append(('error', 'END', key, self.tags, end_time))
 
     def get_events(self, events, maxevents):
         """get available events
@@ -208,23 +212,23 @@ class Stream(Base):
             there might be more after a full hour or when the stream is dying
         :return: True when maxevents is reached
         """
-        for key, value, error, ts, tags in self.generator:
-            ts = max(self.start_time, min(ts or INF, time.time()))
-            if ts >= self.next_hour:
-                ts_ = (ts // 3600) * 3600
-                for key_, value_ in self.cache.items():
-                    events.append((True, value_, ts_, '.'.join(key_), 'float', self.get_tags(key_)))
-                for key_, error_ in self.errors.items():
-                    events.append((False, error_, ts_, '.'.join(key_), 'error', self.get_tags(key_)))
-                self.next_hour = ts_ + 3600
-            if value != self.cache.get(key, None) or error != self.errors.get(key, None):
-                dbkey = '.'.join(key)
-                events.append((True, value, ts, dbkey, 'float', tags))
-                self.cache[key] = value
-                if error and self.errors.get(key) != error:
-                    events.append((False, error, ts, dbkey, 'error', tags))
-                    self.errors[key] = error
-                elif len(events) >= maxevents:
+        for event in self.generator:
+            kind, value, key, tags, ts = event
+            timestamp = max(self.start_time, min(ts or INF, time.time()))
+            if timestamp >= self.next_hour:
+                t = (timestamp // 3600) * 3600
+                events.extend(e[:-1] + (t,) for e in self.cache.values())
+                self.next_hour = ts + 3600
+            prev = self.cache[key][:2] if key in self.cache else (None, None)
+            if (kind, value) != prev:
+                if kind == 'error':
+                    if prev[0] == 'value':
+                        events.append(('value', None, key, tags, timestamp))
+                    self.cache[key] = event
+                elif kind == 'value':
+                    self.cache[key] = event
+                events.append(event)
+                if len(events) >= maxevents:
                     return True
         else:
             if self.dead:
@@ -262,17 +266,25 @@ class EventStream:
             for stream in self.wait_ready(1):
                 if not isinstance(stream, Stream):
                     for streamcls, uri, kwargs in stream.events():
-                        if uri not in self.streams:
+                        stream = self.streams.get(uri)
+                        if stream:
+                            stream.tags.update(kwargs)
+                        else:
                             try:
-                                self.streams[uri] = streamcls(uri, **kwargs)
+                                self.streams[uri] = stream = streamcls(uri, **kwargs)
                                 print('added stream', uri, kwargs)
                             except Exception as e:
                                 print('can not connect to', uri, repr(e))
+                                continue
+                        events.append(('stream', kwargs.get('instrument', '0'),
+                                       {}, uri, int(time.time())))
             for name, stream in self.streams.items():
                 try:
                     if stream.get_events(events, maxevents):
                         return events
                 except StreamDead:
+                    # indicate stream is removed
+                    events.append(('stream', None, {}, uri, int(time.time())))
                     self.streams.pop(name)
                     if events:
                         return events
@@ -285,4 +297,5 @@ class EventStream:
         for stream in self.streams.values():
             stream.close()
             stream.finish_events(events, end_time)
+            events.append(('stream', None, {}, stream.uri, end_time))
         return events