changed started_callback mechanism

- using threading.Events instead of Queue
- started_callback has no more argument
- introduced timeout for starting modules

Change-Id: I5a8b59cf552918cf7e61ae93cda907f7b0d97836
Reviewed-on: https://forge.frm2.tum.de/review/20281
Tested-by: JenkinsCodeReview <bjoern_pedersen@frm2.tum.de>
Reviewed-by: Enrico Faulhaber <enrico.faulhaber@frm2.tum.de>

secop/modules.py

@@ -223,11 +223,12 @@ class Module(object):
         '''runs after init of all modules
 
         started_callback to be called when thread spawned by late_init
-        or, if not implmemented, immediately
+        or, if not implemented, immediately
+        might return a timeout value, if different from default
         '''
 
         self.log.debug('empty %s.start_module()' % self.__class__.__name__)
-        started_callback(self)
+        started_callback()
 
 
 class Readable(Module):
@@ -273,7 +274,7 @@ class Readable(Module):
             except Exception as e:
                 self.log.exception(e)
                 self.status = (self.Status.ERROR, 'polling thread could not start')
-                started_callback(self)
+                started_callback()
                 print(formatException(0, sys.exc_info(), verbose=True))
                 time.sleep(10)
 
@@ -281,7 +282,7 @@ class Readable(Module):
         """super simple and super stupid per-module polling thread"""
         i = 0
         fastpoll = self.poll(i)
-        started_callback(self)
+        started_callback()
         while True:
             i += 1
             try:

secop/server.py

@@ -39,12 +39,6 @@ try:
 except ImportError:
     import ConfigParser as configparser  # py2
 
-try:
-    from queue import Queue # py 3
-except ImportError:
-    from Queue import Queue # py 2
-
-
 try:
     import daemon.pidlockfile as pidlockfile
 except ImportError:
@@ -198,18 +192,14 @@ class Server(object):
         for modname, modobj in self.modules.items():
             modobj.init_module()
 
-        starting_modules = set()
-        finished_modules = Queue()
+        start_events = []
         for modname, modobj in self.modules.items():
-            starting_modules.add(modobj)
-            modobj.start_module(started_callback=finished_modules.put)
-        # remark: it is the module implementors responsibility to call started_callback
-        # within reasonable time (using timeouts). If we find later, that this is not
-        # enough, we might insert checking for a timeout here, and somehow set the remaining
-        # starting_modules to an error state.
-        while starting_modules:
-            finished = finished_modules.get()
-            self.log.info(u'%s has started' % finished.name)
-            # use discard instead of remove here, catching the case when started_callback is called twice
-            starting_modules.discard(finished)
-            finished_modules.task_done()
+            event = threading.Event()
+            # start_module must return either a timeout value or None (default 30 sec)
+            timeout = modobj.start_module(started_callback=event.set) or 30
+            start_events.append((time.time() + timeout, modname, event))
+        self.log.info(u'waiting for modules being started')
+        for deadline, modname, event in sorted(start_events):
+            if not event.wait(timeout=max(0, deadline - time.time())):
+                self.log.info('WARNING: timeout when starting module %s' % modname)
+        self.log.info(u'all modules started')