enhance logging

- bin/secop-server options -v and -q applied to console logger only
- level for logfile taken from general config
- option for automatic deletion of old logfiles
- added 'comlog' level (between debug and info)

This allows to run the servers by default with 'comlog' level on
the logfiles, which helps a lot for analyzing very rare communication
errors in retrospect.

to avoid spamming of the normal log files, comlog data is stored
separately, one file per communicator

+ redesign of remote logging (no more need of LoggerAdapter)

Change-Id: Ie156a202b1e7304e50bbe830901bc75872f6ffe2
Reviewed-on: https://forge.frm2.tum.de/review/c/sine2020/secop/playground/+/27427
Tested-by: Jenkins Automated Tests <pedersen+jenkins@frm2.tum.de>
Reviewed-by: Markus Zolliker <markus.zolliker@psi.ch>

secop/logging.py

@@ -20,11 +20,20 @@
 # *****************************************************************************
 
 
-from logging import LoggerAdapter
-from mlzlog import LOGLEVELS
+import os
+from os.path import dirname, join
+from logging import DEBUG, INFO, addLevelName
+import mlzlog
+
+from secop.lib import generalConfig
+from secop.datatypes import BoolType
+from secop.properties import Property
 
 OFF = 99
-LOG_LEVELS = dict(LOGLEVELS, off=OFF)
+COMLOG = 15
+addLevelName(COMLOG, 'COMLOG')
+assert DEBUG < COMLOG < INFO
+LOG_LEVELS = dict(mlzlog.LOGLEVELS, off=OFF, comlog=COMLOG)
 LEVEL_NAMES = {v: k for k, v in LOG_LEVELS.items()}
 
 
@@ -39,28 +48,120 @@ def check_level(level):
     raise ValueError('%r is not a valid level' % level)
 
 
-class Adapter(LoggerAdapter):
-    def __init__(self, modobj):
-        super().__init__(modobj.log, {})
-        self.subscriptions = {}  # dict [conn] of level
-        self.modobj = modobj
+class RemoteLogHandler(mlzlog.Handler):
+    """handler for remote logging"""
+    def __init__(self):
+        super().__init__()
+        self.subscriptions = {}  # dict[modname] of tuple(mobobj, dict [conn] of level)
 
-    def log(self, level, msg, *args, **kwargs):
-        super().log(level, msg, *args, **kwargs)
-        for conn, lev in self.subscriptions.items():
-            if level >= lev:
-                self.modobj.DISPATCHER.send_log_msg(
-                    conn, self.modobj.name, LEVEL_NAMES[level], msg % args)
+    def emit(self, record):
+        """unused"""
 
-    def set_log_level(self, conn, level):
+    def handle(self, record):
+        modname = record.name.split('.')[-1]
+        try:
+            modobj, subscriptions = self.subscriptions[modname]
+        except KeyError:
+            return
+        for conn, lev in subscriptions.items():
+            if record.levelno >= lev:
+                modobj.DISPATCHER.send_log_msg(
+                    conn, modobj.name, LEVEL_NAMES[record.levelno],
+                    record.getMessage())
+
+    def set_conn_level(self, modobj, conn, level):
         level = check_level(level)
+        modobj, subscriptions = self.subscriptions.setdefault(modobj.name, (modobj, {}))
         if level == OFF:
-            self.subscriptions.pop(conn, None)
+            subscriptions.pop(conn, None)
         else:
-            self.subscriptions[conn] = level
+            subscriptions[conn] = level
+
+    def __repr__(self):
+        return 'RemoteLogHandler()'
 
 
-def set_log_level(modobj, conn, level):
-    if not isinstance(modobj.log, Adapter):
-        modobj.log = Adapter(modobj)
-    modobj.log.set_log_level(conn, level)
+class LogfileHandler(mlzlog.LogfileHandler):
+
+    def __init__(self, logdir, rootname, max_days=0):
+        self.logdir = logdir
+        self.rootname = rootname
+        self.max_days = max_days
+        super().__init__(logdir, rootname)
+
+    def emit(self, record):
+        if record.levelno != COMLOG:
+            super().emit(record)
+
+    def doRollover(self):
+        super().doRollover()
+        if self.max_days:
+            # keep only the last max_days files
+            with os.scandir(dirname(self.baseFilename)) as it:
+                files = sorted(entry.path for entry in it if entry.name != 'current')
+            for filepath in files[-self.max_days:]:
+                os.remove(filepath)
+
+
+class ComLogfileHandler(LogfileHandler):
+    """handler for logging communication"""
+
+    def format(self, record):
+        return '%s %s' % (self.formatter.formatTime(record), record.getMessage())
+
+
+class HasComlog:
+    """mixin for modules with comlog"""
+    comlog = Property('whether communication is logged ', BoolType(),
+                      default=True, export=False)
+    _comLog = None
+
+    def earlyInit(self):
+        super().earlyInit()
+        if self.comlog and generalConfig.initialized and generalConfig.comlog:
+            self._comLog = mlzlog.Logger('COMLOG.%s' % self.name)
+            self._comLog.handlers[:] = []
+            directory = join(logger.logdir, logger.rootname, 'comlog', self.DISPATCHER.name)
+            self._comLog.addHandler(ComLogfileHandler(
+                directory, self.name, max_days=generalConfig.getint('comlog_days', 7)))
+            return
+
+    def comLog(self, msg, *args, **kwds):
+        self.log.log(COMLOG, msg, *args, **kwds)
+        if self._comLog:
+            self._comLog.info(msg, *args)
+
+
+class MainLogger:
+    def __init__(self):
+        self.log = None
+        self.logdir = None
+        self.rootname = None
+        self.console_handler = None
+
+    def init(self, console_level='info'):
+        self.rootname = generalConfig.get('logger_root', 'frappy')
+        # set log level to minimum on the logger, effective levels on the handlers
+        # needed also for RemoteLogHandler
+        # modified from mlzlog.initLogging
+        mlzlog.setLoggerClass(mlzlog.MLZLogger)
+        assert self.log is None
+        self.log = mlzlog.log = mlzlog.MLZLogger(self.rootname)
+
+        self.log.setLevel(DEBUG)
+        self.log.addHandler(mlzlog.ColoredConsoleHandler())
+
+        self.logdir = generalConfig.get('logdir', '/tmp/log')
+        if self.logdir:
+            logfile_days = generalConfig.getint('logfile_days')
+            logfile_handler = LogfileHandler(self.logdir, self.rootname, max_days=logfile_days)
+            if generalConfig.logfile_days:
+                logfile_handler.max_days = int(generalConfig.logfile_days)
+            logfile_handler.setLevel(LOG_LEVELS[generalConfig.get('logfile_level', 'info')])
+            self.log.addHandler(logfile_handler)
+
+        self.log.addHandler(RemoteLogHandler())
+        self.log.handlers[0].setLevel(LOG_LEVELS[console_level])
+
+
+logger = MainLogger()