better caputlog output

iocBoot/ioccaputlog/st.cmd

@@ -5,7 +5,7 @@
 dbLoadDatabase("../../dbd/softIocPy.dbd",0,0)
 softIocPy_registerRecordDeviceDriver(pdbbase) 
 
-dbLoadRecords("../../db/logwatch.db","N=logrec,FNAME=/tmp/testlog")
+dbLoadRecords("../../db/logwatch.db","N=ACC-CT{}Log-I,FNAME=/var/log/epics/epics.log,FILTER=logwatch.caputlog")
 
 iocInit()
 

logApp/Db/logwatch.db

@@ -5,5 +5,6 @@ record(waveform, "$(N)") {
   field(SCAN, "I/O Intr")
   field(FTVL, "CHAR")
   field(NELM, "$(NELM=200)")
+  field(TSE , "-2")
   info("logfilter","$(FILTER=)")
 }

logApp/src/Makefile

@@ -14,7 +14,8 @@ LOADABLE_LIBRARY_HOST = _inotifyy
 _inotifyy_SRCS += inotify_wrap.c
 
 PY += inotifyy.py
-PY += logwatch.py
+PY += logwatch/__init__.py
+PY += logwatch/caputlog.py
 
 #===========================
 

logApp/src/logwatch/__init__.py

@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 from __future__ import print_function
 
-import os.path, errno
+import os.path, errno, time
 
 import numpy as np
 
 import inotifyy as inot
 
 from devsup.hooks import addHook
-from devsup.util import StoppableThread
+from devsup.util import importmod, StoppableThread
 from devsup.db import IOScanListThread
 
 mask=inot.IN_CREATE|inot.IN_DELETE|inot.IN_MOVED_FROM|inot.IN_MODIFY
@@ -34,7 +34,14 @@ class LogWatcher(StoppableThread):
         self.arr = rec.field('VAL').getarray()
         self.fd = None
 
+        filt = rec.info('logfilter',"")
+        if filt:
+            filt = importmod(filt)
+            filt = filt.filter(self.fname)
+        self.filt = filt
+
         print(rec, 'will watch', self.fname)
+        print(rec, 'filtered with',self.filt)
 
     def detach(self, rec):
         pass
@@ -42,11 +49,17 @@ class LogWatcher(StoppableThread):
     def process(self, rec, reason=None):
         if reason is None:
             return
+        ts, reason = reason
         buf = np.frombuffer(reason, dtype=self.arr.dtype)
         buf = buf[:rec.NELM-1]
         self.arr[:buf.size] = buf
         self.arr[buf.size] = 0
         rec.NORD = buf.size+1
+        
+        if ts:
+            rec.setTime(ts)
+        else:
+            rec.setTime(time.time())
 
     def join(self):
         print("Stopping logger for",self.fname)
@@ -82,6 +95,7 @@ class LogWatcher(StoppableThread):
         self.closefile()
         try:
             self.fd = open(self.fname, 'r')
+            self.pos = self.fd.tell()
         except IOError, e:
             if e.errno==errno.ENOENT:
                 return
@@ -95,14 +109,12 @@ class LogWatcher(StoppableThread):
     def catfile(self):
         if not self.fd:
             return
-        op = self.fd.tell()
         self.fd.seek(0,2) # Seek end
         end = self.fd.tell()
-        if end < op:
+        if end < self.pos:
-            self.log("File size decreased, assuming truncation")
+            self.log("File size decreased from %d to %d, assuming truncation"%(self.pos,end))
-            self.buf = None
+            self.pos, self.buf = 0, None
-            op = 0
+        self.fd.seek(self.pos)
-        self.fd.seek(op,0)
 
         for L in self.fd.readlines():
             if L[-1]!='\n':
@@ -115,7 +127,12 @@ class LogWatcher(StoppableThread):
                 L, self.buf = self.buf+L, None
             self.log(L[:-1]) # Skip newline
 
+        self.pos = self.fd.tell()
+
     def log(self, msg):
-        self.scan.interrupt(reason=msg)
+        ts = None
+        if self.filt:
+            ts, msg = self.filt.apply(msg)
+        self.scan.interrupt(reason=(ts, msg))
 
 build = LogWatcher

logApp/src/logwatch/caputlog.py

@@ -0,0 +1,55 @@
+"""
+linacioc01.cs.nsls2.local:39907 Mon Dec  9 07:07:53 2013 09-Dec-13 07:07:48 diagioc-spc softioc LN-DG{SCR:6}In-Cmd.VAL new=0 old=0
+
+Input format is
+ iochost:port DoW Mon day H:M:S Year D-M-Y H:M:S userhost user PV msg
+"""
+
+import re, time
+
+R = re.compile(r"""
+# iochost:port
+ (\S+)\s
+# date 1 (in logger host TZ)
+ (?P<ts>\S+\s\S+\s+\S+\s\S+\s\S+)\s
+# date 2 (in IOC TZ)
+ (\S+\s\S+)\s
+# userhost
+ (?P<host>\S+)\s
+# username
+ (?P<user>\S+)\s
+# PV
+ (?P<pv>\S+)\s
+# message
+ (?P<msg>\S.*)
+""", re.VERBOSE)
+
+class CAPutLogFilter(object):
+    def __init__(self, fname):
+        self.fname = fname
+        self.noise = False
+
+    def __str__(self):
+        return '%s(%s)'%(str(type(self)), self.fname)
+    __repr__ = __str__
+
+    def apply(self, line):
+        M = R.match(line)
+        if not M:
+            # lines not matching the caputlog format
+            # are passed through verbatim
+            return None, line
+        D = M.groupdict()
+        try:
+            ts = time.mktime(time.strptime(D['ts'],'%a %b %d %H:%M:%S %Y'))
+            self.noise = False
+        except ValueError:
+            if not self.noise:
+                print("Failed to parse time",D['ts'])
+                self.noise = True
+            ts = None
+            raise
+        msg = "%(user)s %(host)s    %(pv)s %(msg)s"%D
+        return ts, msg
+
+filter = CAPutLogFilter