change the meaning of pre for regsub slightly

GNUmakefile

@@ -1,3 +1,4 @@
+# Remove this file if not using the PSI build system
 include /ioc/tools/driver.makefile
 EXCLUDE_VERSIONS = 3.13.2
 PROJECT=stream
@@ -39,10 +40,7 @@ HEADERS += StreamFormatConverter.h
 HEADERS += StreamBuffer.h
 HEADERS += StreamError.h
 
-ifeq (${EPICS_BASETYPE},3.13)
-# old gcc needs full path for -include
-CXXFLAGS += -include $(foreach d,${INCLUDES:-I%=%},$(wildcard $d/compat3_13.h))
-else
+ifneq (${EPICS_BASETYPE},3.13)
 RECORDTYPES += calcout
 endif
 
@@ -52,7 +50,7 @@ streamReferences:
 	perl ../src/makeref.pl Interface $(BUSSES) > $@
 	perl ../src/makeref.pl Converter $(FORMATS) >> $@
 
-# have to hack a bit to work with both versions of driver.makefile
-DBDFILES = O.$${EPICSVERSION}_$${T_A}/streamSup.dbd
-../O.${EPICSVERSION}_${T_A}/streamSup.dbd:
+export DBDFILES = streamSup.dbd
+streamSup.dbd:
+	@echo Creating $@
 	perl ../src/makedbd.pl $(RECORDTYPES) > $@

doc/formats.html

@@ -367,7 +367,7 @@ endian</em>, i.e. least significant byte first.
 With the <code>0</code> flag, the value is unsigned, otherwise signed.
 </p>
 <p>
-In output, the <em>prec</em> (or sizeof(long) whatever is less) least
+In output, the <em>precision</em> (or sizeof(long) whatever is less) least
 significant bytes of the value are sign extended or zero extended
 (depending on the <code>0</code> flag) to <em>width</em> bytes.
 </p>
@@ -434,7 +434,7 @@ The <em>width</em> field is the byte number from which to start
 calculating the checksum.
 Default is 0, i.e. the first byte of the input or output of the current
 command.
-The last byte is <em>prec</em> bytes before the checksum (default 0).
+The last byte is <em>precision</em> bytes before the checksum (default 0).
 For example in <code>"abcdefg%&lt;xor&gt;"</code> the checksum is calculated
 from <code>abcdefg</code>,
 but in <code>"abcdefg%2.1&lt;xor&gt;"</code> only from <code>cdef</code>.
@@ -528,41 +528,44 @@ In input, the next byte or bytes must match the checksum.
 </dl>
 
 <a name="regex"></a>
-<h2>12. Regular Expresion STRING Converter (<code>%/<em>regex</em>/</code>)</h2>
+<h2>13. Regular Expresion STRING Converter (<code>%/<em>regex</em>/</code>)</h2>
 <p>
 This input-only format matches <a target="ex"
 href="http://www.pcre.org/" >Perl compatible regular expressions (PCRE)</a>.
 It is only available if a PCRE library is installed.
 </p>
+<div class="box">
 <p>
 If PCRE is not available for your host or cross architecture, download
 the sourcecode from <a target="ex" href="http://www.pcre.org/">www.pcre.org</a>
 and try my EPICS compatible <a target="ex"
 href="http://epics.web.psi.ch/software/streamdevice/pcre/Makefile">Makefile</a>
-to compile it like a normal EPICS application.
+to compile it like a normal EPICS support module.
 The Makefile is known to work with EPICS 3.14.8 and PCRE 7.2.
 In your RELEASE file define the variable <code>PCRE</code> so that
 it points to the install location of PCRE.
 </p>
 <p>
-If PCRE is already installed on your system, use the variables
-<code>PCRE_INCLUDE</code> and <code>PCRE_LIB</code> instead to provide
-the install directories of <code>pcre.h</code> and the library.
-</p>
-<p>
-If you have PCRE installed in different locations for different (cross)
-architectures, define the variables in RELEASE.Common.&lt;architecture&gt;
-instead of the global RELEASE file.
+If PCRE is already installed on (some of) your systems, you may add
+architectures where PCRE can be found in standard include and library
+locations to the variable <code>WITH_SYSTEM_PCRE</code>.
+If either the header file or the library are in a non-standard place,
+set in your RELEASE file the variables <code>PCRE_INCLUDE_<em>arch</em></code>
+and/or <code>PCRE_LIB_<em>arch</em></code> for the respective architectures
+to the correct directories or set
+<code>PCRE_INCLUDE</code> and/or <code>PCRE_LIB</code>
+in architecture specific RELEASE.Common.<em>arch</em> files.
 </p>
+</div>
 <p>
 If the regular expression is not anchored, i.e. does not start with
 <code>^</code>, leading non-matching input is skipped. 
 A maximum of <em>width</em> bytes is matched, if specified.
-If <em>prec</em> is given, it specifies the sub-expression whose match
+If <em>precision</em> is given, it specifies the sub-expression whose match
 is retuned.
 Otherwise the complete match is returned.
 In any case, the complete match is consumed from the input buffer.
-If the expression contains a <code>/</code> it must be escaped.
+If the expression contains a <code>/</code> it must be escaped like <code>\/</code>.
 </p>
 <p>
 Example: <code>%.1/&lt;title&gt;(.*)&lt;\/title&gt;/</code> returns
@@ -570,8 +573,74 @@ the title of an HTML page, skipps anything before the
 <code>&lt;title&gt</code> tag and leaves anything after the
 <code>&lt;/title&gt;</code> tag in the input buffer.
 </p>
+<a name="regsub"></a>
+<h2>14. Regular Expresion Substitution Pseudo-Converter (<code>%#/<em>regex</em>/<em>subst</em>/</code>)</h2>
+<p>
+This is a variant of the previous converter (note the <code>#</code>)
+but instead of returning the matching string,
+it can be used as a pre-processor for input or
+as a post-processor for output.
+</p>
+<p>
+Matches of the <em>regex</em> are replaced by the string <em>subst</em> with all
+<code>&</code> or <code>\0</code> in <em>subst</em> replaced with the match itself and all
+<code>\1</code> through <code>\9</code> replaced with the match of the corresponding sub-expression.
+To get a literal <code>&</code> or <code>\</code> or <code>/</code> in the substitution write
+<code>\&</code> or <code>\\</code> or <code>\/</code>.
+There is no way to specify literal bytes with values less or equal to 9 in the
+substitution!
+</p>
+<p>
+If <em>width</em> is specified, it limits the number of characters processed.
+If the <code>-</code> flag is used (i.e. <em>width</em> looks like a negative number)
+only the last <em>width</em> characters are processed, else the first.
+Without <em>width</em> (or 0) all available characters are processed.
+</p>
+<p>
+If <em>precision</em> is specified, it indicates which matches to replace.
+With the <code>+</code> flag given, <em>precision</em> is the maximum
+number of matches to replace.
+Otherwise <em>precision</em> is the index (counting from 1) of the match to replace. 
+Without <em>precision</em> (or 0), all matches are replaced.
+</p>
+<p>
+In input this converter pre-processes data received from the device before
+following converters read it.
+Converters preceding this one will read unmodified input.
+Thus place this converter before those whose input should be pre-processed.
+</p>
+<p>
+In output it post-processes data already formatted by preceding converters
+before sending it to the device.
+Converters following this one will send their output unmodified.
+Thus place this converter after those whose output should be post-processed.
+</p>
+<p>
+Examples:
+<div class="indent">
+<code>%#+-10.2/ab/X/</code> replaces the string <code>ab</code> with <code>X</code>
+maximal 2 times in the last 10 characters.
+(<code>abcabcabcabc</code> becomes <code>abcXcXcabc</code>)
+</div>
+<div class="indent">
+<code>%#/\\/\//</code> replaces all <code>\</code> with <code>/</code>
+(<code>\dir\file</code> becomes <code>/dir/file</code>)
+</div>
+<div class="indent">
+<code>%#/..\B/&:/</code> inserts <code>:</code> after every second character
+which is not at the end of a word.
+(<code>0b19353134</code> becomes <code>0b:19:35:31:34</code>)
+</div>
+<div class="indent">
+<code>%#/://</code> removes all <code>:</code> characters.
+(<code>0b:19:35:31:34</code> becomes <code>0b19353134</code>)
+</div>
+<div class="indent">
+<code>%#/([^+-])*([+-])/\2\1/</code> moves a postfix sign to the front.
+(<code>1.23-</code> becomes <code>-1.23</code>)<br>
+</div>
 <a name="mantexp"></a>
-<h2>13. MantissaExponent DOUBLE converter (<code>%m</code>)</h2>
+<h2>15. MantissaExponent DOUBLE converter (<code>%m</code>)</h2>
 <p>
 This exotic and experimental format matches numbers in the format
 <i>[sign] mantissa sign exponent</i>, e.g <code>+123-4</code> meaning
@@ -591,7 +660,7 @@ the usual way (always sign, left justified, space instead of + sign).
 Flags <code>#</code> and <code>0</code> are unsupported.
 </p>
 <a name="timestamp"></a>
-<h2>14. Timestamp DOUBLE converter (<code>%T(<em>timeformat</em>)</code>)</h2>
+<h2>16. Timestamp DOUBLE converter (<code>%T(<em>timeformat</em>)</code>)</h2>
 <p>
 This format reads or writes timestamps and converts them to a double number.
 The value represents the number of seconds since 1970 (the UNIX epoch).
@@ -628,7 +697,7 @@ In output, the system function <em>strftime()</em> is used to format the time.
 There may be differences in the implementation between operating systems.
 </p>
 <p>
-In input, <em>StreamDevice</em> used its own implementation because many
+In input, <em>StreamDevice</em> uses its own implementation because many
 systems are missing the <em>strptime()</em> function and additional formats
 are supported.
 </p>

doc/nav.html

@@ -132,6 +132,7 @@ div div div a {list-style-type:circle;}
   <a target="_parent" href="formats.html#bcd"       title="Binary coded decimal LONG converter">%D</a>
   <a target="_parent" href="formats.html#chksum"    title="Checksum pseudo converter">%&lt;<em>checksum</em>&gt;</a>
   <a target="_parent" href="formats.html#regex"     title="Perl regular expression STRING converter">%/<em>regex</em>/</a>
+  <a target="_parent" href="formats.html#regsub"    title="Perl regular expression substitution pseudo converter">%#/<em>regex</em>/<em>subst</em>/</a>
   <a target="_parent" href="formats.html#mantexp"   title="MantissaExponent DOUBLE converter">%m</a>
   <a target="_parent" href="formats.html#timestamp" title="Timestamp DOUBLE converter">%T</a>
  </div>

doc/stream.css

@@ -88,6 +88,16 @@ code {
     text-align:left;
 }
 
+.box {
+    margin-left:1ex;
+    margin-right:1ex;
+    margin-top:0.5ex;
+    padding: 0 1ex;
+    border: 1px solid black;
+    text-align:left;
+    background-color:#f0f0f0;
+}
+
 #navleft {
     position:fixed;
     left:0;

src/ChecksumConverter.cc

@@ -22,9 +22,21 @@
 #include "StreamError.h"
 #if defined(__vxworks) || defined(vxWorks) || defined(_WIN32) || defined(__rtems__)
 // These systems have no strncasecmp
+#include <epicsVersion.h>
+#ifdef BASE_VERSION
+// 3.13
+#include <ctype.h>
+static int strncasecmp(const char *s1, const char *s2, size_t n)
+{
+    int r=0;
+    while (n && (r = toupper(*s1)-toupper(*s2)) == 0) { n--; s1++; s2++; };
+    return r;
+}
+#else
 #include <epicsString.h>
 #define strncasecmp epicsStrnCaseCmp
 #endif
+#endif
 #include <ctype.h>
 
 typedef unsigned int (*checksumFunc)(const unsigned char* data, unsigned int len,  unsigned int init);

src/RegexpConverter.cc

@@ -23,7 +23,7 @@
 #include "string.h"
 #include "pcre.h"
 
-// Perl regular expressions (PCRE)   %/regexp/
+// Perl regular expressions (PCRE) %/regexp/ and  %#/regexp/subst/
 
 /* Notes:
  - Memory for compiled regexp is allocated in parse but never freed.
@@ -37,15 +37,17 @@
 
 class RegexpConverter : public StreamFormatConverter
 {
-    int parse (const StreamFormat&, StreamBuffer&, const char*&, bool);
-    int scanString(const StreamFormat&, const char*, char*, size_t);
+    int parse (const StreamFormat& fmt, StreamBuffer&, const char*&, bool);
+    int scanString(const StreamFormat& fmt, const char*, char*, size_t);
+    int scanPseudo(const StreamFormat& fmt, StreamBuffer& input, long& cursor);
+    bool printPseudo(const StreamFormat& fmt, StreamBuffer& output);
 };
 
 int RegexpConverter::
 parse(const StreamFormat& fmt, StreamBuffer& info,
     const char*& source, bool scanFormat)
 {
-    if (!scanFormat)
+    if (!scanFormat && !(fmt.flags & alt_flag))
     {
         error("Format conversion %%/regexp/ is only allowed in input formats\n");
         return false;
@@ -55,28 +57,31 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
         error("Subexpression index %d too big (>9)\n", fmt.prec);
         return false;
     }    
-    if (fmt.flags & (left_flag|space_flag|zero_flag|alt_flag))
-    {
-        error("Use of modifiers '-', ' ', '0', '#'"
-            "not allowed with %%/regexp/ conversion\n");
-        return false;
-    }
+
     StreamBuffer pattern;
     while (*source != '/')
     {
         if (!*source) {
-            error("Missing closing '/' after %%/ format conversion\n");
+            error("Missing closing '/' after %%/%s format conversion\n", pattern());
             return false;
         }
-        if (*source == esc) {
-            source++;
-            pattern.print("\\x%02x", *source++ & 0xFF);
-            continue;
+        if (*source == esc) {          // handle escaped chars
+            if (*++source != '/')      // just un-escape /
+            {
+                pattern.append('\\');
+                if ((*source & 0x7f) < 0x30) // handle control chars
+                {
+                    pattern.print("x%02x", *source++);
+                    continue;
+                }
+                // fall through for PCRE codes like \B
+            }
         }
         pattern.append(*source++);
     }
     source++;
-    debug("regexp = \"%s\"\n", pattern());
+    debug("regexp = \"%s\"\n", pattern.expand()());
+    
     const char* errormsg;
     int eoffset;
     pcre* code = pcre_compile(pattern(), 0, 
@@ -87,6 +92,26 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
         return false;
     }
     info.append(&code, sizeof(code));
+
+    if (fmt.flags & alt_flag)
+    {
+        StreamBuffer subst;
+        debug("check for subst in \"%s\"\n", StreamBuffer(source).expand()());        
+        while (*source != '/')
+        {
+            if (!*source) {
+                error("Missing closing '/' after %%#/%s/%s format conversion\n", pattern(), subst());
+                return false;
+            }
+            if (*source == esc)
+                subst.append(*source++);
+            subst.append(*source++);
+        }
+        source++;
+        debug("subst = \"%s\"\n", subst.expand()());
+        info.append(subst).append('\0');
+        return pseudo_format;
+    }
     return string_format;
 }
 
@@ -94,32 +119,130 @@ int RegexpConverter::
 scanString(const StreamFormat& fmt, const char* input,
     char* value, size_t maxlen)
 {
-    pcre* code;
-    size_t len;
     int ovector[30];
     int rc;
-    int subexpr = 0;
+    unsigned int l;
     
-    memcpy (&code, fmt.info, sizeof(code));
+    const char* info = fmt.info;
+    pcre* code = extract<pcre*>(info);
+    int length = fmt.width > 0 ? fmt.width : strlen(input);
+    int subexpr = fmt.prec > 0 ? fmt.prec : 0;
     
-    len = fmt.width > 0 ? fmt.width : strlen(input);
-    subexpr = fmt.prec > 0 ? fmt.prec : 0;
-    rc = pcre_exec(code, NULL, input, len, 0, 0, ovector, 30);
-    if (rc < 1) return -1;
-    if (fmt.flags & skip_flag) return ovector[1];
-    len = ovector[subexpr*2+1] - ovector[subexpr*2];
-    if (len >= maxlen) {
+    debug("input = \"%s\"\n", input);
+    debug("length=%d\n", length);
+    
+    rc = pcre_exec(code, NULL, input, length, 0, 0, ovector, 30);
+    debug("pcre_exec match \"%.*s\" result = %d\n", length, input, rc);
+    if ((subexpr && rc <= subexpr) || rc < 0)
+    {
+        // error or no match or not enough sub-expressions
+        return -1;
+    }
+    if (fmt.flags & skip_flag) return ovector[subexpr*2+1];
+
+    l = ovector[subexpr*2+1] - ovector[subexpr*2];
+    if (l >= maxlen) {
         if (!(fmt.flags & sign_flag)) {
-            error("Regexp: Matching string \"%s\" too long (%d>%d bytes). You may want to try the + flag: \"%%+/.../\"\n",
-                StreamBuffer(input+ovector[subexpr*2], len).expand()(),
-                (int)len, (int)maxlen-1);
+            error("Regexp: Matching string \"%s\" too long (%d>%ld bytes). You may want to try the + flag: \"%%+/.../\"\n",
+                StreamBuffer(input + ovector[subexpr*2],l).expand()(),
+                l, (long)maxlen-1);
             return -1;
         }
-        len = maxlen-1;
+        l = maxlen-1;
     }
-    memcpy(value, input+ovector[subexpr*2], len);
-    value[len]=0;
-    return ovector[1];
+    memcpy(value, input + ovector[subexpr*2], l);
+    value[l] = '\0';
+    return ovector[1]; // consume input until end of match 
+}
+
+static void regsubst(const StreamFormat& fmt, StreamBuffer& buffer, long start)
+{
+    const char* subst = fmt.info;
+    pcre* code = extract<pcre*>(subst);
+    long length;
+    int rc, l, c, r, rl, n;
+    int ovector[30];
+    StreamBuffer s;
+
+    length = buffer.length() - start;
+    if (fmt.width && fmt.width < length)
+        length = fmt.width;
+    if (fmt.flags & sign_flag)
+        start = buffer.length() - length;
+
+    debug("regsubst buffer=\"%s\", start=%ld, length=%ld, subst = \"%s\"\n",
+        buffer.expand()(), start, length, subst);
+    
+    for (c = 0, n = 1; c < length; n++)
+    {
+        rc = pcre_exec(code, NULL, buffer(start+c), length-c, 0, 0, ovector, 30);
+        debug("pcre_exec match \"%.*s\" result = %d\n", (int)length-c, buffer(start+c), rc);
+        if (rc < 0) // no match 
+            return;
+            
+        if (!(fmt.flags & sign_flag) && n < fmt.prec) // without + flag
+        {
+            // do not yet replace this match
+            c += ovector[1];
+            continue;
+        }
+        // replace & by match in subst
+        l = ovector[1] - ovector[0];
+        debug("start = \"%s\"\n", buffer(start+c));
+        debug("match = \"%.*s\"\n", l, buffer(start+c+ovector[0]));
+        for (r = 1; r < rc; r++)
+            debug("sub%d = \"%.*s\"\n", r, ovector[r*2+1]-ovector[r*2], buffer(start+c+ovector[r*2]));
+        debug("rest  = \"%s\"\n", buffer(start+c+ovector[1]));
+        s = subst;
+        debug("subs = \"%s\"\n", s.expand()());
+        for (r = 0; r < s.length(); r++)
+        {
+            debug("check \"%s\"\n", s.expand(r)());
+            if (s[r] == esc)
+            {
+                unsigned char ch = s[r+1];
+                if (ch < 9) // escaped 0 - 9 : replace with subexpr
+                {
+                    ch *= 2;
+                    rl = ovector[ch+1] - ovector[ch];
+                    debug("replace \\%d: \"%.*s\"\n", ch/2, rl, buffer(start+c+ovector[ch]));
+                    s.replace(r, 2, buffer(start+c+ovector[ch]), rl);
+                    r += rl - 1;
+                }
+                else
+                    s.remove(r, 1); // just remove escape
+            }
+            else if (s[r] == '&') // unescaped & : replace with match
+            {
+                debug("replace &: \"%.*s\"\n", l,  buffer(start+c+ovector[0]));
+                s.replace(r, 1, buffer(start+c+ovector[0]), l);
+                r += l - 1;
+            }
+            else continue;
+            debug("subs = \"%s\"\n", s());
+        }
+        buffer.replace(start+c+ovector[0], l, s);
+        length += s.length() - l;
+        c += s.length();
+        if (n == fmt.prec) // max match reached
+            return;
+    }
+}
+
+int RegexpConverter::
+scanPseudo(const StreamFormat& fmt, StreamBuffer& input, long& cursor)
+{
+    /* re-write input buffer */
+    regsubst(fmt, input, cursor);
+    return 0;
+}
+
+bool RegexpConverter::
+printPseudo(const StreamFormat& fmt, StreamBuffer& output)
+{
+    /* re-write output buffer */
+    regsubst(fmt, output, 0);
+    return true;
 }
 
 RegisterConverter (RegexpConverter, "/");

src/StreamCore.cc

@@ -832,7 +832,7 @@ lockCallback(StreamIoStatus status)
         case StreamIoSuccess:
             break;
         case StreamIoTimeout:
-            error("%s: Cannot lock device within %ld ms, device seems to be busy\n",
+            debug("%s: length, within %ld ms, device seems to be busy\n",
                 name(), lockTimeout);
             flags &= ~BusOwner;
             finishProtocol(LockTimeout);
@@ -991,7 +991,7 @@ readCallback(StreamIoStatus status,
                 evalIn();
                 return 0;
             }
-            error("%s: No reply from device within %ld ms\n",
+            debug("StreamCore::readCallback(%s): No reply from device within %ld ms\n",
                 name(), replyTimeout);
             inputBuffer.clear();
             finishProtocol(ReplyTimeout);

src/StreamEpics.cc

@@ -115,7 +115,6 @@ class Stream : protected StreamCore
     epicsMutex mutex;
     epicsEvent initDone;
 #endif
-    StreamBuffer fieldBuffer;
     int status;
     int convert;
     long currentValueLength;
@@ -604,7 +603,7 @@ parseLink(const struct link *ioLink, char* filename,
     if (items <= 0)
     {
         error("%s: Empty I/O link. "
-            "Forgot the leading '@' or confused INP with OUT ?\n",
+            "Forgot the leading '@' or confused INP with OUT or link is too long ?\n",
             name());
         return S_dev_badInitRet;
     }
@@ -990,6 +989,7 @@ formatValue(const StreamFormat& format, const void* fieldaddress)
     {
         // Format like "%([record.]field)..." has requested to get value
         // from field of this or other record.
+        StreamBuffer fieldBuffer;
         DBADDR* pdbaddr = (DBADDR*)fieldaddress;
         
         /* Handle time stamps special. %T converter takes double. */
@@ -1134,6 +1134,7 @@ matchValue(const StreamFormat& format, const void* fieldaddress)
     {
         // Format like "%([record.]field)..." has requested to put value
         // to field of this or other record.
+        StreamBuffer fieldBuffer;
         DBADDR* pdbaddr = (DBADDR*)fieldaddress;
         long nord;
         long nelem = pdbaddr->no_elements;

src/devStream.h

@@ -22,7 +22,7 @@
 #define devStream_h
 
 #define STREAM_MAJOR 2
-#define STREAM_MINOR 6
+#define STREAM_MINOR 7
 #define STREAM_PATCHLEVEL 6
 
 #if defined(__vxworks) || defined(vxWorks)