reading web pages

doc/formats.html

@@ -132,6 +132,7 @@ Thus, data type modifiers like <code>l</code> or <code>h</code> do not
 exist in <em>StreamDevice</em> formats.
 </p>
 
+<a name="redirection"></a>
 <h3>Redirection</h3>
 <p>
 To use other fields of the record or even fields of other records on the

doc/nav.html

@@ -178,6 +178,7 @@ div div div a {list-style-type:circle;}
   <a target="_parent" href="tipsandtricks.html#writemany">Write more than one value in one message</a>
   <a target="_parent" href="tipsandtricks.html#readmany">Read more than one value from one message</a>
   <a target="_parent" href="tipsandtricks.html#mixed">Read values of mixed data type</a>
+  <a target="_parent" href="tipsandtricks.html#web">Read a web page</a>
  </div>
 </div>
 

doc/tipsandtricks.html

@@ -103,7 +103,7 @@ an array: (3.14, 17.30, -12.34)
 <h3>B) We have up to 12 numeric values</h3>
 <p>
 Use a <a href="calcout.html">calcout</a> record and
-<a href="formats.html#types">field references</a> in the format.
+<a href="formats.html#redirection">redirection to fields</a>.
 </p>
 <p>
 <code>
@@ -128,7 +128,7 @@ record (calcout, "$(RECORD)") {<br>
 </p>
 <h3>C) Values are in other records on the same IOC</h3>
 <p>
-Use <a href="formats.html#types">record references</a> in the format.
+Use <a href="formats.html#redirection">redirection to records</a>.
 </p>
 <p>
 <code>
@@ -216,7 +216,7 @@ Any non-matching input is ignored by record B.
 </p>
 <h3>C) Values should be stored in other records on the same IOC</h3>
 <p>
-Use <a href="formats.html#types">record references</a> in the format.
+Use <a href="formats.html#redirection">redirection to records</a>.
 To avoid record names in protocol files, use
 <a href="protocol.html#argvar">protocol arguments</a>.
 </p>
@@ -244,11 +244,11 @@ processes record B.
 </p>
 
 <a name="mixed"></a>
-<h2>I have a device that sends mixed data types: numbers and strings</h2>
+<h2>I have a device that sends mixed data types: numbers or strings</h2>
 <p>
 Use a <code>@mismatch</code>
 <a href="protocol.html#except">exception handler</a> and
-<a href="formats.html#types">record references</a> in the format.
+<a href="formats.html#redirection">redirection to records</a>.
 To avoid record names in protocol files, use
 <a href="protocol.html#argvar">protocol arguments</a>.
 </p>
@@ -289,9 +289,124 @@ record (stringout, "$(DEVICE):clean_2") {<br>
 &nbsp;&nbsp;field (VAL, "OK")<br>
 &nbsp;&nbsp;field (OUT,  "$(DEVICE):message PP")<br>
 }<br>
-
 </code>
+<a name="web"></a>
+<h2>I need to read a web page</h2>
+<p>
+First you have to send a correctly formatted HTML request.
+Note that this request must contain the full URL like
+"http://server/page" and must be terminated with <u>two</u> newlines.
+The server should be the same as in the
+<a href="setup.html#sta"><code>drvAsynIPPortConfigure</code></a>
+command (if not using a http proxy).
+
+The web page you get often contains much more information than you need.
+<a href="formats.html#regex">Regular expressions</a> are great
+to find what you are looking for.
+</p>
+<h3>Example 1</h3>
+<p>
+Read the title of a web page.
+</p>
+<p>
+<code>
+get_title {<br>
+&nbsp;&nbsp;extrainput = ignore;<br>
+&nbsp;&nbsp;replyTimeout = 1000;<br>
+&nbsp;&nbsp;out "GET http://\$1\n\n";<br>
+&nbsp;&nbsp;in "%+.1/(?im)&lt;title&gt(.*)&lt\/title&gt;/";<br>
+}
+</code>
+</p>
+<p>
+Terminate the request with two newlines, either explicit like here
+<u>or</u> using an 
+<a href="protocol.html#sysvar"><code>outTerminator</code></a>.
+The URI (without http:// but including the web server host name)
+is passed as <a href="protocol.html#argvar">argument</a> 1 to <code>\$1</code>.
+Note that web servers may be slow, so allow some
+<a href="protocol.html#argvar"><code>replyTimeout</code></a>.
+</p>
+<p>
+If you don't use an <code>inTerminator</code> then the whole page is
+read as one "line" to the <code>in</code> command and can be parsed easily
+with a regular expression.
+We want to see the string between <code>&lt;title&gt</code> and
+<code>&lt;/title&gt;</code>, so we put it into a subexpression in
+<code>()</code> and request the first subexpression with <code>.1</code>.
+Note that the <code>/</code> in the closing tag has be be escaped
+to avoid a misinterpretation as the closing <code>/</code> of the regular
+expression.
+</p>
+<p>
+The tags may be upper or lower case like <code>&lt;TITLE&gt;</code> or
+<code>&lt;Title&gt;</code>, so we ask for case insensitive matching with
+<code>(?i)</code>.
+</p>
+<p>
+The string should be terminated with the first closing
+<code>&lt;/title&gt;</code>, not the last one in the file.
+(There should not be more than one title but you never know.)
+Thus we ask not to be greedy with <code>(?m)</code>.
+<code>(?i)</code> and <code>(?m)</code> can be combined to <code>(?im)</code>.
+See the PCRE documentation for more regexp syntax.
+</p>
+<p>
+The regular expression matcher ignores and discards any content before the
+matching section.
+Content after the match is discarded with <code>extrainput = ignore</code>
+so that it does not trigger errors reporting "surplus input".
+</p>
+<p>
+Finally, the title may be too long for the record.
+The <code>+</code> tells the format matcher not to fail in this case
+but to truncate the string instead.
+You can read the string with a stringin record or for longer strings with
+a waveform record with data type CHAR.
+</p>
+<p>
+<code>
+record (stringin, "$(DEVICE):title") {<br>
+&nbsp;&nbsp;field (DTYP, "stream")<br>
+&nbsp;&nbsp;field (INP,  "@$(DEVICETYPE).proto get_title($(PAGE)) $(BUS)")<br>
+}<br>
+record (waveform, "$(DEVICE):longtitle") {<br>
+&nbsp;&nbsp;field (DTYP, "stream")<br>
+&nbsp;&nbsp;field (INP,  "@$(DEVICETYPE).proto get_title($(PAGE)) $(BUS)")<br>
+&nbsp;&nbsp;field (FTVL, "CHAR")<br>
+&nbsp;&nbsp;field (NELM, "100")<br>
+}<br>
+</code>
+</p>
+
+<h3>Example 2</h3>
+<p>
+Read a number from a web page. First we have to locate the number.
+For that we match against any known string right before the number
+(and <a href="formats.html#syntax">discard the match</a> with <code>*</code>).
+Then we read the number.
+</p>
+<code>
+get_title {<br>
+&nbsp;&nbsp;extrainput = ignore;<br>
+&nbsp;&nbsp;replyTimeout = 1000;<br>
+&nbsp;&nbsp;out "GET http://\$1\n\n";<br>
+&nbsp;&nbsp;in "%*/Interesting value:/%f more text";<br>
+}
+</code>
+<p>
+When using <code>extrainput = ignore;</code>, it is always a good idea to
+match a few bytes after the value, too.
+This catches errors where loading of the page is interrupted in the middle
+of the number. (You don't want to miss the exponent from something like 1.23E-14).
+</p>
+<p>
+You can read more than one value from a file with successive regular expressions
+and <a href="formats.html#redirection">redirections</a>.
+But this only works if the order of the values is predictible.
+<i>StreamDevice</i> is not an XML parser! It always reads sequentially.
+</p>
 <hr>
-<p><small>Dirk Zimoch, 2007</small></p>
+<p><small>Dirk Zimoch, 2012</small></p>
 </body>
 </html>

makefile

@@ -34,7 +34,6 @@ SOURCES += $(FORMATS:%=src/%Converter.cc)
 SOURCES += $(BUSSES:%=src/%Interface.cc)
 SOURCES += $(wildcard src/Stream*.cc)
 SOURCES += src/StreamVersion.c
-SOURCES_3.14 += src/devcalcoutStream.c
 
 HEADERS += StreamFormat.h
 HEADERS += StreamFormatConverter.h
@@ -44,6 +43,9 @@ HEADERS += StreamError.h
 ifeq (${EPICS_BASETYPE},3.13)
 USR_INCLUDES += -include $(INSTALL_INCLUDE)/compat3_13.h
 endif
+ifeq (${EPICS_BASETYPE},3.14)
+RECORDTYPES += calcout
+endif
 
 StreamCore.o: streamReferences
 

src/AsynDriverInterface.cc

@@ -754,6 +754,8 @@ readRequest(unsigned long replyTimeout_ms, unsigned long readTimeout_ms,
         ioAction = Read;
         queueTimeout = replyTimeout;
     }
+    debug("AsynDriverInterface::readRequest %s:  queueRequest(..., priority=%d, queueTimeout=%f)\n",
+        clientName(), priority(), queueTimeout);
     status = pasynManager->queueRequest(pasynUser,
         priority(), queueTimeout);
     if (status != asynSuccess && !async)
@@ -863,27 +865,26 @@ readHandler()
         received = 0;
         eomReason = 0;
         
+        debug("AsynDriverInterface::readHandler(%s): ioAction=%s "
+            "read(..., bytesToRead=%ld, ...) "
+            "[timeout=%f seconds]\n",
+            clientName(), ioActionStr[ioAction],
+            bytesToRead, pasynUser->timeout);
         status = pasynOctet->read(pvtOctet, pasynUser,
             buffer, bytesToRead, &received, &eomReason);
-        if (ioAction == Read || status != asynTimeout)
-        {
         debug("AsynDriverInterface::readHandler(%s): "
-                "read(..., bytesToRead=%ld, received=%ld...) "
+            "read returned %s: ioAction=%s received=%ld, eomReason=%s, buffer=\"%s\"\n",
-                "[timeout=%f seconds] = %s\n",
+            clientName(), asynStatusStr[status], ioActionStr[ioAction],
-                clientName(), bytesToRead, (long)received,
+            (long)received,eomReasonStr[eomReason&0x7],
-                pasynUser->timeout, asynStatusStr[status]);
+            StreamBuffer(buffer, received).expand()());
-        }
+
         pasynManager->isConnected(pasynUser, &connected);
         debug("AsynDriverInterface::readHandler(%s): "
-            "device is %sconnected\n",
+            "device is now %sconnected\n",
             clientName(),connected?"":"dis");        
-        if (!connected) {
+        // asyn 4.16 sets reason to ASYN_EOM_END when device disconnects.
-            error("%s: connection closed in read\n",
+        // What about earlier versions?
-                clientName());
+        if (!connected) eomReason |= ASYN_EOM_END;
-            readCallback(StreamIoFault);
-            return;
-        }
-        // pasynOctet->read() has already cut off terminator.
         
         if (status == asynTimeout &&
             pasynUser->timeout == 0.0 &&
@@ -910,6 +911,7 @@ readHandler()
 #endif
                     // ignore what we got from here.
                     // input was already handeled by asynReadHandler()
+
                     // read until no more input is available
                     readMore = -1;
                     break;

src/BinaryConverter.cc

@@ -130,9 +130,10 @@ scanLong(const StreamFormat& format, const char* input, long& value)
     int width = format.width;
     if (width == 0) width = -1;
     int length = 0;
-    while (isspace(input[length])) length++; // skip whitespaces
     char zero = format.info[0];
     char one = format.info[1];
+    if (!isspace(zero) && !isspace(one))
+        while (isspace(input[length])) length++; // skip whitespaces
     if (input[length] != zero && input[length] != one) return -1;
     if (format.flags & alt_flag)
     {

src/ChecksumConverter.cc

@@ -265,7 +265,7 @@ static ulong crc_0x04C11DB7(const uchar* data, ulong len, ulong crc)
 {
     // x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 +
     //    x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0  (0x04C11DB7)
-    const static ulong table[] = {
+    const static unsigned int table[] = {
         0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9,
         0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005,
         0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61,
@@ -340,7 +340,7 @@ static ulong crc_0x04C11DB7_r(const uchar* data, ulong len, ulong crc)
     // x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 +
     //    x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0  (0x04C11DB7)
     // reflected
-    const static ulong table[] = {
+    const static unsigned int table[] = {
         0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
         0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
         0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
@@ -545,9 +545,9 @@ printPseudo(const StreamFormat& format, StreamBuffer& output)
     debug("ChecksumConverter %s: output to check: \"%s\"\n",
         checksumMap[fnum].name, output.expand(start,length)());
 
-    sum = checksumMap[fnum].xorout ^ checksumMap[fnum].func(
+    sum = (checksumMap[fnum].xorout ^ checksumMap[fnum].func(
         reinterpret_cast<uchar*>(output(start)), length,
-        checksumMap[fnum].init) & mask[checksumMap[fnum].bytes];
+        checksumMap[fnum].init)) & mask[checksumMap[fnum].bytes];
 
     debug("ChecksumConverter %s: output checksum is 0x%lX\n",
         checksumMap[fnum].name, sum);
@@ -555,6 +555,17 @@ printPseudo(const StreamFormat& format, StreamBuffer& output)
     int i;
     unsigned outchar;
     
+    if (format.flags & sign_flag) // decimal
+    {
+        // get number of decimal digits from number of bytes: ceil(xbytes*2.5)
+        i = (checksumMap[fnum].bytes+1)*25/10-2;
+        output.print("%0*ld", i, sum);
+        debug("ChecksumConverter %s: decimal appending %0*ld\n",
+            checksumMap[fnum].name, i, sum);
+        return true;
+    }
+    
+    
     if (format.flags & alt_flag) // lsb first (little endian)
     {
         for (i = 0; i < checksumMap[fnum].bytes; i++)
@@ -563,7 +574,7 @@ printPseudo(const StreamFormat& format, StreamBuffer& output)
             debug("ChecksumConverter %s: little endian appending 0x%X\n",
                 checksumMap[fnum].name, outchar);
             if (format.flags & zero_flag) // ASCII
-                output.printf("%02X", outchar);
+                output.print("%02X", outchar);
             else                          // binary
                 output.append(outchar);
             sum >>= 8;
@@ -578,7 +589,7 @@ printPseudo(const StreamFormat& format, StreamBuffer& output)
             debug("ChecksumConverter %s: big endian appending 0x%X\n",
                 checksumMap[fnum].name, outchar);
             if (format.flags & zero_flag) // ASCII
-                output.printf("%02X", outchar);
+                output.print("%02X", outchar);
             else                          // binary
                 output.append(outchar);
             sum <<= 8;
@@ -606,9 +617,9 @@ scanPseudo(const StreamFormat& format, StreamBuffer& input, long& cursor)
         return -1;
     }
 
-    sum = checksumMap[fnum].xorout ^ checksumMap[fnum].func(
+    sum = (checksumMap[fnum].xorout ^ checksumMap[fnum].func(
         reinterpret_cast<uchar*>(input(start)), length,
-        checksumMap[fnum].init) & mask[checksumMap[fnum].bytes];
+        checksumMap[fnum].init)) & mask[checksumMap[fnum].bytes];
 
     debug("ChecksumConverter %s: input checksum is 0x%0*lX\n",
         checksumMap[fnum].name, 2*checksumMap[fnum].bytes, sum);
@@ -616,6 +627,24 @@ scanPseudo(const StreamFormat& format, StreamBuffer& input, long& cursor)
     int i,j;
     unsigned inchar;
     
+    if (format.flags & sign_flag) // decimal
+    {
+        ulong sumin = 0;
+        // get number of decimal digits from number of bytes: ceil(xbytes*2.5)
+        j = (checksumMap[fnum].bytes+1)*25/10-2;
+        for (i = 0; i < j; i++)
+        {
+            inchar = input[cursor+i];
+            if (isdigit(inchar)) sumin = sumin*10+inchar-'0';
+            else break;
+        }
+        if (sumin==sum) return i;
+        error("Input %0*lu does not match checksum %0*lu\n", 
+            i, sumin, j, sum);
+        return -1;
+    }
+    
+    
     if (format.flags & alt_flag) // lsb first (little endian)
     {
         for (i = 0; i < checksumMap[fnum].bytes; i++)

src/Makefile.Host

@@ -39,7 +39,4 @@ include $(TOP)/config/RULES.Host
 
 # create stream.dbd from all RECORDS
 stream.dbd: ../CONFIG_STREAM
-	@for r in $(RECORDS_3_13); \
+	$(PERL) ../makedbd.pl -3.13 $(RECORDS_3_13) > $@
-	do echo "device($$r,INST_IO,dev$${r}Stream,\"stream\")"; \
-	done > $@
-	@echo "driver(stream)" >> $@

src/MantissaExponentConverter.cc

@@ -75,7 +75,7 @@ printDouble(const StreamFormat& fmt, StreamBuffer& output, double value)
     int prec = fmt.prec;
     
     if (prec < 1) prec = 6;
-    buf.printf("%.*e", prec-1, fabs(value)/pow(10.0, prec-1));
+    buf.print("%.*e", prec-1, fabs(value)/pow(10.0, prec-1));
     buf.remove(1,1);
     buf.remove(buf.find('e'),1);
     

src/RegexpConverter.cc

@@ -70,7 +70,7 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
         }
         if (*source == esc) {
             source++;
-            pattern.printf("\\x%02x", *source++ & 0xFF);
+            pattern.print("\\x%02x", *source++ & 0xFF);
             continue;
         }
         pattern.append(*source++);
@@ -110,7 +110,7 @@ scanString(const StreamFormat& fmt, const char* input,
     len = ovector[subexpr*2+1] - ovector[subexpr*2];
     if (len >= maxlen) {
         if (!(fmt.flags & sign_flag)) {
-            debug("Matching string \"%s\" too long (%d>%d bytes)\n",
+            error("Regexp: Matching string \"%s\" too long (%d>%d bytes). You may want to try the + flag: \"%%+/.../\"\n",
                 StreamBuffer(input+ovector[subexpr*2], len).expand()(),
                 (int)len, (int)maxlen-1);
             return -1;

src/StreamBuffer.cc

@@ -248,7 +248,7 @@ replace(long remstart, long remlen, const void* ins, long inslen)
 }
 
 StreamBuffer& StreamBuffer::
-printf(const char* fmt, ...)
+print(const char* fmt, ...)
 {
     va_list va;
     int printed;
@@ -296,7 +296,7 @@ StreamBuffer StreamBuffer::expand(long start, long length) const
         c = buffer[i];
         if ((c & 0x7f) < 0x20 || (c & 0x7f) == 0x7f)
         {
-            result.printf("<%02x>", c & 0xff);
+            result.print("<%02x>", c & 0xff);
         }
         else
         {
@@ -312,17 +312,17 @@ dump() const
     StreamBuffer result(256+cap*5);
     result.append("\033[0m");
     long i;
-    result.printf("%ld,%ld,%ld:\033[37m", offs, len, cap);
+    result.print("%ld,%ld,%ld:\033[37m", offs, len, cap);
     for (i = 0; i < cap; i++)
     {
         if (i == offs) result.append("\033[34m[\033[0m");
         if ((buffer[i] & 0x7f) < 0x20 || (buffer[i] & 0x7f) == 0x7f)
         {
             if (i < offs || i >= offs+len)
-            result.printf(
+            result.print(
                 "<%02x>", buffer[i] & 0xff);
             else
-            result.printf(
+            result.print(
                 "\033[34m<%02x>\033[37m", buffer[i] & 0xff);
         }
         else

src/StreamBuffer.h

@@ -21,7 +21,6 @@
 #define StreamBuffer_h
 
 #include <string.h>
-#include <stdio.h>
 
 #ifndef __GNUC__
 #define __attribute__(x)
@@ -189,7 +188,7 @@ public:
     StreamBuffer& insert(long pos, char c)
         {return replace(pos, 0, &c, 1);}
 
-    StreamBuffer& printf(const char* fmt, ...)
+    StreamBuffer& print(const char* fmt, ...)
         __attribute__ ((format(printf,2,3)));
 
     // find: get index of data in buffer or -1

src/StreamCore.cc

@@ -57,12 +57,12 @@ static char* printCommands(StreamBuffer& buffer, const char* c)
                 break;
             case wait_cmd:
                 timeout = extract<unsigned long>(c);
-                buffer.printf("    wait %ld;\n # ms", timeout);
+                buffer.print("    wait %ld;\n # ms", timeout);
                 break;
             case event_cmd:
                 eventnumber = extract<unsigned long>(c);
                 timeout = extract<unsigned long>(c);
-                buffer.printf("    event(%ld) %ld; # ms\n", eventnumber, timeout);
+                buffer.print("    event(%ld) %ld; # ms\n", eventnumber, timeout);
                 break;
             case exec_cmd:
                 buffer.append("    exec \"");
@@ -71,7 +71,7 @@ static char* printCommands(StreamBuffer& buffer, const char* c)
                 break;
             case connect_cmd:
                 timeout = extract<unsigned long>(c);
-                buffer.printf("    connect %ld; # ms\n", timeout);
+                buffer.print("    connect %ld; # ms\n", timeout);
                 break;
             case disconnect_cmd:
                 buffer.append("    disconnect;\n");
@@ -118,7 +118,6 @@ printProtocol()
     if (onMismatch)
         printf("  @Mismatch {\n%s  }\n",
         printCommands(buffer.clear(), onMismatch()));
-    debug("StreamCore::printProtocol: commands=%s\n", commands.expand()());
     printf("\n%s}\n",
         printCommands(buffer.clear(), commands()));
 }
@@ -1159,8 +1158,7 @@ matchInput()
     */
     char command;
     const char* fieldName = NULL;
-    const char* formatstring;
+    StreamBuffer formatstring;
-    int formatstringlen;
     
     consumedInput = 0;
     
@@ -1186,22 +1184,14 @@ normal_format:
                 int consumed;
                 // code layout:
                 // formatstring <eos> StreamFormat [info]
-                formatstring = commandIndex;
+                commandIndex = StreamProtocolParser::printString(formatstring, commandIndex);
-                // jump after <eos>
-                while (*commandIndex)
-                {
-                    if (*commandIndex == esc) commandIndex++;
-                    commandIndex++;
-                }
-                formatstringlen = commandIndex-formatstring;
-                commandIndex++;
                 
                 StreamFormat fmt = extract<StreamFormat>(commandIndex);
                 fmt.info = commandIndex; // point to info string
                 commandIndex += fmt.infolen;
 #ifndef NO_TEMPORARY
-                debug("StreamCore::matchInput(%s): format = %%%s\n",
+                debug("StreamCore::matchInput(%s): format = \"%%%s\"\n",
-                    name(), StreamBuffer(formatstring, formatstringlen).expand()());
+                    name(), formatstring());
 #endif
 
                 if (fmt.flags & skip_flag || fmt.type == pseudo_format)
@@ -1243,10 +1233,10 @@ normal_format:
                         {
                             if (!(flags & AsyncMode) && onMismatch[0] != in_cmd)
                             {
-                                error("%s: Input \"%s%s\" does not match format %%%s\n",
+                                error("%s: Input \"%s%s\" does not match format \"%%%s\"\n",
                                     name(), inputLine.expand(consumedInput, 20)(),
                                     inputLine.length()-consumedInput > 20 ? "..." : "",
-                                    formatstring);
+                                    formatstring());
                             }
                             return false;
                         }
@@ -1260,13 +1250,12 @@ normal_format:
                     flags &= ~Separator;
                     if (!formatValue(fmt, fieldAddress ? fieldAddress() : NULL))
                     {
-                        StreamBuffer formatstr(formatstring, formatstringlen);
                         if (fieldAddress)
-                            error("%s: Cannot format field '%s' with '%%%s'\n",
+                            error("%s: Cannot format variable \"%s\" with \"%%%s\"\n",
-                                name(), fieldName, formatstr.expand()());
+                                name(), fieldName, formatstring());
                         else
-                            error("%s: Cannot format value with '%%%s'\n",
+                            error("%s: Cannot format value with \"%%%s\"\n",
-                                name(), formatstr.expand()());
+                                name(), formatstring());
                         return false;
                     }
 #ifndef NO_TEMPORARY
@@ -1279,11 +1268,11 @@ normal_format:
                         if (!(flags & AsyncMode) && onMismatch[0] != in_cmd)
                         {
                             error("%s: Input \"%s%s\" too short."
-                                  " No match for format %%%s (\"%s\")\n",
+                                  " No match for format \"%%%s\" (\"%s\")\n",
                                 name(), 
                                 inputLine.length() > 20 ? "..." : "",
                                 inputLine.expand(-20)(),
-                                formatstring,
+                                formatstring(),
                                 outputLine.expand()());
                         }
                         return false;
@@ -1292,11 +1281,10 @@ normal_format:
                     {
                         if (!(flags & AsyncMode) && onMismatch[0] != in_cmd)
                         {
-                            error("%s: Input \"%s%s\" does not match"
+                            error("%s: Input \"%s%s\" does not match format \"%%%s\" (\"%s\")\n",
-                                  " format %%%s (\"%s\")\n",
                                 name(), inputLine.expand(consumedInput, 20)(),
                                 inputLine.length()-consumedInput > 20 ? "..." : "",
-                                formatstring,
+                                formatstring(),
                                 outputLine.expand()());
                         }
                         return false;
@@ -1310,13 +1298,13 @@ normal_format:
                     if (!(flags & AsyncMode) && onMismatch[0] != in_cmd)
                     {
                         if (flags & ScanTried)
-                            error("%s: Input \"%s%s\" does not match format %%%s\n",
+                            error("%s: Input \"%s%s\" does not match format \"%%%s\"\n",
                                 name(), inputLine.expand(consumedInput, 20)(),
                                 inputLine.length()-consumedInput > 20 ? "..." : "",
-                                formatstring);
+                                formatstring());
                         else
-                            error("%s: Format %%%s has data type %s which does not match this variable.\n",
+                            error("%s: Format \"%%%s\" has data type %s which does not match variable \"%s\".\n",
-                                name(), formatstring, StreamFormatTypeStr[fmt.type] );
+                                name(), formatstring(), StreamFormatTypeStr[fmt.type], fieldName);
                     }
                     return false;
                 }

src/StreamError.cc

@@ -20,6 +20,7 @@
 #include "StreamError.h"
 #include <string.h>
 #include <time.h>
+#include <stdio.h>
 
 int streamDebug = 0;
 extern "C" {

src/StreamError.h

@@ -20,8 +20,8 @@
 #ifndef StreamError_h
 #define StreamError_h
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stddef.h>
 
 #ifndef __GNUC__
 #define __attribute__(x)

src/StreamFormatConverter.cc

@@ -316,7 +316,7 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
 bool StdLongConverter::
 printLong(const StreamFormat& fmt, StreamBuffer& output, long value)
 {
-    output.printf(fmt.info, value);
+    output.print(fmt.info, value);
     return true;
 }
 
@@ -395,7 +395,7 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
 bool StdDoubleConverter::
 printDouble(const StreamFormat& fmt, StreamBuffer& output, double value)
 {
-    output.printf(fmt.info, value);
+    output.print(fmt.info, value);
     return true;
 }
 
@@ -451,7 +451,7 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
 bool StdStringConverter::
 printString(const StreamFormat& fmt, StreamBuffer& output, const char* value)
 {
-    output.printf(fmt.info, value);
+    output.print(fmt.info, value);
     return true;
 }
 
@@ -549,7 +549,7 @@ parse(const StreamFormat& fmt, StreamBuffer& info,
 bool StdCharsConverter::
 printLong(const StreamFormat& fmt, StreamBuffer& output, long value)
 {
-    output.printf(fmt.info, value);
+    output.print(fmt.info, value);
     return true;
 }
 

src/StreamProtocol.cc

@@ -634,7 +634,7 @@ printString(StreamBuffer& buffer, const char* s)
         switch (*s)
         {
             case esc:
-                buffer.printf("\\x%02x", (*++s) & 0xff);
+                buffer.print("\\x%02x", (*++s) & 0xff);
                 break;
             case '\r':
                 buffer.append("\\r");
@@ -655,7 +655,7 @@ printString(StreamBuffer& buffer, const char* s)
                 buffer.append("\\\\");
                 break;
             case format_field:
-                buffer.printf("%%(%s)", ++s);
+                buffer.print("%%(%s)", ++s);
                 while (*s++);
                 s += extract<unsigned short>(s); // skip fieldaddress
                 goto format;
@@ -670,7 +670,7 @@ format:         {
                 continue;
             default:
                 if ((*s & 0x7f) < 0x20 || (*s & 0x7f) == 0x7f)
-                    buffer.printf("\\x%02x", *s & 0xff);
+                    buffer.print("\\x%02x", *s & 0xff);
                 else
                     buffer.append(*s);
         }
@@ -742,7 +742,7 @@ Protocol(const Protocol& p, StreamBuffer& name, int _line)
     int i;
     const char* nextparameter;
     parameter[0] = protocolname();
-    for (i = 0; i < 10; i++)
+    for (i = 0; i < 9; i++)
     {
         debug("StreamProtocolParser::Protocol::Protocol $%d=\"%s\"\n",
             i, parameter[i]);
@@ -1059,7 +1059,6 @@ compileString(StreamBuffer& buffer, const char*& source,
     FormatType formatType, Client* client, int quoted)
 {
     bool escaped = false;
-    int n;
     int newline = 0;
     StreamBuffer formatbuffer;
     int formatpos = buffer.length();
@@ -1136,7 +1135,8 @@ compileString(StreamBuffer& buffer, const char*& source,
         if (escaped)   // char after \ in quoted text
         {
             escaped = false;
-            unsigned int temp;
+            unsigned int c;
+            int n=1;
             switch (*source)
             {
                 case '$': // can't be: readToken would have made a token from this
@@ -1145,61 +1145,50 @@ compileString(StreamBuffer& buffer, const char*& source,
                     return false;
                 case '?':
                     buffer.append(skip);
-                    break;
+                    source++;
+                    continue;
                 case '_':
                     buffer.append(whitespace);
-                    break;
+                    source++;
+                    continue;
                 case 'a':
-                    buffer.append(7);
+                    c=7;
                     break;
                 case 'b':
-                    buffer.append(8);
+                    c=8;
                     break;
                 case 't':
-                    buffer.append(9);
+                    c=9;
                     break;
                 case 'n':
-                    buffer.append('\n');
+                    c='\n';
                     break;
                 case 'r':
-                    buffer.append('\r');
+                    c='\r';
                     break;
                 case 'e':
-                    buffer.append(esc).append(esc);
+                    c=esc;
                     break;
-                case '0':  // octal numbers (max 3 digits after 0)
+                case '0':  // octal numbers (max 4 digits)
-                    sscanf (source, "%3o%n", &temp, &n);
+                    sscanf (source, "%4o%n", &c, &n); //cannot fail
-                    if (temp > 0xFF)
+                    if (c > 0xFF)
                     {
                         error(line, filename(),
-                            "Octal source %#o does not fit in byte: %s\n",
+                            "Octal number %#o does not fit in byte: \"%s\"\n",
-                            temp, source-1);
+                            c, source-1);
                         return false;
                     }
-                    if (formatType != NoFormat &&
+                    break;
-                        (temp < last_function_code || temp == esc))
+                case 'x':  // hex numbers (max 2 digits after x)
-                    {
+                    if (sscanf (source+1, "%2x%n", &c, &n) < 1)
-                        buffer.append(esc);
-                    }
-                    buffer.append(temp);
-                    source += n;
-                    continue;
-                case 'x':  // hex numbers (max 2 digits after 0)
-                    if (sscanf (source+1, "%2x%n", &temp, &n) < 1)
                     {
                         error(line, filename(),
-                            "Hex digit expected after \\x: %s\n",
+                            "Hex digit expected after \\x: \"%s\"\n",
                             source-1);
                         return false;
                     }
-                    if (formatType != NoFormat &&
+                    n++;
-                        (temp < last_function_code || temp == esc))
+                    break;
-                    {
-                        buffer.append(esc);
-                    }
-                    buffer.append(temp);
-                    source += n+1;
-                    continue;
                 case '1': // decimal numbers (max 3 digits)
                 case '2':
                 case '3':
@@ -1209,26 +1198,24 @@ compileString(StreamBuffer& buffer, const char*& source,
                 case '7':
                 case '8':
                 case '9':
-                    sscanf (source, "%3u%n", &temp, &n);
+                    sscanf (source, "%3u%n", &c, &n); //cannot fail
-                    if (temp > 0xFF)
+                    if (c > 0xFF)
                     {
                         error(line, filename(),
-                            "Decimal source %d does not fit in byte: %s\n",
+                            "Decimal number %d does not fit in byte: \"%s\"\n",
-                            temp, source-1);
+                            c, source-1);
                         return false;
                     }
-                    if (formatType != NoFormat &&
+                    break;
-                        (temp < last_function_code || temp == esc))
+                default:  // escaped literals
+                    c=*source;
+            }
+            source+=n;
+            if (formatType != NoFormat)
             {
                 buffer.append(esc);
             }
-                    buffer.append(temp);
+            buffer.append(c);
-                    source += n;
-                    continue;
-                default:  // escaped literals
-                    buffer.append(esc).append(*source);
-            }
-            source++;
             continue;
         }
         if (quoted) // look for ending quotes and escapes
@@ -1281,7 +1268,7 @@ compileString(StreamBuffer& buffer, const char*& source,
         }
         // try numeric byte value
         char* p;
-        int temp = strtol(source, &p, 0);
+        int c = strtol(source, &p, 0);
         if (p != source)
         {
             if (*p != 0)
@@ -1290,18 +1277,17 @@ compileString(StreamBuffer& buffer, const char*& source,
                     "Garbage after numeric source: %s", source);
                 return false;
             }
-            if (temp > 0xFF || temp < -0x80)
+            if (c > 0xFF || c < -0x80)
             {
                 error(line, filename(),
                     "Value %s does not fit in byte\n", source);
                 return false;
             }
-            if (formatType != NoFormat &&
+            if (formatType != NoFormat)
-                (temp < last_function_code || temp == esc))
             {
                 buffer.append(esc);
             }
-            buffer.append(temp);
+            buffer.append(c);
             source = p+1+sizeof(int);
             continue;
         }
@@ -1348,7 +1334,7 @@ compileString(StreamBuffer& buffer, const char*& source,
             {"del",  0x7f}
         };
         size_t i;
-        char c = 0;
+        c=-1;
         for (i = 0; i < sizeof(codes)/sizeof(*codes); i++)
         {
             if (strcmp(source, codes[i].name) == 0)
@@ -1361,8 +1347,7 @@ compileString(StreamBuffer& buffer, const char*& source,
                         source);
                     return false;
                 }
-                if (formatType != NoFormat &&
+                if (formatType != NoFormat)
-                    (temp < last_function_code || temp == esc))
                 {
                     buffer.append(esc);
                 }
@@ -1371,10 +1356,10 @@ compileString(StreamBuffer& buffer, const char*& source,
                 break;
             }
         }
-        if (c) continue;
+        if (c >= 0) continue;
         // source may contain a function name
         error(line, filename(),
-            "Unexpected word: %s\n", source);
+            "Unexpected word: \"%s\"\n", source);
         return false;
     }
     debug("StreamProtocolParser::Protocol::compileString buffer=%s\n", buffer.expand()());
@@ -1493,8 +1478,10 @@ compileFormat(StreamBuffer& buffer, const char*& formatstr,
     buffer.append(&streamFormat, sizeof(streamFormat));
     buffer.append(infoString);
 
-    debug("StreamProtocolParser::Protocol::compileFormat: format.type=%s, infolen=%d\n",
+    debug("StreamProtocolParser::Protocol::compileFormat: format.type=%s, "
-        StreamFormatTypeStr[streamFormat.type], streamFormat.infolen);
+        "infolen=%d infostring=\"%s\"\n",
+        StreamFormatTypeStr[streamFormat.type],
+        streamFormat.infolen, infoString.expand()());
     formatstr = source; // move pointer after parsed format
     return true;
 }

src/StreamVersion.c

@@ -23,6 +23,7 @@
 #define STR2(x) #x
 #define STR(x) STR2(x)
 const char StreamVersion [] =
-    "StreamDevice " STR(STREAM_MAJOR) "." STR(STREAM_MINOR)
+    "StreamDevice " STR(STREAM_MAJOR)
+    "." STR(STREAM_MINOR)
+    "." STR(STREAM_PATCHLEVEL)
     " built " __DATE__ " " __TIME__;
-

src/TimestampConverter.cc

@@ -82,14 +82,14 @@ parse(const StreamFormat&, StreamBuffer& info,
                         if (*c == 'f')
                         {
                             source = c;
-                            info.printf("%%0%uf", n);
+                            info.print("%%0%uf", n);
                             break;
                         }
                     }
                     /* look for nanoseconds %N of %f */
                     if (*source == 'N' || *source == 'f')
                     {
-                        info.printf("%%09f");
+                        info.print("%%09f");
                         break;
                     }
                     /* look for seconds with fractions like %.3S */
@@ -100,7 +100,7 @@ parse(const StreamFormat&, StreamBuffer& info,
                         if (toupper(*c) == 'S')
                         {
                             source = c;
-                            info.printf("%%%c.%%0%uf", *c, n);
+                            info.print("%%%c.%%0%uf", *c, n);
                             break;
                         }
                     }

src/devStream.h

@@ -22,7 +22,8 @@
 #define devStream_h
 
 #define STREAM_MAJOR 2
-#define STREAM_MINOR 4
+#define STREAM_MINOR 5
+#define STREAM_PATCHLEVEL 10
 
 #if defined(__vxworks) || defined(vxWorks)
 #include <vxWorks.h>

src/makedbd.pl

@@ -1,6 +1,10 @@
+if (@ARGV[0] == "-3.13") {
+    shift;
+} else {
+    print "variable(streamDebug, int)\n";
+    print "registrar(streamRegistrar)\n";
+}
+print "driver(stream)\n";
 for (@ARGV) {
      print "device($_,INST_IO,dev${_}Stream,\"stream\")\n";
 }
-print "driver(stream)\n";
-print "variable(streamDebug, int)\n";
-print "registrar(streamRegistrar)\n";

streamApp/terminal.tcl

@@ -1,7 +1,5 @@
 #!/usr/bin/env wish
 
-wm iconify .
-
 proc createTerm {sock} {
     global socket port
     toplevel .$sock
@@ -19,14 +17,6 @@ proc createTerm {sock} {
     wm title .$sock "port $port <-> [fconfigure $sock -peername]" 
 }
 
-set port [lindex $argv 0]
-if {$port == ""} { set port 40000 }
-if [catch {
-    socket -server connect $port
-} msg ] {
-    return -code error "$msg (port $port)"
-}
-
 proc connect {sock addr port} {
     fconfigure $sock -blocking 0 -buffering none
     createTerm $sock
@@ -84,6 +74,10 @@ proc receiveHandler {sock} {
             "echo" {
                 sendReply $sock [string range $a 5 end]
             }
+            "binary" {
+                set x [checkNum [lindex $l 1]]
+                sendReply $sock  [format %c $x]
+            }
             "longmsg" {
                 set length [checkNum [lindex $l 1]]
                 sendReply $sock "[string range x[string repeat 0123456789abcdefghijklmnopqrstuvwxyz [expr $length / 36 + 1]] 1 $length]\n"
@@ -95,7 +89,7 @@ proc receiveHandler {sock} {
             "start" {
                 set wait [checkNum [lindex $l 1]]
                 set ::counter 0
-                after $wait sendAsync $wait [list [lindex $l 2]]
+                after $wait sendAsync $wait [list [lrange $l 2 end-1]]
                 sendReply $sock "Started\n"
             }
             "stop" {
@@ -116,12 +110,16 @@ proc receiveHandler {sock} {
             "help" {
                 sendReply $sock "help             this text\n"
                 sendReply $sock "echo string      reply string\n"
-                sendReply $sock "wait msec        reply Done after some time\n"
+                sendReply $sock "binary number    reply byte with value number\n"
+                sendReply $sock "longmsg length   reply string with length characters\n"
+                sendReply $sock "wait msec        reply \"Done\" after some time\n"
                 sendReply $sock "start msec       start sending messages priodically\n"
                 sendReply $sock "stop             stop sending messages\n"
-                sendReply $sock "set key value    set a value\n"
+                sendReply $sock "set key value    store a value into variable key\n"
-                sendReply $sock "get key          reply value\n"
+                sendReply $sock "get key          reply previously stored value from key\n"
                 sendReply $sock "disconnect       close connection\n"
+                sendReply $sock "exit             kill terminal server\n"
+                
             }
         }
     } msg] {
@@ -138,6 +136,14 @@ proc sendAsync {wait message} {
     after $wait sendAsync $wait [list $message]
 }
 
+proc sendAsyncX {wait} {
+    if {$::counter < 0} return
+    foreach term [array names ::socket] {
+        sendReply $::socket($term) "\u00101\u0004~\u0005~\u00100\u0002|0062|2|1|0|1216|0|0.1087E+0 \u0003\u0012"
+    }
+    after $wait sendAsyncX $wait
+}
+
 if {[info proc tkTextInsert] != ""} {
     set insert tkTextInsert
     set paste tkTextPaste
@@ -153,7 +159,6 @@ rename $paste tkTextPaste_org
 rename $pastesel tkTextPasteSel_org
 
 proc $insert {w s} {
-    puts [list insert $w $s]
     global socket
     if {[string equal $s ""] || [string equal [$w cget -state] "disabled"]} {
         return
@@ -196,3 +201,15 @@ foreach tag {Clear Paste Copy Cut } {
 }
 
 bind Text <Control-Key> [list $insert %W %A]
+
+set port [lindex $argv 0]
+if {$port == ""} { set port 40000 }
+if [catch {
+    socket -server connect $port
+} msg ] {
+    return -code error "$msg (port $port)"
+}
+
+label .info -text "Accepting connections on port $port"
+button .exit -text "Exit" -command exit
+pack .info .exit -expand yes -fill x

streamApp/test.db

@@ -2,17 +2,20 @@ record (stringout, "$(P):cmd")
 {
     field (DTYP, "stream")
     field (OUT,  "@test.proto command terminal")
+    field (PRIO, "HIGH")
     field (VAL, "")
 }
 record (stringout, "$(P):info")
 {
     field (DTYP, "stream")
     field (OUT,  "@test.proto info terminal")
+    field (PRIO, "HIGH")
 }
 record (stringout, "$(P):request")
 {
     field (DTYP, "stream")
     field (OUT,  "@test.proto request($(P):reply.VAL) terminal")
+    field (PRIO, "HIGH")
 }
 record (stringin, "$(P):reply")
 {

streamApp/tests/streamtestlib.tcl

@@ -52,7 +52,7 @@ proc startioc {} {
     
     if [info exists streamversion] {
         puts $fd "#!/usr/local/bin/iocsh"
-        puts $fd "require stream,$streamversion"
+        puts $fd "require stream2,$streamversion"
     } else {
         puts $fd "#!../O.$env(EPICS_HOST_ARCH)/streamApp"
         puts $fd "dbLoadDatabase ../O.Common/streamApp.dbd"

streamApp/tests/testFormats

@@ -7,6 +7,11 @@ source streamtestlib.tcl
 # Send commands to the ioc shell with ioccmd
 
 set records {
+    record (bo, "DZ:percent")
+    {
+        field (DTYP, "stream")
+        field (OUT,  "@test.proto percent device")
+    }
     record (ao, "DZ:ao")
     {
         field (DTYP, "stream")
@@ -34,6 +39,7 @@ set records {
 
 set protocol {
     Terminator = LF;
+    percent {out "\%\x25\37\045" 0x25 37 045;}
     ao {out "%.2f %.2e %.2E %.2g %.2G %i %d %u %o %04x %#.2f %#.2e %#.2E %#.2g %#.2G %#i %#d %#u %#o %#06x";}
     lo {out "%d %(VAL)d %06d %x %06X %b %06b %.6b %B.! %06B.!";}
     bcd {out "%D %6D %.2D %.4D %.6D %.8D %#D %#6D %#.2D %#.4D %#.6D";}
@@ -58,6 +64,8 @@ startioc
 # This is normal. E.g. -1 HAS a different number of 1 bits.
 # Specify the width field in the format if this is a problem.
 
+ioccmd {dbpf DZ:percent 1}
+assure "%%%%%%%\n"
 ioccmd {dbpf DZ:ao 0}
 assure "0.00 0.00e+00 0.00E+00 0 0 32767 32767 32767 77777 7fff 0.00 0.00e+00 0.00E+00 0.0 0.0 32767 32767 32767 077777 0x7fff\n"
 ioccmd {dbpf DZ:ao 10}