checksums after regsub must check original input

docs/formats.html

@@ -485,6 +485,12 @@ than 9.
 <p>
 This is not a normal "converter", because no user data is converted.
 Instead, a checksum is calculated from the input or output.
+<span class="new">
+Any pre-processing of input, e.g. by the <a href="#regsub">regsub</a> converter
+if ignored for the calculation of the checksum.
+</span>
+</p>
+<p>
 The <em>width</em> field is the byte number from which to start
 calculating the checksum.
 Default is 0, i.e. the first byte of the input or output of the current
@@ -711,10 +717,14 @@ However if an empty string is matched, searching advances by 1 character in orde
 avoid matching the same empty string again.</span>
 </p>
 <p>
-In input this converter pre-processes data received from the device before
+In input, this converter pre-processes data received from the device before
 following converters read it.
 Converters preceding this one will read unmodified input.
 Thus place this converter before those whose input should be pre-processed.
+<span class="new">
+However, <a href="#chksum">checksum</a> converters will always use the unmodified
+input as sent by the device because the modified input would not match the checksum.
+</span>
 </p>
 <p>
 In output it post-processes data already formatted by preceding converters

src/ChecksumConverter.cc

@@ -682,13 +682,13 @@ static uint32_t mask[5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF};
 
 class ChecksumConverter : public StreamFormatConverter
 {
-    int parse (const StreamFormat&, StreamBuffer&, const char*&, bool);
+    int parse (const StreamFormat&, StreamBuffer&, const char*&, bool scanFormat);
     bool printPseudo(const StreamFormat&, StreamBuffer&);
     ssize_t scanPseudo(const StreamFormat&, StreamBuffer&, size_t& cursor);
 };
 
 int ChecksumConverter::
-parse(const StreamFormat&, StreamBuffer& info, const char*& source, bool)
+parse(const StreamFormat&, StreamBuffer& info, const char*& source, bool scanFormat)
 {
     const char* p = strchr(source, '>');
     if (!p)
@@ -742,7 +742,7 @@ parse(const StreamFormat&, StreamBuffer& info, const char*& source, bool)
             info.append(&xorout, sizeof(xorout));
             info.append(fnum);
             source = p+1;
-            return pseudo_format;
+            return scanFormat ? needs_original_format : pseudo_format;
         }
     }
 
@@ -847,7 +847,7 @@ scanPseudo(const StreamFormat& format, StreamBuffer& input, size_t& cursor)
         else length = 0;
     }
 
-    debug("ChecksumConverter %s: input to check: \"%s\n",
+    debug("ChecksumConverter %s: input to check: \"%s\"\n",
         checksumMap[fnum].name, input.expand(start,length)());
 
     uint8_t nDigits =
@@ -859,8 +859,8 @@ scanPseudo(const StreamFormat& format, StreamBuffer& input, size_t& cursor)
 
     if ((ssize_t)( input.length() - cursor ) < expectedLength)
     {
-        debug("ChecksumConverter %s: Input '%s' too short for checksum\n",
+        debug("ChecksumConverter %s: Input '%s' too short (%zu-%zu<%zu) for checksum\n",
-            checksumMap[fnum].name, input.expand(cursor)());
+            checksumMap[fnum].name, input.expand(cursor)(), input.length(), cursor, expectedLength);
         return -1;
     }
 

src/StreamCore.cc

@@ -1184,6 +1184,7 @@ matchInput()
     char command;
     const char* fieldName = NULL;
     StreamBuffer formatstring;
+    ssize_t delta = 0;
 
     consumedInput = 0;
 
@@ -1218,7 +1219,7 @@ normal_format:
                 debug("StreamCore::matchInput(%s): format = \"%%%s\"\n",
                     name(), formatstring());
 
-                if (fmt.flags & skip_flag || fmt.type == pseudo_format)
+                if (fmt.flags & skip_flag || fmt.type == pseudo_format || fmt.type == needs_original_format)
                 {
                     long ldummy;
                     double ddummy;
@@ -1240,9 +1241,20 @@ normal_format:
                                 scanString(fmt, inputLine(consumedInput), NULL, size);
                             break;
                         case pseudo_format:
-                            // pass complete input
+                            // pass complete input line for scan and/or re-write
+                            size = inputLine.length();
                             consumed = StreamFormatConverter::find(fmt.conv)->
                                 scanPseudo(fmt, inputLine, consumedInput);
+                            delta += inputLine.length() - size; // track length changes
+                            debug("after rewrite delta=%zi\n", delta);
+                            break;
+                        case needs_original_format:
+                            // pass original input with adjusted current position
+                            debug("before checksum delta=%zi\n", delta);
+                            consumedInput -= delta; // correct for length changes
+                            consumed = StreamFormatConverter::find(fmt.conv)->
+                                scanPseudo(fmt, inputBuffer, consumedInput);
+                            consumedInput += delta;
                             break;
                         default:
                             error("INTERNAL ERROR (%s): illegal format.type 0x%02x\n",

src/StreamFormat.h

@@ -42,7 +42,8 @@ typedef enum {
     enum_format,
     double_format,
     string_format,
-    pseudo_format
+    pseudo_format,
+    needs_original_format
 } StreamFormatType;
 
 extern const char* StreamFormatTypeStr[];

streamApp/tests/testChecksum

@@ -256,6 +256,9 @@ set protocol {
         out "bitsum8  %s %#-9.1<bitsum8>";   in "bitsum8  %=s %#-9.1<bitsum8>";
         out "bitsum16 %s %#-9.1<bitsum16>";  in "bitsum16 %=s %#-9.1<bitsum16>";
         out "bitsum32 %s %#-9.1<bitsum32>";  in "bitsum32 %=s %#-9.1<bitsum32>";
+
+# Check combination of regsub and checksum. Always check what the device sees.
+        out "crc8     %s%#/[0-9]{2}/&:/ %9.1<crc8>";        in "crc8     %#/[0-9]{2}/&:/%s %9.1<crc8>"; out "%s";
         out "DONE";
     }
 }
@@ -740,6 +743,12 @@ assure "bitsum16 123456789 \x32\x31\x30\x30\n"
 send   "bitsum16 123456789 \x32\x31\x30\x30\n"
 assure "bitsum32 123456789 \x32\x31\x30\x30\x30\x30\x30\x30\n"
 send   "bitsum32 123456789 \x32\x31\x30\x30\x30\x30\x30\x30\n"
+
+# check regsub and checksums
+assure "crc8     12:34:56:78:9 \x07\n" ;# modified output string and checksum
+send   "crc8     123456789 \xF4\n"     ;# original input string and checksum
+assure "12:34:56:78:9\n"               ;# modified input string
+
 assure "DONE\n"
                 
 finish