rsrv: avoid possible overflow in vsend_err()

Accounting of message size doesn't take into account
space used by header of failed message (16 or 24 bytes).
This would allow a theoretical really long error message
to overflow the send buffer by 16 or 24 bytes.

src/rsrv/camessage.c

@@ -216,10 +216,10 @@ va_list                 args
     /*
      * add their context string into the protocol
      */
-    localStatus = epicsVsnprintf ( pMsgString, maxDiagLen, pformat, args );
+    localStatus = epicsVsnprintf ( pMsgString, maxDiagLen - size, pformat, args );
     if ( localStatus >= 1 ) {
         unsigned diagLen = ( unsigned ) localStatus;
-        if ( diagLen < maxDiagLen ) {
+        if ( diagLen < maxDiagLen - size ) {
             size += (ca_uint32_t) (diagLen + 1u);
         }
         else {
@@ -227,7 +227,7 @@ va_list                 args
                 "caserver: vsend_err: epicsVsnprintf detected "
                 "error message truncation, pFormat = \"%s\"\n",
                 pformat );
-            size += maxDiagLen;
+            size = maxDiagLen;
             pMsgString [ maxDiagLen - 1 ] = '\0';
         }
     }