Allow configuring of coloured console output with STREAM_DEBUG_COLOR

The STREAM_DEBUG_COLOR environment variable can be set to:
    yes  - always generate ANSI colour escape codes on debug/error output
    no   - use plain text output for debug/error
    auto - (default) output ANSI codes if terminal supports it

src/StreamBuffer.cc

@@ -341,7 +341,10 @@ StreamBuffer StreamBuffer::expand(ssize_t start, ssize_t length) const
     {
         c = buffer[i];
         if (c < 0x20 || c >= 0x7f)
-            result.print("\033[7m<%02x>\033[27m", c & 0xff);
+            result.print("%s<%02x>%s",
+                         ansiEscape(ANSI_REVERSE_VIDEO),
+                         c & 0xff,
+                         ansiEscape(ANSI_NOT_REVERSE_VIDEO));
         else
             result.append(c);
     }
@@ -354,18 +357,21 @@ dump() const
     StreamBuffer result;
     size_t i;
     result.print("%" P "d,%" P "d,%" P "d:", offs, len, cap);
-    if (offs) result.print("\033[47m");
+    if (offs) result.print(ansiEscape(ANSI_BG_WHITE));
     char c;
     for (i = 0; i < cap; i++)
     {
         c = buffer[i];
-        if (offs && i == offs) result.append("\033[0m");
+        if (offs && i == offs) result.append(ansiEscape(ANSI_RESET));
         if (c < 0x20 || c >= 0x7f)
-            result.print("\033[7m<%02x>\033[27m", c & 0xff);
+            result.print("%s<%02x>%s",
+                         ansiEscape(ANSI_REVERSE_VIDEO),
+                         c & 0xff,
+                         ansiEscape(ANSI_NOT_REVERSE_VIDEO));
         else
             result.append(c);
-        if (i == offs+len-1) result.append("\033[47m");
+        if (i == offs+len-1) result.append(ansiEscape(ANSI_BG_WHITE));
     }
-    result.append("\033[0m");
+    result.append(ansiEscape(ANSI_RESET));
     return result;
 }

src/StreamCore.cc

@@ -72,9 +72,9 @@ printCommands(StreamBuffer& buffer, const char* c)
                 buffer.append("    disconnect;\n");
                 break;
             default:
-                buffer.append("\033[31;1mGARBAGE: ");
+                buffer.append(ansiEscape(ANSI_RED_BOLD)).append("GARBAGE: ");
                 c = StreamProtocolParser::printString(buffer, c-1);
-                buffer.append("\033[0m\n");
+                buffer.append(ansiEscape(ANSI_RESET)).append("\n");
         }
     }
 }

src/StreamError.cc

@@ -23,10 +23,13 @@
 #include "StreamError.h"
 #ifdef _WIN32
 #include <windows.h>
-#endif
+#include <io.h>
+#endif /* _WIN32 */
 #include <string.h>
 #include <time.h>
 #include <stdio.h>
+#include <errlog.h>
+#include <epicsStdio.h>
 
 int streamDebug = 0;
 int streamError = 1;
@@ -46,18 +49,31 @@ FILE *StreamDebugFile = NULL;
 #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
 #endif
 
-/* Enable ANSI colors in Windows console */
-static int win_console_init() {
-    DWORD dwMode = 0;
-    HANDLE hCons = GetStdHandle(STD_ERROR_HANDLE);
-    GetConsoleMode(hCons, &dwMode);
-    dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-    SetConsoleMode(hCons, dwMode);
-    return 0;
+/* Enable ANSI color support in Windows console */
+static bool win_console_init() {
+    HANDLE hCons[] = { GetStdHandle(STD_ERROR_HANDLE),
+                       GetStdHandle(STD_OUTPUT_HANDLE) };
+    for(int i=0; i < sizeof(hCons) / sizeof(HANDLE); ++i)
+    {
+        DWORD dwMode = 0;
+        if (hCons[i] == INVALID_HANDLE_VALUE ||
+            !GetConsoleMode(hCons[i], &dwMode))
+        {
+            return false;
+        }
+        dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+        if (!SetConsoleMode(hCons[i], dwMode))
+        {
+            return false;
+        }
+    }
+    return true;
 }
-static int s = win_console_init();
 
-#endif
+/* true if console supports ANSI color codes */
+static bool win_console_colored = win_console_init();
+
+#endif /* _WIN32 */
 
 /* You can globally change the printTimestamp function
    by setting the StreamPrintTimestampFunction variable
@@ -106,14 +122,13 @@ void StreamVError(int line, const char* file, const char* fmt, va_list args)
         va_end(args2);
     }
 #endif
-    fprintf(stderr, "\033[31;1m");
-    fprintf(stderr, "%s ", timestamp);
+    fprintf(stderr, "%s%s", ansiEscape(ANSI_RED_BOLD), timestamp);
     if (file)
     {
         fprintf(stderr, "%s line %d: ", file, line);
     }
     vfprintf(stderr, fmt, args);
-    fprintf(stderr, "\033[0m");
+    fprintf(stderr, "%s", ansiEscape(ANSI_RESET));
 }
 
 int StreamDebugClass::
@@ -133,3 +148,31 @@ print(const char* fmt, ...)
     va_end(args);
     return 1;
 }
+
+/**
+ * Return an ANSI escape code if coloured debug output is enabled
+ * If auto mode if selected (default), return code if a supported console type
+ */
+const char* ansiEscape(AnsiMode mode)
+{
+    static const char* AnsiEscapes[] = { "\033[7m", "\033[27m", "\033[47m",
+                                         "\033[0m", "\033[31;1m" };
+    static const char* stream_debug_color = getenv("STREAM_DEBUG_COLOR");
+    bool color_output = false;
+    if (stream_debug_color == NULL || stream_debug_color[0] == 'A'||
+        stream_debug_color[0] == 'a') // auto
+    {
+#ifdef _WIN32
+        color_output =  win_console_colored &&
+                        _isatty(_fileno(epicsGetStdout()));
+#else
+        color_output =  isatty(fileno(epicsGetStdout()));
+#endif /* _WIN32 */
+    }
+    else if (stream_debug_color[0] == 'Y' || stream_debug_color[0] == 'y' ||
+             stream_debug_color[0] == '1') // yes
+    {
+        color_output = true;
+    }
+    return color_output ? AnsiEscapes[mode] : "";
+}

src/StreamError.h

@@ -66,4 +66,11 @@ StreamDebugObject(const char* file, int line)
 #define error StreamError
 #define debug (!streamDebug)?0:StreamDebugObject(__FILE__,__LINE__).print
 
+/*
+ * ANSI escape sequences for terminal output
+ */
+enum AnsiMode { ANSI_REVERSE_VIDEO, ANSI_NOT_REVERSE_VIDEO, ANSI_BG_WHITE,
+                ANSI_RESET, ANSI_RED_BOLD };
+extern const char* ansiEscape(AnsiMode mode);
+
 #endif