Merged stacktrace-1 branch

configure/os/CONFIG.Common.linuxCommon

@@ -18,6 +18,8 @@ POSIX_LDLIBS = -lpthread
 
 OP_SYS_CPPFLAGS += -Dlinux
 OP_SYS_LDLIBS += -lrt -ldl
+# Use -rdynamic to maximize symbols available for stacktrace
+OP_SYS_LDFLAGS += -rdynamic
 
 # Linker flags for static & shared-library builds
 STATIC_LDFLAGS_YES= -Wl,-Bstatic

configure/os/CONFIG.win32-x86.win32-x86

@@ -40,19 +40,13 @@ WARN_CFLAGS_NO =  -W1
 
 #
 # -Ox maximum optimizations
-# -MD use MSVCRT (run-time as DLL, multi-thread support)
 # -GL whole program optimization
-# -Zi generate program database for debugging information
-OPT_CFLAGS_YES = -Ox -GL
+# -Oy- re-enable creation of frame pointers
+OPT_CFLAGS_YES = -Ox -GL -Oy-
 
 #
 # -Zi generate program database for debugging information
-# -Z7 include debugging info in object files
-# -Fr create source browser file
-# -GZ catch bugs occurring only in optimized code
-# -D_CRTDBG_MAP_ALLOC
 # -RTCsu catch bugs occuring only inoptimized code
-# -DEPICS_FREELIST_DEBUG good for detecting mem mrg bugs
 OPT_CFLAGS_NO = -Zi -RTCsu
 
 # specify object file name and location
@@ -81,14 +75,11 @@ CPP = cl -C -E
 
 # Configure OS vendor C++ compiler
 #
-# __STDC__=0 is a real great idea of Jeff that gives us both:
+# __STDC__=0 gives us both:
 # 1) define STDC for code (pretend ANSI conformance)
 # 2) set it to 0 to use MS C "extensions" (open for _open etc.)
 # because MS uses:   if __STDC__ ... disable many nice things
 #
-# Use of -Za would dissable DLL import/export keywords which
-# include/excludes using architecture neutral macros
-# 
 # -EHsc - generate code for exceptions
 # -GR - generate code for run time type identification
 #
@@ -106,16 +97,12 @@ WARN_CXXFLAGS_NO =  -W1
 #
 # -Ox maximum optimizations
 # -GL whole program optimization
-# -Zi generate program database for debugging information
-OPT_CXXFLAGS_YES = -Ox -GL
+# -Oy- re-enable creation of frame pointers
+OPT_CXXFLAGS_YES = -Ox -GL -Oy-
 
 #
 # -Zi generate program database for debugging information
-# -Z7 include debugging info in object files
-# -Fr create source browser file
-# -D_CRTDBG_MAP_ALLOC
 # -RTCsu catch bugs occurring only in optimized code
-# -DEPICS_FREELIST_DEBUG good for detecting mem mrg bugs
 OPT_CXXFLAGS_NO = -RTCsu -Zi
 
 # specify object file name and location
@@ -138,10 +125,12 @@ STATIC_LDLIBS_NO=
 STATIC_LDFLAGS=
 RANLIB=
 
+#
 # add -profile here to run the ms profiler
-# -LTCG - whole program optimization
-# -fixed:no good for programs such as purify and quantify
-# -debug good for programs such as purify and quantify
+# -LTCG whole program optimization
+# -incremental:no full linking
+# -fixed:no generate relocatable code
+# -debug generate debugging info
 LINK_OPT_FLAGS_YES = -LTCG -incremental:no -opt:ref \
     -release $(PROD_VERSION:%=-version:%)
 LINK_OPT_FLAGS_NO = -debug -incremental:no -fixed:no

src/libCom/misc/cantProceed.c

@@ -19,6 +19,7 @@
 #include "errlog.h"
 #include "cantProceed.h"
 #include "epicsThread.h"
+#include "epicsStackTrace.h"
 
 epicsShareFunc void * callocMustSucceed(size_t count, size_t size, const char *msg)
 {
@@ -62,6 +63,9 @@ epicsShareFunc void cantProceed(const char *msg, ...)
     
     errlogPrintf("Thread %s (%p) can't proceed, suspending.\n",
             epicsThreadGetNameSelf(), (void *)epicsThreadGetIdSelf());
+
+    epicsStackTrace();
+
     errlogFlush();
     
     epicsThreadSleep(1.0);

src/libCom/osi/Makefile

@@ -56,6 +56,7 @@ INC += epicsStdio.h
 INC += epicsStdioRedirect.h
 INC += epicsTempFile.h
 INC += epicsGetopt.h
+INC += epicsStackTrace.h
 
 INC += devLib.h
 INC += devLibVME.h
@@ -140,3 +141,8 @@ Com_SRCS_WIN32 += epicsGetopt.c
 Com_SRCS_WIN32 += setThreadName.cpp
 #Com_SRCS_WIN32 += dllmain.cpp
 Com_SRCS_WIN32 += forceBadAllocException.cpp
+
+#Stack trace support
+Com_SRCS += epicsStackTrace.c
+Com_SRCS += osdBackTrace.cpp
+Com_SRCS += osdFindAddr.cpp

src/libCom/osi/epicsStackTrace.c

@@ -0,0 +1,124 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include <stdlib.h>
+
+#include "epicsStackTracePvt.h"
+#include "epicsThread.h"
+#include "epicsMutex.h"
+#include "errlog.h"
+
+#define epicsExportSharedSymbols
+#include "epicsStackTrace.h"
+
+/* How many stack frames to capture               */
+#define MAXDEPTH 100
+
+static epicsThreadOnceId stackTraceInitId = EPICS_THREAD_ONCE_INIT;
+static epicsMutexId      stackTraceMtx;
+
+static void stackTraceInit(void *unused)
+{
+    stackTraceMtx = epicsMutexMustCreate();
+}
+
+static void stackTraceLock(void)
+{
+    epicsThreadOnce( &stackTraceInitId, stackTraceInit, 0 );
+    epicsMutexLock( stackTraceMtx );
+}
+
+static void stackTraceUnlock(void)
+{
+    epicsMutexUnlock( stackTraceMtx );
+}
+
+static int
+dumpInfo(void *addr, epicsSymbol *sym_p)
+{
+int rval = 0;
+
+	rval += errlogPrintf("[%*p]", (int)(sizeof(addr)*2 + 2), addr);
+	if ( sym_p ) {
+		if ( sym_p->f_nam ) {
+			rval += errlogPrintf(": %s", sym_p->f_nam);
+		}
+		if ( sym_p->s_nam ) {
+			rval += errlogPrintf("(%s+0x%lx)", sym_p->s_nam, (unsigned long)((char*)addr - (char*)sym_p->s_val));
+		} else {
+			rval += errlogPrintf("(<no symbol information>)");
+		}
+	}
+	rval += errlogPrintf("\n");
+	errlogFlush();
+
+    return rval;
+}
+
+void epicsStackTrace(void)
+{
+void        **buf;
+int         i,n;
+epicsSymbol sym;
+
+	if ( 0 == epicsStackTraceGetFeatures() ) {
+		/* unsupported on this platform */
+		return;
+	}
+
+    if ( ! (buf = malloc(sizeof(*buf) * MAXDEPTH))) {
+        free(buf);
+        errlogPrintf("epicsStackTrace(): not enough memory for backtrace\n");
+        return;
+    }
+
+    n = epicsBackTrace(buf, MAXDEPTH);
+
+	if ( n > 0 ) {
+
+		stackTraceLock();
+
+		errlogPrintf("Dumping a stack trace of thread '%s':\n", epicsThreadGetNameSelf());
+
+		errlogFlush();
+
+		for ( i=0; i<n; i++ ) {
+			if ( 0 == epicsFindAddr(buf[i], &sym) )
+				dumpInfo(buf[i], &sym);
+			else
+				dumpInfo(buf[i], 0);
+		}
+
+		errlogFlush();
+
+		stackTraceUnlock();
+
+	}
+
+    free(buf);
+}
+
+int epicsStackTraceGetFeatures()
+{
+void *test[2];
+
+static int initflag = 10; /* init to a value larger than the test snapshot */
+
+	/* don't bother about epicsOnce -- if there should be a race and
+	 * the detection code is executed multiple times that is no big deal.
+	 */
+	if ( 10 == initflag ) {
+		initflag = epicsBackTrace(test, sizeof(test)/sizeof(test[0]));
+	}
+
+	if ( initflag <= 0 )
+		return 0; /* backtrace doesn't work or is not supported */
+
+	return ( EPICS_STACKTRACE_ADDRESSES | epicsFindAddrGetFeatures() );
+}

src/libCom/osi/epicsStackTrace.h

@@ -0,0 +1,43 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#ifndef INC_epicsStackTrace_H
+#define INC_epicsStackTrace_H
+
+#include "shareLib.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Dump a stack trace to the errlog */
+epicsShareFunc void epicsStackTrace(void);
+
+/* Inquire about functionality implemented on your system */
+
+/* StackTrace provides numerical addresses      */
+#define EPICS_STACKTRACE_ADDRESSES   (1<<0)
+
+/* StackTrace is able to lookup dynamic symbols */
+#define EPICS_STACKTRACE_DYN_SYMBOLS (1<<1)
+
+/* StackTrace is able to lookup global symbols  */
+#define EPICS_STACKTRACE_GBL_SYMBOLS (1<<2)
+
+/* StackTrace is able to lookup local symbols   */
+#define EPICS_STACKTRACE_LCL_SYMBOLS (1<<3)
+
+/* returns ORed bitset of supported features    */
+epicsShareFunc int epicsStackTraceGetFeatures(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/libCom/osi/epicsStackTracePvt.h

@@ -0,0 +1,48 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#ifndef INC_epicsStackTracePvt_H
+#define INC_epicsStackTracePvt_H
+
+#include "shareLib.h"
+
+typedef struct epicsSymbol {
+	const char *f_nam;  /* file where the symbol is defined  */
+	const char *s_nam;  /* symbol name                       */
+	void       *s_val;  /* symbol value                      */	
+} epicsSymbol;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Take a snapshot of the stack into 'buf' (limited to buf_sz entries)
+ * RETURNS: actual number of entries in 'buf'
+ */
+epicsShareFunc int epicsBackTrace(void **buf, int buf_sz);
+
+/* Find symbol closest to 'addr'.
+ * 
+ * If successful the routine fills in the members of *sym_p but
+ * note that 'f_nam' and/or 's_nam' may be NULL if the address
+ * cannot be resolved.
+ *
+ * RETURNS: 0 on success, nonzero on failure (not finding an address
+ *          is not considered an error).
+ */
+epicsShareFunc int epicsFindAddr(void *addr, epicsSymbol *sym_p);
+
+/* report supported features (as reported by epicsStackTraceGetFeatures) */
+epicsShareFunc int epicsFindAddrGetFeatures();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/libCom/osi/os/Darwin/osdBackTrace.cpp

@@ -0,0 +1,10 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include "osdExecinfoBackTrace.cpp"

src/libCom/osi/os/Darwin/osdFindAddr.cpp

@@ -0,0 +1,43 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+/* Make sure dladdr() is visible */
+#define _DARWIN_C_SOURCE
+
+#include <dlfcn.h>
+
+#define epicsExportSharedSymbols
+#include "epicsStackTrace.h"
+#include "epicsStackTracePvt.h"
+
+/* Darwin's finds local symbols, too :-) */
+
+int epicsFindAddr(void *addr, epicsSymbol *sym_p)
+{
+Dl_info    inf;
+
+    if ( ! dladdr(addr, &inf) ) {
+		sym_p->f_nam = 0;
+		sym_p->s_nam = 0;
+		sym_p->s_val = 0;
+	} else {
+		sym_p->f_nam = inf.dli_fname;
+		sym_p->s_nam = inf.dli_sname;
+		sym_p->s_val = inf.dli_saddr;
+	}
+
+	return 0;
+}
+
+int epicsFindAddrGetFeatures(void)
+{
+    return  EPICS_STACKTRACE_LCL_SYMBOLS
+          | EPICS_STACKTRACE_GBL_SYMBOLS
+          | EPICS_STACKTRACE_DYN_SYMBOLS;
+}

src/libCom/osi/os/Linux/osdBackTrace.cpp

@@ -0,0 +1,10 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include "osdExecinfoBackTrace.cpp"

src/libCom/osi/os/Linux/osdFindAddr.cpp

@@ -0,0 +1,10 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include "osdElfFindAddr.cpp"

src/libCom/osi/os/WIN32/osdBackTrace.cpp

@@ -0,0 +1,23 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2014
+ */ 
+
+#include <Windows.h>
+
+#define epicsExportSharedSymbols
+#include "epicsStackTracePvt.h"
+
+int epicsBackTrace(void **buf, int buf_sz)
+{
+	/* Docs say that (for some windows versions) the sum of
+	 * skipped + captured frames must be less than 63
+	 */
+	if ( buf_sz >= 63 )
+		buf_sz = 62;
+	return CaptureStackBackTrace(0, buf_sz, buf, 0);
+}

src/libCom/osi/os/default/osdAssert.c

@@ -20,6 +20,7 @@
 #include "epicsThread.h"
 #include "epicsTime.h"
 #include "cantProceed.h"
+#include "epicsStackTrace.h"
 
 
 void epicsAssert (const char *pFile, const unsigned line,
@@ -31,6 +32,9 @@ void epicsAssert (const char *pFile, const unsigned line,
         "A call to 'assert(%s)'\n"
         "    by thread '%s' failed in %s line %u.\n",
         pExp, epicsThreadGetNameSelf(), pFile, line);
+
+    epicsStackTrace();
+
     errlogPrintf("EPICS Release %s.\n", epicsReleaseVersion);
 
     if (epicsTimeGetCurrent(&current) == 0) {

src/libCom/osi/os/default/osdBackTrace.cpp

@@ -0,0 +1,16 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#define epicsExportSharedSymbols
+#include "epicsStackTracePvt.h"
+
+int epicsBackTrace(void **buf, int buf_sz)
+{
+	return -1;
+}

src/libCom/osi/os/default/osdFindAddr.cpp

@@ -0,0 +1,25 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#define epicsExportSharedSymbols
+#include "epicsStackTracePvt.h"
+#include "epicsStackTrace.h"
+
+int epicsFindAddr(void *addr, epicsSymbol *sym_p)
+{
+	sym_p->f_nam = 0;
+	sym_p->s_nam = 0;
+	sym_p->s_val = 0;
+	return -1;
+}
+
+int epicsFindAddrGetFeatures(void)
+{
+	return 0;
+}

src/libCom/osi/os/posix/osdElfFindAddr.cpp

@@ -0,0 +1,655 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include <unistd.h>
+#include <dlfcn.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <string.h>
+#include <elf.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+
+#ifdef _POSIX_MAPPED_FILES
+#include <sys/mman.h>
+#endif
+
+#include "epicsMutex.h"
+#include "epicsThread.h"
+#include <errlog.h>
+
+#define epicsExportSharedSymbols
+#include "epicsStackTrace.h"
+#include "epicsStackTracePvt.h"
+
+#define FIND_ADDR_DEBUG 0
+
+/* 
+ * On some systems (linux, solaris) dladdr doesn't find local symbols
+ * or symbols in the main executable.
+ * Hence, we want to use dladdr() to find the file name
+ * where a symbol is defined and if not more information is available
+ * then proceed to lookup symbols in the ELF symbol tables.
+ */
+
+/* Macros to handle elf32 vs. elf64 access to unions etc. */
+
+#define FLD(c,s,f)   (ELFCLASS32==c ? s.e32.f    : s.e64.f   )
+#define ARR(c,s,i,f) (ELFCLASS32==c ? s.e32[i].f : s.e64[i].f)
+
+/* Elf header      */
+typedef union Ehdr_ {
+    Elf32_Ehdr e32;
+    Elf64_Ehdr e64;
+} Ehdr;
+
+/* Section header  */
+typedef union Shdr_ {
+    Elf32_Shdr e32;
+    Elf64_Shdr e64;
+} Shdr;
+
+/* Elf symbol     */
+typedef union Sym_ {
+    void       *raw;
+    Elf32_Sym  *e32;
+    Elf64_Sym  *e64;
+} Sym;
+
+/* Memory mapped portion of a file; we must
+ * keep additional information because the
+ * map's starting address + length must be
+ * page-aligned (man mmap).
+ */
+typedef struct MMap_ {
+    void    *addr;
+    off_t    off;                        /* offset into the map where 'real' data start            */
+    size_t   len;
+    size_t   max;                        /* max offset: legal data from addr+off .. addr+off+max-1 */
+    void     (*freeMap)(struct MMap_*);  /* 'method' to destroy the mapping                        */
+} *MMap;
+
+/* Structure describing symbol information
+ * contained in a file.
+ * We keep these around (so that the file 
+ * doesn't have to be opened + parsed every
+ * time we do a lookup).
+ */
+typedef struct ESyms_ {
+    struct ESyms_ *next;  /* linked list; one struct per executable */
+    const char    *fname; /* file name                              */
+    int            fd;    /* file descriptor                        */
+    uintptr_t      addr;  /* address where file is loaded           */
+    MMap           symMap;
+    MMap           strMap;
+    size_t         nsyms;
+    uint8_t        eclss;
+} *ESyms;
+
+/* LOCKING NOTE: if the ELF symbol facility is ever expanded to be truly used
+ * in a multithreaded way then proper multiple-readers, single-writer locking
+ * should be implemented.
+ */
+
+/* Linked list where we keep all our ESyms */
+static ESyms             elfs       = 0;
+
+static epicsMutexId      listMtx;
+static epicsThreadOnceId listMtxInitId = EPICS_THREAD_ONCE_INIT;
+
+static time_t            prog_start_time = time(0);
+
+extern "C" {
+
+static void listMtxInit(void *unused)
+{
+    listMtx = epicsMutexMustCreate();
+}
+
+}
+
+static void
+elfsLockWrite()
+{
+    epicsThreadOnce(&listMtxInitId, listMtxInit, 0);
+    epicsMutexMustLock(listMtx);
+}
+
+static void
+elfsUnlockWrite()
+{
+    epicsMutexUnlock(listMtx);
+}
+
+static void
+freeMap(MMap m)
+{
+    if ( m ) {
+        m->freeMap(m);
+        free(m);
+    }
+}
+
+/* Helper to read exactly 'sz' bytes into 'buf'
+ * RETURNS: # chars read or negative value on error.
+ */
+static ssize_t
+do_read(int fd, void *buf, ssize_t sz)
+{
+ssize_t got;
+char   *ptr=(char*)buf;
+    while ( sz > 0 ) {
+        if ( (got=read(fd,ptr,sz)) <= 0 ) {
+            return got;
+        }
+        ptr+=got;
+        sz -=got;
+    }
+    return ptr-(char*)buf;
+}
+
+/* Elf file access -- can either be with mmap or by sequential read */
+
+#ifdef _POSIX_MAPPED_FILES
+/* Destructor for data that is mmap()ed */
+static void
+freeMapMmap(MMap m)
+{
+    if ( MAP_FAILED != m->addr )
+        munmap( m->addr, m->len );
+}
+
+/* Obtain section data with mmap()      */
+static MMap
+getscn_mmap(int fd, uint8_t c, Shdr *shdr_p)
+{
+off_t    n;
+MMap     rval = 0;
+size_t   pgsz  = sysconf(_SC_PAGESIZE);
+
+    if ( 0 == (n = (off_t)FLD(c,(*shdr_p),sh_size)) ) {
+        errlogPrintf("elfRead - getscn() -- no section data\n");
+        goto bail;
+    }
+
+    if ( ! (rval = (MMap) malloc(sizeof(*rval))) ) {
+        errlogPrintf("elfRead - getscn() -- no memory for section map\n");
+        goto bail;
+    }
+
+    rval->freeMap = freeMapMmap;
+
+    rval->off = (off_t) (FLD(c,(*shdr_p),sh_offset)   & (pgsz-1));
+    rval->len = (n + rval->off + (pgsz - 1)) & ~(pgsz - 1);
+    rval->max = rval->len - rval->off;
+
+    if ( MAP_FAILED == (rval->addr = mmap(0, rval->len, PROT_READ, MAP_SHARED, fd, (off_t) (FLD(c,(*shdr_p),sh_offset) & ~(pgsz-1)))) ) {
+        errlogPrintf("elfRead - getscn() -- mapping section contents: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    return rval;
+
+bail:
+    freeMap(rval);
+    return 0;
+}
+#else
+static MMap getscn_mmap(int fd, uint8_t c, Shrd *shdr_p)
+{
+	return 0;
+}
+#endif
+
+/* Destructor for data that is read into a malloc()ed buffer */
+static void
+freeMapMalloc(MMap m)
+{
+    free(m->addr);
+}
+
+/* Read section data into a malloc()ed buffer                */
+static MMap
+getscn_read(int fd, uint8_t c, Shdr *shdr_p)
+{
+ssize_t  n;
+MMap     rval = 0;
+
+    if ( 0 == (n = (ssize_t) FLD(c,(*shdr_p),sh_size)) ) {
+        errlogPrintf("elfRead - getscn() -- no section data\n");
+        goto bail;
+    }
+
+    if ( ! (rval = (MMap) malloc(sizeof(*rval))) ) {
+        errlogPrintf("elfRead - getscn() -- no memory for section map\n");
+        goto bail;
+    }
+
+    rval->freeMap = freeMapMalloc;
+
+    if ( ! (rval->addr = malloc(n)) ) {
+        errlogPrintf("elfRead - getscn() -- no memory for section data\n");
+        goto bail;
+    }
+
+    rval->off = 0;
+    rval->len = n;
+    rval->max = rval->len - rval->off;
+
+    /* seek to symbol table contents */
+    if ( (off_t)-1 == lseek(fd, (off_t) FLD(c,(*shdr_p),sh_offset), SEEK_SET) ) {
+        errlogPrintf("elfRead - getscn() -- seeking to sh_offset: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    if ( n != do_read(fd, rval->addr, n) ) {
+        errlogPrintf("elfRead - getscn() -- reading section contents: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    return rval;
+
+bail:
+    freeMap(rval);
+    return 0;
+}
+
+static MMap
+getscn(int fd, uint8_t c, Shdr *shdr_p)
+{
+MMap rval = getscn_mmap(fd, c, shdr_p);
+
+	if ( ! rval )
+		rval = getscn_read(fd, c, shdr_p);
+	
+	return rval;
+}
+
+/* Release resources but keep filename so that
+ * a file w/o symbol table is not read over and over again.
+ */
+static void
+elfSymsRelease(ESyms es)
+{
+    if ( es ) {
+        freeMap(es->symMap);
+        es->symMap = 0;
+        freeMap(es->strMap);
+        es->strMap = 0;
+        if ( es->fd >= 0 )
+            close(es->fd);
+        es->fd     = -1;
+        es->nsyms  = 0;
+    }
+}
+
+static ESyms
+elfRead(const char *fname, uintptr_t fbase)
+{
+int         i;
+Ehdr        ehdr;
+Shdr        shdr;
+uint8_t     c;
+ESyms       es;
+ssize_t     idx,n;
+const char *cp;
+struct stat stat_b;
+
+    if ( !(es = (ESyms) malloc(sizeof(*es))) ) {
+        /* no memory -- give up */
+        return 0;
+    }
+
+    memset(es, 0, sizeof(*es));
+    es->fd    = -1;
+    es->fname = fname;
+
+    if ( (es->fd = open(fname, O_RDONLY)) < 0 ) {
+        errlogPrintf("elfRead() -- unable to open file: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    if ( EI_NIDENT != do_read(es->fd, &ehdr, EI_NIDENT) ) {
+        errlogPrintf("elfRead() -- unable to read ELF e_ident: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    if (    ELFMAG0 != ehdr.e32.e_ident[EI_MAG0]
+         || ELFMAG1 != ehdr.e32.e_ident[EI_MAG1]
+         || ELFMAG2 != ehdr.e32.e_ident[EI_MAG2]
+         || ELFMAG3 != ehdr.e32.e_ident[EI_MAG3] ) {
+        errlogPrintf("bad ELF magic number\n");
+        goto bail;
+    }
+
+    if ( EV_CURRENT != ehdr.e32.e_ident[EI_VERSION] ) {
+        errlogPrintf("bad ELF version\n");
+        goto bail;
+    }
+
+    switch ( (es->eclss = c = ehdr.e32.e_ident[EI_CLASS]) ) {
+        default:
+            errlogPrintf("bad ELF class\n");
+        goto bail;
+
+        case ELFCLASS32:
+            n = sizeof(Elf32_Ehdr);
+        break;
+        case ELFCLASS64:
+            n = sizeof(Elf64_Ehdr);
+        break;
+    }
+    n -= EI_NIDENT;
+
+	if ( 0 == fstat(es->fd, &stat_b) ) {
+		if ( stat_b.st_mtime >= prog_start_time ) {
+			errlogPrintf("elfRead() -- WARNING: '%s' was modified after program start -- symbol information may be inaccurate or invalid\n", fname);
+		}
+	}
+
+    /* read rest */
+    if ( n != do_read(es->fd, ehdr.e32.e_ident + EI_NIDENT, n) ) {
+        errlogPrintf("elfRead() -- unable to read ELF ehdr: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    /* seek to section header table */
+    if ( (off_t)-1 == lseek(es->fd, (off_t) FLD(c,ehdr,e_shoff), SEEK_SET) ) {
+        errlogPrintf("elfRead() -- unable to seek to shoff: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    n = ELFCLASS32 == c ? sizeof(shdr.e32) : sizeof(shdr.e64);
+
+    for ( i = 0; i<FLD(c,ehdr,e_shnum); i++ ) {
+        if ( n != do_read(es->fd, &shdr, n) ) {
+            errlogPrintf("elfRead() -- unable to read section header: %s\n", strerror(errno));
+            goto bail;
+        }
+        if ( SHT_SYMTAB == FLD(c,shdr,sh_type) )
+            break;
+    }
+
+	if ( i>=FLD(c,ehdr,e_shnum) ) {
+		/* no SYMTAB -- try dynamic symbols */
+
+		if ( (off_t)-1 == lseek(es->fd, (off_t) FLD(c,ehdr,e_shoff), SEEK_SET) ) {
+			errlogPrintf("elfRead() -- unable to seek to shoff: %s\n", strerror(errno));
+			goto bail;
+		}
+
+		for ( i = 0; i<FLD(c,ehdr,e_shnum); i++ ) {
+			if ( n != do_read(es->fd, &shdr, n) ) {
+				errlogPrintf("elfRead() -- unable to read section header: %s\n", strerror(errno));
+				goto bail;
+			}
+			if ( SHT_DYNSYM == FLD(c,shdr,sh_type) )
+				break;
+		}
+    }
+
+    if ( i>=FLD(c,ehdr,e_shnum) ) {
+        errlogPrintf("elfRead() -- no symbol table found\n");
+        goto bail;
+    }
+
+    if ( 0 == (n = (ssize_t) FLD(c,shdr,sh_size)) ) {
+        errlogPrintf("elfRead() -- no symbol table data\n");
+        goto bail;
+    }
+
+    if ( !(es->symMap = getscn(es->fd, c, &shdr)) ) {
+        errlogPrintf("elfRead() -- unable to read ELF symtab\n");
+        goto bail;
+    }
+
+    es->nsyms = n / (ELFCLASS32==c ? sizeof(Elf32_Sym) : sizeof(Elf64_Sym));
+
+    /* find and read string table */
+    
+    n = ELFCLASS32 == c ? sizeof(shdr.e32) : sizeof(shdr.e64);
+
+    /* seek to section header table */
+    if ( (off_t)-1 == lseek(es->fd, (off_t) (FLD(c,ehdr,e_shoff) + n * FLD(c,shdr,sh_link)), SEEK_SET) ) {
+        errlogPrintf("elfRead() -- unable to lseek to ELF e_shoff: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    if ( n != do_read(es->fd, &shdr, n) ) {
+        errlogPrintf("elfRead() -- unable to read ELF strtab section header: %s\n", strerror(errno));
+        goto bail;
+    }
+
+    if ( !(es->strMap = getscn(es->fd,c,&shdr)) ) {
+        errlogPrintf("elfRead() -- unable to read ELF strtab\n");
+        goto bail;
+    }
+
+    /* Make sure there is a terminating NUL - unfortunately, memrchr is not portable */
+    cp = (char*)es->strMap->addr + es->strMap->off;
+    for ( idx = es->strMap->max - 1; i >= 0; i-- ) {
+        if ( !cp[i] )
+            break;
+    }
+    es->strMap->max = idx + 1;
+
+    switch ( FLD(c,ehdr,e_type) ) {
+        case ET_EXEC:
+            /* Symbols in an executable already has absolute addresses   */
+            es->addr = 0;
+            break;
+        case ET_DYN:
+            /* Symbols in an shared library are relative to base address */
+            es->addr  = fbase;
+            break;
+        default:
+            errlogPrintf("dlLookupAddr(): Unexpected ELF object file type %u\n", FLD(c,ehdr,e_type));
+            goto bail;
+    }
+        
+    return es;
+
+bail:
+    elfSymsRelease(es);
+    return es;
+}
+
+/* Destroy a cached ELF symbol table */
+static void
+elfSymsDestroy(ESyms es)
+{
+    if ( es ) {
+        elfSymsRelease(es);
+        free(es);
+    }
+}
+
+/* Destroy all cached ELF symbol tables
+ *
+ * However - w/o proper locking for read access
+ * this must not be used. Otherwise, readers
+ * will hold stale pointers...
+ *
+ * We leave the commented code here to show
+ * how the tables can be torn down.
+
+void
+elfSymTblFlush()
+{
+ESyms es;
+
+    elfsLockWrite();
+    while ( (es = elfs) ) {
+        elfs = es->next; 
+        es->next = 0;
+    	elfsUnlockWrite();
+        elfSymsDestroy(es);
+    	elfsLockWrite();
+    }
+   	elfsUnlockWrite();
+}
+
+*/
+
+
+/* This routine must be called with the write-lock held */
+static ESyms
+elfSymsFind(const char *fname)
+{
+ESyms es;
+    for ( es=elfs; es && strcmp(fname, es->fname); es = es->next )
+        /* nothing else to do */; 
+    return es;
+}
+
+int
+epicsFindAddr(void *addr, epicsSymbol *sym_p)
+{
+Dl_info    inf;
+ESyms      es,nes = 0;
+uintptr_t  minoff,off;
+size_t     i;
+Sym        sym;
+Sym        nearest;
+const char *strtab;
+uint8_t    c;
+size_t     idx;
+
+    if ( ! dladdr(addr, &inf) || (!inf.dli_fname && !inf.dli_sname) ) {
+		sym_p->f_nam = 0;
+		sym_p->s_nam = 0;
+        /* unable to lookup   */
+        return 0;
+    }
+
+	sym_p->f_nam = inf.dli_fname;
+
+	/* If the symbol is in the main executable then solaris' dladdr returns bogus info */
+#ifndef __sun
+    if ( (sym_p->s_nam = inf.dli_sname) ) {
+		sym_p->s_val = inf.dli_saddr;
+        /* Have a symbol name - just use it and be done */
+		return 0;
+    }
+#endif
+
+    /* No symbol info; try to access ELF file and ready symbol table from there */
+
+    elfsLockWrite();
+
+    /* See if we have loaded this file already */
+    es = elfSymsFind(inf.dli_fname);
+
+    if ( !es ) {
+
+    	elfsUnlockWrite();
+
+        if ( ! (nes = elfRead(inf.dli_fname, (uintptr_t)inf.dli_fbase)) )  {
+            /* this path can only be taken if there is no memory for '*nes' */
+            return 0;
+        }
+
+        elfsLockWrite();
+
+        /* Has someone else intervened and already added this file while we were reading ? */
+        es = elfSymsFind(inf.dli_fname);
+
+        if ( es ) {
+            /* will undo our work in the unlikely event... */
+        } else {
+            nes->next = elfs;
+            es  = elfs = nes;
+			nes = 0;
+        }
+    }
+
+    elfsUnlockWrite();
+
+	/* Undo our work in the unlikely event that it was redundant */
+	if ( nes )
+		elfSymsDestroy( nes );
+
+    nearest.raw = 0;
+    minoff      = (uintptr_t)-1LL;
+
+    if ( es->nsyms ) {
+        c       = es->eclss;
+        sym.raw = (char*)es->symMap->addr + es->symMap->off;
+        strtab  = (char*)es->strMap->addr + es->strMap->off;
+
+        /* Do a brute-force search through the symbol table; if this is executed 
+         * very often then it would be worthwhile constructing a sorted list of
+         * symbol addresses but for the stack trace we don't care...
+         */
+#if (FIND_ADDR_DEBUG & 1)
+        printf("Looking for %p\n", addr);
+#endif
+
+        if ( ELFCLASS32 == c ) {
+            for ( i=0; i<es->nsyms; i++ ) {
+                if ( STT_FUNC != ELF32_ST_TYPE(sym.e32[i].st_info) )
+                    continue;
+                /* don't bother about undefined symbols */
+                if ( 0 == sym.e32[i].st_shndx )
+                    continue;
+#if (FIND_ADDR_DEBUG & 1)
+                printf("Trying: %s (0x%lx)\n", strtab + sym.e32[i].st_name, (unsigned long)(sym.e32[i].st_value + es->addr));
+#endif
+                if ( (uintptr_t)addr >= (uintptr_t)sym.e32[i].st_value + es->addr ) {
+                    off = (uintptr_t)addr - ((uintptr_t)sym.e32[i].st_value + es->addr);
+                    if ( off < minoff ) {
+                        minoff      = off;
+                        nearest.e32 = &sym.e32[i];
+                    }
+                }
+            }
+        } else {
+            for ( i=0; i<es->nsyms; i++ ) {
+                if ( STT_FUNC != ELF64_ST_TYPE(sym.e64[i].st_info) )
+                    continue;
+                /* don't bother about undefined symbols */
+                if ( 0 == sym.e64[i].st_shndx )
+                    continue;
+#if (FIND_ADDR_DEBUG & 1)
+                printf("Trying: %s (0x%llx)\n", strtab + sym.e64[i].st_name, (unsigned long long)(sym.e64[i].st_value + es->addr));
+#endif
+                if ( (uintptr_t)addr >= (uintptr_t)sym.e64[i].st_value + es->addr ) {
+                    off = (uintptr_t)addr - ((uintptr_t)sym.e64[i].st_value + es->addr);
+                    if ( off < minoff ) {
+                        minoff      = off;
+                        nearest.e64 = &sym.e64[i];
+                    }
+                }
+            }
+        }
+    }
+
+    if ( nearest.raw && ( (idx = ARR(c,nearest,0,st_name)) < es->strMap->max ) ) {
+		sym_p->s_nam = strtab + idx;
+		sym_p->s_val = (char*) ARR(c, nearest, 0, st_value) + es->addr;
+    }
+
+    return 0;
+}
+
+int epicsFindAddrGetFeatures(void)
+{
+	/* The static information given here may not be correct;
+	 * it also depends on
+	 *  - compilation (frame pointer optimization)
+	 *  - linkage (static vs. dynamic)
+	 *  - stripping
+	 */
+    return  EPICS_STACKTRACE_LCL_SYMBOLS
+          | EPICS_STACKTRACE_GBL_SYMBOLS
+          | EPICS_STACKTRACE_DYN_SYMBOLS;
+}

src/libCom/osi/os/posix/osdExecinfoBackTrace.cpp

@@ -0,0 +1,18 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include <execinfo.h>
+
+#define epicsExportSharedSymbols
+#include "epicsStackTracePvt.h"
+
+int epicsBackTrace(void **buf, int buf_sz)
+{
+	return backtrace(buf, buf_sz);
+}

src/libCom/osi/os/solaris/osdBackTrace.cpp

@@ -0,0 +1,46 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2014
+ */ 
+
+#include <ucontext.h>
+
+#define epicsExportSharedSymbols
+#include "epicsStackTracePvt.h"
+
+struct wlk {
+	void **buf;
+	int   max;
+	int   cur;
+};
+
+
+extern "C" {
+
+static int
+walker(uintptr_t addr, int sig, void *arg)
+{
+struct wlk *w_p = (struct wlk *)arg;
+	if ( w_p->cur < w_p->max )
+		w_p->buf[w_p->cur++] = (void*)addr;
+	return 0;
+}
+
+}
+
+int epicsBackTrace(void **buf, int buf_sz)
+{
+ucontext_t u;
+struct     wlk d;
+	d.buf = buf;
+	d.max = buf_sz;
+	d.cur = 0;
+	if ( getcontext(&u) )
+		return -1;
+	walkcontext( &u, walker, &d );
+	return d.cur;
+}

src/libCom/osi/os/solaris/osdFindAddr.cpp

@@ -0,0 +1,10 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2011, 2014
+ */ 
+
+#include "osdElfFindAddr.cpp"

src/libCom/test/Makefile

@@ -184,6 +184,11 @@ epicsMessageQueueTest_SRCS += epicsMessageQueueTest.cpp
 testHarness_SRCS += epicsMessageQueueTest.cpp
 TESTS += epicsMessageQueueTest
 
+TESTPROD_HOST += epicsStackTraceTest
+epicsStackTraceTest_SRCS += epicsStackTraceTest.c
+testHarness_SRCS += epicsStackTraceTest.c
+TESTS += epicsStackTraceTest
+
 
 # The testHarness runs all the test programs in a known working order.
 testHarness_SRCS += epicsRunLibComTests.c

src/libCom/test/epicsRunLibComTests.c

@@ -29,6 +29,7 @@ int epicsExceptionTest(void);
 int epicsMathTest(void);
 int epicsMessageQueueTest(void);
 int epicsMutexTest(void);
+int epicsStackTraceTest(void);
 int epicsStdioTest(void);
 int epicsStdlibTest(void);
 int epicsStringTest(void);
@@ -84,6 +85,8 @@ void epicsRunLibComTests(void)
 
     runTest(epicsMutexTest);
 
+    runTest(epicsStackTraceTest);
+
     runTest(epicsStdioTest);
 
     runTest(epicsStdlibTest);

src/libCom/test/epicsStackTraceTest.c

@@ -0,0 +1,239 @@
+/* 
+ * Copyright: Stanford University / SLAC National Laboratory.
+ *
+ * EPICS BASE is distributed subject to a Software License Agreement found
+ * in file LICENSE that is included with this distribution. 
+ *
+ * Author: Till Straumann <strauman@slac.stanford.edu>, 2014
+ */ 
+
+/*
+ * Check stack trace functionality
+ */
+
+#include "epicsStackTrace.h"
+#include "errlog.h"
+#include "epicsUnitTest.h"
+#include "testMain.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#define TST_BUFSZ 10000
+
+#define MAXP      10
+
+/* estimated size of (compiled) epicsStackTraceRecurseGbl */
+#define WINDOW_SZ  400
+
+static int test_debug = 0;
+
+typedef struct TestDataRec_ {
+    char buf[TST_BUFSZ];
+    int  pos;
+} TestDataRec, *TestData;
+
+typedef void (*RecFn)(int);
+
+/* We want a stack trace and need a few nested routines.
+ * The whole magic here is intended to prevent a compiler
+ * from optimizing the call stack away:
+ *   - call via a volatile pointer
+ *   - add a call to a no-op function at the end so that
+ *     tail-call optimization doesn't eliminate the call
+ *     stack.
+ *
+ * We use a local (static) and a global routine to test
+ * if the stacktrace supports either flavor.
+ */
+
+void epicsStackTraceRecurseGbl(int lvl);
+static void epicsStackTraceRecurseLcl(int lvl);
+
+void nopFn(int lvl)
+{
+}
+
+RecFn volatile lfp = epicsStackTraceRecurseLcl;
+RecFn volatile gfp = epicsStackTraceRecurseGbl;
+RecFn volatile nfp = nopFn;
+
+static void
+epicsStackTraceRecurseLcl(int lvl)
+{
+    if ( lvl )
+        gfp(lvl-1);
+    else
+        epicsStackTrace();
+
+    /* call something so that the call through gfp() doesn't
+     * get optimized into a jump (tail-call optimization)
+     */
+    nfp(0);
+}
+
+void epicsStackTraceRecurseGbl(int lvl)
+{
+    if ( lvl )
+        lfp(lvl-1);
+    else
+        epicsStackTrace();
+
+    /* call something so that the call through gfp() doesn't
+     * get optimized into a jump (tail-call optimization)
+     */
+    nfp(0);
+}
+
+static void logClient(void *ptr, const char *msg)
+{
+    TestData td = ptr;
+    size_t sz = strlen(msg);
+    size_t mx = sizeof(td->buf) - td->pos - 1;
+
+    if ( sz > mx )
+        sz = mx;
+    strncpy( td->buf+td->pos, msg, sz );
+    td->pos += sz;
+}
+
+static int
+findStringOcc(const char *buf, const char *what)
+{
+    int rval = 0;
+    size_t l = strlen(what);
+    int ch;
+
+    while ( (buf = strstr(buf, what)) ) {
+        /* Is it just a prefix? */
+        ch = buf[l];
+        if ( ! isalnum(ch) && '_' != ch ) {
+            rval++;
+        }
+        buf += l;
+    }
+
+    if ( test_debug )
+        testDiag("found %i x %s\n", rval, what);
+
+    return rval;
+}
+
+static int
+findNumOcc(const char *buf)
+{
+    void *ptrs[MAXP];
+    int n_ptrs = 0;
+    int i,j;
+    int rval = 0;
+
+    while ( n_ptrs < sizeof(ptrs)/sizeof(ptrs[0]) && (buf=strchr(buf,'[')) ) {
+        if ( 1 == sscanf(buf+1,"%p", &ptrs[n_ptrs]) )
+            n_ptrs++;
+        buf++;
+    }
+    /* We should find an address close to epicsStackTraceRecurseGbl twice */
+    for (i=0; i<n_ptrs-1; i++) {
+        /* I got a (unjustified) index-out-of-bound warning 
+         * when setting j=i+1 here. Thus the weird j!= i check...
+         */
+        j = i;
+        while ( j < n_ptrs ) {
+            if ( j != i && ptrs[j] == ptrs[i] ) {
+                if ( (char*)ptrs[i] >= (char*)epicsStackTraceRecurseGbl && (char*)ptrs[i] < (char*)epicsStackTraceRecurseGbl + WINDOW_SZ ) {
+                    rval ++;    
+                    if ( test_debug )
+                        testDiag("found address %p again\n", ptrs[i]);
+                }
+            }
+            j++;
+        }
+    }
+    return rval;
+}
+
+MAIN(epicsStackTraceTest)
+{
+    int features, all_features;
+    TestDataRec testData;
+    int gblFound, lclFound, numFound, dynFound;
+    char *nl, *p;
+
+    if ( getenv("EPICS_STACK_TRACE_TEST_DEBUG") )
+        test_debug = 1;
+
+    testData.pos = 0;
+
+    testPlan(5);
+
+    features = epicsStackTraceGetFeatures(); 
+
+    all_features =   EPICS_STACKTRACE_LCL_SYMBOLS
+                   | EPICS_STACKTRACE_GBL_SYMBOLS
+                   | EPICS_STACKTRACE_DYN_SYMBOLS
+                   | EPICS_STACKTRACE_ADDRESSES;
+
+    if ( ! testOk( (features & ~all_features) == 0,
+              "epicsStackTraceGetFeatures() obtains features") )
+        testAbort("epicsStackTraceGetFeatures() not working as expected");
+
+    testData.pos = 0;
+
+    testDiag("calling a few nested routines and eventually dump a stack trace");
+
+    eltc(0);
+    errlogAddListener( logClient, &testData );
+    gfp(3);
+    errlogRemoveListeners( logClient, &testData );
+    eltc(1);
+
+    /* ensure there's a terminating NUL -- we have reserved space for it */
+    testData.buf[testData.pos] = 0;
+
+    testDiag("now scan the result for what we expect");
+
+    dynFound = findStringOcc( testData.buf, "epicsStackTrace" );
+    gblFound = findStringOcc( testData.buf, "epicsStackTraceRecurseGbl" );
+    lclFound = findStringOcc( testData.buf, "epicsStackTraceRecurseLcl" );
+    numFound = findNumOcc   ( testData.buf );
+
+    if ( (features & EPICS_STACKTRACE_DYN_SYMBOLS) ) {
+        testOk( dynFound == 1, "dumping symbol from library" );
+    } else {
+        testSkip(1, "no support for dumping library symbols on this platform");
+    }
+
+    if ( (features & EPICS_STACKTRACE_GBL_SYMBOLS) ) {
+        testOk( gblFound == 2, "dumping global symbols" );
+    } else {
+        testSkip(1, "no support for dumping global symbols on this platform");
+    }
+
+    if ( (features & EPICS_STACKTRACE_LCL_SYMBOLS) ) {
+        testOk( lclFound == 2, "dumping local symbols" );
+    } else {
+        testSkip(1, "no support for dumping local symbols on this platform");
+    }
+
+    if ( (features & EPICS_STACKTRACE_ADDRESSES) ) {
+        testOk(  numFound > 0, "dumping addresses" );
+    } else {
+        testSkip(1 , "no support for dumping addresses on this platform");
+    }
+
+    if ( test_debug ) {
+        p = testData.buf;
+        while ( (nl = strchr(p,'\n')) ) {
+            *nl = 0;
+            testDiag("%s",p);
+            *nl = '\n';
+            p   = nl+1;
+        }
+        testDiag("%s", p);
+    }
+
+    testDone();
+
+    return 0;
+}