Merge 3.15 branch into 7.0 after ci-scripts added

.gitignore

@@ -9,6 +9,7 @@
 /configure/*.local
 /modules/RELEASE.*.local
 /modules/Makefile.local
+/.tests-failed
 O.*/
 /QtC-*
 *.orig

configure/CONFIG_BASE

@@ -42,8 +42,6 @@ FIND_TOOL = $(firstword $(wildcard $(TOOLS)/$(1) $(EPICS_BASE)/src/tools/$(1)))
 PODTOHTML        = $(PERL) $(TOOLS)/podToHtml.pl
 CONVERTRELEASE   = $(PERL) $(call FIND_TOOL,convertRelease.pl)
 FULLPATHNAME     = $(PERL) $(TOOLS)/fullPathName.pl
-TAPTOJUNIT       = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
-PROVE            = $(PERL) $(TOOLS)/epicsProve.pl
 GENVERSIONHEADER = $(PERL) $(TOOLS)/genVersionHeader.pl $(QUIET_FLAG) $(QUESTION_FLAG)
 
 MAKERPATH = $(PYTHON) $(TOOLS)/makeRPath.py
@@ -63,3 +61,12 @@ REPLACEVAR = $(PERL) $(TOOLS)/replaceVAR.pl
 # tools for cleaning out unwanted files
 CVSCLEAN = $(call FIND_TOOL,cvsclean.pl)
 DEPCLEAN = $(call FIND_TOOL,depclean.pl)
+
+#---------------------------------------------------------------
+# Tools for testing
+TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
+PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
+PROVE.tap                  = $(PROVE) --ext .tap --exec "$(CAT)"
+
+TEST_FAILURE_FILE          = $(TOP)/.tests-failed
+PROVE_FAILURE              = echo $(abspath .)>> $(TEST_FAILURE_FILE)

configure/RULES_BUILD

@@ -108,17 +108,17 @@ PRODTARGETS += $(PRODNAME) $(MUNCHNAME) $(CTDT_SRCS) $(CTDT_OBJS) $(NMS)
 TESTPRODTARGETS += $(TESTPRODNAME) $(TESTMUNCHNAME)
 
 #---------------------------------------------------------------
-#   Test specifications and test result files
+#   Test result files
 #
-ifneq (,$(strip $(TESTS)))
-TARGETS += testspec
-endif
 
-# Enable testing if this host can run tests on the current target
+# Enable testing if this host can run tests for the current target
-ifneq (,$(findstring $(T_A),$(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS)))
+ifneq (,$(filter $(T_A), $(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS)))
 RUNTESTS_ENABLED = YES
-TAPFILES += $(TESTSCRIPTS:.t=.tap)
+TESTSCRIPTS.t = $(filter %.t, $(TESTSCRIPTS))
-JUNITFILES += $(TAPFILES:.tap=.xml)
+TAPFILES.t += $(TESTSCRIPTS.t:.t=.tap)
+JUNITFILES.t += $(TESTSCRIPTS.t:.t=.xml)
+TAPFILES += $(TAPFILES.t)
+JUNITFILES += $(JUNITFILES.t)
 endif
 
 #---------------------------------------------------------------
@@ -354,23 +354,22 @@ $(MODNAME): %$(MODEXT): %$(EXE)
 #---------------------------------------------------------------
 # Automated testing
 
-runtests: $(TESTSCRIPTS)
+runtests: run-tap-tests
+run-tap-tests: $(TESTSCRIPTS.t)
+ifneq ($(TESTSCRIPTS.t),)
 ifdef RUNTESTS_ENABLED
-	$(PERL) -MTest::Harness -e 'runtests @ARGV if @ARGV;' $^
+	$(PROVE) --failures --color $^ || $(PROVE_FAILURE)
+endif
 endif
 
-testspec: $(TESTSCRIPTS)
+tapfiles: $(TAPFILES)
-	@$(RM) $@
+junitfiles: $(JUNITFILES)
-	@echo OS-class: $(OS_CLASS) > $@
-	@echo Target-arch: $(T_A) >> $@
-	$(if $^, @echo Tests: $^ >> $@)
-	$(if $(TESTFILES), @echo Files: $(TESTFILES) >> $@)
-	$(if $(TESTSPEC_$(OS_CLASS)), @echo "Harness: $(TESTSPEC_$(OS_CLASS))" >> $@)
 
-test-results: tapfiles
+test-results: tap-results
-ifneq ($(TAPFILES),)
+tap-results: $(TAPFILES)
+ifneq ($(strip $(TAPFILES)),)
 ifdef RUNTESTS_ENABLED
-	$(PROVE) --failures --ext .tap --exec "$(CAT)" --color $(TAPFILES)
+	$(PROVE.tap) --failures --color $^ || $(PROVE_FAILURE)
 endif
 
 CURRENT_TAPFILES := $(wildcard $(TAPFILES))
@@ -385,16 +384,13 @@ ifneq ($(CURRENT_JUNITFILES),)
 	$(RM) $(CURRENT_JUNITFILES)
 endif
 
-tapfiles: $(TESTSCRIPTS) $(TAPFILES)
-junitfiles: $(JUNITFILES)
-
 # A .tap file is the output from running the associated test script
-%.tap: %.t
+$(TAPFILES.t): %.tap: %.t
 ifdef RUNTESTS_ENABLED
 	$(PERL) $< -tap > $@
 endif
 
-%.xml: %.tap
+$(JUNITFILES.t): %.xml: %.tap
 	$(TAPTOJUNIT) --puretap --output $@ --input $< $*
 
 # If there's a perl test script (.plt) available, use it
@@ -553,8 +549,8 @@ include $(CONFIG)/RULES_EXPAND
 .PRECIOUS: $(COMMON_INC)
 
 .PHONY: all host inc build install clean rebuild buildInstall build_clean
-.PHONY: runtests tapfiles clean-tests test-results junitfiles
+.PHONY: runtests run-tap-tests tapfiles junitfiles test-results tap-results
-.PHONY: checkRelease warnRelease noCheckRelease FORCE
+.PHONY: clean-tests checkRelease warnRelease noCheckRelease FORCE
 
 include $(CONFIG)/RULES_COMMON
 

configure/RULES_DIRS

@@ -54,7 +54,7 @@ $(foreach dir, $(DIRS), \
 
 define DEP_template2
 $(1)$$(DIVIDER)$(2) : $$(foreach ddir, $$($(1)_DEPEND_DIRS), \
-    $$(addsuffix $$(DIVIDER)$(2),$$(ddir)))
+    $$(addsuffix $$(DIVIDER)$(2),$$(ddir))) | before-$(2)
 endef
 $(foreach action, $(ACTIONS), \
     $(foreach dir, $(DIRS), \
@@ -80,15 +80,21 @@ $(foreach arch, $(ARCHS), \
 dirPart = $(join $(dir $@), $(word 1, $(subst $(DIVIDER), ,$(notdir $@))))
 actionArchPart = $(join $(word 2, $(subst $(DIVIDER), ,$(notdir $@))), \
    $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@)))))
+
 $(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets):
 	$(MAKE) -C $(dirPart) $(actionArchPart)
 
+# before-action rules are run once prior to recursing through the
+# list of subdirectories and running the action rule in each one.
+# See DEP_template2 above for how that rule ordering is achieved.
+beforeActions = $(addprefix before-,$(ACTIONS))
+$(beforeActions):
+
 $(ARCHS) $(ACTIONS) $(actionArchTargets) :%: \
     $(foreach dir, $(DIRS), $(dir)$(DIVIDER)%)
 
-
 .PHONY : $(DIRS) all host rebuild
-.PHONY: $(ARCHS) $(ACTIONS)
+.PHONY : $(ARCHS) $(ACTIONS) $(beforeActions)
 .PHONY : $(dirActionTargets) $(dirArchTargets)
 .PHONY : $(dirActionArchTargets)
 .PHONY : $(actionArchTargets)

configure/RULES_TOP

@@ -60,6 +60,11 @@ else
 
 endif # DISABLE_TOP_RULES
 
+before-runtests before-test-results: rm-failure-file
+rm-failure-file:
+	@$(RM) $(TEST_FAILURE_FILE)
+runtests test-results:
+	$(PERL) $(TOOLS)/testFailures.pl $(TEST_FAILURE_FILE)
 
 help:
 	@echo "Usage: gnumake [options] [target] ..."
@@ -99,7 +104,7 @@ endif
 	@echo "Object targets are supported by the O.<arch> level Makefile .e.g"
 	@echo "     xxxRecord.o"
 
-.PHONY: distclean uninstall help
+.PHONY: distclean uninstall rm-failure-file help
 .PHONY: realuninstall archuninstall uninstallDirs
 
 ifndef DISABLE_TOP_RULES

documentation/RELEASE_NOTES.md

@@ -1180,6 +1180,28 @@ header and removed the need for dbScan.c to reach into the internals of its
 
 ## Changes from the 3.15 branch since 3.15.7
 
+### Improvements to the self-test build targets
+
+This release contains changes that make it possible to integrate another test
+running and reporting system (such as Google's gtest) into the EPICS build
+system. The built-in test-runner and reporting system will continue to be used
+by the test programs inside Base however.
+
+These GNUmake `tapfiles` and `test-results` build targets now collect a list of
+the directories that experienced test failures and display those at the end of
+running and/or reporting all of the tests. The GNUmake process will also only
+exit with an error status after running and/or reporting all of the test
+results; previously the `-k` flag to make was needed and even that didn't always
+work.
+
+Continuous Integration systems are recommended to run `make tapfiles` (or if
+they can read junittest output instead of TAP `make junitests`) followed by
+`make -s test-results` to display the results of the tests. If multiple CPUs are
+available the `-j` flag can be used to run tests in parallel, giving the maximum
+jobs that should be allowed so `make -j4 tapfiles` for a system with 4 CPUs say.
+Running many more jobs than you have CPUs is likely to be slower and is not
+recommended.
+
 ### epicsThread: Main thread defaults to allow blocking I/O
 
 VxWorks IOCs (and potentially RTEMS IOCs running GeSys) have had problems with

modules/libcom/src/osi/os/default/osdMessageQueue.cpp

@@ -338,11 +338,10 @@ myReceive(epicsMessageQueueId pmsg, void *message, unsigned int size,
     ellAdd(&pmsg->receiveQueue, &threadNode.link);
     epicsMutexUnlock(pmsg->mutex);
 
-    epicsEventStatus status;
     if (timeout > 0)
-        status = epicsEventWaitWithTimeout(threadNode.evp->event, timeout);
+        epicsEventWaitWithTimeout(threadNode.evp->event, timeout);
     else
-        status = epicsEventWait(threadNode.evp->event);
+        epicsEventWait(threadNode.evp->event);
 
     epicsMutexMustLock(pmsg->mutex);
 
@@ -352,8 +351,7 @@ myReceive(epicsMessageQueueId pmsg, void *message, unsigned int size,
 
     epicsMutexUnlock(pmsg->mutex);
 
-    if (threadNode.eventSent && (threadNode.size <= size) &&
+    if (threadNode.eventSent && (threadNode.size <= size))
-        status == epicsEventOK)
         return threadNode.size;
     return -1;
 }

modules/libcom/test/epicsMessageQueueTest.cpp

@@ -27,6 +27,7 @@ static volatile int sendExit = 0;
 static volatile int recvExit = 0;
 static epicsEventId finished;
 static unsigned int mediumStack;
+static int numReceived;
 
 /*
  * In Numerical Recipes in C: The Art of Scientific Computing (William  H.
@@ -115,6 +116,21 @@ receiver(void *arg)
     epicsEventSignal(finished);
 }
 
+extern "C" void
+fastReceiver(void *arg)
+{
+    epicsMessageQueue *q = (epicsMessageQueue *)arg;
+    char cbuf[80];
+    int len;
+    numReceived = 0;
+    while (!recvExit) {
+        len = q->receive(cbuf, sizeof cbuf, 0.01);
+        if (len > 0) {
+            numReceived++;
+        }
+    }
+}
+
 extern "C" void
 sender(void *arg)
 {
@@ -140,8 +156,10 @@ extern "C" void messageQueueTest(void *parm)
     int len;
     int pass;
     int want;
+    int numSent = 0;
 
     epicsMessageQueue *q1 = new epicsMessageQueue(4, 20);
+    epicsMessageQueue *q2 = new epicsMessageQueue(4, 20);
 
     testDiag("Simple single-thread tests:");
     i = 0;
@@ -251,6 +269,35 @@ extern "C" void messageQueueTest(void *parm)
     testOk(q1->send((void *)msg1, 10) == 0, "Send with no receiver");
     epicsThreadSleep(2.0);
 
+    testDiag("Single receiver with timeout, single sender with sleep tests:");
+    testDiag("These tests last 20 seconds ...");
+    epicsThreadCreate("Fast Receiver", epicsThreadPriorityMedium,
+        mediumStack, fastReceiver, q2);
+    numSent = 0;
+    numReceived = 0;
+    for (i = 0 ; i < 1000 ; i++) {
+        if (q2->send((void *)msg1, 4) == 0) {
+            numSent++;
+        }
+        epicsThreadSleep(0.011);
+    }
+    epicsThreadSleep(1.0);
+    if (!testOk(numSent == 1000 && numReceived == 1000, "sleep=0.011")) {
+        testDiag("numSent should be 1000, actual=%d, numReceived should be 1000, actual=%d", numSent, numReceived);
+    }
+    numSent = 0;
+    numReceived = 0;
+    for (i = 0 ; i < 1000 ; i++) {
+        if (q2->send((void *)msg1, 4) == 0) {
+            numSent++;
+        }
+        epicsThreadSleep(0.010);
+    }
+    epicsThreadSleep(1.0);
+    if (!testOk(numSent == 1000 && numReceived == 1000, "sleep=0.010")) {
+        testDiag("numSent should be 1000, actual=%d, numReceived should be 1000, actual=%d", numSent, numReceived);
+    }
+
     testDiag("Single receiver, single sender tests:");
     epicsThreadSetPriority(myThreadId, epicsThreadPriorityHigh);
     epicsThreadCreate("Receiver one", epicsThreadPriorityMedium,
@@ -285,7 +332,7 @@ extern "C" void messageQueueTest(void *parm)
      * Single receiver, multiple sender tests
      */
     testDiag("Single receiver, multiple sender tests:");
-    testDiag("This test lasts 60 seconds...");
+    testDiag("This test lasts 30 seconds...");
     testOk(!!epicsThreadCreate("Sender 1", epicsThreadPriorityLow,
         mediumStack, sender, q1),
         "Created Sender 1");
@@ -299,9 +346,9 @@ extern "C" void messageQueueTest(void *parm)
         mediumStack, sender, q1),
         "Created Sender 4");
 
-    for (i = 0; i < 10; i++) {
+    for (i = 0; i < 6; i++) {
-        testDiag("... %2d", 10 - i);
+        testDiag("... %2d", 6 - i);
-        epicsThreadSleep(6.0);
+        epicsThreadSleep(5.0);
     }
 
     sendExit = 1;
@@ -312,7 +359,7 @@ extern "C" void messageQueueTest(void *parm)
 
 MAIN(epicsMessageQueueTest)
 {
-    testPlan(62);
+    testPlan(64);
 
     finished = epicsEventMustCreate(epicsEventEmpty);
     mediumStack = epicsThreadGetStackSize(epicsThreadStackMedium);

src/tools/Makefile

@@ -39,6 +39,7 @@ PERL_SCRIPTS += podToHtml.pl
 PERL_SCRIPTS += podRemove.pl
 PERL_SCRIPTS += replaceVAR.pl
 PERL_SCRIPTS += tap-to-junit-xml.pl
+PERL_SCRIPTS += testFailures.pl
 PERL_SCRIPTS += useManifestTool.pl
 PERL_SCRIPTS += genVersionHeader.pl
 

src/tools/podToHtml.pl

@@ -9,8 +9,12 @@
 use strict;
 use warnings;
 
+# To find the EPICS::PodHtml module used below we need to add our lib/perl to
+# the lib search path. If the script is running from the src/tools directory
+# before everything has been installed though, the search path must include
+# our source directory (i.e. $Bin), so we add both here.
 use FindBin qw($Bin);
-use lib "$Bin/../../lib/perl";
+use lib ("$Bin/../../lib/perl", $Bin);
 
 use Getopt::Std;
 $Getopt::Std::STANDARD_HELP_VERSION = 1;

src/tools/testFailures.pl

@@ -0,0 +1,33 @@
+#!/usr/bin/env perl
+#*************************************************************************
+# EPICS BASE is distributed subject to a Software License Agreement found
+# in the file LICENSE that is included with this distribution.
+#*************************************************************************
+
+# This file may appear trivial, but it exists to let the build system
+# fail the 'make test-results' target with a nice output including a
+# summary of the directories where test failures were reported.
+# Test results are collected from the .tap files fed to epicsProve.pl
+# which returns with an exit status of 0 (success) if all tests passed
+# or 1 (failure) if any of the .tap files contained failed tests.
+# When epicsProve.pl indicates a failure, the directory that it was
+# running in is appended to the file $(TOP)/.tests-failed which this
+# program reads in after all the test directories have been visited.
+# The exit status of this program is 1 (failure) if any tests failed,
+# otherwise 0 (success).
+
+use strict;
+use warnings;
+
+die "Usage: testFailures.pl .tests-failed\n"
+    unless @ARGV == 1;
+
+open FAILURES, '<', shift or
+    exit 0;
+my @failures = <FAILURES>;
+close FAILURES;
+
+print "\nTest failures were reported in:\n",
+    (map {"    $_"} @failures), "\n";
+
+exit 1;