Have 'make test-results' fail nicely if any tests did

Lists the directories with failed tests at the end of the build.
It is no longer necessary to use 'make -k' to see the results
of all tests after one or more failures as only the top-level
test-results recipe will generate a build error.

.gitignore

@@ -7,6 +7,7 @@
 /include/
 /templates/
 /configure/*.local
+/.tests-failed
 O.*/
 /QtC-*
 *.orig

configure/CONFIG_BASE

@@ -68,8 +68,6 @@ DBTOMENUH                  = $(PERL) $(TOOLS)/dbdToMenuH.pl
 REGISTERRECORDDEVICEDRIVER = $(PERL) $(TOOLS)/registerRecordDeviceDriver.pl
 CONVERTRELEASE             = $(PERL) $(call FIND_TOOL,convertRelease.pl)
 FULLPATHNAME               = $(PERL) $(TOOLS)/fullPathName.pl
-TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
-PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
 
 #-------------------------------------------------------
 # tools for installing libraries and products
@@ -83,6 +81,15 @@ INSTALL_LIBRARY = $(INSTALL)
 MKMF                       = $(PERL) $(TOOLS)/mkmf.pl
 REPLACEVAR                 = $(PERL) $(TOOLS)/replaceVAR.pl
 
+#---------------------------------------------------------------
+# Tools for testing
+TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
+PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
+PROVE.tap                  = $(PROVE) --ext .tap --exec "$(CAT)"
+
+TEST_FAILURE_FILE          = $(TOP)/.tests-failed
+PROVE_FAILURE              = echo $(abspath .)>> $(TEST_FAILURE_FILE)
+
 #---------------------------------------------------------------
 # private versions of lex/yacc from EPICS
 EYACC = $(TOOLS)/antelope$(HOSTEXE)

configure/RULES_BUILD

@@ -353,7 +353,7 @@ testspec: $(TESTSCRIPTS)
 test-results: tapfiles
 ifneq ($(strip $(TAPFILES)),)
 ifdef RUNTESTS_ENABLED
-	-$(PROVE) --failures --ext .tap --exec "$(CAT)" --color $(TAPFILES)
+	-$(PROVE.tap) --failures --color $(TAPFILES) || $(PROVE_FAILURE)
 endif
 
 CURRENT_TAPFILES := $(wildcard $(TAPFILES))

configure/RULES_DIRS

@@ -4,7 +4,7 @@
 # Copyright (c) 2002 The Regents of the University of California, as
 #     Operator of Los Alamos National Laboratory.
 # EPICS BASE is distributed subject to a Software License Agreement found
-# in the file LICENSE that is included with this distribution. 
+# in the file LICENSE that is included with this distribution.
 #*************************************************************************
 
 ARCHS += $(BUILD_ARCHS)
@@ -79,14 +79,15 @@ $(foreach arch, $(ARCHS), \
 
 dirPart = $(join $(dir $@), $(word 1, $(subst $(DIVIDER), ,$(notdir $@))))
 actionArchPart = $(join $(word 2, $(subst $(DIVIDER), ,$(notdir $@))), \
-   $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) 
-$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets) :
+   $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@)))))
+
+$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets): pre-make
 	$(MAKE) -C $(dirPart) $(actionArchPart)
 
 $(ARCHS) $(ACTIONS) $(actionArchTargets) :%: \
     $(foreach dir, $(DIRS), $(dir)$(DIVIDER)%)
 
-.PHONY : $(DIRS) all host rebuild
+.PHONY : $(DIRS) all host rebuild pre-make
 .PHONY : $(ARCHS) $(ACTIONS)
 .PHONY : $(dirActionTargets) $(dirArchTargets)
 .PHONY : $(dirActionArchTargets)

configure/RULES_TOP

@@ -35,6 +35,11 @@ uninstall$(DIVIDER)%:
 	$(RMDIR) $(addsuffix /$(subst uninstall$(DIVIDER),,$@), \
 	    $(INSTALL_LOCATION_BIN) $(INSTALL_LOCATION_LIB))
 
+pre-make:
+	@$(if $(filter test-results, $(MAKECMDGOALS)), \
+	    $(RM) $(TEST_FAILURE_FILE))
+test-results:
+	$(PERL) $(TOOLS)/testFailures.pl $(TEST_FAILURE_FILE)
 
 help:
 	@echo "Usage: gnumake [options] [target] ..."

src/tools/Makefile

@@ -52,6 +52,7 @@ PERL_SCRIPTS += mkmf.pl
 PERL_SCRIPTS += munch.pl
 PERL_SCRIPTS += replaceVAR.pl
 PERL_SCRIPTS += tap-to-junit-xml.pl
+PERL_SCRIPTS += testFailures.pl
 PERL_SCRIPTS += useManifestTool.pl
 
 PERL_SCRIPTS += dbdToMenuH.pl

src/tools/testFailures.pl

@@ -0,0 +1,31 @@
+#!/usr/bin/env perl
+#*************************************************************************
+# EPICS BASE is distributed subject to a Software License Agreement found
+# in the file LICENSE that is included with this distribution.
+#*************************************************************************
+
+# This file may appear trivial, but it exists to let the build system
+# fail the 'make test-results' target with a nice output including a
+# summary of the directories where test failures were reported.
+# Test results are collected from the .tap files fed to epicsProve.pl
+# which returns with an exit status of 0 (success) if all tests passed
+# or 1 (failure) if any of the .tap files contained failed tests.
+# When epicsProve.pl indicates a failure, the directory that it was
+# running in is appended to the file $(TOP)/.tests-failed which this
+# program reads in after all the test directories have been visited.
+# The exit status of this program is 1 (failure) if any tests failed,
+# otherwise 0 (success).
+
+use strict;
+use warnings;
+
+die "Usage: testFailures.pl .tests-failed\n"
+    unless @ARGV == 1;
+
+open FAILURES, '<', shift or
+    exit 0;
+
+print "\nTest failures were reported in:\n",
+    map {"    $_\n"} <FAILURES>;
+
+exit 1;