From 7beb32e209715424607dc263b7f3244cecdf7844 Mon Sep 17 00:00:00 2001
From: Andrew Johnson <anj@anl.gov>
Date: Thu, 17 Dec 2020 18:34:24 -0600
Subject: [PATCH] Extend testFailures.pl to list the failed test programs

Creates a file .taps-failed in each O.<arch> directory,
appending the name of each tapfile that has failures to it.
The testFailures script now reads the .taps-failed files
from each directory listed in .tests-failed and nicely
displays the failing tests listed in each.
---
 configure/CONFIG_BASE     |  4 +++-
 configure/RULES_BUILD     | 11 +++++----
 src/tools/testFailures.pl | 50 ++++++++++++++++++++++++++-------------
 3 files changed, 44 insertions(+), 21 deletions(-)

diff --git a/configure/CONFIG_BASE b/configure/CONFIG_BASE
index 54747d8e3..f0e739fa1 100644
--- a/configure/CONFIG_BASE
+++ b/configure/CONFIG_BASE
@@ -87,8 +87,10 @@ TAPTOJUNIT                 = $(PERL) $(TOOLS)/tap-to-junit-xml.pl
 PROVE                      = $(PERL) $(TOOLS)/epicsProve.pl
 PROVE.tap                  = $(PROVE) --ext .tap --exec "$(CAT)"
 
-TEST_FAILURE_FILE          = $(TOP)/.tests-failed
+TEST_FAILURE_FILE          = $(abspath $(TOP)/.tests-failed)
 PROVE_FAILURE              = echo $(abspath .)>> $(TEST_FAILURE_FILE)
+TAPS_FAILURE_FILE          = .taps-failed
+TAPFILE_FAILURE            = echo $@>> $(TAPS_FAILURE_FILE)
 
 #---------------------------------------------------------------
 # private versions of lex/yacc from EPICS
diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD
index d23a0684d..eaf51c589 100644
--- a/configure/RULES_BUILD
+++ b/configure/RULES_BUILD
@@ -339,7 +339,8 @@ runtests: run-tap-tests
 run-tap-tests: $(TESTSCRIPTS.t)
 ifneq ($(TESTSCRIPTS.t),)
 ifdef RUNTESTS_ENABLED
-	$(PROVE) --failures --color $^ || $(PROVE_FAILURE)
+	$(ECHO) "$(PROVE) --failures --color $^"
+	@$(PROVE) --failures --color $^ || $(PROVE_FAILURE)
 endif
 endif
 
@@ -350,7 +351,8 @@ test-results: tap-results
 tap-results: $(TAPFILES)
 ifneq ($(strip $(TAPFILES)),)
 ifdef RUNTESTS_ENABLED
-	$(PROVE.tap) --failures --color $^ || $(PROVE_FAILURE)
+	$(ECHO) "$(PROVE.tap) --failures --color $^"
+	@$(PROVE.tap) --failures --color $^ || $(PROVE_FAILURE)
 endif
 
 CURRENT_TAPFILES := $(wildcard $(TAPFILES))
@@ -359,7 +361,7 @@ endif
 
 clean-tests:
 ifneq ($(CURRENT_TAPFILES),)
-	$(RM) $(CURRENT_TAPFILES)
+	$(RM) $(CURRENT_TAPFILES) $(TAPS_FAILURE_FILE)
 endif
 ifneq ($(CURRENT_JUNITFILES),)
 	$(RM) $(CURRENT_JUNITFILES)
@@ -368,7 +370,8 @@ endif
 # A .tap file is the output from running the associated test script
 $(TAPFILES.t): %.tap: %.t
 ifdef RUNTESTS_ENABLED
-	-$(PERL) $< -tap > $@
+	$(ECHO) "$(PERL) $< -tap > $@"
+	@$(PERL) $< -tap > $@ || $(TAPFILE_FAILURE)
 endif
 
 $(JUNITFILES.t): %-results.xml: %.tap
diff --git a/src/tools/testFailures.pl b/src/tools/testFailures.pl
index e19d00006..fed66f8e8 100644
--- a/src/tools/testFailures.pl
+++ b/src/tools/testFailures.pl
@@ -4,30 +4,48 @@
 # in the file LICENSE that is included with this distribution.
 #*************************************************************************
 
-# This file may appear trivial, but it exists to let the build system
-# fail the 'make test-results' target with a nice output including a
-# summary of the directories where test failures were reported.
-# Test results are collected from the .tap files fed to epicsProve.pl
-# which returns with an exit status of 0 (success) if all tests passed
-# or 1 (failure) if any of the .tap files contained failed tests.
-# When epicsProve.pl indicates a failure, the directory that it was
-# running in is appended to the file $(TOP)/.tests-failed which this
-# program reads in after all the test directories have been visited.
+# This file lets the build system fail a top-level 'make test-results'
+# target with output showing the directories where test failures were
+# reported and the test programs that failed there.
+#
 # The exit status of this program is 1 (failure) if any tests failed,
 # otherwise 0 (success).
 
 use strict;
 use warnings;
 
-die "Usage: testFailures.pl .tests-failed\n"
+use File::Basename;
+
+die "Usage: testFailures.pl /path/to/base/.tests-failed\n"
     unless @ARGV == 1;
 
-open FAILURES, '<', shift or
-    exit 0;
-my @failures = <FAILURES>;
-close FAILURES;
+my $path = shift;
+my $base = dirname($path);
 
-print "\nTest failures were reported in:\n",
-    (map {"    $_"} @failures), "\n";
+open(my $failures, '<', $path) or
+    exit 0;
+my @failures = dedup(<$failures>);
+close $failures;
+chomp @failures;
+
+exit 0 unless @failures;
+
+print "\nTests failed:\n";
+for my $dir (@failures) {
+    my $reldir = $dir;
+    $reldir =~ s($base/)();
+    print "    In $reldir:\n";
+    open(my $taps, '<', "$dir/.taps-failed") or next;
+    my @taps = dedup(<$taps>);
+    close $taps;
+    chomp @taps;
+    print '', (map {"        $_\n"} @taps), "\n";
+}
 
 exit 1;
+
+sub dedup {
+    my %dedup;
+    $dedup{$_}++ for @_;
+    return sort keys %dedup;
+}