From 3790ce4452e145d491604724698db2e4ba212358 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 18 Apr 2020 00:18:11 -0500 Subject: [PATCH] Have 'make test-results' fail nicely if any tests did Lists the directories with failed tests at the end of the build. It is no longer necessary to use 'make -k' to see the results of all tests after one or more failures as only the top-level test-results recipe will generate a build error. --- .gitignore | 1 + configure/CONFIG_BASE | 11 +++++++++-- configure/RULES_BUILD | 2 +- configure/RULES_DIRS | 9 +++++---- configure/RULES_TOP | 5 +++++ src/tools/Makefile | 1 + src/tools/testFailures.pl | 31 +++++++++++++++++++++++++++++++ 7 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 src/tools/testFailures.pl diff --git a/.gitignore b/.gitignore index 2dd7304c2..27871fffe 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /include/ /templates/ /configure/*.local +/.tests-failed O.*/ /QtC-* *.orig diff --git a/configure/CONFIG_BASE b/configure/CONFIG_BASE index c979d6d7c..54747d8e3 100644 --- a/configure/CONFIG_BASE +++ b/configure/CONFIG_BASE @@ -68,8 +68,6 @@ DBTOMENUH = $(PERL) $(TOOLS)/dbdToMenuH.pl REGISTERRECORDDEVICEDRIVER = $(PERL) $(TOOLS)/registerRecordDeviceDriver.pl CONVERTRELEASE = $(PERL) $(call FIND_TOOL,convertRelease.pl) FULLPATHNAME = $(PERL) $(TOOLS)/fullPathName.pl -TAPTOJUNIT = $(PERL) $(TOOLS)/tap-to-junit-xml.pl -PROVE = $(PERL) $(TOOLS)/epicsProve.pl #------------------------------------------------------- # tools for installing libraries and products @@ -83,6 +81,15 @@ INSTALL_LIBRARY = $(INSTALL) MKMF = $(PERL) $(TOOLS)/mkmf.pl REPLACEVAR = $(PERL) $(TOOLS)/replaceVAR.pl +#--------------------------------------------------------------- +# Tools for testing +TAPTOJUNIT = $(PERL) $(TOOLS)/tap-to-junit-xml.pl +PROVE = $(PERL) $(TOOLS)/epicsProve.pl +PROVE.tap = $(PROVE) --ext .tap --exec "$(CAT)" + +TEST_FAILURE_FILE = $(TOP)/.tests-failed +PROVE_FAILURE = echo $(abspath .)>> $(TEST_FAILURE_FILE) + #--------------------------------------------------------------- # private versions of lex/yacc from EPICS EYACC = $(TOOLS)/antelope$(HOSTEXE) diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD index 6145927d2..beace64c6 100644 --- a/configure/RULES_BUILD +++ b/configure/RULES_BUILD @@ -353,7 +353,7 @@ testspec: $(TESTSCRIPTS) test-results: tapfiles ifneq ($(strip $(TAPFILES)),) ifdef RUNTESTS_ENABLED - -$(PROVE) --failures --ext .tap --exec "$(CAT)" --color $(TAPFILES) + -$(PROVE.tap) --failures --color $(TAPFILES) || $(PROVE_FAILURE) endif CURRENT_TAPFILES := $(wildcard $(TAPFILES)) diff --git a/configure/RULES_DIRS b/configure/RULES_DIRS index 37cea5e9a..e90006890 100644 --- a/configure/RULES_DIRS +++ b/configure/RULES_DIRS @@ -4,7 +4,7 @@ # Copyright (c) 2002 The Regents of the University of California, as # Operator of Los Alamos National Laboratory. # EPICS BASE is distributed subject to a Software License Agreement found -# in the file LICENSE that is included with this distribution. +# in the file LICENSE that is included with this distribution. #************************************************************************* ARCHS += $(BUILD_ARCHS) @@ -79,14 +79,15 @@ $(foreach arch, $(ARCHS), \ dirPart = $(join $(dir $@), $(word 1, $(subst $(DIVIDER), ,$(notdir $@)))) actionArchPart = $(join $(word 2, $(subst $(DIVIDER), ,$(notdir $@))), \ - $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) -$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets) : + $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) + +$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets): pre-make $(MAKE) -C $(dirPart) $(actionArchPart) $(ARCHS) $(ACTIONS) $(actionArchTargets) :%: \ $(foreach dir, $(DIRS), $(dir)$(DIVIDER)%) -.PHONY : $(DIRS) all host rebuild +.PHONY : $(DIRS) all host rebuild pre-make .PHONY : $(ARCHS) $(ACTIONS) .PHONY : $(dirActionTargets) $(dirArchTargets) .PHONY : $(dirActionArchTargets) diff --git a/configure/RULES_TOP b/configure/RULES_TOP index 176454187..627f4b63c 100644 --- a/configure/RULES_TOP +++ b/configure/RULES_TOP @@ -35,6 +35,11 @@ uninstall$(DIVIDER)%: $(RMDIR) $(addsuffix /$(subst uninstall$(DIVIDER),,$@), \ $(INSTALL_LOCATION_BIN) $(INSTALL_LOCATION_LIB)) +pre-make: + @$(if $(filter test-results, $(MAKECMDGOALS)), \ + $(RM) $(TEST_FAILURE_FILE)) +test-results: + $(PERL) $(TOOLS)/testFailures.pl $(TEST_FAILURE_FILE) help: @echo "Usage: gnumake [options] [target] ..." diff --git a/src/tools/Makefile b/src/tools/Makefile index 26c0ed99a..faa08abbf 100644 --- a/src/tools/Makefile +++ b/src/tools/Makefile @@ -52,6 +52,7 @@ PERL_SCRIPTS += mkmf.pl PERL_SCRIPTS += munch.pl PERL_SCRIPTS += replaceVAR.pl PERL_SCRIPTS += tap-to-junit-xml.pl +PERL_SCRIPTS += testFailures.pl PERL_SCRIPTS += useManifestTool.pl PERL_SCRIPTS += dbdToMenuH.pl diff --git a/src/tools/testFailures.pl b/src/tools/testFailures.pl new file mode 100644 index 000000000..e74dd269f --- /dev/null +++ b/src/tools/testFailures.pl @@ -0,0 +1,31 @@ +#!/usr/bin/env perl +#************************************************************************* +# EPICS BASE is distributed subject to a Software License Agreement found +# in the file LICENSE that is included with this distribution. +#************************************************************************* + +# This file may appear trivial, but it exists to let the build system +# fail the 'make test-results' target with a nice output including a +# summary of the directories where test failures were reported. +# Test results are collected from the .tap files fed to epicsProve.pl +# which returns with an exit status of 0 (success) if all tests passed +# or 1 (failure) if any of the .tap files contained failed tests. +# When epicsProve.pl indicates a failure, the directory that it was +# running in is appended to the file $(TOP)/.tests-failed which this +# program reads in after all the test directories have been visited. +# The exit status of this program is 1 (failure) if any tests failed, +# otherwise 0 (success). + +use strict; +use warnings; + +die "Usage: testFailures.pl .tests-failed\n" + unless @ARGV == 1; + +open FAILURES, '<', shift or + exit 0; + +print "\nTest failures were reported in:\n", + map {" $_\n"} ; + +exit 1;