diff --git a/.gitignore b/.gitignore index af33a79de..dfe2eb44a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /configure/*.local /modules/RELEASE.*.local /modules/Makefile.local +/.tests-failed O.*/ /QtC-* *.orig diff --git a/configure/CONFIG_BASE b/configure/CONFIG_BASE index 8f9fadcdf..ad963ba94 100644 --- a/configure/CONFIG_BASE +++ b/configure/CONFIG_BASE @@ -42,8 +42,6 @@ FIND_TOOL = $(firstword $(wildcard $(TOOLS)/$(1) $(EPICS_BASE)/src/tools/$(1))) PODTOHTML = $(PERL) $(TOOLS)/podToHtml.pl CONVERTRELEASE = $(PERL) $(call FIND_TOOL,convertRelease.pl) FULLPATHNAME = $(PERL) $(TOOLS)/fullPathName.pl -TAPTOJUNIT = $(PERL) $(TOOLS)/tap-to-junit-xml.pl -PROVE = $(PERL) $(TOOLS)/epicsProve.pl GENVERSIONHEADER = $(PERL) $(TOOLS)/genVersionHeader.pl $(QUIET_FLAG) $(QUESTION_FLAG) MAKERPATH = $(PYTHON) $(TOOLS)/makeRPath.py @@ -63,3 +61,12 @@ REPLACEVAR = $(PERL) $(TOOLS)/replaceVAR.pl # tools for cleaning out unwanted files CVSCLEAN = $(call FIND_TOOL,cvsclean.pl) DEPCLEAN = $(call FIND_TOOL,depclean.pl) + +#--------------------------------------------------------------- +# Tools for testing +TAPTOJUNIT = $(PERL) $(TOOLS)/tap-to-junit-xml.pl +PROVE = $(PERL) $(TOOLS)/epicsProve.pl +PROVE.tap = $(PROVE) --ext .tap --exec "$(CAT)" + +TEST_FAILURE_FILE = $(TOP)/.tests-failed +PROVE_FAILURE = echo $(abspath .)>> $(TEST_FAILURE_FILE) diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD index 3df3052b4..682ab8d38 100644 --- a/configure/RULES_BUILD +++ b/configure/RULES_BUILD @@ -108,17 +108,17 @@ PRODTARGETS += $(PRODNAME) $(MUNCHNAME) $(CTDT_SRCS) $(CTDT_OBJS) $(NMS) TESTPRODTARGETS += $(TESTPRODNAME) $(TESTMUNCHNAME) #--------------------------------------------------------------- -# Test specifications and test result files +# Test result files # -ifneq (,$(strip $(TESTS))) -TARGETS += testspec -endif -# Enable testing if this host can run tests on the current target -ifneq (,$(findstring $(T_A),$(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS))) +# Enable testing if this host can run tests for the current target +ifneq (,$(filter $(T_A), $(EPICS_HOST_ARCH) $(CROSS_COMPILER_RUNTEST_ARCHS))) RUNTESTS_ENABLED = YES -TAPFILES += $(TESTSCRIPTS:.t=.tap) -JUNITFILES += $(TAPFILES:.tap=.xml) +TESTSCRIPTS.t = $(filter %.t, $(TESTSCRIPTS)) +TAPFILES.t += $(TESTSCRIPTS.t:.t=.tap) +JUNITFILES.t += $(TESTSCRIPTS.t:.t=.xml) +TAPFILES += $(TAPFILES.t) +JUNITFILES += $(JUNITFILES.t) endif #--------------------------------------------------------------- @@ -354,23 +354,22 @@ $(MODNAME): %$(MODEXT): %$(EXE) #--------------------------------------------------------------- # Automated testing -runtests: $(TESTSCRIPTS) +runtests: run-tap-tests +run-tap-tests: $(TESTSCRIPTS.t) +ifneq ($(TESTSCRIPTS.t),) ifdef RUNTESTS_ENABLED - $(PERL) -MTest::Harness -e 'runtests @ARGV if @ARGV;' $^ + $(PROVE) --failures --color $^ || $(PROVE_FAILURE) +endif endif -testspec: $(TESTSCRIPTS) - @$(RM) $@ - @echo OS-class: $(OS_CLASS) > $@ - @echo Target-arch: $(T_A) >> $@ - $(if $^, @echo Tests: $^ >> $@) - $(if $(TESTFILES), @echo Files: $(TESTFILES) >> $@) - $(if $(TESTSPEC_$(OS_CLASS)), @echo "Harness: $(TESTSPEC_$(OS_CLASS))" >> $@) +tapfiles: $(TAPFILES) +junitfiles: $(JUNITFILES) -test-results: tapfiles -ifneq ($(TAPFILES),) +test-results: tap-results +tap-results: $(TAPFILES) +ifneq ($(strip $(TAPFILES)),) ifdef RUNTESTS_ENABLED - $(PROVE) --failures --ext .tap --exec "$(CAT)" --color $(TAPFILES) + $(PROVE.tap) --failures --color $^ || $(PROVE_FAILURE) endif CURRENT_TAPFILES := $(wildcard $(TAPFILES)) @@ -385,16 +384,13 @@ ifneq ($(CURRENT_JUNITFILES),) $(RM) $(CURRENT_JUNITFILES) endif -tapfiles: $(TESTSCRIPTS) $(TAPFILES) -junitfiles: $(JUNITFILES) - # A .tap file is the output from running the associated test script -%.tap: %.t +$(TAPFILES.t): %.tap: %.t ifdef RUNTESTS_ENABLED $(PERL) $< -tap > $@ endif -%.xml: %.tap +$(JUNITFILES.t): %.xml: %.tap $(TAPTOJUNIT) --puretap --output $@ --input $< $* # If there's a perl test script (.plt) available, use it @@ -553,8 +549,8 @@ include $(CONFIG)/RULES_EXPAND .PRECIOUS: $(COMMON_INC) .PHONY: all host inc build install clean rebuild buildInstall build_clean -.PHONY: runtests tapfiles clean-tests test-results junitfiles -.PHONY: checkRelease warnRelease noCheckRelease FORCE +.PHONY: runtests run-tap-tests tapfiles junitfiles test-results tap-results +.PHONY: clean-tests checkRelease warnRelease noCheckRelease FORCE include $(CONFIG)/RULES_COMMON diff --git a/configure/RULES_DIRS b/configure/RULES_DIRS index db68a84a5..f47df4a53 100644 --- a/configure/RULES_DIRS +++ b/configure/RULES_DIRS @@ -4,7 +4,7 @@ # Copyright (c) 2002 The Regents of the University of California, as # Operator of Los Alamos National Laboratory. # EPICS BASE is distributed subject to a Software License Agreement found -# in the file LICENSE that is included with this distribution. +# in the file LICENSE that is included with this distribution. #************************************************************************* ARCHS += $(BUILD_ARCHS) @@ -54,7 +54,7 @@ $(foreach dir, $(DIRS), \ define DEP_template2 $(1)$$(DIVIDER)$(2) : $$(foreach ddir, $$($(1)_DEPEND_DIRS), \ - $$(addsuffix $$(DIVIDER)$(2),$$(ddir))) + $$(addsuffix $$(DIVIDER)$(2),$$(ddir))) | before-$(2) endef $(foreach action, $(ACTIONS), \ $(foreach dir, $(DIRS), \ @@ -79,18 +79,24 @@ $(foreach arch, $(ARCHS), \ dirPart = $(join $(dir $@), $(word 1, $(subst $(DIVIDER), ,$(notdir $@)))) actionArchPart = $(join $(word 2, $(subst $(DIVIDER), ,$(notdir $@))), \ - $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) -$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets) : + $(addprefix $(DIVIDER),$(word 3, $(subst $(DIVIDER), ,$(notdir $@))))) + +$(DIRS) $(dirActionTargets) $(dirArchTargets) $(dirActionArchTargets): $(MAKE) -C $(dirPart) $(actionArchPart) +# before-action rules are run once prior to recursing through the +# list of subdirectories and running the action rule in each one. +# See DEP_template2 above for how that rule ordering is achieved. +beforeActions = $(addprefix before-,$(ACTIONS)) +$(beforeActions): + $(ARCHS) $(ACTIONS) $(actionArchTargets) :%: \ $(foreach dir, $(DIRS), $(dir)$(DIVIDER)%) - -.PHONY: $(DIRS) all host rebuild -.PHONY: $(ARCHS) $(ACTIONS) -.PHONY: $(dirActionTargets) $(dirArchTargets) -.PHONY: $(dirActionArchTargets) -.PHONY: $(actionArchTargets) +.PHONY : $(DIRS) all host rebuild +.PHONY : $(ARCHS) $(ACTIONS) $(beforeActions) +.PHONY : $(dirActionTargets) $(dirArchTargets) +.PHONY : $(dirActionArchTargets) +.PHONY : $(actionArchTargets) include $(CONFIG)/RULES_COMMON diff --git a/configure/RULES_TOP b/configure/RULES_TOP index 615586d17..c2b77d64c 100644 --- a/configure/RULES_TOP +++ b/configure/RULES_TOP @@ -60,6 +60,11 @@ else endif # DISABLE_TOP_RULES +before-runtests before-test-results: rm-failure-file +rm-failure-file: + @$(RM) $(TEST_FAILURE_FILE) +runtests test-results: + $(PERL) $(TOOLS)/testFailures.pl $(TEST_FAILURE_FILE) help: @echo "Usage: gnumake [options] [target] ..." @@ -99,7 +104,7 @@ endif @echo "Object targets are supported by the O. level Makefile .e.g" @echo " xxxRecord.o" -.PHONY: distclean uninstall help +.PHONY: distclean uninstall rm-failure-file help .PHONY: realuninstall archuninstall uninstallDirs ifndef DISABLE_TOP_RULES diff --git a/documentation/RELEASE_NOTES.md b/documentation/RELEASE_NOTES.md index f25059ebf..a5ddca33d 100644 --- a/documentation/RELEASE_NOTES.md +++ b/documentation/RELEASE_NOTES.md @@ -1180,6 +1180,28 @@ header and removed the need for dbScan.c to reach into the internals of its ## Changes from the 3.15 branch since 3.15.7 +### Improvements to the self-test build targets + +This release contains changes that make it possible to integrate another test +running and reporting system (such as Google's gtest) into the EPICS build +system. The built-in test-runner and reporting system will continue to be used +by the test programs inside Base however. + +These GNUmake `tapfiles` and `test-results` build targets now collect a list of +the directories that experienced test failures and display those at the end of +running and/or reporting all of the tests. The GNUmake process will also only +exit with an error status after running and/or reporting all of the test +results; previously the `-k` flag to make was needed and even that didn't always +work. + +Continuous Integration systems are recommended to run `make tapfiles` (or if +they can read junittest output instead of TAP `make junitests`) followed by +`make -s test-results` to display the results of the tests. If multiple CPUs are +available the `-j` flag can be used to run tests in parallel, giving the maximum +jobs that should be allowed so `make -j4 tapfiles` for a system with 4 CPUs say. +Running many more jobs than you have CPUs is likely to be slower and is not +recommended. + ### epicsThread: Main thread defaults to allow blocking I/O VxWorks IOCs (and potentially RTEMS IOCs running GeSys) have had problems with diff --git a/modules/libcom/src/osi/os/default/osdMessageQueue.cpp b/modules/libcom/src/osi/os/default/osdMessageQueue.cpp index c86d8cc2b..9567c1f09 100644 --- a/modules/libcom/src/osi/os/default/osdMessageQueue.cpp +++ b/modules/libcom/src/osi/os/default/osdMessageQueue.cpp @@ -5,7 +5,7 @@ * Operator of Los Alamos National Laboratory. * EPICS BASE Versions 3.13.7 * and higher are distributed subject to a Software License Agreement found -* in file LICENSE that is included with this distribution. +* in file LICENSE that is included with this distribution. \*************************************************************************/ /* * Author W. Eric Norum @@ -53,7 +53,7 @@ struct epicsMessageQueueOSD { ELLLIST receiveQueue; ELLLIST eventFreeList; int numberOfSendersWaiting; - + epicsMutexId mutex; unsigned long capacity; unsigned long maxMessageSize; @@ -338,11 +338,10 @@ myReceive(epicsMessageQueueId pmsg, void *message, unsigned int size, ellAdd(&pmsg->receiveQueue, &threadNode.link); epicsMutexUnlock(pmsg->mutex); - epicsEventStatus status; if (timeout > 0) - status = epicsEventWaitWithTimeout(threadNode.evp->event, timeout); + epicsEventWaitWithTimeout(threadNode.evp->event, timeout); else - status = epicsEventWait(threadNode.evp->event); + epicsEventWait(threadNode.evp->event); epicsMutexMustLock(pmsg->mutex); @@ -352,8 +351,7 @@ myReceive(epicsMessageQueueId pmsg, void *message, unsigned int size, epicsMutexUnlock(pmsg->mutex); - if (threadNode.eventSent && (threadNode.size <= size) && - status == epicsEventOK) + if (threadNode.eventSent && (threadNode.size <= size)) return threadNode.size; return -1; } diff --git a/modules/libcom/test/epicsMessageQueueTest.cpp b/modules/libcom/test/epicsMessageQueueTest.cpp index ae090fc47..1bee13c36 100644 --- a/modules/libcom/test/epicsMessageQueueTest.cpp +++ b/modules/libcom/test/epicsMessageQueueTest.cpp @@ -27,6 +27,7 @@ static volatile int sendExit = 0; static volatile int recvExit = 0; static epicsEventId finished; static unsigned int mediumStack; +static int numReceived; /* * In Numerical Recipes in C: The Art of Scientific Computing (William H. @@ -115,6 +116,21 @@ receiver(void *arg) epicsEventSignal(finished); } +extern "C" void +fastReceiver(void *arg) +{ + epicsMessageQueue *q = (epicsMessageQueue *)arg; + char cbuf[80]; + int len; + numReceived = 0; + while (!recvExit) { + len = q->receive(cbuf, sizeof cbuf, 0.01); + if (len > 0) { + numReceived++; + } + } +} + extern "C" void sender(void *arg) { @@ -140,8 +156,10 @@ extern "C" void messageQueueTest(void *parm) int len; int pass; int want; + int numSent = 0; epicsMessageQueue *q1 = new epicsMessageQueue(4, 20); + epicsMessageQueue *q2 = new epicsMessageQueue(4, 20); testDiag("Simple single-thread tests:"); i = 0; @@ -251,6 +269,35 @@ extern "C" void messageQueueTest(void *parm) testOk(q1->send((void *)msg1, 10) == 0, "Send with no receiver"); epicsThreadSleep(2.0); + testDiag("Single receiver with timeout, single sender with sleep tests:"); + testDiag("These tests last 20 seconds ..."); + epicsThreadCreate("Fast Receiver", epicsThreadPriorityMedium, + mediumStack, fastReceiver, q2); + numSent = 0; + numReceived = 0; + for (i = 0 ; i < 1000 ; i++) { + if (q2->send((void *)msg1, 4) == 0) { + numSent++; + } + epicsThreadSleep(0.011); + } + epicsThreadSleep(1.0); + if (!testOk(numSent == 1000 && numReceived == 1000, "sleep=0.011")) { + testDiag("numSent should be 1000, actual=%d, numReceived should be 1000, actual=%d", numSent, numReceived); + } + numSent = 0; + numReceived = 0; + for (i = 0 ; i < 1000 ; i++) { + if (q2->send((void *)msg1, 4) == 0) { + numSent++; + } + epicsThreadSleep(0.010); + } + epicsThreadSleep(1.0); + if (!testOk(numSent == 1000 && numReceived == 1000, "sleep=0.010")) { + testDiag("numSent should be 1000, actual=%d, numReceived should be 1000, actual=%d", numSent, numReceived); + } + testDiag("Single receiver, single sender tests:"); epicsThreadSetPriority(myThreadId, epicsThreadPriorityHigh); epicsThreadCreate("Receiver one", epicsThreadPriorityMedium, @@ -285,7 +332,7 @@ extern "C" void messageQueueTest(void *parm) * Single receiver, multiple sender tests */ testDiag("Single receiver, multiple sender tests:"); - testDiag("This test lasts 60 seconds..."); + testDiag("This test lasts 30 seconds..."); testOk(!!epicsThreadCreate("Sender 1", epicsThreadPriorityLow, mediumStack, sender, q1), "Created Sender 1"); @@ -299,9 +346,9 @@ extern "C" void messageQueueTest(void *parm) mediumStack, sender, q1), "Created Sender 4"); - for (i = 0; i < 10; i++) { - testDiag("... %2d", 10 - i); - epicsThreadSleep(6.0); + for (i = 0; i < 6; i++) { + testDiag("... %2d", 6 - i); + epicsThreadSleep(5.0); } sendExit = 1; @@ -312,7 +359,7 @@ extern "C" void messageQueueTest(void *parm) MAIN(epicsMessageQueueTest) { - testPlan(62); + testPlan(64); finished = epicsEventMustCreate(epicsEventEmpty); mediumStack = epicsThreadGetStackSize(epicsThreadStackMedium); diff --git a/src/tools/Makefile b/src/tools/Makefile index e5668f36f..020f612a0 100644 --- a/src/tools/Makefile +++ b/src/tools/Makefile @@ -39,6 +39,7 @@ PERL_SCRIPTS += podToHtml.pl PERL_SCRIPTS += podRemove.pl PERL_SCRIPTS += replaceVAR.pl PERL_SCRIPTS += tap-to-junit-xml.pl +PERL_SCRIPTS += testFailures.pl PERL_SCRIPTS += useManifestTool.pl PERL_SCRIPTS += genVersionHeader.pl diff --git a/src/tools/podToHtml.pl b/src/tools/podToHtml.pl index 5e499443a..d42e20ae3 100644 --- a/src/tools/podToHtml.pl +++ b/src/tools/podToHtml.pl @@ -9,8 +9,12 @@ use strict; use warnings; +# To find the EPICS::PodHtml module used below we need to add our lib/perl to +# the lib search path. If the script is running from the src/tools directory +# before everything has been installed though, the search path must include +# our source directory (i.e. $Bin), so we add both here. use FindBin qw($Bin); -use lib "$Bin/../../lib/perl"; +use lib ("$Bin/../../lib/perl", $Bin); use Getopt::Std; $Getopt::Std::STANDARD_HELP_VERSION = 1; diff --git a/src/tools/testFailures.pl b/src/tools/testFailures.pl new file mode 100644 index 000000000..e19d00006 --- /dev/null +++ b/src/tools/testFailures.pl @@ -0,0 +1,33 @@ +#!/usr/bin/env perl +#************************************************************************* +# EPICS BASE is distributed subject to a Software License Agreement found +# in the file LICENSE that is included with this distribution. +#************************************************************************* + +# This file may appear trivial, but it exists to let the build system +# fail the 'make test-results' target with a nice output including a +# summary of the directories where test failures were reported. +# Test results are collected from the .tap files fed to epicsProve.pl +# which returns with an exit status of 0 (success) if all tests passed +# or 1 (failure) if any of the .tap files contained failed tests. +# When epicsProve.pl indicates a failure, the directory that it was +# running in is appended to the file $(TOP)/.tests-failed which this +# program reads in after all the test directories have been visited. +# The exit status of this program is 1 (failure) if any tests failed, +# otherwise 0 (success). + +use strict; +use warnings; + +die "Usage: testFailures.pl .tests-failed\n" + unless @ARGV == 1; + +open FAILURES, '<', shift or + exit 0; +my @failures = ; +close FAILURES; + +print "\nTest failures were reported in:\n", + (map {" $_"} @failures), "\n"; + +exit 1;