epics/epics-base
0
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

Merge branch 'timeout-tests' into 7.0

Browse Source
This commit is contained in:
Andrew Johnson
2021-04-26 18:16:08 -05:00
parent b00130493f c07ebcee5c
commit 71a1ff1292
5 changed files with 218 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/ci-scripts-build.yml vendored
View File

@ -134,6 +134,11 @@ jobs:
configuration: static
name: "Win2019 MSC-19, static"
- os: windows-2019
cmp: vs2019
configuration: debug
name: "Win2019 MSC-19, debug"
- os: windows-2019
cmp: gcc
configuration: default

27
documentation/RELEASE_NOTES.md
View File

@ -106,6 +106,33 @@ that the variables referenced by output pointers are initialized.
```
### Timeouts for Unit Test Programs
The unit test programs that are run by the `make runtests` or `make tapfiles`
commands get executed by a `.t` wrapper script which is normally generated by
the EPICS `makeTestfile.pl` program. Those generated wrapper scripts now
impose a time-limit on the test program they execute, and will kill it if it
runs for longer than 500 seconds (8 minutes 20) without exiting. That
time-limit can be changed for any such test by modifying the Makefile which
creates and runs the `.t` wrapper script.
Setting the environment variable `EPICS_UNITTEST_TIMEOUT` to the desired
number of seconds while the Makefile is generating the test script changes the
timeout in that script. For example:
```
TESTSCRIPTS_HOST += hourLongTest.t
hourLongTest.t: export EPICS_UNITTEST_TIMEOUT=3600
```
When selecting such a timeout remember that different Continuous Integration
systems such as GitHub Actions and Appveyor run on processors with different
speeds, so allow enough head-room for slower systems to complete the test.
Test programs written directly in Perl as a `.plt` script should implement a
similar timeout for themselves. The "netget" test in Base does this in a way
that works on Windows as well as Unix-like hosts.
-----
## EPICS Release 7.0.5

4
modules/database/test/std/rec/Makefile
View File

@ -175,8 +175,12 @@ dbHeaderTestxx_SRCS += dbHeaderTestxx.cpp
ifeq ($(T_A),$(EPICS_HOST_ARCH))
# Host-only tests of softIoc/softIocPVA, caget and pvget (if present)
# Unfortunately hangs too often on CI systems:
ifndef CI
TESTS += netget
endif
endif
# epicsRunRecordTests runs all the test programs in a known working order.
testHarness_SRCS += epicsRunRecordTests.c

130
modules/database/test/std/rec/netget.plt
View File

@ -3,13 +3,16 @@
use strict;
use warnings;
use if $^O eq 'MSWin32', "Win32::Process";
use if $^O eq 'MSWin32', "Win32";
use lib '@TOP@/lib/perl';
use Test::More tests => 3;
use EPICS::IOC;
# Set to 1 to echo all IOC and client communications
my $debug = 1;
my $debug = 0;
$ENV{HARNESS_ACTIVE} = 1 if scalar @ARGV && shift eq '-tap';
@ -35,7 +38,7 @@ $ioc->debug($debug);
$SIG{__DIE__} = $SIG{INT} = $SIG{QUIT} = sub {
$ioc->exit;
BAIL_OUT('Caught signal');
BAIL_OUT("Caught signal: $_[0]");
};
@ -50,11 +53,21 @@ sub kill_bail {
}
sub watchdog (&$$) {
my ($do, $timeout, $abort) = @_;
$SIG{ALRM} = $abort;
alarm $timeout;
&$do;
alarm 0;
my ($code, $timeout, $fail) = @_;
my $bark = "Woof $$\n";
my $result;
eval {
local $SIG{__DIE__};
local $SIG{ALRM} = sub { die $bark };
alarm $timeout;
$result = &$code;
alarm 0;
};
if ($@) {
die if $@ ne $bark;
$result = &$fail;
}
return $result;
}
@ -90,19 +103,6 @@ like($version, qr/^ \d+ \. \d+ \. \d+ /x,
"Got BaseVersion '$version' from iocsh");
# Client Tests
my $client = EPICS::IOC->new;
$client->debug($debug);
sub close_client {
my $doing = shift;
return sub {
diag("Timeout $doing");
$client->close;
}
}
# Channel Access
SKIP: {
@ -119,17 +119,9 @@ SKIP: {
# CA Client test
watchdog {
$client->start($caget, '-w5', $pv);
my $caVersion = $client->_getline;
like($caVersion, qr/^ $pv \s+ \Q$version\E $/x,
'Got same BaseVersion from caget');
my @errors = $client->_geterrors;
note("Errors from caget:\n",
map(" $_\n", @errors))
if scalar @errors;
$client->close;
} 15, close_client('doing caget');
my $caVersion = qx_timeout(15, "$caget -w5 $pv");
like($caVersion, qr/^ $pv \s+ \Q$version\E $/x,
'Got same BaseVersion from caget');
}
@ -152,17 +144,71 @@ SKIP: {
# PVA Client test
watchdog {
$client->start($pvget, '-w5', $pv);
my $pvaVersion = $client->_getline;
like($pvaVersion, qr/^ $pv \s .* \Q$version\E \s* $/x,
'Got same BaseVersion from pvget');
my @errors = $client->_geterrors;
note("Errors from pvget:\n",
map(" $_\n", @errors))
if scalar @errors;
$client->close;
} 10, close_client('doing pvget');
my $pvaVersion = qx_timeout(15, "$pvget -w5 $pv");
like($pvaVersion, qr/^ $pv \s .* \Q$version\E \s* $/x,
'Got same BaseVersion from pvget');
}
$ioc->exit;
# Process timeout utilities
sub system_timeout {
my ($timeout, $cmdline) = @_;
my $status;
if ($^O eq 'MSWin32') {
my $proc;
(my $app) = split ' ', $cmdline;
if (! Win32::Process::Create($proc, $app, $cmdline,
1, &Win32::Process::NORMAL_PRIORITY_CLASS, '.')) {
my $err = Win32::FormatMessage(Win32::GetLastError());
die "Can't create Process for '$cmdline': $err\n";
}
if (! $proc->Wait(1000 * $timeout)) {
$proc->Kill(1);
note("Timed out '$cmdline' after $timeout seconds\n");
}
my $status;
$proc->GetExitCode($status);
return $status;
}
else {
my $pid;
$status = watchdog {
$pid = fork();
die "Can't fork: $!\n"
unless defined $pid;
exec $cmdline
or die "Can't exec: $!\n"
unless $pid;
waitpid $pid, 0;
return $? >> 8;
} $timeout, sub {
kill 9, $pid if $pid;
note("Timed out '$cmdline' after $timeout seconds\n");
return -2;
};
}
return $status;
}
sub qx_timeout {
my ($timeout, $cmdline) = @_;
open(my $stdout, '>&STDOUT')
or die "Can't save STDOUT: $!\n";
my $outfile = "stdout-$$.txt";
unlink $outfile;
open STDOUT, '>', $outfile;
my $text;
if (system_timeout($timeout, $cmdline) == 0 && -r $outfile) {
open(my $file, '<', $outfile)
or die "Can't open $outfile: $!\n";
$text = join '', <$file>;
close $file;
}
open(STDOUT, '>&', $stdout)
or die "Can't restore STDOUT: $!\n";
unlink $outfile;
return $text;
}

125
src/tools/makeTestfile.pl
View File

@ -21,8 +21,20 @@
# target.t is the name of the Perl script to generate
# executable is the name of the file the script runs
# Test programs that need more than 500 seconds to run should have the
# EPICS_UNITTEST_TIMEOUT environment variable set in their Makefile:
# longRunningTest.t: export EPICS_UNITTEST_TIMEOUT=3600
# That embeds the timeout into the .t file. The timeout variable can also
# be set at runtime, which will override any compiled-in setting but the
# 'make runtests' command can't give a different timeout for each test.
use strict;
use File::Basename;
my $tool = basename($0);
my $timeout = $ENV{EPICS_UNITTEST_TIMEOUT} // 500; # 8 min 20 sec
my ($TA, $HA, $target, $exe) = @ARGV;
my $exec;
@ -43,21 +55,49 @@ if( $TA =~ /^win32-x86/ && $HA !~ /^win/ ) {
# Explicitly fail for other RTEMS targets
} elsif( $TA =~ /^RTEMS-/ ) {
die "$0: I don't know how to create scripts for testing $TA on $HA\n";
die "$tool: I don't know how to create scripts for testing $TA on $HA\n";
} else {
$exec = "./$exe";
}
# Ensure that Windows interactive error handling is disabled.
# This setting is inherited by the test process.
# Set SEM_FAILCRITICALERRORS (1) Disable critical-error-handler dialog
# Clear SEM_NOGPFAULTERRORBOX (2) Enabled WER to allow automatic post mortem debugging (AeDebug)
# Clear SEM_NOALIGNMENTFAULTEXCEPT (4) Allow alignment fixups
# Set SEM_NOOPENFILEERRORBOX (0x8000) Prevent dialog on some I/O errors
# https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-seterrormode?redirectedfrom=MSDN
my $sem = $^O ne 'MSWin32' ? '' : <<ENDBEGIN;
# Create the $target.t file
open(my $OUT, '>', $target)
or die "$tool: Can't create $target: $!\n";
print $OUT <<__EOT__;
#!/usr/bin/env perl
# This file was generated by $tool
use strict;
use Cwd 'abs_path';
use File::Basename;
my \$tool = basename(\$0);
\$ENV{HARNESS_ACTIVE} = 1 if scalar \@ARGV && shift eq '-tap';
\$ENV{TOP} = abs_path(\$ENV{TOP}) if exists \$ENV{TOP};
# The timeout value below can be set in the Makefile that builds
# this test script. Add this line and adjust the value (in seconds):
# $target: export EPICS_UNITTEST_TIMEOUT=$timeout
my \$timeout = \$ENV{EPICS_UNITTEST_TIMEOUT} // $timeout;
__EOT__
if ($^O eq 'MSWin32') {
######################################## Code for Windows run-hosts
print $OUT <<__WIN32__;
use Win32::Process;
use Win32;
BEGIN {
# Ensure that Windows interactive error handling is disabled.
# This setting is inherited by the test process.
# Set SEM_FAILCRITICALERRORS (1) Disable critical-error-handler dialog
# Clear SEM_NOGPFAULTERRORBOX (2) Enabled WER to allow automatic post mortem debugging (AeDebug)
# Clear SEM_NOALIGNMENTFAULTEXCEPT (4) Allow alignment fixups
# Set SEM_NOOPENFILEERRORBOX (0x8000) Prevent dialog on some I/O errors
# https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-seterrormode
my \$sem = 'SetErrorMode';
eval {
require Win32::ErrorMode;
@ -69,30 +109,53 @@ BEGIN {
} if \$@;
SetErrorMode(0x8001) unless \$@;
}
ENDBEGIN
open(my $OUT, '>', $target) or die "Can't create $target: $!\n";
my \$proc;
if (! Win32::Process::Create(\$proc, abs_path('$exec'),
'$exec', 1, NORMAL_PRIORITY_CLASS, '.')) {
my \$err = Win32::FormatMessage(Win32::GetLastError());
die "\$tool: Can't create Process for '$exec': \$err\\n";
}
if (! \$proc->Wait(1000 * \$timeout)) {
\$proc->Kill(1);
print "\\n#### Test stopped by \$tool after \$timeout seconds\\n";
die "\$tool: Timed out '$exec' after \$timeout seconds\\n";
}
my \$status;
\$proc->GetExitCode(\$status);
exit \$status;
print $OUT <<EOF;
#!/usr/bin/env perl
use strict;
use Cwd 'abs_path';
$sem
\$ENV{HARNESS_ACTIVE} = 1 if scalar \@ARGV && shift eq '-tap';
\$ENV{TOP} = abs_path(\$ENV{TOP}) if exists \$ENV{TOP};
if (\$^O eq 'MSWin32') {
# Use system on Windows, exec doesn't work the same there and
# GNUmake thinks the test has finished too soon.
my \$status = system('$exec');
die "Can't run $exec: \$!\\n" if \$status == -1;
exit \$status >> 8;
__WIN32__
}
else {
exec '$exec' or die "Can't run $exec: \$!\\n";
}
EOF
######################################## Code for Unix run-hosts
print $OUT <<__UNIX__;
close $OUT or die "Can't close $target: $!\n";
my \$pid = fork();
die "\$tool: Can't fork for '$exec': \$!\\n"
unless defined \$pid;
if (\$pid) {
# Parent process
\$SIG{ALRM} = sub {
# Time's up, kill the child
kill 9, \$pid;
print "\\n#### Test stopped by \$tool after \$timeout seconds\\n";
die "\$tool: Timed out '$exec' after \$timeout seconds\\n";
};
alarm \$timeout;
waitpid \$pid, 0;
alarm 0;
exit \$? >> 8;
}
else {
# Child process
exec '$exec'
or die "\$tool: Can't run '$exec': \$!\\n";
}
__UNIX__
}
close $OUT
or die "$tool: Can't close '$target': $!\n";