test: explicitly run tests using 'python3'

- 'python' is Python 2.7 on MacOS 12

.github/workflows/build-and-test.yml

@@ -19,13 +19,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04, windows-2022, windows-2019, macos-12, macos-11]
+        os: [ubuntu-22.04, ubuntu-20.04, windows-2022, windows-2019, macos-12, macos-11]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Show initial environment
-      run: python cue-test.py env
+      run: python3 cue-test.py env
     - name: Run unit tests
-      run: python cue-test.py
+      run: python3 cue-test.py
 
   build-linux:
     name: ${{ matrix.cmp }} / ${{ matrix.configuration }} / ${{ matrix.os }}
@@ -36,11 +36,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04]
+        os: [ubuntu-22.04, ubuntu-20.04]
         cmp: [gcc, clang]
         configuration: [default, static, debug, static-debug]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -63,7 +63,7 @@ jobs:
         cmp: [clang]
         configuration: [default, debug]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -93,7 +93,7 @@ jobs:
             cmp: vs2022
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -105,18 +105,19 @@ jobs:
 
   build-cross:
     name: cross ${{ matrix.cross }} / ${{ matrix.cmp }} / ${{ matrix.configuration }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       CMP: gcc
       BCFG: default
       CI_CROSS_TARGETS: ${{ matrix.cross }}
+      TEST: ${{ matrix.test }}
       APT: re2c
     strategy:
       fail-fast: false
       matrix:
         # configuration: [default, static, debug, static-debug]
         configuration: [default]
-        cross: 
+        cross:
         - linux-aarch64
         - linux-arm@arm-linux-gnueabi
         - linux-arm@arm-linux-gnueabihf
@@ -127,8 +128,11 @@ jobs:
         - RTEMS-pc386-qemu@4.9
         - RTEMS-pc386-qemu@4.10
         - RTEMS-pc686-qemu@5
+        include:
+        - cross: RTEMS-pc386-qemu@4.10
+          test: NO
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)

README.md

@@ -189,6 +189,19 @@ Collect the results of your tests and print a summary.
 `exec`\
 Execute the remainder of the line using the default command shell.
 
+## Extra arguments to `make`
+
+You can add additional arguments to the make runs that the `cue.py` script
+starts. Put your additional arguments into environment variables named
+`EXTRA`, `EXTRA1`, ... `EXTRA5`.
+
+The variables may contain multiple arguments, separated by whitespace.
+Use regular shell script quoting (single/double quotes, backslash escapes)
+if you need spaces inside an extra argument.
+
+The YAML syntax needed to set environment variables depends on the CI
+service and platform. (See the full example configuration file.)
+
 ## Setup Files
 
 Your module might depend on EPICS Base and a few other support modules.

cue-test.py

@@ -8,7 +8,6 @@
 from __future__ import print_function
 
 import sys, os, shutil, fileinput
-import distutils.util
 import re
 import subprocess as sp
 import unittest
@@ -81,10 +80,10 @@ class TestSourceSet(unittest.TestCase):
 
     def test_EmptySetupDirsPath(self):
         del os.environ['SETUP_PATH']
-        self.assertRaisesRegexp(NameError, '\(SETUP_PATH\) is empty', cue.source_set, 'test01')
+        self.assertRaisesRegex(NameError, '\(SETUP_PATH\) is empty', cue.source_set, 'test01')
 
     def test_InvalidSetupName(self):
-        self.assertRaisesRegexp(NameError, 'does not exist in SETUP_PATH', cue.source_set, 'xxdoesnotexistxx')
+        self.assertRaisesRegex(NameError, 'does not exist in SETUP_PATH', cue.source_set, 'xxdoesnotexistxx')
 
     def test_ValidSetupName(self):
         capturedOutput = getStringIO()
@@ -121,7 +120,7 @@ class TestSourceSet(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.source_set('test03')
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), 'Ignoring already included setup file')
+        self.assertRegex(capturedOutput.getvalue(), 'Ignoring already included setup file')
 
 
 class TestUpdateReleaseLocal(unittest.TestCase):
@@ -530,7 +529,7 @@ class TestTravisDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
+        self.assertRegex(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
 
     def test_MisspelledBcfgGetsWarning(self):
         os.environ['BCFG'] = 'static-dubug'
@@ -538,7 +537,7 @@ class TestTravisDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting")
+        self.assertRegex(capturedOutput.getvalue(), "Unrecognized build configuration setting")
 
 
 @unittest.skipIf(ci_service != 'appveyor', 'Run appveyor tests only on appveyor')
@@ -691,7 +690,7 @@ class TestAppveyorDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
+        self.assertRegex(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
 
     def test_MisspelledConfigurationGetsWarning(self):
         os.environ['CONFIGURATION'] = 'static-dubug'
@@ -699,7 +698,7 @@ class TestAppveyorDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting")
+        self.assertRegex(capturedOutput.getvalue(), "Unrecognized build configuration setting")
 
 
 class TestSetupForBuild(unittest.TestCase):
@@ -841,7 +840,7 @@ class TestSetupForBuild(unittest.TestCase):
     def test_ExtraMakeArgs(self):
         os.environ['EXTRA'] = 'bla'
         for ind in range(1,5):
-            os.environ['EXTRA{0}'.format(ind)] = 'bla {0}'.format(ind)
+            os.environ['EXTRA{0}'.format(ind)] = '"bla {0}"'.format(ind)
         cue.setup_for_build(self.args)
         self.assertTrue(cue.extra_makeargs[0] == 'bla', 'Extra make arg [0] not set')
         for ind in range(1,5):

cue.py

@@ -4,14 +4,16 @@
 
 from __future__ import print_function
 
-import sys, os, stat, shutil
+import sys, os, stat, shlex, shutil
 import fileinput
 import logging
 import re
+import time
 import threading
 from glob import glob
 import subprocess as sp
-import distutils.util
+import sysconfig
+import shutil
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +33,11 @@ def log_modified():
                 sys.stdout.write(F.read())
             sys.stdout.write(os.linesep)
 
+def whereis(cmd):
+    if hasattr(shutil, 'which'): # >= py3.3
+        loc = shutil.which(cmd)
+        print('{0}Found exec {1} at {2!r} {3}'.format(ANSI_CYAN, cmd, loc, ANSI_RESET))
+
 def prepare_env():
     '''HACK
     github actions yaml configuration doesn't allow
@@ -313,7 +320,7 @@ def host_info():
     print('PYTHONPATH')
     for dname in sys.path:
         print(' ', dname)
-    print('platform =', distutils.util.get_platform())
+    print('platform =', sysconfig.get_platform())
 
     if ci['os'] == 'windows':
         print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET))
@@ -686,8 +693,10 @@ def setup_for_build(args):
     if ci['os'] == 'windows':
         if os.path.exists(r'C:\Strawberry\perl\bin'):
             # Put strawberry perl in front of the PATH (so that Git Perl is further behind)
+            # Put Chocolatey\bin ahead to select correct make.exe
             logger.debug('Adding Strawberry Perl in front of the PATH')
-            os.environ['PATH'] = os.pathsep.join([r'C:\Strawberry\c\bin',
+            os.environ['PATH'] = os.pathsep.join([r'C:\ProgramData\Chocolatey\bin',
+                                                  r'C:\Strawberry\c\bin',
                                                   r'C:\Strawberry\perl\site\bin',
                                                   r'C:\Strawberry\perl\bin',
                                                   os.environ['PATH']])
@@ -755,14 +764,6 @@ def setup_for_build(args):
                     if re.match('^test-results:', line):
                         has_test_results = True
 
-    # Check make version
-    if re.match(r'^GNU Make 3', sp.check_output(['make', '-v']).decode('ascii')):
-        is_make3 = True
-    logger.debug('Check if make is a 3.x series: %s', is_make3)
-
-    # apparently %CD% is handled automagically
-    os.environ['TOP'] = os.getcwd()
-
     addpaths = []
     for path in args.paths:
         try:
@@ -774,11 +775,23 @@ def setup_for_build(args):
 
     os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths)
 
+    logger.debug('Final PATH')
+    for loc in os.environ['PATH'].split(os.pathsep):
+        logger.debug('  %r', loc)
+
+    # Check make version
+    if re.match(r'^GNU Make 3', sp.check_output(['make', '-v']).decode('ascii')):
+        is_make3 = True
+    logger.debug('Check if make is a 3.x series: %s', is_make3)
+
+    # apparently %CD% is handled automagically
+    os.environ['TOP'] = os.getcwd()
+
     # Add EXTRA make arguments
     for tag in ['EXTRA', 'EXTRA1', 'EXTRA2', 'EXTRA3', 'EXTRA4', 'EXTRA5']:
         val = os.environ.get(tag, "")
         if len(val)>0:
-            extra_makeargs.append(val)
+            extra_makeargs.extend(shlex.split(val))
 
 
 def fix_etc_hosts():
@@ -1208,11 +1221,27 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
             with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f:
                 f.write(extra_config)
 
+        # enable color in error and warning messages if the (cross) compiler supports it
+        with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG'), 'a') as f:
+            f.write('''
+ifdef T_A
+  COLOR_FLAG_$(T_A) := $(shell $(CPP) -fdiagnostics-color -E - </dev/null >/dev/null 2>/dev/null && echo -fdiagnostics-color)
+  USR_CPPFLAGS += $(COLOR_FLAG_$(T_A))
+endif''')
+
         fold_end('set.up.epics_build', 'Configuring EPICS build system')
 
     if ci['os'] == 'windows' and ci['choco']:
         fold_start('install.choco', 'Installing CHOCO packages')
-        sp.check_call(['choco', 'install'] + ci['choco'] + ['-y', '--limitoutput', '--no-progress'])
+        for i in range(0,3):
+            try:
+                sp.check_call(['choco', 'install'] + ci['choco'] + ['-y', '--limitoutput', '--no-progress'])
+            except Exception as e:
+                print(e)
+                print("Retrying choco install attempt {} after 30 seconds".format(i+1))
+                time.sleep(30)
+            else:
+                break
         fold_end('install.choco', 'Installing CHOCO packages')
 
     if ci['os'] == 'linux' and ci['apt']:
@@ -1229,23 +1258,28 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
     setup_for_build(args)
 
     print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET))
+    whereis('make')
     print('{0}$ make --version{1}'.format(ANSI_CYAN, ANSI_RESET))
     sys.stdout.flush()
     call_make(['--version'], parallel=0)
+    whereis('perl')
     print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET))
     sys.stdout.flush()
     sp.check_call(['perl', '--version'])
 
     if re.match(r'^vs', ci['compiler']):
+        whereis('cl')
         print('{0}$ cl{1}'.format(ANSI_CYAN, ANSI_RESET))
         sys.stdout.flush()
         sp.check_call(['cl'])
     else:
         cc = ci['compiler']
+        whereis(cc)
         print('{0}$ {1} --version{2}'.format(ANSI_CYAN, cc, ANSI_RESET))
         sys.stdout.flush()
         sp.check_call([cc, '--version'])
         if cxx:
+            whereis(cxx)
             print('{0}$ {1} --version{2}'.format(ANSI_CYAN, cxx, ANSI_RESET))
             sys.stdout.flush()
             sp.check_call([cxx, '--version'])

defaults.set

@@ -18,7 +18,7 @@ NTYPES_REPONAME=normativeTypesCPP
 NTYPES_REPOOWNER=epics-base
 
 # Sequencer
-SNCSEQ_REPOURL=https://www-csr.bessy.de/control/SoftDist/sequencer/repo/branch-2-2.git
+SNCSEQ_REPOURL=https://github.com/mdavidsaver/sequencer-mirror.git
 SNCSEQ_DEPTH=0
 SNCSEQ_DIRNAME=seq
 

github-actions/ci-scripts-build.yml.example-full

@@ -160,7 +160,7 @@ jobs:
             name: "Win2022 MSC-22, static"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Automatic core dumper analysis
@@ -216,7 +216,7 @@ jobs:
         - RTEMS-pc386-qemu@4.10
         - RTEMS-pc686-qemu@5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module