Windows: set CHOCO=NO to not use chocolatey

- needed when using the MSYS2 shell

.github/workflows/build-and-test.yml

@@ -19,13 +19,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04, windows-2022, windows-2019, macos-12, macos-11]
+        os: [ubuntu-22.04, ubuntu-20.04, windows-2022, windows-2019, macos-12, macos-11]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Show initial environment
-      run: python cue-test.py env
+      run: python3 cue-test.py env
     - name: Run unit tests
-      run: python cue-test.py
+      run: python3 cue-test.py
 
   build-linux:
     name: ${{ matrix.cmp }} / ${{ matrix.configuration }} / ${{ matrix.os }}
@@ -36,11 +36,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04]
+        os: [ubuntu-22.04, ubuntu-20.04]
         cmp: [gcc, clang]
         configuration: [default, static, debug, static-debug]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -63,7 +63,7 @@ jobs:
         cmp: [clang]
         configuration: [default, debug]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -93,7 +93,7 @@ jobs:
             cmp: vs2022
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -105,18 +105,19 @@ jobs:
 
   build-cross:
     name: cross ${{ matrix.cross }} / ${{ matrix.cmp }} / ${{ matrix.configuration }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       CMP: gcc
       BCFG: default
       CI_CROSS_TARGETS: ${{ matrix.cross }}
+      TEST: ${{ matrix.test }}
       APT: re2c
     strategy:
       fail-fast: false
       matrix:
         # configuration: [default, static, debug, static-debug]
         configuration: [default]
-        cross: 
+        cross:
         - linux-aarch64
         - linux-arm@arm-linux-gnueabi
         - linux-arm@arm-linux-gnueabihf
@@ -127,8 +128,11 @@ jobs:
         - RTEMS-pc386-qemu@4.9
         - RTEMS-pc386-qemu@4.10
         - RTEMS-pc686-qemu@5
+        include:
+        - cross: RTEMS-pc386-qemu@4.10
+          test: NO
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)

README.md

@@ -189,6 +189,19 @@ Collect the results of your tests and print a summary.
 `exec`\
 Execute the remainder of the line using the default command shell.
 
+## Extra arguments to `make`
+
+You can add additional arguments to the make runs that the `cue.py` script
+starts. Put your additional arguments into environment variables named
+`EXTRA`, `EXTRA1`, ... `EXTRA5`.
+
+The variables may contain multiple arguments, separated by whitespace.
+Use regular shell script quoting (single/double quotes, backslash escapes)
+if you need spaces inside an extra argument.
+
+The YAML syntax needed to set environment variables depends on the CI
+service and platform. (See the full example configuration file.)
+
 ## Setup Files
 
 Your module might depend on EPICS Base and a few other support modules.

cue-test.py

@@ -8,7 +8,6 @@
 from __future__ import print_function
 
 import sys, os, shutil, fileinput
-import distutils.util
 import re
 import subprocess as sp
 import unittest
@@ -81,10 +80,10 @@ class TestSourceSet(unittest.TestCase):
 
     def test_EmptySetupDirsPath(self):
         del os.environ['SETUP_PATH']
-        self.assertRaisesRegexp(NameError, '\(SETUP_PATH\) is empty', cue.source_set, 'test01')
+        self.assertRaisesRegex(NameError, '\(SETUP_PATH\) is empty', cue.source_set, 'test01')
 
     def test_InvalidSetupName(self):
-        self.assertRaisesRegexp(NameError, 'does not exist in SETUP_PATH', cue.source_set, 'xxdoesnotexistxx')
+        self.assertRaisesRegex(NameError, 'does not exist in SETUP_PATH', cue.source_set, 'xxdoesnotexistxx')
 
     def test_ValidSetupName(self):
         capturedOutput = getStringIO()
@@ -121,7 +120,7 @@ class TestSourceSet(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.source_set('test03')
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), 'Ignoring already included setup file')
+        self.assertRegex(capturedOutput.getvalue(), 'Ignoring already included setup file')
 
 
 class TestUpdateReleaseLocal(unittest.TestCase):
@@ -530,7 +529,7 @@ class TestTravisDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
+        self.assertRegex(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
 
     def test_MisspelledBcfgGetsWarning(self):
         os.environ['BCFG'] = 'static-dubug'
@@ -538,7 +537,7 @@ class TestTravisDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting")
+        self.assertRegex(capturedOutput.getvalue(), "Unrecognized build configuration setting")
 
 
 @unittest.skipIf(ci_service != 'appveyor', 'Run appveyor tests only on appveyor')
@@ -691,7 +690,7 @@ class TestAppveyorDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
+        self.assertRegex(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore")
 
     def test_MisspelledConfigurationGetsWarning(self):
         os.environ['CONFIGURATION'] = 'static-dubug'
@@ -699,11 +698,11 @@ class TestAppveyorDetectContext(unittest.TestCase):
         sys.stdout = capturedOutput
         cue.detect_context()
         sys.stdout = sys.__stdout__
-        self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting")
+        self.assertRegex(capturedOutput.getvalue(), "Unrecognized build configuration setting")
 
 
 class TestSetupForBuild(unittest.TestCase):
-    args = Namespace(paths=[])
+    args = Namespace(extra_env_vars=[])
     if ci_os == 'windows':
         choco_installs = ['make']
         if ci_service != 'appveyor':
@@ -722,7 +721,7 @@ class TestSetupForBuild(unittest.TestCase):
 
     def test_AddPathsOption(self):
         os.environ['FOOBAR'] = 'BAR'
-        args = Namespace(paths=['/my/{FOOBAR}/dir', '/my/foobar'])
+        args = Namespace(extra_env_vars=['PATH=/my/{FOOBAR}/dir', 'PATH=/my/foobar'])
         cue.setup_for_build(args)
         self.assertTrue(re.search('/my/BAR/dir', os.environ['PATH']), 'Expanded path not in PATH')
         self.assertTrue(re.search('/foobar', os.environ['PATH']), 'Plain path not in PATH')
@@ -841,7 +840,7 @@ class TestSetupForBuild(unittest.TestCase):
     def test_ExtraMakeArgs(self):
         os.environ['EXTRA'] = 'bla'
         for ind in range(1,5):
-            os.environ['EXTRA{0}'.format(ind)] = 'bla {0}'.format(ind)
+            os.environ['EXTRA{0}'.format(ind)] = '"bla {0}"'.format(ind)
         cue.setup_for_build(self.args)
         self.assertTrue(cue.extra_makeargs[0] == 'bla', 'Extra make arg [0] not set')
         for ind in range(1,5):
@@ -888,7 +887,7 @@ LINE2=NO''')
 
 @unittest.skipIf(ci_os != 'linux', 'CrossCompatibilityHandling tests only apply to linux')
 class TestCrossCompatibilityHandling(unittest.TestCase):
-    args = Namespace(paths=[])
+    args = Namespace(extra_env_vars=[])
 
     def setUp(self):
         cue.clear_lists()

cue.py

@@ -4,14 +4,16 @@
 
 from __future__ import print_function
 
-import sys, os, stat, shutil
+import sys, os, stat, shlex, shutil
 import fileinput
 import logging
 import re
+import time
 import threading
 from glob import glob
 import subprocess as sp
-import distutils.util
+import sysconfig
+import shutil
 
 logger = logging.getLogger(__name__)
 
@@ -31,6 +33,11 @@ def log_modified():
                 sys.stdout.write(F.read())
             sys.stdout.write(os.linesep)
 
+def whereis(cmd):
+    if hasattr(shutil, 'which'): # >= py3.3
+        loc = shutil.which(cmd)
+        print('{0}Found exec {1} at {2!r} {3}'.format(ANSI_CYAN, cmd, loc, ANSI_RESET))
+
 def prepare_env():
     '''HACK
     github actions yaml configuration doesn't allow
@@ -129,7 +136,10 @@ def detect_context():
         ci['cachedir'] = os.environ['CACHEDIR']
 
     if 'CHOCO' in os.environ:
-        ci['choco'].extend(os.environ['CHOCO'].split())
+        if os.environ['CHOCO'] == 'NO':
+            ci['choco'] = []
+        else:
+            ci['choco'].extend(os.environ['CHOCO'].split())
 
     if 'APT' in os.environ:
         ci['apt'].extend(os.environ['APT'].split())
@@ -313,7 +323,7 @@ def host_info():
     print('PYTHONPATH')
     for dname in sys.path:
         print(' ', dname)
-    print('platform =', distutils.util.get_platform())
+    print('platform =', sysconfig.get_platform())
 
     if ci['os'] == 'windows':
         print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET))
@@ -686,8 +696,10 @@ def setup_for_build(args):
     if ci['os'] == 'windows':
         if os.path.exists(r'C:\Strawberry\perl\bin'):
             # Put strawberry perl in front of the PATH (so that Git Perl is further behind)
+            # Put Chocolatey\bin ahead to select correct make.exe
             logger.debug('Adding Strawberry Perl in front of the PATH')
-            os.environ['PATH'] = os.pathsep.join([r'C:\Strawberry\c\bin',
+            os.environ['PATH'] = os.pathsep.join([r'C:\ProgramData\Chocolatey\bin',
+                                                  r'C:\Strawberry\c\bin',
                                                   r'C:\Strawberry\perl\site\bin',
                                                   r'C:\Strawberry\perl\bin',
                                                   os.environ['PATH']])
@@ -755,30 +767,52 @@ def setup_for_build(args):
                     if re.match('^test-results:', line):
                         has_test_results = True
 
-    # Check make version
-    if re.match(r'^GNU Make 3', sp.check_output(['make', '-v']).decode('ascii')):
-        is_make3 = True
-    logger.debug('Check if make is a 3.x series: %s', is_make3)
-
-    # apparently %CD% is handled automagically
+    # apparently %CD% is handled automagically, so use getcwd() instead
     os.environ['TOP'] = os.getcwd()
+    os.environ['MAKE'] = 'make'
+    os.environ['EPICS_BASE'] = places['EPICS_BASE']
 
-    addpaths = []
-    for path in args.paths:
+    changed_vars = set()
+
+    for extra_env_var in args.extra_env_vars:
         try:
-            addpaths.append(path.format(**os.environ))
+            key_value = extra_env_var.split('=')
+            key = key_value[0]
+            value = key_value[1]
+            expanded_value = value.format(**os.environ)
+
+            # Update the environment right now so later variables have access
+            if key in os.environ:
+                old_value = [os.environ[key]]
+            else:
+                old_value = []
+
+            os.environ[key] = os.pathsep.join(old_value + [expanded_value])
+            changed_vars.add(key)
         except KeyError:
             print('Environment')
             [print('  ', K, '=', repr(V)) for K, V in os.environ.items()]
             raise
 
-    os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths)
+    for key in changed_vars:
+        print("{0}{2} = {3}{1}".format(ANSI_CYAN, ANSI_RESET, key, os.environ[key]))
+
+    # os.environ completely updated at this point
+
+    logger.debug('Final PATH')
+    for loc in os.environ['PATH'].split(os.pathsep):
+        logger.debug('  %r', loc)
+
+    # Check make version
+    if re.match(r'^GNU Make 3', sp.check_output(['make', '-v']).decode('ascii')):
+        is_make3 = True
+    logger.debug('Check if make is a 3.x series: %s', is_make3)
 
     # Add EXTRA make arguments
     for tag in ['EXTRA', 'EXTRA1', 'EXTRA2', 'EXTRA3', 'EXTRA4', 'EXTRA5']:
         val = os.environ.get(tag, "")
         if len(val)>0:
-            extra_makeargs.append(val)
+            extra_makeargs.extend(shlex.split(val))
 
 
 def fix_etc_hosts():
@@ -1208,11 +1242,27 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
             with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f:
                 f.write(extra_config)
 
+        # enable color in error and warning messages if the (cross) compiler supports it
+        with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG'), 'a') as f:
+            f.write('''
+ifdef T_A
+  COLOR_FLAG_$(T_A) := $(shell $(CPP) -fdiagnostics-color -E - </dev/null >/dev/null 2>/dev/null && echo -fdiagnostics-color)
+  USR_CPPFLAGS += $(COLOR_FLAG_$(T_A))
+endif''')
+
         fold_end('set.up.epics_build', 'Configuring EPICS build system')
 
     if ci['os'] == 'windows' and ci['choco']:
         fold_start('install.choco', 'Installing CHOCO packages')
-        sp.check_call(['choco', 'install'] + ci['choco'] + ['-y', '--limitoutput', '--no-progress'])
+        for i in range(0,3):
+            try:
+                sp.check_call(['choco', 'install'] + ci['choco'] + ['-y', '--limitoutput', '--no-progress'])
+            except Exception as e:
+                print(e)
+                print("Retrying choco install attempt {} after 30 seconds".format(i+1))
+                time.sleep(30)
+            else:
+                break
         fold_end('install.choco', 'Installing CHOCO packages')
 
     if ci['os'] == 'linux' and ci['apt']:
@@ -1229,23 +1279,28 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
     setup_for_build(args)
 
     print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET))
+    whereis('make')
     print('{0}$ make --version{1}'.format(ANSI_CYAN, ANSI_RESET))
     sys.stdout.flush()
     call_make(['--version'], parallel=0)
+    whereis('perl')
     print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET))
     sys.stdout.flush()
     sp.check_call(['perl', '--version'])
 
     if re.match(r'^vs', ci['compiler']):
+        whereis('cl')
         print('{0}$ cl{1}'.format(ANSI_CYAN, ANSI_RESET))
         sys.stdout.flush()
         sp.check_call(['cl'])
     else:
         cc = ci['compiler']
+        whereis(cc)
         print('{0}$ {1} --version{2}'.format(ANSI_CYAN, cc, ANSI_RESET))
         sys.stdout.flush()
         sp.check_call([cc, '--version'])
         if cxx:
+            whereis(cxx)
             print('{0}$ {1} --version{2}'.format(ANSI_CYAN, cxx, ANSI_RESET))
             sys.stdout.flush()
             sp.check_call([cxx, '--version'])
@@ -1318,8 +1373,6 @@ def test_results(args):
 def doExec(args):
     'exec user command with vcvars'
     setup_for_build(args)
-    os.environ['MAKE'] = 'make'
-    os.environ['EPICS_BASE'] = places['EPICS_BASE']
     fold_start('exec.command', 'Execute command {}'.format(args.cmd))
     sp.check_call(' '.join(args.cmd), shell=True)
     fold_end('exec.command', 'Execute command {}'.format(args.cmd))
@@ -1389,8 +1442,10 @@ def getargs():
     p = ArgumentParser()
     p.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false',
                    help='Assume vcvarsall.bat has already been run')
-    p.add_argument('--add-path', dest='paths', default=[], action='append',
-                   help='Append directory to $PATH or %%PATH%%.  Expands {ENVVAR}')
+    p.add_argument('--add-path', dest='extra_env_vars', type=lambda x: "PATH={}".format(x), default=[], action='append',
+                   help='Append directory to $PATH or %%PATH%%. Expands {ENVVAR}. Equivalent to: "--add-env PATH=<PATHS>"')
+    p.add_argument('--add-env', dest='extra_env_vars', default=[], action='append',
+                   help='Append directory to the specified $ENVVAR or %%ENVVAR%%. Expands {OTHER_ENVVAR}. Example: "--add-env \'LD_LIBRARY_PATH={EPICS_BASE}/lib/{EPICS_HOST_ARCH}\'"')
     p.add_argument('-T', '--timeout', type=timespec, metavar='DLY',
                    help='Terminate make after delay.  DLY interpreted as second, or may be qualified with "S", "M", or "H".  (default no timeout)')
     subp = p.add_subparsers()

defaults.set

@@ -18,7 +18,7 @@ NTYPES_REPONAME=normativeTypesCPP
 NTYPES_REPOOWNER=epics-base
 
 # Sequencer
-SNCSEQ_REPOURL=https://www-csr.bessy.de/control/SoftDist/sequencer/repo/branch-2-2.git
+SNCSEQ_REPOURL=https://github.com/mdavidsaver/sequencer-mirror.git
 SNCSEQ_DEPTH=0
 SNCSEQ_DIRNAME=seq
 

github-actions/ci-scripts-build.yml.example-full

@@ -160,7 +160,7 @@ jobs:
             name: "Win2022 MSC-22, static"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Automatic core dumper analysis
@@ -216,7 +216,7 @@ jobs:
         - RTEMS-pc386-qemu@4.10
         - RTEMS-pc686-qemu@5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module

github-actions/ci-scripts-build.yml.example-mini

@@ -31,6 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+      include:
         - os: ubuntu-18.04
           cmp: gcc
           configuration: default
@@ -68,7 +69,7 @@ jobs:
           name: "3.15 Ub-20 clang-10"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: true
     - name: Prepare and compile dependencies