use cpu_count when available

GitHub announcement:
The macOS 13 runner image will be retired by December 4th, 2025

.appveyor.yml

@@ -45,9 +45,9 @@ skip_commits:
 
 install:
 # fetch submodules (like ci-scripts)
-  - cmd: git submodule update --init --recursive
+  - git submodule update --init --recursive
 # for the sequencer
-  - cinst re2c
+  - choco install re2c
 
 #---------------------------------#
 #   build matrix configuration    #

.github/workflows/build-and-test.yml

@@ -19,9 +19,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04, windows-2022, windows-2019, macos-12, macos-11]
+        os: [ubuntu-24.04, ubuntu-22.04, windows-2025, windows-2022, macos-15, macos-14]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Show initial environment
       run: python3 cue-test.py env
     - name: Run unit tests
@@ -36,11 +36,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04]
+        os: [ubuntu-24.04, ubuntu-22.04]
         cmp: [gcc, clang]
         configuration: [default, static, debug, static-debug]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -59,11 +59,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos-12, macos-11]
+        os: [macos-15, macos-14]
         cmp: [clang]
         configuration: [default, debug]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -82,18 +82,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2022, windows-2019]
-        cmp: [gcc, vs2022, vs2019]
+        os: [windows-2025, windows-2022]
+        cmp: [gcc, vs2022]
         configuration: [default, static, debug, static-debug]
-        exclude:
-          - os: windows-2022
-            cmp: vs2019
-
-          - os: windows-2019
-            cmp: vs2022
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)
@@ -132,7 +126,7 @@ jobs:
         - cross: RTEMS-pc386-qemu@4.10
           test: NO
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Prepare and compile dependencies
       run: python cue.py prepare
     - name: Build main module (example app)

appveyor/.appveyor.yml.example-full

@@ -41,9 +41,9 @@ skip_commits:
 
 install:
 # fetch submodules (like ci-scripts)
-  - cmd: git submodule update --init --recursive
+  - git submodule update --init --recursive
 # for the sequencer
-  - cinst re2c
+  - choco install re2c
 
 #---------------------------------#
 #   build matrix configuration    #

appveyor/.appveyor.yml.example-mini

@@ -18,7 +18,7 @@ skip_commits:
     - '.travis.yml'
 
 install:
-  - cmd: git submodule update --init --recursive
+  - git submodule update --init --recursive
 
 image: Visual Studio 2022
 

cue-test.py

@@ -269,12 +269,12 @@ class TestAddDependencyOptions(unittest.TestCase):
         cue.detect_context()
         if os.path.exists(cue.ci['cachedir']):
             shutil.rmtree(cue.ci['cachedir'], onerror=cue.remove_readonly)
-        self.location = os.path.join(cue.ci['cachedir'], 'mcoreutils-master')
+        self.location = os.path.join(cue.ci['cachedir'], 'mcoreutils-main')
         self.testfile = os.path.join(self.location, '.ci', 'LICENSE')
-        os.path.join(cue.ci['cachedir'], 'mcoreutils-master')
+        os.path.join(cue.ci['cachedir'], 'mcoreutils-main')
         cue.source_set('defaults')
         cue.complete_setup('MCoreUtils')
-        cue.setup['MCoreUtils'] = 'master'
+        cue.setup['MCoreUtils'] = 'main'
 
     def test_Default(self):
         cue.add_dependency('MCoreUtils')
@@ -702,7 +702,7 @@ class TestAppveyorDetectContext(unittest.TestCase):
 
 
 class TestSetupForBuild(unittest.TestCase):
-    args = Namespace(paths=[])
+    args = Namespace(extra_env_vars=[])
     if ci_os == 'windows':
         choco_installs = ['make']
         if ci_service != 'appveyor':
@@ -721,7 +721,7 @@ class TestSetupForBuild(unittest.TestCase):
 
     def test_AddPathsOption(self):
         os.environ['FOOBAR'] = 'BAR'
-        args = Namespace(paths=['/my/{FOOBAR}/dir', '/my/foobar'])
+        args = Namespace(extra_env_vars=['PATH=/my/{FOOBAR}/dir', 'PATH=/my/foobar'])
         cue.setup_for_build(args)
         self.assertTrue(re.search('/my/BAR/dir', os.environ['PATH']), 'Expanded path not in PATH')
         self.assertTrue(re.search('/foobar', os.environ['PATH']), 'Plain path not in PATH')
@@ -863,7 +863,8 @@ class TestHooks(unittest.TestCase):
             pass
         with open(self.bla_file, 'w') as f:
             f.write('''LINE1=YES
-LINE2=NO''')
+LINE2=NO
+''')
 
     def test_patchfile(self):
         hook = os.path.join(builddir, 'test.patch')
@@ -887,7 +888,7 @@ LINE2=NO''')
 
 @unittest.skipIf(ci_os != 'linux', 'CrossCompatibilityHandling tests only apply to linux')
 class TestCrossCompatibilityHandling(unittest.TestCase):
-    args = Namespace(paths=[])
+    args = Namespace(extra_env_vars=[])
 
     def setUp(self):
         cue.clear_lists()

cue.py

@@ -15,6 +15,12 @@ import subprocess as sp
 import sysconfig
 import shutil
 
+try:
+    from os import cpu_count
+except ImportError:
+    def cpu_count():
+        pass # undetermined
+
 logger = logging.getLogger(__name__)
 
 # Keep track of all files we write/append for later logging
@@ -136,7 +142,10 @@ def detect_context():
         ci['cachedir'] = os.environ['CACHEDIR']
 
     if 'CHOCO' in os.environ:
-        ci['choco'].extend(os.environ['CHOCO'].split())
+        if os.environ['CHOCO'] == 'NO':
+            ci['choco'] = []
+        else:
+            ci['choco'].extend(os.environ['CHOCO'].split())
 
     if 'APT' in os.environ:
         ci['apt'].extend(os.environ['APT'].split())
@@ -148,7 +157,7 @@ def detect_context():
     if 'TEST' in os.environ and os.environ['TEST'].lower() == 'no':
         ci['test'] = False
 
-    ci['parallel_make'] = 2
+    ci['parallel_make'] = cpu_count() or 2
     if 'PARALLEL_MAKE' in os.environ:
         ci['parallel_make'] = int(os.environ['PARALLEL_MAKE'])
 
@@ -428,9 +437,11 @@ def update_release_local(var, location):
 def set_setup_from_env(dep):
     for postf in ['', '_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL',
                   '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']:
-        if dep + postf in os.environ:
-            setup[dep + postf] = os.environ[dep + postf]
-            logger.debug('ENV assignment: %s = %s', dep + postf, setup[dep + postf])
+        env = dep + postf
+        val = os.environ.get(env)
+        if val:
+            setup[env] = val
+            logger.debug('ENV assignment: %s = %s', env, setup[env])
 
 
 def call_git(args, **kws):
@@ -764,16 +775,37 @@ def setup_for_build(args):
                     if re.match('^test-results:', line):
                         has_test_results = True
 
-    addpaths = []
-    for path in args.paths:
+    # apparently %CD% is handled automagically, so use getcwd() instead
+    os.environ['TOP'] = os.getcwd()
+    os.environ['MAKE'] = 'make'
+    os.environ['EPICS_BASE'] = places['EPICS_BASE']
+
+    changed_vars = set()
+
+    for extra_env_var in args.extra_env_vars:
         try:
-            addpaths.append(path.format(**os.environ))
+            key_value = extra_env_var.split('=')
+            key = key_value[0]
+            value = key_value[1]
+            expanded_value = value.format(**os.environ)
+
+            # Update the environment right now so later variables have access
+            if key in os.environ:
+                old_value = [os.environ[key]]
+            else:
+                old_value = []
+
+            os.environ[key] = os.pathsep.join(old_value + [expanded_value])
+            changed_vars.add(key)
         except KeyError:
             print('Environment')
             [print('  ', K, '=', repr(V)) for K, V in os.environ.items()]
             raise
 
-    os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths)
+    for key in changed_vars:
+        print("{0}{2} = {3}{1}".format(ANSI_CYAN, ANSI_RESET, key, os.environ[key]))
+
+    # os.environ completely updated at this point
 
     logger.debug('Final PATH')
     for loc in os.environ['PATH'].split(os.pathsep):
@@ -784,9 +816,6 @@ def setup_for_build(args):
         is_make3 = True
     logger.debug('Check if make is a 3.x series: %s', is_make3)
 
-    # apparently %CD% is handled automagically
-    os.environ['TOP'] = os.getcwd()
-
     # Add EXTRA make arguments
     for tag in ['EXTRA', 'EXTRA1', 'EXTRA2', 'EXTRA3', 'EXTRA4', 'EXTRA5']:
         val = os.environ.get(tag, "")
@@ -842,9 +871,8 @@ def handle_old_cross_variables():
         os.environ["CI_CROSS_TARGETS"] = ""
 
     if "RTEMS" in os.environ:
-        if 'RTEMS_TARGET' in os.environ:
-            rtems_target = os.environ['RTEMS_TARGET']
-        else:
+        rtems_target = os.environ.get('RTEMS_TARGET')
+        if not rtems_target:
             if os.environ['RTEMS'] == '5':
                 rtems_target = 'RTEMS-pc686-qemu'
             else:
@@ -868,8 +896,10 @@ def handle_old_cross_variables():
     if "WINE" in os.environ:
         if os.environ['WINE'] == '32':
             new_cross_target = ":win32-x86-mingw"
-        else:
+        elif os.environ['WINE'] == '64':
             new_cross_target = ":windows-x64-mingw"
+        else:
+            raise RuntimeError("Invalid $WINE, must be 32/64")
         os.environ["CI_CROSS_TARGETS"] += new_cross_target
 
         print(
@@ -1352,8 +1382,6 @@ def test_results(args):
 def doExec(args):
     'exec user command with vcvars'
     setup_for_build(args)
-    os.environ['MAKE'] = 'make'
-    os.environ['EPICS_BASE'] = places['EPICS_BASE']
     fold_start('exec.command', 'Execute command {}'.format(args.cmd))
     sp.check_call(' '.join(args.cmd), shell=True)
     fold_end('exec.command', 'Execute command {}'.format(args.cmd))
@@ -1423,8 +1451,10 @@ def getargs():
     p = ArgumentParser()
     p.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false',
                    help='Assume vcvarsall.bat has already been run')
-    p.add_argument('--add-path', dest='paths', default=[], action='append',
-                   help='Append directory to $PATH or %%PATH%%.  Expands {ENVVAR}')
+    p.add_argument('--add-path', dest='extra_env_vars', type=lambda x: "PATH={}".format(x), default=[], action='append',
+                   help='Append directory to $PATH or %%PATH%%. Expands {ENVVAR}. Equivalent to: "--add-env PATH=<PATHS>"')
+    p.add_argument('--add-env', dest='extra_env_vars', default=[], action='append',
+                   help='Append directory to the specified $ENVVAR or %%ENVVAR%%. Expands {OTHER_ENVVAR}. Example: "--add-env \'LD_LIBRARY_PATH={EPICS_BASE}/lib/{EPICS_HOST_ARCH}\'"')
     p.add_argument('-T', '--timeout', type=timespec, metavar='DLY',
                    help='Terminate make after delay.  DLY interpreted as second, or may be qualified with "S", "M", or "H".  (default no timeout)')
     subp = p.add_subparsers()

github-actions/ci-scripts-build.yml.example-full

@@ -41,113 +41,57 @@ jobs:
       matrix:
         # Job names also name artifacts, character limitations apply
         include:
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             cmp: gcc
             configuration: default
             wine: "64"
-            name: "Ub-20 gcc-9 + MinGW"
+            name: "Ub-24 gcc + MinGW"
 
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             cmp: gcc
             configuration: static
             wine: "64"
-            name: "Ub-20 gcc-9 + MinGW, static"
+            name: "Ub-24 gcc + MinGW, static"
 
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             cmp: gcc
             configuration: static
             extra: "CMD_CXXFLAGS=-std=c++11"
-            name: "Ub-20 gcc-9 C++11, static"
+            name: "Ub-24 gcc C++11, static"
 
-          - os: ubuntu-18.04
-            cmp: clang
-            configuration: default
-            name: "Ub-18 clang-9"
-
-          - os: ubuntu-20.04
+          - os: ubuntu-24.04
             cmp: clang
             configuration: default
             extra: "CMD_CXXFLAGS=-std=c++11"
-            name: "Ub-20 clang-10 C++11"
+            name: "Ub-24 clang C++11"
 
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             cmp: gcc
             configuration: default
-            rtems: "4.10"
-            name: "Ub-20 gcc-9 + RT-4.10"
+            cross: "RTEMS-pc686-qemu@5"
+            name: "Ub-22 gcc + RT-5.1 pc686"
 
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             cmp: gcc
             configuration: default
-            rtems: "4.9"
-            name: "Ub-20 gcc-9 + RT-4.9"
-
-          - os: ubuntu-20.04
-            cmp: gcc
-            configuration: default
-            rtems: "5"
-            rtems_target: RTEMS-pc686-qemu
-            name: "Ub-20 gcc-9 + RT-5.1 pc686"
-
-          - os: ubuntu-20.04
-            cmp: gcc
-            configuration: default
-            rtems: "5"
-            rtems_target: RTEMS-beatnik
+            cross: "RTEMS-beatnik@5"
             test: NO
-            name: "Ub-20 gcc-9 + RT-5.1 beatnik"
+            name: "Ub-22 gcc + RT-5.1 beatnik"
 
-          - os: ubuntu-18.04
+          - os: macos-latest
+            cmp: clang
+            configuration: default
+            name: "MacOS clang"
+
+          - os: windows-2022
             cmp: gcc
             configuration: default
-            name: "Ub-18 gcc-7"
+            name: "Win2022 MinGW"
 
-          - os: ubuntu-18.04
-            cmp: gcc-8
-            utoolchain: true
-            configuration: default
-            name: "Ub-18 gcc-8"
-
-          - os: ubuntu-20.04
-            cmp: gcc-8
-            utoolchain: true
-            configuration: default
-            name: "Ub-20 gcc-8"
-
-          - os: ubuntu-20.04
-            cmp: clang
-            configuration: default
-            name: "Ub-20 clang-10"
-
-          - os: macos-11
-            cmp: clang
-            configuration: default
-            name: "MacOS clang-13"
-
-          - os: macos-12
-            cmp: clang
-            configuration: default
-            name: "MacOS clang-13"
-
-          - os: windows-2019
-            cmp: gcc
-            configuration: default
-            name: "Win2019 MinGW"
-
-          - os: windows-2019
+          - os: windows-2022
             cmp: gcc
             configuration: static
-            name: "Win2019 MinGW, static"
-
-          - os: windows-2019
-            cmp: vs2019
-            configuration: default
-            name: "Win2019 MSC-19"
-
-          - os: windows-2019
-            cmp: vs2019
-            configuration: static
-            name: "Win2019 MSC-19, static"
+            name: "Win2022 MinGW, static"
 
           - os: windows-2022
             cmp: vs2022
@@ -159,8 +103,14 @@ jobs:
             configuration: static
             name: "Win2022 MSC-22, static"
 
+          - os: ubuntu-latest
+            cmp: gcc
+            configuration: default
+            name: "Cross linux-aarch64"
+            cross: linux-aarch64
+
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Automatic core dumper analysis
@@ -170,13 +120,6 @@ jobs:
         sudo apt-get update
         sudo apt-get -y install qemu-system-x86 g++-mingw-w64-x86-64 gdb
       if: runner.os == 'Linux'
-    - name: "apt-get install ${{ matrix.cmp }}"
-      run: |
-        sudo apt-get -y install software-properties-common
-        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-        sudo apt-get update
-        sudo apt-get -y install ${{ matrix.cmp }}
-      if: matrix.utoolchain
     - name: Prepare and compile dependencies
       run: python .ci/cue.py prepare
     - name: Build main module
@@ -184,44 +127,10 @@ jobs:
     - name: Run main module tests
       run: python .ci/cue.py -T 15M test
     - name: Upload tapfiles Artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: tapfiles ${{ matrix.name }}
         path: '**/O.*/*.tap'
     - name: Collect and show test results
-      run: python .ci/cue.py test-results
-
-  build-cross:
-    name: cross ${{ matrix.cross }} / ${{ matrix.cmp }} / ${{ matrix.configuration }}
-    runs-on: ubuntu-20.04
-    env:
-      CMP: gcc
-      BCFG: default
-      CI_CROSS_TARGETS: ${{ matrix.cross }}
-      APT: re2c
-    strategy:
-      fail-fast: false
-      matrix:
-        # configuration: [default, static, debug, static-debug]
-        configuration: [default]
-        cross:
-        - linux-aarch64
-        - linux-arm@arm-linux-gnueabi
-        - linux-arm@arm-linux-gnueabihf
-        - linux-ppc
-        - linux-ppc64
-        - win32-x86-mingw
-        - windows-x64-mingw
-        - RTEMS-pc386-qemu@4.9
-        - RTEMS-pc386-qemu@4.10
-        - RTEMS-pc686-qemu@5
-    steps:
-    - uses: actions/checkout@v3
-    - name: Prepare and compile dependencies
-      run: python cue.py prepare
-    - name: Build main module
-      run: python cue.py build
-    - name: Run main module tests
-      run: python cue.py -T 15M test
-    - name: Collect and show test results
-      run: python cue.py test-results
+      if: ${{ always() }}
+      run: python .ci/cue.py -T 5M test-results

github-actions/ci-scripts-build.yml.example-mini

@@ -31,44 +31,39 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        - os: ubuntu-18.04
-          cmp: gcc
-          configuration: default
-          base: "7.0"
-          name: "7.0 Ub-18 gcc-7"
-
-        - os: ubuntu-18.04
-          cmp: gcc
-          configuration: static
-          base: "7.0"
-          name: "7.0 Ub-18 gcc-7 static"
-
-        - os: ubuntu-20.04
+      include:
+        - os: ubuntu-24.04
           cmp: gcc
           configuration: default
           base: "3.15"
-          name: "3.15 Ub-20 gcc-9"
+          name: "3.15 Ub-24 gcc"
 
-        - os: ubuntu-20.04
+        - os: ubuntu-24.04
+          cmp: gcc
+          configuration: default
+          base: "7.0"
+          name: "7.0 Ub-24 gcc"
+
+        - os: ubuntu-24.04
           cmp: gcc
           configuration: static
           base: "3.15"
-          name: "3.15 Ub-20 gcc-9 static"
+          name: "3.15 Ub-24 gcc static"
 
-        - os: ubuntu-20.04
+        - os: ubuntu-24.04
           cmp: gcc
           configuration: debug
           base: "3.15"
-          name: "3.15 Ub-20 gcc-9 debug"
+          name: "3.15 Ub-24 gcc debug"
 
-        - os: ubuntu-20.04
+        - os: ubuntu-24.04
           cmp: clang
           configuration: default
           base: "3.15"
-          name: "3.15 Ub-20 clang-10"
+          name: "3.15 Ub-24 clang"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - name: Prepare and compile dependencies

test.patch

@@ -1,7 +1,7 @@
 diff -ruN a/bla.txt b/bla.txt
 --- a/bla.txt	2020-06-19 18:54:43.129076711 +0200
 +++ b/bla.txt	2020-06-19 18:55:05.093948316 +0200
-@@ -1,3 +1,3 @@
+@@ -1,2 +1,2 @@
 -LINE1=YES
 +LINE1=NO
  LINE2=NO