Merge pull request #69 from minijackson/extra-linux-targets

(closes #69)

.github/workflows/build-and-test.yml

@@ -103,53 +103,30 @@ jobs:
     - name: Collect and show test results
       run: python cue.py test-results
 
-  build-rtems:
+  build-cross:
-    name: RTEMS${{ matrix.rtems }} / ${{ matrix.rtems_target }}
+    name: cross ${{ matrix.cross }} / ${{ matrix.cmp }} / ${{ matrix.configuration }}
     runs-on: ubuntu-20.04
     env:
       CMP: gcc
       BCFG: default
-      RTEMS: ${{ matrix.rtems }}
+      CI_CROSS_TARGETS: ${{ matrix.cross }}
-      RTEMS_TARGET: ${{ matrix.rtems_target }}
+      APT: re2c
-      APT: re2c g++-mingw-w64-i686 g++-mingw-w64-x86-64 qemu-system-x86
     strategy:
       fail-fast: false
       matrix:
-        include:
+        # configuration: [default, static, debug, static-debug]
-          - rtems: "4.9"
+        configuration: [default]
-            rtems_target: RTEMS-pc386-qemu
+        cross: 
-
+        - linux-aarch64
-          - rtems: "4.10"
+        - linux-arm@arm-linux-gnueabi
-            rtems_target: RTEMS-pc386-qemu
+        - linux-arm@arm-linux-gnueabihf
-
+        - linux-ppc
-          - rtems: "5"
+        - linux-ppc64
-            rtems_target: RTEMS-pc686-qemu
+        - win32-x86-mingw
-    steps:
+        - windows-x64-mingw
-    - uses: actions/checkout@v2
+        - RTEMS-pc386-qemu@4.9
-    - name: Prepare and compile dependencies
+        - RTEMS-pc386-qemu@4.10
-      run: python cue.py prepare
+        - RTEMS-pc686-qemu@5
-    - name: Build main module (example app)
-      run: python cue.py build
-    - name: Run main module tests
-      run: python cue.py -T 15M test
-    - name: Collect and show test results
-      run: python cue.py test-results
-
-  build-wine:
-    name: WINE${{ matrix.wine }} / ${{ matrix.configuration }} / ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    env:
-      CMP: ${{ matrix.cmp }}
-      BCFG: ${{ matrix.configuration }}
-      WINE: ${{ matrix.wine }}
-      APT: re2c g++-mingw-w64-i686 g++-mingw-w64-x86-64
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-18.04]
-        cmp: [gcc]
-        configuration: [default, static, debug, static-debug]
-        wine: [32, 64]
     steps:
     - uses: actions/checkout@v2
     - name: Prepare and compile dependencies

README.md

@@ -310,6 +310,18 @@ RTEMS 5 builds need to be switched to a newer ubuntu version
 **dist: focal** with Travis-CI or
 **image: ubuntu:focal** with GitLab CI/CD).
 
+## LINUX_CROSS
+
+Setting the `LINUX_CROSS_<name>` environment variable enables cross-compiling
+to Linux for the provided target. The value of the environment variable should
+be the compiler prefix to be used.
+
+For example, to cross-compile for linux-arm with hard floating points
+
+- ensure you have the `g++-arm-linux-gnueabihf` package installed, if your CI
+  is on Ubuntu or Debian
+- set the environment variable `LINUX_CROSS_arm=arm-linux-gnueabihf`
+
 ## Debugging
 
 Setting `VV=1` in your service configuration (e.g., `.travis.yml`) for a

cue.py

@@ -786,6 +786,243 @@ def fix_etc_hosts():
     logger.debug('EXEC DONE')
 
 
+def edit_make_file(mode, path, values):
+    """Edit an EPICS Make file.
+
+    mode should be either "a" or "w", as for the open function.
+
+    path should be a list, e.g. ["configure", "CONFIG_SITE"]
+
+    values should be a dictionnary of values to edit. If the value starts with
+    a "+" the value will be appended.
+
+    Example usage:
+
+        edit_make_file("a", ["configure", "CONFIG_SITE"], {
+            "VARIABLE": "value",
+            "APPENDED_VARIABLE": "+value",
+        })
+    """
+    with open(os.path.join(places["EPICS_BASE"], *path), mode) as f:
+        for variable, value in values.items():
+            if value.startswith("+"):
+                op = "+="
+                value = value[1:]
+            else:
+                op = "="
+
+            f.write(variable + op + value + "\n")
+
+
+def handle_old_cross_variables():
+    if "CI_CROSS_TARGETS" not in os.environ:
+        os.environ["CI_CROSS_TARGETS"] = ""
+
+    if "RTEMS" in os.environ:
+        print(
+            "{0}WARNING: deprecated RTEMS environment variable was specified, please use CI_CROSS_TARGETS{1}".format(
+                ANSI_RED, ANSI_RESET
+            )
+        )
+        os.environ["CI_CROSS_TARGETS"] += ":RTEMS@" + os.environ["RTEMS"]
+
+    if "WINE" in os.environ:
+        print(
+            "{0}WARNING: deprecated WINE environment variable was specified, please use CI_CROSS_TARGETS{1}".format(
+                ANSI_RED, ANSI_RESET
+            )
+        )
+        os.environ["CI_CROSS_TARGETS"] += ":WINE@" + os.environ["WINE"]
+
+
+def prepare_cross_compilation(cross_target_info):
+    """Prepare the configuration for a single value of the CI_CROSS_TARGETS
+    variable.
+
+    See the README.md file for more information on this variable."""
+    cross_target_info = cross_target_info.split("@")
+    if len(cross_target_info) == 2:
+        target_param = cross_target_info[1]
+    else:
+        target_param = None
+
+    target = cross_target_info[0]
+
+    if target.startswith("RTEMS-"):
+        prepare_rtems_cross(target, target_param)
+    elif target.endswith("-mingw"):
+        prepare_wine_cross(target)
+    elif target.startswith("linux-"):
+        prepare_linux_cross(target, target_param)
+    else:
+        raise ValueError(
+            "Unknown CI_CROSS_TARGETS {0}. "
+            "Please see the ci-scripts README for available values.".format(target)
+        )
+
+
+def prepare_rtems_cross(epics_arch, version):
+    """Prepare the configuration for RTEMS cross-compilation for the given
+    RTEMS version.
+
+    If version is None, it defaults to version 5 for RTEMS-pc686-*, 4.10
+    otherwise."""
+    if version is None:
+        if epics_arch.startswith("RTEMS-pc686"):
+            version = "5"
+        else:
+            version = "4.10"
+
+    # eg. "RTEMS-pc386" or "RTEMS-pc386-qemu" -> "pc386"
+    rtems_bsp = re.match("^RTEMS-([^-]*)(?:-qemu)?$", epics_arch).group(1)
+
+    print("Cross compiler RTEMS{0} @ {1}".format(version, epics_arch))
+
+    if ci["os"] == "linux":
+        download_rtems(version, rtems_bsp)
+
+    edit_make_file(
+        "a",
+        ["configure", "os", "CONFIG_SITE.Common.RTEMS"],
+        {
+            "RTEMS_VERSION": version,
+            "RTEMS_BASE": "/opt/rtems/" + version,
+        },
+    )
+
+    edit_make_file(
+        "a",
+        ["configure", "CONFIG_SITE"],
+        {"CROSS_COMPILER_TARGET_ARCHS": epics_arch},
+    )
+
+    ci["apt"].extend(
+        ["re2c", "g++-mingw-w64-i686", "g++-mingw-w64-x86-64", "qemu-system-x86"]
+    )
+
+def download_rtems(version, rtems_bsp):
+    rsb_release = os.environ.get("RSB_BUILD", "20210306")
+    tar_name = "{0}-rtems{1}.tar.xz".format(rtems_bsp, version)
+    print("Downloading RTEMS {0} cross compiler: {1}".format(version, tar_name))
+    sys.stdout.flush()
+    sp.check_call(
+        [
+            "curl",
+            "-fsSL",
+            "--retry",
+            "3",
+            "-o",
+            tar_name,
+            "https://github.com/mdavidsaver/rsb/releases/download/{0}%2F{1}/{2}".format(
+                version, rsb_release, tar_name
+            ),
+        ],
+        cwd=toolsdir,
+    )
+    sudo_prefix = []
+    if ci["service"] == "github-actions":
+        sudo_prefix = ["sudo"]
+    sp.check_call(
+        sudo_prefix + ["tar", "-C", "/", "-xmJ", "-f", os.path.join(toolsdir, tar_name)]
+    )
+    os.remove(os.path.join(toolsdir, tar_name))
+    for rtems_cc in glob("/opt/rtems/*/bin/*-gcc"):
+        print("{0}{1} --version{2}".format(ANSI_CYAN, rtems_cc, ANSI_RESET))
+        sys.stdout.flush()
+        sp.check_call([rtems_cc, "--version"])
+
+
+def prepare_wine_cross(epics_arch):
+    """Prepare the configuration for Wine cross-compilation for the given mingw
+    architecture."""
+
+    if epics_arch == "win32-x86-mingw":
+        gnu_arch = "i686-w64-mingw32"
+        deb_arch = "mingw-w64-i686"
+        bits = "32"
+    elif epics_arch == "windows-x64-mingw":
+        gnu_arch = "x86_64-w64-mingw32"
+        deb_arch = "mingw-w64-x86-64"
+        bits = "64"
+    else:
+        raise ValueError(
+            "Unknown architecture '{0}' for WINE target. "
+            "Please see the ci-scripts README for available values.".format(epics_arch)
+        )
+
+    print("Cross compiler mingw{} / Wine".format(bits))
+
+    edit_make_file(
+        "a",
+        ["configure", "os", "CONFIG.linux-x86." + epics_arch],
+        {"CMPLR_PREFIX": gnu_arch + "-"},
+    )
+
+    edit_make_file(
+        "a",
+        ["configure", "CONFIG_SITE"],
+        {"CROSS_COMPILER_TARGET_ARCHS": "+" + epics_arch},
+    )
+
+    ci['apt'].extend(["re2c", "g++-" + deb_arch])
+
+
+def prepare_linux_cross(epics_arch, gnu_arch):
+    """Prepare the configuration for Linux cross-compilation for the given
+    architecture.
+
+    If gnu_arch is None, this function will try to guess it using the
+    epics_arch value.
+
+    linux-arm architecture defaults to arm-linux-gnueabi (soft floats)."""
+    # This list is kind of an intersection between the set of cross-compilers
+    # provided by Ubuntu[1] and the list of architectures found in
+    # `epics-base/configure/os`
+    #
+    # [1]: https://packages.ubuntu.com/source/focal/gcc-10-cross
+    if gnu_arch is None:
+        if epics_arch == "linux-x86":
+            gnu_arch = "i686-linux-gnu"
+        elif epics_arch == "linux-arm":
+            gnu_arch = "arm-linux-gnueabi"
+        elif epics_arch == "linux-aarch64":
+            gnu_arch = "aarch64-linux-gnu"
+        elif epics_arch == "linux-ppc":
+            gnu_arch = "powerpc-linux-gnu"
+        elif epics_arch == "linux-ppc64":
+            gnu_arch = "powerpc64le-linux-gnu"
+        else:
+            raise ValueError(
+                "Could not guess the GNU architecture for EPICS arch: {}. "
+                "Please use the '@' syntax of the 'CI_CROSS_TARGETS' variable".format(
+                    epics_arch
+                )
+            )
+
+    print(
+        "Setting up Linux cross-compiling arch {0} with GNU arch {1}".format(
+            epics_arch, gnu_arch
+        )
+    )
+
+    edit_make_file(
+        "w",
+        ["configure", "os", "CONFIG_SITE.linux-x86_64." + epics_arch],
+        {
+            "GNU_TARGET": gnu_arch,
+            "COMMANDLINE_LIBRARY": "EPICS",
+        },
+    )
+
+    edit_make_file(
+        "a",
+        ["configure", "CONFIG_SITE"],
+        {"CROSS_COMPILER_TARGET_ARCHS": "+" + epics_arch},
+    )
+
+    ci["apt"].extend(["re2c", "g++-" + gnu_arch])
+
+
 def prepare(args):
     host_info()
 
@@ -831,20 +1068,8 @@ def prepare(args):
     elif ci['compiler'].startswith('gcc'):
         cxx = re.sub(r'gcc', r'g++', ci['compiler'])
 
-    # Cross compilation on Linux to RTEMS  (set RTEMS to version "4.9", "4.10")
+    if not os.path.isdir(toolsdir):
-    # requires qemu, bison, flex, texinfo, install-info
+        os.makedirs(toolsdir)
-    # rtems_bsp is needed also if Base is from cache
-    if 'RTEMS' in os.environ:
-        if 'RTEMS_TARGET' in os.environ:
-            rtems_target = os.environ['RTEMS_TARGET']
-        elif os.path.exists(os.path.join(places['EPICS_BASE'], 'configure', 'os',
-                                         'CONFIG.Common.RTEMS-pc386-qemu')):
-            # Base 3.15 doesn't have -qemu target architecture
-            rtems_target = 'RTEMS-pc386-qemu'
-        else:
-            rtems_target = 'RTEMS-pc386'
-        # eg. "RTEMS-pc386" or "RTEMS-pc386-qemu" -> "pc386"
-        rtems_bsp = re.match('^RTEMS-([^-]*)(?:-qemu)?$', rtems_target).group(1)
 
     if 'BASE' in modules_to_compile or building_base:
         fold_start('set.up.epics_build', 'Configuring EPICS build system')
@@ -895,44 +1120,12 @@ endif''')
 
         # Cross-compilations from Linux platform
         if ci['os'] == 'linux':
+            handle_old_cross_variables()
 
-            # Cross compilation to Windows/Wine (set WINE to architecture "32", "64")
+            for cross_target_info in os.environ.get("CI_CROSS_TARGETS", "").split(":"):
-            # requires wine and g++-mingw-w64-i686 / g++-mingw-w64-x86-64
+                if cross_target_info == "":
-            if 'WINE' in os.environ:
+                    continue
-                if os.environ['WINE'] == '32':
+                prepare_cross_compilation(cross_target_info)
-                    print('Cross compiler mingw32 / Wine')
-                    with open(os.path.join(places['EPICS_BASE'], 'configure', 'os',
-                                                 'CONFIG.linux-x86.win32-x86-mingw'), 'a') as f:
-                        f.write('''
-CMPLR_PREFIX=i686-w64-mingw32-''')
-                    with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f:
-                        f.write('''
-CROSS_COMPILER_TARGET_ARCHS+=win32-x86-mingw''')
-
-                if os.environ['WINE'] == '64':
-                    print('Cross compiler mingw64 / Wine')
-                    with open(os.path.join(places['EPICS_BASE'], 'configure', 'os',
-                                           'CONFIG.linux-x86.windows-x64-mingw'), 'a') as f:
-                        f.write('''
-CMPLR_PREFIX=x86_64-w64-mingw32-''')
-                    with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f:
-                        f.write('''
-CROSS_COMPILER_TARGET_ARCHS += windows-x64-mingw''')
-
-            # Cross compilation on Linux to RTEMS  (set RTEMS to version "4.9", "4.10")
-            # requires qemu, bison, flex, texinfo, install-info
-            if 'RTEMS' in os.environ:
-                print('Cross compiler RTEMS{0} @ {1}'.format(os.environ['RTEMS'], rtems_target))
-                with open(os.path.join(places['EPICS_BASE'], 'configure', 'os',
-                                       'CONFIG_SITE.Common.RTEMS'), 'a') as f:
-                    f.write('''
-RTEMS_VERSION={0}
-RTEMS_BASE=/opt/rtems/{0}'''.format(os.environ['RTEMS']))
-
-                with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f:
-                    f.write('''
-CROSS_COMPILER_TARGET_ARCHS += {0}
-'''.format(rtems_target))
 
         print('Host compiler', ci['compiler'])
 
@@ -984,9 +1177,6 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
 
         fold_end('set.up.epics_build', 'Configuring EPICS build system')
 
-    if not os.path.isdir(toolsdir):
-        os.makedirs(toolsdir)
-
     if ci['os'] == 'windows' and ci['choco']:
         fold_start('install.choco', 'Installing CHOCO packages')
         sp.check_call(['choco', 'install'] + ci['choco'] + ['-y', '--limitoutput', '--no-progress'])
@@ -1003,26 +1193,6 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
         sp.check_call(['brew', 'install'] + ci['homebrew'])
         fold_end('install.homebrew', 'Installing Homebrew packages')
 
-    if ci['os'] == 'linux' and 'RTEMS' in os.environ:
-        rsb_release = os.environ.get('RSB_BUILD', '20210306')
-        tar_name = '{0}-rtems{1}.tar.xz'.format(rtems_bsp, os.environ['RTEMS'])
-        print('Downloading RTEMS {0} cross compiler: {1}'
-              .format(os.environ['RTEMS'], tar_name))
-        sys.stdout.flush()
-        sp.check_call(['curl', '-fsSL', '--retry', '3', '-o', tar_name,
-                       'https://github.com/mdavidsaver/rsb/releases/download/{0}%2F{1}/{2}'
-                      .format(os.environ['RTEMS'], rsb_release, tar_name)],
-                      cwd=toolsdir)
-        sudo_prefix = []
-        if ci['service'] == 'github-actions':
-            sudo_prefix = ['sudo']
-        sp.check_call(sudo_prefix + ['tar', '-C', '/', '-xmJ', '-f', os.path.join(toolsdir, tar_name)])
-        os.remove(os.path.join(toolsdir, tar_name))
-        for rtems_cc in glob('/opt/rtems/*/bin/*-gcc'):
-            print('{0}{1} --version{2}'.format(ANSI_CYAN, rtems_cc, ANSI_RESET))
-            sys.stdout.flush()
-            sp.check_call([rtems_cc, '--version'])
-
     setup_for_build(args)
 
     print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET))