diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b32ca39 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,160 @@ +# .gitlab-ci.yml for testing EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +# Note: +# Paths to scripts are different in this test configuration +# (your module has one more directory level: .ci) + +image: ubuntu:bionic + +cache: + key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + paths: + - .cache/ + +variables: + GIT_SUBMODULE_STRATEGY: "recursive" + SETUP_PATH: ".:.ci" + BASE_RECURSIVE: "NO" + APT: "libreadline6-dev libncurses5-dev perl clang g++-mingw-w64-i686 g++-mingw-w64-x86-64 qemu-system-x86 re2c" + CMP: "gcc" + BGFC: "default" + VV: "1" + SET: test01 + +# Unit tests +unit_test: + stage: build + variables: + SET: test00 + before_script: + - apt-get update -qq && apt-get install -y -qq make git python curl p7zip-full + - python cue-test.py env + script: + - python cue-test.py + +# Template job for test builds (hidden) +.build: + stage: build + before_script: + - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full + - python cue.py prepare + script: + - python cue.py build + - python cue.py test + - python cue.py test-results + +# gcc builds using four configurations (shared/static, optimized/debug) +gcc_default: + extends: .build + +gcc_static: + extends: .build + variables: + BCFG: "static" + +gcc_debug: + extends: .build + variables: + BCFG: "debug" + +gcc_static_debug: + extends: .build + variables: + BCFG: "static-debug" + +# clang builds using four configurations (shared/static, optimized/debug) +clang_default: + extends: .build + variables: + CMP: "clang" + +clang_static: + extends: .build + variables: + CMP: "clang" + BCFG: "static" + +clang_debug: + extends: .build + variables: + CMP: "clang" + BCFG: "debug" + +clang_static_debug: + extends: .build + variables: + CMP: "clang" + BCFG: "static-debug" + +# WINE based cross-builds to Windows 32bit and 64bit +wine32_default: + extends: .build + variables: + WINE: "32" + +wine32_static: + extends: .build + variables: + WINE: "32" + BCFG: "static" + +wine32_debug: + extends: .build + variables: + WINE: "32" + BCFG: "debug" + +wine32_static_debug: + extends: .build + variables: + WINE: "32" + BCFG: "static-debug" + +wine64_default: + extends: .build + variables: + WINE: "64" + +wine64_static: + extends: .build + variables: + WINE: "64" + BCFG: "static" + +wine64_debug: + extends: .build + variables: + WINE: "64" + BCFG: "debug" + +wine64_static_debug: + extends: .build + variables: + WINE: "64" + BCFG: "static-debug" + +# Cross-builds to RTEMS 4.9 and 4.10 +rtems4.9_default: + extends: .build + variables: + RTEMS: "4.9" + BASE: "3.15" + +rtems4.9_debug: + extends: .build + variables: + RTEMS: "4.9" + BASE: "3.15" + BCFG: "debug" + +rtems4.10_default: + extends: .build + variables: + RTEMS: "4.10" + +rtems4.10_debug: + extends: .build + variables: + RTEMS: "4.10" + BCFG: "debug" diff --git a/README.md b/README.md index d193352..3c50b8b 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,20 @@ See specific **[ci-scripts on GitHub Actions README](github-actions/README.md)** for more details. +### [GitLab CI/CD](https://gitlab.com/) + + - Docker-based runners on Linux (one VM instance per job) + - Can use any Docker image from Dockerhub (the examples use + `ubuntu:bionic`) + - Compile natively using different compilers (gcc, clang) + - Cross-compile for Windows 32bit and 64bit using MinGW and WINE + - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15) + - Built dependencies are cached (for faster builds). + +See specific +**[ci-scripts on GitLab CI/CD README](gitlab/README.md)** +for more details. + ## How to Use the CI-Scripts 1. Get an account on a supported CI service provider platform diff --git a/cue-test.py b/cue-test.py index 809853b..562a6bd 100644 --- a/cue-test.py +++ b/cue-test.py @@ -22,6 +22,10 @@ if 'TRAVIS' in os.environ: ci_service = 'travis' ci_os = os.environ['TRAVIS_OS_NAME'] +if 'GITLAB_CI' in os.environ: + ci_service = 'gitlab' + ci_os = 'linux' + if 'APPVEYOR' in os.environ: ci_service = 'appveyor' if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): @@ -118,7 +122,7 @@ class TestSourceSet(unittest.TestCase): class TestUpdateReleaseLocal(unittest.TestCase): - release_local = os.path.join(cue.cachedir, 'RELEASE.local') + release_local = os.path.join(cue.ci['cachedir'], 'RELEASE.local') def setUp(self): if os.path.exists(self.release_local): @@ -186,16 +190,21 @@ class TestUpdateReleaseLocal(unittest.TestCase): class TestAddDependencyUpToDateCheck(unittest.TestCase): hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" - location = os.path.join(cue.cachedir, 'base-R3.15.6') - licensefile = os.path.join(location, 'LICENSE') - checked_file = os.path.join(location, 'checked_out') - release_file = os.path.join(location, 'configure', 'RELEASE') + location = '' + licensefile = '' + checked_file = '' + release_file = '' def setUp(self): os.environ['SETUP_PATH'] = '.:appveyor' + cue.clear_lists() + cue.detect_context() + self.location = os.path.join(cue.ci['cachedir'], 'base-R3.15.6') + self.licensefile = os.path.join(self.location, 'LICENSE') + self.checked_file = os.path.join(self.location, 'checked_out') + self.release_file = os.path.join(self.location, 'configure', 'RELEASE') if os.path.exists(self.location): shutil.rmtree(self.location, onerror=cue.remove_readonly) - cue.clear_lists() os.chdir(builddir) cue.source_set('defaults') cue.complete_setup('BASE') @@ -249,15 +258,18 @@ def is_shallow_repo(place): class TestAddDependencyOptions(unittest.TestCase): - location = os.path.join(cue.cachedir, 'mcoreutils-master') - testfile = os.path.join(location, '.ci', 'LICENSE') + location = '' + testfile = '' def setUp(self): os.environ['SETUP_PATH'] = '.' - if os.path.exists(cue.cachedir): - shutil.rmtree(cue.cachedir, onerror=cue.remove_readonly) cue.clear_lists() cue.detect_context() + if os.path.exists(cue.ci['cachedir']): + shutil.rmtree(cue.ci['cachedir'], onerror=cue.remove_readonly) + self.location = os.path.join(cue.ci['cachedir'], 'mcoreutils-master') + self.testfile = os.path.join(self.location, '.ci', 'LICENSE') + os.path.join(cue.ci['cachedir'], 'mcoreutils-master') cue.source_set('defaults') cue.complete_setup('MCoreUtils') cue.setup['MCoreUtils'] = 'master' @@ -288,7 +300,7 @@ class TestAddDependencyOptions(unittest.TestCase): def test_AddMsiTo314(self): cue.complete_setup('BASE') cue.setup['BASE'] = 'R3.14.12.1' - msifile = os.path.join(cue.cachedir, 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c') + msifile = os.path.join(cue.ci['cachedir'], 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c') cue.add_dependency('BASE') self.assertTrue(os.path.exists(msifile), 'MSI was not added to Base 3.14') @@ -834,7 +846,7 @@ class TestSetupForBuild(unittest.TestCase): class TestHooks(unittest.TestCase): - location = os.path.join(cue.cachedir, 'hook_test') + location = os.path.join(cue.ci['cachedir'], 'hook_test') bla_file = os.path.join(location, 'bla.txt') new_file = os.path.join(location, 'dd', 'new.txt') diff --git a/cue.py b/cue.py index 5c7475c..48b30b6 100644 --- a/cue.py +++ b/cue.py @@ -16,7 +16,11 @@ logger = logging.getLogger(__name__) # Detect the service and set up context hash accordingly def detect_context(): + global homedir + buildconfig = 'default' + ci['cachedir'] = os.path.join(homedir, '.cache') + if 'TRAVIS' in os.environ: ci['service'] = 'travis' ci['os'] = os.environ['TRAVIS_OS_NAME'] @@ -29,6 +33,17 @@ def detect_context(): if 'BCFG' in os.environ: buildconfig = os.environ['BCFG'].lower() + if 'GITLAB_CI' in os.environ: + ci['service'] = 'gitlab' + ci['os'] = 'linux' + ci['platform'] = 'x64' + ci['sudo'] = [] # No sudo in GitLab Docker containers + ci['cachedir'] = os.path.join(curdir, '.cache') # No caches outside project directory + if 'CMP' in os.environ: + ci['compiler'] = os.environ['CMP'] + if 'BCFG' in os.environ: + buildconfig = os.environ['BCFG'].lower() + if 'APPVEYOR' in os.environ: ci['service'] = 'appveyor' if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): @@ -80,6 +95,9 @@ def detect_context(): ci['scriptsdir'] = os.path.abspath(os.path.dirname(sys.argv[0])) + if 'CACHEDIR' in os.environ: + ci['cachedir'] = os.environ['CACHEDIR'] + if 'CHOCO' in os.environ: ci['choco'].extend(os.environ['CHOCO'].split()) @@ -144,9 +162,11 @@ def clear_lists(): ci['debug'] = False ci['configuration'] = '' ci['scriptsdir'] = '' + ci['cachedir'] = '' ci['choco'] = ['make'] ci['apt'] = [] ci['homebrew'] = [] + ci['sudo'] = ['sudo'] clear_lists() @@ -204,13 +224,9 @@ if 'HomeDrive' in os.environ: homedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath')) elif 'HOME' in os.environ: homedir = os.getenv('HOME') -cachedir = os.path.join(homedir, '.cache') toolsdir = os.path.join(homedir, '.tools') rtemsdir = r'/home/travis/.rtems' # Preliminary, until the next generation of toolchain -if 'CACHEDIR' in os.environ: - cachedir = os.environ['CACHEDIR'] - vcvars_table = { # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History @@ -322,14 +338,14 @@ def source_set(name): # - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end # Set places[var] = location def update_release_local(var, location): - release_local = os.path.join(cachedir, 'RELEASE.local') + release_local = os.path.join(ci['cachedir'], 'RELEASE.local') updated_line = '{0}={1}'.format(var, location.replace('\\', '/')) places[var] = location if not os.path.exists(release_local): logger.debug('RELEASE.local does not exist, creating it') try: - os.makedirs(cachedir) + os.makedirs(ci['cachedir']) except: pass touch = open(release_local, 'w') @@ -484,7 +500,7 @@ def add_dependency(dep): .format(ANSI_RED, tag, dep, setup[dep + '_REPOURL'], ANSI_RESET)) dirname = setup[dep + '_DIRNAME'] + '-{0}'.format(tag) - place = os.path.join(cachedir, dirname) + place = os.path.join(ci['cachedir'], dirname) checked_file = os.path.join(place, "checked_out") if os.path.isdir(place): @@ -506,14 +522,14 @@ def add_dependency(dep): sys.stdout.flush() if not os.path.isdir(place): - if not os.path.isdir(cachedir): - os.makedirs(cachedir) + if not os.path.isdir(ci['cachedir']): + os.makedirs(ci['cachedir']) # clone dependency print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) sys.stdout.flush() call_git(['clone', '--quiet'] + deptharg + recursearg + ['--branch', tag, setup[dep + '_REPOURL'], dirname], - cwd=cachedir) + cwd=ci['cachedir']) sp.check_call(['git', 'log', '-n1'], cwd=place) logger.debug('Setting do_recompile = True (all following modules will be recompiled') @@ -634,7 +650,7 @@ def setup_for_build(args): # Find BASE location if not building_base: - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f: lines = f.readlines() for line in lines: (mod, place) = line.strip().split('=') @@ -649,7 +665,7 @@ def setup_for_build(args): if ci['os'] == 'windows': if not building_base: - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f: lines = f.readlines() for line in lines: (mod, place) = line.strip().split('=') @@ -753,7 +769,7 @@ def prepare(args): targetdir = 'configure' else: targetdir = '.' - shutil.copy(os.path.join(cachedir, 'RELEASE.local'), targetdir) + shutil.copy(os.path.join(ci['cachedir'], 'RELEASE.local'), targetdir) fold_end('check.out.dependencies', 'Checking/cloning dependencies') @@ -909,8 +925,8 @@ PERL = C:/Strawberry/perl/bin/perl -CSD''' if ci['os'] == 'linux' and ci['apt']: fold_start('install.apt', 'Installing APT packages') - sp.check_call(['sudo', 'apt-get', '-y', 'update']) - sp.check_call(['sudo', 'apt-get', '-y', 'install'] + ci['apt']) + sp.check_call(ci['sudo'] + ['apt-get', '-y', 'update']) + sp.check_call(ci['sudo'] + ['apt-get', 'install', '-y', '-qq'] + ci['apt']) fold_end('install.apt', 'Installing APT packages') if ci['os'] == 'osx' and ci['homebrew']: @@ -976,7 +992,7 @@ PERL = C:/Strawberry/perl/bin/perl -CSD''' print("%-10s %-12s %-11s %s" % (mod, setup[mod], stat, commit)) print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET)) - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f: print(f.read().strip()) diff --git a/gitlab/.gitlab-ci.yml.example-full b/gitlab/.gitlab-ci.yml.example-full new file mode 100644 index 0000000..25c19ca --- /dev/null +++ b/gitlab/.gitlab-ci.yml.example-full @@ -0,0 +1,114 @@ +# .gitlab-ci.yml for testing EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +# This is YAML - indentation levels are crucial + +# GitLab runner can use any Docker container, we're using this one +# to be comparable with the other CI services +image: ubuntu:bionic + +cache: + key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" + paths: + - .cache/ + +variables: + GIT_SUBMODULE_STRATEGY: "recursive" + SETUP_PATH: ".ci-local:.ci" + BASE_RECURSIVE: "NO" + # Additional packages needed for + # | EPICS |clang| Windows cross builds |RTEMS testing |sequencer + APT: "libreadline6-dev libncurses5-dev perl clang g++-mingw-w64-i686 g++-mingw-w64-x86-64 qemu-system-x86 re2c" + CMP: "gcc" + BGFC: "default" + +# Template for build jobs (hidden) +.build: + stage: build + before_script: + - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full + - python .ci/cue.py prepare + script: + - python .ci/cue.py build + - python .ci/cue.py test + - python .ci/cue.py test-results + +# If you need to do more during install and build, +# add a local directory to your module and do e.g. +# - ./.ci-local/travis/install-extras.sh + +# Define build jobs + +# Well-known variables to use +# SET source setup file +# ADD_MODULES extra modules (for a specific job) +# BCFG build configuration (static/debug/static-debug; +# default: shared-optimized) +# TEST set to NO to skip running the tests (default: YES) +# VV set to make build scripts verbose (default: unset) +# EXTRA content will be added to make command line +# EXTRA1..5 more additional arguments for the make command +# (one argument per variable) + +# Usually from setup files, but may be specified or overridden +# on a job line +# MODULES list of dependency modules +# BASE branch or release tag name of the EPICS Base to use +# branch or release tag for a specific module +# ... see README for setup file syntax description + +# Different configurations of gcc and clang +gcc_default: + extends: .build + variables: + BASE: "7.0" + +gcc_static: + extends: .build + variables: + BASE: "7.0" + BCFG: "static" + +clang_default: + extends: .build + variables: + BASE: "7.0" + CMP: "clang" + +clang_static_c++11: + extends: .build + variables: + BASE: "7.0" + CMP: "clang" + BCFG: "static" + EXTRA: "CMD_CXXFLAGS=-std=c++11" + +# Cross-compilations to Windows using gcc/MinGW and WINE +wine32_default: + extends: .build + variables: + BASE: "3.15" + BCFG: "static" + WINE: "32" + TEST: "NO" + +wine64_debug: + extends: .build + variables: + BASE: "7.0" + BCFG: "debug" + WINE: "64" + +# Cross-builds to RTEMS 4.9 and 4.10 +rtems4.9_default: + extends: .build + variables: + RTEMS: "4.9" + BASE: "3.15" + +rtems4.10_debug: + extends: .build + variables: + RTEMS: "4.10" + BASE: "7.0" + BCFG: "debug" diff --git a/gitlab/.gitlab-ci.yml.example-mini b/gitlab/.gitlab-ci.yml.example-mini new file mode 100644 index 0000000..e24e82f --- /dev/null +++ b/gitlab/.gitlab-ci.yml.example-mini @@ -0,0 +1,36 @@ +# .gitlab-ci.yml for testing EPICS Base ci-scripts +# (see: https://github.com/epics-base/ci-scripts) + +image: ubuntu:bionic + +variables: + GIT_SUBMODULE_STRATEGY: "recursive" + SETUP_PATH: ".ci-local:.ci" + BASE_RECURSIVE: "NO" + # Minimal set of packages needed to compile EPICS Base + APT: "libreadline6-dev libncurses5-dev perl" + CMP: "gcc" + BGFC: "default" + +# Template for build jobs (hidden) +.build: + stage: build + before_script: + - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full + - python .ci/cue.py prepare + script: + - python .ci/cue.py build + - python .ci/cue.py test + - python .ci/cue.py test-results + +# Build on Linux using default gcc for Base branches 7.0 and 3.15 + +gcc_base_7_0: + extends: .build + variables: + BASE: "7.0" + +gcc_base_3_15: + extends: .build + variables: + BASE: "3.15" diff --git a/gitlab/README.md b/gitlab/README.md new file mode 100644 index 0000000..72ac83f --- /dev/null +++ b/gitlab/README.md @@ -0,0 +1,68 @@ +# GitLab CI/CD Scripts for EPICS Modules + +## Features + + - Docker-based runners on Linux (one VM instance per job) + - Can use any Docker image from Dockerhub (the examples use + `ubuntu:bionic`) + - Compile natively using different compilers (gcc, clang) + - Cross-compile for Windows 32bit and 64bit using MinGW and WINE + - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15) + - Built dependencies are cached (for faster builds). + +## How to Use these Scripts + + 1. Get an account on [GitLab](https://gitlab.com/), create a project + for your support module and have it mirror your upstream GitHub + repository. For more details, please refer to the + [GitLab CI/CD documentation](https://docs.gitlab.com/ee/README.html). + + (This applies when using the free tier offered to open source + projects. Things will be different using an "Enterprise" + installation on customer hardware.) + + 2. Add the ci-scripts respository as a Git Submodule + (see [README](../README.md) one level above). + + 3. Add settings files defining which dependencies in which versions + you want to build against + (see [README](../README.md) one level above). + + 4. Create a GitLab configuration by copying one of the examples into + the root directory of your module. + ``` + $ cp .ci/gitlab/.gitlab-ci.yml.example-full .gitlab-ci.yml + ``` + + 5. Edit the `.gitlab-ci.yml` configuration to include the jobs you want + GitLab CI/CD to run. + + Build jobs are declared in the list at the end of the file. + Each element (starting with the un-indented line) defines the + settings for one build job. `extends:` specifies a template to use as + a default structure, `variables:` controls the setting of environment + variables (overwriting settings from the template). + Also see the comments in the examples for more hints, and the + [GitLab CI/CD documentation](https://docs.gitlab.com/ee/README.html) + for more options and details. + + 6. Push your changes to GitHub, wait for the synchronization (every 5min) + and check [GitLab](https://gitlab.com/) for your build results. + +## Caches + +GitLab is configured to keep the caches separate for different jobs. + +However, changing the job description (in the `.gitlab-ci.yml` +configuration file) or its environment settings or changing a value +inside a setup file will _not_ invalidate the cache - you will +have to manually delete the caches through the GitLab web interface. + +Caches are automatically removed after approx. four weeks. +Your jobs will have to rebuild them once in a while. + +## Miscellanea + +To use the feature to extract `.zip`/`.7z` archives by setting +`*_HOOK` variables, the Linux and MacOS runners need the APT package +`p7zip-full` resp. the Homebrew package `p7zip` installed.