Merge pull request #55 from ralphlange/devel/gitlab

Support for GitLab CI/CD
(closes #51)

.gitlab-ci.yml

@@ -0,0 +1,160 @@
+# .gitlab-ci.yml for testing EPICS Base ci-scripts
+# (see: https://github.com/epics-base/ci-scripts)
+
+# Note:
+# Paths to scripts are different in this test configuration
+# (your module has one more directory level: .ci)
+
+image: ubuntu:bionic
+
+cache:
+  key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
+  paths:
+    - .cache/
+
+variables:
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  SETUP_PATH: ".:.ci"
+  BASE_RECURSIVE: "NO"
+  APT: "libreadline6-dev libncurses5-dev perl clang g++-mingw-w64-i686 g++-mingw-w64-x86-64 qemu-system-x86 re2c"
+  CMP: "gcc"
+  BGFC: "default"
+  VV: "1"
+  SET: test01
+
+# Unit tests
+unit_test:
+  stage: build
+  variables:
+    SET: test00
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq make git python curl p7zip-full
+    - python cue-test.py env
+  script:
+    - python cue-test.py
+
+# Template job for test builds (hidden)
+.build:
+  stage: build
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full
+    - python cue.py prepare
+  script:
+    - python cue.py build
+    - python cue.py test
+    - python cue.py test-results
+
+# gcc builds using four configurations (shared/static, optimized/debug)
+gcc_default:
+  extends: .build
+
+gcc_static:
+  extends: .build
+  variables:
+    BCFG: "static"
+
+gcc_debug:
+  extends: .build
+  variables:
+    BCFG: "debug"
+
+gcc_static_debug:
+  extends: .build
+  variables:
+    BCFG: "static-debug"
+
+# clang builds using four configurations (shared/static, optimized/debug)
+clang_default:
+  extends: .build
+  variables:
+    CMP: "clang"
+
+clang_static:
+  extends: .build
+  variables:
+    CMP: "clang"
+    BCFG: "static"
+
+clang_debug:
+  extends: .build
+  variables:
+    CMP: "clang"
+    BCFG: "debug"
+
+clang_static_debug:
+  extends: .build
+  variables:
+    CMP: "clang"
+    BCFG: "static-debug"
+
+# WINE based cross-builds to Windows 32bit and 64bit
+wine32_default:
+  extends: .build
+  variables:
+    WINE: "32"
+
+wine32_static:
+  extends: .build
+  variables:
+    WINE: "32"
+    BCFG: "static"
+
+wine32_debug:
+  extends: .build
+  variables:
+    WINE: "32"
+    BCFG: "debug"
+
+wine32_static_debug:
+  extends: .build
+  variables:
+    WINE: "32"
+    BCFG: "static-debug"
+
+wine64_default:
+  extends: .build
+  variables:
+    WINE: "64"
+
+wine64_static:
+  extends: .build
+  variables:
+    WINE: "64"
+    BCFG: "static"
+
+wine64_debug:
+  extends: .build
+  variables:
+    WINE: "64"
+    BCFG: "debug"
+
+wine64_static_debug:
+  extends: .build
+  variables:
+    WINE: "64"
+    BCFG: "static-debug"
+
+# Cross-builds to RTEMS 4.9 and 4.10
+rtems4.9_default:
+  extends: .build
+  variables:
+    RTEMS: "4.9"
+    BASE: "3.15"
+
+rtems4.9_debug:
+  extends: .build
+  variables:
+    RTEMS: "4.9"
+    BASE: "3.15"
+    BCFG: "debug"
+
+rtems4.10_default:
+  extends: .build
+  variables:
+    RTEMS: "4.10"
+
+rtems4.10_debug:
+  extends: .build
+  variables:
+    RTEMS: "4.10"
+    BCFG: "debug"

README.md

@@ -105,6 +105,20 @@ See specific
 **[ci-scripts on GitHub Actions README](github-actions/README.md)**
 for more details.
 
+### [GitLab CI/CD](https://gitlab.com/)
+
+ - Docker-based runners on Linux (one VM instance per job)
+ - Can use any Docker image from Dockerhub (the examples use
+  `ubuntu:bionic`)
+ - Compile natively using different compilers (gcc, clang)
+ - Cross-compile for Windows 32bit and 64bit using MinGW and WINE
+ - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15)
+ - Built dependencies are cached (for faster builds).
+
+See specific
+**[ci-scripts on GitLab CI/CD README](gitlab/README.md)**
+for more details.
+
 ## How to Use the CI-Scripts
 
  1. Get an account on a supported CI service provider platform

cue-test.py

@@ -22,6 +22,10 @@ if 'TRAVIS' in os.environ:
     ci_service = 'travis'
     ci_os = os.environ['TRAVIS_OS_NAME']
 
+if 'GITLAB_CI' in os.environ:
+    ci_service = 'gitlab'
+    ci_os = 'linux'
+
 if 'APPVEYOR' in os.environ:
     ci_service = 'appveyor'
     if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']):
@@ -118,7 +122,7 @@ class TestSourceSet(unittest.TestCase):
 
 
 class TestUpdateReleaseLocal(unittest.TestCase):
-    release_local = os.path.join(cue.cachedir, 'RELEASE.local')
+    release_local = os.path.join(cue.ci['cachedir'], 'RELEASE.local')
 
     def setUp(self):
         if os.path.exists(self.release_local):
@@ -186,16 +190,21 @@ class TestUpdateReleaseLocal(unittest.TestCase):
 
 class TestAddDependencyUpToDateCheck(unittest.TestCase):
     hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096"
-    location = os.path.join(cue.cachedir, 'base-R3.15.6')
+    location = ''
-    licensefile = os.path.join(location, 'LICENSE')
+    licensefile = ''
-    checked_file = os.path.join(location, 'checked_out')
+    checked_file = ''
-    release_file = os.path.join(location, 'configure', 'RELEASE')
+    release_file = ''
 
     def setUp(self):
         os.environ['SETUP_PATH'] = '.:appveyor'
+        cue.clear_lists()
+        cue.detect_context()
+        self.location = os.path.join(cue.ci['cachedir'], 'base-R3.15.6')
+        self.licensefile = os.path.join(self.location, 'LICENSE')
+        self.checked_file = os.path.join(self.location, 'checked_out')
+        self.release_file = os.path.join(self.location, 'configure', 'RELEASE')
         if os.path.exists(self.location):
             shutil.rmtree(self.location, onerror=cue.remove_readonly)
-        cue.clear_lists()
         os.chdir(builddir)
         cue.source_set('defaults')
         cue.complete_setup('BASE')
@@ -249,15 +258,18 @@ def is_shallow_repo(place):
 
 
 class TestAddDependencyOptions(unittest.TestCase):
-    location = os.path.join(cue.cachedir, 'mcoreutils-master')
+    location = ''
-    testfile = os.path.join(location, '.ci', 'LICENSE')
+    testfile = ''
 
     def setUp(self):
         os.environ['SETUP_PATH'] = '.'
-        if os.path.exists(cue.cachedir):
-            shutil.rmtree(cue.cachedir, onerror=cue.remove_readonly)
         cue.clear_lists()
         cue.detect_context()
+        if os.path.exists(cue.ci['cachedir']):
+            shutil.rmtree(cue.ci['cachedir'], onerror=cue.remove_readonly)
+        self.location = os.path.join(cue.ci['cachedir'], 'mcoreutils-master')
+        self.testfile = os.path.join(self.location, '.ci', 'LICENSE')
+        os.path.join(cue.ci['cachedir'], 'mcoreutils-master')
         cue.source_set('defaults')
         cue.complete_setup('MCoreUtils')
         cue.setup['MCoreUtils'] = 'master'
@@ -288,7 +300,7 @@ class TestAddDependencyOptions(unittest.TestCase):
     def test_AddMsiTo314(self):
         cue.complete_setup('BASE')
         cue.setup['BASE'] = 'R3.14.12.1'
-        msifile = os.path.join(cue.cachedir, 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c')
+        msifile = os.path.join(cue.ci['cachedir'], 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c')
         cue.add_dependency('BASE')
         self.assertTrue(os.path.exists(msifile), 'MSI was not added to Base 3.14')
 
@@ -834,7 +846,7 @@ class TestSetupForBuild(unittest.TestCase):
 
 
 class TestHooks(unittest.TestCase):
-    location = os.path.join(cue.cachedir, 'hook_test')
+    location = os.path.join(cue.ci['cachedir'], 'hook_test')
     bla_file = os.path.join(location, 'bla.txt')
     new_file = os.path.join(location, 'dd', 'new.txt')
 

cue.py

@@ -16,7 +16,11 @@ logger = logging.getLogger(__name__)
 
 # Detect the service and set up context hash accordingly
 def detect_context():
+    global homedir
+
     buildconfig = 'default'
+    ci['cachedir'] = os.path.join(homedir, '.cache')
+
     if 'TRAVIS' in os.environ:
         ci['service'] = 'travis'
         ci['os'] = os.environ['TRAVIS_OS_NAME']
@@ -29,6 +33,17 @@ def detect_context():
         if 'BCFG' in os.environ:
             buildconfig = os.environ['BCFG'].lower()
 
+    if 'GITLAB_CI' in os.environ:
+        ci['service'] = 'gitlab'
+        ci['os'] = 'linux'
+        ci['platform'] = 'x64'
+        ci['sudo'] = []                                      # No sudo in GitLab Docker containers
+        ci['cachedir'] = os.path.join(curdir, '.cache')    # No caches outside project directory
+        if 'CMP' in os.environ:
+            ci['compiler'] = os.environ['CMP']
+        if 'BCFG' in os.environ:
+            buildconfig = os.environ['BCFG'].lower()
+
     if 'APPVEYOR' in os.environ:
         ci['service'] = 'appveyor'
         if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']):
@@ -80,6 +95,9 @@ def detect_context():
 
     ci['scriptsdir'] = os.path.abspath(os.path.dirname(sys.argv[0]))
 
+    if 'CACHEDIR' in os.environ:
+        ci['cachedir'] = os.environ['CACHEDIR']
+
     if 'CHOCO' in os.environ:
         ci['choco'].extend(os.environ['CHOCO'].split())
 
@@ -144,9 +162,11 @@ def clear_lists():
     ci['debug'] = False
     ci['configuration'] = '<unknown>'
     ci['scriptsdir'] = ''
+    ci['cachedir'] = ''
     ci['choco'] = ['make']
     ci['apt'] = []
     ci['homebrew'] = []
+    ci['sudo'] = ['sudo']
 
 
 clear_lists()
@@ -204,13 +224,9 @@ if 'HomeDrive' in os.environ:
     homedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'))
 elif 'HOME' in os.environ:
     homedir = os.getenv('HOME')
-cachedir = os.path.join(homedir, '.cache')
 toolsdir = os.path.join(homedir, '.tools')
 rtemsdir = r'/home/travis/.rtems'            # Preliminary, until the next generation of toolchain
 
-if 'CACHEDIR' in os.environ:
-    cachedir = os.environ['CACHEDIR']
-
 
 vcvars_table = {
     # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History
@@ -322,14 +338,14 @@ def source_set(name):
 # - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end
 # Set places[var] = location
 def update_release_local(var, location):
-    release_local = os.path.join(cachedir, 'RELEASE.local')
+    release_local = os.path.join(ci['cachedir'], 'RELEASE.local')
     updated_line = '{0}={1}'.format(var, location.replace('\\', '/'))
     places[var] = location
 
     if not os.path.exists(release_local):
         logger.debug('RELEASE.local does not exist, creating it')
         try:
-            os.makedirs(cachedir)
+            os.makedirs(ci['cachedir'])
         except:
             pass
         touch = open(release_local, 'w')
@@ -484,7 +500,7 @@ def add_dependency(dep):
                            .format(ANSI_RED, tag, dep, setup[dep + '_REPOURL'], ANSI_RESET))
 
     dirname = setup[dep + '_DIRNAME'] + '-{0}'.format(tag)
-    place = os.path.join(cachedir, dirname)
+    place = os.path.join(ci['cachedir'], dirname)
     checked_file = os.path.join(place, "checked_out")
 
     if os.path.isdir(place):
@@ -506,14 +522,14 @@ def add_dependency(dep):
             sys.stdout.flush()
 
     if not os.path.isdir(place):
-        if not os.path.isdir(cachedir):
+        if not os.path.isdir(ci['cachedir']):
-            os.makedirs(cachedir)
+            os.makedirs(ci['cachedir'])
         # clone dependency
         print('Cloning {0} of dependency {1} into {2}'
               .format(tag, dep, place))
         sys.stdout.flush()
         call_git(['clone', '--quiet'] + deptharg + recursearg + ['--branch', tag, setup[dep + '_REPOURL'], dirname],
-                 cwd=cachedir)
+                 cwd=ci['cachedir'])
 
         sp.check_call(['git', 'log', '-n1'], cwd=place)
         logger.debug('Setting do_recompile = True (all following modules will be recompiled')
@@ -634,7 +650,7 @@ def setup_for_build(args):
 
     # Find BASE location
     if not building_base:
-        with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
+        with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f:
             lines = f.readlines()
             for line in lines:
                 (mod, place) = line.strip().split('=')
@@ -649,7 +665,7 @@ def setup_for_build(args):
 
     if ci['os'] == 'windows':
         if not building_base:
-            with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
+            with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f:
                 lines = f.readlines()
                 for line in lines:
                     (mod, place) = line.strip().split('=')
@@ -753,7 +769,7 @@ def prepare(args):
             targetdir = 'configure'
         else:
             targetdir = '.'
-        shutil.copy(os.path.join(cachedir, 'RELEASE.local'), targetdir)
+        shutil.copy(os.path.join(ci['cachedir'], 'RELEASE.local'), targetdir)
 
     fold_end('check.out.dependencies', 'Checking/cloning dependencies')
 
@@ -909,8 +925,8 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
 
     if ci['os'] == 'linux' and ci['apt']:
         fold_start('install.apt', 'Installing APT packages')
-        sp.check_call(['sudo', 'apt-get', '-y', 'update'])
+        sp.check_call(ci['sudo'] + ['apt-get', '-y', 'update'])
-        sp.check_call(['sudo', 'apt-get', '-y', 'install'] + ci['apt'])
+        sp.check_call(ci['sudo'] + ['apt-get', 'install', '-y', '-qq'] + ci['apt'])
         fold_end('install.apt', 'Installing APT packages')
 
     if ci['os'] == 'osx' and ci['homebrew']:
@@ -976,7 +992,7 @@ PERL = C:/Strawberry/perl/bin/perl -CSD'''
             print("%-10s %-12s %-11s %s" % (mod, setup[mod], stat, commit))
 
         print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET))
-        with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
+        with open(os.path.join(ci['cachedir'], 'RELEASE.local'), 'r') as f:
             print(f.read().strip())
 
 

gitlab/.gitlab-ci.yml.example-full

@@ -0,0 +1,114 @@
+# .gitlab-ci.yml for testing EPICS Base ci-scripts
+# (see: https://github.com/epics-base/ci-scripts)
+
+# This is YAML - indentation levels are crucial
+
+# GitLab runner can use any Docker container, we're using this one
+# to be comparable with the other CI services
+image: ubuntu:bionic
+
+cache:
+  key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
+  paths:
+    - .cache/
+
+variables:
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  SETUP_PATH: ".ci-local:.ci"
+  BASE_RECURSIVE: "NO"
+  # Additional packages needed for
+  #     |   EPICS                            |clang|   Windows cross builds                |RTEMS testing  |sequencer
+  APT: "libreadline6-dev libncurses5-dev perl clang g++-mingw-w64-i686 g++-mingw-w64-x86-64 qemu-system-x86 re2c"
+  CMP: "gcc"
+  BGFC: "default"
+
+# Template for build jobs (hidden)
+.build:
+  stage: build
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full
+    - python .ci/cue.py prepare
+  script:
+    - python .ci/cue.py build
+    - python .ci/cue.py test
+    - python .ci/cue.py test-results
+
+# If you need to do more during install and build,
+# add a local directory to your module and do e.g.
+#  - ./.ci-local/travis/install-extras.sh
+
+# Define build jobs
+
+# Well-known variables to use
+# SET         source setup file
+# ADD_MODULES extra modules (for a specific job)
+# BCFG        build configuration (static/debug/static-debug;
+#             default: shared-optimized)
+# TEST        set to NO to skip running the tests (default: YES)
+# VV          set to make build scripts verbose (default: unset)
+# EXTRA       content will be added to make command line
+# EXTRA1..5   more additional arguments for the make command
+#             (one argument per variable)
+
+# Usually from setup files, but may be specified or overridden
+#  on a job line
+# MODULES     list of dependency modules
+# BASE        branch or release tag name of the EPICS Base to use
+# <MODULE>    branch or release tag for a specific module
+# ...         see README for setup file syntax description
+
+# Different configurations of gcc and clang
+gcc_default:
+  extends: .build
+  variables:
+    BASE: "7.0"
+
+gcc_static:
+  extends: .build
+  variables:
+    BASE: "7.0"
+    BCFG: "static"
+
+clang_default:
+  extends: .build
+  variables:
+    BASE: "7.0"
+    CMP: "clang"
+
+clang_static_c++11:
+  extends: .build
+  variables:
+    BASE: "7.0"
+    CMP: "clang"
+    BCFG: "static"
+    EXTRA: "CMD_CXXFLAGS=-std=c++11"
+
+# Cross-compilations to Windows using gcc/MinGW and WINE
+wine32_default:
+  extends: .build
+  variables:
+    BASE: "3.15"
+    BCFG: "static"
+    WINE: "32"
+    TEST: "NO"
+
+wine64_debug:
+  extends: .build
+  variables:
+    BASE: "7.0"
+    BCFG: "debug"
+    WINE: "64"
+
+# Cross-builds to RTEMS 4.9 and 4.10
+rtems4.9_default:
+  extends: .build
+  variables:
+    RTEMS: "4.9"
+    BASE: "3.15"
+
+rtems4.10_debug:
+  extends: .build
+  variables:
+    RTEMS: "4.10"
+    BASE: "7.0"
+    BCFG: "debug"

gitlab/.gitlab-ci.yml.example-mini

@@ -0,0 +1,36 @@
+# .gitlab-ci.yml for testing EPICS Base ci-scripts
+# (see: https://github.com/epics-base/ci-scripts)
+
+image: ubuntu:bionic
+
+variables:
+  GIT_SUBMODULE_STRATEGY: "recursive"
+  SETUP_PATH: ".ci-local:.ci"
+  BASE_RECURSIVE: "NO"
+  # Minimal set of packages needed to compile EPICS Base
+  APT: "libreadline6-dev libncurses5-dev perl"
+  CMP: "gcc"
+  BGFC: "default"
+
+# Template for build jobs (hidden)
+.build:
+  stage: build
+  before_script:
+    - apt-get update -qq && apt-get install -y -qq build-essential git python curl p7zip-full
+    - python .ci/cue.py prepare
+  script:
+    - python .ci/cue.py build
+    - python .ci/cue.py test
+    - python .ci/cue.py test-results
+
+# Build on Linux using default gcc for Base branches 7.0 and 3.15
+
+gcc_base_7_0:
+  extends: .build
+  variables:
+    BASE: "7.0"
+
+gcc_base_3_15:
+  extends: .build
+  variables:
+    BASE: "3.15"

gitlab/README.md

@@ -0,0 +1,68 @@
+# GitLab CI/CD Scripts for EPICS Modules
+
+## Features
+
+ - Docker-based runners on Linux (one VM instance per job)
+ - Can use any Docker image from Dockerhub (the examples use
+  `ubuntu:bionic`)
+ - Compile natively using different compilers (gcc, clang)
+ - Cross-compile for Windows 32bit and 64bit using MinGW and WINE
+ - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15)
+ - Built dependencies are cached (for faster builds).
+
+## How to Use these Scripts
+
+ 1. Get an account on [GitLab](https://gitlab.com/), create a project
+    for your support module and have it mirror your upstream GitHub
+    repository. For more details, please refer to the
+    [GitLab CI/CD documentation](https://docs.gitlab.com/ee/README.html).
+    
+    (This applies when using the free tier offered to open source
+    projects. Things will be different using an "Enterprise"
+    installation on customer hardware.)
+
+ 2. Add the ci-scripts respository as a Git Submodule
+    (see [README](../README.md) one level above).
+
+ 3. Add settings files defining which dependencies in which versions
+    you want to build against
+    (see [README](../README.md) one level above).
+
+ 4. Create a GitLab configuration by copying one of the examples into
+    the root directory of your module.
+    ```
+    $ cp .ci/gitlab/.gitlab-ci.yml.example-full .gitlab-ci.yml
+    ```
+	
+ 5. Edit the `.gitlab-ci.yml` configuration to include the jobs you want
+    GitLab CI/CD to run.
+
+    Build jobs are declared in the list at the end of the file.
+    Each element (starting with the un-indented line) defines the
+    settings for one build job. `extends:` specifies a template to use as
+    a default structure, `variables:` controls the setting of environment
+    variables (overwriting settings from the template).
+    Also see the comments in the examples for more hints, and the
+    [GitLab CI/CD documentation](https://docs.gitlab.com/ee/README.html)
+    for more options and details.
+	
+ 6. Push your changes to GitHub, wait for the synchronization (every 5min)
+    and check [GitLab](https://gitlab.com/) for your build results.
+
+## Caches
+
+GitLab is configured to keep the caches separate for different jobs.
+
+However, changing the job description (in the `.gitlab-ci.yml` 
+configuration file) or its environment settings or changing a value
+inside a setup file will _not_ invalidate the cache - you will
+have to manually delete the caches through the GitLab web interface.
+
+Caches are automatically removed after approx. four weeks.
+Your jobs will have to rebuild them once in a while.
+
+## Miscellanea
+
+To use the feature to extract `.zip`/`.7z` archives by setting
+`*_HOOK` variables, the Linux and MacOS runners need the APT package
+`p7zip-full` resp. the Homebrew package `p7zip` installed.