diff --git a/.appveyor.yml b/.appveyor.yml index f7e8f46..0fdbde4 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -55,11 +55,11 @@ install: # Default build worker image image: Visual Studio 2015 -# Build Configurations: dll/static, regular/debug +# Build Configurations: shared/static, optimized/debug configuration: - - dynamic + - default - static - - dynamic-debug + - debug - static-debug # Environment variables: compiler toolchain, base version, setup file, ... @@ -74,7 +74,7 @@ environment: - CMP: vs2019 SET: test00 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - - CMP: mingw + - CMP: gcc - CMP: vs2019 VV: 0 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 @@ -86,6 +86,9 @@ environment: APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMP: vs2017 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + EXTRA: EXTRA_DEF="extra" + EXTRA1: EXTRA1_DEF="extra 1" + EXTRA2: EXTRA2_DEF="extra 2" - CMP: vs2015 - CMP: vs2013 - CMP: vs2012 @@ -111,7 +114,7 @@ matrix: SET: test00 - configuration: static SET: test00 - - configuration: dynamic-debug + - configuration: debug SET: test00 - configuration: static-debug SET: test00 @@ -130,7 +133,7 @@ for: only: - SET: test00 build_script: - - cmd: python appveyor-test.py + - cmd: python cue-test.py test_script: - cmd: echo Tests have been run in the build phase @@ -139,11 +142,12 @@ for: #---------------------------------# build_script: - - cmd: python appveyor/do.py prepare - - cmd: python appveyor/do.py build + - cmd: python cue.py prepare + - cmd: python cue.py build test_script: - - cmd: python appveyor/do.py test + - cmd: python cue.py test + - cmd: python cue.py test-results #---------------------------------# # debugging # diff --git a/.travis.yml b/.travis.yml index 535ec23..aa9d2c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ cache: env: global: - SETUP_PATH=.:.ci + - VV=1 + - CHOCO=re2c + - BASE_RECURSIVE=NO addons: apt: @@ -30,14 +33,15 @@ addons: homebrew: packages: - re2c - - bash update: true install: - - ./travis/prepare.sh + - python cue.py prepare script: - - ./travis-test.sh + - python cue.py build + - python cue.py test + - python cue.py test-results # If you need to do more during install and build, # add a local directory to your module and do e.g. @@ -46,136 +50,125 @@ script: # Define build jobs # Well-known variables to use -# SET source setup file -# EXTRA content will be added to make command line -# STATIC set to YES for static build (default: NO) -# TEST set to NO to skip running the tests (default: YES) -# VV set to make build scripts verbose (default: unset) +# SET source setup file +# ADD_MODULES extra modules (for a specific job) +# EXTRA content will be added to make command line +# BCFG build configuration (static/debug/static-debug; +# default: shared-optimized) +# TEST set to NO to skip running the tests (default: YES) +# VV set to make build scripts verbose (default: unset) # Usually from setup files, but may be specified or overridden # on a job line -# MODULES list of dependency modules -# BASE branch or release tag name of the EPICS Base to use -# branch or release tag for a specific module -# ... see README for setup file syntax description +# MODULES list of dependency modules +# BASE branch or release tag name of the EPICS Base to use +# branch or release tag for a specific module +# ... see README for setup file syntax description jobs: include: -# Run unit tests on Linux and Mac +# Run unit tests - env: SET=test00 + install: python cue-test.py env + script: python cue-test.py + + - env: SET=test00 TRAVIS_COMPILER=gcc + language: python + python: "3.7" + install: python cue-test.py env + script: python cue-test.py - env: SET=test00 os: osx + install: python cue-test.py env + script: python cue-test.py -# Compile the example application -# using the build configurations from full makeBaseApp example + - env: SET=test00 + os: windows + install: python cue-test.py env + script: python cue-test.py + +# Compile example - env: SET=test01 + dist: bionic -# On the side: test ADD_MODULES - - env: SET=test01 ADD_MODULES=ipac - compiler: clang + - env: SET=test01 BCFG=static-debug + dist: bionic - - env: VV="" SET=test01 - - - env: SET=test01 EXTRA="CMD_CXXFLAGS=-std=c++11" - - - env: SET=test01 EXTRA="CMD_CXXFLAGS=-std=c++11" - compiler: clang - -# trusty is pretty close to RHEL7 - env: SET=test01 dist: trusty - - env: SET=test01 EXTRA="CMD_CXXFLAGS=-std=c++11" + - env: SET=test01 BCFG=static-debug dist: trusty -# Cross-compilation to Windows using MinGW and WINE + - env: SET=test01 TRAVIS_COMPILER=gcc + language: python + python: "3.7" - - env: SET=test01 WINE=32 TEST=NO STATIC=YES - compiler: mingw + - env: SET=test01 EXTRA="CMD_CXXFLAGS=-std=c++11" + - env: SET=test01 BCFG=static + - env: SET=test01 BCFG=debug + - env: SET=test01 BCFG=static-debug - - env: SET=test01 WINE=64 TEST=NO STATIC=YES - compiler: mingw + - env: SET=test01 + compiler: clang -# dynamic (DLL) builds are broken on xenial - - env: SET=test01 WINE=32 TEST=NO STATIC=NO - dist: bionic - compiler: mingw + - env: SET=test01 BCFG=static-debug + compiler: clang - - env: SET=test01 WINE=64 TEST=NO STATIC=NO - dist: bionic - compiler: mingw + - os: osx + env: + - SET=test01 + - EXTRA=CMD_CFLAGS="-mmacosx-version-min=10.7" + - EXTRA1=CMD_CXXFLAGS="-mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++" + - EXTRA2=CMD_LDXFLAGS="-mmacosx-version-min=10.7 -std=c++11 -stdlib=libc++" + compiler: clang -# Cross-compilation to RTEMS -# (needs EPICS Base >= 3.16.2) + - env: SET=test01 BCFG=debug + os: osx + compiler: clang + + - env: SET=test01 WINE=32 TEST=NO + - env: SET=test01 WINE=32 TEST=NO BCFG=static + - env: SET=test01 WINE=32 TEST=NO BCFG=debug + - env: SET=test01 WINE=32 TEST=NO BCFG=static-debug + + - env: SET=test01 WINE=64 TEST=NO + - env: SET=test01 WINE=64 TEST=NO BCFG=static + - env: SET=test01 WINE=64 TEST=NO BCFG=debug + - env: SET=test01 WINE=64 TEST=NO BCFG=static-debug + + - env: SET=test01 RTEMS=4.9 BASE=3.15 TEST=NO + - env: SET=test01 RTEMS=4.9 BASE=3.15 TEST=NO BCFG=static + - env: SET=test01 RTEMS=4.9 BASE=3.15 TEST=NO BCFG=debug + - env: SET=test01 RTEMS=4.9 BASE=3.15 TEST=NO BCFG=static-debug - env: SET=test01 RTEMS=4.10 - - - env: SET=test01 RTEMS=4.9 - -# Other gcc versions (adding as an extra package) + - env: SET=test01 RTEMS=4.10 BCFG=static + - env: SET=test01 RTEMS=4.10 BCFG=debug + - env: SET=test01 RTEMS=4.10 BCFG=static-debug - env: SET=test01 - compiler: gcc-6 - addons: { apt: { packages: ["g++-6"], sources: ["ubuntu-toolchain-r-test"] } } + os: windows + - env: SET=test01 BCFG=static + os: windows + - env: SET=test01 BCFG=debug + os: windows + - env: SET=test01 BCFG=static-debug + os: windows - env: SET=test01 - compiler: gcc-7 - addons: { apt: { packages: ["g++-7"], sources: ["ubuntu-toolchain-r-test"] } } - -# MacOS build - -# SNCSEQ 2.2.7 fails to build on MacOS; currently needs master - - env: SET=test01 SNCSEQ=master - os: osx - compiler: clang - -# Base 3.15 builds -# ================ - - - env: BASE=R3.15.7 SET=test01 - - - env: BASE=R3.15.7 SET=test01 WINE=64 TEST=NO STATIC=YES - dist: bionic - compiler: mingw - -# The DLL build for this Base version is known to fail -# - env: BASE=R3.15.7 SET=test01 WINE=64 TEST=NO STATIC=NO -# dist: bionic -# compiler: mingw - -# Cross-compilation to RTEMS -# (needs EPICS Base >= 3.16.2) - - - env: BASE=R3.16.2 SET=test01 RTEMS=4.10 - dist: trusty - - - env: BASE=R3.16.2 SET=test01 RTEMS=4.9 - dist: trusty - -# SNCSEQ 2.2.7 fails to build on MacOS; currently needs master - - env: BASE=R3.15.7 SET=test01 SNCSEQ=master - os: osx - compiler: clang - -# Base 3.14 builds -# ================ - - - env: BASE=R3.14.12.8 SET=test01 - - - env: BASE=R3.14.12.8 SET=test01 WINE=64 TEST=NO STATIC=YES - dist: bionic - compiler: mingw - -# The DLL build for this Base version is known to fail -# - env: BASE=R3.14.12.8 SET=test01 WINE=64 TEST=NO STATIC=NO -# dist: bionic -# compiler: mingw - -# SNCSEQ 2.2.7 fails to build on MacOS; currently needs master - - env: BASE=R3.14.12.8 SET=test01 SNCSEQ=master - os: osx - compiler: clang + compiler: vs2017 + os: windows + - env: SET=test01 BCFG=static + compiler: vs2017 + os: windows + - env: SET=test01 BCFG=debug + compiler: vs2017 + os: windows + - env: SET=test01 BCFG=static-debug + compiler: vs2017 + os: windows diff --git a/README.md b/README.md index 507d6e9..ba68d7d 100644 --- a/README.md +++ b/README.md @@ -20,58 +20,70 @@ never break existing use. ## This Repository -In addition to the scripts themselves (in the subdirectories), -this repository contains the test suite that is used to verify -functionality and features of the ci-scripts. +In addition to the script that runs the builds and tests, this repository +contains service specific documentation and example configuration files +(in the subdirectories), and a small test suite that is used to verify +functionality and features of the ci-scripts module itself You are welcome to use the test suite as a reference, but keep in -mind that in your module the path to the scripts has one level more -(e.g., `./travis/abc` here would be `./.ci/travis/abc` in your +mind that in your main module the path to the scripts has one level more +(e.g., `./abc` here would be `./.ci/abc` in your module). -Also, a test suite might not show the same level of quality as an -example. +Also, a test suite might not show the same quality and documentation levels +as an example. ## Features - Compile against different branches or releases of EPICS Base and - additional dependencies (modules like asyn, std, etc.). + additional dependencies (modules like asyn, std, sequencer, etc.). - Define settings files that declare sets of dependencies with their versions and locations. - - Define hook scripts for any dependency. + - Define hooks for any dependency. Hooks are run on the dependency module before it is compiled, so the module can be patched or further configured. - - Define static or shared builds (executables, libraries). + - Define shared (default) or static builds (for executables and libraries). + + - Define optimized (default) or debug builds. - - Run tests (using the EPICS unit test suite). + - Run tests (using the EPICS build system, i.e., `make runtests` + and friends). ## Supported CI Services ### [Travis-CI](https://travis-ci.org/) + - Five parallel runners on Linux/Windows (one runner on MacOS) - Use different compilers (gcc, clang) - Use different gcc versions - Cross-compile for Windows 32bit and 64bit using MinGW and WINE - - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.16.2) - - Compile on MacOS - - Built dependencies are cached (for faster builds) + - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15) + - Compile natively on MacOS (clang) + - Compile natively on Windows (gcc/MinGW, Visual Studio 2017) + - Built dependencies are cached (for faster builds). -See specific **[ci-scripts on Travis-CI README](travis/README.md)** for more details. +See specific +**[ci-scripts on Travis-CI README](travis/README.md)** +for more details. ### [AppVeyor](https://www.appveyor.com/) - - Use different compilers (Visual Studio, MinGW) + - One parallel runner (all builds are sequential) + - Use different compilers (Visual Studio, gcc/MinGW) - Use different Visual Studio versions: \ - 2008, 2010, 2012, 2013, 2015, 2017, 2019 + 2008, 2010, 2012, 2013, 2015, 2017, 2019 - Compile for Windows 32bit and 64bit + - No useful caching available. -See specific **[ci-scripts on AppVeyor README](appveyor/README.md)** for more details. +See specific +**[ci-scripts on AppVeyor README](appveyor/README.md)** +for more details. ## How to Use the CI-Scripts 1. Get an account on a supported CI service provider platform. (e.g. [Travis-CI](https://travis-ci.org/), - [AppVeyor](https://www.appveyor.com/), Azure Pipelines...) + [AppVeyor](https://www.appveyor.com/), ...) (More details in the specific README of the subdirectory.) @@ -94,8 +106,8 @@ See specific **[ci-scripts on AppVeyor README](appveyor/README.md)** for more de ``` will compile against the EPICS Base 3.15 branch, the Sequencer release 2.2.8 and release 4.34 of asyn. - (Any settings can be overridden from the specific job configuration - in e.g. `.travis.yml`.) + (Any settings can be overridden from the specific job line + in the service configuration, e.g., `.travis.yml`.) 4. Create a configuration for the CI service by copying one of the examples provided in the service specific subdirectory @@ -105,14 +117,41 @@ See specific **[ci-scripts on AppVeyor README](appveyor/README.md)** for more de 5. Push your changes and check the CI service for your build results. +## Calling the cue.py Script + +Independent from CI service and platform, the runner +script is called from your main configuration as: + +`python .ci/cue.py ` + +where `` is one of: + +`prepare`\ +Prepare the build by cloning Base and the configured dependency modules, +set up the EPICS build system, then +compile Base and these modules in the order they appear in the `MODULES` +setting. + +`build`\ +Build your main module. + +`test`\ +Run the tests of your main module. + +`test-results`\ +Collect the results of your tests and print a summary. + +`exec`\ +Execute the remainder of the line using the default command shell. + ## Setup Files Your module might depend on EPICS Base and a few other support modules. (E.g., a specific driver might need StreamDevice, ASYN and the Sequencer.) In that case, building against every possible combination of released versions of those dependencies is not possible: -Base (37) x StreamDevice (50) x ASYN (40) x Sequencer (51) would produce -more than 3.7 million different combinations, i.e. build jobs. +Base (39) x StreamDevice (50) x ASYN (40) x Sequencer (52) would produce +more than 4 million different combinations, i.e. build jobs. A more reasonable approach is to create a few setups, each being a combination of dependency releases, that do a few scans of the available @@ -120,19 +159,23 @@ combination of dependency releases, that do a few scans of the available for stable versions that many of your users have in production, one for the latest released versions and one for the development branches. +A job uses a setup file if `SET=` (without the `.set` extension +of the setup file) is set for the job in the main configuration file. + ## Setup File Syntax -Setup files are loaded by the build scripts. They are found by searching +Setup files are loaded by the build script. They are found by searching the locations in `SETUP_PATH` (space or colon separated list of directories, relative to your module's root directory). Setup files can include other setup files by calling `include ` -(omitting the `.set` extension of the setup file). The configured +(again omitting the `.set` extension of the setup file). The configured `SETUP_PATH` is searched for the include. -Any `VAR=value` setting of a variable is only executed if `VAR` is unset or -empty. That way any settings can be overridden by settings in the main -configuration (e.g., `.travis.yml`). +Any `VAR=value` setting of a variable in a setup file is only executed if +`VAR` is unset or empty. +That way any settings can be overridden by setting them in the job +description inside the main configuration file (e.g., `.travis.yml`). Empty lines or lines starting with `#` are ignored. @@ -141,8 +184,9 @@ by using their well-known slugs, separated by spaces. EPICS Base (slug: `base`) will always be a dependency and will be added and compiled first. The other dependencies are added and compiled in the order they are defined in `MODULES`. + Modules needed only for specific jobs (e.g., on specific architectures) -can be added in the main configuration file by setting `ADD_MODULES` +can be added from the main configuration file by setting `ADD_MODULES` for the specific job(s). `REPOOWNER=` sets the default GitHub owner (or organization) for all @@ -201,29 +245,84 @@ builds to higher verbosity. For debugging on your local machine, you may set `CACHEDIR` to change the location for the dependency builds. [default is `$HOME/.cache`] +Set `PARALLEL_MAKE` to the number of parallel make jobs that you want your +build to use. [default is the number of CPUs on the runner] + +Service specific options are described in the README files +in the service specific subdirectories: + +- [Travis-CI README](travis/README.md) +- [AppVeyor README](appveyor/README.md) + ## References: EPICS Modules Using ci-scripts [EPICS Base](https://github.com/epics-base/epics-base) and its submodules [pvData](https://github.com/epics-base/pvDataCPP), [pvAccess](https://github.com/epics-base/pvAccessCPP), -[pva2pva](https://github.com/epics-base/pva2pva) +[pva2pva](https://github.com/epics-base/pva2pva), +[PVXS](https://github.com/mdavidsaver/pvxs) EPICS Modules: [ASYN](https://github.com/epics-modules/asyn), [devlib2](https://github.com/epics-modules/devlib2), [ecmc](https://github.com/epics-modules/ecmc), +[gtest](https://github.com/epics-modules/gtest), [ip](https://github.com/epics-modules/ip), [lua](https://github.com/epics-modules/lua), [MCoreUtils](https://github.com/epics-modules/MCoreUtils), [modbus](https://github.com/epics-modules/modbus), [motor](https://github.com/epics-modules/motor), +[OPCUA](https://github.com/ralphlange/opcua), [PCAS](https://github.com/epics-modules/pcas), [sscan](https://github.com/epics-modules/sscan), [vac](https://github.com/epics-modules/vac) ESS: [EtherCAT MC Motor Driver][ref.ethercatmc] -ITER: [OPC UA Device Support](https://github.com/ralphlange/opcua) +## Migration Hints + +Look for changes in the example configuration files, and check how they +apply to your module. + +If comments in the example have changed, copy them to your configuration +to always have up-to-date documentation in your file. + +### 2.x to 3.x Migration + +Update the script and test settings in your configuration to call the +new script, following the example file. + +`python .ci/cue.py ` + +#### AppVeyor + +The `configuration:` setting options have changed; they are now +`default`, `static`, `debug` and `static-debug`. + +MinGW builds are now using the `CMP: gcc` compiler setting. + +Adding arguments to make is supported through the `EXTRA` .. `EXTRA5` +variables. Each variable value will be passed as one argument. + +#### Travis + +The new `BCFG` (build configuration) variable accepts the same options as +the AppVeyor `configuration:` setting. Replace any`STATIC=YES` settings with +`BCFG=static`. + +Remove `bash` in the `homebrew:` section of `addons:`. There are no more +bash scripts. + +MinGW builds (cross-builds using WINE as well as native builds on Windows) +are now using the `gcc` compiler setting. +Since `gcc` is the default, you can simply remove `compiler: mingw` lines. + +For Windows, Travis offers native MinGW and Visual Studio 2017 compilers. +Use `os: windows` and set `compiler:` to `gcc` or `vs2017` + for those builds. + +Chocolatey packages to be installed for the Windows jobs are set by adding +them to the environment variable `CHOCO`. ## Frequently Asked Questions @@ -232,7 +331,7 @@ ITER: [OPC UA Device Support](https://github.com/ralphlange/opcua) Set `VV=1` in the configuration line of the job you are interested in. This will make all builds (not just for your module) verbose. -**How do I update my module to use a newer release of ci-scripts?** +**How do I update my module to use a newer minor release of ci-scripts?** Update the submodule in `.ci` first, then change your CI configuration (if needed) and commit both to your module. E.g., to update your Travis @@ -257,12 +356,10 @@ be advisable to clear the CI caches after updating ci-scripts. E.g., a change in setting up EPICS Base will not be applied if Base is found in the cache. -**Why does running the scripts locally on my MacOS machine fail?** +**How do I add a dependency module only for a specific job?** -The ci-scripts for Travis-CI require Bash version 4. -As Apple ships an older Bash for [political reasons][reddit.bash], -you need to install a more recent Bash, e.g. using MacPorts -or Homebrew. +Add the additional dependency in the main configuration file by setting +`ADD_MODULES` for the specific job(s). ## Release Numbering of this Module diff --git a/appveyor-test.py b/appveyor-test.py deleted file mode 100644 index 8cfbf94..0000000 --- a/appveyor-test.py +++ /dev/null @@ -1,437 +0,0 @@ -#!/usr/bin/env python -"""Module ci-scripts AppVeyor unit tests -""" - -# SET=test00 in the environment (.appveyor.yml) runs the tests in this script -# all other jobs are started as compile jobs - -from __future__ import print_function - -import sys, os, shutil, fileinput -import distutils.util -import re -import subprocess as sp -import unittest -import logging -from argparse import Namespace - -builddir = os.getcwd() - -def find_in_file(regex, filename): - file = open (filename, "r") - for line in file: - if re.search(regex, line): - return True - return False - -def getStringIO(): - if (sys.version_info > (3, 0)): - import io - return io.StringIO() - else: - import StringIO - return StringIO.StringIO() - -sys.path.append('appveyor') -import do - -# we're working with tags (detached heads) a lot: suppress advice -do.call_git(['config', '--global', 'advice.detachedHead', 'false']) - -class TestSourceSet(unittest.TestCase): - - def setUp(self): - os.environ['SETUP_PATH'] = '.:appveyor' - if 'BASE' in os.environ: - del os.environ['BASE'] - do.clear_lists() - os.chdir(builddir) - - def test_EmptySetupDirsPath(self): - del os.environ['SETUP_PATH'] - self.assertRaisesRegexp(NameError, '\(SETUP_PATH\) is empty', do.source_set, 'test01') - - def test_InvalidSetupName(self): - self.assertRaisesRegexp(NameError, 'does not exist in SETUP_PATH', do.source_set, 'xxdoesnotexistxx') - - def test_ValidSetupName(self): - capturedOutput = getStringIO() - sys.stdout = capturedOutput - do.source_set('test01') - sys.stdout = sys.__stdout__ - self.assertEqual(do.setup['BASE'], '7.0', 'BASE was not set to \'7.0\'') - - def test_SetupDoesNotOverridePreset(self): - os.environ['BASE'] = 'foo' - capturedOutput = getStringIO() - sys.stdout = capturedOutput - do.source_set('test01') - sys.stdout = sys.__stdout__ - self.assertEqual(do.setup['BASE'], 'foo', - 'Preset BASE was overridden by test01 setup (expected \'foo\' got {0})' - .format(do.setup['BASE'])) - - def test_IncludeSetupFirstSetWins(self): - capturedOutput = getStringIO() - sys.stdout = capturedOutput - do.source_set('test02') - sys.stdout = sys.__stdout__ - self.assertEqual(do.setup['BASE'], 'foo', - 'BASE set in test02 was overridden by test01 setup (expected \'foo\' got {0})' - .format(do.setup['BASE'])) - self.assertEqual(do.setup['FOO'], 'bar', 'Setting of single word does not work') - self.assertEqual(do.setup['FOO2'], 'bar bar2', 'Setting of multiple words does not work') - self.assertEqual(do.setup['FOO3'], 'bar bar2', 'Indented setting of multiple words does not work') - self.assertEqual(do.setup['SNCSEQ'], 'R2-2-7', 'Setup test01 was not included') - - def test_DoubleIncludeGetsIgnored(self): - capturedOutput = getStringIO() - sys.stdout = capturedOutput - do.source_set('test03') - sys.stdout = sys.__stdout__ - self.assertRegexpMatches(capturedOutput.getvalue(), 'Ignoring already included setup file') - -class TestUpdateReleaseLocal(unittest.TestCase): - - release_local = os.path.join(do.cachedir, 'RELEASE.local') - - def setUp(self): - if os.path.exists(self.release_local): - os.remove(self.release_local) - os.chdir(builddir) - - def test_SetModule(self): - do.update_release_local('MOD1', '/foo/bar') - found = 0 - for line in fileinput.input(self.release_local, inplace=1): - if 'MOD1=' in line: - self.assertEqual(line.strip(), 'MOD1=/foo/bar', 'MOD1 not set correctly') - found += 1 - fileinput.close() - self.assertEqual(found, 1, 'MOD1 not written once to RELEASE.local (found {0})'.format(found)) - - def test_SetBaseAndMultipleModules(self): - do.update_release_local('EPICS_BASE', '/bar/foo') - do.update_release_local('MOD1', '/foo/bar') - do.update_release_local('MOD2', '/foo/bar2') - do.update_release_local('MOD1', '/foo/bar1') - found = {} - foundat = {} - for line in fileinput.input(self.release_local, inplace=1): - if 'MOD1=' in line: - self.assertEqual(line.strip(), 'MOD1=/foo/bar1', - 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' - .format(line)) - if 'mod1' in found: - found['mod1'] += 1 - else: - found['mod1'] = 1 - foundat['mod1'] = fileinput.filelineno() - if 'MOD2=' in line: - self.assertEqual(line.strip(), 'MOD2=/foo/bar2', - 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' - .format(line)) - if 'mod2' in found: - found['mod2'] += 1 - else: - found['mod2'] = 1 - foundat['mod2'] = fileinput.filelineno() - if 'EPICS_BASE=' in line: - self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', - 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' - .format(line)) - if 'base' in found: - found['base'] += 1 - else: - found['base'] = 1 - foundat['base'] = fileinput.filelineno() - fileinput.close() - self.assertEqual(found['mod1'], 1, - 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) - self.assertEqual(found['mod2'], 1, - 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) - self.assertEqual(found['base'], 1, - 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) - self.assertGreater(foundat['base'], foundat['mod2'], - 'EPICS_BASE (line {0}) appears before MOD2 (line {1})' - .format(foundat['base'], foundat['mod2'])) - self.assertGreater(foundat['mod2'], foundat['mod1'], - 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) - -class TestAddDependencyUpToDateCheck(unittest.TestCase): - - hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" - location = os.path.join(do.cachedir, 'base-R3.15.6') - licensefile = os.path.join(location, 'LICENSE') - checked_file = os.path.join(location, 'checked_out') - release_file = os.path.join(location, 'configure', 'RELEASE') - - def setUp(self): - os.environ['SETUP_PATH'] = '.:appveyor' - if os.path.exists(self.location): - shutil.rmtree(self.location, onerror=do.remove_readonly) - do.clear_lists() - os.chdir(builddir) - do.source_set('defaults') - do.complete_setup('BASE') - - def test_MissingDependency(self): - do.setup['BASE'] = 'R3.15.6' - do.add_dependency('BASE') - self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out') - self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written') - with open(self.checked_file, 'r') as bfile: - checked_out = bfile.read().strip() - bfile.close() - self.assertEqual(checked_out, self.hash_3_15_6, - 'Wrong commit of dependency checked out (expected=\"{0}\" found=\"{1}\")' - .format(self.hash_3_15_6, checked_out)) - self.assertFalse(find_in_file('include \$\(TOP\)/../RELEASE.local', self.release_file), - 'RELEASE in Base includes TOP/../RELEASE.local') - - def test_UpToDateDependency(self): - do.setup['BASE'] = 'R3.15.6' - do.add_dependency('BASE') - os.remove(self.licensefile) - do.add_dependency('BASE') - self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency') - - def test_OutdatedDependency(self): - do.setup['BASE'] = 'R3.15.6' - do.add_dependency('BASE') - os.remove(self.licensefile) - with open(self.checked_file, "w") as fout: - print('XXX not the right hash XXX', file=fout) - fout.close() - do.add_dependency('BASE') - self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency') - with open(self.checked_file, 'r') as bfile: - checked_out = bfile.read().strip() - bfile.close() - self.assertEqual(checked_out, self.hash_3_15_6, - "Wrong commit of dependency checked out (expected='{0}' found='{1}')" - .format(self.hash_3_15_6, checked_out)) - -def is_shallow_repo(place): - check = sp.check_output(['git', 'rev-parse', '--is-shallow-repository'], cwd=place).strip() - if check == '--is-shallow-repository': - if os.path.exists(os.path.join(place, '.git', 'shallow')): - check = 'true' - else: - check = 'false' - return check == 'true' - -class TestAddDependencyOptions(unittest.TestCase): - - location = os.path.join(do.cachedir, 'mcoreutils-master') - testfile = os.path.join(location, '.ci', 'LICENSE') - - def setUp(self): - os.environ['SETUP_PATH'] = '.:appveyor' - if os.path.exists(do.cachedir): - shutil.rmtree(do.cachedir, onerror=do.remove_readonly) - do.clear_lists() - do.source_set('defaults') - do.complete_setup('MCoreUtils') - do.setup['MCoreUtils'] = 'master' - - def test_Default(self): - do.add_dependency('MCoreUtils') - self.assertTrue(os.path.exists(self.testfile), - 'Submodule (.ci) not checked out recursively (requested: default=YES') - self.assertTrue(is_shallow_repo(self.location), - 'Module not checked out shallow (requested: default=5)') - - def test_SetRecursiveNo(self): - do.setup['MCoreUtils_RECURSIVE'] = 'NO' - do.add_dependency('MCoreUtils') - self.assertFalse(os.path.exists(self.testfile), 'Submodule (.ci) checked out recursively') - - def test_SetDepthZero(self): - do.setup['MCoreUtils_DEPTH'] = '0' - do.add_dependency('MCoreUtils') - self.assertFalse(is_shallow_repo(self.location), 'Module checked out shallow (requested full)') - - def test_SetDepthThree(self): - do.setup['MCoreUtils_DEPTH'] = '3' - do.add_dependency('MCoreUtils') - self.assertTrue(is_shallow_repo(self.location), - 'Module not checked out shallow (requested: default=5)') - - def test_AddMsiTo314(self): - do.complete_setup('BASE') - do.setup['BASE'] = 'R3.14.12.1' - msifile = os.path.join(do.cachedir, 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c') - do.add_dependency('BASE') - self.assertTrue(os.path.exists(msifile), 'MSI was not added to Base 3.14') - -def repo_access(dep): - do.set_setup_from_env(dep) - do.setup.setdefault(dep + "_DIRNAME", dep.lower()) - do.setup.setdefault(dep + "_REPONAME", dep.lower()) - do.setup.setdefault('REPOOWNER', 'epics-modules') - do.setup.setdefault(dep + "_REPOOWNER", do.setup['REPOOWNER']) - do.setup.setdefault(dep + "_REPOURL", 'https://github.com/{0}/{1}.git' - .format(do.setup[dep + '_REPOOWNER'], do.setup[dep + '_REPONAME'])) - with open(os.devnull, 'w') as devnull: - return do.call_git(['ls-remote', '--quiet', '--heads', do.setup[dep + '_REPOURL']], - stdout=devnull, stderr=devnull) - -class TestDefaultModuleURLs(unittest.TestCase): - - modules = ['BASE', 'PVDATA', 'PVACCESS', 'NTYPES', - 'SNCSEQ', 'STREAM', 'ASYN', 'STD', - 'CALC', 'AUTOSAVE', 'BUSY', 'SSCAN', - 'IOCSTATS', 'MOTOR', 'IPAC', ] - - def setUp(self): - os.environ['SETUP_PATH'] = '.:appveyor' - do.clear_lists() - os.chdir(builddir) - do.source_set('defaults') - - def test_Repos(self): - for mod in self.modules: - self.assertEqual(repo_access(mod), 0, 'Defaults for {0} do not point to a valid git repository at {1}' - .format(mod, do.setup[mod + '_REPOURL'])) - -class TestVCVars(unittest.TestCase): - def test_vcvars(self): - if ('CMP' in os.environ and os.environ['CMP'] in ('mingw',)) \ - or distutils.util.get_platform() != "win32": - raise unittest.SkipTest() - - do.with_vcvars('env') - -class TestSetupForBuild(unittest.TestCase): - configuration = os.environ['CONFIGURATION'] - platform = os.environ['PLATFORM'] - cc = os.environ['CMP'] - args = Namespace(paths=[]) - do.building_base = True - - def setUp(self): - os.environ.pop('EPICS_HOST_ARCH', None) - do.clear_lists() - - def tearDown(self): - os.environ['CONFIGURATION'] = self.configuration - os.environ['PLATFORM'] = self.platform - os.environ['CMP'] = self.cc - - def test_AddPathsOption(self): - os.environ['FOOBAR'] = 'BAR' - args = Namespace(paths=['/my/{FOOBAR}/dir', '/my/foobar']) - do.setup_for_build(args) - self.assertTrue(re.search('/my/BAR/dir', os.environ['PATH']), 'Expanded path not in PATH') - self.assertTrue(re.search('/foobar', os.environ['PATH']), 'Plain path not in PATH') - os.environ.pop('FOOBAR', None) - - def test_HostArchConfiguration(self): - for config in ['dynamic', 'dynamic-debug', 'static', 'static-debug']: - os.environ['CONFIGURATION'] = config - do.setup_for_build(self.args) - self.assertTrue('EPICS_HOST_ARCH' in os.environ, - 'EPICS_HOST_ARCH is not set for Configuration={0}'.format(config)) - if re.search('static', config): - self.assertTrue(re.search('-static$', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is not -static for Configuration={0}'.format(config)) - self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) - elif re.search('debug', config): - self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is -static for Configuration={0}'.format(config)) - self.assertTrue(re.search('-debug$', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is not -debug for Configuration={0}'.format(config)) - else: - self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is -static for Configuration={0}'.format(config)) - self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) - - def test_HostArchPlatform(self): - for platform in ['x86', 'x64', 'X64']: - for cc in ['vs2019', 'mingw']: - os.environ['PLATFORM'] = platform - os.environ['CMP'] = cc - os.environ['CONFIGURATION'] = 'dynamic' - do.setup_for_build(self.args) - self.assertTrue('EPICS_HOST_ARCH' in os.environ, - 'EPICS_HOST_ARCH is not set for {0} / {1}'.format(cc, platform)) - if platform == 'x86': - self.assertTrue(re.search('^win32-x86', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is not win32-x86 for {0} / {1}'.format(cc, platform)) - else: - self.assertTrue(re.search('^windows-x64', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is not windows-x64 for {0} / {1}'.format(cc, platform)) - if cc == 'mingw': - self.assertTrue(re.search('-mingw$', os.environ['EPICS_HOST_ARCH']), - 'EPICS_HOST_ARCH is not -mingw for {0} / {1}'.format(cc, platform)) - if platform == 'x86': - pattern = 'mingw32' - else: - pattern = 'mingw64' - self.assertTrue(re.search(pattern, os.environ['PATH']), - 'Binary location for {0} not in PATH'.format(pattern)) - self.assertTrue(re.search(pattern, os.environ['INCLUDE']), - 'Include location for {0} not in INCLUDE'.format(pattern)) - - def test_StrawberryInPath(self): - os.environ['CMP'] = 'vs2019' - do.setup_for_build(self.args) - self.assertTrue(re.search('strawberry', os.environ['PATH'], flags=re.IGNORECASE), - 'Strawberry Perl location not in PATH for vs2019') - - def setBase314(self, yesno): - cfg_base_version = os.path.join('configure', 'CONFIG_BASE_VERSION') - fout = open(cfg_base_version, 'w') - print('# test file for base version detection', file=fout) - print('BASE_3_14={0}'.format(yesno), file=fout) - fout.close() - - def setTestResultsTarget(self, target): - rules_build = os.path.join('configure', 'RULES_BUILD') - fout = open(rules_build, 'w') - print('# test file for target detection', file=fout) - print('{0}: something'.format(target), file=fout) - fout.close() - - def test_DetectionBase314No(self): - self.setBase314('NO') - do.setup_for_build(self.args) - self.assertFalse(do.isbase314, 'Falsely detected Base 3.14') - - def test_DetectionBase314Yes(self): - self.setBase314('YES') - do.setup_for_build(self.args) - self.assertTrue(do.isbase314, 'Base 3.14 = YES not detected') - - def test_DetectionTestResultsTarget314No(self): - self.setBase314('YES') - self.setTestResultsTarget('nottherighttarget') - do.setup_for_build(self.args) - self.assertFalse(do.has_test_results, 'Falsely detected test-results target') - - def test_DetectionTestResultsTarget314Yes(self): - self.setBase314('YES') - self.setTestResultsTarget('test-results') - do.setup_for_build(self.args) - self.assertFalse(do.has_test_results, 'Falsely found test-results on Base 3.14') - - def test_DetectionTestResultsTargetNot314Yes(self): - self.setBase314('NO') - self.setTestResultsTarget('test-results') - do.setup_for_build(self.args) - self.assertTrue(do.has_test_results, 'Target test-results not detected') - -if __name__ == "__main__": - if 'VV' in os.environ and os.environ['VV'] == '1': - logging.basicConfig(level=logging.DEBUG) - do.silent_dep_builds = False - - do.host_info() - if sys.argv[1:]==['env']: - # testing with_vcvars - [print(K,'=',V) for K, V in os.environ.items()] - else: - unittest.main() diff --git a/appveyor/.appveyor.yml.example-full b/appveyor/.appveyor.yml.example-full index 8892f30..36b9e58 100644 --- a/appveyor/.appveyor.yml.example-full +++ b/appveyor/.appveyor.yml.example-full @@ -56,23 +56,45 @@ install: # Default build worker image image: Visual Studio 2015 -# Build Configurations: dll/static, regular/debug +# Build Configurations: shared/static, optimized/debug configuration: - - dynamic + - default - static - - dynamic-debug + - debug - static-debug -# Environment variables: compiler toolchain, base version, setup file, ... +# Environment variables + +# Well-known variables to use +# CMP compiler to use ('gcc' for native MinGW, 'vs...' for Visual Studio) +# SET source setup file +# ADD_MODULES extra modules (for a specific job) +# TEST set to NO to skip running the tests (default: YES) +# VV set VV=1 to make build scripts verbose (default: unset) +# EXTRA content will be added to make command line +# EXTRA1..5 more additional arguments for the make command +# (one argument per variable) + +# Usually from setup files, but may be specified or overridden +# on a job line +# MODULES list of dependency modules +# BASE branch or release tag name of the EPICS Base to use +# branch or release tag for a specific module +# ... see README for setup file syntax description + +# AppVeyor specific +# APPVEYOR_BUILD_WORKER_IMAGE run job using specified VM image +# (not the one from the image: line above) + environment: # common / default variables for all jobs SETUP_PATH: .ci-local:.ci + BASE: 7.0 matrix: - CMP: vs2019 - SET: test00 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - - CMP: mingw + - CMP: gcc - CMP: vs2019 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - CMP: vs2019 @@ -116,15 +138,15 @@ matrix: #---------------------------------# build_script: - - cmd: python .ci/appveyor/do.py prepare - - cmd: python .ci/appveyor/do.py build + - cmd: python .ci/cue.py prepare + - cmd: python .ci/cue.py build test_script: - - cmd: python .ci/appveyor/do.py test + - cmd: python .ci/cue.py test on_finish: - ps: Get-ChildItem *.tap -Recurse -Force | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } - - cmd: python .ci/appveyor/do.py build test-results -s + - cmd: python .ci/cue.py test-results #---------------------------------# # debugging # diff --git a/appveyor/.appveyor.yml.example-mini b/appveyor/.appveyor.yml.example-mini index bec8658..9bb41bc 100644 --- a/appveyor/.appveyor.yml.example-mini +++ b/appveyor/.appveyor.yml.example-mini @@ -1,8 +1,6 @@ # .appveyor.yml for use with EPICS Base ci-scripts # (see: https://github.com/epics-base/ci-scripts) -# This is YAML - indentation levels are crucial - cache: - C:\Users\appveyor\.tools @@ -24,11 +22,11 @@ install: image: Visual Studio 2019 -# Build Configurations: dll/static, regular/debug +# Build Configurations: shared/static, optimized/debug configuration: - - dynamic + - default # - static - - dynamic-debug + - debug # - static-debug environment: @@ -58,15 +56,15 @@ matrix: CMP: vs2008 build_script: - - cmd: python .ci/appveyor/do.py prepare - - cmd: python .ci/appveyor/do.py build + - cmd: python .ci/cue.py prepare + - cmd: python .ci/cue.py build test_script: - - cmd: python .ci/appveyor/do.py test + - cmd: python .ci/cue.py test on_finish: - ps: Get-ChildItem *.tap -Recurse -Force | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } - - cmd: python .ci/appveyor/do.py build test-results -s + - cmd: python .ci/cue.py test-results notifications: - provider: GitHubPullRequest diff --git a/appveyor/README.md b/appveyor/README.md index 2ee6084..e4005f3 100644 --- a/appveyor/README.md +++ b/appveyor/README.md @@ -2,11 +2,12 @@ ## Features - - Use different compilers (Visual Studio, MinGW) - - Use different VS versions (2008, 2010, 2012, 2013, 2015, 2017, 2019) + - One parallel runner (all builds are sequential) + - Use different compilers (Visual Studio, gcc/MinGW) + - Use different Visual Studio versions: \ + 2008, 2010, 2012, 2013, 2015, 2017, 2019 - Compile for Windows 32bit and 64bit - - Create static libraries or DLLs (plus the matching executables) - - Create optimized or debug builds + - No useful caching available. ## How to Use these Scripts @@ -33,24 +34,26 @@ AppVeyor automatically creates a build matrix with the following axes: 1. `configuration:` \ - Select static or dynamic (DLL) as well as regular or debug builds. + Select shared (DLL) or static as well as optimized or debug builds. \ + Default: `shared-optimized` 2. `platform:` \ Select 32bit or 64bit processor architecture. 3. `environment: / matrix:` \ List of environment variable settings. Each list element (starting with a dash) is one step on the axis of the build matrix. \ - Set `CMP` to select the compiler: `mingw` for the native + Set `CMP` to select the compiler: `gcc` for the native [MinGW](http://mingw-w64.org/) GNU compiler, `vs2008` ...`vs2019` (options listed above) for the Microsoft Visual Studio compilers. Your builds will take long. \ - AppVeyor only grants a single worker VM - all jobs of the matrix are - executed sequentially. Each job will take between 6 and 15 minutes, - plus testing time. + AppVeyor only grants a single parallel runner VM - all jobs of the matrix + are executed sequentially. AppVeyor also does not provide a usable cache + mechanism to retain dependency artifacts across builds. + Each job will take between 6 and 15 minutes, plus testing time, every time. The `matrix: / exclude:` setting can be used to reduce the number of jobs. Check the [AppVeyor docs][appveyor.doc.matrix] - for more ways to reduce the build matrix size. + for more ways to reduce the build matrix size. \ E.g., you can opt for not creating matrix axes for `configuration:` and`platform:` by moving these configurations into the job lines under `environment: / matrix:`. diff --git a/appveyor/do.py b/appveyor/do.py deleted file mode 100644 index 8887437..0000000 --- a/appveyor/do.py +++ /dev/null @@ -1,673 +0,0 @@ -#!/usr/bin/env python -"""Windows (AppVeyor) ci build script -""" - -from __future__ import print_function - -import sys, os, stat, shutil -import fileinput -import logging -import re -import subprocess as sp -import distutils.util - -logger = logging.getLogger(__name__) - -# Setup ANSI Colors -ANSI_RED = "\033[31;1m" -ANSI_GREEN = "\033[32;1m" -ANSI_YELLOW = "\033[33;1m" -ANSI_BLUE = "\033[34;1m" -ANSI_MAGENTA = "\033[35;1m" -ANSI_CYAN = "\033[36;1m" -ANSI_RESET = "\033[0m" -ANSI_CLEAR = "\033[0K" - -seen_setups = [] -modules_to_compile = [] -setup = {} -places = {} - -if 'HomeDrive' in os.environ: - cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') - toolsdir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.tools') -elif 'HOME' in os.environ: - cachedir = os.path.join(os.getenv('HOME'), '.cache') - toolsdir = os.path.join(os.getenv('HOME'), '.tools') -else: - cachedir = os.path.join('.', '.cache') - toolsdir = os.path.join('.', '.tools') - -if 'CACHEDIR' in os.environ: - cachedir = os.environ['CACHEDIR'] - -vcvars_table = { - # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History - 'vs2019':r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat', - 'vs2017':r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat', - 'vs2015':r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat', - 'vs2013':r'C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat', - 'vs2012':r'C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat', - 'vs2010':r'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat', - 'vs2008':r'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcvarsall.bat', -} - -ciscriptsdir = os.path.abspath(os.path.dirname(sys.argv[0])) -if os.path.basename(ciscriptsdir) == 'appveyor': - ciscriptsdir = ciscriptsdir.rstrip(os.pathsep+'appveyor') - -if 'BASE' in os.environ and os.environ['BASE'] == 'SELF': - building_base = True - places['EPICS_BASE'] = '.' -else: - building_base = False - -def modlist(): - if building_base: - ret = [] - else: - for var in ['ADD_MODULES', 'MODULES']: - setup.setdefault(var, '') - if var in os.environ: - setup[var] = os.environ[var] - logger.debug('ENV assignment: %s = %s', var, setup[var]) - ret = ['BASE'] + setup['ADD_MODULES'].upper().split() + setup['MODULES'].upper().split() - logger.debug('Effective module list: %s', ret) - return ret - -zip7 = r'C:\Program Files\7-Zip\7z' -make = '' -isbase314 = False -has_test_results = False -silent_dep_builds = True - -def host_info(): - print('{0}AppVeyor Build Worker Image:{1} {2}' - .format(ANSI_CYAN, ANSI_RESET, os.environ['APPVEYOR_BUILD_WORKER_IMAGE'])) - - print('{0}Python setup{1}'.format(ANSI_CYAN, ANSI_RESET)) - print(sys.version) - print('PYTHONPATH') - for dname in sys.path: - print(' ', dname) - print('platform =', distutils.util.get_platform()) - - print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET)) - for key in vcvars_table: - if os.path.exists(vcvars_table[key]): - print('Found', key, 'in', vcvars_table[key]) - sys.stdout.flush() - -# Used from unittests -def clear_lists(): - global isbase314, has_test_results - del seen_setups[:] - del modules_to_compile[:] - setup.clear() - places.clear() - isbase314 = False - has_test_results = False - -# Error-handler to make shutil.rmtree delete read-only files on Windows -def remove_readonly(func, path, excinfo): - os.chmod(path, stat.S_IWRITE) - func(path) - -# source_set(setup) -# -# Source a settings file (extension .set) found in the setup_dirs path -# May be called recursively (from within a setup file) -def source_set(name): - # allowed separators: colon or whitespace - setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() - if len(setup_dirs) == 0: - raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) - - for set_dir in setup_dirs: - set_file = os.path.join(set_dir, name) + ".set" - - if set_file in seen_setups: - print("Ignoring already included setup file {0}".format(set_file)) - return - - if os.path.isfile(set_file): - seen_setups.append(set_file) - print("Loading setup file {0}".format(set_file)) - sys.stdout.flush() - with open(set_file) as fp: - for line in fp: - logger.debug('Next line: %s', line.strip()) - if not line.strip() or line.strip()[0] == '#': - continue - if line.startswith("include"): - logger.debug('Found an include, reading %s', line.split()[1]) - source_set(line.split()[1]) - continue - assign = line.replace('"', '').strip().split("=", 1) - logger.debug('Interpreting as assignment') - setup.setdefault(assign[0], os.getenv(assign[0], "")) - if not setup[assign[0]].strip(): - logger.debug('Doing assignment: %s = %s', assign[0], assign[1]) - setup[assign[0]] = assign[1] - break - else: - raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" - .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) - -# update_release_local(var, location) -# var name of the variable to set in RELEASE.local -# location location (absolute path) of where variable should point to -# -# Manipulate RELEASE.local in the cache location: -# - replace "$var=$location" line if it exists and has changed -# - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end -# Set places[var] = location -def update_release_local(var, location): - release_local = os.path.join(cachedir, 'RELEASE.local') - updated_line = '{0}={1}'.format(var, location.replace('\\', '/')) - places[var] = location - - if not os.path.exists(release_local): - logger.debug('RELEASE.local does not exist, creating it') - try: - os.makedirs(cachedir) - except: - pass - fout = open(release_local, 'w') - fout.close() - base_line = '' - found = False - logger.debug("Opening RELEASE.local for adding '%s'", updated_line) - for line in fileinput.input(release_local, inplace=1): - outputline = line.strip() - if 'EPICS_BASE=' in line: - base_line = line.strip() - logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) - continue - elif '{0}='.format(var) in line: - logger.debug("Found '%s=' line, replacing", var) - found = True - outputline = updated_line - logger.debug("Writing line to RELEASE.local: '%s'", outputline) - print(outputline) - fileinput.close() - fout = open(release_local,"a") - if not found: - logger.debug("Adding new definition: '%s'", updated_line) - print(updated_line, file=fout) - if base_line: - logger.debug("Writing EPICS_BASE line: '%s'", base_line) - print(base_line, file=fout) - fout.close() - -def set_setup_from_env(dep): - for postf in ['', '_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', - '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: - if dep+postf in os.environ: - setup[dep+postf] = os.environ[dep+postf] - logger.debug('ENV assignment: %s = %s', dep+postf, setup[dep+postf]) - -def call_git(args, **kws): - if 'cwd' in kws: - place = kws['cwd'] - else: - place = os.getcwd() - logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), place) - sys.stdout.flush() - exitcode = sp.call(['git'] + args, **kws) - logger.debug('EXEC DONE') - return exitcode - -def call_make(args=[], **kws): - place = kws.get('cwd', os.getcwd()) - parallel = kws.pop('parallel', 2) - silent = kws.pop('silent', False) - # no parallel make for Base 3.14 - if parallel <= 0 or isbase314: - makeargs = [] - else: - makeargs = ['-j{0}'.format(parallel), '-Otarget'] - if silent: - makeargs += ['-s'] - logger.debug("EXEC '%s' in %s", ' '.join([make] + makeargs + args), place) - sys.stdout.flush() - exitcode = sp.call([make] + makeargs + args, **kws) - logger.debug('EXEC DONE') - if exitcode != 0: - sys.exit(exitcode) - -def get_git_hash(place): - logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) - sys.stdout.flush() - head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H'], cwd=place).decode() - logger.debug('EXEC DONE') - return head - -def complete_setup(dep): - set_setup_from_env(dep) - setup.setdefault(dep, 'master') - setup.setdefault(dep+"_DIRNAME", dep.lower()) - setup.setdefault(dep+"_REPONAME", dep.lower()) - setup.setdefault('REPOOWNER', 'epics-modules') - setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) - setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' - .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) - setup.setdefault(dep+"_VARNAME", dep) - setup.setdefault(dep+"_RECURSIVE", 'YES') - setup.setdefault(dep+"_DEPTH", -1) - -# add_dependency(dep, tag) -# -# Add a dependency to the cache area: -# - check out (recursive if configured) in the CACHE area unless it already exists and the -# required commit has been built -# - Defaults: -# $dep_DIRNAME = lower case ($dep) -# $dep_REPONAME = lower case ($dep) -# $dep_REPOURL = GitHub / $dep_REPOOWNER (or $REPOOWNER or epics-modules) / $dep_REPONAME .git -# $dep_VARNAME = $dep -# $dep_DEPTH = 5 -# $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) -# - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) -# - Add full path to $modules_to_compile -def add_dependency(dep): - recurse = setup[dep+'_RECURSIVE'].lower() - if recurse not in ['0', 'no']: - recursearg = ["--recursive"] - elif recurse not in ['1', 'yes']: - recursearg = [] - else: - raise RuntimeError("Invalid value for {}_RECURSIVE='{}' not 0/NO/1/YES".format(dep, recurse)) - deptharg = { - '-1':['--depth', '5'], - '0':[], - }.get(str(setup[dep+'_DEPTH']), ['--depth', str(setup[dep+'_DEPTH'])]) - - tag = setup[dep] - - logger.debug('Adding dependency %s with tag %s', dep, setup[dep]) - - # determine if dep points to a valid release or branch - if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]): - raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" - .format(ANSI_RED, tag, dep, setup[dep+'_REPOURL'], ANSI_RESET)) - - dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag) - place = os.path.join(cachedir, dirname) - checked_file = os.path.join(place, "checked_out") - - if os.path.isdir(place): - logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) - # check HEAD commit against the hash in marker file - if os.path.exists(checked_file): - with open(checked_file, 'r') as bfile: - checked_out = bfile.read().strip() - bfile.close() - else: - checked_out = 'never' - head = get_git_hash(place) - logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) - if head != checked_out: - logger.debug('Dependency %s out of date - removing', dep) - shutil.rmtree(place, onerror=remove_readonly) - else: - print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) - sys.stdout.flush() - - if not os.path.isdir(place): - if not os.path.isdir(cachedir): - os.makedirs(cachedir) - # clone dependency - print('Cloning {0} of dependency {1} into {2}' - .format(tag, dep, place)) - sys.stdout.flush() - call_git(['clone', '--quiet'] + deptharg + recursearg + ['--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) - - sp.check_call(['git', 'log', '-n1'], cwd=place) - modules_to_compile.append(place) - - if dep == 'BASE': - # add MSI 1.7 to Base 3.14 - versionfile = os.path.join(place, 'configure', 'CONFIG_BASE_VERSION') - if os.path.exists(versionfile): - with open(versionfile) as f: - if 'BASE_3_14=YES' in f.read(): - print('Adding MSI 1.7 to {0}'.format(place)) - sys.stdout.flush() - sp.check_call(['patch', '-p1', '-i', os.path.join(ciscriptsdir, 'add-msi-to-314.patch')], - cwd=place) - else: - # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE - release = os.path.join(place, "configure", "RELEASE") - if os.path.exists(release): - with open(release, 'w') as fout: - print('-include $(TOP)/../RELEASE.local', file=fout) - - # run hook if defined - if dep+'_HOOK' in setup: - hook = os.path.join(place, setup[dep+'_HOOK']) - if os.path.exists(hook): - print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) - sys.stdout.flush() - sp.check_call(hook, shell=True, cwd=place) - - # write checked out commit hash to marker file - head = get_git_hash(place) - logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) - with open(checked_file, "w") as fout: - print(head, file=fout) - fout.close() - - update_release_local(setup[dep+"_VARNAME"], place) - -def setup_for_build(args): - global make, isbase314, has_test_results - dllpaths = [] - - # there is no combined static and debug EPICS_HOST_ARCH target, - # so a combined debug and static target will appear to be just static - # but debug will have been specified in CONFIG_SITE by prepare() - hostarchsuffix='' - if re.search('debug', os.environ['CONFIGURATION']): - hostarchsuffix = '-debug' - if re.search('static', os.environ['CONFIGURATION']): - hostarchsuffix = '-static' - - if os.environ['PLATFORM'].lower() == 'x86': - os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + hostarchsuffix - elif os.environ['PLATFORM'].lower() == 'x64': - os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + hostarchsuffix - - if os.environ['CMP'] == 'vs2019': - # put strawberry perl in the PATH - os.environ['PATH'] = os.pathsep.join([os.path.join(r'C:\Strawberry\perl\site\bin'), - os.path.join(r'C:\Strawberry\perl\bin'), - os.environ['PATH']]) - if os.environ['CMP'] == 'mingw': - if 'INCLUDE' not in os.environ: - os.environ['INCLUDE'] = '' - if os.environ['PLATFORM'].lower() == 'x86': - os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' - os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', - os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', - os.environ['PATH']]) - elif os.environ['PLATFORM'].lower() == 'x64': - os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' - os.environ['INCLUDE'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', - os.environ['INCLUDE']]) - os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', - os.environ['PATH']]) - - make = os.path.join(toolsdir, 'make.exe') - - base_place = '.' - if not building_base: - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: - lines = f.readlines() - for line in lines: - (mod, place) = line.strip().split('=') - bindir = os.path.join(place, 'bin', os.environ['EPICS_HOST_ARCH']) - if os.path.isdir(bindir): - dllpaths.append(bindir) - if mod == 'EPICS_BASE': - base_place = place - - cfg_base_version = os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION') - if os.path.exists(cfg_base_version): - with open(cfg_base_version) as myfile: - if 'BASE_3_14=YES' in myfile.read(): - isbase314 = True - - if not isbase314: - rules_build = os.path.join(base_place, 'configure', 'RULES_BUILD') - if os.path.exists(rules_build): - with open(rules_build) as myfile: - for line in myfile: - if re.match('^test-results:', line): - has_test_results = True - - bindir = os.path.join(os.getcwd(), 'bin', os.environ['EPICS_HOST_ARCH']) - if os.path.isdir(bindir): - dllpaths.append(bindir) - - os.environ['PATH'] = os.pathsep.join(dllpaths + [os.environ['PATH']]) - - # apparently %CD% is handled automagically - os.environ['TOP'] = os.getcwd() - - addpaths = [] - for path in args.paths: - try: - addpaths.append(path.format(**os.environ)) - except KeyError: - print('Environment') - [print(' ',K,'=',repr(V)) for K,V in os.environ.items()] - raise - - os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths) - -def prepare(args): - host_info() - - print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) - source_set('defaults') - if 'SET' in os.environ: - source_set(os.environ['SET']) - - [complete_setup(mod) for mod in modlist()] - - logger.debug('Loaded setup') - kvs = list(setup.items()) - kvs.sort() - [logger.debug(' %s = "%s"', *kv) for kv in kvs] - - # we're working with tags (detached heads) a lot: suppress advice - call_git(['config', '--global', 'advice.detachedHead', 'false']) - - print('{0}Checking/cloning dependencies{1}'.format(ANSI_YELLOW, ANSI_RESET)) - sys.stdout.flush() - - [add_dependency(mod) for mod in modlist()] - - if not building_base: - if os.path.isdir('configure'): - targetdir = 'configure' - else: - targetdir = '.' - shutil.copy(os.path.join(cachedir, 'RELEASE.local'), targetdir) - - print('{0}Configuring EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) - - with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: - if re.search('static', os.environ['CONFIGURATION']): - config_site.write('SHARED_LIBRARIES=NO\n') - config_site.write('STATIC_BUILD=YES\n') - linktype = 'static' - else: - linktype = 'dynamic (DLL)' - if re.search('debug', os.environ['CONFIGURATION']): - config_site.write('HOST_OPT=NO\n') - optitype = 'debug' - else: - optitype = 'optimized' - - # Enable/fix parallel build for VisualStudio compiler on older Base versions - add_vs_fix = True - config_win = os.path.join(places['EPICS_BASE'], 'configure', 'os', 'CONFIG.win32-x86.win32-x86') - with open(config_win) as myfile: - for line in myfile: - if re.match(r'^ifneq \(\$\(VisualStudioVersion\),11\.0\)', line): - add_vs_fix = False - if add_vs_fix: - with open(config_win, 'a') as myfile: - myfile.write(''' -# Fix parallel build for some VisualStudio versions -ifneq ($(VisualStudioVersion),) -ifneq ($(VisualStudioVersion),11.0) -ifeq ($(findstring -FS,$(OPT_CXXFLAGS_NO)),) - OPT_CXXFLAGS_NO += -FS - OPT_CFLAGS_NO += -FS -endif -else - OPT_CXXFLAGS_NO := $(filter-out -FS,$(OPT_CXXFLAGS_NO)) - OPT_CFLAGS_NO := $(filter-out -FS,$(OPT_CFLAGS_NO)) -endif -endif''') - - print('EPICS Base build system set up for {0} build with {1} linking' - .format(optitype, linktype)) - - if not os.path.isdir(toolsdir): - os.makedirs(toolsdir) - - makever = '4.2.1' - if not os.path.exists(os.path.join(toolsdir, 'make.exe')): - print('Installing Make 4.2.1 from ANL web site') - sys.stdout.flush() - sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-{0}.zip'.format(makever), - 'https://epics.anl.gov/download/tools/make-{0}-win64.zip'.format(makever)], - cwd=toolsdir) - sp.check_call([zip7, 'e', 'make-{0}.zip'.format(makever)], cwd=toolsdir) - os.remove(os.path.join(toolsdir, 'make-{0}.zip'.format(makever))) - - setup_for_build(args) - - print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET)) - print('{0}$ {1} --version{2}'.format(ANSI_CYAN, make, ANSI_RESET)) - sys.stdout.flush() - call_make(['--version'], parallel=0) - print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET)) - sys.stdout.flush() - sp.check_call(['perl', '--version']) - - if os.environ['CMP'] == 'mingw': - print('{0}$ gcc --version{1}'.format(ANSI_CYAN, ANSI_RESET)) - sys.stdout.flush() - sp.check_call(['gcc', '--version']) - else: - print('{0}$ cl{1}'.format(ANSI_CYAN, ANSI_RESET)) - sys.stdout.flush() - sp.check_call(['cl']) - - if not building_base: - for mod in modlist(): - place = places[setup[mod+"_VARNAME"]] - print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) - call_make(cwd=place, silent=silent_dep_builds) - - print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET)) - print('Module Tag Binaries Commit') - print(100 * '-') - for mod in modlist(): - commit = sp.check_output(['git', 'log', '-n1', '--oneline'], cwd=places[setup[mod+"_VARNAME"]]).strip() - print("%-10s %-12s %-11s %s" % (mod, setup[mod], 'rebuilt', commit)) - - print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET)) - with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: - print(f.read().strip()) - -def build(args): - setup_for_build(args) - print('{0}Building the main module{1}'.format(ANSI_YELLOW, ANSI_RESET)) - call_make(args.makeargs) - -def test(args): - setup_for_build(args) - print('{0}Running the main module tests{1}'.format(ANSI_YELLOW, ANSI_RESET)) - if has_test_results: - call_make(['tapfiles']) - call_make(['test-results'], parallel=0, silent=True) - else: - call_make(['runtests']) - -def doExec(args): - 'exec user command with vcvars' - setup_for_build(args) - os.environ['MAKE'] = make - print('Execute command {}'.format(args.cmd)) - sys.stdout.flush() - sp.check_call(' '.join(args.cmd), shell=True) - -def with_vcvars(cmd): - '''re-exec main script with a (hopefully different) command - ''' - CC = os.environ['CMP'] - - # cf. https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line - - info = { - 'python': sys.executable, - 'self': sys.argv[0], - 'cmd':cmd, - } - - info['arch'] = { - 'x86': 'x86', # 'amd64_x86' ?? - 'x64': 'amd64', - }[os.environ['PLATFORM'].lower()] # 'x86' or 'x64' - - info['vcvars'] = vcvars_table[CC] - - script=''' -call "{vcvars}" {arch} - -"{python}" "{self}" {cmd} -'''.format(**info) - - logger.debug('----- Creating vcvars-trampoline.bat -----') - for line in script.split('\n'): - logger.debug(line) - logger.debug('----- snip -----') - - with open('vcvars-trampoline.bat', 'w') as F: - F.write(script) - - print('{0}Calling vcvars-trampoline.bat to set environment for {1} on {2}{3}' - .format(ANSI_YELLOW, CC, os.environ['PLATFORM'], ANSI_RESET)) - sys.stdout.flush() - returncode = sp.call('vcvars-trampoline.bat', shell=True) - if returncode != 0: - sys.exit(returncode) - -def getargs(): - from argparse import ArgumentParser, REMAINDER - P = ArgumentParser() - P.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false', - help='Assume vcvarsall.bat has already been run') - P.add_argument('--add-path', dest='paths', default=[], action='append', - help='Append directory to %PATH%. Expands {ENVVAR}') - SP = P.add_subparsers() - - CMD = SP.add_parser('prepare') - CMD.set_defaults(func=prepare) - - CMD = SP.add_parser('build') - CMD.add_argument('makeargs', nargs=REMAINDER) - CMD.set_defaults(func=build) - - CMD = SP.add_parser('test') - CMD.set_defaults(func=test) - - CMD = SP.add_parser('exec') - CMD.add_argument('cmd', nargs=REMAINDER) - CMD.set_defaults(func=doExec) - - return P - -def main(raw): - global silent_dep_builds - args = getargs().parse_args(raw) - if 'VV' in os.environ and os.environ['VV'] == '1': - logging.basicConfig(level=logging.DEBUG) - silent_dep_builds = False - - if args.vcvars and os.environ['CMP'].startswith('vs'): - # re-exec with MSVC in PATH - with_vcvars(' '.join(['--no-vcvars']+raw)) - - else: - args.func(args) - -if __name__=='__main__': - main(sys.argv[1:]) diff --git a/cue-test.py b/cue-test.py new file mode 100644 index 0000000..4f7124d --- /dev/null +++ b/cue-test.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python +"""Module ci-scripts unit tests +""" + +# SET=test00 in the environment and run the tests in this script +# all other jobs are started as compile jobs + +from __future__ import print_function + +import sys, os, shutil, fileinput +import distutils.util +import re +import subprocess as sp +import unittest +import logging +from argparse import Namespace + +builddir = os.getcwd() + +# Detect basic context (service, os) +if 'TRAVIS' in os.environ: + ci_service = 'travis' + ci_os = os.environ['TRAVIS_OS_NAME'] + +if 'APPVEYOR' in os.environ: + ci_service = 'appveyor' + if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci_os = 'windows' + elif re.match(r'^Ubuntu', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci_os = 'linux' + elif re.match(r'^macOS', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci_os = 'osx' + + +def find_in_file(regex, filename): + file = open(filename, "r") + for line in file: + if re.search(regex, line): + return True + return False + + +def getStringIO(): + if sys.version_info > (3, 0): + import io + return io.StringIO() + else: + import StringIO + return StringIO.StringIO() + + +sys.path.append('.') +import cue + +# we're working with tags (detached heads) a lot: suppress advice +cue.call_git(['config', '--global', 'advice.detachedHead', 'false']) + + +class TestSourceSet(unittest.TestCase): + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if 'BASE' in os.environ: + del os.environ['BASE'] + cue.clear_lists() + os.chdir(builddir) + + def test_EmptySetupDirsPath(self): + del os.environ['SETUP_PATH'] + self.assertRaisesRegexp(NameError, '\(SETUP_PATH\) is empty', cue.source_set, 'test01') + + def test_InvalidSetupName(self): + self.assertRaisesRegexp(NameError, 'does not exist in SETUP_PATH', cue.source_set, 'xxdoesnotexistxx') + + def test_ValidSetupName(self): + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.source_set('test01') + sys.stdout = sys.__stdout__ + self.assertEqual(cue.setup['BASE'], '7.0', 'BASE was not set to \'7.0\'') + + def test_SetupDoesNotOverridePreset(self): + os.environ['BASE'] = 'foo' + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.source_set('test01') + sys.stdout = sys.__stdout__ + self.assertEqual(cue.setup['BASE'], 'foo', + 'Preset BASE was overridden by test01 setup (expected \'foo\' got {0})' + .format(cue.setup['BASE'])) + + def test_IncludeSetupFirstSetWins(self): + captured_output = getStringIO() + sys.stdout = captured_output + cue.source_set('test02') + sys.stdout = sys.__stdout__ + self.assertEqual(cue.setup['BASE'], 'foo', + 'BASE set in test02 was overridden by test01 setup (expected \'foo\' got {0})' + .format(cue.setup['BASE'])) + self.assertEqual(cue.setup['FOO'], 'bar', 'Setting of single word does not work') + self.assertEqual(cue.setup['FOO2'], 'bar bar2', 'Setting of multiple words does not work') + self.assertEqual(cue.setup['FOO3'], 'bar bar2', 'Indented setting of multiple words does not work') + self.assertEqual(cue.setup['SNCSEQ'], 'R2-2-8', 'Setup test01 was not included') + + def test_DoubleIncludeGetsIgnored(self): + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.source_set('test03') + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), 'Ignoring already included setup file') + + +class TestUpdateReleaseLocal(unittest.TestCase): + release_local = os.path.join(cue.cachedir, 'RELEASE.local') + + def setUp(self): + if os.path.exists(self.release_local): + os.remove(self.release_local) + os.chdir(builddir) + + def test_SetModule(self): + cue.update_release_local('MOD1', '/foo/bar') + found = 0 + for line in fileinput.input(self.release_local, inplace=1): + if 'MOD1=' in line: + self.assertEqual(line.strip(), 'MOD1=/foo/bar', 'MOD1 not set correctly') + found += 1 + fileinput.close() + self.assertEqual(found, 1, 'MOD1 not written once to RELEASE.local (found {0})'.format(found)) + + def test_SetBaseAndMultipleModules(self): + cue.update_release_local('EPICS_BASE', '/bar/foo') + cue.update_release_local('MOD1', '/foo/bar') + cue.update_release_local('MOD2', '/foo/bar2') + cue.update_release_local('MOD1', '/foo/bar1') + found = {} + foundat = {} + for line in fileinput.input(self.release_local, inplace=1): + if 'MOD1=' in line: + self.assertEqual(line.strip(), 'MOD1=/foo/bar1', + 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' + .format(line)) + if 'mod1' in found: + found['mod1'] += 1 + else: + found['mod1'] = 1 + foundat['mod1'] = fileinput.filelineno() + if 'MOD2=' in line: + self.assertEqual(line.strip(), 'MOD2=/foo/bar2', + 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' + .format(line)) + if 'mod2' in found: + found['mod2'] += 1 + else: + found['mod2'] = 1 + foundat['mod2'] = fileinput.filelineno() + if 'EPICS_BASE=' in line: + self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', + 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' + .format(line)) + if 'base' in found: + found['base'] += 1 + else: + found['base'] = 1 + foundat['base'] = fileinput.filelineno() + fileinput.close() + self.assertEqual(found['mod1'], 1, + 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) + self.assertEqual(found['mod2'], 1, + 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) + self.assertEqual(found['base'], 1, + 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) + self.assertGreater(foundat['base'], foundat['mod2'], + 'EPICS_BASE (line {0}) appears before MOD2 (line {1})' + .format(foundat['base'], foundat['mod2'])) + self.assertGreater(foundat['mod2'], foundat['mod1'], + 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) + + +class TestAddDependencyUpToDateCheck(unittest.TestCase): + hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" + location = os.path.join(cue.cachedir, 'base-R3.15.6') + licensefile = os.path.join(location, 'LICENSE') + checked_file = os.path.join(location, 'checked_out') + release_file = os.path.join(location, 'configure', 'RELEASE') + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if os.path.exists(self.location): + shutil.rmtree(self.location, onerror=cue.remove_readonly) + cue.clear_lists() + os.chdir(builddir) + cue.source_set('defaults') + cue.complete_setup('BASE') + + def test_MissingDependency(self): + cue.setup['BASE'] = 'R3.15.6' + cue.add_dependency('BASE') + self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out') + self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + 'Wrong commit of dependency checked out (expected=\"{0}\" found=\"{1}\")' + .format(self.hash_3_15_6, checked_out)) + self.assertFalse(find_in_file('include \$\(TOP\)/../RELEASE.local', self.release_file), + 'RELEASE in Base includes TOP/../RELEASE.local') + + def test_UpToDateDependency(self): + cue.setup['BASE'] = 'R3.15.6' + cue.add_dependency('BASE') + os.remove(self.licensefile) + cue.add_dependency('BASE') + self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency') + + def test_OutdatedDependency(self): + cue.setup['BASE'] = 'R3.15.6' + cue.add_dependency('BASE') + os.remove(self.licensefile) + with open(self.checked_file, "w") as fout: + print('XXX not the right hash XXX', file=fout) + fout.close() + cue.add_dependency('BASE') + self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + "Wrong commit of dependency checked out (expected='{0}' found='{1}')" + .format(self.hash_3_15_6, checked_out)) + + +def is_shallow_repo(place): + check = sp.check_output(['git', 'rev-parse', '--is-shallow-repository'], cwd=place).strip().decode('ascii') + if check == '--is-shallow-repository': + if os.path.exists(os.path.join(place, '.git', 'shallow')): + check = 'true' + else: + check = 'false' + return check == 'true' + + +class TestAddDependencyOptions(unittest.TestCase): + location = os.path.join(cue.cachedir, 'mcoreutils-master') + testfile = os.path.join(location, '.ci', 'LICENSE') + + def setUp(self): + os.environ['SETUP_PATH'] = '.' + if os.path.exists(cue.cachedir): + shutil.rmtree(cue.cachedir, onerror=cue.remove_readonly) + cue.clear_lists() + cue.detect_context() + cue.source_set('defaults') + cue.complete_setup('MCoreUtils') + cue.setup['MCoreUtils'] = 'master' + + def test_Default(self): + cue.add_dependency('MCoreUtils') + self.assertTrue(os.path.exists(self.testfile), + 'Submodule (.ci) not checked out recursively (requested: default=YES') + self.assertTrue(is_shallow_repo(self.location), + 'Module not checked out shallow (requested: default=5)') + + def test_SetRecursiveNo(self): + cue.setup['MCoreUtils_RECURSIVE'] = 'NO' + cue.add_dependency('MCoreUtils') + self.assertFalse(os.path.exists(self.testfile), 'Submodule (.ci) checked out recursively') + + def test_SetDepthZero(self): + cue.setup['MCoreUtils_DEPTH'] = '0' + cue.add_dependency('MCoreUtils') + self.assertFalse(is_shallow_repo(self.location), 'Module checked out shallow (requested full)') + + def test_SetDepthThree(self): + cue.setup['MCoreUtils_DEPTH'] = '3' + cue.add_dependency('MCoreUtils') + self.assertTrue(is_shallow_repo(self.location), + 'Module not checked out shallow (requested: depth=3)') + + def test_AddMsiTo314(self): + cue.complete_setup('BASE') + cue.setup['BASE'] = 'R3.14.12.1' + msifile = os.path.join(cue.cachedir, 'base-R3.14.12.1', 'src', 'dbtools', 'msi.c') + cue.add_dependency('BASE') + self.assertTrue(os.path.exists(msifile), 'MSI was not added to Base 3.14') + + def test_DefaultBaseBranch(self): + cue.complete_setup('BASE') + self.assertEqual(cue.setup['BASE'], '7.0', + 'Default Base branch is not 7.0 (found {0})'.format(cue.setup['BASE'])) + + +def repo_access(dep): + cue.set_setup_from_env(dep) + cue.setup.setdefault(dep + "_DIRNAME", dep.lower()) + cue.setup.setdefault(dep + "_REPONAME", dep.lower()) + cue.setup.setdefault('REPOOWNER', 'epics-modules') + cue.setup.setdefault(dep + "_REPOOWNER", cue.setup['REPOOWNER']) + cue.setup.setdefault(dep + "_REPOURL", 'https://github.com/{0}/{1}.git' + .format(cue.setup[dep + '_REPOOWNER'], cue.setup[dep + '_REPONAME'])) + with open(os.devnull, 'w') as devnull: + return cue.call_git(['ls-remote', '--quiet', '--heads', cue.setup[dep + '_REPOURL']], + stdout=devnull, stderr=devnull) + + +class TestDefaultModuleURLs(unittest.TestCase): + modules = ['BASE', 'PVDATA', 'PVACCESS', 'NTYPES', + 'SNCSEQ', 'STREAM', 'ASYN', 'STD', + 'CALC', 'AUTOSAVE', 'BUSY', 'SSCAN', + 'IOCSTATS', 'MOTOR', 'IPAC', ] + + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + cue.clear_lists() + os.chdir(builddir) + cue.source_set('defaults') + + def test_Repos(self): + for mod in self.modules: + self.assertEqual(repo_access(mod), 0, 'Defaults for {0} do not point to a valid git repository at {1}' + .format(mod, cue.setup[mod + '_REPOURL'])) + +@unittest.skipIf(ci_os != 'windows', 'VCVars test only applies to windows') +class TestVCVars(unittest.TestCase): + def test_vcvars(self): + if ci_service == 'appveyor': + os.environ['CONFIGURATION'] = 'default' + cue.detect_context() + cue.with_vcvars('env') + + +@unittest.skipIf(ci_service != 'travis', 'Run travis tests only on travis') +class TestTravisDetectContext(unittest.TestCase): + def setUp(self): + os.environ['TRAVIS'] = 'true' + os.environ['TRAVIS_OS_NAME'] = 'linux' + os.environ['TRAVIS_COMPILER'] = 'gcc' + + def tearDown(self): + cue.clear_lists() + os.environ.pop('BCFG', None) + os.environ.pop('TEST', None) + os.environ.pop('STATIC', None) + + def test_LinuxGccNone(self): + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'linux', "ci['os'] is {0} (expected: linux)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'gcc', "ci['compiler'] is {0} (expected: gcc)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + + def test_LinuxClangNone(self): + os.environ['TRAVIS_COMPILER'] = 'clang' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'linux', "ci['os'] is {0} (expected: linux)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'clang', "ci['compiler'] is {0} (expected: clang)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + + def test_BcfgShared(self): + os.environ['BCFG'] = 'shared' + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + + def test_BcfgStatic(self): + os.environ['BCFG'] = 'static' + cue.detect_context() + self.assertTrue(cue.ci['static'], "ci['static'] is False (expected: True)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'static-optimized', + "ci['configuration'] is {0} (expected: static-optimized)" + .format(cue.ci['configuration'])) + + def test_BcfgDebug(self): + os.environ['BCFG'] = 'debug' + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertTrue(cue.ci['debug'], "ci['debug'] is False (expected: True)") + self.assertEqual(cue.ci['configuration'], 'shared-debug', + "ci['configuration'] is {0} (expected: shared-debug)" + .format(cue.ci['configuration'])) + + def test_BcfgStaticDebug(self): + os.environ['BCFG'] = 'static-debug' + cue.detect_context() + self.assertTrue(cue.ci['static'], "ci['static'] is False (expected: True)") + self.assertTrue(cue.ci['debug'], "ci['debug'] is False (expected: True)") + self.assertEqual(cue.ci['configuration'], 'static-debug', + "ci['configuration'] is {0} (expected: static-debug)" + .format(cue.ci['configuration'])) + + def test_TestNo(self): + os.environ['TEST'] = 'NO' + cue.detect_context() + self.assertFalse(cue.ci['test'], "ci['test'] is True (expected: False)") + + def test_WindowsGccNone(self): + os.environ['TRAVIS_OS_NAME'] = 'windows' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'gcc', "ci['compiler'] is {0} (expected: gcc)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('strawberryperl', cue.ci['choco'], "'strawberryperl' is not in ci['choco']") + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_WindowsVs2017None(self): + os.environ['TRAVIS_OS_NAME'] = 'windows' + os.environ['TRAVIS_COMPILER'] = 'vs2017' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'vs2017', "ci['compiler'] is {0} (expected: vs2017)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('strawberryperl', cue.ci['choco'], "'strawberryperl' is not in ci['choco']") + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_WindowsVs2019None(self): + os.environ['TRAVIS_OS_NAME'] = 'windows' + os.environ['TRAVIS_COMPILER'] = 'vs2019' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'vs2017', "ci['compiler'] is {0} (expected: vs2017)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('strawberryperl', cue.ci['choco'], "'strawberryperl' is not in ci['choco']") + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_OsxClangNone(self): + os.environ['TRAVIS_OS_NAME'] = 'osx' + os.environ['TRAVIS_COMPILER'] = 'clang' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'travis', "ci['service'] is {0} (expected: travis)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'osx', "ci['os'] is {0} (expected: osx)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'clang', "ci['compiler'] is {0} (expected: clang)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + + def test_StaticGetsWarning(self): + os.environ['STATIC'] = 'YES' + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.detect_context() + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore") + + def test_MisspelledBcfgGetsWarning(self): + os.environ['BCFG'] = 'static-dubug' + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.detect_context() + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting") + + +@unittest.skipIf(ci_service != 'appveyor', 'Run appveyor tests only on appveyor') +class TestAppveyorDetectContext(unittest.TestCase): + def setUp(self): + os.environ['APPVEYOR'] = 'True' + os.environ['APPVEYOR_BUILD_WORKER_IMAGE'] = 'Visual Studio 2019' + os.environ['CMP'] = 'vs2019' + os.environ['CONFIGURATION'] = 'default' + os.environ['PLATFORM'] = 'x64' + + def tearDown(self): + cue.clear_lists() + os.environ.pop('STATIC', None) + os.environ.pop('TEST', None) + + def test_Platform32(self): + os.environ['PLATFORM'] = 'x86' + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertEqual(cue.ci['platform'], 'x86', + "ci['platform'] is {0} (expected: x86)" + .format(cue.ci['platform'])) + + def test_Platform64(self): + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertEqual(cue.ci['platform'], 'x64', + "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + + def test_PlatformX64(self): + os.environ['PLATFORM'] = 'X64' + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertEqual(cue.ci['platform'], 'x64', + "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + + def test_ConfigDefault(self): + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + + def test_ConfigStatic(self): + os.environ['CONFIGURATION'] = 'static' + cue.detect_context() + self.assertTrue(cue.ci['static'], "ci['static'] is False (expected: True)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'static-optimized', + "ci['configuration'] is {0} (expected: static-optimized)" + .format(cue.ci['configuration'])) + + def test_ConfigDebug(self): + os.environ['CONFIGURATION'] = 'debug' + cue.detect_context() + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertTrue(cue.ci['debug'], "ci['debug'] is False (expected: True)") + self.assertEqual(cue.ci['configuration'], 'shared-debug', + "ci['configuration'] is {0} (expected: shared-debug)" + .format(cue.ci['configuration'])) + + def test_ConfigStaticDebug(self): + os.environ['CONFIGURATION'] = 'static-debug' + cue.detect_context() + self.assertTrue(cue.ci['static'], "ci['static'] is False (expected: True)") + self.assertTrue(cue.ci['debug'], "ci['debug'] is False (expected: True)") + self.assertEqual(cue.ci['configuration'], 'static-debug', + "ci['configuration'] is {0} (expected: static-debug)" + .format(cue.ci['configuration'])) + + def test_TestNo(self): + os.environ['TEST'] = 'NO' + cue.detect_context() + self.assertFalse(cue.ci['test'], "ci['test'] is True (expected: False)") + + def test_WindowsGccNone(self): + os.environ['CMP'] = 'gcc' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'appveyor', "ci['service'] is {0} (expected: appveyor)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'gcc', "ci['compiler'] is {0} (expected: gcc)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_WindowsVs2017None(self): + os.environ['APPVEYOR_BUILD_WORKER_IMAGE'] = 'Visual Studio 2017' + os.environ['CMP'] = 'vs2017' + os.environ['PLATFORM'] = 'x86' + cue.detect_context() + self.assertEqual(cue.ci['service'], 'appveyor', "ci['service'] is {0} (expected: appveyor)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'vs2017', "ci['compiler'] is {0} (expected: vs2017)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x86', "ci['platform'] is {0} (expected: x86)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_WindowsVs2019None(self): + cue.detect_context() + self.assertEqual(cue.ci['service'], 'appveyor', "ci['service'] is {0} (expected: appveyor)" + .format(cue.ci['service'])) + self.assertEqual(cue.ci['os'], 'windows', "ci['os'] is {0} (expected: windows)" + .format(cue.ci['os'])) + self.assertEqual(cue.ci['compiler'], 'vs2019', "ci['compiler'] is {0} (expected: vs2019)" + .format(cue.ci['compiler'])) + self.assertEqual(cue.ci['platform'], 'x64', "ci['platform'] is {0} (expected: x64)" + .format(cue.ci['platform'])) + self.assertFalse(cue.ci['static'], "ci['static'] is True (expected: False)") + self.assertFalse(cue.ci['debug'], "ci['debug'] is True (expected: False)") + self.assertEqual(cue.ci['configuration'], 'shared-optimized', + "ci['configuration'] is {0} (expected: shared-optimized)" + .format(cue.ci['configuration'])) + self.assertIn('make', cue.ci['choco'], "'make' is not in ci['choco']") + + def test_StaticGetsWarning(self): + os.environ['STATIC'] = 'YES' + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.detect_context() + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), "Variable 'STATIC' not supported anymore") + + def test_MisspelledConfigurationGetsWarning(self): + os.environ['CONFIGURATION'] = 'static-dubug' + capturedOutput = getStringIO() + sys.stdout = capturedOutput + cue.detect_context() + sys.stdout = sys.__stdout__ + self.assertRegexpMatches(capturedOutput.getvalue(), "Unrecognized build configuration setting") + + +class TestSetupForBuild(unittest.TestCase): + args = Namespace(paths=[]) + cue.building_base = True + if ci_os == 'windows': + sp.check_call(['choco', 'install', 'make']) + + def setUp(self): + if ci_service == 'appveyor': + os.environ['CONFIGURATION'] = 'default' + cue.detect_context() + + def tearDown(self): + os.environ.pop('EPICS_HOST_ARCH', None) + cue.clear_lists() + + def test_AddPathsOption(self): + os.environ['FOOBAR'] = 'BAR' + args = Namespace(paths=['/my/{FOOBAR}/dir', '/my/foobar']) + cue.setup_for_build(args) + self.assertTrue(re.search('/my/BAR/dir', os.environ['PATH']), 'Expanded path not in PATH') + self.assertTrue(re.search('/foobar', os.environ['PATH']), 'Plain path not in PATH') + os.environ.pop('FOOBAR', None) + + @unittest.skipIf(ci_os != 'windows', 'HostArchConfiguration test only applies to windows') + def test_HostArchConfiguration(self): + cue.ci['compiler'] = 'vs2017' + for cue.ci['debug'] in [True, False]: + for cue.ci['static'] in [True, False]: + config_st = {True: 'static', False: 'shared'} + config_db = {True: '-debug', False: '-optimized'} + config = config_st[cue.ci['static']] + config_db[cue.ci['debug']] + cue.setup_for_build(self.args) + self.assertTrue('EPICS_HOST_ARCH' in os.environ, + 'EPICS_HOST_ARCH is not set for Configuration={0}'.format(config)) + if cue.ci['static']: + self.assertTrue(re.search('-static$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is not -static for Configuration={0}'.format(config)) + self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) + elif cue.ci['debug']: + self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH (found {0}) is -static for Configuration={1}' + .format(os.environ['EPICS_HOST_ARCH'], config)) + self.assertTrue(re.search('-debug$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH (found {0}) is not -debug for Configuration={1}' + .format(os.environ['EPICS_HOST_ARCH'], config)) + else: + self.assertFalse(re.search('static', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -static for Configuration={0}'.format(config)) + self.assertFalse(re.search('debug', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH is -debug for Configuration={0}'.format(config)) + + @unittest.skipIf(ci_os != 'windows', 'HostArchPlatform test only applies to windows') + def test_HostArchPlatform(self): + if ci_service == 'travis': + platforms = ['x64'] + else: + platforms = ['x86', 'x64'] + for platform in platforms: + for cc in ['vs2019', 'gcc']: + cue.ci['platform'] = platform + cue.ci['compiler'] = cc + cue.setup_for_build(self.args) + self.assertTrue('EPICS_HOST_ARCH' in os.environ, + 'EPICS_HOST_ARCH is not set for {0} / {1}' + .format(cc, cue.ci['platform'])) + if platform == 'x86': + self.assertTrue(re.search('^win32-x86', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH (found {0}) is not win32-x86 for {1} / {2}' + .format(os.environ['EPICS_HOST_ARCH'], cc, platform)) + else: + self.assertTrue(re.search('^windows-x64', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH (found {0}) is not windows-x64 for {1} / {2}' + .format(os.environ['EPICS_HOST_ARCH'], cc, platform)) + if cc == 'gcc': + self.assertTrue(re.search('-mingw$', os.environ['EPICS_HOST_ARCH']), + 'EPICS_HOST_ARCH (found {0}) is not -mingw for {1} / {2}' + .format(os.environ['EPICS_HOST_ARCH'], cc, platform)) + pattern = {'x86': 'mingw32', 'x64': 'mingw64'} + self.assertTrue(re.search(pattern[platform], os.environ['PATH']), + 'Binary location for {0} not in PATH (found {1})' + .format(pattern[platform], os.environ['PATH'])) + + @unittest.skipIf(ci_os != 'windows', 'Strawberry perl test only applies to windows') + def test_StrawberryInPathVS2019(self): + if 'APPVEYOR' in os.environ: + os.environ['CMP'] = 'vs2019' + cue.setup_for_build(self.args) + self.assertTrue(re.search('strawberry', os.environ['PATH'], flags=re.IGNORECASE), + 'Strawberry Perl installed but location not in PATH (found {0})' + .format(os.environ['PATH'])) + + def setBase314(self, yesno): + cfg_base_version = os.path.join('configure', 'CONFIG_BASE_VERSION') + fout = open(cfg_base_version, 'w') + print('# test file for base version detection', file=fout) + print('BASE_3_14={0}'.format(yesno), file=fout) + fout.close() + + def setTestResultsTarget(self, target): + rules_build = os.path.join('configure', 'RULES_BUILD') + fout = open(rules_build, 'w') + print('# test file for target detection', file=fout) + print('{0}: something'.format(target), file=fout) + fout.close() + + def test_DetectionBase314No(self): + self.setBase314('NO') + cue.setup_for_build(self.args) + self.assertFalse(cue.is_base314, 'Falsely detected Base 3.14') + + def test_DetectionBase314Yes(self): + self.setBase314('YES') + cue.setup_for_build(self.args) + self.assertTrue(cue.is_base314, 'Base 3.14 = YES not detected') + + def test_DetectionTestResultsTarget314No(self): + self.setBase314('YES') + self.setTestResultsTarget('nottherighttarget') + cue.setup_for_build(self.args) + self.assertFalse(cue.has_test_results, 'Falsely detected test-results target') + + def test_DetectionTestResultsTarget314Yes(self): + self.setBase314('YES') + self.setTestResultsTarget('test-results') + cue.setup_for_build(self.args) + self.assertFalse(cue.has_test_results, 'Falsely found test-results on Base 3.14') + + def test_DetectionTestResultsTargetNot314Yes(self): + self.setBase314('NO') + self.setTestResultsTarget('test-results') + cue.setup_for_build(self.args) + self.assertTrue(cue.has_test_results, 'Target test-results not detected') + + def test_ExtraMakeArgs(self): + os.environ['EXTRA'] = 'bla' + for ind in range(1,5): + os.environ['EXTRA{0}'.format(ind)] = 'bla {0}'.format(ind) + cue.setup_for_build(self.args) + self.assertTrue(cue.extra_makeargs[0] == 'bla', 'Extra make arg [0] not set') + for ind in range(1,5): + self.assertTrue(cue.extra_makeargs[ind] == 'bla {0}'.format(ind), + 'Extra make arg [{0}] not set (expected "bla {0}", found "{1}")' + .format(ind, cue.extra_makeargs[ind])) + +if __name__ == "__main__": + if 'VV' in os.environ and os.environ['VV'] == '1': + logging.basicConfig(level=logging.DEBUG) + cue.silent_dep_builds = False + + cue.detect_context() + cue.host_info() + if sys.argv[1:] == ['env']: + # testing with_vcvars + [print(K, '=', V) for K, V in os.environ.items()] + else: + unittest.main() diff --git a/cue.py b/cue.py new file mode 100644 index 0000000..34d59ad --- /dev/null +++ b/cue.py @@ -0,0 +1,1010 @@ +#!/usr/bin/env python +"""CI build script for Linux/MacOS/Windows on Travis/AppVeyor +""" + +from __future__ import print_function + +import sys, os, stat, shutil +import fileinput +import logging +import re +import subprocess as sp +import distutils.util + +logger = logging.getLogger(__name__) + + +# Detect the service and set up context hash accordingly +def detect_context(): + buildconfig = 'default' + if 'TRAVIS' in os.environ: + ci['service'] = 'travis' + ci['os'] = os.environ['TRAVIS_OS_NAME'] + ci['platform'] = 'x64' + ci['compiler'] = os.environ['TRAVIS_COMPILER'] + if ci['os'] == 'windows': + ci['choco'] += ['strawberryperl'] + if re.match(r'^vs', ci['compiler']): + # Only Visual Studio 2017 available + ci['compiler'] = 'vs2017' + if 'BCFG' in os.environ: + buildconfig = os.environ['BCFG'].lower() + + if 'APPVEYOR' in os.environ: + ci['service'] = 'appveyor' + if re.match(r'^Visual', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci['os'] = 'windows' + elif re.match(r'^Ubuntu', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci['os'] = 'linux' + elif re.match(r'^macOS', os.environ['APPVEYOR_BUILD_WORKER_IMAGE']): + ci['os'] = 'osx' + ci['platform'] = os.environ['PLATFORM'].lower() + if 'CMP' in os.environ: + ci['compiler'] = os.environ['CMP'] + buildconfig = os.environ['CONFIGURATION'].lower() + + if 'STATIC' in os.environ: + print("{0}WARNING: Variable 'STATIC' not supported anymore; use 'BCFG' instead{1}" + .format(ANSI_RED, ANSI_RESET)) + sys.stdout.flush() + if not re.match(r'^((default|static|shared|dynamic|optimized|debug)-?)+$', buildconfig): + print("{0}WARNING: Unrecognized build configuration setting '{1}'{2}" + .format(ANSI_RED, buildconfig, ANSI_RESET)) + sys.stdout.flush() + + if re.search('static', buildconfig): + ci['static'] = True + if re.search('debug', buildconfig): + ci['debug'] = True + + if ci['static']: + ci['configuration'] = 'static' + else: + ci['configuration'] = 'shared' + if ci['debug']: + ci['configuration'] += '-debug' + else: + ci['configuration'] += '-optimized' + + ci['scriptsdir'] = os.path.abspath(os.path.dirname(sys.argv[0])) + + if 'CHOCO' in os.environ: + ci['choco'].extend(os.environ['CHOCO'].split()) + + ci['test'] = True + if 'TEST' in os.environ and os.environ['TEST'].lower() == 'no': + ci['test'] = False + + ci['parallel_make'] = 2 + if 'PARALLEL_MAKE' in os.environ: + ci['parallel_make'] = os.environ['PARALLEL_MAKE'] + + logger.debug('Detected a build hosted on %s, using %s on %s (%s) configured as %s (test: %s)', + ci['service'], ci['compiler'], ci['os'], ci['platform'], ci['configuration'], ci['test']) + + +curdir = os.getcwd() + +ci = {} +seen_setups = [] +modules_to_compile = [] +setup = {} +places = {} +extra_makeargs = [] + +is_base314 = False +is_make3 = False +has_test_results = False +silent_dep_builds = True +do_recompile = False + + +def clear_lists(): + global is_base314, has_test_results, silent_dep_builds, is_make3 + del seen_setups[:] + del modules_to_compile[:] + del extra_makeargs[:] + setup.clear() + places.clear() + is_base314 = False + is_make3 = False + has_test_results = False + silent_dep_builds = True + do_recompile = False + ci['service'] = '' + ci['os'] = '' + ci['platform'] = '' + ci['compiler'] = '' + ci['static'] = False + ci['debug'] = False + ci['configuration'] = '' + ci['scriptsdir'] = '' + ci['choco'] = ['make'] + + +clear_lists() + +if 'BASE' in os.environ and os.environ['BASE'] == 'SELF': + building_base = True + places['EPICS_BASE'] = curdir +else: + building_base = False + +# Setup ANSI Colors +ANSI_RED = "\033[31;1m" +ANSI_GREEN = "\033[32;1m" +ANSI_YELLOW = "\033[33;1m" +ANSI_BLUE = "\033[34;1m" +ANSI_MAGENTA = "\033[35;1m" +ANSI_CYAN = "\033[36;1m" +ANSI_RESET = "\033[0m" +ANSI_CLEAR = "\033[0K" + + +# Travis log fold control +# from https://github.com/travis-ci/travis-rubies/blob/build/build.sh + +def fold_start(tag, title): + if ci['service'] == 'travis': + print('travis_fold:start:{0}{1}{2}{3}' + .format(tag, ANSI_YELLOW, title, ANSI_RESET)) + elif ci['service'] == 'appveyor': + print('{0}===== \\/ \\/ \\/ ===== START: {1} ====={2}' + .format(ANSI_YELLOW, title, ANSI_RESET)) + sys.stdout.flush() + + +def fold_end(tag, title): + if ci['service'] == 'travis': + print('\ntravis_fold:end:{0}\r' + .format(tag), end='') + elif ci['service'] == 'appveyor': + print('{0}----- /\\ /\\ /\\ ----- END: {1} -----{2}' + .format(ANSI_YELLOW, title, ANSI_RESET)) + sys.stdout.flush() + + +homedir = curdir +if 'HomeDrive' in os.environ: + homedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath')) +elif 'HOME' in os.environ: + homedir = os.getenv('HOME') +cachedir = os.path.join(homedir, '.cache') +toolsdir = os.path.join(homedir, '.tools') +rtemsdir = os.path.join(homedir, '.rtems') + +if 'CACHEDIR' in os.environ: + cachedir = os.environ['CACHEDIR'] + + +vcvars_table = { + # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History + 'vs2019': [r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat'], + 'vs2017': [r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat', + r'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat'], + 'vs2015': [r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat'], + 'vs2013': [r'C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat'], + 'vs2012': [r'C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat'], + 'vs2010': [r'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat'], + 'vs2008': [r'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcvarsall.bat'], +} + +vcvars_found = {} +for key in vcvars_table: + for dir in vcvars_table[key]: + if os.path.exists(dir): + vcvars_found[key] = dir + + +def modlist(): + if building_base: + ret = [] + else: + for var in ['ADD_MODULES', 'MODULES']: + setup.setdefault(var, '') + if var in os.environ: + setup[var] = os.environ[var] + logger.debug('ENV assignment: %s = %s', var, setup[var]) + ret = ['BASE'] + setup['ADD_MODULES'].upper().split() + setup['MODULES'].upper().split() + return ret + + +def host_info(): + print('{0}Build using {1} compiler on {2} ({3}) hosted by {4}{5}' + .format(ANSI_CYAN, ci['compiler'], ci['os'], ci['platform'], ci['service'], ANSI_RESET)) + + print('{0}Python setup{1}'.format(ANSI_CYAN, ANSI_RESET)) + print(sys.version) + print('PYTHONPATH') + for dname in sys.path: + print(' ', dname) + print('platform =', distutils.util.get_platform()) + + if ci['os'] == 'windows': + print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET)) + for comp in vcvars_found: + print(comp, 'in', vcvars_found[comp]) + + sys.stdout.flush() + + +# Error-handler to make shutil.rmtree delete read-only files on Windows +def remove_readonly(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + + +# source_set(setup) +# +# Source a settings file (extension .set) found in the setup_dirs path +# May be called recursively (from within a setup file) +def source_set(name): + # allowed separators: colon or whitespace + setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() + if len(setup_dirs) == 0: + raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED, ANSI_RESET)) + + for set_dir in setup_dirs: + set_file = os.path.join(set_dir, name) + ".set" + + if set_file in seen_setups: + print("Ignoring already included setup file {0}".format(set_file)) + return + + if os.path.isfile(set_file): + seen_setups.append(set_file) + print("Opening setup file {0}".format(set_file)) + sys.stdout.flush() + with open(set_file) as fp: + for line in fp: + if not line.strip() or line.strip()[0] == '#': + continue + if line.startswith("include"): + logger.debug('%s: Found include directive, reading %s next', + set_file, line.split()[1]) + source_set(line.split()[1]) + continue + assign = line.replace('"', '').strip().split("=", 1) + setup.setdefault(assign[0], os.getenv(assign[0], "")) + if not setup[assign[0]].strip(): + logger.debug('%s: setup[%s] = %s', set_file, assign[0], assign[1]) + setup[assign[0]] = assign[1] + logger.debug('Done with setup file %s', set_file) + break + else: + raise NameError("{0}Setup file {1}.set does not exist in SETUP_PATH search path ({2}){3}" + .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) + + +# update_release_local(var, location) +# var name of the variable to set in RELEASE.local +# location location (absolute path) of where variable should point to +# +# Manipulate RELEASE.local in the cache location: +# - replace "$var=$location" line if it exists and has changed +# - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end +# Set places[var] = location +def update_release_local(var, location): + release_local = os.path.join(cachedir, 'RELEASE.local') + updated_line = '{0}={1}'.format(var, location.replace('\\', '/')) + places[var] = location + + if not os.path.exists(release_local): + logger.debug('RELEASE.local does not exist, creating it') + try: + os.makedirs(cachedir) + except: + pass + touch = open(release_local, 'w') + touch.close() + base_line = '' + found = False + logger.debug("Opening RELEASE.local for adding '%s'", updated_line) + for line in fileinput.input(release_local, inplace=1): + output_line = line.strip() + if 'EPICS_BASE=' in line: + base_line = line.strip() + logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) + continue + elif '{0}='.format(var) in line: + logger.debug("Found '%s=' line, replacing", var) + found = True + output_line = updated_line + logger.debug("Writing line to RELEASE.local: '%s'", output_line) + print(output_line) + fileinput.close() + release_local = open(release_local, "a") + if not found: + logger.debug("Adding new definition: '%s'", updated_line) + print(updated_line, file=release_local) + if base_line: + logger.debug("Writing EPICS_BASE line: '%s'", base_line) + print(base_line, file=release_local) + release_local.close() + + +def set_setup_from_env(dep): + for postf in ['', '_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', + '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: + if dep + postf in os.environ: + setup[dep + postf] = os.environ[dep + postf] + logger.debug('ENV assignment: %s = %s', dep + postf, setup[dep + postf]) + + +def call_git(args, **kws): + if 'cwd' in kws: + place = kws['cwd'] + else: + place = os.getcwd() + logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), place) + sys.stdout.flush() + exitcode = sp.call(['git'] + args, **kws) + logger.debug('EXEC DONE') + return exitcode + + +def call_make(args=[], **kws): + place = kws.get('cwd', os.getcwd()) + parallel = kws.pop('parallel', ci['parallel_make']) + silent = kws.pop('silent', False) + use_extra = kws.pop('use_extra', False) + # no parallel make for Base 3.14 + if parallel <= 0 or is_base314: + makeargs = [] + else: + makeargs = ['-j{0}'.format(parallel)] + if not is_make3: + makeargs += ['-Otarget'] + if silent: + makeargs += ['-s'] + if use_extra: + makeargs += extra_makeargs + logger.debug("EXEC '%s' in %s", ' '.join(['make'] + makeargs + args), place) + sys.stdout.flush() + exitcode = sp.call(['make'] + makeargs + args, **kws) + logger.debug('EXEC DONE') + if exitcode != 0: + sys.exit(exitcode) + + +def get_git_hash(place): + logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) + sys.stdout.flush() + head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H'], cwd=place).decode() + logger.debug('EXEC DONE') + return head + + +def complete_setup(dep): + set_setup_from_env(dep) + setup.setdefault(dep, 'master') + setup.setdefault(dep + "_DIRNAME", dep.lower()) + setup.setdefault(dep + "_REPONAME", dep.lower()) + setup.setdefault('REPOOWNER', 'epics-modules') + setup.setdefault(dep + "_REPOOWNER", setup['REPOOWNER']) + setup.setdefault(dep + "_REPOURL", 'https://github.com/{0}/{1}.git' + .format(setup[dep + '_REPOOWNER'], setup[dep + '_REPONAME'])) + setup.setdefault(dep + "_VARNAME", dep) + setup.setdefault(dep + "_RECURSIVE", 'YES') + setup.setdefault(dep + "_DEPTH", -1) + + +# add_dependency(dep, tag) +# +# Add a dependency to the cache area: +# - check out (recursive if configured) in the CACHE area unless it already exists and the +# required commit has been built +# - Defaults: +# $dep_DIRNAME = lower case ($dep) +# $dep_REPONAME = lower case ($dep) +# $dep_REPOURL = GitHub / $dep_REPOOWNER (or $REPOOWNER or epics-modules) / $dep_REPONAME .git +# $dep_VARNAME = $dep +# $dep_DEPTH = 5 +# $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) +# - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) +# - Add full path to $modules_to_compile +def add_dependency(dep): + global do_recompile + recurse = setup[dep + '_RECURSIVE'].lower() + if recurse not in ['0', 'no']: + recursearg = ["--recursive"] + elif recurse not in ['1', 'yes']: + recursearg = [] + else: + raise RuntimeError("Invalid value for {}_RECURSIVE='{}' not 0/NO/1/YES".format(dep, recurse)) + deptharg = { + '-1': ['--depth', '5'], + '0': [], + }.get(str(setup[dep + '_DEPTH']), ['--depth', str(setup[dep + '_DEPTH'])]) + + tag = setup[dep] + + logger.debug('Adding dependency %s with tag %s', dep, setup[dep]) + + # determine if dep points to a valid release or branch + if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep + '_REPOURL'], tag]): + raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" + .format(ANSI_RED, tag, dep, setup[dep + '_REPOURL'], ANSI_RESET)) + + dirname = setup[dep + '_DIRNAME'] + '-{0}'.format(tag) + place = os.path.join(cachedir, dirname) + checked_file = os.path.join(place, "checked_out") + + if os.path.isdir(place): + logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) + # check HEAD commit against the hash in marker file + if os.path.exists(checked_file): + with open(checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + else: + checked_out = 'never' + head = get_git_hash(place) + logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) + if head != checked_out: + logger.debug('Dependency %s out of date - removing', dep) + shutil.rmtree(place, onerror=remove_readonly) + else: + print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) + sys.stdout.flush() + + if not os.path.isdir(place): + if not os.path.isdir(cachedir): + os.makedirs(cachedir) + # clone dependency + print('Cloning {0} of dependency {1} into {2}' + .format(tag, dep, place)) + sys.stdout.flush() + call_git(['clone', '--quiet'] + deptharg + recursearg + ['--branch', tag, setup[dep + '_REPOURL'], dirname], + cwd=cachedir) + + sp.check_call(['git', 'log', '-n1'], cwd=place) + logger.debug('Setting do_recompile = True (all following modules will be recompiled') + do_recompile = True + + if dep == 'BASE': + # add MSI 1.7 to Base 3.14 + versionfile = os.path.join(place, 'configure', 'CONFIG_BASE_VERSION') + if os.path.exists(versionfile): + with open(versionfile) as f: + if 'BASE_3_14=YES' in f.read(): + print('Adding MSI 1.7 to {0}'.format(place)) + sys.stdout.flush() + sp.check_call(['patch', '-p1', '-i', os.path.join(ci['scriptsdir'], 'add-msi-to-314.patch')], + cwd=place) + else: + # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE + release = os.path.join(place, "configure", "RELEASE") + if os.path.exists(release): + with open(release, 'w') as fout: + print('-include $(TOP)/../RELEASE.local', file=fout) + + # run hook if defined + if dep + '_HOOK' in setup: + hook = os.path.join(place, setup[dep + '_HOOK']) + if os.path.exists(hook): + print('Running hook {0} in {1}'.format(setup[dep + '_HOOK'], place)) + sys.stdout.flush() + sp.check_call(hook, shell=True, cwd=place) + + # write checked out commit hash to marker file + head = get_git_hash(place) + logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) + with open(checked_file, "w") as fout: + print(head, file=fout) + fout.close() + + if do_recompile: + modules_to_compile.append(dep) + update_release_local(setup[dep + "_VARNAME"], place) + + +def detect_epics_host_arch(): + if ci['os'] == 'windows': + if re.match(r'^vs', ci['compiler']): + # there is no combined static and debug EPICS_HOST_ARCH target, + # so a combined debug and static target will appear to be just static + # but debug will have been specified in CONFIG_SITE by prepare() + hostarchsuffix = '' + if ci['debug']: + hostarchsuffix = '-debug' + if ci['static']: + hostarchsuffix = '-static' + + if ci['platform'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86' + hostarchsuffix + elif ci['platform'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + hostarchsuffix + + elif ci['compiler'] == 'gcc': + if ci['platform'] == 'x86': + os.environ['EPICS_HOST_ARCH'] = 'win32-x86-mingw' + elif ci['platform'] == 'x64': + os.environ['EPICS_HOST_ARCH'] = 'windows-x64-mingw' + + if 'EPICS_HOST_ARCH' not in os.environ: + logger.debug('Running script to detect EPICS host architecture in %s', places['EPICS_BASE']) + os.environ['EPICS_HOST_ARCH'] = 'unknown' + eha_scripts = [ + os.path.join(places['EPICS_BASE'], 'src', 'tools', 'EpicsHostArch.pl'), + os.path.join(places['EPICS_BASE'], 'startup', 'EpicsHostArch.pl'), + ] + for eha in eha_scripts: + if os.path.exists(eha): + os.environ['EPICS_HOST_ARCH'] = sp.check_output(['perl', eha]).decode('ascii').strip() + logger.debug('%s returned: %s', + eha, os.environ['EPICS_HOST_ARCH']) + break + + +def setup_for_build(args): + global is_base314, has_test_results, is_make3 + dllpaths = [] + + if ci['os'] == 'windows': + if ci['service'] == 'appveyor': + if ci['compiler'] == 'vs2019': + # put strawberry perl in the PATH + os.environ['PATH'] = os.pathsep.join([os.path.join(r'C:\Strawberry\perl\site\bin'), + os.path.join(r'C:\Strawberry\perl\bin'), + os.environ['PATH']]) + if ci['compiler'] == 'gcc': + if 'INCLUDE' not in os.environ: + os.environ['INCLUDE'] = '' + if ci['platform'] == 'x86': + os.environ['INCLUDE'] = os.pathsep.join( + [r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin', + os.environ['PATH']]) + elif ci['platform'] == 'x64': + os.environ['INCLUDE'] = os.pathsep.join( + [r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\include', + os.environ['INCLUDE']]) + os.environ['PATH'] = os.pathsep.join([r'C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin', + os.environ['PATH']]) + if ci['service'] == 'travis': + os.environ['PATH'] = os.pathsep.join([r'C:\Strawberry\perl\site\bin', r'C:\Strawberry\perl\bin', + os.environ['PATH']]) + + # Find BASE location + if not building_base: + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + lines = f.readlines() + for line in lines: + (mod, place) = line.strip().split('=') + if mod == 'EPICS_BASE': + places['EPICS_BASE'] = place + else: + places['EPICS_BASE'] = '.' + + detect_epics_host_arch() + + if ci['os'] == 'windows': + if not building_base: + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + lines = f.readlines() + for line in lines: + (mod, place) = line.strip().split('=') + bin_dir = os.path.join(place, 'bin', os.environ['EPICS_HOST_ARCH']) + if os.path.isdir(bin_dir): + dllpaths.append(bin_dir) + # Add DLL location to PATH + bin_dir = os.path.join(os.getcwd(), 'bin', os.environ['EPICS_HOST_ARCH']) + if os.path.isdir(bin_dir): + dllpaths.append(bin_dir) + os.environ['PATH'] = os.pathsep.join(dllpaths + [os.environ['PATH']]) + logger.debug('DLL paths added to PATH: %s', os.pathsep.join(dllpaths)) + + cfg_base_version = os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_BASE_VERSION') + if os.path.exists(cfg_base_version): + with open(cfg_base_version) as myfile: + if 'BASE_3_14=YES' in myfile.read(): + is_base314 = True + + if not is_base314: + rules_build = os.path.join(places['EPICS_BASE'], 'configure', 'RULES_BUILD') + if os.path.exists(rules_build): + with open(rules_build) as myfile: + for line in myfile: + if re.match('^test-results:', line): + has_test_results = True + + # Check make version + if re.match(r'^GNU Make 3', sp.check_output(['make', '-v']).decode('ascii')): + is_make3 = True + + # apparently %CD% is handled automagically + os.environ['TOP'] = os.getcwd() + + addpaths = [] + for path in args.paths: + try: + addpaths.append(path.format(**os.environ)) + except KeyError: + print('Environment') + [print(' ', K, '=', repr(V)) for K, V in os.environ.items()] + raise + + os.environ['PATH'] = os.pathsep.join([os.environ['PATH']] + addpaths) + + # Add EXTRA make arguments + for tag in ['EXTRA', 'EXTRA1', 'EXTRA2', 'EXTRA3', 'EXTRA4', 'EXTRA5']: + if tag in os.environ: + extra_makeargs.append(os.environ[tag]) + + +def prepare(args): + host_info() + + fold_start('load.setup', 'Loading setup files') + + if 'SET' in os.environ: + source_set(os.environ['SET']) + source_set('defaults') + + [complete_setup(mod) for mod in modlist()] + + fold_end('load.setup', 'Loading setup files') + + logger.debug('Loaded setup') + kvs = list(setup.items()) + kvs.sort() + [logger.debug(' %s = "%s"', *kv) for kv in kvs] + + logger.debug('Effective module list: %s', modlist()) + + # we're working with tags (detached heads) a lot: suppress advice + call_git(['config', '--global', 'advice.detachedHead', 'false']) + + fold_start('check.out.dependencies', 'Checking/cloning dependencies') + + [add_dependency(mod) for mod in modlist()] + + if not building_base: + if os.path.isdir('configure'): + targetdir = 'configure' + else: + targetdir = '.' + shutil.copy(os.path.join(cachedir, 'RELEASE.local'), targetdir) + + fold_end('check.out.dependencies', 'Checking/cloning dependencies') + + if 'BASE' in modules_to_compile or building_base: + fold_start('set.up.epics_build', 'Configuring EPICS build system') + + detect_epics_host_arch() + + # Set static/debug in CONFIG_SITE + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f: + if ci['static']: + f.write('SHARED_LIBRARIES=NO\n') + f.write('STATIC_BUILD=YES\n') + linktype = 'static' + else: + linktype = 'shared (DLL)' + if ci['debug']: + f.write('HOST_OPT=NO\n') + optitype = 'debug' + else: + optitype = 'optimized' + + print('EPICS Base build system set up for {0} build with {1} linking' + .format(optitype, linktype)) + + # Enable/fix parallel build for VisualStudio compiler on older Base versions + if ci['os'] == 'windows' and re.match(r'^vs', ci['compiler']): + add_vs_fix = True + config_win = os.path.join(places['EPICS_BASE'], 'configure', 'os', 'CONFIG.win32-x86.win32-x86') + with open(config_win) as f: + for line in f: + if re.match(r'^ifneq \(\$\(VisualStudioVersion\),11\.0\)', line): + add_vs_fix = False + if add_vs_fix: + logger.debug('Adding parallel build fix for VisualStudio to %s', config_win) + with open(config_win, 'a') as f: + f.write(''' +# Fix parallel build for some VisualStudio versions +ifneq ($(VisualStudioVersion),) +ifneq ($(VisualStudioVersion),11.0) +ifeq ($(findstring -FS,$(OPT_CXXFLAGS_NO)),) + OPT_CXXFLAGS_NO += -FS + OPT_CFLAGS_NO += -FS +endif +else + OPT_CXXFLAGS_NO := $(filter-out -FS,$(OPT_CXXFLAGS_NO)) + OPT_CFLAGS_NO := $(filter-out -FS,$(OPT_CFLAGS_NO)) +endif +endif''') + + # Cross-compilations from Linux platform + if ci['os'] == 'linux': + + # Cross compilation to Windows/Wine (set WINE to architecture "32", "64") + # requires wine and g++-mingw-w64-i686 / g++-mingw-w64-x86-64 + if 'WINE' in os.environ: + if os.environ['WINE'] == '32': + print('Cross compiler mingw32 / Wine') + with open(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG.linux-x86.win32-x86-mingw'), 'a') as f: + f.write(''' +CMPLR_PREFIX=i686-w64-mingw32-''') + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f: + f.write(''' +CROSS_COMPILER_TARGET_ARCHS+=win32-x86-mingw''') + + if os.environ['WINE'] == '64': + print('Cross compiler mingw64 / Wine') + with open(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG.linux-x86.windows-x64-mingw'), 'a') as f: + f.write(''' +CMPLR_PREFIX=x86_64-w64-mingw32-''') + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f: + f.write(''' +CROSS_COMPILER_TARGET_ARCHS += windows-x64-mingw''') + + # Cross compilation on Linux to RTEMS (set RTEMS to version "4.9", "4.10") + # requires qemu, bison, flex, texinfo, install-info + if 'RTEMS' in os.environ: + print('Cross compiler RTEMS{0} @ pc386',format(os.environ['RTEMS'])) + with open(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG_SITE.Common.RTEMS'), 'a') as f: + f.write(''' +RTEMS_VERSION={0} +RTEMS_BASE={1}'''.format(os.environ['RTEMS'], rtemsdir)) + + # Base 3.15 doesn't have -qemu target architecture + qemu_suffix = '' + if os.path.exists(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG.Common.RTEMS-pc386-qemu')): + qemu_suffix = '-qemu' + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f: + f.write(''' +CROSS_COMPILER_TARGET_ARCHS += RTEMS-pc386{0}'''.format(qemu_suffix)) + + host_ccmplr_name = re.sub(r'^([a-zA-Z][^-]*(-[a-zA-Z][^-]*)*)+(-[0-9.]|)$', r'\1', ci['compiler']) + host_cmplr_ver_suffix = re.sub(r'^([a-zA-Z][^-]*(-[a-zA-Z][^-]*)*)+(-[0-9.]|)$', r'\3', ci['compiler']) + host_cmpl_ver = host_cmplr_ver_suffix[1:] + + if host_ccmplr_name == 'clang': + print('Host compiler clang') + host_cppcmplr_name = re.sub(r'clang', r'clang++', host_ccmplr_name) + with open(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG_SITE.Common.'+os.environ['EPICS_HOST_ARCH']), 'a') as f: + f.write(''' +GNU = NO +CMPLR_CLASS = clang +CC = {0}{2} +CCC = {1}{2}'''.format(host_ccmplr_name, host_cppcmplr_name, host_cmplr_ver_suffix)) + + # hack + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG.gnuCommon'), 'a') as f: + f.write(''' +CMPLR_CLASS = clang''') + + if host_ccmplr_name == 'gcc': + print('Host compiler gcc') + host_cppcmplr_name = re.sub(r'gcc', r'g++', host_ccmplr_name) + with open(os.path.join(places['EPICS_BASE'], 'configure', 'os', + 'CONFIG_SITE.Common.' + os.environ['EPICS_HOST_ARCH']), 'a') as f: + f.write(''' +CC = {0}{2} +CCC = {1}{2}'''.format(host_ccmplr_name, host_cppcmplr_name, host_cmplr_ver_suffix)) + + # Add additional flags to CONFIG_SITE + flags_text = '' + if 'USR_CPPFLAGS' in os.environ: + flags_text += ''' +USR_CPPFLAGS += {0}'''.format(os.environ['USR_CPPFLAGS']) + if 'USR_CFLAGS' in os.environ: + flags_text += ''' +USR_CFLAGS += {0}'''.format(os.environ['USR_CFLAGS']) + if 'USR_CXXFLAGS' in os.environ: + flags_text += ''' +USR_CXXFLAGS += {0}'''.format(os.environ['USR_CXXFLAGS']) + if flags_text: + with open(os.path.join(places['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as f: + f.write(flags_text) + + fold_end('set.up.epics_build', 'Configuring EPICS build system') + + if not os.path.isdir(toolsdir): + os.makedirs(toolsdir) + + if ci['os'] == 'windows' and ci['choco']: + fold_start('install.choco', 'Installing CHOCO packages') + sp.check_call(['choco', 'install'] + ci['choco']) + fold_end('install.choco', 'Installing CHOCO packages') + + if ci['os'] == 'linux' and 'RTEMS' in os.environ: + tar_name = 'i386-rtems{0}-trusty-20171203-{0}.tar.bz2'.format(os.environ['RTEMS']) + print('Downloading RTEMS {0} cross compiler: {1}' + .format(os.environ['RTEMS'], tar_name)) + sys.stdout.flush() + sp.check_call(['curl', '-fsSL', '--retry', '3', '-o', tar_name, + 'https://github.com/mdavidsaver/rsb/releases/download/20171203-{0}/{1}' + .format(os.environ['RTEMS'], tar_name)], + cwd=toolsdir) + sp.check_call(['tar', '-C', '/', '-xmj', '-f', os.path.join(toolsdir, tar_name)]) + os.remove(os.path.join(toolsdir, tar_name)) + + setup_for_build(args) + + print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET)) + print('{0}$ make --version{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + call_make(['--version'], parallel=0) + print('{0}$ perl --version{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + sp.check_call(['perl', '--version']) + + if re.match(r'^vs', ci['compiler']): + print('{0}$ cl{1}'.format(ANSI_CYAN, ANSI_RESET)) + sys.stdout.flush() + sp.check_call(['cl']) + else: + cc = ci['compiler'] + print('{0}$ {1} --version{2}'.format(ANSI_CYAN, cc, ANSI_RESET)) + sys.stdout.flush() + sp.check_call([cc, '--version']) + + if not building_base: + fold_start('build.dependencies', 'Build missing/outdated dependencies') + for mod in modules_to_compile: + place = places[setup[mod + "_VARNAME"]] + print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET)) + call_make(cwd=place, silent=silent_dep_builds) + fold_end('build.dependencies', 'Build missing/outdated dependencies') + + print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET)) + print('Module Tag Binaries Commit') + print(100 * '-') + for mod in modlist(): + if mod in modules_to_compile: + stat = 'rebuilt' + else: + stat = 'from cache' + commit = sp.check_output(['git', 'log', '-n1', '--oneline'], cwd=places[setup[mod + "_VARNAME"]])\ + .decode('ascii').strip() + print("%-10s %-12s %-11s %s" % (mod, setup[mod], stat, commit)) + + print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET)) + with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f: + print(f.read().strip()) + + +def build(args): + setup_for_build(args) + fold_start('build.module', 'Build the main module') + call_make(args.makeargs) + fold_end('build.module', 'Build the main module') + + +def test(args): + if ci['test']: + setup_for_build(args) + fold_start('test.module', 'Run the main module tests') + if has_test_results: + call_make(['tapfiles']) + else: + call_make(['runtests']) + fold_end('test.module', 'Run the main module tests') + else: + print("{0}Action 'test' skipped as per configuration{1}" + .format(ANSI_YELLOW, ANSI_RESET)) + + +def test_results(args): + if ci['test']: + setup_for_build(args) + fold_start('test.results', 'Sum up main module test results') + if has_test_results: + call_make(['test-results'], parallel=0, silent=True) + else: + print("{0}Base in {1} does not implement 'test-results' target{2}" + .format(ANSI_YELLOW, places['EPICS_BASE'], ANSI_RESET)) + fold_end('test.results', 'Sum up main module test results') + else: + print("{0}Action 'test-results' skipped as per configuration{1}" + .format(ANSI_YELLOW, ANSI_RESET)) + + +def doExec(args): + 'exec user command with vcvars' + setup_for_build(args) + os.environ['MAKE'] = 'make' + fold_start('exec.command', 'Execute command {}'.format(args.cmd)) + sp.check_call(' '.join(args.cmd), shell=True) + fold_end('exec.command', 'Execute command {}'.format(args.cmd)) + + +def with_vcvars(cmd): + '''re-exec main script with a (hopefully different) command + ''' + CC = ci['compiler'] + + # cf. https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line + + info = { + 'python': sys.executable, + 'self': sys.argv[0], + 'cmd': cmd, + } + + info['arch'] = { + 'x86': 'x86', # 'amd64_x86' ?? + 'x64': 'amd64', + }[ci['platform']] # 'x86' or 'x64' + + info['vcvars'] = vcvars_found[CC] + + script = ''' +call "{vcvars}" {arch} + +"{python}" "{self}" {cmd} +'''.format(**info) + + print('{0}Calling vcvars-trampoline.bat to set environment for {1} on {2}{3}' + .format(ANSI_YELLOW, CC, ci['platform'], ANSI_RESET)) + sys.stdout.flush() + + logger.debug('----- Creating vcvars-trampoline.bat -----') + for line in script.split('\n'): + logger.debug(line) + logger.debug('----- snip -----') + + with open('vcvars-trampoline.bat', 'w') as F: + F.write(script) + + returncode = sp.call('vcvars-trampoline.bat', shell=True) + if returncode != 0: + sys.exit(returncode) + + +def getargs(): + from argparse import ArgumentParser, REMAINDER + p = ArgumentParser() + p.add_argument('--no-vcvars', dest='vcvars', default=True, action='store_false', + help='Assume vcvarsall.bat has already been run') + p.add_argument('--add-path', dest='paths', default=[], action='append', + help='Append directory to %PATH%. Expands {ENVVAR}') + subp = p.add_subparsers() + + cmd = subp.add_parser('prepare') + cmd.set_defaults(func=prepare) + + cmd = subp.add_parser('build') + cmd.add_argument('makeargs', nargs=REMAINDER) + cmd.set_defaults(func=build) + + cmd = subp.add_parser('test') + cmd.set_defaults(func=test) + + cmd = subp.add_parser('test-results') + cmd.set_defaults(func=test_results) + + cmd = subp.add_parser('exec') + cmd.add_argument('cmd', nargs=REMAINDER) + cmd.set_defaults(func=doExec) + + return p + + +def main(raw): + global silent_dep_builds + args = getargs().parse_args(raw) + if 'VV' in os.environ and os.environ['VV'] == '1': + logging.basicConfig(level=logging.DEBUG) + silent_dep_builds = False + + detect_context() + + if args.vcvars and ci['compiler'].startswith('vs'): + # re-exec with MSVC in PATH + with_vcvars(' '.join(['--no-vcvars'] + raw)) + else: + args.func(args) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/defaults.set b/defaults.set index 7cc9a32..9f433c5 100644 --- a/defaults.set +++ b/defaults.set @@ -1,4 +1,5 @@ # EPICS Base +BASE=7.0 BASE_DIRNAME=base BASE_REPONAME=epics-base BASE_REPOOWNER=epics-base diff --git a/exampleApp/test/Makefile b/exampleApp/test/Makefile index cdfc92a..1eefe57 100644 --- a/exampleApp/test/Makefile +++ b/exampleApp/test/Makefile @@ -38,7 +38,9 @@ testHarness_SRCS += epicsRunExampleTests.c exampleTestHarness_SRCS += $(testHarness_SRCS) exampleTestHarness_SRCS_RTEMS += rtemsTestHarness.c +ifdef BASE_7_0 PROD_SRCS_RTEMS += rtemsTestData.c +endif PROD_vxWorks = exampleTestHarness PROD_RTEMS = exampleTestHarness @@ -54,5 +56,7 @@ endif include $(TOP)/configure/RULES +ifdef BASE_7_0 rtemsTestData.c : $(TESTFILES) $(TOOLS)/epicsMakeMemFs.pl $(PERL) $(TOOLS)/epicsMakeMemFs.pl $@ epicsRtemsFSImage $(TESTFILES) +endif diff --git a/test01.set b/test01.set index 0260e61..2583e99 100644 --- a/test01.set +++ b/test01.set @@ -1,4 +1,4 @@ MODULES="sncseq" BASE=7.0 -SNCSEQ=R2-2-7 +SNCSEQ=R2-2-8 diff --git a/travis/.travis.yml.example-full b/travis/.travis.yml.example-full index 3485ceb..df5491c 100644 --- a/travis/.travis.yml.example-full +++ b/travis/.travis.yml.example-full @@ -14,6 +14,8 @@ cache: env: global: - SETUP_PATH=.ci-local:.ci + # for the sequencer on Windows + - CHOCO=re2c addons: apt: @@ -31,17 +33,17 @@ addons: - qemu-system-x86 homebrew: packages: - # for all EPICS builds - - bash # for the sequencer - re2c update: true install: - - ./.ci/travis/prepare.sh + - python .ci/cue.py prepare script: - - ./.ci/travis/build.sh + - python .ci/cue.py build + - python .ci/cue.py test + - python .ci/cue.py test-results # If you need to do more during install and build, # add a local directory to your module and do e.g. @@ -52,12 +54,13 @@ script: # Well-known variables to use # SET source setup file # ADD_MODULES extra modules (for a specific job) -# EXTRA content will be added to make command line -# EXTRA1..5 additional arguments for the make command -# (one argument per variable) -# STATIC set to YES for static build (default: NO) +# BCFG build configuration (static/debug/static-debug; +# default: shared-optimized) # TEST set to NO to skip running the tests (default: YES) # VV set to make build scripts verbose (default: unset) +# EXTRA content will be added to make command line +# EXTRA1..5 more additional arguments for the make command +# (one argument per variable) # Usually from setup files, but may be specified or overridden # on a job line @@ -76,7 +79,7 @@ jobs: - env: BASE=7.0 compiler: clang - - env: BASE=7.0 EXTRA="CMD_CXXFLAGS=-std=c++11" + - env: BASE=7.0 - env: BASE=7.0 EXTRA="CMD_CXXFLAGS=-std=c++11" compiler: clang @@ -89,19 +92,17 @@ jobs: - env: BASE=7.0 EXTRA="CMD_CXXFLAGS=-std=c++11" dist: trusty -# Cross-compilations to Windows using MinGW and WINE - - - env: BASE=7.0 WINE=32 TEST=NO STATIC=YES - compiler: mingw +# Cross-compilations to Windows using gcc/MinGW and WINE - - env: BASE=7.0 WINE=64 TEST=NO STATIC=NO - compiler: mingw + - env: BASE=7.0 WINE=32 TEST=NO BCFG=static + + - env: BASE=7.0 WINE=64 TEST=NO # Cross-compilation to RTEMS - - env: BASE=7.0 RTEMS=4.10 TEST=NO + - env: BASE=7.0 RTEMS=4.10 - - env: BASE=7.0 RTEMS=4.9 TEST=NO + - env: BASE=7.0 RTEMS=4.9 # Other gcc versions (added as an extra package) @@ -118,3 +119,12 @@ jobs: - env: BASE=7.0 os: osx compiler: clang + +# Windows builds + + - env: BASE=7.0 + os: windows + compiler: vs2017 + + - env: BASE=7.0 + os: windows diff --git a/travis/.travis.yml.example-mini b/travis/.travis.yml.example-mini index d680894..bede258 100644 --- a/travis/.travis.yml.example-mini +++ b/travis/.travis.yml.example-mini @@ -15,12 +15,14 @@ addons: - perl install: - - ./.ci/travis/prepare.sh + - python .ci/cue.py prepare script: - - ./.ci/travis/build.sh + - python .ci/cue.py build + - python .ci/cue.py test + - python .ci/cue.py test-results -# Build using default gcc for Base branches 7.0 and 3.15 +# Build on Linux using default gcc for Base branches 7.0 and 3.15 jobs: include: diff --git a/travis/Add-RTEMS-pc368-qemu-target.patch b/travis/Add-RTEMS-pc368-qemu-target.patch deleted file mode 100644 index de38660..0000000 --- a/travis/Add-RTEMS-pc368-qemu-target.patch +++ /dev/null @@ -1,77 +0,0 @@ -From 00ee7bf7d3618c748491c88742c011a8353abeba Mon Sep 17 00:00:00 2001 -From: Andrew Johnson -Date: Wed, 24 Oct 2018 14:27:15 -0500 -Subject: [PATCH] Add RTEMS-pc368-qemu target, use in Travis-CI builds - ---- - configure/os/CONFIG.Common.RTEMS-pc386-qemu | 11 +++++++++++ - configure/os/CONFIG_SITE.Common.RTEMS-pc386 | 5 ----- - configure/os/CONFIG_SITE.Common.RTEMS-pc386-qemu | 9 +++++++++ - src/libCom/RTEMS/Makefile | 2 +- - src/tools/makeTestfile.pl | 2 +- - 6 files changed, 24 insertions(+), 10 deletions(-) - create mode 100644 configure/os/CONFIG.Common.RTEMS-pc386-qemu - delete mode 100644 configure/os/CONFIG_SITE.Common.RTEMS-pc386 - create mode 100644 configure/os/CONFIG_SITE.Common.RTEMS-pc386-qemu - -diff --git a/configure/os/CONFIG.Common.RTEMS-pc386-qemu b/configure/os/CONFIG.Common.RTEMS-pc386-qemu -new file mode 100644 -index 000000000..684f01a19 ---- /dev/null -+++ b/configure/os/CONFIG.Common.RTEMS-pc386-qemu -@@ -0,0 +1,11 @@ -+# CONFIG.Common.RTEMS-pc386-qemu -+# -+# Definitions for the RTEMS-pc386-qemu target -+# Site-specific overrides go in CONFIG_SITE.Common.RTEMS-pc386-qemu -+# -+#------------------------------------------------------- -+ -+# Include definitions from RTEMS-pc386 -+include $(CONFIG)/os/CONFIG.Common.RTEMS-pc386 -+ -+RTEMS_QEMU_FIXUPS = YES -diff --git a/configure/os/CONFIG_SITE.Common.RTEMS-pc386-qemu b/configure/os/CONFIG_SITE.Common.RTEMS-pc386-qemu -new file mode 100644 -index 000000000..027dcf4ab ---- /dev/null -+++ b/configure/os/CONFIG_SITE.Common.RTEMS-pc386-qemu -@@ -0,0 +1,9 @@ -+# CONFIG_SITE.Common.RTEMS-pc386-qemu -+# -+# Site-specific overrides for the RTEMS-pc386-qemu target -+# -+ -+# If you're building this architecture you _probably_ want to -+# run the tests for it under QEMU, but if not you can turn -+# them off here by commenting out this line: -+CROSS_COMPILER_RUNTEST_ARCHS += RTEMS-pc386-qemu -diff --git a/src/libCom/RTEMS/Makefile b/src/libCom/RTEMS/Makefile -index 2f12b7bf0..22a92733c 100644 ---- a/src/libCom/RTEMS/Makefile -+++ b/src/libCom/RTEMS/Makefile -@@ -24,7 +24,7 @@ rtemsCom_SRCS += epicsRtemsInitHookPre.c - rtemsCom_SRCS += epicsRtemsInitHookPost.c - rtemsCom_SRCS += epicsMemFs.c - --ifeq ($(T_A),RTEMS-pc386) -+ifeq ($(RTEMS_BSP),pc386) - rtemsCom_SRCS += ne2kpci.c - endif - -diff --git a/src/tools/makeTestfile.pl b/src/tools/makeTestfile.pl -index 73f522034..fb431fe7a 100644 ---- a/src/tools/makeTestfile.pl -+++ b/src/tools/makeTestfile.pl -@@ -37,7 +37,7 @@ if( $TA =~ /^win32-x86/ && $HA !~ /^win/ ) { - $exec = "wine64 $exe"; - - # Run pc386 test harness w/ QEMU --} elsif( $TA =~ /^RTEMS-pc386$/ ) { -+} elsif( $TA =~ /^RTEMS-pc386-qemu$/ ) { - $exec = "qemu-system-i386 -m 64 -no-reboot -serial stdio -display none -net nic,model=ne2k_pci -net user,restrict=yes -kernel $exe"; - - # Explicitly fail for other RTEMS targets --- -2.21.0.windows.1 - diff --git a/travis/README.md b/travis/README.md index 1f4bcb0..2d9dac5 100644 --- a/travis/README.md +++ b/travis/README.md @@ -2,12 +2,14 @@ ## Features + - Five parallel runners on Linux/Windows (one runner on MacOS) - Use different compilers (gcc, clang) - Use different gcc versions - Cross-compile for Windows 32bit and 64bit using MinGW and WINE - - Cross-compile for RTEMS 4.9 and 4.10 - - Compile on MacOS - - Released versions of dependencies are cached (for faster builds) + - Cross-compile for RTEMS 4.9 and 4.10 (Base >= 3.15) + - Compile natively on MacOS (clang) + - Compile natively on Windows (gcc/MinGW, Visual Studio 2017) + - Built dependencies are cached (for faster builds). ## How to Use these Scripts @@ -34,7 +36,7 @@ Travis to run. Build jobs are declared in the list following the `jobs: include:` - declaration. Each element (starting with `-` in column 3) defines the + declaration. Each element (starting with a dash) defines the settings for one build job. `env:` controls the setting of environment variables,`dist:` specifies the Linux distribution, `os:` the operating system. diff --git a/travis/build.sh b/travis/build.sh deleted file mode 100755 index f282614..0000000 --- a/travis/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -e - -# Set VV in .travis.yml to make scripts verbose -[ "$VV" ] && set -x - -CURDIR="$PWD" -CACHEDIR=${CACHEDIR:-${HOME}/.cache} - -if [ "$BASE" = "SELF" ] -then - EPICS_BASE=$CURDIR -else - eval $(grep "EPICS_BASE=" ${CACHEDIR}/RELEASE.local) -fi -export EPICS_BASE - -[ -z "$EPICS_HOST_ARCH" -a -f $EPICS_BASE/src/tools/EpicsHostArch.pl ] && EPICS_HOST_ARCH=$(perl $EPICS_BASE/src/tools/EpicsHostArch.pl) -[ -z "$EPICS_HOST_ARCH" -a -f $EPICS_BASE/startup/EpicsHostArch.pl ] && EPICS_HOST_ARCH=$(perl $EPICS_BASE/startup/EpicsHostArch.pl) -export EPICS_HOST_ARCH - -# Base 3.15 doesn't have -qemu target architecture and needs an extra define -[ -e $EPICS_BASE/configure/os/CONFIG.Common.RTEMS-pc386-qemu ] || EXTRA_QEMU=RTEMS_QEMU_FIXUPS=YES - -# use array variable to get the quoting right while using separate words for arguments -[ -n "$EXTRA0" ] && EXTRA[0]="$EXTRA0" -[ -n "$EXTRA1" ] && EXTRA[1]="$EXTRA1" -[ -n "$EXTRA2" ] && EXTRA[2]="$EXTRA2" -[ -n "$EXTRA3" ] && EXTRA[3]="$EXTRA3" -[ -n "$EXTRA4" ] && EXTRA[4]="$EXTRA4" -[ -n "$EXTRA5" ] && EXTRA[5]="$EXTRA5" - -make -j2 $EXTRA_QEMU "${EXTRA[@]}" - -ret=0 - -if [ "$TEST" != "NO" ] -then - if grep -q "BASE_3_14\s*=\s*NO" $EPICS_BASE/configure/CONFIG_BASE_VERSION && grep -q "^test-results:" $EPICS_BASE/configure/RULES_BUILD - then - make -j2 tapfiles || ret=$? - make -sk test-results - else - make runtests - fi -fi - -exit $ret diff --git a/travis/prepare.sh b/travis/prepare.sh deleted file mode 100755 index e313384..0000000 --- a/travis/prepare.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/bin/bash -set -e - -# The following if clause can be removed for ci-scripts major version 3 -if [ "$TRAVIS_OS_NAME" == osx -a "$BASH_VERSINFO" -lt 4 ] -then - brew install bash - if [ $(/usr/local/bin/bash -c 'echo $BASH_VERSINFO') -lt 4 ] - then - echo "Failed to install a recent bash" >&2 - exit 1 - fi - exec /usr/local/bin/bash $0 "$@" -fi - -# Set VV in .travis.yml to make scripts verbose -[ "$VV" ] && set -x - -# Perl version of "readlink -f" (which MacOS does not provide) -readlinkf() { perl -MCwd -e 'print Cwd::abs_path shift' "$1"; } - -SCRIPTDIR=$(dirname $(readlinkf $0)) -CURDIR="$PWD" -CACHEDIR=${CACHEDIR:-${HOME}/.cache} -[ -e ${CACHEDIR} ] || mkdir -p ${CACHEDIR} - -# source functions -. $SCRIPTDIR/utils.sh - -echo -e "${ANSI_YELLOW}Using bash version $BASH_VERSION${ANSI_RESET}" - -if [ -f /etc/hosts ] -then - # The travis-ci "bionic" image throws us a curveball in /etc/hosts - # by including two entries for localhost. The first for 127.0.1.1 - # which causes epicsSockResolveTest to fail. - # cat /etc/hosts - # ... - # 127.0.1.1 localhost localhost ip4-loopback - # 127.0.0.1 localhost nettuno travis vagrant travis-job-.... - - sudo sed -i -e '/^127\.0\.1\.1/ s|localhost\s*||g' /etc/hosts - - echo "==== /etc/hosts" - cat /etc/hosts - echo "====" -fi - -# Load settings -# ------------- - -fold_start load.settings "Loading settings" - -# load default settings for well-known modules -source_set defaults - -# source configured settings -[ -z "${SET+x}" ] || source_set $SET - -fold_end load.settings - -# Check out dependencies -# ---------------------- - -if [ "$BASE" != "SELF" ] -then - fold_start check.out.dependencies "Checking/cloning dependencies" - - for mod in BASE $ADD_MODULES $MODULES - do - mod_uc=${mod^^} - eval add_dependency $mod_uc \${${mod_uc}:=master} - done - [ -d ./configure ] && target=./configure/RELEASE.local || target=./RELEASE.local - cp ${CACHEDIR}/RELEASE.local $target - - fold_end check.out.dependencies -fi - -# Set up compiler -# --------------- - -fold_start set.up.epics_build "Setting up EPICS build system" - -if [ "$BASE" = "SELF" ] -then - EPICS_BASE=$CURDIR -else - eval $(grep "EPICS_BASE=" ${CACHEDIR}/RELEASE.local) -fi -export EPICS_BASE -echo "EPICS_BASE=$EPICS_BASE" - -[ -z "$EPICS_HOST_ARCH" -a -f $EPICS_BASE/src/tools/EpicsHostArch.pl ] && EPICS_HOST_ARCH=$(perl $EPICS_BASE/src/tools/EpicsHostArch.pl) -[ -z "$EPICS_HOST_ARCH" -a -f $EPICS_BASE/startup/EpicsHostArch.pl ] && EPICS_HOST_ARCH=$(perl $EPICS_BASE/startup/EpicsHostArch.pl) -export EPICS_HOST_ARCH -echo "EPICS_HOST_ARCH=$EPICS_HOST_ARCH" - -if echo ${modules_to_compile} | grep -q "$EPICS_BASE" || [ "$BASE" = "SELF" ] -then - - # requires wine and g++-mingw-w64-i686 - if [ "$WINE" = "32" ] - then - echo "Cross mingw32" - sed -i -e '/CMPLR_PREFIX/d' $EPICS_BASE/configure/os/CONFIG_SITE.linux-x86.win32-x86-mingw - cat << EOF >> $EPICS_BASE/configure/os/CONFIG_SITE.linux-x86.win32-x86-mingw -CMPLR_PREFIX=i686-w64-mingw32- -EOF - cat << EOF >> $EPICS_BASE/configure/CONFIG_SITE -CROSS_COMPILER_TARGET_ARCHS+=win32-x86-mingw -EOF - - elif [ "$WINE" = "64" ] - then - echo "Cross mingw64" - sed -i -e '/CMPLR_PREFIX/d' $EPICS_BASE/configure/os/CONFIG_SITE.linux-x86.windows-x64-mingw - cat << EOF >> $EPICS_BASE/configure/os/CONFIG_SITE.linux-x86.windows-x64-mingw -CMPLR_PREFIX=x86_64-w64-mingw32- -EOF - cat << EOF >> $EPICS_BASE/configure/CONFIG_SITE -CROSS_COMPILER_TARGET_ARCHS+=windows-x64-mingw -EOF - fi - - if [ "$STATIC" = "YES" ] - then - echo "Build static libraries/executables" - cat << EOF >> $EPICS_BASE/configure/CONFIG_SITE -SHARED_LIBRARIES=NO -STATIC_BUILD=YES -EOF - fi - - HOST_CCMPLR_NAME=`echo "$TRAVIS_COMPILER" | sed -E 's/^([[:alpha:]][^-]*(-[[:alpha:]][^-]*)*)+(-[0-9\.]+)?$/\1/g'` - HOST_CMPLR_VER_SUFFIX=`echo "$TRAVIS_COMPILER" | sed -E 's/^([[:alpha:]][^-]*(-[[:alpha:]][^-]*)*)+(-[0-9\.]+)?$/\3/g'` - HOST_CMPLR_VER=`echo "$HOST_CMPLR_VER_SUFFIX" | cut -c 2-` - - case "$HOST_CCMPLR_NAME" in - clang) - echo "Host compiler is clang" - HOST_CPPCMPLR_NAME=$(echo "$HOST_CCMPLR_NAME" | sed 's/clang/clang++/g') - cat << EOF >> $EPICS_BASE/configure/os/CONFIG_SITE.Common.$EPICS_HOST_ARCH -GNU = NO -CMPLR_CLASS = clang -CC = ${HOST_CCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX -CCC = ${HOST_CPPCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX -EOF - - # hack - sed -i -e 's/CMPLR_CLASS = gcc/CMPLR_CLASS = clang/' $EPICS_BASE/configure/CONFIG.gnuCommon - - ${HOST_CCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX --version - ;; - gcc) - echo "Host compiler is GCC" - HOST_CPPCMPLR_NAME=$(echo "$HOST_CCMPLR_NAME" | sed 's/gcc/g++/g') - cat << EOF >> $EPICS_BASE/configure/os/CONFIG_SITE.Common.$EPICS_HOST_ARCH -CC = ${HOST_CCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX -CCC = ${HOST_CPPCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX -EOF - - ${HOST_CCMPLR_NAME}$HOST_CMPLR_VER_SUFFIX --version - ;; - *) - echo "Host compiler is default" - gcc --version - ;; - esac - - cat <> $EPICS_BASE/configure/CONFIG_SITE -USR_CPPFLAGS += $USR_CPPFLAGS -USR_CFLAGS += $USR_CFLAGS -USR_CXXFLAGS += $USR_CXXFLAGS -EOF - - # set RTEMS to eg. "4.9" or "4.10" - # requires qemu, bison, flex, texinfo, install-info - if [ -n "$RTEMS" ] - then - echo "Cross RTEMS${RTEMS} for pc386" - sed -i -e '/^RTEMS_VERSION/d' -e '/^RTEMS_BASE/d' $EPICS_BASE/configure/os/CONFIG_SITE.Common.RTEMS - cat << EOF >> $EPICS_BASE/configure/os/CONFIG_SITE.Common.RTEMS -RTEMS_VERSION=$RTEMS -RTEMS_BASE=$HOME/.rtems -EOF - # Base 3.15 doesn't have -qemu target architecture - [ -e $EPICS_BASE/configure/os/CONFIG.Common.RTEMS-pc386-qemu ] && QEMU=-qemu - cat << EOF >> $EPICS_BASE/configure/CONFIG_SITE -CROSS_COMPILER_TARGET_ARCHS += RTEMS-pc386$QEMU -EOF - fi - -else - echo -e "${ANSI_GREEN}EPICS build system already set up (Base was loaded from cache)${ANSI_RESET}" -fi - -# Download RTEMS cross compiler -if [ -n "$RTEMS" ] -then - echo "Downloading RTEMS${RTEMS} cross compiler for pc386" - curl -L "https://github.com/mdavidsaver/rsb/releases/download/20171203-${RTEMS}/i386-rtems${RTEMS}-trusty-20171203-${RTEMS}.tar.bz2" \ - | tar -C / -xmj -fi - -fold_end set.up.compiler - -echo "\$ make --version" -make --version - -[ "$BASE" = "SELF" ] && exit 0 - -# Build required dependencies -# --------------------------- - -fold_start build.dependencies "Build missing/outdated dependencies" - -[ "$VV" ] && silent="-s" || silent= - -[ -z "$modules_to_compile" ] && echo -e "${ANSI_GREEN}All dependency modules are up-to-date (nothing to do)${ANSI_RESET}" - -for module in ${modules_to_compile} -do - eval name=\${module#${CACHEDIR}/} - fold_start build.$name "Build $name" - make -j2 $silent -C $module $EXTRA - fold_end build.$name -done - -fold_end build.dependencies - -echo -e "${ANSI_BLUE}Dependency module information${ANSI_RESET}" - -echo "Module Tag Binaries Commit" -echo "-----------------------------------------------------------------------------------" -for mod in base $MODULES $ADD_MODULES -do - mod_uc=${mod^^} - eval tag=\${${mod_uc}} - eval dir=${CACHEDIR}/\${${mod_uc}_DIRNAME}-$tag - echo "$modules_to_compile" | grep -q "$dir" && stat="rebuilt" || stat="from cache" - commit=$(git -C $dir log -n1 --oneline) - printf "%-10s %-12s %-11s %s\n" "$mod" "$tag" "$stat" "$commit" -done - -echo -e "${ANSI_BLUE}Contents of RELEASE.local${ANSI_RESET}" -cat ${CACHEDIR}/RELEASE.local diff --git a/travis/utils.sh b/travis/utils.sh deleted file mode 100644 index 822e933..0000000 --- a/travis/utils.sh +++ /dev/null @@ -1,218 +0,0 @@ -# Utility functions for Travis scripts in ci-scripts -# -# This file is sourced by the executable scripts -# CACHEDIR must be defined and existing before calling these functions - -# Portable version of 'sed -i' (that MacOS doesn't provide) - -# sedi (cmd, file) -# Do the equivalent of "sed -i cmd file" -sedi () { - cat $2 | sed "$1" > $2.tmp$$; mv -f $2.tmp$$ $2 -} - -# Setup ANSI Colors -export ANSI_RED="\033[31;1m" -export ANSI_GREEN="\033[32;1m" -export ANSI_YELLOW="\033[33;1m" -export ANSI_BLUE="\033[34;1m" -export ANSI_RESET="\033[0m" -export ANSI_CLEAR="\033[0K" - -# Travis log fold control -# from https://github.com/travis-ci/travis-rubies/blob/build/build.sh - -fold_start() { - echo -e "travis_fold:start:$1\\r${ANSI_YELLOW}$2${ANSI_RESET}" -} - -fold_end() { - echo -en "travis_fold:end:$1\\r" -} - -die() { - echo -e "${ANSI_RED}$1${ANSI_RESET}" - [ "$UTILS_UNITTEST" ] || exit 1 -} - -# source_set(settings) -# -# Source a settings file (extension .set) found in SETUP_PATH -# May be called recursively (from within a settings file) -declare -a SEEN_SETUPS -source_set() { - local set_file=${1//[$'\r']} - local set_dir - local found=0 - [ "${SETUP_PATH}" ] || die "Search path for setup files (SETUP_PATH) is empty" - for set_dir in ${SETUP_PATH//:/ } - do - if [ -e $set_dir/$set_file.set ] - then - if [[ " ${SEEN_SETUPS[@]} " =~ " $set_dir/$set_file.set " ]] - then - echo "Ignoring already included setup file $set_dir/$set_file.set" - return - fi - SEEN_SETUPS+=($set_dir/$set_file.set) - echo "Loading setup file $set_dir/$set_file.set" - local line - while read -r line - do - [ -z "$line" ] && continue - echo $line | grep -q "^#" && continue - if echo $line | grep -q "^include\W" - then - source_set $(echo $line | awk '{ print $2 }') - continue - fi - if echo "$line" | grep -q "^\w\+=" - then - IFS== read var value <<< "${line//[$'\r']}" - value=$(sed "s/^\(\"\)\(.*\)\1\$/\2/g" <<< "$value") # remove surrounding quotes - eval [ "\${$var}" ] || eval "$var=\$value" - fi - done < $set_dir/$set_file.set - found=1 - break - fi - done - [ $found -ne 0 ] || die "Setup file $set_file.set does not exist in SETUP_PATH search path ($SETUP_PATH)" -} - -# update_release_local(varname, place) -# varname name of the variable to set in RELEASE.local -# place place (absolute path) of where variable should point to -# -# Manipulate RELEASE.local in the cache location: -# - replace "$varname=$place" line if it exists and has changed -# - otherwise add "$varname=$place" line and possibly move EPICS_BASE=... line to the end -update_release_local() { - local var=$1 - local place=$2 - local release_local=${CACHEDIR}/RELEASE.local - local updated_line="${var}=${place}" - - local ret=0 - [ -e ${release_local} ] && grep -q "${var}=" ${release_local} || ret=$? - if [ $ret -eq 0 ] - then - existing_line=$(grep "${var}=" ${release_local}) - if [ "${existing_line}" != "${updated_line}" ] - then - sedi "s|${var}=.*|${var}=${place}|g" ${release_local} - fi - else - echo "$var=$place" >> ${release_local} - ret=0 - grep -q "EPICS_BASE=" ${release_local} || ret=$? - if [ $ret -eq 0 ] - then - base_line=$(grep "EPICS_BASE=" ${release_local}) - sedi '\|EPICS_BASE=|d' ${release_local} - echo ${base_line} >> ${release_local} - fi - fi -} - -# add_dependency(dep, tag) -# -# Add a dependency to the cache area: -# - check out (recursive if configured) in the CACHE area unless it already exists and the -# required commit has been built -# - Defaults: -# $dep_DIRNAME = lower case ($dep) -# $dep_REPONAME = lower case ($dep) -# $dep_REPOURL = GitHub / $dep_REPOOWNER (or $REPOOWNER or epics-modules) / $dep_REPONAME .git -# $dep_VARNAME = $dep -# $dep_DEPTH = 5 -# $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) -# - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) -# - Add full path to $modules_to_compile -add_dependency() { - curdir="$PWD" - DEP=$1 - TAG=$2 - dep_lc=${DEP,,} - eval dirname=\${${DEP}_DIRNAME:=${dep_lc}} - eval reponame=\${${DEP}_REPONAME:=${dep_lc}} - eval repourl=\${${DEP}_REPOURL:="https://github.com/\${${DEP}_REPOOWNER:=${REPOOWNER:-epics-modules}}/${reponame}.git"} - eval varname=\${${DEP}_VARNAME:=${DEP}} - eval recursive=\${${DEP}_RECURSIVE:=1} - recursive=${recursive,,} - local recurse="" - [ "$recursive" != "0" -a "$recursive" != "no" ] && recurse="--recursive" - - # determine if $DEP points to a valid release or branch - git ls-remote --quiet --exit-code --refs $repourl "$TAG" > /dev/null 2>&1 || - die "$TAG is neither a tag nor a branch name for $DEP ($repourl)" - - if [ -e $CACHEDIR/$dirname-$TAG ] - then - [ -e $CACHEDIR/$dirname-$TAG/built ] && BUILT=$(cat $CACHEDIR/$dirname-$TAG/built) || BUILT="never" - HEAD=$(cd "$CACHEDIR/$dirname-$TAG" && git log -n1 --pretty=format:%H) - if [ "$HEAD" != "$BUILT" ] - then - rm -fr $CACHEDIR/$dirname-$TAG - else - echo "Found $TAG of dependency $DEP in $CACHEDIR/$dirname-$TAG" - fi - fi - - if [ ! -e $CACHEDIR/$dirname-$TAG ] - then - cd $CACHEDIR - eval depth=\${${DEP}_DEPTH:-"-1"} - case ${depth} in - -1 ) - deptharg="--depth 5" - ;; - 0 ) - deptharg="" - ;; - * ) - deptharg="--depth $depth" - ;; - esac - echo "Cloning $TAG of dependency $DEP into $CACHEDIR/$dirname-$TAG" - git clone --quiet $deptharg $recurse --branch "$TAG" $repourl $dirname-$TAG - ( cd $dirname-$TAG && git log -n1 ) - do_recompile=yes - # add MSI to Base 3.14 - if [ $DEP == "BASE" ] - then - versionfile=$CACHEDIR/$dirname-$TAG/configure/CONFIG_BASE_VERSION - if [ -e ${versionfile} ] && grep -q "BASE_3_14=YES" ${versionfile} - then - echo "Adding MSI 1.7 to $CACHEDIR/$dirname-$TAG" - ( cd $dirname-$TAG; patch -p1 < $SCRIPTDIR/../add-msi-to-314.patch ) - fi - else - # fix non-base modules that do not include the .local files in configure/RELEASE - release=$CACHEDIR/$dirname-$TAG/configure/RELEASE - if [ -e $release ] - then - echo "-include \$(TOP)/../RELEASE.local" > $release - fi - fi - # run hook - eval hook="\${${DEP}_HOOK}" - if [ "$hook" ] - then - if [ -x "$curdir/$hook" ] - then - echo "Running hook $hook in $CACHEDIR/$dirname-$TAG" - ( cd $CACHEDIR/$dirname-$TAG; "$curdir/$hook" ) - else - die "Hook script $hook is not executable or does not exist." - fi - fi - HEAD=$(cd "$CACHEDIR/$dirname-$TAG" && git log -n1 --pretty=format:%H) - echo "$HEAD" > "$CACHEDIR/$dirname-$TAG/built" - cd "$curdir" - fi - - [ "${do_recompile}" ] && modules_to_compile="${modules_to_compile} $CACHEDIR/$dirname-$TAG" - - update_release_local ${varname} $CACHEDIR/$dirname-$TAG -}