travis: fix for EXTRA arguments with spaces/quotes

- feed EXTRA variables into an array to be properly expanded

appveyor-test.py

@@ -377,7 +377,7 @@ class TestSetupForBuild(unittest.TestCase):
     def test_StrawberryInPath(self):
         os.environ['CMP'] = 'vs2019'
         do.setup_for_build(self.args)
-        self.assertTrue(re.search('strawberry', os.environ['PATH']),
+        self.assertTrue(re.search('strawberry', os.environ['PATH'], flags=re.IGNORECASE),
                         'Strawberry Perl location not in PATH for vs2019')
 
 

appveyor/README.md

@@ -27,7 +27,7 @@
     ```
     $ cp .ci/appveyor/.appveyor.yml.example-full .appveyor.yml
     ```
-	
+
  5. Edit the `.appveyor.yml` configuration to include the jobs you want
     AppVeyor to run.
 
@@ -38,16 +38,18 @@
     Select 32bit or 64bit processor architecture.
     3. `environment: / matrix:` \
     List of environment variable settings. Each list element (starting with
-    a dash) is one step on the axis of the build matrix.
-    
-    Your builds will take long.
-    
+    a dash) is one step on the axis of the build matrix. \
+    Set `CMP` to select the compiler: `mingw` for the native
+    [MinGW](http://mingw-w64.org/) GNU compiler, `vs2008` ...`vs2019` 
+    (options listed above) for the Microsoft Visual Studio compilers.
+
+    Your builds will take long. \
     AppVeyor only grants a single worker VM - all jobs of the matrix are
     executed sequentially. Each job will take around 10 minutes.
-    
+
     The `matrix: / exclude:` setting can be used to reduce the number of
     jobs. Check the [AppVeyor docs](https://www.appveyor.com/docs/build-configuration/#build-matrix)
     for more ways to reduce the build matrix size.
-	
+
  6. Push your changes and check
     [ci.appveyor.com](https://ci.appveyor.com/) for your build results.

appveyor/do.py

@@ -41,17 +41,37 @@ else:
 if 'CACHEDIR' in os.environ:
     cachedir = os.environ['CACHEDIR']
 
+vcvars_table = {
+    # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History
+    'vs2019':r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat',
+    'vs2017':r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat',
+    'vs2015':r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat',
+    'vs2013':r'C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat',
+    'vs2012':r'C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat',
+    'vs2010':r'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat',
+    'vs2008':r'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcvarsall.bat',
+}
+
 ciscriptsdir = os.path.abspath(os.path.dirname(sys.argv[0]))
 if os.path.basename(ciscriptsdir) == 'appveyor':
     ciscriptsdir = ciscriptsdir.rstrip(os.pathsep+'appveyor')
 
+if 'BASE' not in os.environ or os.environ['BASE'] == 'SELF':
+    building_base = True
+    places['EPICS_BASE'] = '.'
+else:
+    building_base = False
+
 def modlist():
-    for var in ['ADD_MODULES', 'MODULES']:
-        setup.setdefault(var, '')
-        if var in os.environ:
-            setup[var] = os.environ[var]
-            logger.debug('ENV assignment: %s = %s', var, setup[var])
-    ret = ['BASE'] + setup['ADD_MODULES'].upper().split() + setup['MODULES'].upper().split()
+    if building_base:
+        ret = []
+    else:
+        for var in ['ADD_MODULES', 'MODULES']:
+            setup.setdefault(var, '')
+            if var in os.environ:
+                setup[var] = os.environ[var]
+                logger.debug('ENV assignment: %s = %s', var, setup[var])
+        ret = ['BASE'] + setup['ADD_MODULES'].upper().split() + setup['MODULES'].upper().split()
     logger.debug('Effective module list: %s', ret)
     return ret
 
@@ -69,12 +89,9 @@ def host_info():
     print('platform =', distutils.util.get_platform())
 
     print('{0}Available Visual Studio versions{1}'.format(ANSI_CYAN, ANSI_RESET))
-    from fnmatch import fnmatch
-    for base in (r'C:\Program Files (x86)', r'C:\Program Files'):
-        for root, dirs, files in os.walk(base):
-            for fname in files:
-                if fnmatch(fname, 'vcvarsall.bat'):
-                    print('Found', os.path.join(root, fname))
+    for key in vcvars_table:
+        if os.path.exists(vcvars_table[key]):
+            print('Found', key, 'in', vcvars_table[key])
     sys.stdout.flush()
 
 # Used from unittests
@@ -355,9 +372,9 @@ def setup_for_build(args):
         os.environ['EPICS_HOST_ARCH'] = 'windows-x64' + hostarchsuffix
 
     if os.environ['CMP'] == 'vs2019':
-        # put our strawberry 'perl' in the PATH
-        os.environ['PATH'] = os.pathsep.join([os.path.join(toolsdir, 'strawberry', 'perl', 'site', 'bin'),
-                                              os.path.join(toolsdir, 'strawberry', 'perl', 'bin'),
+        # put strawberry perl in the PATH
+        os.environ['PATH'] = os.pathsep.join([os.path.join(r'C:\Strawberry\perl\site\bin'),
+                                              os.path.join(r'C:\Strawberry\perl\bin'),
                                               os.environ['PATH']])
     if os.environ['CMP'] == 'mingw':
         if 'INCLUDE' not in os.environ:
@@ -377,18 +394,23 @@ def setup_for_build(args):
 
     make = os.path.join(toolsdir, 'make.exe')
 
-    with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
-        lines = f.readlines()
-        for line in lines:
-            (mod, place) = line.strip().split('=')
-            bindir = os.path.join(place, 'bin', os.environ['EPICS_HOST_ARCH'])
-            if os.path.isdir(bindir):
-                dllpaths.append(bindir)
-            if mod == 'EPICS_BASE':
-                base_place = place
-    with open(os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')) as myfile:
-        if 'BASE_3_14=YES' in myfile.read():
-            isbase314 = True
+    base_place = '.'
+    if not building_base:
+        with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
+            lines = f.readlines()
+            for line in lines:
+                (mod, place) = line.strip().split('=')
+                bindir = os.path.join(place, 'bin', os.environ['EPICS_HOST_ARCH'])
+                if os.path.isdir(bindir):
+                    dllpaths.append(bindir)
+                if mod == 'EPICS_BASE':
+                    base_place = place
+
+    cfg_base_version = os.path.join(base_place, 'configure', 'CONFIG_BASE_VERSION')
+    if os.path.exists(cfg_base_version):
+        with open(cfg_base_version) as myfile:
+            if 'BASE_3_14=YES' in myfile.read():
+                isbase314 = True
 
     bindir = os.path.join(os.getcwd(), 'bin', os.environ['EPICS_HOST_ARCH'])
     if os.path.isdir(bindir):
@@ -433,9 +455,12 @@ def prepare(args):
 
     [add_dependency(mod) for mod in modlist()]
 
-    if os.path.isdir('configure'):
-        release_local = os.path.join(cachedir, 'RELEASE.local')
-        shutil.copy(release_local, 'configure')
+    if not building_base:
+        if os.path.isdir('configure'):
+            targetdir = 'configure'
+        else:
+            targetdir = '.'
+        shutil.copy(os.path.join(cachedir, 'RELEASE.local'), targetdir)
 
     print('{0}Configuring EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET))
 
@@ -468,20 +493,6 @@ def prepare(args):
         sp.check_call([zip7, 'e', 'make-{0}.zip'.format(makever)], cwd=toolsdir)
         os.remove(os.path.join(toolsdir, 'make-{0}.zip'.format(makever)))
 
-    perlver = '5.30.0.1'
-    if os.environ['CMP'] == 'vs2019':
-        if not os.path.isdir(os.path.join(toolsdir, 'strawberry')):
-            print('Installing Strawberry Perl {0}'.format(perlver))
-            sys.stdout.flush()
-            sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver),
-                           'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)],
-                          cwd=toolsdir)
-            sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir)
-            os.remove(os.path.join(toolsdir, 'perl-{0}.zip'.format(perlver)))
-            with open(os.devnull, 'w') as devnull:
-                sp.check_call('relocation.pl.bat', shell=True, stdout=devnull,
-                              cwd=os.path.join(toolsdir, 'strawberry'))
-
     setup_for_build(args)
 
     print('{0}EPICS_HOST_ARCH = {1}{2}'.format(ANSI_CYAN, os.environ['EPICS_HOST_ARCH'], ANSI_RESET))
@@ -501,21 +512,22 @@ def prepare(args):
         sys.stdout.flush()
         sp.check_call(['cl'])
 
-    for mod in modlist():
-        place = places[setup[mod+"_VARNAME"]]
-        print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET))
-        call_make(cwd=place, silent=silent_dep_builds)
+    if not building_base:
+        for mod in modlist():
+            place = places[setup[mod+"_VARNAME"]]
+            print('{0}Building dependency {1} in {2}{3}'.format(ANSI_YELLOW, mod, place, ANSI_RESET))
+            call_make(cwd=place, silent=silent_dep_builds)
 
-    print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET))
-    print('Module     Tag          Binaries    Commit')
-    print(100 * '-')
-    for mod in modlist():
-        commit = sp.check_output(['git', 'log', '-n1', '--oneline'], cwd=places[setup[mod+"_VARNAME"]]).strip()
-        print("%-10s %-12s %-11s %s" % (mod, setup[mod], 'rebuilt', commit))
+        print('{0}Dependency module information{1}'.format(ANSI_CYAN, ANSI_RESET))
+        print('Module     Tag          Binaries    Commit')
+        print(100 * '-')
+        for mod in modlist():
+            commit = sp.check_output(['git', 'log', '-n1', '--oneline'], cwd=places[setup[mod+"_VARNAME"]]).strip()
+            print("%-10s %-12s %-11s %s" % (mod, setup[mod], 'rebuilt', commit))
 
-    print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET))
-    with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
-        print(f.read().strip())
+        print('{0}Contents of RELEASE.local{1}'.format(ANSI_CYAN, ANSI_RESET))
+        with open(os.path.join(cachedir, 'RELEASE.local'), 'r') as f:
+            print(f.read().strip())
 
 def build(args):
     setup_for_build(args)
@@ -544,26 +556,17 @@ def with_vcvars(cmd):
     # cf. https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line
 
     info = {
-        'python':sys.executable,
-        'self':sys.argv[0],
+        'python': sys.executable,
+        'self': sys.argv[0],
         'cmd':cmd,
     }
 
     info['arch'] = {
-        'x86':'x86',   # 'amd64_x86' ??
-        'x64':'amd64',
+        'x86': 'x86',   # 'amd64_x86' ??
+        'x64': 'amd64',
     }[os.environ['PLATFORM'].lower()] # 'x86' or 'x64'
 
-    info['vcvars'] = {
-        # https://en.wikipedia.org/wiki/Microsoft_Visual_Studio#History
-        'vs2019':r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat',
-        'vs2017':r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat',
-        'vs2015':r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat',
-        'vs2013':r'C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat',
-        'vs2012':r'C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat',
-        'vs2010':r'C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat',
-        'vs2008':r'C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcvarsall.bat',
-    }[CC]
+    info['vcvars'] = vcvars_table[CC]
 
     script='''
 call "{vcvars}" {arch}

travis/.travis.yml.example-full

@@ -53,6 +53,8 @@ script:
 # SET         source setup file
 # ADD_MODULES extra modules (for a specific job)
 # EXTRA       content will be added to make command line
+# EXTRA1..5   additional arguments for the make command
+#             (one argument per variable)
 # STATIC      set to YES for static build (default: NO)
 # TEST        set to NO to skip running the tests (default: YES)
 # VV          set to make build scripts verbose (default: unset)

travis/build.sh

@@ -13,7 +13,15 @@ export EPICS_BASE
 [ -z "$EPICS_HOST_ARCH" -a -f $EPICS_BASE/startup/EpicsHostArch.pl ] && EPICS_HOST_ARCH=$(perl $EPICS_BASE/startup/EpicsHostArch.pl)
 export EPICS_HOST_ARCH
 
-make -j2 $EXTRA
+# use array variable to get the quoting right while using separate words for arguments
+[ -n "$EXTRA0" ] && EXTRA[0]="$EXTRA0"
+[ -n "$EXTRA1" ] && EXTRA[1]="$EXTRA1"
+[ -n "$EXTRA2" ] && EXTRA[2]="$EXTRA2"
+[ -n "$EXTRA3" ] && EXTRA[3]="$EXTRA3"
+[ -n "$EXTRA4" ] && EXTRA[4]="$EXTRA4"
+[ -n "$EXTRA5" ] && EXTRA[5]="$EXTRA5"
+
+make -j2 "${EXTRA[@]}"
 
 ret=0
 
@@ -21,7 +29,7 @@ if [ "$TEST" != "NO" ]
 then
   make tapfiles || ret=$?
 
-  make -s test-results
+  make -sk test-results
 fi
 
 exit $ret

travis/prepare.sh

@@ -29,6 +29,23 @@ CACHEDIR=${CACHEDIR:-${HOME}/.cache}
 
 echo -e "${ANSI_YELLOW}Using bash version $BASH_VERSION${ANSI_RESET}"
 
+if [ -f /etc/hosts ]
+then
+    # The travis-ci "bionic" image throws us a curveball in /etc/hosts
+    # by including two entries for localhost.  The first for 127.0.1.1
+    # which causes epicsSockResolveTest to fail.
+    #  cat /etc/hosts
+    #  ...
+    #  127.0.1.1 localhost localhost ip4-loopback
+    #  127.0.0.1 localhost nettuno travis vagrant travis-job-....
+
+    sudo sed -i -e '/^127\.0\.1\.1/ s|localhost\s*||g' /etc/hosts
+
+    echo "==== /etc/hosts"
+    cat /etc/hosts
+    echo "===="
+fi
+
 # Load settings
 # -------------
 
@@ -45,23 +62,31 @@ fold_end load.settings
 # Check out dependencies
 # ----------------------
 
-fold_start check.out.dependencies "Checking/cloning dependencies"
+if [ "$BASE" != "SELF" ]
+then
+  fold_start check.out.dependencies "Checking/cloning dependencies"
 
-for mod in BASE $ADD_MODULES $MODULES
-do
-  mod_uc=${mod^^}
-  eval add_dependency $mod_uc \${${mod_uc}:=master}
-done
-[ -e ./configure ] && cp ${CACHEDIR}/RELEASE.local ./configure/RELEASE.local
+  for mod in BASE $ADD_MODULES $MODULES
+  do
+    mod_uc=${mod^^}
+    eval add_dependency $mod_uc \${${mod_uc}:=master}
+  done
+  [ -e ./configure ] && cp ${CACHEDIR}/RELEASE.local ./configure/RELEASE.local
 
-fold_end check.out.dependencies
+  fold_end check.out.dependencies
+fi
 
 # Set up compiler
 # ---------------
 
 fold_start set.up.epics_build "Setting up EPICS build system"
 
-eval $(grep "EPICS_BASE=" ${CACHEDIR}/RELEASE.local)
+if [ "$BASE" = "SELF" ]
+then
+  EPICS_BASE=$CURDIR
+else
+  eval $(grep "EPICS_BASE=" ${CACHEDIR}/RELEASE.local)
+fi
 export EPICS_BASE
 echo "EPICS_BASE=$EPICS_BASE"
 
@@ -70,7 +95,7 @@ echo "EPICS_BASE=$EPICS_BASE"
 export EPICS_HOST_ARCH
 echo "EPICS_HOST_ARCH=$EPICS_HOST_ARCH"
 
-if echo ${modules_to_compile} | grep -q "$EPICS_BASE"
+if echo ${modules_to_compile} | grep -q "$EPICS_BASE" || [ "$BASE" = "SELF" ]
 then
 
   # requires wine and g++-mingw-w64-i686
@@ -180,6 +205,8 @@ fold_end set.up.compiler
 echo "\$ make --version"
 make --version
 
+[ "$BASE" = "SELF" ] && exit 0
+
 # Build required dependencies
 # ---------------------------