epics/ci-scripts
0
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

appveyor: add add_dependency()

Browse Source
This commit is contained in:
Ralph Lange
2020-02-17 10:04:02 +01:00
parent 355a5c2fb7
commit cd0becff06
2 changed files with 191 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
appveyor-test.py
View File

@@ -5,9 +5,19 @@
# SET=test00 in .appveyor.yml runs the tests in this script
# all other jobs are started as compile jobs
import sys, os, fileinput
from __future__ import print_function
import sys, os, shutil, fileinput
import re
import unittest
def find_in_file(regex, filename):
file = open (filename, "r")
for line in file:
if re.search(regex, line):
return True
return False
def getStringIO():
if (sys.version_info > (3, 0)):
import io
@@ -19,6 +29,9 @@ def getStringIO():
sys.path.append('appveyor')
import do
# we're working with tags (detached heads) a lot: suppress advice
do.call_git(['config', '--global', 'advice.detachedHead', 'false'])
class TestSourceSet(unittest.TestCase):
def setUp(self):
@@ -101,34 +114,94 @@ class TestUpdateReleaseLocal(unittest.TestCase):
self.assertEqual(line.strip(), 'MOD1=/foo/bar1',
'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')'
.format(line))
found['mod1'] += 1
if 'mod1' in found:
found['mod1'] += 1
else:
found['mod1'] = 1
foundat['mod1'] = fileinput.filelineno()
if 'MOD2=' in line:
self.assertEqual(line.strip(), 'MOD2=/foo/bar2',
'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')'
.format(line))
found['mod2'] += 1
if 'mod2' in found:
found['mod2'] += 1
else:
found['mod2'] = 1
foundat['mod2'] = fileinput.filelineno()
if 'EPICS_BASE=' in line:
self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo',
'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')'
.format(line))
found['base'] += 1
if 'base' in found:
found['base'] += 1
else:
found['base'] = 1
foundat['base'] = fileinput.filelineno()
fileinput.close()
self.assertEqual(found['mod1'], 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1']))
self.assertEqual(found['mod2'], 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2']))
self.assertEqual(found['base'], 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base']))
self.assertEqual(found['mod1'], 1,
'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1']))
self.assertEqual(found['mod2'], 1,
'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2']))
self.assertEqual(found['base'], 1,
'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base']))
self.assertGreater(foundat['base'], foundat['mod2'],
'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundat['base'], foundat['mod2']))
'EPICS_BASE (line {0}) appears before MOD2 (line {1})'
.format(foundat['base'], foundat['mod2']))
self.assertGreater(foundat['mod2'], foundat['mod1'],
'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1']))
class TestAddDependency(unittest.TestCase):
hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096"
location = os.path.join(do.cachedir, 'base-R3.15.6')
licensefile = os.path.join(location, 'LICENSE')
checked_file = os.path.join(location, 'checked_out')
release_file = os.path.join(location, 'configure', 'RELEASE')
def setUp(self):
os.environ['SETUP_PATH'] = '.:appveyor'
if os.path.exists(self.location):
shutil.rmtree(self.location)
do.clear_lists()
do.source_set('defaults')
def test_MissingDependency(self):
do.add_dependency('BASE', 'R3.15.6')
self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out')
self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written')
with open(self.checked_file, 'r') as bfile:
checked_out = bfile.read().strip()
bfile.close()
self.assertEqual(checked_out, self.hash_3_15_6,
'Wrong commit of dependency checked out (expected=\"{0}\" found=\"{1}\")'
.format(self.hash_3_15_6, checked_out))
self.assertFalse(find_in_file('include \$\(TOP\)/../RELEASE.local', self.release_file),
'RELEASE in Base includes TOP/../RELEASE.local')
def test_UpToDateDependency(self):
do.add_dependency('BASE', 'R3.15.6')
os.remove(self.licensefile)
do.add_dependency('BASE', 'R3.15.6')
self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency')
def test_OutdatedDependency(self):
do.add_dependency('BASE', 'R3.15.6')
os.remove(self.licensefile)
with open(self.checked_file, "w") as fout:
print('XXX not the right hash XXX', file=fout)
fout.close()
do.add_dependency('BASE', 'R3.15.6')
self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency')
with open(self.checked_file, 'r') as bfile:
checked_out = bfile.read().strip()
bfile.close()
self.assertEqual(checked_out, self.hash_3_15_6,
"Wrong commit of dependency checked out (expected='{0}' found='{1}')"
.format(self.hash_3_15_6, checked_out))
if __name__ == "__main__":
# suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet)
# suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal)
# suite = unittest.TestLoader().loadTestsFromTestCase(TestAddDependency)
# unittest.TextTestRunner(verbosity=2).run(suite)
unittest.main()

119
appveyor/do.py
View File

@@ -4,9 +4,9 @@
from __future__ import print_function
import sys, os, fileinput
import sys, os, shutil, fileinput
import logging
import subprocess as SP
import subprocess as sp
import distutils.util
logger = logging.getLogger(__name__)
@@ -21,7 +21,9 @@ ANSI_RESET = "\033[0m"
ANSI_CLEAR = "\033[0K"
seen_setups = []
modules_to_compile = []
setup = {}
if 'HomeDrive' in os.environ:
cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache')
elif 'HOME' in os.environ:
@@ -32,21 +34,23 @@ else:
# Used from unittests
def clear_lists():
del seen_setups[:]
del modules_to_compile[:]
setup.clear()
# source_set(setup)
#
# Source a settings file (extension .set) found in the setup_dirs path
# May be called recursively (from within a setup file)
def source_set(set):
def source_set(name):
found = False
# allowed separators: colon or whitespace
setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split()
if len(setup_dirs) == 0:
raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET))
for set_dir in setup_dirs:
set_file = os.path.join(set_dir, set) + ".set"
set_file = os.path.join(set_dir, name) + ".set"
if set_file in seen_setups:
print("Ignoring already included setup file {0}".format(set_file))
@@ -100,16 +104,17 @@ def update_release_local(var, place):
found = False
logger.debug("Opening RELEASE.local for adding '%s'", updated_line)
for line in fileinput.input(release_local, inplace=1):
outputline = line.strip()
if 'EPICS_BASE=' in line:
logger.debug("Found EPICS_BASE line '%s', not writing it", base_line)
base_line = line.strip()
logger.debug("Found EPICS_BASE line '%s', not writing it", base_line)
continue
elif '{0}='.format(var) in line:
logger.debug("Found '%s=' line, replacing", var)
found = True
line = updated_line
outputline = updated_line
logger.debug("Writing line to RELEASE.local: '%s'", outputline)
print(line)
print(outputline)
fileinput.close()
fout = open(release_local,"a")
if not found:
@@ -120,6 +125,19 @@ def update_release_local(var, place):
print(base_line, file=fout)
fout.close()
def set_setup_from_env(dep):
for postf in ['_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL',
'_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']:
if dep+postf in os.environ:
setup[dep+postf] = os.getenv(dep+postf)
def call_git(args, **kws):
logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd())
sys.stdout.flush()
exitcode = sp.call(['git'] + args, **kws)
logger.debug('EXEC DONE')
return exitcode
# add_dependency(dep, tag)
#
# Add a dependency to the cache area:
@@ -135,7 +153,87 @@ def update_release_local(var, place):
# - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there)
# - Add full path to $modules_to_compile
def add_dependency(dep, tag):
pass
curdir = os.getcwd()
set_setup_from_env(dep)
setup.setdefault(dep+"_DIRNAME", dep.lower())
setup.setdefault(dep+"_REPONAME", dep.lower())
setup.setdefault('REPOOWNER', 'epics-modules')
setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER'])
setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git'
.format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME']))
setup.setdefault(dep+"_VARNAME", dep)
setup.setdefault(dep+"_RECURSIVE", 1)
setup.setdefault(dep+"_DEPTH", -1)
if setup[dep+'_RECURSIVE'] not in [0, 'no']:
recursearg = "--recursive"
else:
recursearg = ''
# determine if dep points to a valid release or branch
if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]):
raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}"
.format(ANSI_RED, tag, dep, setup[dep+'_REPOURL'], ANSI_RESET))
dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag)
place = os.path.join(cachedir, dirname)
checked_file = os.path.join(place, "checked_out")
if os.path.isdir(place):
logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place)
# check HEAD commit against the hash in marker file
if os.path.exists(checked_file):
with open(checked_file, 'r') as bfile:
checked_out = bfile.read().strip()
bfile.close()
else:
checked_out = 'never'
head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True)
logger.debug('Found checked_out commit %s, git head is %s', checked_out, head)
if head != checked_out:
logger.debug('Dependency %s out of date - removing', dep)
shutil.rmtree(place)
else:
print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place))
if not os.path.isdir(place):
if not os.path.isdir(cachedir):
os.makedirs(cachedir)
# clone dependency
os.chdir(cachedir)
deptharg = {
-1:['--depth', '5'],
0:[],
}.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']])
print('Cloning {0} of dependency {1} into {2}'
.format(tag, dep, place))
call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname])
sp.check_call(['cd {0}; git log -n1'.format(place)], shell=True)
modules_to_compile.append(place)
# force including RELEASE.local for non-base modules by overwriting their configure/RELEASE
if dep != 'BASE':
release = os.path.join(place, "configure", "RELEASE")
if os.path.exists(release):
fout = open(release, 'w')
print('-include $(TOP)/../RELEASE.local', file=fout)
fout.close()
# run hook if defined
if dep+'_HOOK' in setup:
hook = os.path.join(place, setup[dep+'_HOOK'])
if os.path.exists(hook):
print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place))
os.chdir(place)
sp.check_call(hook, shell=True)
# write checked out commit hash to marker file
head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True)
logger.debug('Writing hash of checked-out dependency (%s) to marker file', head)
with open(checked_file, "w") as fout:
print(head, file=fout)
fout.close()
update_release_local(setup[dep+"_VARNAME"], place)
os.chdir(curdir)
def prepare(args):
print(sys.version)
@@ -145,10 +243,13 @@ def prepare(args):
print('platform = ', distutils.util.get_platform())
print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET))
source_set(default)
source_set('default')
if 'SET' in os.environ:
source_set(os.environ['SET'])
# we're working with tags (detached heads) a lot: suppress advice
call_git(['config', '--global', 'advice.detachedHead', 'false'])
print('Installing dependencies')
def build(args):