From cd0becff0664f0e9997d93115995dadeaa9c9b51 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Mon, 17 Feb 2020 10:04:02 +0100 Subject: [PATCH] appveyor: add add_dependency() --- appveyor-test.py | 89 +++++++++++++++++++++++++++++++---- appveyor/do.py | 119 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 191 insertions(+), 17 deletions(-) diff --git a/appveyor-test.py b/appveyor-test.py index 42478ea..a9643c5 100644 --- a/appveyor-test.py +++ b/appveyor-test.py @@ -5,9 +5,19 @@ # SET=test00 in .appveyor.yml runs the tests in this script # all other jobs are started as compile jobs -import sys, os, fileinput +from __future__ import print_function + +import sys, os, shutil, fileinput +import re import unittest +def find_in_file(regex, filename): + file = open (filename, "r") + for line in file: + if re.search(regex, line): + return True + return False + def getStringIO(): if (sys.version_info > (3, 0)): import io @@ -19,6 +29,9 @@ def getStringIO(): sys.path.append('appveyor') import do +# we're working with tags (detached heads) a lot: suppress advice +do.call_git(['config', '--global', 'advice.detachedHead', 'false']) + class TestSourceSet(unittest.TestCase): def setUp(self): @@ -101,34 +114,94 @@ class TestUpdateReleaseLocal(unittest.TestCase): self.assertEqual(line.strip(), 'MOD1=/foo/bar1', 'MOD1 not set correctly (expected \'MOD1=/foo/bar1\' found \'{0}\')' .format(line)) - found['mod1'] += 1 + if 'mod1' in found: + found['mod1'] += 1 + else: + found['mod1'] = 1 foundat['mod1'] = fileinput.filelineno() if 'MOD2=' in line: self.assertEqual(line.strip(), 'MOD2=/foo/bar2', 'MOD2 not set correctly (expected \'MOD2=/foo/bar2\' found \'{0}\')' .format(line)) - found['mod2'] += 1 + if 'mod2' in found: + found['mod2'] += 1 + else: + found['mod2'] = 1 foundat['mod2'] = fileinput.filelineno() if 'EPICS_BASE=' in line: self.assertEqual(line.strip(), 'EPICS_BASE=/bar/foo', 'EPICS_BASE not set correctly (expected \'EPICS_BASE=/bar/foo\' found \'{0}\')' .format(line)) - found['base'] += 1 + if 'base' in found: + found['base'] += 1 + else: + found['base'] = 1 foundat['base'] = fileinput.filelineno() fileinput.close() - self.assertEqual(found['mod1'], 1, 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) - self.assertEqual(found['mod2'], 1, 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) - self.assertEqual(found['base'], 1, 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) + self.assertEqual(found['mod1'], 1, + 'MOD1 does not appear once in RELEASE.local (found {0})'.format(found['mod1'])) + self.assertEqual(found['mod2'], 1, + 'MOD2 does not appear once in RELEASE.local (found {0})'.format(found['mod2'])) + self.assertEqual(found['base'], 1, + 'EPICS_BASE does not appear once in RELEASE.local (found {0})'.format(found['base'])) self.assertGreater(foundat['base'], foundat['mod2'], - 'EPICS_BASE (line {0}) appears before MOD2 (line {1})'.format(foundat['base'], foundat['mod2'])) + 'EPICS_BASE (line {0}) appears before MOD2 (line {1})' + .format(foundat['base'], foundat['mod2'])) self.assertGreater(foundat['mod2'], foundat['mod1'], 'MOD2 (line {0}) appears before MOD1 (line {1})'.format(foundat['mod2'], foundat['mod1'])) +class TestAddDependency(unittest.TestCase): + hash_3_15_6 = "ce7943fb44beb22b453ddcc0bda5398fadf72096" + location = os.path.join(do.cachedir, 'base-R3.15.6') + licensefile = os.path.join(location, 'LICENSE') + checked_file = os.path.join(location, 'checked_out') + release_file = os.path.join(location, 'configure', 'RELEASE') + def setUp(self): + os.environ['SETUP_PATH'] = '.:appveyor' + if os.path.exists(self.location): + shutil.rmtree(self.location) + do.clear_lists() + do.source_set('defaults') + + def test_MissingDependency(self): + do.add_dependency('BASE', 'R3.15.6') + self.assertTrue(os.path.exists(self.licensefile), 'Missing dependency was not checked out') + self.assertTrue(os.path.exists(self.checked_file), 'Checked-out commit marker was not written') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + 'Wrong commit of dependency checked out (expected=\"{0}\" found=\"{1}\")' + .format(self.hash_3_15_6, checked_out)) + self.assertFalse(find_in_file('include \$\(TOP\)/../RELEASE.local', self.release_file), + 'RELEASE in Base includes TOP/../RELEASE.local') + + def test_UpToDateDependency(self): + do.add_dependency('BASE', 'R3.15.6') + os.remove(self.licensefile) + do.add_dependency('BASE', 'R3.15.6') + self.assertFalse(os.path.exists(self.licensefile), 'Check out on top of existing up-to-date dependency') + + def test_OutdatedDependency(self): + do.add_dependency('BASE', 'R3.15.6') + os.remove(self.licensefile) + with open(self.checked_file, "w") as fout: + print('XXX not the right hash XXX', file=fout) + fout.close() + do.add_dependency('BASE', 'R3.15.6') + self.assertTrue(os.path.exists(self.licensefile), 'No check-out on top of out-of-date dependency') + with open(self.checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + self.assertEqual(checked_out, self.hash_3_15_6, + "Wrong commit of dependency checked out (expected='{0}' found='{1}')" + .format(self.hash_3_15_6, checked_out)) if __name__ == "__main__": # suite = unittest.TestLoader().loadTestsFromTestCase(TestSourceSet) # suite = unittest.TestLoader().loadTestsFromTestCase(TestUpdateReleaseLocal) +# suite = unittest.TestLoader().loadTestsFromTestCase(TestAddDependency) # unittest.TextTestRunner(verbosity=2).run(suite) unittest.main() diff --git a/appveyor/do.py b/appveyor/do.py index 2817315..bf8a822 100644 --- a/appveyor/do.py +++ b/appveyor/do.py @@ -4,9 +4,9 @@ from __future__ import print_function -import sys, os, fileinput +import sys, os, shutil, fileinput import logging -import subprocess as SP +import subprocess as sp import distutils.util logger = logging.getLogger(__name__) @@ -21,7 +21,9 @@ ANSI_RESET = "\033[0m" ANSI_CLEAR = "\033[0K" seen_setups = [] +modules_to_compile = [] setup = {} + if 'HomeDrive' in os.environ: cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') elif 'HOME' in os.environ: @@ -32,21 +34,23 @@ else: # Used from unittests def clear_lists(): del seen_setups[:] + del modules_to_compile[:] setup.clear() # source_set(setup) # # Source a settings file (extension .set) found in the setup_dirs path # May be called recursively (from within a setup file) -def source_set(set): +def source_set(name): found = False + # allowed separators: colon or whitespace setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() if len(setup_dirs) == 0: raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) for set_dir in setup_dirs: - set_file = os.path.join(set_dir, set) + ".set" + set_file = os.path.join(set_dir, name) + ".set" if set_file in seen_setups: print("Ignoring already included setup file {0}".format(set_file)) @@ -100,16 +104,17 @@ def update_release_local(var, place): found = False logger.debug("Opening RELEASE.local for adding '%s'", updated_line) for line in fileinput.input(release_local, inplace=1): + outputline = line.strip() if 'EPICS_BASE=' in line: - logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) base_line = line.strip() + logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) continue elif '{0}='.format(var) in line: logger.debug("Found '%s=' line, replacing", var) found = True - line = updated_line + outputline = updated_line logger.debug("Writing line to RELEASE.local: '%s'", outputline) - print(line) + print(outputline) fileinput.close() fout = open(release_local,"a") if not found: @@ -120,6 +125,19 @@ def update_release_local(var, place): print(base_line, file=fout) fout.close() +def set_setup_from_env(dep): + for postf in ['_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', + '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: + if dep+postf in os.environ: + setup[dep+postf] = os.getenv(dep+postf) + +def call_git(args, **kws): + logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd()) + sys.stdout.flush() + exitcode = sp.call(['git'] + args, **kws) + logger.debug('EXEC DONE') + return exitcode + # add_dependency(dep, tag) # # Add a dependency to the cache area: @@ -135,7 +153,87 @@ def update_release_local(var, place): # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep, tag): - pass + curdir = os.getcwd() + set_setup_from_env(dep) + setup.setdefault(dep+"_DIRNAME", dep.lower()) + setup.setdefault(dep+"_REPONAME", dep.lower()) + setup.setdefault('REPOOWNER', 'epics-modules') + setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) + setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' + .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) + setup.setdefault(dep+"_VARNAME", dep) + setup.setdefault(dep+"_RECURSIVE", 1) + setup.setdefault(dep+"_DEPTH", -1) + if setup[dep+'_RECURSIVE'] not in [0, 'no']: + recursearg = "--recursive" + else: + recursearg = '' + + # determine if dep points to a valid release or branch + if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]): + raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" + .format(ANSI_RED, tag, dep, setup[dep+'_REPOURL'], ANSI_RESET)) + + dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag) + place = os.path.join(cachedir, dirname) + checked_file = os.path.join(place, "checked_out") + if os.path.isdir(place): + logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) + # check HEAD commit against the hash in marker file + if os.path.exists(checked_file): + with open(checked_file, 'r') as bfile: + checked_out = bfile.read().strip() + bfile.close() + else: + checked_out = 'never' + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) + if head != checked_out: + logger.debug('Dependency %s out of date - removing', dep) + shutil.rmtree(place) + else: + print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) + + if not os.path.isdir(place): + if not os.path.isdir(cachedir): + os.makedirs(cachedir) + # clone dependency + os.chdir(cachedir) + deptharg = { + -1:['--depth', '5'], + 0:[], + }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) + print('Cloning {0} of dependency {1} into {2}' + .format(tag, dep, place)) + call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname]) + sp.check_call(['cd {0}; git log -n1'.format(place)], shell=True) + modules_to_compile.append(place) + + # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE + if dep != 'BASE': + release = os.path.join(place, "configure", "RELEASE") + if os.path.exists(release): + fout = open(release, 'w') + print('-include $(TOP)/../RELEASE.local', file=fout) + fout.close() + + # run hook if defined + if dep+'_HOOK' in setup: + hook = os.path.join(place, setup[dep+'_HOOK']) + if os.path.exists(hook): + print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) + os.chdir(place) + sp.check_call(hook, shell=True) + + # write checked out commit hash to marker file + head = sp.check_output(['cd {0}; git log -n1 --pretty=format:%H'.format(place)], shell=True) + logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) + with open(checked_file, "w") as fout: + print(head, file=fout) + fout.close() + + update_release_local(setup[dep+"_VARNAME"], place) + os.chdir(curdir) def prepare(args): print(sys.version) @@ -145,10 +243,13 @@ def prepare(args): print('platform = ', distutils.util.get_platform()) print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) - source_set(default) + source_set('default') if 'SET' in os.environ: source_set(os.environ['SET']) + # we're working with tags (detached heads) a lot: suppress advice + call_git(['config', '--global', 'advice.detachedHead', 'false']) + print('Installing dependencies') def build(args):