#!/usr/bin/env python """Windows (AppVeyor) ci build script """ from __future__ import print_function import sys, os, stat, shutil import fileinput import logging import re import subprocess as sp import distutils.util logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) # Setup ANSI Colors ANSI_RED = "\033[31;1m" ANSI_GREEN = "\033[32;1m" ANSI_YELLOW = "\033[33;1m" ANSI_BLUE = "\033[34;1m" ANSI_RESET = "\033[0m" ANSI_CLEAR = "\033[0K" seen_setups = [] modules_to_compile = [] setup = {} place = {} if 'HomeDrive' in os.environ: cachedir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.cache') toolsdir = os.path.join(os.getenv('HomeDrive'), os.getenv('HomePath'), '.tools') elif 'HOME' in os.environ: cachedir = os.path.join(os.getenv('HOME'), '.cache') toolsdir = os.path.join(os.getenv('HOME'), '.tools') else: cachedir = os.path.join('.', '.cache') toolsdir = os.path.join('.', '.tools') zip7 = 'C:\\Program Files\\7-Zip\\7z' # Used from unittests def clear_lists(): del seen_setups[:] del modules_to_compile[:] setup.clear() place.clear() # Error-handler to make shutil.rmtree delete read-only files on Windows def remove_readonly(func, path, excinfo): os.chmod(path, stat.S_IWRITE) func(path) # source_set(setup) # # Source a settings file (extension .set) found in the setup_dirs path # May be called recursively (from within a setup file) def source_set(name): found = False # allowed separators: colon or whitespace setup_dirs = os.getenv('SETUP_PATH', "").replace(':', ' ').split() if len(setup_dirs) == 0: raise NameError("{0}Search path for setup files (SETUP_PATH) is empty{1}".format(ANSI_RED,ANSI_RESET)) for set_dir in setup_dirs: set_file = os.path.join(set_dir, name) + ".set" if set_file in seen_setups: print("Ignoring already included setup file {0}".format(set_file)) return if os.path.isfile(set_file): seen_setups.append(set_file) print("Loading setup file {0}".format(set_file)) with open(set_file) as fp: for line in fp: logger.debug('Next line: %s', line.strip()) if not line.strip() or line.strip()[0] == '#': continue if line.startswith("include"): logger.debug('Found an include, reading %s', line.split()[1]) source_set(line.split()[1]) continue assign = line.replace('"', '').strip().split("=", 1) logger.debug('Interpreting as assignment') setup.setdefault(assign[0], os.getenv(assign[0], "")) if not setup[assign[0]].strip(): logger.debug('Doing assignment: %s = %s', assign[0], assign[1]) setup[assign[0]] = assign[1] found = True break if not found: raise NameError("{0}Setup file {1} does not exist in SETUP_PATH search path ({2}){3}" .format(ANSI_RED, name, setup_dirs, ANSI_RESET)) # update_release_local(var, location) # var name of the variable to set in RELEASE.local # location location (absolute path) of where variable should point to # # Manipulate RELEASE.local in the cache location: # - replace "$var=$location" line if it exists and has changed # - otherwise add "$var=$location" line and possibly move EPICS_BASE=... line to the end # Set place[var] = location def update_release_local(var, location): release_local = os.path.join(cachedir, 'RELEASE.local') updated_line = '{0}={1}'.format(var, location) place[var] = location if not os.path.exists(release_local): logger.debug('RELEASE.local does not exist, creating it') try: os.makedirs(cachedir) except: pass fout = open(release_local, 'w') fout.close() base_line = '' found = False logger.debug("Opening RELEASE.local for adding '%s'", updated_line) for line in fileinput.input(release_local, inplace=1): outputline = line.strip() if 'EPICS_BASE=' in line: base_line = line.strip() logger.debug("Found EPICS_BASE line '%s', not writing it", base_line) continue elif '{0}='.format(var) in line: logger.debug("Found '%s=' line, replacing", var) found = True outputline = updated_line logger.debug("Writing line to RELEASE.local: '%s'", outputline) print(outputline) fileinput.close() fout = open(release_local,"a") if not found: logger.debug("Adding new definition: '%s'", updated_line) print(updated_line, file=fout) if base_line: logger.debug("Writing EPICS_BASE line: '%s'", base_line) print(base_line, file=fout) fout.close() def set_setup_from_env(dep): for postf in ['_DIRNAME', '_REPONAME', '_REPOOWNER', '_REPOURL', '_VARNAME', '_RECURSIVE', '_DEPTH', '_HOOK']: if dep+postf in os.environ: setup[dep+postf] = os.getenv(dep+postf) def call_git(args, **kws): logger.debug("EXEC '%s' in %s", ' '.join(['git'] + args), os.getcwd()) sys.stdout.flush() exitcode = sp.call(['git'] + args, **kws) logger.debug('EXEC DONE') return exitcode def get_git_hash(place): logger.debug("EXEC 'git log -n1 --pretty=format:%%H' in %s", place) sys.stdout.flush() head = sp.check_output(['git', 'log', '-n1', '--pretty=format:%H'], cwd=place).decode() logger.debug('EXEC DONE') return head # add_dependency(dep, tag) # # Add a dependency to the cache area: # - check out (recursive if configured) in the CACHE area unless it already exists and the # required commit has been built # - Defaults: # $dep_DIRNAME = lower case ($dep) # $dep_REPONAME = lower case ($dep) # $dep_REPOURL = GitHub / $dep_REPOOWNER (or $REPOOWNER or epics-modules) / $dep_REPONAME .git # $dep_VARNAME = $dep # $dep_DEPTH = 5 # $dep_RECURSIVE = 1/YES (0/NO to for a flat clone) # - Add $dep_VARNAME line to the RELEASE.local file in the cache area (unless already there) # - Add full path to $modules_to_compile def add_dependency(dep, tag): set_setup_from_env(dep) setup.setdefault(dep+"_DIRNAME", dep.lower()) setup.setdefault(dep+"_REPONAME", dep.lower()) setup.setdefault('REPOOWNER', 'epics-modules') setup.setdefault(dep+"_REPOOWNER", setup['REPOOWNER']) setup.setdefault(dep+"_REPOURL", 'https://github.com/{0}/{1}.git' .format(setup[dep+'_REPOOWNER'], setup[dep+'_REPONAME'])) setup.setdefault(dep+"_VARNAME", dep) setup.setdefault(dep+"_RECURSIVE", 1) setup.setdefault(dep+"_DEPTH", -1) if setup[dep+'_RECURSIVE'] not in [0, 'no']: recursearg = "--recursive" else: recursearg = '' # determine if dep points to a valid release or branch if call_git(['ls-remote', '--quiet', '--exit-code', '--refs', setup[dep+'_REPOURL'], tag]): raise RuntimeError("{0}{1} is neither a tag nor a branch name for {2} ({3}){4}" .format(ANSI_RED, tag, dep, setup[dep+'_REPOURL'], ANSI_RESET)) dirname = setup[dep+'_DIRNAME']+'-{0}'.format(tag) place = os.path.join(cachedir, dirname) checked_file = os.path.join(place, "checked_out") if os.path.isdir(place): logger.debug('Dependency %s: directory %s exists, comparing checked-out commit', dep, place) # check HEAD commit against the hash in marker file if os.path.exists(checked_file): with open(checked_file, 'r') as bfile: checked_out = bfile.read().strip() bfile.close() else: checked_out = 'never' head = get_git_hash(place) logger.debug('Found checked_out commit %s, git head is %s', checked_out, head) if head != checked_out: logger.debug('Dependency %s out of date - removing', dep) shutil.rmtree(place, onerror=remove_readonly) else: print('Found {0} of dependency {1} up-to-date in {2}'.format(tag, dep, place)) if not os.path.isdir(place): if not os.path.isdir(cachedir): os.makedirs(cachedir) # clone dependency deptharg = { -1:['--depth', '5'], 0:[], }.get(setup[dep+'_DEPTH'], ['--depth', setup[dep+'_DEPTH']]) print('Cloning {0} of dependency {1} into {2}' .format(tag, dep, place)) call_git(['clone', '--quiet'] + deptharg + [recursearg, '--branch', tag, setup[dep+'_REPOURL'], dirname], cwd=cachedir) sp.check_call(['git', 'log', '-n1']) modules_to_compile.append(place) # force including RELEASE.local for non-base modules by overwriting their configure/RELEASE if dep != 'BASE': release = os.path.join(place, "configure", "RELEASE") if os.path.exists(release): fout = open(release, 'w') print('-include $(TOP)/../RELEASE.local', file=fout) fout.close() # run hook if defined if dep+'_HOOK' in setup: hook = os.path.join(place, setup[dep+'_HOOK']) if os.path.exists(hook): print('Running hook {0} in {1}'.format(setup[dep+'_HOOK'], place)) sp.check_call(hook, shell=True, cwd=place) # write checked out commit hash to marker file head = get_git_hash(place) logger.debug('Writing hash of checked-out dependency (%s) to marker file', head) with open(checked_file, "w") as fout: print(head, file=fout) fout.close() update_release_local(setup[dep+"_VARNAME"], place) def prepare(): print(sys.version) print('PYTHONPATH') for dname in sys.path: print(' ', dname) print('platform = ', distutils.util.get_platform()) print('{0}Loading setup files{1}'.format(ANSI_YELLOW, ANSI_RESET)) source_set('defaults') if 'SET' in os.environ: source_set(os.environ['SET']) # we're working with tags (detached heads) a lot: suppress advice call_git(['config', '--global', 'advice.detachedHead', 'false']) print('{0}Checking/cloning dependencies{1}'.format(ANSI_YELLOW, ANSI_RESET)) add_modules = '' if 'ADD_MODULES' in os.environ: add_modules = os.environ['ADD_MODULES'] modules = '' if 'MODULES' in os.environ: modules = os.environ['MODULES'] modlist = 'BASE {0} {1}'.format(add_modules, modules).upper().split() for mod in modlist: if not setup[mod].strip(): setup[mod] = 'master' logger.debug('Adding dependency %s with tag %s', mod, setup[mod]) add_dependency(mod, setup[mod]) if os.path.isdir('configure'): release_local = os.path.join(cachedir, 'RELEASE.local') shutil.copy(release_local, 'configure') print('{0}Setting up EPICS build system{1}'.format(ANSI_YELLOW, ANSI_RESET)) with open(os.path.join(place['EPICS_BASE'], 'configure', 'CONFIG_SITE'), 'a') as config_site: if re.search('static', os.environ['CONFIGURATION']): config_site.write('SHARED_LIBRARIES=NO') config_site.write('STATIC_BUILD=YES') linktype = 'static' else: linktype = 'dynamic (DLL)' if re.search('debug', os.environ['CONFIGURATION']): config_site.write('HOST_OPT=NO') optitype = 'debug' else: optitype = 'optimized' print('EPICS Base set up for {0} build with {1} linking'.format(optitype, linktype)) if not os.path.isdir(toolsdir): os.makedirs(toolsdir) print('Installing Make 4.2.1 from ANL web site') sys.stdout.flush() sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'make-4.2.1.zip', 'https://epics.anl.gov/download/tools/make-4.2.1-win64.zip'], cwd=toolsdir) sp.check_call([zip7, 'e', 'make-4.2.1.zip'], cwd=toolsdir) perlver = '5.30.0.1' if os.environ['CC'] == 'vs2019': print('Installing Strawberry Perl {0}'.format(perlver)) sys.stdout.flush() sp.check_call(['curl', '-fsS', '--retry', '3', '-o', 'perl-{0}.zip'.format(perlver), 'http://strawberryperl.com/download/{0}/strawberry-perl-{0}-64bit.zip'.format(perlver)], cwd=toolsdir) sp.check_call([zip7, 'x', 'perl-{0}.zip'.format(perlver), '-ostrawberry'], cwd=toolsdir) sp.check_call('relocation.pl.bat', shell=True, cwd=os.path.join(toolsdir, 'strawberry')) def build(): print('{0}Building the module{1}'.format(ANSI_YELLOW, ANSI_RESET)) def test(): print('Running the tests') actions = { 'prepare': prepare, 'build': build, 'test': test, } if __name__=='__main__': args = sys.argv[1:] while len(args)>0: name = args.pop(0) print('IN', name, 'with', args) actions[name]()