403 lines
17 KiB
Python
403 lines
17 KiB
Python
# This is the main program to generate the 'Heilige Liste'
|
|
import os
|
|
import glob
|
|
import sys
|
|
import shutil
|
|
import json
|
|
import datetime
|
|
import argparse
|
|
import filecmp
|
|
import re
|
|
import subprocess
|
|
|
|
sys.path.insert(0,os.path.join(os.path.dirname(__file__), 'Python_Code'))
|
|
|
|
import ProtoListe
|
|
import WriteExcel
|
|
import AssemblyList
|
|
from CompareList import compareLists
|
|
|
|
|
|
class HeiligeListe:
|
|
def __init__(self):
|
|
self.lattice = 'None'
|
|
self.workdir = ''
|
|
|
|
self.export = WriteExcel.Export()
|
|
self.assembly = AssemblyList.AssemblyList()
|
|
|
|
|
|
self.PLs = [ProtoListe.ProtoListe(i) for i in range(3)]
|
|
|
|
def HLUpdateLattice(self):
|
|
"""
|
|
Routine to copy the relevant online model files into the reference directories.
|
|
This synchronize the local lattice with the official one. The function is rather administrative
|
|
:return: True or False, depending if the files could be found
|
|
"""
|
|
dirsource = os.path.abspath(
|
|
os.path.join(os.path.abspath(os.sep), 'sf', 'bd', 'applications', 'OnlineModel', 'current'))
|
|
dirtarget = os.path.join(os.path.dirname(__file__), 'Python_Code')
|
|
files = ['OMLayout.py', 'OMFacility.py', 'OMFacilityAccess.py', 'OMAppTemplate.py', 'OMType.py',
|
|
'OMEnergyManager.py']
|
|
for file in files:
|
|
pfile = os.path.join(dirsource, file)
|
|
try:
|
|
shutil.copy(pfile, dirtarget)
|
|
except IOError as e:
|
|
print("Unable to copy file. %s" % e)
|
|
return False
|
|
|
|
return True
|
|
|
|
def HLClone(self, dirname = "Work", phase='All', lattice=''):
|
|
"""
|
|
Routine to take all reference input files and to generate a working directory to copy all relevant files
|
|
to the working directory so that it can be build. Existing working directory are deleted
|
|
|
|
:param dirname: name of working directory, where relevant files are copied to.
|
|
:param phase: either 'Current', 'Planned', 'Future', or 'All' (default). Tag is case-sensitive
|
|
:return: True or False if directory can be generated
|
|
"""
|
|
|
|
if len(dirname) < 1:
|
|
return False
|
|
phases=['Current','Planned','Final']
|
|
if not ( phase == 'All' or phase in phases):
|
|
print('Unrecognized phase. Supported are Current, Planned and Final')
|
|
return False
|
|
|
|
# step 1 - generate working directory or clean it up if it exists
|
|
folder = os.path.join(os.path.dirname(__file__), dirname)
|
|
try:
|
|
os.mkdir(folder)
|
|
except FileExistsError as e0:
|
|
print('Working directory exists. Cleaning up directory')
|
|
for filename in os.listdir(folder):
|
|
file_path = os.path.join(folder, filename)
|
|
try:
|
|
if os.path.isfile(file_path) or os.path.islink(file_path):
|
|
os.unlink(file_path)
|
|
elif os.path.isdir(file_path):
|
|
shutil.rmtree(file_path)
|
|
except Exception as e:
|
|
print('Failed to delete %s. Reason: %s' % (file_path, e))
|
|
|
|
# step 2 - generate json file with version and phase
|
|
verfile = {'Version': self.PLs[0].SF.Version}
|
|
for phs in phases:
|
|
if phase == phs or phase == 'All':
|
|
verfile[phs]=True
|
|
else:
|
|
verfile[phs]=False
|
|
filepath = os.path.join(folder, 'Version.json')
|
|
with open(filepath, 'w') as out:
|
|
json.dump(verfile, out)
|
|
|
|
# step 3 - copy files to working directory
|
|
# substep A - find the best directory to clone
|
|
srel = os.path.join(os.path.dirname(__file__), 'Release')
|
|
Release={}
|
|
search = ['HL-Components','Phase Current','Phase Planned','Phase Final']
|
|
for subdir, dirs, files in os.walk(srel):
|
|
if 'Lattice' in subdir:
|
|
for sea in search:
|
|
if sea in dirs:
|
|
Release[sea] = subdir
|
|
if len(lattice) > 0:
|
|
slat = os.path.join(os.path.dirname(__file__), 'Release','Lattice_%s' % lattice)
|
|
for subdir, dirs, files in os.walk(slat):
|
|
for sea in search:
|
|
if sea in dirs:
|
|
Release[sea] = subdir
|
|
|
|
sdef = os.path.join(os.path.dirname(__file__), 'Sub_assemblies_Lists')
|
|
for sea in search:
|
|
if not sea in Release.keys():
|
|
Release[sea]=sdef
|
|
try:
|
|
print('Copying HL-Components from',Release['HL-Components'])
|
|
shutil.copytree(os.path.join(Release['HL-Components'],'HL-Components'), os.path.join(folder,'HL-Components'), ignore=shutil.ignore_patterns('Old*'))
|
|
except IOError as e:
|
|
print("Unable to copy folder. %s" % e)
|
|
return False
|
|
for phase in phases:
|
|
if verfile[phase]:
|
|
try:
|
|
print('Copying Sub-assembly Lists for Phase %s from %s' % (phase, Release['Phase %s' % phase]))
|
|
shutil.copytree(os.path.join(Release['Phase %s' % phase],'Phase %s' % phase), os.path.join(folder,'Phase %s' % phase),ignore=shutil.ignore_patterns('Old*'))
|
|
except IOError as e:
|
|
print("Unable to copy folder. %s" % e)
|
|
return False
|
|
|
|
# step 4 - remove all Old directories in working directory
|
|
return True
|
|
|
|
def HLRelease(self, dirname='Work', phase='All'):
|
|
"""
|
|
Copies excel files and sub_assembly files into the current directory, add them to git and push it to the git repository
|
|
:param dirname: name of working directory
|
|
:param phase: either 'Current', 'Planned', 'Future', or 'All'. Tag is case-sensitive
|
|
:return:
|
|
"""
|
|
if len(dirname) < 1:
|
|
return False
|
|
|
|
# initiate git link
|
|
from git.repo import Repo
|
|
git = Repo.init(os.path.dirname(__file__))
|
|
|
|
# step 1 - check if working folder exists. Otherwise try to creat it
|
|
folder = os.path.join(os.path.dirname(__file__), dirname)
|
|
if not os.path.exists(folder):
|
|
print('Source Directory does not exist')
|
|
return False
|
|
|
|
# step 2 - read version file
|
|
version = {'Version': None, 'Current': False, 'Planned': False, 'Final': False}
|
|
with open(os.path.join(folder, 'Version.json'), 'r') as fin:
|
|
version = json.load(fin)
|
|
if version['Version'] is None:
|
|
print('Cannot find version file in working directory')
|
|
return False
|
|
print('Preparing files for releasing lattice: %s' % version['Version'])
|
|
phs = ['Current', 'Planned', 'Final']
|
|
output = []
|
|
for ph in phs:
|
|
if version[ph] and (phase == 'All' or phase == ph):
|
|
output.append(ph)
|
|
print('Releasing holy list for phases:')
|
|
for out in output:
|
|
print(' ',out)
|
|
|
|
# step3 - copy files into git repository
|
|
|
|
now = datetime.datetime.now()
|
|
self.date_time = now.strftime("%Y_%m_%d_%H_%M_%S")
|
|
|
|
gitadd=[]
|
|
|
|
ftar = os.path.join(os.path.dirname(__file__), 'Release')
|
|
fver = os.path.join(ftar,'Lattice_%s' % version['Version']) # sub folder in Release
|
|
if not os.path.exists(fver):
|
|
os.makedirs(fver)
|
|
|
|
for out in output: # output has the phases
|
|
# first copy the holy list and the link to the current version
|
|
files = os.path.join(folder, 'HolyList-Lat%s-Phase-%s.xlsx' % (version['Version'], out))
|
|
filet = os.path.join(fver, 'HolyList-Lat%s-Phase-%s.xlsx' % (version['Version'], out))
|
|
filer = os.path.join(ftar, 'HolyList-Latest-Phase-%s.xlsx' % (out))
|
|
print('Copying Holy List file:', files, 'to Release folder')
|
|
if os.path.exists(files):
|
|
gitadd = gitadd+self.copyFileWithBackup(files, filet, fver)
|
|
else:
|
|
print('Holy List File', files, 'not found in working directory', folder)
|
|
print('Cannot release new version. Aborting...')
|
|
return False
|
|
shutil.copy(filet, filer)
|
|
gitadd.append(filer)
|
|
# update the current release)
|
|
# check for assembly lists
|
|
dass = os.path.join(folder,'Phase %s' % out)
|
|
dast = os.path.join(fver,'Phase %s' % out)
|
|
if not os.path.exists(dast):
|
|
os.makedirs(dast)
|
|
for subdir, dirs, files in os.walk(dass):
|
|
for locdir in dirs:
|
|
gitadd = gitadd + self.updateFilesFromDirectory(dass,dast,locdir)
|
|
|
|
# copying the component list
|
|
gitadd = gitadd+self.updateFilesFromDirectory(folder,fver,'HL-Components')
|
|
|
|
# step 4 - update README file
|
|
self.updateREADME(version['Version'], output)
|
|
gitadd.append('README.md')
|
|
|
|
# step 5 - final commit and push to repository
|
|
git.index.add(gitadd)
|
|
git.index.commit("Holy List Release on %s" % self.date_time)
|
|
subprocess.run(["git", "push","origin"])
|
|
# git.remote('origin').push()
|
|
return True
|
|
|
|
|
|
def updateFilesFromDirectory(self,fsrc,ftar,subdir):
|
|
dsrc=os.path.join(fsrc,subdir)
|
|
dtar=os.path.join(ftar,subdir)
|
|
if not os.path.exists(dtar):
|
|
os.makedirs(dtar)
|
|
filter=re.compile('.*xlsx$')
|
|
gitadd = []
|
|
for subdir, dirs, files in os.walk(dsrc):
|
|
for file in files:
|
|
files = os.path.join(dsrc,file)
|
|
filet = os.path.join(dtar,file)
|
|
if filter.match(file):
|
|
print('Copying: %s into release folder' % files)
|
|
shutil.copy(files, filet)
|
|
gitadd.append(filet)
|
|
return gitadd
|
|
|
|
def copyFileWithBackup(self,fsrc,ftar,dtar):
|
|
|
|
gitadd=[]
|
|
if os.path.exists(ftar):
|
|
print('Backing up old holy list file')
|
|
dold = os.path.join(dtar, 'Old')
|
|
if not os.path.exists(dold):
|
|
os.mkdir(dold)
|
|
dold = os.path.join(dold,self.date_time)
|
|
if not os.path.exists(dold):
|
|
os.mkdir(dold)
|
|
hasBackup = True
|
|
try:
|
|
gfile = shutil.move(ftar, dold)
|
|
except IOError as e:
|
|
print("Unable to copy file. %s" % e)
|
|
hasBackup = False
|
|
if hasBackup:
|
|
gitadd.append(gfile)
|
|
shutil.copy(fsrc, ftar)
|
|
gitadd.append(ftar)
|
|
return gitadd
|
|
|
|
|
|
def updateREADME(self,version,phases):
|
|
print('updating README.md file')
|
|
with open('README.md','r') as fin:
|
|
fout=open('README_TMP.md','w')
|
|
for line in fin.readlines():
|
|
if 'Release/HolyList-Latest-Phase-' in line:
|
|
phase0 = line.split(':')[0]
|
|
phase=phase0[2:]
|
|
if phase in phases:
|
|
|
|
now = datetime.datetime.now()
|
|
tag='_(Released: %s)_' % now.strftime("%d/%m/%Y - %H:%M:%S")
|
|
linenew='- %s: [%s](Release/HolyList-Latest-Phase-%s.xlsx) %s\n' % (phase,version,phase,tag)
|
|
print('Updating README.md for',linenew)
|
|
fout.write(linenew)
|
|
else:
|
|
fout.write(line)
|
|
else:
|
|
fout.write(line)
|
|
fout.close()
|
|
shutil.move('README_TMP.md','README.md')
|
|
|
|
|
|
def HLGenerate(self, dirname = 'Work', phase = 'All', filter = None):
|
|
"""
|
|
Generate an Excel File with the tabs for 3DExperience, Holy List and Proto List within the defined working
|
|
directory. If the directory is not found the command HLClone is performed first.
|
|
|
|
:param dirname: name of working directory
|
|
:param phase: either 'Current', 'Planned', 'Future', or 'All'. Tag is case-sensitive
|
|
:return: True or False depending if working directory existed and phase is valid
|
|
"""
|
|
if len(dirname) < 1:
|
|
return False
|
|
|
|
# step 1 - check if working folder exists. Otherwise try to creat it
|
|
folder = os.path.join(os.path.dirname(__file__), dirname)
|
|
if not os.path.exists(folder):
|
|
status = self.HLClone(dirname = dirname, phase = phase)
|
|
if not status:
|
|
return False
|
|
|
|
# step 2 - read in lattice version and phase
|
|
version = {'Version': None, 'Current': False,'Planned': False, 'Final': False}
|
|
with open(os.path.join(folder,'Version.json'), 'r') as fin:
|
|
version = json.load(fin)
|
|
if version['Version'] is None:
|
|
print('Cannot find version file in working directory')
|
|
return False
|
|
vers=['Current','Planned','Final']
|
|
for nphase, ver in enumerate(vers):
|
|
if (phase == 'All' or phase == ver) and version[ver]:
|
|
vercomp = self.PLs[nphase].SF.Version
|
|
if not vercomp == version['Version']:
|
|
print('Lattice mismatch of Cloned Lattice and current Online Model')
|
|
print('Please clone working directory with current model again')
|
|
return False
|
|
|
|
# step 3 - calculate layout for Proto list
|
|
print('\n------------------------------\nGenerating Proto List:\n')
|
|
self.PLs[nphase].generateLayout()
|
|
#excelfile = os.path.join(folder, 'HolyList-Lat%s-Phase-%s' % (version['Version'], ver))
|
|
#self.export.export(excelfile, self.PLs[nphase], None)
|
|
#return True
|
|
|
|
# step 4 - parse assembly and component lists
|
|
print('\n------------------------------\nParsing Assembly and Component Lists:\n')
|
|
assemblyfile = os.path.join(folder,'Phase %s' % ver)
|
|
print('Parsing Assembly Lists in Directory:',assemblyfile)
|
|
self.assembly.readFiles(assemblyfile,folder,nphase)
|
|
|
|
# step 5 - calculate Holy List
|
|
print('\n------------------------------\nGenerating Holy List:\n')
|
|
DL = self.assembly.generateLayout(self.PLs[nphase])
|
|
|
|
# step 6 - write out Excel File
|
|
|
|
if filter is None:
|
|
excelfile=os.path.join(folder,'HolyList-Lat%s-Phase-%s' % (version['Version'],ver))
|
|
else:
|
|
excelfile=os.path.join(folder,'Extract-HolyList-Lat%s-Phase-%s' % (version['Version'],ver))
|
|
|
|
print('\n------------------------------\nWriting Excel File: %s\n' % excelfile)
|
|
self.export.export(excelfile, self.PLs[nphase], DL, Filter = filter)
|
|
return True
|
|
|
|
|
|
|
|
def parse(args):
|
|
HL = HeiligeListe()
|
|
if len(args) < 2:
|
|
print('Command Arguments are missing (e.g. init, generate)')
|
|
cmd=args[1].upper()
|
|
if cmd == 'INIT':
|
|
parser = argparse.ArgumentParser(prog='HLInit', description='Initialize a working directory')
|
|
parser.add_argument('-d','--dir', default='Work', help='Name of working directory')
|
|
parser.add_argument('-l', '--lat', default='', help='Specific lattice to clone from')
|
|
parser.add_argument('-p','--phase', default='Current',
|
|
help = 'Phase: Current, Planned, Final, or All',
|
|
choices=['Current','Planned','Final','All'])
|
|
pargs = parser.parse_args(args[2:])
|
|
HL.HLClone(dirname=pargs.dir,phase=pargs.phase,lattice=pargs.lat)
|
|
elif cmd == 'UPDATE':
|
|
HL.HLUpdateLattice() # this needs to be done - currently only manual.
|
|
elif cmd == 'GENERATE':
|
|
parser = argparse.ArgumentParser(prog='HLGenerate', description='Generate the holy list in the working directory')
|
|
parser.add_argument('-d', '--dir', default='Work', help='Name of working directory')
|
|
parser.add_argument('-p', '--phase', default='Current',
|
|
help='Phase: Current, Planned, Final, or All',
|
|
choices=['Current', 'Planned', 'Final', 'All'])
|
|
pargs = parser.parse_args(args[2:])
|
|
HL.HLGenerate(dirname=pargs.dir, phase=pargs.phase)
|
|
elif cmd == 'EXTRACT':
|
|
parser = argparse.ArgumentParser(prog='HLExtract', description='Extract a subset of the holy list')
|
|
parser.add_argument('-d', '--dir', default='Work', help='Name of working directory')
|
|
parser.add_argument('-f', '--filter', default='.*', help='Regular Expression Filter')
|
|
parser.add_argument('-p', '--phase', default='Current',
|
|
help='Phase: Current, Planned, Final, or All',
|
|
choices=['Current', 'Planned', 'Final', 'All'])
|
|
pargs = parser.parse_args(args[2:])
|
|
HL.HLGenerate(dirname=pargs.dir, phase=pargs.phase,filter=pargs.filter)
|
|
elif cmd == 'COMPARE':
|
|
if len(args) < 4:
|
|
print('Needs argument for the two files to be compared')
|
|
return
|
|
compareLists(args[2:])
|
|
elif cmd == 'RELEASE':
|
|
parser = argparse.ArgumentParser(prog='HLRelease',
|
|
description='Release the holy list as a new version on gitlab')
|
|
parser.add_argument('-d', '--dir', default='Work', help='Name of working directory')
|
|
parser.add_argument('-p', '--phase', default='Current',
|
|
help='Phase: Current, Planned, Final, or All',
|
|
choices=['Current', 'Planned', 'Final', 'All'])
|
|
pargs = parser.parse_args(args[2:])
|
|
HL.HLRelease(dirname=pargs.dir, phase=pargs.phase)
|
|
|
|
# Main Routine
|
|
if __name__ == '__main__':
|
|
parse(sys.argv) |