From ac5d894acc7f85f5fdceba296e9bf414e96c7ecb Mon Sep 17 00:00:00 2001 From: Douglas Clowes Date: Tue, 20 Aug 2013 10:14:59 +1000 Subject: [PATCH] Compare what is in the git tree with what is deployed on the instrument --- site_ansto/instrument/compareSICS.py | 338 +++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100755 site_ansto/instrument/compareSICS.py diff --git a/site_ansto/instrument/compareSICS.py b/site_ansto/instrument/compareSICS.py new file mode 100755 index 00000000..ca3c6ae3 --- /dev/null +++ b/site_ansto/instrument/compareSICS.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python +# vim: ts=8 sts=4 sw=4 et ai si +# Author: Douglas Clowes 2013 (dcl@ansto.gov.au) + +# +# Program to compare what *is* deployed with *was* deployed. +# It uses the file FILEMAP.TXT which can be created after the deployment. +# The location of FILEMAP.TXT may be different to the deployment path. +# +# For example, to compare what is on echidna (without a FILEMAP.TXT file) with +# what existed at the time it was deployed: +# +# sshfs -o ro echidna@ics2-echidna.nbi.ansto.gov.au:/usr/local/sics mnt +# git checkout git checkout 5579f3e +# git checkout merge-replace -- deploySICS.sh +# ./deploySICS.sh -n echidna localhost +# python compare_deploy.py --manifest=FILEMAP.TXT mnt/newserver/ +# ... +# fusermount -u mnt +# +#Notes: +# "sshfs -o ro" mount the echidna tree read only +# the 5579f3e is from the DEPLOYMENT.TXT file on echidna (mnt/server/DEPLOYMENT.TXT) +# The first checkout is to get what *was* deployed to echidna in the working directory +# The second checkout is just to get a deploySICS.sh that produces FILEMAP.TXT in staging +# The "./deploySICS.sh -n" does NOT deploy anywhere but creates FILEMAP.TXT in staging +# The compare_deploy.py uses FILEMAP.TXT from the staging tree produced by the "./deploySICS.sh -n" +# The compare_deploy.py path is to what *is* deployed on echidna +# The untracked files may be manually deployed and missing from MANIFEST.TXT +# The untracked files may be new, modified, test or junk +# You can then "gvim -d " then cut/paste lines emitted under "Changed Files:" (or diff) +# +import os +import re +import sys +import argparse +import difflib +import shutil + +def load_manifest(theManifest): + ''' + This function loads the manifest file ("FILEMAP.TXT") which consists of a + series of shell variables followed by the output of various 'cp -v' + commands and produces two dictionaries, one for the assignments (shell variables) + and one for the filemap (file associations). The shell variables give + context to the file mapping. + ''' + # get the lines from the manifest file with the newline removed + contents = [j[:-1] for j in open(theManifest).readlines()] + + # divide the lines into assignments and filemaps + assignments = [] + filemaps = [] + for line in contents: + if "->" in line: + filemaps.append(line) + else: + assignments.append(line) + + # parse the assignments and create the assignment map which maps shell variables (left) + # from the deploy script to their value at the time of deployment + assignment_map = {} + for line in assignments: + left, right = line.split("=") + assignment_map[left] = right + + # parse the file associations and create the file map which maps the files in the + # destination tree (right) to the file in the source tree from which they came (left) + filemap_map = {} + for line in filemaps: + if line[0] in "`'": + val = re.sub(r"^[`'](.*)[`'] -> [`'](.*)[`'].*", r"\1|\2", line) + else: + val = re.sub(r"^(.*) -> (.*)", r"\1|\2", line) + if val == line: + # unexpected, probably an error + print "Val:", val + else: + left, right = val.split("|") + if (left + "|" + right) != val: + # unexpected, probably an error + print "LR:", left, right + else: + # use the assignment map to remove the staging (temporary) directory + # from the front of the filename and make it independent of that + # and reduce leading slashes to one only + if right.startswith(assignment_map["TEMPDIR"]): + right = right[len(assignment_map["TEMPDIR"]):] + while right.startswith("//"): + right = right[1:] + # use the assignment map to remove the destination directory + # from the front of the filename and make it independent of that + # and remove leading slashes + if right.startswith(assignment_map["DESTDIR"]): + right = right[len(assignment_map["DESTDIR"]):] + while right.startswith("/"): + right = right[1:] + prefix = "newserver/" + if right.startswith(prefix): + right = right[len(prefix):] + else: + # unexpected, probably an error + print "LR+:", left, right + # Note: map[destination] = source + filemap_map[right] = left + + return (assignment_map, filemap_map) + +def load_dir(theDir): + ''' + Walk the directory tree and populate a double map with the filename and directory in which it was found + ''' + myDirs = {} + theLen = len(theDir) + 1 + for root, dirs, files in os.walk(theDir): + for file in files: + # Don't even put these files into the map + if file.endswith((".swp", ".swo", ".hdf", "~", ".bck", ".pyc")): + continue + if file.startswith(("core.", "SICServer")): + continue + if not file in myDirs: + myDirs[file] = {} + myDirs[file][root[theLen:]] = {} + return myDirs + +if __name__ == "__main__": + default_dir = "/usr/local/TEST_SICS/taipan/nbi/sics/taipan" + verbose = False + parser = argparse.ArgumentParser(description = "Compare a deployed instrument tree with a repository") + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="emit tables for debugging") + parser.add_argument("--link", dest="link", action="store_true", help="link deployed files to /tmp tree") + parser.add_argument("--copy", dest="copy", action="store_true", help="copy deployed files to /tmp tree") + parser.add_argument("-m", "--manifest", dest="manifest", help="specify another manifest directory") + parser.add_argument("path", nargs="?", default = default_dir, help="directory containing FILEMAP.TXT") + args = parser.parse_args() + if args.verbose: + verbose = True + default_dir = os.path.abspath(args.path) + if default_dir.endswith("FILEMAP.TXT"): + default_dir = os.path.dirname(default_dir) + if args.manifest: + temp_dir = args.manifest + if temp_dir.endswith("FILEMAP.TXT"): + temp_dir = os.path.dirname(temp_dir) + default_manifest = os.path.join(temp_dir, "FILEMAP.TXT") + else: + default_manifest = os.path.join(default_dir, "FILEMAP.TXT") + root_dir = os.path.dirname(default_dir) + print "path:", args.path + print "dest:", default_dir + print "root:", root_dir + print "--manifest:", args.manifest + print "Manifest:", default_manifest + + amap, fmap = load_manifest(default_manifest) + + if verbose: + print "Assignments:" + for key in sorted(amap): + print " ", key, "=", amap[key] + + print "Contents:" + for key in sorted(fmap): + print " ", key, "=", fmap[key] + + gmap = load_dir(default_dir) + + if verbose: + print "Target:" + for key in sorted(gmap): + print " ", key + for entry in sorted(gmap[key]): + print " ", entry + + print "Untracked Files:" + for key in sorted(gmap): + if key.endswith((".swp", ".swo", ".hdf", "~", ".bck", ".pyc")): + continue + if key.startswith(("core.", "SICServer")): + continue + for entry in sorted(gmap[key]): + if entry.endswith(("/data", "/log")): + continue + if "/data/" in entry: + continue + if os.path.join(entry, key) not in fmap: + #print "Looking in fmap for", os.path.join(entry, key) + print os.path.join(default_dir, os.path.join(entry, key)) + + print "Changed Files:" + for key in sorted(gmap): + if key.endswith((".swp", ".swo", ".hdf", "~", ".bck", ".pyc")): + continue + if key.startswith(("core.", "SICServer")): + continue + for entry in sorted(gmap[key]): + target = os.path.join(entry, key) + #print "Looking in fmap for", os.path.join(entry, key) + if target in fmap: + source = os.path.join(amap["SRCDIR"], fmap[target]) + source = os.path.join(os.getenv("PWD"), fmap[target]) + destin = os.path.join(default_dir, target) + if not (os.path.exists(source) and os.path.exists(destin)): + print " Compare:", source, destin + if not os.path.exists(source): + print " ", source, "does not exist" + if not os.path.exists(destin): + print " ", destin, "does not exist" + continue + delta = list(difflib.unified_diff(open(source).readlines(), open(destin).readlines())) + delta_len = len(delta) + if delta_len > 0: + if verbose: + print delta + print source, destin, "#%d" % delta_len + + if args.link: + print "Linking Files:" + count = 0 + lines = [] + for key in sorted(gmap): + if key.endswith((".swp", ".swo", ".hdf", "~", ".bck", ".pyc")): + continue + if key.startswith(("core.", "SICServer")): + continue + for entry in sorted(gmap[key]): + target = os.path.join(entry, key) + #print "Looking in fmap for", os.path.join(entry, key) + if target in fmap: + source = os.path.join(amap["SRCDIR"], fmap[target]) + source = os.path.join(os.getenv("PWD"), fmap[target]) + destin = os.path.join(default_dir, target) + if not (os.path.exists(source) and os.path.exists(destin)): + print " Linking:", source, destin + if not os.path.exists(source): + print " ", source, "does not exist" + if not os.path.exists(destin): + print " ", destin, "does not exist" + continue + lines.append((destin, fmap[target])) + if len(lines) > 0: + tgt = "/tmp/link_%s" % str(os.getpid()) + if (os.path.exists(tgt)): + for root, disr, files in os.walk(tgt, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.mkdir(tgt) + for line in lines: + source = line[0] + destin = os.path.abspath(os.path.join(tgt, line[1])) + if not destin.startswith(tgt): + print "Cannot link: %s to %s" % (source, destin) + continue + try: + os.makedirs(os.path.dirname(destin)) + except: + pass + if verbose: + cmd = "ln -s %s %s" % (source, destin) + print cmd + if os.path.exists(destin): + delta = list(difflib.unified_diff(open(source).readlines(), open(destin).readlines())) + if len(delta) == 0: + continue + print "Duplicate Link:", destin + continue + try: + os.symlink(source, destin) + count += 1 + except: + pass + print "Links (%d) in: %s" % (count, tgt) + + + if args.copy: + print "Copying Files:" + count = 0 + lines = [] + for key in sorted(gmap): + if key.endswith((".swp", ".swo", ".hdf", "~", ".bck", ".pyc")): + continue + if key.startswith(("core.", "SICServer")): + continue + for entry in sorted(gmap[key]): + target = os.path.join(entry, key) + #print "Looking in fmap for", os.path.join(entry, key) + if target in fmap: + source = os.path.join(amap["SRCDIR"], fmap[target]) + source = os.path.join(os.getenv("PWD"), fmap[target]) + destin = os.path.join(default_dir, target) + if not (os.path.exists(source) and os.path.exists(destin)): + print " Copying:", source, destin + if not os.path.exists(source): + print " ", source, "does not exist" + if not os.path.exists(destin): + print " ", destin, "does not exist" + continue + lines.append((destin, fmap[target])) + if len(lines) > 0: + tgt = "/tmp/copy_%s" % str(os.getpid()) + if (os.path.exists(tgt)): + for root, disr, files in os.walk(tgt, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.mkdir(tgt) + for line in lines: + source = os.path.abspath(line[0]) + destin = os.path.abspath(os.path.join(tgt, line[1])) + if not destin.startswith(tgt): + print "Cannot copy: %s to %s" % (source, destin) + continue + try: + os.makedirs(os.path.dirname(destin)) + except: + pass + if verbose: + cmd = "cp -p %s %s" % (source, destin) + print cmd + if os.path.exists(destin): + delta = list(difflib.unified_diff(open(source).readlines(), open(destin).readlines())) + if len(delta) == 0: + continue + print "Duplicate Copy:", destin + continue + try: + shutil.copyfile(source, destin) + shutil.copystat(source, destin) + count += 1 + except: + pass + print "Copies (%d) in: %s" % (count, tgt) + +