633 lines
19 KiB
Python
Executable File
633 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import sys
|
|
import os
|
|
import time
|
|
from os import chdir, environ, getcwd
|
|
from ast import literal_eval
|
|
from configparser import ConfigParser
|
|
from os.path import expanduser, exists, join, basename, realpath, isdir
|
|
from subprocess import Popen, PIPE, check_output
|
|
from glob import glob
|
|
from socket import gethostname
|
|
|
|
instruments = ['amor', 'camea', 'dmc', 'eiger', 'focus', 'hrpt', 'morpheus', 'sans', 'tasp', 'zebra'] # boa
|
|
|
|
stuffsrc = '/afs/psi.ch/project/sinq/common/lib/servicemanager/'
|
|
|
|
doit = True # False: check only
|
|
sim = False # True: show what to do
|
|
action = ''
|
|
todo = set()
|
|
|
|
home = expanduser('~')
|
|
hostname = gethostname().split('.')[0]
|
|
remote = hostname not in instruments
|
|
|
|
if home.endswith('zolliker'):
|
|
chdir(expanduser('~/servicemanager/'))
|
|
cfg_fil = '/afs/psi.ch/project/sinq/common/stow/markus/lib/servicemanager/', glob('cfg/*.cfg')
|
|
bin_dst = '/afs/psi.ch/project/sinq/common/stow/markus/bin/', ['getsestuff']
|
|
diff = False
|
|
for dstdir, files in cfg_fil, bin_dst:
|
|
for src in files:
|
|
diff = os.system(f'diff {src} {dstdir}')
|
|
if diff and os.system(f'cp {src} {dstdir}'):
|
|
print('can not copy', src, 'to', dstdir)
|
|
if diff:
|
|
print('updated getsestuff, please do again')
|
|
sys.exit(0)
|
|
|
|
|
|
if remote:
|
|
instrument = ''
|
|
nicosroot = ''
|
|
else:
|
|
nicosroot = '/home/nicos/nicos'
|
|
if not exists(f'{nicosroot}/.git/config'):
|
|
print('nicos repo not found at', nicosroot)
|
|
nicosroot = ''
|
|
nicosenv = '/home/software/virtualenv/nicosenv'
|
|
|
|
instrument = environ.get('Instrument')
|
|
if instrument is None:
|
|
instrhome = '/home/%s' % hostname
|
|
if exists(instrhome):
|
|
instrument = hostname
|
|
else:
|
|
instrhome = '/home/%s' % instrument
|
|
if not exists(instrhome):
|
|
instrhome = home
|
|
if not exists(instrhome) or (instrhome != home and not nicosroot.startswith(home)):
|
|
print(instrhome, exists(instrhome), home, nicosroot)
|
|
print('can not guess instrument, please define environment variable "Instrument"')
|
|
instrument = ''
|
|
|
|
|
|
def doget(cmd):
|
|
out = Popen(cmd.split(), stdout=PIPE).communicate()[0]
|
|
return list(out.decode().split('\n'))
|
|
|
|
|
|
def docmd(cmd):
|
|
lines = doget(cmd)
|
|
print('\n'.join(lines), end='')
|
|
|
|
|
|
def from_str(string):
|
|
try:
|
|
return literal_eval(string)
|
|
except Exception as e:
|
|
return string.strip()
|
|
|
|
|
|
def scramble(arg):
|
|
return bytes([(158 - b) for b in arg.encode('ascii')]).decode('ascii')
|
|
|
|
|
|
def do(cmd):
|
|
print('>', cmd)
|
|
if not sim:
|
|
return not os.system(cmd)
|
|
return True
|
|
|
|
|
|
def docopy(src, dst):
|
|
if os.system(f'diff {dst} {src}'):
|
|
if doit:
|
|
if not do(f'cp {src} {dst}'):
|
|
do(f'mv {dst} {dst}0')
|
|
do(f'cp {src} {dst}')
|
|
else:
|
|
todo.add(action)
|
|
|
|
|
|
def dolink(dst, src):
|
|
src = realpath(expanduser(src))
|
|
dst = realpath(expanduser(dst))
|
|
if exists(dst):
|
|
if src != dst:
|
|
if doit:
|
|
do(f'ln -sf {dst} {src}')
|
|
else:
|
|
todo.add(action)
|
|
print('%s !-> %s' % (src, dst))
|
|
else:
|
|
print(f'target {dst} does not exist')
|
|
|
|
|
|
def ch_repo_dir(gitdir):
|
|
"""change working directory to repo dir"""
|
|
# git complains when using it at a directory where we have no write access
|
|
# this can be avoided with adding the command below
|
|
chdir(gitdir)
|
|
if not os.access(gitdir, os.W_OK):
|
|
entry = ['directory', gitdir]
|
|
with open(join(home, ".gitconfig")) as f:
|
|
for line in f:
|
|
if entry == [v.strip() for v in line.split('=')]:
|
|
return
|
|
docmd(f'git config --global --add safe.directory {gitdir}')
|
|
|
|
|
|
def check_repo(root, repo, url=None, branch=None):
|
|
chdir(root)
|
|
created = exists(join(repo, '.git'))
|
|
if url is None:
|
|
url = f'https://gitea.psi.ch/linse/{repo}.git'
|
|
elif url == 'gitlab':
|
|
url = f'git@gitlab.psi.ch-samenv:samenv/{repo}.git'
|
|
|
|
pull = False
|
|
if (doit or sim) and not created:
|
|
todo.add(action)
|
|
short = repo.rpartition('/')[2]
|
|
if exists(repo):
|
|
chdir(repo)
|
|
gitconfig = {}
|
|
for line in doget('git config --list'):
|
|
key, _, value = line.partition('=')
|
|
if value:
|
|
gitconfig[key] = value
|
|
if gitconfig.get('remote.origin.url') != 'url':
|
|
print(gitconfig.get('remote.origin.url'), 'does not match', url)
|
|
print(f'{join(root, repo)} exists already')
|
|
return False
|
|
# if repo == 'sea':
|
|
# do('mkdir seagit')
|
|
# if sim:
|
|
# print('> cd seagit')
|
|
# else:
|
|
# chdir('seagit')
|
|
# do('git clone %s' % url)
|
|
# do(f'mv sea/.git ../sea/.git')
|
|
# if sim:
|
|
# print('> cd ..')
|
|
# else:
|
|
# chdir('..')
|
|
# do(f'rm -rf seagit')
|
|
else:
|
|
do('git clone %s' % url)
|
|
pull = True
|
|
created = doit
|
|
if created:
|
|
chdir(join(root, repo))
|
|
docmd('git remote update')
|
|
gitconfig = {}
|
|
for line in doget('git config --list'):
|
|
key, _, value = line.partition('=')
|
|
if value:
|
|
gitconfig[key] = value
|
|
|
|
prev = dict(gitconfig)
|
|
gitconfig['push.default'] = 'upstream'
|
|
gitconfig['pull.rebase'] = 'True'
|
|
gitconfig['remote.origin.url'] = url
|
|
if 'gitea' in url:
|
|
hooksrc = '/afs/psi.ch/project/sinq/common/stow/markus/lib/gitea'
|
|
docopy(f'{hooksrc}/get_gitea_token', '.git/hooks/')
|
|
docopy(f'{hooksrc}/pre-commit', '.git/hooks/')
|
|
gitconfig['credential.helper'] = f'{getcwd()}/.git/hooks/get_gitea_token'
|
|
diff = {k: v for k, v in gitconfig.items() if v != prev.get(k) and v != prev.get(k.replace('branch.', ''))}
|
|
if doit:
|
|
for kv in diff.items():
|
|
do('git config %s %s' % kv)
|
|
elif diff:
|
|
todo.add(action)
|
|
print('modified git config keys:')
|
|
for key, val in diff.items():
|
|
print('%s=%s -> %s' % (key, prev.get(key), val))
|
|
if branch:
|
|
for line in doget('git rev-parse --abbrev-ref HEAD'):
|
|
if branch != line and line:
|
|
print(f'wrong branch: {line}')
|
|
return False
|
|
pull = False
|
|
show_status = True
|
|
for line in doget('git status'):
|
|
if '"git pull"' in line:
|
|
pull = doit
|
|
todo.add(action)
|
|
break
|
|
elif 'working tree clean' in line:
|
|
show_status = False
|
|
if show_status:
|
|
docmd('git status')
|
|
else:
|
|
todo.add(action)
|
|
print('not yet created', action)
|
|
return pull
|
|
|
|
|
|
SSH_CONFIG = """Host gitlab.psi.ch-samenv
|
|
HostName gitlab.psi.ch
|
|
User zolliker
|
|
IdentityFile ~/.ssh/id_rsa_samenv
|
|
"""
|
|
|
|
|
|
def do_ssh():
|
|
"""ssh config for pushing git to samenv repo"""
|
|
chdir(home)
|
|
try:
|
|
skip = False
|
|
with open('.ssh/config') as f:
|
|
content = f.read()
|
|
except FileNotFoundError:
|
|
content = ''
|
|
lines = []
|
|
for line in content.split('\n'):
|
|
if line.startswith('Host gitlab.psi.ch-samenv'):
|
|
skip = True
|
|
elif line[:1] not in ' \t':
|
|
skip = False
|
|
if not skip:
|
|
lines.append(line)
|
|
|
|
lines.append(SSH_CONFIG)
|
|
result = '\n'.join(lines)
|
|
|
|
if result == content:
|
|
pass
|
|
elif sim:
|
|
todo.add(action)
|
|
print('--- .ssh/config current:')
|
|
print(content)
|
|
print('--- .ssh/config new:')
|
|
print(result)
|
|
print('---')
|
|
elif not doit:
|
|
todo.add(action)
|
|
print('.ssh/config needs update')
|
|
else:
|
|
with open('.ssh/config', 'w') as f:
|
|
f.write(result)
|
|
if not exists('.ssh/id_rsa_samenv'):
|
|
if doit:
|
|
do('scp l_samenv@samenv:.ssh/id_rsa .ssh/id_rsa_samenv')
|
|
do('chmod g-r-x .ssh/id_rsa_samenv')
|
|
else:
|
|
todo.add(action)
|
|
print('missing id_rsa_samenv')
|
|
return
|
|
if not doget('ls -l .ssh/id_rsa_samenv')[0].startswith('-rw------'):
|
|
todo.add(action)
|
|
do('chmod -x .ssh/id_rsa_samenv')
|
|
do('chmod g-r-w .ssh/id_rsa_samenv')
|
|
do('chmod o-r-w .ssh/id_rsa_samenv')
|
|
|
|
|
|
def do_sshnicos():
|
|
do_ssh()
|
|
|
|
|
|
def do_frappy():
|
|
"""Frappy framework"""
|
|
frappydir = join(home, 'frappy')
|
|
if exists(frappydir) and not exists(join(frappydir, '.git')):
|
|
do('rm -rf %s' % frappydir)
|
|
if check_repo(home, 'frappy', 'gitlab'):
|
|
do('git pull')
|
|
if exists(join(frappydir, '.git')):
|
|
sys.path.extend(glob(f'{nicosenv}/lib/*/site-packages'))
|
|
try:
|
|
missing = 'psutil'
|
|
import psutil
|
|
missing = 'mlzlog'
|
|
import mlzlog
|
|
except ImportError:
|
|
print('MISSING', missing, 'in nicosenv (do it manually!)')
|
|
# if doit:
|
|
# do('pip3 install --user psutil')
|
|
# do('pip3 install --user mlzlog')
|
|
# else:
|
|
# todo.add(action)
|
|
# print('missing', missing, 'in nicosenv')
|
|
|
|
|
|
def do_servicemanager():
|
|
"""servicemanager package"""
|
|
if check_repo(home, 'servicemanager', None, 'master'):
|
|
do('git pull')
|
|
|
|
|
|
def do_sehistory():
|
|
"""history feeder"""
|
|
if check_repo(home, 'sehistory', None, 'master'):
|
|
do('git pull')
|
|
|
|
|
|
def do_sea():
|
|
"""SEA server scripts"""
|
|
if check_repo(home, 'sea', 'gitlab'):
|
|
do('git pull')
|
|
# do('git checkout master -- .gitignore') # do not know why this is needed
|
|
# do('chmod -x tcl/config/json_racklist tcl/*.* tcl/*.tcl tcl/*/*.tcl')
|
|
if not os.access('tcl/luft.tclsh', os.X_OK):
|
|
do('chmod +x tcl/luft.tclsh')
|
|
if not exists('tcl/plugin'):
|
|
do('git checkout master -- tcl/plugin')
|
|
if not exists('tcl/calcurves/.git') and not exists('tcl/calcurves/coil.inp'):
|
|
do('rm -rf tcl/calcurves')
|
|
do_calcurves()
|
|
docopy('/afs/psi.ch/user/z/zolliker/public/git.rhel7/sics/SeaServer', join(home, 'sea', 'SeaServer'))
|
|
|
|
|
|
def do_calcurves():
|
|
"""calibration curves"""
|
|
if check_repo(home, 'calcurves', 'gitlab'):
|
|
do('git pull')
|
|
dolink('~/calcurves', '~/sea/tcl/calcurves')
|
|
|
|
|
|
def do_frappysinq():
|
|
"""nicos_sinq/frappy_sinq setups and extensions"""
|
|
if exists(nicosroot):
|
|
if check_repo(join(nicosroot, 'nicos_sinq'), 'frappy_sinq', 'gitlab'):
|
|
do('git pull')
|
|
|
|
|
|
def do_nicosconf():
|
|
"""nicos.conf"""
|
|
# do the change in two places, as there is not always a link
|
|
for nicosconf in [join(nicosroot, 'nicos.conf'),
|
|
join(nicosroot, 'nicos_sinq', instrument, 'nicos.conf')]:
|
|
if not exists(nicosconf):
|
|
continue
|
|
cp = ConfigParser()
|
|
cp.optionxform = str
|
|
cp.read(nicosconf)
|
|
assert from_str(cp['nicos']['instrument']) == instrument
|
|
setup_subdirs = from_str(cp['nicos'].get('setup_subdirs', '["%s"]' % instrument))
|
|
new_subdirs = setup_subdirs[:]
|
|
if "frappy_sinq" not in new_subdirs:
|
|
print("missing frappy_sinq in setup_subdirs")
|
|
new_subdirs.append("frappy_sinq")
|
|
todo.add(action)
|
|
if "frappy" in new_subdirs:
|
|
print("superflous frappy in setup_subdirs")
|
|
new_subdirs.remove("frappy")
|
|
todo.add(action)
|
|
pythonpath = from_str(cp['environment'].get('PYTHONPATH', '""'))
|
|
pythonpath = pythonpath.split(':') if pythonpath else []
|
|
newpath = pythonpath[:]
|
|
frappyhome = join(instrhome, 'frappy')
|
|
localpy = join(instrhome, '.local/lib/python3.6/site-packages')
|
|
if frappyhome not in newpath:
|
|
print("missing %s in PYTHONPATH" % frappyhome)
|
|
newpath.append(frappyhome)
|
|
todo.add(action)
|
|
if instrhome not in newpath:
|
|
print("missing %s in PYTHONPATH" % instrhome)
|
|
newpath.append(instrhome)
|
|
todo.add(action)
|
|
if localpy in newpath:
|
|
print("remove %s from PYTHONPATH" % localpy)
|
|
newpath.remove(localpy)
|
|
todo.add(action)
|
|
if doit:
|
|
dirty = False
|
|
if new_subdirs != setup_subdirs:
|
|
cp['nicos']['setup_subdirs'] = '["%s"]' % '", "'.join(new_subdirs)
|
|
dirty = True
|
|
if newpath != pythonpath:
|
|
cp['environment']['PYTHONPATH'] = '"%s"' % ':'.join(newpath)
|
|
dirty = True
|
|
if dirty and not sim:
|
|
print('modify', nicosconf)
|
|
with open(nicosconf, 'w') as f:
|
|
cp.write(f)
|
|
|
|
|
|
def do_nicosenv():
|
|
"""install python packages needed for frappy/servicemanager"""
|
|
chdir(nicosroot)
|
|
for pkg in ['mlzlog', 'scipy']:
|
|
if not glob(f'{nicosenv}/lib/python3*/site-packages/{pkg}'):
|
|
if doit:
|
|
do(f'{nicosenv}/bin/python3 -m pip install {pkg}')
|
|
else:
|
|
print(f'missing {pkg} in nicosenv')
|
|
|
|
|
|
def do_nicos_pick():
|
|
"""OBSOLETE: cherry-pick all new commits in sinq-3.11 related to nicos/devices/secop"""
|
|
chdir(nicosroot)
|
|
|
|
docmd('git fetch sinq')
|
|
|
|
def get_change_ids(branch):
|
|
command = f'git log --since=2023-09-01 {branch} nicos/devices/secop'
|
|
ps = Popen(command.split(), stdout=PIPE)
|
|
result = []
|
|
try:
|
|
for line in check_output(('grep', 'Change-Id:'), stdin=ps.stdout).decode('latin-1').split('\n'):
|
|
line = line.strip().split()
|
|
if len(line) == 2:
|
|
result.append(line[1])
|
|
except Exception as e:
|
|
print(repr(e))
|
|
ps.wait()
|
|
return result
|
|
|
|
src = 'sinq/sinq-3.11'
|
|
dst = ' '
|
|
srcids = get_change_ids(src)
|
|
dstids = get_change_ids(dst)
|
|
dstset = set(dstids)
|
|
for changeid in reversed(srcids):
|
|
if changeid not in dstset:
|
|
line = check_output(f'git log --oneline {src} --grep={changeid}'.split()).decode('latin-1').strip()
|
|
if line:
|
|
if doit:
|
|
commit = line.split()[0]
|
|
docmd(f'git cherry-pick {commit}')
|
|
else:
|
|
todo.add(action)
|
|
print(line)
|
|
|
|
|
|
BIN = """#!%s
|
|
import sys
|
|
sys.path.append('%s')
|
|
from servicemanager import run
|
|
run('%s', sys.argv[1:])
|
|
"""
|
|
|
|
|
|
def do_bin():
|
|
"""added commands"""
|
|
chdir(home)
|
|
if not exists('bin'):
|
|
do('mkdir -p bin')
|
|
pgms = ['frappy', 'sea']
|
|
if nicosroot:
|
|
executable = f'{nicosenv}/bin/python3'
|
|
else:
|
|
executable = '/usr/bin/env python3.11'
|
|
pgms.append('nicos')
|
|
for pgm in pgms:
|
|
content = BIN % (executable, home, pgm)
|
|
binfile = join(home, 'bin', pgm)
|
|
try:
|
|
with open(binfile) as f:
|
|
writeit = f.read() != content
|
|
except FileNotFoundError:
|
|
writeit = 'create'
|
|
if writeit:
|
|
print('modify', binfile)
|
|
if doit and not sim:
|
|
if writeit != 'create':
|
|
os.remove(binfile)
|
|
with open(binfile, 'w') as f:
|
|
f.write(content)
|
|
do(f'chmod +x {binfile}')
|
|
if nicosroot:
|
|
dolink('~/servicemanager/bin/nicos_sinq', '~/bin/nicos')
|
|
|
|
|
|
def do_scfg():
|
|
"""cfg file for sea and frappy service"""
|
|
chdir(home)
|
|
insfile = f'{stuffsrc}/{instrument}_servicemanager.cfg'
|
|
srcfile = f'{stuffsrc}/servicemanager.cfg'
|
|
if exists(insfile):
|
|
if doit:
|
|
os.system(f'cat {srcfile} {insfile} > servicemanager.cfg')
|
|
else:
|
|
todo.add(action)
|
|
os.system(f'cat {srcfile} {insfile} | diff servicemanager.cfg -')
|
|
return
|
|
docopy(srcfile, 'servicemanager.cfg')
|
|
|
|
|
|
def remove_line(file, content):
|
|
if os.system('grep %s %s' % (content, file)) == 0:
|
|
if doit:
|
|
do('grep -v %s %s > %s_new' % (content, file, file))
|
|
do('mv %s_new %s' % (file, file))
|
|
else:
|
|
todo.add(action)
|
|
print('remove above')
|
|
|
|
#def do_monit():
|
|
# """remove monit support for sea / graph"""
|
|
# remove_line('%s/monitconfig' % home, 'sea')
|
|
# remove_line('%s/monitconfig' % home, 'graph')
|
|
|
|
|
|
selected_instruments = None
|
|
action_arg = ''
|
|
help = True
|
|
for arg in sys.argv[1:]:
|
|
if arg in instruments:
|
|
if remote:
|
|
selected_instruments = [arg]
|
|
elif arg != instrument:
|
|
raise ValueError(f'bad instrument: {arg}')
|
|
elif arg == 'allin':
|
|
if not remote:
|
|
raise ValueError(f'"allin" is only allowed on samenv')
|
|
selected_instruments = instruments
|
|
elif arg == 'check':
|
|
doit = False
|
|
elif arg == 'nohelp':
|
|
help = False
|
|
elif action_arg:
|
|
raise ValueError('only one action is allowed')
|
|
else:
|
|
action_arg = arg
|
|
|
|
|
|
nc_actions = ['sshnicos', 'frappysinq', 'nicosconf', 'nicosenv'] # 'nicos_pick'
|
|
ncactionfuncs = {}
|
|
|
|
with_su = False
|
|
if nicosroot.startswith(home):
|
|
actions = nc_actions
|
|
nc_actions = []
|
|
else:
|
|
if nicosroot or remote:
|
|
with_su = action_arg in nc_actions
|
|
for a in nc_actions:
|
|
ncactionfuncs[a] = locals()['do_%s' % a]
|
|
actions = ['ssh', 'bin', 'scfg', 'frappy', 'servicemanager', 'sea',
|
|
'calcurves', 'sehistory']
|
|
|
|
actionfuncs = {}
|
|
for a in actions:
|
|
actionfuncs[a] = locals()['do_%s' % a]
|
|
# if nicosroot:
|
|
# for a in nc_actions:
|
|
# actionfuncs['su %s' % a] = locals()['do_%s' % a]
|
|
|
|
|
|
def print_help():
|
|
if remote:
|
|
inst = "<instrument> "
|
|
else:
|
|
inst =''
|
|
|
|
def list_actions(funcs, postfix=''):
|
|
return '\n'.join(f" getsestuff {inst}%-15s # %s%s" % (a, f.__doc__, postfix) for a, f in funcs.items()) + '\n'
|
|
|
|
print(f"""-----
|
|
|
|
getsestuff {inst} # status
|
|
getsestuff {inst}all # install all
|
|
getsestuff {inst}sim # show commands to do
|
|
{list_actions(actionfuncs)}{list_actions(ncactionfuncs, ' *')}""", end='')
|
|
print(' ' * (32 + len(inst)), '* executed as user nicos')
|
|
if remote:
|
|
print(" replace <instrument> by allin for applying to all instruments")
|
|
|
|
|
|
if remote:
|
|
if not doit:
|
|
action_arg = 'check ' + action_arg
|
|
if selected_instruments is None:
|
|
print_help()
|
|
else:
|
|
for inst in selected_instruments:
|
|
print(f'===== {inst} ' + '=' * (70-len(inst)))
|
|
os.environ['SSHPASS'] = inst.upper()+'LNS'
|
|
docmd(f'sshpass -e ssh {inst}@{inst} getsestuff {action_arg} nohelp')
|
|
sys.exit(0)
|
|
|
|
def su_action(action):
|
|
os.environ['SSHPASS'] = scramble('P[d20+2:1eh')
|
|
docmd(f'sshpass -e ssh nicos@localhost {sys.argv[0]} {action} nohelp')
|
|
|
|
if with_su:
|
|
su_action(action_arg)
|
|
sys.exit(0)
|
|
if action_arg == 'sim':
|
|
sim = True
|
|
doit = False
|
|
elif action_arg in actions:
|
|
action = action_arg
|
|
actionfuncs[action]()
|
|
sys.exit()
|
|
elif action_arg != 'all':
|
|
if action_arg != '':
|
|
print(f'unknown action {action}, not in', actions)
|
|
print_help()
|
|
sys.exit(1)
|
|
doit = False
|
|
|
|
print(f'===== user {os.environ.get("USER", home)}')
|
|
|
|
for action, func in actionfuncs.items():
|
|
print(f'----- {action} ({home})')
|
|
func()
|
|
|
|
if action_arg in ('', 'all', 'sim') and nc_actions:
|
|
su_action(action_arg)
|
|
print_help()
|
|
sys.exit(0)
|
|
|
|
if help:
|
|
print_help()
|
|
if sim or not action_arg and todo:
|
|
print('needs updates: %s' % ' '.join(todo))
|
|
|