From a3d2ef3f2773492b2c5f51c39a3ea2f9d8c2cf42 Mon Sep 17 00:00:00 2001 From: Markus Zolliker Date: Tue, 29 Apr 2025 09:27:58 +0200 Subject: [PATCH] keep gesestuff in servicemanager - still need to copy to /afs/psi.ch/project/sinq/common/stow/markus/bin/ --- getsestuff | 617 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 617 insertions(+) create mode 100755 getsestuff diff --git a/getsestuff b/getsestuff new file mode 100755 index 0000000..608dd5c --- /dev/null +++ b/getsestuff @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +import sys +import os +import time +from os import chdir, environ, getcwd +from ast import literal_eval +from configparser import ConfigParser +from os.path import expanduser, exists, join, basename, realpath, isdir +from subprocess import Popen, PIPE, check_output +from glob import glob +from socket import gethostname + +instruments = ['amor', 'camea', 'dmc', 'eiger', 'focus', 'hrpt', 'morpheus', 'sans', 'tasp', 'zebra'] # boa + +stuffsrc = realpath(join(sys.modules['__main__'].__file__, '..', '..', 'lib', 'servicemanager')) +# this should be equivalent to /afs/psi.ch/project/sinq/common/lib/servicemanager/ +doit = True # False: check only +sim = False # True: show what to do +action = '' +todo = set() + +home = expanduser('~') +hostname = gethostname().split('.')[0] +remote = hostname not in instruments + +if remote: + instrument = '' + nicosroot = '' +else: + nicosroot = '/home/nicos/nicos' + if not exists(f'{nicosroot}/.git/config'): + print('nicos repo not found at', nicosroot) + nicosroot = '' + nicosenv = '/home/software/virtualenv/nicosenv' + + instrument = environ.get('Instrument') + if instrument is None: + instrhome = '/home/%s' % hostname + if exists(instrhome): + instrument = hostname + else: + instrhome = '/home/%s' % instrument + if not exists(instrhome): + instrhome = home + if not exists(instrhome) or (instrhome != home and not nicosroot.startswith(home)): + print(instrhome, exists(instrhome), home, nicosroot) + print('can not guess instrument, please define environment variable "Instrument"') + instrument = '' + + +def doget(cmd): + out = Popen(cmd.split(), stdout=PIPE).communicate()[0] + return list(out.decode().split('\n')) + + +def docmd(cmd): + lines = doget(cmd) + print('\n'.join(lines), end='') + + +def from_str(string): + try: + return literal_eval(string) + except Exception as e: + return string.strip() + + +def scramble(arg): + return bytes([(158 - b) for b in arg.encode('ascii')]).decode('ascii') + + +def do(cmd): + print('>', cmd) + if not sim: + return not os.system(cmd) + return True + + +def docopy(src, dst): + if os.system(f'diff {dst} {src}'): + if doit: + if not do(f'cp {src} {dst}'): + do(f'mv {dst} {dst}0') + do(f'cp {src} {dst}') + else: + todo.add(action) + + +def dolink(dst, src): + src = realpath(expanduser(src)) + dst = realpath(expanduser(dst)) + if exists(dst): + if src != dst: + if doit: + do(f'ln -sf {dst} {src}') + else: + todo.add(action) + print('%s !-> %s' % (src, dst)) + else: + print(f'target {dst} does not exist') + + +def ch_repo_dir(gitdir): + """change working directory to repo dir""" + # git complains when using it at a directory where we have no write access + # this can be avoided with adding the command below + chdir(gitdir) + if not os.access(gitdir, os.W_OK): + entry = ['directory', gitdir] + with open(join(home, ".gitconfig")) as f: + for line in f: + if entry == [v.strip() for v in line.split('=')]: + return + docmd(f'git config --global --add safe.directory {gitdir}') + + +def check_repo(root, repo, url=None, branch=None): + chdir(root) + created = exists(join(repo, '.git')) + if url is None: + url = f'https://gitea.psi.ch/linse/{repo}.git' + elif url == 'gitlab': + url = f'git@gitlab.psi.ch-samenv:samenv/{repo}.git' + + pull = False + if (doit or sim) and not created: + todo.add(action) + short = reponame.rpartition('/')[2] + if exists(repo): + chdir(join(root, repo)) + gitconfig = {} + for line in doget('git config --list'): + key, _, value = line.partition('=') + if value: + gitconfig[key] = value + if gitconfig.get('remote.origin.url') != 'url': + print(gitconfig.get('remote.origin.url'), 'does not match', url) + print(f'{join(root, repo)} exists already') + return False + # if repo == 'sea': + # do('mkdir seagit') + # if sim: + # print('> cd seagit') + # else: + # chdir('seagit') + # do('git clone %s' % url) + # do(f'mv sea/.git ../sea/.git') + # if sim: + # print('> cd ..') + # else: + # chdir('..') + # do(f'rm -rf seagit') + else: + do('git clone %s' % url) + pull = True + created = doit + if created: + chdir(join(root, repo)) + docmd('git remote update') + gitconfig = {} + for line in doget('git config --list'): + key, _, value = line.partition('=') + if value: + gitconfig[key] = value + + prev = dict(gitconfig) + gitconfig['push.default'] = 'upstream' + gitconfig['pull.rebase'] = 'True' + gitconfig['remote.origin.url'] = url + if 'gitea' in url: + hooksrc = '/afs/psi.ch/project/sinq/common/stow/markus/lib/gitea' + docopy(f'{hooksrc}/get_gitea_token', '.git/hooks/') + docopy(f'{hooksrc}/pre-commit', '.git/hooks/') + gitconfig['credential.helper'] = f'{getcwd()}/.git/hooks/get_gitea_token' + diff = {k: v for k, v in gitconfig.items() if v != prev.get(k) and v != prev.get(k.replace('branch.', ''))} + if doit: + for kv in diff.items(): + do('git config %s %s' % kv) + elif diff: + todo.add(action) + print('modified git config keys:') + for key, val in diff.items(): + print('%s=%s -> %s' % (key, prev.get(key), val)) + if branch: + for line in doget('git rev-parse --abbrev-ref HEAD'): + if branch != line and line: + print(f'wrong branch: {line}') + return False + pull = False + show_status = True + for line in doget('git status'): + if '"git pull"' in line: + pull = doit + todo.add(action) + break + elif 'working tree clean' in line: + show_status = False + if show_status: + docmd('git status') + else: + todo.add(action) + print('not yet created', action) + return pull + + +SSH_CONFIG = """Host gitlab.psi.ch-samenv + HostName gitlab.psi.ch + User zolliker + IdentityFile ~/.ssh/id_rsa_samenv +""" + + +def do_ssh(): + """ssh config for pushing git to samenv repo""" + chdir(home) + try: + skip = False + with open('.ssh/config') as f: + content = f.read() + except FileNotFoundError: + content = '' + lines = [] + for line in content.split('\n'): + if line.startswith('Host gitlab.psi.ch-samenv'): + skip = True + elif line[:1] not in ' \t': + skip = False + if not skip: + lines.append(line) + + lines.append(SSH_CONFIG) + result = '\n'.join(lines) + + if result == content: + pass + elif sim: + todo.add(action) + print('--- .ssh/config current:') + print(content) + print('--- .ssh/config new:') + print(result) + print('---') + elif not doit: + todo.add(action) + print('.ssh/config needs update') + else: + with open('.ssh/config', 'w') as f: + f.write(result) + if not exists('.ssh/id_rsa_samenv'): + if doit: + do('scp l_samenv@samenv:.ssh/id_rsa .ssh/id_rsa_samenv') + do('chmod g-r-x .ssh/id_rsa_samenv') + else: + todo.add(action) + print('missing id_rsa_samenv') + return + if not doget('ls -l .ssh/id_rsa_samenv')[0].startswith('-rw------'): + todo.add(action) + do('chmod -x .ssh/id_rsa_samenv') + do('chmod g-r-w .ssh/id_rsa_samenv') + do('chmod o-r-w .ssh/id_rsa_samenv') + + +def do_sshnicos(): + do_ssh() + + +def do_frappy(): + """Frappy framework""" + frappydir = join(home, 'frappy') + if exists(frappydir) and not exists(join(frappydir, '.git')): + do('rm -rf %s' % frappydir) + if check_repo(home, 'frappy', 'gitlab'): + do('git pull') + if exists(join(frappydir, '.git')): + sys.path.extend(glob(f'{nicosenv}/lib/*/site-packages')) + try: + missing = 'psutil' + import psutil + missing = 'mlzlog' + import mlzlog + except ImportError: + print('MISSING', missing, 'in nicosenv (do it manually!)') + # if doit: + # do('pip3 install --user psutil') + # do('pip3 install --user mlzlog') + # else: + # todo.add(action) + # print('missing', missing, 'in nicosenv') + + +def do_servicemanager(): + """servicemanager package""" + if check_repo(home, 'servicemanager', None, 'master'): + do('git pull') + + +def do_sehistory(): + """history feeder""" + if check_repo(home, 'sehistory', None, 'master'): + do('git pull') + + +def do_sea(): + """SEA server scripts""" + if check_repo(home, 'sea', 'gitlab'): + do('git pull') + # do('git checkout master -- .gitignore') # do not know why this is needed + # do('chmod -x tcl/config/json_racklist tcl/*.* tcl/*.tcl tcl/*/*.tcl') + if not os.access('tcl/luft.tclsh', os.X_OK): + do('chmod +x tcl/luft.tclsh') + if not exists('tcl/plugin'): + do('git checkout master -- tcl/plugin') + if not exists('tcl/calcurves/.git') and not exists('tcl/calcurves/coil.inp'): + do('rm -rf tcl/calcurves') + do_calcurves() + docopy('/afs/psi.ch/user/z/zolliker/public/git.rhel7/sics/SeaServer', join(home, 'sea', 'SeaServer')) + + +def do_calcurves(): + """calibration curves""" + if check_repo(home, 'calcurves', 'gitlab'): + do('git pull') + dolink('~/calcurves', '~/sea/tcl/calcurves') + + +def do_frappysinq(): + """nicos_sinq/frappy_sinq setups and extensions""" + if exists(nicosroot): + if check_repo(join(nicosroot, 'nicos_sinq'), 'frappy_sinq', 'gitlab'): + do('git pull') + + +def do_nicosconf(): + """nicos.conf""" + # do the change in two places, as there is not always a link + for nicosconf in [join(nicosroot, 'nicos.conf'), + join(nicosroot, 'nicos_sinq', instrument, 'nicos.conf')]: + if not exists(nicosconf): + continue + cp = ConfigParser() + cp.optionxform = str + cp.read(nicosconf) + assert from_str(cp['nicos']['instrument']) == instrument + setup_subdirs = from_str(cp['nicos'].get('setup_subdirs', '["%s"]' % instrument)) + new_subdirs = setup_subdirs[:] + if "frappy_sinq" not in new_subdirs: + print("missing frappy_sinq in setup_subdirs") + new_subdirs.append("frappy_sinq") + todo.add(action) + if "frappy" in new_subdirs: + print("superflous frappy in setup_subdirs") + new_subdirs.remove("frappy") + todo.add(action) + pythonpath = from_str(cp['environment'].get('PYTHONPATH', '""')) + pythonpath = pythonpath.split(':') if pythonpath else [] + newpath = pythonpath[:] + frappyhome = join(instrhome, 'frappy') + localpy = join(instrhome, '.local/lib/python3.6/site-packages') + if frappyhome not in newpath: + print("missing %s in PYTHONPATH" % frappyhome) + newpath.append(frappyhome) + todo.add(action) + if instrhome not in newpath: + print("missing %s in PYTHONPATH" % instrhome) + newpath.append(instrhome) + todo.add(action) + if localpy in newpath: + print("remove %s from PYTHONPATH" % localpy) + newpath.remove(localpy) + todo.add(action) + if doit: + dirty = False + if new_subdirs != setup_subdirs: + cp['nicos']['setup_subdirs'] = '["%s"]' % '", "'.join(new_subdirs) + dirty = True + if newpath != pythonpath: + cp['environment']['PYTHONPATH'] = '"%s"' % ':'.join(newpath) + dirty = True + if dirty and not sim: + print('modify', nicosconf) + with open(nicosconf, 'w') as f: + cp.write(f) + + +def do_nicosenv(): + """install python packages needed for frappy/servicemanager""" + chdir(nicosroot) + for pkg in ['mlzlog', 'scipy']: + if not glob(f'{nicosenv}/lib/python3*/site-packages/{pkg}'): + if doit: + do(f'{nicosenv}/bin/python3 -m pip install {pkg}') + else: + print(f'missing {pkg} in nicosenv') + + +def do_nicos_pick(): + """OBSOLETE: cherry-pick all new commits in sinq-3.11 related to nicos/devices/secop""" + chdir(nicosroot) + + docmd('git fetch sinq') + + def get_change_ids(branch): + command = f'git log --since=2023-09-01 {branch} nicos/devices/secop' + ps = Popen(command.split(), stdout=PIPE) + result = [] + try: + for line in check_output(('grep', 'Change-Id:'), stdin=ps.stdout).decode('latin-1').split('\n'): + line = line.strip().split() + if len(line) == 2: + result.append(line[1]) + except Exception as e: + print(repr(e)) + ps.wait() + return result + + src = 'sinq/sinq-3.11' + dst = ' ' + srcids = get_change_ids(src) + dstids = get_change_ids(dst) + dstset = set(dstids) + for changeid in reversed(srcids): + if changeid not in dstset: + line = check_output(f'git log --oneline {src} --grep={changeid}'.split()).decode('latin-1').strip() + if line: + if doit: + commit = line.split()[0] + docmd(f'git cherry-pick {commit}') + else: + todo.add(action) + print(line) + + +BIN = """#!%s +import sys +sys.path.append('%s') +from servicemanager import run +run('%s', sys.argv[1:]) +""" + + +def do_bin(): + """added commands""" + chdir(home) + if not exists('bin'): + do('mkdir -p bin') + pgms = ['frappy', 'sea'] + if nicosroot: + executable = f'{nicosenv}/bin/python3' + else: + executable = '/usr/bin/env python3.11' + pgms.append('nicos') + for pgm in pgms: + content = BIN % (executable, home, pgm) + binfile = join(home, 'bin', pgm) + try: + with open(binfile) as f: + writeit = f.read() != content + except FileNotFoundError: + writeit = 'create' + if writeit: + print('modify', binfile) + if doit and not sim: + if writeit != 'create': + os.remove(binfile) + with open(binfile, 'w') as f: + f.write(content) + do(f'chmod +x {binfile}') + if nicosroot: + dolink('~/servicemanager/bin/nicos_sinq', '~/bin/nicos') + + +def do_scfg(): + """cfg file for sea and frappy service""" + chdir(home) + insfile = f'{stuffsrc}/{instrument}_servicemanager.cfg' + srcfile = f'{stuffsrc}/servicemanager.cfg' + if exists(insfile): + if doit: + os.system(f'cat {srcfile} {insfile} > servicemanager.cfg') + else: + todo.add(action) + os.system(f'cat {srcfile} {insfile} | diff servicemanager.cfg -') + return + docopy(srcfile, 'servicemanager.cfg') + + +def remove_line(file, content): + if os.system('grep %s %s' % (content, file)) == 0: + if doit: + do('grep -v %s %s > %s_new' % (content, file, file)) + do('mv %s_new %s' % (file, file)) + else: + todo.add(action) + print('remove above') + +#def do_monit(): +# """remove monit support for sea / graph""" +# remove_line('%s/monitconfig' % home, 'sea') +# remove_line('%s/monitconfig' % home, 'graph') + + +selected_instruments = None +action_arg = '' +help = True +for arg in sys.argv[1:]: + if arg in instruments: + if remote: + selected_instruments = [arg] + elif arg != instrument: + raise ValueError(f'bad instrument: {arg}') + elif arg == 'allin': + if not remote: + raise ValueError(f'"allin" is only allowed on samenv') + selected_instruments = instruments + elif arg == 'check': + doit = False + elif arg == 'nohelp': + help = False + elif action_arg: + raise ValueError('only one action is allowed') + else: + action_arg = arg + + +nc_actions = ['sshnicos', 'frappysinq', 'nicosconf', 'nicosenv'] # 'nicos_pick' +ncactionfuncs = {} + +with_su = False +if nicosroot.startswith(home): + actions = nc_actions + nc_actions = [] +else: + if nicosroot or remote: + with_su = action_arg in nc_actions + for a in nc_actions: + ncactionfuncs[a] = locals()['do_%s' % a] + actions = ['ssh', 'bin', 'scfg', 'frappy', 'servicemanager', 'sea', + 'calcurves', 'sehistory'] + +actionfuncs = {} +for a in actions: + actionfuncs[a] = locals()['do_%s' % a] +# if nicosroot: +# for a in nc_actions: +# actionfuncs['su %s' % a] = locals()['do_%s' % a] + + +def print_help(): + if remote: + inst = " " + else: + inst ='' + + def list_actions(funcs, postfix=''): + return '\n'.join(f" getsestuff {inst}%-15s # %s%s" % (a, f.__doc__, postfix) for a, f in funcs.items()) + '\n' + + print(f"""----- + + getsestuff {inst} # status + getsestuff {inst}all # install all + getsestuff {inst}sim # show commands to do +{list_actions(actionfuncs)}{list_actions(ncactionfuncs, ' *')}""", end='') + print(' ' * (32 + len(inst)), '* executed as user nicos') + if remote: + print(" replace by allin for applying to all instruments") + + +if remote: + if not doit: + action_arg = 'check ' + action_arg + if selected_instruments is None: + print_help() + else: + for inst in selected_instruments: + print(f'===== {inst} ' + '=' * (70-len(inst))) + os.environ['SSHPASS'] = inst.upper()+'LNS' + docmd(f'sshpass -e ssh {inst}@{inst} getsestuff {action_arg} nohelp') + sys.exit(0) + +def su_action(action): + os.environ['SSHPASS'] = scramble('P[d20+2:1eh') + docmd(f'sshpass -e ssh nicos@localhost {sys.argv[0]} {action} nohelp') + +if with_su: + su_action(action_arg) + sys.exit(0) +if action_arg == 'sim': + sim = True + doit = False +elif action_arg in actions: + action = action_arg + actionfuncs[action]() + sys.exit() +elif action_arg != 'all': + if action_arg != '': + print(f'unknown action {action}, not in', actions) + print_help() + sys.exit(1) + doit = False + +print(f'===== user {os.environ.get("USER", home)}') + +for action, func in actionfuncs.items(): + print(f'----- {action} ({home})') + func() + +if action_arg in ('', 'all', 'sim') and nc_actions: + su_action(action_arg) + print_help() + sys.exit(0) + +if help: + print_help() + if sim or not action_arg and todo: + print('needs updates: %s' % ' '.join(todo)) +