use sevenv for frappy

currently, the firewall rules allow only a small number of ports
8642 and 8742 were used for the second SEA instance on
hrpt2 and zebra2 - changed them to 8652 and 8752, e.g. they are
now only available within the instrument network.

README.md

@@ -3,3 +3,5 @@
 A manager for starting, stopping and listing services/servers like frappy, nicos and sea.
 
 Several instances of nicos, frappy, sea and seweb might run on the same machine.
+
+contains also getsestuff, the script to manage the repos on all instruments

__init__.py

@@ -1,4 +1,3 @@
-#  -*- coding: utf-8 -*-
 # *****************************************************************************
 #
 # This program is free software; you can redistribute it and/or modify it under
@@ -27,28 +26,19 @@ this code is currently used:
 - from a script allowing to start/stop/list (and more) multiple frappy and nicos servers
 """
 
+import sys
 from servicemanager.base import ServiceManager, ServiceDown, UsageError, get_config
 from servicemanager.nicosman import NicosManager
 from servicemanager.seaman import SeaManager
 from servicemanager.frappyman import FrappyManager, Reconnect, Keep
+from servicemanager.single import SewebManager, FeederManager
+#from servicemanager.racks import RackConfig
 
 
-class SewebManager(ServiceManager):
+#rack = RackConfig()
-    group = 'seweb'
-    services = ('',)
-    USAGE = """
-    Usage:
 
-      seaweb list [instance]
+all = NicosManager, FrappyManager, SeaManager, SewebManager, FeederManager
-      seaweb start <instance>
+KINDS = 'action', 'ins', 'service'
-      seaweb restart <instance>
-      seaweb stop <instance>
-
-      %s
-    """
-
-
-all = NicosManager, FrappyManager, SeaManager, SewebManager
 
 
 def run(group, arglist):
@@ -68,7 +58,7 @@ def run(group, arglist):
            'service': lambda arg: arg in serv.services,
         }
 
-        for kind in 'action', 'ins', 'service':
+        for kind in KINDS:
             if arglist and arg_is[kind](arglist[0]):  # arg is of expected kind
                 args[kind] = arglist.pop(0)
                 continue
@@ -91,8 +81,11 @@ def run(group, arglist):
             args.setdefault('ins', serv.main_ins)
         if guessed_args:
             args.setdefault('action', 'gui')
-            print('do you mean:\n   %s %s %s %s %s' %
+            guessed = [group] + [args.get(k, '') for k in KINDS] + extra
-                  (group, args.get('action', ''), args.get('ins', ''), args.get('service', ''), ' '.join(extra)))
+            while not guessed[-1]:
+                guessed.pop()
+            guessed = ' '.join(a or '?' for a in guessed)
+            print(f'do you mean:\n   {guessed}')
         else:
             try:
                 serv.action(args['action'], *serv.treat_args(args, extra + arglist))
@@ -101,5 +94,4 @@ def run(group, arglist):
     except UsageError as e:
         serv.do_help()
         print('ERROR:', str(e))
-
+        sys.exit(1)
-

allins.py

@@ -0,0 +1,12 @@
+import os
+
+instruments = ['amor', 'camea', 'dmc', 'eiger', 'focus', 'hrpt', 'boa', 'sans', 'tasp', 'zebra']  # morpheus
+
+
+def influx_setup(instrument=None):
+    ins_list = instruments if instrument is None else [instrument]
+    for ins in ins_list:
+        print(ins)
+        os.system(f'ssh {ins} influx setup --username l_samenv --password LIN3601se '
+                  '--token lZpLL_FrPw3cgkeuOWo_DcwjFZerrVTa1QQk6bcjSOYgjQ8W0eyvsp4Z4FULvsz2XEWTZZ8OPengYfLaWU-gdA== '
+                  f'--org linse --bucket sehistory --force --name {ins}')

base.py

@@ -24,7 +24,7 @@
 import sys
 import json
 import os
-from os.path import expanduser, basename
+from os.path import expanduser, basename, exists
 import subprocess
 import time
 import re
@@ -84,6 +84,7 @@ class ServiceManager:
     revcmd = {}
     USAGE = None
     main_ins = None
+    single_ins = None
 
     def __init__(self):
         self.env = {}
@@ -146,12 +147,18 @@ class ServiceManager:
             nr = section.get(self.group)
             if nr is not None:
                 nr = '%02d' % int(nr)
-                self.commands[ins] = command.replace('~', expanduser('~'))
                 services = self.get_services(section)
                 env = {k: get_subs(section, k, ins, nr) for k in section if k.isupper()}
                 result[ins] = services
                 self.env[ins] = env
+                cmd =  command.replace('~', expanduser('~'))
+                if cmd.startswith('PY '):
+                    cmd = env.get('PY', 'python3') + cmd[2:]
+                self.commands[ins] = cmd
         self.info = result
+        if len(self.info) == 1:
+            self.single_ins = list(self.info)[0]
+        return result
 
     #def get_cmdpats(self, groups):
     #    return self.cmdpats
@@ -207,7 +214,7 @@ class ServiceManager:
                     match = cmdpat.match(cmd)
                     if match:
                         gdict = match.groupdict()
-                        ins = gdict.get('ins', self.main_ins)
+                        ins = gdict.get('ins', self.main_ins) or self.single_ins
                         serv = gdict.get('serv', '')
                         if cfginfo is not None and 'cfg' in gdict:
                             cfginfo[ins, serv] = gdict['cfg']
@@ -219,7 +226,7 @@ class ServiceManager:
 
         or None, when no wildcard character in ins
         """
-        if ins is None or ins == 'all':
+        if not ins or ins == 'all':
             return list(self.info)
         pat = re.sub(r'(\.|\*)', '.*', ins)
         if pat == ins:
@@ -279,9 +286,9 @@ class ServiceManager:
         return done
 
     def do_stop(self, ins, service=None, *args):
+        if not ins:
+            raise UsageError(f'need instrument or "all" to stop all')
         self.get_info()
-        if ins is None:
-            raise ValueError('use stop all if you really want to stop all')
         ins_list = self.wildcard(ins)
         if ins_list is not None:
             return ins_list
@@ -298,6 +305,8 @@ class ServiceManager:
         return env.get('%s_ROOT' % gr, ''), env
 
     def do_start(self, ins, service=None, cfg='', restart=False, wait=False, logger=None, opts=''):
+        if not ins:
+            raise UsageError(f'need instrument or "all" to start all')
         ins_list = self.wildcard(ins)
         if ins_list is not None:
             return ins_list
@@ -316,10 +325,10 @@ class ServiceManager:
             return
         try:
             service_ports = self.get_ins_info(ins)
-        except ValueError:
+        except (KeyError, ValueError):
-            raise ValueError('do not know %r' % ins)
+            raise UsageError('do not know %r' % ins)
         if ins in self.remote_hosts:
-            raise ValueError('can not start, %s is running on a remote host' % self.group)
+            raise UsageError('can not start, %s is running on a remote host' % self.group)
         services = list(service_ports) if service is None else [service]
         if restart:
             self.stop(ins, service)
@@ -336,6 +345,7 @@ class ServiceManager:
             services = to_start
         for service_i in services:
             port = service_ports[service_i]
+            # TODO: remove unused pkg
             cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg or ins)
             if opts:
                 cmd = f'{cmd} {opts}'
@@ -353,6 +363,9 @@ class ServiceManager:
                 os.chdir(start_dir)
                 if start_dir not in sys.path:
                     sys.path.insert(0, start_dir)
+                nicosenv = '/home/nicos/nicos/nicosenv/bin/'
+                if exists(nicosenv):
+                    env['PATH'] = f"{nicosenv}:{env['PATH']}"
                 if wait:
                     proc = subprocess.Popen(cmd.split(), env=env)
                     for _ in range(3):
@@ -399,8 +412,8 @@ class ServiceManager:
                 os.chdir(wd)
 
     def do_restart(self, ins, service=None, cfg=None, logger=None):
-        if ins is None:
+        if not ins:
-            raise UsageError("need instance or 'all' or wildcard")
+            raise UsageError("need instrument or 'all' or wildcard")
         ins_list = self.wildcard(ins)
         if ins_list is not None:
             if cfg is not None:
@@ -411,7 +424,7 @@ class ServiceManager:
     def do_run(self, ins, service=None, cfg=None, opts=''):
         """for tests: run and wait"""
         if self.wildcard(ins) is not None:
-            raise UsageError('no wildcards allowed with %s run' % self.group)
+            raise UsageError('need instrument and service for "%s run"' % self.group)
         if not service:
             try:
                 service, = self.services
@@ -421,6 +434,7 @@ class ServiceManager:
 
     def do_list(self, ins=None, *args):
         """info about running services"""
+        self.get_info()
         show_unused = ins == 'all'
         if show_unused:
             ins = None

bin/feeder

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import sys
+from os.path import expanduser
+sys.path.append(expanduser('~'))
+from servicemanager import run
+
+run('feeder', sys.argv[1:])

bin/nicos_sinq

@@ -1,4 +1,4 @@
-#!/home/nicos/nicos/nicosenv/bin/python3
+#!/home/software/virtualenv/nicosenv/bin/python3
 import sys
 import time
 from os import path
@@ -15,8 +15,8 @@ Usage:
   nicos cli     start nicos command line client and connect without asking for password
 """
 
-yr = time.strftime('%y')
+# yr = time.strftime('%y')
-connect = f'user:{yr}lns1@localhost'
+connect = f'user:sinq@localhost'
 
 if sys.argv[-1] == 'cli':
    from nicos.clients.cli import main

cfg/feeder@.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=sehistory feeder process
+
+[Service]
+Type=simple
+WorkingDirectory=%h/sehistory
+ExecStart=%h/sevenv/bin/python feeder.py -d %i
+Restart=always
+
+[Install]
+WantedBy=default.target

cfg/hrpt_servicemanager.cfg

@@ -0,0 +1,7 @@
+
+[hrpt2]
+SEA_SEA_PORT = 8652
+SEA_GRAPH_PORT = 8752
+sea_command = ./SeaServer %(serv)s_%(ins)s.tcl
+sea = 2
+

cfg/sehistory

@@ -0,0 +1,11 @@
+[local]
+uri=http://localhost:8086
+bucket=sehistory
+org=linse
+token=lZpLL_FrPw3cgkeuOWo_DcwjFZerrVTa1QQk6bcjSOYgjQ8W0eyvsp4Z4FULvsz2XEWTZZ8OPengYfLaWU-gdA==
+
+[central]
+uri=http://linse-c:8086
+bucket=sehistory
+org=linse
+token=ggo2cRoCIkgniXdUGi-s6oxXJPPDzgKEFHh6PXuBg1QQa3aBo36tGFP_6cf50FRNoDQgGcelh3xBM5QsVw4rHA==

cfg/servicemanager.cfg

@@ -0,0 +1,34 @@
+[DEFAULT]
+VENV = /home/nicos/nicos/nicosenv/bin/
+FRAPPY_MAIN_PORT = 151nr
+FRAPPY_STICK_PORT = 152nr
+FRAPPY_ADDONS_PORT = 153nr
+FRAPPY_ROOT = ~/frappy
+#FRAPPY_HISTORY = ~/frappy/history/
+FRAPPY_CONFDIR =  ~/frappy/cfg/<SERV>:~/frappy/cfg/develop
+FRAPPY_LOGDIR =  ~/frappylog
+FRAPPY_PIDDIR =  ~/frappylog/pid
+FRAPPY_SEA_DIR = ~/frappy/cfg/sea
+FRAPPY_CALIB_PATH = ~/calcurves
+frappy_command = ~/sevenv/bin/python bin/frappy-server %(ins)s_%(serv)s -p=%(port)s -c=%(cfg)s
+
+SEA_SEA_PORT = 8641
+SEA_GRAPH_PORT = 8741
+SEA_ROOT = ~/sea
+sea_command = ./SeaServer %(serv)s.tcl
+
+SEWEB_ROOT = ~/seweb
+SEWEB_PORT = 8642
+seweb_command = ~/sevenv/bin/python secop-webserver %(port)s -i %(ins)s
+
+FEEDER_ROOT = ~/sehistory
+FEEDER_PORT = 0
+feeder_command = ~/sevenv/bin/python feeder.py -d %(serv)s
+
+PYTHONPATH = ~:~/frappy
+
+[MAIN]
+frappy = 1
+sea = 1
+seweb = 1
+feeder = 1

cfg/seweb@.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=SE web server
+
+[Service]
+Type=simple
+WorkingDirectory=%h/seweb
+ExecStart=%h/sevenv/bin/python secop-webserver 8642 -i %i
+Restart=always
+
+[Install]
+WantedBy=default.target

cfg/zebra_servicemanager.cfg

@@ -0,0 +1,7 @@
+
+[zebra2]
+SEA_SEA_PORT = 8652
+SEA_GRAPH_PORT = 8752
+sea_command = ./SeaServer %(serv)s_%(ins)s.tcl
+sea = 2
+

frappyman.py

@@ -37,9 +37,12 @@ STICK = 2
 
 
 class Namespace(dict):
-    def __init__(self):
+    def __init__(self, frappymanager, *args):
+        self.fm = frappymanager
+        self.args = args
         self['Node'] = self.node
         self['Mod'] = self.mod
+        self['Include'] = self.include
         for fun in 'Param', 'Command', 'Group':
             self[fun] = self.dummy
         self.init()
@@ -59,6 +62,12 @@ class Namespace(dict):
     def dummy(self, *args, **kwds):
         return None
 
+    def include(self, cfg):
+        local = {}
+        print('INCLUDE', self.args, cfg, self.fm.get_cfg_file(*self.args, cfg, True))
+        with open(self.fm.get_cfg_file(*self.args, cfg, True), encoding='utf-8') as f:
+            exec(f.read(), self, local)
+
     __builtins__ = builtins
 
 
@@ -114,16 +123,19 @@ class FrappyManager(ServiceManager):
         cfgpaths = []
         cfgparser = ConfigParser()
         cfgparser.optionxform = str
-        cfgfile = self.env[ins].get('FRAPPY_CONFIG_FILE')
+        env = self.env.get(ins, {})
-        confdir = self.env[ins].get('FRAPPY_CONFDIR')
+        cfgfile = env.get('FRAPPY_CONFIG_FILE')
+        confdir = env.get('FRAPPY_CONFDIR')
         if cfgfile:
-            cfgfile = self.env[ins]['FRAPPY_CONFIG_FILE'].replace('<SERV>', service)
+            cfgfile = cfgfile.replace('<SERV>', service)
             cfgparser.read(cfgfile)
             try:
                 section = cfgparser['FRAPPY']
             except KeyError:
                 raise ValueError('%s does not exist or has no FRAPPY section' % cfgfile)
             confdir = section.get('confdir', confdir)
+        if not confdir:
+            return []
         for cfgpath in confdir.split(os.pathsep):
             if cfgpath.endswith('<SERV>'):
                 cfgpaths.append(expanduser(cfgpath[:-6] + service))
@@ -207,6 +219,9 @@ class FrappyManager(ServiceManager):
         nodes = self.get_nodes(ins, service)
         from frappy.client.interactive import init, interact
         init(*nodes)
+        try:
+            interact(appname=ins)
+        except TypeError:  # older frappy client
             interact()
 
     @staticmethod
@@ -219,19 +234,25 @@ class FrappyManager(ServiceManager):
         return namespace.description, local.get('sea_cfg', namespace.sea_cfg)
 
     def cfg_details(self, ins, service, cfg):
-        namespace = Namespace()
+        namespace = Namespace(self, ins, service)
-        for cfgdir in self.config_dirs(ins, service):
+        if cfgfile:
-            cfgfile = join(cfgdir, f'{cfg}_cfg.py')
-            if exists(cfgfile):
             return self.get_cfg_details(namespace, cfgfile)
         raise FileNotFoundError(f'{cfg} not found')
 
+    def get_cfg_file(self, ins, service, cfg, lazy=False):
+        filenames = [f'{cfg}_cfg.py']
+        if lazy:
+            filenames.extend([f'{cfg}.py', cfg])
+        for cfgdir in self.config_dirs(ins, service):
+            for filename in filenames:
+                cfgfile = join(cfgdir, filename)
+                if exists(cfgfile):
+                    return cfgfile
+        print('NOT FOUND', cfg, self.config_dirs(ins, service), filenames)
+        return None
+
     def is_cfg(self, ins, service, cfg):
-        try:
+        return bool(self.get_cfg_file(ins, service, cfg))
-            self.cfg_details(ins, service, cfg)
-            return True
-        except Exception:
-            return False
 
     def all_cfg(self, ins, service, details=False):
         """get available cfg files
@@ -249,7 +270,7 @@ class FrappyManager(ServiceManager):
         all_cfg = set()
         if not ins:
             return {}
-        namespace = Namespace()
+        namespace = Namespace(self, ins, service)
         if details:
             self.frappy2sea = f2s = {}
             self.sea2frappy = s2f = {}
@@ -320,7 +341,7 @@ class FrappyManager(ServiceManager):
                 argdict['service'] = cfg
                 return super().treat_args(argdict, (), extra)
             if (',' in cfg or cfg.endswith('.cfg') or
-                    self.is_cfg(argdict.get('ins'), argdict.get('service'), cfg)):
+                    self.get_cfg_file(argdict.get('ins'), argdict.get('service'), cfg, True)):
                 return super().treat_args(argdict, (), [cfg] + extra)
         return super().treat_args(argdict, unknown, extra)
 
@@ -404,7 +425,6 @@ class FrappyManager(ServiceManager):
         if proposed_addons:  # and set(proposed_addons) != set(running_addons):
             proposed_cfg['addons'] = {','.join(proposed_addons)}
 
-        self._debug = {}
         for service in FrappyManager.services:
             given = givencfgs.get(service)
             running = self.frappy_cfgs.get(service)
@@ -413,8 +433,6 @@ class FrappyManager(ServiceManager):
             if running:
                 self.state[f'frappy {service}'] = running
 
-            self._debug[service] = (seaconfig, available, running, running in self.frappy2sea)
-
             if seaconfig and (available or running in self.frappy2sea):
                 # we get here when the sea server is running and either at least one of:
                 # - the sea config is matching any frappy cfg

getsestuff

@@ -0,0 +1,664 @@
+#!/usr/bin/env python3
+import sys
+import os
+import time
+from os import chdir, environ, getcwd
+from ast import literal_eval
+from configparser import ConfigParser
+from os.path import expanduser, exists, join, basename, realpath, isdir
+from subprocess import Popen, PIPE, check_output
+from glob import glob
+from socket import gethostname
+
+instruments = ['camea', 'dmc', 'eiger', 'focus', 'hrpt', 'sans', 'tasp', 'zebra', 'boa', 'amor']
+
+stuffsrc = '/afs/psi.ch/project/sinq/common/lib/servicemanager/'
+
+doit = True  # False: check only
+sim = False  # True: show what to do
+action = ''
+todo = set()
+
+home = expanduser('~')
+hostname = gethostname().split('.')[0]
+remote = hostname not in instruments
+
+if home.endswith('zolliker'):
+    chdir(expanduser('~/servicemanager/'))
+    cfg_fil = '/afs/psi.ch/project/sinq/common/stow/markus/lib/servicemanager/', glob('cfg/*.cfg')
+    bin_dst = '/afs/psi.ch/project/sinq/common/stow/markus/bin/', ['getsestuff']
+    diff = False
+    for dstdir, files in cfg_fil, bin_dst:
+        for src in files:
+            diff = os.system(f'diff {src} {dstdir}')
+            if diff and os.system(f'cp {src} {dstdir}'):
+                print('can not copy', src, 'to', dstdir)
+    if diff:
+        print('updated getsestuff, please do again')
+        sys.exit(0)
+
+
+if remote:
+    instrument = ''
+    nicosroot = ''
+    nicosenv = ''
+else:
+    nicosroot = '/home/nicos/nicos'
+    if not exists(f'{nicosroot}/.git/config'):
+        print('nicos repo not found at', nicosroot)
+        nicosroot = ''
+    nicosenv = '/home/software/virtualenv/nicosenv'
+
+    instrument = environ.get('Instrument')
+    if instrument is None:
+        instrhome = '/home/%s' % hostname
+        if exists(instrhome):
+            instrument = hostname
+    else:
+        instrhome = '/home/%s' % instrument
+        if not exists(instrhome):
+            instrhome = home
+    if not exists(instrhome) or (instrhome != home and not nicosroot.startswith(home)):
+        print(instrhome, exists(instrhome), home, nicosroot)
+        print('can not guess instrument, please define environment variable "Instrument"')
+        instrument = ''
+
+
+def doget(cmd):
+    out = Popen(cmd.split(), stdout=PIPE).communicate()[0]
+    return list(out.decode().split('\n'))
+
+
+def docmd(cmd):
+    lines = doget(cmd)
+    print('\n'.join(lines), end='')
+
+
+def from_str(string):
+    try:
+        return literal_eval(string)
+    except Exception as e:
+        return string.strip()
+
+
+def scramble(arg):
+    return bytes([(158 - b) for b in arg.encode('ascii')]).decode('ascii')
+
+
+def do(cmd):
+    print('>', cmd)
+    if not sim:
+        return not os.system(cmd)
+    return True
+
+
+def docopy(src, dst):
+    if not os.system(f'diff {dst} {src}'):
+        return False
+    if doit:
+        if not do(f'cp {src} {dst}'):
+            do(f'mv {dst} {dst}0')
+            do(f'cp {src} {dst}')
+    else:
+        todo.add(action)
+    return True
+
+
+def dolink(dst, src):
+    src = realpath(expanduser(src))
+    dst = realpath(expanduser(dst))
+    if exists(dst):
+        if src != dst:
+            if doit:
+                do(f'ln -sf {dst} {src}')
+            else:
+                todo.add(action)
+                print('%s !-> %s' % (src, dst))
+    else:
+        print(f'target {dst} does not exist')
+
+
+def ch_repo_dir(gitdir):
+    """change working directory to repo dir"""
+    # git complains when using it at a directory where we have no write access
+    # this can be avoided with adding the command below
+    chdir(gitdir)
+    if not os.access(gitdir, os.W_OK):
+        entry = ['directory', gitdir]
+        with open(join(home, ".gitconfig")) as f:
+            for line in f:
+                if entry == [v.strip() for v in line.split('=')]:
+                    return
+        docmd(f'git config --global --add safe.directory {gitdir}')
+
+
+def check_repo(root, repo, url=None, branch=None):
+    chdir(root)
+    created = exists(join(repo, '.git'))
+    if url is None:
+        url = f'https://gitea.psi.ch/linse/{repo}.git'
+    elif url == 'gitlab':
+        url = f'git@gitlab.psi.ch-samenv:samenv/{repo}.git'
+
+    pull = False
+    if (doit or sim) and not created:
+        todo.add(action)
+        short = repo.rpartition('/')[2]
+        if exists(repo):
+            chdir(repo)
+            gitconfig = {}
+            for line in doget('git config --list'):
+                key, _, value = line.partition('=')
+                if value:
+                    gitconfig[key] = value
+            if gitconfig.get('remote.origin.url') != 'url':
+                print(gitconfig.get('remote.origin.url'), 'does not match', url)
+            print(f'{join(root, repo)} exists already')
+            return False
+        else:
+            do('git clone %s' % url)
+        pull = True
+        created = doit
+    if created:
+        chdir(join(root, repo))
+        docmd('git remote update')
+        gitconfig = {}
+        for line in doget('git config --list'):
+            key, _, value = line.partition('=')
+            if value:
+                gitconfig[key] = value
+
+        prev = dict(gitconfig)
+        gitconfig['push.default'] = 'upstream'
+        gitconfig['pull.rebase'] = 'True'
+        gitconfig['remote.origin.url'] = url
+        if 'gitea' in url:
+            hooksrc = '/afs/psi.ch/project/sinq/common/stow/markus/lib/gitea'
+            docopy(f'{hooksrc}/get_gitea_token', '.git/hooks/')
+            docopy(f'{hooksrc}/pre-commit', '.git/hooks/')
+            gitconfig['credential.helper'] = f'{getcwd()}/.git/hooks/get_gitea_token'
+        diff = {k: v for k, v in gitconfig.items() if v != prev.get(k) and v != prev.get(k.replace('branch.', ''))}
+        if doit:
+            for kv in diff.items():
+                do('git config %s %s' % kv)
+        elif diff:
+            todo.add(action)
+            print('modified git config keys:')
+            for key, val in diff.items():
+                print('%s=%s -> %s' % (key, prev.get(key), val))
+        if branch:
+            for line in doget('git rev-parse --abbrev-ref HEAD'):
+                if branch != line and line:
+                     print(f'wrong branch: {line}')
+                     return False
+        pull = False
+        show_status = True
+        for line in doget('git status'):
+            if '"git pull"' in line:
+               pull = doit
+               todo.add(action)
+               break
+            elif 'working tree clean' in line:
+               show_status = False
+        if show_status:
+            docmd('git status')
+    else:
+        todo.add(action)
+        print('not yet created', action)
+    return pull
+
+
+SSH_CONFIG = """Host gitlab.psi.ch-samenv
+     HostName gitlab.psi.ch
+     User zolliker
+     IdentityFile ~/.ssh/id_rsa_samenv
+"""
+
+
+def do_ssh():
+    """ssh config for pushing git to samenv repo"""
+    chdir(home)
+    try:
+        skip = False
+        with open('.ssh/config') as f:
+            content = f.read()
+    except FileNotFoundError:
+        content = ''
+    lines = []
+    for line in content.split('\n'):
+        if line.startswith('Host gitlab.psi.ch-samenv'):
+            skip = True
+        elif line[:1] not in ' \t':
+            skip = False
+        if not skip:
+            lines.append(line)
+
+    lines.append(SSH_CONFIG)
+    result = '\n'.join(lines)
+
+    if result == content:
+        pass
+    elif sim:
+        todo.add(action)
+        print('--- .ssh/config current:')
+        print(content)
+        print('--- .ssh/config new:')
+        print(result)
+        print('---')        
+    elif not doit:
+        todo.add(action)
+        print('.ssh/config needs update')
+    else:
+        with open('.ssh/config', 'w') as f:
+            f.write(result)
+    if not exists('.ssh/id_rsa_samenv'):
+        if doit:
+            do('scp l_samenv@samenv:.ssh/id_rsa .ssh/id_rsa_samenv')
+            do('chmod g-r-x .ssh/id_rsa_samenv')
+        else:
+            todo.add(action)
+            print('missing id_rsa_samenv')
+            return
+    if not doget('ls -l .ssh/id_rsa_samenv')[0].startswith('-rw------'):
+        todo.add(action)
+        do('chmod -x .ssh/id_rsa_samenv')
+        do('chmod g-r-w .ssh/id_rsa_samenv')
+        do('chmod o-r-w .ssh/id_rsa_samenv')
+
+
+def do_sshnicos():
+    do_ssh()
+
+
+def do_sevenv():
+    """install python packages needed for seweb etc."""
+    sevenv = join(home, 'sevenv')
+    if not exists(join(sevenv, 'bin')):
+        if exists(nicosenv):
+            # start from nicos env if possible
+            py = join(nicosenv, 'bin/python')
+        else:
+            py = 'python3'
+        do(f'{py} -m venv {sevenv}')
+    if exists(join(sevenv, 'bin')):
+        upgrade = True
+        for pkg in ['mlzlog', 'scipy', 'psutil', 'flask', 'gevent', 'influxdb_client']:
+            if not glob(f'{sevenv}/lib/python3*/site-packages/{pkg}'):
+                if doit:
+                    if upgrade:
+                        do(f'{sevenv}/bin/python3 -m pip install --upgrade pip')
+                        upgrade = False
+                    do(f'{sevenv}/bin/python3 -m pip install {pkg}')
+                else:
+                    print(f'missing {pkg} in sevenv')
+
+
+def do_frappy():
+    """Frappy framework"""
+    frappydir = join(home, 'frappy')
+    if exists(frappydir) and not exists(join(frappydir, '.git')):
+        do('rm -rf %s' % frappydir)
+    if check_repo(home, 'frappy'):
+        do('git pull')
+    if exists(join(frappydir, '.git')):
+        sys.path.extend(glob(f'{nicosenv}/lib/*/site-packages'))
+        try:
+            missing = 'psutil'
+            import psutil
+            missing = 'mlzlog'
+            import mlzlog
+        except ImportError:
+            print('MISSING', missing, 'in nicosenv (do it manually!)')
+            # if doit:
+            #     do('pip3 install --user psutil')
+            #     do('pip3 install --user mlzlog')
+            # else:
+            #     todo.add(action)
+            #     print('missing', missing, 'in nicosenv')
+
+
+def do_servicemanager():
+    """servicemanager package"""
+    if check_repo(home, 'servicemanager', None, 'master'):
+        do('git pull')
+
+
+def do_feeder():
+    """history feeder"""
+    if check_repo(home, 'sehistory', None, 'master'):
+        do('git pull')
+    configdir = join(home, '.config')
+    systemddir = join(configdir, 'systemd/user')
+    if not exists(systemddir):
+        do(f'mkdir -p {systemddir}')
+    docopy(join(home, 'servicemanager/cfg/sehistory'), join(configdir, 'sehistory'))
+    if docopy(join(home, 'servicemanager/cfg/feeder@.service'),
+              join(systemddir, 'feeder@.service')):
+        do('systemctl --user daemon-reload')
+
+
+def do_seweb():
+    """SE web client"""
+    if check_repo(home, 'seweb', None, 'master'):
+        do('git pull')
+    systemddir = join(home, '.config/systemd/user')
+    if not exists(systemddir):
+        do(f'mkdir -p {systemddir}')
+    if docopy(join(home, 'servicemanager/cfg/seweb@.service'),
+              join(systemddir, 'seweb@.service')):
+        do('systemctl --user daemon-reload')
+
+
+def do_sea():
+    """SEA server scripts"""
+    if check_repo(home, 'sea'):
+        do('git pull')
+        # do('git checkout master -- .gitignore')  # do not know why this is needed
+        # do('chmod -x tcl/config/json_racklist tcl/*.* tcl/*.tcl tcl/*/*.tcl')
+        if not os.access('tcl/luft.tclsh', os.X_OK):
+            do('chmod +x tcl/luft.tclsh')
+        if not exists('tcl/plugin'):
+            do('git checkout master -- tcl/plugin')
+        if not exists('tcl/calcurves/.git') and not exists('tcl/calcurves/coil.inp'):
+            do('rm -rf tcl/calcurves')
+            do_calcurves()
+    docopy('/afs/psi.ch/user/z/zolliker/public/git.rhel7/sics/SeaServer', join(home, 'sea', 'SeaServer'))
+    dolink('/afs/psi.ch/project/sinq/rhel7/bin/SeaClient', join(home, 'bin', 'SeaClient'))
+
+
+def do_calcurves():
+    """calibration curves"""
+    if check_repo(home, 'calcurves'):
+        do('git pull')
+    dolink('~/calcurves', '~/sea/tcl/calcurves')
+
+
+def do_frappysinq():
+    """nicos_sinq/frappy_sinq setups and extensions"""
+    if exists(nicosroot):
+        if check_repo(join(nicosroot, 'nicos_sinq'), 'frappy_sinq'):
+            do('git pull')
+
+
+def do_nicosconf():
+    """nicos.conf"""
+    # do the change in two places, as there is not always a link
+    for nicosconf in [join(nicosroot, 'nicos.conf'),
+                      join(nicosroot, 'nicos_sinq', instrument, 'nicos.conf')]:
+        if not exists(nicosconf):
+            continue
+        cp = ConfigParser()
+        cp.optionxform = str
+        cp.read(nicosconf)
+        assert from_str(cp['nicos']['instrument']) == instrument
+        setup_subdirs = from_str(cp['nicos'].get('setup_subdirs', '["%s"]' % instrument))
+        new_subdirs = setup_subdirs[:]
+        if "frappy_sinq" not in new_subdirs:
+            print("missing frappy_sinq in setup_subdirs")
+            new_subdirs.append("frappy_sinq")
+            todo.add(action)
+        if "frappy" in new_subdirs:
+            print("superflous frappy in setup_subdirs")
+            new_subdirs.remove("frappy")
+            todo.add(action)
+        pythonpath = from_str(cp['environment'].get('PYTHONPATH', '""'))
+        pythonpath = pythonpath.split(':') if pythonpath else []
+        newpath = pythonpath[:]
+        frappyhome = join(instrhome, 'frappy')
+        localpy = join(instrhome, '.local/lib/python3.6/site-packages')
+        if frappyhome not in newpath:
+            print("missing %s in PYTHONPATH" % frappyhome)
+            newpath.append(frappyhome)
+            todo.add(action)
+        if instrhome not in newpath:
+            print("missing %s in PYTHONPATH" % instrhome)
+            newpath.append(instrhome)
+            todo.add(action)
+        if localpy in newpath:
+            print("remove  %s from PYTHONPATH" % localpy)
+            newpath.remove(localpy)
+            todo.add(action)
+        if doit:
+            dirty = False
+            if new_subdirs != setup_subdirs:
+                cp['nicos']['setup_subdirs'] = '["%s"]' % '", "'.join(new_subdirs)
+                dirty = True
+            if newpath != pythonpath:
+                cp['environment']['PYTHONPATH'] = '"%s"' % ':'.join(newpath)
+                dirty = True
+            if dirty and not sim:
+                print('modify', nicosconf)
+                with open(nicosconf, 'w') as f:
+                    cp.write(f)
+
+
+def do_nicosenv():
+    """install python packages needed for frappy/servicemanager"""
+    chdir(nicosroot)
+    for pkg in ['mlzlog', 'scipy']:
+        if not glob(f'{nicosenv}/lib/python3*/site-packages/{pkg}'):
+            if doit:
+                do(f'{nicosenv}/bin/python3 -m pip install {pkg}')
+            else:
+                print(f'missing {pkg} in nicosenv')
+
+
+def do_nicos_pick():
+    """OBSOLETE: cherry-pick all new commits in sinq-3.11 related to nicos/devices/secop"""
+    chdir(nicosroot)
+
+    docmd('git fetch sinq')
+
+    def get_change_ids(branch):
+        command = f'git log --since=2023-09-01 {branch} nicos/devices/secop'
+        ps = Popen(command.split(), stdout=PIPE)
+        result = []
+        try:
+            for line in check_output(('grep', 'Change-Id:'), stdin=ps.stdout).decode('latin-1').split('\n'):
+                line = line.strip().split()
+                if len(line) == 2:
+                    result.append(line[1])
+        except Exception as e:
+            print(repr(e))
+        ps.wait()
+        return result
+
+    src = 'sinq/sinq-3.11'
+    dst = ' '
+    srcids = get_change_ids(src)
+    dstids = get_change_ids(dst)
+    dstset = set(dstids)
+    for changeid in reversed(srcids):
+        if changeid not in dstset:
+            line = check_output(f'git log --oneline {src} --grep={changeid}'.split()).decode('latin-1').strip()
+            if line:
+                if doit:
+                    commit = line.split()[0]
+                    docmd(f'git cherry-pick {commit}')
+                else:
+                    todo.add(action)
+                    print(line)
+
+
+BIN = """#!%s
+import sys
+sys.path.append('%s')
+from servicemanager import run
+run('%s', sys.argv[1:])
+"""
+
+
+def do_bin():
+    """added commands"""
+    chdir(home)
+    if not exists('bin'):
+        do('mkdir -p bin')
+    pgms = ['frappy', 'sea', 'seweb', 'feeder']
+    if nicosroot:
+        executable = f'{nicosenv}/bin/python3'
+    else:
+        executable = '/usr/bin/env python3.11'
+        pgms.append('nicos')
+    for pgm in pgms:
+        content = BIN % (executable, home, pgm)
+        binfile = join(home, 'bin', pgm)
+        try:
+            with open(binfile) as f:
+                writeit = f.read() != content
+        except FileNotFoundError:
+            writeit = 'create'
+        if writeit:
+            print('modify', binfile)
+            if doit and not sim:
+                if writeit != 'create':
+                    os.remove(binfile)
+                with open(binfile, 'w') as f:
+                    f.write(content)
+                do(f'chmod +x {binfile}')
+    if nicosroot:
+        dolink('~/servicemanager/bin/nicos_sinq', '~/bin/nicos')
+
+
+def do_scfg():
+    """cfg file for sea and frappy service"""
+    chdir(home)
+    insfile = f'{stuffsrc}/{instrument}_servicemanager.cfg'
+    srcfile = f'{stuffsrc}/servicemanager.cfg'
+    if exists(insfile):
+        if doit:
+            os.system(f'cat {srcfile} {insfile} > servicemanager.cfg')
+        else:
+            todo.add(action)
+            os.system(f'cat {srcfile} {insfile} | diff servicemanager.cfg -')
+        return
+    docopy(srcfile, 'servicemanager.cfg')
+
+
+def remove_line(file, content):
+    if os.system('grep %s %s' % (content, file)) == 0:
+        if doit:
+            do('grep -v %s %s > %s_new' % (content, file, file))
+            do('mv %s_new %s' % (file, file))
+        else:
+            todo.add(action)
+            print('remove above')
+
+
+selected_instruments = None
+action_arg = ''
+help = True
+for arg in sys.argv[1:]:
+    if arg in instruments:
+        if remote:
+            selected_instruments = [arg]
+        elif arg != instrument:
+            raise ValueError(f'bad instrument: {arg}')
+    elif arg == 'allin':
+       if not remote:
+            raise ValueError(f'"allin" is only allowed on samenv')
+       selected_instruments = instruments
+    elif arg == 'check':
+       doit = False
+    elif arg == 'nohelp':
+       help = False
+    elif action_arg:
+       raise ValueError('only one action is allowed')
+    else:
+       action_arg = arg
+
+
+nc_actions = ['frappysinq', 'nicosconf', 'nicosenv']  # 'sshnicos', 'nicos_pick'
+ncactionfuncs = {}
+
+with_su = False
+if nicosroot.startswith(home):
+    actions = nc_actions
+    nc_actions = []
+else:
+    if nicosroot or remote:
+        with_su = action_arg in nc_actions
+        for a in nc_actions:
+            ncactionfuncs[a] = locals()['do_%s' % a]
+    actions = ['bin', 'scfg', 'frappy', 'servicemanager', 'sea',
+               'calcurves', 'feeder', 'seweb', 'sevenv']  # 'ssh'
+
+actionfuncs = {}
+for a in actions:
+    actionfuncs[a] = locals()['do_%s' % a]
+# if nicosroot:
+#     for a in nc_actions:
+#         actionfuncs['su %s' % a] = locals()['do_%s' % a]
+
+
+def print_help():
+    if not help:
+        return
+    if remote:
+        inst = "<instrument> "
+    else:
+        inst =''
+
+    def list_actions(funcs, postfix=''):
+        return '\n'.join(f"    getsestuff {inst}%-15s # %s%s" % (a, f.__doc__, postfix) for a, f in funcs.items()) + '\n'
+
+    print(f"""-----
+
+    getsestuff {inst}                # status
+    getsestuff {inst}all             # install all
+    getsestuff {inst}sim             # show commands to do
+{list_actions(actionfuncs)}{list_actions(ncactionfuncs, ' *')}""", end='')
+    print(' ' * (32 + len(inst)), '* executed as user nicos')
+    if remote:
+        print("    replace <instrument> by allin for applying to all instruments")
+
+
+if remote:
+    if not doit:
+        action_arg = 'check ' + action_arg
+    if selected_instruments is None:
+        print_help()
+    else:
+        for inst in selected_instruments:
+            print(f'===== {inst} ' + '=' * (70-len(inst)))
+            os.environ['SSHPASS'] = inst.upper()+'LNS'
+            docmd(f'sshpass -e ssh {inst}@{inst} getsestuff {action_arg} nohelp')
+    sys.exit(0)
+
+
+def su_action(action):
+    os.environ['SSHPASS'] = scramble('P[d20+2:1eh')
+    docmd(f'sshpass -e ssh nicos@localhost {sys.argv[0]} {action} nohelp')
+
+
+if with_su:
+    su_action(action_arg)
+    sys.exit(0)
+if action_arg == 'sim':
+    sim = True
+    doit = False
+elif action_arg in actions:
+    action = action_arg
+    actionfuncs[action]()
+    sys.exit()
+elif action_arg != 'all':
+    if action_arg != '':
+        print(f'unknown action {action}, not in', actions)
+        print_help()
+        sys.exit(1)
+    doit = False
+
+print(f'===== user {os.environ.get("USER", home)}')
+
+for action, func in actionfuncs.items():
+    print(f'----- {action} ({home})')
+    func()
+
+if action_arg in ('', 'all', 'sim') and nc_actions:
+    su_action(action_arg)
+    print_help()
+    sys.exit(0)
+
+print_help()
+if sim or not action_arg and todo:
+    print('needs updates: %s' % ' '.join(todo))
+

nicosman.py

@@ -84,6 +84,7 @@ class NicosManager(ServiceManager):
                     print('ambiguos package: %s' % ', '.join(instdir))
                 else:
                     env['NICOS_PACKAGE'] = basename(dirname(instdir[0]))
+        return self.info
 
     def do_create(self, ins, *args):
         if ins == 'check':

seaman.py

@@ -1,4 +1,3 @@
-#  -*- coding: utf-8 -*-
 # *****************************************************************************
 #
 # This program is free software; you can redistribute it and/or modify it under
@@ -34,6 +33,16 @@ CFGLINE = re.compile(r'(?:device makeitem (name|stick_name|confirmed) "(.*)" ""'
                      r'|addon_list makeitem (.*) (?:"permanent"|"volatile"))')
 
 
+INIFILE_CONTENT = """
+set instrument {ins_i}
+set serverport {port}
+set logbase {insdir}
+{extraline}cd {seadir}tcl
+exe echo 1
+exe {service}init.tcl
+"""
+
+
 def run_command(cmd, wait=False):
     if wait:
         old = termios.tcgetattr(sys.stdin)
@@ -68,7 +77,49 @@ class SeaManager(ServiceManager):
 %(legend)s
     """
 
-    def do_cli(self, ins):
+    def do_create(self, ins, *args):
+        if ins == 'check':
+            ins_list = self.wildcard(None)
+        else:
+            ins_list = self.wildcard(ins)
+            if ins_list is None:
+                ins_list = [ins]
+        self.get_info()
+        for ins_i in ins_list:
+            seadir = f"{expanduser('~')}/sea/"
+            if not exists(seadir):
+                print(f'get {seadir} from git first')
+                return
+            insdir = seadir if ins_i == self.main_ins else f'{seadir}{ins_i}/'
+            if ins != 'check':
+                os.makedirs(f'{insdir}/logger', exist_ok=True)
+                os.makedirs(f'{insdir}/log', exist_ok=True)
+                os.makedirs(f'{insdir}/status', exist_ok=True)
+            extraline = 'set connect_sea_to_sics 0\n'
+            for service in 'sea', 'graph':
+                port = self.info[ins_i][service]
+                if not port:
+                    continue
+                sea_init = INIFILE_CONTENT.format(**locals()).strip()
+                extraline = ''
+                filename = f'{seadir}{service}_{ins_i}.tcl'
+                try:
+                    with open(filename) as f:
+                        content = f.read().strip()
+                except FileNotFoundError:
+                    content = ''
+                if sea_init != content:
+                    if ins == 'check':
+                        if content:
+                            print(filename, 'does not match')
+                        else:
+                            print(filename, 'does not exist')
+                    else:
+                        with open(filename, 'w') as f:
+                            f.write(sea_init)
+                            f.write('\n')
+
+    def do_cli(self, ins, service=None):
         if self.wildcard(ins):
             raise UsageError('wildcards not allowed in sea cli')
         try:
@@ -78,7 +129,8 @@ class SeaManager(ServiceManager):
             print(str(e))
         except KeyError:  # running on an other machine?
             self.do_help()
-        run_command('six -sea %s' % ins, wait=True)
+        service = service or 'sea'
+        run_command(f'six -{service} {ins}', wait=True)
 
     def do_gui(self, ins='', *args):
         if ins and self.wildcard(ins):
@@ -117,7 +169,7 @@ class SeaManager(ServiceManager):
             seastatus = self.get_status_filename(ins)
             if seastatus:
                 boot_time = time.strftime("%Y-%m-%dT%H-%M-%S", time.localtime(psutil.boot_time()))
-                dst = seastatus.replace('.tcl', '') + '.' + boot_time
+                dst = seastatus.replace('.tcl', '') + '.before_boot_at' + boot_time
                 if not exists(dst):
                     os.system(f'cp {seastatus} {dst}')
         return start_dir, env
@@ -196,6 +248,8 @@ class SeaManager(ServiceManager):
                     argdict['ins'] = arg
                 else:
                     raise UsageError('unknown argument: %s' % arg)
+            else:
+                print('env variable "InstrumentHostList" is not defined')
         result = [argdict.pop('ins', '')]
         service = argdict.pop('service', '')
         if service:

single.py

@@ -0,0 +1,104 @@
+# *****************************************************************************
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+# Module authors:
+#   Markus Zolliker <markus.zolliker@psi.ch>
+#
+# *****************************************************************************
+
+import os
+from os.path import expanduser
+from configparser import ConfigParser
+from servicemanager.base import ServiceManager
+
+
+class SingleManager(ServiceManager):
+    """service to be started with systemd, only one per computer
+
+    running as systemd (user level)
+    """
+
+    def systemd_service(self, service):
+        """return the name of the systemd service"""
+        raise NotImplementedError
+
+    def treat_args(self, argdict, unknown=(), extra=()):
+        if not argdict.get('ins'):
+            argdict['ins'] = self.single_ins
+        return super().treat_args(argdict, unknown, extra)
+
+    def systemd_action(self, service, action):
+        service_ports = self.get_ins_info(self.single_ins)
+        services = list(service_ports) if service is None else [service]
+        for service in services:
+            print(f'systemctl --user {action} {self.systemd_service(service)}')
+            os.system(f'systemctl --user {action} {self.systemd_service(service)}')
+
+    def do_start(self, ins, service=None, cfg='', restart=False, wait=False, logger=None, opts=''):
+        if wait:
+            super().do_start(ins, service, cfg, False, True, logger, opts)
+            return
+        if restart:
+            action = 'restart'
+        else:
+            action = 'start'
+        self.systemd_action(service, action)
+
+    def do_stop(self, ins, service=None, *args):
+        self.systemd_action(service, 'stop')
+
+
+class FeederManager(SingleManager):
+    group = 'feeder'
+    services = ('central', 'local')
+    USAGE = """
+    Usage:
+
+      feeder start <instance> <service>
+      feeder restart <instance> <service>
+      feeder stop <instance> <service>
+      feeder list [<instance>]
+
+      %s
+    """
+
+    def __init__(self):
+        parser = ConfigParser()
+        parser.read(expanduser('~/.config/sehistory'))
+        services = tuple(s for s in self.services if s in parser.sections())
+        self.services = services
+        super().__init__()
+
+    def systemd_service(self, service):
+        return f'{self.group}@{service}'
+
+
+class SewebManager(SingleManager):
+    group = 'seweb'
+    services = ('',)
+    USAGE = """
+    Usage:
+
+      seweb start <instance>
+      seweb restart <instance>
+      seweb stop <instance>
+      seweb list [<instance>]
+
+      %s
+    """
+
+    def systemd_service(self, service):
+        return f'{self.group}@{self.single_ins}'