include changes from tmp branch

__init__.py

@@ -30,7 +30,7 @@ this code is currently used:
 from servicemanager.base import ServiceManager, ServiceDown, UsageError, get_config
 from servicemanager.nicosman import NicosManager
 from servicemanager.seaman import SeaManager
-from servicemanager.frappyman import FrappyManager
+from servicemanager.frappyman import FrappyManager, Reconnect, Keep
 
 
 class SewebManager(ServiceManager):

base.py

@@ -146,7 +146,7 @@ class ServiceManager:
             nr = section.get(self.group)
             if nr is not None:
                 nr = '%02d' % int(nr)
-                self.commands[ins] = command
+                self.commands[ins] = command.replace('~', expanduser('~'))
                 services = self.get_services(section)
                 env = {k: get_subs(section, k, ins, nr) for k in section if k.isupper()}
                 result[ins] = services
@@ -156,8 +156,12 @@ class ServiceManager:
     #def get_cmdpats(self, groups):
     #    return self.cmdpats
 
-    def get_ins_info(self, ins):
+    def get_ins_info(self, ins=None):
         self.get_info()
+        if ins is None:
+            ins = os.environ.get('Instrument')
+            if ins is None:
+                return {}
         return self.info[ins]
 
     def get_cfg(self, ins, service):
@@ -175,7 +179,7 @@ class ServiceManager:
     def get_procs(self, groups=None, cfginfo=None):
         """return processes
 
-        :param groups: group to look for or None for all groups
+        :param groups: group to look for or None for own groups
         :param cfginfo: cfginfo dict to be populated
         :result: a dict[ins] of dict[service] of list of tuples (process, cfg)
         """
@@ -211,6 +215,10 @@ class ServiceManager:
         return result
 
     def wildcard(self, ins):
+        """return a list of matching instruments
+
+        or None, when no wildcard character in ins
+        """
         if ins is None or ins == 'all':
             return list(self.info)
         pat = re.sub(r'(\.|\*)', '.*', ins)
@@ -248,13 +256,16 @@ class ServiceManager:
                         continue  # already killed
                     for i in range(10):  # total 0.1 * 10 * 9 / 2 = 4.5 sec
                         try:
-                            p.wait(0.1 * i)
+                            try:
-                        except psutil.TimeoutExpired:
+                                p.wait(0.1 * i)
-                            if p.status() != psutil.STATUS_ZOMBIE:
+                            except psutil.TimeoutExpired:
-                                if print_wait and i > 4:
+                                if p.status() != psutil.STATUS_ZOMBIE:
-                                    print('wait for %s %s' % (ins, service))
+                                    if print_wait and i > 4:
-                                    print_wait = False
+                                        print('wait for %s %s' % (ins, service))
-                                continue
+                                        print_wait = False
+                                    continue
+                        except psutil.NoSuchProcess:
+                            pass  # process stopped in the meantime
                         self.stopped[ins][service] = ' '.join(p.info['cmdline'])
                         break
                     else:
@@ -325,9 +336,10 @@ class ServiceManager:
             services = to_start
         for service_i in services:
             port = service_ports[service_i]
-            cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg)
+            cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg or ins)
             if opts:
                 cmd = f'{cmd} {opts}'
+            print(cmd)
             if '%(cfg)s' in self.commands[ins] and not cfg:
                 cmd = self.stopped[ins].get(service_i)
                 if not cmd:
@@ -337,7 +349,7 @@ class ServiceManager:
             wd = os.getcwd()
             try:
                 start_dir, env = self.prepare_start(ins, service_i, cfg)
-                env = dict(os.environ, **env)
+                env = dict(os.environ, **env, Instrument=ins)
                 os.chdir(start_dir)
                 if start_dir not in sys.path:
                     sys.path.insert(0, start_dir)
@@ -414,8 +426,7 @@ class ServiceManager:
             ins = None
         instances = self.wildcard(ins)
         if instances is None:
-            # ins_set = {ins}
+            ins_set = {ins}
-            ins_set = set(self.info)
         else:
             ins_set = set(instances)
         cfginfo = {}
@@ -441,7 +452,7 @@ class ServiceManager:
                     plist = procs_dict.get(serv)
                     if plist:
                         cfg = cfginfo.get((ins_i, serv), '') or sm.get_cfg(ins_i, serv)
-                        if sm == self:
+                        if sm == self or instances is None:
                             show_ins = True
                         gs = '%s %s' % (group, serv)
                         port = str(port or '')

frappyman.py

@@ -1,4 +1,3 @@
-#  -*- coding: utf-8 -*-
 # *****************************************************************************
 #
 # This program is free software; you can redistribute it and/or modify it under
@@ -26,6 +25,7 @@ import re
 import builtins
 from glob import glob
 from itertools import zip_longest
+from collections import defaultdict
 from os.path import join, isdir, basename, expanduser, exists
 from configparser import ConfigParser
 from .base import ServiceManager, ServiceDown, UsageError
@@ -65,112 +65,22 @@ class Namespace(dict):
 SEAEXT = {'main': '.config', 'stick': '.stick'}
 
 
-def summarize_server_state(givencfgs, ourcfgs, sealist, sea_info, strict=False):
+class Reconnect(str):
-    """get a guess for the configuration from information about running services
+    """a string tagged with the information that the frappy server is already running
 
-    :param givencfgs:  dict <service> of given configuration (from nicos cache)
+    isinstance(cfg, Reconnect) means: cfg may need reconnection, but not restart
-    :param outcfgs: dict <service> of running configuration (frappy)
-    :param sealist: list of running sea configuration [<config>, <stick>, <addon1>, <addon2> ...]
-    :param sea_info: dict <seacfg> of <frappycfg>
-    :param strict: when True return empty cfg result on error
-    :return: tuple (<error>, <proposed>, (<frappyitems>, <seaitems>), <remarks> where:
-        <error>: proposed config not sure
-        <proposed>: dict <service> of proposed cfg (cfg is True: no restart needed)
-        <frappyitems>: dict of items runnning in frappy servers (addons are separated)
-        <seaitems>: dict of items running on the sea server
-        <remarks>: dict of actions / remarks
     """
-    givencfgs = dict(givencfgs)
+    def __repr__(self):
-    addons = givencfgs.pop('addons', '')
+        return f'Reconnect({str(self)!r})'
-    for addon in addons.split(','):
-        addon = addon.strip()
-        if addon:
-            givencfgs[addon] = addon
-    frappycfgs = dict(ourcfgs)
-    addons = frappycfgs.pop('addons', '')
-    for addon in addons.split(','):
-        addon = addon.strip()
-        if addon:
-            frappycfgs[addon] = addon
 
-    seacfgfiles = [(c or '') + SEAEXT.get(s, '.addon')
-                   for c, s in zip_longest(sealist, FrappyManager.services)]
-    error = False
-    result = {}
-    addons = set()
-    seacfgs = {}
-    remarks = {}
-    inverted = defaultdict(set)
-    all_of = defaultdict(set)
-    for cfg, seacfg in sea_info.items():
-        inverted[seacfg].add(cfg)
-        all_of[seacfg.rsplit('.', 1)[-1]].add(cfg)
-    for seacfg, seacfgfile, key in zip_longest(sealist, seacfgfiles, ('main', 'stick')):
-        if not seacfg:
-            continue
-        available = inverted[seacfg + SEAEXT.get(key, '.addon')]
-        if available:
-            proposed = list(available)[0] if len(available) == 1 else None
-            if not proposed:
-                running = None
-                for item in available:
-                    if item == givencfgs.get(key or item):
-                        proposed = item
-                    if item == frappycfgs.get(key or item):
-                        running = item
-                if running and not proposed:
-                    proposed = running
-            if proposed:
-                pkey = key or proposed
-                given = givencfgs.get(pkey)
-                running = frappycfgs.get(pkey)
-                if running == proposed:
-                    remarks[pkey] = '' if given == running else 'reconnect'
-                elif running:
-                    remarks[pkey] = 'frappy restart needed'
-                else:
-                    remarks[pkey] = 'frappy start needed'
 
-                seacfgs[pkey] = seacfg
+class Keep(Reconnect):
-                if key:
+    """a string tagged with the information to keep the connection as given
-                    result[key] = proposed
-                else:
-                    addons.add(proposed)
-            else:
-                for item in available:
-                    remarks[item] = 'ambiguous'
-                    seacfgs[item] = seacfg
-                error = True
-        else:
-            pkey = key or seacfg
-            seacfgs[pkey] = seacfg
-            remarks[pkey] = 'missing frappy config'
-            error = True
-    restart = set()
-    for key, running in frappycfgs.items():
-        if running:
-            if key in ('main', 'stick'):
-                service = key
-            else:
-                service = 'addons'
-                addons.add(running)
-            if not seacfgs.get(key):
-                if running in all_of[SEAEXT.get(key, '.addon')]:
-                    restart.add(service)
-                    remarks[key] = 'restart to start sea'
-                elif givencfgs.get(key) != running:
-                    remarks[key] = 'reconnect'
-                elif remarks.get(key) is None:
-                    remarks[key] = ''
-    if addons:
-        result['addons'] = ','.join(addons)
 
-    for service, cfg in ourcfgs.items():
+    isinstance(cfg, Reconnect) means: the given cfg is already running, so no reconnection is needed
-        if cfg:
+    """
-            prop = result.get(service, '')
+    def __repr__(self):
-            if prop == cfg and service not in restart:
+        return f'Keep({str(self)!r})'
-                result[service] = True
-    return error, result, (frappycfgs, seacfgs), remarks
 
 
 class FrappyManager(ServiceManager):
@@ -188,6 +98,17 @@ class FrappyManager(ServiceManager):
       <service> is one of main, stick, addons
 %(legend)s
     """
+    # changed in all_info (see docstring there)
+    frappy2sea = None
+    sea2frappy = None
+    list_info = None
+    # changed in get_server_state:
+    frappy_cfgs = None
+    sea_cfgs = None
+    state = None
+    error = None
+    remarks = None
+    sea = None
 
     def config_dirs(self, ins, service):
         cfgpaths = []
@@ -288,7 +209,9 @@ class FrappyManager(ServiceManager):
         init(*nodes)
         interact()
 
-    def get_cfg_details(self, namespace, cfgfile):
+    @staticmethod
+    def get_cfg_details(namespace, cfgfile):
+        # get sea_cfg option from frappy cfg file
         namespace.init()
         local = {}
         with open(cfgfile, encoding='utf-8') as f:
@@ -310,22 +233,27 @@ class FrappyManager(ServiceManager):
         except Exception:
             return False
 
-    def all_cfg(self, ins, service, list_info=None, sea_info=None):
+    def all_cfg(self, ins, service, details=False):
         """get available cfg files
 
         :param ins: instance
         :param service: service nor None for all services
-        :param list_info: None or a dict to collect info in the form
+        :param details: get details about relation to sea
-            dict <cfgdir> of <cfg> of <description>
+        :return: see param:`what`
-        :param sea_info: None or a dict <frappycfg> of <seacfg> with info about sea configs
+
-                         in frappy cfgs to be populated
+        implicit results:
-        :return: set of available config
+        self.frappy2sea: a dict <frappycfg> of <seacfg> (including extension .config/.stick/.addon)
+        self.sea2frappy: a dict <seacfg> of set of <frappycfg>
+        self.list_info: dict <cfgdir> of <cfg> of <description>
         """
         all_cfg = set()
         if not ins:
             return {}
         namespace = Namespace()
-        details = sea_info is not None or list_info is not None
+        if details:
+            self.frappy2sea = f2s = {}
+            self.sea2frappy = s2f = {}
+            self.list_info = list_info = {}
         for service in [service] if service else self.services:
             for cfgdir in self.config_dirs(ins, service):
                 for cfgfile in glob(join(cfgdir, '*_cfg.py')):
@@ -337,58 +265,45 @@ class FrappyManager(ServiceManager):
                             sea_cfg = None
                             desc = repr(e)
                         if cfg not in all_cfg:
-                            if sea_cfg and sea_info is not None:
+                            if sea_cfg:
                                 # sea_info.setdefault(sea_cfg, set()).add(cfg)
-                                sea_info[cfg] = sea_cfg
+                                f2s[cfg] = sea_cfg
-                            all_cfg.add(cfg)
+                                s2f.setdefault(sea_cfg, set()).add(cfg)
-                            if list_info is not None:
+                            list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0]
-                                list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0]
+                    all_cfg.add(cfg)
+            # if service == 'main':
+            #    sea_info['none.config'] = {''}
         return all_cfg
 
     def do_listcfg(self, ins='', service='', prt=print):
         if not ins:
              raise UsageError('missing instance')
-        list_info = {}
+        self.all_cfg(ins, service, True)
-        sea_info = {}
-        if service:
-            self.all_cfg(ins, service, list_info, sea_info)
-        else:
-            for service in self.services:
-                self.all_cfg(ins, service, list_info, sea_info)
         seacfgpat = re.compile(r'(.*)(\.config|\.stick|\.addon)')
-        # inverted_sea_info = {}
+        keylen = max((max(len(k) for k in cfgs) for cfgs in self.list_info.values()), default=1)
-        # for seacfg, cfgset in sea_info.items():
+        ambiguous = set()
-        #     for cfg in cfgset:
+        for cfgdir, cfgs in self.list_info.items():
-        #         inverted_sea_info[cfg] = seacfg
-        cfgset = set()
-        ambiguous = {}
-        for cfg, seacfg in sea_info.items():
-            if cfg in cfgset:
-                ambiguous[cfg] = ambiguous.get(cfg, 1) + 1
-            else:
-                cfgset.add(cfg)
-        keylen = max(max(len(k) for k in cfgs) for cfgs in list_info.values())
-        for cfgdir, cfgs in list_info.items():
             if cfgs:
                 prt('')
                 prt('--- %s:' % cfgdir)
                 for cfg, desc in sorted(cfgs.items(), key=lambda v: (v[0].lower(), v)):
-                    seacfg = sea_info.get(cfg)
+                    seacfg = self.frappy2sea.get(cfg)
                     if seacfg:
                         name, ext = seacfgpat.match(seacfg).groups()
                         if name == cfg or name + 'stick' == cfg:
                             prefix = '* '
                         else:
                             prefix = f'* ({name}{ext}) '
-                        if cfg in ambiguous:
+                        if len(self.sea2frappy[seacfg]) > 1:
                             prefix = '!' + prefix[1:]
+                            ambiguous.add(seacfg)
                         desc = prefix + desc
                     prt('%s %s' % (cfg.ljust(keylen), desc))
         prt(' ')
         gap = ' ' * keylen
         prt(f'{gap} * need sea')
         if ambiguous:
-            print(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy')
+            prt(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy')
 
     def treat_args(self, argdict, unknown=(), extra=()):
         cfg = None
@@ -410,7 +325,7 @@ class FrappyManager(ServiceManager):
         return super().treat_args(argdict, unknown, extra)
 
     def check_cfg_file(self, ins, service, cfg, needsea=False):
-        if cfg is 'none':
+        if cfg == 'none':
             return ''
         try:
             desc, sea_cfg = self.cfg_details(self, ins, service, cfg)
@@ -425,19 +340,137 @@ class FrappyManager(ServiceManager):
 
         :param ins: the instance to be checked for
         :param givencfgs: a dict <service> of cfg given by the ECS
+        :return: a dict <service> of cfg, where cfg is either:
+                - a bare string: this cfg is proposed to change to this value
+                - Reconnect(cfg): the frappy server is running as expected, but the given cfg does not match
+                - Keep(cfg): no change needed
+                remark: Reconnect amd Keep inherit from str, so Reconnect(cfg) == cfg is always True
 
-        :return: tuple (<error>, <proposed>, (<frappyitems>, <seaitems>), <remarks> where:
+        implicit results:
-            <error>: proposed config not sure
+        self.remarks: a dict <service> of remark (why should this be changed?)
-            <proposed>: dict <service> of proposed cfg
+        self.frappy_cfgs: a dict <service> of running cfgs
-            <frappyitems>: dict of items runnning in frappy servers (addons are separated)
+        self.sea_cfgs: a dict <service> of sea cfgs (without ending .config/.stick/.addon)
-            <seaitems>: dict of items running on the sea server
+        self.state: a dict ('sea <service>' | 'frappy <service>') of cfg summarizing the state of all servers
-            <remarks>: dict of actions to do / remarks
+                    a change of self.state indicates that the configuration may need to be reevaluated
+        self.error: there is an ambiguity for the mapping seacfg -> frappycfg
+        self.sea: a fresh SeaManager instance
         """
-        ourcfgs = self.get_cfg(ins, None)
+        self.frappy_cfgs = self.get_cfg(ins, None)  # dict <service> of running frappy servers
-        sea = SeaManager()
+        self.sea = SeaManager()
-        seacfgs = sea.get_cfg(ins, 'sea').split('/')
+        seaconfig = self.sea.get_cfg(ins, 'sea')
-        if len(seacfgs) < 2:
+        sealist = seaconfig.split('/')  # <config>, <stick>, <addon1>, <addon2> ...
-            seacfgs.append('')
+        if len(sealist) < 2:
-        sea_info = {}
+            sealist.append('')
-        self.all_cfg(ins, None, sea_info=sea_info)
+
-        return summarize_server_state(givencfgs, ourcfgs, seacfgs, sea_info)
+        all_cfg = self.all_cfg(ins, None, True)
+        proposed_cfg = {}
+        self.sea_cfgs = {}
+        self.remarks = {}
+        self.error = False
+        self.state = {}
+        result = {}
+        for (service, ext), seacfg in zip_longest(SEAEXT.items(), sealist[0:2], fillvalue=(None, None)):
+            if service:
+                self.sea_cfgs[service] = seacfg
+                proposed = self.sea2frappy.get(seacfg + ext) or set()
+                if seaconfig:
+                    proposed_cfg[service] = proposed
+                    self.state[f'sea {service}'] = seacfg
+
+        running_addons = self.frappy_cfgs.get('addons')
+        running_addons = [v.strip() for v in running_addons.split(',') if v.strip()]
+        proposed_addons = []  # use list instead of set for keeping order
+        running_sea_addons = set()
+        for cfg in running_addons:
+            seacfg = self.frappy2sea.get(cfg.strip())
+            if seacfg is None:
+                proposed_addons.append(cfg)  # addons with no sea cfg should be kept
+            elif seacfg.endswith('.addon') and seacfg[:-6] in sealist[2:]:
+                proposed_addons.append(cfg)
+                running_sea_addons.add(seacfg)
+
+        for scfg in sealist[2:]:
+            seacfg = scfg + '.addon'
+            if seacfg not in running_sea_addons:
+                proposed = self.sea2frappy.get(seacfg) or set()
+                if proposed:
+                    if len(proposed) > 1:
+                        self.error = f'ambiguous frappy cfg for {seacfg}.addon: {proposed}'
+                    else:
+                        proposed = list(proposed)[0]
+                        if proposed not in proposed_addons:
+                            proposed_addons.append(proposed)
+        self.sea_cfgs['addons'] = ','.join(sealist[2:])
+        if proposed_addons:  # and set(proposed_addons) != set(running_addons):
+            proposed_cfg['addons'] = {','.join(proposed_addons)}
+
+        self._debug = {}
+        for service in FrappyManager.services:
+            given = givencfgs.get(service)
+            running = self.frappy_cfgs.get(service)
+            available = proposed_cfg.get(service)
+
+            if running:
+                self.state[f'frappy {service}'] = running
+
+            self._debug[service] = (seaconfig, available, running, running in self.frappy2sea)
+
+            if seaconfig and (available or running in self.frappy2sea):
+                # we get here when the sea server is running and either at least one of:
+                # - the sea config is matching any frappy cfg
+                # - the running frappy cfg is matching a sea config
+                if not available:
+                    self.remarks[service] = 'no frappy config'
+                    continue
+                proposed = list(available)[0] if len(available) == 1 else None
+                if proposed is None:
+                    running = None
+                    for item in available:
+                        if item == given:
+                            proposed = item
+                        if item == running:
+                            running = item
+                    if running and not proposed:
+                        proposed = running
+                if proposed is None:
+                    self.remarks[service] = 'ambiguous frappy cfg'
+                    self.error = f'ambiguous frappy cfg for {seacfg}: {available}'
+                else:
+                    self.remarks[service] = ''
+                    if running == proposed:
+                        if given == running:
+                            result[service] = Keep(running)
+                    elif running:
+                        result[service] = proposed
+                        self.remarks[service] = 'restart frappy'
+                    else:
+                        result[service] = proposed
+            else:
+                # we get here when sea is not running or all of:
+                # - the sea config has no matching frappy cfg
+                # - the frappy cfg has no matching sea config
+                if running:
+                    if self.frappy2sea.get(running, '').endswith(ext):
+                        result[service] = running
+                        self.remarks[service] = 'restart frappy to start sea'
+                    elif given != running:
+                        result[service] = Reconnect(running)
+                        self.remarks[service] = 'reconnect needed'
+                    else:
+                        result[service] = Keep(running)
+                        self.remarks[service] = ''
+                elif given:
+                    if self.frappy2sea.get(given, '').endswith(ext):
+                        result[service] = running
+                        self.remarks[service] = 'restart frappy and sea'
+                    elif ':' in given:
+                        self.remarks[service] = 'external device'
+                    elif given in all_cfg:
+                        result[service] = running
+                        self.remarks[service] = 'restart frappy'
+                    else:
+                        result[service] = ''
+                        self.remarks[service] = f'unknown frappy cfg {given}'
+
+        return result
+

nicosman.py

@@ -261,6 +261,17 @@ class NicosManager(ServiceManager):
 
     def do_gui(self, ins):
         self.prepare_client(ins)
+
+        # patch QApplication to use 'nicos_<instr>' instead of 'nicos' as organizationName
+        # for different settings and log files
+        import nicos.guisupport.qt
+
+        class QApplication(nicos.guisupport.qt.QApplication):
+            def __init__(self, *args, organizationName=None, **kwds):
+                super().__init__(*args, organizationName=f'nicos_{ins}', **kwds)
+
+        nicos.guisupport.qt.QApplication = QApplication
+
         from nicos.clients.gui.main import main
         print('starting nicos gui %s' % ins)
         try: