diff --git a/__init__.py b/__init__.py index 4f64add..c17b2b5 100644 --- a/__init__.py +++ b/__init__.py @@ -30,7 +30,7 @@ this code is currently used: from servicemanager.base import ServiceManager, ServiceDown, UsageError, get_config from servicemanager.nicosman import NicosManager from servicemanager.seaman import SeaManager -from servicemanager.frappyman import FrappyManager +from servicemanager.frappyman import FrappyManager, Reconnect, Keep class SewebManager(ServiceManager): diff --git a/base.py b/base.py index 1c591c0..37a6430 100644 --- a/base.py +++ b/base.py @@ -146,7 +146,7 @@ class ServiceManager: nr = section.get(self.group) if nr is not None: nr = '%02d' % int(nr) - self.commands[ins] = command + self.commands[ins] = command.replace('~', expanduser('~')) services = self.get_services(section) env = {k: get_subs(section, k, ins, nr) for k in section if k.isupper()} result[ins] = services @@ -156,8 +156,12 @@ class ServiceManager: #def get_cmdpats(self, groups): # return self.cmdpats - def get_ins_info(self, ins): + def get_ins_info(self, ins=None): self.get_info() + if ins is None: + ins = os.environ.get('Instrument') + if ins is None: + return {} return self.info[ins] def get_cfg(self, ins, service): @@ -175,7 +179,7 @@ class ServiceManager: def get_procs(self, groups=None, cfginfo=None): """return processes - :param groups: group to look for or None for all groups + :param groups: group to look for or None for own groups :param cfginfo: cfginfo dict to be populated :result: a dict[ins] of dict[service] of list of tuples (process, cfg) """ @@ -211,6 +215,10 @@ class ServiceManager: return result def wildcard(self, ins): + """return a list of matching instruments + + or None, when no wildcard character in ins + """ if ins is None or ins == 'all': return list(self.info) pat = re.sub(r'(\.|\*)', '.*', ins) @@ -248,13 +256,16 @@ class ServiceManager: continue # already killed for i in range(10): # total 0.1 * 10 * 9 / 2 = 4.5 sec try: - p.wait(0.1 * i) - except psutil.TimeoutExpired: - if p.status() != psutil.STATUS_ZOMBIE: - if print_wait and i > 4: - print('wait for %s %s' % (ins, service)) - print_wait = False - continue + try: + p.wait(0.1 * i) + except psutil.TimeoutExpired: + if p.status() != psutil.STATUS_ZOMBIE: + if print_wait and i > 4: + print('wait for %s %s' % (ins, service)) + print_wait = False + continue + except psutil.NoSuchProcess: + pass # process stopped in the meantime self.stopped[ins][service] = ' '.join(p.info['cmdline']) break else: @@ -325,9 +336,10 @@ class ServiceManager: services = to_start for service_i in services: port = service_ports[service_i] - cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg) + cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg or ins) if opts: cmd = f'{cmd} {opts}' + print(cmd) if '%(cfg)s' in self.commands[ins] and not cfg: cmd = self.stopped[ins].get(service_i) if not cmd: @@ -337,7 +349,7 @@ class ServiceManager: wd = os.getcwd() try: start_dir, env = self.prepare_start(ins, service_i, cfg) - env = dict(os.environ, **env) + env = dict(os.environ, **env, Instrument=ins) os.chdir(start_dir) if start_dir not in sys.path: sys.path.insert(0, start_dir) @@ -414,8 +426,7 @@ class ServiceManager: ins = None instances = self.wildcard(ins) if instances is None: - # ins_set = {ins} - ins_set = set(self.info) + ins_set = {ins} else: ins_set = set(instances) cfginfo = {} @@ -441,7 +452,7 @@ class ServiceManager: plist = procs_dict.get(serv) if plist: cfg = cfginfo.get((ins_i, serv), '') or sm.get_cfg(ins_i, serv) - if sm == self: + if sm == self or instances is None: show_ins = True gs = '%s %s' % (group, serv) port = str(port or '') diff --git a/frappyman.py b/frappyman.py index 97c5abb..5fc2686 100644 --- a/frappyman.py +++ b/frappyman.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # ***************************************************************************** # # This program is free software; you can redistribute it and/or modify it under @@ -26,6 +25,7 @@ import re import builtins from glob import glob from itertools import zip_longest +from collections import defaultdict from os.path import join, isdir, basename, expanduser, exists from configparser import ConfigParser from .base import ServiceManager, ServiceDown, UsageError @@ -65,112 +65,22 @@ class Namespace(dict): SEAEXT = {'main': '.config', 'stick': '.stick'} -def summarize_server_state(givencfgs, ourcfgs, sealist, sea_info, strict=False): - """get a guess for the configuration from information about running services +class Reconnect(str): + """a string tagged with the information that the frappy server is already running - :param givencfgs: dict of given configuration (from nicos cache) - :param outcfgs: dict of running configuration (frappy) - :param sealist: list of running sea configuration [, , , ...] - :param sea_info: dict of - :param strict: when True return empty cfg result on error - :return: tuple (, , (, ), where: - : proposed config not sure - : dict of proposed cfg (cfg is True: no restart needed) - : dict of items runnning in frappy servers (addons are separated) - : dict of items running on the sea server - : dict of actions / remarks + isinstance(cfg, Reconnect) means: cfg may need reconnection, but not restart """ - givencfgs = dict(givencfgs) - addons = givencfgs.pop('addons', '') - for addon in addons.split(','): - addon = addon.strip() - if addon: - givencfgs[addon] = addon - frappycfgs = dict(ourcfgs) - addons = frappycfgs.pop('addons', '') - for addon in addons.split(','): - addon = addon.strip() - if addon: - frappycfgs[addon] = addon + def __repr__(self): + return f'Reconnect({str(self)!r})' - seacfgfiles = [(c or '') + SEAEXT.get(s, '.addon') - for c, s in zip_longest(sealist, FrappyManager.services)] - error = False - result = {} - addons = set() - seacfgs = {} - remarks = {} - inverted = defaultdict(set) - all_of = defaultdict(set) - for cfg, seacfg in sea_info.items(): - inverted[seacfg].add(cfg) - all_of[seacfg.rsplit('.', 1)[-1]].add(cfg) - for seacfg, seacfgfile, key in zip_longest(sealist, seacfgfiles, ('main', 'stick')): - if not seacfg: - continue - available = inverted[seacfg + SEAEXT.get(key, '.addon')] - if available: - proposed = list(available)[0] if len(available) == 1 else None - if not proposed: - running = None - for item in available: - if item == givencfgs.get(key or item): - proposed = item - if item == frappycfgs.get(key or item): - running = item - if running and not proposed: - proposed = running - if proposed: - pkey = key or proposed - given = givencfgs.get(pkey) - running = frappycfgs.get(pkey) - if running == proposed: - remarks[pkey] = '' if given == running else 'reconnect' - elif running: - remarks[pkey] = 'frappy restart needed' - else: - remarks[pkey] = 'frappy start needed' - seacfgs[pkey] = seacfg - if key: - result[key] = proposed - else: - addons.add(proposed) - else: - for item in available: - remarks[item] = 'ambiguous' - seacfgs[item] = seacfg - error = True - else: - pkey = key or seacfg - seacfgs[pkey] = seacfg - remarks[pkey] = 'missing frappy config' - error = True - restart = set() - for key, running in frappycfgs.items(): - if running: - if key in ('main', 'stick'): - service = key - else: - service = 'addons' - addons.add(running) - if not seacfgs.get(key): - if running in all_of[SEAEXT.get(key, '.addon')]: - restart.add(service) - remarks[key] = 'restart to start sea' - elif givencfgs.get(key) != running: - remarks[key] = 'reconnect' - elif remarks.get(key) is None: - remarks[key] = '' - if addons: - result['addons'] = ','.join(addons) +class Keep(Reconnect): + """a string tagged with the information to keep the connection as given - for service, cfg in ourcfgs.items(): - if cfg: - prop = result.get(service, '') - if prop == cfg and service not in restart: - result[service] = True - return error, result, (frappycfgs, seacfgs), remarks + isinstance(cfg, Reconnect) means: the given cfg is already running, so no reconnection is needed + """ + def __repr__(self): + return f'Keep({str(self)!r})' class FrappyManager(ServiceManager): @@ -188,6 +98,17 @@ class FrappyManager(ServiceManager): is one of main, stick, addons %(legend)s """ + # changed in all_info (see docstring there) + frappy2sea = None + sea2frappy = None + list_info = None + # changed in get_server_state: + frappy_cfgs = None + sea_cfgs = None + state = None + error = None + remarks = None + sea = None def config_dirs(self, ins, service): cfgpaths = [] @@ -288,7 +209,9 @@ class FrappyManager(ServiceManager): init(*nodes) interact() - def get_cfg_details(self, namespace, cfgfile): + @staticmethod + def get_cfg_details(namespace, cfgfile): + # get sea_cfg option from frappy cfg file namespace.init() local = {} with open(cfgfile, encoding='utf-8') as f: @@ -310,22 +233,27 @@ class FrappyManager(ServiceManager): except Exception: return False - def all_cfg(self, ins, service, list_info=None, sea_info=None): + def all_cfg(self, ins, service, details=False): """get available cfg files :param ins: instance :param service: service nor None for all services - :param list_info: None or a dict to collect info in the form - dict of of - :param sea_info: None or a dict of with info about sea configs - in frappy cfgs to be populated - :return: set of available config + :param details: get details about relation to sea + :return: see param:`what` + + implicit results: + self.frappy2sea: a dict of (including extension .config/.stick/.addon) + self.sea2frappy: a dict of set of + self.list_info: dict of of """ all_cfg = set() if not ins: return {} namespace = Namespace() - details = sea_info is not None or list_info is not None + if details: + self.frappy2sea = f2s = {} + self.sea2frappy = s2f = {} + self.list_info = list_info = {} for service in [service] if service else self.services: for cfgdir in self.config_dirs(ins, service): for cfgfile in glob(join(cfgdir, '*_cfg.py')): @@ -337,58 +265,45 @@ class FrappyManager(ServiceManager): sea_cfg = None desc = repr(e) if cfg not in all_cfg: - if sea_cfg and sea_info is not None: + if sea_cfg: # sea_info.setdefault(sea_cfg, set()).add(cfg) - sea_info[cfg] = sea_cfg - all_cfg.add(cfg) - if list_info is not None: - list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0] + f2s[cfg] = sea_cfg + s2f.setdefault(sea_cfg, set()).add(cfg) + list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0] + all_cfg.add(cfg) + # if service == 'main': + # sea_info['none.config'] = {''} return all_cfg def do_listcfg(self, ins='', service='', prt=print): if not ins: raise UsageError('missing instance') - list_info = {} - sea_info = {} - if service: - self.all_cfg(ins, service, list_info, sea_info) - else: - for service in self.services: - self.all_cfg(ins, service, list_info, sea_info) + self.all_cfg(ins, service, True) seacfgpat = re.compile(r'(.*)(\.config|\.stick|\.addon)') - # inverted_sea_info = {} - # for seacfg, cfgset in sea_info.items(): - # for cfg in cfgset: - # inverted_sea_info[cfg] = seacfg - cfgset = set() - ambiguous = {} - for cfg, seacfg in sea_info.items(): - if cfg in cfgset: - ambiguous[cfg] = ambiguous.get(cfg, 1) + 1 - else: - cfgset.add(cfg) - keylen = max(max(len(k) for k in cfgs) for cfgs in list_info.values()) - for cfgdir, cfgs in list_info.items(): + keylen = max((max(len(k) for k in cfgs) for cfgs in self.list_info.values()), default=1) + ambiguous = set() + for cfgdir, cfgs in self.list_info.items(): if cfgs: prt('') prt('--- %s:' % cfgdir) for cfg, desc in sorted(cfgs.items(), key=lambda v: (v[0].lower(), v)): - seacfg = sea_info.get(cfg) + seacfg = self.frappy2sea.get(cfg) if seacfg: name, ext = seacfgpat.match(seacfg).groups() if name == cfg or name + 'stick' == cfg: prefix = '* ' else: prefix = f'* ({name}{ext}) ' - if cfg in ambiguous: + if len(self.sea2frappy[seacfg]) > 1: prefix = '!' + prefix[1:] + ambiguous.add(seacfg) desc = prefix + desc prt('%s %s' % (cfg.ljust(keylen), desc)) prt(' ') gap = ' ' * keylen prt(f'{gap} * need sea') if ambiguous: - print(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy') + prt(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy') def treat_args(self, argdict, unknown=(), extra=()): cfg = None @@ -410,7 +325,7 @@ class FrappyManager(ServiceManager): return super().treat_args(argdict, unknown, extra) def check_cfg_file(self, ins, service, cfg, needsea=False): - if cfg is 'none': + if cfg == 'none': return '' try: desc, sea_cfg = self.cfg_details(self, ins, service, cfg) @@ -425,19 +340,137 @@ class FrappyManager(ServiceManager): :param ins: the instance to be checked for :param givencfgs: a dict of cfg given by the ECS + :return: a dict of cfg, where cfg is either: + - a bare string: this cfg is proposed to change to this value + - Reconnect(cfg): the frappy server is running as expected, but the given cfg does not match + - Keep(cfg): no change needed + remark: Reconnect amd Keep inherit from str, so Reconnect(cfg) == cfg is always True - :return: tuple (, , (, ), where: - : proposed config not sure - : dict of proposed cfg - : dict of items runnning in frappy servers (addons are separated) - : dict of items running on the sea server - : dict of actions to do / remarks + implicit results: + self.remarks: a dict of remark (why should this be changed?) + self.frappy_cfgs: a dict of running cfgs + self.sea_cfgs: a dict of sea cfgs (without ending .config/.stick/.addon) + self.state: a dict ('sea ' | 'frappy ') of cfg summarizing the state of all servers + a change of self.state indicates that the configuration may need to be reevaluated + self.error: there is an ambiguity for the mapping seacfg -> frappycfg + self.sea: a fresh SeaManager instance """ - ourcfgs = self.get_cfg(ins, None) - sea = SeaManager() - seacfgs = sea.get_cfg(ins, 'sea').split('/') - if len(seacfgs) < 2: - seacfgs.append('') - sea_info = {} - self.all_cfg(ins, None, sea_info=sea_info) - return summarize_server_state(givencfgs, ourcfgs, seacfgs, sea_info) + self.frappy_cfgs = self.get_cfg(ins, None) # dict of running frappy servers + self.sea = SeaManager() + seaconfig = self.sea.get_cfg(ins, 'sea') + sealist = seaconfig.split('/') # , , , ... + if len(sealist) < 2: + sealist.append('') + + all_cfg = self.all_cfg(ins, None, True) + proposed_cfg = {} + self.sea_cfgs = {} + self.remarks = {} + self.error = False + self.state = {} + result = {} + for (service, ext), seacfg in zip_longest(SEAEXT.items(), sealist[0:2], fillvalue=(None, None)): + if service: + self.sea_cfgs[service] = seacfg + proposed = self.sea2frappy.get(seacfg + ext) or set() + if seaconfig: + proposed_cfg[service] = proposed + self.state[f'sea {service}'] = seacfg + + running_addons = self.frappy_cfgs.get('addons') + running_addons = [v.strip() for v in running_addons.split(',') if v.strip()] + proposed_addons = [] # use list instead of set for keeping order + running_sea_addons = set() + for cfg in running_addons: + seacfg = self.frappy2sea.get(cfg.strip()) + if seacfg is None: + proposed_addons.append(cfg) # addons with no sea cfg should be kept + elif seacfg.endswith('.addon') and seacfg[:-6] in sealist[2:]: + proposed_addons.append(cfg) + running_sea_addons.add(seacfg) + + for scfg in sealist[2:]: + seacfg = scfg + '.addon' + if seacfg not in running_sea_addons: + proposed = self.sea2frappy.get(seacfg) or set() + if proposed: + if len(proposed) > 1: + self.error = f'ambiguous frappy cfg for {seacfg}.addon: {proposed}' + else: + proposed = list(proposed)[0] + if proposed not in proposed_addons: + proposed_addons.append(proposed) + self.sea_cfgs['addons'] = ','.join(sealist[2:]) + if proposed_addons: # and set(proposed_addons) != set(running_addons): + proposed_cfg['addons'] = {','.join(proposed_addons)} + + self._debug = {} + for service in FrappyManager.services: + given = givencfgs.get(service) + running = self.frappy_cfgs.get(service) + available = proposed_cfg.get(service) + + if running: + self.state[f'frappy {service}'] = running + + self._debug[service] = (seaconfig, available, running, running in self.frappy2sea) + + if seaconfig and (available or running in self.frappy2sea): + # we get here when the sea server is running and either at least one of: + # - the sea config is matching any frappy cfg + # - the running frappy cfg is matching a sea config + if not available: + self.remarks[service] = 'no frappy config' + continue + proposed = list(available)[0] if len(available) == 1 else None + if proposed is None: + running = None + for item in available: + if item == given: + proposed = item + if item == running: + running = item + if running and not proposed: + proposed = running + if proposed is None: + self.remarks[service] = 'ambiguous frappy cfg' + self.error = f'ambiguous frappy cfg for {seacfg}: {available}' + else: + self.remarks[service] = '' + if running == proposed: + if given == running: + result[service] = Keep(running) + elif running: + result[service] = proposed + self.remarks[service] = 'restart frappy' + else: + result[service] = proposed + else: + # we get here when sea is not running or all of: + # - the sea config has no matching frappy cfg + # - the frappy cfg has no matching sea config + if running: + if self.frappy2sea.get(running, '').endswith(ext): + result[service] = running + self.remarks[service] = 'restart frappy to start sea' + elif given != running: + result[service] = Reconnect(running) + self.remarks[service] = 'reconnect needed' + else: + result[service] = Keep(running) + self.remarks[service] = '' + elif given: + if self.frappy2sea.get(given, '').endswith(ext): + result[service] = running + self.remarks[service] = 'restart frappy and sea' + elif ':' in given: + self.remarks[service] = 'external device' + elif given in all_cfg: + result[service] = running + self.remarks[service] = 'restart frappy' + else: + result[service] = '' + self.remarks[service] = f'unknown frappy cfg {given}' + + return result + diff --git a/nicosman.py b/nicosman.py index ec3c6a7..7799cdc 100644 --- a/nicosman.py +++ b/nicosman.py @@ -261,6 +261,17 @@ class NicosManager(ServiceManager): def do_gui(self, ins): self.prepare_client(ins) + + # patch QApplication to use 'nicos_' instead of 'nicos' as organizationName + # for different settings and log files + import nicos.guisupport.qt + + class QApplication(nicos.guisupport.qt.QApplication): + def __init__(self, *args, organizationName=None, **kwds): + super().__init__(*args, organizationName=f'nicos_{ins}', **kwds) + + nicos.guisupport.qt.QApplication = QApplication + from nicos.clients.gui.main import main print('starting nicos gui %s' % ins) try: