include changes from tmp branch

This commit is contained in:
l_samenv 2024-06-10 10:19:22 +02:00
parent 3d8f220182
commit e31d0248d7
4 changed files with 224 additions and 169 deletions

View File

@ -30,7 +30,7 @@ this code is currently used:
from servicemanager.base import ServiceManager, ServiceDown, UsageError, get_config
from servicemanager.nicosman import NicosManager
from servicemanager.seaman import SeaManager
from servicemanager.frappyman import FrappyManager
from servicemanager.frappyman import FrappyManager, Reconnect, Keep
class SewebManager(ServiceManager):

27
base.py
View File

@ -146,7 +146,7 @@ class ServiceManager:
nr = section.get(self.group)
if nr is not None:
nr = '%02d' % int(nr)
self.commands[ins] = command
self.commands[ins] = command.replace('~', expanduser('~'))
services = self.get_services(section)
env = {k: get_subs(section, k, ins, nr) for k in section if k.isupper()}
result[ins] = services
@ -156,8 +156,12 @@ class ServiceManager:
#def get_cmdpats(self, groups):
# return self.cmdpats
def get_ins_info(self, ins):
def get_ins_info(self, ins=None):
self.get_info()
if ins is None:
ins = os.environ.get('Instrument')
if ins is None:
return {}
return self.info[ins]
def get_cfg(self, ins, service):
@ -175,7 +179,7 @@ class ServiceManager:
def get_procs(self, groups=None, cfginfo=None):
"""return processes
:param groups: group to look for or None for all groups
:param groups: group to look for or None for own groups
:param cfginfo: cfginfo dict to be populated
:result: a dict[ins] of dict[service] of list of tuples (process, cfg)
"""
@ -211,6 +215,10 @@ class ServiceManager:
return result
def wildcard(self, ins):
"""return a list of matching instruments
or None, when no wildcard character in ins
"""
if ins is None or ins == 'all':
return list(self.info)
pat = re.sub(r'(\.|\*)', '.*', ins)
@ -247,6 +255,7 @@ class ServiceManager:
except psutil.NoSuchProcess:
continue # already killed
for i in range(10): # total 0.1 * 10 * 9 / 2 = 4.5 sec
try:
try:
p.wait(0.1 * i)
except psutil.TimeoutExpired:
@ -255,6 +264,8 @@ class ServiceManager:
print('wait for %s %s' % (ins, service))
print_wait = False
continue
except psutil.NoSuchProcess:
pass # process stopped in the meantime
self.stopped[ins][service] = ' '.join(p.info['cmdline'])
break
else:
@ -325,9 +336,10 @@ class ServiceManager:
services = to_start
for service_i in services:
port = service_ports[service_i]
cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg)
cmd = self.commands[ins] % dict(ins=ins, serv=service_i, port=port, cfg=cfg, pkg=self.pkg or ins)
if opts:
cmd = f'{cmd} {opts}'
print(cmd)
if '%(cfg)s' in self.commands[ins] and not cfg:
cmd = self.stopped[ins].get(service_i)
if not cmd:
@ -337,7 +349,7 @@ class ServiceManager:
wd = os.getcwd()
try:
start_dir, env = self.prepare_start(ins, service_i, cfg)
env = dict(os.environ, **env)
env = dict(os.environ, **env, Instrument=ins)
os.chdir(start_dir)
if start_dir not in sys.path:
sys.path.insert(0, start_dir)
@ -414,8 +426,7 @@ class ServiceManager:
ins = None
instances = self.wildcard(ins)
if instances is None:
# ins_set = {ins}
ins_set = set(self.info)
ins_set = {ins}
else:
ins_set = set(instances)
cfginfo = {}
@ -441,7 +452,7 @@ class ServiceManager:
plist = procs_dict.get(serv)
if plist:
cfg = cfginfo.get((ins_i, serv), '') or sm.get_cfg(ins_i, serv)
if sm == self:
if sm == self or instances is None:
show_ins = True
gs = '%s %s' % (group, serv)
port = str(port or '')

View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# *****************************************************************************
#
# This program is free software; you can redistribute it and/or modify it under
@ -26,6 +25,7 @@ import re
import builtins
from glob import glob
from itertools import zip_longest
from collections import defaultdict
from os.path import join, isdir, basename, expanduser, exists
from configparser import ConfigParser
from .base import ServiceManager, ServiceDown, UsageError
@ -65,112 +65,22 @@ class Namespace(dict):
SEAEXT = {'main': '.config', 'stick': '.stick'}
def summarize_server_state(givencfgs, ourcfgs, sealist, sea_info, strict=False):
"""get a guess for the configuration from information about running services
class Reconnect(str):
"""a string tagged with the information that the frappy server is already running
:param givencfgs: dict <service> of given configuration (from nicos cache)
:param outcfgs: dict <service> of running configuration (frappy)
:param sealist: list of running sea configuration [<config>, <stick>, <addon1>, <addon2> ...]
:param sea_info: dict <seacfg> of <frappycfg>
:param strict: when True return empty cfg result on error
:return: tuple (<error>, <proposed>, (<frappyitems>, <seaitems>), <remarks> where:
<error>: proposed config not sure
<proposed>: dict <service> of proposed cfg (cfg is True: no restart needed)
<frappyitems>: dict of items runnning in frappy servers (addons are separated)
<seaitems>: dict of items running on the sea server
<remarks>: dict of actions / remarks
isinstance(cfg, Reconnect) means: cfg may need reconnection, but not restart
"""
givencfgs = dict(givencfgs)
addons = givencfgs.pop('addons', '')
for addon in addons.split(','):
addon = addon.strip()
if addon:
givencfgs[addon] = addon
frappycfgs = dict(ourcfgs)
addons = frappycfgs.pop('addons', '')
for addon in addons.split(','):
addon = addon.strip()
if addon:
frappycfgs[addon] = addon
def __repr__(self):
return f'Reconnect({str(self)!r})'
seacfgfiles = [(c or '') + SEAEXT.get(s, '.addon')
for c, s in zip_longest(sealist, FrappyManager.services)]
error = False
result = {}
addons = set()
seacfgs = {}
remarks = {}
inverted = defaultdict(set)
all_of = defaultdict(set)
for cfg, seacfg in sea_info.items():
inverted[seacfg].add(cfg)
all_of[seacfg.rsplit('.', 1)[-1]].add(cfg)
for seacfg, seacfgfile, key in zip_longest(sealist, seacfgfiles, ('main', 'stick')):
if not seacfg:
continue
available = inverted[seacfg + SEAEXT.get(key, '.addon')]
if available:
proposed = list(available)[0] if len(available) == 1 else None
if not proposed:
running = None
for item in available:
if item == givencfgs.get(key or item):
proposed = item
if item == frappycfgs.get(key or item):
running = item
if running and not proposed:
proposed = running
if proposed:
pkey = key or proposed
given = givencfgs.get(pkey)
running = frappycfgs.get(pkey)
if running == proposed:
remarks[pkey] = '' if given == running else 'reconnect'
elif running:
remarks[pkey] = 'frappy restart needed'
else:
remarks[pkey] = 'frappy start needed'
seacfgs[pkey] = seacfg
if key:
result[key] = proposed
else:
addons.add(proposed)
else:
for item in available:
remarks[item] = 'ambiguous'
seacfgs[item] = seacfg
error = True
else:
pkey = key or seacfg
seacfgs[pkey] = seacfg
remarks[pkey] = 'missing frappy config'
error = True
restart = set()
for key, running in frappycfgs.items():
if running:
if key in ('main', 'stick'):
service = key
else:
service = 'addons'
addons.add(running)
if not seacfgs.get(key):
if running in all_of[SEAEXT.get(key, '.addon')]:
restart.add(service)
remarks[key] = 'restart to start sea'
elif givencfgs.get(key) != running:
remarks[key] = 'reconnect'
elif remarks.get(key) is None:
remarks[key] = ''
if addons:
result['addons'] = ','.join(addons)
class Keep(Reconnect):
"""a string tagged with the information to keep the connection as given
for service, cfg in ourcfgs.items():
if cfg:
prop = result.get(service, '')
if prop == cfg and service not in restart:
result[service] = True
return error, result, (frappycfgs, seacfgs), remarks
isinstance(cfg, Reconnect) means: the given cfg is already running, so no reconnection is needed
"""
def __repr__(self):
return f'Keep({str(self)!r})'
class FrappyManager(ServiceManager):
@ -188,6 +98,17 @@ class FrappyManager(ServiceManager):
<service> is one of main, stick, addons
%(legend)s
"""
# changed in all_info (see docstring there)
frappy2sea = None
sea2frappy = None
list_info = None
# changed in get_server_state:
frappy_cfgs = None
sea_cfgs = None
state = None
error = None
remarks = None
sea = None
def config_dirs(self, ins, service):
cfgpaths = []
@ -288,7 +209,9 @@ class FrappyManager(ServiceManager):
init(*nodes)
interact()
def get_cfg_details(self, namespace, cfgfile):
@staticmethod
def get_cfg_details(namespace, cfgfile):
# get sea_cfg option from frappy cfg file
namespace.init()
local = {}
with open(cfgfile, encoding='utf-8') as f:
@ -310,22 +233,27 @@ class FrappyManager(ServiceManager):
except Exception:
return False
def all_cfg(self, ins, service, list_info=None, sea_info=None):
def all_cfg(self, ins, service, details=False):
"""get available cfg files
:param ins: instance
:param service: service nor None for all services
:param list_info: None or a dict to collect info in the form
dict <cfgdir> of <cfg> of <description>
:param sea_info: None or a dict <frappycfg> of <seacfg> with info about sea configs
in frappy cfgs to be populated
:return: set of available config
:param details: get details about relation to sea
:return: see param:`what`
implicit results:
self.frappy2sea: a dict <frappycfg> of <seacfg> (including extension .config/.stick/.addon)
self.sea2frappy: a dict <seacfg> of set of <frappycfg>
self.list_info: dict <cfgdir> of <cfg> of <description>
"""
all_cfg = set()
if not ins:
return {}
namespace = Namespace()
details = sea_info is not None or list_info is not None
if details:
self.frappy2sea = f2s = {}
self.sea2frappy = s2f = {}
self.list_info = list_info = {}
for service in [service] if service else self.services:
for cfgdir in self.config_dirs(ins, service):
for cfgfile in glob(join(cfgdir, '*_cfg.py')):
@ -337,58 +265,45 @@ class FrappyManager(ServiceManager):
sea_cfg = None
desc = repr(e)
if cfg not in all_cfg:
if sea_cfg and sea_info is not None:
if sea_cfg:
# sea_info.setdefault(sea_cfg, set()).add(cfg)
sea_info[cfg] = sea_cfg
all_cfg.add(cfg)
if list_info is not None:
f2s[cfg] = sea_cfg
s2f.setdefault(sea_cfg, set()).add(cfg)
list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0]
all_cfg.add(cfg)
# if service == 'main':
# sea_info['none.config'] = {''}
return all_cfg
def do_listcfg(self, ins='', service='', prt=print):
if not ins:
raise UsageError('missing instance')
list_info = {}
sea_info = {}
if service:
self.all_cfg(ins, service, list_info, sea_info)
else:
for service in self.services:
self.all_cfg(ins, service, list_info, sea_info)
self.all_cfg(ins, service, True)
seacfgpat = re.compile(r'(.*)(\.config|\.stick|\.addon)')
# inverted_sea_info = {}
# for seacfg, cfgset in sea_info.items():
# for cfg in cfgset:
# inverted_sea_info[cfg] = seacfg
cfgset = set()
ambiguous = {}
for cfg, seacfg in sea_info.items():
if cfg in cfgset:
ambiguous[cfg] = ambiguous.get(cfg, 1) + 1
else:
cfgset.add(cfg)
keylen = max(max(len(k) for k in cfgs) for cfgs in list_info.values())
for cfgdir, cfgs in list_info.items():
keylen = max((max(len(k) for k in cfgs) for cfgs in self.list_info.values()), default=1)
ambiguous = set()
for cfgdir, cfgs in self.list_info.items():
if cfgs:
prt('')
prt('--- %s:' % cfgdir)
for cfg, desc in sorted(cfgs.items(), key=lambda v: (v[0].lower(), v)):
seacfg = sea_info.get(cfg)
seacfg = self.frappy2sea.get(cfg)
if seacfg:
name, ext = seacfgpat.match(seacfg).groups()
if name == cfg or name + 'stick' == cfg:
prefix = '* '
else:
prefix = f'* ({name}{ext}) '
if cfg in ambiguous:
if len(self.sea2frappy[seacfg]) > 1:
prefix = '!' + prefix[1:]
ambiguous.add(seacfg)
desc = prefix + desc
prt('%s %s' % (cfg.ljust(keylen), desc))
prt(' ')
gap = ' ' * keylen
prt(f'{gap} * need sea')
if ambiguous:
print(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy')
prt(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy')
def treat_args(self, argdict, unknown=(), extra=()):
cfg = None
@ -410,7 +325,7 @@ class FrappyManager(ServiceManager):
return super().treat_args(argdict, unknown, extra)
def check_cfg_file(self, ins, service, cfg, needsea=False):
if cfg is 'none':
if cfg == 'none':
return ''
try:
desc, sea_cfg = self.cfg_details(self, ins, service, cfg)
@ -425,19 +340,137 @@ class FrappyManager(ServiceManager):
:param ins: the instance to be checked for
:param givencfgs: a dict <service> of cfg given by the ECS
:return: a dict <service> of cfg, where cfg is either:
- a bare string: this cfg is proposed to change to this value
- Reconnect(cfg): the frappy server is running as expected, but the given cfg does not match
- Keep(cfg): no change needed
remark: Reconnect amd Keep inherit from str, so Reconnect(cfg) == cfg is always True
:return: tuple (<error>, <proposed>, (<frappyitems>, <seaitems>), <remarks> where:
<error>: proposed config not sure
<proposed>: dict <service> of proposed cfg
<frappyitems>: dict of items runnning in frappy servers (addons are separated)
<seaitems>: dict of items running on the sea server
<remarks>: dict of actions to do / remarks
implicit results:
self.remarks: a dict <service> of remark (why should this be changed?)
self.frappy_cfgs: a dict <service> of running cfgs
self.sea_cfgs: a dict <service> of sea cfgs (without ending .config/.stick/.addon)
self.state: a dict ('sea <service>' | 'frappy <service>') of cfg summarizing the state of all servers
a change of self.state indicates that the configuration may need to be reevaluated
self.error: there is an ambiguity for the mapping seacfg -> frappycfg
self.sea: a fresh SeaManager instance
"""
ourcfgs = self.get_cfg(ins, None)
sea = SeaManager()
seacfgs = sea.get_cfg(ins, 'sea').split('/')
if len(seacfgs) < 2:
seacfgs.append('')
sea_info = {}
self.all_cfg(ins, None, sea_info=sea_info)
return summarize_server_state(givencfgs, ourcfgs, seacfgs, sea_info)
self.frappy_cfgs = self.get_cfg(ins, None) # dict <service> of running frappy servers
self.sea = SeaManager()
seaconfig = self.sea.get_cfg(ins, 'sea')
sealist = seaconfig.split('/') # <config>, <stick>, <addon1>, <addon2> ...
if len(sealist) < 2:
sealist.append('')
all_cfg = self.all_cfg(ins, None, True)
proposed_cfg = {}
self.sea_cfgs = {}
self.remarks = {}
self.error = False
self.state = {}
result = {}
for (service, ext), seacfg in zip_longest(SEAEXT.items(), sealist[0:2], fillvalue=(None, None)):
if service:
self.sea_cfgs[service] = seacfg
proposed = self.sea2frappy.get(seacfg + ext) or set()
if seaconfig:
proposed_cfg[service] = proposed
self.state[f'sea {service}'] = seacfg
running_addons = self.frappy_cfgs.get('addons')
running_addons = [v.strip() for v in running_addons.split(',') if v.strip()]
proposed_addons = [] # use list instead of set for keeping order
running_sea_addons = set()
for cfg in running_addons:
seacfg = self.frappy2sea.get(cfg.strip())
if seacfg is None:
proposed_addons.append(cfg) # addons with no sea cfg should be kept
elif seacfg.endswith('.addon') and seacfg[:-6] in sealist[2:]:
proposed_addons.append(cfg)
running_sea_addons.add(seacfg)
for scfg in sealist[2:]:
seacfg = scfg + '.addon'
if seacfg not in running_sea_addons:
proposed = self.sea2frappy.get(seacfg) or set()
if proposed:
if len(proposed) > 1:
self.error = f'ambiguous frappy cfg for {seacfg}.addon: {proposed}'
else:
proposed = list(proposed)[0]
if proposed not in proposed_addons:
proposed_addons.append(proposed)
self.sea_cfgs['addons'] = ','.join(sealist[2:])
if proposed_addons: # and set(proposed_addons) != set(running_addons):
proposed_cfg['addons'] = {','.join(proposed_addons)}
self._debug = {}
for service in FrappyManager.services:
given = givencfgs.get(service)
running = self.frappy_cfgs.get(service)
available = proposed_cfg.get(service)
if running:
self.state[f'frappy {service}'] = running
self._debug[service] = (seaconfig, available, running, running in self.frappy2sea)
if seaconfig and (available or running in self.frappy2sea):
# we get here when the sea server is running and either at least one of:
# - the sea config is matching any frappy cfg
# - the running frappy cfg is matching a sea config
if not available:
self.remarks[service] = 'no frappy config'
continue
proposed = list(available)[0] if len(available) == 1 else None
if proposed is None:
running = None
for item in available:
if item == given:
proposed = item
if item == running:
running = item
if running and not proposed:
proposed = running
if proposed is None:
self.remarks[service] = 'ambiguous frappy cfg'
self.error = f'ambiguous frappy cfg for {seacfg}: {available}'
else:
self.remarks[service] = ''
if running == proposed:
if given == running:
result[service] = Keep(running)
elif running:
result[service] = proposed
self.remarks[service] = 'restart frappy'
else:
result[service] = proposed
else:
# we get here when sea is not running or all of:
# - the sea config has no matching frappy cfg
# - the frappy cfg has no matching sea config
if running:
if self.frappy2sea.get(running, '').endswith(ext):
result[service] = running
self.remarks[service] = 'restart frappy to start sea'
elif given != running:
result[service] = Reconnect(running)
self.remarks[service] = 'reconnect needed'
else:
result[service] = Keep(running)
self.remarks[service] = ''
elif given:
if self.frappy2sea.get(given, '').endswith(ext):
result[service] = running
self.remarks[service] = 'restart frappy and sea'
elif ':' in given:
self.remarks[service] = 'external device'
elif given in all_cfg:
result[service] = running
self.remarks[service] = 'restart frappy'
else:
result[service] = ''
self.remarks[service] = f'unknown frappy cfg {given}'
return result

View File

@ -261,6 +261,17 @@ class NicosManager(ServiceManager):
def do_gui(self, ins):
self.prepare_client(ins)
# patch QApplication to use 'nicos_<instr>' instead of 'nicos' as organizationName
# for different settings and log files
import nicos.guisupport.qt
class QApplication(nicos.guisupport.qt.QApplication):
def __init__(self, *args, organizationName=None, **kwds):
super().__init__(*args, organizationName=f'nicos_{ins}', **kwds)
nicos.guisupport.qt.QApplication = QApplication
from nicos.clients.gui.main import main
print('starting nicos gui %s' % ins)
try: