500 lines
20 KiB
Python
500 lines
20 KiB
Python
# *****************************************************************************
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it under
|
|
# the terms of the GNU General Public License as published by the Free Software
|
|
# Foundation; either version 2 of the License, or (at your option) any later
|
|
# version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
|
# details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along with
|
|
# this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#
|
|
# Module authors:
|
|
# Markus Zolliker <markus.zolliker@psi.ch>
|
|
#
|
|
# *****************************************************************************
|
|
|
|
import sys
|
|
import os
|
|
import re
|
|
import builtins
|
|
from glob import glob
|
|
from socket import gethostbyname, gethostname
|
|
from itertools import zip_longest
|
|
from pathlib import Path
|
|
from collections import defaultdict
|
|
from os.path import join, isdir, basename, expanduser, exists
|
|
from configparser import ConfigParser
|
|
from .base import ServiceManager, ServiceDown, UsageError
|
|
from .seaman import SeaManager
|
|
|
|
|
|
MAIN = 1
|
|
STICK = 2
|
|
|
|
|
|
class Config:
|
|
log = None
|
|
process_file = None
|
|
|
|
@classmethod
|
|
def get(cls, cfgfile):
|
|
if not cls.process_file:
|
|
import logging
|
|
try:
|
|
from frappy.config import process_file
|
|
except Exception as e:
|
|
print(sys.path)
|
|
raise
|
|
from frappy.lib import generalConfig
|
|
generalConfig.init()
|
|
cls.log = logging.getLogger('frappyman')
|
|
cls.process_file = process_file
|
|
return cls.process_file(Path(cfgfile), cls.log)
|
|
|
|
|
|
SEAEXT = {'main': '.config', 'stick': '.stick'}
|
|
|
|
|
|
class Reconnect(str):
|
|
"""a string tagged with the information that the frappy server is already running
|
|
|
|
isinstance(cfg, Reconnect) means: cfg may need reconnection, but not restart
|
|
"""
|
|
def __repr__(self):
|
|
return f'Reconnect({str(self)!r})'
|
|
|
|
|
|
class Keep(Reconnect):
|
|
"""a string tagged with the information to keep the connection as given
|
|
|
|
isinstance(cfg, Reconnect) means: the given cfg is already running, so no reconnection is needed
|
|
"""
|
|
def __repr__(self):
|
|
return f'Keep({str(self)!r})'
|
|
|
|
|
|
class FrappyManager(ServiceManager):
|
|
group = 'frappy'
|
|
services = ('main', 'stick', 'addons')
|
|
USAGE = """
|
|
Usage:
|
|
|
|
frappy list %(optional_ins)s
|
|
frappy start %(ins)s<service> <cfgfiles>
|
|
frappy restart %(ins)s[<service>] [<cfgfiles>] %(remark)s
|
|
frappy stop %(ins)s[<service>] %(remark)s
|
|
frappy listcfg %(ins)s[<service> | develop] # list available cfg files
|
|
|
|
<service> is one of main, stick, addons
|
|
%(legend)s
|
|
"""
|
|
# changed in all_info (see docstring there)
|
|
frappy2sea = None
|
|
sea2frappy = None
|
|
list_info = None
|
|
# changed in get_server_state:
|
|
frappy_cfgs = None
|
|
sea_cfgs = None
|
|
state = None
|
|
error = None
|
|
remarks = None
|
|
sea = None
|
|
|
|
def config_dirs(self, ins, service):
|
|
cfgpaths = []
|
|
cfgparser = ConfigParser()
|
|
cfgparser.optionxform = str
|
|
env = self.env.get(ins, {})
|
|
cfgfile = env.get('FRAPPY_CONFIG_FILE')
|
|
confdir = env.get('FRAPPY_CONFDIR')
|
|
if cfgfile:
|
|
cfgfile = cfgfile.replace('<SERV>', service)
|
|
cfgparser.read(cfgfile)
|
|
try:
|
|
section = cfgparser['FRAPPY']
|
|
except KeyError:
|
|
raise ValueError('%s does not exist or has no FRAPPY section' % cfgfile)
|
|
confdir = section.get('confdir', confdir)
|
|
if not confdir:
|
|
return []
|
|
for cfgpath in confdir.split(os.pathsep):
|
|
if cfgpath.endswith('<SERV>'):
|
|
cfgpaths.append(expanduser(cfgpath[:-6] + service))
|
|
else:
|
|
scfg = join(cfgpath, service)
|
|
if isdir(scfg):
|
|
cfgpaths.append(scfg)
|
|
cfgpaths.append(expanduser(cfgpath))
|
|
return cfgpaths
|
|
|
|
def prepare_start(self, ins, service, cfg=''):
|
|
start_dir, env = super().prepare_start(ins, service)
|
|
env_update = {}
|
|
for key, value in env.items():
|
|
if '<SERV>' in value:
|
|
env_update[key] = value.replace('<SERV>', service)
|
|
os.environ[key] = env[key]
|
|
cfgpaths = self.config_dirs(ins, service)
|
|
if cfgpaths:
|
|
env_update['FRAPPY_CONFDIR'] = os.pathsep.join(cfgpaths)
|
|
# merge PYTHONPATH from servicemanager.cfg with the one from environment
|
|
pypathlist = []
|
|
for pypath in env.get('PYTHONPATH'), os.environ.get('PYTHONPATH'):
|
|
if pypath is not None:
|
|
pypathlist.extend(p for p in pypath.split(':') if p not in pypathlist)
|
|
if pypathlist:
|
|
env_update['PYTHONPATH'] = os.environ['PYTHONPATH'] = ':'.join(pypathlist)
|
|
return start_dir, dict(env, **env_update)
|
|
|
|
def do_start(self, ins, service=None, cfg='', restart=False, wait=False, logger=None, opts=''):
|
|
if self.wildcard(ins) is not None:
|
|
raise UsageError('no wildcards allowed with %s start' % self.group)
|
|
if cfg and not service and len(self.services) != 1:
|
|
raise UsageError('need service to start (one of %s)' % ', '.join(self.services))
|
|
super().do_start(ins, service, cfg, restart, wait, logger, opts=opts)
|
|
|
|
def do_restart(self, ins, service=None, cfg=None, logger=None):
|
|
ins_list = super().do_restart(ins, service, cfg, logger)
|
|
if ins_list: # wildcard used
|
|
# determine running processes with cfg
|
|
cfginfo = {}
|
|
self.get_procs(None, cfginfo)
|
|
cfgs = {i for i, s in cfginfo if s == service or service is None}
|
|
return [i for i in ins_list if i in cfgs]
|
|
|
|
def get_nodes(self, ins='', service=None):
|
|
ServiceManager.prepare_start(self, ins, None)
|
|
nodes = []
|
|
services = self.services if service is None else [service]
|
|
for service in services:
|
|
try:
|
|
self.check_running(ins, service)
|
|
nodes.append('localhost:%d' % self.info[ins][service])
|
|
except ServiceDown:
|
|
if len(services) == 1:
|
|
raise UsageError('frappy %s %s is not running' % (ins, service))
|
|
except KeyError:
|
|
if ins:
|
|
raise UsageError('unknown instance %s' % ins)
|
|
raise UsageError('missing instance')
|
|
if not nodes:
|
|
raise UsageError(f"frappy {ins}: none of {'/'.join(services)} is running")
|
|
return nodes
|
|
|
|
def do_gui(self, ins='', service=None):
|
|
print(f'starting frappy gui {ins} {service or ""}')
|
|
nodes = self.get_nodes(ins, service)
|
|
|
|
import logging
|
|
from frappy.gui.qt import QApplication
|
|
from frappy.gui.mainwindow import MainWindow
|
|
|
|
app = QApplication([])
|
|
args = type('args', (), dict(detailed=True, node=nodes))
|
|
win = MainWindow(args, logging.getLogger('gui'))
|
|
win.show()
|
|
return app.exec_()
|
|
|
|
def do_cli(self, ins='', service=None):
|
|
nodes = self.get_nodes(ins, service)
|
|
from frappy.client.interactive import init, interact
|
|
from frappy.protocol.discovery import scan
|
|
if ins == self.single_ins:
|
|
all_nodes = {}
|
|
for node in nodes:
|
|
host, port = node.split(':')
|
|
if host == 'localhost':
|
|
host = gethostname()
|
|
all_nodes[gethostbyname(host), int(port)] = node
|
|
for a in scan():
|
|
all_nodes.setdefault((a.address, a.port), f'{a.hostname}:{a.port}')
|
|
nodes = list(all_nodes.values())
|
|
init(*nodes)
|
|
try:
|
|
interact(appname=ins)
|
|
except TypeError: # older frappy client
|
|
interact()
|
|
|
|
@staticmethod
|
|
def get_cfg_details(cfgfile):
|
|
mods = Config.get(cfgfile)
|
|
node = mods.pop('node') or {}
|
|
sea_cfg = None
|
|
for mod, config in mods.items():
|
|
cls = config['cls']
|
|
cls = getattr(cls, '__name__', cls)
|
|
if cls.endswith('SeaClient'):
|
|
try:
|
|
sea_cfg = config['config']['value']
|
|
except KeyError:
|
|
sea_cfg = None
|
|
return node.get('description', '').strip(), sea_cfg
|
|
|
|
def cfg_details(self, ins, service, cfgfile):
|
|
if cfgfile:
|
|
return self.get_cfg_details(cfgfile)
|
|
raise FileNotFoundError(f'{cfgfile} not found')
|
|
|
|
def get_cfg_file(self, ins, service, cfg, lazy=False):
|
|
if service is None:
|
|
return None
|
|
filenames = [f'{cfg}_cfg.py']
|
|
if lazy:
|
|
filenames.extend([f'{cfg}.py', cfg])
|
|
for cfgdir in self.config_dirs(ins, service):
|
|
for filename in filenames:
|
|
cfgfile = join(cfgdir, filename)
|
|
if exists(cfgfile):
|
|
return cfgfile
|
|
return None
|
|
|
|
def is_cfg(self, ins, service, cfg):
|
|
return bool(self.get_cfg_file(ins, service, cfg))
|
|
|
|
def all_cfg(self, ins, service, details=False):
|
|
"""get available cfg files
|
|
|
|
:param ins: instance
|
|
:param service: service nor None for all services
|
|
:param details: get details about relation to sea
|
|
:return: see param:`what`
|
|
|
|
implicit results:
|
|
self.frappy2sea: a dict <frappycfg> of <seacfg> (including extension .config/.stick/.addon)
|
|
self.sea2frappy: a dict <seacfg> of set of <frappycfg>
|
|
self.list_info: dict <cfgdir> of <cfg> of <description>
|
|
"""
|
|
all_cfg = set()
|
|
if not ins:
|
|
return {}
|
|
if details:
|
|
self.frappy2sea = f2s = {}
|
|
self.sea2frappy = s2f = {}
|
|
self.list_info = list_info = {}
|
|
for service in [service] if service else self.services:
|
|
for cfgdir in self.config_dirs(ins, service):
|
|
for cfgfile in glob(join(cfgdir, '*_cfg.py')):
|
|
cfg = basename(cfgfile)[:-7]
|
|
if details:
|
|
try:
|
|
desc, sea_cfg = self.get_cfg_details(cfgfile)
|
|
except TypeError:
|
|
raise
|
|
except Exception as e:
|
|
sea_cfg = None
|
|
desc = repr(e)
|
|
if cfg not in all_cfg:
|
|
if sea_cfg:
|
|
# sea_info.setdefault(sea_cfg, set()).add(cfg)
|
|
f2s[cfg] = sea_cfg
|
|
s2f.setdefault(sea_cfg, set()).add(cfg)
|
|
list_info.setdefault(cfgdir, {})[cfg] = desc.split('\n', 1)[0]
|
|
all_cfg.add(cfg)
|
|
# if service == 'main':
|
|
# sea_info['none.config'] = {''}
|
|
return all_cfg
|
|
|
|
def do_listcfg(self, ins='', service='', prt=print):
|
|
if not ins:
|
|
raise UsageError('missing instance')
|
|
ServiceManager.prepare_start(self, ins, service)
|
|
self.all_cfg(ins, service, True)
|
|
seacfgpat = re.compile(r'(.*)(\.config|\.stick|\.addon)')
|
|
keylen = max((max(len(k) for k in cfgs) for cfgs in self.list_info.values()), default=1)
|
|
ambiguous = set()
|
|
for cfgdir, cfgs in self.list_info.items():
|
|
if cfgs:
|
|
prt('')
|
|
prt('--- %s:' % cfgdir)
|
|
for cfg, desc in sorted(cfgs.items(), key=lambda v: (v[0].lower(), v)):
|
|
seacfg = self.frappy2sea.get(cfg)
|
|
if seacfg:
|
|
name, ext = seacfgpat.match(seacfg).groups()
|
|
if name == cfg or name + 'stick' == cfg:
|
|
prefix = '* '
|
|
else:
|
|
prefix = f'* ({name}{ext}) '
|
|
if len(self.sea2frappy[seacfg]) > 1:
|
|
prefix = '!' + prefix[1:]
|
|
ambiguous.add(seacfg)
|
|
desc = prefix + desc
|
|
prt('%s %s' % (cfg.ljust(keylen), desc))
|
|
prt(' ')
|
|
gap = ' ' * keylen
|
|
prt(f'{gap} * need sea')
|
|
if ambiguous:
|
|
prt(f'{gap} ! {len(ambiguous)} ambiguous mappings sea -> frappy')
|
|
|
|
def treat_args(self, argdict, unknown=(), extra=()):
|
|
cfg = None
|
|
extra = list(extra)
|
|
for arg in unknown:
|
|
if arg.startswith('-'): # this is an option
|
|
extra.append(arg)
|
|
elif cfg is None:
|
|
cfg = arg
|
|
else:
|
|
cfg = ''
|
|
if cfg:
|
|
if cfg == 'develop':
|
|
argdict['service'] = cfg
|
|
return super().treat_args(argdict, (), extra)
|
|
if (',' in cfg or cfg.endswith('.cfg') or
|
|
self.get_cfg_file(argdict.get('ins'), argdict.get('service'), cfg, True)):
|
|
return super().treat_args(argdict, (), [cfg] + extra)
|
|
return super().treat_args(argdict, unknown, extra)
|
|
|
|
def check_cfg_file(self, ins, service, cfg, needsea=False):
|
|
if cfg == 'none':
|
|
return ''
|
|
try:
|
|
desc, sea_cfg = self.cfg_details(self, ins, service, cfg)
|
|
if needsea and not sea_cfg:
|
|
return None
|
|
return cfg
|
|
except Exception:
|
|
return cfg + '?'
|
|
|
|
def get_server_state(self, ins, givencfgs):
|
|
"""get proposed configuration and an overview of the running servers
|
|
|
|
:param ins: the instance to be checked for
|
|
:param givencfgs: a dict <service> of cfg given by the ECS
|
|
:return: a dict <service> of cfg, where cfg is either:
|
|
- a bare string: this cfg is proposed to change to this value
|
|
- Reconnect(cfg): the frappy server is running as expected, but the given cfg does not match
|
|
- Keep(cfg): no change needed
|
|
remark: Reconnect amd Keep inherit from str, so Reconnect(cfg) == cfg is always True
|
|
|
|
implicit results:
|
|
self.remarks: a dict <service> of remark (why should this be changed?)
|
|
self.frappy_cfgs: a dict <service> of running cfgs
|
|
self.sea_cfgs: a dict <service> of sea cfgs (without ending .config/.stick/.addon)
|
|
self.state: a dict ('sea <service>' | 'frappy <service>') of cfg summarizing the state of all servers
|
|
a change of self.state indicates that the configuration may need to be reevaluated
|
|
self.error: there is an ambiguity for the mapping seacfg -> frappycfg
|
|
self.sea: a fresh SeaManager instance
|
|
"""
|
|
self.frappy_cfgs = self.get_cfg(ins, None) # dict <service> of running frappy servers
|
|
self.sea = SeaManager()
|
|
seaconfig = self.sea.get_cfg(ins, 'sea')
|
|
sealist = seaconfig.split('/') # <config>, <stick>, <addon1>, <addon2> ...
|
|
if len(sealist) < 2:
|
|
sealist.append('')
|
|
|
|
all_cfg = self.all_cfg(ins, None, True)
|
|
proposed_cfg = {}
|
|
self.sea_cfgs = {}
|
|
self.remarks = {}
|
|
self.error = False
|
|
self.state = {}
|
|
result = {}
|
|
for (service, ext), seacfg in zip_longest(SEAEXT.items(), sealist[0:2], fillvalue=(None, None)):
|
|
if service:
|
|
self.sea_cfgs[service] = seacfg
|
|
proposed = self.sea2frappy.get(seacfg + ext) or set()
|
|
if seaconfig:
|
|
proposed_cfg[service] = proposed
|
|
self.state[f'sea {service}'] = seacfg
|
|
|
|
running_addons = self.frappy_cfgs.get('addons')
|
|
running_addons = [v.strip() for v in running_addons.split(',') if v.strip()]
|
|
proposed_addons = [] # use list instead of set for keeping order
|
|
running_sea_addons = set()
|
|
for cfg in running_addons:
|
|
seacfg = self.frappy2sea.get(cfg.strip())
|
|
if seacfg is None:
|
|
proposed_addons.append(cfg) # addons with no sea cfg should be kept
|
|
elif seacfg.endswith('.addon') and seacfg[:-6] in sealist[2:]:
|
|
proposed_addons.append(cfg)
|
|
running_sea_addons.add(seacfg)
|
|
|
|
for scfg in sealist[2:]:
|
|
seacfg = scfg + '.addon'
|
|
if seacfg not in running_sea_addons:
|
|
proposed = self.sea2frappy.get(seacfg) or set()
|
|
if proposed:
|
|
if len(proposed) > 1:
|
|
self.error = f'ambiguous frappy cfg for {seacfg}.addon: {proposed}'
|
|
else:
|
|
proposed = list(proposed)[0]
|
|
if proposed not in proposed_addons:
|
|
proposed_addons.append(proposed)
|
|
self.sea_cfgs['addons'] = ','.join(sealist[2:])
|
|
if proposed_addons: # and set(proposed_addons) != set(running_addons):
|
|
proposed_cfg['addons'] = {','.join(proposed_addons)}
|
|
|
|
for service in FrappyManager.services:
|
|
given = givencfgs.get(service)
|
|
running = self.frappy_cfgs.get(service)
|
|
available = proposed_cfg.get(service)
|
|
|
|
if running:
|
|
self.state[f'frappy {service}'] = running
|
|
|
|
if seaconfig and (available or running in self.frappy2sea):
|
|
# we get here when the sea server is running and either at least one of:
|
|
# - the sea config is matching any frappy cfg
|
|
# - the running frappy cfg is matching a sea config
|
|
if not available:
|
|
self.remarks[service] = 'no frappy config'
|
|
continue
|
|
proposed = list(available)[0] if len(available) == 1 else None
|
|
if proposed is None:
|
|
running = None
|
|
for item in available:
|
|
if item == given:
|
|
proposed = item
|
|
if item == running:
|
|
running = item
|
|
if running and not proposed:
|
|
proposed = running
|
|
if proposed is None:
|
|
self.remarks[service] = 'ambiguous frappy cfg'
|
|
self.error = f'ambiguous frappy cfg for {seacfg}: {available}'
|
|
else:
|
|
self.remarks[service] = ''
|
|
if running == proposed:
|
|
if given == running:
|
|
result[service] = Keep(running)
|
|
elif running:
|
|
result[service] = proposed
|
|
self.remarks[service] = 'restart frappy'
|
|
else:
|
|
result[service] = proposed
|
|
else:
|
|
# we get here when sea is not running or all of:
|
|
# - the sea config has no matching frappy cfg
|
|
# - the frappy cfg has no matching sea config
|
|
if running:
|
|
if self.frappy2sea.get(running, '').endswith(ext):
|
|
result[service] = running
|
|
self.remarks[service] = 'restart frappy to start sea'
|
|
elif given != running:
|
|
result[service] = Reconnect(running)
|
|
self.remarks[service] = 'reconnect needed'
|
|
else:
|
|
result[service] = Keep(running)
|
|
self.remarks[service] = ''
|
|
elif given:
|
|
if self.frappy2sea.get(given, '').endswith(ext):
|
|
result[service] = running
|
|
self.remarks[service] = 'restart frappy and sea'
|
|
elif ':' in given:
|
|
self.remarks[service] = 'external device'
|
|
elif given in all_cfg:
|
|
result[service] = running
|
|
self.remarks[service] = 'restart frappy'
|
|
else:
|
|
result[service] = ''
|
|
self.remarks[service] = f'unknown frappy cfg {given}'
|
|
|
|
return result
|
|
|