boxtools/install.py

#!/usr/bin/python3
"""install.py

- copy files from to_system into system directories
- set host name / network settings from aputools/servercfg file
"""

if bytes == str:
    raise NotImplementedError('python2 not supported')

import sys
import os
import filecmp
import shutil
from subprocess import Popen, PIPE
from glob import glob
from ipaddress import IPv4Interface
from configparser import ConfigParser
from os.path import join, getmtime, exists, basename

more_info = sys.argv[1] if len(sys.argv) > 1 else 'NONE'
os.chdir('/root/aputools/to_system')

DEL = '__to_delete__'
CFGPATH = '/root/aputools/servercfg/%s_%6.6x.cfg'

COMMENT = "; please refer to README.md for help"

CONFIG_TEMPLATE = """[NETWORK]
%s
enp1s0=192.168.127.254
enp2s0=192.168.2.2
enp3s0=192.168.3.3
enp4s0=dhcp

[ROUTER]
%s
3001=192.168.127.254:3001
""" % (COMMENT, COMMENT)

DHCP_HEADER = """
default-lease-time 600;
max-lease-time 7200;
authoritative;
"""

ROUTER_TEMPLATE = """[Unit]
Description = Routing to locally connected hardware
After = network.target

[Service]
ExecStart = /usr/bin/python3 /root/aputools/router.py

[Install]
WantedBy = multi-user.target
"""

FRAPPY_TEMPLATE = """[Unit]
Description = Running frappy server
After = network.target

[Service]
User = l_samenv
ExecStart = /usr/bin/python3 /home/l_samenv/frappy/bin/secop-server %s

[Install]
WantedBy = multi-user.target
"""
pip_requirements = {
    'l_samenv': {},
    'root': {}
}


def frappy(cfg=None, port=None, requirements='', **kwds):
    if not cfg:
        return None
    req = pip_requirements['l_samenv']
    req[cfg] = '\n'.join(requirements.split(','))
    if port:
        cfg = '-p %s %s' % (port, cfg)
    with open('/home/l_samenv/frappy/requirements.txt') as f:
        req['frappy main'] = f.read()
    return FRAPPY_TEMPLATE % cfg


def router(**opts):
    if not opts:
        return None
    try:
        with open('/root/aputools/requirements.txt') as f:
            pip_requirements['root']['aputools'] = f.read()
    except FileNotFoundError:
        pass
    return ROUTER_TEMPLATE


def pip():
    for user, requirements in pip_requirements.items():
        if user == 'root':
            tmpname = join('/root', 'pip_requirements.tmp')
            pipcmd = 'pip3 install -r %s' % tmpname
        else:
            tmpname = join('/home', user, 'pip_requirements.tmp')
            pipcmd = 'sudo --user %s pip3 install --user -r %s' % (user, tmpname)
        filename = tmpname.replace('.tmp', '.txt')
        content = ''.join('# --- for %s ---\n%s\n' % kv for kv in requirements.items())
        if write_when_new(filename, content, True):
            if doit:
                os.rename(filename, tmpname)
                if os.system(pipcmd) == 0:
                    os.rename(tmpname, filename)
                else:
                    os.remove(tmpname)
            else:
                print(pipcmd)
            # unix_cmd(pipcmd, stdout=None)
            show.dirty = True


SERVICES = dict(router=router, frappy=frappy)


def unix_cmd(command, always=False, stdout=PIPE):
    if doit or always:
        if not always:
            print('$ %s' % command)
        result = Popen(command.split(), stdout=stdout).communicate()[0]
        return (result or b'').decode()
    else:
        print('> %s' % command)


def write_when_new(filename, content, ignore_reduction=False):
    if content is None:
        lines = []
    else:
        if not content.endswith('\n'):
            content += '\n'
        lines = content.split('\n')
    try:
        with open(filename) as fil:
            old = fil.read()
    except FileNotFoundError:
        old = None
    if old == content:
        return False
    if ignore_reduction:
        content_set = set(v for v in lines if not v.startswith('#'))
        old_set = set(v for v in (old or '').split('\n') if not v.startswith('#'))
        content_set -= old_set
        if content_set:
            print('missing packages: %s' % (', '.join(content_set - old_set)))
        else:
            return False
    if doit:
        if lines:
            with open(filename, 'w') as fil:
                fil.write(content)
        else:
            os.remove(filename)
    elif filename.endswith(more_info):
        print('changes in', filename)
        old = [] if old is None else old.split('\n')
        top_lines = 0  # in case of empty loop
        for top_lines, (ol, ll) in enumerate(zip(old, lines)):
            if ol != ll:
                break
        bottom_lines = 0
        for bottom_lines, (ol, ll) in enumerate(zip(reversed(old[top_lines:]), reversed(lines[top_lines:]))):
            if ol != ll:
                break
        if bottom_lines == 0:
            old.append('')
            lines.append('')
            bottom_lines += 1
        print("===")
        print('\n'.join(old[:top_lines]))
        print('<<<')
        print('\n'.join(old[top_lines:-bottom_lines]))
        print('---')
        print('\n'.join(lines[top_lines:-bottom_lines]))
        print('>>>')
        print('\n'.join(old[-bottom_lines:-1]))
        print("===")
    return True


def create_if(name, cfg, mac, dhcp_server_cfg):
    result = dict(
        TYPE='Ethernet',
        NAME=name,
        DEVICE=name,
        BOOTPROTO='none',
        ONBOOT='yes',
        PROXY_METHOD='none',
        BROWSER_ONLY='no',
        IPV4_FAILURE_FATAL='yes',
        IPV6INIT='no')
    if cfg == 'off':
        result['ONBOOT'] = 'no'
    elif cfg.startswith('wan') or cfg == 'dhcp':
        result['BOOTPROTO'] = 'dhcp'
        # default: all <= 192.0.0.0
        # others have to be added explicitly
        dhcp_server_cfg.append(('0.0.0.0/128.0.0.0', []))
        dhcp_server_cfg.append(('128.0.0.0/192.0.0.0', []))
        for nw in cfg.split(',')[1:]:
            nw = IPv4Interface(cfg).network
            dhcp_server_cfg.append((nw.with_netmask.split('/'), []))
    else:
        cfgip = IPv4Interface(cfg)
        network = cfgip.network
        if network.prefixlen == 32:  # or no prefix specified
            otherip = IPv4Interface('%s/24' % cfgip.ip)
            network = otherip.network
            if str(cfgip.ip).endswith('.1'):
                cfgip = network.network_address + 2
            else:
                cfgip = network.network_address + 1
            dhcp_server_cfg.append((network.with_netmask, [(str(otherip.ip), str(otherip.ip))]))
            result['IPADDR'] = str(cfgip)
        else:  # subnet with multiple adresses -> static adresses only. dhcp range not yet implemented
            result['IPADDR'] = str(cfgip.ip)
        result['NETMASK'] = network.netmask
        result['PREFIX'] = str(network.prefixlen)
    return result


def walk(action):
    for dirpath, _, files in os.walk('.'):
        syspath = dirpath[1:]  # remove leading '.'
        action.dirpath = dirpath
        action.syspath = syspath
        if files:
            match, mismatch, missing = filecmp.cmpfiles(dirpath, syspath, files)
            if mismatch:
                newer = [f for f in mismatch if getmtime(join(syspath, f)) > getmtime(join(dirpath, f))]
                if newer:
                    action.newer(newer)
                if len(newer) < len(mismatch):
                    newer = set(newer)
                    action.older([f for f in mismatch if f not in newer])
            if missing:
                if DEL in missing:
                    missing.remove(DEL)
                    with open(join(dirpath, DEL)) as fil:
                        to_delete = []
                        for f in fil:
                            f = f.strip()
                            if f and exists(join(syspath, f)):
                                if exists(join(dirpath, f)):
                                    print('ERROR: %s in %s, but also in repo -> ignored' % (f, DEL))
                                else:
                                    to_delete.append(f)
                        action.delete(to_delete)
                if missing:
                    action.missing(missing)


class Show:
    dirty = False

    def diff(self, title, files):
        self.dirty = True
        if more_info == 'NONE':
            print('%s %s:\n  %s' % (title, self.syspath, ' '.join(files)))
        else:
            for f in files:
                if f.endswith(MORE_INFO):
                    print('diff %s %s' % (join(self.dirpath, f), join(self.syspath, f)))

    def show(self, title, dirpath, files):
        if more_info == 'NONE':
            print('%s %s:\n  %s' % (title, self.syspath, ' '.join(files)))
        else:
            for f in files:
                if f.endswith(MORE_INFO):
                    print('cat %s' % join(dirpath, f))

    def newer(self, files):
        self.show('get from', self.dirpath, files)

    def older(self, files):
        self.show('replace in', self.dirpath, files)

    def missing(self, files):
        self.show('install in', self.dirpath, files)

    def delete(self, files):
        self.show('remove from', self.syspath, files)


class Do:
    def newer(self, files):
        for file in files:
            shutil.copy(join(self.syspath, file), join(self.dirpath, file))

    def older(self, files):
        for file in files:
            shutil.copy(join(self.dirpath, file), join(self.syspath, file))

    def missing(self, files):
        self.older(files)

    def delete(self, files):
        for file in files:
            os.remove(join(self.syspath, file))


IFNAMES = ['enp%ds0' % i for i in range(1,5)]


def handle_config():
    netaddr_dict = {}
    for ifname in IFNAMES:
        with open('/sys/class/net/%s/address' % ifname) as f:
            netaddr_dict[ifname] = f.read().strip().lower()
    netaddr = netaddr_dict[IFNAMES[0]]
    apuid = int(''.join(netaddr.split(':')[-3:]), 16) & 0xfffffc
    cfgfile = None
    cfgfiles = []
    for i in range(4):
        # goodie: look for mac addresses of all 4 ports
        cfgfiles += glob(CFGPATH % ('*', apuid + i))
    with open('/etc/hostname') as f:
        hostname = f.read().strip()
    if not cfgfiles:
        print('no cfg file found for %s' % hostname)
        newname = input('enter host name: ')
        if not newname:
            print('no hostname given')
            return False
        cfgfile = CFGPATH % (newname, apuid)
        with open(cfgfile, 'w') as f:
            f.write(CONFIG_TEMPLATE)
    elif len(cfgfiles) > 1:
        print('ERROR: ambiguous cfg files: %s' % ', '.join(cfgfiles))
    else:
        cfgfile = cfgfiles[0]
    if cfgfile != CFGPATH % (hostname, apuid):
        if doit:
            os.system('sh ../sethostname.sh')
        else:
            if cfgfile:
                print('replace host name %r by %r' % (hostname, basename(cfgfile).rpartition('_')[0]))
                show.dirty = True
    if cfgfile is None:
        return False
    to_start = {}
    ifname = ''
    parser = ConfigParser()
    dhcp_server_cfg = []
    try:
        parser.read(cfgfile)
        network = dict(parser['NETWORK'])
        for ifname in IFNAMES:
            content = create_if(ifname, network.pop(ifname, 'off'), netaddr_dict[ifname], dhcp_server_cfg)
            content = '\n'.join('%s=%s' % kv for kv in content.items())
            todo = write_when_new('/etc/sysconfig/network-scripts/ifcfg-%s' % ifname, content)
            if todo and more_info == 'NONE':
                print('change', ifname)
                show.dirty = True
                to_start[ifname] = 'if_restart'
        if dhcp_server_cfg:
            content = [DHCP_HEADER]
            for subnet, rangelist in dhcp_server_cfg:
                adr, mask = subnet.split('/')
                content.append('subnet %s netmask %s {\n' % (adr, mask))
                #content.append('    option netmask %s;\n' % mask)
                for rng in rangelist:
                    content.append('    range %s %s;\n' % rng)
                content.append('}\n')
            content = ''.join(content)
            todo = write_when_new('/etc/dhcp/dhcpd.conf', content)
            if todo:
                print('change dhcpd.conf')
                to_start['dhcpd'] = 'restart'
            show.dirty = True
        elif doit:
            unix_cmd('systemctl stop dhcpd')
            unix_cmd('systemctl disable dhcpd')
        content = []
        dirty = False
        for section in parser.sections():
            if COMMENT not in parser[section]:
                dirty = True
            content.append('[%s]' % section)
            content.append(COMMENT)
            for key, value in parser[section].items():
                content.append('%s=%s' % (key, value))
            content.append('')
        if dirty:
            with open(cfgfile, 'w') as f:
                f.write('\n'.join(content))
    except Exception as e:
        print('ERROR: can not handle %s %s: %r' % (hostname, ifname, e))
        raise
        return False

    reload_systemd = False
    for service, template_func in SERVICES.items():
        section = service.upper()
        if parser.has_section(section):
           template = template_func(**dict(parser[section]))
        else:
           template = None
        result = unix_cmd('systemctl show -p WantedBy -p ActiveState %s' % service, True)
        active = False
        enabled = False
        for line in result.split('\n'):
            if line.startswith('WantedBy=m'):
                enabled = True
            elif line.strip() == 'ActiveState=active':
                active = True
        if template is None:
            if active:
                unix_cmd('systemctl stop %s' % service)
                show.dirty = True
            if enabled:
                unix_cmd('systemctl disable %s' % service)
                show.dirty = True
        else:
            if not enabled:
                to_start[service] = 'enable'
            elif not active:
                to_start[service] = 'restart'
        if write_when_new('/etc/systemd/system/%s.service' % service, template):
            show.dirty = True
            reload_systemd = True
            if template and to_start.get('service') is None:
                to_start[service] = 'restart'

    pip()
    if reload_systemd:
        unix_cmd('systemctl daemon-reload')
    for service, action in to_start.items():
        show.dirty = True
        if action == 'if_restart':
            unix_cmd('ifdown %s' % service)
            unix_cmd('ifup %s' % service)
        else:
            if action == 'restart':
                unix_cmd('systemctl restart %s' % service)
            unix_cmd('systemctl enable %s' % service)
    return True


doit = False
print('---')
show = Show()
walk(show)
result = handle_config()

if not result:
    print('fix first above errors')
elif show.dirty and more_info == 'NONE':
    print('---')
    answer = input('do above? ')
    doit = True
    if answer.lower().startswith('y'):
        handle_config()
        walk(Do())
else:
    print('nothing to do')