From b9262ed3c3ff00c0e75d08663c3168f06783d7a8 Mon Sep 17 00:00:00 2001 From: l_samenv Date: Mon, 19 Oct 2026 15:11:31 +0200 Subject: [PATCH] add repo-sync sync frappy, boxtools and boxweb repos automatically to linse-c:boxes/repos/ --- install.py | 23 +- .../systemd/user/network_online.service | 29 +++ .../.config/systemd/user/repo-restore.service | 12 + to_home/.local/bin/repo-sync | 227 ++++++++++++++++++ 4 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 to_home/.config/systemd/user/network_online.service create mode 100644 to_home/.config/systemd/user/repo-restore.service create mode 100755 to_home/.local/bin/repo-sync diff --git a/install.py b/install.py index 06fc101..806648c 100755 --- a/install.py +++ b/install.py @@ -136,10 +136,11 @@ WantedBy = multi-user.target FRAPPY_SERVICE = """[Unit] Description = Running frappy server +# this waits also for network_online: +After=repo-restore.service +Wants=repo-restore.service [Service] -# wait for network is online -ExecStartPre=/bin/sh -c 'until systemctl is-active --quiet network-online.target; do sleep 1; done' Type=simple Environment=PYTHONPATH=/home/l_samenv/.local/lib/python3.11/site-packages/ ExecStart = /home/l_samenv/boxtools/start_frappy @@ -174,6 +175,18 @@ ExecStart = /usr/bin/python3 flaskserver.py {page}.{port}.{wsport} WantedBy = default.target """ +REPOSYNC_SERVICE = """[Unit] +Description=sync repos to central computer +Wants=repo-restore.service +After=repo-restore.service + +[Service] +ExecStart=%h/.local/bin/repo-sync watch + +[Install] +WantedBy=default.target +""" + pip_requirements = {} box = BoxInfo() @@ -228,6 +241,10 @@ def router(**opts): return True, ROUTER_SERVICE +def reposync(): + return False, REPOSYNC_SERVICE + + def display_update(cfg): text = '\n'.join(cfg.get('startup_text', '').split('|')[:3]) text = text.replace('HOST', box.hostname) \ @@ -287,7 +304,7 @@ def boxweb(port=8080, wsport=8051, page=None): return port <= 1024, servicecfg -SERVICES = dict(router=router, display=display, frappy=frappy, boxweb=boxweb) +SERVICES = dict(router=router, display=display, frappy=frappy, boxweb=boxweb, reposync=reposync) AS_ROOT = {'router', 'display'} diff --git a/to_home/.config/systemd/user/network_online.service b/to_home/.config/systemd/user/network_online.service new file mode 100644 index 0000000..c817335 --- /dev/null +++ b/to_home/.config/systemd/user/network_online.service @@ -0,0 +1,29 @@ +[Unit] +Description=replacement for system service network-online +Documentation=man:nm-online(1) + +[Service] +# `nm-online -s` waits until the point when NetworkManager logs +# "startup complete". That is when startup actions are settled and +# devices and profiles reached a conclusive activated or deactivated +# state. It depends on which profiles are configured to autoconnect and +# also depends on profile settings like ipv4.may-fail/ipv6.may-fail, +# which affect when a profile is considered fully activated. +# Check NetworkManager logs to find out why wait-online takes a certain +# time. + +Type=oneshot +ExecStart=/usr/bin/nm-online -s -q +RemainAfterExit=yes +Environment=NM_ONLINE_TIMEOUT=60 + +# Set $NM_ONLINE_TIMEOUT variable for timeout in seconds. +# Edit with `systemctl edit NetworkManager-wait-online`. +# +# Note, this timeout should commonly not be reached. If your boot +# gets delayed too long, then the solution is usually not to decrease +# the timeout, but to fix your setup so that the connected state +# gets reached earlier. + +[Install] +WantedBy=default.target diff --git a/to_home/.config/systemd/user/repo-restore.service b/to_home/.config/systemd/user/repo-restore.service new file mode 100644 index 0000000..87f2424 --- /dev/null +++ b/to_home/.config/systemd/user/repo-restore.service @@ -0,0 +1,12 @@ +[Unit] +Description=Restore repos from central computer if requested +Wants=network_online.service +After=network_online.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=%h/.local/bin/repo-sync restore + +[Install] +WantedBy=default.target diff --git a/to_home/.local/bin/repo-sync b/to_home/.local/bin/repo-sync new file mode 100755 index 0000000..023dcff --- /dev/null +++ b/to_home/.local/bin/repo-sync @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +import re +import subprocess +import socket +import sys +import threading +import time +from pathlib import Path + +HOME = Path.home() +DIRS = ["frappy", "boxtools", "boxweb"] # relative to $HOME + +CENTRAL = "l_samenv@linse-c" +BASE = "boxes/repos" +NAME = socket.gethostname() +REMOTE_DIR = f"{BASE}/{NAME}" +STATE_FILE = f"{BASE}/{NAME}.state" +STATES = ["stopped", "restore", "backup"] # ordered by priority + +QUIET = 2.0 # seconds without events before syncing +RETRY_DELAY = 10.0 # seconds before retrying after a failure / non-backup state + +EXCLUDE_DIRS = [".git", "__pycache__", ".venv", ".mypy_cache"] +EXCLUDE_FILES = ["*.pyc", "*.swp", "*~"] +INOTIFY_EXCLUDE = ( + r"(^|/)(" + "|".join(re.escape(d) for d in EXCLUDE_DIRS) + r")(/|$)" + r"|\.(pyc|swp)$|~$" +) + +SSH_OPTS = "ssh -o BatchMode=yes -o ConnectTimeout=10" + +changed = threading.Event() +last_event = 0.0 + + +def log(msg: str) -> None: + print(msg, flush=True) + + +def get_dirs(): + return [v for v in DIRS if (HOME / v).is_dir()] + + +# ---------- state handling on the central computer ---------- + +def ssh(remote_cmd: str) -> subprocess.CompletedProcess: + print(remote_cmd) + return subprocess.run( + SSH_OPTS.split() + [CENTRAL, remote_cmd], + capture_output=True, text=True, + ) + + +#def get_state() -> str | None: +# """Return the state, or None if central is unreachable.""" +# r = ssh(f"cat {shlex.quote(STATE_FILE)} 2>/dev/null || echo backup") +# if r.returncode != 0: +# return None +# state = r.stdout.split()[0] +# if state not in STATES: +# return "backup" +# return state + + +def get_or_set_state(newstate=None) -> str | None: + """Return the state, or None if central is unreachable.""" + cmd = f"ls {BASE}/{NAME}.* 2> /dev/null" + r = ssh(cmd) + if r.returncode != 0: + # check if remote_dir is available at all + if ssh(f"ls {BASE}").returncode != 0: + log(f'failed: {cmd}') + return None + files = set(r.stdout.split()) + states = set(v.rsplit('.')[-1] for v in files) + for state in STATES: + if state in states: + break + else: + state = 'backup' + print('current', state) + if newstate: + statefile = f'{BASE}/{NAME}.{newstate}' + ssh(f'touch {statefile}') + files.discard(statefile) + if files: + ssh(f'rm {" ".join(files)}') + return newstate + return state + + +def get_state(): + return get_or_set_state() + + +def set_state(state): + return get_or_set_state(state) + + +#def set_state(state: str) -> bool: +# f = shlex.quote(STATE_FILE) +# # write to a temp file and rename, so readers never see a partial file +# r = ssh(f"echo {state} > {f}.tmp && mv {f}.tmp {f}") +# return r.returncode == 0 + + +def wait_for_state() -> str: + while True: + state = get_state() + if state is not None: + return state + log("central unreachable, retrying") + time.sleep(RETRY_DELAY) + + +# ---------- rsync in both directions ---------- + +def exclude_args() -> list[str]: + return [*[f"--exclude={d}/" for d in EXCLUDE_DIRS], + *[f"--exclude={p}" for p in EXCLUDE_FILES]] + + +def backup() -> bool: + cmd = ["rsync", "-aR", '--delete', *exclude_args(), "-e", SSH_OPTS, + *get_dirs(), f"{CENTRAL}:{REMOTE_DIR}/"] + return subprocess.run(cmd, cwd=HOME).returncode == 0 + + +def restore() -> bool: + # "/./" marks where the relative path starts, so with -R each subtree + # lands under $HOME with its original relative path + cmd = ["rsync", "-aR", *exclude_args(), "-e", SSH_OPTS, + *[f"{CENTRAL}:{REMOTE_DIR}/./{d}" for d in DIRS], f"{HOME}/"] + return subprocess.run(cmd).returncode == 0 + + +def do_restore(deadline: float | None = None) -> bool: + while True: + log("restoring from central") + if restore() and set_state("backup"): + log("restore done, state set to 'backup'") + return True + if deadline is not None and time.monotonic() + RETRY_DELAY > deadline: + return False + log("restore failed, retrying") + time.sleep(RETRY_DELAY) + + +# ---------- watcher ---------- + +def reader(proc: subprocess.Popen) -> None: + global last_event + for _ in proc.stdout: + last_event = time.monotonic() + changed.set() + changed.set() # EOF: inotifywait exited, wake the main loop + + +def cmd_watch() -> None: + """Long-running: watch and back up (only while state == 'backup').""" + watcher = subprocess.Popen( + ["inotifywait", "-m", "-r", "-q", + "-e", "close_write,create,delete,move,attrib", + "--exclude", INOTIFY_EXCLUDE, + "--format", "%w%f", + *get_dirs()], + cwd=HOME, stdout=subprocess.PIPE, text=True, + ) + threading.Thread(target=reader, args=(watcher,), daemon=True).start() + changed.set() # initial full backup (if state allows) + try: + while True: + changed.wait() + if watcher.poll() is not None: + sys.exit("inotifywait exited, letting systemd restart us") + while time.monotonic() - last_event < QUIET: + time.sleep(0.2) + changed.clear() + + if get_state() != "backup": # stopped, restore pending, or unreachable + time.sleep(RETRY_DELAY) + changed.set() + continue + if backup(): + log('BACKUP success') + set_state('backup') # update creation date of state file + else: + log('BACKUP failed') + time.sleep(RETRY_DELAY) + changed.set() + finally: + watcher.terminate() + + +def wait_for_state(deadline: float | None = None) -> str | None: + """Return the state; None if central is still unreachable at the deadline.""" + while True: + state = get_state() + if state is not None: + return state + if deadline is not None and time.monotonic() + RETRY_DELAY > deadline: + return None + log("central unreachable, retrying") + time.sleep(RETRY_DELAY) + + +def cmd_restore(timeout: float | None) -> None: + deadline = None if timeout is None else time.monotonic() + timeout + state = wait_for_state(deadline) + if state is None: + sys.exit("central unreachable, giving up") + log(f"state: {state}") + if state == "restore" and not do_restore(deadline): + sys.exit("restore did not complete") + + +def main() -> None: + args = sys.argv[1:] + if args[:1] == ["restore"] and len(args) <= 2: + cmd_restore(float(args[1]) if len(args) == 2 else None) + elif args == ["watch"]: + cmd_watch() + else: + sys.exit(f"usage: {sys.argv[0]} restore [timeout_s] | watch") + +if __name__ == "__main__": + main()