From 7717efd19a0ef5886f647e91397b23d18445afa7 Mon Sep 17 00:00:00 2001
From: x01dc <x01dc@psi.ch>
Date: Wed, 22 Jul 2026 15:49:33 +0200
Subject: [PATCH] feat(webpage): mirror status-page uploads to a second URL

HttpUploader now accepts a single URL or a list, fanning every upload
(dir/changed/single-file/cleanup) out to each configured server with an
independent per-URL mtime cache, so a newly added mirror backfills on
its own and a failing one doesn't block the others. Session-auth
endpoints (session_query/set_password) still derive from the primary
URL only.

flomni.py/lamni.py now mirror to https://omny-test.psi.ch/upload.php
alongside production, marked TEMP for easy removal once that server
is evaluated. Adds OMNY_shared/AI_docs/WEBPAGE_UPLOAD_MECHANISM.md
consolidating the upload mechanism (the prior doc predates this file
and still called LamNI a stub).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../bec_ipython_client/plugins/LamNI/lamni.py |   7 +-
 .../AI_docs/WEBPAGE_UPLOAD_MECHANISM.md       | 110 ++++++++++++++++++
 .../plugins/OMNY_shared/web_common.py         |  88 ++++++++------
 .../OMNY_shared/webpage_generator_base.py     |  19 ++-
 .../plugins/flomni/flomni.py                  |   7 +-
 5 files changed, 189 insertions(+), 42 deletions(-)
 create mode 100644 csaxs_bec/bec_ipython_client/plugins/OMNY_shared/AI_docs/WEBPAGE_UPLOAD_MECHANISM.md

diff --git a/csaxs_bec/bec_ipython_client/plugins/LamNI/lamni.py b/csaxs_bec/bec_ipython_client/plugins/LamNI/lamni.py
index c1c7f18..bc7105a 100644
--- a/csaxs_bec/bec_ipython_client/plugins/LamNI/lamni.py
+++ b/csaxs_bec/bec_ipython_client/plugins/LamNI/lamni.py
@@ -233,7 +233,12 @@ class LamNI(TomoQueueMixin, LamNIAlignmentMixin, LamNIOpticsMixin, LamniGuiTools
         self._webpage_gen = LamniWebpageGenerator(
             bec_client=client,
             output_dir="~/data/raw/webpage/",
-            upload_url="https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php",
+            upload_url=[
+                "https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php",
+                # TEMP: mirroring to the test server while it's evaluated --
+                # drop this line once omny-test.psi.ch is confirmed or the trial ends.
+                "https://omny-test.psi.ch/upload.php",
+            ],
             local_port=8080,
         )
         self._webpage_gen.start()
diff --git a/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/AI_docs/WEBPAGE_UPLOAD_MECHANISM.md b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/AI_docs/WEBPAGE_UPLOAD_MECHANISM.md
new file mode 100644
index 0000000..268a6cc
--- /dev/null
+++ b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/AI_docs/WEBPAGE_UPLOAD_MECHANISM.md
@@ -0,0 +1,110 @@
+# Webpage generator upload mechanism
+
+Consolidated reference for how the flOMNI/LamNI/OMNY status pages get uploaded to the
+remote web server(s). Supersedes the upload-related parts of
+`flomni/AI_docs/Webpage.md`, which predates `OMNY_shared/web_common.py` and still
+describes LamNI as an unimplemented stub.
+
+## Live files vs. dead duplicate
+
+The live implementation is:
+
+- `OMNY_shared/web_common.py` -- `HttpUploader`, `LocalHttpServer`.
+- `OMNY_shared/webpage_generator_base.py` -- `WebpageGeneratorBase`, `make_webpage_generator()`.
+- Per-setup subclasses: `flomni/flomni_webpage_generator.py` (`FlomniWebpageGenerator`),
+  `LamNI/LamNI_webpage_generator.py` (`LamniWebpageGenerator`). These are thin subclasses
+  that forward constructor kwargs straight to `WebpageGeneratorBase.__init__` -- neither
+  overrides upload handling.
+- `OMNY_shared/eps/eps_status_generator.py` -- `EpsStatusGenerator`, the separate
+  EPS/machine-status daemon. It has its own `HttpUploader` construction for standalone
+  use, but when launched from `WebpageGeneratorBase._launch()` it's always given
+  `upload_url=None` -- its files live in the same `output_dir` and ride along on the
+  parent generator's own `upload_changed_async()` scan every cycle. Giving it its own
+  uploader too would re-POST the same directory redundantly.
+
+**`omny/omny_webpage_generator.py` is a stale, unwired duplicate.** It defines its own
+copies of `HttpUploader`/`LocalHttpServer`/`WebpageGeneratorBase`, is never imported by
+anything, and its own docstring header still reads `"""flomni/webpage_generator.py"""`
+(a leftover from an earlier version of the flomni file). `make_webpage_generator()`
+explicitly falls back to the plain base class for the `"omny"` session rather than using
+it. **Do not edit this file** -- a `grep -rn "HttpUploader"` across `plugins/` will
+surface it as a false positive; always check the import path before editing.
+
+## `HttpUploader` contract (`web_common.py`)
+
+Uploads files from a local directory to one or more remote servers via HTTP POST, in a
+background daemon thread so uploads never block the generator's cycle.
+
+- `_UPLOAD_SUFFIXES = {".html", ".json", ".jpg", ".png", ".pdf", ".txt", ".svg"}` --
+  files with any other extension are silently skipped by the directory-scanning methods
+  (`upload_dir_async`/`upload_changed_async`). `upload_file_async` bypasses this whitelist
+  entirely (used only for `session.htpasswd`, which has no matching extension).
+- **Fire-and-forget, drop-if-busy**: `_dispatch()` guards a single `_busy` flag under a
+  lock. If a previous upload from this `HttpUploader` is still running when the next
+  cycle fires, the new request is dropped (logged at DEBUG) -- there is no queueing or
+  retry. One dispatched thread does all configured URLs sequentially.
+- **Per-URL mtime cache**: `self._uploaded` is `dict[url, dict[path_str, mtime]]`. A file
+  is only re-uploaded to a given URL once its mtime changes since the last *successful*
+  upload to that specific URL. This means each configured server tracks "changed" fully
+  independently -- a URL that's been failing doesn't block another URL from proceeding,
+  and a brand-new URL (empty cache) naturally gets backfilled the next time a full
+  (`force=True`) upload runs.
+- **Rate-limited warning logging**: `_warn(key, msg)` logs at most once per 600s per key.
+  Keys fold in the URL (e.g. `f"{url}|upload_{filename}"`, `f"{url}|cleanup"`) so a
+  persistent failure against one server doesn't suppress the warning for another.
+- **SSRF hardening**: every POST uses `allow_redirects=False` -- a compromised public
+  server must not be able to redirect the internal process to an arbitrary internal
+  address. `verify=False` accepts self-signed certs (the fallback Raspberry Pi target
+  historically used one).
+- **Ptycho cleanup**: `cleanup_ptycho_images_async()` POSTs `{"action": "cleanup"}` to
+  each URL, asking the server to delete stale `S*_*.png`/`S*_*.jpg` files, then forgets
+  the local mtime records for any cached path containing `/S` or `\S` so those files get
+  re-uploaded on the next cycle.
+
+### Multi-URL mirroring
+
+`HttpUploader(url)` accepts either a single URL string or a list of URLs. When a list is
+given, **every** method (`upload_dir_async`, `upload_changed_async`, `upload_file_async`,
+`cleanup_ptycho_images_async`) mirrors uniformly to all of them, including
+`session.htpasswd` pushes -- there is no special-casing of any file or action per URL.
+
+`WebpageGeneratorBase.__init__`'s `_session_query_url`/`_set_password_url` (used for
+account-match checking and `set_web_password()`) are **not** part of `HttpUploader` and
+are derived only from the *first/primary* URL, even when `upload_url` is a list -- they
+govern access control for the production page and have no meaning for a secondary/test
+mirror.
+
+As of 2026-07, both `flomni.py` and `lamni.py` pass a two-element list -- production
+(`https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php`) plus a temporary evaluation target
+(`https://omny-test.psi.ch/upload.php`) -- marked with a `# TEMP` comment at the call
+site. Drop the second entry once the trial ends.
+
+## Call-site map (`webpage_generator_base.py`)
+
+- `_launch()`: on startup (unless `local_only`), `self._uploader.upload_dir_async(...)`
+  uploads every eligible file once. Also prints `self._uploader._urls` (joined) in the
+  startup log line.
+- `_cycle()` (every ~15s): if the ptycho scan id changed since the last cycle, triggers
+  `cleanup_ptycho_images_async()` then a delayed (2s) `upload_dir_async()` full re-upload;
+  otherwise just `upload_changed_async()`. Skipped entirely when `self._uploader is None`
+  or `self._local_only` is True.
+- `_clear_session_htpasswd()`: writes an emptied `session.htpasswd` locally and pushes it
+  via `upload_file_async()` whenever the active BEC account changes.
+- `status()`: prints `self._uploader._urls` (joined) as the "Upload URL" line.
+
+`local_only` (the account-mismatch fallback, entered automatically by `start()` unless
+`force=True`) suppresses **all** uploads regardless of how many URLs are configured --
+it's a binary "upload at all vs. not" switch, orthogonal to how many mirrors exist.
+
+## Known issue: `.svg` upload failing on `omny-test.psi.ch`
+
+Both the live `HttpUploader._UPLOAD_SUFFIXES` and the in-repo reference
+`OMNY_shared/server_side/webserver/upload.php` already allow `.svg`. Manual testing
+against `omny-test.psi.ch` showed `.json`/`.html` status updates succeeding but `.svg`
+(the EPS synoptic image, generated by `OMNY_shared/eps/eps_synoptic_svg.py`) failing.
+Since nothing in this repo's client or reference server code excludes `.svg`, the most
+likely cause is whatever `upload.php`/WAF is actually deployed on the test box differing
+from the reference copy here (a stricter filename regex, or a WAF rule flagging SVG's
+potential for embedded `<script>` content). This is a server-side configuration question
+for the test box, not a client bug -- follow up there before assuming a code fix is
+needed.
diff --git a/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/web_common.py b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/web_common.py
index 0b5ae4f..2770ca0 100644
--- a/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/web_common.py
+++ b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/web_common.py
@@ -47,20 +47,24 @@ def _epoch() -> float:
 
 class HttpUploader:
     """
-    Uploads files from a local directory to a remote server via HTTP POST.
+    Uploads files from a local directory to one or more remote servers via
+    HTTP POST.
 
     Uploads run in a daemon thread so they never block the caller's cycle.
     If an upload is already in progress when the next cycle fires, the new
-    request is dropped (logged at DEBUG) rather than queuing up.
+    request is dropped (logged at DEBUG) rather than queuing up. Passing a
+    list of URLs mirrors every upload to each of them independently (each
+    URL gets its own mtime cache, so a newly-added URL is backfilled with a
+    full upload and a persistently failing URL doesn't affect the others).
     """
 
     _UPLOAD_SUFFIXES = {".html", ".json", ".jpg", ".png", ".pdf", ".txt", ".svg"}
     _WARN_COOLDOWN_S = 600
 
-    def __init__(self, url: str, timeout: float = 20.0):
-        self._url = url
+    def __init__(self, url: str | list[str], timeout: float = 20.0):
+        self._urls = [url] if isinstance(url, str) else list(url)
         self._timeout = timeout
-        self._uploaded: dict[str, float] = {}
+        self._uploaded: dict[str, dict[str, float]] = {u: {} for u in self._urls}
         self._lock = threading.Lock()
         self._busy = False
         self._warn_at: dict[str, float] = {}
@@ -75,22 +79,21 @@ class HttpUploader:
     # -- public API ---------------------------------------------------------
 
     def upload_dir_async(self, directory: Path) -> None:
-        """Upload ALL eligible files in directory, in a background thread."""
-        self._dispatch(self._upload_files, self._eligible_files(directory), True)
+        """Upload ALL eligible files in directory to every URL, in a background thread."""
+        files = self._eligible_files(directory)
+        self._dispatch(self._upload_files_all, files, True)
 
     def upload_changed_async(self, directory: Path) -> None:
-        """Upload only files whose mtime changed since the last upload."""
-        files = self._changed_files(directory)
-        if files:
-            self._dispatch(self._upload_files, files, False)
+        """Upload, to every URL, only the files whose mtime changed since that URL's last upload."""
+        self._dispatch(self._upload_changed_all, directory)
 
     def upload_file_async(self, path: Path) -> None:
-        """Upload one specific file, bypassing suffix whitelist and mtime check."""
-        self._dispatch(self._upload_files, [Path(path)], True)
+        """Upload one specific file to every URL, bypassing suffix whitelist and mtime check."""
+        self._dispatch(self._upload_files_all, [Path(path)], True)
 
     def cleanup_ptycho_images_async(self) -> None:
-        """Ask the server to delete all S*_*.png / S*_*.jpg files (background)."""
-        self._dispatch(self._do_cleanup)
+        """Ask every server to delete all S*_*.png / S*_*.jpg files (background)."""
+        self._dispatch(self._do_cleanup_all)
 
     # -- internals ----------------------------------------------------------
 
@@ -117,18 +120,33 @@ class HttpUploader:
         except OSError:
             return []
 
-    def _changed_files(self, directory: Path) -> list:
+    def _changed_files(self, directory: Path, url: str) -> list:
         out = []
+        uploaded = self._uploaded[url]
         for path in self._eligible_files(directory):
             try:
                 mtime = path.stat().st_mtime
             except OSError:
                 continue
-            if self._uploaded.get(str(path)) != mtime:
+            if uploaded.get(str(path)) != mtime:
                 out.append(path)
         return out
 
-    def _upload_files(self, files: list, force: bool = False) -> None:
+    def _upload_changed_all(self, directory: Path) -> None:
+        for url in self._urls:
+            files = self._changed_files(directory, url)
+            if files:
+                self._upload_files(url, files, force=False)
+
+    def _upload_files_all(self, files: list, force: bool) -> None:
+        for url in self._urls:
+            self._upload_files(url, files, force=force)
+
+    def _do_cleanup_all(self) -> None:
+        for url in self._urls:
+            self._do_cleanup(url)
+
+    def _upload_files(self, url: str, files: list, force: bool = False) -> None:
         try:
             import requests as _requests
             import urllib3
@@ -137,37 +155,38 @@ class HttpUploader:
             self._warn("no_requests", "HttpUploader: 'requests' library not installed")
             return
 
+        uploaded = self._uploaded[url]
         for path in files:
             try:
                 mtime = path.stat().st_mtime
             except OSError:
                 continue
-            if not force and self._uploaded.get(str(path)) == mtime:
+            if not force and uploaded.get(str(path)) == mtime:
                 continue
             try:
                 with open(path, "rb") as fh:
                     r = _requests.post(
-                        self._url,
+                        url,
                         files={"file": (path.name, fh)},
                         timeout=self._timeout,
                         verify=False,           # accept self-signed certs
                         allow_redirects=False,  # SSRF hardening
                     )
                 if r.status_code == 200:
-                    self._uploaded[str(path)] = mtime
-                    self._warn_at.pop(f"upload_{path.name}", None)
-                    logger.debug(f"HttpUploader: OK {path.name}")
+                    uploaded[str(path)] = mtime
+                    self._warn_at.pop(f"{url}|upload_{path.name}", None)
+                    logger.debug(f"HttpUploader: OK {path.name} -> {url}")
                 else:
                     self._warn(
-                        f"upload_{path.name}",
-                        f"HttpUploader: {path.name} -> HTTP {r.status_code}: "
+                        f"{url}|upload_{path.name}",
+                        f"HttpUploader: {path.name} -> {url} -> HTTP {r.status_code}: "
                         f"{r.text[:120]}",
                     )
             except Exception as exc:
-                self._warn(f"upload_{path.name}",
-                           f"HttpUploader: {path.name} failed: {exc}")
+                self._warn(f"{url}|upload_{path.name}",
+                           f"HttpUploader: {path.name} -> {url} failed: {exc}")
 
-    def _do_cleanup(self) -> None:
+    def _do_cleanup(self, url: str) -> None:
         try:
             import requests as _requests
             import urllib3
@@ -176,21 +195,22 @@ class HttpUploader:
             return
         try:
             r = _requests.post(
-                self._url,
+                url,
                 data={"action": "cleanup"},
                 timeout=self._timeout,
                 verify=False,           # accept self-signed certs
                 allow_redirects=False,  # SSRF hardening
             )
-            logger.info(f"HttpUploader cleanup: {r.text[:120]}")
-            self._warn_at.pop("cleanup", None)
+            logger.info(f"HttpUploader cleanup ({url}): {r.text[:120]}")
+            self._warn_at.pop(f"{url}|cleanup", None)
             # Forget mtime records for ptycho files so they get re-uploaded
+            uploaded = self._uploaded[url]
             with self._lock:
-                to_remove = [k for k in self._uploaded if "/S" in k or "\\S" in k]
+                to_remove = [k for k in uploaded if "/S" in k or "\\S" in k]
             for k in to_remove:
-                self._uploaded.pop(k, None)
+                uploaded.pop(k, None)
         except Exception as exc:
-            self._warn("cleanup", f"HttpUploader cleanup failed: {exc}")
+            self._warn(f"{url}|cleanup", f"HttpUploader cleanup ({url}) failed: {exc}")
 
 
 # ---------------------------------------------------------------------------
diff --git a/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/webpage_generator_base.py b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/webpage_generator_base.py
index bd6d450..25e8192 100644
--- a/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/webpage_generator_base.py
+++ b/csaxs_bec/bec_ipython_client/plugins/OMNY_shared/webpage_generator_base.py
@@ -464,7 +464,7 @@ class WebpageGeneratorBase:
         output_dir: str = "~/data/raw/webpage/",
         cycle_interval: float = _CYCLE_INTERVAL_S,
         verbosity: int = VERBOSITY_NORMAL,
-        upload_url: str = None,
+        upload_url: str | list[str] = None,
         local_port: int = 8080,
         mismatch_local_port: int = _MISMATCH_LOCAL_PORT,
     ):
@@ -479,10 +479,17 @@ class WebpageGeneratorBase:
         self._local_only     = False  # True while running the account-mismatch fallback
         self._eps            = None   # EpsStatusGenerator, constructed in _launch()
 
-        # Derive companion URLs from upload_url
+        # Derive companion URLs from upload_url. When upload_url is a list
+        # (mirroring uploads to several servers), the account-match /
+        # password-setting endpoints are only ever derived from the primary
+        # (first) URL -- they govern access to the production page and are
+        # not part of HttpUploader's mirrored-upload path.
         if upload_url is not None:
-            self._session_query_url = upload_url.replace("upload.php", "session_query.php")
-            self._set_password_url  = upload_url.replace("upload.php", "set_password.php")
+            primary_upload_url = (
+                upload_url[0] if isinstance(upload_url, (list, tuple)) else upload_url
+            )
+            self._session_query_url = primary_upload_url.replace("upload.php", "session_query.php")
+            self._set_password_url  = primary_upload_url.replace("upload.php", "set_password.php")
         else:
             self._session_query_url = None
             self._set_password_url  = None
@@ -639,7 +646,7 @@ class WebpageGeneratorBase:
         self._log(VERBOSITY_NORMAL,
                   f"WebpageGenerator started{mode_note}  owner={self._owner_id}  "
                   f"output={self._output_dir}  interval={self._cycle_interval}s"
-                  + (f"  upload={self._uploader._url}"
+                  + (f"  upload={', '.join(self._uploader._urls)}"
                      if (self._uploader and not local_only) else "  upload=disabled")
                   + f"\n  ➜  Status page:{local_url_msg}")
 
@@ -692,7 +699,7 @@ class WebpageGeneratorBase:
             f"  Output dir           : {self._output_dir}\n"
             f"  Local URL            : {local}\n"
             f"  Cycle interval       : {self._cycle_interval}s\n"
-            f"  Upload URL           : {self._uploader._url if self._uploader else 'disabled'}\n"
+            f"  Upload URL           : {', '.join(self._uploader._urls) if self._uploader else 'disabled'}\n"
             f"  Session query URL    : {self._session_query_url or 'disabled'}\n"
             f"  BEC active account   : {active_account}\n"
             f"  Server session user  : {remote_user}\n"
diff --git a/csaxs_bec/bec_ipython_client/plugins/flomni/flomni.py b/csaxs_bec/bec_ipython_client/plugins/flomni/flomni.py
index b8693a1..cdd3958 100644
--- a/csaxs_bec/bec_ipython_client/plugins/flomni/flomni.py
+++ b/csaxs_bec/bec_ipython_client/plugins/flomni/flomni.py
@@ -1642,7 +1642,12 @@ class Flomni(
             bec_client=client,
             output_dir="~/data/raw/webpage/",
             # upload_url="http://s1090968537.online.de/upload.php",   # optional
-            upload_url="https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php",
+            upload_url=[
+                "https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php",
+                # TEMP: mirroring to the test server while it's evaluated --
+                # drop this line once omny-test.psi.ch is confirmed or the trial ends.
+                "https://omny-test.psi.ch/upload.php",
+            ],
             local_port=8080,
         )
         self._webpage_gen.start()