bec/csaxs_bec
0
0
Fork 1
Code Issues 16 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix/webpage_generator #179

Merged
holler merged 7 commits from fix/webpage_generator into main 2026-03-30 13:07:47 +02:00
Conversation 0 Commits 7 Files Changed 2
+191 -102
2 changed files with 191 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
csaxs_bec/bec_ipython_client/plugins/flomni/flomni.py
+3 -1
View File
@@ -1317,7 +1317,9 @@ class Flomni(
self._webpage_gen = FlomniWebpageGenerator(
bec_client=client,
output_dir="~/data/raw/webpage/",
upload_url="http://s1090968537.online.de/upload.php", # optional
#upload_url="http://s1090968537.online.de/upload.php", # optional
upload_url="https://v1p0zyg2w9n2k9c1.myfritz.net/upload.php",
local_port=8080
)
self._webpage_gen.start()
csaxs_bec/bec_ipython_client/plugins/flomni/flomni_webpage_generator.py
+188 -101
View File
@@ -5,10 +5,14 @@ Background thread that reads tomo progress from the BEC global variable store
and writes status.json (every cycle) + status.html (once at startup) to a
staging directory. An optional HttpUploader sends those files to a web host
after every cycle, running in a separate daemon thread so uploads never block
the generator cycle.
the generator cycle. A built-in LocalHttpServer always serves the output
directory locally (default port 8080) so the page can be accessed on the
lab network without any extra setup.
Architecture
------------
LocalHttpServer -- built-in HTTP server; serves output_dir on port 8080.
Always started at _launch(); URL printed to console.
HttpUploader -- non-blocking HTTP uploader (fire-and-forget thread).
Tracks file mtimes; only uploads changed files.
Sends a cleanup request to the server when the
@@ -31,19 +35,24 @@ Integration (inside Flomni.__init__, after self._progress_proxy.reset()):
bec_client=client,
output_dir="~/data/raw/webpage/",
upload_url="http://omny.online/upload.php", # optional
local_port=8080, # optional, default 8080
)
self._webpage_gen.start()
# On start(), the console prints:
# ➜ Status page: http://hostname:8080/status.html
Interactive helpers (optional, in the iPython session):
-------------------------------------------------------
flomni._webpage_gen.status() # print current status
flomni._webpage_gen.status() # print current status + local URL
flomni._webpage_gen.verbosity = 2 # VERBOSE: one-line summary per cycle
flomni._webpage_gen.verbosity = 3 # DEBUG: full JSON per cycle
flomni._webpage_gen.stop() # release lock
flomni._webpage_gen.stop() # release lock, stop local server
flomni._webpage_gen.start() # restart after stop()
"""
import datetime
import functools
import http.server
import json
import os
import shutil
@@ -235,6 +244,16 @@ class HttpUploader:
self._uploaded: dict[str, float] = {} # abs path -> mtime at last upload
self._lock = threading.Lock()
self._busy = False # True while an upload thread is running
self._warn_at: dict[str, float] = {} # key -> epoch of last warning
_WARN_COOLDOWN_S = 600 # only repeat the same warning once per minute
def _warn(self, key: str, msg: str) -> None:
"""Log a warning at most once per _WARN_COOLDOWN_S for a given key."""
now = _epoch()
if now - self._warn_at.get(key, 0) >= self._WARN_COOLDOWN_S:
self._warn_at[key] = now
logger.warning(msg)
# ── Public API ──────────────────────────────────────────────────────────
@@ -294,8 +313,10 @@ class HttpUploader:
def _upload_files(self, files: list, force: bool = False) -> None:
try:
import requests as _requests
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
logger.warning("HttpUploader: 'requests' library not installed")
self._warn("no_requests", "HttpUploader: 'requests' library not installed")
return
for path in files:
@@ -312,21 +333,26 @@ class HttpUploader:
self._url,
files={"file": (path.name, f)},
timeout=self._timeout,
verify=False, # accept self-signed / untrusted certs
)
if r.status_code == 200:
self._uploaded[str(path)] = mtime
self._warn_at.pop(f"upload_{path.name}", None) # clear on success
logger.debug(f"HttpUploader: OK {path.name}")
else:
logger.warning(
self._warn(
f"upload_{path.name}",
f"HttpUploader: {path.name} -> HTTP {r.status_code}: "
f"{r.text[:120]}"
)
except Exception as exc:
logger.warning(f"HttpUploader: {path.name} failed: {exc}")
self._warn(f"upload_{path.name}", f"HttpUploader: {path.name} failed: {exc}")
def _do_cleanup(self) -> None:
try:
import requests as _requests
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
except ImportError:
return
try:
@@ -334,15 +360,84 @@ class HttpUploader:
self._url,
data={"action": "cleanup"},
timeout=self._timeout,
verify=False, # accept self-signed / untrusted certs
)
logger.info(f"HttpUploader cleanup: {r.text[:120]}")
self._warn_at.pop("cleanup", None) # clear on success
# Forget mtime records for ptycho files so they get re-uploaded
with self._lock:
to_remove = [k for k in self._uploaded if "/S" in k or "\\S" in k]
for k in to_remove:
self._uploaded.pop(k, None)
except Exception as exc:
logger.warning(f"HttpUploader cleanup failed: {exc}")
self._warn("cleanup", f"HttpUploader cleanup failed: {exc}")
# ---------------------------------------------------------------------------
# Local HTTP server (serves output_dir over http://hostname:port/)
# ---------------------------------------------------------------------------
class LocalHttpServer:
"""
Serves the generator's output directory over plain HTTP in a daemon thread.
Uses Python's built-in http.server — no extra dependencies.
Request logging is suppressed so the BEC console stays clean.
The server survives stop()/start() cycles: _launch() creates a fresh
instance each time start() is called.
Usage:
srv = LocalHttpServer(output_dir, port=8080)
srv.start()
print(srv.url) # http://hostname:8080/status.html
srv.stop()
"""
def __init__(self, directory: Path, port: int = 8080):
self._directory = Path(directory)
self._port = port
self._server = None
self._thread = None
# ── silence the per-request log lines in the iPython console ──────────
class _QuietHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, *args):
pass
def start(self) -> None:
Handler = functools.partial(
self._QuietHandler,
directory=str(self._directory),
)
try:
self._server = http.server.HTTPServer(("", self._port), Handler)
except OSError as exc:
raise RuntimeError(
f"LocalHttpServer: cannot bind port {self._port}: {exc}"
) from exc
self._thread = threading.Thread(
target=self._server.serve_forever,
name="LocalHttpServer",
daemon=True,
)
self._thread.start()
def stop(self) -> None:
if self._server is not None:
self._server.shutdown() # blocks until serve_forever() returns
self._server = None
def is_alive(self) -> bool:
return self._thread is not None and self._thread.is_alive()
@property
def port(self) -> int:
return self._port
@property
def url(self) -> str:
"""Best-guess URL for printing. Uses the machine's hostname."""
return f"http://{socket.gethostname()}:{self._port}/status.html"
# ---------------------------------------------------------------------------
@@ -363,12 +458,15 @@ class WebpageGeneratorBase:
cycle_interval: float = _CYCLE_INTERVAL_S,
verbosity: int = VERBOSITY_NORMAL,
upload_url: str = None,
local_port: int = 8080,
):
self._bec = bec_client
self._output_dir = Path(output_dir).expanduser().resolve()
self._cycle_interval = cycle_interval
self._verbosity = verbosity
self._uploader = HttpUploader(upload_url) if upload_url else None
self._local_port = local_port
self._local_server = None # created fresh each _launch()
self._thread = None
self._stop_event = threading.Event()
@@ -418,6 +516,17 @@ class WebpageGeneratorBase:
self._copy_logo()
(self._output_dir / "status.html").write_text(_render_html(_PHONE_NUMBERS))
# Start local HTTP server (always on; a fresh instance per _launch).
if self._local_server is not None and self._local_server.is_alive():
self._local_server.stop()
self._local_server = LocalHttpServer(self._output_dir, self._local_port)
try:
self._local_server.start()
local_url_msg = f" local={self._local_server.url}"
except RuntimeError as exc:
local_url_msg = f" local=ERROR({exc})"
self._log(VERBOSITY_NORMAL, str(exc), level="warning")
# Upload static files (html + logo) once at startup
if self._uploader is not None:
self._uploader.upload_dir_async(self._output_dir)
@@ -430,13 +539,16 @@ class WebpageGeneratorBase:
self._log(VERBOSITY_NORMAL,
f"WebpageGenerator started owner={self._owner_id} "
f"output={self._output_dir} interval={self._cycle_interval}s"
+ (f" upload={self._uploader._url}" if self._uploader else " upload=disabled"))
+ (f" upload={self._uploader._url}" if self._uploader else " upload=disabled")
+ f"\n ➜ Status page:{local_url_msg}")
def stop(self) -> None:
"""Stop the generator thread and release the singleton lock."""
"""Stop the generator thread, local HTTP server, and release the singleton lock."""
self._stop_event.set()
if self._thread is not None:
self._thread.join(timeout=self._cycle_interval + 5)
if self._local_server is not None:
self._local_server.stop()
self._release_lock()
self._log(VERBOSITY_NORMAL, "WebpageGenerator stopped.")
@@ -453,12 +565,14 @@ class WebpageGeneratorBase:
"""Print a human-readable status summary to the console."""
lock = self._read_lock()
running = self._thread is not None and self._thread.is_alive()
local = self._local_server.url if (self._local_server and self._local_server.is_alive()) else "stopped"
print(
f"WebpageGenerator\n"
f" This session running : {running}\n"
f" Lock owner : {lock.get('owner_id', 'none')}\n"
f" Lock heartbeat : {lock.get('heartbeat', 'never')}\n"
f" Output dir : {self._output_dir}\n"
f" Local URL : {local}\n"
f" Cycle interval : {self._cycle_interval}s\n"
f" Upload URL : {self._uploader._url if self._uploader else 'disabled'}\n"
f" Verbosity : {self._verbosity}\n"
@@ -1245,21 +1359,6 @@ def _render_html(phone_numbers: list) -> str:
}}
.info-item .value {{ font-size: 0.9rem; font-weight: 600; color: var(--text); }}
.bar-wrap {{ grid-column: 1 / -1; display: flex; flex-direction: column; gap: 0.35rem; }}
.bar-label {{
display: flex; justify-content: space-between;
font-family: var(--mono); font-size: 0.6rem; color: var(--text-dim);
letter-spacing: 0.06em; text-transform: uppercase;
}}
.bar-track {{ height: 5px; background: var(--surface2); border-radius: 99px; overflow: hidden; }}
.bar-fill {{
height: 100%;
background: var(--ring-blend);
background: color-mix(in srgb, var(--status-color) 65%, var(--surface2));
border-radius: 99px;
transition: width 0.8s cubic-bezier(.4,0,.2,1), background 0.6s;
}}
/* ── Recon card ── */
.recon-stats {{ display: flex; gap: 2rem; flex-wrap: wrap; }}
.recon-stat {{ display: flex; flex-direction: column; gap: 0.15rem; }}
@@ -1502,10 +1601,7 @@ def _render_html(phone_numbers: list) -> str:
<div class="info-item"><span class="label">ETA</span><span class="value" id="pi-eta">-</span></div>
<div class="info-item"><span class="label">Started</span><span class="value" id="pi-start">-</span></div>
</div>
<div class="bar-wrap">
<div class="bar-label"><span>Sub-tomo progress</span><span id="bar-sub-label">-</span></div>
<div class="bar-track"><div class="bar-fill" id="bar-sub-fill" style="width:0%"></div></div>
</div>
</div>
@@ -1667,90 +1763,80 @@ function initDrag() {{
}}
initDrag();
// ── Audio ─────────────────────────────────────────────────────────────────
// iOS (all browsers on iPhone use WebKit) requires:
// 1. AudioContext created inside a user gesture.
// 2. A real (even silent) BufferSource started synchronously in the gesture.
// 3. ctx.resume() awaited before scheduling audible nodes.
// We combine all three: silent unlock buffer + resume promise + .then(beeps).
// iOS (all browsers on iPhone use WebKit) strict rules:
// 1. AudioContext must be created inside a user gesture handler.
// 2. A real BufferSource must be started SYNCHRONOUSLY in the gesture —
// .then() / microtasks run outside the gesture and are rejected.
// 3. ctx.resume() is called fire-and-forget; beeps are delayed 80ms by
// setTimeout so the engine has time to start before nodes are scheduled.
//
// unlockAudio() handles all of this and must be called at the TOP of any
// onclick handler that wants audio — before any other logic.
let audioCtx = null, audioEnabled = false;
let audioArmed = false, warningActive = false, warningTimer = null, lastStatus = null;
let staleActive = false, staleTimer = null, staleConfirmed = false;
let audioCtx=null, audioEnabled=false;
let audioArmed=false, warningActive=false, warningTimer=null, lastStatus=null;
let staleActive=false, staleTimer=null, staleConfirmed=false;
function getCtx() {{
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
function getCtx(){{
if(!audioCtx) audioCtx=new(window.AudioContext||window.webkitAudioContext)();
return audioCtx;
}}
function unlockAudio() {{
// Must be called synchronously inside a user gesture handler.
// Plays a 1-sample silent buffer — the most reliable iOS unlock method.
const ctx = getCtx();
const buf = ctx.createBuffer(1, 1, ctx.sampleRate);
const src = ctx.createBufferSource();
src.buffer = buf;
src.connect(ctx.destination);
src.start(0);
// resume() is async; return the promise so callers can chain.
return ctx.state === 'suspended' ? ctx.resume() : Promise.resolve();
function unlockAudio(){{
// Synchronous silent 1-sample buffer — the only reliable iOS unlock.
// Must be called synchronously at the start of a user gesture handler.
const ctx=getCtx();
const buf=ctx.createBuffer(1,1,ctx.sampleRate);
const src=ctx.createBufferSource();
src.buffer=buf; src.connect(ctx.destination); src.start(0);
if(ctx.state==='suspended') ctx.resume(); // fire-and-forget
}}
function beep(freq, dur, vol) {{
try {{
const ctx = getCtx();
const o = ctx.createOscillator();
const g = ctx.createGain();
function beep(freq,dur,vol){{
try{{
const ctx=getCtx(),o=ctx.createOscillator(),g=ctx.createGain();
o.connect(g); g.connect(ctx.destination);
o.type = 'sine';
o.frequency.setValueAtTime(freq, ctx.currentTime);
g.gain.setValueAtTime(vol, ctx.currentTime);
g.gain.exponentialRampToValueAtTime(0.001, ctx.currentTime + dur);
o.start(); o.stop(ctx.currentTime + dur);
}} catch(e) {{ console.warn('Audio beep:', e); }}
o.type='sine'; o.frequency.setValueAtTime(freq,ctx.currentTime);
g.gain.setValueAtTime(vol,ctx.currentTime);
g.gain.exponentialRampToValueAtTime(0.001,ctx.currentTime+dur);
o.start(); o.stop(ctx.currentTime+dur);
}}catch(e){{console.warn('Audio:',e);}}
}}
function warningChime() {{
beep(660, 0.3, 0.4);
setTimeout(() => beep(440, 0.5, 0.4), 350);
function warningChime(){{
beep(660,0.3,0.4); setTimeout(()=>beep(440,0.5,0.4),350);
}}
function staleChime(){{
beep(1200,0.12,0.35);
setTimeout(()=>beep(1200,0.12,0.35),180);
setTimeout(()=>beep(1200,0.25,0.35),360);
}}
function staleChime() {{
beep(1200, 0.12, 0.35);
setTimeout(() => beep(1200, 0.12, 0.35), 180);
setTimeout(() => beep(1200, 0.25, 0.35), 360);
function testSound(){{
// Gesture handler — unlock first, then delay beeps 80ms for resume().
unlockAudio();
setTimeout(()=>beep(880, 0.15,0.4), 80);
setTimeout(()=>beep(1100,0.15,0.4),260);
setTimeout(()=>beep(880, 0.3, 0.4),440);
}}
function testSound() {{
// Called directly from onclick — gesture is active here.
unlockAudio().then(() => {{
beep(880, 0.15, 0.4);
setTimeout(() => beep(1100, 0.15, 0.4), 180);
setTimeout(() => beep(880, 0.3, 0.4), 360);
}});
function toggleAudio(){{
// Gesture handler — unlock first (synchronous), then do logic.
unlockAudio();
audioEnabled=!audioEnabled;
localStorage.setItem('audioEnabled',audioEnabled);
if(!audioEnabled){{
stopWarning(); audioArmed=false; warningActive=false;
document.getElementById('btn-confirm').style.display='none';
stopStaleWarning(); staleActive=false; staleConfirmed=false;
document.getElementById('btn-confirm-stale').style.display='none';
}} else {{
if(lastStatus==='scanning' && !audioArmed) audioArmed=true;
}}
updateAudioUI();
}}
function toggleAudio() {{
// Called directly from onclick — gesture is active here.
unlockAudio().then(() => {{
audioEnabled = !audioEnabled;
localStorage.setItem('audioEnabled', audioEnabled);
if (!audioEnabled) {{
stopWarning();
audioArmed = false; warningActive = false;
document.getElementById('btn-confirm').style.display = 'none';
stopStaleWarning();
staleActive = false; staleConfirmed = false;
document.getElementById('btn-confirm-stale').style.display = 'none';
}} else {{
if (lastStatus === 'scanning' && !audioArmed) audioArmed = true;
}}
updateAudioUI();
}});
}}
function confirmWarning(){{
stopWarning();
warningActive=false;
@@ -1759,9 +1845,10 @@ function confirmWarning(){{
}}
function startWarning(){{
// Not a gesture handler — context already unlocked by Enable button click.
if(warningActive) return;
warningActive=true;
if(audioEnabled) warningChime(); // returns promise; chime plays after unlock
if(audioEnabled) warningChime();
warningTimer=setInterval(()=>{{ if(audioEnabled) warningChime(); }},30000);
document.getElementById('btn-confirm').style.display='inline-block';
updateAudioUI();
@@ -1780,9 +1867,10 @@ function confirmStale(){{
}}
function startStaleWarning(){{
// Not a gesture handler — context already unlocked by Enable button click.
if(staleActive || staleConfirmed) return;
staleActive=true;
if(audioEnabled) staleChime(); // returns promise; chime plays after unlock
if(audioEnabled) staleChime();
staleTimer=setInterval(()=>{{ if(audioEnabled) staleChime(); }},30000);
document.getElementById('btn-confirm-stale').style.display='inline-block';
updateAudioUI();
@@ -1964,8 +2052,7 @@ function render(d){{
document.getElementById('pi-type').textContent=p.tomo_type||'-';
document.getElementById('pi-eta').textContent=p.estimated_remaining_human||'-';
document.getElementById('pi-start').textContent=fmtTime(p.tomo_start_time);
document.getElementById('bar-sub-label').textContent=(p.subtomo_projection||0)+' / '+(p.subtomo_total_projections||0);
document.getElementById('bar-sub-fill').style.width=(sPct*100).toFixed(1)+'%';
if(d.recon){{
document.getElementById('recon-waiting').textContent=d.recon.waiting;
const fv=document.getElementById('recon-failed');
@@ -1990,7 +2077,7 @@ function render(d){{
async function poll(){{
try{{
const r=await fetch(STATUS_JSON+'?t='+Date.now());
const r=await fetch(STATUS_JSON, {{cache:'no-store'}});
if(!r.ok) throw new Error('HTTP '+r.status);
render(await r.json());
}}catch(e){{
@@ -2011,4 +2098,4 @@ poll();
</script>
</body>
</html>
"""
"""