use http.HTTPStatus for readability

.gitignore

@@ -138,6 +138,9 @@ dmypy.json
 cython_debug/
 
 # stand
+adb/
+.nicegui/
+.secret
 *.h5
 *.xls?
 

.streamlit/config.toml

@@ -1,5 +0,0 @@
-[browser]
-
-gatherUsageStats = false
-
-

README.md

@@ -1,54 +1,30 @@
 # stand
 
-An editable, filterable and sortable spreadsheet (via [Ag Grid](https://github.com/PablocFonseca/streamlit-aggrid/)) webapp (via [Streamlit](https://streamlit.io/)) with REST API (via [CherryPy](https://cherrypy.dev/)) using [Pandas](https://pandas.pydata.org/) [DataFrames](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).
+An editable, filterable and sortable spreadsheet (via [AG Grid](https://www.ag-grid.com/)) webapp (via [NiceGUI](https://nicegui.io/)) with REST API (via [FastAPI](https://fastapi.tiangolo.com/)) using [Pandas](https://pandas.pydata.org/) [DataFrames](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) backed by [ArcticDB](https://arcticdb.io/).
-
-<img src="https://gitea.psi.ch/SwissFEL/stand/wiki/raw/uploads%2Fea8ad99871c9f2fd8fcdff38da028211%2Fscreenshot.png" width="50%" />
 
 ## Setup the environment
 
 ```bash
-conda create -n stand
+pixi install
-conda activate stand
-
-conda install cherrypy
-conda install streamlit=1.9.2 altair=4
-conda install streamlit-aggrid=0.3.4.post3
-conda install openpyxl # excel writing
-conda install pytables # hdf5 writing
 ```
 
 ## Start the webapp
 
 ```bash
-$ ./stand.py
+pixi run stand
 ```
 
-Check the parameters with
+or
 
 ```bash
-$ ./stand.py -h
+pixi shell
+./stand/main.py
 ```
 
+
 ## Interact in the browser
 
 - **edit**: double click a cell
-
+- **filter**: click the filter icon on a column head
-- **filter**: hover a column head and click the appearing menu icon
-
 - **sort**: click a column head (cycles through ascending, descending, chronological)
 
-- **download**: click the *Downloads* button and choose from the given options
-
-## Use the python client
-
-```python
-from client import Client
-
-c = Client()
-
-c.add_row(a=1, b=2.3, c="three")
-c.add_row(a=2, b=3.4, c="four", x=123)
-
-c.clear()
-```
-

check_ldap.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+import argparse
+import timeit
+from functools import partial
+from getpass import getpass
+
+from stand.auth.psildap import get_data
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("username")
+clargs = parser.parse_args()
+
+username = clargs.username
+password = getpass()
+
+data = get_data(username, password)
+print(data)
+
+
+times = timeit.repeat(partial(get_data, username, password), number=1, repeat=5)
+
+for i, t in enumerate(times, 1):
+    print(f"run {i}:   {t:.6f}s")
+
+avg = sum(times) / len(times)
+print(f"average: {avg:.6f}s")
+
+
+

client.py

@@ -1,42 +0,0 @@
-import requests
-import json
-
-
-class Client:
-
-    def __init__(self, host="127.0.0.1", port=9090):
-        self.host = host
-        self.port = port
-        self.session = requests.Session()
-
-    def add_row(self, **kwargs):
-        addr = self._make_addr()
-        resp = self.session.patch(addr, json=kwargs)
-        return ResponseWrapper(resp)
-
-    def get(self):
-        addr = self._make_addr()
-        resp = self.session.get(addr)
-        return ResponseWrapper(resp)
-
-    def clear(self):
-        addr = self._make_addr()
-        resp = self.session.delete(addr)
-        return ResponseWrapper(resp)
-
-    def _make_addr(self):
-        return f"http://{self.host}:{self.port}/"
-
-
-
-class ResponseWrapper:
-
-    def __init__(self, resp):
-        resp.raise_for_status()
-        self.resp = resp
-
-    def __repr__(self):
-        return self.resp.text
-
-
-

pixi.toml

@@ -0,0 +1,17 @@
+[workspace]
+authors = ["Sven Augustin <sven.augustin@psi.ch>"]
+channels = ["conda-forge"]
+name = "stand"
+platforms = ["linux-64"]
+version = "0.0.2"
+
+[tasks]
+stand = "stand/main.py"
+
+[dependencies]
+python = ">=3.13,<3.14"
+nicegui = ">=3.11.0,<4"
+arcticdb = ">=6.13.0,<7"
+python-gssapi = ">=1.11.1,<2"
+ldap3 = ">=2.9.1,<3"
+

print_adb.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import arcticdb as adb
+
+
+uri = "lmdb://adb"
+ac = adb.Arctic(uri)
+libs = ac.list_libraries()
+
+for n in sorted(libs):
+    print(n)
+    print("=" * len(n))
+
+    lib = ac.get_library(n)
+    symbols = lib.list_symbols()
+    metas = [lib.read_metadata(i) for i in sorted(symbols)] or ["nothing"]
+
+    for i in metas:
+        print("-", i)
+
+    print()
+
+
+

stand.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import socket
-
-
-parser = argparse.ArgumentParser(
-    description="✋ stand",
-    formatter_class=argparse.ArgumentDefaultsHelpFormatter
-)
-
-hostname = socket.gethostname()
-
-msg = " of the Streamlit app"
-parser.add_argument("--app-host", default=hostname,       help="host"+msg)
-parser.add_argument("--app-port", default=8080, type=int, help="port"+msg)
-
-msg = " of the CherryPy API"
-parser.add_argument("--api-host", default=hostname,       help="host"+msg)
-parser.add_argument("--api-port", default=9090, type=int, help="port"+msg)
-
-clargs = parser.parse_args()
-
-
-streamlit_params = f"""
-    --browser.gatherUsageStats false
-    --server.headless true
-    --server.address {clargs.app_host}
-    --server.port {clargs.app_port}
-""".split()
-
-cherrypy_params = f"""
-    --host {clargs.api_host}
-    --port {clargs.api_port}
-""".split()
-
-
-import sys
-
-# combine the folder that contains this script
-
-base = sys.path[0] or "."
-script = f"{base}/stand/webapp.py"
-
-
-import subprocess
-
-# arguments before "--" will be parsed by streamlit
-# arguments after  "--" will be passed into the script
-# the inner argparse is in restapi.py
-
-cmd = [
-#    "echo",
-    "streamlit", "run", *streamlit_params, script,
-    "--", *cherrypy_params
-]
-
-subprocess.run(cmd)
-
-
-

stand.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-RESET="\033[0m"
-RED="\033[0;31m"
-echo -e "${RED}please switch to the stand.py launcher${RESET}"
-
-base=$(dirname $0)
-
-params=(
-  --browser.gatherUsageStats false
-  --server.headless true
-)
-
-streamlit run "${params[@]}" $@ $base/stand/webapp.py
-
-

stand/adb.py

@@ -0,0 +1,18 @@
+import arcticdb as adb
+
+
+class ArcticDB(adb.Arctic):
+
+    exceptions = adb.exceptions
+
+    def get(self, name):
+        return self.get_library(
+            name,
+            create_if_missing=True,
+            library_options=adb.LibraryOptions(
+                dynamic_schema=True
+            )
+        )
+
+
+

stand/aggridx.py

@@ -0,0 +1,135 @@
+from nicegui import ui
+
+
+class aggridx(ui.aggrid):
+
+#    #TODO: this should be done in the constructor via a keyword argument switch
+#    #TODO: this would need the from_pandas/from_polars classmethods to pass through that switch
+#    def setup_sync_edits(self):
+#        """
+#        automatically sync data to the server after edits
+#        this operation maintains the current view on the data
+#        """
+#        self.on("cellValueChanged", self.set_cell_server)
+
+
+    def set_cell_server(self, row_index, col_id, new_val):
+        with self.props.suspend_updates():
+            self.options["rowData"][row_index][col_id] = new_val
+
+    def set_cell_client(self, row_id, col_id, new_val):
+        self.run_row_method(row_id, "setDataValue", col_id, new_val)
+
+
+#    def set_row_server_from_event(self, evt):
+#        index = evt.args["rowIndex"]
+#        data = evt.args["data"]
+#        self.set_row_server(index, data)
+
+#    def set_row_server(self, index, data):
+#        """
+#        overwrite row at index with data
+#        this operation maintains the current view on the data
+#        """
+#        with self.props.suspend_updates():
+#            self.options["rowData"][index] = data
+
+
+    def append(self, row):
+        """
+        append row to the bottom of the table and
+        append any missing columns on the right side of the table
+        this operation maintains the current view on the data
+        """
+        self.ensure_column_defs(row)
+        self.add_row_data([row])
+
+#    def insert(self, index, row):
+#        """
+#        insert row at the provided index
+#        append any missing columns on the right side of the table
+#        this operation maintains the current view on the data
+#        """
+#        self.ensure_column_defs(row)
+#        self.add_row_data([row], index=index)
+
+#    def extend(self, rows):
+#        """
+#        append rows to the bottom of the table and
+#        append any missing columns on the right side of the table
+#        this operation maintains the current view on the data
+#        """
+#        columns = dict.fromkeys(k for d in rows for k in d) # dict.fromkeys acts as ordered set
+#        self.ensure_column_defs(columns)
+#        self.add_row_data(rows)
+
+
+    def ensure_column_defs(self, columns):
+        """
+        append any missing columns on the right side of the table
+        this operation maintains the current view on the data
+        """
+        current_column_defs = self.options["columnDefs"]
+        current_fields = [i["field"] for i in current_column_defs]
+        added_fields = [i for i in columns if i not in current_fields]
+        if not added_fields:
+            return
+        added_column_defs = [{"field": n} for n in added_fields]
+        new_column_defs = current_column_defs + added_column_defs
+        # update server without re-draw
+        with self.props.suspend_updates():
+            self.options["columnDefs"] = new_column_defs
+        # update client
+        self.run_grid_method("setGridOption", "columnDefs", new_column_defs)
+
+    def add_row_data(self, rows, index=-1):
+        """
+        insert rows at the provided index,
+        with the default being appending at the bottom of the table
+        this operation maintains the current view on the data
+        """
+        # update server without re-draw
+        with self.props.suspend_updates():
+            self.options["rowData"].extend(rows)
+        # update client
+        self.run_grid_method("applyTransaction", {"add": rows, "addIndex": index})
+
+
+#    def scroll_to_row_index(self, index):
+#        """
+#        scroll the view on the data such that the row with the given index is visible
+#        negative indices count from the back
+#        """
+#        nrows = len(self.options["rowData"])
+#        index %= nrows # move index into [0, nrows)
+#        self.run_grid_method("ensureIndexVisible", index)
+
+#    def scroll_to_row_key(self, key):
+#        """
+#        scroll the view on the data such that the row with the given key is visible
+#        consistent with tables generated from dataframes, the column called index is used to match the key
+#        if the key is not found, the view is not scrolled
+#        """
+#        indices = [i.get("index") for i in self.options["rowData"]]
+#        try:
+#            index = indices.index(key)
+#        except ValueError:
+#            return
+#        self.scroll_to_row_index(index)
+
+#    def scroll_to_column_index(self, index):
+#        """
+#        scroll the view on the data such that the column with the given index is visible
+#        negative indices count from the back
+#        """
+#        key = self.options["columnDefs"][index]["field"]
+#        self.scroll_to_column_key(key)
+
+#    def scroll_to_column_key(self, key):
+#        """
+#        scroll the view on the data such that the column with the given key is visible
+#        """
+#        self.run_grid_method("ensureColumnVisible", key)
+
+
+

stand/api.py

@@ -0,0 +1,43 @@
+from datetime import datetime
+from http import HTTPStatus
+from typing import Any
+
+import pandas as pd
+from fastapi import APIRouter, HTTPException
+
+from state import grids, adb
+from utils.annotations import Beamline, PGroup
+
+
+router = APIRouter()
+
+
+@router.post("/tables/{pgroup}/append")
+def append(beamline: Beamline, pgroup: PGroup, run: int, row: dict[str, Any]):
+    lib = adb.get(beamline)
+
+    index = pd.Timestamp(run) # adb supports update only for timeseries indexes
+    index = pd.Index([index], name="run")
+
+    run_exists = not lib.read(pgroup, date_range=index, columns=[]).data.index.empty
+    if run_exists:
+        raise HTTPException(HTTPStatus.CONFLICT, f"run {run} exists already in {beamline}/{pgroup}")
+
+    timestamp = row.pop("timestamp", None)
+    timestamp = datetime.fromisoformat(timestamp) if timestamp else datetime.now()
+    row = {"timestamp": timestamp, **row} # setdefault would not force timestamp to be the first column
+
+    df = pd.DataFrame(row, index=index)
+    lib.append(pgroup, df)
+
+    row = {"run": run, **row} # setdefault would not force run to be the first column
+
+    res = []
+    for grid in grids[pgroup]:
+        grid.append(row)
+        res.append(grid.options)
+
+    return res
+
+
+

stand/auth/fakeldap.py

@@ -0,0 +1,28 @@
+PASSWORDS = {
+    "a": "a",
+    "b": "b",
+    "c": "c"
+}
+
+PGROUPS = {
+    "a": {
+        "p11111": "stand",
+        "p22222": "SATESE"
+    },
+    "b": {
+        "p33333": "SARESA"
+    }
+}
+
+
+def get_data(username, password):
+    if PASSWORDS.get(username) != password:
+        raise RuntimeError(f'authentication failed for user "{username}"')
+
+    return {
+        "username": username,
+        "pgroups": PGROUPS.get(username, {})
+    }
+
+
+

stand/auth/krbldap.py

@@ -0,0 +1,26 @@
+import gssapi
+from ldap3 import GSSAPI, NONE, ROUND_ROBIN, SASL, Connection, Server, ServerPool
+
+
+def Konnection(hosts, username, password):
+    server_pool = make_server_pool(hosts)
+    creds = make_creds(username, password)
+    return Connection(
+        server_pool,
+        authentication=SASL,
+        sasl_mechanism=GSSAPI,
+        sasl_credentials=(None, None, creds)
+    )
+
+
+def make_server_pool(hosts):
+    servers = [Server(host, get_info=NONE, connect_timeout=3) for host in hosts]
+    return ServerPool(servers, ROUND_ROBIN, active=3, exhaust=True)
+
+
+def make_creds(username, password):
+    user = gssapi.Name(base=username, name_type=gssapi.NameType.user)
+    return gssapi.raw.acquire_cred_with_password(user, password.encode()).creds
+
+
+

stand/auth/login.py

@@ -0,0 +1,57 @@
+import logging
+
+from fastapi.responses import RedirectResponse
+from nicegui import APIRouter, app, ui
+
+from state import config
+
+
+log = logging.getLogger(__name__)
+router = APIRouter()
+
+
+if config.fake:
+    log.warning("using fake LDAP")
+    from .fakeldap import get_data
+else:
+    from .psildap import get_data
+
+
+@router.page("/login")
+def login(redirect_to: str = "/") -> RedirectResponse | None:
+    if app.storage.user.get("authenticated"):
+        return RedirectResponse("/")
+
+    def try_login(): # local function to avoid passing username and password as arguments
+        if not username.value:
+            ui.notify("Missing username", color="negative")
+            username.run_method("focus")
+            return
+
+        if not password.value:
+            password.run_method("focus")
+            return
+
+        try:
+            data = get_data(username.value, password.value)
+        except Exception:
+            log.exception(f'login failed for user "{username.value}"')
+            ui.notify("Wrong username or password", color="negative")
+            return
+
+        app.storage.user.update(data, authenticated=True)
+
+        ui.navigate.to(redirect_to) # go back to where the user wanted to go
+
+    with ui.card().classes("absolute-center items-stretch"):
+        ui.image("assets/icon.png")
+        username = ui.input("Username").props("autofocus")
+        password = ui.input("Password", password=True, password_toggle_button=True)
+        username.on("keydown.enter", try_login)
+        password.on("keydown.enter", try_login)
+        ui.button("log in", icon="login", on_click=try_login)
+
+    return None
+
+
+

stand/auth/logout.py

@@ -0,0 +1,9 @@
+from nicegui import app, ui
+
+
+def logout():
+    app.storage.user.clear()
+    ui.navigate.to("/login")
+
+
+

stand/auth/mw.py

@@ -0,0 +1,30 @@
+from fastapi import Request
+from fastapi.responses import RedirectResponse
+from nicegui import app
+from starlette.middleware.base import BaseHTTPMiddleware
+
+
+unrestricted_page_routes = {"/favicon.ico", "/login"}
+
+
+class AuthMiddleware(BaseHTTPMiddleware):
+    """
+    This middleware restricts access to all NiceGUI pages.
+    It redirects the user to the login page if they are not authenticated.
+    """
+
+    async def dispatch(self, request: Request, call_next):
+        path = request.url.path
+
+        if (
+            app.storage.user.get("authenticated")
+            or path in unrestricted_page_routes
+            or path.startswith("/_nicegui")
+            or path.startswith("/api/")
+        ):
+            return await call_next(request)
+
+        return RedirectResponse(f"/login?redirect_to={path}")
+
+
+

stand/auth/psildap.py

@@ -0,0 +1,96 @@
+import re
+
+from ldap3 import BASE, SUBTREE
+
+from .krbldap import Konnection
+
+
+USER_ATTRIBUTE_MAP = {
+    "cn": "username",
+    "displayName": "display name",
+    "givenName": "first name",
+#    "mail": "email",
+#    "sn": "last name"
+}
+
+USER_ATTRIBUTES = [*USER_ATTRIBUTE_MAP, "memberOf"]
+COMMON_USER_DN = "OU=users,OU=psi,DC=d,DC=psi,DC=ch"
+BASE_PGROUP_DN = "OU=Groups,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch"
+PGROUP_PATTERN = re.compile(r"CN=(p\d{5}),OU=Groups,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch")
+BEAMLINE_PATTERN = re.compile(r"CN=([^,]+),OU=Beamlines,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch")
+SERVERS = [f"dc{i:02}.d.psi.ch" for i in range(3)]
+
+
+def get_data(username, password):
+    with Konnection(SERVERS, username, password) as conn:
+        conn.search(
+            make_user_search_base(username),
+            search_filter="(objectClass=*)",
+            search_scope=BASE,
+            attributes=USER_ATTRIBUTES
+        )
+
+        entry = ensure_single(conn.entries, "users")
+        res = repack_user(entry)
+
+        res_username = res["username"]
+        if res_username != username:
+            raise RuntimeError(f"username mismatch: {res_username} != {username}")
+
+        conn.search(
+            BASE_PGROUP_DN,
+            search_filter=make_pgroups_search_filter(res["pgroups"]),
+            search_scope=SUBTREE,
+            attributes=["cn", "memberOf"]
+        )
+
+        res["pgroups"] = repack_pgroups(conn.entries)
+
+    return res
+
+
+def make_user_search_base(cn):
+    res = [f"CN={cn}"]
+    if cn.startswith("ext-"):
+        res.append("OU=external")
+    elif cn.startswith("gac-"):
+        res.append("OU=nonpersons")
+    res.append(COMMON_USER_DN)
+    return ",".join(res)
+
+
+def repack_user(entry):
+    attrs = entry.entry_attributes_as_dict
+    res = {USER_ATTRIBUTE_MAP[old]: item[0] for old, item in attrs.items() if old in USER_ATTRIBUTE_MAP if item}
+    res["pgroups"] = extract_matches(attrs["memberOf"], PGROUP_PATTERN)
+    return res
+
+
+def make_pgroups_search_filter(pgroups):
+    search_filter = "".join(f"(cn={cn})" for cn in pgroups)
+    return f"(|{search_filter})"
+
+
+def repack_pgroups(entries):
+    res = {}
+    for entry in entries:
+        attrs = entry.entry_attributes_as_dict
+        beamlines = extract_matches(attrs["memberOf"], BEAMLINE_PATTERN)
+        beamline = ensure_single(beamlines, "beamlines")
+        pgroup = ensure_single(attrs["cn"], "pgroup CNs")
+        res[pgroup] = beamline
+    return res
+
+
+def ensure_single(seq, what):
+    n = len(seq)
+    if n == 1:
+        return seq[0]
+    raise RuntimeError(f"received ambiguous {what} ({n} entries)")
+
+
+def extract_matches(seq, pattern):
+    return sorted(m.group(1) for i in seq if (m := pattern.search(i)))
+
+
+

stand/auth/secret.py

@@ -0,0 +1,23 @@
+import os
+import secrets
+
+
+def get_secret(fn=".secret"):
+    if os.path.exists(fn):
+        res = read(fn)
+    else:
+        res = secrets.token_hex(32)
+        write(res, fn)
+    return res.strip()
+
+
+def read(fn):
+    with open(fn) as f:
+        return f.read()
+
+def write(s, fn):
+    with open(fn, "w") as f:
+        f.write(s)
+
+
+

stand/beamline.py

@@ -0,0 +1,19 @@
+
+SUBNET_TO_BEAMLINE = {
+    "129.129.242": "SARESA",
+    "129.129.243": "SARESB",
+    "129.129.244": "SARESC",
+    "129.129.245": "SATESD",
+    "129.129.246": "SATESE",
+    "129.129.247": "SATESF"
+}
+
+
+def get_beamline(ip):
+    if ip == "127.0.0.1":
+        return "stand"
+    subnet = ".".join(ip.split(".")[:3])
+    return SUBNET_TO_BEAMLINE[subnet]
+
+
+

stand/cli.py

@@ -0,0 +1,11 @@
+import argparse
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-f", "--fake", action="store_true")
+
+clargs = parser.parse_args()
+
+
+

stand/hacks/__init__.py

@@ -1,4 +0,0 @@
-
-from . import patch_st_cp_stop
-
-

stand/hacks/patch_st_cp_stop.py

@@ -1,21 +0,0 @@
-"""
-Monkey patch the streamlit server to stop cherrypy upon its own stop
-"""
-
-
-import cherrypy
-from streamlit.server.server import Server
-
-
-old_fn = Server._on_stopped
-
-def new_fn(*args, **kwargs):
-    print("exit cherrypy")
-    cherrypy.engine.exit()
-    print("exit streamlit")
-    return old_fn(*args, **kwargs)
-
-Server._on_stopped = new_fn
-
-
-

stand/home.py

@@ -0,0 +1,25 @@
+from nicegui import APIRouter, app, ui
+
+from auth.logout import logout
+
+
+router = APIRouter()
+
+
+@router.page("/")
+def home():
+    with ui.column().classes("absolute-center items-center"):
+        username = app.storage.user.get("username", "unknown user")
+        ui.label(f"Hello {username}!").classes("text-2xl")
+        ui.button("log out", icon="logout", on_click=logout)
+
+        pgroups = app.storage.user.get("pgroups", {})
+        ui.select(
+            label="pgroup",
+            options=sorted(pgroups, reverse=True),
+            with_input=True,
+            on_change=lambda e: ui.navigate.to(f"/tables/{e.value}")
+        )
+
+
+

stand/logcfg.py

@@ -0,0 +1,29 @@
+import logging
+
+from uvicorn.config import LOGGING_CONFIG
+from uvicorn.logging import ColourizedFormatter
+
+
+LOGFMT = "%(asctime)s %(levelprefix)s [%(name)s] %(message)s"
+
+
+def logcfg(level=logging.INFO, fmt=LOGFMT, **kwargs):
+    formatter = ColourizedFormatter(fmt=fmt, **kwargs)
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+
+    logging.basicConfig(level=level, handlers=[handler])
+
+    # add timestamps to uvicorn logs
+    formatters = LOGGING_CONFIG["formatters"]
+    for formatter in formatters.values():
+        if "fmt" not in formatter:
+            continue
+        fmt = formatter["fmt"]
+        if "%(asctime)s" in fmt:
+            continue
+        formatter["fmt"] = "%(asctime)s " + fmt
+
+
+

stand/main.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+from nicegui import app, ui
+
+from cli import clargs
+from state import config
+config.update(vars(clargs))
+
+from logcfg import logcfg
+logcfg()
+
+from api import router as api_router
+from auth.login import router as login_router
+from auth.mw import AuthMiddleware
+from auth.secret import get_secret
+from home import router as home_router
+from table import router as table_router
+
+
+app.include_router(login_router)
+app.add_middleware(AuthMiddleware)
+
+app.include_router(api_router, prefix="/api")
+app.include_router(home_router)
+app.include_router(table_router)
+
+
+ui.run(
+    title="stand",
+    favicon="assets/favicon.png",
+    storage_secret=get_secret(),
+    fastapi_docs=True,
+    dark=True,
+    show=False
+)
+
+
+

stand/restapi.py

@@ -1,44 +0,0 @@
-import argparse
-from threading import Thread
-
-import cherrypy as cp
-
-from tblctrl import TableController
-
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument("--host", default="127.0.0.1")
-parser.add_argument("--port", default=9090, type=int)
-
-clargs = parser.parse_args()
-
-
-cp.config.update({
-    "server.socket_host": clargs.host,
-    "server.socket_port": clargs.port,
-})
-
-
-# creating instances here allows to use restapi etc. as singletons
-
-print(f">>> start restapi on {clargs.host}:{clargs.port}")
-
-restapi = TableController()
-
-conf = {
-    "/": {
-        "request.dispatch": cp.dispatch.MethodDispatcher(),
-        "tools.sessions.on": True,
-        "tools.response_headers.on": True,
-        "tools.response_headers.headers": [("Content-Type", "text/plain")],
-    }
-}
-
-args = (restapi, "/", conf)
-
-thread = Thread(target=cp.quickstart, args=args, daemon=True)
-thread.start()
-
-
-

stand/state.py

@@ -0,0 +1,20 @@
+from collections import defaultdict
+
+from adb import ArcticDB
+
+from utils.config import Config
+from utils.registry import Registry
+
+
+adb = ArcticDB("lmdb://adb")
+
+
+config = Config(
+    fake=False
+)
+
+
+grids = defaultdict(Registry)
+
+
+

stand/table.py

@@ -0,0 +1,118 @@
+import logging
+
+import pandas as pd
+from nicegui import APIRouter, app, ui
+
+from aggridx import aggridx
+from state import grids, adb
+from utils.annotations import PGroup
+
+
+OPTIONS = {
+    ":getRowId": "(params) => params.data.run", # set row ID to index column
+    "context": {
+        "beamline": None,
+        "pgroup": None
+    },
+    "defaultColDef": {
+        "filter": True,
+        "editable": True,
+        "sortable": True,
+        "resizable": True
+    },
+    "pagination": True,
+    "paginationAutoPageSize": True,
+    "theme": "balham"
+}
+
+
+log = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.page("/tables/{pgroup}")
+def table(pgroup: PGroup):
+    pgroups = app.storage.user.get("pgroups", {})
+    if pgroup in pgroups:
+        beamline = pgroups[pgroup]
+        table_show(pgroup, beamline)
+    else:
+        table_deny(pgroup)
+
+
+def table_show(pgroup, beamline):
+#    with ui.left_drawer(value=False) as ld:
+#        dark = ui.dark_mode(value=True)
+#        ui.switch("dark mode").bind_value(dark)
+
+#    with ui.page_sticky(position="left", x_offset=-12.5):
+#        def cb():
+#            ld.toggle()
+#            btn.icon = "sym_o_left_panel_close" if ld.value else "sym_o_left_panel_open"
+#        btn = ui.button(icon="sym_o_left_panel_open", on_click=cb).props("flat dense")
+
+    try:
+        df = adb.get(beamline).read(pgroup).data
+    except adb.exceptions.NoSuchVersionException:
+        df = pd.DataFrame()
+
+    df.index = df.index.view("int64") # adb supports update only for timeseries indexes
+    df.index.name = "run"
+    df = df.reset_index()
+
+    options = OPTIONS.copy()
+    options["context"].update(
+        beamline=beamline,
+        pgroup=pgroup
+    )
+
+    grid = aggridx.from_pandas(df, options=options)
+    grid.classes("h-[calc(100vh-2rem)]") # full height minus padding
+    grid.on("cellValueChanged", update_adb)
+    grid.on("cellValueChanged", update_grids)
+    grids[pgroup].add(grid)
+
+
+def table_deny(pgroup):
+    username = app.storage.user.get("username", "unknown user")
+    log.error(f'access to pgroup {pgroup} denied for user "{username}"')
+    with ui.column().classes("absolute-center items-center gap-8"):
+        ui.icon("sym_o_front_hand", size="xl")
+        ui.label(f"access to {pgroup} denied").classes("text-2xl")
+
+
+def update_adb(evt):
+    if evt.args.get("source") != "edit":
+        # ignore event if it was no direct edit
+        return
+
+    beamline = evt.args["context"]["beamline"]
+    pgroup = evt.args["context"]["pgroup"]
+    row_id = evt.args["rowId"]
+    col_id = evt.args["colId"]
+    new_val = evt.args["newValue"]
+
+    index = pd.Timestamp(int(row_id)) # adb supports update only for timeseries indexes
+    lib = adb.get(beamline)
+    df = lib.read(pgroup, date_range=[index]).data
+    df.at[index, col_id] = new_val
+    lib.update(pgroup, df)
+
+
+def update_grids(evt):
+    sender = evt.sender
+    pgroup = evt.args["context"]["pgroup"]
+    row_index = evt.args["rowIndex"]
+    row_id = evt.args["rowId"]
+    col_id = evt.args["colId"]
+    new_val = evt.args["newValue"]
+
+    for grid in grids[pgroup]:
+        grid.set_cell_server(row_index, col_id, new_val)
+        if grid == sender:
+            # the sender is already up-to-date
+            continue
+        grid.set_cell_client(row_id, col_id, new_val)
+
+
+

stand/tblctrl.py

@@ -1,63 +0,0 @@
-import cherrypy as cp
-
-from utils.dfh import DateFrameHolder
-from utils.st_utils import rerunall, get_session_ids
-
-
-@cp.expose
-class TableController:
-
-    def __init__(self):
-        self.dfh = DateFrameHolder("output.h5")
-#        self.changed = True
-        self.sids = get_session_ids()
-
-    def get(self, sid):
-        changed = self.get_changed(sid)
-        data = self.get_data(sid)
-        return changed, data
-
-    def get_changed(self, sid):
-        return (sid in self.sids)
-
-    def get_data(self, sid):
-#        self.changed = False
-        try:
-            self.sids.remove(sid)
-        except KeyError as e:
-            if self.sids:
-                raise RuntimeError(f"{self.sids} is not empty") from e
-        return self.dfh.df
-
-    def set_data(self, df):
-        if self.dfh.df.equals(df):
-            print("<<< skipping dump because dataframe did not change")
-            return
-        self.dfh.df = df
-        self.dfh.dump()
-        self._trigger_changed()
-
-    def GET(self):
-        return str(self.dfh.df)
-
-    @cp.tools.json_in()
-    def PATCH(self, **kwargs):
-        kwargs = kwargs or cp.request.json
-        self.dfh.append(kwargs)
-        self._trigger_changed()
-        self.dfh.dump()
-        return str(self.dfh.df)
-
-    def DELETE(self):
-        self.dfh.clear()
-        self._trigger_changed()
-        return "cleared"
-
-
-    def _trigger_changed(self):
-#        self.changed = True
-        self.sids = get_session_ids()
-        rerunall()
-
-
-

stand/utils/annotations.py

@@ -0,0 +1,17 @@
+from typing import Annotated
+
+from fastapi import Depends, Path, Request
+
+from beamline import get_beamline
+
+
+def beamline_dependency(request: Request) -> str:
+    return get_beamline(request.client.host)
+
+Beamline = Annotated[str, Depends(beamline_dependency)]
+
+
+PGroup = Annotated[str, Path(pattern=r"^p\d{5}$")]
+
+
+

stand/utils/config.py

@@ -0,0 +1,8 @@
+
+class Config(dict):
+    __getattr__ = dict.__getitem__
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+
+

stand/utils/df2bin.py

@@ -1,27 +0,0 @@
-from io import BytesIO
-
-
-def to_excel_binary(df, **kwargs):
-    with BytesIO() as out:
-        df.to_excel(out, **kwargs)
-        res = out.getvalue()
-    return res
-
-
-
-from pandas import HDFStore
-
-
-def to_hdf_binary(df):
-    with HDFStore(
-        "wontbewritten.h5",
-        mode="a",
-        driver="H5FD_CORE",
-        driver_core_backing_store=0
-    ) as out:
-        out["/data"] = df
-        res = out._handle.get_file_image()
-    return res
-
-
-

stand/utils/dfh.py

@@ -1,72 +0,0 @@
-import pandas as pd
-
-
-class DateFrameHolder:
-
-    def __init__(self, fn):
-        self.fn = fn
-        self.df = try_load_df(fn)
-
-    def dump(self):
-        dump_non_empty_df(self.df, self.fn)
-
-
-    def append(self, data):
-        data = pd.DataFrame.from_records([data])
-        self.df = pd.concat([self.df, data], ignore_index=True)
-
-    def clear(self):
-        self.df = pd.DataFrame()
-
-
-    def __repr__(self):
-        head = self.fn + ":"
-        line = "-" * len(head)
-        df = str(self.df)
-        return "\n".join((head, line, df))
-
-
-
-def try_load_df(fn):
-    try:
-        df = pd.read_hdf(fn)
-        print(f">>> loaded dataframe from {fn}")
-    except FileNotFoundError:
-        df = pd.DataFrame()
-        print(">>> created empty dataframe")
-    return df
-
-def dump_non_empty_df(df, fn, key="data"):
-    if df.empty:
-        print("<<< skip dumping empty dataframe")
-        return
-    print(f"<<< dump dataframe to {fn}")
-    df.to_hdf(fn, key=key)
-    backup(df, fn, key) #TODO: test then remove
-
-
-
-# the following is just a safety precaution
-# if everything works, it can be removed
-
-from datetime import datetime
-from pathlib import Path
-
-FMT = "%Y-%m-%d_%H-%M-%S-%f"
-BAKDIR = ".backup"
-
-def backup(df, fn, key):
-    Path(BAKDIR).mkdir(parents=True, exist_ok=True)
-    bfn = backup_filename(fn)
-    print(f"<<< backup dataframe to {bfn}")
-    df.to_hdf(bfn, key=key)
-
-def backup_filename(fn):
-    p = Path(fn)
-    fn = p.stem
-    ext = p.suffix
-    ts = datetime.now().strftime(FMT)
-    return f"{BAKDIR}/{fn}_{ts}{ext}"
-
-
-

stand/utils/registry.py

@@ -0,0 +1,18 @@
+from nicegui import ui
+
+
+class Registry:
+
+    def __init__(self):
+        self.data = {}
+
+    def __iter__(self):
+        return iter(self.data.values())
+
+    def add(self, widget):
+        client = ui.context.client
+        self.data[client.id] = widget
+        client.on_disconnect(lambda: self.data.pop(client.id, None))
+
+
+

stand/utils/st_utils.py

@@ -1,64 +0,0 @@
-import streamlit as st
-
-
-get_server = st.server.server.Server.get_current
-
-
-def rerunall():
-    sibi = get_session_info_by_id()
-    for si in sibi.values():
-        print("rerun:", si.session.id)
-        client_state = None
-        si.session.request_rerun(client_state)
-
-
-def get_session_ids():
-    sibi = get_session_info_by_id()
-    sids = sibi.keys()
-    return set(sids)
-
-
-def rerun(session_id=None):
-    if session_id is None:
-        session_id = get_session_id()
-
-    server = get_server()
-    session = server.get_session_by_id(session_id)
-
-    client_state = None
-    session.request_rerun(client_state)
-
-
-def get_session_id():
-    ctx = st.scriptrunner.script_run_context.get_script_run_ctx()
-    return ctx.session_id
-
-
-def get_session_info_by_id():
-    server = get_server()
-    return server._session_info_by_id
-
-
-
-def hide_UI_elements(menu=True, header=True, footer=True):
-    HIDDEN = " {visibility: hidden;}"
-    res = []
-
-    if menu:
-        res.append("#MainMenu" + HIDDEN)
-
-    if header:
-        res.append("header" + HIDDEN)
-
-    if footer:
-        res.append("footer" + HIDDEN)
-
-    if not res:
-        return
-
-    res = ["<style>"] + res + ["</style>"]
-    res = "\n".join(res)
-    return st.markdown(res, unsafe_allow_html=True)
-
-
-

stand/webapp.py

@@ -1,84 +0,0 @@
-from pathlib import Path
-
-import streamlit as st
-
-import hacks
-
-from widgets.aggrid import aggrid
-from widgets.download import download
-from restapi import restapi
-from utils.st_utils import get_session_id, hide_UI_elements
-
-
-print(">>> start of streamlit run")
-
-icon = Path(__file__).parent.parent / "icon.png"
-
-st.set_page_config(
-    page_title="stand",
-    page_icon=str(icon),
-    layout="wide",
-    initial_sidebar_state="collapsed"
-)
-
-hide_UI_elements()
-
-
-sid = get_session_id() # rest api needs current session IDs to trigger reruns
-changed, df = restapi.get(sid)
-
-
-download(df)
-
-
-auto_height = st.sidebar.checkbox(
-    "Auto Height",
-    value=False,
-    key="auto_height"
-)
-
-pagination = st.sidebar.checkbox(
-    "Pagination",
-    value=True,
-    disabled=auto_height,
-    key="pagination"
-)
-
-height = st.sidebar.slider(
-    "Grid Height",
-    min_value=100, max_value=1500, value=500, step=10,
-    disabled=auto_height,
-    key="grid_height"
-)
-
-if auto_height:
-    pagination = False # would only show "1 of 1"
-    height = "auto"
-
-
-# if df in restapi changed, we need to reload it into aggrid.
-# if df in restapi did not change, we do not reload to ensure changes in the browser persist.
-
-response = aggrid(
-    df,
-    reload_data=changed,
-    pagination=pagination,
-    height=height,
-    enable_enterprise_modules=False,
-    key="stand"
-)
-
-
-# if we reloaded, aggrid returns the df from before (why?), thus we do not update the restapi.
-# if we did not reload, aggrid may return an updated df from an edit in the browser,
-# and thus we need to update the restapi.
-
-new_df = response["data"]
-if not new_df.equals(df) and not changed:
-    restapi.set_data(new_df)
-
-
-print(">>> end of streamlit run")
-
-
-

stand/widgets/aggrid.py

@@ -1,59 +0,0 @@
-from st_aggrid import AgGrid, GridOptionsBuilder
-
-
-def aggrid(df, reload_data=False, height="auto", pagination=True, key=None, **kwargs):
-    df = df[::-1] # display in reversed chronological order
-
-    gob = GridOptionsBuilder.from_dataframe(
-        df,
-        editable=True,
-        filterable=True,
-        groupable=True,
-        resizable=True,
-        sortable=True
-    )
-
-    auto_height = (height == "auto")
-    gob.configure_auto_height(auto_height)
-    gob.configure_pagination(pagination)
-
-    go = gob.build()
-
-    response = AgGrid(
-        df,
-        go,
-        height=height,
-        theme="streamlit",
-        reload_data=reload_data,
-        key=make_key(key, df, auto_height, pagination),
-        **kwargs
-    )
-
-    df = response.get("data", df)
-    df = df[::-1] # undo reversed chronological order
-
-    #TODO: streamlit-aggrid changed return type
-    if isinstance(response, dict):
-        response["data"] = df # <= 0.2.3.post2
-    else:
-        response.data = df # >= 0.3.0
-
-    return response
-
-
-
-def make_key(prefix, df, auto_height, pagination):
-    """
-    encode the dataframe's column names into a key,
-    as well as the state of the auto height and pagination settings,
-    this triggers a hard reload (like F5) of the AgGrid if the columns/settings change
-    """
-    if prefix is None:
-        return None
-    items = list(df.columns)
-    items.append(auto_height)
-    items.append(pagination)
-    return str(prefix) + ":" + "+".join(str(i) for i in items)
-
-
-

stand/widgets/download.py

@@ -1,31 +0,0 @@
-import streamlit as st
-
-from utils.df2bin import to_excel_binary, to_hdf_binary
-
-
-state = st.session_state
-
-
-def download(df):
-    showing_downloads = state.get("showing_downloads", False)
-
-    col0, col1, col2, col3 = st.columns(4)
-    if col0.button("Downloads") and not showing_downloads:
-        state.showing_downloads = True
-
-        h5 = to_hdf_binary(df)
-        col1.download_button("📥 hdf5", h5, file_name="output.h5")
-
-        xlsx = to_excel_binary(df)
-        col2.download_button("📥 xlsx", xlsx, file_name="output.xlsx")
-
-        csv = df.to_csv()
-        col3.download_button("📥 csv", csv, file_name="output.csv")
-
-#        st.stop()
-
-    else:
-        state.showing_downloads = False
-
-
-

test.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-
-from itertools import count, cycle
-from random import random, randint
-from string import ascii_letters
-from time import sleep
-
-from client import Client
-
-
-def irandom():
-    while True:
-        yield random()
-
-num = count()
-ran = irandom()
-let = cycle(ascii_letters)
-
-c = Client()
-
-c.clear()
-sleep(1)
-
-for i, r, l in zip(num, ran, let):
-    l = l * randint(1, 5)
-
-    kwargs = {} if i < 10 else {"x": 123}
-
-    c.add_row(index=i, random=r, string=l, **kwargs)
-    sleep(1)
-
-
-