Merge pull request 'gitlab refs removed' (#1) from gitlab_hunt into master

Reviewed-on: #1

.gitignore

@@ -138,9 +138,6 @@ dmypy.json
 cython_debug/
 
 # stand
-adb/
-.nicegui/
-.secret
 *.h5
 *.xls?
 

.streamlit/config.toml

@@ -0,0 +1,5 @@
+[browser]
+
+gatherUsageStats = false
+
+

README.md

@@ -1,30 +1,54 @@
 # stand
 
-An editable, filterable and sortable spreadsheet (via [AG Grid](https://www.ag-grid.com/)) webapp (via [NiceGUI](https://nicegui.io/)) with REST API (via [FastAPI](https://fastapi.tiangolo.com/)) using [Pandas](https://pandas.pydata.org/) [DataFrames](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) backed by [ArcticDB](https://arcticdb.io/).
+An editable, filterable and sortable spreadsheet (via [Ag Grid](https://github.com/PablocFonseca/streamlit-aggrid/)) webapp (via [Streamlit](https://streamlit.io/)) with REST API (via [CherryPy](https://cherrypy.dev/)) using [Pandas](https://pandas.pydata.org/) [DataFrames](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html).
+
+<img src="https://gitea.psi.ch/SwissFEL/stand/wiki/raw/uploads%2Fea8ad99871c9f2fd8fcdff38da028211%2Fscreenshot.png" width="50%" />
 
 ## Setup the environment
 
 ```bash
-pixi install
+conda create -n stand
+conda activate stand
+
+conda install cherrypy
+conda install streamlit=1.9.2 altair=4
+conda install streamlit-aggrid=0.3.4.post3
+conda install openpyxl # excel writing
+conda install pytables # hdf5 writing
 ```
 
 ## Start the webapp
 
 ```bash
-pixi run stand
+$ ./stand.py
 ```
 
-or
+Check the parameters with
 
 ```bash
-pixi shell
+$ ./stand.py -h
-./stand/main.py
 ```
 
-
 ## Interact in the browser
 
 - **edit**: double click a cell
-- **filter**: click the filter icon on a column head
+
+- **filter**: hover a column head and click the appearing menu icon
+
 - **sort**: click a column head (cycles through ascending, descending, chronological)
 
+- **download**: click the *Downloads* button and choose from the given options
+
+## Use the python client
+
+```python
+from client import Client
+
+c = Client()
+
+c.add_row(a=1, b=2.3, c="three")
+c.add_row(a=2, b=3.4, c="four", x=123)
+
+c.clear()
+```
+

check_ldap.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import timeit
-from functools import partial
-from getpass import getpass
-
-from stand.auth.psildap import get_data
-
-
-parser = argparse.ArgumentParser()
-parser.add_argument("username")
-clargs = parser.parse_args()
-
-username = clargs.username
-password = getpass()
-
-data = get_data(username, password)
-print(data)
-
-
-times = timeit.repeat(partial(get_data, username, password), number=1, repeat=5)
-
-for i, t in enumerate(times, 1):
-    print(f"run {i}:   {t:.6f}s")
-
-avg = sum(times) / len(times)
-print(f"average: {avg:.6f}s")
-
-
-

client.py

@@ -0,0 +1,42 @@
+import requests
+import json
+
+
+class Client:
+
+    def __init__(self, host="127.0.0.1", port=9090):
+        self.host = host
+        self.port = port
+        self.session = requests.Session()
+
+    def add_row(self, **kwargs):
+        addr = self._make_addr()
+        resp = self.session.patch(addr, json=kwargs)
+        return ResponseWrapper(resp)
+
+    def get(self):
+        addr = self._make_addr()
+        resp = self.session.get(addr)
+        return ResponseWrapper(resp)
+
+    def clear(self):
+        addr = self._make_addr()
+        resp = self.session.delete(addr)
+        return ResponseWrapper(resp)
+
+    def _make_addr(self):
+        return f"http://{self.host}:{self.port}/"
+
+
+
+class ResponseWrapper:
+
+    def __init__(self, resp):
+        resp.raise_for_status()
+        self.resp = resp
+
+    def __repr__(self):
+        return self.resp.text
+
+
+

pixi.toml

@@ -1,17 +0,0 @@
-[workspace]
-authors = ["Sven Augustin <sven.augustin@psi.ch>"]
-channels = ["conda-forge"]
-name = "stand"
-platforms = ["linux-64"]
-version = "0.0.2"
-
-[tasks]
-stand = "stand/main.py"
-
-[dependencies]
-python = ">=3.13,<3.14"
-nicegui = ">=3.11.0,<4"
-arcticdb = ">=6.13.0,<7"
-python-gssapi = ">=1.11.1,<2"
-ldap3 = ">=2.9.1,<3"
-

print_adb.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-
-import arcticdb as adb
-
-
-uri = "lmdb://adb"
-ac = adb.Arctic(uri)
-libs = ac.list_libraries()
-
-for n in sorted(libs):
-    print(n)
-    print("=" * len(n))
-
-    lib = ac.get_library(n)
-    symbols = lib.list_symbols()
-    metas = [lib.read_metadata(i) for i in sorted(symbols)] or ["nothing"]
-
-    for i in metas:
-        print("-", i)
-
-    print()
-
-
-

stand.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import argparse
+import socket
+
+
+parser = argparse.ArgumentParser(
+    description="✋ stand",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter
+)
+
+hostname = socket.gethostname()
+
+msg = " of the Streamlit app"
+parser.add_argument("--app-host", default=hostname,       help="host"+msg)
+parser.add_argument("--app-port", default=8080, type=int, help="port"+msg)
+
+msg = " of the CherryPy API"
+parser.add_argument("--api-host", default=hostname,       help="host"+msg)
+parser.add_argument("--api-port", default=9090, type=int, help="port"+msg)
+
+clargs = parser.parse_args()
+
+
+streamlit_params = f"""
+    --browser.gatherUsageStats false
+    --server.headless true
+    --server.address {clargs.app_host}
+    --server.port {clargs.app_port}
+""".split()
+
+cherrypy_params = f"""
+    --host {clargs.api_host}
+    --port {clargs.api_port}
+""".split()
+
+
+import sys
+
+# combine the folder that contains this script
+
+base = sys.path[0] or "."
+script = f"{base}/stand/webapp.py"
+
+
+import subprocess
+
+# arguments before "--" will be parsed by streamlit
+# arguments after  "--" will be passed into the script
+# the inner argparse is in restapi.py
+
+cmd = [
+#    "echo",
+    "streamlit", "run", *streamlit_params, script,
+    "--", *cherrypy_params
+]
+
+subprocess.run(cmd)
+
+
+

stand.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+RESET="\033[0m"
+RED="\033[0;31m"
+echo -e "${RED}please switch to the stand.py launcher${RESET}"
+
+base=$(dirname $0)
+
+params=(
+  --browser.gatherUsageStats false
+  --server.headless true
+)
+
+streamlit run "${params[@]}" $@ $base/stand/webapp.py
+
+

stand/adb.py

@@ -1,18 +0,0 @@
-import arcticdb as adb
-
-
-class ArcticDB(adb.Arctic):
-
-    exceptions = adb.exceptions
-
-    def get(self, name):
-        return self.get_library(
-            name,
-            create_if_missing=True,
-            library_options=adb.LibraryOptions(
-                dynamic_schema=True
-            )
-        )
-
-
-

stand/aggridx.py

@@ -1,135 +0,0 @@
-from nicegui import ui
-
-
-class aggridx(ui.aggrid):
-
-#    #TODO: this should be done in the constructor via a keyword argument switch
-#    #TODO: this would need the from_pandas/from_polars classmethods to pass through that switch
-#    def setup_sync_edits(self):
-#        """
-#        automatically sync data to the server after edits
-#        this operation maintains the current view on the data
-#        """
-#        self.on("cellValueChanged", self.set_cell_server)
-
-
-    def set_cell_server(self, row_index, col_id, new_val):
-        with self.props.suspend_updates():
-            self.options["rowData"][row_index][col_id] = new_val
-
-    def set_cell_client(self, row_id, col_id, new_val):
-        self.run_row_method(row_id, "setDataValue", col_id, new_val)
-
-
-#    def set_row_server_from_event(self, evt):
-#        index = evt.args["rowIndex"]
-#        data = evt.args["data"]
-#        self.set_row_server(index, data)
-
-#    def set_row_server(self, index, data):
-#        """
-#        overwrite row at index with data
-#        this operation maintains the current view on the data
-#        """
-#        with self.props.suspend_updates():
-#            self.options["rowData"][index] = data
-
-
-    def append(self, row):
-        """
-        append row to the bottom of the table and
-        append any missing columns on the right side of the table
-        this operation maintains the current view on the data
-        """
-        self.ensure_column_defs(row)
-        self.add_row_data([row])
-
-#    def insert(self, index, row):
-#        """
-#        insert row at the provided index
-#        append any missing columns on the right side of the table
-#        this operation maintains the current view on the data
-#        """
-#        self.ensure_column_defs(row)
-#        self.add_row_data([row], index=index)
-
-#    def extend(self, rows):
-#        """
-#        append rows to the bottom of the table and
-#        append any missing columns on the right side of the table
-#        this operation maintains the current view on the data
-#        """
-#        columns = dict.fromkeys(k for d in rows for k in d) # dict.fromkeys acts as ordered set
-#        self.ensure_column_defs(columns)
-#        self.add_row_data(rows)
-
-
-    def ensure_column_defs(self, columns):
-        """
-        append any missing columns on the right side of the table
-        this operation maintains the current view on the data
-        """
-        current_column_defs = self.options["columnDefs"]
-        current_fields = [i["field"] for i in current_column_defs]
-        added_fields = [i for i in columns if i not in current_fields]
-        if not added_fields:
-            return
-        added_column_defs = [{"field": n} for n in added_fields]
-        new_column_defs = current_column_defs + added_column_defs
-        # update server without re-draw
-        with self.props.suspend_updates():
-            self.options["columnDefs"] = new_column_defs
-        # update client
-        self.run_grid_method("setGridOption", "columnDefs", new_column_defs)
-
-    def add_row_data(self, rows, index=-1):
-        """
-        insert rows at the provided index,
-        with the default being appending at the bottom of the table
-        this operation maintains the current view on the data
-        """
-        # update server without re-draw
-        with self.props.suspend_updates():
-            self.options["rowData"].extend(rows)
-        # update client
-        self.run_grid_method("applyTransaction", {"add": rows, "addIndex": index})
-
-
-#    def scroll_to_row_index(self, index):
-#        """
-#        scroll the view on the data such that the row with the given index is visible
-#        negative indices count from the back
-#        """
-#        nrows = len(self.options["rowData"])
-#        index %= nrows # move index into [0, nrows)
-#        self.run_grid_method("ensureIndexVisible", index)
-
-#    def scroll_to_row_key(self, key):
-#        """
-#        scroll the view on the data such that the row with the given key is visible
-#        consistent with tables generated from dataframes, the column called index is used to match the key
-#        if the key is not found, the view is not scrolled
-#        """
-#        indices = [i.get("index") for i in self.options["rowData"]]
-#        try:
-#            index = indices.index(key)
-#        except ValueError:
-#            return
-#        self.scroll_to_row_index(index)
-
-#    def scroll_to_column_index(self, index):
-#        """
-#        scroll the view on the data such that the column with the given index is visible
-#        negative indices count from the back
-#        """
-#        key = self.options["columnDefs"][index]["field"]
-#        self.scroll_to_column_key(key)
-
-#    def scroll_to_column_key(self, key):
-#        """
-#        scroll the view on the data such that the column with the given key is visible
-#        """
-#        self.run_grid_method("ensureColumnVisible", key)
-
-
-

stand/api.py

@@ -1,43 +0,0 @@
-from datetime import datetime
-from http import HTTPStatus
-from typing import Any
-
-import pandas as pd
-from fastapi import APIRouter, HTTPException
-
-from state import grids, adb
-from utils.annotations import Beamline, PGroup
-
-
-router = APIRouter()
-
-
-@router.post("/tables/{pgroup}/append")
-def append(beamline: Beamline, pgroup: PGroup, run: int, row: dict[str, Any]):
-    lib = adb.get(beamline)
-
-    index = pd.Timestamp(run) # adb supports update only for timeseries indexes
-    index = pd.Index([index], name="run")
-
-    run_exists = not lib.read(pgroup, date_range=index, columns=[]).data.index.empty
-    if run_exists:
-        raise HTTPException(HTTPStatus.CONFLICT, f"run {run} exists already in {beamline}/{pgroup}")
-
-    timestamp = row.pop("timestamp", None)
-    timestamp = datetime.fromisoformat(timestamp) if timestamp else datetime.now()
-    row = {"timestamp": timestamp, **row} # setdefault would not force timestamp to be the first column
-
-    df = pd.DataFrame(row, index=index)
-    lib.append(pgroup, df)
-
-    row = {"run": run, **row} # setdefault would not force run to be the first column
-
-    res = []
-    for grid in grids[pgroup]:
-        grid.append(row)
-        res.append(grid.options)
-
-    return res
-
-
-

stand/auth/fakeldap.py

@@ -1,28 +0,0 @@
-PASSWORDS = {
-    "a": "a",
-    "b": "b",
-    "c": "c"
-}
-
-PGROUPS = {
-    "a": {
-        "p11111": "stand",
-        "p22222": "SATESE"
-    },
-    "b": {
-        "p33333": "SARESA"
-    }
-}
-
-
-def get_data(username, password):
-    if PASSWORDS.get(username) != password:
-        raise RuntimeError(f'authentication failed for user "{username}"')
-
-    return {
-        "username": username,
-        "pgroups": PGROUPS.get(username, {})
-    }
-
-
-

stand/auth/krbldap.py

@@ -1,26 +0,0 @@
-import gssapi
-from ldap3 import GSSAPI, NONE, ROUND_ROBIN, SASL, Connection, Server, ServerPool
-
-
-def Konnection(hosts, username, password):
-    server_pool = make_server_pool(hosts)
-    creds = make_creds(username, password)
-    return Connection(
-        server_pool,
-        authentication=SASL,
-        sasl_mechanism=GSSAPI,
-        sasl_credentials=(None, None, creds)
-    )
-
-
-def make_server_pool(hosts):
-    servers = [Server(host, get_info=NONE, connect_timeout=3) for host in hosts]
-    return ServerPool(servers, ROUND_ROBIN, active=3, exhaust=True)
-
-
-def make_creds(username, password):
-    user = gssapi.Name(base=username, name_type=gssapi.NameType.user)
-    return gssapi.raw.acquire_cred_with_password(user, password.encode()).creds
-
-
-

stand/auth/login.py

@@ -1,57 +0,0 @@
-import logging
-
-from fastapi.responses import RedirectResponse
-from nicegui import APIRouter, app, ui
-
-from state import config
-
-
-log = logging.getLogger(__name__)
-router = APIRouter()
-
-
-if config.fake:
-    log.warning("using fake LDAP")
-    from .fakeldap import get_data
-else:
-    from .psildap import get_data
-
-
-@router.page("/login")
-def login(redirect_to: str = "/") -> RedirectResponse | None:
-    if app.storage.user.get("authenticated"):
-        return RedirectResponse("/")
-
-    def try_login(): # local function to avoid passing username and password as arguments
-        if not username.value:
-            ui.notify("Missing username", color="negative")
-            username.run_method("focus")
-            return
-
-        if not password.value:
-            password.run_method("focus")
-            return
-
-        try:
-            data = get_data(username.value, password.value)
-        except Exception:
-            log.exception(f'login failed for user "{username.value}"')
-            ui.notify("Wrong username or password", color="negative")
-            return
-
-        app.storage.user.update(data, authenticated=True)
-
-        ui.navigate.to(redirect_to) # go back to where the user wanted to go
-
-    with ui.card().classes("absolute-center items-stretch"):
-        ui.image("assets/icon.png")
-        username = ui.input("Username").props("autofocus")
-        password = ui.input("Password", password=True, password_toggle_button=True)
-        username.on("keydown.enter", try_login)
-        password.on("keydown.enter", try_login)
-        ui.button("log in", icon="login", on_click=try_login)
-
-    return None
-
-
-

stand/auth/logout.py

@@ -1,9 +0,0 @@
-from nicegui import app, ui
-
-
-def logout():
-    app.storage.user.clear()
-    ui.navigate.to("/login")
-
-
-

stand/auth/mw.py

@@ -1,30 +0,0 @@
-from fastapi import Request
-from fastapi.responses import RedirectResponse
-from nicegui import app
-from starlette.middleware.base import BaseHTTPMiddleware
-
-
-unrestricted_page_routes = {"/favicon.ico", "/login"}
-
-
-class AuthMiddleware(BaseHTTPMiddleware):
-    """
-    This middleware restricts access to all NiceGUI pages.
-    It redirects the user to the login page if they are not authenticated.
-    """
-
-    async def dispatch(self, request: Request, call_next):
-        path = request.url.path
-
-        if (
-            app.storage.user.get("authenticated")
-            or path in unrestricted_page_routes
-            or path.startswith("/_nicegui")
-            or path.startswith("/api/")
-        ):
-            return await call_next(request)
-
-        return RedirectResponse(f"/login?redirect_to={path}")
-
-
-

stand/auth/psildap.py

@@ -1,96 +0,0 @@
-import re
-
-from ldap3 import BASE, SUBTREE
-
-from .krbldap import Konnection
-
-
-USER_ATTRIBUTE_MAP = {
-    "cn": "username",
-    "displayName": "display name",
-    "givenName": "first name",
-#    "mail": "email",
-#    "sn": "last name"
-}
-
-USER_ATTRIBUTES = [*USER_ATTRIBUTE_MAP, "memberOf"]
-COMMON_USER_DN = "OU=users,OU=psi,DC=d,DC=psi,DC=ch"
-BASE_PGROUP_DN = "OU=Groups,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch"
-PGROUP_PATTERN = re.compile(r"CN=(p\d{5}),OU=Groups,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch")
-BEAMLINE_PATTERN = re.compile(r"CN=([^,]+),OU=Beamlines,OU=Experiment,OU=IT,DC=d,DC=psi,DC=ch")
-SERVERS = [f"dc{i:02}.d.psi.ch" for i in range(3)]
-
-
-def get_data(username, password):
-    with Konnection(SERVERS, username, password) as conn:
-        conn.search(
-            make_user_search_base(username),
-            search_filter="(objectClass=*)",
-            search_scope=BASE,
-            attributes=USER_ATTRIBUTES
-        )
-
-        entry = ensure_single(conn.entries, "users")
-        res = repack_user(entry)
-
-        res_username = res["username"]
-        if res_username != username:
-            raise RuntimeError(f"username mismatch: {res_username} != {username}")
-
-        conn.search(
-            BASE_PGROUP_DN,
-            search_filter=make_pgroups_search_filter(res["pgroups"]),
-            search_scope=SUBTREE,
-            attributes=["cn", "memberOf"]
-        )
-
-        res["pgroups"] = repack_pgroups(conn.entries)
-
-    return res
-
-
-def make_user_search_base(cn):
-    res = [f"CN={cn}"]
-    if cn.startswith("ext-"):
-        res.append("OU=external")
-    elif cn.startswith("gac-"):
-        res.append("OU=nonpersons")
-    res.append(COMMON_USER_DN)
-    return ",".join(res)
-
-
-def repack_user(entry):
-    attrs = entry.entry_attributes_as_dict
-    res = {USER_ATTRIBUTE_MAP[old]: item[0] for old, item in attrs.items() if old in USER_ATTRIBUTE_MAP if item}
-    res["pgroups"] = extract_matches(attrs["memberOf"], PGROUP_PATTERN)
-    return res
-
-
-def make_pgroups_search_filter(pgroups):
-    search_filter = "".join(f"(cn={cn})" for cn in pgroups)
-    return f"(|{search_filter})"
-
-
-def repack_pgroups(entries):
-    res = {}
-    for entry in entries:
-        attrs = entry.entry_attributes_as_dict
-        beamlines = extract_matches(attrs["memberOf"], BEAMLINE_PATTERN)
-        beamline = ensure_single(beamlines, "beamlines")
-        pgroup = ensure_single(attrs["cn"], "pgroup CNs")
-        res[pgroup] = beamline
-    return res
-
-
-def ensure_single(seq, what):
-    n = len(seq)
-    if n == 1:
-        return seq[0]
-    raise RuntimeError(f"received ambiguous {what} ({n} entries)")
-
-
-def extract_matches(seq, pattern):
-    return sorted(m.group(1) for i in seq if (m := pattern.search(i)))
-
-
-

stand/auth/secret.py

@@ -1,23 +0,0 @@
-import os
-import secrets
-
-
-def get_secret(fn=".secret"):
-    if os.path.exists(fn):
-        res = read(fn)
-    else:
-        res = secrets.token_hex(32)
-        write(res, fn)
-    return res.strip()
-
-
-def read(fn):
-    with open(fn) as f:
-        return f.read()
-
-def write(s, fn):
-    with open(fn, "w") as f:
-        f.write(s)
-
-
-

stand/beamline.py

@@ -1,19 +0,0 @@
-
-SUBNET_TO_BEAMLINE = {
-    "129.129.242": "SARESA",
-    "129.129.243": "SARESB",
-    "129.129.244": "SARESC",
-    "129.129.245": "SATESD",
-    "129.129.246": "SATESE",
-    "129.129.247": "SATESF"
-}
-
-
-def get_beamline(ip):
-    if ip == "127.0.0.1":
-        return "stand"
-    subnet = ".".join(ip.split(".")[:3])
-    return SUBNET_TO_BEAMLINE[subnet]
-
-
-

stand/cli.py

@@ -1,11 +0,0 @@
-import argparse
-
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument("-f", "--fake", action="store_true")
-
-clargs = parser.parse_args()
-
-
-

stand/hacks/__init__.py

@@ -0,0 +1,4 @@
+
+from . import patch_st_cp_stop
+
+

stand/hacks/patch_st_cp_stop.py

@@ -0,0 +1,21 @@
+"""
+Monkey patch the streamlit server to stop cherrypy upon its own stop
+"""
+
+
+import cherrypy
+from streamlit.server.server import Server
+
+
+old_fn = Server._on_stopped
+
+def new_fn(*args, **kwargs):
+    print("exit cherrypy")
+    cherrypy.engine.exit()
+    print("exit streamlit")
+    return old_fn(*args, **kwargs)
+
+Server._on_stopped = new_fn
+
+
+

stand/home.py

@@ -1,25 +0,0 @@
-from nicegui import APIRouter, app, ui
-
-from auth.logout import logout
-
-
-router = APIRouter()
-
-
-@router.page("/")
-def home():
-    with ui.column().classes("absolute-center items-center"):
-        username = app.storage.user.get("username", "unknown user")
-        ui.label(f"Hello {username}!").classes("text-2xl")
-        ui.button("log out", icon="logout", on_click=logout)
-
-        pgroups = app.storage.user.get("pgroups", {})
-        ui.select(
-            label="pgroup",
-            options=sorted(pgroups, reverse=True),
-            with_input=True,
-            on_change=lambda e: ui.navigate.to(f"/tables/{e.value}")
-        )
-
-
-

stand/logcfg.py

@@ -1,29 +0,0 @@
-import logging
-
-from uvicorn.config import LOGGING_CONFIG
-from uvicorn.logging import ColourizedFormatter
-
-
-LOGFMT = "%(asctime)s %(levelprefix)s [%(name)s] %(message)s"
-
-
-def logcfg(level=logging.INFO, fmt=LOGFMT, **kwargs):
-    formatter = ColourizedFormatter(fmt=fmt, **kwargs)
-
-    handler = logging.StreamHandler()
-    handler.setFormatter(formatter)
-
-    logging.basicConfig(level=level, handlers=[handler])
-
-    # add timestamps to uvicorn logs
-    formatters = LOGGING_CONFIG["formatters"]
-    for formatter in formatters.values():
-        if "fmt" not in formatter:
-            continue
-        fmt = formatter["fmt"]
-        if "%(asctime)s" in fmt:
-            continue
-        formatter["fmt"] = "%(asctime)s " + fmt
-
-
-

stand/main.py

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-from nicegui import app, ui
-
-from cli import clargs
-from state import config
-config.update(vars(clargs))
-
-from logcfg import logcfg
-logcfg()
-
-from api import router as api_router
-from auth.login import router as login_router
-from auth.mw import AuthMiddleware
-from auth.secret import get_secret
-from home import router as home_router
-from table import router as table_router
-
-
-app.include_router(login_router)
-app.add_middleware(AuthMiddleware)
-
-app.include_router(api_router, prefix="/api")
-app.include_router(home_router)
-app.include_router(table_router)
-
-
-ui.run(
-    title="stand",
-    favicon="assets/favicon.png",
-    storage_secret=get_secret(),
-    fastapi_docs=True,
-    dark=True,
-    show=False
-)
-
-
-

stand/restapi.py

@@ -0,0 +1,44 @@
+import argparse
+from threading import Thread
+
+import cherrypy as cp
+
+from tblctrl import TableController
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--host", default="127.0.0.1")
+parser.add_argument("--port", default=9090, type=int)
+
+clargs = parser.parse_args()
+
+
+cp.config.update({
+    "server.socket_host": clargs.host,
+    "server.socket_port": clargs.port,
+})
+
+
+# creating instances here allows to use restapi etc. as singletons
+
+print(f">>> start restapi on {clargs.host}:{clargs.port}")
+
+restapi = TableController()
+
+conf = {
+    "/": {
+        "request.dispatch": cp.dispatch.MethodDispatcher(),
+        "tools.sessions.on": True,
+        "tools.response_headers.on": True,
+        "tools.response_headers.headers": [("Content-Type", "text/plain")],
+    }
+}
+
+args = (restapi, "/", conf)
+
+thread = Thread(target=cp.quickstart, args=args, daemon=True)
+thread.start()
+
+
+

stand/state.py

@@ -1,20 +0,0 @@
-from collections import defaultdict
-
-from adb import ArcticDB
-
-from utils.config import Config
-from utils.registry import Registry
-
-
-adb = ArcticDB("lmdb://adb")
-
-
-config = Config(
-    fake=False
-)
-
-
-grids = defaultdict(Registry)
-
-
-

stand/table.py

@@ -1,118 +0,0 @@
-import logging
-
-import pandas as pd
-from nicegui import APIRouter, app, ui
-
-from aggridx import aggridx
-from state import grids, adb
-from utils.annotations import PGroup
-
-
-OPTIONS = {
-    ":getRowId": "(params) => params.data.run", # set row ID to index column
-    "context": {
-        "beamline": None,
-        "pgroup": None
-    },
-    "defaultColDef": {
-        "filter": True,
-        "editable": True,
-        "sortable": True,
-        "resizable": True
-    },
-    "pagination": True,
-    "paginationAutoPageSize": True,
-    "theme": "balham"
-}
-
-
-log = logging.getLogger(__name__)
-router = APIRouter()
-
-
-@router.page("/tables/{pgroup}")
-def table(pgroup: PGroup):
-    pgroups = app.storage.user.get("pgroups", {})
-    if pgroup in pgroups:
-        beamline = pgroups[pgroup]
-        table_show(pgroup, beamline)
-    else:
-        table_deny(pgroup)
-
-
-def table_show(pgroup, beamline):
-#    with ui.left_drawer(value=False) as ld:
-#        dark = ui.dark_mode(value=True)
-#        ui.switch("dark mode").bind_value(dark)
-
-#    with ui.page_sticky(position="left", x_offset=-12.5):
-#        def cb():
-#            ld.toggle()
-#            btn.icon = "sym_o_left_panel_close" if ld.value else "sym_o_left_panel_open"
-#        btn = ui.button(icon="sym_o_left_panel_open", on_click=cb).props("flat dense")
-
-    try:
-        df = adb.get(beamline).read(pgroup).data
-    except adb.exceptions.NoSuchVersionException:
-        df = pd.DataFrame()
-
-    df.index = df.index.view("int64") # adb supports update only for timeseries indexes
-    df.index.name = "run"
-    df = df.reset_index()
-
-    options = OPTIONS.copy()
-    options["context"].update(
-        beamline=beamline,
-        pgroup=pgroup
-    )
-
-    grid = aggridx.from_pandas(df, options=options)
-    grid.classes("h-[calc(100vh-2rem)]") # full height minus padding
-    grid.on("cellValueChanged", update_adb)
-    grid.on("cellValueChanged", update_grids)
-    grids[pgroup].add(grid)
-
-
-def table_deny(pgroup):
-    username = app.storage.user.get("username", "unknown user")
-    log.error(f'access to pgroup {pgroup} denied for user "{username}"')
-    with ui.column().classes("absolute-center items-center gap-8"):
-        ui.icon("sym_o_front_hand", size="xl")
-        ui.label(f"access to {pgroup} denied").classes("text-2xl")
-
-
-def update_adb(evt):
-    if evt.args.get("source") != "edit":
-        # ignore event if it was no direct edit
-        return
-
-    beamline = evt.args["context"]["beamline"]
-    pgroup = evt.args["context"]["pgroup"]
-    row_id = evt.args["rowId"]
-    col_id = evt.args["colId"]
-    new_val = evt.args["newValue"]
-
-    index = pd.Timestamp(int(row_id)) # adb supports update only for timeseries indexes
-    lib = adb.get(beamline)
-    df = lib.read(pgroup, date_range=[index]).data
-    df.at[index, col_id] = new_val
-    lib.update(pgroup, df)
-
-
-def update_grids(evt):
-    sender = evt.sender
-    pgroup = evt.args["context"]["pgroup"]
-    row_index = evt.args["rowIndex"]
-    row_id = evt.args["rowId"]
-    col_id = evt.args["colId"]
-    new_val = evt.args["newValue"]
-
-    for grid in grids[pgroup]:
-        grid.set_cell_server(row_index, col_id, new_val)
-        if grid == sender:
-            # the sender is already up-to-date
-            continue
-        grid.set_cell_client(row_id, col_id, new_val)
-
-
-

stand/tblctrl.py

@@ -0,0 +1,63 @@
+import cherrypy as cp
+
+from utils.dfh import DateFrameHolder
+from utils.st_utils import rerunall, get_session_ids
+
+
+@cp.expose
+class TableController:
+
+    def __init__(self):
+        self.dfh = DateFrameHolder("output.h5")
+#        self.changed = True
+        self.sids = get_session_ids()
+
+    def get(self, sid):
+        changed = self.get_changed(sid)
+        data = self.get_data(sid)
+        return changed, data
+
+    def get_changed(self, sid):
+        return (sid in self.sids)
+
+    def get_data(self, sid):
+#        self.changed = False
+        try:
+            self.sids.remove(sid)
+        except KeyError as e:
+            if self.sids:
+                raise RuntimeError(f"{self.sids} is not empty") from e
+        return self.dfh.df
+
+    def set_data(self, df):
+        if self.dfh.df.equals(df):
+            print("<<< skipping dump because dataframe did not change")
+            return
+        self.dfh.df = df
+        self.dfh.dump()
+        self._trigger_changed()
+
+    def GET(self):
+        return str(self.dfh.df)
+
+    @cp.tools.json_in()
+    def PATCH(self, **kwargs):
+        kwargs = kwargs or cp.request.json
+        self.dfh.append(kwargs)
+        self._trigger_changed()
+        self.dfh.dump()
+        return str(self.dfh.df)
+
+    def DELETE(self):
+        self.dfh.clear()
+        self._trigger_changed()
+        return "cleared"
+
+
+    def _trigger_changed(self):
+#        self.changed = True
+        self.sids = get_session_ids()
+        rerunall()
+
+
+

stand/utils/annotations.py

@@ -1,17 +0,0 @@
-from typing import Annotated
-
-from fastapi import Depends, Path, Request
-
-from beamline import get_beamline
-
-
-def beamline_dependency(request: Request) -> str:
-    return get_beamline(request.client.host)
-
-Beamline = Annotated[str, Depends(beamline_dependency)]
-
-
-PGroup = Annotated[str, Path(pattern=r"^p\d{5}$")]
-
-
-

stand/utils/config.py

@@ -1,8 +0,0 @@
-
-class Config(dict):
-    __getattr__ = dict.__getitem__
-    __setattr__ = dict.__setitem__
-    __delattr__ = dict.__delitem__
-
-
-

stand/utils/df2bin.py

@@ -0,0 +1,27 @@
+from io import BytesIO
+
+
+def to_excel_binary(df, **kwargs):
+    with BytesIO() as out:
+        df.to_excel(out, **kwargs)
+        res = out.getvalue()
+    return res
+
+
+
+from pandas import HDFStore
+
+
+def to_hdf_binary(df):
+    with HDFStore(
+        "wontbewritten.h5",
+        mode="a",
+        driver="H5FD_CORE",
+        driver_core_backing_store=0
+    ) as out:
+        out["/data"] = df
+        res = out._handle.get_file_image()
+    return res
+
+
+

stand/utils/dfh.py

@@ -0,0 +1,72 @@
+import pandas as pd
+
+
+class DateFrameHolder:
+
+    def __init__(self, fn):
+        self.fn = fn
+        self.df = try_load_df(fn)
+
+    def dump(self):
+        dump_non_empty_df(self.df, self.fn)
+
+
+    def append(self, data):
+        data = pd.DataFrame.from_records([data])
+        self.df = pd.concat([self.df, data], ignore_index=True)
+
+    def clear(self):
+        self.df = pd.DataFrame()
+
+
+    def __repr__(self):
+        head = self.fn + ":"
+        line = "-" * len(head)
+        df = str(self.df)
+        return "\n".join((head, line, df))
+
+
+
+def try_load_df(fn):
+    try:
+        df = pd.read_hdf(fn)
+        print(f">>> loaded dataframe from {fn}")
+    except FileNotFoundError:
+        df = pd.DataFrame()
+        print(">>> created empty dataframe")
+    return df
+
+def dump_non_empty_df(df, fn, key="data"):
+    if df.empty:
+        print("<<< skip dumping empty dataframe")
+        return
+    print(f"<<< dump dataframe to {fn}")
+    df.to_hdf(fn, key=key)
+    backup(df, fn, key) #TODO: test then remove
+
+
+
+# the following is just a safety precaution
+# if everything works, it can be removed
+
+from datetime import datetime
+from pathlib import Path
+
+FMT = "%Y-%m-%d_%H-%M-%S-%f"
+BAKDIR = ".backup"
+
+def backup(df, fn, key):
+    Path(BAKDIR).mkdir(parents=True, exist_ok=True)
+    bfn = backup_filename(fn)
+    print(f"<<< backup dataframe to {bfn}")
+    df.to_hdf(bfn, key=key)
+
+def backup_filename(fn):
+    p = Path(fn)
+    fn = p.stem
+    ext = p.suffix
+    ts = datetime.now().strftime(FMT)
+    return f"{BAKDIR}/{fn}_{ts}{ext}"
+
+
+

stand/utils/registry.py

@@ -1,18 +0,0 @@
-from nicegui import ui
-
-
-class Registry:
-
-    def __init__(self):
-        self.data = {}
-
-    def __iter__(self):
-        return iter(self.data.values())
-
-    def add(self, widget):
-        client = ui.context.client
-        self.data[client.id] = widget
-        client.on_disconnect(lambda: self.data.pop(client.id, None))
-
-
-

stand/utils/st_utils.py

@@ -0,0 +1,64 @@
+import streamlit as st
+
+
+get_server = st.server.server.Server.get_current
+
+
+def rerunall():
+    sibi = get_session_info_by_id()
+    for si in sibi.values():
+        print("rerun:", si.session.id)
+        client_state = None
+        si.session.request_rerun(client_state)
+
+
+def get_session_ids():
+    sibi = get_session_info_by_id()
+    sids = sibi.keys()
+    return set(sids)
+
+
+def rerun(session_id=None):
+    if session_id is None:
+        session_id = get_session_id()
+
+    server = get_server()
+    session = server.get_session_by_id(session_id)
+
+    client_state = None
+    session.request_rerun(client_state)
+
+
+def get_session_id():
+    ctx = st.scriptrunner.script_run_context.get_script_run_ctx()
+    return ctx.session_id
+
+
+def get_session_info_by_id():
+    server = get_server()
+    return server._session_info_by_id
+
+
+
+def hide_UI_elements(menu=True, header=True, footer=True):
+    HIDDEN = " {visibility: hidden;}"
+    res = []
+
+    if menu:
+        res.append("#MainMenu" + HIDDEN)
+
+    if header:
+        res.append("header" + HIDDEN)
+
+    if footer:
+        res.append("footer" + HIDDEN)
+
+    if not res:
+        return
+
+    res = ["<style>"] + res + ["</style>"]
+    res = "\n".join(res)
+    return st.markdown(res, unsafe_allow_html=True)
+
+
+

stand/webapp.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+import streamlit as st
+
+import hacks
+
+from widgets.aggrid import aggrid
+from widgets.download import download
+from restapi import restapi
+from utils.st_utils import get_session_id, hide_UI_elements
+
+
+print(">>> start of streamlit run")
+
+icon = Path(__file__).parent.parent / "icon.png"
+
+st.set_page_config(
+    page_title="stand",
+    page_icon=str(icon),
+    layout="wide",
+    initial_sidebar_state="collapsed"
+)
+
+hide_UI_elements()
+
+
+sid = get_session_id() # rest api needs current session IDs to trigger reruns
+changed, df = restapi.get(sid)
+
+
+download(df)
+
+
+auto_height = st.sidebar.checkbox(
+    "Auto Height",
+    value=False,
+    key="auto_height"
+)
+
+pagination = st.sidebar.checkbox(
+    "Pagination",
+    value=True,
+    disabled=auto_height,
+    key="pagination"
+)
+
+height = st.sidebar.slider(
+    "Grid Height",
+    min_value=100, max_value=1500, value=500, step=10,
+    disabled=auto_height,
+    key="grid_height"
+)
+
+if auto_height:
+    pagination = False # would only show "1 of 1"
+    height = "auto"
+
+
+# if df in restapi changed, we need to reload it into aggrid.
+# if df in restapi did not change, we do not reload to ensure changes in the browser persist.
+
+response = aggrid(
+    df,
+    reload_data=changed,
+    pagination=pagination,
+    height=height,
+    enable_enterprise_modules=False,
+    key="stand"
+)
+
+
+# if we reloaded, aggrid returns the df from before (why?), thus we do not update the restapi.
+# if we did not reload, aggrid may return an updated df from an edit in the browser,
+# and thus we need to update the restapi.
+
+new_df = response["data"]
+if not new_df.equals(df) and not changed:
+    restapi.set_data(new_df)
+
+
+print(">>> end of streamlit run")
+
+
+

stand/widgets/aggrid.py

@@ -0,0 +1,59 @@
+from st_aggrid import AgGrid, GridOptionsBuilder
+
+
+def aggrid(df, reload_data=False, height="auto", pagination=True, key=None, **kwargs):
+    df = df[::-1] # display in reversed chronological order
+
+    gob = GridOptionsBuilder.from_dataframe(
+        df,
+        editable=True,
+        filterable=True,
+        groupable=True,
+        resizable=True,
+        sortable=True
+    )
+
+    auto_height = (height == "auto")
+    gob.configure_auto_height(auto_height)
+    gob.configure_pagination(pagination)
+
+    go = gob.build()
+
+    response = AgGrid(
+        df,
+        go,
+        height=height,
+        theme="streamlit",
+        reload_data=reload_data,
+        key=make_key(key, df, auto_height, pagination),
+        **kwargs
+    )
+
+    df = response.get("data", df)
+    df = df[::-1] # undo reversed chronological order
+
+    #TODO: streamlit-aggrid changed return type
+    if isinstance(response, dict):
+        response["data"] = df # <= 0.2.3.post2
+    else:
+        response.data = df # >= 0.3.0
+
+    return response
+
+
+
+def make_key(prefix, df, auto_height, pagination):
+    """
+    encode the dataframe's column names into a key,
+    as well as the state of the auto height and pagination settings,
+    this triggers a hard reload (like F5) of the AgGrid if the columns/settings change
+    """
+    if prefix is None:
+        return None
+    items = list(df.columns)
+    items.append(auto_height)
+    items.append(pagination)
+    return str(prefix) + ":" + "+".join(str(i) for i in items)
+
+
+

stand/widgets/download.py

@@ -0,0 +1,31 @@
+import streamlit as st
+
+from utils.df2bin import to_excel_binary, to_hdf_binary
+
+
+state = st.session_state
+
+
+def download(df):
+    showing_downloads = state.get("showing_downloads", False)
+
+    col0, col1, col2, col3 = st.columns(4)
+    if col0.button("Downloads") and not showing_downloads:
+        state.showing_downloads = True
+
+        h5 = to_hdf_binary(df)
+        col1.download_button("📥 hdf5", h5, file_name="output.h5")
+
+        xlsx = to_excel_binary(df)
+        col2.download_button("📥 xlsx", xlsx, file_name="output.xlsx")
+
+        csv = df.to_csv()
+        col3.download_button("📥 csv", csv, file_name="output.csv")
+
+#        st.stop()
+
+    else:
+        state.showing_downloads = False
+
+
+

test.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+from itertools import count, cycle
+from random import random, randint
+from string import ascii_letters
+from time import sleep
+
+from client import Client
+
+
+def irandom():
+    while True:
+        yield random()
+
+num = count()
+ran = irandom()
+let = cycle(ascii_letters)
+
+c = Client()
+
+c.clear()
+sleep(1)
+
+for i, r, l in zip(num, ran, let):
+    l = l * randint(1, 5)
+
+    kwargs = {} if i < 10 else {"x": 123}
+
+    c.add_row(index=i, random=r, string=l, **kwargs)
+    sleep(1)
+
+
+