Erstes Frontend Layout erstellt.

.gitignore

@@ -4,4 +4,7 @@ logic/scripts/.idea
 gaga*
 *backup
 .idea/
+.jbeval
 *.log*
+*.bak
+*.backup

TODO.md

@@ -0,0 +1,32 @@
+# Projekt ToDo-Liste (ArchiveCostWebapp)
+
+## 📊 Streamlit Frontend
+- [x] Erstes Tabellen-Layout mit Pandas erstellen
+- [x] Navigation über die Sidebar implementieren
+- [ ] Uebersichtsteil erstellen
+- [ ] Bereichsteil erstellen
+- [ ] Teil ueber nicht zuweisbare Daten
+- [ ] Kosten auf allen Seiten einblenden
+- [ ] Uebersicht ueber single, double Kopien
+- [ ] Benutzerseite, wo er selber Filter erstellen kann, z.B. Anzahl Datasets groesser als ...
+- [ ] Diagramm für die Department-Verteilung einbauen (Plotly)
+- [ ] Export-Button für gefilterte Excel-Dateien hinzufügen
+
+## ⚙️ Node-RED & AD-Skript
+- [x] Python-Skript auf lokale Datei-Pipes umstellen
+- [x] Zweistufige p-Gruppen-Abfrage stabilisieren
+- [ ] Cronjob/Intervall in Node-RED für tägliches Update einrichten
+
+## 🐳 Docker & Deployment
+- [ ] Docker-Volumes für Produktion prüfen
+- [ ] Deployment auf unserer git-Umgebung erstellen. Automatisiert.
+- [ ] requirements.txt File erstellen, falls es dies ueberhaupt braucht.
+
+## 😪 Fehlerbehandlung
+- [ ] Fehler beim periodischen Update der Daten abfangen
+
+## 📜 Dokumentation
+- [ ] Ausfuehrliche Dokumentation fuer Kollegen
+
+## 🙋‍♂️Abstimmung mit Benutzern
+- [ ] Mit Markus

analytics/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Erfordert keine schweren Systempakete, das beschleunigt auch den Build!
+# Wir installieren direkt die benötigten Python-Bibliotheken
+RUN pip3 install --no-cache-dir streamlit pandas plotly
+
+# Port für Streamlit öffnen
+EXPOSE 8501
+
+# Streamlit starten und an alle Netzwerk-Schnittstellen binden
+ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]

analytics/app.py

@@ -0,0 +1,82 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+
+# Umrechnungsfaktor von Terabytes nach Bytes
+TB2B = 1024**4
+
+# Seitenkonfiguration (gibt uns ein schönes breites Layout)
+st.set_page_config(layout="wide")
+
+# 1. Daten laden (Pfad aus deinem Docker-Setup)
+FILE_PATH = "/data/json_cache.json"
+
+
+@st.cache_data
+def load_data():
+    # Falls die Datei existiert, laden, sonst leeres Dataframe für den Test
+    try:
+        return pd.read_json(FILE_PATH)
+    except Exception:
+        # Dummy-Daten, falls Node-RED noch nichts abgelegt hat
+        return pd.DataFrame([
+            {"ownerGroup": "a-123", "department": "4000", "size": 500000},
+            {"ownerGroup": "p9999", "department": "6000", "size": 1200000}
+        ])
+
+
+df = load_data()
+
+# 2. NAVIGATION LINKS (Sidebar)
+st.sidebar.title("Navigation")
+st.sidebar.markdown("Wählen Sie eine Ansicht:")
+
+# Ein Radio-Button verhält sich wie eine klickbare Liste
+auswahl = st.sidebar.radio(
+    label="Ansichten",
+    options=["Übersicht & Metriken", "Bereichs-Analyse", "Nicht zuweisbare Daten", "Rohdaten"],
+    label_visibility="collapsed"  # Versteckt das Label für eine sauberere Optik
+)
+
+# 3. ANZEIGE RECHTS (Hauptbildschirm)
+# Hier steuern wir den Inhalt basierend auf der Auswahl links
+
+if auswahl == "Übersicht & Metriken":
+    st.title("Übersicht")
+
+    # Beispiel für schnelle Metriken rechts
+    col1, col2 = st.columns(2)
+    col1.metric("Anzahl Archivgruppen", len(df))
+    col2.metric("Gesamtvolumen (TB)", f'{df["size"].sum()/TB2B:.2f}' if "size" in df else "0.00")
+
+    # Hier könnte eine Grafik hin
+    if "department" in df:
+        fig = px.pie(df, names="department", title="Verteilung nach Departments")
+        st.plotly_chart(fig, use_container_width=True)
+
+elif auswahl == "Bereichs-Analyse":
+    st.title("Bereichs-Analyse")
+    st.write("Filtern Sie die Daten gezielt nach einzelnen Abteilungen.")
+
+    # Ein zusätzlicher Filter, der nur in dieser Ansicht erscheint
+    if "department" in df:
+        depts = sorted(df["department"].unique().tolist())
+        gewaehltes_dept = st.selectbox("Wählen Sie ein Department:", depts)
+
+        filtered_df = df[df["department"] == gewaehltes_dept]
+        st.metric(f"Gruppen in Department {gewaehltes_dept}", len(filtered_df))
+        st.dataframe(filtered_df, use_container_width=True)
+
+elif auswahl == "Nicht zuweisbare Daten":
+    st.title("Nicht zuweisbare Daten")
+
+    # Die Tabelle, die du schon gebaut hast
+    st.dataframe(df, use_container_width=True)
+
+elif auswahl == "Rohdaten":
+    st.title("Rohdaten")
+    st.write("Durchsuchen und filtern Sie die vollständige Liste der Gruppen.")
+
+    # Die Tabelle, die du schon gebaut hast
+    st.dataframe(df, use_container_width=True)
+

docker-compose.yml

@@ -4,18 +4,14 @@ services:
     volumes:
       - ./logic/node-red-data:/data
       - ./shared_data:/data/results  # Hier landen die JSON/SQLite Dateien
-      - ./logic/scripts:/data/scripts # Hier liegen Ihre Python-Skripte
+      - ./logic/scripts:/data/scripts 
     environment:
       - TZ=Europe/Zurich
-      # Link to the newest archivegroup information file downloaded
-      # from metabase.psi.ch
-      #- GROUPINFO_JSON_LNK_NAME=archivegroup_information.json
       - GROUPINFO_JSON_LNK_NAME_WITH_PATH=/data/archivegroup_information.json
       - NON_SPECIFIED_GROUPS=/data/non_specified_groups.csv
-      # Beginning of name of the downloaded archivebroup information file
       - BEGIN_NAME_GROUPINFO_JSON_FILE=size_by_ownergroup_and_number_of_copies_
       - ERROR_LOGFILE=error.log
-      # AD variables
+      - JSON_CACHE=json_cache.json
       - AD_SERVER=d.psi.ch
       - AD_SEARCH_BASE=DC=d,DC=psi,DC=ch
       - AD_USER=${AD_USER}
@@ -23,9 +19,13 @@ services:
     ports:
       - "1880:1880"
 
-  frontend-app:
-    build: ./frontend
+  # --- NEUER STREAMLIT CONTAINER ---
+  analytics-app:
+    build: ./analytics
     ports:
-      - "8080:80"
+      - "8501:8501"  # Standard-Port für Streamlit im Browser
     volumes:
-      - ./shared_data:/usr/share/nginx/html/data:ro # Nur lesend für die Webseite
+      - ./analytics:/app  # Mappt deinen Code, damit Änderungen live ohne Rebuild sichtbar sind
+      - ./shared_data:/data:ro  # Bindet die JSON-Daten NUR LESEND unter /data ein
+    environment:
+      - TZ=Europe/Zurich

logic/node-red-data/.config.nodes.json

@@ -1,7 +1,7 @@
 {
     "node-red": {
         "name": "node-red",
-        "version": "4.1.10",
+        "version": "5.0.0",
         "local": false,
         "user": false,
         "nodes": {

logic/node-red-data/flows.json

@@ -173,11 +173,11 @@
         "property": "payload",
         "action": "",
         "pretty": false,
-        "x": 1060,
-        "y": 600,
+        "x": 840,
+        "y": 880,
         "wires": [
             [
-                "426e46f53063d368"
+                "8f50bc0cb9009461"
             ]
         ]
     },
@@ -213,11 +213,11 @@
         "winHide": false,
         "oldrc": false,
         "name": "",
-        "x": 620,
+        "x": 540,
         "y": 620,
         "wires": [
             [
-                "e8f1570441f300ee"
+                "dd3574493b97a5f4"
             ],
             [
                 "444c29de997bd1b6"
@@ -237,8 +237,8 @@
         "complete": "false",
         "statusVal": "",
         "statusType": "auto",
-        "x": 1320,
-        "y": 600,
+        "x": 1240,
+        "y": 620,
         "wires": []
     },
     {
@@ -253,8 +253,92 @@
         "complete": "false",
         "statusVal": "",
         "statusType": "auto",
-        "x": 1060,
-        "y": 720,
+        "x": 920,
+        "y": 700,
+        "wires": []
+    },
+    {
+        "id": "dd3574493b97a5f4",
+        "type": "file",
+        "z": "f6f2187d.f17ca8",
+        "name": "Write the new JSON object to cache",
+        "filename": "\"/data/results/\" & $env(\"JSON_CACHE\")",
+        "filenameType": "jsonata",
+        "appendNewline": true,
+        "createDir": false,
+        "overwriteFile": "true",
+        "encoding": "utf8",
+        "x": 970,
+        "y": 620,
+        "wires": [
+            [
+                "426e46f53063d368"
+            ]
+        ]
+    },
+    {
+        "id": "5c695a90a30b61d5",
+        "type": "file in",
+        "z": "f6f2187d.f17ca8",
+        "name": "Read cached json file",
+        "filename": "\"/data/results/\" & $env(\"JSON_CACHE\")",
+        "filenameType": "jsonata",
+        "format": "utf8",
+        "chunk": false,
+        "sendError": false,
+        "encoding": "utf8",
+        "allProps": false,
+        "x": 480,
+        "y": 880,
+        "wires": [
+            [
+                "e8f1570441f300ee"
+            ]
+        ]
+    },
+    {
+        "id": "6c8e4b6960b242ec",
+        "type": "inject",
+        "z": "f6f2187d.f17ca8",
+        "name": "",
+        "props": [
+            {
+                "p": "payload"
+            },
+            {
+                "p": "topic",
+                "vt": "str"
+            }
+        ],
+        "repeat": "",
+        "crontab": "",
+        "once": false,
+        "onceDelay": 0.1,
+        "topic": "",
+        "payload": "",
+        "payloadType": "date",
+        "x": 200,
+        "y": 880,
+        "wires": [
+            [
+                "5c695a90a30b61d5"
+            ]
+        ]
+    },
+    {
+        "id": "8f50bc0cb9009461",
+        "type": "debug",
+        "z": "f6f2187d.f17ca8",
+        "name": "debug 1",
+        "active": true,
+        "tosidebar": true,
+        "console": false,
+        "tostatus": false,
+        "complete": "false",
+        "statusVal": "",
+        "statusType": "auto",
+        "x": 1120,
+        "y": 880,
         "wires": []
     },
     {

shared_data/json_cache.json

@@ -0,0 +1,2 @@
+[{"ownerGroup": "p18973", "copies": "oneCopyBig", "size": "6.85468397803E11", "packedSize": "6.8563388416E11", "beamline": "lem", "department": "3000"}, {"ownerGroup": "p22124", "copies": "oneCopyBig", "size": "134335306", "packedSize": "134420480", "beamline": "FOCUS", "department": "3000"}, {"ownerGroup": "p22354", "copies": "oneCopyBig", "size": "83105124", "packedSize": "83599360", "beamline": "SANS-I", "department": "3000"}, {"ownerGroup": "p19138", "copies": "oneCopyBig", "size": "3.6115314583E10", "packedSize": "3.612919808E10", "beamline": "X12SA", "department": "6000"}, {"ownerGroup": "p19548", "copies": "oneCopyBig", "size": "6.62762759871E12", "packedSize": "6.62763145216E12", "beamline": "X02DA", "department": "6000"}, {"ownerGroup": "p22089", "copies": "oneCopyBig", "size": "994151259", "packedSize": "994744320", "beamline": "gps", "department": "3000"}, {"ownerGroup": "p22293", "copies": "oneCopyBig", "size": "114067605", "packedSize": "114196480", "beamline": "CAMEA", "department": "3000"}, {"ownerGroup": "p21555", "copies": "oneCopyBig", "size": "74373374", "packedSize": "74833920", "beamline": "X07MA", "department": "6000"}, {"ownerGroup": "p17926", "copies": "oneCopyBig", "size": "2143208567", "packedSize": "2.1508608E9", "beamline": "X04SA", "department": "6000"}, {"ownerGroup": "p21716", "copies": "oneCopyBig", "size": "29499839596", "packedSize": "29501122560", "beamline": "lem", "department": "3000"}, {"ownerGroup": "p22579", "copies": "oneCopySmall", "size": "1043132", "packedSize": "1495040", "beamline": "TASP", "department": "3000"}, {"ownerGroup": "p20497", "copies": "oneCopyBig", "size": "61524056356", "packedSize": "61527848960", "beamline": "lem", "department": "3000"}, {"ownerGroup": "p16099", "copies": "oneCopyBig", "size": "2.45503897305E11", "packedSize": "2.455212032E11", "beamline": "X12SA", "department": "6000"}, {"ownerGroup": "p15793", "copies": "oneCopyBig", "size": "109713853", "packedSize": "110735360", "beamline": "X09LA", "department": "6000"}, {"ownerGroup": "p19521", "copies": "oneCopyBig", "size": "2.51420504044E11", "packedSize": "2.5144180736E11", "beamline": "X12SA", "department": "6000"}, {"ownerGroup": "p22799", "copies": "oneCopyBig", "size": "219528490", "packedSize": "220856320", "beamline": "flame", "department": "3000"}, {"ownerGroup": "p19467", "copies": "oneCopyBig", "size": "3441829176", "packedSize": "3441960960", "beamline": "X05LA", "department": "6000"}, {"ownerGroup": "p22865", "copies": "oneCopySmall", "size": "587325", "packedSize": "604160", "beamline": "SANS-I", "department": "3000"}, {"ownerGroup": "p22172", "copies": "oneCopyBig", "size": "128212442", "packedSize": "128419840", "beamline": "CAMEA", "department": "3000"}, {"ownerGroup": "p22653", "copies": "oneCopySmall", "size": "7756909", "packedSize": "7895040", "beamline": "DMC", "department": "3000"}, {"ownerGroup": "p21903", "copies": "oneCopyBig", "size": "459850003", "packedSize": "470886400", "beamline": "SANS-I", "department": "3000"}, {"ownerGroup": "p21458", "copies": "oneCopyBig", "size": "3189274499", "packedSize": "3191828480", "beamline": "flame", "department": "3000"}, {"ownerGroup": "p20558", "copies": "oneCopyBig", "size": "4.802741590201E12", "packedSize": "4.80274899968E12", "beamline": "SARESC", "department": "6000"}, {"ownerGroup": "p21457", "copies": "oneCopyBig", "size": "2016161396", "packedSize": "2017044480", "beamline": "flame", "department": "3000"}, {"ownerGroup": "p18517", "copies": "oneCopySmall", "size": "3392816", "packedSize": "4597760", "beamline": "X12SA", "department": "6000"}, {"ownerGroup": "p21174", "copies": "twoCopiesBig", "size": "4.665185642E9", "packedSize": "4.6651904E9", "beamline": "X12SA", "department": "6000"}, {"ownerGroup": "p22301", "copies": "oneCopyBig", "size": "746899935", "packedSize": "746956800", "beamline": "DMC", "department": "3000"}, {"ownerGroup": "p22365", "copies": "oneCopySmall", "size": "833784", "packedSize": "1187840", "beamline": "TASP", "department": "3000"}, {"ownerGroup": "p12657", "copies": "twoCopiesBig", "size": "2.1248152411E12", "packedSize": "2.12481383424E12", "beamline": "X02DA", "department": "6000"}, {"ownerGroup": "p19816", "copies": "oneCopyBig", "size": "2956146681", "packedSize": "2957690880", "beamline": "dolly", "department": "3000"}, {"ownerGroup": "p17919", "copies": "oneCopyBig", "size": "2.2781733726067E13", "packedSize": "2.278183621632E13", "beamline": "SARESB", "department": "6000"}, {"ownerGroup": "p18603", "copies": "oneCopySmall", "size": "671271", "packedSize": "911360", "beamline": "X03MA", "department": "6000"}, {"ownerGroup": "p10042", "copies": "oneCopyBig", "size": "172581107", "packedSize": "172718080", "beamline": "X04SA", "department": "6000"}, {"ownerGroup": "p21406", "copies": "oneCopySmall", "size": "11352178", "packedSize": "11683840", "beamline": "SANS-I", "department": "3000"}, {"ownerGroup": "p19342", "copies": "oneCopyBig", "size": "1.3209839361696E13", "packedSize": "1.Line truncated
+