data-catalog-services/ArchiveCostWebapp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Taeglicher Update des Caches implementiert

Browse Source
This commit is contained in:
huesser
2026-06-23 17:30:09 +02:00
parent ae5af28782
commit f95055c510
5 changed files with 80 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO.md
+3 -1
View File
@@ -17,8 +17,9 @@
## ⚙️ Node-RED & AD-Skript
- [x] Python-Skript auf lokale Datei-Pipes umstellen
- [x] Zweistufige p-Gruppen-Abfrage stabilisieren
- [ ] Cronjob/Intervall in Node-RED für tägliches Update einrichten
- [x] Cronjob/Intervall in Node-RED für tägliches Update einrichten
- [ ] Fehlerhandling
- [ ] Bereinigung der Workflows
- [x] Zeit wird falsch angegeben (Zeit fuer Filenamen mit $now)
- [x] Aktuell Fehler beim berechnen der Zeit fuer das File
- [x] In "Set Filename" wird soll eine Umgebungsvariable verwendet werden
@@ -29,6 +30,7 @@
- [ ] requirements.txt File erstellen, falls es dies ueberhaupt braucht.
## 😪 Fehlerbehandlung
- [ ] Wo braucht es ueberall Fehlerbehandlungen?
- [ ] Fehler beim periodischen Update der Daten abfangen
## 📜 Dokumentation
logic/node-red-data/.config.users.json
+5 -4
View File
@@ -27,7 +27,7 @@
"v": 4,
"tabs": [],
"primary": {
"width": 276,
"width": 304,
"top": {
"tabs": [
"info",
@@ -35,7 +35,7 @@
"config",
"context"
],
"height": 153,
"height": 311,
"hidden": false,
"active": "help"
},
@@ -53,7 +53,7 @@
"tabs": [
"explorer"
],
"height": 304,
"height": 306,
"hidden": false,
"active": "explorer"
},
@@ -67,6 +67,7 @@
}
}
}
}
},
"menu-menu-item-left-sidebar": true
}
}
logic/node-red-data/archivegroup_information.json
+1 -1
View File
@@ -1 +1 @@
size_by_ownergroup_and_number_of_copies_2026-06-23T11_26_07.json
size_by_ownergroup_and_number_of_copies_2026-06-23T17_18_00.json
logic/node-red-data/flows.json
+70 -221
View File
@@ -140,7 +140,7 @@
"id": "metabase_output_debug",
"type": "debug",
"z": "5a9dd923b7459820",
"name": "Written File",
"name": "Written file content",
"active": true,
"tosidebar": true,
"console": false,
@@ -169,23 +169,6 @@
]
]
},
{
"id": "cc064f55fd438347",
"type": "file",
"z": "5a9dd923b7459820",
"name": "Write json to disk old",
"filename": "filename",
"filenameType": "msg",
"appendNewline": true,
"createDir": false,
"overwriteFile": "true",
"encoding": "none",
"x": 900,
"y": 1000,
"wires": [
[]
]
},
{
"id": "91e9f1c0e52a0592",
"type": "change",
@@ -235,7 +218,7 @@
"timer": "",
"winHide": false,
"oldrc": false,
"name": "create link to newest archivegroup information json",
"name": "Create link to newest archivegroup information json",
"x": 610,
"y": 500,
"wires": [
@@ -260,8 +243,8 @@
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 1080,
@@ -272,14 +255,14 @@
"id": "7fa57e2c57d1e6eb",
"type": "file",
"z": "5a9dd923b7459820",
"name": "write log error",
"name": "Write to error log",
"filename": "\"/data/\" & $env(\"ERROR_LOGFILE\")",
"filenameType": "jsonata",
"appendNewline": false,
"createDir": false,
"overwriteFile": "false",
"encoding": "none",
"x": 740,
"x": 750,
"y": 660,
"wires": [
[
@@ -291,7 +274,7 @@
"id": "2d8f3b73dc80bf4a",
"type": "change",
"z": "5a9dd923b7459820",
"name": "",
"name": "Create error log entry",
"rules": [
{
"t": "set",
@@ -306,7 +289,7 @@
"from": "",
"to": "",
"reg": false,
"x": 520,
"x": 540,
"y": 660,
"wires": [
[
@@ -318,16 +301,16 @@
"id": "394f5382cb6182dd",
"type": "debug",
"z": "5a9dd923b7459820",
"name": "data ingest error debug",
"name": "Error debug",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 990,
"x": 950,
"y": 660,
"wires": []
},
@@ -360,41 +343,21 @@
"id": "c62bf4e1cfd12c03",
"type": "status",
"z": "5a9dd923b7459820",
"name": "workflow status",
"name": "\"Get New Groupinformation\" workflow status",
"scope": [
"b8cc9c157460e2c5",
"63ed8c262fa7753b",
"metabase_api_flow",
"metabase_http_node",
"metabase_output_debug",
"b8cc9c157460e2c5",
"cc064f55fd438347",
"91e9f1c0e52a0592",
"63ed8c262fa7753b"
"metabase_output_debug"
],
"x": 240,
"x": 270,
"y": 760,
"wires": [
[
"dfd30c3d6f180d27"
]
[]
]
},
{
"id": "dfd30c3d6f180d27",
"type": "debug",
"z": "5a9dd923b7459820",
"name": "debug workflow",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 460,
"y": 760,
"wires": []
},
{
"id": "2d607e7db0d4c765",
"type": "moment",
@@ -424,105 +387,7 @@
"id": "7e0c38cadf8c5ebe",
"type": "debug",
"z": "5a9dd923b7459820",
"name": "Show filename",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 1000,
"y": 260,
"wires": []
},
{
"id": "4c790912771b89dc",
"type": "moment",
"z": "5a9dd923b7459820",
"name": "Prepare date format for filename",
"topic": "",
"input": "start_time",
"inputType": "msg",
"inTz": "Europe/Zurich",
"adjAmount": 0,
"adjType": "days",
"adjDir": "add",
"format": "YYYY-MM-DD[T]HH_mm_ss",
"locale": "en-US",
"output": "date_string",
"outputType": "msg",
"outTz": "Europe/Zurich",
"x": 430,
"y": 880,
"wires": [
[
"c55ac9de5736d58c"
]
]
},
{
"id": "c55ac9de5736d58c",
"type": "change",
"z": "5a9dd923b7459820",
"name": "Set Filename",
"rules": [
{
"t": "set",
"p": "filename",
"pt": "msg",
"to": "$env(\"BEGIN_NAME_GROUPINFO_JSON_FILE\") & msg.date_string & \".json\"",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 710,
"y": 880,
"wires": [
[
"5ede70b6fab13b03"
]
]
},
{
"id": "3061839cda32600d",
"type": "inject",
"z": "5a9dd923b7459820",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 180,
"y": 880,
"wires": [
[
"4c790912771b89dc"
]
]
},
{
"id": "5ede70b6fab13b03",
"type": "debug",
"z": "5a9dd923b7459820",
"name": "debug 1",
"name": "Show download filename",
"active": true,
"tosidebar": true,
"console": false,
@@ -531,8 +396,8 @@
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 900,
"y": 880,
"x": 1030,
"y": 260,
"wires": []
},
{
@@ -562,7 +427,7 @@
"name": "WARNING: please check you have started this container with a volume that is mounted to /data\\n otherwise any flow changes are lost when you redeploy or upgrade the container\\n (e.g. upgrade to a more recent node-red docker image).\\n If you are using named volumes you can ignore this warning.\\n Double click or see info side panel to learn how to start Node-RED in Docker to save your work",
"info": "\nTo start docker with a bind mount volume (-v option), for example:\n\n```\ndocker run -it -p 1880:1880 -v /home/user/node_red_data:/data --name mynodered nodered/node-red\n```\n\nwhere `/home/user/node_red_data` is a directory on your host machine where you want to store your flows.\n\nIf you do not do this then you can experiment and redploy flows, but if you restart or upgrade the container the flows will be disconnected and lost. \n\nThey will still exist in a hidden data volume, which can be recovered using standard docker techniques, but that is much more complex than just starting with a named volume as described above.",
"x": 350,
"y": 80,
"y": 60,
"wires": []
},
{
@@ -605,8 +470,8 @@
"timer": "",
"winHide": false,
"oldrc": false,
"name": "",
"x": 380,
"name": "Call add_lookup.py",
"x": 330,
"y": 340,
"wires": [
[
@@ -622,28 +487,30 @@
"id": "426e46f53063d368",
"type": "debug",
"z": "f6f2187d.f17ca8",
"name": "debug 3",
"name": "Write cache payload",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "false",
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 1020,
"y": 260,
"x": 1060,
"y": 280,
"wires": []
},
{
"id": "444c29de997bd1b6",
"type": "debug",
"z": "f6f2187d.f17ca8",
"name": "debug 4",
"name": "add_lookup.py errors",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "false",
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 700,
@@ -692,43 +559,24 @@
"mode": "return",
"links": [],
"x": 1015,
"y": 400,
"y": 360,
"wires": []
},
{
"id": "845b63e247001e57",
"type": "status",
"z": "f6f2187d.f17ca8",
"name": "workflow status",
"name": "\"Process Groupinformation\" workflow status",
"scope": [
"fb9ff82281dc83e0",
"dd3574493b97a5f4"
],
"x": 640,
"y": 220,
"x": 210,
"y": 520,
"wires": [
[
"1e2a3c41555c2d9b"
]
[]
]
},
{
"id": "1e2a3c41555c2d9b",
"type": "debug",
"z": "f6f2187d.f17ca8",
"name": "debug workflow",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 880,
"y": 220,
"wires": []
},
{
"id": "44c678d93545a6f5",
"type": "inject",
@@ -767,7 +615,7 @@
"913d320bfbe13635"
],
"linkType": "static",
"timeout": "300",
"timeout": "0",
"x": 380,
"y": 280,
"wires": [
@@ -785,7 +633,7 @@
"eb6d443e97fedbee"
],
"linkType": "static",
"timeout": "300",
"timeout": "0",
"x": 700,
"y": 280,
"wires": [
@@ -798,13 +646,13 @@
"id": "4d7bd4a079e1c50d",
"type": "debug",
"z": "1593c8c9a04da215",
"name": "End of workflow",
"name": "Workflow ended",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 960,
@@ -815,36 +663,17 @@
"id": "cd1bcf72369d7e25",
"type": "status",
"z": "1593c8c9a04da215",
"name": "workflow status",
"name": "\"Run\" workflow status",
"scope": [
"b994acfe29648a35",
"5dab821150d36d52"
],
"x": 400,
"y": 360,
"x": 140,
"y": 380,
"wires": [
[
"a613cbef5aae458b"
]
[]
]
},
{
"id": "a613cbef5aae458b",
"type": "debug",
"z": "1593c8c9a04da215",
"name": "debug workflow",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 620,
"y": 360,
"wires": []
},
{
"id": "2f498ce2ead2e74c",
"type": "inject",
@@ -859,17 +688,37 @@
"vt": "str"
}
],
"repeat": "300",
"crontab": "",
"repeat": "",
"crontab": "18 17 * * *",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 180,
"x": 160,
"y": 180,
"wires": [
[]
[
"b994acfe29648a35",
"c544bf44687bf00d"
]
]
},
{
"id": "c544bf44687bf00d",
"type": "debug",
"z": "1593c8c9a04da215",
"name": "Trigger startet",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "true",
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 460,
"y": 180,
"wires": []
}
]
shared_data/json_cache.json
+1 -1
View File
File diff suppressed because one or more lines are too long