data-catalog-services/ArchiveCostWebapp
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

- Cleanup Workflow um aeltere Files des Typ's size_by_ownergroup_and_number_of_copies_* zu loeschen

Browse Source 
- Trigger der Workflows: Zeit wird nun im docker-compose.yml File eingestellt.
This commit is contained in:
huesser
2026-06-25 14:50:55 +02:00
parent f95055c510
commit a77bc1b90b
11 changed files with 262 additions and 8958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker-compose.yml
+13 -1
View File
@@ -12,7 +12,7 @@ services:
- NON_SPECIFIED_GROUPS=/data/non_specified_groups.csv
- BEGIN_NAME_GROUPINFO_JSON_FILE=size_by_ownergroup_and_number_of_copies_
- BEGIN_NAME_GROUPINFO_JSON_FILE_WITH_PATH=/data/size_by_ownergroup_and_number_of_copies_
- ERROR_LOGFILE=error.log
- ERROR_LOGFILE=/data/logs/get_new_groupinfo_error.log
- JSON_CACHE=json_cache.json
- JSON_CACHE_WITH_PATH=/data/json_cache.json
- AD_SERVER=d.psi.ch
@@ -20,6 +20,16 @@ services:
- AD_USER=${AD_USER}
- AD_PASSWORD=${AD_PASSWORD}
- METABASE_API_KEY=${METABASE_API_KEY}
# Behalte size_by_ownergroup_and_number_of_copies_ Files juenger als CLEANUP_TIME Tage
- CLEANUP_TIME=20
# Ausfuehrungszeitpunkt fuer den Cleanup im Cron-Format (Minuten Stunden * * *)
- CLEANUP_CRON=0 7 * * *
# Pfad zum Cleanup-Skript
- CLEANUP_SCRIPT_PATH=/data/scripts/cleanup.sh
# Pfad zum Logfile des Cleanup-Skripts
- CLEANUP_LOGFILE_PATH=/data/logs/cleanup.log
# Ausfuehrungszeitpunkt fuer den Haupt-Workflow im Cron-Format
- MAIN_WORKFLOW_CRON=0 6 * * *
ports:
- "1880:1880"
@@ -31,5 +41,7 @@ services:
volumes:
- ./analytics:/app # Mappt deinen Code, damit Änderungen live ohne Rebuild sichtbar sind
- ./shared_data:/data:ro # Bindet die JSON-Daten NUR LESEND unter /data ein
- ./analytics/config:/app/config # Mappt den config Ordner
- ./analytics/data:/app/data # Mappt den data Ordner
environment:
- TZ=Europe/Zurich
logic/Dockerfile
+2 -2
View File
@@ -31,5 +31,5 @@ RUN pip3 install --no-cache-dir --break-system-packages \
# Zurück zum Node-RED User für die Sicherheit
USER node-red
# Installiere "moment" Node um die richtige Zeitzone verwenden zu koennen
RUN npm install node-red-contrib-moment
# Installiere zusaetzliche Node-RED Module (moment fuer Zeitzonen, cron-plus fuer erweiterte Zeitsteuerung)
RUN npm install node-red-contrib-moment node-red-contrib-cron-plus
logic/node-red-data/.config.nodes.json
+19
View File
@@ -456,5 +456,24 @@
"file": "/usr/src/node-red/node_modules/node-red-contrib-moment/moment/nrhumanizer.js"
}
}
},
"node-red-contrib-cron-plus": {
"name": "node-red-contrib-cron-plus",
"version": "2.2.4",
"local": false,
"user": false,
"nodes": {
"cronplus": {
"name": "cronplus",
"types": [
"cronplus"
],
"enabled": true,
"local": false,
"user": false,
"module": "node-red-contrib-cron-plus",
"file": "/usr/src/node-red/node_modules/node-red-contrib-cron-plus/cronplus.js"
}
}
}
}
logic/node-red-data/.config.users.json
+2 -2
View File
@@ -35,7 +35,7 @@
"config",
"context"
],
"height": 311,
"height": 466,
"hidden": false,
"active": "help"
},
@@ -53,7 +53,7 @@
"tabs": [
"explorer"
],
"height": 306,
"height": 308,
"hidden": false,
"active": "explorer"
},
logic/node-red-data/archivegroup_information.json
+1 -1
View File
@@ -1 +1 @@
size_by_ownergroup_and_number_of_copies_2026-06-23T17_18_00.json
size_by_ownergroup_and_number_of_copies_2026-06-25T14_46_23.json
logic/node-red-data/error.log
-4
View File
@@ -1,4 +0,0 @@
2026-05-07T15:50:03.349Z - ERROR in node: create link to newest archivegroup informationUnbekannter Node - message: manual test error
2026-05-07T15:51:14.151Z - ERROR in node: create link to newest archivegroup information - message: manual test error
2026-05-07T15:51:48.982Z - ERROR in node: 'create link to newest archivegroup information' - message: manual test error
2026-06-18T09:06:21.303Z - ERROR in node: 'create link to newest archivegroup information' - message: ls: size_by_ownergroup_and_number_of_copies_2026-06-18T09_03_11.261Z.json: Symbolic link loop
logic/node-red-data/flows.json
+220 -86
View File
@@ -22,6 +22,14 @@
"info": "",
"env": []
},
{
"id": "0e1f41f8e2a82a01",
"type": "tab",
"label": "Periodical Cleanup Filesystem",
"disabled": false,
"info": "",
"env": []
},
{
"id": "c87ef57b2517bbd5",
"type": "global-config",
@@ -256,7 +264,7 @@
"type": "file",
"z": "5a9dd923b7459820",
"name": "Write to error log",
"filename": "\"/data/\" & $env(\"ERROR_LOGFILE\")",
"filename": "$env(\"ERROR_LOGFILE\")",
"filenameType": "jsonata",
"appendNewline": false,
"createDir": false,
@@ -352,8 +360,8 @@
"91e9f1c0e52a0592",
"metabase_output_debug"
],
"x": 270,
"y": 760,
"x": 930,
"y": 160,
"wires": [
[]
]
@@ -571,43 +579,39 @@
"fb9ff82281dc83e0",
"dd3574493b97a5f4"
],
"x": 210,
"y": 520,
"x": 910,
"y": 200,
"wires": [
[]
]
},
{
"id": "44c678d93545a6f5",
"type": "inject",
"id": "cd1bcf72369d7e25",
"type": "status",
"z": "1593c8c9a04da215",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
"name": "\"Run\" workflow status",
"scope": [
"b994acfe29648a35",
"5dab821150d36d52"
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 140,
"y": 280,
"x": 900,
"y": 160,
"wires": [
[
"b994acfe29648a35"
]
[]
]
},
{
"id": "b994acfe29648a35",
"id": "25cacb1fa8cab069",
"type": "comment",
"z": "1593c8c9a04da215",
"name": "ZWECK: Erstelle einen neuen json-Cache mit den aktuellen Archivdaten, \\n damit der Webservice auf dies zugreiffen kann.",
"info": "",
"x": 280,
"y": 60,
"wires": []
},
{
"id": "d64ab3e4b088d812",
"type": "link call",
"z": "1593c8c9a04da215",
"name": "",
@@ -616,16 +620,16 @@
],
"linkType": "static",
"timeout": "0",
"x": 380,
"y": 280,
"x": 620,
"y": 340,
"wires": [
[
"5dab821150d36d52"
"6a59e981543cfc24"
]
]
},
{
"id": "5dab821150d36d52",
"id": "6a59e981543cfc24",
"type": "link call",
"z": "1593c8c9a04da215",
"name": "",
@@ -634,16 +638,16 @@
],
"linkType": "static",
"timeout": "0",
"x": 700,
"y": 280,
"x": 620,
"y": 440,
"wires": [
[
"4d7bd4a079e1c50d"
"3dc41b840224bcad"
]
]
},
{
"id": "4d7bd4a079e1c50d",
"id": "3dc41b840224bcad",
"type": "debug",
"z": "1593c8c9a04da215",
"name": "Workflow ended",
@@ -655,57 +659,12 @@
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 960,
"y": 280,
"x": 940,
"y": 440,
"wires": []
},
{
"id": "cd1bcf72369d7e25",
"type": "status",
"z": "1593c8c9a04da215",
"name": "\"Run\" workflow status",
"scope": [
"b994acfe29648a35",
"5dab821150d36d52"
],
"x": 140,
"y": 380,
"wires": [
[]
]
},
{
"id": "2f498ce2ead2e74c",
"type": "inject",
"z": "1593c8c9a04da215",
"name": "Periodischer Trigger",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "18 17 * * *",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 160,
"y": 180,
"wires": [
[
"b994acfe29648a35",
"c544bf44687bf00d"
]
]
},
{
"id": "c544bf44687bf00d",
"id": "7c99c77f41ddd3ac",
"type": "debug",
"z": "1593c8c9a04da215",
"name": "Trigger startet",
@@ -717,8 +676,183 @@
"targetType": "full",
"statusVal": "",
"statusType": "auto",
"x": 460,
"x": 560,
"y": 260,
"wires": []
},
{
"id": "0d879d4015da10a1",
"type": "cronplus",
"z": "1593c8c9a04da215",
"name": "Periodischer Trigger",
"outputField": "payload",
"timeZone": "",
"storeName": "",
"commandResponseMsgOutput": "output1",
"defaultLocation": "",
"defaultLocationType": "default",
"outputs": 1,
"options": [
{
"name": "schedule1",
"topic": "trigger",
"payloadType": "date",
"payload": "",
"expressionType": "cron",
"expression": "${MAIN_WORKFLOW_CRON}",
"location": "",
"offset": "0",
"solarType": "all",
"solarEvents": "sunrise,sunset"
}
],
"x": 240,
"y": 340,
"wires": [
[
"d64ab3e4b088d812",
"7c99c77f41ddd3ac"
]
]
},
{
"id": "80c7fa47b298de99",
"type": "exec",
"z": "0e1f41f8e2a82a01",
"command": "${CLEANUP_SCRIPT_PATH}",
"addpay": false,
"append": "",
"useSpawn": "false",
"timer": "",
"winHide": false,
"oldrc": false,
"name": "Führe Cleanup Skript aus",
"x": 610,
"y": 280,
"wires": [
[
"a3dccf14812e1a8a",
"2f1e47d10d6e77c3"
],
[
"a7d69391d5bb7e2c",
"2f1e47d10d6e77c3"
],
[
"7e5af8a04444818d"
]
]
},
{
"id": "a3dccf14812e1a8a",
"type": "debug",
"z": "0e1f41f8e2a82a01",
"name": "cleanup stdout",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 880,
"y": 220,
"wires": []
},
{
"id": "a7d69391d5bb7e2c",
"type": "debug",
"z": "0e1f41f8e2a82a01",
"name": "cleanup stderr",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 880,
"y": 260,
"wires": []
},
{
"id": "7e5af8a04444818d",
"type": "debug",
"z": "0e1f41f8e2a82a01",
"name": "cleanup return code",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 900,
"y": 300,
"wires": []
},
{
"id": "2f1e47d10d6e77c3",
"type": "file",
"z": "0e1f41f8e2a82a01",
"name": "Output -> Logfile",
"filename": "CLEANUP_LOGFILE_PATH",
"filenameType": "env",
"appendNewline": true,
"createDir": true,
"overwriteFile": "false",
"encoding": "none",
"x": 890,
"y": 180,
"wires": [
[]
]
},
{
"id": "72a71b1fd9b4c317",
"type": "cronplus",
"z": "0e1f41f8e2a82a01",
"name": "Periodischer Cleanup",
"outputField": "payload",
"timeZone": "",
"storeName": "",
"commandResponseMsgOutput": "output1",
"defaultLocation": "",
"defaultLocationType": "default",
"outputs": 1,
"options": [
{
"name": "schedule1",
"topic": "cleanup",
"payloadType": "date",
"payload": "",
"expressionType": "cron",
"expression": "${CLEANUP_CRON}",
"location": "",
"offset": "0",
"solarType": "all",
"solarEvents": "sunrise,sunset"
}
],
"x": 300,
"y": 280,
"wires": [
[
"80c7fa47b298de99"
]
]
},
{
"id": "2bacfef9837edc2b",
"type": "comment",
"z": "0e1f41f8e2a82a01",
"name": "ZWECK: Regelmaessiges Cleanup von alteren Files.",
"info": "",
"x": 230,
"y": 120,
"wires": []
}
]
logic/node-red-data/size_by_ownergroup_and_number_of_copies_2026-05-05T18_20_50.165030095Z.json
-4430
View File
File diff suppressed because it is too large Load Diff
logic/node-red-data/size_by_ownergroup_and_number_of_copies_2026-05-06T09_20_13.580575106Z.json
-4431
View File
File diff suppressed because it is too large Load Diff
logic/scripts/cleanup.sh
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/bash
echo "--- Starte Cleanup am $(date) ---"
find /data/results -maxdepth 1 -name "size_by_ownergroup_and_number_of_copies_*.json" -type f -mtime +${CLEANUP_TIME} -delete -print
echo "--- Cleanup abgeschlossen ---"
shared_data/json_cache.json
+1 -1
View File
File diff suppressed because one or more lines are too long