[ { "id": "5a9dd923b7459820", "type": "tab", "label": "Get New Groupinformation", "disabled": false, "info": "", "env": [] }, { "id": "f6f2187d.f17ca8", "type": "tab", "label": "Process Groupinformation", "disabled": false, "info": " - Reads the ingested/downloaded data\n - Adds division information to each group\n - Calculates the total archive volume for a division\n - Handles error in a detailed way" }, { "id": "1593c8c9a04da215", "type": "tab", "label": "Run", "disabled": false, "info": "", "env": [] }, { "id": "0e1f41f8e2a82a01", "type": "tab", "label": "Periodical Cleanup Filesystem", "disabled": false, "info": "", "env": [] }, { "id": "c87ef57b2517bbd5", "type": "global-config", "env": [], "modules": {} }, { "id": "eaa3eb4a201883c0", "type": "inject", "z": "5a9dd923b7459820", "name": "", "props": [ { "p": "payload" }, { "p": "topic", "vt": "str" } ], "repeat": "", "crontab": "", "once": false, "onceDelay": 0.1, "topic": "", "payload": "", "payloadType": "date", "x": 180, "y": 260, "wires": [ [ "metabase_api_flow" ] ] }, { "id": "metabase_api_flow", "type": "change", "z": "5a9dd923b7459820", "name": "Metabase configuration of request", "rules": [ { "t": "set", "p": "start_time", "pt": "msg", "to": "$millis()", "tot": "jsonata" }, { "t": "set", "p": "method", "pt": "msg", "to": "POST", "tot": "str" }, { "t": "set", "p": "headers", "pt": "msg", "to": "{\t \"x-api-key\": $env('METABASE_API_KEY'),\t \"Content-Type\": \"application/x-www-form-urlencoded;charset=UTF-8\",\t \"Accept\": \"application/json\"\t}", "tot": "jsonata" }, { "t": "set", "p": "payload", "pt": "msg", "to": "parameters=%5B%5D&format_rows=true&pivot_results=false", "tot": "str" }, { "t": "set", "p": "requestTimeout", "pt": "msg", "to": "300000", "tot": "num" } ], "action": "", "property": "", "from": "", "to": "", "reg": false, "x": 440, "y": 260, "wires": [ [ "metabase_http_node" ] ] }, { "id": "metabase_http_node", "type": "http request", "z": "5a9dd923b7459820", "name": "Metabase request", "method": "use", "ret": "obj", "paytoqs": "ignore", "url": "https://metabase.psi.ch/api/card/94/query/json", "tls": "", "persist": false, "proxy": "", "insecureHTTPParser": false, "authType": "", "senderr": false, "headers": [], "x": 730, "y": 260, "wires": [ [ "b8cc9c157460e2c5" ] ] }, { "id": "metabase_output_debug", "type": "debug", "z": "5a9dd923b7459820", "name": "Written file content", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 310, "y": 600, "wires": [] }, { "id": "b8cc9c157460e2c5", "type": "json", "z": "5a9dd923b7459820", "name": "Convert JSON object to JSON string", "property": "payload", "action": "", "pretty": false, "x": 250, "y": 360, "wires": [ [ "2d607e7db0d4c765" ] ] }, { "id": "91e9f1c0e52a0592", "type": "change", "z": "5a9dd923b7459820", "name": "Set Filename", "rules": [ { "t": "set", "p": "filename", "pt": "msg", "to": "$env(\"BEGIN_NAME_GROUPINFO_JSON_FILE_WITH_PATH\") & msg.date_string & \".json\"", "tot": "jsonata" } ], "action": "", "property": "", "from": "", "to": "", "reg": false, "x": 810, "y": 360, "wires": [ [ "7e0c38cadf8c5ebe", "737f4bb0df2e799c" ] ] }, { "id": "cdbf736ee149ac8f", "type": "comment", "z": "5a9dd923b7459820", "name": "ZWECK: Kontaktiert metabase und laedt eine Liste von JSON Dictionaires herunter,\\n welche Informationen ueber die einzelnen Archivgruppen enthalten. \\n Diese JSON-Struktur wird dann in ein File abgespeichert.", "info": "", "x": 350, "y": 100, "wires": [] }, { "id": "63ed8c262fa7753b", "type": "exec", "z": "5a9dd923b7459820", "command": "cd /data/; ln -sf $(ls $BEGIN_NAME_GROUPINFO_JSON_FILE*.json | sort | tail -n 1) $GROUPINFO_JSON_LNK_NAME_WITH_PATH", "addpay": "", "append": "", "useSpawn": "false", "timer": "", "winHide": false, "oldrc": false, "name": "Create link to newest archivegroup information json", "x": 610, "y": 500, "wires": [ [ "0957c917da9bc8b3", "3e09805d0a8e88f9" ], [ "2d8f3b73dc80bf4a" ], [ "0957c917da9bc8b3" ] ] }, { "id": "0957c917da9bc8b3", "type": "debug", "z": "5a9dd923b7459820", "name": "See external commands non errors", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "true", "targetType": "full", "statusVal": "", "statusType": "auto", "x": 1080, "y": 500, "wires": [] }, { "id": "7fa57e2c57d1e6eb", "type": "file", "z": "5a9dd923b7459820", "name": "Write to error log", "filename": "$env(\"ERROR_LOGFILE\")", "filenameType": "jsonata", "appendNewline": false, "createDir": false, "overwriteFile": "false", "encoding": "none", "x": 750, "y": 660, "wires": [ [ "394f5382cb6182dd" ] ] }, { "id": "2d8f3b73dc80bf4a", "type": "change", "z": "5a9dd923b7459820", "name": "Create error log entry", "rules": [ { "t": "set", "p": "payload", "pt": "msg", "to": "$now() & \" - ERROR in node: 'create link to newest archivegroup information'\" & \" - message: \" & payload", "tot": "jsonata" } ], "action": "", "property": "", "from": "", "to": "", "reg": false, "x": 540, "y": 660, "wires": [ [ "7fa57e2c57d1e6eb" ] ] }, { "id": "394f5382cb6182dd", "type": "debug", "z": "5a9dd923b7459820", "name": "Error debug", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "true", "targetType": "full", "statusVal": "", "statusType": "auto", "x": 950, "y": 660, "wires": [] }, { "id": "913d320bfbe13635", "type": "link in", "z": "5a9dd923b7459820", "name": "link in Get New Groupinformation", "links": [], "x": 185, "y": 180, "wires": [ [ "metabase_api_flow" ] ] }, { "id": "3e09805d0a8e88f9", "type": "link out", "z": "5a9dd923b7459820", "name": "link out Get New Groupinformation", "mode": "return", "links": [], "x": 975, "y": 420, "wires": [] }, { "id": "c62bf4e1cfd12c03", "type": "status", "z": "5a9dd923b7459820", "name": "\"Get New Groupinformation\" workflow status", "scope": [ "b8cc9c157460e2c5", "63ed8c262fa7753b", "metabase_api_flow", "metabase_http_node", "91e9f1c0e52a0592", "metabase_output_debug" ], "x": 930, "y": 160, "wires": [ [] ] }, { "id": "2d607e7db0d4c765", "type": "moment", "z": "5a9dd923b7459820", "name": "Prepare date format for filename", "topic": "", "input": "start_time", "inputType": "msg", "inTz": "Europe/Zurich", "adjAmount": 0, "adjType": "days", "adjDir": "add", "format": "YYYY-MM-DD[T]HH_mm_ss", "locale": "en-US", "output": "date_string", "outputType": "msg", "outTz": "Europe/Zurich", "x": 570, "y": 360, "wires": [ [ "91e9f1c0e52a0592" ] ] }, { "id": "7e0c38cadf8c5ebe", "type": "debug", "z": "5a9dd923b7459820", "name": "Show download filename", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "filename", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 1030, "y": 260, "wires": [] }, { "id": "737f4bb0df2e799c", "type": "file", "z": "5a9dd923b7459820", "name": "Write json to disk", "filename": "filename", "filenameType": "msg", "appendNewline": true, "createDir": false, "overwriteFile": true, "encoding": "none", "x": 190, "y": 500, "wires": [ [ "metabase_output_debug", "63ed8c262fa7753b" ] ] }, { "id": "3cc11d24.ff01a2", "type": "comment", "z": "f6f2187d.f17ca8", "name": "WARNING: please check you have started this container with a volume that is mounted to /data\\n otherwise any flow changes are lost when you redeploy or upgrade the container\\n (e.g. upgrade to a more recent node-red docker image).\\n If you are using named volumes you can ignore this warning.\\n Double click or see info side panel to learn how to start Node-RED in Docker to save your work", "info": "\nTo start docker with a bind mount volume (-v option), for example:\n\n```\ndocker run -it -p 1880:1880 -v /home/user/node_red_data:/data --name mynodered nodered/node-red\n```\n\nwhere `/home/user/node_red_data` is a directory on your host machine where you want to store your flows.\n\nIf you do not do this then you can experiment and redploy flows, but if you restart or upgrade the container the flows will be disconnected and lost. \n\nThey will still exist in a hidden data volume, which can be recovered using standard docker techniques, but that is much more complex than just starting with a named volume as described above.", "x": 350, "y": 60, "wires": [] }, { "id": "a918100ba6063585", "type": "inject", "z": "f6f2187d.f17ca8", "name": "", "props": [ { "p": "payload" }, { "p": "topic", "vt": "str" } ], "repeat": "", "crontab": "", "once": false, "onceDelay": 0.1, "topic": "", "payload": "", "payloadType": "date", "x": 120, "y": 340, "wires": [ [ "fb9ff82281dc83e0" ] ] }, { "id": "fb9ff82281dc83e0", "type": "exec", "z": "f6f2187d.f17ca8", "command": "python3 /data/scripts/ad_lookup.py", "addpay": "", "append": "", "useSpawn": "false", "timer": "", "winHide": false, "oldrc": false, "name": "Call add_lookup.py", "x": 330, "y": 340, "wires": [ [ "dd3574493b97a5f4" ], [ "444c29de997bd1b6" ], [] ] }, { "id": "426e46f53063d368", "type": "debug", "z": "f6f2187d.f17ca8", "name": "Write cache payload", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 1060, "y": 280, "wires": [] }, { "id": "444c29de997bd1b6", "type": "debug", "z": "f6f2187d.f17ca8", "name": "add_lookup.py errors", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 700, "y": 420, "wires": [] }, { "id": "dd3574493b97a5f4", "type": "file", "z": "f6f2187d.f17ca8", "name": "Write the new JSON object to cache", "filename": "\"/data/results/\" & $env(\"JSON_CACHE\")", "filenameType": "jsonata", "appendNewline": true, "createDir": false, "overwriteFile": "true", "encoding": "utf8", "x": 750, "y": 320, "wires": [ [ "426e46f53063d368", "dbc196df1931ee06" ] ] }, { "id": "eb6d443e97fedbee", "type": "link in", "z": "f6f2187d.f17ca8", "name": "link in Process Groupinformation", "links": [], "x": 115, "y": 260, "wires": [ [ "fb9ff82281dc83e0" ] ] }, { "id": "dbc196df1931ee06", "type": "link out", "z": "f6f2187d.f17ca8", "name": "link out Process Groupinformation", "mode": "return", "links": [], "x": 1015, "y": 360, "wires": [] }, { "id": "845b63e247001e57", "type": "status", "z": "f6f2187d.f17ca8", "name": "\"Process Groupinformation\" workflow status", "scope": [ "fb9ff82281dc83e0", "dd3574493b97a5f4" ], "x": 910, "y": 200, "wires": [ [] ] }, { "id": "cd1bcf72369d7e25", "type": "status", "z": "1593c8c9a04da215", "name": "\"Run\" workflow status", "scope": [ "b994acfe29648a35", "5dab821150d36d52" ], "x": 900, "y": 160, "wires": [ [] ] }, { "id": "25cacb1fa8cab069", "type": "comment", "z": "1593c8c9a04da215", "name": "ZWECK: Erstelle einen neuen json-Cache mit den aktuellen Archivdaten, \\n damit der Webservice auf dies zugreiffen kann.", "info": "", "x": 280, "y": 60, "wires": [] }, { "id": "d64ab3e4b088d812", "type": "link call", "z": "1593c8c9a04da215", "name": "", "links": [ "913d320bfbe13635" ], "linkType": "static", "timeout": "0", "x": 620, "y": 340, "wires": [ [ "6a59e981543cfc24" ] ] }, { "id": "6a59e981543cfc24", "type": "link call", "z": "1593c8c9a04da215", "name": "", "links": [ "eb6d443e97fedbee" ], "linkType": "static", "timeout": "0", "x": 620, "y": 440, "wires": [ [ "3dc41b840224bcad" ] ] }, { "id": "3dc41b840224bcad", "type": "debug", "z": "1593c8c9a04da215", "name": "Workflow ended", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "true", "targetType": "full", "statusVal": "", "statusType": "auto", "x": 940, "y": 440, "wires": [] }, { "id": "7c99c77f41ddd3ac", "type": "debug", "z": "1593c8c9a04da215", "name": "Trigger startet", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "true", "targetType": "full", "statusVal": "", "statusType": "auto", "x": 560, "y": 260, "wires": [] }, { "id": "0d879d4015da10a1", "type": "cronplus", "z": "1593c8c9a04da215", "name": "Periodischer Trigger", "outputField": "payload", "timeZone": "", "storeName": "", "commandResponseMsgOutput": "output1", "defaultLocation": "", "defaultLocationType": "default", "outputs": 1, "options": [ { "name": "schedule1", "topic": "trigger", "payloadType": "date", "payload": "", "expressionType": "cron", "expression": "${MAIN_WORKFLOW_CRON}", "location": "", "offset": "0", "solarType": "all", "solarEvents": "sunrise,sunset" } ], "x": 240, "y": 340, "wires": [ [ "d64ab3e4b088d812", "7c99c77f41ddd3ac" ] ] }, { "id": "80c7fa47b298de99", "type": "exec", "z": "0e1f41f8e2a82a01", "command": "${CLEANUP_SCRIPT_PATH}", "addpay": false, "append": "", "useSpawn": "false", "timer": "", "winHide": false, "oldrc": false, "name": "Führe Cleanup Skript aus", "x": 610, "y": 280, "wires": [ [ "a3dccf14812e1a8a", "2f1e47d10d6e77c3" ], [ "a7d69391d5bb7e2c", "2f1e47d10d6e77c3" ], [ "7e5af8a04444818d" ] ] }, { "id": "a3dccf14812e1a8a", "type": "debug", "z": "0e1f41f8e2a82a01", "name": "cleanup stdout", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 880, "y": 220, "wires": [] }, { "id": "a7d69391d5bb7e2c", "type": "debug", "z": "0e1f41f8e2a82a01", "name": "cleanup stderr", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 880, "y": 260, "wires": [] }, { "id": "7e5af8a04444818d", "type": "debug", "z": "0e1f41f8e2a82a01", "name": "cleanup return code", "active": true, "tosidebar": true, "console": false, "tostatus": false, "complete": "payload", "targetType": "msg", "statusVal": "", "statusType": "auto", "x": 900, "y": 300, "wires": [] }, { "id": "2f1e47d10d6e77c3", "type": "file", "z": "0e1f41f8e2a82a01", "name": "Output -> Logfile", "filename": "CLEANUP_LOGFILE_PATH", "filenameType": "env", "appendNewline": true, "createDir": true, "overwriteFile": "false", "encoding": "none", "x": 890, "y": 180, "wires": [ [] ] }, { "id": "72a71b1fd9b4c317", "type": "cronplus", "z": "0e1f41f8e2a82a01", "name": "Periodischer Cleanup", "outputField": "payload", "timeZone": "", "storeName": "", "commandResponseMsgOutput": "output1", "defaultLocation": "", "defaultLocationType": "default", "outputs": 1, "options": [ { "name": "schedule1", "topic": "cleanup", "payloadType": "date", "payload": "", "expressionType": "cron", "expression": "${CLEANUP_CRON}", "location": "", "offset": "0", "solarType": "all", "solarEvents": "sunrise,sunset" } ], "x": 300, "y": 280, "wires": [ [ "80c7fa47b298de99" ] ] }, { "id": "2bacfef9837edc2b", "type": "comment", "z": "0e1f41f8e2a82a01", "name": "ZWECK: Regelmaessiges Cleanup von alteren Files.", "info": "", "x": 230, "y": 120, "wires": [] } ]