760 lines
19 KiB
JSON
760 lines
19 KiB
JSON
[
|
|
{
|
|
"id": "5a9dd923b7459820",
|
|
"type": "tab",
|
|
"label": "Get New Groupinformation",
|
|
"disabled": false,
|
|
"info": "",
|
|
"env": []
|
|
},
|
|
{
|
|
"id": "f6f2187d.f17ca8",
|
|
"type": "tab",
|
|
"label": "Process Groupinformation",
|
|
"disabled": false,
|
|
"info": " - Reads the ingested/downloaded data\n - Adds division information to each group\n - Calculates the total archive volume for a division\n - Handles error in a detailed way"
|
|
},
|
|
{
|
|
"id": "1593c8c9a04da215",
|
|
"type": "tab",
|
|
"label": "Run",
|
|
"disabled": false,
|
|
"info": "",
|
|
"env": []
|
|
},
|
|
{
|
|
"id": "c87ef57b2517bbd5",
|
|
"type": "global-config",
|
|
"env": [],
|
|
"modules": {}
|
|
},
|
|
{
|
|
"id": "eaa3eb4a201883c0",
|
|
"type": "inject",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "",
|
|
"props": [
|
|
{
|
|
"p": "payload"
|
|
},
|
|
{
|
|
"p": "topic",
|
|
"vt": "str"
|
|
}
|
|
],
|
|
"repeat": "",
|
|
"crontab": "",
|
|
"once": false,
|
|
"onceDelay": 0.1,
|
|
"topic": "",
|
|
"payload": "",
|
|
"payloadType": "date",
|
|
"x": 120,
|
|
"y": 260,
|
|
"wires": [
|
|
[
|
|
"metabase_api_flow"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "metabase_api_flow",
|
|
"type": "change",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Metabase configuration of request",
|
|
"rules": [
|
|
{
|
|
"t": "set",
|
|
"p": "start_time",
|
|
"pt": "msg",
|
|
"to": "$millis()",
|
|
"tot": "jsonata"
|
|
},
|
|
{
|
|
"t": "set",
|
|
"p": "method",
|
|
"pt": "msg",
|
|
"to": "POST",
|
|
"tot": "str"
|
|
},
|
|
{
|
|
"t": "set",
|
|
"p": "headers",
|
|
"pt": "msg",
|
|
"to": "{\t \"x-api-key\": $env('METABASE_API_KEY'),\t \"Content-Type\": \"application/x-www-form-urlencoded;charset=UTF-8\",\t \"Accept\": \"application/json\"\t}",
|
|
"tot": "jsonata"
|
|
},
|
|
{
|
|
"t": "set",
|
|
"p": "payload",
|
|
"pt": "msg",
|
|
"to": "parameters=%5B%5D&format_rows=true&pivot_results=false",
|
|
"tot": "str"
|
|
},
|
|
{
|
|
"t": "set",
|
|
"p": "requestTimeout",
|
|
"pt": "msg",
|
|
"to": "300000",
|
|
"tot": "num"
|
|
}
|
|
],
|
|
"action": "",
|
|
"property": "",
|
|
"from": "",
|
|
"to": "",
|
|
"reg": false,
|
|
"x": 380,
|
|
"y": 260,
|
|
"wires": [
|
|
[
|
|
"metabase_http_node"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "metabase_http_node",
|
|
"type": "http request",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Metabase request",
|
|
"method": "use",
|
|
"ret": "obj",
|
|
"paytoqs": "ignore",
|
|
"url": "https://metabase.psi.ch/api/card/94/query/json",
|
|
"tls": "",
|
|
"persist": false,
|
|
"proxy": "",
|
|
"insecureHTTPParser": false,
|
|
"authType": "",
|
|
"senderr": false,
|
|
"headers": [],
|
|
"x": 670,
|
|
"y": 260,
|
|
"wires": [
|
|
[
|
|
"b8cc9c157460e2c5"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "metabase_output_debug",
|
|
"type": "debug",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Written File",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "payload",
|
|
"targetType": "msg",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 250,
|
|
"y": 600,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "b8cc9c157460e2c5",
|
|
"type": "json",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Convert JSON object to JSON string",
|
|
"property": "payload",
|
|
"action": "",
|
|
"pretty": false,
|
|
"x": 190,
|
|
"y": 360,
|
|
"wires": [
|
|
[
|
|
"2d607e7db0d4c765"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "cc064f55fd438347",
|
|
"type": "file",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Write json to disk",
|
|
"filename": "filename",
|
|
"filenameType": "msg",
|
|
"appendNewline": true,
|
|
"createDir": false,
|
|
"overwriteFile": "true",
|
|
"encoding": "none",
|
|
"x": 130,
|
|
"y": 500,
|
|
"wires": [
|
|
[
|
|
"metabase_output_debug",
|
|
"63ed8c262fa7753b"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "91e9f1c0e52a0592",
|
|
"type": "change",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Set Filename",
|
|
"rules": [
|
|
{
|
|
"t": "set",
|
|
"p": "filename",
|
|
"pt": "msg",
|
|
"to": "\"/data/size_by_ownergroup_and_number_of_copies_\" & msg.date_string & \".json\"",
|
|
"tot": "jsonata"
|
|
}
|
|
],
|
|
"action": "",
|
|
"property": "",
|
|
"from": "",
|
|
"to": "",
|
|
"reg": false,
|
|
"x": 750,
|
|
"y": 360,
|
|
"wires": [
|
|
[
|
|
"cc064f55fd438347",
|
|
"7e0c38cadf8c5ebe"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "cdbf736ee149ac8f",
|
|
"type": "comment",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "ZWECK: Kontaktiert metabase und laedt eine Liste von JSON Dictionaires herunter,\\n welche Informationen ueber die einzelnen Archivgruppen enthalten. \\n Diese JSON-Struktur wird dann in ein File abgespeichert.",
|
|
"info": "",
|
|
"x": 350,
|
|
"y": 100,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "63ed8c262fa7753b",
|
|
"type": "exec",
|
|
"z": "5a9dd923b7459820",
|
|
"command": "cd /data/; ln -sf $(ls $BEGIN_NAME_GROUPINFO_JSON_FILE*.json | sort | tail -n 1) $GROUPINFO_JSON_LNK_NAME_WITH_PATH",
|
|
"addpay": "",
|
|
"append": "",
|
|
"useSpawn": "false",
|
|
"timer": "",
|
|
"winHide": false,
|
|
"oldrc": false,
|
|
"name": "create link to newest archivegroup information json",
|
|
"x": 550,
|
|
"y": 500,
|
|
"wires": [
|
|
[
|
|
"0957c917da9bc8b3",
|
|
"3e09805d0a8e88f9"
|
|
],
|
|
[
|
|
"2d8f3b73dc80bf4a"
|
|
],
|
|
[
|
|
"0957c917da9bc8b3"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "0957c917da9bc8b3",
|
|
"type": "debug",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "See external commands non errors",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "payload",
|
|
"targetType": "msg",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 1020,
|
|
"y": 500,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "7fa57e2c57d1e6eb",
|
|
"type": "file",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "write log error",
|
|
"filename": "\"/data/\" & $env(\"ERROR_LOGFILE\")",
|
|
"filenameType": "jsonata",
|
|
"appendNewline": false,
|
|
"createDir": false,
|
|
"overwriteFile": "false",
|
|
"encoding": "none",
|
|
"x": 680,
|
|
"y": 660,
|
|
"wires": [
|
|
[
|
|
"394f5382cb6182dd"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "2d8f3b73dc80bf4a",
|
|
"type": "change",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "",
|
|
"rules": [
|
|
{
|
|
"t": "set",
|
|
"p": "payload",
|
|
"pt": "msg",
|
|
"to": "$now() & \" - ERROR in node: 'create link to newest archivegroup information'\" & \" - message: \" & payload",
|
|
"tot": "jsonata"
|
|
}
|
|
],
|
|
"action": "",
|
|
"property": "",
|
|
"from": "",
|
|
"to": "",
|
|
"reg": false,
|
|
"x": 460,
|
|
"y": 660,
|
|
"wires": [
|
|
[
|
|
"7fa57e2c57d1e6eb"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "394f5382cb6182dd",
|
|
"type": "debug",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "data ingest error debug",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "payload",
|
|
"targetType": "msg",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 930,
|
|
"y": 660,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "913d320bfbe13635",
|
|
"type": "link in",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "link in Get New Groupinformation",
|
|
"links": [],
|
|
"x": 125,
|
|
"y": 180,
|
|
"wires": [
|
|
[
|
|
"metabase_api_flow"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "3e09805d0a8e88f9",
|
|
"type": "link out",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "link out Get New Groupinformation",
|
|
"mode": "return",
|
|
"links": [],
|
|
"x": 915,
|
|
"y": 420,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "c62bf4e1cfd12c03",
|
|
"type": "status",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "workflow status",
|
|
"scope": [
|
|
"metabase_api_flow",
|
|
"metabase_http_node",
|
|
"metabase_output_debug",
|
|
"b8cc9c157460e2c5",
|
|
"cc064f55fd438347",
|
|
"91e9f1c0e52a0592",
|
|
"63ed8c262fa7753b"
|
|
],
|
|
"x": 240,
|
|
"y": 760,
|
|
"wires": [
|
|
[
|
|
"dfd30c3d6f180d27"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "dfd30c3d6f180d27",
|
|
"type": "debug",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "debug workflow",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "true",
|
|
"targetType": "full",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 460,
|
|
"y": 760,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "2d607e7db0d4c765",
|
|
"type": "moment",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Prepare date format for filename",
|
|
"topic": "",
|
|
"input": "start_time",
|
|
"inputType": "msg",
|
|
"inTz": "Europe/Zurich",
|
|
"adjAmount": 0,
|
|
"adjType": "days",
|
|
"adjDir": "add",
|
|
"format": "YYYY-MM-DD[T]HH_mm_ss",
|
|
"locale": "en-US",
|
|
"output": "date_string",
|
|
"outputType": "msg",
|
|
"outTz": "Europe/Zurich",
|
|
"x": 510,
|
|
"y": 360,
|
|
"wires": [
|
|
[
|
|
"91e9f1c0e52a0592"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "7e0c38cadf8c5ebe",
|
|
"type": "debug",
|
|
"z": "5a9dd923b7459820",
|
|
"name": "Show filename",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "filename",
|
|
"targetType": "msg",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 940,
|
|
"y": 260,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "3cc11d24.ff01a2",
|
|
"type": "comment",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "WARNING: please check you have started this container with a volume that is mounted to /data\\n otherwise any flow changes are lost when you redeploy or upgrade the container\\n (e.g. upgrade to a more recent node-red docker image).\\n If you are using named volumes you can ignore this warning.\\n Double click or see info side panel to learn how to start Node-RED in Docker to save your work",
|
|
"info": "\nTo start docker with a bind mount volume (-v option), for example:\n\n```\ndocker run -it -p 1880:1880 -v /home/user/node_red_data:/data --name mynodered nodered/node-red\n```\n\nwhere `/home/user/node_red_data` is a directory on your host machine where you want to store your flows.\n\nIf you do not do this then you can experiment and redploy flows, but if you restart or upgrade the container the flows will be disconnected and lost. \n\nThey will still exist in a hidden data volume, which can be recovered using standard docker techniques, but that is much more complex than just starting with a named volume as described above.",
|
|
"x": 350,
|
|
"y": 80,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "a918100ba6063585",
|
|
"type": "inject",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "",
|
|
"props": [
|
|
{
|
|
"p": "payload"
|
|
},
|
|
{
|
|
"p": "topic",
|
|
"vt": "str"
|
|
}
|
|
],
|
|
"repeat": "",
|
|
"crontab": "",
|
|
"once": false,
|
|
"onceDelay": 0.1,
|
|
"topic": "",
|
|
"payload": "",
|
|
"payloadType": "date",
|
|
"x": 120,
|
|
"y": 340,
|
|
"wires": [
|
|
[
|
|
"fb9ff82281dc83e0"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "fb9ff82281dc83e0",
|
|
"type": "exec",
|
|
"z": "f6f2187d.f17ca8",
|
|
"command": "python3 /data/scripts/ad_lookup.py",
|
|
"addpay": "",
|
|
"append": "",
|
|
"useSpawn": "false",
|
|
"timer": "",
|
|
"winHide": false,
|
|
"oldrc": false,
|
|
"name": "",
|
|
"x": 380,
|
|
"y": 340,
|
|
"wires": [
|
|
[
|
|
"dd3574493b97a5f4"
|
|
],
|
|
[
|
|
"444c29de997bd1b6"
|
|
],
|
|
[]
|
|
]
|
|
},
|
|
{
|
|
"id": "426e46f53063d368",
|
|
"type": "debug",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "debug 3",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "false",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 1020,
|
|
"y": 260,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "444c29de997bd1b6",
|
|
"type": "debug",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "debug 4",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "false",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 700,
|
|
"y": 420,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "dd3574493b97a5f4",
|
|
"type": "file",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "Write the new JSON object to cache",
|
|
"filename": "\"/data/results/\" & $env(\"JSON_CACHE\")",
|
|
"filenameType": "jsonata",
|
|
"appendNewline": true,
|
|
"createDir": false,
|
|
"overwriteFile": "true",
|
|
"encoding": "utf8",
|
|
"x": 750,
|
|
"y": 320,
|
|
"wires": [
|
|
[
|
|
"426e46f53063d368",
|
|
"dbc196df1931ee06"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "eb6d443e97fedbee",
|
|
"type": "link in",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "link in Process Groupinformation",
|
|
"links": [],
|
|
"x": 115,
|
|
"y": 260,
|
|
"wires": [
|
|
[
|
|
"fb9ff82281dc83e0"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "dbc196df1931ee06",
|
|
"type": "link out",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "link out Process Groupinformation",
|
|
"mode": "return",
|
|
"links": [],
|
|
"x": 1015,
|
|
"y": 400,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "845b63e247001e57",
|
|
"type": "status",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "workflow status",
|
|
"scope": [
|
|
"fb9ff82281dc83e0",
|
|
"dd3574493b97a5f4"
|
|
],
|
|
"x": 640,
|
|
"y": 220,
|
|
"wires": [
|
|
[
|
|
"1e2a3c41555c2d9b"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "1e2a3c41555c2d9b",
|
|
"type": "debug",
|
|
"z": "f6f2187d.f17ca8",
|
|
"name": "debug workflow",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "true",
|
|
"targetType": "full",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 880,
|
|
"y": 220,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "44c678d93545a6f5",
|
|
"type": "inject",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "",
|
|
"props": [
|
|
{
|
|
"p": "payload"
|
|
},
|
|
{
|
|
"p": "topic",
|
|
"vt": "str"
|
|
}
|
|
],
|
|
"repeat": "",
|
|
"crontab": "",
|
|
"once": false,
|
|
"onceDelay": 0.1,
|
|
"topic": "",
|
|
"payload": "",
|
|
"payloadType": "date",
|
|
"x": 140,
|
|
"y": 280,
|
|
"wires": [
|
|
[
|
|
"b994acfe29648a35"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "b994acfe29648a35",
|
|
"type": "link call",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "",
|
|
"links": [
|
|
"913d320bfbe13635"
|
|
],
|
|
"linkType": "static",
|
|
"timeout": "300",
|
|
"x": 380,
|
|
"y": 280,
|
|
"wires": [
|
|
[
|
|
"5dab821150d36d52"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "5dab821150d36d52",
|
|
"type": "link call",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "",
|
|
"links": [
|
|
"eb6d443e97fedbee"
|
|
],
|
|
"linkType": "static",
|
|
"timeout": "300",
|
|
"x": 700,
|
|
"y": 280,
|
|
"wires": [
|
|
[
|
|
"4d7bd4a079e1c50d"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "4d7bd4a079e1c50d",
|
|
"type": "debug",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "End of workflow",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "payload",
|
|
"targetType": "msg",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 960,
|
|
"y": 280,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "cd1bcf72369d7e25",
|
|
"type": "status",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "workflow status",
|
|
"scope": [
|
|
"b994acfe29648a35",
|
|
"5dab821150d36d52"
|
|
],
|
|
"x": 400,
|
|
"y": 360,
|
|
"wires": [
|
|
[
|
|
"a613cbef5aae458b"
|
|
]
|
|
]
|
|
},
|
|
{
|
|
"id": "a613cbef5aae458b",
|
|
"type": "debug",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "debug workflow",
|
|
"active": true,
|
|
"tosidebar": true,
|
|
"console": false,
|
|
"tostatus": false,
|
|
"complete": "true",
|
|
"targetType": "full",
|
|
"statusVal": "",
|
|
"statusType": "auto",
|
|
"x": 620,
|
|
"y": 360,
|
|
"wires": []
|
|
},
|
|
{
|
|
"id": "2f498ce2ead2e74c",
|
|
"type": "inject",
|
|
"z": "1593c8c9a04da215",
|
|
"name": "Periodischer Trigger",
|
|
"props": [
|
|
{
|
|
"p": "payload"
|
|
},
|
|
{
|
|
"p": "topic",
|
|
"vt": "str"
|
|
}
|
|
],
|
|
"repeat": "300",
|
|
"crontab": "",
|
|
"once": false,
|
|
"onceDelay": 0.1,
|
|
"topic": "",
|
|
"payload": "",
|
|
"payloadType": "date",
|
|
"x": 180,
|
|
"y": 180,
|
|
"wires": [
|
|
[]
|
|
]
|
|
}
|
|
] |