From 188c9abb55a0dbea4d70c8c500e86d895eecca23 Mon Sep 17 00:00:00 2001 From: Simon Ebner Date: Tue, 16 Mar 2021 16:03:37 +0100 Subject: [PATCH] started bufferutils command --- Config Tests.ipynb | 257 ++++++++++++++++++++++++++++++++++------ bufferutils.py | 220 ++++++++++++++++++++++++++++++++++ sources/SARES20.sources | 7 -- sources/image.sources | 6 +- 4 files changed, 443 insertions(+), 47 deletions(-) create mode 100644 bufferutils.py diff --git a/Config Tests.ipynb b/Config Tests.ipynb index 6c82e31..dc4a3eb 100644 --- a/Config Tests.ipynb +++ b/Config Tests.ipynb @@ -84,14 +84,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ "def read_files(files_dir, file_type):\n", " sources = []\n", " for file in files_dir.iterdir():\n", - "# print(file)\n", + " print(file)\n", " config = load_file(file)\n", " sources.extend(config[file_type])\n", "# print(config)\n", @@ -100,16 +100,45 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 89, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sources/gunlaser.sources\n", + "sources/SARES20.sources\n", + "sources/bam.sources\n", + "sources/mps.sources\n", + "sources/pumplaser.sources\n", + "sources/timing.sources\n", + "sources/ict.sources\n", + "sources/SATES20.sources\n", + "sources/blm.sources\n", + "sources/rf_modulator.sources\n", + "sources/ECMC.sources\n", + "sources/rf_interlock.sources\n", + "sources/pho_keysight.sources\n", + "sources/bcm.sources\n", + "sources/image.sources\n", + "sources/SARES10.sources\n", + "sources/spectrometer.sources\n", + "sources/dwsc.sources\n", + "sources/llrf.sources\n", + "sources/pho_digitizer.sources\n", + "sources/SATFE10.sources\n", + "sources/bpm.sources\n" + ] + } + ], "source": [ "sources = read_files(Path(\"sources/\"), \"sources\")" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 63, "metadata": {}, "outputs": [ { @@ -130,13 +159,11 @@ " {'stream': 'tcp://SARES20-CVME-01:9999'},\n", " {'stream': 'tcp://sf-daqsync-06.psi.ch:9015'},\n", " {'stream': 'tcp://sf-daqsync-06.psi.ch:9016'},\n", - " {'stream': 'tcp://sf-daqsync-06.psi.ch:9017'},\n", " {'stream': 'tcp://sf-daqsync-06.psi.ch:9018'},\n", " {'stream': 'tcp://SINLH01-CVME-DBAM12:9999'},\n", " {'stream': 'tcp://S10BC01-CVME-DBAM32:9999'},\n", " {'stream': 'tcp://SARCL01-CVME-DBAM42:9999'},\n", " {'stream': 'tcp://SARUN20-CVME-DBAM52:9999'},\n", - " {'stream': 'tcp://sf-daqsync-02:8889'},\n", " {'stream': 'tcp://SIN-CVME-MPS0151:9999'},\n", " {'stream': 'tcp://SLAAR-CVME-LAS6861:9999'},\n", " {'stream': 'tcp://SLAAR-CVME-LAS6891:9999'},\n", @@ -155,8 +182,6 @@ " {'stream': 'tcp://sates20-cvme-maloja1:9999'},\n", " {'stream': 'tcp://sates21-cpcl-ges1:9999'},\n", " {'stream': 'tcp://sates21-cpcl-ges1:9989'},\n", - " {'stream': 'tcp://sf-daqsync-04.psi.ch:9001'},\n", - " {'stream': 'tcp://sf-daqsync-04.psi.ch:9011'},\n", " {'stream': 'tcp://SINEG01-CVME-DBLM009:9999'},\n", " {'stream': 'tcp://SINLH02-CVME-DBLM018:9999'},\n", " {'stream': 'tcp://SINDI02-CVME-DBLM084:9999'},\n", @@ -485,15 +510,38 @@ " {'stream': 'tcp://daqsf-sioc-cs-c2.psi.ch:9070',\n", " 'split': 4,\n", " 'backend': 'sf-imagebuffer'},\n", + " {'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARFE10-PSSS059']},\n", + " {'stream': 'tcp://sf-daqsync-02:8889', 'labels': ['SARFE10-PSSS059']},\n", + " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SATES21-CAMS154-M1']},\n", + " {'stream': 'tcp://sf-daqsync-04.psi.ch:9001',\n", + " 'labels': ['SATES21-CAMS154-M1']},\n", + " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SATES24-CAMS161-M1']},\n", + " {'stream': 'tcp://sf-daqsync-04.psi.ch:9011',\n", + " 'labels': ['SATES24-CAMS161-M1']},\n", " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9000',\n", " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES11-SPEC125-M1']},\n", + " {'stream': 'tcp://sf-daqsync-05:9001', 'labels': ['SARES11-SPEC125-M1']},\n", " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9010',\n", " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES11-SPEC125-M2']},\n", + " {'stream': 'tcp://sf-daqsync-05:9011', 'labels': ['SARES11-SPEC125-M2']},\n", " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9002',\n", " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES12-CAMS128-M1']},\n", + " {'stream': 'tcp://sf-daqsync-05:9003', 'labels': ['SARES12-CAMS128-M1']},\n", " {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n", " 'split': 4,\n", " 'backend': 'sf-imagebuffer',\n", @@ -509,18 +557,17 @@ " 'backend': 'sf-imagebuffer',\n", " 'labels': ['SARES20-CAMS142-M5']},\n", " {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']},\n", - " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n", - " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", - " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n", - " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", - " {'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n", - " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", " {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9005',\n", " 'split': 4,\n", - " 'backend': 'sf-imagebuffer'},\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES20-PROF142-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9017',\n", + " 'labels': ['SARES20-PROF142-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9015', 'labels': ['SAROP21-PPRM138']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9016',\n", + " 'labels': ['SARES20-PROF141-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9018',\n", + " 'labels': ['SARES20-PROF146-M1']},\n", " {'stream': 'tcp://SARES11-CPCL-GES1:9999'},\n", " {'stream': 'tcp://SARES11-CPCL-GES1:9990'},\n", " {'stream': 'tcp://SARES11-CPCL-GES1:9980'},\n", @@ -532,9 +579,6 @@ " {'stream': 'tcp://SARES12-CVME-FLEX1:9999'},\n", " {'stream': 'tcp://SARES11-CPPM-MOT6912:9999'},\n", " {'stream': 'tcp://SARES12-CPPM-MOT6952:9999'},\n", - " {'stream': 'tcp://sf-daqsync-05:9001'},\n", - " {'stream': 'tcp://sf-daqsync-05:9011'},\n", - " {'stream': 'tcp://sf-daqsync-05:9003'},\n", " {'stream': 'tcp://sf-sioc-cs-61:9030'},\n", " {'stream': 'tcp://SIN-CPPM-MOT0841:9999'},\n", " {'stream': 'tcp://S10-CPPM-MOT0991:9999'},\n", @@ -662,7 +706,7 @@ " {'stream': 'tcp://SFTEST-CVME-DBPM1:9000'}]}" ] }, - "execution_count": 50, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -11869,7 +11913,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 65, "metadata": {}, "outputs": [], "source": [ @@ -11879,13 +11923,45 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 66, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n", + "[{'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARFE10-PSSS059']},\n", + " {'stream': 'tcp://sf-daqsync-02:8889', 'labels': ['SARFE10-PSSS059']},\n", + " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SATES21-CAMS154-M1']},\n", + " {'stream': 'tcp://sf-daqsync-04.psi.ch:9001',\n", + " 'labels': ['SATES21-CAMS154-M1']},\n", + " {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SATES24-CAMS161-M1']},\n", + " {'stream': 'tcp://sf-daqsync-04.psi.ch:9011',\n", + " 'labels': ['SATES24-CAMS161-M1']},\n", + " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9000',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES11-SPEC125-M1']},\n", + " {'stream': 'tcp://sf-daqsync-05:9001', 'labels': ['SARES11-SPEC125-M1']},\n", + " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9010',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES11-SPEC125-M2']},\n", + " {'stream': 'tcp://sf-daqsync-05:9011', 'labels': ['SARES11-SPEC125-M2']},\n", + " {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9002',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES12-CAMS128-M1']},\n", + " {'stream': 'tcp://sf-daqsync-05:9003', 'labels': ['SARES12-CAMS128-M1']},\n", + " {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n", " 'split': 4,\n", " 'backend': 'sf-imagebuffer',\n", " 'labels': ['SARES20-CAMS142-M1']},\n", @@ -11899,10 +11975,21 @@ " 'split': 4,\n", " 'backend': 'sf-imagebuffer',\n", " 'labels': ['SARES20-CAMS142-M5']},\n", - " {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']}]" + " {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']},\n", + " {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9005',\n", + " 'split': 4,\n", + " 'backend': 'sf-imagebuffer',\n", + " 'labels': ['SARES20-PROF142-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9017',\n", + " 'labels': ['SARES20-PROF142-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9015', 'labels': ['SAROP21-PPRM138']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9016',\n", + " 'labels': ['SARES20-PROF141-M1']},\n", + " {'stream': 'tcp://sf-daqsync-06.psi.ch:9018',\n", + " 'labels': ['SARES20-PROF146-M1']}]" ] }, - "execution_count": 52, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } @@ -11913,7 +12000,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 67, "metadata": {}, "outputs": [], "source": [ @@ -11923,7 +12010,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -11936,7 +12023,7 @@ " {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']}]" ] }, - "execution_count": 54, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -11947,7 +12034,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -11957,16 +12044,28 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 70, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'SARES20-CAMS142-M1', 'SARES20-CAMS142-M4', 'SARES20-CAMS142-M5'}" + "{'SARES11-SPEC125-M1',\n", + " 'SARES11-SPEC125-M2',\n", + " 'SARES12-CAMS128-M1',\n", + " 'SARES20-CAMS142-M1',\n", + " 'SARES20-CAMS142-M4',\n", + " 'SARES20-CAMS142-M5',\n", + " 'SARES20-PROF141-M1',\n", + " 'SARES20-PROF142-M1',\n", + " 'SARES20-PROF146-M1',\n", + " 'SARFE10-PSSS059',\n", + " 'SAROP21-PPRM138',\n", + " 'SATES21-CAMS154-M1',\n", + " 'SATES24-CAMS161-M1'}" ] }, - "execution_count": 56, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -11975,6 +12074,88 @@ "labels" ] }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "# remove labels\n", + "# sources2 = [x if \"labels\" not in x else x.pop('labels', None) for x in sources[\"sources\"]]\n", + "def remove_labels(sources):\n", + " return {\"sources\": [x if \"labels\" not in x else x.pop('labels', None) for x in sources[\"sources\"]]}" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "363" + ] + }, + "execution_count": 82, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(remove_labels(sources)[\"sources\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "363" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(sources[\"sources\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": {}, + "outputs": [], + "source": [ + "def remove_labeled_source(sources, label):\n", + " return {\"sources\": [x for x in sources[\"sources\"] if \"labels\" not in x or (label not in x['labels'])]}" + ] + }, + { + "cell_type": "code", + "execution_count": 95, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "361" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(remove_labeled_source(sources, \"SARES11-SPEC125-M1\")[\"sources\"])" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/bufferutils.py b/bufferutils.py new file mode 100644 index 0000000..42dfa85 --- /dev/null +++ b/bufferutils.py @@ -0,0 +1,220 @@ +import argparse +import re +import json +import logging +import requests +from io import StringIO +from pathlib import Path + +base_directory = Path(".") +upload_url = "https://dispatcher-api.psi.ch/sf/configuration/upload" +# upload_url = "http://localhost:1234" + + +def _remove_comments(text): + """ remove c-style comments. + text: blob of text with comments (can include newlines) + returns: text with comments removed + + # Stolen from https://www.saltycrane.com/blog/2007/11/remove-c-comments-python/ + """ + + pattern = r""" + ## --------- COMMENT --------- + /\* ## Start of /* ... */ comment + [^*]*\*+ ## Non-* followed by 1-or-more *'s + ( ## + [^/*][^*]*\*+ ## + )* ## 0-or-more things which don't start with / + ## but do end with '*' + / ## End of /* ... */ comment + | ## -OR- various things which aren't comments: + ( ## + ## ------ " ... " STRING ------ + " ## Start of " ... " string + ( ## + \\. ## Escaped char + | ## -OR- + [^"\\] ## Non "\ characters + )* ## + " ## End of " ... " string + | ## -OR- + ## + ## ------ ' ... ' STRING ------ + ' ## Start of ' ... ' string + ( ## + \\. ## Escaped char + | ## -OR- + [^'\\] ## Non '\ characters + )* ## + ' ## End of ' ... ' string + | ## -OR- + ## + ## ------ ANYTHING ELSE ------- + . ## Anything other char + [^/"'\\]* ## Chars which doesn't start a comment, string + ) ## or escape + """ + regex = re.compile(pattern, re.VERBOSE | re.MULTILINE | re.DOTALL) + non_comments = [m.group(2) for m in regex.finditer(text) if m.group(2)] + + return "".join(non_comments) + + +def _load_file(file_path): + with open(file_path) as file_h: + text = file_h.read() + text = _remove_comments(text) + config = json.loads(text) + return config + + +def remove_labels(sources): + """ + Remove labels from the source config + :param sources: + :return: source config without labels + """ + new_sources = [] + for source in sources["sources"]: + source.pop('labels', None) + new_sources.append(source) + return {"sources": new_sources} + + +def remove_labeled_source(sources, label): + """ + Remove (a) source(s) with a specific label + :param sources: + :param label: + :return: source config without the source(s) with the given label + """ + return {"sources": [x for x in sources["sources"] if "labels" not in x or (label not in x['labels'])]} + + +def get_labels(sources): + """ + Retrieve all used labels in the source configurations + :param sources: source config + :return: list of available labels + """ + labels = set([item for x in sources["sources"] if "labels" in x for item in x["labels"]]) + return labels + + +def get_labeled_sources(sources, label): + """ + Get source(s) with the given label + :param sources: + :param label: + :return: list of source config that contains label + """ + return [x for x in sources["sources"] if "labels" in x and 'SARES20-CAMS142-M5' in x['labels']] + + +def read_files(files_dir, file_type): + """ + Read sources or policies files + :param files_dir: + :param file_type: "sources" or "policies" + :return: + """ + sources = [] + for file in files_dir.iterdir(): + logging.info(f"Read file: {file}") + config = _load_file(file) + sources.extend(config[file_type]) + return {file_type: sources} + + +def upload_sources_and_policies(sources, policies): + """ + Upload sources and policies definition to the data/imagebuffer + :param sources: sources to upload + :param policies: policies to upload + :return: + """ + + upload_files = [("files", ("all.sources", StringIO(json.dumps(sources)))), + ("files", ("all.policies", StringIO(json.dumps(policies))))] + + test_response = requests.post(upload_url, files=upload_files) + + if test_response.ok: + print("Upload completed successfully!") + # print(test_response.text) + else: + print("Something went wrong!") + + +def main(): + parser = argparse.ArgumentParser(description="Utility commands to work with the databuffer", + formatter_class=argparse.RawTextHelpFormatter, + epilog=f"--------\n\n" + f"Examples:\n" + "bufferutils upload\n" + "bufferutils restart --label