forked from archiver_config/sf_databuffer
started bufferutils command
This commit is contained in:
@@ -84,14 +84,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 60,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def read_files(files_dir, file_type):\n",
|
||||
" sources = []\n",
|
||||
" for file in files_dir.iterdir():\n",
|
||||
"# print(file)\n",
|
||||
" print(file)\n",
|
||||
" config = load_file(file)\n",
|
||||
" sources.extend(config[file_type])\n",
|
||||
"# print(config)\n",
|
||||
@@ -100,16 +100,45 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 49,
|
||||
"execution_count": 89,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"sources/gunlaser.sources\n",
|
||||
"sources/SARES20.sources\n",
|
||||
"sources/bam.sources\n",
|
||||
"sources/mps.sources\n",
|
||||
"sources/pumplaser.sources\n",
|
||||
"sources/timing.sources\n",
|
||||
"sources/ict.sources\n",
|
||||
"sources/SATES20.sources\n",
|
||||
"sources/blm.sources\n",
|
||||
"sources/rf_modulator.sources\n",
|
||||
"sources/ECMC.sources\n",
|
||||
"sources/rf_interlock.sources\n",
|
||||
"sources/pho_keysight.sources\n",
|
||||
"sources/bcm.sources\n",
|
||||
"sources/image.sources\n",
|
||||
"sources/SARES10.sources\n",
|
||||
"sources/spectrometer.sources\n",
|
||||
"sources/dwsc.sources\n",
|
||||
"sources/llrf.sources\n",
|
||||
"sources/pho_digitizer.sources\n",
|
||||
"sources/SATFE10.sources\n",
|
||||
"sources/bpm.sources\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"sources = read_files(Path(\"sources/\"), \"sources\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 50,
|
||||
"execution_count": 63,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -130,13 +159,11 @@
|
||||
" {'stream': 'tcp://SARES20-CVME-01:9999'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9015'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9016'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9017'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9018'},\n",
|
||||
" {'stream': 'tcp://SINLH01-CVME-DBAM12:9999'},\n",
|
||||
" {'stream': 'tcp://S10BC01-CVME-DBAM32:9999'},\n",
|
||||
" {'stream': 'tcp://SARCL01-CVME-DBAM42:9999'},\n",
|
||||
" {'stream': 'tcp://SARUN20-CVME-DBAM52:9999'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-02:8889'},\n",
|
||||
" {'stream': 'tcp://SIN-CVME-MPS0151:9999'},\n",
|
||||
" {'stream': 'tcp://SLAAR-CVME-LAS6861:9999'},\n",
|
||||
" {'stream': 'tcp://SLAAR-CVME-LAS6891:9999'},\n",
|
||||
@@ -155,8 +182,6 @@
|
||||
" {'stream': 'tcp://sates20-cvme-maloja1:9999'},\n",
|
||||
" {'stream': 'tcp://sates21-cpcl-ges1:9999'},\n",
|
||||
" {'stream': 'tcp://sates21-cpcl-ges1:9989'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9001'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9011'},\n",
|
||||
" {'stream': 'tcp://SINEG01-CVME-DBLM009:9999'},\n",
|
||||
" {'stream': 'tcp://SINLH02-CVME-DBLM018:9999'},\n",
|
||||
" {'stream': 'tcp://SINDI02-CVME-DBLM084:9999'},\n",
|
||||
@@ -485,15 +510,38 @@
|
||||
" {'stream': 'tcp://daqsf-sioc-cs-c2.psi.ch:9070',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARFE10-PSSS059']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-02:8889', 'labels': ['SARFE10-PSSS059']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SATES21-CAMS154-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9001',\n",
|
||||
" 'labels': ['SATES21-CAMS154-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SATES24-CAMS161-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9011',\n",
|
||||
" 'labels': ['SATES24-CAMS161-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9000',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES11-SPEC125-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9001', 'labels': ['SARES11-SPEC125-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9010',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES11-SPEC125-M2']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9011', 'labels': ['SARES11-SPEC125-M2']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9002',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES12-CAMS128-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9003', 'labels': ['SARES12-CAMS128-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
@@ -509,18 +557,17 @@
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES20-CAMS142-M5']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9005',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer'},\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES20-PROF142-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9017',\n",
|
||||
" 'labels': ['SARES20-PROF142-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9015', 'labels': ['SAROP21-PPRM138']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9016',\n",
|
||||
" 'labels': ['SARES20-PROF141-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9018',\n",
|
||||
" 'labels': ['SARES20-PROF146-M1']},\n",
|
||||
" {'stream': 'tcp://SARES11-CPCL-GES1:9999'},\n",
|
||||
" {'stream': 'tcp://SARES11-CPCL-GES1:9990'},\n",
|
||||
" {'stream': 'tcp://SARES11-CPCL-GES1:9980'},\n",
|
||||
@@ -532,9 +579,6 @@
|
||||
" {'stream': 'tcp://SARES12-CVME-FLEX1:9999'},\n",
|
||||
" {'stream': 'tcp://SARES11-CPPM-MOT6912:9999'},\n",
|
||||
" {'stream': 'tcp://SARES12-CPPM-MOT6952:9999'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9001'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9011'},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9003'},\n",
|
||||
" {'stream': 'tcp://sf-sioc-cs-61:9030'},\n",
|
||||
" {'stream': 'tcp://SIN-CPPM-MOT0841:9999'},\n",
|
||||
" {'stream': 'tcp://S10-CPPM-MOT0991:9999'},\n",
|
||||
@@ -662,7 +706,7 @@
|
||||
" {'stream': 'tcp://SFTEST-CVME-DBPM1:9000'}]}"
|
||||
]
|
||||
},
|
||||
"execution_count": 50,
|
||||
"execution_count": 63,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -11869,7 +11913,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 51,
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -11879,13 +11923,45 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 52,
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[{'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n",
|
||||
"[{'stream': 'tcp://daqsf-daqsync-02.psi.ch:8890',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARFE10-PSSS059']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-02:8889', 'labels': ['SARFE10-PSSS059']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9000',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SATES21-CAMS154-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9001',\n",
|
||||
" 'labels': ['SATES21-CAMS154-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-04.psi.ch:9010',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SATES24-CAMS161-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-04.psi.ch:9011',\n",
|
||||
" 'labels': ['SATES24-CAMS161-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9000',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES11-SPEC125-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9001', 'labels': ['SARES11-SPEC125-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9010',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES11-SPEC125-M2']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9011', 'labels': ['SARES11-SPEC125-M2']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-05.psi.ch:9002',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES12-CAMS128-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-05:9003', 'labels': ['SARES12-CAMS128-M1']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9002',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES20-CAMS142-M1']},\n",
|
||||
@@ -11899,10 +11975,21 @@
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES20-CAMS142-M5']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']}]"
|
||||
" {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']},\n",
|
||||
" {'stream': 'tcp://daqsf-daqsync-06.psi.ch:9005',\n",
|
||||
" 'split': 4,\n",
|
||||
" 'backend': 'sf-imagebuffer',\n",
|
||||
" 'labels': ['SARES20-PROF142-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9017',\n",
|
||||
" 'labels': ['SARES20-PROF142-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9015', 'labels': ['SAROP21-PPRM138']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9016',\n",
|
||||
" 'labels': ['SARES20-PROF141-M1']},\n",
|
||||
" {'stream': 'tcp://sf-daqsync-06.psi.ch:9018',\n",
|
||||
" 'labels': ['SARES20-PROF146-M1']}]"
|
||||
]
|
||||
},
|
||||
"execution_count": 52,
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -11913,7 +12000,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 53,
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -11923,7 +12010,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 54,
|
||||
"execution_count": 68,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -11936,7 +12023,7 @@
|
||||
" {'stream': 'tcp://sf-daqsync-06:9011', 'labels': ['SARES20-CAMS142-M5']}]"
|
||||
]
|
||||
},
|
||||
"execution_count": 54,
|
||||
"execution_count": 68,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -11947,7 +12034,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 55,
|
||||
"execution_count": 69,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -11957,16 +12044,28 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 56,
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'SARES20-CAMS142-M1', 'SARES20-CAMS142-M4', 'SARES20-CAMS142-M5'}"
|
||||
"{'SARES11-SPEC125-M1',\n",
|
||||
" 'SARES11-SPEC125-M2',\n",
|
||||
" 'SARES12-CAMS128-M1',\n",
|
||||
" 'SARES20-CAMS142-M1',\n",
|
||||
" 'SARES20-CAMS142-M4',\n",
|
||||
" 'SARES20-CAMS142-M5',\n",
|
||||
" 'SARES20-PROF141-M1',\n",
|
||||
" 'SARES20-PROF142-M1',\n",
|
||||
" 'SARES20-PROF146-M1',\n",
|
||||
" 'SARFE10-PSSS059',\n",
|
||||
" 'SAROP21-PPRM138',\n",
|
||||
" 'SATES21-CAMS154-M1',\n",
|
||||
" 'SATES24-CAMS161-M1'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 56,
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -11975,6 +12074,88 @@
|
||||
"labels"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 80,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# remove labels\n",
|
||||
"# sources2 = [x if \"labels\" not in x else x.pop('labels', None) for x in sources[\"sources\"]]\n",
|
||||
"def remove_labels(sources):\n",
|
||||
" return {\"sources\": [x if \"labels\" not in x else x.pop('labels', None) for x in sources[\"sources\"]]}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 82,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"363"
|
||||
]
|
||||
},
|
||||
"execution_count": 82,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(remove_labels(sources)[\"sources\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 79,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"363"
|
||||
]
|
||||
},
|
||||
"execution_count": 79,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(sources[\"sources\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 93,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def remove_labeled_source(sources, label):\n",
|
||||
" return {\"sources\": [x for x in sources[\"sources\"] if \"labels\" not in x or (label not in x['labels'])]}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 95,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"361"
|
||||
]
|
||||
},
|
||||
"execution_count": 95,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(remove_labeled_source(sources, \"SARES11-SPEC125-M1\")[\"sources\"])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
|
||||
220
bufferutils.py
Normal file
220
bufferutils.py
Normal file
@@ -0,0 +1,220 @@
|
||||
import argparse
|
||||
import re
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
|
||||
base_directory = Path(".")
|
||||
upload_url = "https://dispatcher-api.psi.ch/sf/configuration/upload"
|
||||
# upload_url = "http://localhost:1234"
|
||||
|
||||
|
||||
def _remove_comments(text):
|
||||
""" remove c-style comments.
|
||||
text: blob of text with comments (can include newlines)
|
||||
returns: text with comments removed
|
||||
|
||||
# Stolen from https://www.saltycrane.com/blog/2007/11/remove-c-comments-python/
|
||||
"""
|
||||
|
||||
pattern = r"""
|
||||
## --------- COMMENT ---------
|
||||
/\* ## Start of /* ... */ comment
|
||||
[^*]*\*+ ## Non-* followed by 1-or-more *'s
|
||||
( ##
|
||||
[^/*][^*]*\*+ ##
|
||||
)* ## 0-or-more things which don't start with /
|
||||
## but do end with '*'
|
||||
/ ## End of /* ... */ comment
|
||||
| ## -OR- various things which aren't comments:
|
||||
( ##
|
||||
## ------ " ... " STRING ------
|
||||
" ## Start of " ... " string
|
||||
( ##
|
||||
\\. ## Escaped char
|
||||
| ## -OR-
|
||||
[^"\\] ## Non "\ characters
|
||||
)* ##
|
||||
" ## End of " ... " string
|
||||
| ## -OR-
|
||||
##
|
||||
## ------ ' ... ' STRING ------
|
||||
' ## Start of ' ... ' string
|
||||
( ##
|
||||
\\. ## Escaped char
|
||||
| ## -OR-
|
||||
[^'\\] ## Non '\ characters
|
||||
)* ##
|
||||
' ## End of ' ... ' string
|
||||
| ## -OR-
|
||||
##
|
||||
## ------ ANYTHING ELSE -------
|
||||
. ## Anything other char
|
||||
[^/"'\\]* ## Chars which doesn't start a comment, string
|
||||
) ## or escape
|
||||
"""
|
||||
regex = re.compile(pattern, re.VERBOSE | re.MULTILINE | re.DOTALL)
|
||||
non_comments = [m.group(2) for m in regex.finditer(text) if m.group(2)]
|
||||
|
||||
return "".join(non_comments)
|
||||
|
||||
|
||||
def _load_file(file_path):
|
||||
with open(file_path) as file_h:
|
||||
text = file_h.read()
|
||||
text = _remove_comments(text)
|
||||
config = json.loads(text)
|
||||
return config
|
||||
|
||||
|
||||
def remove_labels(sources):
|
||||
"""
|
||||
Remove labels from the source config
|
||||
:param sources:
|
||||
:return: source config without labels
|
||||
"""
|
||||
new_sources = []
|
||||
for source in sources["sources"]:
|
||||
source.pop('labels', None)
|
||||
new_sources.append(source)
|
||||
return {"sources": new_sources}
|
||||
|
||||
|
||||
def remove_labeled_source(sources, label):
|
||||
"""
|
||||
Remove (a) source(s) with a specific label
|
||||
:param sources:
|
||||
:param label:
|
||||
:return: source config without the source(s) with the given label
|
||||
"""
|
||||
return {"sources": [x for x in sources["sources"] if "labels" not in x or (label not in x['labels'])]}
|
||||
|
||||
|
||||
def get_labels(sources):
|
||||
"""
|
||||
Retrieve all used labels in the source configurations
|
||||
:param sources: source config
|
||||
:return: list of available labels
|
||||
"""
|
||||
labels = set([item for x in sources["sources"] if "labels" in x for item in x["labels"]])
|
||||
return labels
|
||||
|
||||
|
||||
def get_labeled_sources(sources, label):
|
||||
"""
|
||||
Get source(s) with the given label
|
||||
:param sources:
|
||||
:param label:
|
||||
:return: list of source config that contains label
|
||||
"""
|
||||
return [x for x in sources["sources"] if "labels" in x and 'SARES20-CAMS142-M5' in x['labels']]
|
||||
|
||||
|
||||
def read_files(files_dir, file_type):
|
||||
"""
|
||||
Read sources or policies files
|
||||
:param files_dir:
|
||||
:param file_type: "sources" or "policies"
|
||||
:return:
|
||||
"""
|
||||
sources = []
|
||||
for file in files_dir.iterdir():
|
||||
logging.info(f"Read file: {file}")
|
||||
config = _load_file(file)
|
||||
sources.extend(config[file_type])
|
||||
return {file_type: sources}
|
||||
|
||||
|
||||
def upload_sources_and_policies(sources, policies):
|
||||
"""
|
||||
Upload sources and policies definition to the data/imagebuffer
|
||||
:param sources: sources to upload
|
||||
:param policies: policies to upload
|
||||
:return:
|
||||
"""
|
||||
|
||||
upload_files = [("files", ("all.sources", StringIO(json.dumps(sources)))),
|
||||
("files", ("all.policies", StringIO(json.dumps(policies))))]
|
||||
|
||||
test_response = requests.post(upload_url, files=upload_files)
|
||||
|
||||
if test_response.ok:
|
||||
print("Upload completed successfully!")
|
||||
# print(test_response.text)
|
||||
else:
|
||||
print("Something went wrong!")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Utility commands to work with the databuffer",
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
epilog=f"--------\n\n"
|
||||
f"Examples:\n"
|
||||
"bufferutils upload\n"
|
||||
"bufferutils restart --label <label>\n"
|
||||
)
|
||||
subparsers = parser.add_subparsers(title='command',
|
||||
description='valid commands',
|
||||
dest='command',
|
||||
help='command to execute')
|
||||
|
||||
parser_upload = subparsers.add_parser('upload',
|
||||
help="upload configuration",
|
||||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
|
||||
parser_restart = subparsers.add_parser('restart',
|
||||
help="restart a source",
|
||||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
|
||||
parser_restart.add_argument('-l',
|
||||
'--label',
|
||||
default=None,
|
||||
help="label that identifies the source(s) to restart")
|
||||
|
||||
parser_list = subparsers.add_parser('list',
|
||||
help="list",
|
||||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser_list.add_argument('--label',
|
||||
action="store_true",
|
||||
help="list labels")
|
||||
|
||||
arguments = parser.parse_args()
|
||||
|
||||
#
|
||||
# UPLOAD
|
||||
if arguments.command == 'upload':
|
||||
print("upload")
|
||||
sources = read_files(base_directory / Path("sources"), "sources")
|
||||
policies = read_files(base_directory / Path("policies"), "policies")
|
||||
|
||||
# Just to make sure that the additional labels entry does not break the backend - remove it
|
||||
# sources = remove_labels(sources)
|
||||
|
||||
upload_sources_and_policies(sources, policies)
|
||||
|
||||
#
|
||||
# RESTART
|
||||
elif arguments.command == 'restart':
|
||||
print("restart")
|
||||
pass
|
||||
|
||||
#
|
||||
# LIST
|
||||
elif arguments.command == 'list':
|
||||
if arguments.label:
|
||||
sources = read_files(base_directory / Path("sources"), "sources")
|
||||
for label in get_labels(sources):
|
||||
print(label)
|
||||
else:
|
||||
print("Not yet implemented")
|
||||
parser_list.print_usage()
|
||||
|
||||
else:
|
||||
parser.print_usage()
|
||||
return -1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -8,12 +8,5 @@
|
||||
,{"stream": "tcp://SLAAR21-CPPM-MOT6991:9999"}
|
||||
,{"stream": "tcp://SLAAR21-CVME-LAS6991:9999"}
|
||||
,{"stream": "tcp://SARES20-CVME-01:9999"}
|
||||
|
||||
/* SAROP21-PPRM138_proc1 */
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9015"}
|
||||
/* SARES20-PROF141-M1_proc1 */
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9016"}
|
||||
/* SARES20-PROF146-M1_proc1 */
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9018"}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
/*
|
||||
Get the bsread stream address from an image name (e.g. for SARES20-PROF142-M3:FPICTURE)
|
||||
caget SARES20-PROF142-M3:BSREADCONFIG
|
||||
|
||||
the current camserver/pipline configuration can be found here: https://git.psi.ch/controls_highlevel_applications/cam_server_configuration/blob/master/configuration/pipeline_config/servers.json
|
||||
*/
|
||||
|
||||
{
|
||||
@@ -143,7 +145,7 @@
|
||||
|
||||
/* PSEN SARES11-SPEC125-M1 */
|
||||
,{"stream":"tcp://daqsf-daqsync-05.psi.ch:9000","split":4,"backend":"sf-imagebuffer", "labels": ["SARES11-SPEC125-M1"]}
|
||||
,{"stream": "tcp://sf-daqsync-05:9001", "labels": ["SARES11-SPEC125-M1"]
|
||||
,{"stream": "tcp://sf-daqsync-05:9001", "labels": ["SARES11-SPEC125-M1"]}
|
||||
|
||||
/* SARES11-SPEC125-M2 */
|
||||
,{"stream":"tcp://daqsf-daqsync-05.psi.ch:9010","split":4,"backend":"sf-imagebuffer", "labels": ["SARES11-SPEC125-M2"]}
|
||||
@@ -172,7 +174,7 @@
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9017", "labels": ["SARES20-PROF142-M1"]}
|
||||
|
||||
/* SAROP21-PPRM138 pipeline */
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9015", "labels": ["SAROP21-PPRM138"]
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9015", "labels": ["SAROP21-PPRM138"]}
|
||||
|
||||
/* SARES20-PROF141-M1 pipeline */
|
||||
, {"stream": "tcp://sf-daqsync-06.psi.ch:9016", "labels": ["SARES20-PROF141-M1"]}
|
||||
|
||||
Reference in New Issue
Block a user