sf-daq/sf_daq_buffer
0
0
Fork 0
mirror of https://github.com/paulscherrerinstitute/sf_daq_buffer.git synced 2026-05-03 15:24:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Repo refactoring

Browse Source
This commit is contained in:
babic_a
2020-08-12 15:37:41 +02:00
parent cb2a69dc17
commit 511a25e04e
6 changed files with 54 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
epics-writer/.gitignore
-5
View File
@@ -1,5 +0,0 @@
build
__pycache__
.idea
.pytest_cache
*.h5
epics-writer/api.http
-23
View File
@@ -1,23 +0,0 @@
# For a quick start check out our HTTP Requests collection (Tools|HTTP Client|Open HTTP Requests Collection) or
# paste cURL into the file and request will be converted to HTTP Request format.
#
# Following HTTP Request Live Templates are available:
# * 'gtrp' and 'gtr' create a GET request with or without query parameters;
# * 'ptr' and 'ptrp' create a POST request with a simple or parameter-like body;
# * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data);
# curl -XPUT -d '{"range":{"startPulseId": 11281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify
PUT http://localhost:10200/notify
Content-Type: application/json
{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}}
###
PUT http://localhost:10200/notify
Content-Type: application/json
{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}, "channels": ["ABC"]}
###
epics-writer/ca_default_channel_list
-179
View File
@@ -1,179 +0,0 @@
SARES11-VPIG124-360:PRESSURE
SARES11-VMFR124-350:PRESSURE
SARES11-VMFR125-600:PRESSURE
SARES11-VMCP125-610:PRESSURE
SARES11-VMFR125-400:PRESSURE
SARES11-VMCP125-410:PRESSURE
SARES11-VMCP125-510:PRESSURE
SARES11-EVSP-SAMPLE:SET_PRES
SARES11-EVSP-010:SAMPLE
SARES11-EVSP-010:INTERMEDIATE
SARES11-EVSP-010:DIFFERENT
SARES11-EVGG-A010:PRESSURE
SARES11-EVGG-B010:PRESSURE
SARES11-VMDI125-400:PRESSURE
SARES11-VMDI125-500:PRESSURE
SARES11-EVGG-C010:PRESSURE
SARES11-EVGV-C010:PLC_OPEN
SARES11-EVGV-D010:PLC_OPEN
SARES11-EVGV-E010:PLC_OPEN
SARES11-EVGV-E020:PLC_OPEN
SARES11-EVGV-F010:PLC_OPEN
SLAAR11-LMOT-M421:MOT.RBV
SLAAR11-LMOT-M422:MOT.RBV
SLAAR11-LMOT-M423:MOT.RBV
SLAAR11-LMOT-M431:MOT.RBV
SLAAR11-LMOT-M432:MOT.RBV
SLAAR11-LMOT-M441:MOT.RBV
SLAAR11-LMOT-M442:MOT.RBV
SLAAR11-LMOT-M444:MOT.RBV
SLAAR11-LENG-R452:VAL_GET
SLAAR11-LMOT-M451:MOTOR_1.RBV
SLAAR11-LDIO-LAS6891:SET_BO02
SLAAR11-LPSYS-ESA:TRANS_OPEN
SAROP11-PALMK118:RAVE
SAROP11-PALMK118:LIVE
SLAAR11-L-LAMCALC2:MOTOR_POS_SMAR_SC
SLAAR11-LENG-R452:VAL_GET
SARES11-XFOC125:LIN1:MOTRBV
SARES11-XICM125:ROX1:MOTRBV
SARES11-XICM125:TRX1:MOTRBV
SARES11-XICM125:ROY1:MOTRBV
SARES11-XMI125-C4P1:EXPOSURE
SARES11-XMI125:FOCUS.RBV
SARES11-XMI125:ZOOM.RBV
SARES11-XILL125:LIGHTLEVEL_SP
SARES11-XMI125:ROY1:MOTRBV
SARES11-XMI125:ROZ1:MOTRBV
SARES11-XOTA125:W_X.RBV
SARES11-XOTA125:W_Y.RBV
SARES11-XOTA125:W_Z.RBV
SARES11-XOTA125:W_RX.RBV
SARES11-XOTA125:W_RY.RBV
SARES11-XOTA125:W_RZ.RBV
SARES11-XOTA125:RAIL.RBV
SARES11-XOTA125:MOTOR_Y1.RBV
SARES11-XOTA125:MOTOR_Y2.RBV
SARES11-XOTA125:MOTOR_Y3.RBV
SARES11-XOTA125:MOTOR_X1.RBV
SARES11-XOTA125:MOTOR_Z1.RBV
SARES11-XOTA125:MOTOR_Z2.RBV
SARES11-XOTA125:MODE_SP
SARES11-XIZO125:TRX1:MOTRBV
SARES11-XIZO125:TRX2:MOTRBV
SARES11-XIZO125:TRY1:MOTRBV
SARES11-XIZO125:TRY2:MOTRBV
SARES10-XHPLC125:01:FLOW
SARES10-XHPLC125:01:PRESSURE
SARES10-XHPLC125:02:FLOW
SARES10-XHPLC125:02:PRESSURE
SARES11-XSAM125:MOTOR_X1.RBV
SARES11-XSAM125:MOTOR_Y1.RBV
SARES11-XSAM125:MOTOR_Z1.RBV
SARES11-XCRY125:CRY_1.RBV
SARES11-XCRY125:CRY_2.RBV
SARES11-XCRY125:TRZ11:MOTRBV
SARES11-XCRY125:ROY11:MOTRBV
SARES11-XCRY125:ROX11:MOTRBV
SARES11-XCRY125:TRZ21:MOTRBV
SARES11-XCRY125:ROY21:MOTRBV
SARES11-XCRY125:ROX21:MOTRBV
SARES11-XCRY125:TRZ12:MOTRBV
SARES11-XCRY125:ROY12:MOTRBV
SARES11-XCRY125:ROX12:MOTRBV
SARES11-XCRY125:TRZ22:MOTRBV
SARES11-XCRY125:ROY22:MOTRBV
SARES11-XCRY125:ROX22:MOTRBV
SARBD01-DBPM040:Q1
SAROP11-ARAMIS:ENERGY
SAROP11-ARAMIS:MODE
SARFE10-PBPG050:PHOTON-ENERGY-PER-PULSE-AVG
SGE-OP2E-ARAMIS:ESA_PH2E_Y1
SGE-OP2E-ARAMIS:ESA_PH2E_Y2
SGE-OP2E-ARAMIS:ESA_PH2E_X1
SGE-OP2E-ARAMIS:ESA_PH2E_X2
SARCL02-DCOL290:ENE
SARBD01-DBPM040:Q1
SARBD01-MBND100:P-READ
SARUN03-UIND030:K_SET
SARUN04-UIND030:K_SET
SARUN05-UIND030:K_SET
SARUN06-UIND030:K_SET
SARUN07-UIND030:K_SET
SARUN08-UIND030:K_SET
SARUN09-UIND030:K_SET
SARUN10-UIND030:K_SET
SARUN11-UIND030:K_SET
SARUN12-UIND030:K_SET
SARUN13-UIND030:K_SET
SARUN14-UIND030:K_SET
SIN-TIMAST-TMA:Beam-Appl-Freq-RB
SIN-TIMAST-TMA:Evt-20-Delay-RB
SIN-TIMAST-TMA:Evt-20-Freq-I
SIN-TIMAST-TMA:Evt-50-Delay-RB
SIN-TIMAST-TMA:Evt-50-Freq-I
SIN-TIMAST-TMA:Evt-29-Delay-RB
SIN-TIMAST-TMA:Evt-29-Freq-I
SIN-TIMAST-TMA:Evt-24-Delay-RB
SIN-TIMAST-TMA:Evt-24-Freq-I
SARFE10-OATT053:TRANS_SP
SARFE10-OATT053:TRANS_RB
SARFE10-OATT053:TRANS3EDHARM_RB
SARFE10-OATT053:ENERGY
SAROP11-OATT120:TRANS_SP
SAROP11-OATT120:TRANS_RB
SAROP11-OATT120:TRANS3EDHARM_RB
SAROP11-OATT120:ENERGY
SAROP11-OPPI110:MOTOR_X1.RBV
SAROP11-OPPI110:MOTOR_Y1.RBV
SAROP11-OPPI110:PRESET_SP
SGE-CPCW-71-EVR0:FrontUnivOut15-Ena-SP
SGE-CPCW-71-EVR0:FrontUnivOut15-Src-SP
SARFE10-OAPU044:MOTOR_X
SARFE10-OAPU044:MOTOR_Y
SARFE10-OAPU044:MOTOR_W
SARFE10-OAPU044:MOTOR_H
SARFE10-OAPU044:MOTOR_X.RBV
SARFE10-OAPU044:MOTOR_Y.RBV
SARFE10-OAPU044:MOTOR_W.RBV
SARFE10-OAPU044:MOTOR_H.RBV
SARFE10-OAPU044:MOTOR_AX1.RBV
SARFE10-OAPU044:MOTOR_AX2.RBV
SARFE10-OAPU044:MOTOR_AY1.RBV
SARFE10-OAPU044:MOTOR_AY2.RBV
SARFE10-OAPU044:MOTOR_BX1.RBV
SARFE10-OAPU044:MOTOR_BX2.RBV
SARFE10-OAPU044:MOTOR_BY1.RBV
SARFE10-OAPU044:MOTOR_BY2.RBV
SAROP11-OAPU104:MOTOR_X
SAROP11-OAPU104:MOTOR_Y
SAROP11-OAPU104:MOTOR_W
SAROP11-OAPU104:MOTOR_H
SAROP11-OAPU120:MOTOR_X
SAROP11-OAPU120:MOTOR_Y
SAROP11-OAPU120:MOTOR_W
SAROP11-OAPU120:MOTOR_H
SLAAR01-TLSY-EPL:JITTERMON
epics-writer/epics-writer/__init__.py → epics-writer/epics_writer/__init__.py
View File
epics-writer/epics-writer/start.py → epics-writer/epics_writer/start.py
+8 -3
View File
@@ -1,7 +1,8 @@
import json
from pika import BlockingConnection, ConnectionParameters, BasicProperties
from epics_writer import writer
DEFAULT_BROKER_URL = "127.0.0.1"
REQUEST_EXCHANGE = "request"
STATUS_EXCHANGE = "status"
@@ -12,7 +13,7 @@ OUTPUT_FILE_SUFFIX = ".PVCHANNELS.h5"
def update_status(channel, body, action, file, message=None):
status_header = {
"action": action,
"source": "epics-writer",
"source": "epics_writer",
"routing_key": QUEUE_NAME,
"file": file,
"message": message
@@ -41,7 +42,11 @@ def on_message(channel, method_frame, header_frame, body):
update_status(channel, body, "write_start", output_file)
# TODO: Call the actual writing.
writer.write_epics_pvs(output_file=output_file,
start_pulse_id=start_pulse_id,
stop_pulse_id=stop_pulse_id,
metadata=metadata,
epics_pvs=epics_pvs)
except Exception as e:
channel.basic_reject(delivery_tag=method_frame.delivery_tag,
epics-writer/epics-writer/epics_writer.py → epics-writer/epics_writer/writer.py
+46 -47
View File
@@ -1,13 +1,11 @@
import data_api
import data_api.client
# import data_api
# import data_api.client
import requests
import datetime
import time
import logging
logger = logging.getLogger("logger")
@@ -26,28 +24,29 @@ logger = logging.getLogger("logger")
# 'output_file': '/bla/test.h5'} # this is usually the full path
# }
def write_epics_pvs(
output_file, start_pulse_id, stop_pulse_id, metadata, epics_pvs):
def download_data(start_pulse, end_pulse, channels):
logger.info("Dump data to hdf5 ...")
logger.info("Retrieve data for channels: %s" % channels)
logger.info("Retrieve data for channels: %s" % epics_pvs)
logger.info("Retrieve pulse-id / data mapping for pulse ids")
start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse])
start_date, end_date = get_pulse_id_date_mapping([start_pulse_id, stop_pulse_id])
logger.info("Retrieving data for interval start: " + str(
start_date) + " end: " + str(end_date) + " . From " + new_base_url)
data = get_data(channels, start=start_date, end=end_date,
base_url=new_base_url)
# logger.info("Retrieving data for interval start: " + str(
# start_date) + " end: " + str(end_date) + " . From " + new_base_url)
# data = get_data(channels, start=start_date, end=end_date,
# base_url=new_base_url)
if len(data) < 1:
logger.error("No data retrieved")
open(new_filename + "_NO_DATA", 'a').close()
else:
if new_filename:
logger.info("Persist data to hdf5 file")
data_api.to_hdf5(data, new_filename, overwrite=True,
compression=None, shuffle=False)
# if len(data) < 1:
# logger.error("No data retrieved")
# open(new_filename + "_NO_DATA", 'a').close()
#
# else:
# if new_filename:
# logger.info("Persist data to hdf5 file")
# data_api.to_hdf5(data, new_filename, overwrite=True,
# compression=None, shuffle=False)
def get_data(channel_list, start=None, end=None, base_url=None):
@@ -79,11 +78,11 @@ def get_data(channel_list, start=None, end=None, base_url=None):
raise RuntimeError("Unable to retrieve data from server: ", response)
logger.info("Data retieval is successful")
data = response.json()
return data_api.client._build_pandas_data_frame(data,
index_field="globalDate")
#
# data = response.json()
#
# return data_api.client._build_pandas_data_frame(data,
# index_field="globalDate")
def get_pulse_id_date_mapping(pulse_ids):
@@ -123,28 +122,28 @@ def get_pulse_id_date_mapping(pulse_ids):
logger.info("retrieval failed")
if c == 0:
ref_date = data[0]["data"][0]["globalDate"]
ref_date = dateutil.parser.parse(ref_date)
now_date = datetime.datetime.now()
now_date = pytz.timezone('Europe/Zurich').localize(
now_date)
check_date = ref_date + datetime.timedelta(
seconds=24) # 20 seconds should be enough
delta_date = check_date - now_date
s = delta_date.seconds
logger.info("retry in " + str(s) + " seconds ")
if not s <= 0:
time.sleep(s)
continue
raise RuntimeError('Unable to retrieve mapping')
date = data[0]["data"][0]["globalDate"]
date = dateutil.parser.parse(date)
dates.append(date)
break
# ref_date = dateutil.parser.parse(ref_date)
#
# now_date = datetime.datetime.now()
# now_date = pytz.timezone('Europe/Zurich').localize(
# now_date)
#
# check_date = ref_date + datetime.timedelta(
# seconds=24) # 20 seconds should be enough
# delta_date = check_date - now_date
#
# s = delta_date.seconds
# logger.info("retry in " + str(s) + " seconds ")
# if not s <= 0:
# time.sleep(s)
# continue
#
# raise RuntimeError('Unable to retrieve mapping')
#
# date = data[0]["data"][0]["globalDate"]
# date = dateutil.parser.parse(date)
# dates.append(date)
# break
return dates
except Exception: