diff --git a/epics-writer/.gitignore b/epics-writer/.gitignore deleted file mode 100644 index 3ed1302..0000000 --- a/epics-writer/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -build -__pycache__ -.idea -.pytest_cache -*.h5 diff --git a/epics-writer/api.http b/epics-writer/api.http deleted file mode 100644 index 0fd0d2e..0000000 --- a/epics-writer/api.http +++ /dev/null @@ -1,23 +0,0 @@ -# For a quick start check out our HTTP Requests collection (Tools|HTTP Client|Open HTTP Requests Collection) or -# paste cURL into the file and request will be converted to HTTP Request format. -# -# Following HTTP Request Live Templates are available: -# * 'gtrp' and 'gtr' create a GET request with or without query parameters; -# * 'ptr' and 'ptrp' create a POST request with a simple or parameter-like body; -# * 'mptr' and 'fptr' create a POST request to submit a form with a text or file field (multipart/form-data); - -# curl -XPUT -d '{"range":{"startPulseId": 11281433214, "endPulseId": 7281489688}, "parameters":{"output_file":"test.h5"}}' http://localhost:10200/notify -PUT http://localhost:10200/notify -Content-Type: application/json - -{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}} - -### - -PUT http://localhost:10200/notify -Content-Type: application/json - -{"range":{"startPulseId": 11876043548, "endPulseId": 11876044548}, "parameters":{"output_file":"test.h5"}, "channels": ["ABC"]} - -### - diff --git a/epics-writer/ca_default_channel_list b/epics-writer/ca_default_channel_list deleted file mode 100644 index cd23afe..0000000 --- a/epics-writer/ca_default_channel_list +++ /dev/null @@ -1,179 +0,0 @@ -SARES11-VPIG124-360:PRESSURE -SARES11-VMFR124-350:PRESSURE - -SARES11-VMFR125-600:PRESSURE -SARES11-VMCP125-610:PRESSURE -SARES11-VMFR125-400:PRESSURE -SARES11-VMCP125-410:PRESSURE -SARES11-VMCP125-510:PRESSURE -SARES11-EVSP-SAMPLE:SET_PRES -SARES11-EVSP-010:SAMPLE -SARES11-EVSP-010:INTERMEDIATE -SARES11-EVSP-010:DIFFERENT - - -SARES11-EVGG-A010:PRESSURE -SARES11-EVGG-B010:PRESSURE -SARES11-VMDI125-400:PRESSURE -SARES11-VMDI125-500:PRESSURE -SARES11-EVGG-C010:PRESSURE -SARES11-EVGV-C010:PLC_OPEN -SARES11-EVGV-D010:PLC_OPEN -SARES11-EVGV-E010:PLC_OPEN -SARES11-EVGV-E020:PLC_OPEN -SARES11-EVGV-F010:PLC_OPEN - -SLAAR11-LMOT-M421:MOT.RBV -SLAAR11-LMOT-M422:MOT.RBV -SLAAR11-LMOT-M423:MOT.RBV - -SLAAR11-LMOT-M431:MOT.RBV -SLAAR11-LMOT-M432:MOT.RBV -SLAAR11-LMOT-M441:MOT.RBV -SLAAR11-LMOT-M442:MOT.RBV -SLAAR11-LMOT-M444:MOT.RBV -SLAAR11-LENG-R452:VAL_GET -SLAAR11-LMOT-M451:MOTOR_1.RBV - -SLAAR11-LDIO-LAS6891:SET_BO02 -SLAAR11-LPSYS-ESA:TRANS_OPEN - -SAROP11-PALMK118:RAVE -SAROP11-PALMK118:LIVE -SLAAR11-L-LAMCALC2:MOTOR_POS_SMAR_SC -SLAAR11-LENG-R452:VAL_GET -SARES11-XFOC125:LIN1:MOTRBV -SARES11-XICM125:ROX1:MOTRBV -SARES11-XICM125:TRX1:MOTRBV -SARES11-XICM125:ROY1:MOTRBV - -SARES11-XMI125-C4P1:EXPOSURE -SARES11-XMI125:FOCUS.RBV -SARES11-XMI125:ZOOM.RBV -SARES11-XILL125:LIGHTLEVEL_SP -SARES11-XMI125:ROY1:MOTRBV -SARES11-XMI125:ROZ1:MOTRBV - -SARES11-XOTA125:W_X.RBV -SARES11-XOTA125:W_Y.RBV -SARES11-XOTA125:W_Z.RBV -SARES11-XOTA125:W_RX.RBV -SARES11-XOTA125:W_RY.RBV -SARES11-XOTA125:W_RZ.RBV -SARES11-XOTA125:RAIL.RBV -SARES11-XOTA125:MOTOR_Y1.RBV -SARES11-XOTA125:MOTOR_Y2.RBV -SARES11-XOTA125:MOTOR_Y3.RBV -SARES11-XOTA125:MOTOR_X1.RBV -SARES11-XOTA125:MOTOR_Z1.RBV -SARES11-XOTA125:MOTOR_Z2.RBV -SARES11-XOTA125:MODE_SP - -SARES11-XIZO125:TRX1:MOTRBV -SARES11-XIZO125:TRX2:MOTRBV -SARES11-XIZO125:TRY1:MOTRBV -SARES11-XIZO125:TRY2:MOTRBV - -SARES10-XHPLC125:01:FLOW -SARES10-XHPLC125:01:PRESSURE - -SARES10-XHPLC125:02:FLOW -SARES10-XHPLC125:02:PRESSURE - -SARES11-XSAM125:MOTOR_X1.RBV -SARES11-XSAM125:MOTOR_Y1.RBV -SARES11-XSAM125:MOTOR_Z1.RBV - -SARES11-XCRY125:CRY_1.RBV -SARES11-XCRY125:CRY_2.RBV -SARES11-XCRY125:TRZ11:MOTRBV -SARES11-XCRY125:ROY11:MOTRBV -SARES11-XCRY125:ROX11:MOTRBV -SARES11-XCRY125:TRZ21:MOTRBV -SARES11-XCRY125:ROY21:MOTRBV -SARES11-XCRY125:ROX21:MOTRBV -SARES11-XCRY125:TRZ12:MOTRBV -SARES11-XCRY125:ROY12:MOTRBV -SARES11-XCRY125:ROX12:MOTRBV -SARES11-XCRY125:TRZ22:MOTRBV -SARES11-XCRY125:ROY22:MOTRBV -SARES11-XCRY125:ROX22:MOTRBV - -SARBD01-DBPM040:Q1 -SAROP11-ARAMIS:ENERGY -SAROP11-ARAMIS:MODE -SARFE10-PBPG050:PHOTON-ENERGY-PER-PULSE-AVG -SGE-OP2E-ARAMIS:ESA_PH2E_Y1 -SGE-OP2E-ARAMIS:ESA_PH2E_Y2 -SGE-OP2E-ARAMIS:ESA_PH2E_X1 -SGE-OP2E-ARAMIS:ESA_PH2E_X2 -SARCL02-DCOL290:ENE -SARBD01-DBPM040:Q1 -SARBD01-MBND100:P-READ -SARUN03-UIND030:K_SET -SARUN04-UIND030:K_SET -SARUN05-UIND030:K_SET -SARUN06-UIND030:K_SET -SARUN07-UIND030:K_SET -SARUN08-UIND030:K_SET -SARUN09-UIND030:K_SET -SARUN10-UIND030:K_SET -SARUN11-UIND030:K_SET -SARUN12-UIND030:K_SET -SARUN13-UIND030:K_SET -SARUN14-UIND030:K_SET - -SIN-TIMAST-TMA:Beam-Appl-Freq-RB -SIN-TIMAST-TMA:Evt-20-Delay-RB -SIN-TIMAST-TMA:Evt-20-Freq-I -SIN-TIMAST-TMA:Evt-50-Delay-RB -SIN-TIMAST-TMA:Evt-50-Freq-I -SIN-TIMAST-TMA:Evt-29-Delay-RB -SIN-TIMAST-TMA:Evt-29-Freq-I -SIN-TIMAST-TMA:Evt-24-Delay-RB -SIN-TIMAST-TMA:Evt-24-Freq-I - -SARFE10-OATT053:TRANS_SP -SARFE10-OATT053:TRANS_RB -SARFE10-OATT053:TRANS3EDHARM_RB -SARFE10-OATT053:ENERGY -SAROP11-OATT120:TRANS_SP -SAROP11-OATT120:TRANS_RB -SAROP11-OATT120:TRANS3EDHARM_RB -SAROP11-OATT120:ENERGY - -SAROP11-OPPI110:MOTOR_X1.RBV -SAROP11-OPPI110:MOTOR_Y1.RBV -SAROP11-OPPI110:PRESET_SP -SGE-CPCW-71-EVR0:FrontUnivOut15-Ena-SP -SGE-CPCW-71-EVR0:FrontUnivOut15-Src-SP - -SARFE10-OAPU044:MOTOR_X -SARFE10-OAPU044:MOTOR_Y -SARFE10-OAPU044:MOTOR_W -SARFE10-OAPU044:MOTOR_H -SARFE10-OAPU044:MOTOR_X.RBV -SARFE10-OAPU044:MOTOR_Y.RBV -SARFE10-OAPU044:MOTOR_W.RBV -SARFE10-OAPU044:MOTOR_H.RBV -SARFE10-OAPU044:MOTOR_AX1.RBV -SARFE10-OAPU044:MOTOR_AX2.RBV -SARFE10-OAPU044:MOTOR_AY1.RBV -SARFE10-OAPU044:MOTOR_AY2.RBV -SARFE10-OAPU044:MOTOR_BX1.RBV -SARFE10-OAPU044:MOTOR_BX2.RBV -SARFE10-OAPU044:MOTOR_BY1.RBV -SARFE10-OAPU044:MOTOR_BY2.RBV - -SAROP11-OAPU104:MOTOR_X -SAROP11-OAPU104:MOTOR_Y -SAROP11-OAPU104:MOTOR_W -SAROP11-OAPU104:MOTOR_H - -SAROP11-OAPU120:MOTOR_X -SAROP11-OAPU120:MOTOR_Y -SAROP11-OAPU120:MOTOR_W -SAROP11-OAPU120:MOTOR_H - -SLAAR01-TLSY-EPL:JITTERMON - diff --git a/epics-writer/epics-writer/__init__.py b/epics-writer/epics_writer/__init__.py similarity index 100% rename from epics-writer/epics-writer/__init__.py rename to epics-writer/epics_writer/__init__.py diff --git a/epics-writer/epics-writer/start.py b/epics-writer/epics_writer/start.py similarity index 89% rename from epics-writer/epics-writer/start.py rename to epics-writer/epics_writer/start.py index de84e51..feac307 100644 --- a/epics-writer/epics-writer/start.py +++ b/epics-writer/epics_writer/start.py @@ -1,7 +1,8 @@ import json - from pika import BlockingConnection, ConnectionParameters, BasicProperties +from epics_writer import writer + DEFAULT_BROKER_URL = "127.0.0.1" REQUEST_EXCHANGE = "request" STATUS_EXCHANGE = "status" @@ -12,7 +13,7 @@ OUTPUT_FILE_SUFFIX = ".PVCHANNELS.h5" def update_status(channel, body, action, file, message=None): status_header = { "action": action, - "source": "epics-writer", + "source": "epics_writer", "routing_key": QUEUE_NAME, "file": file, "message": message @@ -41,7 +42,11 @@ def on_message(channel, method_frame, header_frame, body): update_status(channel, body, "write_start", output_file) - # TODO: Call the actual writing. + writer.write_epics_pvs(output_file=output_file, + start_pulse_id=start_pulse_id, + stop_pulse_id=stop_pulse_id, + metadata=metadata, + epics_pvs=epics_pvs) except Exception as e: channel.basic_reject(delivery_tag=method_frame.delivery_tag, diff --git a/epics-writer/epics-writer/epics_writer.py b/epics-writer/epics_writer/writer.py similarity index 62% rename from epics-writer/epics-writer/epics_writer.py rename to epics-writer/epics_writer/writer.py index 45ca24d..aa7e346 100644 --- a/epics-writer/epics-writer/epics_writer.py +++ b/epics-writer/epics_writer/writer.py @@ -1,13 +1,11 @@ -import data_api -import data_api.client +# import data_api +# import data_api.client import requests import datetime import time import logging - - logger = logging.getLogger("logger") @@ -26,28 +24,29 @@ logger = logging.getLogger("logger") # 'output_file': '/bla/test.h5'} # this is usually the full path # } +def write_epics_pvs( + output_file, start_pulse_id, stop_pulse_id, metadata, epics_pvs): -def download_data(start_pulse, end_pulse, channels): logger.info("Dump data to hdf5 ...") - logger.info("Retrieve data for channels: %s" % channels) + logger.info("Retrieve data for channels: %s" % epics_pvs) logger.info("Retrieve pulse-id / data mapping for pulse ids") - start_date, end_date = get_pulse_id_date_mapping([start_pulse, end_pulse]) + start_date, end_date = get_pulse_id_date_mapping([start_pulse_id, stop_pulse_id]) - logger.info("Retrieving data for interval start: " + str( - start_date) + " end: " + str(end_date) + " . From " + new_base_url) - data = get_data(channels, start=start_date, end=end_date, - base_url=new_base_url) + # logger.info("Retrieving data for interval start: " + str( + # start_date) + " end: " + str(end_date) + " . From " + new_base_url) + # data = get_data(channels, start=start_date, end=end_date, + # base_url=new_base_url) - if len(data) < 1: - logger.error("No data retrieved") - open(new_filename + "_NO_DATA", 'a').close() - - else: - if new_filename: - logger.info("Persist data to hdf5 file") - data_api.to_hdf5(data, new_filename, overwrite=True, - compression=None, shuffle=False) + # if len(data) < 1: + # logger.error("No data retrieved") + # open(new_filename + "_NO_DATA", 'a').close() + # + # else: + # if new_filename: + # logger.info("Persist data to hdf5 file") + # data_api.to_hdf5(data, new_filename, overwrite=True, + # compression=None, shuffle=False) def get_data(channel_list, start=None, end=None, base_url=None): @@ -79,11 +78,11 @@ def get_data(channel_list, start=None, end=None, base_url=None): raise RuntimeError("Unable to retrieve data from server: ", response) logger.info("Data retieval is successful") - - data = response.json() - - return data_api.client._build_pandas_data_frame(data, - index_field="globalDate") + # + # data = response.json() + # + # return data_api.client._build_pandas_data_frame(data, + # index_field="globalDate") def get_pulse_id_date_mapping(pulse_ids): @@ -123,28 +122,28 @@ def get_pulse_id_date_mapping(pulse_ids): logger.info("retrieval failed") if c == 0: ref_date = data[0]["data"][0]["globalDate"] - ref_date = dateutil.parser.parse(ref_date) - - now_date = datetime.datetime.now() - now_date = pytz.timezone('Europe/Zurich').localize( - now_date) - - check_date = ref_date + datetime.timedelta( - seconds=24) # 20 seconds should be enough - delta_date = check_date - now_date - - s = delta_date.seconds - logger.info("retry in " + str(s) + " seconds ") - if not s <= 0: - time.sleep(s) - continue - - raise RuntimeError('Unable to retrieve mapping') - - date = data[0]["data"][0]["globalDate"] - date = dateutil.parser.parse(date) - dates.append(date) - break + # ref_date = dateutil.parser.parse(ref_date) + # + # now_date = datetime.datetime.now() + # now_date = pytz.timezone('Europe/Zurich').localize( + # now_date) + # + # check_date = ref_date + datetime.timedelta( + # seconds=24) # 20 seconds should be enough + # delta_date = check_date - now_date + # + # s = delta_date.seconds + # logger.info("retry in " + str(s) + " seconds ") + # if not s <= 0: + # time.sleep(s) + # continue + # + # raise RuntimeError('Unable to retrieve mapping') + # + # date = data[0]["data"][0]["globalDate"] + # date = dateutil.parser.parse(date) + # dates.append(date) + # break return dates except Exception: