diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..70697c9 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.linting.pylintEnabled": true, + "python.linting.enabled": true, + "python.pythonPath": "/sf/bernina/anaconda/bernina_envs/bernina37/bin/python" +} \ No newline at end of file diff --git a/eco/.vscode/settings.json b/eco/.vscode/settings.json new file mode 100644 index 0000000..de288e1 --- /dev/null +++ b/eco/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} \ No newline at end of file diff --git a/eco/acquisition/daq_client.py b/eco/acquisition/daq_client.py new file mode 100644 index 0000000..0225740 --- /dev/null +++ b/eco/acquisition/daq_client.py @@ -0,0 +1,158 @@ +import requests +from pathlib import Path +from time import sleep +from ..devices_general.detectors import PvDataStream + + +class Daq: + def __init__( + self, + broker_address="http://sf-daq-1:10002", + timeout=10, + pgroup=None, + pulse_id_adj=None, + event_master=None, + detectors_event_code=None, + instrument=None, + channels_JF=None, + channels_BS=None, + channels_BSCAM=None, + ): + self.channels = {} + if channels_JF: + self.channels["channels_JF"] = channels_JF + if channels_BS: + self.channels["channels_BS"] = channels_BS + if channels_BSCAM: + self.channels["channels_BSCAM"] = channels_BSCAM + self.broker_address = broker_address + self.timeout = timeout + self.pgroup = pgroup + if type(pulse_id_adj) is str: + self.pulse_id = PvDataStream(pulse_id_adj, name="pulse_id") + else: + self.pulse_id = pulse_id_adj + self.running = [] + self._event_master = event_master + self._detectors_event_code = detectors_event_code + + def acquire(self, Npulses, label=None, wait=True, **kwargs): + ix = self.start(label=label, **kwargs) + return self.stop(stop_id=ix, wait=wait) + + def start(self, label=None, **kwargs): + start_id = self.pulse_id.get_current_value() + acq_pars = { + "label": label, + "start_id": start_id, + } + acq_pars.update(kwargs) + self.running.append(acq_pars) + return self.running.index(acq_pars) + + def stop( + self, + stop_id=None, + acq_ix=None, + label=None, + wait=True, + wait_cycle_sleep=0.01, + **kwargs, + ): + if not stop_id: + stop_id = int(self.pulse_id.get_current_value()) + if not acq_ix: + acq_ix = -1 + + acq_pars = self.running.pop(acq_ix) + acq_pars["stop_id"] = stop_id + acq_pars.update(kwargs) + label = acq_pars.pop("label") + if wait: + while int(self.pulse_id.get_current_value()) < stop_id: + sleep(wait_cycle_sleep) + return self.retrieve(**acq_pars) + + def retrieve( + self, + *, + start_id, + stop_id, + directory_relative=None, + channels_CA=None, + channels_JF=None, + channels_BS=None, + channels_BSCAM=None, + pgroup=None, + pgroup_base_path="/sf/bernina/data/{:s}/raw", + filename_format="run_{:06d}", + ): + if not pgroup: + pgroup = self.pgroup + if not pgroup: + raise Exception("a pgroup needs to be defined") + if not directory_relative: + directory_relative = "" + directory_relative = Path(directory_relative) + directory_base = Path(pgroup_base_path.format(pgroup)) / directory_relative + files_extensions = [] + parameters = {"start_pulseid": start_id, "stop_pulseid": stop_id} + if channels_CA: + parameters["pv_list"] = channels_CA + files_extensions.append("CADUMP") + if channels_BS: + parameters["channels_list"] = channels_BS + files_extensions.append("BSREAD") + if channels_JF: + parameters["detectors"] = {tn: {} for tn in channels_JF} + for ch in channels_JF: + files_extensions.append(ch) + if channels_BSCAM: + parameters["camera_list"] = channels_BSCAM + files_extensions.append("BSREAD.CAMERAS") + if directory_relative: + parameters["directory_name"] = directory_relative.as_posix() + + parameters["pgroup"] = pgroup + + runno = validate_response( + requests.post( + f"{self.broker_address}/retrieve_from_buffers", + json=parameters, + timeout=self.timeout, + ).json() + ) + + filenames = [ + (directory_base / Path(filename_format.format(runno))).with_suffix( + f".{ext}.h5" + ) + for ext in files_extensions + ] + + return runno, filenames + + def get_detector_frequency(self): + return self._event_master.get_evtcode_frequency(self._detectors_event_code) + + +def validate_response(resp): + if resp.get("status") == "ok": + return int(resp.get("message")) + + message = resp.get("message", "Unknown error") + msg = "An error happened on the server:\n{}".format(message) + raise Exception(msg) + + +# parameters = { +# "pgroup":"p16584", +# "start_pulseid":12054777413-2000, +# "stop_pulseid":12054777413-1000, +# "channels_list":[ +# "SAR-CVME-TIFALL5:EvtSet" +# ], +# "run_number" +# } + +# r = requests.post(f'{broker_address}/retrieve_from_buffers',json=parameters, timeout=TIMEOUT_DAQ).json() diff --git a/eco/acquisition/epics_data.py b/eco/acquisition/epics_data.py index ae8af50..203d05e 100644 --- a/eco/acquisition/epics_data.py +++ b/eco/acquisition/epics_data.py @@ -16,8 +16,10 @@ class Epicstools: default_channel_list={"listname": []}, default_file_path="%s", elog=None, + name=None, channel_list=None, ): + self.name = name self._default_file_path = default_file_path self._default_channel_list = default_channel_list self._elog = elog diff --git a/eco/acquisition/scan.py b/eco/acquisition/scan.py index 9877fc6..f4737e7 100755 --- a/eco/acquisition/scan.py +++ b/eco/acquisition/scan.py @@ -23,8 +23,10 @@ class Scan: callbackStartStep=None, checker_sleep_time=0.2, return_at_end="question", + run_table=None, ): self.Nsteps = len(values) + self._run_table = run_table self.pulses_per_step = Npulses self.adjustables = adjustables self.values_todo = values @@ -57,6 +59,27 @@ class Scan: tv = adj.get_current_value() self.initial_values.append(adj.get_current_value()) print("Initial value of %s : %g" % (adj.name, tv)) + + if self._run_table: + runname = os.path.basename(fina).split('.')[0] + runno = int(runname.split('run')[1].split('_')[0]) + metadata = { + "type": "scan", + "name": runname.split('_', 1)[1], + + } + for n, adj in enumerate(self.adjustables): + metadata.update({ + f'scan_motor_{n}': adj.name, + f'from_motor_{n}': self.values_todo[0][n], + f'to_motor_{n}': self.values_todo[-1][n] + }) + metadata.update({ + "steps": len(self.values_todo), + "pulses_per_step": Npulses, + "counters": [daq.name for daq in counterCallers], + }) + run_table.append_run(runno, metadata=metadata) def get_filename(self, stepNo, Ndigits=4): fina = os.path.join(self.basepath, Path(self.fina).stem) @@ -181,7 +204,9 @@ class Scans: default_counters=[], checker=None, scan_directories=False, + run_table=None, ): + self._run_table=run_table self.data_base_dir = data_base_dir scan_info_dir = Path(scan_info_dir) if not scan_info_dir.exists(): @@ -242,6 +267,7 @@ class Scans: scan_info_dir=self.scan_info_dir, checker=self.checker, scan_directories=self._scan_directories, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) @@ -275,6 +301,7 @@ class Scans: scan_info_dir=self.scan_info_dir, checker=self.checker, scan_directories=self._scan_directories, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) @@ -309,6 +336,7 @@ class Scans: checker=self.checker, scan_directories=self._scan_directories, return_at_end=return_at_end, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) @@ -344,6 +372,7 @@ class Scans: checker=self.checker, scan_directories=self._scan_directories, return_at_end=return_at_end, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) @@ -382,6 +411,7 @@ class Scans: checker=self.checker, scan_directories=self._scan_directories, return_at_end=return_at_end, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) @@ -419,6 +449,7 @@ class Scans: checker=self.checker, scan_directories=self._scan_directories, return_at_end=return_at_end, + run_table=self._run_table, ) if start_immediately: s.scanAll(step_info=step_info) diff --git a/eco/bernina/config.py b/eco/bernina/config.py index bfa8073..762bb4e 100755 --- a/eco/bernina/config.py +++ b/eco/bernina/config.py @@ -32,6 +32,14 @@ components = [ # 'args' : ['all','the','requires','args'], # 'kwargs': {} # } + { + "args": [config["pgroup"]], + "name": "run_table", + "desc": "run table for data acquisition", + "type": "eco.utilities.runtable:Run_Table", + "kwargs": {"devices": None, "alias_namespace": None}, + "lazy": False, + }, { "type": "eco.utilities.config:append_to_path", "args": config["path_exp"], @@ -467,6 +475,7 @@ components = [ "default_counters": [Component("daq")], "checker": Component("checker"), "scan_directories": True, + "run_table": Component("run_table"), }, }, { @@ -480,6 +489,7 @@ components = [ "default_counters": [Component("epics_daq")], "checker": Component("checker"), "scan_directories": True, + "run_table": Component("run_table"), }, }, { @@ -643,9 +653,15 @@ components = [ "z_und": 142, "desc": "electro optic sampling stages", "type": "eco.endstations.bernina_sample_environments:electro_optic_sampling", - "kwargs": {"Id": "SARES23", - 'pgroup':config['pgroup'], - 'diode_channels': {'d1': 'SARES20-LSCP9-FNS:CH1:VAL_GET','d2': 'SARES20-LSCP9-FNS:CH2:VAL_GET','diff': 'SARES20-LSCP9-FNS:CH3:VAL_GET'}}, + "kwargs": { + "Id": "SARES23", + "pgroup": config["pgroup"], + "diode_channels": { + "d1": "SARES20-LSCP9-FNS:CH1:VAL_GET", + "d2": "SARES20-LSCP9-FNS:CH2:VAL_GET", + "diff": "SARES20-LSCP9-FNS:CH3:VAL_GET", + }, + }, "lazy": True, }, { @@ -658,12 +674,18 @@ components = [ "lazy": True, }, { - "args": [config['pgroup']], - "name": "run_table", - "desc": "run table for data acquisition", - "type": "eco.utilities.runtable:Run_Table", - "kwargs": {'devices':None, 'alias_namespace': None}, - "lazy": False, + "args": [], + "name": "daq_new", + "desc": "server based acquisition new JF stuff", + "type": "eco.acquisition.daq_client:Daq", + "kwargs": { + "instrument": "bernina", + "pgroup": config["pgroup"], + "channels_JF": config["jf_channels"], + "pulse_id_adj": "SLAAR21-LTIM01-EVR0:RX-PULSEID", + "event_master": Component("event_master"), + "detectors_event_code": 50, + }, }, ] @@ -672,5 +694,3 @@ try: print("Did append additional components!") except: print("Could not append components from config.") - - diff --git a/eco/devices_general/.smaract.py.swo b/eco/devices_general/.smaract.py.swo new file mode 100644 index 0000000..ecdd4dd Binary files /dev/null and b/eco/devices_general/.smaract.py.swo differ diff --git a/eco/endstations/bernina_sample_environments.py b/eco/endstations/bernina_sample_environments.py index 7d179af..57aafb3 100644 --- a/eco/endstations/bernina_sample_environments.py +++ b/eco/endstations/bernina_sample_environments.py @@ -430,10 +430,11 @@ class electro_optic_sampling: dat2 = datmean['d2'] diff = datmean['diff'] diffOverSum = (dat1-dat2) / (dat1+dat2) - - freq_0 , ampl_0 = self.calcFFT(x,diff.T) - - freq_1, ampl_1 = self.calcFFT(x,diffOverSum.T) + if 'delay' in x_motor: + freq_0 , ampl_0 = self.calcFFT(x,diff.T) + freq_1, ampl_1 = self.calcFFT(x,diffOverSum.T) + else: + freq_0 = 0; freq_1 =0 ; ampl_0 = 0; ampl_1 = 0 ax[0,0].set_title(f'Run_{runno}') ax[0,0].plot(x,diff,'k-',label = 'Channel3 (diff)') ax[1,0].plot(x,diffOverSum,'r-',label='Diff / Sum ') @@ -448,7 +449,7 @@ class electro_optic_sampling: ax[ii,0].legend() ax[ii,0].set_xlabel(x_motor) ax[0,ii].legend() - return + return x,diffOverSum def calcFFT(self,x,y,norm=True, lim=[0.1, 15]): # lim: min and max in THz for normalization and plotting diff --git a/eco/timing/event_timing.py b/eco/timing/event_timing.py index 1f09606..3bf927e 100644 --- a/eco/timing/event_timing.py +++ b/eco/timing/event_timing.py @@ -273,7 +273,9 @@ class EvrOutput: self._pvs[pvname] = PV(pvname) return self._pvs[pvname] - # def _update_connected_pulsers(self): + # def _update_connected_pulsers(self): event_master=None, + detectors_event_code = (None,) + # self._get_pv() # self.pulsers = () diff --git a/eco/timing/event_timing_new.py b/eco/timing/event_timing_new.py index 63e1fcc..425015a 100644 --- a/eco/timing/event_timing_new.py +++ b/eco/timing/event_timing_new.py @@ -5,6 +5,7 @@ from ..devices_general.adjustable import PvEnum, PvRecord from ..devices_general.detectors import PvDataStream from ..eco_epics.utilities_epics import EpicsString from cta_lib import CtaLib +from numbers import Number class TimingSystem: @@ -195,7 +196,22 @@ class MasterEventSystem: def get_evtcode_period(self, evtcode): """ in s""" Id = self._get_evtcode_Id(evtcode) - return self._get_Id_period(Id) / 1000 + return self._get_Id_period(Id) + + def get_evt_code_status(self, codes=None): + if not codes: + codes = sorted(eventcodes) + if isinstance(codes, Number): + codes = [codes] + s = [] + for c in codes: + s.append( + f"{c:3d}: delay = {self.get_evtcode_delay(c)*1e6:9.3f} us; frequency: {self.get_evtcode_frequency(c):5.1f} Hz; Desc.: {self.get_evtcode_description(c)}" + ) + return s + + def status(self, codes=None): + print("\n".join(self.get_evt_code_status(codes))) / 1000 class EvrPulser: diff --git a/eco/utilities/runtable.py b/eco/utilities/runtable.py index 947cb0a..e92e8dd 100644 --- a/eco/utilities/runtable.py +++ b/eco/utilities/runtable.py @@ -10,17 +10,19 @@ import gspread_dataframe as gd import gspread_formatting as gf import gspread_formatting.dataframe as gf_dataframe from datetime import datetime - +import xlwt +import openpyxl class Run_Table(): - def __init__(self, pgroup, devices=None, alias_namespace=None, add_pvs={'pulse_id': 'SLAAR11-LTIM01-EVR0:RX-PULSEID'}): + def __init__(self, pgroup, devices=None, alias_namespace=None, add_pvs={'pulse_id': 'SLAAR11-LTIM01-EVR0:RX-PULSEID'}, name=None): ''' Additional PVs is a dictionary holding additional PVs to be saved to the dataframes ''' + self.name=name self.alias_df = DataFrame() self.adj_df = DataFrame() - self.alias_file_name = f'/sf/bernina/data/{pgroup}/res/runtables/{pgroup}_alias_runtable.pkl' - self.adj_file_name = f'/sf/bernina/data/{pgroup}/res/runtables/{pgroup}_adjustable_runtable.pkl' + self.alias_file_name = f'/sf/bernina/data/{pgroup}/res/runtables/{pgroup}_alias_runtable' + self.adj_file_name = f'/sf/bernina/data/{pgroup}/res/runtables/{pgroup}_adjustable_runtable' self._add_pvs = add_pvs if not devices: from eco import bernina @@ -31,8 +33,8 @@ class Run_Table(): 'https://www.googleapis.com/auth/drive'] self._credentials = ServiceAccountCredentials.from_json_keyfile_name('/sf/bernina/config/src/python/gspread/pandas_push', self._scope) self.gc = gspread.authorize(self._credentials) - self.keys = 'gps thc hex energy transmission hpos vpos hgap vgap shut delay lxt phase_shifter type name scan_motor pulse_id shut att_self att_fe_self las' - self.key_order = 'metadata time type name scan_motor from to steps gps gps_hex thc las lxt phase_shifter xrd mono att att_fe slit_und slit_switch slit_att slit_kb slit_cleanup pulse_id mono_energy_rbk att_transmission att_fe_transmission' + self.keys = 'metadata gps thc hex tht eos energy transmission hpos vpos hgap vgap shut delay lxt pulse_id att_self att_fe_self' + self.key_order = 'metadata time name gps gps_hex thc tht eos las lxt phase_shifter xrd mono att att_fe slit_und slit_switch slit_att slit_kb slit_cleanup pulse_id mono_energy_rbk att_transmission att_fe_transmission' if not alias_namespace: from eco.aliases import NamespaceCollection alias_namespace = NamespaceCollection().bernina @@ -42,14 +44,7 @@ class Run_Table(): self.adjustables = {} self._parse_parent() pd.options.display.max_rows = 999 - - if os.path.exists(self.alias_file_name): - #self.alias_df = pd.read_hdf(self.alias_file_name, key=None) - self.alias_df = pd.read_pickle(self.alias_file_name) - - if os.path.exists(self.adj_file_name): - #self.adj_df = pd.read_hdf(self.adj_file_name, key=None) - self.adj_df = pd.read_pickle(self.adj_file_name) + self.load() def _query_by_keys(self, keys='', df=None): if df is None: @@ -88,8 +83,7 @@ class Run_Table(): return filtered_dict def save(self): - data_dir = Path(os.path.dirname(self.alias_file_name)) - + data_dir = Path(os.path.dirname(self.alias_file_name+'.pkl')) if not data_dir.exists(): print( f"Path {data_dir.absolute().as_posix()} does not exist, will try to create it..." @@ -98,10 +92,19 @@ class Run_Table(): print(f"Tried to create {data_dir.absolute().as_posix()}") data_dir.chmod(0o775) print(f"Tried to change permissions to 775") - self.alias_df.to_pickle(self.alias_file_name) - self.adj_df.to_pickle(self.adj_file_name) - #self.alias_df.to_hdf(self.alias_file_name, key=None) - #self.adj_df.to_hdf(self.adj_file_name, key=None) + self.alias_df.to_pickle(self.alias_file_name+'.pkl') + self.adj_df.to_pickle(self.adj_file_name+'.pkl') + self.alias_df.to_hdf(self.alias_file_name+'.h5', key='data') + self.adj_df.to_hdf(self.adj_file_name+'.h5', key='data') + self.alias_df.to_excel(self.alias_file_name+'.xlsx') + self.adj_df.to_excel(self.adj_file_name+'.xlsx') + + def load(self): + if os.path.exists(self.alias_file_name+'.pkl'): + self.alias_df = pd.read_pickle(self.alias_file_name+'.pkl') + if os.path.exists(self.adj_file_name+'.pkl'): + self.adj_df = pd.read_pickle(self.adj_file_name+'.pkl') + def append_run(self, runno, metadata={'type': 'ascan', 'name': 'phi scan (001)', @@ -109,6 +112,9 @@ class Run_Table(): 'from': 1, 'to': 2, 'steps': 51}): + self.load() + if len(self.adjustables) == 0: + self._parse_parent() dat = self._get_values() dat.update(metadata) dat['time'] = datetime.now() @@ -126,6 +132,9 @@ class Run_Table(): self.save() def append_pos(self, name=''): + self.load() + if len(self.adjustables) == 0: + self._parse_parent() try: posno = int(self.alias_df.query('type == "pos"').index[-1].split('p')[1])+1 except: diff --git a/eco/xoptics/reflaser_new.py b/eco/xoptics/reflaser_new.py index bb3f4c3..e261327 100644 --- a/eco/xoptics/reflaser_new.py +++ b/eco/xoptics/reflaser_new.py @@ -9,7 +9,7 @@ class RefLaser_Aramis: self.elog = elog self.name = name self.alias = Alias(name) - append_object_to_object(self, + # append_object_to_object(self, self._inpos = inpos self._outpos = outpos