first version
This commit is contained in:
84
rewrite_epicsdata_pids.py
Executable file
84
rewrite_epicsdata_pids.py
Executable file
@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
from glob import glob
|
||||
import numpy as np
|
||||
from scipy.spatial import KDTree
|
||||
import h5py
|
||||
|
||||
|
||||
class KDTree1D(KDTree):
|
||||
|
||||
def __init__(self, arr):
|
||||
arr = arr.reshape(-1, 1)
|
||||
super().__init__(arr)
|
||||
|
||||
def query(self, arr):
|
||||
arr = arr.reshape(-1, 1)
|
||||
return super().query(arr)
|
||||
|
||||
|
||||
def get_fn(fns, contains):
|
||||
fn = [fn for fn in fns if contains in fn]
|
||||
assert len(fn) == 1
|
||||
return fn[0]
|
||||
|
||||
|
||||
print("globbing")
|
||||
fns = "/sf/alvra/data/p18938/raw/**/*.PVCHANNELS.h5"
|
||||
fns = sorted(glob(fns, recursive=True))
|
||||
|
||||
for i, fn_pv in enumerate(fns):
|
||||
fn_bs = fn_pv.replace(".PVCHANNELS.h5", ".BSDATA.h5")
|
||||
|
||||
new_fn_pv = fn_pv.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5")
|
||||
folder = os.path.dirname(new_fn_pv)
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
|
||||
print(i)
|
||||
print(fn_bs, fn_pv)
|
||||
print("->", new_fn_pv)
|
||||
|
||||
if os.path.exists(new_fn_pv):
|
||||
print(new_fn_pv, "exists... skipping")
|
||||
continue
|
||||
|
||||
ch_evts = "SAR-CVME-TIFALL4:EvtSet"
|
||||
#ch_evts = "SAR-CVME-TIFALL5:EvtSet"
|
||||
|
||||
with h5py.File(fn_bs, "r") as f:
|
||||
try:
|
||||
bs_ps = f[ch_evts + "/pulse_id"][:]
|
||||
bs_ts = f[ch_evts + "/timestamp"][:]
|
||||
except KeyError:
|
||||
print("broken file", fn_bs)
|
||||
continue
|
||||
|
||||
# print(bs_ts)
|
||||
|
||||
bs_ts_tree = KDTree1D(bs_ts)
|
||||
|
||||
empty_chs = []
|
||||
|
||||
with h5py.File(fn_pv, "r") as f, h5py.File(new_fn_pv, "x") as n:
|
||||
if not len(f.keys()):
|
||||
print("empty file")
|
||||
for ch in f:
|
||||
pv_data = f[ch + "/data"][:]
|
||||
pv_ts = f[ch + "/timestamp"][:]
|
||||
if not pv_ts.size:
|
||||
empty_chs.append(ch)
|
||||
continue
|
||||
|
||||
_dists, indices = bs_ts_tree.query(pv_ts)
|
||||
pv_ps = bs_ps[indices]
|
||||
# print(ch, indices, pv_ps)
|
||||
|
||||
n.create_dataset(ch + "/data", data=pv_data)
|
||||
n.create_dataset(ch + "/timestamp", data=pv_ts)
|
||||
n.create_dataset(ch + "/pulse_id", data=pv_ps)
|
||||
|
||||
print("#empty", len(empty_chs))
|
||||
|
||||
|
||||
|
32
rewrite_scaninfo.py
Executable file
32
rewrite_scaninfo.py
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
from glob import glob
|
||||
import json
|
||||
|
||||
fns = "/sf/alvra/data/p18938/raw/scan_info/**.json"
|
||||
for fn_si in sorted(glob(fns, recursive=True)):
|
||||
new_fn_si = fn_si.replace("/raw/", "/res/epics/")
|
||||
print(fn_si, new_fn_si)
|
||||
|
||||
folder = os.path.dirname(new_fn_si)
|
||||
os.makedirs(folder, exist_ok=True)
|
||||
|
||||
with open(fn_si, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
scan_files = data["scan_files"].copy()
|
||||
for i, sfs in enumerate(scan_files):
|
||||
for j, sf in enumerate(sfs):
|
||||
if sf.endswith(".PVCHANNELS.h5"):
|
||||
new_sf = sf.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5")
|
||||
scan_files[i][j] = new_sf
|
||||
|
||||
new_data = data.copy()
|
||||
new_data["scan_files"] = scan_files
|
||||
|
||||
with open(new_fn_si, "w") as f:
|
||||
json.dump(new_data, f, indent=4)
|
||||
|
||||
|
||||
|
Reference in New Issue
Block a user