diff --git a/rewrite_epicsdata_pids.py b/rewrite_epicsdata_pids.py new file mode 100755 index 0000000..434278d --- /dev/null +++ b/rewrite_epicsdata_pids.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import os +from glob import glob +import numpy as np +from scipy.spatial import KDTree +import h5py + + +class KDTree1D(KDTree): + + def __init__(self, arr): + arr = arr.reshape(-1, 1) + super().__init__(arr) + + def query(self, arr): + arr = arr.reshape(-1, 1) + return super().query(arr) + + +def get_fn(fns, contains): + fn = [fn for fn in fns if contains in fn] + assert len(fn) == 1 + return fn[0] + + +print("globbing") +fns = "/sf/alvra/data/p18938/raw/**/*.PVCHANNELS.h5" +fns = sorted(glob(fns, recursive=True)) + +for i, fn_pv in enumerate(fns): + fn_bs = fn_pv.replace(".PVCHANNELS.h5", ".BSDATA.h5") + + new_fn_pv = fn_pv.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5") + folder = os.path.dirname(new_fn_pv) + os.makedirs(folder, exist_ok=True) + + print(i) + print(fn_bs, fn_pv) + print("->", new_fn_pv) + + if os.path.exists(new_fn_pv): + print(new_fn_pv, "exists... skipping") + continue + + ch_evts = "SAR-CVME-TIFALL4:EvtSet" + #ch_evts = "SAR-CVME-TIFALL5:EvtSet" + + with h5py.File(fn_bs, "r") as f: + try: + bs_ps = f[ch_evts + "/pulse_id"][:] + bs_ts = f[ch_evts + "/timestamp"][:] + except KeyError: + print("broken file", fn_bs) + continue + +# print(bs_ts) + + bs_ts_tree = KDTree1D(bs_ts) + + empty_chs = [] + + with h5py.File(fn_pv, "r") as f, h5py.File(new_fn_pv, "x") as n: + if not len(f.keys()): + print("empty file") + for ch in f: + pv_data = f[ch + "/data"][:] + pv_ts = f[ch + "/timestamp"][:] + if not pv_ts.size: + empty_chs.append(ch) + continue + + _dists, indices = bs_ts_tree.query(pv_ts) + pv_ps = bs_ps[indices] +# print(ch, indices, pv_ps) + + n.create_dataset(ch + "/data", data=pv_data) + n.create_dataset(ch + "/timestamp", data=pv_ts) + n.create_dataset(ch + "/pulse_id", data=pv_ps) + + print("#empty", len(empty_chs)) + + + diff --git a/rewrite_scaninfo.py b/rewrite_scaninfo.py new file mode 100755 index 0000000..ed74f3e --- /dev/null +++ b/rewrite_scaninfo.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import os +from glob import glob +import json + +fns = "/sf/alvra/data/p18938/raw/scan_info/**.json" +for fn_si in sorted(glob(fns, recursive=True)): + new_fn_si = fn_si.replace("/raw/", "/res/epics/") + print(fn_si, new_fn_si) + + folder = os.path.dirname(new_fn_si) + os.makedirs(folder, exist_ok=True) + + with open(fn_si, "r") as f: + data = json.load(f) + + scan_files = data["scan_files"].copy() + for i, sfs in enumerate(scan_files): + for j, sf in enumerate(sfs): + if sf.endswith(".PVCHANNELS.h5"): + new_sf = sf.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5") + scan_files[i][j] = new_sf + + new_data = data.copy() + new_data["scan_files"] = scan_files + + with open(new_fn_si, "w") as f: + json.dump(new_data, f, indent=4) + + +