first version

rewrite_epicsdata_pids.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import os
+from glob import glob
+import numpy as np
+from scipy.spatial import KDTree
+import h5py
+
+
+class KDTree1D(KDTree):
+
+    def __init__(self, arr):
+        arr = arr.reshape(-1, 1)
+        super().__init__(arr)
+
+    def query(self, arr):
+        arr = arr.reshape(-1, 1)
+        return super().query(arr)
+
+
+def get_fn(fns, contains):
+    fn = [fn for fn in fns if contains in fn]
+    assert len(fn) == 1
+    return fn[0]
+
+
+print("globbing")
+fns = "/sf/alvra/data/p18938/raw/**/*.PVCHANNELS.h5"
+fns = sorted(glob(fns, recursive=True))
+
+for i, fn_pv in enumerate(fns):
+    fn_bs = fn_pv.replace(".PVCHANNELS.h5", ".BSDATA.h5")
+
+    new_fn_pv = fn_pv.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5")
+    folder = os.path.dirname(new_fn_pv)
+    os.makedirs(folder, exist_ok=True)
+
+    print(i)
+    print(fn_bs, fn_pv)
+    print("->", new_fn_pv)
+
+    if os.path.exists(new_fn_pv):
+        print(new_fn_pv, "exists... skipping")
+        continue
+
+    ch_evts = "SAR-CVME-TIFALL4:EvtSet"
+    #ch_evts = "SAR-CVME-TIFALL5:EvtSet"
+
+    with h5py.File(fn_bs, "r") as f:
+        try:
+            bs_ps = f[ch_evts + "/pulse_id"][:]
+            bs_ts = f[ch_evts + "/timestamp"][:]
+        except KeyError:
+            print("broken file", fn_bs)
+            continue
+
+#    print(bs_ts)
+
+    bs_ts_tree = KDTree1D(bs_ts)
+
+    empty_chs = []
+
+    with h5py.File(fn_pv, "r") as f, h5py.File(new_fn_pv, "x") as n:
+        if not len(f.keys()):
+            print("empty file")
+        for ch in f:
+            pv_data = f[ch + "/data"][:]
+            pv_ts = f[ch + "/timestamp"][:]
+            if not pv_ts.size:
+                empty_chs.append(ch)
+                continue
+
+            _dists, indices = bs_ts_tree.query(pv_ts)
+            pv_ps = bs_ps[indices]
+#            print(ch, indices, pv_ps)
+
+            n.create_dataset(ch + "/data",      data=pv_data)
+            n.create_dataset(ch + "/timestamp", data=pv_ts)
+            n.create_dataset(ch + "/pulse_id",  data=pv_ps)
+
+    print("#empty", len(empty_chs))
+
+
+

rewrite_scaninfo.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+import os
+from glob import glob
+import json
+
+fns = "/sf/alvra/data/p18938/raw/scan_info/**.json"
+for fn_si in sorted(glob(fns, recursive=True)):
+    new_fn_si = fn_si.replace("/raw/", "/res/epics/")
+    print(fn_si, new_fn_si)
+
+    folder = os.path.dirname(new_fn_si)
+    os.makedirs(folder, exist_ok=True)
+
+    with open(fn_si, "r") as f:
+        data = json.load(f)
+
+    scan_files = data["scan_files"].copy()
+    for i, sfs in enumerate(scan_files):
+        for j, sf in enumerate(sfs):
+            if sf.endswith(".PVCHANNELS.h5"):
+                new_sf = sf.replace("/raw/", "/res/epics/").replace(".PVCHANNELS.h5", ".EPICSDATA.h5")
+                scan_files[i][j] = new_sf
+
+    new_data = data.copy()
+    new_data["scan_files"] = scan_files
+
+    with open(new_fn_si, "w") as f:
+        json.dump(new_data, f, indent=4)
+
+
+