From fc11bb3576b3f150cb843ffd37c9a21a788fd767 Mon Sep 17 00:00:00 2001 From: Sven Augustin Date: Tue, 22 Sep 2020 12:35:43 +0200 Subject: [PATCH] prototype --- execute.py | 16 +- sani.py | 290 +++++++++++++++--------------------- test.csv | 12 ++ test1.csv | 12 ++ test2.csv | 12 ++ test3.csv | 12 ++ test4.csv | 12 ++ chans.txt => test_chans.txt | 0 8 files changed, 186 insertions(+), 180 deletions(-) create mode 100644 test.csv create mode 100644 test1.csv create mode 100644 test2.csv create mode 100644 test3.csv create mode 100644 test4.csv rename chans.txt => test_chans.txt (100%) diff --git a/execute.py b/execute.py index b0df236..722fbc6 100644 --- a/execute.py +++ b/execute.py @@ -1,16 +1,20 @@ from concurrent.futures import ThreadPoolExecutor -def parallel(func, targets, names): +def parallel(func, targets, names=None): with ThreadPoolExecutor() as executor: results = executor.map(func, targets) -# return list(results) - return dict(zip(names, results)) + if names: + return dict(zip(names, results)) + else: + return list(results) -def serial(func, targets): - return [func(t) for t in targets] -# return {t: func(t) for t in targets} +def serial(func, targets, names=None): + if names: + return {n: func(t) for n, t in zip(names, targets)} + else: + return [func(t) for t in targets] diff --git a/sani.py b/sani.py index 1c23904..41bc045 100755 --- a/sani.py +++ b/sani.py @@ -1,177 +1,54 @@ #!/usr/bin/env python - import argparse -parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser = argparse.ArgumentParser() +subparsers = parser.add_subparsers(title="command", description="valid commands", dest="command", help="commands") -parser.add_argument("filename") +parser_check = subparsers.add_parser("check", help="check!", formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser_check.add_argument("filename", help="name of input channel-list file") +parser_check.add_argument("-o", "--output", help="output CSV file", default=None) +parser_check.add_argument("-s", "--silent", help="do not show each channel's answer", action="store_true") +parser_check.add_argument("-t", "--timeout", help="connection timeout in seconds", type=float, default=1) -#parser.add_argument("-v", "--verbose", action="count", default=0) -parser.add_argument("-s", "--show", action="store_true") -parser.add_argument("-t", "--timeout", type=float, default=2) +parser_compare = subparsers.add_parser("compare", help="compare!") +parser_compare.add_argument("filenames", metavar="filename", nargs=2, help="name of input CSV file, two are needed") +parser_compare.add_argument("-v", "--ignore-values", help="do not check values", action="store_true") clargs = parser.parse_args() +if not clargs.command: + parser.print_help() + raise SystemExit - - - - -import enum - +from datetime import datetime import epics +import numpy as np import pandas as pd +from colorama import Fore, Style -from pvcollection import PVCollection from alarms import message from config import load from execute import parallel - - - - - - - - - - +#from execute import serial as parallel MSG_NOT_CONNECTED = "did not connect" MSG_SUCCESS = "OK" +SYM_GOOD = "👍" +SYM_BAD = "💔" +COL_NOT_CONNECTED = Fore.RED +COL_SUCCESS = Fore.GREEN +COL_ALARM = Fore.YELLOW -def v1a(): - cs = load(clargs.filename) - pvs = [epics.PV(c) for c in cs] - for pv in pvs: - c = pv.pvname - connected = pv.wait_for_connection(clargs.timeout) - if not connected: - msg = MSG_NOT_CONNECTED - elif pv.status != 0 or pv.severity != 0: - msg = message(pv.status, pv.severity) - else: - if not clargs.show: - continue - msg = MSG_SUCCESS -# print(c, msg) - - -def v1b(): - chans = load(clargs.filename) - pvs = [epics.PV(ch) for ch in chans] - connected = parallel(lambda pv: pv.wait_for_connection(clargs.timeout), pvs) - - for ch, pv, con in zip(chans, pvs, connected): - if not con: - msg = MSG_NOT_CONNECTED - elif pv.status != 0 or pv.severity != 0: - msg = message(pv.status, pv.severity) - else: - if not clargs.show: - continue - msg = MSG_SUCCESS -# print(ch, msg) - - - - - -def v1c(): - chans = load(clargs.filename) - length = maxstrlen(chans) - pvs = [epics.PV(ch) for ch in chans] - connected = parallel(lambda pv: pv.wait_for_connection(clargs.timeout), pvs) - - data = {} - for ch, pv, con in zip(chans, pvs, connected): - if not con: - status = severity = -1 - value = None - msg = MSG_NOT_CONNECTED - else: - status = pv.status - severity = pv.severity - if status == 0 and severity == 0: - if not clargs.show: - continue - msg = MSG_SUCCESS - else: - msg = message(status, severity) - value = pv.value - -# print(ch.ljust(length), msg) - data[ch] = (con, status, severity, value) - - columns = ("connected", "status", "severity", "value") - df = pd.DataFrame.from_dict(data, columns=columns, orient="index") -# print() -# print(df) -# print() -# print(df.dtypes) - - - -def maxstrlen(seq): - return max(strlen(i) for i in seq) - -def strlen(val): - return len(str(val)) - - - -#cs = load(clargs.filename) -#pvs = [epics.PV(c) for c in cs] -#data = [] -#for pv in pvs: -# c = pv.pvname -# connected = pv.wait_for_connection(clargs.timeout) - -# if not connected: -# print(c, "didn't connect") -# data.append((connected, -1, -1, None)) -# continue -# if pv.status != 0 or pv.severity != 0: -# print(c, message(pv.status, pv.severity)) - -# data.append((connected, pv.status, pv.severity, pv.value)) - -#df = pd.DataFrame(data, index=cs, columns=["connected", "status", "severity", "value"]) - -#print(df) -#print(df.dtypes) - - - - - - - -def v2a(): - pvc = PVCollection.from_file(clargs.filename) - - data = { - "connected": pvc.connected(), - "status": pvc.status(), - "severity": pvc.severity(), - "value": pvc.value() - } - - df = pd.DataFrame(data, index=pvc.chans) - -# print(df) -# print(df.dtypes) - - - - +#COL_COMP_LEFT = Fore.MAGENTA +#COL_COMP_LEFT = Fore.CYAN +COL_RESET = Fore.RESET @@ -179,17 +56,20 @@ def get_data(pv): connected = pv.wait_for_connection(clargs.timeout) if not connected: - value = None + value = np.nan status = severity = -1 msg = MSG_NOT_CONNECTED + col = COL_NOT_CONNECTED else: - value = pv.value #TODO: not needed if shot not ok + value = pv.value status = pv.status severity = pv.severity if status == 0 and severity == 0: msg = MSG_SUCCESS + col = COL_SUCCESS else: msg = message(status, severity) + col = COL_ALARM data = { "connected": connected, @@ -198,44 +78,106 @@ def get_data(pv): "severity": severity } - if clargs.show: + if not clargs.silent: + msg = colored(col, msg) print(pv.pvname, msg) - return data#, msg + return data + + +def colored(color, msg): + return color + str(msg) + COL_RESET - - -def v3(): - chans = load(clargs.filename) - pvs = (epics.PV(ch) for ch in chans) # putting PV constructors into threads has weird effects +def run_check(): + filename = clargs.filename + chans = load(filename) + pvs = (epics.PV(ch) for ch in chans) # putting PV constructors into ThreadPoolExecutor has weird effects data = parallel(get_data, pvs, chans) + df = pd.DataFrame(data).T - return df - - - - - - - - -res = v3() -print(res) -#for m in res: -# print(m) - - - - + df = df.infer_objects() #TODO: why is this needed? +# print(df) +# print(df.dtypes) + + connection_state = df["connected"] + if connection_state.all(): + print(f"{SYM_GOOD} all connections OK") + else: + total = connection_state.index + good = total[connection_state] + + ntotal = len(total) + ngood = len(good) + + print(f"{SYM_BAD} only {ngood}/{ntotal} connections OK") + + + output = clargs.output + if not output: + return + + timestamp = datetime.now() + meta = f"{filename} / {timestamp}" + store_csv(df, output, meta) + + +def run_compare(): + fn1, fn2 = clargs.filenames + df1 = load_csv(fn1) + df2 = load_csv(fn2) + + if clargs.ignore_values: + df1.drop("value", axis="columns", inplace=True) + df2.drop("value", axis="columns", inplace=True) + + def report_diff(x): + return "" if equal(*x) else " {} | {}".format(*x) + + def equal(a, b): + return a == b or (np.isnan(a) and np.isnan(b)) + + df = pd.concat((df1, df2)) + changes = df.groupby(level=0).agg(report_diff) + + changes.replace("", np.nan, inplace=True) + changes.dropna(axis="columns", how="all", inplace=True) + changes.dropna(axis="index", how="all", inplace=True) + changes.replace(np.nan, "", inplace=True) + + if changes.empty: + print(f'{SYM_GOOD} "{fn1}" and "{fn2}" are identical') + else: + print(f'{SYM_BAD} "{fn1}" and "{fn2}" differ:') + print(changes) +def store_csv(df, fname, meta): + fname = fix_file_ext(fname, "csv") + with open(fname, "w") as f: + f.write(f"# {meta}\n") + df.to_csv(f) + +def load_csv(fname): + fname = fix_file_ext(fname, "csv") + return pd.read_csv(fname, index_col=0, comment="#", float_precision="high") +def fix_file_ext(fn, ext): + if not ext.startswith("."): + ext = "." + ext + if not fn.endswith(ext): + fn += ext + return fn +if __name__ == "__main__": + if clargs.command == "check": + run_check() + elif clargs.command == "compare": + run_compare() diff --git a/test.csv b/test.csv new file mode 100644 index 0000000..cce6f5a --- /dev/null +++ b/test.csv @@ -0,0 +1,12 @@ +#metameta +,connected,value,status,severity +CRAP:CRAP0,False,,-1,-1 +CRAP:CRAP1,False,,-1,-1 +CRAP:CRAP2,False,,-1,-1 +CRAP:CRAP3,False,,-1,-1 +SAROP21-OKB:MODE,True,0.0,0,0 +SATOP11-VVPG092-A010:PLC_OPEN,True,0.0,7,2 +SATOP21-OAPU161:MOTOR_W.RBV,True,10.0,0,0 +SATOP21-VMCC165-A010:PRESSURE,True,1.88e-07,0,0 +SATOP21-VMFR165-A010:PRESSURE,True,3.49e-08,0,0 +SIN-TIMAST-TMA:Bunch-2-Appl-Freq-RB,True,25.0,0,0 diff --git a/test1.csv b/test1.csv new file mode 100644 index 0000000..a2be4c1 --- /dev/null +++ b/test1.csv @@ -0,0 +1,12 @@ +# test_chans.txt / 2020-09-20 11:40:26.560030 +,connected,value,status,severity +CRAP:CRAP0,False,,-1,-1 +CRAP:CRAP1,False,,-1,-1 +CRAP:CRAP2,False,,-1,-1 +CRAP:CRAP3,False,,-1,-1 +SAROP21-OKB:MODE,True,0.0,0,0 +SATOP11-VVPG092-A010:PLC_OPEN,True,0.0,7,2 +SATOP21-OAPU161:MOTOR_W.RBV,True,10.0,0,0 +SATOP21-VMCC165-A010:PRESSURE,True,1.88e-07,0,0 +SATOP21-VMFR165-A010:PRESSURE,True,3.49e-08,0,0 +SIN-TIMAST-TMA:Bunch-2-Appl-Freq-RB,True,25.0,0,0 diff --git a/test2.csv b/test2.csv new file mode 100644 index 0000000..f27927d --- /dev/null +++ b/test2.csv @@ -0,0 +1,12 @@ +# test_chans.txt / 2020-09-20 11:41:38.343128 +,connected,value,status,severity +CRAP:CRAP0,False,,-1,-1 +CRAP:CRAP1,False,,-1,-1 +CRAP:CRAP2,False,,-1,-1 +CRAP:CRAP3,False,,-1,-1 +SAROP21-OKB:MODE,True,0.0,0,0 +SATOP11-VVPG092-A010:PLC_OPEN,True,0.0,7,2 +SATOP21-OAPU161:MOTOR_W.RBV,True,10.0,0,0 +SATOP21-VMCC165-A010:PRESSURE,True,1.88e-07,0,0 +SATOP21-VMFR165-A010:PRESSURE,True,3.49e-08,0,0 +SIN-TIMAST-TMA:Bunch-2-Appl-Freq-RB,True,25.0,0,0 diff --git a/test3.csv b/test3.csv new file mode 100644 index 0000000..b4ab888 --- /dev/null +++ b/test3.csv @@ -0,0 +1,12 @@ +# test_chans.txt / 2020-09-20 11:41:38.343128 +,connected,value,status,severity +CRAP:CRAP0,False,123,-1,-1 +CRAP:CRAP1,False,,-1,-1 +CRAP:CRAP2,False,,-1,-1 +CRAP:CRAP3,False,,-1,-1 +SAROP21-OKB:MODE,True,0.0,0,0 +SATOP11-VVPG092-A010:PLC_OPEN,True,0.0,8,2 +SATOP21-OAPU161:MOTOR_W.RBV,True,10.0,0,0 +SATOP21-VMCC165-A010:PRESSURE,True,1.8e-07,0,0 +SATOP21-VMFR165-A010:PRESSURE,True,3.49e-08,0,0 +SIN-TIMAST-TMA:Bunch-2-Appl-Freq-RB,True,25.0,0,0 diff --git a/test4.csv b/test4.csv new file mode 100644 index 0000000..64288ce --- /dev/null +++ b/test4.csv @@ -0,0 +1,12 @@ +# test_chans.txt / 2020-09-20 16:26:00.235687 +,connected,value,status,severity +CRAP:CRAP0,False,,-1,-1 +CRAP:CRAP1,False,,-1,-1 +CRAP:CRAP2,False,,-1,-1 +CRAP:CRAP3,False,,-1,-1 +SAROP21-OKB:MODE,True,0.0,0,0 +SATOP11-VVPG092-A010:PLC_OPEN,True,0.0,7,2 +SATOP21-OAPU161:MOTOR_W.RBV,True,10.0,0,0 +SATOP21-VMCC165-A010:PRESSURE,True,1.85e-07,0,0 +SATOP21-VMFR165-A010:PRESSURE,True,3.51e-08,0,0 +SIN-TIMAST-TMA:Bunch-2-Appl-Freq-RB,True,25.0,0,0 diff --git a/chans.txt b/test_chans.txt similarity index 100% rename from chans.txt rename to test_chans.txt