May updates

This commit is contained in:
gac-furka
2023-05-10 15:05:22 +02:00
parent b037315ac8
commit f1887e5f22
7 changed files with 968 additions and 89 deletions

573
Run_table_update.py Normal file
View File

@ -0,0 +1,573 @@
from sfdata import SFDataFiles
from sfdata import SFScanInfo
import pandas as pd
import glob
import datetime
import warnings
import io
import os
import time
import numpy as np
## You may need this !pip install lxml
warnings.filterwarnings('ignore')
def PVchs():
pv_name_dict={"SATUN:FELPHOTENE":'FEL_en (keV)',
"SATUN13-UIND030:POL-SET": 'Polarisation',
"SATOP11-OSGM087:exitslit": 'Exit_slit (um)',
"SATOP11-OSGM087:photonenergy": "Mono_en (eV)",
"SATFE10-OGAT053:transmission": "Gas_trans",
"SATFE10-PEPG046:PHOTON-ENERGY-PER-PULSE-AVG": "FEL_pulse_en (uJ)",
"SATES30-ARES:MOT_2TRY.RBV": "diode_otr_tth (deg)",
"SATES30-ARES:MOT_DRY.RBV": "diode_inr_tth (deg)",
"SATES30-ARES:MOT_SRZ.RBV": "sample_chi (deg)",
"SATES30-ARES:MOT_SRY.RBV": "sample_th (deg)",
"SATES30-ARES:MOT_SRX.RBV": "sample_phi (deg)",
"SATES30-ARES:MOT_STX.RBV": "sample_x (mm)",
"SATES30-ARES:MOT_STY.RBV": "sample_y (mm)",
"SATES30-ARES:MOT_STZ.RBV": "sample_z (mm)",
"SATES30-LS336:C_RBV": "sample_temp (K)",
"SATES30-LS336:A_RBV": "cryo_temp (K)",
"SLAAT-LGEN:DLY_OFFS1": "LXT_off1 (us)",
"SLAAT01-LTIM-PDLY:DELAY": "LXT_delay (ps)",
"SLAAT01-LTIM-PDLY:DELAY_Z_OFFS": "LXT_off1 (ps)",
"SLAAT31-LMOT-M801:MOT.RBV": "800_WP (deg)",
"SLAAT31-LMOT-M802:MOT.RBV": "comp1",
"SLAAT31-LMOT-M803:MOT.RBV": "comp2",
"SLAAT31-LMOT-M804:MOT.RBV": "comp3",
"SLAAT31-LMOT-M805:MOT.RBV": "comp4",
"SLAAT31-LMOT-M806:MOT.RBV": "EOS_delay (mm)",
"SLAAT31-LMOT-M807:MOT.RBV": "WL_delay (mm)",
"SLAAT31-LMOT-M808:MOT.RBV": "800_delay (mm)",
}
return pv_name_dict
def colors():
dict_colors = {
'yellow_light' : ('#BF8F00', '2px solid #BF8F00', '#FFF2CC', '#FFFFFF'),
'grey_light' : ('#808080', '2px solid #808080', '#EDEDED', '#FFFFFF'),
'blue_light' : ('#305496', '2px solid #305496', '#D9E1F2', '#FFFFFF'),
'orange_light' : ('#C65911', '2px solid #C65911', '#FCE4D6', '#FFFFFF'),
'green_light' : ('#548235', '2px solid #548235', '#E2EFDA', '#FFFFFF'),
'red_light' : ('#823535', '2px solid #823535', '#efdada', '#FFFFFF'),
'yellow_dark' : ('#FFFFFF', '2px solid #BF8F00', '#FFF2CC', '#BF8F00'),
'grey_dark' : ('#FFFFFF', '2px solid #808080', '#EDEDED', '#808080'),
'blue_dark': ('#FFFFFF', '2px solid #305496', '#D9E1F2', '#305496'),
'blue_yellow': ('#FFFFFF', '2px solid #305496', '#ecedbe', '#305496'),
'orange_dark' : ('#FFFFFF', '2px solid #C65911', '#FCE4D6', '#C65911'),
'green_dark' : ('#FFFFFF', '2px solid #548235', '#E2EFDA', '#548235'),
'red_dark' : ('#FFFFFF', '2px solid #823535', '#efdada', '#823535')
}
return (dict_colors)
def build_table_nocondition(
df,
color,
font_size='12px',
font_family='Century Gothic, sans-serif',
text_align='center',
width='600px',
index=False,
even_color='black',
even_bg_color='white',
odd_bg_color=None,
border_bottom_color=None,
escape=True,
width_dict=[],
padding="10px 10px 3px 3px",
float_format=None):
dict_colors = colors()
if df.empty:
return ''
# Set color
color, border_bottom, odd_background_color, header_background_color = dict_colors[color]
if odd_bg_color:
odd_background_color = odd_bg_color
if border_bottom_color:
border_bottom = border_bottom_color
a = 0
while a != len(df):
if a == 0:
df_html_output = df.iloc[[a]].to_html(
na_rep="",
index=index,
border=0,
escape=escape,
float_format=float_format,
)
# change format of header
if index:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">', len(df.columns)+1)
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
else:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = """<p>""" + format(df_html_output)
a = 1
elif a % 2 == 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
elif a % 2 != 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
body = body + """</p>"""
body = body.replace("""</td>
</tr>
</tbody>
</table>
<table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""").replace("""</td>
</tr>
</tbody>
</table><table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""")
if len(width_dict) == len(df.columns):
width_body = ''
w = 0
for line in io.StringIO(body):
line = line.replace("\n", "")
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
return width_body[:len(width_body)-1].replace("'", "")
else:
return body.replace(r"\n'", "")
def build_table(
df,
color,
font_size='12px',
font_family='Century Gothic, sans-serif',
text_align='center',
width='600px',
index=False,
even_color='black',
even_bg_color='white',
odd_bg_color=None,
border_bottom_color=None,
escape=True,
width_dict=[],
padding="10px 10px 3px 3px",
float_format=None,
conditions={}):
dict_colors = colors()
if df.empty:
return ''
# Set color
color, border_bottom, odd_background_color, header_background_color = dict_colors[color]
if odd_bg_color:
odd_background_color = odd_bg_color
if border_bottom_color:
border_bottom = border_bottom_color
a = 0
while a != len(df):
if a == 0:
df_html_output = df.iloc[[a]].to_html(
na_rep="",
index=index,
border=0,
escape=escape,
float_format=float_format,
)
# change format of header
if index:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">', len(df.columns)+1)
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
else:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = """<p>""" + format(df_html_output)
a = 1
elif a % 2 == 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
elif a % 2 != 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
body = body + """</p>"""
body = body.replace("""</td>
</tr>
</tbody>
</table>
<table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""").replace("""</td>
</tr>
</tbody>
</table><table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""")
if conditions:
for k in conditions.keys():
try:
conditions[k]['index'] = list(df.columns).index(k)
width_body = ''
w = 0
for line in io.StringIO(body):
updated_body = False
if w == conditions[k]['index']:
try:
if float(repr(line).split('>')[1].split('<')[0]) < conditions[k]['min']:
if 'color: black' in repr(line):
width_body = width_body + repr(line).replace("color: black", 'color: ' + conditions[k]['min_color'])[1:]
elif 'color: white' in repr(line):
width_body = width_body + repr(line).replace("color: white", 'color: ' + conditions[k]['min_color'])[1:]
else:
width_body = width_body + repr(line).replace('">', '; color: ' + conditions[k]['min_color'] + '">')[1:]
updated_body = True
elif float(repr(line).split('>')[1].split('<')[0]) > conditions[k]['max']:
if 'color: black' in repr(line):
width_body = width_body + repr(line).replace("color: black", 'color: ' + conditions[k]['max_color'])[1:]
elif 'color: white' in repr(line):
width_body = width_body + repr(line).replace("color: white", 'color: ' + conditions[k]['max_color'])[1:]
else:
width_body = width_body + repr(line).replace('">', '; color: ' + conditions[k]['max_color'] + '">')[1:]
updated_body = True
except:
pass
if not updated_body:
width_body = width_body + repr(line)[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th":
if w == len(df.columns) -1:
w = 0
else:
w += 1
body = width_body[:len(width_body)-1]
except:
pass
if len(width_dict) == len(df.columns):
width_body = ''
w = 0
if conditions:
for line in body.split(r"\n'"):
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
else:
for line in io.StringIO(body):
line = line.replace("\n", "")
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
return width_body[:len(width_body)-1].replace("'", "")
else:
return body.replace(r"\n'", "")
def get_filename(run,dir_raw):
file_list=glob.glob(dir_raw+f'run{str(run).zfill(4)}*')
if len(file_list)!=1:
print(f'No/duplicate file name for {run}')
return False
else:
filename=file_list[0]
return filename
def get_latest_run(dir_raw):
with open(dir_raw+'run_info/LAST_RUN') as f:
return int(f.readlines()[0])
def get_scan_info_dict(dir_raw,run):
scan_SFSI=SFScanInfo(get_filename(run,dir_raw)+"/meta/scan.json")
scan_SFDF=SFDataFiles(get_filename(run,dir_raw)+"/data/*.PVDATA.h5")
pv_name_dict = PVchs()
pv_info_dict = {k+' '+pv_name_dict[k]: np.round(v.data[0],6) if isinstance(v.data[0], float) else v.data[0] for k,v in scan_SFDF.items()}
scan_info_dict={'Run':run,
'date':scan_SFSI.fs.atime.strftime("%d/%m/%Y"),
'time':scan_SFSI.fs.atime.strftime("%H:%M:%S"),
'duration':f'{str(scan_SFSI.fs.mtime-scan_SFSI.fs.atime)[:-7:]}',
'scan_name':scan_SFSI.parameters['scan_name'],
'scanned':scan_SFSI.parameters['name'][0],
'start':scan_SFSI.readbacks[0],
'end':scan_SFSI.readbacks[-1]}
if len(scan_SFSI.readbacks)!=1:
scan_info_dict['step']=np.abs(np.diff(scan_SFSI.readbacks)[0])
elif len(scan_SFSI.readbacks)==1:
scan_info_dict['step']=0
scan_info_dict['steps']=len(scan_SFSI.readbacks)
scan_info_dict['shots']=len(scan_SFSI[0].all_pids)
scan_info_dict['scanned_ID']=scan_SFSI.parameters['Id'][0]
try:
scan_info_dict['units']=scan_SFSI.parameters['units'][0]
except KeyError:
scan_info_dict['units']='au'
scan_info_dict.update(pv_info_dict)
scanned_id=scan_info_dict['scanned_ID']
if scanned_id=='ATHOS_Mon_Und':
scan_info_dict['SATUN:FELPHOTENE FEL_en (keV)']='scanned'
scan_info_dict['SATOP11-OSGM087:photonenergy Mono_en (eV)']='scanned'
elif scanned_id=='SLAAT01-LTIM-PDLY:DELAY':
scan_info_dict["SLAAT01-LTIM-PDLY:DELAY LXT_delay (ps)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STZ.VAL':
scan_info_dict["SATES30-ARES:MOT_STZ.RBV sample_z (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STY.VAL':
scan_info_dict["SATES30-ARES:MOT_STY.RBV sample_y (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STX.VAL':
scan_info_dict["SATES30-ARES:MOT_STX.RBV sample_x (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRX.VAL':
scan_info_dict["SATES30-ARES:MOT_SRX.RBV sample_phi (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRY.VAL':
scan_info_dict["SATES30-ARES:MOT_SRY.RBV sample_th (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRZ.VAL':
scan_info_dict["SATES30-ARES:MOT_SRZ.RBV sample_chi (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_DRY.VAL':
scan_info_dict["SATES30-ARES:MOT_DRY.RBV diode_inr_tth (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_2TRY.VAL':
scan_info_dict["SATES30-ARES:MOT_2TRY.RBV diode_otr_tth (deg)"]='scanned'
else:
matching_keys=[k for k in pv_name_dict.keys() if k.startswith(scanned_id)]
if len(matching_keys)==1:
scan_info_dict[matching_keys[0]+' '+pv_name_dict[matching_keys[0]]]='scanned'
else:
xyz=2
# print(f'Channel naming inconsistency in {run}')
# print(scanned_id)
# print(matching_keys)
return scan_info_dict
def make_run_table(dir_raw,path,runs,htmlfilename='Few_runs_run_table',color_scheme='blue_yellow'):
if os.path.exists(path+htmlfilename+'.html'):
info_df = pd.read_html(path+htmlfilename+'.html', header=0, index_col=None)[0]
else:
info_df = pd.DataFrame(columns=['Run'])
for run in runs:
if run not in info_df['Run'].values:
if get_filename(run,dir_raw) is not False:
new_row = pd.DataFrame([get_scan_info_dict(dir_raw,run)])
info_df = pd.concat([info_df, new_row], ignore_index=True, sort=False)
info_df = info_df.sort_values('Run')
# conditions={'SATES30-LS336:C_RBV sample_temp (K)': {'min': 40,'max': 60, 'min_color': 'red','max_color': 'green'},
# }
# html_table = build_table(info_df, color_scheme,conditions=conditions)
html_table = build_table_nocondition(info_df, color_scheme)
with open(path+htmlfilename+'.html', 'w') as f:
f.write(path+html_table)
def make_run_table_live(dir_raw,path,check_interval=5,keep_running_for=4,htmlfilename='updating_run_table',color_scheme='blue_yellow'):
run_last = 0
time_last_change = time.time()
while True:
if run_last != get_latest_run(dir_raw):
run_last = get_latest_run(dir_raw)
time_last_change = time.time()
if os.path.exists(path+htmlfilename+'.html'):
info_df=pd.read_html(path+htmlfilename+'.html',header=0, index_col=None)[0]
else:
info_df=pd.DataFrame(columns=['Run'])
for run in range(run_last + 1):
if run in info_df['Run'].values:
continue
filename = get_filename(run, dir_raw)
if filename == False:
continue
new_row = pd.DataFrame([get_scan_info_dict(dir_raw,run)])
info_df = pd.concat([info_df, new_row], ignore_index=True, sort=False)
info_df = info_df.sort_values('Run')
# html_table = build_table(info_df, color_scheme)
html_table = build_table_nocondition(info_df, color_scheme)
with open(path+htmlfilename+'.html', 'w') as f:
f.write(html_table)
if time.time() - time_last_change > keep_running_for*60*60:
print('Not running any more.')
break
print('Waiting to check.')
time.sleep(check_interval*60)