Files
furka/Run_table_update.py
2023-07-24 11:55:55 +02:00

562 lines
27 KiB
Python

from sfdata import SFDataFiles
from sfdata import SFScanInfo
import pandas as pd
import glob
import datetime
import warnings
import io
import os
import time
import numpy as np
## You may need this !pip install lxml
warnings.filterwarnings('ignore')
def PVchs():
pv_name_dict={"SATUN:FELPHOTENE":'FEL_en (keV)',
"SATUN13-UIND030:POL-SET": 'Polarisation',
"SATOP11-OSGM087:exitslit": 'Exit_slit (um)',
"SATOP11-OSGM087:photonenergy": "Mono_en (eV)",
"SATFE10-OGAT053:transmission": "Gas_trans",
"SATFE10-PEPG046:PHOTON-ENERGY-PER-PULSE-AVG": "FEL_pulse_en (uJ)",
"SATES30-ARES:MOT_2TRY.RBV": "diode_otr_tth (deg)",
"SATES30-ARES:MOT_DRY.RBV": "diode_inr_tth (deg)",
"SATES30-ARES:MOT_SRZ.RBV": "sample_phi (deg)",
"SATES30-ARES:MOT_SRY.RBV": "sample_th (deg)",
"SATES30-ARES:MOT_SRX.RBV": "sample_chi (deg)",
"SATES30-ARES:MOT_STX.RBV": "sample_x (mm)",
"SATES30-ARES:MOT_STY.RBV": "sample_y (mm)",
"SATES30-ARES:MOT_STZ.RBV": "sample_z (mm)",
"SATES30-LS336:D_RBV": "sample_temp (K)",
"SATES30-LS336:A_RBV": "cryo_temp (K)",
"SATES30-LS336:C_RBV": "dummy_temp (K)",
"SATES30-LS336:B_RBV": "dummy_temp (K)",
"SLAAT-LGEN:DLY_OFFS1": "LXT_off1 (us)",
"SLAAT01-LTIM-PDLY:DELAY": "LXT_delay (ps)",
"SLAAT01-LTIM-PDLY:DELAY_Z_OFFS": "LXT_off1 (ps)",
"SLAAT31-LMOT-M801:MOT.RBV": "800_WP (deg)",
"SLAAT31-LMOT-M802:MOT.RBV": "comp1",
"SLAAT31-LMOT-M803:MOT.RBV": "comp2",
"SLAAT31-LMOT-M804:MOT.RBV": "comp3",
"SLAAT31-LMOT-M805:MOT.RBV": "comp4",
"SLAAT31-LMOT-M806:MOT.RBV": "THz_delay (mm)",
"SLAAT31-LMOT-M807:MOT.RBV": "WL_delay (mm)",
"SLAAT31-LMOT-M808:MOT.RBV": "800_delay (mm)",
"SATOP21-PMOS127-2D:SPECTRUM_X":"PMOS127-2D:X",
"SATOP21-PMOS127-2D:SPECTRUM_Y":"PMOS127-2D:Y",
"SLAAT31-LCAM-C801:FIT-XPOS_EGU":"Cam_801_x",
"SLAAT31-LCAM-C801:FIT-YPOS_EGU":"Cam_801_y",
"SLAAT31-LCAM-C802:FIT-XPOS_EGU":"Cam_802_x",
"SLAAT31-LCAM-C802:FIT-YPOS_EGU":"Cam_802_y",
'SLAAT31-LCAM-C801:FIT-XEGU-ARRAY':"Cam_801_xy",
'SLAAT31-LCAM-C802:FIT-XEGU-ARRAY':"Cam_802_xy",
"SIN-LPOSIC1:POS1":"PICO X not sure",
"SIN-LPOSIC1:POS2":"PICO Y not sure",
"SATES30-CAMS182-GIGE2_sp1:x_center_of_mass":"",
"SATES30-CAMS182-GIGE2_sp1:y_center_of_mass":"",
}
return pv_name_dict
def colors():
dict_colors = {
'yellow_light' : ('#BF8F00', '2px solid #BF8F00', '#FFF2CC', '#FFFFFF'),
'grey_light' : ('#808080', '2px solid #808080', '#EDEDED', '#FFFFFF'),
'blue_light' : ('#305496', '2px solid #305496', '#D9E1F2', '#FFFFFF'),
'orange_light' : ('#C65911', '2px solid #C65911', '#FCE4D6', '#FFFFFF'),
'green_light' : ('#548235', '2px solid #548235', '#E2EFDA', '#FFFFFF'),
'red_light' : ('#823535', '2px solid #823535', '#efdada', '#FFFFFF'),
'yellow_dark' : ('#FFFFFF', '2px solid #BF8F00', '#FFF2CC', '#BF8F00'),
'grey_dark' : ('#FFFFFF', '2px solid #808080', '#EDEDED', '#808080'),
'blue_dark': ('#FFFFFF', '2px solid #305496', '#D9E1F2', '#305496'),
'blue_yellow': ('#FFFFFF', '2px solid #305496', '#ecedbe', '#305496'),
'orange_dark' : ('#FFFFFF', '2px solid #C65911', '#FCE4D6', '#C65911'),
'green_dark' : ('#FFFFFF', '2px solid #548235', '#E2EFDA', '#548235'),
'red_dark' : ('#FFFFFF', '2px solid #823535', '#efdada', '#823535')
}
return (dict_colors)
def build_table_nocondition(
df,
color,
font_size='12px',
font_family='Century Gothic, sans-serif',
text_align='center',
width='600px',
index=False,
even_color='black',
even_bg_color='white',
odd_bg_color=None,
border_bottom_color=None,
escape=True,
width_dict=[],
padding="10px 10px 3px 3px",
float_format=None):
dict_colors = colors()
if df.empty:
return ''
# Set color
color, border_bottom, odd_background_color, header_background_color = dict_colors[color]
if odd_bg_color:
odd_background_color = odd_bg_color
if border_bottom_color:
border_bottom = border_bottom_color
a = 0
while a != len(df):
if a == 0:
df_html_output = df.iloc[[a]].to_html(
na_rep="",
index=index,
border=0,
escape=escape,
float_format=float_format,
)
# change format of header
if index:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">', len(df.columns)+1)
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
else:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = """<p>""" + format(df_html_output)
a = 1
elif a % 2 == 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
elif a % 2 != 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
body = body + """</p>"""
body = body.replace("""</td>
</tr>
</tbody>
</table>
<table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""").replace("""</td>
</tr>
</tbody>
</table><table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""")
if len(width_dict) == len(df.columns):
width_body = ''
w = 0
for line in io.StringIO(body):
line = line.replace("\n", "")
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
return width_body[:len(width_body)-1].replace("'", "")
else:
return body.replace(r"\n'", "")
def build_table(
df,
color,
font_size='12px',
font_family='Century Gothic, sans-serif',
text_align='center',
width='600px',
index=False,
even_color='black',
even_bg_color='white',
odd_bg_color=None,
border_bottom_color=None,
escape=True,
width_dict=[],
padding="10px 10px 3px 3px",
float_format=None,
conditions={}):
dict_colors = colors()
if df.empty:
return ''
# Set color
color, border_bottom, odd_background_color, header_background_color = dict_colors[color]
if odd_bg_color:
odd_background_color = odd_bg_color
if border_bottom_color:
border_bottom = border_bottom_color
a = 0
while a != len(df):
if a == 0:
df_html_output = df.iloc[[a]].to_html(
na_rep="",
index=index,
border=0,
escape=escape,
float_format=float_format,
)
# change format of header
if index:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">', len(df.columns)+1)
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
else:
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + header_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';color: ' + color
+ ';text-align: ' + text_align
+ ';border-bottom: ' + border_bottom
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = """<p>""" + format(df_html_output)
a = 1
elif a % 2 == 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + odd_background_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
elif a % 2 != 0:
df_html_output = df.iloc[[a]].to_html(na_rep = "", index = index, header = False, escape=escape)
# change format of index
df_html_output = df_html_output.replace('<th>'
,'<th style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
#change format of table
df_html_output = df_html_output.replace('<td>'
,'<td style = "background-color: ' + even_bg_color
+ '; color: ' + even_color
+ ';font-family: ' + font_family
+ ';font-size: ' + str(font_size)
+ ';text-align: ' + text_align
+ ';padding: ' + padding
+ ';width: ' + str(width) + '">')
body = body + format(df_html_output)
a += 1
body = body + """</p>"""
body = body.replace("""</td>
</tr>
</tbody>
</table>
<table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""").replace("""</td>
</tr>
</tbody>
</table><table border="1" class="dataframe">
<tbody>
<tr>""","""</td>
</tr>
<tr>""")
if conditions:
for k in conditions.keys():
try:
conditions[k]['index'] = list(df.columns).index(k)
width_body = ''
w = 0
for line in io.StringIO(body):
updated_body = False
if w == conditions[k]['index']:
try:
if float(repr(line).split('>')[1].split('<')[0]) < conditions[k]['min']:
if 'color: black' in repr(line):
width_body = width_body + repr(line).replace("color: black", 'color: ' + conditions[k]['min_color'])[1:]
elif 'color: white' in repr(line):
width_body = width_body + repr(line).replace("color: white", 'color: ' + conditions[k]['min_color'])[1:]
else:
width_body = width_body + repr(line).replace('">', '; color: ' + conditions[k]['min_color'] + '">')[1:]
updated_body = True
elif float(repr(line).split('>')[1].split('<')[0]) > conditions[k]['max']:
if 'color: black' in repr(line):
width_body = width_body + repr(line).replace("color: black", 'color: ' + conditions[k]['max_color'])[1:]
elif 'color: white' in repr(line):
width_body = width_body + repr(line).replace("color: white", 'color: ' + conditions[k]['max_color'])[1:]
else:
width_body = width_body + repr(line).replace('">', '; color: ' + conditions[k]['max_color'] + '">')[1:]
updated_body = True
except:
pass
if not updated_body:
width_body = width_body + repr(line)[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th":
if w == len(df.columns) -1:
w = 0
else:
w += 1
body = width_body[:len(width_body)-1]
except:
pass
if len(width_dict) == len(df.columns):
width_body = ''
w = 0
if conditions:
for line in body.split(r"\n'"):
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
else:
for line in io.StringIO(body):
line = line.replace("\n", "")
width_body = width_body + repr(line).replace("width: auto", 'width: ' + width_dict[w])[1:]
if str(repr(line))[:10] == "' <td" or str(repr(line))[:10] == "' <th" :
if w == len(df.columns) -1:
w = 0
else:
w += 1
return width_body[:len(width_body)-1].replace("'", "")
else:
return body.replace(r"\n'", "")
def get_filename(run,dir_raw):
file_list=glob.glob(dir_raw+f'run{str(run).zfill(4)}*')
if len(file_list)!=1:
print(f'No/duplicate file name for {run}')
return False
else:
filename=file_list[0]
return filename
def get_latest_run(dir_raw):
with open(dir_raw+'run_info/LAST_RUN') as f:
return int(f.readlines()[0])
def get_scan_info_dict(dir_raw,run):
scan_SFSI=SFScanInfo(get_filename(run,dir_raw)+"/meta/scan.json")
scan_SFDF=SFDataFiles(get_filename(run,dir_raw)+"/data/*.PVDATA.h5")
pv_name_dict = PVchs()
pv_info_dict = {k+' '+pv_name_dict[k]: np.round(v.data.flat[0],6) if isinstance(v.data[0], float) else v.data.flat[0] for k,v in scan_SFDF.items()}
scan_info_dict={'Run':run,
'date':scan_SFSI.fs.mtime.strftime("%d/%m/%Y"),
'time':scan_SFSI.fs.mtime.strftime("%H:%M:%S"),
'scan_name':scan_SFSI.parameters['scan_name'],
'scanned':scan_SFSI.parameters['name'][0],
'start':scan_SFSI.readbacks[0],
'end':scan_SFSI.readbacks[-1]}
print('Updating run',run)
if len(scan_SFSI.readbacks)!=1:
try:
scan_info_dict['step']=np.abs(scan_SFSI.readbacks[1]-scan_SFSI.readbacks[0])
except TypeError:
scan_info_dict['step']=np.nan
elif len(scan_SFSI.readbacks)==1:
scan_info_dict['step']=0
scan_info_dict['steps']=len(scan_SFSI.readbacks)
scan_info_dict['shots']=len(scan_SFSI[0].all_pids)
scan_info_dict['scanned_ID']=scan_SFSI.parameters['Id'][0]
try:
scan_info_dict['units']=scan_SFSI.parameters['units'][0]
except KeyError:
scan_info_dict['units']='au'
scan_info_dict.update(pv_info_dict)
scanned_id=scan_info_dict['scanned_ID']
if scanned_id=='ATHOS_Mon_Und':
scan_info_dict['SATUN:FELPHOTENE FEL_en (keV)']='scanned'
scan_info_dict['SATOP11-OSGM087:photonenergy Mono_en (eV)']='scanned'
elif scanned_id=='SLAAT01-LTIM-PDLY:DELAY':
scan_info_dict["SLAAT01-LTIM-PDLY:DELAY LXT_delay (ps)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STZ.VAL':
scan_info_dict["SATES30-ARES:MOT_STZ.RBV sample_z (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STY.VAL':
scan_info_dict["SATES30-ARES:MOT_STY.RBV sample_y (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_STX.VAL':
scan_info_dict["SATES30-ARES:MOT_STX.RBV sample_x (mm)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRX.VAL':
scan_info_dict["SATES30-ARES:MOT_SRX.RBV sample_chi (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRY.VAL':
scan_info_dict["SATES30-ARES:MOT_SRY.RBV sample_th (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_SRZ.VAL':
scan_info_dict["SATES30-ARES:MOT_SRZ.RBV sample_phi (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_DRY.VAL':
scan_info_dict["SATES30-ARES:MOT_DRY.RBV diode_inr_tth (deg)"]='scanned'
elif scanned_id=='SATES30-ARES:MOT_2TRY.VAL':
scan_info_dict["SATES30-ARES:MOT_2TRY.RBV diode_otr_tth (deg)"]='scanned'
else:
matching_keys=[k for k in pv_name_dict.keys() if k.startswith(scanned_id)]
if len(matching_keys)==1:
scan_info_dict[matching_keys[0]+' '+pv_name_dict[matching_keys[0]]]='scanned'
else:
xyz=2
# print(f'Channel naming inconsistency in {run}')
# print(scanned_id)
# print(matching_keys)
return scan_info_dict
def make_run_table(dir_raw,path,runs,htmlfilename,color_scheme='blue_yellow'):
if os.path.exists(path+htmlfilename+'.html'):
info_df = pd.read_html(path+htmlfilename+'.html', header=0, index_col=None)[0]
else:
info_df = pd.DataFrame(columns=['Run'])
for run in runs:
if run not in info_df['Run'].values:
if get_filename(run,dir_raw) is not False:
new_row = pd.DataFrame([get_scan_info_dict(dir_raw,run)])
info_df = pd.concat([info_df, new_row], ignore_index=True, sort=False)
info_df = info_df.sort_values('Run')
# conditions={'SATES30-LS336:C_RBV sample_temp (K)': {'min': 40,'max': 60, 'min_color': 'red','max_color': 'green'},
# }
# html_table = build_table(info_df, color_scheme,conditions=conditions)
html_table = build_table_nocondition(info_df, color_scheme)
with open(path+htmlfilename+'.html', 'w') as f:
f.write(path+html_table)