mirror of
https://github.com/paulscherrerinstitute/sf_daq_broker.git
synced 2026-08-10 01:00:30 +02:00
double quotes
This commit is contained in:
NichtJens
committed by
Auf keinen Fall Jens
Auf keinen Fall Jens
parent
61b5e2947e
commit
2f6c2b78e5
+42
-42
@@ -9,7 +9,7 @@ import numpy as np
|
||||
|
||||
def run():
|
||||
|
||||
parser = argparse.ArgumentParser(description='check consistency of produced files')
|
||||
parser = argparse.ArgumentParser(description="check consistency of produced files")
|
||||
|
||||
parser.add_argument("-r", "--run_file", help="JSON file from the retrieve process", default=None)
|
||||
parser.add_argument("--frequency_reduction_factor", help="beam rate, default 1 means 100Hz (2: 50Hz, 4: 25Hz....) (overwrites one from json file)", default=0, type=int)
|
||||
@@ -36,7 +36,7 @@ def check_consistency(run_file=None, rate_multiplicator=0):
|
||||
return {"check" : False, "reason" : problems}
|
||||
|
||||
if not os.path.exists(run_file):
|
||||
problems.append(f'{run_file} does not exist')
|
||||
problems.append(f"{run_file} does not exist")
|
||||
return {"check" : False, "reason" : problems}
|
||||
|
||||
try:
|
||||
@@ -58,9 +58,9 @@ def check_consistency(run_file=None, rate_multiplicator=0):
|
||||
pgroup = parameters["pgroup"]
|
||||
beamline = parameters["beamline"]
|
||||
run_number = parameters["run_number"]
|
||||
request_time = datetime.datetime.strptime(parameters["request_time"], '%Y-%m-%d %H:%M:%S.%f')
|
||||
request_time = datetime.datetime.strptime(parameters["request_time"], "%Y-%m-%d %H:%M:%S.%f")
|
||||
|
||||
full_directory = f'/sf/{beamline}/data/{pgroup}/raw/'
|
||||
full_directory = f"/sf/{beamline}/data/{pgroup}/raw/"
|
||||
if "directory_name" in parameters:
|
||||
full_directory = f'{full_directory}{parameters["directory_name"]}'
|
||||
|
||||
@@ -73,23 +73,23 @@ def check_consistency(run_file=None, rate_multiplicator=0):
|
||||
expected_number_measurements = len(expected_pulse_id)
|
||||
|
||||
if "channels_list" in parameters:
|
||||
bsread_file = f'{full_directory}/run_{run_number:06}.BSREAD.h5'
|
||||
bsread_file = f"{full_directory}/run_{run_number:06}.BSREAD.h5"
|
||||
if not os.path.exists(bsread_file):
|
||||
problems.append(f'bsread file {bsread_file} does not exist')
|
||||
problems.append(f"bsread file {bsread_file} does not exist")
|
||||
else:
|
||||
try:
|
||||
bsread_h5py = h5py.File(bsread_file,"r")
|
||||
inside_file = list(bsread_h5py.keys())
|
||||
if 'data' not in inside_file:
|
||||
problems.append(f'BSREAD file {bsread_file} has bad content {inside_file}')
|
||||
if "data" not in inside_file:
|
||||
problems.append(f"BSREAD file {bsread_file} has bad content {inside_file}")
|
||||
else:
|
||||
channels_inside_file = list(bsread_h5py['data'].keys())
|
||||
channels_inside_file = list(bsread_h5py["data"].keys())
|
||||
for channel in parameters["channels_list"]:
|
||||
if channel not in channels_inside_file:
|
||||
problems.append(f'channel {channel} requested but not present in cameras file')
|
||||
problems.append(f"channel {channel} requested but not present in cameras file")
|
||||
else:
|
||||
pulse_id_raw = bsread_h5py[f'/data/{channel}/pulse_id'][:]
|
||||
is_data_present = bsread_h5py[f'/data/{channel}/is_data_present'][:]
|
||||
pulse_id_raw = bsread_h5py[f"/data/{channel}/pulse_id"][:]
|
||||
is_data_present = bsread_h5py[f"/data/{channel}/is_data_present"][:]
|
||||
# pulse_id = pulse_id_raw[is_data_present]
|
||||
pulse_id = []
|
||||
for n_p,p in enumerate(pulse_id_raw):
|
||||
@@ -97,92 +97,92 @@ def check_consistency(run_file=None, rate_multiplicator=0):
|
||||
pulse_id.append(p)
|
||||
n_pulse_id = len(pulse_id)
|
||||
if n_pulse_id != expected_number_measurements:
|
||||
problems.append(f'{channel} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}')
|
||||
problems.append(f"{channel} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}")
|
||||
else:
|
||||
if pulse_id[0] != expected_pulse_id[0] or pulse_id[-1] != expected_pulse_id[-1]:
|
||||
problems.append(f'{channel} start/stop pulse_id are not the one which are requested (requested : {expected_pulse_id[0]},{expected_pulse_id[-1]}, got: {pulse_id[0]},{pulse_id[-1]}) ')
|
||||
problems.append(f"{channel} start/stop pulse_id are not the one which are requested (requested : {expected_pulse_id[0]},{expected_pulse_id[-1]}, got: {pulse_id[0]},{pulse_id[-1]}) ")
|
||||
pulse_id_check = True # this is for 100Hz only, todo: to make for different rate
|
||||
for i in range(n_pulse_id):
|
||||
if pulse_id[i] != expected_pulse_id[i]:
|
||||
pulse_id_check = False
|
||||
#print(channel, i, pulse_id[i], expected_pulse_id[i])
|
||||
if not pulse_id_check:
|
||||
problems.append(f'{channel} pulse_id are not monotonic')
|
||||
problems.append(f"{channel} pulse_id are not monotonic")
|
||||
bsread_h5py.close()
|
||||
except:
|
||||
problems.append(f'Can not read from BSREAD file {bsread_file} may be too early')
|
||||
problems.append(f"Can not read from BSREAD file {bsread_file} may be too early")
|
||||
|
||||
|
||||
if "camera_list" in parameters:
|
||||
cameras_file = f'{full_directory}/run_{run_number:06}.CAMERAS.h5'
|
||||
cameras_file = f"{full_directory}/run_{run_number:06}.CAMERAS.h5"
|
||||
if not os.path.exists(cameras_file):
|
||||
problems.append(f'camera file {cameras_file} does not exist')
|
||||
problems.append(f"camera file {cameras_file} does not exist")
|
||||
else:
|
||||
try:
|
||||
cameras_h5py = h5py.File(cameras_file,"r")
|
||||
cameras_inside_file = list(cameras_h5py.keys())
|
||||
for camera in parameters["camera_list"]:
|
||||
if camera not in cameras_inside_file:
|
||||
problems.append(f'camera {camera} requested but not present in cameras file')
|
||||
problems.append(f"camera {camera} requested but not present in cameras file")
|
||||
else:
|
||||
pulse_id = cameras_h5py[f'/{camera}/pulse_id'][:]
|
||||
pulse_id = cameras_h5py[f"/{camera}/pulse_id"][:]
|
||||
n_pulse_id = len(pulse_id)
|
||||
if n_pulse_id != expected_number_measurements:
|
||||
problems.append(f'{camera} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}')
|
||||
problems.append(f"{camera} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}")
|
||||
else:
|
||||
if expected_pulse_id[0] != pulse_id[0] or expected_pulse_id[-1] != pulse_id[-1]:
|
||||
problems.append(f'{camera} start/stop pulse_id are not the one which are requested')
|
||||
problems.append(f"{camera} start/stop pulse_id are not the one which are requested")
|
||||
pulse_id_check = True # this is for 100Hz only, todo: to make for different rate
|
||||
for i in range(n_pulse_id):
|
||||
if pulse_id[i] != expected_pulse_id[i]:
|
||||
pulse_id_check = False
|
||||
if not pulse_id_check:
|
||||
problems.append(f'{camera} pulse_id are not monotonic')
|
||||
problems.append(f"{camera} pulse_id are not monotonic")
|
||||
n_images_corrupted = 0
|
||||
image_data = cameras_h5py[f'/{camera}/data']
|
||||
image_data = cameras_h5py[f"/{camera}/data"]
|
||||
for i_image in range(n_pulse_id):
|
||||
try:
|
||||
image_try = image_data[i_image]
|
||||
except:
|
||||
n_images_corrupted += 1
|
||||
if n_images_corrupted != 0:
|
||||
problems.append(f'{camera} {n_images_corrupted} images (from {n_pulse_id}) corrupted, can not read them')
|
||||
problems.append(f"{camera} {n_images_corrupted} images (from {n_pulse_id}) corrupted, can not read them")
|
||||
cameras_h5py.close()
|
||||
except:
|
||||
problems.append(f'Can not read from cameras file {cameras_file} may be too early')
|
||||
problems.append(f"Can not read from cameras file {cameras_file} may be too early")
|
||||
|
||||
if "detectors" in parameters:
|
||||
for detector in parameters["detectors"]:
|
||||
|
||||
detector_file = f'{full_directory}/run_{run_number:06}.{detector}.h5'
|
||||
detector_file = f"{full_directory}/run_{run_number:06}.{detector}.h5"
|
||||
if not os.path.exists(detector_file):
|
||||
problems.append(f'detector file {detector_file} does not exist')
|
||||
problems.append(f"detector file {detector_file} does not exist")
|
||||
else:
|
||||
try:
|
||||
detector_h5py = h5py.File(detector_file,"r")
|
||||
pulse_id = detector_h5py[f'/data/{detector}/pulse_id'][:]
|
||||
pulse_id = detector_h5py[f"/data/{detector}/pulse_id"][:]
|
||||
n_pulse_id = len(pulse_id)
|
||||
# in case of converted data, frame_index, is_good_frame and daq_rec may be missing
|
||||
if f'data/{detector}/frame_index' in detector_h5py.keys():
|
||||
frame_index = detector_h5py[f'data/{detector}/frame_index'][:]
|
||||
if f"data/{detector}/frame_index" in detector_h5py.keys():
|
||||
frame_index = detector_h5py[f"data/{detector}/frame_index"][:]
|
||||
else:
|
||||
frame_index = [0] * n_pulse_id
|
||||
if f'/data/{detector}/is_good_frame' in detector_h5py.keys():
|
||||
is_good_frame = detector_h5py[f'/data/{detector}/is_good_frame'][:]
|
||||
if f"/data/{detector}/is_good_frame" in detector_h5py.keys():
|
||||
is_good_frame = detector_h5py[f"/data/{detector}/is_good_frame"][:]
|
||||
else:
|
||||
is_good_frame = [1] * n_pulse_id
|
||||
if f'/data/{detector}/daq_rec' in detector_h5py.keys():
|
||||
daq_rec = detector_h5py[f'/data/{detector}/daq_rec'][:]
|
||||
if f"/data/{detector}/daq_rec" in detector_h5py.keys():
|
||||
daq_rec = detector_h5py[f"/data/{detector}/daq_rec"][:]
|
||||
else:
|
||||
daq_rec = [0] * n_pulse_id
|
||||
|
||||
if len(frame_index) != n_pulse_id or len(is_good_frame) != n_pulse_id or len(daq_rec) != n_pulse_id:
|
||||
problems.append(f'{detector} length of frame_index,is_good_frame,daq_rec is not consistent with pulse_id')
|
||||
problems.append(f"{detector} length of frame_index,is_good_frame,daq_rec is not consistent with pulse_id")
|
||||
if n_pulse_id != expected_number_measurements:
|
||||
problems.append(f'{detector} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}')
|
||||
problems.append(f"{detector} number of pulse_id is different from expected : {n_pulse_id} vs {expected_number_measurements}")
|
||||
else:
|
||||
if expected_pulse_id[0] != pulse_id[0] or expected_pulse_id[-1] != pulse_id[-1]:
|
||||
problems.append(f'{detector} start/stop pulse_id are not the one which are requested')
|
||||
problems.append(f"{detector} start/stop pulse_id are not the one which are requested")
|
||||
# todo: check on nan's for pulse_id's
|
||||
frame_index_check = True
|
||||
n_frames_bad = 0
|
||||
@@ -196,14 +196,14 @@ def check_consistency(run_file=None, rate_multiplicator=0):
|
||||
if pulse_id[i] != expected_pulse_id[i]:
|
||||
pulse_id_check = False
|
||||
if not frame_index_check:
|
||||
problems.append(f'{detector} frame_index is not monotonic')
|
||||
problems.append(f"{detector} frame_index is not monotonic")
|
||||
if n_frames_bad != 0:
|
||||
problems.append(f'{detector} there are bad frames : {n_frames_bad} out of {n_pulse_id}')
|
||||
problems.append(f"{detector} there are bad frames : {n_frames_bad} out of {n_pulse_id}")
|
||||
if not pulse_id_check:
|
||||
problems.append(f'{detector} pulse_id are not monotonic')
|
||||
problems.append(f"{detector} pulse_id are not monotonic")
|
||||
detector_h5py.close()
|
||||
except:
|
||||
problems.append(f'Can not read from detector file {detector_file} may be too early')
|
||||
problems.append(f"Can not read from detector file {detector_file} may be too early")
|
||||
|
||||
|
||||
if len(problems) > 0:
|
||||
|
||||
@@ -63,9 +63,9 @@ class BrokerClient:
|
||||
self.start_pulseid = None
|
||||
|
||||
beamline=get_beamline()
|
||||
raw_directory = f'/sf/{beamline}/data/{self.pgroup}/raw/'
|
||||
raw_directory = f"/sf/{beamline}/data/{self.pgroup}/raw/"
|
||||
if not os.path.isdir(raw_directory):
|
||||
raise NameError(f'{raw_directory} doesnt exist or accessible')
|
||||
raise NameError(f"{raw_directory} doesnt exist or accessible")
|
||||
|
||||
def configure(self,
|
||||
channels_file=None, epics_file=None,
|
||||
@@ -80,7 +80,7 @@ class BrokerClient:
|
||||
|
||||
try:
|
||||
beamline=get_beamline()
|
||||
last_run_file = f'/sf/{beamline}/data/{self.pgroup}/raw/run_info/LAST_RUN'
|
||||
last_run_file = f"/sf/{beamline}/data/{self.pgroup}/raw/run_info/LAST_RUN"
|
||||
if os.path.exists(last_run_file):
|
||||
run_file = open(last_run_file, "r")
|
||||
self.last_run = int(run_file.read())
|
||||
@@ -122,10 +122,10 @@ class BrokerClient:
|
||||
last_known_run = int(self.last_run) if self.last_run is not None else -1
|
||||
def signal_handler(sig, frame):
|
||||
current_pulseid = get_current_pulseid()
|
||||
print('\nYou pressed Ctrl+C!')
|
||||
print(f'what do you want me to do with already collected up to now frames (pulseids: {self.start_pulseid}-{current_pulseid})')
|
||||
answer=input('[s]-save them into; any other key - discard : ')
|
||||
if answer == 's':
|
||||
print("\nYou pressed Ctrl+C!")
|
||||
print(f"what do you want me to do with already collected up to now frames (pulseids: {self.start_pulseid}-{current_pulseid})")
|
||||
answer=input("[s]-save them into; any other key - discard : ")
|
||||
if answer == "s":
|
||||
self.stop(stop_pulseid=current_pulseid)
|
||||
raise NameError("Ctrl-c is called")
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
@@ -8,7 +8,7 @@ TIMEOUT_DAQ = 10
|
||||
|
||||
def run():
|
||||
|
||||
parser = argparse.ArgumentParser(description='simple daq client example')
|
||||
parser = argparse.ArgumentParser(description="simple daq client example")
|
||||
|
||||
parser.add_argument("-p", "--pgroup", help="pgroup, example p12345", default="p18493")
|
||||
|
||||
@@ -129,7 +129,7 @@ def retrieve_data_from_buffer(pgroup=None,
|
||||
parameters["scan_info"] = scan_step_info
|
||||
|
||||
try:
|
||||
r = requests.post(f'{broker_address}/retrieve_from_buffers',json=parameters, timeout=TIMEOUT_DAQ)
|
||||
r = requests.post(f"{broker_address}/retrieve_from_buffers",json=parameters, timeout=TIMEOUT_DAQ)
|
||||
except:
|
||||
raise NameError("Cant connect to daq")
|
||||
|
||||
@@ -141,18 +141,18 @@ def retrieve_data_from_buffer(pgroup=None,
|
||||
run_number = responce.get("run_number", None)
|
||||
if run_number is not None:
|
||||
run_number = int(run_number)
|
||||
run_number_print = f'{run_number:04}'
|
||||
run_number_print = f"{run_number:04}"
|
||||
else:
|
||||
run_number_print = None
|
||||
acq_number = responce.get("acquisition_number", None)
|
||||
unq_acq_number = responce.get("unique_acquisition_number", None)
|
||||
files_daq = responce.get("files", [])
|
||||
print(f'success: {message=} {run_number=} {acq_number=} {unq_acq_number=}')
|
||||
print(f' these files to expect in raw/{pgroup}/run{run_number_print}/data/ directory : {files_daq}')
|
||||
print(f"success: {message=} {run_number=} {acq_number=} {unq_acq_number=}")
|
||||
print(f" these files to expect in raw/{pgroup}/run{run_number_print}/data/ directory : {files_daq}")
|
||||
else:
|
||||
message = responce.get("message", None)
|
||||
print(f' Error, reason : {message=}')
|
||||
print(f' whole responce : {responce=}')
|
||||
print(f" Error, reason : {message=}")
|
||||
print(f" whole responce : {responce=}")
|
||||
else:
|
||||
print("Bad responce from request")
|
||||
|
||||
|
||||
@@ -50,10 +50,10 @@ Utility to read binary Jungfrau gain maps from PSI Detectors Group and save them
|
||||
|
||||
for i in range(n_modules):
|
||||
if maps[i].shape[0] == 3 * 512 * 1024:
|
||||
print(f'{i}-module gain coefficients are only for G0,G1,G2. Expanding them to HG0,HG1,HG2 (copy G0,G1,G2)')
|
||||
print(f"{i}-module gain coefficients are only for G0,G1,G2. Expanding them to HG0,HG1,HG2 (copy G0,G1,G2)")
|
||||
maps[i] = np.append(maps[i], maps[i][:3 * 512 * 1024])
|
||||
if maps[i].shape[0] == 4 * 512 * 1024:
|
||||
print(f'{i}-module gain coefficients are only for G0,G1,G2,HG0. Expanding them to HG1,HG2 (copy G1,G2)')
|
||||
print(f"{i}-module gain coefficients are only for G0,G1,G2,HG0. Expanding them to HG1,HG2 (copy G1,G2)")
|
||||
maps[i] = np.append(maps[i], maps[i][1024*512:3*1024*512])
|
||||
print(maps[i].shape[0])
|
||||
maps = [i.reshape(module_shape) for i in maps]
|
||||
|
||||
Reference in New Issue
Block a user