sf-mx/clara
0
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

JF adaptations

Browse Source
This commit is contained in:
assman_g
2024-06-20 16:48:20 +02:00
parent 87082d7de7
commit a2f05b768d
2 changed files with 84 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
src/clara.py
View File

@@ -84,19 +84,20 @@ class CollectedH5:
def mk_cd_output_dir_bl(self): def mk_cd_output_dir_bl(self):
""" """
mk putput dir with pathlib and change into this dir. mk putput dir with pathlib and change into this dir.
Output dir to /sf/cristallina/data.. will be written as gac-account Output dir to /sf/cristallina/data/.. will be written as gac-account
pa = "/sf/cristallina/data/"
:return: None :return: None
""" """
# Assemble output dir path # Assemble output dir path
now = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") now = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
pgroup = str(self.message["user_data"]["paccount"]) pgroup = str(self.message["experiment_group"])
merge_id = str(self.message["user_data"]["mergeID"])
res = "res" res = "res"
data_path = "data" #data_path = "data"
file_name = Path(str(self.message["filename"][:-3]) + "_" + str(now)) file_name = Path(str(self.message["filename"][:-3]) + "_" + str(now))
out_path = pa / pgroup / res / merge_id/ data_path /file_name out_path = pa / pgroup / res / file_name
logger.info(f"processing folder will be created at : {out_path}") logger.info(f"processing folder will be created at : {out_path}")
try: try:
out_path.mkdir(parents=True, exist_ok=True) out_path.mkdir(parents=True, exist_ok=True)
@@ -250,28 +251,35 @@ class CollectedH5:
processing folder with the corresponding mergeID for processing processing folder with the corresponding mergeID for processing
:return: - :return: -
""" """
merge_id = str(self.message["user_data"]["mergeID"])
f = open(merge_id + ".cell", "w")
# start writing the cell file
f.write("CrystFEL unit cell file version 1.0\n\n")
# get lattice params and write to file
space_group = self.convert_spg_num(self.message["user_data"]["spaceGroupNumber"])
lat_type, unique_a, cent = self.get_spaceg_params(space_group)
f.write("lattice_type = " + lat_type[2:].lower() + "\n")
f.write("centering = " + cent + "\n")
if unique_a != "*":
f.write("unique_axis = " + unique_a + "\n\n")
else:
f.write("\n\n")
# print unit cell constants
f.write("a = " + str(self.message["user_data"]["unitCell"]["a"]) + " A\n") #check if cell parameters were sent with the message or not
f.write("b = " + str(self.message["user_data"]["unitCell"]["b"]) + " A\n") if "space_group_number" in self.message:
f.write("c = " + str(self.message["user_data"]["unitCell"]["c"]) + " A\n") name = str(self.message["run_number"])
f.write("al = " + str(self.message["user_data"]["unitCell"]["alpha"]) + " deg\n") f = open(name + ".cell", "w")
f.write("be = " + str(self.message["user_data"]["unitCell"]["beta"]) + " deg\n") # start writing the cell file
f.write("ga = " + str(self.message["user_data"]["unitCell"]["gamma"]) + " deg\n") f.write("CrystFEL unit cell file version 1.0\n\n")
f.close() # get lattice params and write to file
space_group = self.convert_spg_num(self.message["space_group_number"])
lat_type, unique_a, cent = self.get_spaceg_params(space_group)
f.write("lattice_type = " + lat_type[2:].lower() + "\n")
f.write("centering = " + cent + "\n")
if unique_a != "*":
f.write("unique_axis = " + unique_a + "\n\n")
else:
f.write("\n\n")
# print unit cell constants
f.write("a = " + str(self.message["unit_cell"]["a"]) + " A\n")
f.write("b = " + str(self.message["unit_cell"]["b"]) + " A\n")
f.write("c = " + str(self.message["unit_cell"]["c"]) + " A\n")
f.write("al = " + str(self.message["unit_cell"]["alpha"]) + " deg\n")
f.write("be = " + str(self.message["unit_cell"]["beta"]) + " deg\n")
f.write("ga = " + str(self.message["unit_cell"]["gamma"]) + " deg\n")
f.close()
else:
logger.info("no space group sent, no cell file written")
return None
return None return None
def create_geom_from_master(self): def create_geom_from_master(self):
@@ -282,20 +290,20 @@ class CollectedH5:
:return: none :return: none
""" """
merge_id = str(self.message["user_data"]["mergeID"]) name = str(self.message["run_number"])
f2 = open(merge_id + "_jf.geom", "w") f2 = open(name + "_jf.geom", "w")
f2.write("; PSI JF8M \n") f2.write("; PSI JF8M \n")
f2.write("\n") f2.write("\n")
f2.write("\n") f2.write("\n")
f2.write("; Camera length (in m) and photon energy (eV) \n") f2.write("; Camera length (in m) and photon energy (eV) \n")
f2.write("clen = " + str(self.message["user_data"]["detectorDistance_mm"] * 0.001) + "\n") f2.write("clen = " + str(self.message["detector_distance_m"]) + "\n")
f2.write("photon_energy = " + str(self.message["user_data"]["enery_kev"] * 1000) + "\n") f2.write("photon_energy = " + str(self.message["photon_energy_eV"]) + "\n")
f2.write("flag_lessthan = " + str(self.message["user_data"]["underload"]) + "\n") f2.write("flag_lessthan = " + str(self.message["underload"]) + "\n")
f2.write("\n") f2.write("\n")
#f2.write("adu_per_eV = 0.00008065\n") #f2.write("adu_per_eV = 0.00008065\n")
f2.write("adu_per_eV = 0.00008285\n") f2.write("adu_per_eV = 0.00008285\n")
# f2.write("adu_per_photon = 1\n") # f2.write("adu_per_photon = 1\n")
f2.write("res = 13333.3 ; " + str(self.message["user_data"]["pixelSize_um"]) + " micron pixel size\n") f2.write("res = 13333.3 ; " + str(self.message["pixel_size_m"] *1000000) + " micron pixel size\n")
f2.write("\n") f2.write("\n")
f2.write("rigid_group_0 = 0 \n") f2.write("rigid_group_0 = 0 \n")
f2.write("rigid_group_collection_0 = 0 \n") f2.write("rigid_group_collection_0 = 0 \n")
@@ -309,27 +317,25 @@ class CollectedH5:
f2.write("\n") f2.write("\n")
# Assembling path for master file # Assembling path for master file
pgroup = str(self.message["user_data"]["paccount"]) pgroup = str(self.message["experiment_group"])
raw = "raw" raw = "raw"
data_path = "data"
merge_id = str(self.message["user_data"]["mergeID"])
master_file = str(self.message["filename"][:-11]) + str("master.h5") master_file = str(self.message["filename"][:-11]) + str("master.h5")
master_file_path = pa / pgroup / raw / merge_id / data_path / master_file master_file_path = pa / pgroup / raw / master_file
f2.write("mask_file ="+ str(master_file_path) +"\n") f2.write(";mask_file ="+ str(master_file_path) +"\n")
f2.write("mask = /entry/instrument/detector/pixel_mask \n") f2.write(";mask = /entry/instrument/detector/pixel_mask \n")
f2.write("mask_good = 0x0 \n") f2.write(";mask_good = 0x0 \n")
f2.write("mask_bad = 0xFFFFFFFF\n") f2.write(";mask_bad = 0xFFFFFFFF\n")
f2.write("\n") f2.write("\n")
f2.write("; corner_{x,y} set the position of the corner of the detector (in pixels) \n") f2.write("; corner_{x,y} set the position of the corner of the detector (in pixels) \n")
f2.write("; relative to the beam \n") f2.write("; relative to the beam \n")
f2.write("\n") f2.write("\n")
f2.write("0/min_fs = 0 \n") f2.write("0/min_fs = 0 \n")
f2.write("0/min_ss = 0 \n") f2.write("0/min_ss = 0 \n")
f2.write("0/max_fs =" + str(self.message["user_data"]["detectorWidth_pxl"] - 1) + "\n") f2.write("0/max_fs =" + str(self.message["detector_height_pxl"] ) + "\n")
f2.write("0/max_ss =" + str(self.message["user_data"]["detectorHeight_pxl"] - 1) + "\n") f2.write("0/max_ss =" + str(self.message["detector_width_pxl"] ) + "\n")
f2.write("0/corner_x = -" + str(self.message["user_data"]["beamCenterX_pxl"]) + "\n") f2.write("0/corner_x = -" + str(self.message["beam_x_pxl"]) + "\n")
f2.write("0/corner_y = -" + str(self.message["user_data"]["beamCenterY_pxl"]) + "\n") f2.write("0/corner_y = -" + str(self.message["beam_y_pxl"]) + "\n")
f2.write("0/fs = x \n") f2.write("0/fs = x \n")
f2.write("0/ss = y \n") f2.write("0/ss = y \n")
f2.write("\n") f2.write("\n")
@@ -350,20 +356,21 @@ class CollectedH5:
:return:None :return:None
""" """
#Assemble path for raw data #Assemble path for raw data
pgroup = str(self.message["user_data"]["paccount"]) pgroup = str(self.message["experiment_group"])
raw = "raw" raw = "raw"
data_path = "data"
merge_id = str(self.message["user_data"]["mergeID"])
filen = str(self.message["filename"]) filen = str(self.message["filename"])
file_path = pa / pgroup / raw / merge_id / data_path / filen file_path = pa / pgroup / raw / filen
# write to cell file in output folder # write to cell file in output folder
f = open(merge_id + ".list", "w") name = str(self.message["run_number"])
f = open(name + ".list", "w")
f.write(str(file_path)) f.write(str(file_path))
f.close() f.close()
#Insert new key into message for triggered/non tirggered
self.message["trigger"] = "on"
return None return None
def create_slurm_script(self): def create_slurm_script(self):
@@ -381,12 +388,15 @@ class CollectedH5:
""" """
# get dat file name without any preceding paths.. # get dat file name without any preceding paths..
#last_pos = str(self.message["dataFileName"]).rfind("/") #last_pos = str(self.message["dataFileName"]).rfind("/")
data_file_name = str(self.message["filename"][: -3]) #data_file_name = str(self.message["filename"][: -3])
data_file_name = str(self.message["run_number"])
# write file # write file
f = open("run_SLURM", "w") f = open("run_SLURM", "w")
f.write("#!/bin/bash \n") f.write("#!/bin/bash \n")
f.write("#SBATCH --job-name=index \n") f.write("#SBATCH --job-name=index \n")
f.write("#SBATCH --partition=prod-aramis \n")
#f.write("#SBATCH --nodelist=sf-cn-21 \n")
# uncomment if on RA # uncomment if on RA
# f.write("#SBATCH --partition=hour \n") # f.write("#SBATCH --partition=hour \n")
f.write("#SBATCH --cpus-per-task=32 \n") f.write("#SBATCH --cpus-per-task=32 \n")
@@ -397,7 +407,7 @@ class CollectedH5:
f.write("# Load modules \n") f.write("# Load modules \n")
f.write("module purge \n") f.write("module purge \n")
f.write("module use MX unstable \n") f.write("module use MX unstable \n")
f.write("module load crystfel/0.10.2 \n") f.write("module load crystfel/0.10.2-rhel8 \n")
# f.write( # f.write(
# "module load crystfel/0.10.1-2 xgandalf/2018.01 HDF5_bitshuffle/2018.05 HDF5_LZ4/2018.05 gcc/4.8.5 hdf5_serial/1.10.3 \n" # "module load crystfel/0.10.1-2 xgandalf/2018.01 HDF5_bitshuffle/2018.05 HDF5_LZ4/2018.05 gcc/4.8.5 hdf5_serial/1.10.3 \n"
# ) # )
@@ -406,18 +416,19 @@ class CollectedH5:
f.write( f.write(
" indexamajig --peaks=peakfinder8 --indexing=xgandalf --xgandalf-fast-execution --threshold=" " indexamajig --peaks=peakfinder8 --indexing=xgandalf --xgandalf-fast-execution --threshold="
+ str(int(self.message["user_data"]["crystfelTreshold"])) + str(int(self.message["user_data"]["crystfelTreshold"]))
+ " --int-radius=2,3,5 -p " #+ " --int-radius=2,3,5 "
+ str(self.message["user_data"]["mergeID"]) + " -p "
+ str(self.message["run_number"])
+ ".cell --min-snr=" + ".cell --min-snr="
+ str(self.message["user_data"]["crystfelMinSNR"]) + str(self.message["user_data"]["crystfelMinSNR"])
+ " --min-peaks=6 --min-pix-count=" + " --min-peaks=6 --min-pix-count="
+ str(self.message["user_data"]["crystfelMinPixCount"]) + str(self.message["user_data"]["crystfelMinPixCount"])
+ " -i " + " -i "
+ str(self.message["user_data"]["mergeID"]) + str(self.message["run_number"])
+ ".list -o " + ".list -o "
+ data_file_name + data_file_name
+ ".stream -g " + ".stream -g "
+ str(self.message["user_data"]["mergeID"]) + str(self.message["run_number"])
+ "_jf.geom " + "_jf.geom "
+ " -j `nproc` --min-res=75 " + " -j `nproc` --min-res=75 "
) )
@@ -542,13 +553,28 @@ if __name__ == "__main__":
elif sys.argv[1] == "z": elif sys.argv[1] == "z":
pa = Path("/sf/cristallina/data/") pa = Path("/sf/cristallina/data/")
# potential message recieving: # potential message recieving:
logger.info("SUBscribing to ZeroMQ PUBlisher") logger.info("SUBscribing to ZeroMQ PUBlisher.. connecting ...")
context = zmq.Context() context = zmq.Context()
subscriber = context.socket(zmq.SUB) subscriber = context.socket(zmq.SUB)
subscriber.connect("tcp://sf-broker-01.psi.ch:5555") subscriber.connect("tcp://sf-broker-01.psi.ch:5555")
subscriber.setsockopt_string(zmq.SUBSCRIBE, "") subscriber.setsockopt_string(zmq.SUBSCRIBE, "")
#--------------------------------------------------
#UNcomment if handshake between publisher and subscriber is not neccesary
ready_socket = context.socket(zmq.REQ)
ready_socket.connect("tcp://sf-broker-01.psi.ch:5556")
# Notify publisher that subscriber is ready
ready_socket.send(b"READY")
start_message= ready_socket.recv()
print(f"Received start message: {start_message}")
#----------------------------------------------------------------------
time.sleep(3)
logger.info("connected to ZeroMQ PUBlisher") logger.info("connected to ZeroMQ PUBlisher")
TERMINATE_SERVER = False TERMINATE_SERVER = False
signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGINT, sigint_handler)
@@ -573,7 +599,7 @@ if __name__ == "__main__":
logger.info("list file created") logger.info("list file created")
mess_inp.create_slurm_script() mess_inp.create_slurm_script()
logger.info("slurm submission script created") logger.info("slurm submission script created")
#mess_inp.submit_job_to_slurm() mess_inp.submit_job_to_slurm()
mess_inp.create_msg_file() mess_inp.create_msg_file()
logger.info("message dumped in processing folder") logger.info("message dumped in processing folder")
#code BLOCK what is happening to message #code BLOCK what is happening to message

30
src/results.py
View File

@@ -13,10 +13,7 @@ import MxdbVdpTools
# import matplotlib.pyplot as plt # import matplotlib.pyplot as plt
LOG_FILENAME = time.strftime("/sls/MX/Data10/e20233/log/vdp_%Y%m.log") # as eaccount at beamline LOG_FILENAME = time.strftime("/sf/cristallina/applications/mx/clara_tools/log/clara_%Y%m.log")
# LOG_FILENAME = time.strftime("/sls/MX/Data10-staff/e19370/log/vdp_%Y%m.log") # as eaccount at beamline
# LOG_FILENAME = time.strftime("/home/assman_g/Documents/log/vdp_%Y%m.log") # as assman_g at beamline
# LOG_FILENAME = time.strftime("/das/home/assman_g/vdp_%Y%m.log") # on RA
logger.add(LOG_FILENAME, level="INFO", rotation="100MB") logger.add(LOG_FILENAME, level="INFO", rotation="100MB")
@@ -353,31 +350,6 @@ if __name__ == "__main__":
# insert message to DB # insert message to DB
_id = mxdb.insert(results_message) _id = mxdb.insert(results_message)
#EXAMPLE MESSAGE {
# "mergeID": "something_mergeID",
# "trackingId": "something_track",
# "eaccount": "e19370",
# "masterFileName": "Lyso_12p4keV_1kHz_150mm_run000026_master.h5",
# "dataFileName": "Lyso_12p4keV_1kHz_150mm_run000026_data_000010.h5",
# "filesystemPath": "/das/work/p19/p19607/FromGreta/REDML_indx_data/lyso/processing/",
# "detectorDistance_mm": 150.0,
# "beamCenterX_pxl": 1103.7,
# "beamCenterY_pxl": 1175.1,
# "pixelSize_um": 75,
# "numberOfImages": 10000,
# "imageTime_us": 100,
# "enery_kev": 12398.0,
# "detectorWidth_pxl": 2067,
# "detectorHeight_pxl": 2163,
# "underload": -30000,
# "overload": 30000,
# "unitCell": {"a": 79.5, "b": 79.5, "c": 38.6, "alpha": 90.0, "beta": 90.0, "gamma": 90.0},
# "spaceGroupNumber": 96,
# "crystfelTreshold": 6.0,
# "crystfelMinSNR": 4.0,
# "crystfelMinPixCount": 1,
# "crystfelMultiCrystal": False,
#}
logger.info("message inserted to DB") logger.info("message inserted to DB")