diff --git a/beamline_tools/swissmx.sh b/beamline_tools/swissmx.sh index 2f4d8d7..cb01779 100644 --- a/beamline_tools/swissmx.sh +++ b/beamline_tools/swissmx.sh @@ -1,7 +1,7 @@ #!/bin/bash -source /sf/cristallina/applications/mx/conda/miniconda/bin/activate -conda activate crmx38 +source /sf/daq/source-conda +conda activate /sf/cristallina/config/python/.pixi/envs/crmx-proc cd /sf/cristallina/applications/mx/zamofing_t/ESB_MX/python/SwissMX/ python swissmx.py diff --git a/clen_tools/detector-distance-refinement.py b/clen_tools/detector-distance-refinement.py index b4f8531..74502d3 100644 --- a/clen_tools/detector-distance-refinement.py +++ b/clen_tools/detector-distance-refinement.py @@ -32,6 +32,9 @@ import argparse from scipy.optimize import curve_fit from scipy.signal import peak_widths, find_peaks +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def h5_sample( lst, sample ): # create sample of images from run @@ -78,7 +81,7 @@ def geom_amend( lab6_geom_file, clen ): # return clen_geom file name return clen_geom_file -def write_crystfel_run( clen, sample_h5_file, clen_geom_file, cell_file, threshold ): +def write_crystfel_run( clen, sample_h5_file, clen_geom_file, cell_file, threshold, crystfel_version ): # crystfel file name cryst_run_file = "{0}_run.sh".format( clen ) @@ -87,7 +90,8 @@ def write_crystfel_run( clen, sample_h5_file, clen_geom_file, cell_file, thresho run_sh = open( cryst_run_file, "w" ) run_sh.write( "#!/bin/sh\n\n" ) run_sh.write( "module purge\n" ) - run_sh.write( "module load crystfel/0.11.1\n" ) + run_sh.write( "module use MX\n" ) + run_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) run_sh.write( "indexamajig -i {0} \\\n".format( sample_h5_file ) ) run_sh.write( " --output={0}.stream \\\n".format( clen ) ) run_sh.write( " --geometry={0}\\\n".format( clen_geom_file ) ) @@ -376,7 +380,7 @@ def plot_indexed_std_alpha_beta_gamma( stats_df, ax1, ax2 ): color = "darkolivegreen" ax2.plot(stats_df.clen, stats_df.std_gamma, color=color, label="gamma" ) -def scan( cwd, lst, sample, lab6_geom_file, centre_clen, cell_file, threshold, step_size ): +def scan( cwd, lst, sample, lab6_geom_file, centre_clen, cell_file, threshold, step_size, crystfel_version ): # define coarse or fine scan if step_size == "coarse": @@ -417,7 +421,7 @@ def scan( cwd, lst, sample, lab6_geom_file, centre_clen, cell_file, threshold, s clen_geom_file = geom_amend( lab6_geom_file, clen ) # make crystfel run file - cryst_run_file = write_crystfel_run( clen, sample_h5_file, clen_geom_file, cell_file, threshold ) + cryst_run_file = write_crystfel_run( clen, sample_h5_file, clen_geom_file, cell_file, threshold, crystfel_version ) # run crystfel file job_list.append( cryst_run_file ) @@ -452,17 +456,17 @@ def scrub_scan( scan_top_dir, scan ): return suggested_clen -def main( cwd, lst, sample, geom, centre_clen, cell_file, threshold ): +def main( cwd, lst, sample, geom, centre_clen, cell_file, threshold, crystfel_version ): # run scan if not os.path.isdir( "{0}/scan".format( cwd ) ): # run inital coarse scan - scan( cwd, lst, sample, geom, centre_clen, cell_file, threshold, "coarse" ) + scan( cwd, lst, sample, geom, centre_clen, cell_file, threshold, "coarse", crystfel_version ) # get approx centre from new results coarse_clen = scrub_scan( cwd, scan="scan" ) # perfom 2nd scan - scan( cwd, lst, sample, geom, coarse_clen, cell_file, threshold, "fine" ) + scan( cwd, lst, sample, geom, coarse_clen, cell_file, threshold, "fine", crystfel_version ) else: print( "scan already performed" ) @@ -516,6 +520,14 @@ if __name__ == "__main__": type=str, default="cristallina" ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # set threshold based on endstation if args.endstation == "alvra": @@ -528,6 +540,7 @@ if __name__ == "__main__": # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.lst, args.sample, args.geom, args.central_distance, args.cell_file, threshold ) + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.lst, args.sample, args.geom, args.central_distance, args.cell_file, threshold, crystfel_version ) diff --git a/reduction_tools/crystfel_run.sh b/reduction_tools/crystfel_run.sh index a1e8adf..3126349 100644 --- a/reduction_tools/crystfel_run.sh +++ b/reduction_tools/crystfel_run.sh @@ -1,8 +1,16 @@ #!/bin/bash +# crystfel version: 10 = 0.10.2-rhel8, 12 = 0.12.0 +CRYSTFEL_VERSION=10 +if [ "$CRYSTFEL_VERSION" = "10" ]; then + CRYSTFEL_MODULE=0.10.2-rhel8 +elif [ "$CRYSTFEL_VERSION" = "12" ]; then + CRYSTFEL_MODULE=0.12.0 +fi + module purge -module use MX unstable -module load crystfel/0.10.2-rhel8 +module use MX +module load crystfel/${CRYSTFEL_MODULE} # add path to images # add output stream name diff --git a/reduction_tools/crystfel_split.py b/reduction_tools/crystfel_split.py index 4a39ff3..29b3b0b 100644 --- a/reduction_tools/crystfel_split.py +++ b/reduction_tools/crystfel_split.py @@ -34,6 +34,9 @@ import regex as re import numpy as np from loguru import logger +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def scrub_cells( line ): # get uc values from stream file @@ -129,7 +132,7 @@ def h5_split( lst, chunk_size ): return list_df -def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, threshold ): +def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, threshold, crystfel_version ): """ crystfel run file - spot-finding and indexing parameters may need some editing @@ -145,8 +148,8 @@ def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_f run_sh = open( cryst_run_file, "w" ) run_sh.write( "#!/bin/sh\n\n" ) run_sh.write( "module purge\n" ) - run_sh.write( "module use MX unstable\n" ) - run_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + run_sh.write( "module use MX\n" ) + run_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) run_sh.write( "indexamajig -i {0} \\\n".format( chunk_lst_file ) ) run_sh.write( " --output={0} \\\n".format( stream_file ) ) run_sh.write( " --geometry={0} \\\n".format( geom_file ) ) @@ -222,7 +225,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update(completed_jobs) time.sleep(2) -def run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation ): +def run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation, crystfel_version ): # set chunk counter chunk = 0 @@ -250,7 +253,7 @@ def run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation chunk_lst.to_csv( chunk_lst_file, index=False, header=False ) # write crystfel file and append path to list - cryst_run_file, stream_file = write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, threshold ) + cryst_run_file, stream_file = write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, threshold, crystfel_version ) stream_lst.append( "{0}/{1}".format( proc_dir, stream_file ) ) # submit jobs @@ -265,7 +268,7 @@ def run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation return submitted_job_ids, chunk, stream_lst -def main( cwd, name, lst, chunk_size, geom_file, cell_file, threshold, reservation ): +def main( cwd, name, lst, chunk_size, geom_file, cell_file, threshold, reservation, crystfel_version ): print( "reading SwissFEL lst file" ) print( "creating {0} image chunks of lst".format( chunk_size ) ) @@ -274,7 +277,7 @@ def main( cwd, name, lst, chunk_size, geom_file, cell_file, threshold, reservati # run crystfel runs on individual splits print( "submitting jobs to cluster" ) - submitted_job_ids, chunk, stream_lst = run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation ) + submitted_job_ids, chunk, stream_lst = run_splits( list_df, cwd, name, geom_file, cell_file, threshold, reservation, crystfel_version ) # monitor progress of jobs time.sleep( 30 ) @@ -399,6 +402,14 @@ if __name__ == "__main__": type=bool, default=False ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # set current working directory cwd = os.getcwd() @@ -415,5 +426,6 @@ if __name__ == "__main__": threshold = 3000 elif args.photons_or_energy == "photons": threshold = 15 - main( cwd, args.job_name, args.lst_file, args.chunk_size, args.geom_file, args.cell_file, threshold, args.reservation ) + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.job_name, args.lst_file, args.chunk_size, args.geom_file, args.cell_file, threshold, args.reservation, crystfel_version ) diff --git a/reduction_tools/crystfel_split_var.py b/reduction_tools/crystfel_split_var.py index 94043e5..2b8f479 100644 --- a/reduction_tools/crystfel_split_var.py +++ b/reduction_tools/crystfel_split_var.py @@ -36,6 +36,9 @@ from tqdm import tqdm import regex as re from loguru import logger +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def count_chunks( stream ): # get number of chunks @@ -142,7 +145,8 @@ def h5_split( lst, chunk_size ): def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, indexer, peakfinder, integrator, tolerance, threshold, min_snr, - int_rad, multi, retry, min_pix, bg_rad, min_res ): + int_rad, multi, retry, min_pix, bg_rad, min_res, + crystfel_version ): # stream file name stream_file = "{0}_{1}.stream".format( name, chunk ) @@ -153,8 +157,8 @@ def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, run_sh = open( cryst_run_file, "w" ) run_sh.write( "#!/bin/sh\n\n" ) run_sh.write( "module purge\n" ) - run_sh.write( "module use MX unstable\n" ) - run_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + run_sh.write( "module use MX\n" ) + run_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) run_sh.write( "indexamajig -i {0} \\\n".format( chunk_lst_file ) ) run_sh.write( " --output={0} \\\n".format( stream_file ) ) run_sh.write( " --geometry={0} \\\n".format( geom_file ) ) @@ -233,7 +237,7 @@ def wait_for_jobs( job_ids, total_jobs ): def run_splits( list_df, cwd, name, geom_file, cell_file, indexer, peakfinder, integrator, tolerance, threshold, min_snr, int_rad, multi, retry, min_pix, bg_rad, - min_res, reservation ): + min_res, reservation, crystfel_version ): # set chunk counter chunk = 0 @@ -264,7 +268,8 @@ def run_splits( list_df, cwd, name, geom_file, cell_file, cryst_run_file, stream_file = write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_file, indexer, peakfinder, integrator, tolerance, threshold, min_snr, - int_rad, multi, retry, min_pix, bg_rad, min_res ) + int_rad, multi, retry, min_pix, bg_rad, min_res, + crystfel_version ) stream_lst.append( "{0}/{1}".format( proc_dir, stream_file ) ) # submit jobs @@ -279,10 +284,10 @@ def run_splits( list_df, cwd, name, geom_file, cell_file, return submitted_job_ids, chunk, stream_lst -def main( cwd, name, lst, chunk_size, geom_file, cell_file, +def main( cwd, name, lst, chunk_size, geom_file, cell_file, indexer, peakfinder, integrator, tolerance, threshold, min_snr, int_rad, multi, retry, min_pix, bg_rad, - min_res, reservation ): + min_res, reservation, crystfel_version ): print( "reading SwissFEL lst file" ) print( "creating {0} image chunks of lst".format( chunk_size ) ) @@ -294,7 +299,7 @@ def main( cwd, name, lst, chunk_size, geom_file, cell_file, submitted_job_ids, chunk, stream_lst = run_splits( list_df, cwd, name, geom_file, cell_file, indexer, peakfinder, integrator, tolerance, threshold, min_snr, int_rad, multi, retry, min_pix, bg_rad, - min_res, reservation ) + min_res, reservation, crystfel_version ) # monitor progress of jobs time.sleep( 30 ) @@ -502,6 +507,14 @@ if __name__ == "__main__": type=bool, default=False ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # run geom converter cwd = os.getcwd() @@ -521,8 +534,9 @@ if __name__ == "__main__": retry = "retry" else: retry = "no-retry" + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] main( cwd, args.job_name, args.lst_file, args.chunk_size, args.geom_file, args.cell_file, args.indexer, args.peakfinder, - args.integrator, args.tolerance, args.threshold, + args.integrator, args.tolerance, args.threshold, args.min_snr, args.int_radius, multi, retry, args.min_pix, args.bg_rad, - args.min_res, args.reservation ) \ No newline at end of file + args.min_res, args.reservation, crystfel_version ) \ No newline at end of file diff --git a/reduction_tools/make_mtz.py b/reduction_tools/make_mtz.py index a530de8..85c968a 100644 --- a/reduction_tools/make_mtz.py +++ b/reduction_tools/make_mtz.py @@ -92,6 +92,7 @@ def make_mtz( hklout_file, mtzout_file, project, crystal, dataset, cell, spacegr mtz_sh = open( mtz_run_file, "w" ) mtz_sh.write( "#!/bin/sh\n\n" ) mtz_sh.write( "module purge\n" ) + mtz_sh.write( "module use MX\n" ) mtz_sh.write( "module load ccp4/8.0\n\n" ) mtz_sh.write( "f2mtz HKLIN {0} HKLOUT {1} << EOF_hkl > f2mtz.log\n".format( hklout_file, mtzout_file ) ) mtz_sh.write( "TITLE Reflections from CrystFEL\n" ) diff --git a/reduction_tools/partialator.py b/reduction_tools/partialator.py index 578606e..eef320a 100644 --- a/reduction_tools/partialator.py +++ b/reduction_tools/partialator.py @@ -42,6 +42,9 @@ import warnings warnings.filterwarnings( "ignore", category=RuntimeWarning ) from loguru import logger +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def submit_job( job_file, reservation ): # submit the job @@ -80,7 +83,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update( completed_jobs ) time.sleep( 2 ) -def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu ): +def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu, crystfel_version ): # partialator file name part_run_file = "{0}/partialator_{1}.sh".format( proc_dir, name ) @@ -89,8 +92,8 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu part_sh = open( part_run_file, "w" ) part_sh.write( "#!/bin/sh\n\n" ) part_sh.write( "module purge\n" ) - part_sh.write( "module use MX unstable\n" ) - part_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + part_sh.write( "module use MX\n" ) + part_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) part_sh.write( "partialator -i {0} \\\n".format( stream ) ) part_sh.write( " -o {0}.hkl \\\n".format( name ) ) part_sh.write( " -y {0} \\\n".format( pointgroup ) ) @@ -111,7 +114,7 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu # return partialator file name return part_run_file -def run_compare_check( proc_dir, name, cell, shells, part_h_res ): +def run_compare_check( proc_dir, name, cell, shells, part_h_res, crystfel_version ): # check file name check_run_file = "{0}/check_{1}.sh".format( proc_dir, name ) @@ -120,8 +123,8 @@ def run_compare_check( proc_dir, name, cell, shells, part_h_res ): check_sh = open( check_run_file, "w" ) check_sh.write( "#!/bin/sh\n\n" ) check_sh.write( "module purge\n" ) - check_sh.write( "module use MX unstable\n" ) - check_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + check_sh.write( "module use MX\n" ) + check_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) check_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) ) @@ -326,7 +329,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): plt.tight_layout() plt.savefig( "plots.png" ) -def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu, reservation ): +def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu, reservation, crystfel_version ): # submitted job set submitted_job_ids = set() @@ -340,8 +343,8 @@ def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h print( "making partialator file" ) # make partialator run file - part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu ) - check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res ) + part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu, crystfel_version ) + check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res, crystfel_version ) # submit job job_id = submit_job( part_run_file, reservation ) @@ -481,6 +484,14 @@ if __name__ == "__main__": type=bool, default=False ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # set loguru if not args.debug: @@ -490,4 +501,5 @@ if __name__ == "__main__": # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation ) + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation, crystfel_version ) diff --git a/reduction_tools/partialator_mtz.py b/reduction_tools/partialator_mtz.py index a238927..ca5f5fe 100644 --- a/reduction_tools/partialator_mtz.py +++ b/reduction_tools/partialator_mtz.py @@ -45,6 +45,9 @@ import warnings warnings.filterwarnings( "ignore", category=RuntimeWarning ) from loguru import logger +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def submit_job( job_file, reservation ): # submit the job @@ -83,7 +86,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update( completed_jobs ) time.sleep( 2 ) -def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu ): +def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu, crystfel_version ): # partialator file name part_run_file = "{0}/partialator_{1}.sh".format( proc_dir, name ) @@ -92,8 +95,8 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu part_sh = open( part_run_file, "w" ) part_sh.write( "#!/bin/sh\n\n" ) part_sh.write( "module purge\n" ) - part_sh.write( "module use MX unstable\n" ) - part_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + part_sh.write( "module use MX\n" ) + part_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) part_sh.write( "partialator -i {0} \\\n".format( stream ) ) part_sh.write( " -o {0}.hkl \\\n".format( name ) ) part_sh.write( " -y {0} \\\n".format( pointgroup ) ) @@ -114,7 +117,7 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu # return partialator file name return part_run_file -def run_compare_check( proc_dir, name, cell, shells, part_h_res ): +def run_compare_check( proc_dir, name, cell, shells, part_h_res, crystfel_version ): # check file name check_run_file = "{0}/check_{1}.sh".format( proc_dir, name ) @@ -123,8 +126,8 @@ def run_compare_check( proc_dir, name, cell, shells, part_h_res ): check_sh = open( check_run_file, "w" ) check_sh.write( "#!/bin/sh\n\n" ) check_sh.write( "module purge\n" ) - check_sh.write( "module use MX unstable\n" ) - check_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + check_sh.write( "module use MX\n" ) + check_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) check_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) ) @@ -364,6 +367,7 @@ def make_mtz( hklout_file, mtzout_file, cell_constants, spacegroup, residues, re mtz_sh = open( mtz_run_file, "w" ) mtz_sh.write( "#!/bin/sh\n\n" ) mtz_sh.write( "module purge\n" ) + mtz_sh.write( "module use MX\n" ) mtz_sh.write( "module load ccp4/8.0\n\n" ) mtz_sh.write( "f2mtz HKLIN {0} HKLOUT {1} << EOF_hkl > f2mtz.log\n".format( hklout_file, mtzout_file ) ) mtz_sh.write( "TITLE Reflections from CrystFEL\n" ) @@ -470,7 +474,7 @@ def read_cell( cell_file ): return cell_constants -def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu, spacegroup, residues, reservation ): +def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu, spacegroup, residues, reservation, crystfel_version ): # submitted job set submitted_job_ids = set() @@ -484,8 +488,8 @@ def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h print( "making partialator file" ) # make partialator run file - part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu ) - check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res ) + part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu, crystfel_version ) + check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res, crystfel_version ) # submit job job_id = submit_job( part_run_file, reservation ) @@ -651,6 +655,14 @@ if __name__ == "__main__": type=bool, default=False ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # set loguru if not args.debug: @@ -660,4 +672,5 @@ if __name__ == "__main__": # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.spacegroup, args.residues, args.reservation ) \ No newline at end of file + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.spacegroup, args.residues, args.reservation, crystfel_version ) \ No newline at end of file diff --git a/reduction_tools/partialator_summary.py b/reduction_tools/partialator_summary.py index bcbb9ea..af88b22 100644 --- a/reduction_tools/partialator_summary.py +++ b/reduction_tools/partialator_summary.py @@ -42,6 +42,9 @@ import warnings warnings.filterwarnings( "ignore", category=RuntimeWarning ) from loguru import logger +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def submit_job( job_file ): @@ -72,7 +75,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update( completed_jobs ) time.sleep( 2 ) -def run_compare_check( proc_dir, name, cell, shells, part_h_res ): +def run_compare_check( proc_dir, name, cell, shells, part_h_res, crystfel_version ): # check file name check_run_file = "{0}/check_{1}.sh".format( proc_dir, name ) @@ -81,8 +84,8 @@ def run_compare_check( proc_dir, name, cell, shells, part_h_res ): check_sh = open( check_run_file, "w" ) check_sh.write( "#!/bin/sh\n\n" ) check_sh.write( "module purge\n" ) - check_sh.write( "module use MX unstable\n" ) - check_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + check_sh.write( "module use MX\n" ) + check_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) check_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) ) check_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) ) @@ -279,14 +282,14 @@ def summary_fig( name, stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): plt.tight_layout() plt.savefig( "{0}_plots.png".format( name ) ) -def main( cwd, name, cell, shells, part_h_res ): +def main( cwd, name, cell, shells, part_h_res, crystfel_version ): # submitted job set submitted_job_ids = set() - + # now run the check and compare scripts print( "running check/compare" ) - check_run_file = run_compare_check( cwd, name, cell, shells, part_h_res ) + check_run_file = run_compare_check( cwd, name, cell, shells, part_h_res, crystfel_version ) check_job_id = submit_job( check_run_file ) print( f"job submitted: {0}".format( check_job_id ) ) submitted_job_ids.add( check_job_id ) @@ -371,6 +374,14 @@ if __name__ == "__main__": type=bool, default=False ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # set loguru if not args.debug: @@ -380,4 +391,5 @@ if __name__ == "__main__": # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.name, args.cell_file, args.bins, args.resolution ) + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.name, args.cell_file, args.bins, args.resolution, crystfel_version ) diff --git a/reduction_tools/push_res_scan.py b/reduction_tools/push_res_scan.py index c4da4bb..6b0928b 100644 --- a/reduction_tools/push_res_scan.py +++ b/reduction_tools/push_res_scan.py @@ -35,6 +35,9 @@ import regex as re import matplotlib.pyplot as plt from scipy.optimize import curve_fit +# maps short CLI version selector to full CrystFEL module version +CRYSTFEL_VERSIONS = { 10: "0.10.2-rhel8", 12: "0.12.0" } + def submit_job( job_file ): # submit the job @@ -61,7 +64,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update(completed_jobs) time.sleep(5) -def run_partialator( proc_dir, stream, pointgroup, model, iterations, push_res, cell, bins, part_h_res, flag ): +def run_partialator( proc_dir, stream, pointgroup, model, iterations, push_res, cell, bins, part_h_res, flag, crystfel_version ): # partialator file name if flag == "push-res": @@ -73,8 +76,8 @@ def run_partialator( proc_dir, stream, pointgroup, model, iterations, push_res, part_sh = open( part_run_file, "w" ) part_sh.write( "#!/bin/sh\n\n" ) part_sh.write( "module purge\n" ) - part_sh.write( "module use MX unstable\n" ) - part_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + part_sh.write( "module use MX\n" ) + part_sh.write( "module load crystfel/{0}\n".format( crystfel_version ) ) part_sh.write( "partialator -i {0} \\\n".format( stream ) ) part_sh.write( " -o push-res_{0}.hkl \\\n".format( push_res ) ) part_sh.write( " -y {0} \\\n".format( pointgroup ) ) @@ -103,7 +106,7 @@ def make_process_dir( dir ): if e.errno != errno.EEXIST: raise -def run_push_res( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res ): +def run_push_res( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res, crystfel_version ): # list of push-res values to try push_res_list = np.arange( 0.0, 3.2, 0.2 ) @@ -127,7 +130,7 @@ def run_push_res( cwd, name, stream, pointgroup, model, iterations, cell, bins, os.chdir( push_res_dir ) # make partialator run file - part_run_file = run_partialator( push_res_dir, stream, pointgroup, model, iterations, res, cell, bins, part_h_res, flag="push-res" ) + part_run_file = run_partialator( push_res_dir, stream, pointgroup, model, iterations, res, cell, bins, part_h_res, flag="push-res", crystfel_version=crystfel_version ) # submit job job_id = submit_job( part_run_file ) @@ -339,11 +342,11 @@ def push_res_analysis( cwd, name ): print( "use push-res = {0}".format( push_res ) ) return push_res, high_res -def main( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res ): +def main( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res, crystfel_version ): # run push-res scan print( "begin push-res scan" ) - run_push_res( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res ) + run_push_res( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_res, crystfel_version ) # run analysis push_res, high_res = push_res_analysis( cwd, name ) @@ -352,18 +355,20 @@ def main( cwd, name, stream, pointgroup, model, iterations, cell, bins, part_h_r print( "best push-res = {0} and a resolution of {1}".format( push_res, high_res ) ) # make final run file final_dir = "{0}/{1}/final".format( cwd, name ) - make_process_dir( final_dir ) print( "rerunning partialator with {0} A cut off".format( high_res ) ) # check to see if directories already made if not os.path.exists( final_dir ): + # make final directory + make_process_dir( final_dir ) + # move to push-res directory os.chdir( final_dir ) # re-run final push-res submitted_job_ids = set() - final_file = run_partialator( final_dir, stream, pointgroup, model, iterations, push_res, cell, bins, part_h_res=high_res, flag="final" ) + final_file = run_partialator( final_dir, stream, pointgroup, model, iterations, push_res, cell, bins, part_h_res=high_res, flag="final", crystfel_version=crystfel_version ) # submit job job_id = submit_job( final_file ) @@ -462,8 +467,17 @@ if __name__ == "__main__": type=float, default=1.3 ) + parser.add_argument( + "-V", + "--crystfel_version", + help="which CrystFEL version to use, 10 = 0.10.2-rhel8 or 12 = 0.12.0. Default = 10.", + type=int, + choices=[ 10, 12 ], + default=10 + ) args = parser.parse_args() # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution ) + crystfel_version = CRYSTFEL_VERSIONS[ args.crystfel_version ] + main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, crystfel_version ) diff --git a/reduction_tools/stream_random_mtz.py b/reduction_tools/stream_random_mtz.py index 62220f2..04fe964 100644 --- a/reduction_tools/stream_random_mtz.py +++ b/reduction_tools/stream_random_mtz.py @@ -163,8 +163,8 @@ def run_partialator_mtz( part_mtz_py, stream_file, part_name, pointgroup, cell_f # write file part_sh = open( part_run_file, "w" ) part_sh.write( "#!/bin/sh\n\n" ) - part_sh.write( "source /sf/cristallina/applications/mx/conda/miniconda/bin/activate\n" ) - part_sh.write( "conda activate crmx38-analysis\n\n" ) + part_sh.write( "source /sf/daq/source-conda\n" ) + part_sh.write( "conda activate /sf/cristallina/config/python/.pixi/envs/crmx-proc\n\n" ) part_sh.write( "python {0} -n merge_{1}".format( part_mtz_py, part_name ) ) part_sh.write( " -s {0}".format( stream_file ) ) part_sh.write( " -p {0}".format( pointgroup ) )