diff --git a/reduction_tools/partialator.py b/reduction_tools/partialator.py index 1b856f1..fb30cae 100644 --- a/reduction_tools/partialator.py +++ b/reduction_tools/partialator.py @@ -16,7 +16,7 @@ python partialator.py -s -b number of resolution bins - must be > 20 -r high-res limt. Needs a default. Default set to 1.3 -a max-adu. Default = 12000 - -R ra reservation name if available + -v ra reservation name if available # output - scaled/merged files @@ -57,14 +57,14 @@ def submit_job( job_file, reservation ): job_output = subprocess.check_output( submit_cmd ) logger.info( "submited job = {0}".format( job_output ) ) except subprocess.CalledProcessError as e: - print( "please give the correct ra reservation or remove the -R from the arguements" ) + print( "please give the correct ra reservation or remove the -v from the arguements" ) exit() # scrub job id from - example Submitted batch job 742403 pattern = r"Submitted batch job (\d+)" job_id = re.search( pattern, job_output.decode().strip() ).group(1) - return int(job_id) + return int( job_id ) def wait_for_jobs( job_ids, total_jobs ): @@ -80,7 +80,7 @@ def wait_for_jobs( job_ids, total_jobs ): job_ids.difference_update( completed_jobs ) time.sleep( 2 ) -def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu ): +def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu ): # partialator file name part_run_file = "{0}/partialator_{1}.sh".format( proc_dir, name ) @@ -98,12 +98,6 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell part_sh.write( " --max-adu={0} \\\n".format( adu ) ) part_sh.write( " -j 32 \\\n" ) part_sh.write( " --iterations={0}\n\n".format( iterations ) ) - part_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) ) - part_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) ) - part_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) ) - part_sh.write( "compare_hkl --fom=Rsplit --shell-file=Rsplit.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> Rsplit.log\n".format( cell, shells, part_h_res ) ) - part_sh.write( "compare_hkl --fom=cc --shell-file=cc.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> cc.log\n".format( cell, shells, part_h_res ) ) - part_sh.write( "compare_hkl --fom=ccstar --shell-file=ccstar.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> ccstar.log\n".format( cell, shells, part_h_res ) ) part_sh.close() # make file executable @@ -117,6 +111,36 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell # return partialator file name return part_run_file +def run_compare_check( proc_dir, name, cell, shells, part_h_res ): + + # check file name + check_run_file = "{0}/check_{1}.sh".format( proc_dir, name ) + + # write file + check_sh = open( check_run_file, "w" ) + check_sh.write( "#!/bin/sh\n\n" ) + check_sh.write( "module purge\n" ) + check_sh.write( "module use MX unstable\n" ) + check_sh.write( "module load crystfel/0.10.2-rhel8\n" ) + check_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) ) + check_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) ) + check_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) ) + check_sh.write( "compare_hkl --fom=Rsplit --shell-file=Rsplit.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> Rsplit.log\n".format( cell, shells, part_h_res ) ) + check_sh.write( "compare_hkl --fom=cc --shell-file=cc.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> cc.log\n".format( cell, shells, part_h_res ) ) + check_sh.write( "compare_hkl --fom=ccstar --shell-file=ccstar.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> ccstar.log\n".format( cell, shells, part_h_res ) ) + check_sh.close() + + # make file executable + subprocess.call( [ "chmod", "+x", "{0}".format( check_run_file ) ] ) + + # add check script to log + check_input = open( check_run_file, "r" ) + logger.info( "check input file =\n{0}".format( check_input.read() ) ) + check_input.close() + + # return check file name + return check_run_file + def make_process_dir( dir ): # make process directory try: @@ -204,8 +228,10 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): def dto1_d( x ): return 1/x - def dto1_d2( x ): + def dto1_d2_rev( x ): return 1/x**2 + def dto1_d2_for( x ): + return 1/x**0.5 # plot results cc_fig, axs = plt.subplots(2, 2) @@ -221,7 +247,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): axs[0,0].plot( stats_df[ "1_d" ], stats_df.cc, color=color ) # plot fit axs[0,0].plot( stats_df[ "1_d" ], cc_tanh, color="tab:grey", linestyle = "dashed" ) - sax1 = axs[0,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ) ) + sax1 = axs[0,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ), rotation=90 ) sax1.set_xlabel('d (A)') axs[0,0].tick_params( axis="y", labelcolor=color ) axs[0,0].text( 0.1, 0.1, "CC @ 0.2 = {0}".format( cc_cut ), fontsize = 8 ) @@ -246,7 +272,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): axs[1,0].set_ylabel( "Rsplit", color=color ) axs[1,0].plot( stats_df[ "1_d" ], stats_df.rsplit, color=color ) sax3 = axs[1,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ) ) - sax3.set_xlabel('d (A)') + sax3.set_xlabel( 'd (A)' ) axs[1,0].tick_params( axis='y', labelcolor=color ) @@ -255,7 +281,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ): axs[1,1].set_xlabel( "1/d**2 (1/A**2)" ) axs[1,1].set_ylabel( "lnI", color=color ) axs[1,1].plot( stats_df[ "1_d2" ], stats_df.lnI, color=color ) - sax4 = axs[1,1].secondary_xaxis( 'top', functions=( dto1_d2, dto1_d2 ) ) + sax4 = axs[1,1].secondary_xaxis( 'top', functions=( dto1_d2_for, dto1_d2_rev ) ) sax4.set_xlabel( "d (A)" ) axs[1,1].tick_params( axis='y', labelcolor=color ) @@ -277,17 +303,24 @@ def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h print( "making partialator file" ) # make partialator run file - part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu ) + part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu ) + check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res ) # submit job job_id = submit_job( part_run_file, reservation ) - print(f"job submitted: {0}".format( job_id ) ) + print( f"job submitted: {0}".format( job_id ) ) submitted_job_ids.add( job_id ) # use progress bar to track job completion time.sleep(10) - wait_for_jobs(submitted_job_ids, 1 ) - print("slurm processing done") + wait_for_jobs( submitted_job_ids, 1 ) + print( "slurm processing done" ) + + # now run the check and compare scripts + print( "running check/compare" ) + submit_cmd = [ "{0}".format( check_run_file ) ] + subprocess.call( submit_cmd ) + print( "done" ) # stats files names cc_dat = "cc.dat" @@ -385,7 +418,7 @@ if __name__ == "__main__": default=12000 ) parser.add_argument( - "-R", + "-v", "--reservation", help="reservation name for ra cluster. Usually along the lines of P11111_2024-12-10", type=str, @@ -407,8 +440,4 @@ if __name__ == "__main__": # run main cwd = os.getcwd() print( "top working directory = {0}".format( cwd ) ) - main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation ) - - - - + main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation ) \ No newline at end of file