sf-mx/crystfel_tools
0
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changes names of output files

Browse Source
This commit is contained in:
Beale John Henry
2025-01-19 22:43:34 +01:00
parent 753f0665c3
commit db017a52e1
1 changed files with 53 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
reduction_tools/partialator.py
View File

@@ -16,7 +16,7 @@ python partialator.py -s <path-to-stream-file>
-b number of resolution bins - must be > 20
-r high-res limt. Needs a default. Default set to 1.3
-a max-adu. Default = 12000
-R ra reservation name if available
-v ra reservation name if available
# output
- scaled/merged files
@@ -57,14 +57,14 @@ def submit_job( job_file, reservation ):
job_output = subprocess.check_output( submit_cmd )
logger.info( "submited job = {0}".format( job_output ) )
except subprocess.CalledProcessError as e:
print( "please give the correct ra reservation or remove the -R from the arguements" )
print( "please give the correct ra reservation or remove the -v from the arguements" )
exit()
# scrub job id from - example Submitted batch job 742403
pattern = r"Submitted batch job (\d+)"
job_id = re.search( pattern, job_output.decode().strip() ).group(1)
return int(job_id)
return int( job_id )
def wait_for_jobs( job_ids, total_jobs ):
@@ -80,7 +80,7 @@ def wait_for_jobs( job_ids, total_jobs ):
job_ids.difference_update( completed_jobs )
time.sleep( 2 )
def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu ):
def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, adu ):
# partialator file name
part_run_file = "{0}/partialator_{1}.sh".format( proc_dir, name )
@@ -98,12 +98,6 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell
part_sh.write( " --max-adu={0} \\\n".format( adu ) )
part_sh.write( " -j 32 \\\n" )
part_sh.write( " --iterations={0}\n\n".format( iterations ) )
part_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) )
part_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) )
part_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) )
part_sh.write( "compare_hkl --fom=Rsplit --shell-file=Rsplit.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> Rsplit.log\n".format( cell, shells, part_h_res ) )
part_sh.write( "compare_hkl --fom=cc --shell-file=cc.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> cc.log\n".format( cell, shells, part_h_res ) )
part_sh.write( "compare_hkl --fom=ccstar --shell-file=ccstar.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> ccstar.log\n".format( cell, shells, part_h_res ) )
part_sh.close()
# make file executable
@@ -117,6 +111,36 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell
# return partialator file name
return part_run_file
def run_compare_check( proc_dir, name, cell, shells, part_h_res ):
# check file name
check_run_file = "{0}/check_{1}.sh".format( proc_dir, name )
# write file
check_sh = open( check_run_file, "w" )
check_sh.write( "#!/bin/sh\n\n" )
check_sh.write( "module purge\n" )
check_sh.write( "module use MX unstable\n" )
check_sh.write( "module load crystfel/0.10.2-rhel8\n" )
check_sh.write( "check_hkl --shell-file=mult.dat *.hkl -p {0} --nshells={1} --highres={2} &> check_hkl.log\n".format( cell, shells, part_h_res ) )
check_sh.write( "check_hkl --ltest --ignore-negs --shell-file=ltest.dat *.hkl -p {0} --nshells={1} --highres={2} &> ltest.log\n".format( cell, shells, part_h_res ) )
check_sh.write( "check_hkl --wilson --shell-file=wilson.dat *.hkl -p {0} --nshells={1} --highres={2} &> wilson.log\n".format( cell, shells, part_h_res ) )
check_sh.write( "compare_hkl --fom=Rsplit --shell-file=Rsplit.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> Rsplit.log\n".format( cell, shells, part_h_res ) )
check_sh.write( "compare_hkl --fom=cc --shell-file=cc.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> cc.log\n".format( cell, shells, part_h_res ) )
check_sh.write( "compare_hkl --fom=ccstar --shell-file=ccstar.dat *.hkl1 *hkl2 -p {0} --nshells={1} --highres={2} &> ccstar.log\n".format( cell, shells, part_h_res ) )
check_sh.close()
# make file executable
subprocess.call( [ "chmod", "+x", "{0}".format( check_run_file ) ] )
# add check script to log
check_input = open( check_run_file, "r" )
logger.info( "check input file =\n{0}".format( check_input.read() ) )
check_input.close()
# return check file name
return check_run_file
def make_process_dir( dir ):
# make process directory
try:
@@ -204,8 +228,10 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ):
def dto1_d( x ):
return 1/x
def dto1_d2( x ):
def dto1_d2_rev( x ):
return 1/x**2
def dto1_d2_for( x ):
return 1/x**0.5
# plot results
cc_fig, axs = plt.subplots(2, 2)
@@ -221,7 +247,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ):
axs[0,0].plot( stats_df[ "1_d" ], stats_df.cc, color=color )
# plot fit
axs[0,0].plot( stats_df[ "1_d" ], cc_tanh, color="tab:grey", linestyle = "dashed" )
sax1 = axs[0,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ) )
sax1 = axs[0,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ), rotation=90 )
sax1.set_xlabel('d (A)')
axs[0,0].tick_params( axis="y", labelcolor=color )
axs[0,0].text( 0.1, 0.1, "CC @ 0.2 = {0}".format( cc_cut ), fontsize = 8 )
@@ -246,7 +272,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ):
axs[1,0].set_ylabel( "Rsplit", color=color )
axs[1,0].plot( stats_df[ "1_d" ], stats_df.rsplit, color=color )
sax3 = axs[1,0].secondary_xaxis( 'top', functions=( dto1_d, dto1_d ) )
sax3.set_xlabel('d (A)')
sax3.set_xlabel( 'd (A)' )
axs[1,0].tick_params( axis='y', labelcolor=color )
@@ -255,7 +281,7 @@ def summary_fig( stats_df, cc_tanh, ccstar_tanh, cc_cut, ccstar_cut ):
axs[1,1].set_xlabel( "1/d**2 (1/A**2)" )
axs[1,1].set_ylabel( "lnI", color=color )
axs[1,1].plot( stats_df[ "1_d2" ], stats_df.lnI, color=color )
sax4 = axs[1,1].secondary_xaxis( 'top', functions=( dto1_d2, dto1_d2 ) )
sax4 = axs[1,1].secondary_xaxis( 'top', functions=( dto1_d2_for, dto1_d2_rev ) )
sax4.set_xlabel( "d (A)" )
axs[1,1].tick_params( axis='y', labelcolor=color )
@@ -277,17 +303,24 @@ def main( cwd, name, stream, pointgroup, model, iterations, cell, shells, part_h
print( "making partialator file" )
# make partialator run file
part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, cell, shells, part_h_res, adu )
part_run_file = run_partialator( part_dir, name, stream, pointgroup, model, iterations, adu )
check_run_file = run_compare_check( part_dir, name, cell, shells, part_h_res )
# submit job
job_id = submit_job( part_run_file, reservation )
print(f"job submitted: {0}".format( job_id ) )
print( f"job submitted: {0}".format( job_id ) )
submitted_job_ids.add( job_id )
# use progress bar to track job completion
time.sleep(10)
wait_for_jobs(submitted_job_ids, 1 )
print("slurm processing done")
wait_for_jobs( submitted_job_ids, 1 )
print( "slurm processing done" )
# now run the check and compare scripts
print( "running check/compare" )
submit_cmd = [ "{0}".format( check_run_file ) ]
subprocess.call( submit_cmd )
print( "done" )
# stats files names
cc_dat = "cc.dat"
@@ -385,7 +418,7 @@ if __name__ == "__main__":
default=12000
)
parser.add_argument(
"-R",
"-v",
"--reservation",
help="reservation name for ra cluster. Usually along the lines of P11111_2024-12-10",
type=str,
@@ -407,8 +440,4 @@ if __name__ == "__main__":
# run main
cwd = os.getcwd()
print( "top working directory = {0}".format( cwd ) )
main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation )
main( cwd, args.name, args.stream_file, args.pointgroup, args.model, args.iterations, args.cell_file, args.bins, args.resolution, args.max_adu, args.reservation )