updated with temporay crystfel version

reduction_tools/crystfel_split.py

@@ -150,7 +150,8 @@ def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file, geom_file, cell_f
     run_sh = open( cryst_run_file, "w" )
     run_sh.write( "#!/bin/sh\n\n" )
     run_sh.write( "module purge\n" )
-    run_sh.write( "module load crystfel/0.10.2\n" )
+    run_sh.write( "module use MX unstable\n" )
+    run_sh.write( "module load crystfel/0.10.2-rhel8\n" )
     run_sh.write( "indexamajig -i {0} \\\n".format( chunk_lst_file ) )
     run_sh.write( "  --output={0} \\\n".format( stream_file ) )
     run_sh.write( "  --geometry={0} \\\n".format( geom_file ) )

reduction_tools/crystfel_split_var.py

@@ -5,7 +5,8 @@
 """
 # aim
 to process a batch of data very fast by splitting it into a number of chunks and submitting 
-these jobs separately to the cluster
+these jobs separately to the cluster - but now all with the ability to change crystfel parameters
+from the command line
 
 # usage
 python crystfel_split.py -l <path-to-list-file>
@@ -13,7 +14,6 @@ python crystfel_split.py -l <path-to-list-file>
                          -g <path-to-geom-file>                          
                          -c <path-to-cell-file>
                          -n <job-name>                              -default split 
-# note                   -p progress bar (True/False)               -default True
                          -t crystfel threshold                      -default 10
                          -s crystfel min-snr                        -default 5
                          -i crystfel int-radius                     -default 3,5,9
@@ -54,7 +54,7 @@ def h5_split( lst, chunk_size ):
 
 def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file,
                         geom_file, cell_file, threshold, min_snr,
-                        int_rad, multi, retry, min_pix ):
+                        int_rad, multi, retry, min_pix, bg_rad, min_res ):
 
     # stream file name
     stream_file = "{0}_{1}.stream".format( name, chunk )
@@ -65,20 +65,27 @@ def write_crystfel_run( proc_dir, name, chunk, chunk_lst_file,
     run_sh = open( cryst_run_file, "w" )
     run_sh.write( "#!/bin/sh\n\n" )
     run_sh.write( "module purge\n" )
-    run_sh.write( "module load crystfel/0.10.2\n" )
+    run_sh.write( "module use MX unstable\n" )
+    run_sh.write( "module load crystfel/0.10.2-rhel8\n" )
     run_sh.write( "indexamajig -i {0} \\\n".format( chunk_lst_file ) )
     run_sh.write( "  --output={0} \\\n".format( stream_file ) )
     run_sh.write( "  --geometry={0} \\\n".format( geom_file ) )
     run_sh.write( "  --pdb={0} \\\n".format( cell_file ) )
-    run_sh.write( "  --indexing=xgandalf-latt-cell --peaks=peakfinder8 \\\n" )
+    run_sh.write( "  --indexing=xgandalf-latt-cell \\\n" )
+    run_sh.write( "  --peaks=peakfinder8 \\\n" )
+    run_sh.write( "  --integration=rings-grad \\\n" )
+    run_sh.write( "  --tolerance=10.0,10.0,10.0,2,3,2 \\\n" )
     run_sh.write( "  --threshold={0} \\\n".format( threshold ) )
     run_sh.write( "  --min-snr={0} \\\n".format( min_snr ) )
     run_sh.write( "  --int-radius={0},{1},{2} \\\n".format( int_rad[0], int_rad[1], int_rad[2] ) )
-    run_sh.write( "  -j 32 --max-res=3000\\\n" )
+    run_sh.write( "  -j 32 \\\n" )
     run_sh.write( "  --{0} \\\n".format( multi ) )
+    run_sh.write( "  --check-peaks \\\n" )
     run_sh.write( "  --{0} \\\n".format( retry ) )
-    run_sh.write( "  --min-pix-count={0}\\\n".format( min_pix ) )
+    run_sh.write( "  --max-res=3000 \\\n" )
-    run_sh.write( "  --min-res=85\\\n" )
+    run_sh.write( "  --min-pix-count={0} \\\n".format( min_pix ) )
+    run_sh.write( "  --local-bg-radius={0} \\\n".format( bg_rad ) )
+    run_sh.write( "  --min-res={0}".format( min_res ) )
     run_sh.close()
 
     # make file executable
@@ -175,10 +182,8 @@ def run_splits( cwd, name, lst, chunk_size, geom_file,
 
     print( "DONE" )
 
-    # include progress bar if required
+    wait_for_jobs(submitted_job_ids, chunk)
-    if progress==True:
+    print("slurm processing done")
-        wait_for_jobs(submitted_job_ids, chunk)
-        print("slurm processing done")
     
     # make composite .stream file
     output_file = "{0}.stream".format( name )
@@ -237,13 +242,6 @@ if __name__ == "__main__":
         type=str,
         default="split"
     )
-    parser.add_argument(
-        "-p",
-        "--progress",
-        help="gives you the option of also having a progress bar",
-        type=bool,
-        default=True
-    )
     parser.add_argument(
         "-t",
         "--threshold",
@@ -286,6 +284,20 @@ if __name__ == "__main__":
         type=int,
         default=2
     )
+    parser.add_argument(
+        "-b",
+        "--bg_rad",
+        help="crystfel background radius flag, radius (in pixels) used for the estimation of the local background",
+        type=int,
+        default=2
+    )
+    parser.add_argument(
+        "-q",
+        "--min-res",
+        help="m",
+        type=int,
+        default=2
+    )
     args = parser.parse_args()
     # run geom converter
     cwd = os.getcwd()
@@ -298,6 +310,6 @@ if __name__ == "__main__":
     else:
         retry = "no-retry"
     run_splits( cwd, args.job_name, args.lst_file, args.chunk_size,
-                args.geom_file, args.cell_file, args.progress,
+                args.geom_file, args.cell_file,
                 args.threshold, args.min_snr, args.int_radius, 
                 multi, retry, args.min_pix )

reduction_tools/partialator.py

@@ -71,7 +71,8 @@ def run_partialator( proc_dir, name, stream, pointgroup, model, iterations, cell
     part_sh = open( part_run_file, "w" )
     part_sh.write( "#!/bin/sh\n\n" )
     part_sh.write( "module purge\n" )
-    part_sh.write( "module load crystfel/0.10.2\n" )
+    part_sh.write( "module use MX unstable\n" )
+    part_sh.write( "module load crystfel/0.10.2-rhel8\n" )
     part_sh.write( "partialator -i {0} \\\n".format( stream ) )
     part_sh.write( "  -o merged_{0}.hkl \\\n".format( name ) )
     part_sh.write( "  -y {0} \\\n".format( pointgroup ) )

reduction_tools/push_res_scan.py

@@ -73,7 +73,8 @@ def run_partialator( proc_dir, stream, pointgroup, model, iterations, push_res,
     part_sh = open( part_run_file, "w" )
     part_sh.write( "#!/bin/sh\n\n" )
     part_sh.write( "module purge\n" )
-    part_sh.write( "module load crystfel/0.10.2\n" )
+    part_sh.write( "module use MX unstable\n" )
+    part_sh.write( "module load crystfel/0.10.2-rhel8\n" )
     part_sh.write( "partialator -i {0} \\\n".format( stream ) )
     part_sh.write( "  -o push-res_{0}.hkl \\\n".format( push_res ) )
     part_sh.write( "  -y {0} \\\n".format( pointgroup ) )