adaptation of expert script to jungfrau_utils changes by Ivan; use of second half of machine for conversion

scripts/export_file.py

@@ -46,11 +46,6 @@ if not mask and mask_double_pixels:
     warnings.warn("mask_double_pixels set to False")
     mask_double_pixels = False
 
-if factor:
-    dtype = np.int32
-else:
-    dtype = None
-
 with ju.File(
     args.file_in,
     gain_file=gain_file,
@@ -59,7 +54,7 @@ with ju.File(
     mask=mask,
     gap_pixels=gap_pixels,
     geometry=geometry,
-    parallel=True,
+    parallel=False,
 ) as juf:
     n_input_frames = len(juf["data"])
     good_frames = np.nonzero(juf["is_good_frame"])[0]
@@ -72,8 +67,8 @@ with ju.File(
         roi=None,
         compression=compression,
         factor=factor,
-        dtype=dtype,
-        batch_size=25,
+        dtype=None,
+        batch_size=35,
     )
 
     pixel_mask = juf.handler.get_pixel_mask(gap_pixels=gap_pixels, geometry=geometry)

scripts/retrieve_detector_data.sh

@@ -90,7 +90,7 @@ do
 #    PREVIOUS_STILL_RUN=$? # not found == 1
 #    PREVIOUS_STILL_RUN=1
     n=`ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer | wc -l`
-    if [ ${n} -le 10 ]
+    if [ ${n} -lt 9 ]
     then
         PREVIOUS_STILL_RUN=1
     fi
@@ -125,6 +125,8 @@ wait
 
 #coreAssociatedConversion="35,34,33,32,31,30,29,28,27"
 coreAssociatedConversion="35,34,33,32,31,30,29,28,27,26,25,24,23,22,21,20,19,18"
+#TODO: calculate this number from coreAssociatedConversion
+#export NUMBA_NUM_THREADS=18
 
 date3=$(date +%s)
 echo "Finished                : "`date`
@@ -142,7 +144,7 @@ else
     do
         sleep 15 # we need to sleep at least to make sure that we don't read from CURRENT file
         n=`ps -fe | grep "scripts/export_file.py " | grep -v grep | grep export | wc -l`
-        if [ ${n} -le 10 ]
+        if [ ${n} -lt 18 ]
         then
             PREVIOUS_STILL_RUN=1
         fi
@@ -154,7 +156,7 @@ else
     export PATH=/home/dbe/miniconda3/bin:$PATH
     source deactivate >/dev/null 2>&1
     source activate conversion
-    taskset -c ${coreAssociatedConversion} python /home/dbe/git/sf_daq_buffer/scripts/export_file.py ${OUTFILE_RAW} ${OUTFILE} ${RUN_FILE} ${DET_CONFIG_FILE} 
+    taskset -c ${coreAssociatedConversion} python /home/dbe/git/sf_daq_buffer/scripts/export_file.py ${OUTFILE_RAW} ${OUTFILE} ${RUN_FILE} ${DET_CONFIG_FILE}
     python /home/dbe/git/sf_daq_buffer/scripts/make_crystfel_list.py ${OUTFILE} ${RUN_FILE}
     date5=$(date +%s)
     echo "Finished                : "`date`