diff --git a/scripts/export_file.py b/scripts/export_file.py index 5e0a21b..c8eb70c 100644 --- a/scripts/export_file.py +++ b/scripts/export_file.py @@ -59,14 +59,23 @@ with ju.File( mask=mask, gap_pixels=gap_pixels, geometry=geometry, - parallel=False, + parallel=True, ) as juf: n_input_frames = len(juf["data"]) good_frames = np.nonzero(juf["is_good_frame"])[0] n_output_frames = len(good_frames) juf.handler.mask_double_pixels = mask_double_pixels - juf.export(args.file_out, index=good_frames, roi=None, compression=compression, factor=factor, dtype=dtype, batch_size=500) + juf.export( + args.file_out, + index=good_frames, + roi=None, + compression=compression, + factor=factor, + dtype=dtype, + batch_size=25, + ) + pixel_mask = juf.handler.get_pixel_mask(gap_pixels=gap_pixels, geometry=geometry) # Postprocessing diff --git a/scripts/make_crystfel_list.py b/scripts/make_crystfel_list.py index 3bc9c41..77ae477 100644 --- a/scripts/make_crystfel_list.py +++ b/scripts/make_crystfel_list.py @@ -62,7 +62,7 @@ except: print(f"Can't open {data_file}") exit() -detector = 'JF06T08V01' +detector = 'JF06T32V02' pulseids = f[f'/data/{detector}/pulse_id'][:] n_pulse_id = len(pulseids) diff --git a/scripts/retrieve_detector_data.sh b/scripts/retrieve_detector_data.sh index 362f27f..228f71d 100755 --- a/scripts/retrieve_detector_data.sh +++ b/scripts/retrieve_detector_data.sh @@ -55,11 +55,11 @@ case ${DETECTOR} in NM=9 DET_CONFIG_FILE=/gpfs/photonics/swissfel/buffer/config/stream-JF02.json ;; -'JF06T32V01') +'JF06T32V02') NM=32 DET_CONFIG_FILE=/gpfs/photonics/swissfel/buffer/config/stream-JF06.json ;; -'JF06T08V01') +'JF06T08V02') NM=8 DET_CONFIG_FILE=/gpfs/photonics/swissfel/buffer/config/stream-JF06_4M.json ;; @@ -75,7 +75,8 @@ case ${DETECTOR} in NM=1 esac -coreAssociated="7,8,9,10,11,12,13,14" +#coreAssociated="7,8,9,10,11,12,13,14" +coreAssociated="9,10,11,12,13,14,15,16,17" touch /tmp/detector_retrieve.log @@ -85,8 +86,14 @@ PREVIOUS_STILL_RUN=0 while [ ${PREVIOUS_STILL_RUN} == 0 ] do sleep 15 # we need to sleep at least to make sure that we don't read from CURRENT file - ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer > /dev/null - PREVIOUS_STILL_RUN=$? # not found == 1 +# ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer > /dev/null +# PREVIOUS_STILL_RUN=$? # not found == 1 +# PREVIOUS_STILL_RUN=1 + n=`ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer | wc -l` + if [ ${n} -le 10 ] + then + PREVIOUS_STILL_RUN=1 + fi done date2=$(date +%s) @@ -116,7 +123,8 @@ taskset -c ${coreAssociated} /usr/bin/sf_writer ${OUTFILE_RAW} /gpfs/photonics/s wait -coreAssociatedConversion="35,34,33,32,31,30,29,28,27" +#coreAssociatedConversion="35,34,33,32,31,30,29,28,27" +coreAssociatedConversion="35,34,33,32,31,30,29,28,27,26,25,24,23,22,21,20,19,18" date3=$(date +%s) echo "Finished : "`date` @@ -128,13 +136,29 @@ then echo "File is written in raw format, no compression" else echo "Will call compression/convertion ${OUTFILE_RAW} --> ${OUTFILE}" + + PREVIOUS_STILL_RUN=0 + while [ ${PREVIOUS_STILL_RUN} == 0 ] + do + sleep 15 # we need to sleep at least to make sure that we don't read from CURRENT file + n=`ps -fe | grep "scripts/export_file.py " | grep -v grep | grep export | wc -l` + if [ ${n} -le 10 ] + then + PREVIOUS_STILL_RUN=1 + fi + done + date4=$(date +%s) + echo -n "Sleep Time : " + echo $((date4-date3)) | awk '{print int($1/60)":"int($1%60)}' + export PATH=/home/dbe/miniconda3/bin:$PATH source deactivate >/dev/null 2>&1 source activate conversion taskset -c ${coreAssociatedConversion} python /home/dbe/git/sf_daq_buffer/scripts/export_file.py ${OUTFILE_RAW} ${OUTFILE} ${RUN_FILE} ${DET_CONFIG_FILE} python /home/dbe/git/sf_daq_buffer/scripts/make_crystfel_list.py ${OUTFILE} ${RUN_FILE} - date4=$(date +%s) + date5=$(date +%s) echo "Finished : "`date` - echo -n "Convertion Time : " - echo $((date4-date3)) | awk '{print int($1/60)":"int($1%60)}' + echo -n "Conversion Time : " + echo $((date5-date4)) | awk '{print int($1/60)":"int($1%60)}' + fi