Merge branch 'master' of https://github.com/paulscherrerinstitute/sf_daq_buffer

scripts/JF01-stream.service

@@ -8,7 +8,7 @@ User=root
 ExecStart=/usr/bin/sh /home/writer/git/sf_daq_buffer/scripts/JF01-stream.sh
 TimeoutStartSec=10
 Restart=on-failure
-RestartSec=10
+RestartSec=1
 
 [Install]
 WantedBy=multi-user.target

scripts/retrieve_detector_data.sh

@@ -86,9 +86,6 @@ PREVIOUS_STILL_RUN=0
 while [ ${PREVIOUS_STILL_RUN} == 0 ]
 do
     sleep 15 # we need to sleep at least to make sure that we don't read from CURRENT file
-#    ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer > /dev/null 
-#    PREVIOUS_STILL_RUN=$? # not found == 1
-#    PREVIOUS_STILL_RUN=1
     n=`ps -fe | grep "bin/sf_writer " | grep -v grep | grep sf_writer | wc -l`
     if [ ${n} -lt 9 ]
     then
@@ -119,7 +116,7 @@ else
     fi
 fi
 
-taskset -c ${coreAssociated} /usr/bin/sf_writer ${OUTFILE_RAW} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${NM} ${START_PULSE_ID} ${STOP_PULSE_ID} ${PULSE_ID_STEP}>> /tmp/detector_retrieve.log &
+taskset -c ${coreAssociated} /usr/local/bin/sf_writer ${OUTFILE_RAW} /gpfs/photonics/swissfel/buffer/${DETECTOR} ${NM} ${START_PULSE_ID} ${STOP_PULSE_ID} ${PULSE_ID_STEP}>> /tmp/detector_retrieve.log &
 
 wait
 
@@ -129,6 +126,9 @@ coreAssociatedConversion="35,34,33,32,31,30,29,28,27,26,25,24,23,22,21,20,19,18"
 #TODO: calculate this number from coreAssociatedConversion
 #export NUMBA_NUM_THREADS=18
 
+#not clear why, but bitshuffle doesn't respect OMP_NUM_THREADS set in jungfrau_utils anymore, thus we set it here
+export OMP_NUM_THREADS=1
+
 date3=$(date +%s)
 echo "Finished                : "`date`
 echo -n "Retrieve Time : "
@@ -159,7 +159,7 @@ else
     source /home/dbe/miniconda3/etc/profile.d/conda.sh
 
     conda deactivate
-    conda activate bsread
+    conda activate sf-daq
 
     taskset -c ${coreAssociatedConversion} python /home/dbe/git/sf_daq_buffer/scripts/export_file.py ${OUTFILE_RAW} ${OUTFILE} ${RUN_FILE} ${DET_CONFIG_FILE}
     if [ ${DETECTOR} == "JF06T32V02" ] || [ ${DETECTOR} == "JF06T08V02" ]

scripts/test_conversion.sh

@@ -3,12 +3,13 @@
 export PATH=/home/dbe/miniconda3/bin:$PATH
 source /home/dbe/miniconda3/etc/profile.d/conda.sh
 conda deactivate
-conda activate bsread
+conda activate sf-daq
 
+export OMP_NUM_THREADS=1
 
 #export NUMBA_NUM_THREADS=$1
 #OUTDIR=/sf/alvra/data/p18674/raw/run_info/003000/CONVERSION-PAR-${NUMBA_NUM_THREADS}
-OUTDIR=/sf/alvra/data/p18674/raw/run_info/003000/CONVERSION-NEW.20-daq2
+OUTDIR=/sf/alvra/data/p18674/raw/run_info/003000/CONVERSION-NEW.21-daq1
 
 #coreAssociatedBuffer=(35 34 33 32 31 30 29 28 27 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 8 7 6 5 4 3 2 1 0)
 #coreAssociatedBuffer=(35 34 33 32 31 30 29 28 27 18 19 20 21 22 23 24 25 26 9 10 11 12 13 14 15 16 17 8 7 6 5 4 3 2 1 0)

sf-stream/src/ZmqLiveSender.cpp

@@ -4,6 +4,8 @@
 #include "zmq.h"
 #include <stdexcept>
 
+#include <iostream>
+//
 using namespace std;
 using namespace stream_config;
 
@@ -98,6 +100,9 @@ void ZmqLiveSender::send(const ModuleFrameBuffer *meta, const char *data)
 
             if (module_metadata.n_recv_packets != 128 ) is_good_frame = false;
         }
+        if (pulse_id % 10000 == 0 && is_good_frame != true) {
+            cout << "Frame is not good " << pulse_id << " module : " << i_module << " frame_index(0) : " << frame_index << " frame_index : " << module_metadata.frame_index << endl;
+        }
     }
 
     // TODO: Here we need to send to streamvis and live analysis metadata(probably need to operate still on them) and data(not every frame)