copied files from sf-dap repository
This commit is contained in:
Dmitry Ozerov
50
scripts/ap.sh
Executable file
50
scripts/ap.sh
Executable file
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
|
||||
. ./env_setup.sh
|
||||
|
||||
while true
|
||||
do
|
||||
echo "Running "`date`
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/automatic2.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK}
|
||||
cd output
|
||||
for i in `ls *base.out 2>/dev/null`
|
||||
do
|
||||
if [ -s $i ]
|
||||
then
|
||||
f=`echo $i | sed 's/\.out//'`
|
||||
cp $f a.sh
|
||||
chmod +x a.sh
|
||||
cd ..
|
||||
output/a.sh > output/$i.out 2>&1
|
||||
cd output
|
||||
rm -rf a.sh
|
||||
rm -rf $i
|
||||
if [ -s $i.out ]
|
||||
then
|
||||
mv $i.out $i
|
||||
else
|
||||
rm -rf $i.out
|
||||
fi
|
||||
else
|
||||
rm -rf $i
|
||||
fi
|
||||
done
|
||||
cd ..
|
||||
|
||||
rm -rf CURRENT_JOBS.txt
|
||||
echo " statistics at "`date` >> CURRENT_JOBS.txt
|
||||
echo " Running jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt
|
||||
echo " " >> CURRENT_JOBS.txt
|
||||
echo " Pending jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt
|
||||
|
||||
cd output
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.sh
|
||||
cd ..
|
||||
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.2.sh
|
||||
|
||||
echo "Sleeping "`date`
|
||||
sleep 120
|
||||
done
|
||||
28
scripts/env_setup.sh
Executable file
28
scripts/env_setup.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
|
||||
DIRNAME=`dirname ${BASH_SOURCE}`
|
||||
|
||||
#export PGROUP=`cat ${DIRNAME}/PGROUP`
|
||||
export PGROUP=...
|
||||
|
||||
export BEAMLINE=alvra
|
||||
# alvra bernina cristallina furka maloja
|
||||
|
||||
BASEDIR=/sf/${BEAMLINE}/data/${PGROUP}/res
|
||||
|
||||
source /sf/jungfrau/applications/miniconda3/etc/profile.d/conda.sh
|
||||
conda activate sf-dap
|
||||
|
||||
export DETN=JF06T08V03
|
||||
# JF06T08V03 JF06T32V03 JF17T16V01
|
||||
|
||||
export GEOM_FILE=${DETN}.geom
|
||||
|
||||
#for Alvra(rounding 0.25): 4000 and lower
|
||||
#for Cristallina (photon counts): 10
|
||||
# edit run_index.*.sh file(s) for indexing options
|
||||
export THRESHOLD_INDEXING=10
|
||||
|
||||
#LOGBOOK="https://docs.google.com/spreadsheets/...."
|
||||
|
||||
SRCDIR=$PWD
|
||||
140
scripts/index_data.sh
Normal file
140
scripts/index_data.sh
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
##SBATCH -p hour
|
||||
##SBATCH -t 1:00:00
|
||||
#SBATCH -e output/index_data.%j.err
|
||||
#SBATCH -o output/index_data.%j.out
|
||||
##SBATCH -x sf-cn-[5-7]
|
||||
|
||||
. ./env_setup.sh
|
||||
|
||||
DIRN=$1
|
||||
RUN=$2
|
||||
FILELST=$3
|
||||
PROTEIN_NAME=$4
|
||||
UNIQUERUN=$5
|
||||
|
||||
OUTDIR=`echo ${DIRN} | sed 's/data/index/'`
|
||||
|
||||
FR=`printf "%04d" ${RUN}`
|
||||
FILEN=acq${FR}
|
||||
|
||||
RUNNR=`echo ${DIRN} | awk -F "/" '{print $1}' | awk -F "-" '{print $1}'`
|
||||
|
||||
hostname
|
||||
|
||||
if [ -e ${FILELST} ]
|
||||
then
|
||||
|
||||
mkdir -p ${BASEDIR}/${OUTDIR}
|
||||
|
||||
cd ${BASEDIR}/${OUTDIR}
|
||||
|
||||
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh
|
||||
|
||||
cp ${SRCDIR}/${GEOM_FILE} ${FILEN}.geom
|
||||
|
||||
grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
DETECTOR_DISTANCE=`grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
else
|
||||
DETECTOR_DISTANCE=`grep DEFAULT ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
fi
|
||||
sed -i "s:clen.*:clen = ${DETECTOR_DISTANCE}:g" ${FILEN}.geom
|
||||
|
||||
grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
BEAM_ENERGY=`grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
else
|
||||
BEAM_ENERGY=`grep DEFAULT ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
fi
|
||||
sed -i "s:photon_energy.*:photon_energy = ${BEAM_ENERGY}:g" ${FILEN}.geom
|
||||
|
||||
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ]
|
||||
then
|
||||
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell
|
||||
fi
|
||||
|
||||
rm -rf ${FILEN}.stream
|
||||
|
||||
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
then
|
||||
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
else
|
||||
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
fi
|
||||
|
||||
module purge
|
||||
|
||||
laser=dark
|
||||
echo ${FILELST} | grep light.lst > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
laser=light
|
||||
fi
|
||||
grep Final ${FILEN}.err > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
n_indexed=`grep Final ${FILEN}.err | awk '{print $8}'`
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json
|
||||
fi
|
||||
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
mean_res=`grep Mean ${FILEN}.resolution.tmp | awk '{print $5}'`
|
||||
best_res=`grep Best ${FILEN}.resolution.tmp | awk '{print $5}'`
|
||||
worst_res=`grep Worst ${FILEN}.resolution.tmp | awk '{print $5}'`
|
||||
echo $mean_res $best_res $worst_res > ${FILEN}.resolution
|
||||
else
|
||||
echo "nan nan nan" > ${FILEN}.resolution
|
||||
fi
|
||||
|
||||
read r1 r2 r3 <<< $(cat ${FILEN}.resolution)
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json
|
||||
|
||||
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ]
|
||||
then
|
||||
echo "Running alternative cell indexing"
|
||||
|
||||
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh
|
||||
|
||||
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative
|
||||
diff ${FILEN}.cell ${FILEN}.cell_alternative > /dev/null
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
rm -rf ${FILEN}.stream_alternative
|
||||
|
||||
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
then
|
||||
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
else
|
||||
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
fi
|
||||
rm -rf ${FILEN}.cell_alternative
|
||||
|
||||
grep Final ${FILEN}.err_alternative > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
n_indexed_alternative=`grep Final ${FILEN}.err_alternative | awk '{print $8}'`
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json
|
||||
fi
|
||||
|
||||
else
|
||||
echo "Alternative cell is the same as main cell"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
rm -rf ${FILEN}.list ${FILEN}.geom ${FILEN}.cell ${FILEN}.resolution.tmp
|
||||
|
||||
else
|
||||
|
||||
echo "File ${FILELST} doesnt exists"
|
||||
|
||||
fi
|
||||
|
||||
2
scripts/load_crystfel.sh
Normal file
2
scripts/load_crystfel.sh
Normal file
@@ -0,0 +1,2 @@
|
||||
module use MX
|
||||
module load crystfel/0.10.2
|
||||
48
scripts/prepare.sh
Executable file
48
scripts/prepare.sh
Executable file
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
|
||||
SRC=/sf/jungfrau/applications/sf-dap/sfx/processing
|
||||
|
||||
chmod g+w `pwd`
|
||||
|
||||
for d in CELL output
|
||||
do
|
||||
if [ ! -d ${d} ]
|
||||
then
|
||||
mkdir ${d}
|
||||
fi
|
||||
done
|
||||
|
||||
for f in env_setup.sh run_index.sh
|
||||
do
|
||||
if [ ! -e $f ]
|
||||
then
|
||||
cp ${SRC}/$f $f
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Edit file env_setup.sh with correct information (PGROUP, DETECTOR_NAME, BEAMLINE, THRESHOLD, LOGBOOK_URL)"
|
||||
|
||||
echo "Put proper geom file as <DETECTOR_NAME>.geom (e.g. JF17T16V01.geom)"
|
||||
|
||||
echo "Copy cell files to CELL/ directory (example : lyso.cell, hewl.cell....)"
|
||||
|
||||
echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api key to access logbook"
|
||||
|
||||
if [ ! -e DETECTOR_DISTANCE.txt ]
|
||||
then
|
||||
echo "DEFAULT 0.09369" > DETECTOR_DISTANCE.txt
|
||||
echo Make some good guess for detector distance in DETECTOR_DISTANCE.txt file
|
||||
fi
|
||||
|
||||
if [ ! -e BEAM_ENERGY.txt ]
|
||||
then
|
||||
echo "DEFAULT 11330.0" > BEAM_ENERGY.txt
|
||||
echo Make proper beam energy in BEAM_ENERGY.txt file
|
||||
fi
|
||||
|
||||
for f in env_setup.sh run_index.sh DETECTOR_DISTANCE.txt BEAM_ENERGY.txt
|
||||
do
|
||||
chmod g+w $f
|
||||
done
|
||||
chmod -R g+w CELL
|
||||
|
||||
25
scripts/re-insert-spearsheet.2.sh
Executable file
25
scripts/re-insert-spearsheet.2.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
for i in {0001..5000}
|
||||
do
|
||||
for l in light dark
|
||||
do
|
||||
f=output/run00${i}.index.${l}.out
|
||||
if [ -e ${f} ]
|
||||
then
|
||||
grep -v docs.google ${f} | egrep " google|Cannot find|gspread.exceptions.APIError" > /dev/null
|
||||
a=$?
|
||||
if [ $a == 0 ]
|
||||
then
|
||||
b=`echo $f | sed 's/\.out//'`
|
||||
grep python $b | grep credentials > a
|
||||
if [ -s a ]
|
||||
then
|
||||
chmod +x a
|
||||
./a > $f
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
33
scripts/re-insert-spearsheet.sh
Executable file
33
scripts/re-insert-spearsheet.sh
Executable file
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
|
||||
mkdir -p re-try
|
||||
|
||||
for i in {0001..5000}
|
||||
do
|
||||
for l in light dark
|
||||
do
|
||||
f=run00${i}.index.${l}.slurm.out
|
||||
if [ -e ${f} ]
|
||||
then
|
||||
grep -v docs.google ${f} | egrep " google|Cannot find" > /dev/null
|
||||
a=$?
|
||||
b=1
|
||||
if [ -s run00${i}.index.${l}.slurm.err ]
|
||||
then
|
||||
b=0
|
||||
fi
|
||||
if [ $a == 0 ] || [ $b == 0 ]
|
||||
then
|
||||
grep python $f | grep credentials.json > a
|
||||
if [ -s a ]
|
||||
then
|
||||
chmod +x a
|
||||
./a > run00${i}.index.${l}.slurm.err
|
||||
grep -v " google" $f | grep -v "Cannot find" > b
|
||||
mv b $f
|
||||
mv a re-try/run00${i}.`date +%s`.sh
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
81
scripts/run_index.sh
Executable file
81
scripts/run_index.sh
Executable file
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
|
||||
NP=`grep processor /proc/cpuinfo | wc -l`
|
||||
|
||||
if [ $# -ge 1 ]
|
||||
then
|
||||
FILELST=$1
|
||||
else
|
||||
FILELST=file.lst
|
||||
fi
|
||||
|
||||
if [ $# -ge 2 ]
|
||||
then
|
||||
FILEGEOM=$2
|
||||
else
|
||||
FILEGEOM=geom.geom
|
||||
fi
|
||||
|
||||
if [ $# -ge 3 ]
|
||||
then
|
||||
FILECELL=$3
|
||||
else
|
||||
FILECELL=cell.cell
|
||||
fi
|
||||
|
||||
if [ $# -ge 4 ]
|
||||
then
|
||||
FILESTREAM=$4
|
||||
else
|
||||
FILESTREAM=stream.stream
|
||||
fi
|
||||
|
||||
# list of indexing methods : xgandalf-latt-cell,asdf-latt-cell,dirax-latt-cell,mosflm-latt-cell
|
||||
# but usually all is indexed by xgandalf (99.99%)
|
||||
|
||||
# to speed up indexing, one can use : --xgandalf-fast-execution, --highres=1.6 ...
|
||||
|
||||
if [ -e ${FILECELL} ]
|
||||
then
|
||||
|
||||
indexamajig -i ${FILELST} \
|
||||
-o ${FILESTREAM} \
|
||||
--geometry=${FILEGEOM} \
|
||||
--pdb=${FILECELL} \
|
||||
--indexing=xgandalf-latt-cell \
|
||||
--peaks=peakfinder8 \
|
||||
--integration=rings-grad \
|
||||
--tolerance=10.0,10.0,10.0,2,3,2 \
|
||||
--threshold=${THRESHOLD_INDEXING} \
|
||||
--min-snr=5 \
|
||||
--int-radius=2,3,6 \
|
||||
-j ${NP} \
|
||||
--no-multi \
|
||||
--no-retry \
|
||||
--check-peaks \
|
||||
--max-res=3000 \
|
||||
--min-pix-count=1 \
|
||||
--local-bg-radius=4
|
||||
|
||||
else
|
||||
|
||||
indexamajig -i ${FILELST} \
|
||||
-o ${FILESTREAM} \
|
||||
--geometry=${FILEGEOM} \
|
||||
--indexing=xgandalf-latt-cell \
|
||||
--peaks=peakfinder8 \
|
||||
--integration=rings-grad \
|
||||
--tolerance=10.0,10.0,10.0,2,2,2 \
|
||||
--threshold=${THRESHOLD_INDEXING} \
|
||||
--min-snr=5 \
|
||||
--int-radius=2,3,6 \
|
||||
-j ${NP} \
|
||||
--no-multi \
|
||||
--no-retry \
|
||||
--check-peaks \
|
||||
--max-res=3000 \
|
||||
--min-pix-count=1 \
|
||||
--local-bg-radius=4
|
||||
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user