remove hardcoded path dependence
This commit is contained in:
12
ap/ap.py
12
ap/ap.py
@ -14,6 +14,7 @@ parser.add_argument("--pgroup", default="p18674", help="pgroup")
|
||||
parser.add_argument("--detector", default="JF06T08V02", help="detector")
|
||||
parser.add_argument("--logbook", default=None, help="url to logbook")
|
||||
parser.add_argument("--online_hits_threshold", default = 15, type=int, help="number of peaks to consider frame a hit")
|
||||
parser.add_argument("--dir_ap", default=None, help="path to ap code")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@ -22,8 +23,8 @@ pgroup = args.pgroup
|
||||
detector = args.detector
|
||||
online_hits_threshold = args.online_hits_threshold
|
||||
logbook_url = args.logbook
|
||||
dir_ap = args.dir_ap
|
||||
|
||||
#credential_files = ["credentials-3.json", "credentials-1.json", "credentials-2.json"]
|
||||
credential_files = glob("credentials-*.json")
|
||||
|
||||
raw_directory = f'/sf/{beamline}/data/{pgroup}/raw'
|
||||
@ -35,13 +36,11 @@ last_run_file = f'{raw_directory}/run_info/LAST_RUN'
|
||||
if not os.path.exists(last_run_file):
|
||||
print(f'last run file does not exists {last_run_file}')
|
||||
last_run=0
|
||||
#exit()
|
||||
else:
|
||||
with open(last_run_file, "r") as run_file:
|
||||
last_run = int(run_file.read())
|
||||
|
||||
for run in range(last_run,0,-1):
|
||||
#for run in range(last_run,360,-1):
|
||||
|
||||
data_directory = glob(f'{raw_directory}/run{run:04}*')
|
||||
|
||||
@ -103,7 +102,7 @@ for run in range(last_run,0,-1):
|
||||
|
||||
credential_file = choice(credential_files)
|
||||
|
||||
log_run = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}'
|
||||
log_run = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}'
|
||||
process=Popen(log_run, shell=True, stdout=process_log_file, stderr=process_log_file)
|
||||
print(log_run, file = f_log)
|
||||
|
||||
@ -206,15 +205,14 @@ for run in range(1, last_run+1):
|
||||
|
||||
credential_file = choice(credential_files)
|
||||
|
||||
log_laser = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}'
|
||||
log_laser = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}'
|
||||
process=Popen(log_laser, shell=True, stdout=process_log_file, stderr=process_log_file)
|
||||
print(log_laser, file = f_log)
|
||||
|
||||
#slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis", "prod-athos"])
|
||||
slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis"])
|
||||
|
||||
#log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {user_tag} {unique_run_number}'
|
||||
log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}'
|
||||
log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out {dir_ap}/scripts/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}'
|
||||
print(log_index, file = f_log)
|
||||
process=Popen(log_index, shell=True, stdout=process_log_file, stderr=process_log_file)
|
||||
|
||||
|
@ -5,45 +5,15 @@
|
||||
while true
|
||||
do
|
||||
echo "Running "`date`
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/automatic2.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK}
|
||||
cd output
|
||||
for i in `ls *base.out 2>/dev/null`
|
||||
do
|
||||
if [ -s $i ]
|
||||
then
|
||||
f=`echo $i | sed 's/\.out//'`
|
||||
cp $f a.sh
|
||||
chmod +x a.sh
|
||||
cd ..
|
||||
output/a.sh > output/$i.out 2>&1
|
||||
cd output
|
||||
rm -rf a.sh
|
||||
rm -rf $i
|
||||
if [ -s $i.out ]
|
||||
then
|
||||
mv $i.out $i
|
||||
else
|
||||
rm -rf $i.out
|
||||
fi
|
||||
else
|
||||
rm -rf $i
|
||||
fi
|
||||
done
|
||||
cd ..
|
||||
python ${DIR_AP}/ap/ap.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK} --dir_ap ${DIR_AP}
|
||||
|
||||
rm -rf CURRENT_JOBS.txt
|
||||
echo " statistics at "`date` >> CURRENT_JOBS.txt
|
||||
echo " Running jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt
|
||||
echo " " >> CURRENT_JOBS.txt
|
||||
echo " Pending jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt
|
||||
${DIR_AP}/scripts/jobs_overview.sh
|
||||
|
||||
cd output
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.sh
|
||||
cd ..
|
||||
${DIR_AP}/scripts/re-insert-spearsheet.base.sh
|
||||
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.2.sh
|
||||
${DIR_AP}/scripts/re-insert-spearsheet.sh
|
||||
|
||||
${DIR_AP}/scripts/re-insert-spearsheet.2.sh
|
||||
|
||||
echo "Sleeping "`date`
|
||||
sleep 120
|
||||
|
@ -2,19 +2,18 @@
|
||||
|
||||
DIRNAME=`dirname ${BASH_SOURCE}`
|
||||
|
||||
#export PGROUP=`cat ${DIRNAME}/PGROUP`
|
||||
export PGROUP=...
|
||||
export PGROUP=<EDIT HERE>
|
||||
|
||||
export BEAMLINE=alvra
|
||||
# alvra bernina cristallina furka maloja
|
||||
# Choice: alvra bernina cristallina furka maloja
|
||||
|
||||
BASEDIR=/sf/${BEAMLINE}/data/${PGROUP}/res
|
||||
|
||||
source /sf/jungfrau/applications/miniconda3/etc/profile.d/conda.sh
|
||||
conda activate sf-dap
|
||||
|
||||
export DETN=JF06T08V03
|
||||
# JF06T08V03 JF06T32V03 JF17T16V01
|
||||
export DETN=<EDIT HERE>
|
||||
# Possible choice: JF06T08V04 JF06T32V04 JF17T16V01
|
||||
|
||||
export GEOM_FILE=${DETN}.geom
|
||||
|
||||
@ -25,4 +24,4 @@ export THRESHOLD_INDEXING=10
|
||||
|
||||
#LOGBOOK="https://docs.google.com/spreadsheets/...."
|
||||
|
||||
SRCDIR=$PWD
|
||||
CONFIG_DIR=$PWD
|
||||
|
@ -29,40 +29,40 @@ then
|
||||
|
||||
cd ${BASEDIR}/${OUTDIR}
|
||||
|
||||
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh
|
||||
. ${DIR_AP}/scripts/load_crystfel.sh
|
||||
|
||||
cp ${SRCDIR}/${GEOM_FILE} ${FILEN}.geom
|
||||
cp ${CONFIG_DIR}/${GEOM_FILE} ${FILEN}.geom
|
||||
|
||||
grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt > /dev/null
|
||||
grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
DETECTOR_DISTANCE=`grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
DETECTOR_DISTANCE=`grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
else
|
||||
DETECTOR_DISTANCE=`grep DEFAULT ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
DETECTOR_DISTANCE=`grep DEFAULT ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
|
||||
fi
|
||||
sed -i "s:clen.*:clen = ${DETECTOR_DISTANCE}:g" ${FILEN}.geom
|
||||
|
||||
grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt > /dev/null
|
||||
grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt > /dev/null
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
BEAM_ENERGY=`grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
BEAM_ENERGY=`grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
else
|
||||
BEAM_ENERGY=`grep DEFAULT ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
BEAM_ENERGY=`grep DEFAULT ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
|
||||
fi
|
||||
sed -i "s:photon_energy.*:photon_energy = ${BEAM_ENERGY}:g" ${FILEN}.geom
|
||||
|
||||
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ]
|
||||
if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ]
|
||||
then
|
||||
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell
|
||||
cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell
|
||||
fi
|
||||
|
||||
rm -rf ${FILEN}.stream
|
||||
|
||||
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
then
|
||||
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
else
|
||||
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
|
||||
fi
|
||||
|
||||
module purge
|
||||
@ -77,11 +77,11 @@ then
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
n_indexed=`grep Final ${FILEN}.err | awk '{print $8}'`
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json
|
||||
echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json
|
||||
python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json
|
||||
fi
|
||||
|
||||
/sf/jungfrau/applications/sf-dap/sfx/processing/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp
|
||||
${DIR_AP}/scripts/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp
|
||||
if [ $? == 0 ]
|
||||
then
|
||||
mean_res=`grep Mean ${FILEN}.resolution.tmp | awk '{print $5}'`
|
||||
@ -93,26 +93,26 @@ then
|
||||
fi
|
||||
|
||||
read r1 r2 r3 <<< $(cat ${FILEN}.resolution)
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json
|
||||
echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json
|
||||
python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json
|
||||
|
||||
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ]
|
||||
if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ]
|
||||
then
|
||||
echo "Running alternative cell indexing"
|
||||
|
||||
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh
|
||||
. ${DIR_AP}/scripts/load_crystfel.sh
|
||||
|
||||
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative
|
||||
cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative
|
||||
diff ${FILEN}.cell ${FILEN}.cell_alternative > /dev/null
|
||||
if [ $? != 0 ]
|
||||
then
|
||||
rm -rf ${FILEN}.stream_alternative
|
||||
|
||||
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ]
|
||||
then
|
||||
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
else
|
||||
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
|
||||
fi
|
||||
rm -rf ${FILEN}.cell_alternative
|
||||
|
||||
@ -120,8 +120,8 @@ then
|
||||
if [ $? = 0 ]
|
||||
then
|
||||
n_indexed_alternative=`grep Final ${FILEN}.err_alternative | awk '{print $8}'`
|
||||
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json
|
||||
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json
|
||||
echo python ${DIP_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json
|
||||
python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json
|
||||
fi
|
||||
|
||||
else
|
||||
|
10
scripts/jobs_overview.sh
Normal file
10
scripts/jobs_overview.sh
Normal file
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf CURRENT_JOBS.txt
|
||||
echo " statistics at "`date` >> CURRENT_JOBS.txt
|
||||
echo " Running jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt
|
||||
echo " " >> CURRENT_JOBS.txt
|
||||
echo " Pending jobs " >> CURRENT_JOBS.txt
|
||||
squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt
|
||||
|
@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
SRC=/sf/jungfrau/applications/sf-dap/sfx/processing
|
||||
SRC=`dirname ${BASH_SOURCE}`
|
||||
|
||||
chmod g+w `pwd`
|
||||
|
||||
@ -20,13 +20,17 @@ do
|
||||
fi
|
||||
done
|
||||
|
||||
# path to the installed ap package
|
||||
DIR_AP=`dirname ${SRC}`
|
||||
echo "export DIR_AP=${DIR_AP}" >> env_setup.sh
|
||||
|
||||
echo "Edit file env_setup.sh with correct information (PGROUP, DETECTOR_NAME, BEAMLINE, THRESHOLD, LOGBOOK_URL)"
|
||||
|
||||
echo "Put proper geom file as <DETECTOR_NAME>.geom (e.g. JF17T16V01.geom)"
|
||||
|
||||
echo "Copy cell files to CELL/ directory (example : lyso.cell, hewl.cell....)"
|
||||
|
||||
echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api key to access logbook"
|
||||
echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api keys to access logbook"
|
||||
|
||||
if [ ! -e DETECTOR_DISTANCE.txt ]
|
||||
then
|
||||
|
27
scripts/re-insert-spearsheet.base.sh
Normal file
27
scripts/re-insert-spearsheet.base.sh
Normal file
@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd output
|
||||
for i in `ls *base.out 2>/dev/null`
|
||||
do
|
||||
if [ -s $i ]
|
||||
then
|
||||
f=`echo $i | sed 's/\.out//'`
|
||||
cp $f a.sh
|
||||
chmod +x a.sh
|
||||
cd ..
|
||||
output/a.sh > output/$i.out 2>&1
|
||||
cd output
|
||||
rm -rf a.sh
|
||||
rm -rf $i
|
||||
if [ -s $i.out ]
|
||||
then
|
||||
mv $i.out $i
|
||||
else
|
||||
rm -rf $i.out
|
||||
fi
|
||||
else
|
||||
rm -rf $i
|
||||
fi
|
||||
done
|
||||
cd ..
|
||||
|
@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd output
|
||||
|
||||
mkdir -p re-try
|
||||
|
||||
for i in {0001..5000}
|
||||
@ -31,3 +33,4 @@ do
|
||||
fi
|
||||
done
|
||||
done
|
||||
cd ..
|
||||
|
Reference in New Issue
Block a user