sf-daq/ap
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove hardcoded path dependence

Browse Source
This commit is contained in:
Dmitry Ozerov
2023-08-14 13:23:08 +02:00
parent 307202e657
commit 094922735e
8 changed files with 88 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ap/ap.py
View File

@ -14,6 +14,7 @@ parser.add_argument("--pgroup", default="p18674", help="pgroup")
parser.add_argument("--detector", default="JF06T08V02", help="detector") parser.add_argument("--detector", default="JF06T08V02", help="detector")
parser.add_argument("--logbook", default=None, help="url to logbook") parser.add_argument("--logbook", default=None, help="url to logbook")
parser.add_argument("--online_hits_threshold", default = 15, type=int, help="number of peaks to consider frame a hit") parser.add_argument("--online_hits_threshold", default = 15, type=int, help="number of peaks to consider frame a hit")
parser.add_argument("--dir_ap", default=None, help="path to ap code")
args = parser.parse_args() args = parser.parse_args()
@ -22,8 +23,8 @@ pgroup = args.pgroup
detector = args.detector detector = args.detector
online_hits_threshold = args.online_hits_threshold online_hits_threshold = args.online_hits_threshold
logbook_url = args.logbook logbook_url = args.logbook
dir_ap = args.dir_ap
#credential_files = ["credentials-3.json", "credentials-1.json", "credentials-2.json"]
credential_files = glob("credentials-*.json") credential_files = glob("credentials-*.json")
raw_directory = f'/sf/{beamline}/data/{pgroup}/raw' raw_directory = f'/sf/{beamline}/data/{pgroup}/raw'
@ -35,13 +36,11 @@ last_run_file = f'{raw_directory}/run_info/LAST_RUN'
if not os.path.exists(last_run_file): if not os.path.exists(last_run_file):
print(f'last run file does not exists {last_run_file}') print(f'last run file does not exists {last_run_file}')
last_run=0 last_run=0
#exit()
else: else:
with open(last_run_file, "r") as run_file: with open(last_run_file, "r") as run_file:
last_run = int(run_file.read()) last_run = int(run_file.read())
for run in range(last_run,0,-1): for run in range(last_run,0,-1):
#for run in range(last_run,360,-1):
data_directory = glob(f'{raw_directory}/run{run:04}*') data_directory = glob(f'{raw_directory}/run{run:04}*')
@ -103,7 +102,7 @@ for run in range(last_run,0,-1):
credential_file = choice(credential_files) credential_file = choice(credential_files)
log_run = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}' log_run = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}'
process=Popen(log_run, shell=True, stdout=process_log_file, stderr=process_log_file) process=Popen(log_run, shell=True, stdout=process_log_file, stderr=process_log_file)
print(log_run, file = f_log) print(log_run, file = f_log)
@ -206,15 +205,14 @@ for run in range(1, last_run+1):
credential_file = choice(credential_files) credential_file = choice(credential_files)
log_laser = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}' log_laser = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}'
process=Popen(log_laser, shell=True, stdout=process_log_file, stderr=process_log_file) process=Popen(log_laser, shell=True, stdout=process_log_file, stderr=process_log_file)
print(log_laser, file = f_log) print(log_laser, file = f_log)
#slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis", "prod-athos"]) #slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis", "prod-athos"])
slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis"]) slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis"])
#log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {user_tag} {unique_run_number}' log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out {dir_ap}/scripts/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}'
log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}'
print(log_index, file = f_log) print(log_index, file = f_log)
process=Popen(log_index, shell=True, stdout=process_log_file, stderr=process_log_file) process=Popen(log_index, shell=True, stdout=process_log_file, stderr=process_log_file)

42
scripts/ap.sh
View File

@ -5,45 +5,15 @@
while true while true
do do
echo "Running "`date` echo "Running "`date`
python /sf/jungfrau/applications/sf-dap/sfx/processing/automatic2.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK} python ${DIR_AP}/ap/ap.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK} --dir_ap ${DIR_AP}
cd output
for i in `ls *base.out 2>/dev/null`
do
if [ -s $i ]
then
f=`echo $i | sed 's/\.out//'`
cp $f a.sh
chmod +x a.sh
cd ..
output/a.sh > output/$i.out 2>&1
cd output
rm -rf a.sh
rm -rf $i
if [ -s $i.out ]
then
mv $i.out $i
else
rm -rf $i.out
fi
else
rm -rf $i
fi
done
cd ..
rm -rf CURRENT_JOBS.txt ${DIR_AP}/scripts/jobs_overview.sh
echo " statistics at "`date` >> CURRENT_JOBS.txt
echo " Running jobs " >> CURRENT_JOBS.txt
squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt
echo " " >> CURRENT_JOBS.txt
echo " Pending jobs " >> CURRENT_JOBS.txt
squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt
cd output ${DIR_AP}/scripts/re-insert-spearsheet.base.sh
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.sh
cd ..
/sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.2.sh ${DIR_AP}/scripts/re-insert-spearsheet.sh
${DIR_AP}/scripts/re-insert-spearsheet.2.sh
echo "Sleeping "`date` echo "Sleeping "`date`
sleep 120 sleep 120

11
scripts/env_setup.sh
View File

@ -2,19 +2,18 @@
DIRNAME=`dirname ${BASH_SOURCE}` DIRNAME=`dirname ${BASH_SOURCE}`
#export PGROUP=`cat ${DIRNAME}/PGROUP` export PGROUP=<EDIT HERE>
export PGROUP=...
export BEAMLINE=alvra export BEAMLINE=alvra
# alvra bernina cristallina furka maloja # Choice: alvra bernina cristallina furka maloja
BASEDIR=/sf/${BEAMLINE}/data/${PGROUP}/res BASEDIR=/sf/${BEAMLINE}/data/${PGROUP}/res
source /sf/jungfrau/applications/miniconda3/etc/profile.d/conda.sh source /sf/jungfrau/applications/miniconda3/etc/profile.d/conda.sh
conda activate sf-dap conda activate sf-dap
export DETN=JF06T08V03 export DETN=<EDIT HERE>
# JF06T08V03 JF06T32V03 JF17T16V01 # Possible choice: JF06T08V04 JF06T32V04 JF17T16V01
export GEOM_FILE=${DETN}.geom export GEOM_FILE=${DETN}.geom
@ -25,4 +24,4 @@ export THRESHOLD_INDEXING=10
#LOGBOOK="https://docs.google.com/spreadsheets/...." #LOGBOOK="https://docs.google.com/spreadsheets/...."
SRCDIR=$PWD CONFIG_DIR=$PWD

52
scripts/index_data.sh
View File

@ -29,40 +29,40 @@ then
cd ${BASEDIR}/${OUTDIR} cd ${BASEDIR}/${OUTDIR}
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh . ${DIR_AP}/scripts/load_crystfel.sh
cp ${SRCDIR}/${GEOM_FILE} ${FILEN}.geom cp ${CONFIG_DIR}/${GEOM_FILE} ${FILEN}.geom
grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt > /dev/null grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt > /dev/null
if [ $? = 0 ] if [ $? = 0 ]
then then
DETECTOR_DISTANCE=`grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` DETECTOR_DISTANCE=`grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
else else
DETECTOR_DISTANCE=`grep DEFAULT ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` DETECTOR_DISTANCE=`grep DEFAULT ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'`
fi fi
sed -i "s:clen.*:clen = ${DETECTOR_DISTANCE}:g" ${FILEN}.geom sed -i "s:clen.*:clen = ${DETECTOR_DISTANCE}:g" ${FILEN}.geom
grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt > /dev/null grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt > /dev/null
if [ $? = 0 ] if [ $? = 0 ]
then then
BEAM_ENERGY=`grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` BEAM_ENERGY=`grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
else else
BEAM_ENERGY=`grep DEFAULT ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` BEAM_ENERGY=`grep DEFAULT ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'`
fi fi
sed -i "s:photon_energy.*:photon_energy = ${BEAM_ENERGY}:g" ${FILEN}.geom sed -i "s:photon_energy.*:photon_energy = ${BEAM_ENERGY}:g" ${FILEN}.geom
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ] if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ]
then then
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell
fi fi
rm -rf ${FILEN}.stream rm -rf ${FILEN}.stream
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ] if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ]
then then
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
else else
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err ${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err
fi fi
module purge module purge
@ -77,11 +77,11 @@ then
if [ $? = 0 ] if [ $? = 0 ]
then then
n_indexed=`grep Final ${FILEN}.err | awk '{print $8}'` n_indexed=`grep Final ${FILEN}.err | awk '{print $8}'`
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json
fi fi
/sf/jungfrau/applications/sf-dap/sfx/processing/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp ${DIR_AP}/scripts/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp
if [ $? == 0 ] if [ $? == 0 ]
then then
mean_res=`grep Mean ${FILEN}.resolution.tmp | awk '{print $5}'` mean_res=`grep Mean ${FILEN}.resolution.tmp | awk '{print $5}'`
@ -93,26 +93,26 @@ then
fi fi
read r1 r2 r3 <<< $(cat ${FILEN}.resolution) read r1 r2 r3 <<< $(cat ${FILEN}.resolution)
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json
if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ] if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ]
then then
echo "Running alternative cell indexing" echo "Running alternative cell indexing"
. /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh . ${DIR_AP}/scripts/load_crystfel.sh
cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative
diff ${FILEN}.cell ${FILEN}.cell_alternative > /dev/null diff ${FILEN}.cell ${FILEN}.cell_alternative > /dev/null
if [ $? != 0 ] if [ $? != 0 ]
then then
rm -rf ${FILEN}.stream_alternative rm -rf ${FILEN}.stream_alternative
if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ] if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ]
then then
${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
else else
${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative ${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative
fi fi
rm -rf ${FILEN}.cell_alternative rm -rf ${FILEN}.cell_alternative
@ -120,8 +120,8 @@ then
if [ $? = 0 ] if [ $? = 0 ]
then then
n_indexed_alternative=`grep Final ${FILEN}.err_alternative | awk '{print $8}'` n_indexed_alternative=`grep Final ${FILEN}.err_alternative | awk '{print $8}'`
echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json echo python ${DIP_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json
python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json
fi fi
else else

10
scripts/jobs_overview.sh Normal file
View File

@ -0,0 +1,10 @@
#!/bin/bash
rm -rf CURRENT_JOBS.txt
echo " statistics at "`date` >> CURRENT_JOBS.txt
echo " Running jobs " >> CURRENT_JOBS.txt
squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt
echo " " >> CURRENT_JOBS.txt
echo " Pending jobs " >> CURRENT_JOBS.txt
squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt

8
scripts/prepare.sh
View File

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
SRC=/sf/jungfrau/applications/sf-dap/sfx/processing SRC=`dirname ${BASH_SOURCE}`
chmod g+w `pwd` chmod g+w `pwd`
@ -20,13 +20,17 @@ do
fi fi
done done
# path to the installed ap package
DIR_AP=`dirname ${SRC}`
echo "export DIR_AP=${DIR_AP}" >> env_setup.sh
echo "Edit file env_setup.sh with correct information (PGROUP, DETECTOR_NAME, BEAMLINE, THRESHOLD, LOGBOOK_URL)" echo "Edit file env_setup.sh with correct information (PGROUP, DETECTOR_NAME, BEAMLINE, THRESHOLD, LOGBOOK_URL)"
echo "Put proper geom file as <DETECTOR_NAME>.geom (e.g. JF17T16V01.geom)" echo "Put proper geom file as <DETECTOR_NAME>.geom (e.g. JF17T16V01.geom)"
echo "Copy cell files to CELL/ directory (example : lyso.cell, hewl.cell....)" echo "Copy cell files to CELL/ directory (example : lyso.cell, hewl.cell....)"
echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api key to access logbook" echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api keys to access logbook"
if [ ! -e DETECTOR_DISTANCE.txt ] if [ ! -e DETECTOR_DISTANCE.txt ]
then then

27
scripts/re-insert-spearsheet.base.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
cd output
for i in `ls *base.out 2>/dev/null`
do
if [ -s $i ]
then
f=`echo $i | sed 's/\.out//'`
cp $f a.sh
chmod +x a.sh
cd ..
output/a.sh > output/$i.out 2>&1
cd output
rm -rf a.sh
rm -rf $i
if [ -s $i.out ]
then
mv $i.out $i
else
rm -rf $i.out
fi
else
rm -rf $i
fi
done
cd ..

3
scripts/re-insert-spearsheet.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash #!/bin/bash
cd output
mkdir -p re-try mkdir -p re-try
for i in {0001..5000} for i in {0001..5000}
@ -31,3 +33,4 @@ do
fi fi
done done
done done
cd ..