From 094922735e35939fa133ee73b9d72479a5fe708b Mon Sep 17 00:00:00 2001 From: Dmitry Ozerov Date: Mon, 14 Aug 2023 13:23:08 +0200 Subject: [PATCH] remove hardcoded path dependence --- ap/ap.py | 12 +++---- scripts/ap.sh | 42 ++++------------------ scripts/env_setup.sh | 11 +++--- scripts/index_data.sh | 52 ++++++++++++++-------------- scripts/jobs_overview.sh | 10 ++++++ scripts/prepare.sh | 8 +++-- scripts/re-insert-spearsheet.base.sh | 27 +++++++++++++++ scripts/re-insert-spearsheet.sh | 3 ++ 8 files changed, 88 insertions(+), 77 deletions(-) create mode 100644 scripts/jobs_overview.sh create mode 100644 scripts/re-insert-spearsheet.base.sh diff --git a/ap/ap.py b/ap/ap.py index db03c83..2c18084 100644 --- a/ap/ap.py +++ b/ap/ap.py @@ -14,6 +14,7 @@ parser.add_argument("--pgroup", default="p18674", help="pgroup") parser.add_argument("--detector", default="JF06T08V02", help="detector") parser.add_argument("--logbook", default=None, help="url to logbook") parser.add_argument("--online_hits_threshold", default = 15, type=int, help="number of peaks to consider frame a hit") +parser.add_argument("--dir_ap", default=None, help="path to ap code") args = parser.parse_args() @@ -22,8 +23,8 @@ pgroup = args.pgroup detector = args.detector online_hits_threshold = args.online_hits_threshold logbook_url = args.logbook +dir_ap = args.dir_ap -#credential_files = ["credentials-3.json", "credentials-1.json", "credentials-2.json"] credential_files = glob("credentials-*.json") raw_directory = f'/sf/{beamline}/data/{pgroup}/raw' @@ -35,13 +36,11 @@ last_run_file = f'{raw_directory}/run_info/LAST_RUN' if not os.path.exists(last_run_file): print(f'last run file does not exists {last_run_file}') last_run=0 - #exit() else: with open(last_run_file, "r") as run_file: last_run = int(run_file.read()) for run in range(last_run,0,-1): -#for run in range(last_run,360,-1): data_directory = glob(f'{raw_directory}/run{run:04}*') @@ -103,7 +102,7 @@ for run in range(last_run,0,-1): credential_file = choice(credential_files) - log_run = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}' + log_run = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --run_number {run_number} --acq_number {acq_number} --user_tag {user_tag} --cell_name {cell_name} --time_run_taken {trun} --motor_name {motor_name} --motor_value {motor_value} --credentials {credential_file}' process=Popen(log_run, shell=True, stdout=process_log_file, stderr=process_log_file) print(log_run, file = f_log) @@ -206,15 +205,14 @@ for run in range(1, last_run+1): credential_file = choice(credential_files) - log_laser = f'python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}' + log_laser = f'python {dir_ap}/ap/update-spreadsheet.py --url {logbook_url} --unique_run {unique_run_number} --number_frames {nframes} --hits_rate {hitrate_value} --laser {laser} --credentials {credential_file}' process=Popen(log_laser, shell=True, stdout=process_log_file, stderr=process_log_file) print(log_laser, file = f_log) #slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis", "prod-athos"]) slurm_partition = choice(["prod-aramis", "prod-aramis", "prod-aramis"]) - #log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {user_tag} {unique_run_number}' - log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out /sf/jungfrau/applications/sf-dap/sfx/processing/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}' + log_index = f'sbatch --exclusive -p {slurm_partition} -J {unique_run_number}_{laser} -e output/run{unique_run_number:06}.index.{laser}.slurm.err -o output/run{unique_run_number:06}.index.{laser}.slurm.out {dir_ap}/scripts/index_data.sh {dir_name}/index/{laser} {acq_number} {frame_list_file} {cell_name} {unique_run_number}' print(log_index, file = f_log) process=Popen(log_index, shell=True, stdout=process_log_file, stderr=process_log_file) diff --git a/scripts/ap.sh b/scripts/ap.sh index 11a6e21..43b9dfc 100755 --- a/scripts/ap.sh +++ b/scripts/ap.sh @@ -5,45 +5,15 @@ while true do echo "Running "`date` - python /sf/jungfrau/applications/sf-dap/sfx/processing/automatic2.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK} - cd output - for i in `ls *base.out 2>/dev/null` - do - if [ -s $i ] - then - f=`echo $i | sed 's/\.out//'` - cp $f a.sh - chmod +x a.sh - cd .. - output/a.sh > output/$i.out 2>&1 - cd output - rm -rf a.sh - rm -rf $i - if [ -s $i.out ] - then - mv $i.out $i - else - rm -rf $i.out - fi - else - rm -rf $i - fi - done - cd .. + python ${DIR_AP}/ap/ap.py --pgroup ${PGROUP} --beamline ${BEAMLINE} --detector ${DETN} --logbook ${LOGBOOK} --dir_ap ${DIR_AP} - rm -rf CURRENT_JOBS.txt - echo " statistics at "`date` >> CURRENT_JOBS.txt - echo " Running jobs " >> CURRENT_JOBS.txt - squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt - echo " " >> CURRENT_JOBS.txt - echo " Pending jobs " >> CURRENT_JOBS.txt - squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt + ${DIR_AP}/scripts/jobs_overview.sh - cd output - /sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.sh - cd .. + ${DIR_AP}/scripts/re-insert-spearsheet.base.sh - /sf/jungfrau/applications/sf-dap/sfx/processing/wip/re-insert-spearsheet.2.sh + ${DIR_AP}/scripts/re-insert-spearsheet.sh + + ${DIR_AP}/scripts/re-insert-spearsheet.2.sh echo "Sleeping "`date` sleep 120 diff --git a/scripts/env_setup.sh b/scripts/env_setup.sh index ac9d569..9abd21e 100755 --- a/scripts/env_setup.sh +++ b/scripts/env_setup.sh @@ -2,19 +2,18 @@ DIRNAME=`dirname ${BASH_SOURCE}` -#export PGROUP=`cat ${DIRNAME}/PGROUP` -export PGROUP=... +export PGROUP= export BEAMLINE=alvra -# alvra bernina cristallina furka maloja +# Choice: alvra bernina cristallina furka maloja BASEDIR=/sf/${BEAMLINE}/data/${PGROUP}/res source /sf/jungfrau/applications/miniconda3/etc/profile.d/conda.sh conda activate sf-dap -export DETN=JF06T08V03 -# JF06T08V03 JF06T32V03 JF17T16V01 +export DETN= +# Possible choice: JF06T08V04 JF06T32V04 JF17T16V01 export GEOM_FILE=${DETN}.geom @@ -25,4 +24,4 @@ export THRESHOLD_INDEXING=10 #LOGBOOK="https://docs.google.com/spreadsheets/...." -SRCDIR=$PWD +CONFIG_DIR=$PWD diff --git a/scripts/index_data.sh b/scripts/index_data.sh index 771b702..4368cd6 100644 --- a/scripts/index_data.sh +++ b/scripts/index_data.sh @@ -29,40 +29,40 @@ then cd ${BASEDIR}/${OUTDIR} - . /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh + . ${DIR_AP}/scripts/load_crystfel.sh - cp ${SRCDIR}/${GEOM_FILE} ${FILEN}.geom + cp ${CONFIG_DIR}/${GEOM_FILE} ${FILEN}.geom - grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt > /dev/null + grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt > /dev/null if [ $? = 0 ] then - DETECTOR_DISTANCE=`grep ${RUNNR} ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` + DETECTOR_DISTANCE=`grep ${RUNNR} ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` else - DETECTOR_DISTANCE=`grep DEFAULT ${SRCDIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` + DETECTOR_DISTANCE=`grep DEFAULT ${CONFIG_DIR}/DETECTOR_DISTANCE.txt | tail -1 | awk '{print $2}'` fi sed -i "s:clen.*:clen = ${DETECTOR_DISTANCE}:g" ${FILEN}.geom - grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt > /dev/null + grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt > /dev/null if [ $? = 0 ] then - BEAM_ENERGY=`grep ${RUNNR} ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` + BEAM_ENERGY=`grep ${RUNNR} ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` else - BEAM_ENERGY=`grep DEFAULT ${SRCDIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` + BEAM_ENERGY=`grep DEFAULT ${CONFIG_DIR}/BEAM_ENERGY.txt | tail -1 | awk '{print $2}'` fi sed -i "s:photon_energy.*:photon_energy = ${BEAM_ENERGY}:g" ${FILEN}.geom - if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ] + if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ] then - cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell + cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell ${FILEN}.cell fi rm -rf ${FILEN}.stream - if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ] + if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ] then - ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err + ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err else - ${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err + ${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell ${FILEN}.stream > ${FILEN}.out 2>${FILEN}.err fi module purge @@ -77,11 +77,11 @@ then if [ $? = 0 ] then n_indexed=`grep Final ${FILEN}.err | awk '{print $8}'` - echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json - python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${SRCDIR}/credentials.json + echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json + python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed ${n_indexed} --credentials ${CONFIG_DIR}/credentials.json fi - /sf/jungfrau/applications/sf-dap/sfx/processing/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp + ${DIR_AP}/scripts/ave-resolution ${FILEN}.stream > ${FILEN}.resolution.tmp if [ $? == 0 ] then mean_res=`grep Mean ${FILEN}.resolution.tmp | awk '{print $5}'` @@ -93,26 +93,26 @@ then fi read r1 r2 r3 <<< $(cat ${FILEN}.resolution) - echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json - python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${SRCDIR}/credentials.json + echo python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json + python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --resolution_min ${r3} --resolution_max ${r2} --resolution_mean ${r1} --credentials ${CONFIG_DIR}/credentials.json - if [ -e ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ] + if [ -e ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ] then echo "Running alternative cell indexing" - . /sf/jungfrau/applications/sf-dap/sfx/processing/load_crystfel.sh + . ${DIR_AP}/scripts/load_crystfel.sh - cp ${SRCDIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative + cp ${CONFIG_DIR}/CELL/${PROTEIN_NAME}.cell_alternative ${FILEN}.cell_alternative diff ${FILEN}.cell ${FILEN}.cell_alternative > /dev/null if [ $? != 0 ] then rm -rf ${FILEN}.stream_alternative - if [ -e ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ] + if [ -e ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ] then - ${SRCDIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative + ${CONFIG_DIR}/run_index.${PROTEIN_NAME}.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative else - ${SRCDIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative + ${CONFIG_DIR}/run_index.sh ${FILELST} ${FILEN}.geom ${FILEN}.cell_alternative ${FILEN}.stream_alternative > ${FILEN}.out_alternative 2>${FILEN}.err_alternative fi rm -rf ${FILEN}.cell_alternative @@ -120,8 +120,8 @@ then if [ $? = 0 ] then n_indexed_alternative=`grep Final ${FILEN}.err_alternative | awk '{print $8}'` - echo python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json - python /sf/jungfrau/applications/sf-dap/sfx/processing/update-spreadsheet-per-run.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${SRCDIR}/credentials.json + echo python ${DIP_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json + python ${DIR_AP}/ap/update-spreadsheet.py --url ${LOGBOOK} --unique_run ${UNIQUERUN} --laser ${laser} --number_indexed_alternative ${n_indexed_alternative} --credentials ${CONFIG_DIR}/credentials.json fi else diff --git a/scripts/jobs_overview.sh b/scripts/jobs_overview.sh new file mode 100644 index 0000000..89d535b --- /dev/null +++ b/scripts/jobs_overview.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +rm -rf CURRENT_JOBS.txt +echo " statistics at "`date` >> CURRENT_JOBS.txt +echo " Running jobs " >> CURRENT_JOBS.txt +squeue | grep " R " | awk '{print $3" "$6}' >> CURRENT_JOBS.txt +echo " " >> CURRENT_JOBS.txt +echo " Pending jobs " >> CURRENT_JOBS.txt +squeue | grep " PD " | awk '{print $3}' >> CURRENT_JOBS.txt + diff --git a/scripts/prepare.sh b/scripts/prepare.sh index 6d3fd5f..104d9bb 100755 --- a/scripts/prepare.sh +++ b/scripts/prepare.sh @@ -1,6 +1,6 @@ #!/bin/bash -SRC=/sf/jungfrau/applications/sf-dap/sfx/processing +SRC=`dirname ${BASH_SOURCE}` chmod g+w `pwd` @@ -20,13 +20,17 @@ do fi done +# path to the installed ap package +DIR_AP=`dirname ${SRC}` +echo "export DIR_AP=${DIR_AP}" >> env_setup.sh + echo "Edit file env_setup.sh with correct information (PGROUP, DETECTOR_NAME, BEAMLINE, THRESHOLD, LOGBOOK_URL)" echo "Put proper geom file as .geom (e.g. JF17T16V01.geom)" echo "Copy cell files to CELL/ directory (example : lyso.cell, hewl.cell....)" -echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api key to access logbook" +echo "Make files credentials.json, credentials-*.json (e.g. credentials-1.json, credentials-2.json) with the api keys to access logbook" if [ ! -e DETECTOR_DISTANCE.txt ] then diff --git a/scripts/re-insert-spearsheet.base.sh b/scripts/re-insert-spearsheet.base.sh new file mode 100644 index 0000000..4de453a --- /dev/null +++ b/scripts/re-insert-spearsheet.base.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +cd output +for i in `ls *base.out 2>/dev/null` +do + if [ -s $i ] + then + f=`echo $i | sed 's/\.out//'` + cp $f a.sh + chmod +x a.sh + cd .. + output/a.sh > output/$i.out 2>&1 + cd output + rm -rf a.sh + rm -rf $i + if [ -s $i.out ] + then + mv $i.out $i + else + rm -rf $i.out + fi + else + rm -rf $i + fi +done +cd .. + diff --git a/scripts/re-insert-spearsheet.sh b/scripts/re-insert-spearsheet.sh index 2d65b60..7e1926a 100755 --- a/scripts/re-insert-spearsheet.sh +++ b/scripts/re-insert-spearsheet.sh @@ -1,5 +1,7 @@ #!/bin/bash +cd output + mkdir -p re-try for i in {0001..5000} @@ -31,3 +33,4 @@ do fi done done +cd ..