diff --git a/EM/relion/README.md b/EM/relion/README.md index 095d00f..5af3571 100644 --- a/EM/relion/README.md +++ b/EM/relion/README.md @@ -12,4 +12,42 @@ and MKL, but this would require additional openmpi variants. After running ./build, install relion scripts from the repository at https://git.psi.ch/lsm-hpce/relion-scripts -From version 5.0-beta on, a conda env is required. conda env is build in the folder $SRC\_DIR/conda\_envs. Also, weights for BLUSH, MODEL-ANGELO and CLASS-RANKER are downloaded in the installation step. +## Important info per version! + +### 5.0-beta (OLD, Merlin6) +- From version 5.0-beta on, a conda env is required (done with central anaconda module, needs to be done on pmod7 because of AFS hardlinks) +- the conda env is build in the folder $SRC\_DIR/conda\_envs. Also, weights for BLUSH, MODEL-ANGELO and CLASS-RANKER are downloaded in the installation step. + +### 5.0-1beta (OLD, Merlin6) +- the conda env is made with a miniconda installation, and because of upgrading to Pmodules/1.1.21 , the variants file was dropped and changed into the config.yaml file. +- The build script was also updated accordingly +- Also, cryocare is now available in a wrapper from relion, the cryocare installation executables need to be provided (seperate cryocare module). It is crucial to add the cryocare lib path (from the cryocare conda env) to the LD_LIBRARY_PATH when loading the relion module +- tiff module no longer required, as this conflicts with the system shared tiff libs (not 100% sure why this canged from previous installations though) +### 5.0-2beta (ON MERLIN7) +- no fetching of weights with ftp possible, so cp from other machines into BUILD DIR and using the -DFETCH\_WEIGHTS=OFF option +- link : ftp://ftp.mrc-lmb.cam.ac.uk/pub/scheres/fltk-1.3.5-source.tar.gz +- link : ftp://ftp.mrc-lmb.cam.ac.uk/pub/dari/classranker\_v1.0.ckpt.gz + +### 5.0.0 A100 +- same as 5.0-2beta (especially when it comes to the weights...) +- copy the fltk and the classranker file into the BUILD Directory. The build script assumes that these two files are in the BUILD\_DIR. +- sha256 errors occured during installation: +``` +ERROR: THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE. If you have updated the package versions, please update the hashes. Otherwise, examine the package contents carefully; someone may have tampered with them. + unknown package: + Expected sha256 20abd2cae58e55ca1af8a8dcf43293336a59adf0391f1917bf8518633cfc2cdf + Got e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` +- The download of some packages (.whl files) was blocked by the merlin7 firewall. Google the shasum and find the package name. Try to manually download with curl. If this fails, this is a strong indication for network/firewall issues that need to be adressed by the networking team. .whl files from conda +pip installations are ususally stored in the env in lib/pythonXXX. +- Also, to not clean up build directories: +``` +--no-clean +Don't clean up build directories. +(environment variable: PIP_NO_CLEAN) + +``` +### 5.0.0 for GH + +no successful installation so far - failing for conda env. Elsa G. build some hacky hack that works kind of.. +needs to be investigated further. + diff --git a/EM/relion/build b/EM/relion/build index 6e3c435..e2388ce 100755 --- a/EM/relion/build +++ b/EM/relion/build @@ -1,7 +1,5 @@ #!/usr/bin/env modbuild -pbuild::add_to_group 'EM' -pbuild::use_cmake PROFILE="no" SP="no" @@ -23,15 +21,22 @@ pbuild::prep() { BRANCH="${V_PKG}" fi - git clone --depth=1 -b "$BRANCH" https://github.com/3dem/relion.git "$SRC_DIR" || return $? + if [[ -d "$SRC_DIR/.git" ]]; then + cd "$SRC_DIR" + git pull + else + git clone --depth=1 -b "$BRANCH" https://github.com/3dem/relion.git "$SRC_DIR" || return $? + fi } + pbuild::pre_configure() { # Section will be switched on for -profile version #----------------------------------------------------------- if [[ "$PROFILE" == "yes" ]]; then - pbuild::add_configure_args "-DCMAKE_BUILD_TYPE=BENCHMARKING" + # this needs to go into the config.yml file for >5.0-1beta + : #pbuild::add_configure_args "-DCMAKE_BUILD_TYPE=BENCHMARKING" fi #------------------------------------------------------------ @@ -40,55 +45,49 @@ pbuild::pre_configure() { # Safe memory usage for multibody refinements #------------------------------------------------------------ if [[ "$SP" == "yes" ]]; then - pbuild::add_configure_args "-DDoublePrec_CPU=OFF" + # this needs to go into the config.yml file for >5.0-1beta + : #pbuild::add_configure_args "-DDoublePrec_CPU=OFF" fi #------------------------------------------------------------ - # The following section will only work for versions >= 5.0-beta # ------------------------------------------------------------- if [[ "$V_MAJOR" -ge "5" ]]; then - - #download blush weights - wget ftp://ftp.mrc-lmb.cam.ac.uk/pub/dari/blush_v1.0.ckpt.gz -P "$PREFIX/torch/hub/checkpoints/relion_blush" \ + #download blush weights, from zenodo + #mkdir -p "${PREFIX}" + wget -q https://zenodo.org/records/10072731/files/blush_v1.0.ckpt.gz -P "$PREFIX/torch/hub/checkpoints/relion_blush" \ && gunzip "$PREFIX/torch/hub/checkpoints/relion_blush/blush_v1.0.ckpt.gz" mv "$PREFIX/torch/hub/checkpoints/relion_blush/blush_v1.0.ckpt" "$PREFIX/torch/hub/checkpoints/relion_blush/v1.0.ckpt" touch "$PREFIX/torch/hub/checkpoints/relion_blush/v1.0_installed.txt" + + #cp classranker that was downloaded on another machine before from BUILDBLOCK_DIR + mkdir -p "$PREFIX/torch/hub/checkpoints/relion_class_ranker" + cp "$BUILDBLOCK_DIR/classranker_v1.0.ckpt.gz" "$PREFIX/torch/hub/checkpoints/relion_class_ranker" + gunzip "$PREFIX/torch/hub/checkpoints/relion_class_ranker/classranker_v1.0.ckpt.gz" + mv "$PREFIX/torch/hub/checkpoints/relion_class_ranker/classranker_v1.0.ckpt" "$PREFIX/torch/hub/checkpoints/relion_class_ranker/v1.0.ckpt" + touch "$PREFIX/torch/hub/checkpoints/relion_class_ranker/v1.0_installed.txt" + + # fltk stuff + mkdir -p "$SRC_DIR/external/fltk" + cp "$BUILDBLOCK_DIR/fltk-1.3.5-source.tar.gz" "$SRC_DIR/external/fltk/" + - #Set Torch Home - pbuild::add_configure_args "-DTORCH_HOME_PATH=$PREFIX/torch" + #INSTALL Miniconda + echo "installing miniconda" + mkdir -p "$PREFIX/miniconda" + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O "$PREFIX/miniconda/miniconda.sh" + #wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O "$PREFIX/miniconda/miniconda.sh" + bash "$PREFIX/miniconda/miniconda.sh" -b -u -p "$PREFIX/miniconda/" - #Conda - conda env create -f "$SRC_DIR/environment.yml" -p "$PREFIX/conda_env/$P_$V" - pbuild::add_configure_args "-DPYTHON_EXE_PATH=$PREFIX/conda_env/$P_$V/bin/python" + # Copy manually edited environment.yml file (with cryocare specs) into the folder to substitute the default one + #Conda create env with miniconda installation + "$PREFIX/miniconda/condabin/conda" env create --name "${P}-${V}" -f "$SRC_DIR/environment.yml" + "$PREFIX/miniconda/condabin/conda" env list fi #--------------------------------------------------------------- - # CMake-based build - - pbuild::add_configure_args "-DFORCE_OWN_FFTW=ON" - pbuild::add_configure_args "-DFORCE_OWN_FLTK=ON" - pbuild::add_configure_args "-DFORCE_OWN_TBB=ON" - - # CUDA related (set lowest CC to 6.1 for 1080 compatibility) - pbuild::add_configure_args "-DCUDA=ON" - pbuild::add_configure_args "-DCUDA_ARCH=61" - pbuild::add_configure_args "-DCudaTexture=ON" - - # Since ver3.0 relion cannot have ALTCPU and CUDA activated - # at the same time. Otherwise the configuration exits with - # an error. - if [[ "$V_MAJOR" -le "3" ]]; then - pbuild::add_configure_args "-DALTCPU=ON" - fi - - if [[ -n "$INTEL_VERSION" ]]; then - # Requires intel compiler - pbuild::add_configure_args "-DMKLFFT=ON" - fi } -# vim: ts=4 sw=4 et diff --git a/EM/relion/environment.yml b/EM/relion/environment.yml new file mode 100644 index 0000000..549b89b --- /dev/null +++ b/EM/relion/environment.yml @@ -0,0 +1,34 @@ +name: relion-5.0 +channels: + - conda-forge +dependencies: + - pip + - python=3.9 + - setuptools=59.5.0 + - pip: + - torch==2.0.1 + - torchvision==0.15.2 + - tqdm==4.65.0 + - mrcfile==1.4.3 + - starfile>=0.5.6 + - loguru==0.7.0 + - scikit-learn==1.3.0 + - umap-learn==0.5.3 + - matplotlib==3.7.2 + - morphosamplers==0.0.13 + - pydantic==1.10.18 + - napari[all]==0.4.18 + - tsnecuda==3.0.1 + - PyQt5==5.15.9 + - typer==0.9.0 + - biopython==1.81 + - fastcluster==1.2.6 + - seaborn==0.12.2 + - dill==0.3.7 + - numpy<2 + - git+https://github.com/3dem/relion-classranker@b6e751e5cb4205d8e9b36d0ae38c3687b3395acb + - git+https://github.com/3dem/relion-blush + - git+https://github.com/3dem/DynaMight + - git+https://github.com/3dem/topaz + - git+https://github.com/3dem/model-angelo + - ".[vis]" diff --git a/EM/relion/files/config.yaml b/EM/relion/files/config.yaml new file mode 100644 index 0000000..e51829f --- /dev/null +++ b/EM/relion/files/config.yaml @@ -0,0 +1,109 @@ +--- +format: 1 +relion: + defaults: + group: EM + overlay: base + relstage: unstable + build_variants: first_match + versions: + 5.0.0: + variants: + - overlay: Alps + target_cpus: [x86_64] + systems: [.*.merlin7.psi.ch] + build_requires: + - cmake/3.23.2 + - git/2.37.0 + - openmpi/5.0.5 + relstage: unstable + runtime_deps: + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + configure_with: cmake + configure_args: + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DFORCE_OWN_TBB=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=80" # dep on the cuda arch of the oldest GPU + - "-DFETCH_WEIGHTS=OFF" + # does not work yet- just an idea... + - overlay: Alps + target_cpus: [aarch64] + systems: [gpu0.*.merlin7.psi.ch] + build_requires: + #- cmake/3.23.2 + #- git/2.37.0 + - openmpi/5.0.5 + - gcc/12.3.0 + - cuda/12.2.0 + relstage: unstable + runtime_deps: + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + configure_with: cmake + configure_args: + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DFORCE_OWN_TBB=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=90" # dep on the cuda arch of the oldest GPU + - "-DFETCH_WEIGHTS=OFF" + #MERLIN7 old (cray) + 5.0-2beta: + variants: + - overlay: Alps + systems: [gpu.*.merlin7.psi.ch] + build_requires: + - cmake/3.23.2 + - git/2.37.0 + - libfabric/1.22.0 + relstage: unstable + runtime_deps: + - gcc/12.3.0 + # - cudatoolkit/23.9_12.2 + - cuda/12.2.0 + configure_with: cmake + configure_args: + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DFORCE_OWN_TBB=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=80" # dep on the cuda arch of the oldest GPU + #- "-DCudaTexture=ON" + - "-DFETCH_WEIGHTS=OFF" + #MERLIN6 + 5.0-1beta: + variants: + - systems: [merlin-.*.psi.ch] + build_requires: + - gcc/10.4.0 + - openmpi/4.1.4_slurm + - cuda/11.5.1 + - cmake/3.23.2 + - git/2.37.0 + relstage: unstable + runtime_deps: + - cuda/11.5.1 + - gcc/10.4.0 + - openmpi/4.1.4_slurm + configure_with: cmake + configure_args: + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DFORCE_OWN_TBB=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=61" # dep on the cuda arch of the oldest GPU + - "-DCudaTexture=ON" + diff --git a/EM/relion/files/variants b/EM/relion/files/variants deleted file mode 100644 index bbe791e..0000000 --- a/EM/relion/files/variants +++ /dev/null @@ -1,18 +0,0 @@ -relion/2.1.b1 deprecated gcc/4.9.4 openmpi/2.0.1 cuda/8.0.44 -relion/2.1 deprecated gcc/9.3.0 openmpi/4.0.4_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.0_beta deprecated gcc/4.9.4 openmpi/2.0.1 cuda/8.0.44 -relion/3.0.8 deprecated gcc/7.4.0 openmpi/3.1.4_merlin6 cuda/9.2.148 b:cmake/3.14.0 b:tiff/4.0.9 -relion/3.0.8-1 stable gcc/9.2.0 openmpi/4.0.5_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1-beta deprecated gcc/7.4.0 openmpi/3.1.4_merlin6 cuda/9.2.148 b:cmake/3.14.0 b:tiff/4.0.9 -relion/3.1.0 deprecated gcc/7.5.0 openmpi/4.0.4_slurm cuda/10.0.130 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1.0-1 deprecated gcc/7.5.0 openmpi/4.0.4_slurm cuda/10.1.105 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1.1 deprecated gcc/9.3.0 openmpi/4.0.4_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1.1-1 stable gcc/9.2.0 openmpi/4.0.5_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1.2 stable gcc/9.2.0 openmpi/4.0.5_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/3.1.3 stable gcc/9.2.0 openmpi/4.0.5_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/4.0-beta deprecated gcc/9.2.0 openmpi/4.0.5_slurm cuda/11.0.3 b:cmake/3.15.5 b:tiff/4.0.9 b:git/2.22.0 -relion/4.0.0 unstable gcc/11.2.0 openmpi/4.1.3_slurm cuda/11.5.1 b:cmake/3.23.2 b:tiff/4.3.0 b:git/2.37.0 -relion/4.0.1 unstable gcc/10.4.0 openmpi/4.1.4_slurm cuda/11.5.1 b:cmake/3.23.2 b:tiff/4.3.0 b:git/2.37.0 -relion/4.0.1-profile unstable gcc/10.4.0 openmpi/4.1.4_slurm cuda/11.5.1 b:cmake/3.23.2 b:tiff/4.3.0 b:git/2.37.0 -relion/5.0-beta unstable gcc/10.4.0 openmpi/4.1.4_slurm cuda/11.5.1 b:cmake/3.23.2 b:tiff/4.3.0 b:git/2.37.0 anaconda/2019.07 -relion/5.0-sp unstable gcc/10.4.0 openmpi/4.1.4_slurm cuda/11.5.1 b:cmake/3.23.2 b:tiff/4.3.0 b:git/2.37.0 anaconda/2019.07 diff --git a/EM/relion/modulefile b/EM/relion/modulefile index ed09173..1dac00d 100644 --- a/EM/relion/modulefile +++ b/EM/relion/modulefile @@ -25,10 +25,10 @@ Scheres(2012) JSB http://www.sciencedirect.com/science/article/pii/S104784771200 " setenv RELION_QUEUE_USE yes -setenv RELION_QUEUE_NAME gpu +setenv RELION_QUEUE_NAME general-x64 setenv RELION_QSUB_COMMAND sbatch setenv RELION_THREAD_MAX 44 -setenv RELION_QSUB_EXTRA_COUNT 2 +setenv RELION_QSUB_EXTRA_COUNT 4 setenv RELION_QSUB_EXTRA1 "Max Time" setenv RELION_QSUB_EXTRA1_DEFAULT "1-00:00:00" setenv RELION_QSUB_EXTRA1_HELP "Maximum time before the job is killed by slurm. Default: 1-00:00:00" @@ -36,32 +36,15 @@ setenv RELION_QSUB_EXTRA2 "Extra Slurm Args" setenv RELION_QSUB_EXTRA2_DEFAULT "" setenv RELION_QSUB_EXTRA2_HELP "Any additional arguments to pass to slurm" setenv RELION_SCRATCH_DIR /scratch -setenv RELION_UNBLUR_EXECUTABLE /opt/psi/EM/Unblur/1.0.2/bin/unblur_openmp_7_17_15.exe -setenv RELION_SUMMOVIE_EXECUTABLE /opt/psi/EM/Summovie/1.0.2/bin/sum_movie_openmp_7_17_15.exe setenv RELION_CTFFIND_EXECUTABLE /opt/psi/EM/ctffind4/4.1.8/bin/ctffind -setenv RELION_GCTF_EXECUTABLE /opt/psi/EM/Gctf/1.06/bin/Gctf +#also has internal version, depending on what users want... dont know if it works setenv RELION_RESMAP_EXECUTABLE /opt/psi/EM/ResMap/1.1.4/bin/ResMap -setenv RELION_TOPAZ_EXECUTABLE /opt/psi/EM/relion/5.0-sp/conda_env/5.0-beta/bin/topaz -setenv RELION_PYTHON_EXECUTABLE /opt/psi/EM/relion/5.0-sp/conda_env/5.0-beta/bin/python -setenv RELION_TORCH_HOME_PATH /opt/psi/EM/relion/5.0-sp/torch # Version-specific variables -setenv RELION_QSUB_TEMPLATE /opt/psi/EM/relion/$V/scripts/multi_gpu.sh +setenv RELION_QSUB_TEMPLATE /opt/psi/overlays/Alps/EM/relion/$V/scripts/multi_gpu.sh -# See MC2 compatibility matrix: https://intranet.psi.ch/en/bio/motioncor2 -if {[string match "8.0*" $env(CUDA_VERSION)]} { - setenv RELION_MOTIONCOR2_EXECUTABLE "/opt/psi/EM/MotionCor2/1.3.2/bin/MotionCor2_1.3.2-Cuda80" -} elseif {[string match "11.0*" $env(CUDA_VERSION)]} { - setenv RELION_MOTIONCOR2_EXECUTABLE "/opt/psi/EM/MotionCor2/1.4.0/bin/MotionCor2" -} else { - setenv RELION_MOTIONCOR2_EXECUTABLE "/opt/psi/EM/MotionCor2/1.6.4/bin/MotionCor2" -} +setenv RELION_MOTIONCOR2_EXECUTABLE "/opt/psi/EM/MotionCor2/1.6.4/bin/MotionCor2" -# relion 5 features -if {[file exists /opt/psi/EM/relion/$V/conda_env/$V]} { - setenv RELION_TOPAZ_EXECUTABLE /opt/psi/EM/relion/$V/conda_env/$V/bin/topaz - setenv RELION_PYTHON_EXECUTABLE /opt/psi/EM/relion/$V/conda_env/$V/bin/python -} -if {[file exists /opt/psi/EM/relion/$V/torch]} { - setenv RELION_TORCH_HOME_PATH /opt/psi/EM/relion/$V/torch -} +setenv RELION_TOPAZ_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/$P-$V/bin/topaz +setenv RELION_PYTHON_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/$P-$V/bin/python +setenv RELION_TORCH_HOME_PATH /opt/psi/overlays/Alps/EM/relion/$V/torch/