diff --git a/EM/relion/README.md b/EM/relion/README.md index 5af3571..f1c1701 100644 --- a/EM/relion/README.md +++ b/EM/relion/README.md @@ -23,6 +23,7 @@ https://git.psi.ch/lsm-hpce/relion-scripts - The build script was also updated accordingly - Also, cryocare is now available in a wrapper from relion, the cryocare installation executables need to be provided (seperate cryocare module). It is crucial to add the cryocare lib path (from the cryocare conda env) to the LD_LIBRARY_PATH when loading the relion module - tiff module no longer required, as this conflicts with the system shared tiff libs (not 100% sure why this canged from previous installations though) + ### 5.0-2beta (ON MERLIN7) - no fetching of weights with ftp possible, so cp from other machines into BUILD DIR and using the -DFETCH\_WEIGHTS=OFF option - link : ftp://ftp.mrc-lmb.cam.ac.uk/pub/scheres/fltk-1.3.5-source.tar.gz @@ -32,7 +33,7 @@ https://git.psi.ch/lsm-hpce/relion-scripts - same as 5.0-2beta (especially when it comes to the weights...) - copy the fltk and the classranker file into the BUILD Directory. The build script assumes that these two files are in the BUILD\_DIR. - sha256 errors occured during installation: -``` +```text ERROR: THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE. If you have updated the package versions, please update the hashes. Otherwise, examine the package contents carefully; someone may have tampered with them. unknown package: Expected sha256 20abd2cae58e55ca1af8a8dcf43293336a59adf0391f1917bf8518633cfc2cdf @@ -40,14 +41,17 @@ ERROR: THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE. If you ``` - The download of some packages (.whl files) was blocked by the merlin7 firewall. Google the shasum and find the package name. Try to manually download with curl. If this fails, this is a strong indication for network/firewall issues that need to be adressed by the networking team. .whl files from conda +pip installations are ususally stored in the env in lib/pythonXXX. - Also, to not clean up build directories: -``` +```text --no-clean Don't clean up build directories. (environment variable: PIP_NO_CLEAN) ``` +- adding flags to activate more aggressive compiler optimsations (affixing to `x86-64-v3` which is fully support on all EPYC nodes on Merlin7) + ### 5.0.0 for GH no successful installation so far - failing for conda env. Elsa G. build some hacky hack that works kind of.. needs to be investigated further. +- will attempt to build with more aggressive compiler optimisations as proscribed in: diff --git a/EM/relion/build b/EM/relion/build index e2388ce..d66be30 100755 --- a/EM/relion/build +++ b/EM/relion/build @@ -1,5 +1,6 @@ #!/usr/bin/env modbuild +PERFORMANCE="yes" PROFILE="no" SP="no" @@ -8,21 +9,18 @@ pbuild::prep() { local BRANCH if [[ "$V" =~ beta$ ]]; then BRANCH="ver${V_MAJOR}.${V_MINOR}" - elif [[ "$V" =~ profile$ ]]; then BRANCH="${V_PKG}" PROFILE="yes" - elif [[ "$V" =~ sp$ ]]; then BRANCH="ver${V_MAJOR}.${V_MINOR}" SP="yes" - else BRANCH="${V_PKG}" fi if [[ -d "$SRC_DIR/.git" ]]; then - cd "$SRC_DIR" + cd "$SRC_DIR" git pull else git clone --depth=1 -b "$BRANCH" https://github.com/3dem/relion.git "$SRC_DIR" || return $? @@ -31,63 +29,111 @@ pbuild::prep() { pbuild::pre_configure() { - # Section will be switched on for -profile version #----------------------------------------------------------- if [[ "$PROFILE" == "yes" ]]; then # this needs to go into the config.yml file for >5.0-1beta - : #pbuild::add_configure_args "-DCMAKE_BUILD_TYPE=BENCHMARKING" + if [[ "$V_MAJOR" -lt "5" ]]; then + : pbuild::add_configure_args "-DCMAKE_BUILD_TYPE=BENCHMARKING" + fi fi #------------------------------------------------------------ - # Section will be switched on for single-precision version # Safe memory usage for multibody refinements #------------------------------------------------------------ if [[ "$SP" == "yes" ]]; then # this needs to go into the config.yml file for >5.0-1beta - : #pbuild::add_configure_args "-DDoublePrec_CPU=OFF" + if [[ "$V_MAJOR" -lt "5" ]]; then + : pbuild::add_configure_args "-DDoublePrec_CPU=OFF" + fi fi #------------------------------------------------------------ + # Set performance oriented compiler flags, specific for a given + # architecture. + # ------------------------------------------------------------- + if [[ "$PERFORMANCE" == "yes" && "$PROFILE" == "no" ]]; then + # set compiler flags per arch + echo "> setting performance system compiler flags" + case "$(uname -m)" in + "aarch64") + export CC=gcc + export CFLAGS='-mcpu=neoverse-v2 -O3 -pipe -Wno-error' + export CXX=g++ + export CXXFLAGS='-mcpu=neoverse-v2 -O3 -pipe -Wno-error' + export FCFLAGS='-mcpu=neoverse-v2 -O3 -pipe' + export FFLAGS='-mcpu=neoverse-v2 -O3 -pipe' + export LDFLAGS='-Wl,--as-needed' + ;; + "x86_64") + export CC=gcc + export CFLAGS='-march=x86-64-v3 -mtune=znver1 -O3 -pipe -Wno-error' + export CXX=g++ + export CXXFLAGS='-march=x86-64-v3 -mtune=znver1 -O3 -pipe -Wno-error' + export FCFLAGS='-march=x86-64-v3 -mtune=znver1 -O3 -pipe' + export FFLAGS='-march=x86-64-v3 -mtune=znver1 -O3 -pipe' + export LDFLAGS='-Wl,--as-needed' + ;; + *) + echo ">> Unknown system arch found $(uname -m)! Using basic flags only!" + export CC=gcc + export CFLAGS='-O3 -pipe -Wno-error' + export CXX=g++ + export CXXFLAGS='-O3 -pipe -Wno-error' + export FCFLAGS='-O3 -pipe' + export FFLAGS='-O3 -pipe' + export LDFLAGS='-Wl,--as-needed' + ;; + esac + echo ">> ${CC} ${CFLAGS}" + echo ">> ${CXX} ${CXXFLAGS}" + fi + # ------------------------------------------------------------- + # The following section will only work for versions >= 5.0-beta # ------------------------------------------------------------- if [[ "$V_MAJOR" -ge "5" ]]; then - #download blush weights, from zenodo - #mkdir -p "${PREFIX}" + #download blush weights, from zenodo + echo "> downloading blush weights" + mkdir -p "$PREFIX/torch/hub/checkpoints/relion_blush" wget -q https://zenodo.org/records/10072731/files/blush_v1.0.ckpt.gz -P "$PREFIX/torch/hub/checkpoints/relion_blush" \ && gunzip "$PREFIX/torch/hub/checkpoints/relion_blush/blush_v1.0.ckpt.gz" mv "$PREFIX/torch/hub/checkpoints/relion_blush/blush_v1.0.ckpt" "$PREFIX/torch/hub/checkpoints/relion_blush/v1.0.ckpt" touch "$PREFIX/torch/hub/checkpoints/relion_blush/v1.0_installed.txt" - #cp classranker that was downloaded on another machine before from BUILDBLOCK_DIR + #cp classranker that was downloaded on another machine before from BUILDBLOCK_DIR + echo "> downloading classranker" mkdir -p "$PREFIX/torch/hub/checkpoints/relion_class_ranker" - cp "$BUILDBLOCK_DIR/classranker_v1.0.ckpt.gz" "$PREFIX/torch/hub/checkpoints/relion_class_ranker" - gunzip "$PREFIX/torch/hub/checkpoints/relion_class_ranker/classranker_v1.0.ckpt.gz" + wget -q ftp://ftp.mrc-lmb.cam.ac.uk/pub/dari/classranker_v1.0.ckpt.gz -P "$PREFIX/torch/hub/checkpoints/relion_class_ranker" \ + && gunzip "$PREFIX/torch/hub/checkpoints/relion_class_ranker/classranker_v1.0.ckpt.gz" mv "$PREFIX/torch/hub/checkpoints/relion_class_ranker/classranker_v1.0.ckpt" "$PREFIX/torch/hub/checkpoints/relion_class_ranker/v1.0.ckpt" touch "$PREFIX/torch/hub/checkpoints/relion_class_ranker/v1.0_installed.txt" # fltk stuff + echo "> downloading FLTK" mkdir -p "$SRC_DIR/external/fltk" - cp "$BUILDBLOCK_DIR/fltk-1.3.5-source.tar.gz" "$SRC_DIR/external/fltk/" - + wget -q ftp://ftp.mrc-lmb.cam.ac.uk/pub/scheres/fltk-1.3.5-source.tar.gz -P "$SRC_DIR/external/fltk/" - #INSTALL Miniconda - echo "installing miniconda" + # Install Miniconda + echo "> installing miniconda" mkdir -p "$PREFIX/miniconda" - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O "$PREFIX/miniconda/miniconda.sh" - #wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O "$PREFIX/miniconda/miniconda.sh" + if [[ "$(uname -m)" =~ "aarch64" ]]; then + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O "$PREFIX/miniconda/miniconda.sh" + else + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O "$PREFIX/miniconda/miniconda.sh" + fi bash "$PREFIX/miniconda/miniconda.sh" -b -u -p "$PREFIX/miniconda/" - # Copy manually edited environment.yml file (with cryocare specs) into the folder to substitute the default one - #Conda create env with miniconda installation - "$PREFIX/miniconda/condabin/conda" env create --name "${P}-${V}" -f "$SRC_DIR/environment.yml" + # Setup conda env; switch into relion repo directory to capture local + # `.[vis]` pip package dependencies + echo "> building conda env" + pushd "$SRC_DIR" + "$PREFIX/miniconda/condabin/conda" env create -q -f environment.yml "$PREFIX/miniconda/condabin/conda" env list + popd fi - + #--------------------------------------------------------------- - - } - diff --git a/EM/relion/environment.yml b/EM/relion/environment.yml deleted file mode 100644 index 549b89b..0000000 --- a/EM/relion/environment.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: relion-5.0 -channels: - - conda-forge -dependencies: - - pip - - python=3.9 - - setuptools=59.5.0 - - pip: - - torch==2.0.1 - - torchvision==0.15.2 - - tqdm==4.65.0 - - mrcfile==1.4.3 - - starfile>=0.5.6 - - loguru==0.7.0 - - scikit-learn==1.3.0 - - umap-learn==0.5.3 - - matplotlib==3.7.2 - - morphosamplers==0.0.13 - - pydantic==1.10.18 - - napari[all]==0.4.18 - - tsnecuda==3.0.1 - - PyQt5==5.15.9 - - typer==0.9.0 - - biopython==1.81 - - fastcluster==1.2.6 - - seaborn==0.12.2 - - dill==0.3.7 - - numpy<2 - - git+https://github.com/3dem/relion-classranker@b6e751e5cb4205d8e9b36d0ae38c3687b3395acb - - git+https://github.com/3dem/relion-blush - - git+https://github.com/3dem/DynaMight - - git+https://github.com/3dem/topaz - - git+https://github.com/3dem/model-angelo - - ".[vis]" diff --git a/EM/relion/files/config.yaml b/EM/relion/files/config.yaml index e51829f..e0242f6 100644 --- a/EM/relion/files/config.yaml +++ b/EM/relion/files/config.yaml @@ -7,38 +7,94 @@ relion: relstage: unstable build_variants: first_match versions: - 5.0.0: + 5.0.0-perf: variants: - - overlay: Alps + - overlay: Alps target_cpus: [x86_64] systems: [.*.merlin7.psi.ch] build_requires: - cmake/3.23.2 - git/2.37.0 - - openmpi/5.0.5 - relstage: unstable + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + relstage: stable runtime_deps: - gcc/12.3.0 - cuda/12.2.0 - openmpi/5.0.5 configure_with: cmake configure_args: + - "-DCMAKE_BUILD_TYPE=RELEASE" + - "-DCMAKE_CXX_FLAGS=${CXXFLAGS} -DNDEBUG" + - "-DCMAKE_C_FLAGS=${CFLAGS} -DNDEBUG" - "-DTORCH_HOME_PATH=${PREFIX}/torch" - - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" - "-DFORCE_OWN_FFTW=ON" - "-DFORCE_OWN_FLTK=ON" - - "-DFORCE_OWN_TBB=ON" - "-DCUDA=ON" - "-DCUDA_ARCH=80" # dep on the cuda arch of the oldest GPU - "-DFETCH_WEIGHTS=OFF" - # does not work yet- just an idea... - - overlay: Alps + # does not work yet- just an idea... + - overlay: Alps target_cpus: [aarch64] systems: [gpu0.*.merlin7.psi.ch] build_requires: #- cmake/3.23.2 #- git/2.37.0 - - openmpi/5.0.5 + - openmpi/5.0.5 + - gcc/12.3.0 + - cuda/12.2.0 + relstage: unstable + runtime_deps: + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + configure_with: cmake + configure_args: + - "-DCMAKE_BUILD_TYPE=RELEASE" + - "-DCMAKE_CXX_FLAGS=${CXXFLAGS} -DNDEBUG" + - "-DCMAKE_C_FLAGS=${CFLAGS} -DNDEBUG" + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=90" # dep on the cuda arch of the oldest GPU + - "-DFETCH_WEIGHTS=OFF" + 5.0.0: + variants: + - overlay: Alps + target_cpus: [x86_64] + systems: [.*.merlin7.psi.ch] + build_requires: + - cmake/3.23.2 + - git/2.37.0 + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + relstage: unstable + runtime_deps: + - gcc/12.3.0 + - cuda/12.2.0 + - openmpi/5.0.5 + configure_with: cmake + configure_args: + - "-DTORCH_HOME_PATH=${PREFIX}/torch" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" + - "-DFORCE_OWN_FFTW=ON" + - "-DFORCE_OWN_FLTK=ON" + - "-DCUDA=ON" + - "-DCUDA_ARCH=80" # dep on the cuda arch of the oldest GPU + - "-DFETCH_WEIGHTS=OFF" + # does not work yet- just an idea... + - overlay: Alps + target_cpus: [aarch64] + systems: [gpu0.*.merlin7.psi.ch] + build_requires: + #- cmake/3.23.2 + #- git/2.37.0 + - openmpi/5.0.5 - gcc/12.3.0 - cuda/12.2.0 relstage: unstable @@ -49,17 +105,16 @@ relion: configure_with: cmake configure_args: - "-DTORCH_HOME_PATH=${PREFIX}/torch" - - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" - "-DFORCE_OWN_FFTW=ON" - "-DFORCE_OWN_FLTK=ON" - - "-DFORCE_OWN_TBB=ON" - "-DCUDA=ON" - "-DCUDA_ARCH=90" # dep on the cuda arch of the oldest GPU - "-DFETCH_WEIGHTS=OFF" #MERLIN7 old (cray) 5.0-2beta: variants: - - overlay: Alps + - overlay: Alps systems: [gpu.*.merlin7.psi.ch] build_requires: - cmake/3.23.2 @@ -73,10 +128,9 @@ relion: configure_with: cmake configure_args: - "-DTORCH_HOME_PATH=${PREFIX}/torch" - - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" - "-DFORCE_OWN_FFTW=ON" - "-DFORCE_OWN_FLTK=ON" - - "-DFORCE_OWN_TBB=ON" - "-DCUDA=ON" - "-DCUDA_ARCH=80" # dep on the cuda arch of the oldest GPU #- "-DCudaTexture=ON" @@ -99,10 +153,9 @@ relion: configure_with: cmake configure_args: - "-DTORCH_HOME_PATH=${PREFIX}/torch" - - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/${P}-${V}/bin/python" + - "-DPYTHON_EXE_PATH=${PREFIX}/miniconda/envs/relion-5.0/bin/python" - "-DFORCE_OWN_FFTW=ON" - "-DFORCE_OWN_FLTK=ON" - - "-DFORCE_OWN_TBB=ON" - "-DCUDA=ON" - "-DCUDA_ARCH=61" # dep on the cuda arch of the oldest GPU - "-DCudaTexture=ON" diff --git a/EM/relion/modulefile b/EM/relion/modulefile index 1dac00d..53760e2 100644 --- a/EM/relion/modulefile +++ b/EM/relion/modulefile @@ -44,7 +44,7 @@ setenv RELION_QSUB_TEMPLATE /opt/psi/overlays/Alps/EM/relion/$V/scripts/multi_gp setenv RELION_MOTIONCOR2_EXECUTABLE "/opt/psi/EM/MotionCor2/1.6.4/bin/MotionCor2" -setenv RELION_TOPAZ_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/$P-$V/bin/topaz -setenv RELION_PYTHON_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/$P-$V/bin/python +setenv RELION_TOPAZ_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/relion-5.0/bin/topaz +setenv RELION_PYTHON_EXECUTABLE /opt/psi/overlays/Alps/EM/relion/$V/miniconda/envs/relion-5.0/bin/python setenv RELION_TORCH_HOME_PATH /opt/psi/overlays/Alps/EM/relion/$V/torch/