Merge branch 'gpu_updates' into 'master'

ADD: nvhidia 4.3 and 23.5 SDKK related openmpi and qe/7.3.1 CPU&GPU See merge request Pmodules/buildblocks!434
2024-07-24 11:30:26 +02:00
parent 666b93065f 993a57c8da
commit b40d6fd4dd
9 changed files with 202 additions and 19 deletions
--- a/Compiler/openmpi/build
+++ b/Compiler/openmpi/build
@@ -58,6 +58,13 @@ pbuild::pre_configure() {
                pbuild::add_configure_args "LDFLAGS=-Wc,-static-intel,-O0"
        fi

+        if [[ -v NVHPC_VERSION ]]; then
+                pbuild::add_configure_args "--with-cuda=${CUDA_HOME}"
+                pbuild::add_configure_args "CFLAGS=-fPIC -DPIC"
+				pbuild::add_configure_args "CXXFLAGS=-fPIC -DPIC"
+                pbuild::add_configure_args "FCFLAGS=-Mstandard -fPIC -DPIC"
+        fi
+
        if pbuild::use_flag slurm || pbuild::use_flag dgx || pbuild::use_flag merlin6; then
 		pbuild::add_configure_args "--with-gpfs=/usr/lpp/mmfs"
 		pbuild::add_configure_args "--with-pmi"
--- a/Compiler/openmpi/files/config.yaml
+++ b/Compiler/openmpi/files/config.yaml
@@ -9,7 +9,7 @@ openmpi:
    systems: [rhel7, rhel8, rhel9]
    docfiles: [AUTHORS, LICENSE, NEWS, README]
    urls:
-      - url: https://download.open-mpi.org/release/open-mpi/v${V_MAJOR}.${V_MINOR}/openmpi-${V_PKG}.tar.bz
+      - url: https://download.open-mpi.org/release/open-mpi/v${V_MAJOR}.${V_MINOR}/openmpi-${V_PKG}.tar.bz2
  shasums:
    openmpi-1.10.0.tar.bz2: 26b432ce8dcbad250a9787402f2c999ecb6c25695b00c9c6ee05a306c78b6490
    openmpi-1.10.4.tar.bz2: fb3c0c4c77896185013b6091b306d29ba592eb40d8395533da5c8bc300d922db
@@ -54,6 +54,14 @@ openmpi:
    openmpi-4.1.5.tar.bz2: a640986bc257389dd379886fdae6264c8cfa56bc98b71ce3ae3dfbd8ce61dbe3

  versions:
+    4.1.4:
+      variants:
+        - systems: [merlin-*, ra-*]
+          relstage: unstable
+          suffix: _slurm
+          group_deps:
+            compiler: {nvhpc: [24.3_cuda_12.3, 23.5_cuda_12.1]}
+          build_requires: [pmix/4.2.4, ucx/1.14.1_slurm, libfabric/1.18.0, hwloc/2.9.1, patchelf/0.14.5]

    4.1.5:
      variants:
--- a/MPI/hdf5/build
+++ b/MPI/hdf5/build
@@ -45,6 +45,11 @@ pbuild::pre_configure() {
 			pbuild::add_configure_args "CXXFLAGS=-fPIC"
 			pbuild::add_configure_args "FCFLAGS=-fPIC"
 			;;
+		nvhpc )
+			pbuild::add_configure_args "CFLAGS=-fPIC"
+			pbuild::add_configure_args "CXXFLAGS=-fPIC"
+			pbuild::add_configure_args "FCFLAGS=-fPIC"
+			;;
 	esac

 	if [[ "${enable_fortran}" == 'yes' ]]; then
--- a/MPI/qe/build
+++ b/MPI/qe/build
@@ -1,23 +1,22 @@
 #!/usr/bin/env modbuild

-pbuild::set_download_url "https://github.com/QEF/q-e/archive/$P-$V.tar.gz"
-
-pbuild::add_to_group 'MPI'
-
-pbuild::install_docfiles License README.md CONTRIBUTING.md
-
-pbuild::compile_in_sourcetree
-
 pbuild::pre_configure() {
-        pbuild::add_configure_args "-enable-openmp"
-}
-
-pbuild::compile() {
-        make all
-        make install
-}
-
-pbuild::install() {
-	:
+        pbuild::add_configure_args "-DQE_ENABLE_OPENMP=ON"
+        pbuild::add_configure_args "-DQE_ENABLE_MPI=ON"
+        # Recommended by the Quantum Espresso Developers
+        pbuild::add_configure_args "-DCMAKE_C_COMPILER=mpicc"
+        pbuild::add_configure_args "-DCMAKE_Fortran_COMPILER=mpif90"
+        pbuild::add_configure_args "-DCMAKE_SYSTEM_PROCESSOR=x86_64"
+
+        case "${COMPILER}" in
+                nvhpc )
+                        pbuild::add_configure_args "-DQE_ENABLE_CUDA=ON"
+                        pbuild::add_configure_args "-DQE_ENABLE_MPI_GPU_AWARE=ON"
+                        pbuild::add_configure_args "-DQE_ENABLE_OPENACC=ON"
+                        pbuild::add_configure_args "-DNVFORTRAN_CUDA_VERSION=12.1"
+                        pbuild::add_configure_args "-DNVFORTRAN_CUDA_CC=60"
+                        ;;
+        esac
+
 }

--- a/MPI/qe/files/config.yaml
+++ b/MPI/qe/files/config.yaml
@@ -0,0 +1,32 @@
+---
+# yamllint disable rule:line-length
+format: 1
+qe:
+  defaults:
+    group: MPI
+    overlay: base
+    relstage: stable
+    systems: [rhel7, rhel8, rhel9]
+    docfiles: [AUTHORS, LICENSE, NEWS, README]
+    configure_with: cmake
+    urls:
+      - url: https://gitlab.com/QEF/q-e/-/archive/qe-${V_PKG}/q-e-qe-${V_PKG}.tar.bz2
+  shasums:
+    qe-7.3.1.tar.bz2: 4955e4ca5e47a0faafefc72442155bef02ff94c8b796698ad81d4804a1cf6d88
+
+  versions:
+    7.3.1:
+      variants:
+        - systems: [merlin-*, ra-*]
+          group_deps:
+            compiler: {gcc: [13.1.0]}
+            mpi: {openmpi: [4.1.5_slurm]}
+          relstage: unstable
+          build_requires: [cmake/3.26.3, git/2.39.1, lapack/3.10.0, OpenBLAS/0.3.23, hdf5/1.13.1_slurm]
+        - systems: [merlin-*, ra-*]
+          group_deps:
+            compiler: {nvhpc: [23.5_cuda_12.1]}
+            mpi: {openmpi: [4.1.4_slurm]}
+          relstage: unstable
+          build_requires: [cmake/3.26.3, git/2.39.1]
+          runtime_deps: [libfabric/1.18.0]
--- a/Programming/nvhpc/Readme.md
+++ b/Programming/nvhpc/Readme.md
@@ -0,0 +1,5 @@
+After installing PGI, the localrc file of the pgi installation needs to be modified by issuing the following commands in the directory where the localrc file is located:
+
+    module add gcc/<appropriate version>  # currently 7.3.0
+    ./makelocalrc -x -gcc $(which gcc) -gpp $(which g++) -g77 $(which gfortran)
+
--- a/Programming/nvhpc/build
+++ b/Programming/nvhpc/build
@@ -0,0 +1,23 @@
+#!/usr/bin/env modbuild
+
+pbuild::configure() {
+	:
+}
+
+pbuild::compile() {
+	:
+}
+
+pbuild::install() {
+	export NVHPC_SILENT="true"
+	export NVHPC_INSTALL_DIR=$PREFIX
+	export NVHPC_INSTALL_TYPE="network"
+	./install
+}
+
+pbuild::post_install() {	
+	cd $PREFIX/Linux_x86_64/$V_MAJOR.$V_MINOR/compilers/bin
+	rm -rf localrc*
+	./makelocalrc -x $PWD
+}
+
--- a/Programming/nvhpc/files/config.yaml
+++ b/Programming/nvhpc/files/config.yaml
@@ -0,0 +1,33 @@
+---
+# yamllint disable rule:line-length
+format: 1
+nvhpc:
+  defaults:
+    group: Programming
+    overlay: base
+    relstage: stable
+    systems: [rhel7, rhel8, rhel9]
+    docfiles: [AUTHORS, LICENSE, NEWS, README]
+    compile_in_sourcetree: true 
+  shasums:
+    nvhpc-24.3.tar.gz: 4fb8949ba8cef73b28818bd9375c9420ec48fab1c64e71315a7c1984f5329d6b
+    nvhpc-23.5.tar.gz: 3196309b38df6f0a27e18b80f6a40e23cab8195cad43fc6bbd569af7d8ca5742
+
+  versions:
+    24.3:
+      variants:
+        - systems: [merlin-*, ra-*]
+          suffix: _cuda_12.3
+          urls:
+            - url: https://developer.download.nvidia.com/hpc-sdk/${V_PKG}/nvhpc_2024_243_Linux_x86_64_cuda_12.3.tar.gz
+          suffix: _cuda_12.3
+          relstage: unstable
+          build_requires: [gcc/10.4.0]
+    23.5:
+      variants:
+        - systems: [merlin-*, ra-*]
+          suffix: _cuda_12.1
+          urls:
+            - url: https://developer.download.nvidia.com/hpc-sdk/${V_PKG}/nvhpc_2023_235_Linux_x86_64_cuda_12.1.tar.gz
+          relstage: unstable
+          build_requires: [gcc/10.4.0]
--- a/Programming/nvhpc/modulefile
+++ b/Programming/nvhpc/modulefile
@@ -0,0 +1,71 @@
+#%Module1.0
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# NVIDIA CORPORATION and its licensors retain all intellectual property
+# and proprietary rights in and to this software, related documentation
+# and any modifications thereto.  Any use, reproduction, disclosure or
+# distribution of this software and related documentation without an express
+# license agreement from NVIDIA CORPORATION is strictly prohibited.
+
+module-whatis		"NVIDIA HPC SDK: A Comprehensive Suite of Compilers, Libraries and Tools for HPC"
+module-url		    "https://developer.nvidia.com/hpc-sdk"
+module-license		"# Copyright (c) 2023 NVIDIA CORPORATION.  All rights reserved.
+                     #
+                     # NVIDIA CORPORATION and its licensors retain all intellectual property
+                     # and proprietary rights in and to this software, related documentation
+                     # and any modifications thereto.  Any use, reproduction, disclosure or
+                     # distribution of this software and related documentation without an express
+                     # license agreement from NVIDIA CORPORATION is strictly prohibited."
+module-maintainer	"Elsa Germann <elsa.germann@psi.ch>"
+
+module-addgroup	"Compiler"
+
+set nvhome $PREFIX
+set target Linux_x86_64
+set version $V_MAJOR.3
+
+set nvcudadir $nvhome/$target/$version/cuda
+set nvcompdir $nvhome/$target/$version/compilers
+set nvmathdir $nvhome/$target/$version/math_libs
+set nvcommdir $nvhome/$target/$version/comm_libs
+
+setenv NVHPC $nvhome
+setenv NVHPC_ROOT $nvhome/$target/$version
+setenv CUDA_HOME $nvcudadir
+setenv CC $nvcompdir/bin/nvc
+setenv CXX $nvcompdir/bin/nvc++
+setenv FC $nvcompdir/bin/nvfortran
+setenv F90 $nvcompdir/bin/nvfortran
+setenv F77 $nvcompdir/bin/nvfortran
+setenv CPP cpp
+
+prepend-path PATH $nvcudadir/bin
+prepend-path PATH $nvcompdir/bin
+prepend-path PATH $nvcompdir/extras/qd/bin
+
+prepend-path LD_LIBRARY_PATH $nvcudadir/lib64
+prepend-path LD_LIBRARY_PATH $nvcudadir/extras/CUPTI/lib64
+prepend-path LD_LIBRARY_PATH $nvcompdir/extras/qd/lib
+prepend-path LD_LIBRARY_PATH $nvcompdir/lib
+prepend-path LD_LIBRARY_PATH $nvmathdir/lib64
+prepend-path LD_LIBRARY_PATH $nvcommdir/nccl/lib
+prepend-path LD_LIBRARY_PATH $nvcommdir/nvshmem/lib
+prepend-path LD_LIBRARY_PATH /opt/psi/Programming/gcc/10.4.0/lib64
+
+prepend-path LIBRARY_PATH /opt/psi/Programming/gcc/10.4.0/lib64
+prepend-path LIBRARY_PATH $nvmathdir/lib64
+prepend-path LIBRARY_PATH $nvcudadir/lib64
+prepend-path LIBRARY_PATH $nvcudadir/extras/CUPTI/lib64
+prepend-path LIBRARY_PATH $nvcompdir/extras/qd/lib
+prepend-path LIBRARY_PATH $nvcompdir/lib
+prepend-path LIBRARY_PATH $nvmathdir/lib64
+prepend-path LIBRARY_PATH $nvcommdir/nccl/lib
+prepend-path LIBRARY_PATH $nvcommdir/nvshmem/lib
+
+prepend-path CPATH $nvmathdir/include
+prepend-path CPATH $nvcommdir/nccl/include
+prepend-path CPATH $nvcommdir/nvshmem/include
+prepend-path CPATH $nvcompdir/extras/qd/include/qd
+
+prepend-path MANPATH $nvcompdir/man