diff --git a/Compiler/openmpi/build b/Compiler/openmpi/build index a09afbd..796fa90 100755 --- a/Compiler/openmpi/build +++ b/Compiler/openmpi/build @@ -7,11 +7,13 @@ pbuild::pre_configure() { if (( V_MAJOR < 5 )); then pbuild::add_configure_args "--enable-mpi-cxx" pbuild::add_configure_args "--enable-mpi-cxx-seek" - pbuild::add_configure_args "--with-pmi=/usr" - pbuild::add_configure_args "--with-pmi-libdir=/usr/lib64/slurmpmi" pbuild::add_configure_args "--enable-orterun-prefix-by-default" fi - if (( V_MAJOR >= 5 )); then + if (( V_MAJOR < 5 )) && pbuild::use_flag 'slurm'; then + pbuild::add_configure_args "--with-pmi=/usr" + pbuild::add_configure_args "--with-pmi-libdir=/usr/lib64/slurmpmi" +fi + if (( V_MAJOR >= 5 )); then pbuild::add_configure_args "--enable-prte-prefix-by-default" fi if (( V_MAJOR < 4 )); then @@ -24,43 +26,44 @@ pbuild::pre_configure() { pbuild::add_configure_args "--without-verbs" fi - if [[ -v CUDA_VERSION ]]; then + if [[ -v CUDA_VERSION ]]; then + std::info "Enabling CUDA ${CUDA_VERSION}." pbuild::add_configure_args "--with-cuda=${CUDA_HOME}" fi if [[ -v HWLOC_VERSION ]]; then unset HWLOC_VERSION + std::info "Enabling external hwloc ${HWLOC_PREFIX}." pbuild::add_configure_args "--with-hwloc=${HWLOC_PREFIX}" else pbuild::add_configure_args "--with-hwloc=internal" fi if [[ -v LIBEVENT_VERSION ]]; then + std::info "Enabling libevent ${LIBEVENT_VERSION}." pbuild::add_configure_args "--with-libevent=${LIBEVENT_PREFIX}" fi if [[ -v PMIX_VERSION ]]; then - std::info "Enabling PMIX ${PMIX_VERSION}" + std::info "Enabling PMIX ${PMIX_VERSION}." unset PMIX_VERSION pbuild::add_configure_args "--with-pmix=${PMIX_PREFIX}" fi - if [[ ! -v LIBFABRIC_VERSION ]]; then + if pbuild::use_flag merlin7 && [[ ! -v LIBFABRIC_VERSION ]]; then if pkg-config --exists libfabric; then std::info "Get LIBFABRIC_VERSION via pkg-config" LIBFABRIC_PREFIX=$(pkg-config --variable=prefix libfabric) LIBFABRIC_VERSION="$(LIBFABRIC_PREFIX##*/)" fi fi - std::info "LIBFABRIC_PREFIX: ${LIBFABRIC_PREFIX}" - std::info "LIBFABRIC_VERSION=: ${LIBFABRIC_VERSION}" if [[ -v LIBFABRIC_VERSION ]]; then - std::info "Enabling libfabric ${LIBFABRIC_VERSION}" + std::info "Enabling libfabric ${LIBFABRIC_VERSION}." pbuild::add_configure_args "--with-ofi=${LIBFABRIC_PREFIX}" fi if [[ -v UCX_VERSION ]]; then - std::info "Enabling UCX ${UCX_VERSION}" + std::info "Enabling UCX ${UCX_VERSION}." pbuild::add_configure_args "--with-ucx=${UCX_PREFIX}" fi @@ -73,9 +76,12 @@ pbuild::pre_configure() { pbuild::add_configure_args "LDFLAGS=-Wc,-static-intel,-O0" fi - if pbuild::use_flag slurm || pbuild::use_flag dgx || pbuild::use_flag merlin6 || pbuild::use_flag merlin7; then - std::info "Enabling SLURM." - pbuild::add_configure_args "--with-slurm=yes" + if pbuild::use_flag slurm || \ + pbuild::use_flag dgx || \ + pbuild::use_flag merlin6 || \ + pbuild::use_flag merlin7; then + std::info "Enabling SLURM." + pbuild::add_configure_args "--with-slurm=yes" fi if pbuild::use_flag slurm || pbuild::use_flag dgx || pbuild::use_flag merlin6; then std::info "Enabling GPFS." @@ -89,31 +95,44 @@ pbuild::pre_configure() { } pbuild::post_install_no_slurm() { - if ! pbuild::use_flag slurm && ! pbuild::use_flag dgx && ! pbuild::use_flag merlin6 && ! pbuild::use_flag merlin7; then - mkdir -p "${PREFIX}/lib/fallback" - local -r binary=$(ls "${PREFIX}"/lib/libmpi.so.*.*.*) - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libuc[mpst].so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libuct_ib.so.0' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libnuma.so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libibverbs.so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/librdmacm.so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi.so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi2.so' - pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi2.so' + if ! pbuild::use_flag slurm && \ + ! pbuild::use_flag dgx && \ + ! pbuild::use_flag merlin6 && \ + ! pbuild::use_flag merlin7; then + mkdir -p "${PREFIX}/lib/fallback" + local -r binary=$(ls "${PREFIX}"/lib/libmpi.so.*.*.*) + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libuc[mpst].so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libuct_ib.so.0' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libnuma.so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libibverbs.so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/librdmacm.so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi.so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi2.so' + pbuild::install_shared_libs "${binary}" "${PREFIX}/lib/fallback" '/libpmi2.so' fi } pbuild::post_install() { if [[ -v CUDA_VERSION ]]; then - echo "opal_warn_on_missing_libcuda = 0" >> ${PREFIX}/etc/openmpi-mca-params.conf + echo "opal_warn_on_missing_libcuda = 0" >> ${PREFIX}/etc/openmpi-mca-params.conf fi -} -# for FILE in $(find $PREFIX -type f \( ! -name "*.a" -and ! -name "*.mod" \) -exec grep -IL . "{}" \;) -# do -# OLD_RPATH=$(objdump -a -x $FILE | grep RPATH | awk '{print $2}') -# NEW_RPATH=$(echo $OLD_RPATH | sed 's/:\/usr\/lib64:/:/g') -# if [[ "${OLD_RPATH}" != "${NEW_RPATH}" ]]; then -# patchelf --force-rpath --set-rpath "${NEW_RPATH}" "${FILE}" -# fi -# done + local old_rpath='' + local new_rpath='' + while read file; do + old_rpath=$(objdump -x "${file}" | awk '/RPATH|RUNPATH/ {print $2}') + new_rpath=$(echo "${old_rpath}" | sed 's/:\/usr\/lib64:/:/g') + if [[ "${old_rpath}" != "${new_rpath}" ]]; then + patchelf \ + --force-rpath \ + --set-rpath "${new_rpath}" \ + "${file}" + fi + done < <(find "{PREFIX}" -type f \( ! -name "*.a" -and ! -name "*.mod" \) \ + -exec grep -IL . "{}" \;) + +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# tab-width: 8 +# End: diff --git a/Compiler/openmpi/files/config.yaml b/Compiler/openmpi/files/config.yaml index 0d9dfc8..2b21844 100644 --- a/Compiler/openmpi/files/config.yaml +++ b/Compiler/openmpi/files/config.yaml @@ -69,10 +69,9 @@ openmpi: group_deps: compiler: {gcc: [12.3.0]} build_requires: [pmix/5.0.3, hwloc/2.11.1, patchelf/0.14.5] - # build_requires: [pmix/5.0.3, ucx/1.15.0_slurm, hwloc/2.9.1, patchelf/0.14.5] runtime_deps: [cuda/12.2.0] configure_args+: - - --with-ofi=/opt/cray/libfabric/1.15.2.0 + - --with-ofi=${LIBFABRIC_PREFIX} - --enable-mpi1-compatibility - --with-cxi - --with-lustre @@ -86,20 +85,27 @@ openmpi: overlay: private use_overlays: [private] group_deps: - compiler: {gcc: [9.5.0, 10.4.0, 11.4.0, 12.3.0, 12.4.0, 13.1.0]} - build_requires: [pmix/4.2.4, ucx/1.14.1_slurm, libfabric/1.18.0, hwloc/2.9.1, patchelf/0.14.5] + compiler: {gcc: [9.5.0, 10.4.0, 11.4.0, 12.3.0, 12.4.0, 13.2.0, 14.2.0]} + build_requires: + - pmix/4.2.4 + - ucx/1.14.1_slurm + - libfabric/1.18.0 + - hwloc/2.9.1 + - patchelf/0.14.5 runtime_deps: [cuda/12.1.1] configure_args+: - CFLAGS=-g -O0 - LDFLAGS=-g - --with-gpfs=/usr/lpp/mmfs - --with-pmi=/usr + - systems: [ra-.*] suffix: _ra group_deps: compiler: {gcc: [9.5.0, 10.4.0, 11.4.0, 12.3.0, 13.1.0]} build_requires: [pmix/4.2.4, ucx/1.14.1_slurm, hwloc/2.9.1, patchelf/0.14.5] runtime_deps: [cuda/12.1.1] + - systems: [.*.merlin7.psi.ch] relstage: unstable overlay: Alps @@ -108,11 +114,11 @@ openmpi: build_requires: [pmix/4.2.9, hwloc/2.11.1, patchelf/0.14.5] runtime_deps: [cuda/12.2.0] configure_args+: - - --with-ofi=/opt/cray/libfabric/1.15.2.0 + - --with-ofi=${LIBFABRIC_PREFIX} - --enable-mpi1-compatibility - --with-lustre use_overlays: [PSI, Alps] - use_flags: [merlin7] + use_flags: [merlin7, slurm] 4.1.4: variants: