From eb488b2274b7e6c2e74b34cd19455335dd2d827e Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 11 Jul 2019 17:55:42 +0200 Subject: [PATCH 01/40] options for help command fixed --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 3b227f4..9dbaa7c 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1698,7 +1698,7 @@ subcommand_search() { # help [module|sub-command] # Subcommands[help]='help' -Options[help]='-o hHV? -l version -l help' +Options[help]='-o hHV\? -l version -l help' Help[help]=' USAGE: module [ switches ] [ subcommand ] [subcommand-args ] From fb28efc834625b05bee45bb7426ae9b29a9f7025 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 19 Jul 2019 17:01:15 +0200 Subject: [PATCH 02/40] function to get OS releases for Linux and macOS added to libstd --- Pmodules/libstd.bash | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 846484d..b3e5e1b 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -271,6 +271,31 @@ There is NO WARRANTY, to the extent permitted by law." done } +std.get_os_release_linux() { + source /etc/os-release + case "${ID}" in + science | rhel | centos | fedora ) + echo "rhel${VERSION_ID%.*}" + ;; + * ) + echo "Unknown" + exit 1 + ;; + esac +} +std.get_os_release_macos() { + VERSION_ID=$(sw_vers -productVersion) + echo "macOS${VERSION_ID%.*}" +} + +std::get_os_release() { + local -r OS=$(uname -s) + local -A func_map; + func_map['Linux']=std.get_os_release_linux + func_map['Darwin']=std.get_os_release_macos + ${func_map[${OS}]} +} + # Local Variables: # mode: sh # sh-basic-offset: 8 From 58359b837e764ae3002e6f04ee95718c25fff1ca Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 19 Jul 2019 17:17:23 +0200 Subject: [PATCH 03/40] review 'system' feature - 'system' and 'OS' have different meaning now 'system' defaults to a string derived from the Linux distribution (like rhel7) or on Mac to the macOS version (like macOS10.14) - OS is equivalent to the output of 'uname -s' - move dynamically created functions to new library libpbuild_dyn.bash - do not set defaults in libpbuild.bash - use the string returned by std::get_os_release() as system string if not passed as argument - --- Pmodules/libpbuild.bash | 52 ++++++-------------- Pmodules/libpbuild_dyn.bash | 24 +++++++++ Pmodules/modbuild.in | 98 ++++++++++++++++++++++++------------- build | 1 + 4 files changed, 104 insertions(+), 71 deletions(-) create mode 100644 Pmodules/libpbuild_dyn.bash diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 9f33653..83562ad 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -76,59 +76,48 @@ declare bootstrap='no' #.............................................................................. # global variables -declare force_rebuild='no' pbuild.force_rebuild() { - force_rebuild="$1" + declare -gr force_rebuild="$1" } -declare dry_run='no' pbuild.dry_run() { - dry_run="$1" + declare -gr dry_run="$1" } -declare enable_cleanup_build='yes' pbuild.enable_cleanup_build() { - enable_cleanup_build="$1" + declare -gr enable_cleanup_build="$1" } -declare enable_cleanup_src='yes' pbuild.enable_cleanup_src() { - enable_cleanup_src="$1" + declare -gr enable_cleanup_src="$1" } -declare build_target='all' pbuild.build_target() { - build_target="$1" + declare -gr build_target="$1" } -declare opt_update_modulefiles='no' pbuild.update_modulefiles() { - opt_update_modulefiles="$1" + declare -gr opt_update_modulefiles="$1" } # number of parallel make jobs -declare -i JOBS=3 pbuild.jobs() { - JOBS="$1" + declare -gr JOBS="$1" } -declare system=$(uname -s) pbuild.system() { - system="$1" + declare -gr system="$1" } -declare TEMP_DIR="${PMODULES_TMPDIR:-/var/tmp/${USER}}" pbuild.temp_dir() { - TEMP_DIR="$1" + declare -gr TEMP_DIR="$1" } -declare PMODULES_DISTFILESDIR="${PMODULES_ROOT}/var/distfiles" pbuild.pmodules_distfilesdir() { - PMODULES_DISTFILESDIR="$1" + declare -gr PMODULES_DISTFILESDIR="$1" } -declare verbose='no' pbuild.verbose() { - verbose='yes' + declare -gr verbose="$1" } # module name including path in hierarchy and version @@ -156,10 +145,7 @@ declare -r _DOCDIR='share/doc' # install prefix of module. declare -x PREFIX='' -# :FIXME: -# OS is still used in some build-scripts. We have to implement a getter -# and use this getter in the build-scripts. -declare -r OS="${system}" +declare -r OS=$(uname -s) ############################################################################## @@ -206,7 +192,9 @@ set_full_module_name_and_prefix() { echo "$*" } - [[ -n ${GROUP} ]] || std::die 1 "${module_name}/${module_version}: group not set." + [[ -n ${GROUP} ]] || std::die 1 \ + "${module_name}/${module_version}:" \ + "group not set." # build module name # :FIXME: this should be read from a configuration file @@ -327,12 +315,10 @@ pbuild::use_cc() { pbuild::pre_prep() { : } -eval "pbuild::pre_prep_${system}() { :; }" pbuild::post_prep() { : } -eval "pbuild::post_prep_${system}() { :; }" ############################################################################### # @@ -488,7 +474,6 @@ pbuild::add_patch() { PATCH_FILES+=( "$1" ) PATCH_STRIPS+=( "$2" ) } -eval "pbuild::add_patch_${system}() { pbuild::add_patch \"\$@\"; }" pbuild::set_default_patch_strip() { [[ -n "$1" ]] || \ @@ -509,7 +494,6 @@ pbuild::use_flag() { pbuild::pre_configure() { : } -eval "pbuild::pre_configure_${system}() { :; }" pbuild::set_configure_args() { CONFIGURE_ARGS+=( "$@" ) @@ -575,12 +559,10 @@ pbuild::configure() { pbuild::post_configure() { : } -eval "pbuild::post_configure_${system}() { :; }" pbuild::pre_compile() { : } -eval "pbuild::pre_compile_${system}() { :; }" pbuild::compile() { make -j${JOBS} @@ -589,12 +571,10 @@ pbuild::compile() { pbuild::post_compile() { : } -eval "pbuild::post_compile_${system}() { :; }" pbuild::pre_install() { : } -eval "pbuild::pre_install_${system}() { :; }" pbuild::install() { make install @@ -636,8 +616,6 @@ pbuild::install_shared_libs() { pbuild::post_install() { : } -eval "pbuild::post_install_${system}() { :; }" - # # The 'do it all' function. diff --git a/Pmodules/libpbuild_dyn.bash b/Pmodules/libpbuild_dyn.bash new file mode 100644 index 0000000..85f1d76 --- /dev/null +++ b/Pmodules/libpbuild_dyn.bash @@ -0,0 +1,24 @@ +#!/bin/bash + +eval "pbuild::pre_prep_${system}() { :; }" +eval "pbuild::pre_prep_${OS}() { :; }" +eval "pbuild::post_prep_${system}() { :; }" +eval "pbuild::post_prep_${OS}() { :; }" + +eval "pbuild::add_patch_${system}() { pbuild::add_patch \"\$@\"; }" +eval "pbuild::add_patch_${OD}() { pbuild::add_patch \"\$@\"; }" + +eval "pbuild::pre_configure_${system}() { :; }" +eval "pbuild::pre_configure_${OS}() { :; }" +eval "pbuild::post_configure_${system}() { :; }" +eval "pbuild::post_configure_${OS}() { :; }" + +eval "pbuild::pre_compile_${system}() { :; }" +eval "pbuild::pre_compile_${OS}() { :; }" +eval "pbuild::post_compile_${system}() { :; }" +eval "pbuild::post_compile_${OS}() { :; }" + +eval "pbuild::pre_install_${system}() { :; }" +eval "pbuild::pre_install_${OS}() { :; }" +eval "pbuild::post_install_${system}() { :; }" +eval "pbuild::post_install_${OS}() { :; }" diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b231656..e13b498 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -1,4 +1,4 @@ -#!/usr/bin/env bash -- +#!/usr/bin/env bash # # The following build specific variables are set and used in libpbuild.bash: # ARGS @@ -21,11 +21,11 @@ PATH+=":${mydir}" PATH+=":${mydir}/../lib:${mydir}/../config" source libstd.bash || { - echo "Oops: library '$_' cannot be loaded!" 1>&2; exit 3; + echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } source libpbuild.bash || \ - std::die 3 "Oops: Cannot source library -- '$_'" + std::die 3 "Oops: cannot source library -- '$_'" # save arguments, (still) required for building dependencies declare -r ARGS="$@" @@ -127,30 +127,40 @@ MISCELLANEOUS OPTIONS: declare -a versions=() declare opt_all_variants='no' declare opt_bootstrap='no' +declare opt_build_config='modbuild.conf' +declare opt_build_target='all' +declare opt_distfiles_dir="${PMODULES_ROOT}/var/distfiles" +declare opt_dry_run='no' +declare opt_enable_cleanup_build='yes' +declare opt_enable_cleanup_src='yes' +declare opt_force_rebuild='no' +declare -i opt_jobs=3 +declare opt_update_modulefiles='no' +declare opt_system='' +declare opt_temp_dir="${PMODULES_TMPDIR:-/var/tmp/${USER}}" +declare opt_verbose='no' # array collecting all modules specified on the command line via '--with=module' declare -a opt_with_modules=() -declare build_config='modbuild.conf' -declare system="$(uname -s)" parse_args() { while (( $# > 0 )); do case $1 in -j ) - pbuild.jobs "$2" + opt_jobs="$2" shift ;; --jobs=[0-9]* ) - pbuild.jobs "${1/--jobs=}" + opt_jobs="${1/--jobs=}" ;; -v | --verbose ) trap 'echo "$BASH_COMMAND"' DEBUG - pbuild.verbose 'yes' + opt_verbose='yes' ;; --debug ) set -x ;; -f | --force-rebuild ) - pbuild.force_rebuild 'yes' + opt_force_rebuild='yes' ;; -\? | -h | --help ) usage @@ -159,57 +169,55 @@ parse_args() { std::die 0 "\nPmodules version ${VERSION}\nCopyright GNU GPL v2\n" ;; --dry-run ) - pbuild.dry_run 'yes' + opt_dry_run='yes' ;; --config ) - build_config="$2" + opt_build_config="$2" shift 1 ;; --config=* ) - build_config="${1#*=}" + opt_build_config="${1#*=}" ;; --enable-cleanup ) - pbuild.enable_cleanup_build 'yes' - pbuild.enable_cleanup_src 'yes' + opt_enable_cleanup_build 'yes' + opt_enable_cleanup_src 'yes' ;; --disable-cleanup ) - pbuild.enable_cleanup_build 'no' - pbuild.enable_cleanup_src 'no' + opt_enable_cleanup_build 'no' + opt_enable_cleanup_src 'no' ;; --enable-cleanup-build ) - pbuild.enable_cleanup_build 'yes' + opt_enable_cleanup_build 'yes' ;; --disable-cleanup-build ) - pbuild.enable_cleanup_build 'no' + opt_enable_cleanup_build 'no' ;; --enable-cleanup-src ) - pbuild.enable_cleanup_src 'yes' + opt_enable_cleanup_src 'yes' ;; --disable-cleanup-src ) - pbuild.enable_cleanup_src 'no' + opt_enable_cleanup_src 'no' ;; --distdir ) - pbuild.pmodules_distfilesdir "$2" + opt_distfiles_dir "$2" shift ;; --distdir=* ) - pbuild.pmodules_distfilesdir "${1/--distdir=}" + opt_distfiles_dir "${1/--distdir=}" ;; --tmpdir ) - pbuild.temp_dir "$2" + opt_temp_dir="$2" shift ;; --tmpdir=* ) - pbuilf.temp_dir "${1/--tmpdir=}" + opt_temp_dir="${1/--tmpdir=}" ;; --system ) - system=".$2" - pbuild.system "${system}" + opt_system="$2" shift ;; --system=* ) - system=".${1/*=}" - pbuild.system "${system}" + opt_system="${1/*=}" ;; --with ) opt_with_modules+=( "$2" ) @@ -220,13 +228,13 @@ parse_args() { opt_with_modules+=( ${m} ) ;; --prep | --configure | --compile | --install | --all ) - pbuild.build_target ${1:2} + opt_build_target=${1:2} ;; --bootstrap ) opt_bootstrap='yes' ;; --update-modulefiles ) - pbuild.update_modulefiles 'yes' + opt_update_modulefiles='yes' ;; -- ) : @@ -263,10 +271,13 @@ find_variants_files(){ shopt -q nullglob || : local -i nullglob_set=$? shopt -s nullglob - local files=( "${BUILDBLOCK_DIR}"/*/variants.${system} ) + local files=( "${BUILDBLOCK_DIR}"/*/variants.${opt_system} ) + files+=( "${BUILDBLOCK_DIR}"/*/variants.$(uname -s) ) local f for f in "${BUILDBLOCK_DIR}"/*/variants; do - [[ -e "${f}.${system}" ]] || files+=( "$f" ) + [[ -e "${f}.${opt_system}" ]] \ + || [[ -e "${f}.$(uname -s)" ]] \ + || files+=( "$f" ) done (( nullglob_set == 1 )) && shopt -u nullglob std::upvar "$1" "${files[@]}" @@ -373,6 +384,25 @@ build_modules() { parse_args "$@" +if [[ -z "${opt_system}" ]]; then + opt_system=$(std::get_os_release) +fi + +pbuild.jobs "${opt_jobs}" +pbuild.force_rebuild "${opt_force_rebuild}" +pbuild.build_target "${opt_build_target}" +pbuild.dry_run "${opt_dry_run}" +pbuild.enable_cleanup_build "${opt_enable_cleanup_build}" +pbuild.enable_cleanup_src "${opt_enable_cleanup_src}" +pbuild.pmodules_distfilesdir "${opt_distfiles_dir}" +pbuild.update_modulefiles "${opt_update_modulefiles}" +pbuild.temp_dir "${opt_temp_dir}" +pbuild.system "${opt_system}" +pbuild.verbose "${opt_verbose}" + +source libpbuild_dyn.bash || \ + std::die 3 "Oops: cannot source library -- '$_'" + # source build configuration, # must be done before sourcing libpbuild! if [[ "${opt_bootstrap}" == 'yes' ]]; then @@ -380,8 +410,8 @@ if [[ "${opt_bootstrap}" == 'yes' ]]; then else test -d "${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}" && PATH+=":$_" fi -source "${build_config}" || \ - std::die 3 "Oops: Cannot source configuration file -- '${build_config}'" +source "${opt_build_config}" || \ + std::die 3 "Oops: Cannot source configuration file -- '$_'" declare -r BUILD_SCRIPT declare -r BUILDBLOCK_DIR diff --git a/build b/build index a858aef..c34dd90 100755 --- a/build +++ b/build @@ -313,6 +313,7 @@ pmodules::install() { install -m 0644 "${SRC_DIR}/libpmodules.bash" "${PMODULES_HOME}/lib" install -m 0644 "${SRC_DIR}/libpbuild.bash" "${PMODULES_HOME}/lib" + install -m 0644 "${SRC_DIR}/libpbuild_dyn.bash" "${PMODULES_HOME}/lib" install -m 0644 "${SRC_DIR}/libstd.bash" "${PMODULES_HOME}/lib" install -m 0755 -d "${PMODULES_HOME}/lib/Pmodules" install -m 0644 "${SRC_DIR}/libmodules.tcl" "${PMODULES_HOME}/lib/Pmodules" From 74db99c87e128ed3fd16cc045b6a57673bfc8add Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 23 Jul 2019 17:20:09 +0200 Subject: [PATCH 04/40] fixes for isses 52 to 55 - ignore multiple load commands (#52) - do not print empty line if output to stderr is empty (#53) - loading dependencies review (#54) - rescan group in 'use' sub-command if newly created group is requested --- Pmodules/modulecmd.bash.in | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 9dbaa7c..2481169 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -294,8 +294,10 @@ subcommand_load() { # 0: module is loadable # 1: either not a modulefile or unsused release # - # Notes: - # The variable 'release' in function 'subcommand_load()' will be set. + # The following variables in the enclosing function are set: + # current_modulefile + # prefix + # release # is_available() { local m=$1 @@ -359,13 +361,7 @@ subcommand_load() { [[ -z ${dep} ]] && continue [[ ${dep:0:1} == \# ]] && continue module_is_loaded "${dep}" && continue - local output=$( subcommand_load 'bash' "${dep}") - eval ${output} - if [[ "${Shell}" == "bash" ]]; then - echo ${output} - else - subcommand_load "${Shell}" "${dep}" - fi + subcommand_load "${dep}" done < "${fname}" } @@ -468,6 +464,8 @@ subcommand_load() { fi local found='' for flag in "${UseFlags[@]/#/_}" ""; do + # :FIXME: this doesn't work if ${m} is a + # modulename without version if is_available "${m}${flag}"; then m+="${flag}" found=':' @@ -480,11 +478,12 @@ subcommand_load() { [[ ${verbosity_lvl} == 'verbose' ]] && output_load_hints std::die 3 "" fi + if [[ ${current_modulefile} =~ ${PMODULES_ROOT} ]] \ + && [[ ! ${m} =~ / ]]; then + m+="/${current_modulefile##*/}" + fi if [[ ":${LOADEDMODULES}:" =~ ":${m}:" ]]; then - std::die 3 "%s %s: %s -- %s\n" \ - "${CMD}" "${subcommand}" \ - "module conflicts with already loaded module" \ - "${m}" + continue fi if [[ ${current_modulefile} =~ ${PMODULES_ROOT} ]]; then # modulefile is in our hierarchy @@ -519,7 +518,9 @@ subcommand_load() { fi if [[ "${Shell}" == "bash" ]]; then echo "${output}" - echo "${error}" 1>&2 + if [[ -n "${error}" ]]; then + echo "${error}" 1>&2 + fi else "${modulecmd}" "${Shell}" ${opts} 'load' \ "${current_modulefile}" @@ -1050,6 +1051,10 @@ subcommand_use() { std::append_path UseFlags "${arg/flag=}" return fi + if [[ -z ${GroupDepths[${arg}]} ]] && [[ -d "${PMODULES_ROOT}/${arg}" ]]; then + scan_groups "${PMODULES_ROOT}" + fi + if [[ -n ${GroupDepths[${arg}]} ]] && (( ${GroupDepths[${arg}]} == 0 )); then # argument is group in our root with depth 0 From 52648a3274da7e9adb9aee19f965b584729ad648 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 23 Jul 2019 17:47:11 +0200 Subject: [PATCH 05/40] libpbuild: missing targets added (#56) --- Pmodules/libpbuild.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 83562ad..86dd8d1 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -983,9 +983,9 @@ pbuild::make_all() { return 0 fi local targets=() - targets+=( "pre_${target}_${system}" "pre_${target}" ) + targets+=( "pre_${target}_${system}" "pre_${target}_${OS}" "pre_${target}" ) targets+=( "${target}" ) - targets+=( "post_${target}_${system}" "post_${target}" ) + targets+=( "post_${target}_${system}" "post_${target}_${OS}" "post_${target}" ) for t in "${targets[@]}"; do # We cd into the dir before calling the function - From 85fc352f10504f23a794e00fa74a365d6d21229d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 23 Jul 2019 17:48:11 +0200 Subject: [PATCH 06/40] libstd.bash: setting of unused variable OS removed --- Pmodules/libstd.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index b3e5e1b..8447915 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -289,7 +289,6 @@ std.get_os_release_macos() { } std::get_os_release() { - local -r OS=$(uname -s) local -A func_map; func_map['Linux']=std.get_os_release_linux func_map['Darwin']=std.get_os_release_macos From 38a09a8ba36df6eebdc7b91c7951e605603a501a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 23 Jul 2019 17:49:54 +0200 Subject: [PATCH 07/40] fixes for issues #57 and #58 - fixes building dependencies on macOS (#57) - fixes errors in parsing arguments (#58) --- Pmodules/modbuild.in | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index e13b498..9b60313 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -16,7 +16,13 @@ declare -r mydir=$(cd ${mydir} && pwd -P) # initialize PATH, # add library installation directories to the PATH, # so 'source' is able find them -PATH="/usr/bin:/bin:/usr/sbin:/sbin" + +if [[ $(uname -s) == 'Darwin' ]]; then + PATH='/usr/local/bin:' +else + PATH='' +fi +PATH+='/usr/bin:/bin:/usr/sbin:/sbin' PATH+=":${mydir}" PATH+=":${mydir}/../lib:${mydir}/../config" @@ -179,31 +185,31 @@ parse_args() { opt_build_config="${1#*=}" ;; --enable-cleanup ) - opt_enable_cleanup_build 'yes' - opt_enable_cleanup_src 'yes' + opt_enable_cleanup_build='yes' + opt_enable_cleanup_src='yes' ;; --disable-cleanup ) - opt_enable_cleanup_build 'no' - opt_enable_cleanup_src 'no' + opt_enable_cleanup_build='no' + opt_enable_cleanup_src='no' ;; --enable-cleanup-build ) - opt_enable_cleanup_build 'yes' + opt_enable_cleanup_build='yes' ;; --disable-cleanup-build ) - opt_enable_cleanup_build 'no' + opt_enable_cleanup_build='no' ;; --enable-cleanup-src ) - opt_enable_cleanup_src 'yes' + opt_enable_cleanup_src='yes' ;; --disable-cleanup-src ) - opt_enable_cleanup_src 'no' + opt_enable_cleanup_src='no' ;; --distdir ) - opt_distfiles_dir "$2" + opt_distfiles_dir="$2" shift ;; --distdir=* ) - opt_distfiles_dir "${1/--distdir=}" + opt_distfiles_dir="${1/--distdir=}" ;; --tmpdir ) opt_temp_dir="$2" @@ -408,8 +414,14 @@ source libpbuild_dyn.bash || \ if [[ "${opt_bootstrap}" == 'yes' ]]; then test -d "${BUILDBLOCK_DIR}/../../${PMODULES_CONFIG_DIR}" && PATH+=":$_" else - test -d "${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}" && PATH+=":$_" + # Please note: if we trap DEBUG a statement like + # test -d ... && PATH+=$_ + # does not work (at least on macOS with bash 4 and 5) + if [[ -d "${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}" ]]; then + PATH+=":${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}" + fi fi +echo PATH=$PATH source "${opt_build_config}" || \ std::die 3 "Oops: Cannot source configuration file -- '$_'" From c4a7c48dd9061acaeb0d828b5fe307fd0bf8d1e2 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 24 Jul 2019 16:51:20 +0200 Subject: [PATCH 08/40] libstd.bash: fix issue in getting OS release - try to get the OS release via lsb_release first - then via /etc/os-release - abort if both fail --- Pmodules/libstd.bash | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 8447915..544f435 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -272,7 +272,19 @@ There is NO WARRANTY, to the extent permitted by law." } std.get_os_release_linux() { - source /etc/os-release + local lsb_release=$(which lsb_release) + local ID='' + local VERSION_ID='' + + if [[ -n $(which lsb_release) ]]; then + ID=$(lsb_release -is) + VERSION_ID=$(lsb_release -rs) + elif [[ -r '/etc/os-release' ]]; then + source /etc/os-release + else + std::die 4 "Cannot determin OS release!\n" + fi + case "${ID}" in science | rhel | centos | fedora ) echo "rhel${VERSION_ID%.*}" From 7df9fe71118380a4207140e917449614e5a3b479 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 24 Jul 2019 17:26:37 +0200 Subject: [PATCH 09/40] libstd.bash: list of possible OS ID's reviewed --- Pmodules/libstd.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 544f435..e54f4b7 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -286,7 +286,7 @@ std.get_os_release_linux() { fi case "${ID}" in - science | rhel | centos | fedora ) + RedHatEnterpriseServer | RedHatEnterprise | Scientific | rhel | centos | fedora ) echo "rhel${VERSION_ID%.*}" ;; * ) From a1697cd0c5a8075f468f043e337d916f8804edf3 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 24 Jul 2019 17:34:40 +0200 Subject: [PATCH 10/40] libpbuild: option -g is not supported in declare on RHEL/SL 6 --- Pmodules/libpbuild.bash | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 86dd8d1..15d080c 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -76,48 +76,60 @@ declare bootstrap='no' #.............................................................................. # global variables +declare force_rebuild='' pbuild.force_rebuild() { - declare -gr force_rebuild="$1" + declare -r force_rebuild="$3" } +declare dry_run='' pbuild.dry_run() { - declare -gr dry_run="$1" + declare -r dry_run="$1" } +declare enable_cleanup_build='' pbuild.enable_cleanup_build() { - declare -gr enable_cleanup_build="$1" + declare -r enable_cleanup_build="$1" } +declare enable_cleanup_src='' pbuild.enable_cleanup_src() { - declare -gr enable_cleanup_src="$1" + declare -r enable_cleanup_src="$1" } +declare build_target='' pbuild.build_target() { - declare -gr build_target="$1" + declare -r build_target="$1" } +declare opt_update_modulefiles='' pbuild.update_modulefiles() { - declare -gr opt_update_modulefiles="$1" + declare -r opt_update_modulefiles="$1" } # number of parallel make jobs +declare -i JOBS=3 pbuild.jobs() { - declare -gr JOBS="$1" + declare -r JOBS="$1" } +declare system='' pbuild.system() { - declare -gr system="$1" + declare -r system="$1" } +declare TEMP_DIR='' pbuild.temp_dir() { - declare -gr TEMP_DIR="$1" + declare -r TEMP_DIR="$1" } +declare PMODULES_DISTFILESDIR='' pbuild.pmodules_distfilesdir() { - declare -gr PMODULES_DISTFILESDIR="$1" + declare -r PMODULES_DISTFILESDIR="$1" } + +declare verbose='' pbuild.verbose() { - declare -gr verbose="$1" + declare -r verbose="$1" } # module name including path in hierarchy and version From 98fac76798bcbfffdcdfafeee4f42ea2ce4f9d87 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 25 Jul 2019 11:09:10 +0200 Subject: [PATCH 11/40] libpbuild.bash: do not use declare for global variables inside functions. --- Pmodules/libpbuild.bash | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 15d080c..126c52f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -78,58 +78,58 @@ declare bootstrap='no' # global variables declare force_rebuild='' pbuild.force_rebuild() { - declare -r force_rebuild="$3" + force_rebuild="$1" } declare dry_run='' pbuild.dry_run() { - declare -r dry_run="$1" + dry_run="$1" } declare enable_cleanup_build='' pbuild.enable_cleanup_build() { - declare -r enable_cleanup_build="$1" + enable_cleanup_build="$1" } declare enable_cleanup_src='' pbuild.enable_cleanup_src() { - declare -r enable_cleanup_src="$1" + enable_cleanup_src="$1" } declare build_target='' pbuild.build_target() { - declare -r build_target="$1" + build_target="$1" } declare opt_update_modulefiles='' pbuild.update_modulefiles() { - declare -r opt_update_modulefiles="$1" + opt_update_modulefiles="$1" } # number of parallel make jobs declare -i JOBS=3 pbuild.jobs() { - declare -r JOBS="$1" + JOBS="$1" } declare system='' pbuild.system() { - declare -r system="$1" + system="$1" } declare TEMP_DIR='' pbuild.temp_dir() { - declare -r TEMP_DIR="$1" + TEMP_DIR="$1" } declare PMODULES_DISTFILESDIR='' pbuild.pmodules_distfilesdir() { - declare -r PMODULES_DISTFILESDIR="$1" + PMODULES_DISTFILESDIR="$1" } declare verbose='' pbuild.verbose() { - declare -r verbose="$1" + verbose="$1" } # module name including path in hierarchy and version From f5dd0aa60af05529c853a0c2ecd14eb25362a589 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 25 Jul 2019 11:09:37 +0200 Subject: [PATCH 12/40] modbuild.in: debug echo statement removed --- Pmodules/modbuild.in | 1 - 1 file changed, 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 9b60313..4fafb73 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -421,7 +421,6 @@ else PATH+=":${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}" fi fi -echo PATH=$PATH source "${opt_build_config}" || \ std::die 3 "Oops: Cannot source configuration file -- '$_'" From 47723622cb1185226a0e45196f41bd9c3f240633 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 9 Aug 2019 16:51:53 +0200 Subject: [PATCH 13/40] load runtime dependencies before sourcing the build-script --- Pmodules/libpbuild.bash | 311 ++++++++++++++++++++-------------------- 1 file changed, 156 insertions(+), 155 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 126c52f..2bbbda1 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -634,6 +634,7 @@ pbuild::post_install() { # pbuild::make_all() { local -a runtime_dependencies=() + source "${BUILD_SCRIPT}" # # everything set up? @@ -659,158 +660,6 @@ pbuild::make_all() { "Not available for ${system}." } - #...................................................................... - # - # test whether a module is loaded or not - # - # $1: module name - # - is_loaded() { - [[ :${LOADEDMODULES}: =~ :$1: ]] - } - - #...................................................................... - # - # build a dependency - # - # $1: name of module to build - # - # :FIXME: needs testing - # - build_dependency() { - #.............................................................. - # - # Test whether a module with the given name already exists. - # - # Arguments: - # $1: module name - # - # Notes: - # The passed module name should be NAME/VERSION - # :FIXME: this does not really work in a hierarchical group - # without adding the dependencies... - # - module_exists() { - [[ -n $("${MODULECMD}" bash search -a --no-header "$1" \ - 2>&1 1>/dev/null) ]] - } - - - local -r m=$1 - std::debug "${m}: module not available" - local rels=( ${PMODULES_DEFINED_RELEASES//:/ } ) - [[ ${dry_run} == yes ]] && \ - std::die 1 \ - "%s " \ - "${m}: module does not exist," \ - "cannot continue with dry run..." - - std::info "$m: module does not exist, trying to build it..." - local args=( '' ) - set -- ${ARGS[@]} - while (( $# > 0 )); do - case $1 in - -j ) - args+=( "-j $2" ) - shift - ;; - --jobs=[0-9]* ) - args+=( $1 ) - ;; - -v | --verbose) - args+=( $1 ) - ;; - --with=*/* ) - args+=( $1 ) - ;; - esac - shift - done - - find_build_script(){ - local p=$1 - local script=$(find "${BUILDBLOCK_DIR}/../.." -path "*/$p/build") - std::get_abspath "${script}" - } - local buildscript=$(find_build_script "${m%/*}") - [[ -x "${buildscript}" ]] || \ - std::die 1 \ - "$m: build-block not found!" - "${buildscript}" "${m#*/}" ${args[@]} - module_exists "$m" || \ - std::die 1 \ - "$m: oops: build failed..." - } - - #...................................................................... - # - # Load build- and run-time dependencies. - # - # Arguments: - # none - # - # Variables - # [r] module_release set if defined in a variants file - # runtime_dependencies runtime dependencies from variants added - # - load_build_dependencies() { - local m='' - for m in "${with_modules[@]}"; do - - # module name prefixes in dependency declarations: - # 'b:' this is a build dependency - # 'r:' this a run-time dependency, *not* required for - # building - # without prefix: this is a build and - # run-time dependency - if [[ "${m:0:2}" == "b:" ]]; then - m=${m#*:} # remove 'b:' - elif [[ "${m:0:2}" == "r:" ]]; then - m=${m#*:} # remove 'r:' - runtime_dependencies+=( "$m" ) - else - runtime_dependencies+=( "$m" ) - fi - is_loaded "$m" && continue - - # 'module avail' might output multiple matches if module - # name and version are not fully specified or in case - # modules with and without a release number exist. - # Example: - # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list - # from 'module avail' and the full version should be set - # in the variants file, we look for the first exact - # match. - local release_of_dependency='' - if ! pbuild::module_is_avail "$m" release_of_dependency; then - build_dependency "$m" - pbuild::module_is_avail "$m" release_of_dependency || \ - std::die 6 "Oops" - fi - # should be set, just in case it is not... - : ${release_of_dependency:='unstable'} - - # for a stable module all dependencies must be stable - if [[ "${module_release}" == 'stable' ]] \ - && [[ "${release_of_dependency}" != 'stable' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - # for a unstable module no dependency must be deprecated - elif [[ "${module_release}" == 'unstable' ]] \ - && [[ "${release_of_dependency}" == 'deprecated' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - fi - - echo "Loading module: ${m}" - module load "${m}" - done - } - #...................................................................... # non-redefinable post-install post_install() { @@ -1104,7 +953,6 @@ pbuild::make_all() { # setup module specific environment if [[ "${bootstrap}" == 'no' ]]; then check_supported_systems - load_build_dependencies set_full_module_name_and_prefix if [[ "${module_release}" == 'removed' ]]; then remove_module @@ -1224,6 +1072,158 @@ pbuild.build_module() { shift 3 with_modules=( "$@" ) + #...................................................................... + # + # test whether a module is loaded or not + # + # $1: module name + # + is_loaded() { + [[ :${LOADEDMODULES}: =~ :$1: ]] + } + + #...................................................................... + # + # build a dependency + # + # $1: name of module to build + # + # :FIXME: needs testing + # + build_dependency() { + #.............................................................. + # + # Test whether a module with the given name already exists. + # + # Arguments: + # $1: module name + # + # Notes: + # The passed module name should be NAME/VERSION + # :FIXME: this does not really work in a hierarchical group + # without adding the dependencies... + # + module_exists() { + [[ -n $("${MODULECMD}" bash search -a --no-header "$1" \ + 2>&1 1>/dev/null) ]] + } + + + local -r m=$1 + std::debug "${m}: module not available" + local rels=( ${PMODULES_DEFINED_RELEASES//:/ } ) + [[ ${dry_run} == yes ]] && \ + std::die 1 \ + "%s " \ + "${m}: module does not exist," \ + "cannot continue with dry run..." + + std::info "$m: module does not exist, trying to build it..." + local args=( '' ) + set -- ${ARGS[@]} + while (( $# > 0 )); do + case $1 in + -j ) + args+=( "-j $2" ) + shift + ;; + --jobs=[0-9]* ) + args+=( $1 ) + ;; + -v | --verbose) + args+=( $1 ) + ;; + --with=*/* ) + args+=( $1 ) + ;; + esac + shift + done + + find_build_script(){ + local p=$1 + local script=$(find "${BUILDBLOCK_DIR}/../.." -path "*/$p/build") + std::get_abspath "${script}" + } + local buildscript=$(find_build_script "${m%/*}") + [[ -x "${buildscript}" ]] || \ + std::die 1 \ + "$m: build-block not found!" + "${buildscript}" "${m#*/}" ${args[@]} + module_exists "$m" || \ + std::die 1 \ + "$m: oops: build failed..." + } + + #...................................................................... + # + # Load build- and run-time dependencies. + # + # Arguments: + # none + # + # Variables + # [r] module_release set if defined in a variants file + # runtime_dependencies runtime dependencies from variants added + # + load_build_dependencies() { + local m='' + for m in "${with_modules[@]}"; do + + # module name prefixes in dependency declarations: + # 'b:' this is a build dependency + # 'r:' this a run-time dependency, *not* required for + # building + # without prefix: this is a build and + # run-time dependency + if [[ "${m:0:2}" == "b:" ]]; then + m=${m#*:} # remove 'b:' + elif [[ "${m:0:2}" == "r:" ]]; then + m=${m#*:} # remove 'r:' + runtime_dependencies+=( "$m" ) + else + runtime_dependencies+=( "$m" ) + fi + is_loaded "$m" && continue + + # 'module avail' might output multiple matches if module + # name and version are not fully specified or in case + # modules with and without a release number exist. + # Example: + # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list + # from 'module avail' and the full version should be set + # in the variants file, we look for the first exact + # match. + local release_of_dependency='' + if ! pbuild::module_is_avail "$m" release_of_dependency; then + build_dependency "$m" + pbuild::module_is_avail "$m" release_of_dependency || \ + std::die 6 "Oops" + fi + # should be set, just in case it is not... + : ${release_of_dependency:='unstable'} + + # for a stable module all dependencies must be stable + if [[ "${module_release}" == 'stable' ]] \ + && [[ "${release_of_dependency}" != 'stable' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + # for a unstable module no dependency must be deprecated + elif [[ "${module_release}" == 'unstable' ]] \ + && [[ "${release_of_dependency}" == 'deprecated' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + fi + + echo "Loading module: ${m}" + module load "${m}" + done + } + MODULECMD="${PMODULES_HOME}/bin/modulecmd" [[ -x ${MODULECMD} ]] || \ std::die 2 "No such file or executable -- '${MODULECMD}'" @@ -1231,12 +1231,14 @@ pbuild.build_module() { eval $( "${MODULECMD}" bash use unstable ) eval $( "${MODULECMD}" bash use deprecated ) eval $( "${MODULECMD}" bash purge ) + # :FIXME: this is a hack!!! # shouldn't this be set in the build-script? eval $( "${MODULECMD}" bash use Libraries ) + load_build_dependencies + pbuild.init_env "${module_name}" "${module_version}" - source "${BUILD_SCRIPT}" pbuild::make_all } @@ -1262,7 +1264,6 @@ pbuild.bootstrap() { PATH+=":${PREFIX}/bin" PATH+=":${PREFIX}/sbin" - source "${BUILD_SCRIPT}" pbuild::make_all } From 695fe498b481909a3302dbca2e4bec591743ed3c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 9 Aug 2019 17:32:46 +0200 Subject: [PATCH 14/40] fix not called 'post_install_linux()' --- Pmodules/libpbuild.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 2bbbda1..d717faf 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -728,7 +728,7 @@ pbuild::make_all() { post_install_linux() { std::info \ "%s " "${module_name}/${module_version}:" \ - "running post-installation for ${system} ..." + "running post-installation for ${OS} ..." cd "${PREFIX}" # solve multilib problem with LIBRARY_PATH # on 64bit Linux @@ -737,7 +737,7 @@ pbuild::make_all() { } cd "${BUILD_DIR}" - [[ "${system}" == "Linux" ]] && post_install_linux + [[ "${OS}" == "Linux" ]] && post_install_linux install_doc install_pmodules_files write_runtime_dependencies From b2f2ebe0618f55d4f44a0e175693edf298f3f2a6 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 16 Aug 2019 15:27:03 +0200 Subject: [PATCH 15/40] abort build on error in a target --- Pmodules/libpbuild.bash | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index d717faf..03a0b09 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -635,6 +635,7 @@ pbuild::post_install() { pbuild::make_all() { local -a runtime_dependencies=() source "${BUILD_SCRIPT}" + local -r logfile="${BUILD_DIR}/pbuild.log" # # everything set up? @@ -856,7 +857,13 @@ pbuild::make_all() { # work because in some function global variables # might/need to be set. # - cd "${dir}" && "pbuild::$t" || std::die 42 "Aborting ..." + cd "${dir}" + if [[ "${verbose}" = 'yes' ]]; then + "pbuild::$t" 2>&1 | tee -a "${logfile}" + else + { "pbuild::$t" >> "${logfile}"; } 2>&1 | tee -a "${logfile}" + fi + (( ${PIPESTATUS[0]} == 0 )) || std::die 42 "Aborting ..." done touch "${BUILD_DIR}/.${target}" } @@ -864,12 +871,6 @@ pbuild::make_all() { #...................................................................... # build module ${module_name}/${module_version} build_module() { - local -r logfile="${BUILD_DIR}/pbuild.log" - if [[ "${verbose}" = 'yes' ]]; then - local -r output="/dev/fd/1" - else - local -r output="/dev/null" - fi std::info \ "%s " "${module_name}/${module_version}:" \ "start building ..." @@ -878,29 +879,33 @@ pbuild::make_all() { mkdir -p "${SRC_DIR}" mkdir -p "${BUILD_DIR}" + echo -n > "${logfile}" + std::info \ "%s " "${module_name}/${module_version}:" \ "preparing sources ..." - build_target "${SRC_DIR}" prep | tee "${logfile}" > ${output} + # write stdout and stderr to logfile, stderr to terminal + # write all to logfile and terminal + build_target "${SRC_DIR}" prep [[ "${build_target}" == "prep" ]] && return 0 std::info \ "%s " "${module_name}/${module_version}:" \ "configuring ..." - build_target "${BUILD_DIR}" configure | tee "${logfile}" >> ${output} + build_target "${BUILD_DIR}" configure [[ "${build_target}" == "configure" ]] && return 0 std::info \ "%s " "${module_name}/${module_version}:" \ "compiling ..." - build_target "${BUILD_DIR}" compile | tee "${logfile}" >> ${output} + build_target "${BUILD_DIR}" compile [[ "${build_target}" == "compile" ]] && return 0 std::info \ "%s " "${module_name}/${module_version}:" \ "installing ..." mkdir -p "${PREFIX}" - build_target "${BUILD_DIR}" install | tee "${logfile}" >> ${output} + build_target "${BUILD_DIR}" install post_install [[ "${build_target}" == "install" ]] && return 0 From c89c2c42385777163d8216a1a84aa402bf27367a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 20 Aug 2019 14:33:54 +0200 Subject: [PATCH 16/40] unsetting variables like C_INCLUDE_PATH before loading dependencies --- Pmodules/libpbuild.bash | 74 +++++++++++------------------------------ 1 file changed, 20 insertions(+), 54 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 03a0b09..b4ba7e4 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -21,6 +21,7 @@ unset __path #............................................................................. # disable auto-echo feature of 'cd' unset CDPATH +declare -A SOURCE_UNPACK_DIRS #............................................................................. # @@ -38,42 +39,8 @@ error_handler() { trap "error_handler" ERR -############################################################################### -# -# unset environment variables used for compiling -unset C_INCLUDE_PATH -unset CPLUS_INCLUDE_PATH -unset CPP_INCLUDE_PATH -unset LIBRARY_PATH -unset LD_LIBRARY_PATH -unset DYLD_LIBRARY_PATH - -unset CFLAGS -unset CPPFLAGS -unset CXXFLAGS -unset LIBS -unset LDFLAGS - -unset CC -unset CXX -unset FC -unset F77 -unset F90 - -declare SOURCE_URLS=() -declare SOURCE_SHA256_SUMS=() -declare SOURCE_NAMES=() -declare -A SOURCE_UNPACK_DIRS - -declare CONFIGURE_ARGS=() -declare SUPPORTED_SYSTEMS=() -declare PATCH_FILES=() -declare PATCH_STRIPS=() -declare PATCH_STRIP_DEFAULT='1' declare configure_with='undef' -declare bootstrap='no' - #.............................................................................. # global variables declare force_rebuild='' @@ -1037,29 +1004,9 @@ pbuild.init_env() { V="${module_version}" parse_version "${module_version}" - unset C_INCLUDE_PATH - unset CPLUS_INCLUDE_PATH - unset CPP_INCLUDE_PATH - unset LIBRARY_PATH - unset LD_LIBRARY_PATH - unset DYLD_LIBRARY_PATH - - unset CFLAGS - unset CPPFLAGS - unset CXXFLAGS - unset LIBS - unset LDFLAGS - - unset CC - unset CXX - unset FC - unset F77 - unset F90 - SOURCE_URLS=() SOURCE_SHA256_SUMS=() SOURCE_NAMES=() - CONFIGURE_ARGS=() SUPPORTED_SYSTEMS=() PATCH_FILES=() @@ -1241,6 +1188,25 @@ pbuild.build_module() { # shouldn't this be set in the build-script? eval $( "${MODULECMD}" bash use Libraries ) + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + unset CPP_INCLUDE_PATH + unset LIBRARY_PATH + unset LD_LIBRARY_PATH + unset DYLD_LIBRARY_PATH + + unset CFLAGS + unset CPPFLAGS + unset CXXFLAGS + unset LIBS + unset LDFLAGS + + unset CC + unset CXX + unset FC + unset F77 + unset F90 + load_build_dependencies pbuild.init_env "${module_name}" "${module_version}" From b706e1002b4066c33cbaddafaf32c544d8bf5803 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 20 Aug 2019 16:58:58 +0200 Subject: [PATCH 17/40] do not use functions in pipes --- Pmodules/libpbuild.bash | 98 +++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 32 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 03a0b09..4d13d8c 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -447,7 +447,8 @@ pbuild::prep() { local i=0 for ((_i = 0; _i < ${#PATCH_FILES[@]}; _i++)); do std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" @@ -563,7 +564,8 @@ pbuild::configure() { "cmake failed" else std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "${FUNCNAME[0]}: skipping..." fi } @@ -635,7 +637,21 @@ pbuild::post_install() { pbuild::make_all() { local -a runtime_dependencies=() source "${BUILD_SCRIPT}" - local -r logfile="${BUILD_DIR}/pbuild.log" + + set -e + local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + + # + # To be able to set environment variables in one of the 'pbuild::TARGET' + # function we cannot use PIPE's like + # pbuild::configure | tee -a ... + # + if [[ "${verbose}" == 'yes' ]]; then + exec > >(tee -a "${logfile}") + else + exec > >(cat >> "${logfile}") + fi + exec 2> >(tee -a "${logfile}" >&2) # # everything set up? @@ -669,7 +685,8 @@ pbuild::make_all() { local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "Installing documentation to ${docdir}" install -m 0755 -d \ "${docdir}" @@ -708,7 +725,8 @@ pbuild::make_all() { write_runtime_dependencies() { local -r fname="${PREFIX}/.dependencies" std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "writing run-time dependencies to ${fname} ..." local dep echo -n "" > "${fname}" @@ -728,7 +746,8 @@ pbuild::make_all() { # sometimes we need an system depended post-install post_install_linux() { std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "running post-installation for ${OS} ..." cd "${PREFIX}" # solve multilib problem with LIBRARY_PATH @@ -751,7 +770,8 @@ pbuild::make_all() { local -r src="${BUILDBLOCK_DIR}/modulefile" if [[ ! -r "${src}" ]]; then std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "skipping modulefile installation ..." return fi @@ -765,7 +785,8 @@ pbuild::make_all() { local -r dstdir=${dst%/*} std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "installing modulefile in '${dstdir}' ..." mkdir -p "${dstdir}" install -m 0444 "${src}" "${dst}" @@ -788,14 +809,16 @@ pbuild::make_all() { read release < "${release_file}" if [[ "${release}" != "${module_release}" ]]; then std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s %s\n" \ + "${module_name}/${module_version}:" \ "changing release from" \ "'${release}' to '${module_release}' ..." echo "${module_release}" > "${release_file}" fi else std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "setting release to '${module_release}' ..." echo "${module_release}" > "${release_file}" fi @@ -813,7 +836,8 @@ pbuild::make_all() { "BUILD_DIR is set to '/'" std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "Cleaning up '${BUILD_DIR}'..." rm -rf "${BUILD_DIR##*/}" }; @@ -830,7 +854,8 @@ pbuild::make_all() { "Oops: internal error:" \ "SRC_DIR is set to '/'" std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "Cleaning up '${SRC_DIR}'..." rm -rf "${SRC_DIR##*/}" }; @@ -858,12 +883,7 @@ pbuild::make_all() { # might/need to be set. # cd "${dir}" - if [[ "${verbose}" = 'yes' ]]; then - "pbuild::$t" 2>&1 | tee -a "${logfile}" - else - { "pbuild::$t" >> "${logfile}"; } 2>&1 | tee -a "${logfile}" - fi - (( ${PIPESTATUS[0]} == 0 )) || std::die 42 "Aborting ..." + "pbuild::$t" done touch "${BUILD_DIR}/.${target}" } @@ -871,18 +891,19 @@ pbuild::make_all() { #...................................................................... # build module ${module_name}/${module_version} build_module() { + echo C_INCLUDE_PATH=$C_INCLUDE_PATH std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "start building ..." [[ ${dry_run} == yes ]] && std::die 0 "" mkdir -p "${SRC_DIR}" mkdir -p "${BUILD_DIR}" - echo -n > "${logfile}" - std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "preparing sources ..." # write stdout and stderr to logfile, stderr to terminal # write all to logfile and terminal @@ -890,19 +911,22 @@ pbuild::make_all() { [[ "${build_target}" == "prep" ]] && return 0 std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "configuring ..." build_target "${BUILD_DIR}" configure [[ "${build_target}" == "configure" ]] && return 0 std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "compiling ..." build_target "${BUILD_DIR}" compile [[ "${build_target}" == "compile" ]] && return 0 std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "installing ..." mkdir -p "${PREFIX}" build_target "${BUILD_DIR}" install @@ -914,13 +938,17 @@ pbuild::make_all() { install_release_file cleanup_build cleanup_src - std::info "%s" "${module_name}/${module_version}: Done ..." + std::info \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ + "Done ..." return 0 } remove_module() { if [[ -d "${PREFIX}" ]]; then std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "removing all files in '${PREFIX}' ..." [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} fi @@ -936,14 +964,16 @@ pbuild::make_all() { if [[ -e "${dst}" ]]; then std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "removing modulefile '${dst}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${dst}" fi local release_file="${dstdir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi @@ -966,7 +996,8 @@ pbuild::make_all() { build_module else std::info \ - "%s " "${module_name}/${module_version}:" \ + "%s %s\n" \ + "${module_name}/${module_version}:" \ "already exists, not rebuilding ..." if [[ "${opt_update_modulefiles}" == "yes" ]]; then install_modulefile @@ -1069,7 +1100,6 @@ pbuild.init_env() { configure_with='undef' } - pbuild.build_module() { module_name="$1" module_version="$2" @@ -1077,6 +1107,9 @@ pbuild.build_module() { shift 3 with_modules=( "$@" ) + # used in pbuild::make_all + declare bootstrap='no' + #...................................................................... # # test whether a module is loaded or not @@ -1123,7 +1156,8 @@ pbuild.build_module() { "${m}: module does not exist," \ "cannot continue with dry run..." - std::info "$m: module does not exist, trying to build it..." + std::info "%s\n" \ + $m: module does not exist, trying to build it..." local args=( '' ) set -- ${ARGS[@]} while (( $# > 0 )); do From a2d9f6a665df0bd6d2121bbb3eb431dc3553ef81 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 21 Aug 2019 13:21:20 +0200 Subject: [PATCH 18/40] missing quote in libpbuild.bash fixed --- Pmodules/libpbuild.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 2281c46..0bf6446 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1,4 +1,3 @@ - #!/bin/bash #............................................................................. @@ -1104,7 +1103,7 @@ pbuild.build_module() { "cannot continue with dry run..." std::info "%s\n" \ - $m: module does not exist, trying to build it..." + "$m: module does not exist, trying to build it..." local args=( '' ) set -- ${ARGS[@]} while (( $# > 0 )); do From 1dc54d0264a5aef46ea01b0c3d6935406b852b83 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 22 Aug 2019 16:40:25 +0200 Subject: [PATCH 19/40] use System group in build environment --- Pmodules/libpbuild.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 0bf6446..da9c91c 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1220,6 +1220,7 @@ pbuild.build_module() { # :FIXME: this is a hack!!! # shouldn't this be set in the build-script? eval $( "${MODULECMD}" bash use Libraries ) + eval $( "${MODULECMD}" bash use System ) unset C_INCLUDE_PATH unset CPLUS_INCLUDE_PATH From de95904aadaef1f410cf73d8f69385ae39ba69d8 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 4 Sep 2019 14:41:31 +0200 Subject: [PATCH 20/40] bugfix in finding variants files with two dots --- Pmodules/modbuild.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 4fafb73..7ba6feb 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -273,11 +273,12 @@ parse_args() { (( ${#versions[@]} > 0)) || std::die 1 "Module version not specified!" } +shopt -s nocaseglob find_variants_files(){ shopt -q nullglob || : local -i nullglob_set=$? shopt -s nullglob - local files=( "${BUILDBLOCK_DIR}"/*/variants.${opt_system} ) + local files=( "${BUILDBLOCK_DIR}"/*/variants\.${opt_system} ) files+=( "${BUILDBLOCK_DIR}"/*/variants.$(uname -s) ) local f for f in "${BUILDBLOCK_DIR}"/*/variants; do From a32070a6584cb113f39341934b86dcddff824d9c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 4 Sep 2019 15:27:24 +0200 Subject: [PATCH 21/40] modbuild: slash removed from PREFIX --- Pmodules/libpbuild.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index da9c91c..892f4d1 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -212,10 +212,10 @@ set_full_module_name_and_prefix() { # set full module name fully_qualified_module_name=$( join_by '/' "${name[@]}" ) # set PREFIX of module - PREFIX="${PMODULES_ROOT}/${GROUP}/" + PREFIX="${PMODULES_ROOT}/${GROUP}" local -i i=0 for ((i=${#name[@]}-1; i >= 0; i--)); do - PREFIX+="${name[i]}/" + PREFIX+="/${name[i]}" done } From 22ee989ad2960143ceb2beb78d6ae13d82e18a99 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 4 Sep 2019 15:34:10 +0200 Subject: [PATCH 22/40] modbuild: rmdir on macOS doesn't support --ignore-fail-on-non-empty option fixed --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 892f4d1..8ed0ebc 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -943,7 +943,7 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p --ignore-fail-on-non-empty "${dstdir}" 2>/dev/null + rmdir -p "${dstdir}" 2>/dev/null || : } ######################################################################## From 593d00357db17fb7467cd4375acaec9ed1bc47cb Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 4 Sep 2019 15:49:52 +0200 Subject: [PATCH 23/40] modbuild: implementation of system and OS specific targets --- Pmodules/libpbuild.bash | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 8ed0ebc..fa86150 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -837,7 +837,13 @@ pbuild::make_all() { fi local targets=() targets+=( "pre_${target}_${system}" "pre_${target}_${OS}" "pre_${target}" ) - targets+=( "${target}" ) + if typeset -F pbuild::${target}_${system} 1>/dev/null 2>&1; then + targets+=( "${target}_${system}" ) + elif typeset -F pbuild::${target}_${OS} 1>/dev/null 2>&1; then + targets+=( "${target}_${OS}" ) + else + targets+=( "${target}" ) + fi targets+=( "post_${target}_${system}" "post_${target}_${OS}" "post_${target}" ) for t in "${targets[@]}"; do From 0214be689c6cf34dcf5effbaed4bc53584134ece Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 6 Sep 2019 17:56:20 +0200 Subject: [PATCH 24/40] modbuild: error writing dependencies fixed, debug output removed --- Pmodules/libpbuild.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index fa86150..6fcecb9 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -601,7 +601,6 @@ pbuild::post_install() { # The 'do it all' function. # pbuild::make_all() { - local -a runtime_dependencies=() source "${BUILD_SCRIPT}" set -e @@ -612,6 +611,7 @@ pbuild::make_all() { # function we cannot use PIPE's like # pbuild::configure | tee -a ... # + rm -f "${logfile}" if [[ "${verbose}" == 'yes' ]]; then exec > >(tee -a "${logfile}") else @@ -863,7 +863,6 @@ pbuild::make_all() { #...................................................................... # build module ${module_name}/${module_version} build_module() { - echo C_INCLUDE_PATH=$C_INCLUDE_PATH std::info \ "%s %s\n" \ "${module_name}/${module_version}:" \ @@ -1061,6 +1060,7 @@ pbuild.build_module() { # used in pbuild::make_all declare bootstrap='no' + declare -a runtime_dependencies=() #...................................................................... # From 46bfc351ddbc647c10d678a6ae203f38936b1a2a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 12 Sep 2019 17:44:42 +0200 Subject: [PATCH 25/40] modbuild: compute default number of parallel make jobs from number of cores --- Pmodules/libpbuild.bash | 47 ++++++++++++++++++++++++++++------------- Pmodules/modbuild.in | 2 +- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 6fcecb9..32d6a3d 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -41,7 +41,32 @@ trap "error_handler" ERR declare configure_with='undef' #.............................................................................. -# global variables +# +# The following variables are available in build-blocks and set read-only +# :FIXME: do we have to export them? + +# install prefix of module. +declare -x PREFIX='' + +declare -r OS=$(uname -s) + +pbuild::get_num_cores() { + case "${OS}" in + Linux ) + echo $(grep -c ^processor /proc/cpuinfo) + ;; + Darwin ) + echo $(sysctl -n hw.ncpu) + ;; + * ) + std::die 1 "OS ${OS} is not supported\n" + ;; + esac +} + +#.............................................................................. +# global variables which can be set/overwritten by command line args + declare force_rebuild='' pbuild.force_rebuild() { force_rebuild="$1" @@ -73,9 +98,13 @@ pbuild.update_modulefiles() { } # number of parallel make jobs -declare -i JOBS=3 +declare -i JOBS=$(pbuild::get_num_cores) pbuild.jobs() { - JOBS="$1" + if (( $1 == 0 )); then + JOBS=$(pbuild::get_num_cores) + else + JOBS="$1" + fi } declare system='' @@ -114,18 +143,6 @@ declare -x module_release='' # abs. path is "${PREFIX}/${_docdir}/${module_name}" declare -r _DOCDIR='share/doc' -#.............................................................................. -# -# The following variables are available in build-blocks and set read-only -# :FIXME: do we have to export them? -# - -# install prefix of module. -declare -x PREFIX='' - -declare -r OS=$(uname -s) - - ############################################################################## # # Set flag to build module in source tree. diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 7ba6feb..e43f41a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -140,7 +140,7 @@ declare opt_dry_run='no' declare opt_enable_cleanup_build='yes' declare opt_enable_cleanup_src='yes' declare opt_force_rebuild='no' -declare -i opt_jobs=3 +declare -i opt_jobs=0 declare opt_update_modulefiles='no' declare opt_system='' declare opt_temp_dir="${PMODULES_TMPDIR:-/var/tmp/${USER}}" From 7fe59f99e287ef54ea041eab8448b7bc4a242a36 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 17 Sep 2019 13:54:41 +0200 Subject: [PATCH 26/40] libpbuild.bash: check supported system, os and compiler - functions pbuild::supported_os() and pbuild::supported_compiler() added - bugfix in check of supported system --- Pmodules/libpbuild.bash | 54 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 32d6a3d..e0fe47e 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -159,12 +159,36 @@ pbuild::compile_in_sourcetree() { # Check whether the script is running on a supported OS. # # Arguments: -# $@: supported opertating systems (as printed by 'uname -s') +# $@: supported opertating systems (something like RHEL6, macOS10.14, ...). +# Default is all. # pbuild::supported_systems() { SUPPORTED_SYSTEMS+=( "$@" ) } +############################################################################## +# +# Check whether the script is running on a supported OS. +# +# Arguments: +# $@: supported opertating systems (like Linux, Darwin). +# Default is all. +# +pbuild::supported_os() { + SUPPORTED_OS+=( "$@" ) +} + +############################################################################## +# +# Check whether the loaded compiler is supported. +# +# Arguments: +# $@: supported compiler (like GCC, Intel, PGI). +# Default is all. +# +pbuild::supported_compiler() { + SUPPORTED_COMPILERS+=( "$@" ) +} #...................................................................... # # compute full module name and installation prefix @@ -653,13 +677,35 @@ pbuild::make_all() { check_supported_systems() { [[ -z "${SUPPORTED_SYSTEMS}" ]] && return 0 for sys in "${SUPPORTED_SYSTEMS[@]}"; do - [[ ${sys} == ${system} ]] && return 0 + [[ ${sys,,} == ${system,,} ]] && return 0 done std::die 1 \ "%s " "${module_name}/${module_version}:" \ "Not available for ${system}." } + #...................................................................... + check_supported_os() { + [[ -z "${SUPPORTED_OS}" ]] && return 0 + for os in "${SUPPORTED_OS[@]}"; do + [[ ${os,,} == ${OS,,} ]] && return 0 + done + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "Not available for ${OS}." + } + + #...................................................................... + check_supported_compilers() { + [[ -z "${SUPPORTED_COMPILERS}" ]] && return 0 + for compiler in "${SUPPORTED_COMPILERS[@]}"; do + [[ ${compiler,,} == ${COMPILER,,} ]] && return 0 + done + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "Not available for ${COMPILER}." + } + #...................................................................... # non-redefinable post-install post_install() { @@ -976,6 +1022,8 @@ pbuild::make_all() { # setup module specific environment if [[ "${bootstrap}" == 'no' ]]; then check_supported_systems + check_supported_os + check_supported_compilers set_full_module_name_and_prefix if [[ "${module_release}" == 'removed' ]]; then remove_module @@ -1061,6 +1109,8 @@ pbuild.init_env() { SOURCE_NAMES=() CONFIGURE_ARGS=() SUPPORTED_SYSTEMS=() + SUPPORTED_OS=() + SUPPORTED_COMPILERS=() PATCH_FILES=() PATCH_STRIPS=() PATCH_STRIP_DEFAULT='1' From e9e1268137911050c02d0010608383677e0312cc Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 19 Sep 2019 13:00:06 +0200 Subject: [PATCH 27/40] modbuild: typo in definition of pbuild::supported_compilers fixed --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index e0fe47e..df9ad06 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -186,7 +186,7 @@ pbuild::supported_os() { # $@: supported compiler (like GCC, Intel, PGI). # Default is all. # -pbuild::supported_compiler() { +pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } #...................................................................... From 1c82df4b102652898b4b558f04a4797f5cdce446 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 19 Sep 2019 13:23:12 +0200 Subject: [PATCH 28/40] modbuild: '.*' is now a valid version argument --- Pmodules/modbuild.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index e43f41a..6c83e69 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -248,7 +248,7 @@ parse_args() { -* ) std::die 1 "Invalid option -- '$1'" ;; - [=0-9]* ) + [=0-9]* | '.*' ) versions+=( "$1" ) ;; '') From 83dc1a3de6b8be51a572c25e24558d41ea7167ed Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 Sep 2019 10:10:38 +0200 Subject: [PATCH 29/40] libpbuild: limit parallel make jobs to 10, unless specified on cmd line --- Pmodules/libpbuild.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index df9ad06..12230ff 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -102,6 +102,7 @@ declare -i JOBS=$(pbuild::get_num_cores) pbuild.jobs() { if (( $1 == 0 )); then JOBS=$(pbuild::get_num_cores) + (( JOBS > 10 )) && JOBS=10 || : else JOBS="$1" fi From 611aae3b1bf8a6be224f633c2aa5a1ba6b8a419a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 Sep 2019 16:15:07 +0200 Subject: [PATCH 30/40] modulecmd: log loaded modules to syslog --- Pmodules/modulecmd.bash.in | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 2481169..03e9fe0 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -526,13 +526,15 @@ subcommand_load() { "${current_modulefile}" fi - if [[ ${verbosity_lvl} != silent ]] && \ - [[ ${release} != stable ]]; then - std::info "%s %s: %s -- %s\n" \ + local msg=$(printf "%s %s: %s -- %s\n" \ "${CMD}" 'load' \ "${release} module has been loaded" \ - "${m}" + "${m}") + if [[ ${verbosity_lvl} != silent ]] && \ + [[ ${release} != stable ]]; then + std::info "${msg}" fi + logger "${msg}" done # fix LOADEDMODULES LOADEDMODULES="${_LMFILES_}" From a58f142c935771350072eea15aa91fb3ae871bd0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 Sep 2019 16:27:00 +0200 Subject: [PATCH 31/40] CHANGES added --- CHANGES | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 CHANGES diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..c6dac62 --- /dev/null +++ b/CHANGES @@ -0,0 +1,20 @@ +Changes in version 1.0.0rc5 (since 1.0.0rc2) + - build-system + * lot of cleanup, refactoring and bugfixes + * '.*' is now a valid version argument + * 'system' isn't a synonym for OS/kernel any more. It defines a target operating system like RHEL6, macOS1014. + * checks for supported system, OS and compiler + * compute default for the number of parallel make jobs + * calling pbuild::make_all in a build-script is now deprecate + + - modulecmd + * --with option of sub-command search now accepts a comma separated list of strings + * better load hints + * log modules loaded to syslog + * PMODULES_ENV is exported only on content changes + * bugfixes and cleanup + +1.0.0rc4 + - never tagged +1.0.0rc3 + - never tagged From 3a59d8f630ba31d86ac83222e0c3710d93a69d4c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 27 Sep 2019 10:38:24 +0200 Subject: [PATCH 32/40] output of build process 'pbuild.log' added to git ignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c8133ac..1c5135b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ profile.zsh Downloads tmp tclIndex +pbuild.log *~ From 77275b9c660b5adb45c057b2546db5f1cd9a6f29 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 27 Sep 2019 10:41:00 +0200 Subject: [PATCH 33/40] modbuild: bootstrapping error fixed --- Pmodules/libpbuild.bash | 14 ++------------ Pmodules/modbuild.in | 29 +++++++++++++++++------------ 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 12230ff..d17926b 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -113,16 +113,6 @@ pbuild.system() { system="$1" } -declare TEMP_DIR='' -pbuild.temp_dir() { - TEMP_DIR="$1" -} - -declare PMODULES_DISTFILESDIR='' -pbuild.pmodules_distfilesdir() { - PMODULES_DISTFILESDIR="$1" -} - declare verbose='' pbuild.verbose() { verbose="$1" @@ -1097,8 +1087,8 @@ pbuild.init_env() { local -r module_name="$1" local -r module_version="$2" - SRC_DIR="${TEMP_DIR}/${module_name}-${module_version}/src" - BUILD_DIR="${TEMP_DIR}/${module_name}-${module_version}/build" + SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" + BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" # P and V can be used in the build-script, so we have to set them here P="${module_name}" diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 6c83e69..1e6831e 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -30,6 +30,10 @@ source libstd.bash || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } +# can be set in the configuration file +declare PMODULES_DISTFILESDIR='' +declare PMODULES_TMPDIR='' + source libpbuild.bash || \ std::die 3 "Oops: cannot source library -- '$_'" @@ -128,14 +132,15 @@ MISCELLANEOUS OPTIONS: # # parse options and arguments # +# command line arguments are taken first +# then configuration file +# last default -# multiple version can be passed via comand line +# versions to be build, '.*' or none means all declare -a versions=() -declare opt_all_variants='no' declare opt_bootstrap='no' declare opt_build_config='modbuild.conf' declare opt_build_target='all' -declare opt_distfiles_dir="${PMODULES_ROOT}/var/distfiles" declare opt_dry_run='no' declare opt_enable_cleanup_build='yes' declare opt_enable_cleanup_src='yes' @@ -143,7 +148,6 @@ declare opt_force_rebuild='no' declare -i opt_jobs=0 declare opt_update_modulefiles='no' declare opt_system='' -declare opt_temp_dir="${PMODULES_TMPDIR:-/var/tmp/${USER}}" declare opt_verbose='no' # array collecting all modules specified on the command line via '--with=module' declare -a opt_with_modules=() @@ -205,18 +209,18 @@ parse_args() { opt_enable_cleanup_src='no' ;; --distdir ) - opt_distfiles_dir="$2" + PMODULES_DISTFILESDIR="$2" shift ;; --distdir=* ) - opt_distfiles_dir="${1/--distdir=}" + PMODULES_DISTFILESDIR="${1/--distdir=}" ;; --tmpdir ) - opt_temp_dir="$2" + PMODULES_TMPDIR="$2" shift ;; --tmpdir=* ) - opt_temp_dir="${1/--tmpdir=}" + PMODULES_TMPDIR="${1/--tmpdir=}" ;; --system ) opt_system="$2" @@ -270,7 +274,7 @@ parse_args() { shift done [[ -n ${BUILD_SCRIPT} ]] || std::die 1 "No build-block specified!" - (( ${#versions[@]} > 0)) || std::die 1 "Module version not specified!" + (( ${#versions[@]} > 0)) || versions+=( '.*' ) } shopt -s nocaseglob @@ -401,9 +405,7 @@ pbuild.build_target "${opt_build_target}" pbuild.dry_run "${opt_dry_run}" pbuild.enable_cleanup_build "${opt_enable_cleanup_build}" pbuild.enable_cleanup_src "${opt_enable_cleanup_src}" -pbuild.pmodules_distfilesdir "${opt_distfiles_dir}" pbuild.update_modulefiles "${opt_update_modulefiles}" -pbuild.temp_dir "${opt_temp_dir}" pbuild.system "${opt_system}" pbuild.verbose "${opt_verbose}" @@ -413,7 +415,7 @@ source libpbuild_dyn.bash || \ # source build configuration, # must be done before sourcing libpbuild! if [[ "${opt_bootstrap}" == 'yes' ]]; then - test -d "${BUILDBLOCK_DIR}/../../${PMODULES_CONFIG_DIR}" && PATH+=":$_" + test -d "${BUILDBLOCK_DIR}/../../config" && PATH+=":$_" else # Please note: if we trap DEBUG a statement like # test -d ... && PATH+=$_ @@ -425,6 +427,9 @@ fi source "${opt_build_config}" || \ std::die 3 "Oops: Cannot source configuration file -- '$_'" +: ${PMODULES_DISTFILESDIR:=${PMODULES_ROOT}/var/distfiles} +: ${PMODULES_TMPDIR:=/var/tmp/${USER}} + declare -r BUILD_SCRIPT declare -r BUILDBLOCK_DIR From 6746ac30506bdf27ea9eddb4ee77eeb2f0c0d135 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 27 Sep 2019 10:43:41 +0200 Subject: [PATCH 34/40] version bumped to 1.0.0rc6 --- config/versions.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/versions.conf b/config/versions.conf index aef2bef..f6eedf4 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -3,5 +3,5 @@ coreutils 8.31 getopt 1.1.6 gettext 0.19.8 modules 3.2.10.1 -Pmodules 1.0.0rc5 +Pmodules 1.0.0rc6 Tcl 8.6.9 From cd13cfb33cae5f88115ea1206f0a68e7efadf277 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 27 Sep 2019 10:52:17 +0200 Subject: [PATCH 35/40] modulecmd: missing newline at end of warning fixed --- Pmodules/modulecmd.bash.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 03e9fe0..b462502 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -526,13 +526,13 @@ subcommand_load() { "${current_modulefile}" fi - local msg=$(printf "%s %s: %s -- %s\n" \ + local msg=$(printf "%s %s: %s -- %s" \ "${CMD}" 'load' \ "${release} module has been loaded" \ "${m}") if [[ ${verbosity_lvl} != silent ]] && \ [[ ${release} != stable ]]; then - std::info "${msg}" + std::info "%s\n" "${msg}" fi logger "${msg}" done From 965ab75ce7e1b048a01652022e4a0a659f0d5f43 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 6 Jan 2020 14:53:34 +0100 Subject: [PATCH 36/40] fix: support for shell 'sh' added --- Pmodules/modulecmd.bash.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index b462502..2bac147 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -55,7 +55,7 @@ print_help() { export_env() { case "${Shell}" in - bash | zsh ) + sh | bash | zsh ) local -r fmt="export %s=\"%s\"; " ;; csh | tcsh ) @@ -1960,8 +1960,8 @@ subcommand_initclear() { # main # case "$1" in - bash | zsh ) - declare Shell="$1" + sh | bash | zsh ) + declare Shell="sh" ;; csh | tcsh ) declare Shell='csh' From 4a7c3f7122de70da5b279d80362cf80d9862d419 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 6 Jan 2020 14:59:01 +0100 Subject: [PATCH 37/40] fix: prepend Pmodules bin directory to PATH --- Pmodules/csh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pmodules/csh b/Pmodules/csh index 00d781e..be69b42 100644 --- a/Pmodules/csh +++ b/Pmodules/csh @@ -46,3 +46,5 @@ end if (! $?LOADEDMODULES ) then setenv LOADEDMODULES "" endif + +setenv PATH "${PMODULES_HOME}/bin:${PATH}" From aefdd4a711aff16b719fd2fa2bbfb632fd7a259a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 6 Jan 2020 15:02:51 +0100 Subject: [PATCH 38/40] fix: broken module load with csh --- Pmodules/modulecmd.bash.in | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 2bac147..b91da3e 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -495,7 +495,6 @@ subcommand_load() { || std::die 1 "Oops: unable to create tmp file!\n" local output=$("${modulecmd}" 'bash' ${opts} 'load' \ "${current_modulefile}" 2> "${tmpfile}") - eval "${output}" # we do not want to print the error message we got from # modulecmd, they are a bit ugly @@ -516,15 +515,18 @@ subcommand_load() { "${error_txt}" \ "${m}" fi - if [[ "${Shell}" == "bash" ]]; then + if [[ "${Shell}" == "sh" ]]; then + # for sh-like shells just echo echo "${output}" - if [[ -n "${error}" ]]; then - echo "${error}" 1>&2 - fi else + # re-run with right shell "${modulecmd}" "${Shell}" ${opts} 'load' \ "${current_modulefile}" fi + eval "${output}" + if [[ -n "${error}" ]]; then + echo "${error}" 1>&2 + fi local msg=$(printf "%s %s: %s -- %s" \ "${CMD}" 'load' \ From 47ed9cda1bd659e757e293e87ce6a3677d0b73f0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jan 2020 17:21:07 +0100 Subject: [PATCH 39/40] bugfix: handle empty list of be installed shared libs --- Pmodules/libpbuild.bash | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index d17926b..cff6983 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -593,21 +593,21 @@ pbuild::install() { } pbuild::install_shared_libs() { - local -r binary="${PREFIX}/$1" - local -r pattern="${2//\//\\/}" # escape slash - local -r dstdir="${3:-${PREFIX}/lib}" + local -r binary="$1" + local -r dstdir="$2" + local -r pattern="${3//\//\\/}" # escape slash install_shared_libs_Linux() { local libs=( $(ldd "${binary}" | \ awk "/ => \// && /${pattern}/ {print \$3}") ) - cp -avL "${libs[@]}" "${dstdir}" + [[ -n "${libs}" ]] && cp -vL "${libs[@]}" "${dstdir}" } install_shared_libs_Darwin() { # https://stackoverflow.com/questions/33991581/install-name-tool-to-update-a-executable-to-search-for-dylib-in-mac-os-x local libs=( $(otool -L "${binary}" | \ awk "/${pattern}/ {print \$1}")) - cp -avL "${libs[@]}" "${dstdir}" + [[ -n "${libs}" ]] && cp -vL "${libs[@]}" "${dstdir}" } test -e "${binary}" || \ @@ -1268,7 +1268,7 @@ pbuild.build_module() { "since the dependency '$m' is ${release_of_dependency}" fi - echo "Loading module: ${m}" + std::info "Loading module: ${m}\n" module load "${m}" done } From fe1ab6d35cc8863d1c4c84660817bf6f88795fb5 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 15 Jan 2020 13:01:41 +0100 Subject: [PATCH 40/40] bugfix: broken sub-cmd unload fixed --- Pmodules/modulecmd.bash.in | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index b91da3e..5319f01 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -594,11 +594,14 @@ subcommand_unload() { for arg in "${args[@]}"; do local output=$("${modulecmd}" "${Shell}" 'unload' "${arg}") eval "${output}" - if [[ "${Shell}" == "bash" ]]; then + case ${Shell} in + sh | bash | zsh ) echo "${output}" - else + ;; + * ) "${modulecmd}" "${Shell}" 'unload' "${arg}" - fi + ;; + esac done }