From d371fc78ab7627843b683f5de5d1485cde2add84 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 23 May 2022 23:47:28 +0200 Subject: [PATCH 01/97] shopt errexit disabled --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 165d84b..6773026 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -37,7 +37,7 @@ declare -A SOURCE_UNPACK_DIRS # # $1 exit code # -set -o errexit +#set -o errexit error_handler() { local -i ec=$? From 454ae6ec154f2e4f93d18f811add119d7be59cc2 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 23 May 2022 23:48:14 +0200 Subject: [PATCH 02/97] modulecmd: bugfix in listing unused overlays --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index aa93c93..532553e 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1355,7 +1355,7 @@ subcommand_use() { print_ol_info(){ local only_used="$1" local ol='' - for ol in "${OverlayList[@]}"; do + for ol in "${Overlays[@]}"; do [[ ${OverlayDict[${ol}:used]} == ${only_used} ]] || continue local inst_root="${OverlayDict[${ol}:inst_root]}" local mod_root="${OverlayDict[${ol}:mod_root]}" From 2e131c1f2ae1342d31d5ce23bdad4b4a42af2fa1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 23 May 2022 23:49:21 +0200 Subject: [PATCH 03/97] definition of overlay types moved to libpmodule.bash --- Pmodules/libpmodules.bash.in | 8 +++++++- Pmodules/modulecmd.bash.in | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 8b6992b..8801f39 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -12,6 +12,11 @@ declare -A OverlayDict declare -a OverlayList declare -Ag Dir2OverlayMap +declare -r ol_normal='n' +declare -r ol_hiding='h' +declare -r ol_replacing='r' + + # initialize help text of 'module --version' Help['version']=" Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @@ -113,7 +118,8 @@ pm::read_config(){ ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${ol}" \ "${config_files[@]}" | \ sed 's/: /=/; s/\(.*\)/local \1/') - : ${type:=n} + : ${type:=${ol_normal}} + echo type=${type} case ${type} in ${ol_normal} | ${ol_replacing} | ${ol_hiding} ) : diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 532553e..f1564f1 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -44,10 +44,6 @@ else declare -r modulecmd="${libexecdir}/modulecmd.bin" fi -declare -r ol_normal='n' -declare -r ol_hiding='h' -declare -r ol_replacing='r' - declare verbosity_lvl=${PMODULES_VERBOSITY:-'verbose'} declare Shell='' From c4592a5c3238380215bc3e9fb03b30cd1a2690b7 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 23 May 2022 23:50:35 +0200 Subject: [PATCH 04/97] modbuild: support for YAML variant files and splitted overlays --- Pmodules/modbuild.in | 202 ++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 88 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 78a600e..5039ce1 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -27,10 +27,16 @@ PATH+=":${mydir}" PATH+=":${mydir}/../lib:${mydir}/../config" PATH+=":${PMODULES_DIR}/libexec" +path=$PATH + source libstd.bash || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } +std::def_cmds "${path}" \ + 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ + 'rm' 'sort' 'find' 'yq' + # can be set in the configuration file declare PMODULES_DISTFILESDIR='' declare PMODULES_TMPDIR='' @@ -47,6 +53,9 @@ declare -r ARGS="$@" shopt -s nocaseglob shopt -s extglob +declare ol_dir +declare ol_install_dir + ############################################################################## # usage() { @@ -163,6 +172,7 @@ declare -a opt_with_modules=() declare opt_ol_name_or_dir='' +echo "parse_args()" parse_args() { while (( $# > 0 )); do @@ -317,7 +327,7 @@ parse_args() { # "text gcc/10.3.0 openmpi/4.0.5" # "text gcc/10.3.0 openmpi/4.1.0" # -brace_expand_with_prefix(){ +bash_expand(){ local text="$1" shift local to_expand=( "${@}" ) @@ -328,14 +338,12 @@ brace_expand_with_prefix(){ eval list=( ${to_expand[0]} ) local s for s in ${list[*]}; do - brace_expand_with_prefix "${text} ${s}" "${to_expand[@]:1}" + bash_expand "${text} ${s}" "${to_expand[@]:1}" done; fi; } -brace_expand(){ - brace_expand_with_prefix "" "$@" -} +echo "build_modules_legacy()" build_modules_legacy() { find_variants_files(){ @@ -361,7 +369,7 @@ build_modules_legacy() { [[ -z ${toks} ]] && continue [[ ${toke:0:1} == '#' ]] && continue local -a deps=( ${toks[*]:2} ) - brace_expand_with_prefix "${toks[0]} ${toks[1]}" "${deps[@]}" + bash_expand "${toks[0]} ${toks[1]}" "${deps[@]}" done < "${input}" } @@ -427,107 +435,125 @@ build_modules_legacy() { done } -build_modules_yaml(){ - local variants_file="${BUILDBLOCK_DIR}/files/" - variants_file+="${BNAME_VARIANTS}.${opt_system}.yaml" +echo "build_modules_yaml()" - # - # $1 file name - # $2 module name - # $3 index - # - yaml_get_overlay(){ - local ol=$(yq e ".$2.variants[$3].overlay" "$1") - if [[ "${ol}" != 'null' ]]; then - echo "${ol}" - return - fi - ol=$(yq e ".$2.overlay" "$1") - if [[ "${ol}" != 'null' ]]; then - echo "${ol}" - return - fi - echo 'base' - } +build_modules_yaml(){ + local fnames=() + fnames+=( "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.${opt_system}.yaml" ) + fnames+=( "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.yaml" ) + fnames+=( '__zzzzz__' ) + for fname in "${fnames[@]}"; do + [[ -r "${fname}" ]] && break + done + [[ ${fname} == '__zzzzz__' ]] && \ + std::die 3 "New suitable YAML variants file found" + echo "Using ${fname}..." yaml_get_versions(){ - yq e ".$2.variants[].version" "$1" + local -n _result="$1" + local fname="$2" + local version="$3" + _result=( $(yq -Ne e \ + "with_entries(select(.key | test(\"^${version}\$\")))|keys" \ + "${fname}" 2>/dev/null | awk '{print $2}') ) } yaml_get_num_variants(){ - yq e ".$2.variants | length" "$1" - } - - yaml_get_version(){ - yq e ".$2.variants[$3].version" "$1" - } - - yaml_get_dependencies(){ - yq e ".$2.variants[$2].dependencies[]" "$1" + local -n _result="$1" + local fname="$2" + local version="$3" + _result=$(yq -Ne e ".\"${version}\"|length" \ + "${fname}" 2>/dev/null) + if (( $? != 0 )); then + _result=0 + fi } yaml_get_relstage(){ - local relstage=$(yq e ".$2.variants[$3].relstage" "$1") - if [[ "${relstage}" != 'null' ]]; then - echo "${relstage}" - return - fi - relstage=$(yq e ".$2.relstage" "$1") - if [[ "${relstage}" != 'null' ]]; then - echo "${relstage}" - return - fi - echo 'unstable' + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=$(yq -Ne e ".\"${version}\"[${idx}].relstage" \ + "${fname}" 2>/dev/null) + (( $? != 0 )) && relstage='unstable' || : } - local name="$1" + yaml_get_overlay(){ + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=$(yq -Ne e ".\"${version}\"[${idx}].overlay" \ + "${fname}" 2>/dev/null) + (( $? == 0 )) && return || : + _result=$(yq -Ne e ".overlay" "${fname}" 2>/dev/null) + (( $? == 0 )) && return || : + _result='base' + } + + yaml_get_dependencies(){ + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=( $(yq -Ne e ".\"${version}\"[${idx}].dependencies[]" \ + "${fname}" 2>/dev/null) ) + (( $? == 0 )) && return || : + _result=() + } + + local name="$1" local version="$2" shift 2 local with_modules=( $* ) - # pattern used in the for-loop to filter dependencies via awk + local m local pattern="//" for m in "${with_modules[@]}"; do - pattern+=" && /${m//\//\\/}( |$)/" + if [[ -n $(awk "/${m%/*}[\/ ]/" "${files[@]}") ]]; then + pattern+=" && /${m//\//\\/}/" + fi done - local -i n_variants=$(yaml_get_num_variants \ - "${variants_file}" "${name}") - local i - for (( i=0; i 0 )); then - with_modules=( $(brace_expand ${deps[@]} | awk "${pattern}") ) - (( ${#with_modules[@]} == 0 )) && continue - fi - local relstage=$(yaml_get_relstage \ - "${variants_file}" \ - "${name}" \ - "$i") - local ol=$(yaml_get_overlay \ - "${variants_file}" \ - "${name}" \ - "$i") - get_ol_info "${ol}" \ - ol_name \ - ol_type \ - ol_install_dir \ - ol_dir \ - || std::die 3 "${variants_file##*/}: unknown overlay -- ${ol_name_or_dir}" - pbuild.build_module \ - "${name}" "${version}" \ - "${relstage}" "${with_modules[@]}" + local -a versions + yaml_get_versions versions "${fname}" "${name}/${version}" + echo versions=${versions[@]} + echo ${#versions[@]} + for v in "${versions[@]}"; do + echo version=$v + local -i n_variants + yaml_get_num_variants n_variants "${fname}" "${v}" + local -i i + local -a deps=() + local relstage + local ol_name + for (( i=0; i 0 )); then + while read -a with_modules; do + pbuild.build_module \ + "${name}" "${v##*/}" \ + "${relstage}" "${with_modules[@]}" + done < <(bash_expand "" ${deps[@]}|awk "${pattern}") + else + pbuild.build_module \ + "${name}" "${v##*/}" \ + "${relstage}" + fi + done done } + +echo "build_modules_yaml()" build_modules() { - local variants_file="${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.${opt_system}" - if [[ -e "${variants_file}.yaml" ]]; then + local variants_file="${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}" + if [[ -n "${variants_file}*.yaml" ]]; then build_modules_yaml "$@" else build_modules_legacy "$@" @@ -555,11 +581,11 @@ pbuild.verbose "${opt_verbose}" # read configuration for modbuild # if [[ "${opt_bootstrap}" == 'yes' ]]; then - test -d "${BUILDBLOCK_DIR}/../../config" && PATH+=":$_" + pm::read_config "${BUILDBLOCK_DIR}/../../config/Pmodules.yaml" +else + pm::read_config fi -read_config_file "${pm_root}/${PMODULES_CONFIG_DIR}/${opt_build_config}" - # :FIXME: should go dist files to # ${pm_root}/var/distfiles # or From 7d9fe02b39cf5b31e664859c04737f5b2d3fa7cc Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 23 May 2022 23:51:36 +0200 Subject: [PATCH 05/97] YAML configurarion file added --- config/Pmodules.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 config/Pmodules.yaml diff --git a/config/Pmodules.yaml b/config/Pmodules.yaml new file mode 100644 index 0000000..43ca4f8 --- /dev/null +++ b/config/Pmodules.yaml @@ -0,0 +1,7 @@ +DefaultGroups: Tools:Programming +DefaultReleaseStages: stable +ReleaseStages: unstable:stable:deprecated + +Overlays: + base: + install_root: @INSTALL_ROOT@ From 25c302d91cd700c22b6d54ed57542af1a86e0ca4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 May 2022 10:10:20 +0200 Subject: [PATCH 06/97] modulecmd: PMODULES_ENV initialisation fixed PMODULES_ENV must be initialised before modulecmd.bin is called --- Pmodules/modulecmd.bash.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index f1564f1..688dbc9 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -131,12 +131,12 @@ save_env() { vars+=( OverlayDict Dir2OverlayMap) local s=$(typeset -p ${vars[@]}) - declare -g PMODULES_ENV=$( encode_base64 "$s" ) - export_env 'PMODULES_ENV' + declare -gx PMODULES_ENV=$( encode_base64 "$s" ) } _exit() { save_env "${g_env_must_be_saved}" + export_env 'PMODULES_ENV' if [[ -n "${tmpfile}" ]] && [[ -e "${tmpfile}" ]]; then ${rm} -f "${tmpfile}" || : fi @@ -1852,6 +1852,7 @@ pmodules_init() { init_overlay_vars init_modulepath init_manpath + save_env export_env \ LOADEDMODULES \ _LMFILES_ \ From bd0efa648ba4163eb7591eae3b1f0ff56cfe014b Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 May 2022 10:13:19 +0200 Subject: [PATCH 07/97] libstd.bash: use reference variable in std:prepend_path() not upvar --- Pmodules/libstd.bash | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index b39cced..cbe2121 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -91,25 +91,24 @@ std::append_path () { } std::prepend_path () { - local -r P="$1" + local -nr P="$1" shift 1 - local new_path="$1" + local path="$1" shift 1 local dir - if [[ -z ${!P} ]]; then + if [[ -z ${P} ]]; then for dir in "$@"; do - new_path+=":${dir}" + path+=":${dir}" done - export "$P=${new_path}" + P="${path}" else for dir in "$@"; do - [[ "${!P}" == @(|*:)${dir}@(|:*) ]] && continue - new_path+=":${dir}" + [[ "${P}" == @(|*:)${dir}@(|:*) ]] && continue + path+=":${dir}" done - new_path+=":${!P}" + P="${path}:${P}" fi - std::upvar "$P" "${new_path}" } std::remove_path() { From 944c1ab558ae57a788c2392d74e20d3d1505c02b Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 May 2022 11:17:12 +0200 Subject: [PATCH 08/97] libstd.bash: upvar replaced with reference variables --- Pmodules/libstd.bash | 73 +++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index cbe2121..3a8dae4 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -78,41 +78,42 @@ std::get_abspath() { } std::append_path () { - local -r P=$1 - local -r d=$2 + local -nr P="$1" + shift 1 + local dir + local dirs='' + for dir in "$@"; do + [[ "${P}" == @(|*:)${dir}@(|:*) ]] && continue + dirs+=":${dir}" + done - if ! echo ${!P} | egrep -q "(^|:)${d}($|:)" ; then - if [[ -z ${!P} ]]; then - export "$P=${d}" - else - export "$P=${!P}:${d}" - fi - fi + if [[ -z ${P} ]]; then + P="${dirs:1}" # remove leading ':' + else + P="${P}${dirs}" + fi } std::prepend_path () { local -nr P="$1" shift 1 - local path="$1" - shift 1 + local dir + local dirs='' + for dir in "$@"; do + [[ "${P}" == @(|*:)${dir}@(|:*) ]] && continue + dirs+="${dir}:" + done if [[ -z ${P} ]]; then - for dir in "$@"; do - path+=":${dir}" - done - P="${path}" + P="${dirs:0:-1}" # remove trailing ':' else - for dir in "$@"; do - [[ "${P}" == @(|*:)${dir}@(|:*) ]] && continue - path+=":${dir}" - done - P="${path}:${P}" + P="${dirs}${P}" fi } std::remove_path() { - local -r P=$1 + local -nr P="$1" shift 1 local -ar dirs="$@" local new_path='' @@ -124,8 +125,7 @@ std::remove_path() { [[ "${entry}" != "${dir}" ]] && new_path+=":${entry}" done done - # remove leading ':' - std::upvar "$P" "${new_path:1}" + P="${new_path:1}" # remove leading ':' } # @@ -175,22 +175,23 @@ std::replace_path () { # analog to std::split_abspath() with a relative path. # std::split_path() { - local parts="$1" - local -r path="$2" + local -n parts="$1" + local -r path="$2" IFS='/' local std__split_path_result=( ${std__split_path_tmp} ) unset IFS - std::upvar ${parts} "${std__split_path_result[@]}" + parts="${std__split_path_result[@]}" if (( $# >= 3 )); then # return number of parts - std::upvar "$3" ${#std__split_path_result[@]} + local -n num="$3" + num="${#std__split_path_result[@]}" fi } std::split_abspath() { - local parts="$1" - local -r path="$2" + local -n parts="$1" + local -r path="$2" if [[ "${path:0:1}" == '/' ]]; then local -r std__split_path_tmp="${path:1}" else @@ -200,16 +201,17 @@ std::split_abspath() { IFS='/' local std__split_path_result=( ${std__split_path_tmp} ) unset IFS - std::upvar ${parts} "${std__split_path_result[@]}" + parts="${std__split_path_result[@]}" if (( $# >= 3 )); then # return number of parts - std::upvar "$3" ${#std__split_path_result[@]} + local -n num="$3" + num="${#std__split_path_result[@]}" fi } std::split_relpath() { - local parts="$1" - local -r path="$2" + local -n parts="$1" + local -r path="$2" if [[ "${path:0:1}" == '/' ]]; then std::die 255 "Oops: Internal error in '${FUNCNAME[0]}' called by '${FUNCNAME[1]}' }" else @@ -219,10 +221,11 @@ std::split_relpath() { IFS='/' local std__split_path_result=( ${std__split_path_tmp} ) unset IFS - std::upvar ${parts} "${std__split_path_result[@]}" + parts="${std__split_path_result[@]}" if (( $# >= 3 )); then # return number of parts - std::upvar "$3" ${#std__split_path_result[@]} + local -n num="$3" + num="${#std__split_path_result[@]}" fi } From 1c98c1268424f8cac1b4d2edb032dde183797591 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 24 May 2022 11:52:35 +0200 Subject: [PATCH 09/97] some varibales renamed - OverlayList -> UsedOverlays - OverlayDict -> OverlayInfo --- Pmodules/libmodules.tcl | 22 ++++---- Pmodules/libpmodules.bash.in | 16 +++--- Pmodules/modbuild.in | 4 +- Pmodules/modulecmd.bash.in | 106 +++++++++++++++++------------------ 4 files changed, 74 insertions(+), 74 deletions(-) diff --git a/Pmodules/libmodules.tcl b/Pmodules/libmodules.tcl index b4ef6e2..7a82ba3 100644 --- a/Pmodules/libmodules.tcl +++ b/Pmodules/libmodules.tcl @@ -43,15 +43,15 @@ proc _pmodules_parse_pmodules_env { } { Dir2OverlayMap { array set ::Dir2OverlayMap [regsub -all {[]=[]} $value " "] } - OverlayDict { - array set ::OverlayDict [regsub -all {[]=[]} $value " "] + OverlayInfo { + array set ::OverlayInfo [regsub -all {[]=[]} $value " "] } - OverlayList { + UsedOverlays { array set tmp [regsub -all {[]=[]} $value " "] - set ::OverlayList {} + set ::UsedOverlays {} set l [lsort [array names tmp]] foreach k $l { - lappend ::OverlayList $tmp($k) + lappend ::UsedOverlays $tmp($k) } } UsedGroups { @@ -79,9 +79,9 @@ proc module-addgroup { group } { if { [module-info mode load] } { set overlays_to_add {} - foreach overlay $::OverlayList { + foreach overlay $::UsedOverlays { lappend overlays_to_add $overlay - set ol_type [lindex [split $::OverlayDict($overlay) ":"] 0] + set ol_type [lindex [split $::OverlayInfo($overlay) ":"] 0] if { [string compare $ol_type $::ol_replacing] == 0 } { break } @@ -122,7 +122,7 @@ proc module-addgroup { group } { debug "mode=remove: no orphan modules to unload" } debug "mode=remove: $env(MODULEPATH)" - foreach overlay $::OverlayList { + foreach overlay $::UsedOverlays { set dir [file join \ $overlay \ $group \ @@ -325,9 +325,9 @@ proc ModulesHelp { } { # proc _find_overlay { modulefile_components } { debug "_find_overlay()" - foreach ol $::OverlayList { + foreach ol $::UsedOverlays { debug "$ol" - set ol_dir $::OverlayDict(${ol}:mod_root) + set ol_dir $::OverlayInfo(${ol}:mod_root) if { [string range $ol_dir end end] == "/" } { set ol_dir [string range $ol_dir 0 end-1] } @@ -388,7 +388,7 @@ proc _pmodules_init_global_vars { } { debug "mod_root=$mod_root" set ol $::Dir2OverlayMap($mod_root) debug "ol=$ol" - set install_prefix [file split $::OverlayDict(${ol}:inst_root)] + set install_prefix [file split $::OverlayInfo(${ol}:inst_root)] set prefix "$install_prefix $group [lreverse_n $variant 2]" set PREFIX [file join {*}$prefix] debug "PREFIX=$PREFIX" diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 8801f39..4ec0d1d 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -8,9 +8,9 @@ declare -A Options=() declare -A Help=() declare -a Overlays=() -declare -A OverlayDict -declare -a OverlayList -declare -Ag Dir2OverlayMap +declare -A OverlayInfo +declare -a UsedOverlays +declare -A Dir2OverlayMap declare -r ol_normal='n' declare -r ol_hiding='h' @@ -61,7 +61,7 @@ compute_group_depth () { scan_groups () { local ol for ol in "$@"; do - local mod_root="${OverlayDict[${ol}:mod_root]}" + local mod_root="${OverlayInfo[${ol}:mod_root]}" local moduledir for moduledir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do compute_group_depth "${moduledir}" @@ -128,23 +128,23 @@ pm::read_config(){ std::die 3 "Invalid type for overlay '${ol}' -- ${type}" ;; esac - OverlayDict[${ol}:type]="${type}" + OverlayInfo[${ol}:type]="${type}" : ${modulefiles_root:=${PMODULES_HOME%%/Tools*}} [[ -d ${modulefiles_root} ]] || \ std::die 3 \ "Invalid modulefiles root directory for overlay '${ol}' -- ${modulefiles_root}" : ${modulefiles_root:=${PMODULES_HOME%%/Tools*}} - OverlayDict[${ol}:mod_root]="${modulefiles_root}" + OverlayInfo[${ol}:mod_root]="${modulefiles_root}" Dir2OverlayMap[${modulefiles_root}]="${ol}" : ${install_root:=${PMODULES_HOME%%/Tools*}} [[ -d ${install_root} ]] || \ std::die 3 \ "Invalid installation root directory for overlay '${ol}' -- ${install_root}" - OverlayDict[${ol}:inst_root]="${install_root:-${PMODULES_HOME%%/Tools*}}" + OverlayInfo[${ol}:inst_root]="${install_root:-${PMODULES_HOME%%/Tools*}}" - OverlayDict[${ol}:used]='no' + OverlayInfo[${ol}:used]='no' unset type modulefiles_root install_root done } diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 5039ce1..7318673 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -531,8 +531,8 @@ build_modules_yaml(){ yaml_get_relstage relstage "${fname}" "${v}" $i yaml_get_overlay ol_name "${fname}" "${v}" $i yaml_get_dependencies deps "${fname}" "${v}" $i - ol_install_dir="${OverlayDict[${ol_name}:inst_root]}" - ol_dir="${OverlayDict[${ol_name}:mod_root]}" + ol_install_dir="${OverlayInfo[${ol_name}:inst_root]}" + ol_dir="${OverlayInfo[${ol_name}:mod_root]}" if (( ${#deps[@]} > 0 )); then while read -a with_modules; do diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 688dbc9..65d17a7 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -127,8 +127,8 @@ save_env() { vars+=( ReleaseStages ) vars+=( GroupDepths ) vars+=( Overlays ) - vars+=( OverlayList ) - vars+=( OverlayDict Dir2OverlayMap) + vars+=( UsedOverlays ) + vars+=( OverlayInfo Dir2OverlayMap) local s=$(typeset -p ${vars[@]}) declare -gx PMODULES_ENV=$( encode_base64 "$s" ) @@ -158,8 +158,8 @@ map_moduledir2ol_name() { if [[ ! -v Dir2OverlayMap[${moduledir}] ]]; then local ol - for ol in "${OverlayList[@]}" 'other'; do - local mod_root="${OverlayDict[${ol}:mod_root]}" + for ol in "${UsedOverlays[@]}" 'other'; do + local mod_root="${OverlayInfo[${ol}:mod_root]}" [[ ${moduledir} == ${mod_root}/* ]] && break done Dir2OverlayMap[${moduledir}]="${ol}" @@ -170,8 +170,8 @@ map_moduledir2ol_name() { # # get release stage of module # Note: -# - the release stage of a module outside ${OverlayDict[@]} is always 'stable' -# - the release stage of a module inside ${OverlayDict[@]} without a +# - the release stage of a module outside ${OverlayInfo[@]} is always 'stable' +# - the release stage of a module inside ${OverlayInfo[@]} without a # coresponding file is always 'unstable' # # Args: @@ -231,7 +231,7 @@ find_overlay () { [[ "${ol}" == 'other' ]] && return 1 - local group="${path#${OverlayDict[${ol}:mod_root]}/}" + local group="${path#${OverlayInfo[${ol}:mod_root]}/}" group=${group%%/*} std::upvar $2 "${group}" return 0 @@ -497,8 +497,8 @@ subcommand_load() { local "$1" local -r group="$2/${PMODULES_MODULEFILES_DIR}" local ol - for ol in "${OverlayList[@]}"; do - local inst_root="${OverlayDict[${ol}:inst_root]}" + for ol in "${UsedOverlays[@]}"; do + local inst_root="${OverlayInfo[${ol}:inst_root]}" if [[ -d "${inst_root}/${group}" ]]; then std::upvar $1 "${ol}" return 0 @@ -511,7 +511,7 @@ subcommand_load() { [[ -n ${group} ]] && [[ -n ${GroupDepths[${group}]} ]] && return 0 local ol='' find_overlay_with_group ol "${group}" || return 1 - local moduledir="${OverlayDict[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" + local moduledir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" compute_group_depth "${moduledir}" || return 1 g_env_must_be_saved='yes' } @@ -612,8 +612,8 @@ subcommand_load() { MODULEPATH="" modulepath=() group+="${PMODULES_MODULEFILES_DIR}" - for overlay in "${OverlayList[@]}"; do - local mod_root="${OverlayDict[${overlay}:mod_root]}" + for overlay in "${UsedOverlays[@]}"; do + local mod_root="${OverlayInfo[${overlay}:mod_root]}" MODULEPATH="${mod_root}/${group}/:${MODULEPATH}" modulepath=( "${mod_root}/${group}/" "${modulepath[@]}" ) done @@ -949,7 +949,7 @@ get_available_modules() { local name="${mod%/*}" local key="${dir##/${PMODULES_MODULEFILES_DIR}}/${name}" if [[ -z "${modulenames[${key}]}" ]]; then - if [[ "${OverlayDict[${ol}:type]}" == "${ol_hiding}" ]]; then + if [[ "${OverlayInfo[${ol}:type]}" == "${ol_hiding}" ]]; then modulenames[${key}]="${ol}" else modulenames[${key}]='0' @@ -1352,9 +1352,9 @@ subcommand_use() { local only_used="$1" local ol='' for ol in "${Overlays[@]}"; do - [[ ${OverlayDict[${ol}:used]} == ${only_used} ]] || continue - local inst_root="${OverlayDict[${ol}:inst_root]}" - local mod_root="${OverlayDict[${ol}:mod_root]}" + [[ ${OverlayInfo[${ol}:used]} == ${only_used} ]] || continue + local inst_root="${OverlayInfo[${ol}:inst_root]}" + local mod_root="${OverlayInfo[${ol}:mod_root]}" local txt="\t${ol}" if [[ ${inst_root} == ${mod_root} ]]; then txt+="\n\t\t${inst_root}" @@ -1362,7 +1362,7 @@ subcommand_use() { txt+="\n\t\t${inst_root} (install root)" txt+="\n\t\t${mod_root} (modulefiles root)" fi - case "${OverlayDict[${ol}:type]}" in + case "${OverlayInfo[${ol}:type]}" in "${ol_hiding}" ) txt+='\n\t\t(hiding modules with same name)' ;; @@ -1441,7 +1441,7 @@ subcommand_use() { "modules are already loaded!" fi - if [[ ${OverlayDict[${ol_name}:used]} == 'yes' ]]; then + if [[ ${OverlayInfo[${ol_name}:used]} == 'yes' ]]; then std::die 3 "%s %s: %s -- %s" \ "${CMD}" "${subcommand}" \ "overlay already in use" \ @@ -1449,27 +1449,27 @@ subcommand_use() { return 0 fi - if [[ "${OverlayDict[${ol_name}:type]}" == "${ol_replacing}" ]]; then + if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then # if this overlay replaces groups, we have # to remove the modules made available by # other overlays in these groups for group in ${UsedGroups//:/ }; do # is this group in the to be added overlay? - local dir="${OverlayDict[${ol_name}:mod_root]}/" + local dir="${OverlayInfo[${ol_name}:mod_root]}/" dir+="${group}/${PMODULES_MODULEFILES_DIR}" [[ -d "${dir}" ]] || continue # no dir="/${group}/${PMODULES_MODULEFILES_DIR}" local -a dirs=() - for ol in "${OverlayList[@]}"; do - dirs+=( ${OverlayDict[${ol}:mod_root]}${dir} ) + for ol in "${UsedOverlays[@]}"; do + dirs+=( ${OverlayInfo[${ol}:mod_root]}${dir} ) done std::remove_path MODULEPATH "${dirs[@]}" done fi for group in ${UsedGroups//:/ }; do - local dir="${OverlayDict[${ol_name}:mod_root]}/" + local dir="${OverlayInfo[${ol_name}:mod_root]}/" dir+="${group}/${PMODULES_MODULEFILES_DIR}" if [[ -d "${dir}" ]]; then std::prepend_path MODULEPATH "${dir}" @@ -1477,11 +1477,11 @@ subcommand_use() { fi done - OverlayList=( "${ol_name}" "${OverlayList[@]}" ) - OverlayDict[${ol_name}:used]='yes' - export_env OverlayList + UsedOverlays=( "${ol_name}" "${UsedOverlays[@]}" ) + OverlayInfo[${ol_name}:used]='yes' + export_env UsedOverlays g_env_must_be_saved='yes' - scan_groups "${OverlayList[@]}" + scan_groups "${UsedOverlays[@]}" } #.............................................................. @@ -1495,13 +1495,13 @@ subcommand_use() { fi std::append_path UsedGroups "$1" local ol_name - for ol_name in "${OverlayList[@]}"; do - local dir="${OverlayDict[${ol_name}:mod_root]}/$1/${PMODULES_MODULEFILES_DIR}" + for ol_name in "${UsedOverlays[@]}"; do + local dir="${OverlayInfo[${ol_name}:mod_root]}/$1/${PMODULES_MODULEFILES_DIR}" [[ -d "${dir}" ]] || continue std::prepend_path MODULEPATH "${dir}" Dir2OverlayMap[${dir}]=${ol_name} - [[ "${OverlayDict[${ol_name}:type]}" == "${ol_replacing}" ]] && break + [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]] && break done } @@ -1518,7 +1518,7 @@ subcommand_use() { UsedFlags+=( "${arg/flag=}" ) return fi - if [[ -v OverlayDict[${arg}:type] ]]; then + if [[ -v OverlayInfo[${arg}:type] ]]; then use_overlay "${arg}" return 0 fi @@ -1529,7 +1529,7 @@ subcommand_use() { if [[ ! -v GroupDepths[${arg}] ]]; then # this scan is required if a new group has been # create inside an used overlay - scan_groups "${!OverlayDict[@]}" + scan_groups "${!OverlayInfo[@]}" g_env_must_be_saved='yes' fi if [[ -n ${GroupDepths[${arg}]} ]]; then @@ -1611,46 +1611,46 @@ subcommand_unuse() { "some modules are still loaded!" fi - [[ "${OverlayDict[ol_name]:mod_root}" == "${PMODULES_HOME%%/Tools*}" ]] && \ + [[ "${OverlayInfo[ol_name]:mod_root}" == "${PMODULES_HOME%%/Tools*}" ]] && \ std::die 3 "%s %s: %s -- %s" \ "${CMD}" "${subcommand}" \ "cannot remove base overlay" \ "${ol_name}" - [[ ${OverlayDict[${ol_name}:used]} != 'yes' ]] && \ + [[ ${OverlayInfo[${ol_name}:used]} != 'yes' ]] && \ std::die 3 "%s %s: %s -- %s" \ "${CMD}" "${subcommand}" \ "not an used overlay" \ "${ol_name}" # make sure first index is '0' (it should, but you never know) - OverlayList=( "${OverlayList[@]}" ) - [[ "${ol_name}" != "${OverlayList[0]}" ]] && \ + UsedOverlays=( "${UsedOverlays[@]}" ) + [[ "${ol_name}" != "${UsedOverlays[0]}" ]] && \ std::die 3 "%s %s: %s %s -- %s" \ "${CMD}" "${subcommand}" \ "overlay cannot be removed since" \ "it not on top of the stack" \ "${ol_name}" - if [[ "${OverlayDict[${ol_name}:type]}" == "${ol_replacing}" ]]; then + if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then # if this overlay hides groups, we have to re-add # the modules made available by other overlays - local mod_root=${OverlayDict[${ol_name}:mod_root]} + local mod_root=${OverlayInfo[${ol_name}:mod_root]} for group in ${UsedGroups//:/ }; do # is this group in the to be removed overlay? local dir="${mod_root}/${group}/${PMODULES_MODULEFILES_DIR}" [[ -d "${dir}" ]] || continue # no dir="/${group}/${PMODULES_MODULEFILES_DIR}" - std::prepend_path MODULEPATH "${OverlayList[@]/%/${dir}}" + std::prepend_path MODULEPATH "${UsedOverlays[@]/%/${dir}}" done fi - OverlayDict[${ol_name}:used]='no' - OverlayList=( "${OverlayList[@]:1}") + OverlayInfo[${ol_name}:used]='no' + UsedOverlays=( "${UsedOverlays[@]:1}") g_env_must_be_saved='yes' - export_env OverlayList + export_env UsedOverlays local dir for dir in "${modulepath[@]}"; do - [[ "${dir}" =~ "${OverlayDict[${ol_name}:mod_root]}" ]] && \ + [[ "${dir}" =~ "${OverlayInfo[${ol_name}:mod_root]}" ]] && \ std::remove_path MODULEPATH "${dir}" done } @@ -1674,7 +1674,7 @@ subcommand_unuse() { fi std::remove_path UsedGroups "${arg}" local overlay - for overlay in "${OverlayList[@]}"; do + for overlay in "${UsedOverlays[@]}"; do local dir="${overlay}/${arg}/${PMODULES_MODULEFILES_DIR}" std::remove_path MODULEPATH "${dir}" done @@ -1697,7 +1697,7 @@ subcommand_unuse() { done return fi - if [[ -v OverlayDict[${arg}:type] ]]; then + if [[ -v OverlayInfo[${arg}:type] ]]; then unuse_overlay "${arg}" return 0 fi @@ -1792,9 +1792,9 @@ init_modulepath() { declare -gx MODULEPATH='' local group local ol - for ol in "${OverlayList[@]}"; do + for ol in "${UsedOverlays[@]}"; do for group in ${UsedGroups//:/ }; do - local dir="${OverlayDict[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" + local dir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" if [[ -d "${dir}" ]]; then std::prepend_path MODULEPATH "${dir}" Dir2OverlayMap[${dir}]="${ol}" @@ -2402,7 +2402,7 @@ subcommand_search() { shift done if [[ -z "${src_prefix}" ]]; then - local -a src_prefix=( "${OverlayList[@]}" ) + local -a src_prefix=( "${UsedOverlays[@]}" ) fi if [[ "${opt_use_rel_stages}" == ":" ]]; then @@ -2738,13 +2738,13 @@ if [[ -z "${Subcommands[${subcommand}]}" ]]; then fi init_overlay_vars() { - declare -ag OverlayList=( 'base' ) - OverlayDict['base:used']='yes' + declare -ag UsedOverlays=( 'base' ) + OverlayInfo['base:used']='yes' #declare -Ag Dir2OverlayMap=() - for ol in "${OverlayList[@]}"; do + for ol in "${UsedOverlays[@]}"; do local group for group in ${UsedGroups//:/ }; do - local dir="${OverlayDict[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" + local dir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" if [[ -d "${dir}" ]]; then Dir2OverlayMap[${dir}]=${ol} fi @@ -2818,7 +2818,7 @@ fi #fi if (( ${#GroupDepths[@]} == 0 )); then - scan_groups "${OverlayList[@]}" + scan_groups "${UsedOverlays[@]}" g_env_must_be_saved='yes' fi From 1f7e9340e4397afe34d86a9232952d619121cd47 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 09:47:10 +0200 Subject: [PATCH 10/97] default values for overlays changed - install_root is now mandatory - if modulefiles_root is not specified it defaults to install_root --- Pmodules/libpmodules.bash.in | 38 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 4ec0d1d..4a78529 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -111,6 +111,13 @@ pm::read_config(){ [[ -v cfg_ReleaseStages ]] && ReleaseStages="${cfg_ReleaseStages}" unset ${!cfg_*} + # + # loop over all overlays and save config in OverlayInfo and Dir2OverlayMap + # + # - at least install_root must be specified + # - modulefiles_root default so install_root + # - the type defaults to ${ol_normal} + # Overlays=( $(_get_ol_names) ) local ol='' for ol in "${Overlays[@]}"; do @@ -118,8 +125,23 @@ pm::read_config(){ ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${ol}" \ "${config_files[@]}" | \ sed 's/: /=/; s/\(.*\)/local \1/') + + [[ -n ${install_root} ]] || \ + std::die 3 \ + "install_root missing for overlay -- ${ol}" + [[ -d ${install_root} ]] || \ + std::die 3 \ + "Invalid installation root directory for overlay '${ol}' -- ${install_root}" + OverlayInfo[${ol}:inst_root]="${install_root}" + + : ${modulefiles_root:=${install_root}} + [[ -d ${modulefiles_root} ]] || \ + std::die 3 \ + "Invalid modulefiles root directory for overlay '${ol}' -- ${modulefiles_root}" + OverlayInfo[${ol}:mod_root]="${modulefiles_root}" + Dir2OverlayMap[${modulefiles_root}]="${ol}" + : ${type:=${ol_normal}} - echo type=${type} case ${type} in ${ol_normal} | ${ol_replacing} | ${ol_hiding} ) : @@ -130,20 +152,6 @@ pm::read_config(){ esac OverlayInfo[${ol}:type]="${type}" - : ${modulefiles_root:=${PMODULES_HOME%%/Tools*}} - [[ -d ${modulefiles_root} ]] || \ - std::die 3 \ - "Invalid modulefiles root directory for overlay '${ol}' -- ${modulefiles_root}" - : ${modulefiles_root:=${PMODULES_HOME%%/Tools*}} - OverlayInfo[${ol}:mod_root]="${modulefiles_root}" - Dir2OverlayMap[${modulefiles_root}]="${ol}" - - : ${install_root:=${PMODULES_HOME%%/Tools*}} - [[ -d ${install_root} ]] || \ - std::die 3 \ - "Invalid installation root directory for overlay '${ol}' -- ${install_root}" - OverlayInfo[${ol}:inst_root]="${install_root:-${PMODULES_HOME%%/Tools*}}" - OverlayInfo[${ol}:used]='no' unset type modulefiles_root install_root done From 71671f9ff33d5c05741d0d800130fa244e512ecc Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 09:49:11 +0200 Subject: [PATCH 11/97] always add Pmodules libexec dir to search path to find and set binaries --- Pmodules/modulecmd.bash.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 65d17a7..55532a3 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -27,8 +27,7 @@ source "${libdir}/libpmodules.bash" declare -r os_name="$(uname -s)" -path="/bin:/usr/bin" -[[ ${os_name} == 'Darwin' ]] && path="${libexecdir}:${path}" +path="${libexecdir}:/bin:/usr/bin" std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ 'rm' 'sort' 'find' 'yq' From 184ceb73d5a2abf3aabd88b80b0746c092d0434c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 11:55:58 +0200 Subject: [PATCH 12/97] Changelog for version 1.1.8 added --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5759b8..25bcc61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog of Pmodules +## Version 1.1.8 +* **modulecmd** + * *User visible changes* + * configuration in YAML files + * modulefiles and software must not + have a common root directory + * the installation root must be specified, it doesn't default + to the base 'overlay' any more. + * zsh initialisation fixed. + * *Internal changes and fixes* + * std::upvar() replaced with reference variables in part of the + code. + * environment variable `PMODULES_ROOT` removed. + * unsetting aliases fixed. + * update to bash 5.1.16 + * update to findutils 4.9 (macOS only) + * minor fixes +* **build-system** + * *User visible changes* + * YAML format for variants files + * *Internal changes and fixes* + * use lib `libpmodules.bash` + * bugfixes +* **modmanage** + * *User visible changes* + * none, support for overlays still missing + * *Internal changes and fixes* + * none + ## Version 1.1.7 * **modulecmd** * list of available overlays in subcommand `use` is now better readable From 81486a0606fe881891d3c9dab20d6476efa1dc12 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 15:45:09 +0200 Subject: [PATCH 13/97] Pmodules version bumped to 1.1.9 --- config/versions.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/versions.conf b/config/versions.conf index e315957..96769f7 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -2,6 +2,6 @@ bash 5.1.16 findutils 4.9.0 getopt 1.1.6 modules 3.2.10.1 -Pmodules 1.1.8 +Pmodules 1.1.9 Tcl 8.6.12 tcllib 1.20 From cae2a10b4e08ff9083a09c53b072073336236be0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 15:46:05 +0200 Subject: [PATCH 14/97] modbuild: bugfix in build_modules_yaml() - quering dependencies fixed --- Pmodules/modbuild.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 7318673..0bbcea4 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -496,7 +496,7 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=( $(yq -Ne e ".\"${version}\"[${idx}].dependencies[]" \ + _result=( $(yq -Ne e ".\"${version}\"[${idx}].dependencies" \ "${fname}" 2>/dev/null) ) (( $? == 0 )) && return || : _result=() From 436cbcf1eefe4c3f5813801bd2dfcf68a6bde3ad Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 17:29:29 +0200 Subject: [PATCH 15/97] modbuild: type fixed in message "No YAML variants file found" --- Pmodules/modbuild.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 0bbcea4..2224939 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -446,7 +446,7 @@ build_modules_yaml(){ [[ -r "${fname}" ]] && break done [[ ${fname} == '__zzzzz__' ]] && \ - std::die 3 "New suitable YAML variants file found" + std::die 3 "No suitable YAML variants file found" echo "Using ${fname}..." yaml_get_versions(){ local -n _result="$1" From 3e1d8020123ec07ce74bd4d76915a85632d9e3dc Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 7 Jun 2022 17:30:43 +0200 Subject: [PATCH 16/97] modubuild: error exit if overlay does not exist If an overlay is requested in an variants file which does not exist, print a message and exit with exit code > 0 --- Pmodules/modbuild.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 2224939..901258c 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -528,8 +528,11 @@ build_modules_yaml(){ local relstage local ol_name for (( i=0; i Date: Tue, 7 Jun 2022 17:33:31 +0200 Subject: [PATCH 17/97] libpbuild.bash: UTF encode space replaced with ASCII space char --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 6773026..51c5ce2 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -601,7 +601,7 @@ pbuild::pre_install() { } pbuild::install() { - make install || \ + make install || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" From aa8fcd7a8afbdf0f7512b9d71d5f7d7c46216803 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 10:32:05 +0200 Subject: [PATCH 18/97] modbuild: is now defined as function analog to module() --- Pmodules/bash | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Pmodules/bash b/Pmodules/bash index 9f0963a..b9cbaf9 100644 --- a/Pmodules/bash +++ b/Pmodules/bash @@ -43,6 +43,14 @@ module() { } export -f module +modbuild(){ + ( + eval $("${PMODULES_HOME}/bin/modulecmd" bash load System:bash) + "${PMODULES_HOME}/bin/modbuild" "$@" + ) +} +export -f modbuild + # Local Variables: # mode: sh # sh-basic-offset: 8 From 5e01557aadd73cb533dca12759dc0a338cb8d51c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 10:42:43 +0200 Subject: [PATCH 19/97] modbuild: is now defined as function analog to module() --- Pmodules/zsh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Pmodules/zsh b/Pmodules/zsh index 0b5d282..b6b800b 100644 --- a/Pmodules/zsh +++ b/Pmodules/zsh @@ -34,6 +34,14 @@ module() { } export -f module +modbuild(){ + ( + eval $("${PMODULES_HOME}/bin/modulecmd" bash load System:bash) + "${PMODULES_HOME}/bin/modbuild" "$@" + ) +} +export -f modbuild + # Local Variables: # mode: sh # sh-basic-offset: 8 From 38cd9b1583e407c9569fdc48a36f5533ef0adaae Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 11:59:09 +0200 Subject: [PATCH 20/97] modulefile: shell init file must be sourced while loading --- Pmodules/modulefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Pmodules/modulefile b/Pmodules/modulefile index 40eba48..16df9dc 100644 --- a/Pmodules/modulefile +++ b/Pmodules/modulefile @@ -10,9 +10,8 @@ Pmodules are a hierarchical module environment based on Environment Modules. " # -# It might be that '${PMODULES_HOME}' is in PATH. -# Why? With older version the PATH might have been set without loading -# a module. +# Older versions add '$PREFIX/bin' to PATH without loading Pmodules as +# a module. Remove it ... # if { [module-info mode load] } { set PATH ":$::env(PATH):" @@ -22,3 +21,6 @@ if { [module-info mode load] } { } } } + +set shell [module-info shell] +puts "source \"$PREFIX/init/$shell\"" From 3c434c48f7beba21ba452f8ceebb947f96507b14 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 12:03:54 +0200 Subject: [PATCH 21/97] modulecmd: bugfix in module-addgroup() Type of overlay is now stored with key 'overlay_name:type'. The old method was still used. --- Pmodules/libmodules.tcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pmodules/libmodules.tcl b/Pmodules/libmodules.tcl index 7a82ba3..74f8752 100644 --- a/Pmodules/libmodules.tcl +++ b/Pmodules/libmodules.tcl @@ -81,7 +81,8 @@ proc module-addgroup { group } { set overlays_to_add {} foreach overlay $::UsedOverlays { lappend overlays_to_add $overlay - set ol_type [lindex [split $::OverlayInfo($overlay) ":"] 0] + set ol_type $::OverlayInfo($overlay:type) + debug "ol_type=$ol_type" if { [string compare $ol_type $::ol_replacing] == 0 } { break } From cede8559a64c93ad2e300039c779bbf52c3eb1f0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:06:02 +0200 Subject: [PATCH 22/97] modbuild: abort build on error errors in the build-steps have been ignored --- Pmodules/libpbuild.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 51c5ce2..c43b0c1 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1063,7 +1063,10 @@ pbuild::make_all() { # might/need to be set. # cd "${dir}" - typeset -F "$t" 2>/dev/null && "$t" || : + if typeset -F "$t" 2>/dev/null; then + "$t" || \ + std::die 10 "Aborting..." + fi done touch "${BUILD_DIR}/.${target}" } From a8e5f8bf22bd556cb29cede76796040d1f9bf4f4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:15:21 +0200 Subject: [PATCH 23/97] modbuild: fixed bug in setting PATH obsolete variable PMODULES_DIR was still used --- Pmodules/modbuild.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 901258c..e456735 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -25,7 +25,7 @@ fi PATH+='/usr/bin:/bin:/usr/sbin:/sbin' PATH+=":${mydir}" PATH+=":${mydir}/../lib:${mydir}/../config" -PATH+=":${PMODULES_DIR}/libexec" +PATH+=":${mydir}/../libexec" path=$PATH From e52a03ff143a70eca29b5c8a1dfbe84bae6b24d1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:19:10 +0200 Subject: [PATCH 24/97] modbuild: cleanup code for debugging --- Pmodules/modbuild.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index e456735..cef62a7 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -7,6 +7,7 @@ # #............................................................................. +#set -x declare VERSION='@PMODULES_VERSION@' # get absolute path of script @@ -343,8 +344,6 @@ bash_expand(){ fi; } -echo "build_modules_legacy()" - build_modules_legacy() { find_variants_files(){ shopt -q nullglob || : @@ -435,8 +434,6 @@ build_modules_legacy() { done } -echo "build_modules_yaml()" - build_modules_yaml(){ local fnames=() fnames+=( "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.${opt_system}.yaml" ) From 5337160b7a467325c99700b97ee35cc630c33e1d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:21:15 +0200 Subject: [PATCH 25/97] modbuild: test for bash 5 or later, otherwise abort --- Pmodules/modbuild.in | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index cef62a7..9b85ddd 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -34,6 +34,14 @@ source libstd.bash || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } +if (( ${BASH_VERSINFO[0]} < 5 )); then + std::info "bash >= 5 is required! You are running bash ${BASH_VERSION} ..." + std::info "Make sure that bash >= 5 is in your PATH." + std::info "bash >= 5 is available as Pmodule:" + std::info " module load System:bash" + std::die 3 "" +fi + std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ 'rm' 'sort' 'find' 'yq' From 80a050e7fad7c0674dd0f9fe59a895e97cebaedd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:24:16 +0200 Subject: [PATCH 26/97] modbuild: can now be called without specifying a build-script if no build-script has been specified, a file with the name 'build' must exist in the current directory. --- Pmodules/modbuild.in | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 9b85ddd..1c0cf1a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -320,7 +320,14 @@ parse_args() { esac shift done - [[ -n ${BUILD_SCRIPT} ]] || std::die 1 "No build-block specified!" + if [[ -z ${BUILD_SCRIPT} ]]; then + if [[ -r "${PWD}/build" ]]; then + BUILD_SCRIPT="${PWD}/build" + BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}") + else + std::die 1 "No build-block specified!" + fi + fi (( ${#versions[@]} > 0)) || versions+=( '.*' ) } From 07fcd427d514dcd8c01a2323cebc14e8e6bd4d88 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:30:55 +0200 Subject: [PATCH 27/97] modbuild: option added to set/override overlay --- Pmodules/modbuild.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 1c0cf1a..b042cdb 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -175,6 +175,7 @@ declare opt_force_rebuild='no' declare -i opt_jobs=0 declare opt_update_modulefiles='no' declare opt_system='' +declare opt_overlay='' declare opt_verbose='no' # array collecting all modules specified on the command line via '--with=module' declare -a opt_with_modules=() @@ -540,7 +541,11 @@ build_modules_yaml(){ local relstage local ol_name for (( i=0; i Date: Thu, 9 Jun 2022 14:36:47 +0200 Subject: [PATCH 28/97] modulecmd: bugfix if MODULEPATH starts or ends with a colon --- Pmodules/modulecmd.bash.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 55532a3..5eb3cb1 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -715,6 +715,9 @@ subcommand_load() { LOADEDMODULES="${_LMFILES_}" local dir while read dir; do + # if the first or last character of MODULEPATH is ':', + # we read an empty string. + [[ -z ${dir} ]] && continue [[ "${dir: -1}" == "/" ]] || dir+="/" LOADEDMODULES="${LOADEDMODULES//${dir}}" map_moduledir2ol_name overlay "${dir}" From 047e7161b52fb197c1e8dbcc718bfa343cc2ddd8 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:41:22 +0200 Subject: [PATCH 29/97] modulecmd: initialise Dir2OverlayMap for all available groups till now Dir2OverlayMap was only initialise for used groups. This is not sufficient, if a module outside the used groups is loaded via module load group:name. --- Pmodules/libpmodules.bash.in | 7 ++++--- Pmodules/modulecmd.bash.in | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 4a78529..f3aa5b5 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -62,9 +62,10 @@ scan_groups () { local ol for ol in "$@"; do local mod_root="${OverlayInfo[${ol}:mod_root]}" - local moduledir - for moduledir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do - compute_group_depth "${moduledir}" + local dir + for dir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do + compute_group_depth "${dir}" + Dir2OverlayMap[${dir}]="${ol}" done done } diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 5eb3cb1..43a9cbe 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -608,13 +608,13 @@ subcommand_load() { "${CMD}" "${subcommand}" \ "illegal group name" \ "${group}" - MODULEPATH="" + #MODULEPATH="" modulepath=() - group+="${PMODULES_MODULEFILES_DIR}" + group+="/${PMODULES_MODULEFILES_DIR}" for overlay in "${UsedOverlays[@]}"; do local mod_root="${OverlayInfo[${overlay}:mod_root]}" - MODULEPATH="${mod_root}/${group}/:${MODULEPATH}" - modulepath=( "${mod_root}/${group}/" "${modulepath[@]}" ) + #MODULEPATH="${mod_root}/${group}:${MODULEPATH}" + modulepath=( "${mod_root}/${group}" "${modulepath[@]}" ) done fi if [[ -n ${rel_stage} ]]; then From 36453ff20a36627478eabefef6d4f2f304847869 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:45:54 +0200 Subject: [PATCH 30/97] modulecmd: some comments added --- Pmodules/modulecmd.bash.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 43a9cbe..bc2a2b5 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -907,6 +907,12 @@ subcommand_show() { # Find all modules in a given modulepath matching a specific string. # The search can be restricted to certain release stages. # +# Args: +# $1 upvar for results +# $2 search pattern +# $3 release stages +# $4... module directories +# # return list like # modulename_1 rel_stage_1 modulefile_1 ... # @@ -1042,12 +1048,18 @@ find_module() { # no version has been specified. This makes it more # difficult. We have to load the newest version taking # the used release stages and flags into account. + + # get list of reverse sorted version numbers (( col += ${#module} + 1 )) modules=( $(${find} -L "${dir}" -type f -not -name ".*" \ -ipath "${dir}/${module}/*" \ | cut -b${col}- \ | sort -rV ) ) + # prepend module name modules=( "${modules[@]/#/${module}/}" ) + + # now modules contains a reverse sorted list of + # available modules in the form name/version for mod in "${modules[@]}"; do # # loop over all used flags. If a module with From 1c39e8ded0627b5b27bef49466548efb36de8d5d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:46:27 +0200 Subject: [PATCH 31/97] modulecmd: bugfix in initialising src_prefix in sub-cmd search --- Pmodules/modulecmd.bash.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index bc2a2b5..20359ff 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -2416,7 +2416,10 @@ subcommand_search() { shift done if [[ -z "${src_prefix}" ]]; then - local -a src_prefix=( "${UsedOverlays[@]}" ) + local ol='' + for ol in "${UsedOverlays[@]}"; do + src_prefix+=( "${OverlayInfo[${ol}:mod_root]}" ) + done fi if [[ "${opt_use_rel_stages}" == ":" ]]; then From d5ab3dd2095c19ac388fca1b71f4f98a70828d0e Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:48:39 +0200 Subject: [PATCH 32/97] modulecmd: add overlay info to output of sub-command 'search' --- Pmodules/modulecmd.bash.in | 46 ++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 20359ff..5492772 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -945,6 +945,7 @@ get_available_modules() { # implementations. local entries=$(echo *) [[ -n ${entries} ]] || continue + local mod='' # module_name/module_version while read mod; do local add='no' if [[ -n "${ol}" ]]; then @@ -980,7 +981,7 @@ get_available_modules() { "${mod}" [[ :${used_rel_stages}: =~ :${rel_stage}: ]] || continue - mods+=( "${mod}" ${rel_stage} "${dir}/${mod}" ) + mods+=( "${mod}" ${rel_stage} "${dir}/${mod}" "${ol}" ) dict[${mod}]=1 done < <(${find} -L ${entries} \ \( -type f -o -type l \) \ @@ -1073,7 +1074,7 @@ find_module() { "${dir}" \ "${mod}" std::upvar $2 "${rel_stage}" - [[ :${release}: =~ :${UsedReleaseStages}: ]] && \ + [[ :${rel_stage}: =~ :${UsedReleaseStages}: ]] && \ return 0 done done @@ -1144,7 +1145,7 @@ subcommand_avail() { terse_output() { output_header "$1" local -i i=0 - for (( i=0; i<${#mods[@]}; i+=3 )); do + for (( i=0; i<${#mods[@]}; i+=4 )); do local mod=${mods[i]} local rel_stage=${mods[i+1]} case ${rel_stage} in @@ -1162,7 +1163,7 @@ subcommand_avail() { #...................................................................... machine_output() { - for (( i=0; i<${#mods[@]}; i+=3 )); do + for (( i=0; i<${#mods[@]}; i+=4 )); do printf "%-20s\t%s\n" "${mods[i]}" "${mods[i+1]}" 1>&2 done } @@ -1171,7 +1172,7 @@ subcommand_avail() { # :FIXME: for the time being, this is the same as terse_output! long_output() { output_header "$1" - for (( i=0; i<${#mods[@]}; i+=3 )); do + for (( i=0; i<${#mods[@]}; i+=4 )); do local mod=${mods[i]} local rel_stage=${mods[i+1]} case ${rel_stage} in @@ -1194,7 +1195,7 @@ subcommand_avail() { local -a available_modules=() local mod='' local -i max_length=1 - for ((i=0; i<${#mods[@]}; i+=3)); do + for ((i=0; i<${#mods[@]}; i+=4)); do if [[ ${verbosity_lvl} == 'verbose' ]]; then local rel_stage=${mods[i+1]} case ${rel_stage} in @@ -2141,7 +2142,7 @@ subcommand_search() { local fmt='' print_default() { - fmt="%-${max_len_modulename}s %-10s %-12s %-s" + fmt="%-${max_len_modulename}s %-10s %-12s %-12s %-s" if [[ ${opt_print_header} == 'yes' ]]; then func_print_header='print_header_default' else @@ -2152,7 +2153,7 @@ subcommand_search() { print_header_default() { std::info '' - std::info "${fmt}" "Module" "Rel.stage" "Group" "Requires" + std::info "${fmt}" "Module" "Rel.stage" "Group" "Overlay" "Requires" std::info '-%.0s' $(seq 1 ${cols}) } @@ -2165,12 +2166,12 @@ subcommand_search() { std::info "${str}" } if [[ "${opt_wrap}" == 'no' ]]; then - local deps="${@:5}" - local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[@]}") + local deps="${@:6}" + local str=$(printf "${fmt}" "$1" "$2" "$3" "$5" "${deps[@]}") write_line "${str}" else - local deps=( "${@:5}" ) - local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[0]}") + local deps=( "${@:6}" ) + local str=$(printf "${fmt}" "$1" "$2" "$3" "$5" "${deps[0]}") for (( i = 1; i < ${#deps[@]}; i++ )); do if (( ${#str} + ${#deps[i]} + 1 <= cols )); then str+=" ${deps[i]}" @@ -2184,21 +2185,26 @@ subcommand_search() { } print_verbose() { - fmt="%-${max_len_modulename}s %-10s %-12s %-s" + fmt="%-${max_len_modulename}s %-12s %-14s %-s" func_print_header='print_header_verbose' func_print_line='print_line_verbose' } print_header_verbose() { std::info '' - std::info "${fmt}" "Module" "Rel.stage" "Group" "Dependencies/Modulefile" - std::info '-%.0s' $(seq 1 ${cols}) + #std::info "${fmt}" "Module" "Rel.stage" "Group" "Overlay" "Dependencies/Modulefile" + #std::info '-%.0s' $(seq 1 ${cols}) } print_line_verbose() { - local deps="${@:5}" - std::info "${fmt}" "$1" "$2" "$3" "dependencies: ${deps}" - std::info "${fmt}" "" "" "" "modulefile: $4" + local deps="${@:6}" + [[ -z ${deps} ]] && deps="(none)" + std::info "$1:" + std::info " release stage: $2" + std::info " group: $3" + std::info " overlay: $5" + std::info " modulefile: $4" + std::info " dependencies: ${deps}" } # print full modulefile names only @@ -2282,10 +2288,11 @@ subcommand_search() { "${opt_use_rel_stages}" \ "${modulepath[@]}" \ - for (( i=0; i<${#mods[@]}; i+=3 )); do + for (( i=0; i<${#mods[@]}; i+=4 )); do local name=${mods[i]} local rel_stage=${mods[i+1]} local modulefile=${mods[i+2]} + local ol=${mods[i+3]} if (( ${#name} > max_len_modulename)); then max_len_modulename=${#name} @@ -2313,6 +2320,7 @@ subcommand_search() { fi echo ${name} ${rel_stage} ${group} ${modulefile} \ + ${ol} \ ${deps[@]} >> "${tmpfile}" done done From 4ac251341c35c2e6ea732f82fa14f19f5ecbb888 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 9 Jun 2022 14:49:33 +0200 Subject: [PATCH 33/97] CHANGELOG updated --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25bcc61..b52fdb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog of Pmodules +## Version 1.1.9 +* **modulecmd** + * *User visible changes* + * ... + * *Internal changes and fixes* + * the shell's init file is sourced, when Pmodules is loaded as module + * bug fixed in libmodules.tcl:module-addgroup() which crashed + "module load". + * don't crash if MODULEPATH starts or ends with a colon + * add overlay info to output of sub-command 'search' +* **build-system** + * *User visible changes* + * modbuild is now defined as shell function analog to module() + * modbuild can be called without specifying a build-script, + if a build-script exist in the current working directory + * abort build on error + * with option `--overlay` + - an overlay can be set if legacy variants files are used + - the overlay in the YAML variants file can overriden + * *Internal changes and fixes* + * bugfix in setting PATH + * requires bash 5 or later + ## Version 1.1.8 * **modulecmd** * *User visible changes* From 9991dda16c0ae29656e69abab3d9a1189b933f66 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 10:04:20 +0200 Subject: [PATCH 34/97] modulecmd: bugfix in module-addgroup the modulefiles root is now in the array OverlayInfo --- Pmodules/libmodules.tcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pmodules/libmodules.tcl b/Pmodules/libmodules.tcl index 74f8752..074550e 100644 --- a/Pmodules/libmodules.tcl +++ b/Pmodules/libmodules.tcl @@ -92,10 +92,11 @@ proc module-addgroup { group } { debug "group=$group" debug "::variant=$::variant" set dir [file join \ - $overlay \ + $::OverlayInfo($overlay:mod_root) \ $group \ $::MODULEFILES_DIR \ {*}$::variant] + debug "dir=$dir" if { [file isdirectory $dir] } { debug "prepend $dir to MODULEPATH " prepend-path MODULEPATH $dir From 67df5ba324adf0112b3543b273b0b41532864cac Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 10:06:28 +0200 Subject: [PATCH 35/97] modbuild: don't exit on error there are some issues if exit on error (set -o errexit) is set: A statement like var=$(cmd) causes the program to terminate, if 'cmd' exit with an exit code > 0. Using var=$(cmd || :) cannot be used in all cases because $? will be 0 even if 'cmd' exits with an error. --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index c43b0c1..ad62e80 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -656,7 +656,7 @@ pbuild::post_install() { pbuild::make_all() { source "${BUILD_SCRIPT}" - set -e + #set -o errexit local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" # module name including path in hierarchy and version # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) From 27018d9be03fb389058e6e608d6ef05738fe13ae Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 10:14:36 +0200 Subject: [PATCH 36/97] libstd.bash: redirect output of 'which lsb_release' to /dev/null lsb_release might not be installed! --- Pmodules/libstd.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 3a8dae4..95e70c7 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -289,11 +289,11 @@ std::upvar() { } std.get_os_release_linux() { - local lsb_release=$(which lsb_release) + #local lsb_release=$(which lsb_release) local ID='' local VERSION_ID='' - if [[ -n $(which lsb_release) ]]; then + if [[ -n $(which lsb_release 2>/dev/null) ]]; then ID=$(lsb_release -is) VERSION_ID=$(lsb_release -rs) elif [[ -r '/etc/os-release' ]]; then From ae56b59ef5afb5908446f7e031795a19ecd06aaf Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 10:23:58 +0200 Subject: [PATCH 37/97] modbuild: new key 'with' to specify hierarchical dependencies --- Pmodules/modbuild.in | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b042cdb..995b817 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -509,10 +509,21 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=( $(yq -Ne e ".\"${version}\"[${idx}].dependencies" \ + _result=( $(yq -Ne e ".\"${version}\"[${idx}]|(.with, .dependencies)" \ "${fname}" 2>/dev/null) ) - (( $? == 0 )) && return || : - _result=() + if (( $? != 0 )); then + # neither .with nor .dependencies are set + _result=() + return + fi + # if one of .with, .dependencies is not set, the vaulue is + # returned as 'null'. + local -i i + for ((i=0; i<${#_result[@]}; i++)); do + if [[ ${_result[$i]} == 'null' ]]; then + unset _result[$i] + fi + done } local name="$1" From 8c247f673a9dd7573204ca945ebcd38ce839d6b3 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 11:06:37 +0200 Subject: [PATCH 38/97] code cleanup there is one important change: The keys in Dir2OverlayMap are now the modulefiles root + group name. Example: New '/opt/psi/Tools', old '/opt/psi/Tools/modulefiles' --- Pmodules/libpmodules.bash.in | 4 +- Pmodules/modbuild.in | 6 +- Pmodules/modulecmd.bash.in | 115 +++++------------------------------ 3 files changed, 20 insertions(+), 105 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index f3aa5b5..8a430a5 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -56,7 +56,7 @@ compute_group_depth () { # (Re-)Scan available groups in given overlays and compute group depth's # # Args: -# $1: array of overlays +# $@: overlay names # scan_groups () { local ol @@ -65,7 +65,7 @@ scan_groups () { local dir for dir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do compute_group_depth "${dir}" - Dir2OverlayMap[${dir}]="${ol}" + Dir2OverlayMap[${dir%/${PMODULES_MODULEFILES_DIR}*}]="${ol}" done done } diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 995b817..fa1afb6 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -498,9 +498,9 @@ build_modules_yaml(){ local idx="$4" _result=$(yq -Ne e ".\"${version}\"[${idx}].overlay" \ "${fname}" 2>/dev/null) - (( $? == 0 )) && return || : + (( $? == 0 )) && return _result=$(yq -Ne e ".overlay" "${fname}" 2>/dev/null) - (( $? == 0 )) && return || : + (( $? == 0 )) && return _result='base' } @@ -542,7 +542,6 @@ build_modules_yaml(){ local -a versions yaml_get_versions versions "${fname}" "${name}/${version}" echo versions=${versions[@]} - echo ${#versions[@]} for v in "${versions[@]}"; do echo version=$v local -i n_variants @@ -581,7 +580,6 @@ build_modules_yaml(){ } -echo "build_modules_yaml()" build_modules() { local variants_file="${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}" if [[ -n "${variants_file}*.yaml" ]]; then diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 5492772..4e041ba 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -143,29 +143,6 @@ _exit() { trap '_exit' EXIT -# -# map a moduledir to an overlay -# -# Args: -# $1 upvar for overlay -# $2 moduledir -# -map_moduledir2ol_name() { - local "$1" - local moduledir="${2//+(\/)/\/}" # replace multpile '/' with one - moduledir="${moduledir/%\/}" # remove trailing slash if exist - - if [[ ! -v Dir2OverlayMap[${moduledir}] ]]; then - local ol - for ol in "${UsedOverlays[@]}" 'other'; do - local mod_root="${OverlayInfo[${ol}:mod_root]}" - [[ ${moduledir} == ${mod_root}/* ]] && break - done - Dir2OverlayMap[${moduledir}]="${ol}" - fi - std::upvar $1 "${Dir2OverlayMap[${moduledir}]}" -} - # # get release stage of module # Note: @@ -180,13 +157,10 @@ map_moduledir2ol_name() { # get_release_stage() { local "$1" - local -r moduledir=$2 - local -r modulefile="$2/$3" + local -r dir="$2" + local -r modulefile="${dir}/$3" - local ol_name - map_moduledir2ol_name ol_name "${moduledir}" - - if [[ "${ol_name}" == 'other' ]]; then + if [[ ! -v Dir2OverlayMap[${dir%/${PMODULES_MODULEFILES_DIR}*}] ]]; then std::upvar $1 'stable' return fi @@ -223,15 +197,14 @@ find_overlay () { local "$2" local path="${3//+(\/)/\/}" # replace multpile '/' with one path="${path/%\/}" # remove trailing slash if exist + path="${path%/${PMODULES_MODULEFILES_DIR}*}" + + [[ -v Dir2OverlayMap[${path}] ]] || return 1 local ol="${Dir2OverlayMap[${path}]}" - map_moduledir2ol_name ol "${path}" std::upvar $1 "${ol}" - [[ "${ol}" == 'other' ]] && return 1 - local group="${path#${OverlayInfo[${ol}:mod_root]}/}" - group=${group%%/*} std::upvar $2 "${group}" return 0 } @@ -608,12 +581,10 @@ subcommand_load() { "${CMD}" "${subcommand}" \ "illegal group name" \ "${group}" - #MODULEPATH="" modulepath=() group+="/${PMODULES_MODULEFILES_DIR}" for overlay in "${UsedOverlays[@]}"; do local mod_root="${OverlayInfo[${overlay}:mod_root]}" - #MODULEPATH="${mod_root}/${group}:${MODULEPATH}" modulepath=( "${mod_root}/${group}" "${modulepath[@]}" ) done fi @@ -720,7 +691,6 @@ subcommand_load() { [[ -z ${dir} ]] && continue [[ "${dir: -1}" == "/" ]] || dir+="/" LOADEDMODULES="${LOADEDMODULES//${dir}}" - map_moduledir2ol_name overlay "${dir}" done <<< "${MODULEPATH//:/$'\n'}" g_env_must_be_saved='yes' export_env 'LOADEDMODULES' @@ -935,8 +905,8 @@ get_available_modules() { # - after loading the parent of a hierarchical group # - if we do a search # - if we create a new hierarchical group - local ol - local group + local ol='' + local group='' find_overlay ol group "${dir}" # if no modules are installed in ${dir}, '*' expands to @@ -1482,13 +1452,12 @@ subcommand_use() { std::remove_path MODULEPATH "${dirs[@]}" done fi - + scan_groups "${ol_name}" for group in ${UsedGroups//:/ }; do local dir="${OverlayInfo[${ol_name}:mod_root]}/" dir+="${group}/${PMODULES_MODULEFILES_DIR}" if [[ -d "${dir}" ]]; then std::prepend_path MODULEPATH "${dir}" - Dir2OverlayMap[${dir}]="${ol_name}" fi done @@ -1515,7 +1484,6 @@ subcommand_use() { [[ -d "${dir}" ]] || continue std::prepend_path MODULEPATH "${dir}" - Dir2OverlayMap[${dir}]=${ol_name} [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]] && break done } @@ -1812,7 +1780,6 @@ init_modulepath() { local dir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" if [[ -d "${dir}" ]]; then std::prepend_path MODULEPATH "${dir}" - Dir2OverlayMap[${dir}]="${ol}" fi done done @@ -1853,6 +1820,10 @@ pmodules_init() { done g_env_must_be_saved='yes' } + init_overlay_vars() { + declare -ag UsedOverlays=( 'base' ) + OverlayInfo['base:used']='yes' + } pm::read_config @@ -1964,10 +1935,11 @@ subcommand_purge() { # we may get output like # 'unstable module has been loaded' subcommand_load "${pmodule}" 2> /dev/null + # export_env PMODULES_HOME fi init_modulepath - export_env MODULEPATH PMODULES_HOME + export_env MODULEPATH } ############################################################################## @@ -2057,7 +2029,6 @@ subcommand_clear() { "no arguments allowed" fi pmodules_init - export_env LOADEDMODULES MODULEPATH _LMFILES_ } ############################################################################## @@ -2762,21 +2733,6 @@ if [[ -z "${Subcommands[${subcommand}]}" ]]; then std::die 1 "${CMD}: unknown sub-command -- ${subcommand}" fi -init_overlay_vars() { - declare -ag UsedOverlays=( 'base' ) - OverlayInfo['base:used']='yes' - #declare -Ag Dir2OverlayMap=() - for ol in "${UsedOverlays[@]}"; do - local group - for group in ${UsedGroups//:/ }; do - local dir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" - if [[ -d "${dir}" ]]; then - Dir2OverlayMap[${dir}]=${ol} - fi - done - done -} - case ${subcommand} in add ) subcommand='load' @@ -2800,48 +2756,9 @@ if [[ -n ${PMODULES_ENV} ]]; then fi if [[ -z ${Version} ]] || [[ ${Version} != ${PMODULES_VERSION} ]]; then pmodules_init - g_env_must_be_saved='yes' + #g_env_must_be_saved='yes' fi -#if [[ -n ${PMODULES_ENV} ]]; then -# eval "$("${base64}" -d <<< "${PMODULES_ENV}" 2>/dev/null)" -# if [[ -z ${Version} ]] || [[ ${Version} != ${PMODULES_VERSION} ]]; then -# # the Pmodules version changed! -# declare -g Version="${PMODULES_VERSION}" -# pm::read_config -# init_overlay_vars -# # renamed in version 1.0.0rc10 and type changed from -# # associative array to normal array -# if [[ -v UseFlags ]]; then -# declare -a UsedFlags=( "${!UseFlags[@]}" ) -# unset UseFlags -# fi -# if [[ ! -v UsedFlags ]]; then -# declare -a UsedFlags=() -# fi -# if [[ -v UsedReleases ]]; then -# declare -- UsedReleaseStages="${UsedReleases}" -# unset UsedReleases -# fi -# if [[ -v PMODULES_DEFAULT_GROUPS ]]; then -# declare -- DefaultGroups="${PMODULES_DEFAULT_GROUPS}" -# unset PMODULES_DEFAULT_GROUPS -# fi -# if [[ -v PMODULES_DEFINED_RELEASES ]]; then -# declare -- ReleaseStages="${PMODULES_DEFINED_RELEASES}" -# unset PMODULES_DEFINED_RELEASES -# fi -# if [[ -v PMODULES_DEFAULT_RELEASES ]]; then -# declare -- DefaultReleaseStages="${PMODULES_DEFAULT_RELEASES}" -# unset PMODULES_DEFAULT_RELEASES -# fi -# g_env_must_be_saved='yes' -# fi -#else -# pmodules_init -# g_env_must_be_saved='yes' -#fi - if (( ${#GroupDepths[@]} == 0 )); then scan_groups "${UsedOverlays[@]}" g_env_must_be_saved='yes' From d916dd1dfe4b8b9e5b6d42420917e0eca6920444 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 13 Jun 2022 11:11:28 +0200 Subject: [PATCH 39/97] refactor: ol_dir -> ol_mod_root and ol_install_dir -> ol_inst_root --- Pmodules/libmodules.tcl | 24 ++++++++++++------------ Pmodules/libpbuild.bash | 4 ++-- Pmodules/modbuild.in | 18 +++++++++--------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Pmodules/libmodules.tcl b/Pmodules/libmodules.tcl index 074550e..942ae33 100644 --- a/Pmodules/libmodules.tcl +++ b/Pmodules/libmodules.tcl @@ -329,20 +329,20 @@ proc _find_overlay { modulefile_components } { debug "_find_overlay()" foreach ol $::UsedOverlays { debug "$ol" - set ol_dir $::OverlayInfo(${ol}:mod_root) - if { [string range $ol_dir end end] == "/" } { - set ol_dir [string range $ol_dir 0 end-1] + set ol_mod_root $::OverlayInfo(${ol}:mod_root) + if { [string range $ol_mod_root end end] == "/" } { + set ol_mod_root [string range $ol_mod_root 0 end-1] } - debug "$ol_dir" - set ol_dir_splitted [file split $ol_dir] + debug "$ol_mod_root" + set ol_mod_root_splitted [file split $ol_mod_root] set modulefile_root [file join \ {*}[lrange \ $modulefile_components \ - 0 [expr [llength $ol_dir_splitted] - 1]]] + 0 [expr [llength $ol_mod_root_splitted] - 1]]] debug "$modulefile_root" - if { [string compare $ol_dir $modulefile_root] == 0 } { - debug "$ol_dir_splitted" - return $ol_dir_splitted + if { [string compare $ol_mod_root $modulefile_root] == 0 } { + debug "$ol_mod_root_splitted" + return $ol_mod_root_splitted } } debug "not found" @@ -374,8 +374,8 @@ proc _pmodules_init_global_vars { } { set modulefile_splitted [file split $::ModulesCurrentModulefile] - set ol_dir_splitted [_find_overlay ${modulefile_splitted}] - set rel_modulefile [lrange $modulefile_splitted [llength $ol_dir_splitted] end] + set ol_mod_root_splitted [_find_overlay ${modulefile_splitted}] + set rel_modulefile [lrange $modulefile_splitted [llength $ol_mod_root_splitted] end] set group [lindex $rel_modulefile 0] set GROUP "${group}" set name [lindex $modulefile_splitted end-1] @@ -386,7 +386,7 @@ proc _pmodules_init_global_vars { } { set V_RELEASE [lindex [split $tmp _] 0] lassign [split $V_PKG .] V_MAJOR V_MINOR V_PATCHLVL set variant [lrange $rel_modulefile 2 end] - set mod_root [file join {*}$ol_dir_splitted] + set mod_root [file join {*}$ol_mod_root_splitted] debug "mod_root=$mod_root" set ol $::Dir2OverlayMap($mod_root) debug "ol=$ol" diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index ad62e80..c9abe8a 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -769,7 +769,7 @@ pbuild::make_all() { "${module_name}") modulefile_name="${modulefile_dir}/${module_version}" - PREFIX="${ol_install_dir}/${GROUP}/${module_name}/${module_version}" + PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}" local -i i=0 for ((i=${#names[@]}-1; i >= 0; i--)); do PREFIX+="/${names[i]}" @@ -1482,7 +1482,7 @@ pbuild.bootstrap() { MODULECMD=$(which true) GROUP='Tools' - PREFIX="${ol_dir}/${GROUP}/Pmodules/${PMODULES_VERSION}" + PREFIX="${ol_mod_root}/${GROUP}/Pmodules/${PMODULES_VERSION}" C_INCLUDE_PATH="${PREFIX}/include" CPLUS_INCLUDE_PATH="${PREFIX}/include" diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index fa1afb6..4c0665a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -62,8 +62,8 @@ declare -r ARGS="$@" shopt -s nocaseglob shopt -s extglob -declare ol_dir -declare ol_install_dir +declare ol_mod_root +declare ol_inst_root ############################################################################## # @@ -434,8 +434,8 @@ build_modules_legacy() { fi declare ol_name='base' declare ol_type='' - declare ol_dir="${pm_root}" - declare ol_install_dir="${pm_root}" + declare ol_mod_root="${pm_root}" + declare ol_inst_root="${pm_root}" local -i i=0 local -i num_variants=${#variants[@]} for ((i = 0; i < num_variants; i++)); do @@ -534,7 +534,7 @@ build_modules_yaml(){ local m local pattern="//" for m in "${with_modules[@]}"; do - if [[ -n $(awk "/${m%/*}[\/ ]/" "${files[@]}") ]]; then + if [[ -n $(awk "/${m%/*}[\/ ]/" "${fname}") ]]; then pattern+=" && /${m//\//\\/}/" fi done @@ -561,8 +561,8 @@ build_modules_yaml(){ fi yaml_get_relstage relstage "${fname}" "${v}" $i yaml_get_dependencies deps "${fname}" "${v}" $i - ol_install_dir="${OverlayInfo[${ol_name}:inst_root]}" - ol_dir="${OverlayInfo[${ol_name}:mod_root]}" + ol_inst_root="${OverlayInfo[${ol_name}:inst_root]}" + ol_mod_root="${OverlayInfo[${ol_name}:mod_root]}" if (( ${#deps[@]} > 0 )); then while read -a with_modules; do @@ -636,8 +636,8 @@ module_name=${fname[${#fname[@]}-2]} if [[ "${opt_bootstrap}" == 'yes' ]]; then declare ol_name='base' declare ol_type='' - declare ol_dir="${pm_root}" - declare ol_install_dir="${pm_root}" + declare ol_mod_root="${pm_root}" + declare ol_inst_root="${pm_root}" pbuild.bootstrap "${module_name}" "${versions[0]}" 'stable' exit $? fi From 9ff2289116ce0e2564b3749a9c1065b95d4b1bac Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:15:36 +0200 Subject: [PATCH 40/97] cleanup unused packages: coreutils and gettext --- Tools/coreutils/build | 4 ---- Tools/coreutils/build.new | 10 ---------- Tools/gettext/build | 19 ------------------- 3 files changed, 33 deletions(-) delete mode 100755 Tools/coreutils/build delete mode 100755 Tools/coreutils/build.new delete mode 100755 Tools/gettext/build diff --git a/Tools/coreutils/build b/Tools/coreutils/build deleted file mode 100755 index c561e2a..0000000 --- a/Tools/coreutils/build +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "http://ftp.gnu.org/gnu/coreutils/$P-$V.tar.xz" -pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" diff --git a/Tools/coreutils/build.new b/Tools/coreutils/build.new deleted file mode 100755 index 63030c7..0000000 --- a/Tools/coreutils/build.new +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env pbuild - -pmodules.configure() { - "${MODULE_SRCDIR}"/configure \ - --prefix="${PREFIX}" \ - || exit 1 -} - -pmodules.add_to_group 'Tools' -pmodules.make_all diff --git a/Tools/gettext/build b/Tools/gettext/build deleted file mode 100755 index 776185e..0000000 --- a/Tools/gettext/build +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.gz" - -pbuild::add_configure_args "--prefix=${PREFIX}" -pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" -pbuild::add_configure_args "--disable-java" -pbuild::add_configure_args "--disable-threads" -pbuild::add_configure_args "--disable-shared" -pbuild::add_configure_args "--enable-relocatable" -pbuild::add_configure_args "--disable-openmp" -pbuild::add_configure_args "--disable-acl" -pbuild::add_configure_args "--disable-curses" -pbuild::add_configure_args "--with-included-gettext" -pbuild::add_configure_args "--without-libiconv-prefix" -pbuild::add_configure_args "--without-libintl-prefix" -pbuild::add_configure_args "--with-included-libxml" -pbuild::add_configure_args "--with-pic=yes" - From ac318f2e83a7144bcfa7a2ff12a17dbd6c879dc5 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:17:19 +0200 Subject: [PATCH 41/97] function read_config_file moved from libpbuild.bash to build --- Pmodules/libpbuild.bash | 20 -------------------- build | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index c9abe8a..feb0129 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1496,26 +1496,6 @@ pbuild.bootstrap() { pbuild::make_all } -#----------------------------------------------------------------------------- -# -read_config_file() { - local fname="$1" - if [[ ! -r "${fname}" ]]; then - std::die 1 "Configuration file '${fname}' does not exist or is not readable!" - fi - - eval $(std::parse_yaml "${fname}" '') || std::die 1 "Cannot read configuration file '${fname}'" - - PMODULES_ROOT="${Overlays_base_install_root}" - PMODULES_DISTFILESDIR="${DistfilesDir}" - PMODULES_TMPDIR="${TmpDir}" - PMODULES_HOME="${PMODULES_ROOT}/Tools/Pmodules/${PMODULES_VERSION}" - - if [[ -z "${PMODULES_HOME}" ]]; then - std::die 1 "Error in configuration file '${fname}': PMODULE_HOME not defined!" - fi -} - # Local Variables: # mode: sh # sh-basic-offset: 8 diff --git a/build b/build index 232a760..21932ba 100755 --- a/build +++ b/build @@ -72,6 +72,28 @@ to get help for a specific sub-command. std::die 1 "" } +#----------------------------------------------------------------------------- +# +read_config_file() { + local fname="$1" + if [[ ! -r "${fname}" ]]; then + std::die 1 "Configuration file '${fname}' does not exist or is not readable!" + fi + + eval $(std::parse_yaml "${fname}" '') || std::die 1 "Cannot read configuration file '${fname}'" + + PMODULES_ROOT="${Overlays_base_install_root}" + PMODULES_DISTFILESDIR="${DistfilesDir}" + PMODULES_TMPDIR="${TmpDir}" + PMODULES_HOME="${PMODULES_ROOT}/Tools/Pmodules/${PMODULES_VERSION}" + + if [[ -z "${PMODULES_HOME}" ]]; then + std::die 1 "Error in configuration file '${fname}': PMODULE_HOME not defined!" + fi +} + + + #----------------------------------------------------------------------------- # pmodules::help() { From 6401d2e1f7f8564b9c933c5d1c521e4ca6b224ed Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:19:51 +0200 Subject: [PATCH 42/97] modbuild: bugfix in decision with to use legacy or YAML variants file --- Pmodules/modbuild.in | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 4c0665a..6b66305 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -433,7 +433,7 @@ build_modules_legacy() { std::die 10 "Aborting..." fi declare ol_name='base' - declare ol_type='' + declare ol_type='n' declare ol_mod_root="${pm_root}" declare ol_inst_root="${pm_root}" local -i i=0 @@ -581,8 +581,7 @@ build_modules_yaml(){ build_modules() { - local variants_file="${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}" - if [[ -n "${variants_file}*.yaml" ]]; then + if [[ -n $(ls "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}"*.yaml 2>/dev/null) ]]; then build_modules_yaml "$@" else build_modules_legacy "$@" From 49826b60ec6853595fda8728b48ebd3d28abc4ad Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:21:40 +0200 Subject: [PATCH 43/97] libpbuild: bugfix: ol_dir replaced by ol_mod_root --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index feb0129..aceecfd 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -764,7 +764,7 @@ pbuild::make_all() { fi modulefile_dir=$(join_by '/' \ - "${ol_dir}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ "${names[@]}" \ "${module_name}") modulefile_name="${modulefile_dir}/${module_version}" From 430632def927c83586cb3f4c35407e5d02b23cd5 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:23:19 +0200 Subject: [PATCH 44/97] libpbuild: set/remove modulefiles in overlays according configuration --- Pmodules/libpbuild.bash | 95 +++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index aceecfd..119e35f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -958,11 +958,35 @@ pbuild::make_all() { std::info \ "%s " \ "${module_name}/${module_version}:" \ - "installing modulefile '${modulefile_name}' ..." + "adding modulefile to overlay '${ol_name}' ..." mkdir -p "${modulefile_dir}" install -m 0644 "${src}" "${modulefile_name}" } + cleanup_modulefiles(){ + local ol='' + for ol in "${Overlays[@]}"; do + [[ "${ol}" == "${ol_name}" ]] && continue + local mod_root="${OverlayInfo[${ol}:mod_root]}" + local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" + local fname="${dir}/${module_version}" + if [[ -e "${fname}" ]]; then + std::info "%s "\ + "${module_name}/${module_version}:" \ + "removing modulefile from overlay '${ol}' ..." + rm "${fname}" + fi + fname="${dir}/.release-${module_version}" + if [[ -e "${fname}" ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "removing release file from overlay '${ol}' ..." + rm "${fname}" + fi + done + } + install_release_file() { local -r release_file="${modulefile_dir}/.release-${module_version}" @@ -1074,12 +1098,6 @@ pbuild::make_all() { #...................................................................... # build module ${module_name}/${module_version} build_module() { - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "start building" \ - ${with_modules:+with ${with_modules[@]}} \ - "..." [[ ${dry_run} == yes ]] && std::die 0 "" mkdir -p "${SRC_DIR}" @@ -1154,37 +1172,50 @@ pbuild::make_all() { check_supported_os check_supported_compilers set_full_module_name_and_prefix - if [[ -e "${modulefile_name}" ]] \ - && [[ -d ${PREFIX} ]] \ - && [[ ${force_rebuild} != 'yes' ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + ${with_modules:+build with ${with_modules[@]}} + if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then + # don't (re-)build the module, but + # - if the release stage has been changed to 'removed', + # remove the module. + # - if requested, update the modulefile. + # - if modulefile does not exist, install it. + # - update release stage. if [[ "${module_release}" == 'removed' ]]; then remove_module - else - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "already exists, not rebuilding ..." - if [[ "${opt_update_modulefiles}" == "yes" ]]; then - install_modulefile - fi - install_release_file + return $? fi - else - if [[ "${module_release}" == 'deprecated' ]]; then - std::info \ - "%s " "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "is deprecated, skiping!" - install_release_file - else - build_module + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "already exists, not rebuilding ..." + if [[ "${opt_update_modulefiles}" == "yes" ]] || \ + [[ ! -e "${modulefile_name}" ]]; then + install_modulefile fi + install_release_file + cleanup_modulefiles + return $? + fi + if [[ "${module_release}" == 'deprecated' ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "is deprecated, skiping!" + install_release_file + cleanup_modulefiles + return $? fi - else - build_module fi - return 0 + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "start building ..." + + build_module + return $? } pbuild.init_env() { From fbc64b01361bd6b00d51a4d24eabbc0a094a73dd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 22 Jun 2022 22:24:57 +0200 Subject: [PATCH 45/97] modulecmd: bugfix in sub-cmd search - sorting fixed --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 4e041ba..ea7f5c0 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -2216,7 +2216,7 @@ subcommand_search() { ${func_print_header} while read -a toks; do ${func_print_line} "${toks[@]}" - done < <("${sort}" --version-sort -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ + done < <("${sort}" --version-sort -k 1,1 -k 6,6 -k 7,7 "${tmpfile}" | \ ${awk} "${with_modules}") } From bec2795be59eacbd9221c3f1e3b86fb13b4c6537 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 23 Jun 2022 14:58:28 +0200 Subject: [PATCH 46/97] CHANGELOG.md for version 1.1.9 --- CHANGELOG.md | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b52fdb2..26854bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,25 +3,38 @@ ## Version 1.1.9 * **modulecmd** * *User visible changes* - * ... + * Overlay info added to output of sub-command 'search' + * Output of 'module search --verbose' revised for better readability. * *Internal changes and fixes* - * the shell's init file is sourced, when Pmodules is loaded as module - * bug fixed in libmodules.tcl:module-addgroup() which crashed - "module load". - * don't crash if MODULEPATH starts or ends with a colon - * add overlay info to output of sub-command 'search' + * The shell`s init file is sourced, when Pmodules is loaded as module. + This is required if there are changes in the module function or too + define new shell functions. + * A bug in libmodules.tcl:module-addgroup() which crashed + 'module load ...' has been fixed. + * In versions before 1.1.9 a colon at the beginning or end of MODULEPATH + crashed the module function. This has been fixed. + * **build-system** * *User visible changes* - * modbuild is now defined as shell function analog to module() - * modbuild can be called without specifying a build-script, - if a build-script exist in the current working directory - * abort build on error - * with option `--overlay` - - an overlay can be set if legacy variants files are used - - the overlay in the YAML variants file can overriden - * *Internal changes and fixes* - * bugfix in setting PATH - * requires bash 5 or later + * The command 'modbuild' is now defined as shell function analog to + the 'module' command. The main reason to introduce this function + is due to the fact that Bash version 5 or newer is now required + by 'modbuild'. The function 'modbuild' load Bash 5.x as module + before calling the modbuild-script. If you want to use the script + directly, a Bash binary with version 5.x must be in PATH. + * If a build-script is in the current working directory, + 'modbuild' can now be called without specifying the build-script. + * In case of an error in a build-step the build process did not + abort as it should. This has been fixed. + * The option `--overlay` can now be used + - to define an overlay if legacy variants files are used + - to override the overlay in a YAML variants file. + * The new keyword `with` has been introduced in YAML variants file + to specified hierarchical dependencies. + +* *Internal changes and fixes* + * bugfix in setting PATH + * requires bash 5 or later ## Version 1.1.8 * **modulecmd** @@ -29,7 +42,7 @@ * configuration in YAML files * modulefiles and software must not have a common root directory - * the installation root must be specified, it doesn't default + * the installation root must be specified, it doesn`t default to the base 'overlay' any more. * zsh initialisation fixed. * *Internal changes and fixes* From 564bad3652fdb7899fa6d5fae4be7cb8ed4a83bf Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 23 Jun 2022 15:07:40 +0200 Subject: [PATCH 47/97] CHANGELOG.md for version 1.1.9 --- CHANGELOG.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26854bf..c217d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,27 +3,27 @@ ## Version 1.1.9 * **modulecmd** * *User visible changes* - * Overlay info added to output of sub-command 'search' - * Output of 'module search --verbose' revised for better readability. + * Overlay info added to output of sub-command `search`. + * Output of `module search --verbose` revised for better readability. * *Internal changes and fixes* * The shell`s init file is sourced, when Pmodules is loaded as module. This is required if there are changes in the module function or too define new shell functions. - * A bug in libmodules.tcl:module-addgroup() which crashed - 'module load ...' has been fixed. - * In versions before 1.1.9 a colon at the beginning or end of MODULEPATH + * A bug in `libmodules.tcl:module-addgroup()` which crashed + `module load ...` has been fixed. + * In versions before 1.1.9 a colon at the beginning or end of `MODULEPATH` crashed the module function. This has been fixed. * **build-system** * *User visible changes* - * The command 'modbuild' is now defined as shell function analog to - the 'module' command. The main reason to introduce this function + * The command `modbuild` is now defined as shell function analog to + the `module` command. The main reason to introduce this function is due to the fact that Bash version 5 or newer is now required - by 'modbuild'. The function 'modbuild' load Bash 5.x as module + by `modbuild`. The function `modbuild` load Bash 5.x as module before calling the modbuild-script. If you want to use the script directly, a Bash binary with version 5.x must be in PATH. * If a build-script is in the current working directory, - 'modbuild' can now be called without specifying the build-script. + `modbuild` can now be called without specifying the build-script. * In case of an error in a build-step the build process did not abort as it should. This has been fixed. * The option `--overlay` can now be used @@ -32,8 +32,8 @@ * The new keyword `with` has been introduced in YAML variants file to specified hierarchical dependencies. -* *Internal changes and fixes* - * bugfix in setting PATH +* **Internal changes and fixes** + * bugfix in setting `PATH` * requires bash 5 or later ## Version 1.1.8 From da87be53de8c64ab043e7b5ae12411486c236308 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 23 Jun 2022 15:11:17 +0200 Subject: [PATCH 48/97] version set to 1.1.10, stub entry in CHANGELOG.md --- CHANGELOG.md | 16 ++++++++++++++++ config/versions.conf | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c217d13..9bc8906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog of Pmodules +## Version 1.1.10 +* **modulecmd** + * *User visible changes* + * ... + * *Internal changes and fixes* + * ... + +* **build-system** + * *User visible changes* + * ... + * *Internal changes and fixes* + * ... + +* **other changes** + * ... + ## Version 1.1.9 * **modulecmd** * *User visible changes* diff --git a/config/versions.conf b/config/versions.conf index 96769f7..2cbdbab 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -2,6 +2,6 @@ bash 5.1.16 findutils 4.9.0 getopt 1.1.6 modules 3.2.10.1 -Pmodules 1.1.9 +Pmodules 1.1.10 Tcl 8.6.12 tcllib 1.20 From 724fc93ad2099c84f6c09c382af5cde10a73d99d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 24 Jun 2022 18:54:33 +0200 Subject: [PATCH 49/97] build script: don't use modbuild for Linux tools --- 10-build-bash | 48 +++++++++++++++++++++++++++ 20-build-tcl | 61 ++++++++++++++++++++++++++++++++++ 30-build-tcllib | 44 +++++++++++++++++++++++++ 40-build-modules | 79 +++++++++++++++++++++++++++++++++++++++++++++ Tools/Tcl/build | 25 -------------- Tools/bash/build | 5 --- Tools/modules/build | 56 -------------------------------- Tools/tcllib/build | 5 --- build | 73 ++++++++++++++++++++++++++++++++++------- 9 files changed, 294 insertions(+), 102 deletions(-) create mode 100755 10-build-bash create mode 100755 20-build-tcl create mode 100755 30-build-tcllib create mode 100755 40-build-modules delete mode 100755 Tools/Tcl/build delete mode 100755 Tools/bash/build delete mode 100755 Tools/modules/build delete mode 100755 Tools/tcllib/build diff --git a/10-build-bash b/10-build-bash new file mode 100755 index 0000000..8ac20b7 --- /dev/null +++ b/10-build-bash @@ -0,0 +1,48 @@ +#!/bin/bash +# +# https://www.gnu.org/software/bash/ +# + +set -x + +P=bash +V=${BASH5_VERSION:-5.1.16} + +FNAME="$P-$V.tar.gz" +DOWNLOAD_URL="https://ftp.gnu.org/gnu/$P/${FNAME}" + +PREFIX="${PMODULES_HOME}" +SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" +SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" +BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" + +trap "pb_exit" EXIT + +# download +test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} + +# unpack +mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} + +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +loadablesdir="${PREFIX}/${UTILBIN_DIR}/builtins" \ +"${SRC_DIR}/configure" \ + --prefix="${PREFIX}" \ + --bindir="${PREFIX}/${UTILBIN_DIR}" \ + || exit ${PB_ERR_CONFIGURE} + +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make -C examples/loadables -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +# post-install +rm -rf "${PREFIX}/include/bash" +rm -rf "${PREFIX}/share/locale" + +# Local Variables: +# mode: shell-script-mode +# sh-basic-offset: 8 +# End: diff --git a/20-build-tcl b/20-build-tcl new file mode 100755 index 0000000..6b54d94 --- /dev/null +++ b/20-build-tcl @@ -0,0 +1,61 @@ +#!/bin/bash +# +# https://www.tcl.tk +# + +set -x + +P=tcl +V=${TCL_VERSION:-8.6.12} + +FNAME="$P$V-src.tar.gz" +DOWNLOAD_URL="https://prdownloads.sourceforge.net/tcl/${FNAME}" + +PREFIX="${PMODULES_HOME}" +SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" +SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" +BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" + +trap "pb_exit" EXIT + +# download +test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} + +# unpack +mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} + +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +case $(uname -s) in +Linux ) + srcdir="${SRC_DIR}/unix" + ;; +Darwin ) + srcdir="${SRC_DIR}/macosx" + ;; +* ) + echo "Oops: unsupported OS!" 1>&2 + exit ${PB_ERR_SYSTEM} + ;; +esac +"${srcdir}"/configure \ + --prefix="${PREFIX}" \ + --bindir="${PREFIX}/${UTILBIN_DIR}" \ + --mandir="${PREFIX}/share/man" \ + --enable-shared=no \ + || exit ${PB_ERR_CONFIGURE} + +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +# post-install +{ cd "${PREFIX}/${UTILBIN_DIR}" && rm -f tclsh && ln -fs tclsh${V%.*} tclsh; }; +rm -rf "${PREFIX}/share/man/man3" +rm -rf "${PREFIX}/share/man/mann" + +# Local Variables: +# mode: shell-script-mode +# sh-basic-offset: 8 +# End: diff --git a/30-build-tcllib b/30-build-tcllib new file mode 100755 index 0000000..c81a0a0 --- /dev/null +++ b/30-build-tcllib @@ -0,0 +1,44 @@ +#!/bin/bash +# +# https://core.tcl-lang.org +# + +set -x + +P=tcllib +V=${TCLLIB_VERSION:-1.21} + +FNAME="$P-$V.tar.gz" +DOWNLOAD_URL="https://core.tcl-lang.org/tcllib/uv/${FNAME}" +PREFIX="${PMODULES_HOME}" +SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" +SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" +BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" + +trap "pb_exit" EXIT + +# download +test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} + +# unpack +mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} + +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +"${SRC_DIR}"/configure \ + --prefix="${PREFIX}" \ + --mandir="${PREFIX}/share/man" \ + || exit ${PB_ERR_CONFIGURE} + +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +# post-install +rm -rf "${PREFIX}/share/man/mann" + +# Local Variables: +# mode: shell-script-mode +# sh-basic-offset: 8 +# End: diff --git a/40-build-modules b/40-build-modules new file mode 100755 index 0000000..73ee00d --- /dev/null +++ b/40-build-modules @@ -0,0 +1,79 @@ +#!/bin/bash +# +# https://core.tcl-lang.org +# + +set -x + +P=modules +V=${MODULES_VERSION:-3.2.10.1} + +FNAME="$P-$V.tar.gz" +DOWNLOAD_URL="https://amas.web.psi.ch/Downloads/$P/$P-$V.tar.bz2" + +PREFIX="${PMODULES_HOME}" +SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" +SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" +BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" + +PATH="${PREFIX}/bin:${PATH}" + +trap "pb_exit" EXIT + +# download +test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} + +# unpack +mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} + +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +case $(uname -s) in + Linux ) + declare -x LIBS="-lz -lpthread" + ;; + Darwin ) + declare -x LIBS="-lz -framework CoreFoundation" + ;; + * ) + echo "Oops: unsupported OS!" 1>&2 + exit ${PB_ERR_SYSTEM} + ;; +esac +CPPFLAGS="-DUSE_INTERP_ERRORLINE" \ +"${SRC_DIR}"/configure \ + --prefix="${PREFIX}" \ + --exec-prefix="${PREFIX}" \ + --with-module-path="${PREFIX%%/Tools*}/Tools/${PMODULES_MODULEFILES_DIR}" \ + --with-tcl="${PREFIX}/lib" \ + --without-x \ + --disable-versioning \ + || exit ${PB_ERR_CONFIGURE} + +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +# post-install +mkdir -p "${PREFIX}/share/man/man1" +mkdir -p "${PREFIX}/share/man/man4" +rm -v "${PREFIX}/Modules/bin/add.modules" +rm -v "${PREFIX}/Modules/bin/mkroot" +rm -rfv "${PREFIX}/Modules/modulefiles" +mv -v "${PREFIX}/Modules/share/man/man1/module.1" "${PREFIX}/share/man/man1" +mv -v "${PREFIX}/Modules/share/man/man4/modulefile.4" "${PREFIX}/share/man/man4" +rmdir "${PREFIX}/Modules/bin" +rmdir "${PREFIX}/Modules/share/man/man1" +rmdir "${PREFIX}/Modules/share/man/man4" +rmdir "${PREFIX}/Modules/share/man" +rmdir "${PREFIX}/Modules/share" +rmdir "${PREFIX}/Modules" +rm -f "${PREIX}/init/{ksh,perl.pm,python.py,ruby.rb,cmake,.modulespath}" +cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 + +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# tab-width: 8 +# End: diff --git a/Tools/Tcl/build b/Tools/Tcl/build deleted file mode 100755 index d1ceea6..0000000 --- a/Tools/Tcl/build +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "https://prdownloads.sourceforge.net/tcl/tcl$V-src.tar.gz" -pbuild::install_docfiles 'license.terms' 'README.md' - -pbuild::configure() { - case ${OS} in - Linux ) - srcdir="${SRC_DIR}/unix" - ;; - Darwin ) - srcdir="${SRC_DIR}/macosx" - ;; - esac - "${srcdir}"/configure \ - --prefix="${PREFIX}" \ - --bindir="${PREFIX}/${UTILBIN_DIR}" \ - --enable-shared=no \ - || exit 1 -} - -pbuild::post_install() { - { cd "${PREFIX}/${UTILBIN_DIR}" && rm -f tclsh && ln -fs tclsh${V%.*} tclsh; }; -} - diff --git a/Tools/bash/build b/Tools/bash/build deleted file mode 100755 index ff8489f..0000000 --- a/Tools/bash/build +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "https://ftp.gnu.org/gnu/$P/$P-$V.tar.gz" -pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" - diff --git a/Tools/modules/build b/Tools/modules/build deleted file mode 100755 index 14399de..0000000 --- a/Tools/modules/build +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env modbuild - -# use system gcc to compile -declare -rx CC=gcc - -pbuild::set_download_url "https://amas.web.psi.ch/Downloads/$P/$P-$V.tar.bz2" - -TCL_DIR="${PMODULES_HOME}" - -PATH="${TCL_DIR}/bin:${PATH}" - -pbuild::configure() { - case ${OS} in - Linux ) - declare -x LIBS="-lz -lpthread" - ;; - Darwin ) - declare -x LIBS="-lz -framework CoreFoundation" - ;; - esac - CPPFLAGS="-DUSE_INTERP_ERRORLINE" "${SRC_DIR}"/configure \ - --prefix="${PREFIX}" \ - --exec-prefix="${PREFIX}" \ - --with-module-path="${PMODULES_HOME%%/Tools*}/Tools/${PMODULES_MODULEFILES_DIR}" \ - --with-tcl="${TCL_DIR}/lib" \ - --without-x \ - --disable-versioning \ - || exit 1 -} - -pbuild::post_install() { - rm -v "${PREFIX}/Modules/bin/add.modules" - rm -v "${PREFIX}/Modules/bin/mkroot" - rm -rfv "${PREFIX}/Modules/modulefiles" - mv -v "${PREFIX}/Modules/share/man/man1/module.1 ${PREFIX}/share/man/man1" - mv -v "${PREFIX}/Modules/share/man/man4/modulefile.4 ${PREFIX}/share/man/man4" - rmdir "${PREFIX}/Modules/bin" - rmdir "${PREFIX}/Modules/share/man/man1" - rmdir "${PREFIX}/Modules/share/man/man4" - rmdir "${PREFIX}/Modules/share/man" - rmdir "${PREFIX}/Modules/share" - rmdir "${PREFIX}/Modules" - rm -f "${PREIX}/init/{ksh,perl.pm,python.py,ruby.rb,cmake,.modulespath}" - cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 -} - -# fake module command -module() { - : -} - -# Local Variables: -# mode: sh -# sh-basic-offset: 8 -# tab-width: 8 -# End: diff --git a/Tools/tcllib/build b/Tools/tcllib/build deleted file mode 100755 index ea151ba..0000000 --- a/Tools/tcllib/build +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "https://core.tcl-lang.org/tcllib/uv/$P-$V.tar.xz" - - diff --git a/build b/build index 21932ba..0646e56 100755 --- a/build +++ b/build @@ -29,6 +29,57 @@ declare -rx DEFAULT_TMPDIR='var/tmp/${USER}' # directory where the required tools will be installed (like bash, tclsh, etc) declare -rx UTILBIN_DIR='libexec' +declare -ix PB_ERR_ARG=1 +declare -ix PB_ERR_SETUP=2 +declare -ix PB_ERR_SYSTEM=3 +declare -ix PB_ERR_DOWNLOAD=4 +declare -ix PB_ERR_UNTAR=5 +declare -ix PB_ERR_CONFIGURE=6 +declare -ix PB_ERR_MAKE=7 +declare -ix PB_ERR_PRE_INSTALL=8 +declare -ix PB_ERR_INSTALL=9 +declare -ix PB_ERR_POST_INSTALL=10 +declare -ix PB_ERR=255 +declare -ix NJOBS=4 + +pb_exit() { + local -i ec=$? + if [[ -n "${BASH_VERSION}" ]]; then + local -i n=${#BASH_SOURCE[@]} + local -r recipe_name="${BASH_SOURCE[n]}" + else + local -r recipe_name="${ZSH_ARGZERO}" + fi + echo -n "${recipe_name}: " + if (( ec == 0 )); then + echo "done!" + elif (( ec == PB_ERR_ARG )); then + echo "argument error!" + elif (( ec == PB_ERR_SETUP )); then + echo "error in setting everything up!" + elif (( ec == PB_ERR_SYSTEM )); then + echo "unexpected system error!" + elif (( ec == PB_ERR_DOWNLOAD )); then + echo "error in downloading the source file!" + elif (( ec == PB_ERR_UNTAR )); then + echo "error in un-taring the source file!" + elif (( ec == PB_ERR_CONFIGURE )); then + echo "error in configuring the software!" + elif (( ec == PB_ERR_MAKE )); then + echo "error in compiling the software!" + elif (( ec == PB_ERR_PRE_INSTALL )); then + echo "error in pre-installing the software!" + elif (( ec == PB_ERR_INSTALL )); then + echo "error in installing the software!" + elif (( ec == PB_ERR_POST_INSTALL )); then + echo "error in post-installing the software!" + else + echo "oops, unknown error!!!" + fi + exit ${ec} +} +export -f pb_exit > /dev/null + #----------------------------------------------------------------------------- # # Get version from config file. @@ -354,27 +405,27 @@ pmodules::compile() { fi if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/bash" ]] || [[ ${opt_force} == 'yes' ]]; then - build bash + ./10-build-bash fi if [[ ! -e "${PMODULES_HOME}/${UTILBIN_DIR}/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then - build Tcl + ./20-build-tcl fi if [[ ! -e "${PMODULES_HOME}/lib/tcllib1.20" ]] || [[ ${opt_force} == 'yes' ]]; then - build tcllib + ./30-build-tcllib fi if [[ ! -e "${PMODULES_HOME}/libexec/modulecmd.bin" ]] || [[ ${opt_force} == 'yes' ]]; then - build modules + ./40-build-modules fi - rm -rf "${PMODULES_HOME}/include" - rm -rf "${PMODULES_HOME}/lib/"*.a - rm -rf "${PMODULES_HOME}/lib/"*.la - rm -rf "${PMODULES_HOME}/lib/bash" - rm -rf "${PMODULES_HOME}/lib/pkginfo" - rm -rf "${PMODULES_HOME}/man" - rm -rf "${PMODULES_HOME}/share" + #rm -rf "${PMODULES_HOME}/include" + #rm -rf "${PMODULES_HOME}/lib/"*.a + #rm -rf "${PMODULES_HOME}/lib/"*.la + #rm -rf "${PMODULES_HOME}/lib/bash" + #rm -rf "${PMODULES_HOME}/lib/pkginfo" + #rm -rf "${PMODULES_HOME}/man" + #rm -rf "${PMODULES_HOME}/share" echo "Done..." } From 868da78ecc43ac198e43f8b9e5c724a2cf20c4e2 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 24 Jun 2022 18:55:35 +0200 Subject: [PATCH 50/97] modulefile: unset C_INCLUDE_PATH and CPLUS_INCLUDE_PATH --- Pmodules/modulefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pmodules/modulefile b/Pmodules/modulefile index 16df9dc..8dec4df 100644 --- a/Pmodules/modulefile +++ b/Pmodules/modulefile @@ -20,6 +20,8 @@ if { [module-info mode load] } { remove-path PATH $str } } + remove-path C_INCLUDE_PATH "$PREFIX/include" + remove-path CPLUS_INCLUDE_PATH "$PREFIX/include" } set shell [module-info shell] From d1a557f6633dbe58f6a576f36fc4d72a6914ee2f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 24 Jun 2022 18:56:50 +0200 Subject: [PATCH 51/97] bash initialisation: don't unset PMODULES_ENV --- Pmodules/bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/bash b/Pmodules/bash index b9cbaf9..4e9044c 100644 --- a/Pmodules/bash +++ b/Pmodules/bash @@ -29,7 +29,7 @@ fi unset MODULE_VERSION unset MODULE_VERSION_STACK unset MODULESHOME -unset PMODULES_ENV +#unset PMODULES_ENV declare -x PMODULES_DIR="${PMODULES_HOME}" From 809c283990bfe991c28bd9bd4f2e6cb301070019 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 24 Jun 2022 19:42:05 +0200 Subject: [PATCH 52/97] build-script: unused build via modbuild function removed --- build | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/build b/build index 0646e56..12d737b 100755 --- a/build +++ b/build @@ -330,24 +330,6 @@ Options: } pmodules::compile() { - build () { - local -r name="$1" - local -r version=$(get_version "${name}") - shift - - "${BOOTSTRAP_DIR}/Pmodules/modbuild.in" \ - "--config=${config_file}" \ - "--disable-cleanup" \ - "--force-rebuild" \ - "--debug" \ - "--verbose" \ - "${BOOTSTRAP_DIR}/Tools/${name}/build" \ - "${build_opts[@]}" "$@" "${version}" || \ - std::die 3 "Compiling '${name}' failed!" - # :FIXME: this must be fixed in modbuild! - rm -f "${BOOTSTRAP_DIR}/Tools/modulefiles/.release-$V" - } - local prefix="${PMODULES_ROOT:-${DEFAULT_PMODULES_ROOT}}" local opt_force='no' local config_file='' @@ -396,11 +378,11 @@ pmodules::compile() { if [[ "${OS}" == 'Darwin' ]]; then if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then - build getopt + : build getopt fi if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/find" ]] || [[ ${opt_force} == 'yes' ]]; then - build findutils + : build findutils fi fi From 13a9ada59d7ac518a683cd4ff3bdf724343276a9 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 24 Jun 2022 19:42:53 +0200 Subject: [PATCH 53/97] modbuild: all bootstrapping code removed --- Pmodules/libpbuild.bash | 102 ++++++++++++++-------------------------- Pmodules/modbuild.in | 25 +--------- 2 files changed, 37 insertions(+), 90 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 119e35f..6ff5762 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -927,10 +927,8 @@ pbuild::make_all() { "${PREFIX}/${FNAME_IDEPS}" \ "${install_dependencies[@]}" fi - if [[ "${bootstrap}" == 'no' ]]; then - install_modulefile - install_release_file - fi + install_modulefile + install_release_file cleanup_build cleanup_src std::info \ @@ -1167,47 +1165,45 @@ pbuild::make_all() { # # setup module specific environment - if [[ "${bootstrap}" == 'no' ]]; then - check_supported_systems - check_supported_os - check_supported_compilers - set_full_module_name_and_prefix + check_supported_systems + check_supported_os + check_supported_compilers + set_full_module_name_and_prefix + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + ${with_modules:+build with ${with_modules[@]}} + if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then + # don't (re-)build the module, but + # - if the release stage has been changed to 'removed', + # remove the module. + # - if requested, update the modulefile. + # - if modulefile does not exist, install it. + # - update release stage. + if [[ "${module_release}" == 'removed' ]]; then + remove_module + return $? + fi std::info \ "%s " \ "${module_name}/${module_version}:" \ - ${with_modules:+build with ${with_modules[@]}} - if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then - # don't (re-)build the module, but - # - if the release stage has been changed to 'removed', - # remove the module. - # - if requested, update the modulefile. - # - if modulefile does not exist, install it. - # - update release stage. - if [[ "${module_release}" == 'removed' ]]; then - remove_module - return $? - fi - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "already exists, not rebuilding ..." - if [[ "${opt_update_modulefiles}" == "yes" ]] || \ + "already exists, not rebuilding ..." + if [[ "${opt_update_modulefiles}" == "yes" ]] || \ [[ ! -e "${modulefile_name}" ]]; then - install_modulefile - fi - install_release_file - cleanup_modulefiles - return $? - fi - if [[ "${module_release}" == 'deprecated' ]]; then - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "is deprecated, skiping!" - install_release_file - cleanup_modulefiles - return $? + install_modulefile fi + install_release_file + cleanup_modulefiles + return $? + fi + if [[ "${module_release}" == 'deprecated' ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "is deprecated, skiping!" + install_release_file + cleanup_modulefiles + return $? fi std::info \ "%s " \ @@ -1312,7 +1308,6 @@ pbuild.build_module() { with_modules=( "$@" ) # used in pbuild::make_all - declare bootstrap='no' declare -a runtime_dependencies=() declare -a install_dependencies=() @@ -1502,31 +1497,6 @@ pbuild.build_module() { std::info "* * * * *\n" } -pbuild.bootstrap() { - local -r module_name="$1" - local -r module_version="$2" - - # used in pbuild::make_all - bootstrap='yes' - - pbuild.init_env "${module_name}" "${module_version}" - - MODULECMD=$(which true) - GROUP='Tools' - PREFIX="${ol_mod_root}/${GROUP}/Pmodules/${PMODULES_VERSION}" - - C_INCLUDE_PATH="${PREFIX}/include" - CPLUS_INCLUDE_PATH="${PREFIX}/include" - CPP_INCLUDE_PATH="${PREFIX}/include" - LIBRARY_PATH="${PREFIX}/lib" - LD_LIBRARY_PATH="${PREFIX}/lib" - DYLD_LIBRARY_PATH="${PREFIX}/lib" - - PATH+=":${PREFIX}/bin" - PATH+=":${PREFIX}/sbin" - pbuild::make_all -} - # Local Variables: # mode: sh # sh-basic-offset: 8 diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 6b66305..adcb5be 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -165,7 +165,6 @@ MISCELLANEOUS OPTIONS: # versions to be build, '.*' or none means all declare -a versions=() -declare opt_bootstrap='no' declare opt_build_config='Pmodules.yaml' declare opt_build_target='all' declare opt_dry_run='no' @@ -288,9 +287,6 @@ parse_args() { --prep | --configure | --compile | --install | --all ) opt_build_target=${1:2} ;; - --bootstrap ) - opt_bootstrap='yes' - ;; --update-modulefiles ) opt_update_modulefiles='yes' ;; @@ -608,11 +604,7 @@ pbuild.verbose "${opt_verbose}" # # read configuration for modbuild # -if [[ "${opt_bootstrap}" == 'yes' ]]; then - pm::read_config "${BUILDBLOCK_DIR}/../../config/Pmodules.yaml" -else - pm::read_config -fi +pm::read_config # :FIXME: should go dist files to # ${pm_root}/var/distfiles @@ -629,21 +621,6 @@ declare -r BUILDBLOCK_DIR IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}" module_name=${fname[${#fname[@]}-2]} -# -# are we bootstrapping? If yes, go for it... -# -if [[ "${opt_bootstrap}" == 'yes' ]]; then - declare ol_name='base' - declare ol_type='' - declare ol_mod_root="${pm_root}" - declare ol_inst_root="${pm_root}" - pbuild.bootstrap "${module_name}" "${versions[0]}" 'stable' - exit $? -fi - -# -# else -# for version in "${versions[@]}"; do build_modules "${module_name}" "${version}" "${opt_with_modules[@]}" done From c428887a8e91f50ea43524b42eb37050e2d1bc3f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 27 Jun 2022 19:22:29 +0200 Subject: [PATCH 54/97] major review/rewrite of the build script --- 40-build-modules | 79 ---- build | 644 ++++++++++++-------------- config/Pmodules.yaml | 7 - config/modbuild.conf.in | 13 - config/versions.conf | 7 - 10-build-bash => recipes/010-bash | 30 +- 20-build-tcl => recipes/020-tcl | 19 +- 30-build-tcllib => recipes/030-tcllib | 24 +- recipes/040-modules | 59 +++ 9 files changed, 379 insertions(+), 503 deletions(-) delete mode 100755 40-build-modules delete mode 100644 config/Pmodules.yaml delete mode 100644 config/modbuild.conf.in delete mode 100644 config/versions.conf rename 10-build-bash => recipes/010-bash (60%) rename 20-build-tcl => recipes/020-tcl (86%) rename 30-build-tcllib => recipes/030-tcllib (73%) create mode 100755 recipes/040-modules diff --git a/40-build-modules b/40-build-modules deleted file mode 100755 index 73ee00d..0000000 --- a/40-build-modules +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -# -# https://core.tcl-lang.org -# - -set -x - -P=modules -V=${MODULES_VERSION:-3.2.10.1} - -FNAME="$P-$V.tar.gz" -DOWNLOAD_URL="https://amas.web.psi.ch/Downloads/$P/$P-$V.tar.bz2" - -PREFIX="${PMODULES_HOME}" -SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" -SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" -BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" - -PATH="${PREFIX}/bin:${PATH}" - -trap "pb_exit" EXIT - -# download -test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} - -# unpack -mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} -tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} - -# configure -mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} -case $(uname -s) in - Linux ) - declare -x LIBS="-lz -lpthread" - ;; - Darwin ) - declare -x LIBS="-lz -framework CoreFoundation" - ;; - * ) - echo "Oops: unsupported OS!" 1>&2 - exit ${PB_ERR_SYSTEM} - ;; -esac -CPPFLAGS="-DUSE_INTERP_ERRORLINE" \ -"${SRC_DIR}"/configure \ - --prefix="${PREFIX}" \ - --exec-prefix="${PREFIX}" \ - --with-module-path="${PREFIX%%/Tools*}/Tools/${PMODULES_MODULEFILES_DIR}" \ - --with-tcl="${PREFIX}/lib" \ - --without-x \ - --disable-versioning \ - || exit ${PB_ERR_CONFIGURE} - -# compile & install -make -j ${NJOBS} || exit ${PB_ERR_MAKE} -make install || exit ${PB_ERR_INSTALL} - -# post-install -mkdir -p "${PREFIX}/share/man/man1" -mkdir -p "${PREFIX}/share/man/man4" -rm -v "${PREFIX}/Modules/bin/add.modules" -rm -v "${PREFIX}/Modules/bin/mkroot" -rm -rfv "${PREFIX}/Modules/modulefiles" -mv -v "${PREFIX}/Modules/share/man/man1/module.1" "${PREFIX}/share/man/man1" -mv -v "${PREFIX}/Modules/share/man/man4/modulefile.4" "${PREFIX}/share/man/man4" -rmdir "${PREFIX}/Modules/bin" -rmdir "${PREFIX}/Modules/share/man/man1" -rmdir "${PREFIX}/Modules/share/man/man4" -rmdir "${PREFIX}/Modules/share/man" -rmdir "${PREFIX}/Modules/share" -rmdir "${PREFIX}/Modules" -rm -f "${PREIX}/init/{ksh,perl.pm,python.py,ruby.rb,cmake,.modulespath}" -cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 - -# Local Variables: -# mode: sh -# sh-basic-offset: 8 -# tab-width: 8 -# End: diff --git a/build b/build index 12d737b..4c867ab 100755 --- a/build +++ b/build @@ -1,107 +1,84 @@ #!/usr/bin/env bash # -if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then - echo "BASH version >= 4 is required and must be available in PATH!" 1>&2 + +declare -rx VERSION='1.1.10' + +if (( "${BASH_VERSINFO[0]}" < 5 )); then + echo "BASH version 5.0 or newer is required and must be available in PATH!" 1>&2 exit 1 fi -declare BOOTSTRAP_DIR=$(dirname "$0") -source "${BOOTSTRAP_DIR}/Pmodules/libstd.bash" || { echo "Oops!" 1>&2; exit 42; } -source "${BOOTSTRAP_DIR}/Pmodules/libpbuild.bash" || { echo "Oops!" 1>&2; exit 42; } +set -o nounset +set -o pipefail +shopt -s nullglob -declare -r BOOTSTRAP_DIR=$(std::get_abspath "${BOOTSTRAP_DIR}") +declare -r BOOTSTRAP_DIR="$(cd "$(dirname "$0")" && pwd -P)" declare -r SRC_DIR="${BOOTSTRAP_DIR}/Pmodules" -declare -x PMODULES_ROOT -declare -x PMODULES_HOME -declare -x PMODULES_DISTFILESDIR -declare -x PMODULES_TMPDIR +source "${SRC_DIR}/libstd.bash" || { echo "Oops!" 1>&2; exit 42; } +source "${SRC_DIR}/libpbuild.bash" || { echo "Oops!" 1>&2; exit 42; } +declare -r PMOD_DIR="Tools/Pmodules/${VERSION}" +# config directory and file relative to install root declare -rx CONFIG_DIR='config' -declare -rx CONFIG_FILE='Pmodules.yaml' -declare -rx VERSIONS_CFG_FILE="${CONFIG_DIR}/versions.conf" - -# defaults -declare -rx DEFAULT_PMODULES_ROOT='/opt/psi' -declare -rx DEFAULT_DISTFILES_DIR="var/distfiles" -declare -rx DEFAULT_TMPDIR='var/tmp/${USER}' +declare -rx CONFIG_FILE="${CONFIG_DIR}/Pmodules.yaml" # directory where the required tools will be installed (like bash, tclsh, etc) declare -rx UTILBIN_DIR='libexec' -declare -ix PB_ERR_ARG=1 -declare -ix PB_ERR_SETUP=2 -declare -ix PB_ERR_SYSTEM=3 -declare -ix PB_ERR_DOWNLOAD=4 -declare -ix PB_ERR_UNTAR=5 -declare -ix PB_ERR_CONFIGURE=6 -declare -ix PB_ERR_MAKE=7 -declare -ix PB_ERR_PRE_INSTALL=8 -declare -ix PB_ERR_INSTALL=9 -declare -ix PB_ERR_POST_INSTALL=10 -declare -ix PB_ERR=255 -declare -ix NJOBS=4 - -pb_exit() { - local -i ec=$? - if [[ -n "${BASH_VERSION}" ]]; then - local -i n=${#BASH_SOURCE[@]} - local -r recipe_name="${BASH_SOURCE[n]}" - else - local -r recipe_name="${ZSH_ARGZERO}" - fi - echo -n "${recipe_name}: " - if (( ec == 0 )); then - echo "done!" - elif (( ec == PB_ERR_ARG )); then - echo "argument error!" - elif (( ec == PB_ERR_SETUP )); then - echo "error in setting everything up!" - elif (( ec == PB_ERR_SYSTEM )); then - echo "unexpected system error!" - elif (( ec == PB_ERR_DOWNLOAD )); then - echo "error in downloading the source file!" - elif (( ec == PB_ERR_UNTAR )); then - echo "error in un-taring the source file!" - elif (( ec == PB_ERR_CONFIGURE )); then - echo "error in configuring the software!" - elif (( ec == PB_ERR_MAKE )); then - echo "error in compiling the software!" - elif (( ec == PB_ERR_PRE_INSTALL )); then - echo "error in pre-installing the software!" - elif (( ec == PB_ERR_INSTALL )); then - echo "error in installing the software!" - elif (( ec == PB_ERR_POST_INSTALL )); then - echo "error in post-installing the software!" - else - echo "oops, unknown error!!!" - fi - exit ${ec} -} -export -f pb_exit > /dev/null +# defaults +declare -rx DEFAULT_INSTALL_ROOT='/opt/psi' +declare -rx DEFAULT_DISTFILES_DIR='var/distfiles' +declare -rx DEFAULT_TMP_DIR='var/tmp/${USER}' #----------------------------------------------------------------------------- # -# Get version from config file. -# -# The format of the config file is: -# -# -# -get_version() { - local -r name="$1" - echo $(awk "/^$1[[:blank:]]/ {print \$2}" "${VERSIONS_CFG_FILE}") +read_config_file() { + local fname="$1" + if [[ ! -r "${fname}" ]]; then + std::die 1 "Configuration file '${fname}' does not exist or is not readable!" + fi + + eval $(std::parse_yaml "${fname}" '') || \ + std::die 1 "Cannot read configuration file '${fname}'" + + declare -xg INSTALL_ROOT="${Overlays_base_install_root}" + if [[ -z "${INSTALL_ROOT}" ]]; then + std::die 1 "Error in configuration file '${fname}': install root not defined!" + fi + declare -xg PREFIX="${INSTALL_ROOT}/${PMOD_DIR}" + declare -xg DOWNLOADS_DIR="${DistfilesDir:-${INSTALL_ROOT}/${DEFAULT_DISTFILES_DIR}}" + declare -xg TMP_DIR="${TmpDir:-${INSTALL_ROOT}/${DEFAULT_TMP_DIR}}" } +#----------------------------------------------------------------------------- +# The next functions are used in the sub-commands, if an illegal option +# or argument has been passed. # -# the Pmodules version is defined in the config file -# -declare -rx PMODULES_VERSION=$(get_version 'Pmodules') +illegal_option(){ + local subcmd="$1" + local opt="$2" + std::die 1 \ + "%s: %s -- %s" \ + "$(basename $0) ${subcmd}" \ + "Illegal option" \ + "${opt}" +} + +illegal_arg(){ + local subcmd="$1" + local arg="$2" + std::die 1 \ + "%s: %s -- %s" \ + "$(basename $0) ${subcmd}" \ + "Illegal argument" \ + "${arg}" +} #----------------------------------------------------------------------------- -# -usage() { - local prog=$(basename $0) +# help for sub-command 'help' (usage) +build::help_help(){ + local prog="$(basename "$0")" echo " Usage: ${prog} help|configure|compile|install @@ -124,48 +101,36 @@ to get help for a specific sub-command. } #----------------------------------------------------------------------------- +# sub-command 'help' # -read_config_file() { - local fname="$1" - if [[ ! -r "${fname}" ]]; then - std::die 1 "Configuration file '${fname}' does not exist or is not readable!" - fi - - eval $(std::parse_yaml "${fname}" '') || std::die 1 "Cannot read configuration file '${fname}'" - - PMODULES_ROOT="${Overlays_base_install_root}" - PMODULES_DISTFILESDIR="${DistfilesDir}" - PMODULES_TMPDIR="${TmpDir}" - PMODULES_HOME="${PMODULES_ROOT}/Tools/Pmodules/${PMODULES_VERSION}" - - if [[ -z "${PMODULES_HOME}" ]]; then - std::die 1 "Error in configuration file '${fname}': PMODULE_HOME not defined!" +# print help for sub-commands +# +build::help() { + if (( $# == 0 )); then + build::help_help + else + case $1 in + configure|compile|install ) + build::help_$1 + ;; + help ) + build::help_help + ;; + -* ) + illegal_option 'help' "$1" + ;; + * ) + std::error "No such command -- $1" + build::help_help + ;; + esac fi } - - #----------------------------------------------------------------------------- +# help for sub-command 'configure' # -pmodules::help() { - if (( $# > 1 )); then - usage - fi - case $1 in - configure|compile|install ) - pmodules::help_$1 - ;; - * ) - echo -en "$1 - invalid sub-command!\n" 1>&2 - usage - ;; - esac -} - - -#----------------------------------------------------------------------------- -# -pmodules::help_configure() { +build::help_configure() { echo " Usage: $(basename $0) configure [OPTION...] @@ -173,21 +138,24 @@ Configure and setup a new Pmodules environment. You need permissions to write to the installation root. Options: ---prefix=DIR +--install_root=DIR Root of the Pmodules environment installation. Everything will be installed in a directory hierarchy with 'DIR' as prefix. - The default is '${DEFAULT_PMODULES_ROOT}'. + The default is '${DEFAULT_INSTALL_ROOT}'. --distfilesdir=DIR Directory where downloaded files are stored. - The default is 'PREFIX/${DEFAULT_DISTFILES_DIR}' in the + The default is '${DEFAULT_INSTALL_ROOT}/${DEFAULT_DISTFILES_DIR}' in the Pmodules root directory. --tmpdir=DIR Directory for temporary files. - The default is 'PREFIX/${DEFAULT_TMPDIR}' + The default is '${DEFAULT_INSTALL_ROOT}/${DEFAULT_TMP_DIR}' ---help +--force|-f + Override existing configuration. + +--help|-h|-? Print this help text. " 1>&2 @@ -195,110 +163,120 @@ Options: } #----------------------------------------------------------------------------- +# sub-command 'configure' # -pmodules::configure() { - local prefix="${PMODULES_ROOT:-${DEFAULT_PMODULES_ROOT}}" - local distfilesdir='' - local tmpdir='' - local config_file='' +build::configure() { local opt_force='no' while (( $# > 0 )); do case "$1" in - --prefix ) - prefix="$2" - shift 1 - ;; - --prefix=* ) - prefix="${1#*=}" + --install_root | --install_root=* ) + if [[ $1 == *=* ]]; then + INSTALL_ROOT="${1#*=}" + else + INSTALL_ROOT="$2" + shift 1 + fi ;; --distfilesdir ) - distfilesdir="$2" - shift 1 - ;; - --distfilesdir=* ) - distfilesdir="${1#*=}" + if [[ $1 == *=* ]]; then + DOWNLOADS_DIR="${1#*=}" + else + DOWNLOADS_DIR="$2" + shift 1 + fi ;; -f | --force ) opt_force='yes' ;; - --tmpdir ) - tmpdir="$2" - shift 1 - ;; - --tmpdir=* ) - tmpdir="${1#*=}" + --tmpdir | --tmpdir=* ) + if [[ $1 == *=* ]]; then + TMP_DIR="${1#*=}" + else + TMP_DIR="$2" + shift 1 + fi ;; --help | -h | -\? ) - pmodules::help_configure + build::help_configure ;; -* ) - std::die 1 "$1: illegal option" + illegal_option 'configure' "$1" ;; * ) - std::die 1 "$1: illegal argument to sub-command 'configure'." + illegal_arg 'configure' "$1" ;; esac shift 1 done - if [[ ! -d ${prefix} ]]; then - echo "The root directory '${prefix}' does not exist, trying to create it..." - if ! mkdir -p "${prefix}"; then - std::die 1 "Creating the root directory failed!\nAborting..." + : ${INSTALL_ROOT:=${DEFAULT_INSTALL_ROOT}} + : ${DOWNLOADS_DIR:=${INSTALL_ROOT}/${DEFAULT_DISTFILES_DIR}} + : ${TMP_DIR:=${INSTALL_ROOT}/${DEFAULT_TMP_DIR}} + PREFIX="${INSTALL_ROOT}/${PMOD_DIR}" + + #--- + # check/create the install root + if [[ ! -d ${INSTALL_ROOT} ]]; then + std::info "%s\n%s" \ + "The root directory '${INSTALL_ROOT}' does not exist!" \ + "Trying to create it..." + if ! mkdir -p "${INSTALL_ROOT}"; then + std::die 1 "%s\n%s" \ + "Creating the root directory failed!" \ + "Aborting..." fi fi - if [[ ! -w ${prefix} ]]; then - std::die 1 "The root directory '${prefix}' is not writable!\nAborting..." + if [[ ! -w ${INSTALL_ROOT} ]]; then + std::die 1 "%s\n%s" \ + "The root directory '${INSTALL_ROOT}' is not writable!" \ + "Aborting..." fi - mkdir -p "${prefix}/${CONFIG_DIR}" || \ + + #--- + # check/create YAML config file in install root + mkdir -p "${INSTALL_ROOT}/${CONFIG_DIR}" || \ std::die 1 "Aborting..." - local config_file="${prefix}/${CONFIG_DIR}/${CONFIG_FILE}" - if [[ "${opt_force}" == 'yes' ]]; then - rm -f "${config_file}" + local config_file="${INSTALL_ROOT}/${CONFIG_FILE}" + if [[ "${opt_force}" != 'yes' ]] && [[ -e "${config_file}" ]]; then + std::die 1 "%s\n%s" \ + "The Pmodules environment in '${INSTALL_ROOT}' has already been configured!" \ + "Use the option --force to override. Aborting..." fi - if [[ -e "${config_file}" ]]; then - std::die 1 "The Pmodules environment has already been configured! -Use the option --force to override.\nAborting..." - fi - - [[ -z "${distfilesdir}" ]] && distfilesdir="${prefix}/${DEFAULT_DISTFILES_DIR}" - [[ -z "${tmpdir}" ]] && tmpdir="${prefix}/${DEFAULT_TMPDIR}" - - sed_cmd="s:@INSTALL_ROOT@:${prefix}:g;" - sed_cmd+="s:@PMODULES_DISTFILESDIR@:${distfilesdir}:g;" - sed_cmd+="s:@PMODULES_TMPDIR@:${tmpdir}:g;" - sed_cmd+="s:@PMODULES_VERSION@:${PMODULES_VERSION}:g" + sed_cmd="s:@INSTALL_ROOT@:${INSTALL_ROOT}:g;" + sed_cmd+="s:@PMODULES_DISTFILESDIR@:${DOWNLOADS_DIR}:g;" + sed_cmd+="s:@PMODULES_TMPDIR@:${TMP_DIR}:g;" + sed_cmd+="s:@PMODULES_VERSION@:${VERSION}:g" - sed "${sed_cmd}" "${BOOTSTRAP_DIR}/${CONFIG_DIR}/${CONFIG_FILE}.in" \ + sed "${sed_cmd}" "${BOOTSTRAP_DIR}/${CONFIG_FILE}.in" \ > "${config_file}" || \ std::die 1 "Cannot create configuration file in Pmodules root\nAborting..." - sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" > "${SRC_DIR}/libpmodules.bash" - chmod 0755 "${SRC_DIR}/libpmodules.bash" - - read_config_file "${config_file}" - install -d -m 0755 "${PMODULES_HOME}/bin" - install -d -m 0755 "${PMODULES_HOME}/init" - install -d -m 0755 "${PMODULES_HOME}/lib" - install -d -m 0755 "${PMODULES_HOME}/libexec" - install -d -m 0755 "${PMODULES_ROOT}/Tools/modulefiles/Pmodules" - install -d -m 0755 "${PMODULES_ROOT}/Libraries/modulefiles" - install -d -m 0755 "${PMODULES_ROOT}/Programming/modulefiles" - install -d -m 0755 "${distfilesdir}" + #--- + # create basic directories + install -d -m 0755 \ + "${INSTALL_ROOT}/Tools/modulefiles/Pmodules" \ + "${INSTALL_ROOT}/Libraries/modulefiles" \ + "${INSTALL_ROOT}/Programming/modulefiles" \ + "${DOWNLOADS_DIR}" || \ + std::die 1 "%s" \ + "Creating basic directories failed\n" \ + "Aborting..." + #--- echo "Configuration:" - echo " root of Pmodules environment: ${prefix}" - echo " Pmodule prefix: ${PMODULES_HOME}" - echo " tmp directory: ${tmpdir}" - echo " store for downloaded files: ${distfilesdir}" + echo " root of Pmodules environment: ${INSTALL_ROOT}" + echo " Pmodule prefix: ${PREFIX}" + echo " tmp directory: ${TMP_DIR}" + echo " store for downloaded files: ${DOWNLOADS_DIR}" echo "Done..." } #----------------------------------------------------------------------------- +# help for sub-command 'compile' # -pmodules::help_compile() { +build::help_compile() { echo " Usage: $(basename $0) compile [OPTION...] @@ -306,22 +284,14 @@ Compile and install the required tools for a new Pmodules environment. ou need the permissions to write to the installation root. Options: ---prefix=DIR - Root of the Pmodules environment installation. The root of the - installation must be either specified via this option or the - environment variable PMODULES_ROOT. If this option is used and - the PMODULES_ROOT is set, the directory specified with this - option will be used. +--install_root=DIR + Root of the Pmodules environment installation. Everything will be + installed in a directory hierarchy with 'DIR' as prefix. + The default is '${DEFAULT_INSTALL_ROOT}'. ---debug - Enable verbose/debug output. - ---disable-cleanup ) +--disable-cleanup Do not cleanup the tmp directory after compilation and installation. ---force | -f - Force compilation. - --help Print this help text. @@ -329,252 +299,208 @@ Options: std::die 1 "" } -pmodules::compile() { - local prefix="${PMODULES_ROOT:-${DEFAULT_PMODULES_ROOT}}" +#----------------------------------------------------------------------------- +# sub-command 'compile' +# +build::compile() { local opt_force='no' - local config_file='' + local opt_cleanup='yes' while (( $# > 0 )); do case $1 in - --prefix ) - prefix="$2" - shift 1 - ;; - --prefix=* ) - prefix="${1#*=}" + --install_root | --install_root=* ) + if [[ $1 == *=* ]]; then + INSTALL_ROOT="${1#*=}" + else + INSTALL_ROOT="$2" + shift 1 + fi ;; --disable-cleanup ) - build_opts+=( "$1" ) - ;; - --debug ) - build_opts+=( "$1" ) - ;; - -f | --force ) - opt_force='yes' + opt_cleanup='no' ;; --help | -h | -\? ) - pmodules::help_compile + build::help_compile ;; -* ) - std::die 1 "$1: illegal option" + illegal_option 'compile' "$1" ;; * ) - std::die 1 "$1: illegal argument for sub-command 'compile'." + illegal_arg 'compile' "$1" ;; esac shift 1 done + : ${INSTALL_ROOT:=${DEFAULT_INSTALL_ROOT}} + PREFIX="${INSTALL_ROOT}/${PMOD_DIR}" - local config_file="${prefix}/${CONFIG_DIR}/${CONFIG_FILE}" - read_config_file "${config_file}" - install -d -m 0755 "${PMODULES_HOME}/bin" - install -d -m 0755 "${PMODULES_HOME}/init" - install -d -m 0755 "${PMODULES_HOME}/lib" - install -d -m 0755 "${PMODULES_HOME}/libexec" + read_config_file "${INSTALL_ROOT}/${CONFIG_FILE}" echo "Configuration:" - echo " root of Pmodules environment: ${prefix}" - echo " Pmodule prefix: ${PMODULES_HOME}" + echo " root of Pmodules environment: ${INSTALL_ROOT}" + echo " Pmodule prefix: ${PREFIX}" - if [[ "${OS}" == 'Darwin' ]]; then - if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then - : build getopt - fi + install -m 0755 -d "${PREFIX}"/{bin,init,lib,libexec} \ - if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/find" ]] || [[ ${opt_force} == 'yes' ]]; then - : build findutils - fi + for recipe in recipes/[0-9]*; do + "./${recipe}" "${PREFIX}" + done + if [[ "${opt_cleanup}" == 'yes' ]]; then + rm -rf "${TMP_DIR}/*" + rm -f "${PREFIX}/lib/libtcl*.a" + rm -rf "${PREFIX}/include" fi - - if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/bash" ]] || [[ ${opt_force} == 'yes' ]]; then - ./10-build-bash - fi - - if [[ ! -e "${PMODULES_HOME}/${UTILBIN_DIR}/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then - ./20-build-tcl - fi - - if [[ ! -e "${PMODULES_HOME}/lib/tcllib1.20" ]] || [[ ${opt_force} == 'yes' ]]; then - ./30-build-tcllib - fi - - if [[ ! -e "${PMODULES_HOME}/libexec/modulecmd.bin" ]] || [[ ${opt_force} == 'yes' ]]; then - ./40-build-modules - fi - #rm -rf "${PMODULES_HOME}/include" - #rm -rf "${PMODULES_HOME}/lib/"*.a - #rm -rf "${PMODULES_HOME}/lib/"*.la - #rm -rf "${PMODULES_HOME}/lib/bash" - #rm -rf "${PMODULES_HOME}/lib/pkginfo" - #rm -rf "${PMODULES_HOME}/man" - #rm -rf "${PMODULES_HOME}/share" - echo "Done..." } #----------------------------------------------------------------------------- +# help for sub-command 'install' # -pmodules::help_install() { +build::help_install() { echo " Usage: $(basename $0) install [OPTION...] Install a new Pmodules version. Options: ---prefix=DIR - Root of the Pmodules environment installation. The root of the - installation must be either specified via this option or the - environment variable PMODULES_ROOT. If this option is used and - the PMODULES_ROOT is set, the directory specified with this - option will be used. +--install_root=DIR + Root of the Pmodules environment installation. Everything will be + installed in a directory hierarchy with 'DIR' as prefix. + The default is '${DEFAULT_INSTALL_ROOT}'. --debug Enable verbose/debug output. ---disable-cleanup ) - Do not cleanup the tmp directory after compilation and installation. - ---force | -f - Force compilation. - ---help +--help|-h|-? Print this help text. " 1>&2 std::die 1 "" } -pmodules::install() { - if [[ -v PMOULES_HOME ]]; then - local prefix="${PMODULES_HOME%%/Tools*}" - else - local prefix="${DEFAULT_PMODULES_ROOT}" - fi - local config_file='' - local opt_force='no' - +#----------------------------------------------------------------------------- +# sub-command 'install' +# +build::install() { while (( $# > 0 )); do case $1 in --debug ) set -x ;; - --prefix ) - prefix="$2" - shift 1 - ;; - --prefix=* ) - prefix="${1#*=}" - ;; - -f | --force ) - opt_force='yes' + --install_root | --install_root=* ) + if [[ $1 == *=* ]]; then + INSTALL_ROOT="${1#*=}" + else + INSTALL_ROOT="$2" + shift 1 + fi ;; --help | -h | -\? ) - pmodules::help_install + build::help_install ;; -* ) - std::die 1 "$1: illegal option" + illegal_option 'install' "$1" ;; * ) - std::die 1 "$1: illegal argument to sub-command 'install'." + illegal_arg 'install' "$1" ;; esac shift 1 done - local config_file="${prefix}/${CONFIG_DIR}/${CONFIG_FILE}" - read_config_file "${config_file}" + : ${INSTALL_ROOT:=${DEFAULT_INSTALL_ROOT}} + PREFIX="${INSTALL_ROOT}/${PMOD_DIR}" + + read_config_file "${INSTALL_ROOT}/${CONFIG_FILE}" ### # # begin installation # echo "Configuration:" - echo " root of Pmodules environment: ${prefix}" - echo " Pmodule prefix: ${PMODULES_HOME}" - sed_cmd="s:@PMODULES_HOME@:${PMODULES_HOME}:g;" - sed_cmd+="s:@PMODULES_VERSION@:${PMODULES_VERSION}:g;" - sed_cmd+="s:@MODULES_VERSION@:${MODULES_VERSION}:g;" - sed_cmd+="s:@PMODULES_DISTFILESDIR@:${PMODULES_DISTFILESDIR}:g;" - sed_cmd+="s:@PMODULES_TMPDIR@:${PMODULES_TMPDIR}:g;" - sed_cmd+="s:@TCLSHDIR@:${PMODULES_HOME}/${UTILBIN_DIR}:g;" - sed_cmd+="s:@pager@::g;" - sed_cmd+="s:@pageropts@::g;" - sed_cmd+="s:@etcdir@:${PMODULES_ROOT}/${CONFIG_DIR}:g;" + echo " root of Pmodules environment: ${INSTALL_ROOT}" + echo " Pmodule prefix: ${PREFIX}" + sed_cmd+="s:@PMODULES_VERSION@:${VERSION}:g;" sed_cmd+="s:@VERSIONING@:#:g;" - sed_cmd+="s:@prefix@:${PMODULES_HOME}:g;" - sed_cmd+="s:@initdir@:${PMODULES_HOME}/init:g;" - sed_cmd+="s:@MODULES_RELEASE@:${PMODULES_VERSION}:g;" - sed_cmd+="s:@BASH@:${PMODULES_HOME}/${UTILBIN_DIR}/bash:g;" - sed_cmd+="s:@MODULECMD@:${PMODULES_HOME}/${UTILBIN_DIR}/modulecmd.bash:g;" - sed_cmd+="s:@MODMANAGE@:${PMODULES_HOME}/${UTILBIN_DIR}/modmanage.bash:g;" - - sed "${sed_cmd}" "${SRC_DIR}/profile.bash.in" > "${PMODULES_ROOT}/${CONFIG_DIR}/profile.bash-${PMODULES_VERSION}" - sed "${sed_cmd}" "${SRC_DIR}/profile.csh.in" > "${PMODULES_ROOT}/${CONFIG_DIR}/profile.csh-${PMODULES_VERSION}" - sed "${sed_cmd}" "${SRC_DIR}/profile.zsh.in" > "${PMODULES_ROOT}/${CONFIG_DIR}/profile.zsh-${PMODULES_VERSION}" - chmod 0644 "${PMODULES_ROOT}/${CONFIG_DIR}"/*-${PMODULES_VERSION} + sed_cmd+="s:@BASH@:${PREFIX}/${UTILBIN_DIR}/bash:g;" + sed_cmd+="s:@MODULECMD@:${PREFIX}/${UTILBIN_DIR}/modulecmd.bash:g;" - test -e "${PMODULES_ROOT}/${CONFIG_DIR}/profile.bash" || \ - install -m 0644 "$_-${PMODULES_VERSION}" "$_" + sed "${sed_cmd}" "${SRC_DIR}/profile.bash.in" \ + > "${INSTALL_ROOT}/${CONFIG_DIR}/profile.bash-${VERSION}" + sed "${sed_cmd}" "${SRC_DIR}/profile.csh.in" \ + > "${INSTALL_ROOT}/${CONFIG_DIR}/profile.csh-${VERSION}" + sed "${sed_cmd}" "${SRC_DIR}/profile.zsh.in" \ + > "${INSTALL_ROOT}/${CONFIG_DIR}/profile.zsh-${VERSION}" + chmod 0644 "${INSTALL_ROOT}/${CONFIG_DIR}"/*-${VERSION} - test -e "${PMODULES_ROOT}/${CONFIG_DIR}/profile.csh" || \ - install -m 0644 "$_-${PMODULES_VERSION}" "$_" + test -e "${INSTALL_ROOT}/${CONFIG_DIR}/profile.bash" || \ + install -m 0644 "$_-${VERSION}" "$_" - test -e "${PMODULES_ROOT}/${CONFIG_DIR}/profile.zsh" || \ - install -m 0644 "$_-${PMODULES_VERSION}" "$_" + test -e "${INSTALL_ROOT}/${CONFIG_DIR}/profile.csh" || \ + install -m 0644 "$_-${VERSION}" "$_" - sed "${sed_cmd}" "${SRC_DIR}/modulecmd.in" > "${PMODULES_HOME}/bin/modulecmd" - chmod 0755 "${PMODULES_HOME}/bin/modulecmd" - sed "${sed_cmd}" "${SRC_DIR}/modulecmd.bash.in" > "${PMODULES_HOME}/libexec/modulecmd.bash" - chmod 0755 "${PMODULES_HOME}/libexec/modulecmd.bash" - sed "${sed_cmd}" "${SRC_DIR}/modulecmd.tcl.in" > "${PMODULES_HOME}/libexec/modulecmd.tcl" - chmod 0755 "${PMODULES_HOME}/libexec/modulecmd.tcl" + test -e "${INSTALL_ROOT}/${CONFIG_DIR}/profile.zsh" || \ + install -m 0644 "$_-${VERSION}" "$_" - sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" > "${SRC_DIR}/libpmodules.bash" + sed "${sed_cmd}" "${SRC_DIR}/modulecmd.in" \ + > "${PREFIX}/bin/modulecmd" + chmod 0755 "${PREFIX}/bin/modulecmd" + sed "${sed_cmd}" "${SRC_DIR}/modulecmd.bash.in" \ + > "${PREFIX}/libexec/modulecmd.bash" + chmod 0755 "${PREFIX}/libexec/modulecmd.bash" + sed "${sed_cmd}" "${SRC_DIR}/modulecmd.tcl.in" \ + > "${PREFIX}/libexec/modulecmd.tcl" + chmod 0755 "${PREFIX}/libexec/modulecmd.tcl" + + sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" \ + > "${SRC_DIR}/libpmodules.bash" chmod 0755 "${SRC_DIR}/libpmodules.bash" - sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" > "${PMODULES_HOME}/lib/libpmodules.bash" - chmod 0755 "${PMODULES_HOME}/lib/libpmodules.bash" + sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" \ + > "${PREFIX}/lib/libpmodules.bash" + chmod 0755 "${PREFIX}/lib/libpmodules.bash" - sed "${sed_cmd}" "${SRC_DIR}/modbuild.in" > "${PMODULES_HOME}/bin/modbuild" - chmod 0755 "${PMODULES_HOME}/bin/modbuild" + sed "${sed_cmd}" "${SRC_DIR}/modbuild.in" \ + > "${PREFIX}/bin/modbuild" + chmod 0755 "${PREFIX}/bin/modbuild" - sed "${sed_cmd}" "${SRC_DIR}/modmanage.in" > "${PMODULES_HOME}/bin/modmanage" - chmod 0755 "${PMODULES_HOME}/bin/modmanage" - sed "${sed_cmd}" "${SRC_DIR}/modmanage.bash.in" > "${PMODULES_HOME}/libexec/modmanage.bash" - chmod 0755 "${PMODULES_HOME}/libexec/modmanage.bash" + sed "${sed_cmd}" "${SRC_DIR}/modmanage.in" \ + > "${PREFIX}/bin/modmanage" + chmod 0755 "${PREFIX}/bin/modmanage" + sed "${sed_cmd}" "${SRC_DIR}/modmanage.bash.in" \ + > "${PREFIX}/libexec/modmanage.bash" + chmod 0755 "${PREFIX}/libexec/modmanage.bash" - test -e "${PMODULES_ROOT}/${CONFIG_DIR}/Pmodules.yaml" || \ - install -m 0644 "$_" "${PMODULES_ROOT}/${CONFIG_DIR}" + test -e "${INSTALL_ROOT}/${CONFIG_FILE}" || \ + install -m 0644 "$_" "${INSTALL_ROOT}/${CONFIG_DIR}" - install -m 0755 "${SRC_DIR}/yq.$(uname -m)_$(uname -s)" "${PMODULES_HOME}/libexec/yq" - install -m 0644 "${SRC_DIR}/bash" "${PMODULES_HOME}/init" - install -m 0644 "${SRC_DIR}/bash_completion" "${PMODULES_HOME}/init" - install -m 0644 "${SRC_DIR}/csh" "${PMODULES_HOME}/init" - install -m 0644 "${SRC_DIR}/zsh" "${PMODULES_HOME}/init" + install -m 0755 "${SRC_DIR}/yq.$(uname -m)_$(uname -s)" "${PREFIX}/libexec/yq" + install -m 0644 "${SRC_DIR}/bash" "${PREFIX}/init" + install -m 0644 "${SRC_DIR}/bash_completion" "${PREFIX}/init" + install -m 0644 "${SRC_DIR}/csh" "${PREFIX}/init" + install -m 0644 "${SRC_DIR}/zsh" "${PREFIX}/init" - install -m 0644 "${SRC_DIR}/libpbuild.bash" "${PMODULES_HOME}/lib" - install -m 0644 "${SRC_DIR}/libpbuild_dyn.bash" "${PMODULES_HOME}/lib" - install -m 0644 "${SRC_DIR}/libstd.bash" "${PMODULES_HOME}/lib" - install -m 0755 -d "${PMODULES_HOME}/lib/Pmodules" - install -m 0644 "${SRC_DIR}/libmodules.tcl" "${PMODULES_HOME}/lib/Pmodules" + install -m 0644 "${SRC_DIR}/libpbuild.bash" "${PREFIX}/lib" + install -m 0644 "${SRC_DIR}/libpbuild_dyn.bash" "${PREFIX}/lib" + install -m 0644 "${SRC_DIR}/libstd.bash" "${PREFIX}/lib" + install -m 0755 -d "${PREFIX}/lib/Pmodules" + install -m 0644 "${SRC_DIR}/libmodules.tcl" "${PREFIX}/lib/Pmodules" { - PATH="${PMODULES_HOME}/${UTILBIN_DIR}:${PATH}" - cd "${PMODULES_HOME}/lib/Pmodules" + PATH="${PREFIX}/${UTILBIN_DIR}:${PATH}" + cd "${PREFIX}/lib/Pmodules" "${BOOTSTRAP_DIR}/mkindex.tcl" } - install -m 0644 "${SRC_DIR}/modulefile" "${PMODULES_ROOT}/Tools/modulefiles/Pmodules/${PMODULES_VERSION}" + install -m 0644 \ + "${SRC_DIR}/modulefile" \ + "${INSTALL_ROOT}/Tools/modulefiles/Pmodules/${VERSION}" echo "Done..." } #============================================================================= # - -declare -a build_opts=() -build_opts+=( '--bootstrap' ) - declare subcmd='' declare -a subcmd_args=() @@ -605,7 +531,7 @@ done [[ -n "${subcmd}" ]] || std::die 1 "Missing sub-command.\n\nUse 'build --help' to get help..." -pmodules::${subcmd} "${subcmd_args[@]}" +build::${subcmd} "${subcmd_args[@]}" # Local Variables: # mode: sh diff --git a/config/Pmodules.yaml b/config/Pmodules.yaml deleted file mode 100644 index 43ca4f8..0000000 --- a/config/Pmodules.yaml +++ /dev/null @@ -1,7 +0,0 @@ -DefaultGroups: Tools:Programming -DefaultReleaseStages: stable -ReleaseStages: unstable:stable:deprecated - -Overlays: - base: - install_root: @INSTALL_ROOT@ diff --git a/config/modbuild.conf.in b/config/modbuild.conf.in deleted file mode 100644 index 6292ea6..0000000 --- a/config/modbuild.conf.in +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# -: ${PMODULES_DISTFILESDIR:=@PMODULES_DISTFILESDIR@} -: ${PMODULES_TMPDIR:=@PMODULES_TMPDIR@} -export PMODULES_DISTFILESDIR -export PMODULES_TMPDIR - -declare -x PMODULES_HOME="@PMODULES_ROOT@/Tools/Pmodules/${PMODULES_VERSION}" - -declare -x DefaultGroups='Tools Programming' -declare ReleaseStages=':unstable:stable:deprecated:' -declare DefaultReleaseStages='stable' - diff --git a/config/versions.conf b/config/versions.conf deleted file mode 100644 index 2cbdbab..0000000 --- a/config/versions.conf +++ /dev/null @@ -1,7 +0,0 @@ -bash 5.1.16 -findutils 4.9.0 -getopt 1.1.6 -modules 3.2.10.1 -Pmodules 1.1.10 -Tcl 8.6.12 -tcllib 1.20 diff --git a/10-build-bash b/recipes/010-bash similarity index 60% rename from 10-build-bash rename to recipes/010-bash index 8ac20b7..4956448 100755 --- a/10-build-bash +++ b/recipes/010-bash @@ -2,29 +2,14 @@ # # https://www.gnu.org/software/bash/ # - -set -x - P=bash V=${BASH5_VERSION:-5.1.16} - FNAME="$P-$V.tar.gz" DOWNLOAD_URL="https://ftp.gnu.org/gnu/$P/${FNAME}" -PREFIX="${PMODULES_HOME}" -SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" -SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" -BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" - -trap "pb_exit" EXIT - -# download -test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} - -# unpack -mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} -tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} +source "$(dirname "$0")/librecipes.bash" +#--- # configure mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} loadablesdir="${PREFIX}/${UTILBIN_DIR}/builtins" \ @@ -33,15 +18,24 @@ loadablesdir="${PREFIX}/${UTILBIN_DIR}/builtins" \ --bindir="${PREFIX}/${UTILBIN_DIR}" \ || exit ${PB_ERR_CONFIGURE} -# compile & install +#--- +# compile make -j ${NJOBS} || exit ${PB_ERR_MAKE} make -C examples/loadables -j ${NJOBS} || exit ${PB_ERR_MAKE} + +#--- +# install make install || exit ${PB_ERR_INSTALL} +#--- # post-install rm -rf "${PREFIX}/include/bash" rm -rf "${PREFIX}/share/locale" +rm -rf "${PREFIX}/share/doc" +rm -rf "${PREFIX}/share/info" +rm -rf "${PREFIX}/share/man/man1/bash"* +#--- # Local Variables: # mode: shell-script-mode # sh-basic-offset: 8 diff --git a/20-build-tcl b/recipes/020-tcl similarity index 86% rename from 20-build-tcl rename to recipes/020-tcl index 6b54d94..0e380e0 100755 --- a/20-build-tcl +++ b/recipes/020-tcl @@ -2,29 +2,23 @@ # # https://www.tcl.tk # - -set -x - P=tcl V=${TCL_VERSION:-8.6.12} - FNAME="$P$V-src.tar.gz" DOWNLOAD_URL="https://prdownloads.sourceforge.net/tcl/${FNAME}" -PREFIX="${PMODULES_HOME}" -SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" -SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" -BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" - -trap "pb_exit" EXIT +source "$(dirname "$0")/librecipes.bash" +#--- # download test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} +#--- # unpack mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} +#--- # configure mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} case $(uname -s) in @@ -46,15 +40,20 @@ esac --enable-shared=no \ || exit ${PB_ERR_CONFIGURE} +#--- # compile & install make -j ${NJOBS} || exit ${PB_ERR_MAKE} make install || exit ${PB_ERR_INSTALL} +#--- # post-install { cd "${PREFIX}/${UTILBIN_DIR}" && rm -f tclsh && ln -fs tclsh${V%.*} tclsh; }; + +rm -rf "${PREFIX}/share/man/man1/tclsh.1" rm -rf "${PREFIX}/share/man/man3" rm -rf "${PREFIX}/share/man/mann" +#--- # Local Variables: # mode: shell-script-mode # sh-basic-offset: 8 diff --git a/30-build-tcllib b/recipes/030-tcllib similarity index 73% rename from 30-build-tcllib rename to recipes/030-tcllib index c81a0a0..6497227 100755 --- a/30-build-tcllib +++ b/recipes/030-tcllib @@ -2,28 +2,21 @@ # # https://core.tcl-lang.org # - -set -x - P=tcllib V=${TCLLIB_VERSION:-1.21} - FNAME="$P-$V.tar.gz" DOWNLOAD_URL="https://core.tcl-lang.org/tcllib/uv/${FNAME}" -PREFIX="${PMODULES_HOME}" -SRC_FILE="${PMODULES_DISTFILESDIR}/${FNAME}" -SRC_DIR="${PMODULES_TMPDIR}/$P-$V/src" -BUILD_DIR="${PMODULES_TMPDIR}/$P-$V/build" -trap "pb_exit" EXIT +source "$(dirname "$0")/librecipes.bash" +#--- # download test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} +#--- # unpack mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} - # configure mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} "${SRC_DIR}"/configure \ @@ -31,13 +24,24 @@ mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} --mandir="${PREFIX}/share/man" \ || exit ${PB_ERR_CONFIGURE} +#--- # compile & install make -j ${NJOBS} || exit ${PB_ERR_MAKE} make install || exit ${PB_ERR_INSTALL} +#--- # post-install rm -rf "${PREFIX}/share/man/mann" +rm -f "${PREFIX}/bin/dtplite" +rm -f "${PREFIX}/bin/mkdoc" +rm -f "${PREFIX}/bin/nns" +rm -f "${PREFIX}/bin/nnsd" +rm -f "${PREFIX}/bin/nnslog" +rm -f "${PREFIX}/bin/page" +rm -f "${PREFIX}/bin/pt" +rm -f "${PREFIX}/bin/tcldocstrip" +#--- # Local Variables: # mode: shell-script-mode # sh-basic-offset: 8 diff --git a/recipes/040-modules b/recipes/040-modules new file mode 100755 index 0000000..5c8dca5 --- /dev/null +++ b/recipes/040-modules @@ -0,0 +1,59 @@ +#!/bin/bash +# +# https://core.tcl-lang.org +# +P=modules +V=${MODULES_VERSION:-3.2.10.1} +FNAME="$P-$V.tar.gz" +DOWNLOAD_URL="https://amas.web.psi.ch/Downloads/$P/$P-$V.tar.bz2" + +source "$(dirname "$0")/librecipes.bash" + +#--- +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +case $(uname -s) in + Linux ) + declare -x LIBS="-lz -lpthread" + ;; + Darwin ) + declare -x LIBS="-lz -framework CoreFoundation" + ;; + * ) + echo "Oops: unsupported OS!" 1>&2 + exit ${PB_ERR_SYSTEM} + ;; +esac +CPPFLAGS="-DUSE_INTERP_ERRORLINE" \ +"${SRC_DIR}"/configure \ + --prefix="${PREFIX}" \ + --exec-prefix="${PREFIX}" \ + --with-module-path="${PREFIX%%/Tools*}/Tools/${PMODULES_MODULEFILES_DIR}" \ + --with-tcl="${PREFIX}/lib" \ + --without-x \ + --disable-versioning \ + || exit ${PB_ERR_CONFIGURE} + +#--- +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +#--- +# post-install +mkdir -p "${PREFIX}/share/man/man1" +mkdir -p "${PREFIX}/share/man/man4" +mv -v "${PREFIX}/Modules/share/man/man1/module.1" "${PREFIX}/share/man/man1" +mv -v "${PREFIX}/Modules/share/man/man4/modulefile.4" "${PREFIX}/share/man/man4" +mkdir -p "${PREFIX}/libexec" +cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 + +rm -rf "${PREFIX}/Modules" +rm -f "${PREIX}"/init/{ksh,perl.pm,python.py,ruby.rb,cmake,.modulespath} + +#--- +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# tab-width: 8 +# End: From 7af9f9af2fdcf9341aac3b3b77fccb087e6edb00 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 27 Jun 2022 19:30:56 +0200 Subject: [PATCH 55/97] build: some comments added --- build | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build b/build index 4c867ab..0f36b34 100755 --- a/build +++ b/build @@ -165,6 +165,9 @@ Options: #----------------------------------------------------------------------------- # sub-command 'configure' # +# Create basic directory hierachy and the configuration file in the given +# installation root directory +# build::configure() { local opt_force='no' @@ -302,6 +305,9 @@ Options: #----------------------------------------------------------------------------- # sub-command 'compile' # +# compile all required tools like bash, tclsh etc for a Pmodules module. +# The version is defined at the beginning of this file. +# build::compile() { local opt_force='no' local opt_cleanup='yes' @@ -381,6 +387,8 @@ Options: #----------------------------------------------------------------------------- # sub-command 'install' # +# Install Pmodules files. +# build::install() { while (( $# > 0 )); do case $1 in From 4f4b926ef31890c1b04ba128d3f05e96ee322e29 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 28 Jun 2022 14:24:57 +0200 Subject: [PATCH 56/97] modbuild: add checks whether all required hierarchical modules are loaded --- Pmodules/libpbuild.bash | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 6ff5762..6a98169 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -746,21 +746,38 @@ pbuild::make_all() { } [[ -n ${GROUP} ]] || std::die 1 \ - "${module_name}/${module_version}:" \ + "%s: %s" \ + "${module_name}/${module_version}" \ "group not set." - # define defaults if not set in configuration file - : ${Compiler_HIERARCHY:='${COMPILER}/${COMPILER_VERSION}'} - : ${CUDA_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} cuda/${CUDA_VERSION}'} - : ${MPI_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION}'} - : ${HDF5_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION} hdf5/${HDF5_VERSION}'} - : ${HDF5_serial_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} hdf5_serial/${HDF5_SERIAL_VERSION}'} + # define hierarchies + if [[ -v COMPILER_VERSION ]]; then + Compiler_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then + HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then + MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then + HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' + fi # evaluate local names=() local vname="${GROUP}_HIERARCHY" if [[ -n ${!vname} ]]; then names=( $(eval echo ${!vname}) ) + else + std::die 1 \ + "%s: %s" \ + "${module_name}/${module_version}" \ + "not all hierarchical dependencies loaded!" fi modulefile_dir=$(join_by '/' \ From ce6a17f39484a067fd6d9388ff3c1fc4959516e1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 28 Jun 2022 14:43:59 +0200 Subject: [PATCH 57/97] build-script: add librecipes and required macOS tools --- recipes/100-getopt | 52 +++++++++++++++++++++++ recipes/110-findutils | 43 +++++++++++++++++++ recipes/librecipes.bash | 91 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100755 recipes/100-getopt create mode 100755 recipes/110-findutils create mode 100644 recipes/librecipes.bash diff --git a/recipes/100-getopt b/recipes/100-getopt new file mode 100755 index 0000000..17979c8 --- /dev/null +++ b/recipes/100-getopt @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# https://frodo.looijaard.name/project/getopt +# +P=getopt +V=${GETOPT_VERSION:-1.1.6} +FNAME="$P-$V.tar.gz" +DOWNLOAD_URL="http://frodo.looijaard.name/system/files/software/$P/$P-$V.tar.gz" + +#--- +# build on macOS only +[[ $(uname -s) == 'Darwin' ]] || exit 0 + +#--- +source "$(dirname "$0")/librecipes.bash" + +#--- +# configure +# nothing to configure but we need gettext from Macports +if [[ ! -d '/opt/local/bin' ]] || [[ ! -x '/opt/local/bin/msgfmt' ]]; then + echo "gettext port from Macports is required to build 'getopt'!" 1>&2 + exit 1 +fi + +#--- +# compile +PATH+=':/opt/local/bin' +declare -x C_INCLUDE_PATH="${PREFIX}/include:/opt/local/include" +declare -x LDFLAGS="/opt/local/lib/libintl.a /opt/local/lib/libiconv.a -framework CoreFoundation" +declare -x LIBRARY_PATH="${PREFIX}/lib" + +make -e all || exit 1 + +#--- +# install +declare -x DESTDIR="${PREFIX}" +declare -x prefix='' +#PATH="${PREFIX}/${UTILBIN_DIR}:${PATH}" +make -e install + +#--- +# post-install +mv "${PREFIX}/bin/getopt" "${PREFIX}/${UTILBIN_DIR}" +rm -rf "${PREFIX}/man" +rm -rf "${PREFIX}/share/locale" + +#--- +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# tab-width: 8 +# End: diff --git a/recipes/110-findutils b/recipes/110-findutils new file mode 100755 index 0000000..3f2ef6c --- /dev/null +++ b/recipes/110-findutils @@ -0,0 +1,43 @@ +#!/bin/bash +# +# https://www.gnu.org/software/findutils/ +# +P=findutils +V=${FINDUTILS_VERSION:-4.9.0} +FNAME="$P-$V.tar.xz" +DOWNLOAD_URL="https://ftp.gnu.org/gnu/$P/${FNAME}" + +#--- +# build on macOS only +[[ $(uname -s) == 'Darwin' ]] || exit 0 + +#--- +source "$(dirname "$0")/librecipes.bash" + +#--- +# configure +mkdir -p "${BUILD_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +loadablesdir="${PREFIX}/${UTILBIN_DIR}/builtins" \ +"${SRC_DIR}/configure" \ + --prefix="${PREFIX}" \ + --bindir="${PREFIX}/${UTILBIN_DIR}" \ + || exit ${PB_ERR_CONFIGURE} + +#--- +# compile & install +make -j ${NJOBS} || exit ${PB_ERR_MAKE} +make install || exit ${PB_ERR_INSTALL} + +#--- +# post-install +rm -vf "${PREFIX}/share/man/man5/locatedb.5" +rm -vf "${PREFIX}/share/man/man1/updatedb.1" +rm -vf "${PREFIX}/share/man/man1/xargs.1" +rm -vf "${PREFIX}/share/man/man1/locate.1" +rm -vf "${PREFIX}/share/man/man1/find.1" + +#--- +# Local Variables: +# mode: shell-script-mode +# sh-basic-offset: 8 +# End: diff --git a/recipes/librecipes.bash b/recipes/librecipes.bash new file mode 100644 index 0000000..c696aaf --- /dev/null +++ b/recipes/librecipes.bash @@ -0,0 +1,91 @@ +#!/bin/bash + +set -x +set -o errexit +set -o pipefail +shopt -s nullglob + +if (( $# == 0 )); then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +PREFIX="$1" +if [[ ! -d ${PREFIX} ]]; then + echo "Destinstion directory '${PREFIX}' does not exist! Aborting..." 1>&2 + exit 2 +fi +TMP_DIR="${PMODULES_TMPDIR:-/var/tmp/${USER}}" +DOWNLOADS_DIR="${PMODULES_DISTFILESDIR:-${TMP_DIR}/Downloads}" +SRC_DIR="${TMP_DIR}/$P-$V/src" +BUILD_DIR="${TMP_DIR}/$P-$V/build" +SRC_FILE="${DOWNLOADS_DIR}/${FNAME}" + + +declare -ix PB_ERR_ARG=1 +declare -ix PB_ERR_SETUP=2 +declare -ix PB_ERR_SYSTEM=3 +declare -ix PB_ERR_DOWNLOAD=4 +declare -ix PB_ERR_UNTAR=5 +declare -ix PB_ERR_CONFIGURE=6 +declare -ix PB_ERR_MAKE=7 +declare -ix PB_ERR_PRE_INSTALL=8 +declare -ix PB_ERR_INSTALL=9 +declare -ix PB_ERR_POST_INSTALL=10 +declare -ix PB_ERR=255 +declare -ix NJOBS=4 + +pb_exit() { + local -i ec=$? + if [[ -n "${BASH_VERSION}" ]]; then + local -i n=${#BASH_SOURCE[@]} + local -r recipe_name="${BASH_SOURCE[n]}" + else + local -r recipe_name="${ZSH_ARGZERO}" + fi + echo -n "${recipe_name}: " + if (( ec == 0 )); then + echo "done!" + elif (( ec == PB_ERR_ARG )); then + echo "argument error!" + elif (( ec == PB_ERR_SETUP )); then + echo "error in setting everything up!" + elif (( ec == PB_ERR_SYSTEM )); then + echo "unexpected system error!" + elif (( ec == PB_ERR_DOWNLOAD )); then + echo "error in downloading the source file!" + elif (( ec == PB_ERR_UNTAR )); then + echo "error in un-taring the source file!" + elif (( ec == PB_ERR_CONFIGURE )); then + echo "error in configuring the software!" + elif (( ec == PB_ERR_MAKE )); then + echo "error in compiling the software!" + elif (( ec == PB_ERR_PRE_INSTALL )); then + echo "error in pre-installing the software!" + elif (( ec == PB_ERR_INSTALL )); then + echo "error in installing the software!" + elif (( ec == PB_ERR_POST_INSTALL )); then + echo "error in post-installing the software!" + else + echo "oops, unknown error!!!" + fi + exit ${ec} +} +#export -f pb_exit > /dev/null +trap "pb_exit" EXIT + +#--- +# download +mkdir -p "${DOWNLOADS_DIR}" || exit ${PB_ERR_SYSTEM} +test -r "${SRC_FILE}" || curl -L --output "$_" "${DOWNLOAD_URL}" || exit ${PB_ERR_DOWNLOAD} + +#--- +# unpack +mkdir -p "${SRC_DIR}" && cd "$_" || exit ${PB_ERR_SYSTEM} +tar --directory "${SRC_DIR}" --strip-components 1 -xv -f "${SRC_FILE}" || exit ${PB_ERR_UNTAR} + +#--- +# Local Variables: +# mode: shell-script-mode +# sh-basic-offset: 8 +# End: From cbd6de07698419c2ad1fdb72c81a0cc4c33292f0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 28 Jun 2022 16:18:05 +0200 Subject: [PATCH 58/97] modbuild: use bash from Pmodules module --- Pmodules/modbuild.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index adcb5be..b489513 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!@BASH@ # # The following build specific variables are set and used in libpbuild.bash: # ARGS From efff990f0aa9d543190865f8bd273cc4979c8530 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 28 Jun 2022 16:21:21 +0200 Subject: [PATCH 59/97] modbuild: bash 5 check removed --- Pmodules/modbuild.in | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b489513..a9896ce 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -34,14 +34,6 @@ source libstd.bash || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } -if (( ${BASH_VERSINFO[0]} < 5 )); then - std::info "bash >= 5 is required! You are running bash ${BASH_VERSION} ..." - std::info "Make sure that bash >= 5 is in your PATH." - std::info "bash >= 5 is available as Pmodule:" - std::info " module load System:bash" - std::die 3 "" -fi - std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ 'rm' 'sort' 'find' 'yq' From a5afd75a5e4f7d4f1ad9c7cf63ed3364a634ba4b Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:33:22 +0200 Subject: [PATCH 60/97] modbuild(function): no need to load System:bash any more --- Pmodules/bash | 1 - 1 file changed, 1 deletion(-) diff --git a/Pmodules/bash b/Pmodules/bash index 4e9044c..1adbc65 100644 --- a/Pmodules/bash +++ b/Pmodules/bash @@ -45,7 +45,6 @@ export -f module modbuild(){ ( - eval $("${PMODULES_HOME}/bin/modulecmd" bash load System:bash) "${PMODULES_HOME}/bin/modbuild" "$@" ) } From 7a7137a2e2b0191d415bed95c0272e65d98fc5bb Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:34:59 +0200 Subject: [PATCH 61/97] build-script: installation of libpbuild_dyn.bash removed --- build | 1 - 1 file changed, 1 deletion(-) diff --git a/build b/build index 0f36b34..7a5ab08 100755 --- a/build +++ b/build @@ -489,7 +489,6 @@ build::install() { install -m 0644 "${SRC_DIR}/zsh" "${PREFIX}/init" install -m 0644 "${SRC_DIR}/libpbuild.bash" "${PREFIX}/lib" - install -m 0644 "${SRC_DIR}/libpbuild_dyn.bash" "${PREFIX}/lib" install -m 0644 "${SRC_DIR}/libstd.bash" "${PREFIX}/lib" install -m 0755 -d "${PREFIX}/lib/Pmodules" install -m 0644 "${SRC_DIR}/libmodules.tcl" "${PREFIX}/lib/Pmodules" From cee0704a0bdc2eb6068daac24bc0e520a17528db Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:36:02 +0200 Subject: [PATCH 62/97] build-script: no need for a local libpmodules.bash any more --- build | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build b/build index 7a5ab08..99f8959 100755 --- a/build +++ b/build @@ -460,10 +460,6 @@ build::install() { > "${PREFIX}/libexec/modulecmd.tcl" chmod 0755 "${PREFIX}/libexec/modulecmd.tcl" - sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" \ - > "${SRC_DIR}/libpmodules.bash" - chmod 0755 "${SRC_DIR}/libpmodules.bash" - sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" \ > "${PREFIX}/lib/libpmodules.bash" chmod 0755 "${PREFIX}/lib/libpmodules.bash" From 6e125835b519de25538099c90822f2237d2cfaea Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:37:55 +0200 Subject: [PATCH 63/97] modubuild: cleanup --- Pmodules/modbuild.in | 64 ++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index a9896ce..d82c41a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -53,6 +53,7 @@ declare -r ARGS="$@" shopt -s nocaseglob shopt -s extglob +shopt -s nullglob declare ol_mod_root declare ol_inst_root @@ -157,7 +158,6 @@ MISCELLANEOUS OPTIONS: # versions to be build, '.*' or none means all declare -a versions=() -declare opt_build_config='Pmodules.yaml' declare opt_build_target='all' declare opt_dry_run='no' declare opt_enable_cleanup_build='yes' @@ -173,8 +173,6 @@ declare -a opt_with_modules=() declare opt_ol_name_or_dir='' -echo "parse_args()" - parse_args() { while (( $# > 0 )); do case $1 in @@ -204,13 +202,6 @@ parse_args() { --dry-run ) opt_dry_run='yes' ;; - --config ) - opt_build_config="$2" - shift 1 - ;; - --config=* ) - opt_build_config="${1#*=}" - ;; --enable-cleanup ) opt_enable_cleanup_build='yes' opt_enable_cleanup_src='yes' @@ -231,19 +222,21 @@ parse_args() { --disable-cleanup-src ) opt_enable_cleanup_src='no' ;; - --distdir ) - PMODULES_DISTFILESDIR="$2" - shift + --distdir | --distdir=* ) + if [[ $1 == *=* ]]; then + PMODULES_DISTFILESDIR="${1/--distdir=}" + else + PMODULES_DISTFILESDIR="$2" + shift + fi ;; - --distdir=* ) - PMODULES_DISTFILESDIR="${1/--distdir=}" - ;; - --tmpdir ) - PMODULES_TMPDIR="$2" - shift - ;; - --tmpdir=* ) - PMODULES_TMPDIR="${1/--tmpdir=}" + --tmpdir | --tmpdir=* ) + if [[ $1 == *=* ]]; then + PMODULES_TMPDIR="${1#--*=}" + else + PMODULES_TMPDIR="$2" + shift + fi ;; --system | --system=* ) if [[ $1 == *=* ]]; then @@ -261,20 +254,21 @@ parse_args() { shift fi ;; - --use-flags ) - USE_FLAGS="y:$2:" - shift + --use-flags | --use-flags=* ) + if [[ $1 == *=* ]]; then + USE_FLAGS=":${1#--*=}:" + else + USE_FLAGS=":$2:" + shift + fi ;; - --use-flags=* ) - USE_FLAGS=":${1/--use-flags=}:" - ;; - --with ) - opt_with_modules+=( "$2" ) - shift - ;; - --with=*/* ) - m="${1/--with=}" - opt_with_modules+=( ${m} ) + --with | --with=*/* ) + if [[ $1 == *=* ]]; then + opt_with_modules+=( "${1#--*=}" ) + else + opt_with_modules+=( "$2" ) + shift + fi ;; --prep | --configure | --compile | --install | --all ) opt_build_target=${1:2} From dcec3671d8becfa88b6a47bf03ecfd348a81381b Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:38:35 +0200 Subject: [PATCH 64/97] modbuild: argument parsing fixed --- Pmodules/modbuild.in | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index d82c41a..ace74f8 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -305,10 +305,13 @@ parse_args() { done if [[ -z ${BUILD_SCRIPT} ]]; then if [[ -r "${PWD}/build" ]]; then - BUILD_SCRIPT="${PWD}/build" - BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}") - else - std::die 1 "No build-block specified!" + if grep -q '#!.* modbuild' "${PWD}/build"; then + BUILD_SCRIPT="${PWD}/build" + BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}") + fi + fi + if [[ -z ${BUILD_SCRIPT} ]]; then + std::die 1 "Don't know what to build!" fi fi (( ${#versions[@]} > 0)) || versions+=( '.*' ) From 44e5edeebb48769e42d00f53bbd54c47fcbd4d04 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:43:10 +0200 Subject: [PATCH 65/97] modulecmd: reading the YAML config files reviewed --- Pmodules/libpmodules.bash.in | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 8a430a5..f118b8d 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -79,17 +79,24 @@ pm::read_config(){ # # Args: # $1 [upvar] result + local -n fnames="$1" + + # user defined via environment variable if [[ -v PMODULES_OVERLAYS_DEF ]]; then test -r "${PMODULES_OVERLAYS_DEF}" || \ std::die 3 \ "%s -- %s" \ "overlay definition file is not readable" \ "$_" - config_files+=("${PMODULES_OVERLAYS_DEF}") + fnames+=("${PMODULES_OVERLAYS_DEF}") fi + + # user defined if [[ -r "${HOME}/.Pmodules/Pmodules.yaml" ]]; then - config_files+=("${HOME}/.Pmodules/Pmodules.yaml") + fnames+=("${HOME}/.Pmodules/Pmodules.yaml") fi + + # system config file test -r "${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml" || \ std::die 3 \ "%s %s -- %s" \ @@ -97,14 +104,19 @@ pm::read_config(){ "does not exist or is not readable" \ "$_" - config_files+=("${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml") + fnames+=("${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml") } _get_ol_names(){ - ${yq} -Ne eval-all '. as $item ireduce ({}; . *+ $item) |.Overlays|keys()' "${config_files[@]}" | awk '{print $2}' + # + # get the names of all overlays + # + local -n fnames="$1" + ${yq} -Ne eval-all '. as $item ireduce ({}; . *+ $item) |.Overlays|keys()' \ + "${fnames[@]}" | awk '{print $2}' } - _get_config_files + _get_config_files config_files eval $(std::parse_yaml "${config_files[-1]}" 'cfg_') [[ -v cfg_DefaultGroups ]] && DefaultGroups="${cfg_DefaultGroups}" @@ -119,7 +131,7 @@ pm::read_config(){ # - modulefiles_root default so install_root # - the type defaults to ${ol_normal} # - Overlays=( $(_get_ol_names) ) + Overlays=( $(_get_ol_names config_files) ) local ol='' for ol in "${Overlays[@]}"; do eval $(${yq} -Ne eval-all \ From d408a73d896e23e58f9eda8b0749a428c3cce62f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:44:56 +0200 Subject: [PATCH 66/97] modulecmd/libpmodules.bash: interface of compute_group_depth() changed retrun result in a reference variable --- Pmodules/libpmodules.bash.in | 17 ++++++++++------- Pmodules/modulecmd.bash.in | 4 +++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index f118b8d..ea15fbc 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -38,18 +38,17 @@ print_help() { # $1: absolute path of a modulefile directory # compute_group_depth () { - local -r dir=$1 + local -n result="$1" + local -r dir="$2" test -d "${dir}" || return 1 local group=${dir%/*} local group=${group##*/} - [[ -n "${GroupDepths[${group}]}" ]] && return 0 - local -i depth=$(${find} "${dir}" -depth \( -type f -o -type l \) \ + local -i result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ -printf "%d" -quit 2>/dev/null) - (( depth-=2 )) + (( result-=2 )) # if a group doesn't contain a modulefile, depth is negativ # :FIXME: better solution? - (( depth < 0 )) && (( depth = 0 )) - GroupDepths[$group]=${depth} + (( result < 0 )) && (( result = 0 )) } # @@ -62,9 +61,13 @@ scan_groups () { local ol for ol in "$@"; do local mod_root="${OverlayInfo[${ol}:mod_root]}" + local -i depth local dir for dir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do - compute_group_depth "${dir}" + if [[ -n "${GroupDepths[${group}]}" ]]; then + compute_group_depth depth "${dir}" + GroupDepths[$group]=${depth} + fi Dir2OverlayMap[${dir%/${PMODULES_MODULEFILES_DIR}*}]="${ol}" done done diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index ea7f5c0..42e9471 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -484,7 +484,9 @@ subcommand_load() { local ol='' find_overlay_with_group ol "${group}" || return 1 local moduledir="${OverlayInfo[${ol}:mod_root]}/${group}/${PMODULES_MODULEFILES_DIR}" - compute_group_depth "${moduledir}" || return 1 + local -i depth + compute_group_depth depth "${moduledir}" || return 1 + GroupDepths[${group}]=${depth} g_env_must_be_saved='yes' } From 423a5dea672f60b9322d3540d41b1839992f4880 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 10:46:42 +0200 Subject: [PATCH 67/97] modbuild:libpbuild.bash: setting the full module name and prefix reviewed --- Pmodules/libpbuild.bash | 102 +++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 43 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 6a98169..8b6030b 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -739,10 +739,60 @@ pbuild::make_all() { # PREFIX # set_full_module_name_and_prefix() { - join_by() { - local IFS="$1" - shift - echo "$*" + do_simple_group(){ + modulefile_dir=$(join_by '/' \ + "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${module_name}") + modulefile_name="${modulefile_dir}/${module_version}" + PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}" + } + do_hierarchical_group(){ + join_by() { + local IFS="$1" + shift + echo "$*" + } + # define hierarchies + if [[ -v COMPILER_VERSION ]]; then + Compiler_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then + HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then + MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + fi + if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then + HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' + fi + + # evaluate + local names=() + local vname="${GROUP}_HIERARCHY" + if [[ -n ${!vname} ]]; then + names=( $(eval echo ${!vname}) ) + else + std::die 1 \ + "%s: %s" \ + "${module_name}/${module_version}" \ + "not all hierarchical dependencies loaded!" + fi + + modulefile_dir=$(join_by '/' \ + "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${names[@]}" \ + "${module_name}") + modulefile_name="${modulefile_dir}/${module_version}" + + PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}" + local -i i=0 + for ((i=${#names[@]}-1; i >= 0; i--)); do + PREFIX+="/${names[i]}" + done } [[ -n ${GROUP} ]] || std::die 1 \ @@ -750,47 +800,13 @@ pbuild::make_all() { "${module_name}/${module_version}" \ "group not set." - # define hierarchies - if [[ -v COMPILER_VERSION ]]; then - Compiler_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' - fi - if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then - HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' - HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' - fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then - MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' - MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' - fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then - HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' - HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' - HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' - fi - - # evaluate - local names=() - local vname="${GROUP}_HIERARCHY" - if [[ -n ${!vname} ]]; then - names=( $(eval echo ${!vname}) ) + local -i grp_depth + compute_group_depth grp_depth "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" + if (( grp_depth == 0 )); then + do_simple_group else - std::die 1 \ - "%s: %s" \ - "${module_name}/${module_version}" \ - "not all hierarchical dependencies loaded!" + do_hierarchical_group fi - - modulefile_dir=$(join_by '/' \ - "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ - "${names[@]}" \ - "${module_name}") - modulefile_name="${modulefile_dir}/${module_version}" - - PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}" - local -i i=0 - for ((i=${#names[@]}-1; i >= 0; i--)); do - PREFIX+="/${names[i]}" - done } #...................................................................... From 6bfd3f5cf23f88d2003596b9f0f1a0e93b6da40c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 17:50:53 +0200 Subject: [PATCH 68/97] init/bash: cleanup --- Pmodules/bash | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Pmodules/bash b/Pmodules/bash index 1adbc65..4b826b1 100644 --- a/Pmodules/bash +++ b/Pmodules/bash @@ -29,11 +29,9 @@ fi unset MODULE_VERSION unset MODULE_VERSION_STACK unset MODULESHOME -#unset PMODULES_ENV declare -x PMODULES_DIR="${PMODULES_HOME}" - ############################################################################# # implement module comand as shell function # @@ -44,9 +42,7 @@ module() { export -f module modbuild(){ - ( - "${PMODULES_HOME}/bin/modbuild" "$@" - ) + "${PMODULES_HOME}/bin/modbuild" "$@" } export -f modbuild From 5727cc9ef8affab8d3867c30aac4e7d2428ff043 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 17:52:26 +0200 Subject: [PATCH 69/97] modulecmd: bugfix in sub-cmd use UsedOverlays must be used instead of OverlayInfo --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 42e9471..4c4a362 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1514,7 +1514,7 @@ subcommand_use() { if [[ ! -v GroupDepths[${arg}] ]]; then # this scan is required if a new group has been # create inside an used overlay - scan_groups "${!OverlayInfo[@]}" + scan_groups "${UsedOverlays[@]}" g_env_must_be_saved='yes' fi if [[ -n ${GroupDepths[${arg}]} ]]; then From aa4838ed2e0017102b9c574994d7d9ddb40c0a76 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 17:54:04 +0200 Subject: [PATCH 70/97] modulecmd: bugfix if a Pmodule module have been loaded after loading a Pmodule module the module was not displayed with 'module list', this has been fixed --- Pmodules/modulecmd.bash.in | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 4c4a362..133a8b8 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -2757,8 +2757,21 @@ if [[ -n ${PMODULES_ENV} ]]; then eval "$("${base64}" -d <<< "${PMODULES_ENV}" 2>/dev/null)" fi if [[ -z ${Version} ]] || [[ ${Version} != ${PMODULES_VERSION} ]]; then + # this can only happen if the last command was + # module load Pmodules/${PMODULES_VERSION} + # + # the values these two variables must be saved before initialising + declare _tmp_loaded_modules_="${LOADEDMODULES}" + declare _tmp_lmfiles_="${_LMFILES_}" + pmodules_init - #g_env_must_be_saved='yes' + + # restore and export + LOADEDMODULES="${_tmp_loaded_modules_}" + _LMFILES_="${_tmp_lmfiles_}" + export_env \ + LOADEDMODULES \ + _LMFILES_ fi if (( ${#GroupDepths[@]} == 0 )); then From 5dbc1cf9a384846833e2c5ff50daf51967f68af2 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 18:19:28 +0200 Subject: [PATCH 71/97] modbuild: some cleanup --- Pmodules/modbuild.in | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index ace74f8..7ca8915 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -519,16 +519,12 @@ build_modules_yaml(){ local m local pattern="//" for m in "${with_modules[@]}"; do - if [[ -n $(awk "/${m%/*}[\/ ]/" "${fname}") ]]; then - pattern+=" && /${m//\//\\/}/" - fi + pattern+=" && /${m//\//\\/}/" done local -a versions yaml_get_versions versions "${fname}" "${name}/${version}" - echo versions=${versions[@]} for v in "${versions[@]}"; do - echo version=$v local -i n_variants yaml_get_num_variants n_variants "${fname}" "${v}" local -i i From 81a10f5b557d635f18fb70738fc3106d8688dc64 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 18:20:26 +0200 Subject: [PATCH 72/97] modbuild: new keyword 'systems' YAML variants files implemented The new keyword obsoletes the need of having differend variants files for different systems. --- Pmodules/modbuild.in | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 7ca8915..c392ce9 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -489,6 +489,19 @@ build_modules_yaml(){ _result='base' } + yaml_get_systems(){ + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=$(yq -Ne e ".\"${version}\"[${idx}].systems" \ + "${fname}" 2>/dev/null) + (( $? == 0 )) && return + _result=$(yq -Ne e ".systems" "${fname}" 2>/dev/null) + (( $? == 0 )) && return + _result='any' + } + yaml_get_dependencies(){ local -n _result="$1" local fname="$2" @@ -544,7 +557,21 @@ build_modules_yaml(){ yaml_get_dependencies deps "${fname}" "${v}" $i ol_inst_root="${OverlayInfo[${ol_name}:inst_root]}" ol_mod_root="${OverlayInfo[${ol_name}:mod_root]}" - + + local systems + yaml_get_systems systems "${fname}" "${v}" $i + # build module if + # - systems is any + # - opt_system is listed in systems + local build_it='no' + if [[ "${systems}" == 'any' ]]; then + build_it='yes' + fi + if [[ ,${systems}, == *,${opt_system},* ]]; then + build_it='yes' + fi + [[ ${build_it} == 'no' ]] && continue + if (( ${#deps[@]} > 0 )); then while read -a with_modules; do pbuild.build_module \ From d9a1771838f8c1da82b2ae1c362c57240be4c9cd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 18:23:20 +0200 Subject: [PATCH 73/97] modulecmd/libpmodules: bugfixes in scaning group depths --- Pmodules/libpmodules.bash.in | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index ea15fbc..f394df5 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -43,7 +43,7 @@ compute_group_depth () { test -d "${dir}" || return 1 local group=${dir%/*} local group=${group##*/} - local -i result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ + result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ -printf "%d" -quit 2>/dev/null) (( result-=2 )) # if a group doesn't contain a modulefile, depth is negativ @@ -59,12 +59,14 @@ compute_group_depth () { # scan_groups () { local ol + local depth for ol in "$@"; do local mod_root="${OverlayInfo[${ol}:mod_root]}" - local -i depth local dir for dir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do - if [[ -n "${GroupDepths[${group}]}" ]]; then + local group="${dir%/*}" + group="${group##*/}" + if [[ ! -v GroupDepths[${group}] ]]; then compute_group_depth depth "${dir}" GroupDepths[$group]=${depth} fi From 7d9e32f63363d2b8f6e960e0c3967188a0eb7afa Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 18:41:26 +0200 Subject: [PATCH 74/97] modbuild/libpbuild: some code polishing --- Pmodules/libpbuild.bash | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 8b6030b..92b28ce 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -740,9 +740,8 @@ pbuild::make_all() { # set_full_module_name_and_prefix() { do_simple_group(){ - modulefile_dir=$(join_by '/' \ - "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ - "${module_name}") + modulefile_dir="${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}/" + modulefile_dir+="${module_name}" modulefile_name="${modulefile_dir}/${module_version}" PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}" } @@ -783,7 +782,9 @@ pbuild::make_all() { fi modulefile_dir=$(join_by '/' \ - "${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${ol_mod_root}" \ + "${GROUP}" \ + "${PMODULES_MODULEFILES_DIR}" \ "${names[@]}" \ "${module_name}") modulefile_name="${modulefile_dir}/${module_version}" From 17d0d12de842b31bfe3637f91d1278d819a415d9 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 30 Jun 2022 22:44:42 +0200 Subject: [PATCH 75/97] Overlays doc added --- doc/Overlays.adoc | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 doc/Overlays.adoc diff --git a/doc/Overlays.adoc b/doc/Overlays.adoc new file mode 100644 index 0000000..d08004b --- /dev/null +++ b/doc/Overlays.adoc @@ -0,0 +1,72 @@ += Overlays +:TOC: +:sectnums: + +Please note: This document is work in progress! + +== What are Overlays and for what can they be used? + +== Overlay configuration + +=== System wide + +.A system wide configuration file `/opt/psi/config/Pmodules.yaml` +==== +.... +DefaultGroups: Tools:Programming +DefaultReleaseStages: stable +ReleaseStages: unstable:stable:deprecated +TmpDir: /opt/psi/var/tmp/${USER} +DistfilesDir: /opt/psi/var/distfiles + +Overlays: + base: + install_root: /opt/psi + devel: + install_root: /opt/psi + modulefiles_root: ${HOME}/modulefiles +.... +==== +=== User defined overlays + +Each user can define his own overlays in `$HOME/.Pmodules/Pmodules.yaml`. + +== Working with overlays + +== Building modules + +The old format of the variants file is simple but very limited and almost impossible to extend for new features. To overcome the limitations a new format using YAML for variants files has been introduced. For the time being both format are supported. But it is highly recommended to use the YAML format for new modules and to migrate existing variants files in the old format to the new. + +=== With a YAML variants file + +.A YAML variants file +==== +.... +overlay: base + +hdf5_serial/1.12.2: +- with: gcc/{5.5.0,6.5.0,7.5.0,10.2.0,10.3.0} + relstage: stable +- with: gcc/{8.5.0,9.5.0,11.3.0,12.1.0} + relstage: unstable + overlay: devel +.... +==== + +.YAML format of varaints file +.... +overlay: +systems: + +/: + - with: + dependencies: + relstate: + overlay: + systems: + ... +... +.... + + +=== With a legacy variants file From bf17ee607515dd5e3b6fbce236d2edb9eb54a850 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 1 Jul 2022 16:26:58 +0200 Subject: [PATCH 76/97] modbuild: cleanup modulefiles must be done after building all variants --- Pmodules/libpbuild.bash | 25 ------------------------- Pmodules/modbuild.in | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 92b28ce..4770e9e 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -995,29 +995,6 @@ pbuild::make_all() { install -m 0644 "${src}" "${modulefile_name}" } - cleanup_modulefiles(){ - local ol='' - for ol in "${Overlays[@]}"; do - [[ "${ol}" == "${ol_name}" ]] && continue - local mod_root="${OverlayInfo[${ol}:mod_root]}" - local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" - local fname="${dir}/${module_version}" - if [[ -e "${fname}" ]]; then - std::info "%s "\ - "${module_name}/${module_version}:" \ - "removing modulefile from overlay '${ol}' ..." - rm "${fname}" - fi - fname="${dir}/.release-${module_version}" - if [[ -e "${fname}" ]]; then - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "removing release file from overlay '${ol}' ..." - rm "${fname}" - fi - done - } install_release_file() { local -r release_file="${modulefile_dir}/.release-${module_version}" @@ -1227,7 +1204,6 @@ pbuild::make_all() { install_modulefile fi install_release_file - cleanup_modulefiles return $? fi if [[ "${module_release}" == 'deprecated' ]]; then @@ -1236,7 +1212,6 @@ pbuild::make_all() { "${module_name}/${module_version}:" \ "is deprecated, skiping!" install_release_file - cleanup_modulefiles return $? fi std::info \ diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c392ce9..990a943 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -436,7 +436,9 @@ build_modules_legacy() { } build_modules_yaml(){ - local fnames=() + local -a fnames=() + local -A mod_overlays=() + fnames+=( "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.${opt_system}.yaml" ) fnames+=( "${BUILDBLOCK_DIR}/files/${BNAME_VARIANTS}.yaml" ) fnames+=( '__zzzzz__' ) @@ -524,6 +526,30 @@ build_modules_yaml(){ done } + cleanup_modulefiles(){ + local ol='' + for ol in "${Overlays[@]}"; do + [[ -v mod_overlays[${ol}] ]] && continue + local mod_root="${OverlayInfo[${ol}:mod_root]}" + local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" + local fname="${dir}/${module_version}" + if [[ -e "${fname}" ]]; then + std::info "%s "\ + "${module_name}/${module_version}:" \ + "removing modulefile from overlay '${ol}' ..." + rm "${fname}" + fi + fname="${dir}/.release-${module_version}" + if [[ -e "${fname}" ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "removing release file from overlay '${ol}' ..." + rm "${fname}" + fi + done + } + local name="$1" local version="$2" shift 2 @@ -550,6 +576,7 @@ build_modules_yaml(){ else ol_name="${opt_overlay}" fi + mod_overlays[${ol_name}]=1 if [[ ! -v OverlayInfo[${ol_name}:inst_root] ]]; then std::die 3 "Overlay is not defined -- ${ol_name}" fi @@ -570,8 +597,11 @@ build_modules_yaml(){ if [[ ,${systems}, == *,${opt_system},* ]]; then build_it='yes' fi - [[ ${build_it} == 'no' ]] && continue - + if [[ ${build_it} == 'no' ]]; then + # we don't build, but we remember the overlay for + # cleaning up the modulefiles later. + continue + fi if (( ${#deps[@]} > 0 )); then while read -a with_modules; do pbuild.build_module \ @@ -584,6 +614,7 @@ build_modules_yaml(){ "${relstage}" fi done + cleanup_modulefiles done } From 9a8fd163727aab3e140bbfb811ec99d454217cde Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 1 Jul 2022 16:29:17 +0200 Subject: [PATCH 77/97] modbuild: use full path to call yq --- Pmodules/modbuild.in | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 990a943..9a0385c 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -452,7 +452,7 @@ build_modules_yaml(){ local -n _result="$1" local fname="$2" local version="$3" - _result=( $(yq -Ne e \ + _result=( $(${yq} -Ne e \ "with_entries(select(.key | test(\"^${version}\$\")))|keys" \ "${fname}" 2>/dev/null | awk '{print $2}') ) } @@ -461,7 +461,7 @@ build_modules_yaml(){ local -n _result="$1" local fname="$2" local version="$3" - _result=$(yq -Ne e ".\"${version}\"|length" \ + _result=$(${yq} -Ne e ".\"${version}\"|length" \ "${fname}" 2>/dev/null) if (( $? != 0 )); then _result=0 @@ -473,7 +473,7 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=$(yq -Ne e ".\"${version}\"[${idx}].relstage" \ + _result=$(${yq} -Ne e ".\"${version}\"[${idx}].relstage" \ "${fname}" 2>/dev/null) (( $? != 0 )) && relstage='unstable' || : } @@ -483,10 +483,10 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=$(yq -Ne e ".\"${version}\"[${idx}].overlay" \ + _result=$(${yq} -Ne e ".\"${version}\"[${idx}].overlay" \ "${fname}" 2>/dev/null) (( $? == 0 )) && return - _result=$(yq -Ne e ".overlay" "${fname}" 2>/dev/null) + _result=$(${yq} -Ne e ".overlay" "${fname}" 2>/dev/null) (( $? == 0 )) && return _result='base' } @@ -496,10 +496,10 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=$(yq -Ne e ".\"${version}\"[${idx}].systems" \ + _result=$(${yq} -Ne e ".\"${version}\"[${idx}].systems" \ "${fname}" 2>/dev/null) (( $? == 0 )) && return - _result=$(yq -Ne e ".systems" "${fname}" 2>/dev/null) + _result=$(${yq} -Ne e ".systems" "${fname}" 2>/dev/null) (( $? == 0 )) && return _result='any' } @@ -509,7 +509,7 @@ build_modules_yaml(){ local fname="$2" local version="$3" local idx="$4" - _result=( $(yq -Ne e ".\"${version}\"[${idx}]|(.with, .dependencies)" \ + _result=( $(${yq} -Ne e ".\"${version}\"[${idx}]|(.with, .dependencies)" \ "${fname}" 2>/dev/null) ) if (( $? != 0 )); then # neither .with nor .dependencies are set From 17ebf5c1ae667bc5725c169b1d9031888dea6b6b Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 1 Jul 2022 16:30:45 +0200 Subject: [PATCH 78/97] modbuild: code beautifying --- Pmodules/modbuild.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 9a0385c..a36ef48 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -49,11 +49,11 @@ source libpmodules.bash || \ std::die 3 "Oops: cannot source library -- '$_'" # save arguments, (still) required for building dependencies -declare -r ARGS="$@" +declare -r ARGS="$@" -shopt -s nocaseglob -shopt -s extglob -shopt -s nullglob +shopt -s nocaseglob +shopt -s extglob +shopt -s nullglob declare ol_mod_root declare ol_inst_root From 06491fde932ad154d2fbfcf5013d3025cefc0125 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 1 Jul 2022 16:35:15 +0200 Subject: [PATCH 79/97] modulecmd: issue with PMODULES_HOME fixed in sub-cmd purge the env.variable PMODULES_HOME must be preserved. --- Pmodules/modulecmd.bash.in | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 133a8b8..24f7dcb 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1895,13 +1895,15 @@ subcommand_purge() { # is a Pmodule module loaded? # if yes, save name in variable 'pmodule' - local pmodule='' + # We also have to save PMODULES_HOME since it will be + # unset while sourcing the shell's init script. IFS=':' local -a lmfiles=($_LMFILES_) unset IFS for f in "${lmfiles[@]}"; do if [[ $f == */${PMODULES_MODULEFILES_DIR}/Pmodules/* ]]; then - pmodule="${f##*/${PMODULES_MODULEFILES_DIR}/}" + local pm_home="${PMODULES_HOME}" + local pmodule="${f##*/${PMODULES_MODULEFILES_DIR}/}" break; fi done @@ -1931,13 +1933,14 @@ subcommand_purge() { echo "${error}" 1>&2 fi - if [[ -n "${pmodule}" ]]; then + if [[ -v pmodule ]]; then # reload a previously loaded Pmodule module # stderr is redirected to /dev/null, otherwise # we may get output like # 'unstable module has been loaded' + PMODULES_HOME="${pm_home}" + export_env PMODULES_HOME subcommand_load "${pmodule}" 2> /dev/null - # export_env PMODULES_HOME fi init_modulepath From c80d57d7b744abee129d01e0e5514a3d69ab3314 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 1 Jul 2022 16:37:39 +0200 Subject: [PATCH 80/97] modulefile: source shell init script only in mode 'load' --- Pmodules/modulefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pmodules/modulefile b/Pmodules/modulefile index 8dec4df..866568f 100644 --- a/Pmodules/modulefile +++ b/Pmodules/modulefile @@ -22,7 +22,7 @@ if { [module-info mode load] } { } remove-path C_INCLUDE_PATH "$PREFIX/include" remove-path CPLUS_INCLUDE_PATH "$PREFIX/include" -} -set shell [module-info shell] -puts "source \"$PREFIX/init/$shell\"" + set shell [module-info shell] + puts "source \"$PREFIX/init/$shell\"" +} \ No newline at end of file From cebae11b3886b1d0e98f8f19686cad021c7eeb4d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 10:22:48 +0200 Subject: [PATCH 81/97] cleanup unused files --- Pmodules/libpbuild_dyn.bash | 24 ----------------- Tools/findutils/build | 4 --- Tools/getopt/Makefile.patch | 12 --------- Tools/getopt/build | 54 ------------------------------------- 4 files changed, 94 deletions(-) delete mode 100644 Pmodules/libpbuild_dyn.bash delete mode 100755 Tools/findutils/build delete mode 100644 Tools/getopt/Makefile.patch delete mode 100755 Tools/getopt/build diff --git a/Pmodules/libpbuild_dyn.bash b/Pmodules/libpbuild_dyn.bash deleted file mode 100644 index 85f1d76..0000000 --- a/Pmodules/libpbuild_dyn.bash +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -eval "pbuild::pre_prep_${system}() { :; }" -eval "pbuild::pre_prep_${OS}() { :; }" -eval "pbuild::post_prep_${system}() { :; }" -eval "pbuild::post_prep_${OS}() { :; }" - -eval "pbuild::add_patch_${system}() { pbuild::add_patch \"\$@\"; }" -eval "pbuild::add_patch_${OD}() { pbuild::add_patch \"\$@\"; }" - -eval "pbuild::pre_configure_${system}() { :; }" -eval "pbuild::pre_configure_${OS}() { :; }" -eval "pbuild::post_configure_${system}() { :; }" -eval "pbuild::post_configure_${OS}() { :; }" - -eval "pbuild::pre_compile_${system}() { :; }" -eval "pbuild::pre_compile_${OS}() { :; }" -eval "pbuild::post_compile_${system}() { :; }" -eval "pbuild::post_compile_${OS}() { :; }" - -eval "pbuild::pre_install_${system}() { :; }" -eval "pbuild::pre_install_${OS}() { :; }" -eval "pbuild::post_install_${system}() { :; }" -eval "pbuild::post_install_${OS}() { :; }" diff --git a/Tools/findutils/build b/Tools/findutils/build deleted file mode 100755 index d63f114..0000000 --- a/Tools/findutils/build +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env modbuild - -pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.xz" -pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" diff --git a/Tools/getopt/Makefile.patch b/Tools/getopt/Makefile.patch deleted file mode 100644 index 6aa2403..0000000 --- a/Tools/getopt/Makefile.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -u getopt-1.1.6.orig//Makefile getopt-1.1.6/Makefile ---- getopt-1.1.6.orig//Makefile 2014-11-24 04:33:39.000000000 -0800 -+++ getopt-1.1.6/Makefile 2016-10-27 08:27:52.000000000 -0700 -@@ -61,7 +61,7 @@ - -$(RM) $(objects) $(binaries) - - getopt: $(objects) -- $(CC) $(LDFLAGS) -o $@ $(objects) -+ $(CC) -o $@ $(objects) $(LDFLAGS) - - install: getopt install_po - $(INSTALL) -m 755 -d $(DESTDIR)$(bindir) $(DESTDIR)$(man1dir) diff --git a/Tools/getopt/build b/Tools/getopt/build deleted file mode 100755 index 81d8433..0000000 --- a/Tools/getopt/build +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env modbuild -set -x - -pbuild::set_download_url "http://frodo.looijaard.name/system/files/software/getopt/getopt-1.1.6.tar.gz" -pbuild::compile_in_sourcetree - -pbuild::pre_prep_Linux() { - pbuild::add_patch "Makefile.patch" -} - -pbuild::configure() { - : -} - -pbuild::compile() { - declare -x C_INCLUDE_PATH="${PREFIX}/include" - declare -x LIBRARY_PATH="${PREFIX}/lib" - case ${OS} in - Linux ) - declare -x C_INCLUDE_PATH="${PREFIX}/include" - declare -x LIBRARY_PATH="${PREFIX}/lib" - declare -x LDFLAGS="-lintl -L${PREFIX}/lib" - ;; - Darwin ) - if [[ ! -d '/opt/local/bin' ]] || [[ ! -x '/opt/local/bin/msgfmt' ]]; then - std::die 1 "gettext port from Macports is required to build 'getopt'!" - fi - PATH+=':/opt/local/bin' - declare -x C_INCLUDE_PATH='/opt/local/include' - declare -x LDFLAGS="/opt/local/lib/libintl.a /opt/local/lib/libiconv.a -framework CoreFoundation" - ;; - esac - - make -e all || exit 1 - declare -x DESTDIR="${PREFIX}" - declare -x prefix='' || exit 1 - PATH="${PREFIX}/${UTILBIN_DIR}:${PATH}" - make -e install - mv "${PREFIX}/bin/getopt" "${PREFIX}/${UTILBIN_DIR}" -} - -pbuild::install() { - : -} - -pbuild::cleanup_build() { - : -} - -# Local Variables: -# mode: sh -# sh-basic-offset: 8 -# tab-width: 8 -# End: From dca348c4e3ce93ef881d3c1adda37f0ab85dbc8c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:03:19 +0200 Subject: [PATCH 82/97] accessing unset variables fixed --- Pmodules/libpbuild.bash | 12 ++++++++---- Pmodules/libstd.bash | 2 +- Pmodules/modbuild.in | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 4770e9e..f104874 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -301,7 +301,11 @@ pbuild::module_is_avail() { pbuild::set_download_url() { local -i _i=${#SOURCE_URLS[@]} SOURCE_URLS[_i]="$1" - SOURCE_NAMES[_i]="$2" + if (( $# == 2 )); then + SOURCE_NAMES[$_i]="$2" + else + SOURCE_NAMES[$_i]='' + fi } pbuild::set_sha256sum() { @@ -691,7 +695,7 @@ pbuild::make_all() { #...................................................................... check_supported_systems() { - [[ -z "${SUPPORTED_SYSTEMS}" ]] && return 0 + (( ${#SUPPORTED_SYSTEMS[@]} == 0 )) && return 0 for sys in "${SUPPORTED_SYSTEMS[@]}"; do [[ ${sys,,} == ${system,,} ]] && return 0 done @@ -702,7 +706,7 @@ pbuild::make_all() { #...................................................................... check_supported_os() { - [[ -z "${SUPPORTED_OS}" ]] && return 0 + (( ${#SUPPORTED_OS[@]} == 0 )) && return 0 for os in "${SUPPORTED_OS[@]}"; do [[ ${os,,} == ${OS,,} ]] && return 0 done @@ -713,7 +717,7 @@ pbuild::make_all() { #...................................................................... check_supported_compilers() { - [[ -z "${SUPPORTED_COMPILERS}" ]] && return 0 + (( ${#SUPPORTED_COMPILERS[@]} == 0 )) && return 0 for compiler in "${SUPPORTED_COMPILERS[@]}"; do [[ ${compiler,,} == ${COMPILER,,} ]] && return 0 done diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 95e70c7..6a52073 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -20,7 +20,7 @@ std::error() { } std::debug() { - [[ ${PMODULES_DEBUG} ]] || return 0 + [[ -v PMODULES_DEBUG ]] || return 0 std::log 2 "$@" } diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index a36ef48..819b521 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -171,6 +171,8 @@ declare opt_verbose='no' # array collecting all modules specified on the command line via '--with=module' declare -a opt_with_modules=() +declare BUILD_SCRIPT='' + declare opt_ol_name_or_dir='' parse_args() { From cd1788704d0dc195442ced887bfee24105fcd82a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:07:24 +0200 Subject: [PATCH 83/97] modbuild: issue with cleaning up modulefiles in overlays fixed --- Pmodules/libpbuild.bash | 29 +++++++++++++++++++++++++++++ Pmodules/modbuild.in | 7 +++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index f104874..bdaed40 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -967,6 +967,7 @@ pbuild::make_all() { fi install_modulefile install_release_file + cleanup_modulefiles cleanup_build cleanup_src std::info \ @@ -999,6 +1000,33 @@ pbuild::make_all() { install -m 0644 "${src}" "${modulefile_name}" } + cleanup_modulefiles(){ + local ol='' + for ol in "${Overlays[@]}"; do + local i + for ((i=0; i<${#mod_overlays}; i++ )); do + [[ "${ol}" == "{mod_overlays[i]}" ]] && continue 2 + done + [[ "${ol}" == "${ol_name}" ]] && continue + local mod_root="${OverlayInfo[${ol}:mod_root]}" + local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" + local fname="${dir}/${module_version}" + if [[ -e "${fname}" ]]; then + std::info "%s "\ + "${module_name}/${module_version}:" \ + "removing modulefile from overlay '${ol}' ..." + rm "${fname}" + fi + fname="${dir}/.release-${module_version}" + if [[ -e "${fname}" ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "removing release file from overlay '${ol}' ..." + rm "${fname}" + fi + done + } install_release_file() { local -r release_file="${modulefile_dir}/.release-${module_version}" @@ -1208,6 +1236,7 @@ pbuild::make_all() { install_modulefile fi install_release_file + cleanup_modulefiles return $? fi if [[ "${module_release}" == 'deprecated' ]]; then diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 819b521..98570c9 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -578,10 +578,14 @@ build_modules_yaml(){ else ol_name="${opt_overlay}" fi - mod_overlays[${ol_name}]=1 if [[ ! -v OverlayInfo[${ol_name}:inst_root] ]]; then std::die 3 "Overlay is not defined -- ${ol_name}" fi + mod_overlays[$i]="${ol_name}" + done + + for (( i=0; i Date: Mon, 4 Jul 2022 19:08:10 +0200 Subject: [PATCH 84/97] modbuild: handling of empty variant array fixed an entry /: without further information is legal and must be handled properly. --- Pmodules/modbuild.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 98570c9..b80afb1 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -568,6 +568,7 @@ build_modules_yaml(){ for v in "${versions[@]}"; do local -i n_variants yaml_get_num_variants n_variants "${fname}" "${v}" + (( n_variants == 0 )) && (( n_variants = 1 )) local -i i local -a deps=() local relstage From e9cfc1fee438d771548066910a1a2a2dc0c6f28d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:13:59 +0200 Subject: [PATCH 85/97] modbuild: a default release stage can now be set in the header --- Pmodules/modbuild.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b80afb1..0e39831 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -477,7 +477,10 @@ build_modules_yaml(){ local idx="$4" _result=$(${yq} -Ne e ".\"${version}\"[${idx}].relstage" \ "${fname}" 2>/dev/null) - (( $? != 0 )) && relstage='unstable' || : + (( $? == 0 )) && return + _result=$(${yq} -Ne e ".relstage" "${fname}" 2>/dev/null) + (( $? == 0 )) && return + _result='unstable' } yaml_get_overlay(){ From 9e63b4571bc821de0fdd4a900f90b35baaf50074 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:14:51 +0200 Subject: [PATCH 86/97] modbuild: querying dependencies fixed and old cleanup_modulefiles() removed --- Pmodules/modbuild.in | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 0e39831..ddec29a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -524,35 +524,10 @@ build_modules_yaml(){ # if one of .with, .dependencies is not set, the vaulue is # returned as 'null'. local -i i - for ((i=0; i<${#_result[@]}; i++)); do - if [[ ${_result[$i]} == 'null' ]]; then - unset _result[$i] - fi - done - } - - cleanup_modulefiles(){ - local ol='' - for ol in "${Overlays[@]}"; do - [[ -v mod_overlays[${ol}] ]] && continue - local mod_root="${OverlayInfo[${ol}:mod_root]}" - local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" - local fname="${dir}/${module_version}" - if [[ -e "${fname}" ]]; then - std::info "%s "\ - "${module_name}/${module_version}:" \ - "removing modulefile from overlay '${ol}' ..." - rm "${fname}" - fi - fname="${dir}/.release-${module_version}" - if [[ -e "${fname}" ]]; then - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "removing release file from overlay '${ol}' ..." - rm "${fname}" - fi + for i in "${!_result[@]}"; do + [[ ${_result[$i]} == 'null' ]] && unset -v '_result[$i]' done + _result=("${_result[@]}") } local name="$1" From 6eced654ebf77d78b7df648c3f55b68dfec46b1a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:15:43 +0200 Subject: [PATCH 87/97] modbuild: for the time being we don't set 'nounset' --- Pmodules/modbuild.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index ddec29a..b8c050c 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -51,6 +51,7 @@ source libpmodules.bash || \ # save arguments, (still) required for building dependencies declare -r ARGS="$@" +#set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob From 48ea630332d5d898800be5a0f6c2af6ec9f050b0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 8 Jul 2022 11:43:08 +0200 Subject: [PATCH 88/97] modbuild: fixed some issues if nounset is set --- Pmodules/libpbuild.bash | 21 +++++++++++++++------ Pmodules/modbuild.in | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index bdaed40..4feab50 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -483,7 +483,8 @@ pbuild::prep() { std::die 4 \ "%s " "${module_name}/${module_version}:" \ "sources for not found." - unpack "${source_fname}" "${SOURCE_UNPACK_DIRS[${source_fname##*/}]}" + local dir=${SOURCE_UNPACK_DIRS[${source_fname##*/}]:-''} + unpack "${source_fname}" "${dir}" done patch_sources # create build directory @@ -758,26 +759,34 @@ pbuild::make_all() { # define hierarchies if [[ -v COMPILER_VERSION ]]; then Compiler_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + else + unset Compiler_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' + else + unset HDF5_serial_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + else + unset MPI_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' + else + unset HDF5_HIERARCHY fi # evaluate local names=() - local vname="${GROUP}_HIERARCHY" - if [[ -n ${!vname} ]]; then - names=( $(eval echo ${!vname}) ) + local -n vname="${GROUP}"_HIERARCHY + if [[ -v vname ]]; then + names=( $(eval echo ${vname}) ) else std::die 1 \ "%s: %s" \ @@ -955,12 +964,12 @@ pbuild::make_all() { [[ "${OS}" == "Linux" ]] && post_install_linux install_doc install_pmodules_files - if [[ -n "${runtime_dependencies}" ]]; then + if [[ -v runtime_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_RDEPS}" \ "${runtime_dependencies[@]}" fi - if [[ -n "${install_dependencies}" ]]; then + if [[ -v install_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_IDEPS}" \ "${install_dependencies[@]}" diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b8c050c..c5813d1 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -51,7 +51,7 @@ source libpmodules.bash || \ # save arguments, (still) required for building dependencies declare -r ARGS="$@" -#set -o nounset +set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob From a7c50a94598658ac1621b7fbb746fa64473b49e4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 12 Jul 2022 16:21:21 +0200 Subject: [PATCH 89/97] cleanup, re-factoring, minor fixes, comments added --- Pmodules/libpbuild.bash | 1530 ++++++++++++++++++------------------ Pmodules/libstd.bash | 4 +- Pmodules/modbuild.in | 85 +- Pmodules/modulecmd.bash.in | 2 +- 4 files changed, 831 insertions(+), 790 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 4feab50..72f3fd3 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1,22 +1,5 @@ #!/bin/bash -#............................................................................. -# -# We need GNU versions of the following utilities. This code works -# well on Linux and Mac OS X with MacPorts. -# :FIXME: implement a smarter, portable solution. -# -shopt -s expand_aliases -unalias -a - -__path=$(which gsed 2>/dev/null || : ) -if [[ $__path ]]; then - alias sed=$__path -else - alias sed=$(which sed 2>/dev/null) -fi -unset __path - #............................................................................. # disable auto-echo feature of 'cd' unset CDPATH @@ -28,8 +11,9 @@ declare -r FNAME_RDEPS='.dependencies' declare -r FNAME_IDEPS='.install_dependencies' declare -r FNAME_BDEPS='.build_dependencies' -#............................................................................. -declare -A SOURCE_UNPACK_DIRS +# relative path of documentation +# abs. path is "${PREFIX}/${_docdir}/${module_name}" +declare -r _DOCDIR='share/doc' #............................................................................. # @@ -39,15 +23,104 @@ declare -A SOURCE_UNPACK_DIRS # #set -o errexit -error_handler() { +_error_handler() { local -i ec=$? std::die ${ec} "Oops" } +readonly -f _error_handler -trap "error_handler" ERR +trap "_error_handler" ERR -declare configure_with='undef' +#.............................................................................. +# +# write number of cores to stdout +# +_get_num_cores() { + case "${OS}" in + Linux ) + ${grep} -c ^processor /proc/cpuinfo + ;; + Darwin ) + ${sysctl} -n hw.ncpu + ;; + * ) + std::die 1 "OS ${OS} is not supported\n" + ;; + esac +} +readonly -f _get_num_cores + +#.............................................................................. +# global variables which can be set/overwritten by command line args +# and their corresponding functions +# +declare force_rebuild='' +pbuild.force_rebuild() { + force_rebuild="$1" +} +readonly -f pbuild.force_rebuild + +declare dry_run='' +pbuild.dry_run() { + dry_run="$1" +} +readonly -f pbuild.dry_run + +declare enable_cleanup_build='' +pbuild.enable_cleanup_build() { + enable_cleanup_build="$1" +} +readonly -f pbuild.enable_cleanup_build + +declare enable_cleanup_src='' +pbuild.enable_cleanup_src() { + enable_cleanup_src="$1" +} +readonly -f pbuild.enable_cleanup_src + +declare build_target='' +pbuild.build_target() { + build_target="$1" +} +readonly -f pbuild.build_target + +declare opt_update_modulefiles='' +pbuild.update_modulefiles() { + opt_update_modulefiles="$1" +} +readonly -f pbuild.update_modulefiles + +# number of parallel make jobs +declare -i JOBS=0 +pbuild.jobs() { + if (( $1 == 0 )); then + JOBS=$(_get_num_cores) + (( JOBS > 10 )) && JOBS=10 || : + else + JOBS="$1" + fi +} +readonly -f pbuild.jobs + +declare system='' +pbuild.system() { + system="$1" +} +readonly -f pbuild.system + +declare verbose='' +pbuild.verbose() { + verbose="$1" +} +readonly -f pbuild.verbose + + +############################################################################### +# +# function in the "namespace" (with prefix) 'pbuild::' can be used in +# build-scripts +# #.............................................................................. # @@ -81,120 +154,53 @@ pbuild::version_compare () { done return 0 } +readonly -f pbuild::version_compare +#.............................................................................. +# version less than +# +# return 0 if version passed in $1 is older then $2 +# pbuild::version_lt() { pbuild::version_compare "$1" "$2" (( $? == 2 )) } +readonly -f pbuild::version_lt +#.............................................................................. +# version less than or equal +# +# return 0 if version passed in $1 is older or equal then $2 +# pbuild::version_le() { pbuild::version_compare "$1" "$2" local -i exit_code=$? (( exit_code == 0 || exit_code == 2 )) } +readonly -f pbuild::version_le - +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 is newer then $2 +# pbuild::version_gt() { pbuild::version_compare "$1" "$2" (( $? == 1 )) local -i exit_code=$? (( exit_code == 0 || exit_code == 1 )) } +readonly -f pbuild::version_gt +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 and $2 are equal +# pbuild::version_eq() { pbuild::version_compare "$1" "$2" } - -#.............................................................................. -# -# The following variables are available in build-blocks and set read-only -# :FIXME: do we have to export them? - -# install prefix of module. -declare -x PREFIX='' - -declare -r OS=$(uname -s) - -pbuild::get_num_cores() { - case "${OS}" in - Linux ) - echo $(grep -c ^processor /proc/cpuinfo) - ;; - Darwin ) - echo $(sysctl -n hw.ncpu) - ;; - * ) - std::die 1 "OS ${OS} is not supported\n" - ;; - esac -} - -#.............................................................................. -# global variables which can be set/overwritten by command line args - -declare force_rebuild='' -pbuild.force_rebuild() { - force_rebuild="$1" -} - -declare dry_run='' -pbuild.dry_run() { - dry_run="$1" -} - -declare enable_cleanup_build='' -pbuild.enable_cleanup_build() { - enable_cleanup_build="$1" -} - -declare enable_cleanup_src='' -pbuild.enable_cleanup_src() { - enable_cleanup_src="$1" -} - -declare build_target='' -pbuild.build_target() { - build_target="$1" -} - -declare opt_update_modulefiles='' -pbuild.update_modulefiles() { - opt_update_modulefiles="$1" -} - -# number of parallel make jobs -declare -i JOBS=$(pbuild::get_num_cores) -pbuild.jobs() { - if (( $1 == 0 )); then - JOBS=$(pbuild::get_num_cores) - (( JOBS > 10 )) && JOBS=10 || : - else - JOBS="$1" - fi -} - -declare system='' -pbuild.system() { - system="$1" -} - -declare verbose='' -pbuild.verbose() { - verbose="$1" -} - - -# group this module is in (ex: 'Programming') -declare -x GROUP='' - -# name, version and release of module -declare -x module_name='' -declare -x module_version='' -declare -x module_release='' - -# relative path of documentation -# abs. path is "${PREFIX}/${_docdir}/${module_name}" -declare -r _DOCDIR='share/doc' +readonly -f pbuild::version_eq ############################################################################## # @@ -206,6 +212,7 @@ declare -r _DOCDIR='share/doc' pbuild::compile_in_sourcetree() { BUILD_DIR="${SRC_DIR}" } +readonly -f pbuild::compile_in_sourcetree ############################################################################## # @@ -218,6 +225,7 @@ pbuild::compile_in_sourcetree() { pbuild::supported_systems() { SUPPORTED_SYSTEMS+=( "$@" ) } +readonly -f pbuild::supported_systems ############################################################################## # @@ -230,6 +238,7 @@ pbuild::supported_systems() { pbuild::supported_os() { SUPPORTED_OS+=( "$@" ) } +readonly -f pbuild::supported_os ############################################################################## # @@ -242,6 +251,7 @@ pbuild::supported_os() { pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } +readonly -f pbuild::supported_compilers ############################################################################## # @@ -258,6 +268,7 @@ pbuild::add_to_group() { fi GROUP="$1" } +readonly -f pbuild::add_to_group ############################################################################## # @@ -269,6 +280,7 @@ pbuild::add_to_group() { pbuild::install_docfiles() { MODULE_DOCFILES+=("$@") } +readonly -f pbuild::install_docfiles ############################################################################## # @@ -277,45 +289,85 @@ pbuild::install_docfiles() { # # Arguments: # $1: module name -# $2: optional variable name to return release via upvar +# $2: optional variable name to return release # # Notes: -# The passed module name must be NAME/VERSION! +# The passed module name must be module/version! +# +# Exit codes: +# 0 if module/version is available +# 1 otherwise # pbuild::module_is_avail() { - local "$2" - local uvar="$2" - [[ -n "${uvar}" ]] || uvar="__unused__" local output=( $("${MODULECMD}" bash avail -a -m "$1" \ 2>&1 1>/dev/null) ) local i for (( i = 0; i < ${#output[@]}; i += 2 )); do if [[ "${output[$i]}" == "$1" ]]; then - std::upvar "${uvar}" "${output[i+1]}" + if (( $# > 1 )); then + local -n _result="$2" + _result="${output[i+1]}" + fi return 0 fi done return 1 } +readonly -f pbuild::module_is_avail +############################################################################## +# +# Set the download URL and name of downloaded file. +# +# Arguments: +# $1 download URL +# $2 optional file-name (of) pbuild::set_download_url() { local -i _i=${#SOURCE_URLS[@]} SOURCE_URLS[_i]="$1" - if (( $# == 2 )); then - SOURCE_NAMES[$_i]="$2" + if (( $# > 1 )); then + SOURCE_NAMES[$_i]="${2:-${1##*/}}" else - SOURCE_NAMES[$_i]='' + SOURCE_NAMES[$_i]="${1##*/}" fi } +readonly -f pbuild::set_download_url +############################################################################## +# +# Set hash sum for file. +# +# Arguments: +# $1 filen-name:hash-sum +# +# :FIXME: +# Maybe we should use a dictionary in the future. +# pbuild::set_sha256sum() { SOURCE_SHA256_SUMS+=("$1") } +readonly -f pbuild::set_sha256sum +############################################################################## +# +# Unpack file $1 in directory $2 +# +# Arguments: +# $1 file-name +# $2 directory +# pbuild::set_unpack_dir() { SOURCE_UNPACK_DIRS[$1]=$2 } +readonly -f pbuild::set_unpack_dir +############################################################################## +# +# Use this C-compiler +# +# Arguments: +# $1 C-compiler to use. +# pbuild::use_cc() { [[ -x "$1" ]] || std::die 3 \ "%s " "${module_name}/${module_version}:" \ @@ -323,14 +375,59 @@ pbuild::use_cc() { "'$1' is not an executable!" CC="$1" } +readonly -f pbuild::use_cc -pbuild::pre_prep() { - : +############################################################################### +# +pbuild::add_patch() { + [[ -z "$1" ]] && \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + PATCH_FILES+=( "$1" ) + PATCH_STRIPS+=( "$2" ) } +readonly -f pbuild::add_patch -pbuild::post_prep() { - : +############################################################################### +# +pbuild::set_default_patch_strip() { + [[ -n "$1" ]] || \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + + PATCH_STRIP_DEFAULT="$1" } +readonly -f pbuild::set_default_patch_strip + +############################################################################### +# +pbuild::use_flag() { + [[ "${USE_FLAGS}" =~ ":${1}:" ]] +} +readonly -f pbuild::use_flag + +############################################################################### +# +pbuild::add_configure_args() { + CONFIGURE_ARGS+=( "$@" ) +} +readonly -f pbuild::add_configure_args + +############################################################################### +# +pbuild::use_autotools() { + configure_with='autotools' +} +readonly -f pbuild::use_autotools + +############################################################################### +# +pbuild::use_cmake() { + configure_with='cmake' +} +readonly -f pbuild::use_cmake ############################################################################### # @@ -339,63 +436,58 @@ pbuild::post_prep() { pbuild::prep() { #...................................................................... # - # Find/download tarball for given module. + # download the source file if not already downloaded and validate + # checksum (if known). + # Abort on any error! # # Arguments: - # $1: store file name with upvar here - # $2: download URL - # $3: output filename (can be empty string) - # $4...: download directories + # $1 reference varibale to return result + # $2 download URL + # $3 save downloaded file with this name. If the empty + # string is passed, derive file name from URL + # $4... directories the source file might be already in. If the + # file does not exist in one of these directories, it + # is downloaded and stored in the first given directory. # - # Returns: - # 0 on success otherwise a value > 0 - # - download_with_curl() { - local -r output="$1" - local -r url="$2" - curl \ - -L \ - --output "${output}" \ - "${url}" - if (( $? != 0 )); then - curl \ - --insecure \ + download_source_file() { + download_with_curl() { + local -r output="$1" + local -r url="$2" + ${curl} \ + --location \ + --fail \ + --remove-on-error \ --output "${output}" \ "${url}" - fi - } + # :FIXME: How to handle insecure downloads? + #if (( $? != 0 )); then + # curl \ + # --insecure \ + # --output "${output}" \ + # "${url}" + #fi + } - check_hash_sum() { - local -r fname="$1" - local -r expected_hash_sum="$2" - local hash_sum='' + check_hash_sum() { + local -r fname="$1" + local -r expected_hash_sum="$2" + local hash_sum='' - if which 'sha256sum' 1>/dev/null; then - hash_sum=$(sha256sum "${fname}" | awk '{print $1}') - elif which 'shasum' 1>/dev/null; then - hash_sum=$(shasum -a 256 "${fname}" | awk '{print $1}') - else - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "Binary to compute SHA256 sum missing!" - fi - test "${hash_sum}" == "${expected_hash_sum}" || \ - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "hash-sum missmatch for file '%s'" "${fname}" - } + hash_sum=$(${sha256sum} "${fname}" | awk '{print $1}') + test "${hash_sum}" == "${expected_hash_sum}" || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "hash-sum missmatch for file '${fname}'!" + } - download_source_file() { - local "$1" - local var="$1" + local -n _result="$1" local -r url="$2" local fname="$3" shift 3 dirs+=( "$@" ) [[ -n "${fname}" ]] || fname="${url##*/}" - local expr='s/.*\(.tar.bz2\|.tbz2\|.tar.gz\|.tgz\|.tar.xz\|.zip\)/\1/' - local -r extension=$(echo ${fname} | sed "${expr}") local dir='' dirs+=( 'not found' ) for dir in "${dirs[@]}"; do @@ -403,19 +495,20 @@ pbuild::prep() { done if [[ "${dir}" == 'not found' ]]; then dir="${dirs[0]}" - local -r method="${url%:*}" - case "${method}" in - http | https | ftp ) - download_with_curl "${dir}/${fname}" "${url}" - ;; - * ) - std::die 4 \ - "%s " "${module_name}/${module_version}:" \ - "Error in download URL:" \ - "unknown download method '${method}'!" - ;; - esac + download_with_curl "${dir}/${fname}" "${url}" + (( $? == 0 )) || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "downloading source file '${fname}' failed!" fi + _result="${dir}/${fname}" + [[ -r "${_result}" ]] || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "source file '${_result}' is not readable!" + local sha256_sum='' local hash='' for hash in "${SOURCE_SHA256_SUMS[@]}"; do @@ -426,15 +519,13 @@ pbuild::prep() { if [[ -n "${sha256_sum}" ]]; then check_hash_sum "${dir}/${fname}" "${sha256_sum}" fi - std::upvar "${var}" "${dir}/${fname}" - [[ -r "${dir}/${fname}" ]] } unpack() { local -r file="$1" local -r dir="${2:-${SRC_DIR}}" - tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || { - rm -f "${file}" + ${tar} --directory="${dir}" -xv --strip-components 1 -f "${file}" || { + ${rm} -f "${file}" std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -451,7 +542,7 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" - patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ + ${patch} -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -470,7 +561,7 @@ pbuild::prep() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "Download source not set!" - mkdir -p "${PMODULES_DISTFILESDIR}" + ${mkdir} -p "${PMODULES_DISTFILESDIR}" local i=0 local source_fname for ((i = 0; i < ${#SOURCE_URLS[@]}; i++)); do @@ -483,59 +574,29 @@ pbuild::prep() { std::die 4 \ "%s " "${module_name}/${module_version}:" \ "sources for not found." - local dir=${SOURCE_UNPACK_DIRS[${source_fname##*/}]:-''} + local dir='' + local key="${SOURCE_URLS[i]##*/}" + if [[ -v SOURCE_UNPACK_DIRS[${key}] ]]; then + echo "dir specified" + dir="${SOURCE_UNPACK_DIRS[${SOURCE_URLS[i]##*/}]}" + else + echo "use SRC_DIR" + dir="${SRC_DIR}" + fi unpack "${source_fname}" "${dir}" done patch_sources # create build directory - mkdir -p "${BUILD_DIR}" -} - -pbuild::add_patch() { - [[ -z "$1" ]] && \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - PATCH_FILES+=( "$1" ) - PATCH_STRIPS+=( "$2" ) -} - -pbuild::set_default_patch_strip() { - [[ -n "$1" ]] || \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - - PATCH_STRIP_DEFAULT="$1" -} - -pbuild::use_flag() { - [[ "${USE_FLAGS}" =~ ":${1}:" ]] + ${mkdir} -p "${BUILD_DIR}" } ############################################################################### # +# Configure the software to be compiled. +# +# Arguments: +# none # -pbuild::pre_configure() { - : -} - -pbuild::set_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::add_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::use_autotools() { - configure_with='autotools' -} - -pbuild::use_cmake() { - configure_with='cmake' -} - pbuild::configure() { case "${configure_with}" in autotools ) @@ -567,6 +628,7 @@ pbuild::configure() { elif [[ -r "${SRC_DIR}/CMakeLists.txt" ]] && \ [[ "${configure_with}" == 'undef' ]] || \ [[ "${configure_with}" == "cmake" ]]; then + # note: in most/many cases a cmake module is used! cmake \ -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ "${CONFIGURE_ARGS[@]}" \ @@ -582,36 +644,38 @@ pbuild::configure() { fi } -pbuild::post_configure() { - : -} - -pbuild::pre_compile() { - : -} +############################################################################### +# +# Default compile function. +# +# Arguments: +# none +# pbuild::compile() { - make -j${JOBS} || \ + (( JOBS == 0 )) && JOBS=$(_get_num_cores) + ${make} -j${JOBS} || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } -pbuild::post_compile() { - : -} - -pbuild::pre_install() { - : -} - +############################################################################### +# +# Default install function. +# +# Arguments: +# none +# pbuild::install() { - make install || \ + ${make} install || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } +############################################################################### +# pbuild::install_shared_libs() { local -r binary="$1" local -r dstdir="$2" @@ -619,19 +683,19 @@ pbuild::install_shared_libs() { install_shared_libs_Linux() { local libs=( $(ldd "${binary}" | \ - awk "/ => \// && /${pattern}/ {print \$3}") ) + ${awk} "/ => \// && /${pattern}/ {print \$3}") ) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } install_shared_libs_Darwin() { # https://stackoverflow.com/questions/33991581/install-name-tool-to-update-a-executable-to-search-for-dylib-in-mac-os-x - local libs=( $(otool -L "${binary}" | \ - awk "/${pattern}/ {print \$1}")) + local libs=( $(${otool} -L "${binary}" | \ + ${awk} "/${pattern}/ {print \$1}")) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } @@ -640,7 +704,7 @@ pbuild::install_shared_libs() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "${binary}: does not exist or is not executable!" - mkdir -p "${dstdir}" + ${mkdir} -p "${dstdir}" case "${OS}" in Linux ) install_shared_libs_Linux @@ -651,48 +715,292 @@ pbuild::install_shared_libs() { esac } -pbuild::post_install() { - : -} - +############################################################################### # -# The 'do it all' function. +# This is the main entry function called by modbuild! # -pbuild::make_all() { - source "${BUILD_SCRIPT}" +pbuild.build_module() { + declare -gx module_name="$1" + declare -gx module_version="$2" + declare -gx module_release="$3" + shift 3 + with_modules=( "$@" ) - #set -o errexit - local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" - # module name including path in hierarchy and version - # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) - local modulefile_dir='' - local modulefile_name='' + # used in _make_all + declare -a runtime_dependencies=() + declare -a install_dependencies=() + #...................................................................... # - # To be able to set environment variables in one of the 'pbuild::TARGET' - # function we cannot use PIPE's like - # pbuild::configure | tee -a ... + # test whether a module is loaded or not # - rm -f "${logfile}" - if [[ "${verbose}" == 'yes' ]]; then - exec > >(tee -a "${logfile}") - else - exec > >(cat >> "${logfile}") - fi - exec 2> >(tee -a "${logfile}" >&2) + # Arguments: + # $1 module name + # + is_loaded() { + [[ :${LOADEDMODULES}: =~ :$1: ]] + } + #...................................................................... # - # everything set up? + # Initialise environment modules. # - [[ -n ${GROUP} ]] || \ - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "Module group not set! Aborting ..." + # Arguments: + # none + # + init_module_environment(){ + eval $( "${MODULECMD}" bash use unstable ) + eval $( "${MODULECMD}" bash use deprecated ) + eval $( "${MODULECMD}" bash purge ) + # :FIXME: this is a hack!!! + # shouldn't this be set in the build-script? + if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then + eval $( "${MODULECMD}" bash use Libraries ) + fi + if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then + eval $( "${MODULECMD}" bash use System ) + fi + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + unset CPP_INCLUDE_PATH + unset LIBRARY_PATH + unset LD_LIBRARY_PATH + unset DYLD_LIBRARY_PATH + + unset CFLAGS + unset CPPFLAGS + unset CXXFLAGS + unset LIBS + unset LDFLAGS + unset CC + unset CXX + unset FC + unset F77 + unset F90 + } + + #...................................................................... # - # helper functions + # Load build- and run-time dependencies. # + # Arguments: + # none + # + # Variables + # module_release set if defined in a variants file + # runtime_dependencies runtime dependencies from variants added + # + load_build_dependencies() { + + #.............................................................. + # + # build a dependency + # + # $1: name of module to build + # + # :FIXME: needs testing + # + build_dependency() { + find_build_script(){ + local p=$1 + local script=$(${find} "${BUILDBLOCK_DIR}/../.." \ + -path "*/$p/build") + std::get_abspath "${script}" + } + + local -r m=$1 + std::debug "${m}: module not available" + [[ ${dry_run} == yes ]] && \ + std::die 1 \ + "%s " \ + "${m}: module does not exist," \ + "cannot continue with dry run..." + + std::info "%s " \ + "$m: module does not exist, trying to build it..." + local args=( '' ) + set -- ${ARGS[@]} + while (( $# > 0 )); do + case $1 in + -j ) + args+=( "-j $2" ) + shift + ;; + --jobs=[0-9]* ) + args+=( $1 ) + ;; + -v | --verbose) + args+=( $1 ) + ;; + --with=*/* ) + args+=( $1 ) + ;; + esac + shift + done + + local buildscript=$(find_build_script "${m%/*}") + [[ -x "${buildscript}" ]] || \ + std::die 1 \ + "$m: build-block not found!" + if ! "${buildscript}" "${m#*/}" ${args[@]}; then + std::die 1 \ + "$m: oops: build failed..." + fi + } + + local m='' + for m in "${with_modules[@]}"; do + + # module name prefixes in dependency declarations: + # 'b:' this is a build dependency + # 'r:' this a run-time dependency, *not* required for + # building + # without prefix: this is a build and + # run-time dependency + if [[ "${m:0:2}" == "b:" ]]; then + m=${m#*:} # remove 'b:' + elif [[ "${m:0:2}" == "r:" ]]; then + m=${m#*:} # remove 'r:' + runtime_dependencies+=( "$m" ) + elif [[ "${m:0:2}" == "R:" ]]; then + m=${m#*:} # remove 'R:' + install_dependencies+=( "$m" ) + continue + else + runtime_dependencies+=( "$m" ) + fi + is_loaded "$m" && continue + + # 'module avail' might output multiple matches if module + # name and version are not fully specified or in case + # modules with and without a release number exist. + # Example: + # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list + # from 'module avail' and the full version should be set + # in the variants file, we look for the first exact + # match. + local release_of_dependency='' + if ! pbuild::module_is_avail "$m" release_of_dependency; then + build_dependency "$m" + pbuild::module_is_avail "$m" release_of_dependency || \ + std::die 6 "Oops" + fi + # should be set, just in case it is not... + : ${release_of_dependency:='unstable'} + + # for a stable module all dependencies must be stable + if [[ "${module_release}" == 'stable' ]] \ + && [[ "${release_of_dependency}" != 'stable' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + # for a unstable module no dependency must be deprecated + elif [[ "${module_release}" == 'unstable' ]] \ + && [[ "${release_of_dependency}" == 'deprecated' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + fi + + std::info "Loading module: ${m}" + eval $( "${MODULECMD}" bash load "${m}" ) + done + } + + init_build_environment() { + #...................................................................... + # + # parse the passed version string + # + # the following global variables will be set in this function: + # V_MAJOR + # V_MINOR + # V_PATCHLVL + # V_RELEASE + # USE_FLAGS + # + parse_version() { + local v="$1" + V_MAJOR='' # first number in version string + V_MINOR='' # second number in version string (or empty) + V_PATCHLVL='' # third number in version string (or empty) + V_RELEASE='' # module release (or empty) + : ${USE_FLAGS:=''} # architectures (or empty) + + local tmp='' + + if [[ "$v" =~ "_" ]]; then + tmp="${v#*_}" + USE_FLAGS+=":${tmp//_/:}:" + v="${v%%_*}" + fi + V_PKG="${v%%-*}" # version without the release number + if [[ $v == *-* ]]; then + V_RELEASE="${v#*-}" # release number + else + V_RELEASE='' + fi + case "${V_PKG}" in + *.*.* ) + V_MAJOR="${V_PKG%%.*}" + tmp="${V_PKG#*.}" + V_MINOR="${tmp%%.*}" + V_PATCHLVL="${tmp#*.}" + ;; + *.* ) + V_MAJOR="${V_PKG%.*}" + V_MINOR="${V_PKG#*.}" + ;; + * ) + V_MAJOR="${V_PKG}" + ;; + esac + + VERSIONS=() + if [[ -n ${V_RELEASE} ]]; then + VERSIONS+=( ${V_PKG}-${V_RELEASE} ) + fi + if [[ -n ${V_PATCHLVL} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ) + fi + if [[ -n ${V_MINOR} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR} ) + fi + VERSIONS+=( ${V_MAJOR} ) + } + + local -r module_name="$1" + local -r module_version="$2" + + SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" + BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" + + # P and V can be used in the build-script, so we have to set them here + P="${module_name}" + V="${module_version}" + parse_version "${module_version}" + declare -gx GROUP='' + declare -g PREFIX='' + + SOURCE_URLS=() + SOURCE_SHA256_SUMS=() + SOURCE_NAMES=() + declare -Ag SOURCE_UNPACK_DIRS=() + CONFIGURE_ARGS=() + SUPPORTED_SYSTEMS=() + SUPPORTED_OS=() + SUPPORTED_COMPILERS=() + PATCH_FILES=() + PATCH_STRIPS=() + PATCH_STRIP_DEFAULT='1' + MODULE_DOCFILES=() + configure_with='undef' + } # init_build_environment() #...................................................................... check_supported_systems() { @@ -762,19 +1070,23 @@ pbuild::make_all() { else unset Compiler_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v HDF5_SERIAL_VERSION ]]; then HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' else unset HDF5_serial_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]]; then MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' else unset MPI_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]] && \ + [[ HDF5_VERSION ]]; then HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' @@ -821,110 +1133,57 @@ pbuild::make_all() { else do_hierarchical_group fi - } + } # set_full_module_name_and_prefix #...................................................................... - # Select the modulefile to install. Modulefiles can be versioned like - # modulefile-10.2.0 - # modulefile-10.2 - # modulefile-10 - # modulefile - # the most specific modulefile will be selected. Example: - # For a version 10.2.1 the file moduelfile-10.2 would be selected. + # post-install. # # Arguments: - # $1 upvar to return the filename - # - # Used gloabal variables: - # VERSIONS - # BUILDBLOCK_DIR - # - find_modulefile() { - local "$1" - local fname='' - local modulefile='' - for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do - if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then - modulefile="${BUILDBLOCK_DIR}/${fname}" - break; - fi - done - std::upvar $1 "${modulefile}" - [[ -n "${modulefile}" ]] - } - - #...................................................................... - # non-redefinable post-install. Install: - # - documentation files as defined in the build-script - # - modulefile and file with release - # . + # none post_install() { #.............................................................. - # install the doc-files specified in the build-script + # post-install: + # - build-script + # - list of loaded modules while building + # - doc-files specified in the build-script # # Arguments: # none # install_doc() { - if [[ -z "${MODULE_DOCFILES}" ]]; then + local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "Installing documentation to ${docdir}" + ${install} -m 0755 -d "${docdir}" + ${install} -m0644 "${BUILD_SCRIPT}" "${docdir}" + "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ + ${grep} -v "Currently Loaded" > \ + "${docdir}/dependencies" || : + + if [[ ! -v MODULE_DOCFILES[0] ]]; then + # loop over version specific functions. In these function + # more MODULE_DOCFILES can be defined. + # :FIXME: maybe we find a better solution. for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do if typeset -F "$f" 2>/dev/null; then $f break fi done - fi - [[ -n "${MODULE_DOCFILES}" ]] || return 0 - local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "Installing documentation to ${docdir}" - install -m 0755 -d \ - "${docdir}" - install -m0644 \ + if [[ ! -v MODULE_DOCFILES[0] ]]; then + return 0 + fi + ${install} -m0644 \ "${MODULE_DOCFILES[@]/#/${SRC_DIR}/}" \ "${docdir}" return 0 } #.............................................................. - # install build-block files - # - modulefile - # - build-script - # - run-time and build dependencies - # in ${PREFIX}/share/${GROUP}/${module_name} - # - # Arguments: - # none - # - install_pmodules_files() { - local modulefile='' - find_modulefile modulefile || return 0 - - local -r target_dir="${PREFIX}/share/$GROUP/${module_name}" - mkdir -p "${target_dir}" - install -m0644 \ - "${BUILD_SCRIPT}" \ - "${target_dir}" - install -m0644 \ - "${modulefile}" \ - "${target_dir}" - #install -m 0755 \ - # -d "${target_dir}/files" - #install -m0644 \ - # "${variants_file}" \ - # "${target_dir}/files" - - local -r fname="${target_dir}/dependencies" - "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ - grep -v "Currently Loaded" > "${fname}" || : - } - - #.............................................................. - # write run time dependencies to file + # post-install: write file with required modules write_runtime_dependencies() { local -r fname="$1" shift @@ -947,8 +1206,8 @@ pbuild::make_all() { } #.............................................................. - # for Linux we need a special post-install to solve the - # multilib problem with LIBRARY_PATH on 64-bit systems + # post-install: for Linux we need a special post-install to + # solve the multilib problem with LIBRARY_PATH on 64-bit systems post_install_linux() { std::info \ "%s " \ @@ -960,10 +1219,10 @@ pbuild::make_all() { } #.............................................................. + # post-install cd "${BUILD_DIR}" [[ "${OS}" == "Linux" ]] && post_install_linux install_doc - install_pmodules_files if [[ -v runtime_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_RDEPS}" \ @@ -976,7 +1235,6 @@ pbuild::make_all() { fi install_modulefile install_release_file - cleanup_modulefiles cleanup_build cleanup_src std::info \ @@ -984,14 +1242,47 @@ pbuild::make_all() { "${module_name}/${module_version}:" \ "Done ..." return 0 - } + } # post_install #...................................................................... # Install modulefile in ${pm_root}/${GROUP}/modulefiles/... + # The modulefiles in the build-block can be + # versioned like + # modulefile-10.2.0 + # modulefile-10.2 + # modulefile-10 + # modulefile + # the most specific modulefile will be selected. Example: + # For a version 10.2.1 the file moduelfile-10.2 would be + # selected. # # Arguments # none + # + # Used gloabal variables: + # VERSIONS + # BUILDBLOCK_DIR + # modulefile_name + # install_modulefile() { + #.............................................................. + # Select the modulefile to install. + # + # Arguments: + # $1 upvar to return the filename + # + find_modulefile() { + local -n _modulefile="$1" + local fname='' + for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do + if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then + _modulefile="${BUILDBLOCK_DIR}/${fname}" + break; + fi + done + [[ -n "${_modulefile}" ]] + } + local src='' find_modulefile src if (( $? != 0 )); then @@ -1005,8 +1296,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "adding modulefile to overlay '${ol_name}' ..." - mkdir -p "${modulefile_dir}" - install -m 0644 "${src}" "${modulefile_name}" + ${mkdir} -p "${modulefile_dir}" + ${install} -m 0644 "${src}" "${modulefile_name}" } cleanup_modulefiles(){ @@ -1024,7 +1315,7 @@ pbuild::make_all() { std::info "%s "\ "${module_name}/${module_version}:" \ "removing modulefile from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi fname="${dir}/.release-${module_version}" if [[ -e "${fname}" ]]; then @@ -1032,7 +1323,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing release file from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi done } @@ -1065,7 +1356,7 @@ pbuild::make_all() { [[ "${BUILD_DIR}" == "${SRC_DIR}" ]] && return 0 { cd "/${BUILD_DIR}/.." || std::die 42 "Internal error" - [[ "$(pwd)" == "/" ]] && \ + [[ "$(${pwd})" == "/" ]] && \ std::die 1 \ "%s " "${module_name}/${module_version}:" \ "Oops: internal error:" \ @@ -1075,7 +1366,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "Cleaning up '${BUILD_DIR}'..." - rm -rf "${BUILD_DIR##*/}" + ${rm} -rf "${BUILD_DIR##*/}" }; return 0 } @@ -1098,67 +1389,66 @@ pbuild::make_all() { return 0 } - build_target() { - local dir="$1" # src or build directory, depends on target - local target="$2" # prep, configure, compile or install - - if [[ -e "${BUILD_DIR}/.${target}" ]] && \ - [[ ${force_rebuild} != 'yes' ]]; then - return 0 - fi - local targets=() - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) - targets+=( pbuild::pre_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) - targets+=( pbuild::pre_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) - targets+=( pbuild::pre_${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) - targets+=( pbuild::${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) - targets+=( pbuild::${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) - targets+=( pbuild::${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) - targets+=( pbuild::post_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) - targets+=( pbuild::post_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) - targets+=( pbuild::post_${target} ) - - for t in "${targets[@]}"; do - # We cd into the dir before calling the function - - # just to be sure we are in the right directory. - # - # Executing the function in a sub-process doesn't - # work because in some function global variables - # might/need to be set. - # - cd "${dir}" - if typeset -F "$t" 2>/dev/null; then - "$t" || \ - std::die 10 "Aborting..." - fi - done - touch "${BUILD_DIR}/.${target}" - } - #...................................................................... # build module ${module_name}/${module_version} - build_module() { + compile_and_install() { + build_target() { + local dir="$1" # src or build directory, depends on target + local target="$2" # prep, configure, compile or install + + if [[ -e "${BUILD_DIR}/.${target}" ]] && \ + [[ ${force_rebuild} != 'yes' ]]; then + return 0 + fi + local targets=() + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) + targets+=( pbuild::pre_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) + targets+=( pbuild::pre_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) + targets+=( pbuild::pre_${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) + targets+=( pbuild::${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) + targets+=( pbuild::${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) + targets+=( pbuild::${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) + targets+=( pbuild::post_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) + targets+=( pbuild::post_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) + targets+=( pbuild::post_${target} ) + + for t in "${targets[@]}"; do + # We cd into the dir before calling the function - + # just to be sure we are in the right directory. + # + # Executing the function in a sub-process doesn't + # work because in some function global variables + # might/need to be set. + # + cd "${dir}" + if typeset -F "$t" 2>/dev/null; then + "$t" || \ + std::die 10 "Aborting..." + fi + done + touch "${BUILD_DIR}/.${target}" + } # compile_and_install():build_target() + [[ ${dry_run} == yes ]] && std::die 0 "" - mkdir -p "${SRC_DIR}" - mkdir -p "${BUILD_DIR}" + ${mkdir} -p "${SRC_DIR}" + ${mkdir} -p "${BUILD_DIR}" + ${mkdir} -p "${PREFIX}" std::info \ "%s " \ "${module_name}/${module_version}:" \ "preparing sources ..." - # write stdout and stderr to logfile, stderr to terminal - # write all to logfile and terminal build_target "${SRC_DIR}" prep [[ "${build_target}" == "prep" ]] && return 0 @@ -1180,10 +1470,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "installing ..." - mkdir -p "${PREFIX}" build_target "${BUILD_DIR}" install - post_install - } + } # compile_and_install() remove_module() { if [[ -d "${PREFIX}" ]]; then @@ -1191,14 +1479,14 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing all files in '${PREFIX}' ..." - [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} + [[ "${dry_run}" == 'no' ]] && ${rm} -rf ${PREFIX} fi if [[ -e "${modulefile_name}" ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ "removing modulefile '${modulefile_name}' ..." - [[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}" + [[ "${dry_run}" == 'no' ]] && ${rm} -v "${modulefile_name}" fi local release_file="${modulefile_dir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then @@ -1208,34 +1496,70 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p "${modulefile_dir}" 2>/dev/null || : + ${rmdir} -p "${modulefile_dir}" 2>/dev/null || : } - ######################################################################## - # - # here we really start with make_all() - # + deprecate_module(){ + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "is deprecated, skiping!" + install_release_file + } - # setup module specific environment + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + ${with_modules:+with ${with_modules[@]}} \ + "building ..." + + init_module_environment + load_build_dependencies + init_build_environment "${module_name}" "${module_version}" + + source "${BUILD_SCRIPT}" + + # module name including path in hierarchy and version + # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) + local modulefile_dir='' + local modulefile_name='' + + # + # :FIXME: add comments what and why we are doing this. + # + local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + rm -f "${logfile}" + if [[ "${verbose}" == 'yes' ]]; then + exec > >(${tee} -a "${logfile}") + else + exec > >(${cat} >> "${logfile}") + fi + exec 2> >(${tee} -a "${logfile}" >&2) + + # the group must have been defined - otherwise we cannot continue + [[ -n ${GROUP} ]] || \ + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "Module group not set! Aborting ..." + + # check whether this module is supported check_supported_systems check_supported_os check_supported_compilers + # setup module name and prefix set_full_module_name_and_prefix + + # ok, finally we can start ... std::info \ "%s " \ "${module_name}/${module_version}:" \ ${with_modules:+build with ${with_modules[@]}} - if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then - # don't (re-)build the module, but - # - if the release stage has been changed to 'removed', - # remove the module. - # - if requested, update the modulefile. - # - if modulefile does not exist, install it. - # - update release stage. - if [[ "${module_release}" == 'removed' ]]; then - remove_module - return $? - fi + + if [[ "${module_release}" == 'removed' ]]; then + remove_module + elif [[ "${module_release}" == 'deprecated' ]]; then + deprecate_module + elif [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ @@ -1245,308 +1569,18 @@ pbuild::make_all() { install_modulefile fi install_release_file - cleanup_modulefiles - return $? - fi - if [[ "${module_release}" == 'deprecated' ]]; then + else std::info \ "%s " \ "${module_name}/${module_version}:" \ - "is deprecated, skiping!" - install_release_file - return $? + "start building ..." + compile_and_install + post_install fi - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "start building ..." - - build_module - return $? -} - -pbuild.init_env() { - #...................................................................... - # - # parse the passed version string - # - # the following global variables will be set in this function: - # V_MAJOR - # V_MINOR - # V_PATCHLVL - # V_RELEASE - # USE_FLAGS - # - parse_version() { - local v="$1" - V_MAJOR='' # first number in version string - V_MINOR='' # second number in version string (or empty) - V_PATCHLVL='' # third number in version string (or empty) - V_RELEASE='' # module release (or empty) - : ${USE_FLAGS:=''} # architectures (or empty) - - local tmp='' - - if [[ "$v" =~ "_" ]]; then - tmp="${v#*_}" - USE_FLAGS+=":${tmp//_/:}:" - v="${v%%_*}" - fi - V_PKG="${v%%-*}" # version without the release number - if [[ $v == *-* ]]; then - V_RELEASE="${v#*-}" # release number - else - V_RELEASE='' - fi - case "${V_PKG}" in - *.*.* ) - V_MAJOR="${V_PKG%%.*}" - tmp="${V_PKG#*.}" - V_MINOR="${tmp%%.*}" - V_PATCHLVL="${tmp#*.}" - ;; - *.* ) - V_MAJOR="${V_PKG%.*}" - V_MINOR="${V_PKG#*.}" - ;; - * ) - V_MAJOR="${V_PKG}" - ;; - esac - - VERSIONS=( ${V_MAJOR} ) - if [[ -n ${V_MINOR} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR} ${VERSIONS[@]} ) - fi - if [[ -n ${V_PATCHLVL} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ${VERSIONS[@]} ) - fi - if [[ -n ${V_RELEASE} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL}-${V_RELEASE} ${VERSIONS[@]} ) - fi - } - - local -r module_name="$1" - local -r module_version="$2" - - SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" - BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" - - # P and V can be used in the build-script, so we have to set them here - P="${module_name}" - V="${module_version}" - parse_version "${module_version}" - - SOURCE_URLS=() - SOURCE_SHA256_SUMS=() - SOURCE_NAMES=() - CONFIGURE_ARGS=() - SUPPORTED_SYSTEMS=() - SUPPORTED_OS=() - SUPPORTED_COMPILERS=() - PATCH_FILES=() - PATCH_STRIPS=() - PATCH_STRIP_DEFAULT='1' - MODULE_DOCFILES=() - configure_with='undef' -} - -pbuild.build_module() { - module_name="$1" - module_version="$2" - module_release="$3" - shift 3 - with_modules=( "$@" ) - - # used in pbuild::make_all - declare -a runtime_dependencies=() - declare -a install_dependencies=() - - #...................................................................... - # - # test whether a module is loaded or not - # - # $1: module name - # - is_loaded() { - [[ :${LOADEDMODULES}: =~ :$1: ]] - } - - #...................................................................... - # - # build a dependency - # - # $1: name of module to build - # - # :FIXME: needs testing - # - build_dependency() { - local -r m=$1 - std::debug "${m}: module not available" - local rels=( ${ReleaseStages//:/ } ) - [[ ${dry_run} == yes ]] && \ - std::die 1 \ - "%s " \ - "${m}: module does not exist," \ - "cannot continue with dry run..." - - std::info "%s " \ - "$m: module does not exist, trying to build it..." - local args=( '' ) - set -- ${ARGS[@]} - while (( $# > 0 )); do - case $1 in - -j ) - args+=( "-j $2" ) - shift - ;; - --jobs=[0-9]* ) - args+=( $1 ) - ;; - -v | --verbose) - args+=( $1 ) - ;; - --with=*/* ) - args+=( $1 ) - ;; - esac - shift - done - - find_build_script(){ - local p=$1 - local script=$(find "${BUILDBLOCK_DIR}/../.." -path "*/$p/build") - std::get_abspath "${script}" - } - local buildscript=$(find_build_script "${m%/*}") - [[ -x "${buildscript}" ]] || \ - std::die 1 \ - "$m: build-block not found!" - if ! "${buildscript}" "${m#*/}" ${args[@]}; then - std::die 1 \ - "$m: oops: build failed..." - fi - } - - #...................................................................... - # - # Load build- and run-time dependencies. - # - # Arguments: - # none - # - # Variables - # [r] module_release set if defined in a variants file - # runtime_dependencies runtime dependencies from variants added - # - load_build_dependencies() { - local m='' - for m in "${with_modules[@]}"; do - - # module name prefixes in dependency declarations: - # 'b:' this is a build dependency - # 'r:' this a run-time dependency, *not* required for - # building - # without prefix: this is a build and - # run-time dependency - if [[ "${m:0:2}" == "b:" ]]; then - m=${m#*:} # remove 'b:' - elif [[ "${m:0:2}" == "r:" ]]; then - m=${m#*:} # remove 'r:' - runtime_dependencies+=( "$m" ) - elif [[ "${m:0:2}" == "R:" ]]; then - m=${m#*:} # remove 'R:' - install_dependencies+=( "$m" ) - continue - else - runtime_dependencies+=( "$m" ) - fi - is_loaded "$m" && continue - - # 'module avail' might output multiple matches if module - # name and version are not fully specified or in case - # modules with and without a release number exist. - # Example: - # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list - # from 'module avail' and the full version should be set - # in the variants file, we look for the first exact - # match. - local release_of_dependency='' - if ! pbuild::module_is_avail "$m" release_of_dependency; then - build_dependency "$m" - pbuild::module_is_avail "$m" release_of_dependency || \ - std::die 6 "Oops" - fi - # should be set, just in case it is not... - : ${release_of_dependency:='unstable'} - - # for a stable module all dependencies must be stable - if [[ "${module_release}" == 'stable' ]] \ - && [[ "${release_of_dependency}" != 'stable' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - # for a unstable module no dependency must be deprecated - elif [[ "${module_release}" == 'unstable' ]] \ - && [[ "${release_of_dependency}" == 'deprecated' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - fi - - std::info "Loading module: ${m}" - eval $( "${MODULECMD}" bash load "${m}" ) - done - } - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "building ..." - MODULECMD="${PMODULES_HOME}/bin/modulecmd" - [[ -x ${MODULECMD} ]] || \ - std::die 2 "No such file or executable -- '${MODULECMD}'" - - eval $( "${MODULECMD}" bash use unstable ) - eval $( "${MODULECMD}" bash use deprecated ) - eval $( "${MODULECMD}" bash purge ) - - # :FIXME: this is a hack!!! - # shouldn't this be set in the build-script? - if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then - eval $( "${MODULECMD}" bash use Libraries ) - fi - if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then - eval $( "${MODULECMD}" bash use System ) - fi - unset C_INCLUDE_PATH - unset CPLUS_INCLUDE_PATH - unset CPP_INCLUDE_PATH - unset LIBRARY_PATH - unset LD_LIBRARY_PATH - unset DYLD_LIBRARY_PATH - - unset CFLAGS - unset CPPFLAGS - unset CXXFLAGS - unset LIBS - unset LDFLAGS - - unset CC - unset CXX - unset FC - unset F77 - unset F90 - - load_build_dependencies - - pbuild.init_env "${module_name}" "${module_version}" - pbuild::make_all + cleanup_modulefiles std::info "* * * * *\n" } +readonly -f pbuild.build_module # Local Variables: # mode: sh diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 6a52073..eb3b6ba 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -39,7 +39,7 @@ std::def_cmds(){ local path="$1" shift for cmd in "$@"; do - eval declare -g ${cmd}=$(PATH="${path}" which $cmd 2>/dev/null) + eval declare -gr ${cmd}=$(PATH="${path}" /usr/bin/which $cmd 2>/dev/null) if [[ -z "${!cmd}" ]]; then std::die 255 "${cmd} not found" fi @@ -324,7 +324,7 @@ std::get_os_release() { local -A func_map; func_map['Linux']=std.get_os_release_linux func_map['Darwin']=std.get_os_release_macos - ${func_map[${OS}]} + ${func_map[$(uname -s)]} } std::get_type() { diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c5813d1..7f92148 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -7,58 +7,69 @@ # #............................................................................. -#set -x +set -x declare VERSION='@PMODULES_VERSION@' # get absolute path of script declare mydir=$(dirname "$0") declare -r mydir=$(cd ${mydir} && pwd -P) -# initialize PATH, -# add library installation directories to the PATH, -# so 'source' is able find them - -if [[ $(uname -s) == 'Darwin' ]]; then - PATH='/opt/local/bin:' -else - PATH='' -fi -PATH+='/usr/bin:/bin:/usr/sbin:/sbin' -PATH+=":${mydir}" -PATH+=":${mydir}/../lib:${mydir}/../config" -PATH+=":${mydir}/../libexec" - -path=$PATH - -source libstd.bash || { +source "${mydir}/../lib/libstd.bash" || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } -std::def_cmds "${path}" \ - 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' +############################################################################## +# +# check availability of used commands and set environment variables +# cmd=$(which cmd) +# in the following we use these environment variable to call binaries. +# +declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd" +[[ -x ${MODULECMD} ]] || \ + std::die 1 "Oops: modulecmd binary not available!" -# can be set in the configuration file +std::def_cmds "${mydir}/../libexec" \ + 'yq' + +std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'awk' 'base64' 'cat' 'cp' 'find' 'getopt' 'grep' \ + 'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \ + 'rm' 'rmdir' 'sort' 'tar' 'tee' 'uname' + +declare -r OS="$(${uname} -s)" +if [[ ${OS} == 'Darwin' ]]; then + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \ + 'curl' 'otool' 'shasum' 'sysctl' + declare -r sha256sum="${shasum -a 256}" +else + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'ldd' 'curl' 'sha256sum' +fi + +# for the time being, we still set PATH. Just in case we forgot a binary +PATH='/usr/bin:/bin:/usr/sbin:/sbin' + +############################################################################## +# save arguments, required for building dependencies +declare -r ARGS="$@" declare PMODULES_DISTFILESDIR='' declare PMODULES_TMPDIR='' declare pm_root="${PMODULES_HOME%%/Tools*}" +declare ol_mod_root='' +declare ol_inst_root='' -source libpbuild.bash || \ +source "${mydir}/../lib/libpbuild.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -source libpmodules.bash || \ +source "${mydir}/../lib/libpmodules.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -# save arguments, (still) required for building dependencies -declare -r ARGS="$@" - +############################################################################## +# set some shell options set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob -declare ol_mod_root -declare ol_inst_root - ############################################################################## # usage() { @@ -350,9 +361,6 @@ bash_expand(){ build_modules_legacy() { find_variants_files(){ - shopt -q nullglob || : - local -i nullglob_set=$? - shopt -s nullglob local files=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}"\.${opt_system} ) files+=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}.$(uname -s)" ) local f @@ -361,7 +369,6 @@ build_modules_legacy() { || [[ -e "${f}.$(uname -s)" ]] \ || files+=( "$f" ) done - (( nullglob_set == 1 )) && shopt -u nullglob std::upvar "$1" "${files[@]}" } @@ -395,7 +402,7 @@ build_modules_legacy() { local m local pattern="/^${name}\/${version}[[:blank:]]/" for m in "${with_modules[@]}"; do - if [[ -n $(awk "/${m%/*}[\/ ]/" "${files[@]}") ]]; then + if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then pattern+=" && /${m//\//\\/}/" fi done @@ -405,7 +412,7 @@ build_modules_legacy() { local line='' while read line; do variants+=( "${line}" ) - done < <(expand_variants_file "${f}" | awk "${pattern}") + done < <(expand_variants_file "${f}" | ${awk} "${pattern}") done if (( ${#variants[@]} == 0 )); then std::info "%s " \ @@ -457,7 +464,7 @@ build_modules_yaml(){ local version="$3" _result=( $(${yq} -Ne e \ "with_entries(select(.key | test(\"^${version}\$\")))|keys" \ - "${fname}" 2>/dev/null | awk '{print $2}') ) + "${fname}" 2>/dev/null | ${awk} '{print $2}') ) } yaml_get_num_variants(){ @@ -593,7 +600,7 @@ build_modules_yaml(){ pbuild.build_module \ "${name}" "${v##*/}" \ "${relstage}" "${with_modules[@]}" - done < <(bash_expand "" ${deps[@]}|awk "${pattern}") + done < <(bash_expand "" ${deps[@]}|${awk} "${pattern}") else pbuild.build_module \ "${name}" "${v##*/}" \ @@ -634,7 +641,7 @@ pbuild.verbose "${opt_verbose}" # pm::read_config -# :FIXME: should go dist files to +# :FIXME: should dist files go to # ${pm_root}/var/distfiles # or # ${overlay}/var/distfiles diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 24f7dcb..b4a2035 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -30,7 +30,7 @@ declare -r os_name="$(uname -s)" path="${libexecdir}:/bin:/usr/bin" std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' + 'rm' 'sort' 'yq' if [[ ${PMODULES_PURETCL} == yes ]]; then declare -r modulecmd="${libexecdir}/modulecmd.tcl" From 2112a05dea3c2292b9dc47481aaaa86ad67bb1fe Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:07:05 +0200 Subject: [PATCH 90/97] modbuild: issues with nounset fixed --- Pmodules/libpbuild.bash | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 72f3fd3..87e1dc3 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -261,7 +261,7 @@ readonly -f pbuild::supported_compilers # $1: group # pbuild::add_to_group() { - if [[ -z ${1} ]]; then + if (( $# == 0 )); then std::die 42 \ "%s " "${module_name}/${module_version}:" \ "${FUNCNAME}: missing group argument." @@ -385,7 +385,11 @@ pbuild::add_patch() { "%s " "${module_name}/${module_version}:" \ "${FUNCNAME}: missing argument!" PATCH_FILES+=( "$1" ) - PATCH_STRIPS+=( "$2" ) + if (( $# >= 2 )); then + PATCH_STRIPS+=( "$2" ) + else + PATCH_STRIPS+=( "${PATCH_STRIP_DEFAULT}" ) + fi } readonly -f pbuild::add_patch From 4e2150b6d733bc4cefad5700c34f20bd67ee4a72 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:10:58 +0200 Subject: [PATCH 91/97] libpmodules: create directory in compute_group_depth() if missing --- Pmodules/libpmodules.bash.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index f394df5..30d2f25 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -40,7 +40,10 @@ print_help() { compute_group_depth () { local -n result="$1" local -r dir="$2" - test -d "${dir}" || return 1 + if [[ ! -d "${dir}" ]]; then + ${mkdir} -p "${dir}" || \ + std::die 1 "Cannot create directory -- ${dir}" + fi local group=${dir%/*} local group=${group##*/} result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ From 18284822f881021b4fa9ecba74c06c37b2c41136 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:14:26 +0200 Subject: [PATCH 92/97] libpbuild.bash: curl on RHEL7 doesn't support the opt '--remove-on-error' --- Pmodules/libpbuild.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 87e1dc3..b111021 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -460,7 +460,6 @@ pbuild::prep() { ${curl} \ --location \ --fail \ - --remove-on-error \ --output "${output}" \ "${url}" # :FIXME: How to handle insecure downloads? From 3b0ed98524733bbae9d3953d22dc207319834f8e Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:15:34 +0200 Subject: [PATCH 93/97] libpbuild.bash: create $PREFIX dir before install target The directory $PREFIX should not created to early. If $PREFIX is created at the beginning and a build target fails, the next build attempt will skip all build targets. --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index b111021..d775a5f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1446,7 +1446,6 @@ pbuild.build_module() { ${mkdir} -p "${SRC_DIR}" ${mkdir} -p "${BUILD_DIR}" - ${mkdir} -p "${PREFIX}" std::info \ "%s " \ @@ -1469,6 +1468,7 @@ pbuild.build_module() { build_target "${BUILD_DIR}" compile [[ "${build_target}" == "compile" ]] && return 0 + ${mkdir} -p "${PREFIX}" std::info \ "%s " \ "${module_name}/${module_version}:" \ From fa84196a04bf80a54d1d9a646d00fe76a516d22d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:20:53 +0200 Subject: [PATCH 94/97] libpbuild.bash: fix in comment --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index d775a5f..ddb954f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1248,7 +1248,7 @@ pbuild.build_module() { } # post_install #...................................................................... - # Install modulefile in ${pm_root}/${GROUP}/modulefiles/... + # Install modulefile in ${ol_mod_root}/${GROUP}/modulefiles/... # The modulefiles in the build-block can be # versioned like # modulefile-10.2.0 From 3a7746805d13a4092aa36caea175daccedf94cd4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:21:55 +0200 Subject: [PATCH 95/97] modbuild: cleanup --- Pmodules/modbuild.in | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 7f92148..c7597b4 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -50,21 +50,12 @@ fi PATH='/usr/bin:/bin:/usr/sbin:/sbin' ############################################################################## -# save arguments, required for building dependencies -declare -r ARGS="$@" -declare PMODULES_DISTFILESDIR='' -declare PMODULES_TMPDIR='' -declare pm_root="${PMODULES_HOME%%/Tools*}" -declare ol_mod_root='' -declare ol_inst_root='' - source "${mydir}/../lib/libpbuild.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" source "${mydir}/../lib/libpmodules.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" ############################################################################## -# set some shell options set -o nounset shopt -s nocaseglob shopt -s extglob @@ -168,6 +159,11 @@ MISCELLANEOUS OPTIONS: # then configuration file # last default +# save arguments, required for building dependencies +declare -r ARGS="$@" +declare PMODULES_DISTFILESDIR='' +declare PMODULES_TMPDIR='' + # versions to be build, '.*' or none means all declare -a versions=() declare opt_build_target='all' @@ -185,8 +181,6 @@ declare -a opt_with_modules=() declare BUILD_SCRIPT='' -declare opt_ol_name_or_dir='' - parse_args() { while (( $# > 0 )); do case $1 in @@ -429,8 +423,8 @@ build_modules_legacy() { fi declare ol_name='base' declare ol_type='n' - declare ol_mod_root="${pm_root}" - declare ol_inst_root="${pm_root}" + declare ol_mod_root="${PMODULES_HOME%%/Tools*}" + declare ol_inst_root="${PMODULES_HOME%%/Tools*}" local -i i=0 local -i num_variants=${#variants[@]} for ((i = 0; i < num_variants; i++)); do @@ -646,7 +640,7 @@ pm::read_config # or # ${overlay}/var/distfiles # ? -: ${PMODULES_DISTFILESDIR:=${pm_root}/var/distfiles} +: ${PMODULES_DISTFILESDIR:="${PMODULES_HOME%%/Tools*}/var/distfiles"} : ${PMODULES_TMPDIR:=/var/tmp/${USER}} declare -r BUILD_SCRIPT From cddc91865271ba85e06876a01504c4d756d9eccd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:23:11 +0200 Subject: [PATCH 96/97] modbuild: new key 'build_requires' in YAML variant files --- Pmodules/modbuild.in | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c7597b4..6620c9b 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -532,6 +532,19 @@ build_modules_yaml(){ _result=("${_result[@]}") } + yaml_get_build_requirements(){ + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=( $(${yq} -Ne e ".\"${version}\"[${idx}].build_requires" \ + "${fname}" 2>/dev/null) ) + if (( $? != 0 )); then + _result=() + return + fi + } + local name="$1" local version="$2" shift 2 @@ -551,6 +564,7 @@ build_modules_yaml(){ (( n_variants == 0 )) && (( n_variants = 1 )) local -i i local -a deps=() + local -a build_requires=() local relstage local ol_name for (( i=0; i Date: Mon, 18 Jul 2022 15:58:10 +0200 Subject: [PATCH 97/97] overlay doc updated --- doc/Overlays.adoc | 53 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/doc/Overlays.adoc b/doc/Overlays.adoc index d08004b..b486cc6 100644 --- a/doc/Overlays.adoc +++ b/doc/Overlays.adoc @@ -10,6 +10,7 @@ Please note: This document is work in progress! === System wide +==== Example .A system wide configuration file `/opt/psi/config/Pmodules.yaml` ==== .... @@ -22,11 +23,34 @@ DistfilesDir: /opt/psi/var/distfiles Overlays: base: install_root: /opt/psi + modulefiles_root: /opt/psi devel: install_root: /opt/psi modulefiles_root: ${HOME}/modulefiles .... ==== +==== YAML Format +.Format +==== +.... +Overlays: + : + install_root: + modulefiles_root: + type: + ... +.... +==== +``:: Name of overlay. Note: whitespace in the name is not supported. +`.install_root`:: The root of the software installation. This key is mandatory. +`.modulefiles_root`:: The root of the modulefile hierarchy. This key is optional and defaults to `.install_root> +`.type`:: The type of the overlay, see below. This key is optional and the default value is `n`. + +==== Overlay types +`n`:: Normal overlay. +`h`:: Hiding overlay. +`r`:: Replacing overlay. + === User defined overlays Each user can define his own overlays in `$HOME/.Pmodules/Pmodules.yaml`. @@ -39,6 +63,7 @@ The old format of the variants file is simple but very limited and almost imposs === With a YAML variants file +==== Example of a variants file in YAML format .A YAML variants file ==== .... @@ -53,13 +78,15 @@ hdf5_serial/1.12.2: .... ==== -.YAML format of varaints file +==== Format specification +.YAML format .... +relstage: overlay: systems: -/: - - with: +/: + - with: dependencies: relstate: overlay: @@ -67,6 +94,24 @@ systems: ... ... .... +==== Defaults +Default values can be overriden per version/variant. +`overlay`:: The default overlay the module will be installed in. This value can be overriden for dedicated versions/variants. +`systems`:: The default for supported systems. -=== With a legacy variants file +==== Versions and Variants + +`/`:: An array with variants for this version. + +`/.[i].with`:: Hierarchical dependencies for variant `i`. + +`/.[i].dependencies`:: Build/run-time dependencies for variant `i`. + +`/.[i].relstage`:: Relase stage of variant `i`. + +`/.[i].overlay`:: Overlay of variant `i`. + +`/.[i].systems`:: Supported systems. + +=== Legacy format