From ebdd93051b52537f4ab317e07d915583380390ce Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 5 Mar 2021 19:03:59 +0100 Subject: [PATCH 01/30] find added to required tools --- build | 4 ++++ config/versions.conf | 1 + 2 files changed, 5 insertions(+) diff --git a/build b/build index d85440e..163a928 100755 --- a/build +++ b/build @@ -345,6 +345,10 @@ pmodules::compile() { build bash fi + if [[ ! -f "${PMODULES_HOME}/sbin/find" ]] || [[ ${opt_force} == 'yes' ]]; then + build findutils + fi + if [[ ! -e "${PMODULES_HOME}/sbin/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then build Tcl fi diff --git a/config/versions.conf b/config/versions.conf index 9aa7e9d..00777ad 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -1,5 +1,6 @@ bash 5.1-rc3 coreutils 8.31 +findutils 4.7.0 getopt 1.1.6 gettext 0.21 modules 3.2.10.1 From 90d9cee20c39d462c1c2dbdc90efa9eca4787ad8 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 5 Mar 2021 19:04:54 +0100 Subject: [PATCH 02/30] module search: output all dependencies --- Pmodules/modulecmd.bash.in | 256 +++++++++++++++++++++++++------------ 1 file changed, 172 insertions(+), 84 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 676e083..d7dce81 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -715,22 +715,29 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local -r dir="$1" + local saved_IFS=${IFS}; + IFS=':' + local -a dirs=($1) + IFS=${saved_IFS} local -r module="$2" local -r use_releases="${3:-${UsedReleases}}" local -a mods=() local release - test -d "${dir}" || return 0 - { - cd "${dir}" - while read mod; do - get_release release "${dir}/${mod}" - - if [[ :${use_releases}: =~ :${release}: ]]; then - mods+=( "${mod}" ${release} ) - fi - done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") - } + local dir='' + + for dir in "${dirs[@]}"; do + test -d "${dir}" || return 0 + { + cd "${dir}" + while read mod; do + get_release release "${dir}/${mod}" + + if [[ :${use_releases}: =~ :${release}: ]]; then + mods+=( "${mod}" ${release} "${dir}/${mod}") + fi + done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") + } + done echo "${mods[@]}" } @@ -790,7 +797,7 @@ subcommand_avail() { terse_output() { output_header - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do local mod=${mods[i]} local release=${mods[i+1]} case $release in @@ -807,7 +814,7 @@ subcommand_avail() { } machine_output() { - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do printf "%-20s\t%s\n" "${mods[i]}" "${mods[i+1]}" 1>&2 done } @@ -816,7 +823,7 @@ subcommand_avail() { # :FIXME: for the time being, this is the same as terse_output! long_output() { output_header - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do local mod=${mods[i]} local release=${mods[i+1]} case $release in @@ -837,7 +844,7 @@ subcommand_avail() { local -i column=$cols local -i colsize=16 - for ((i=0; i<${#mods[@]}; i+=2)); do + for ((i=0; i<${#mods[@]}; i+=3 )); do if [[ ${verbosity_lvl} == 'verbose' ]]; then local release=${mods[i+1]} case ${mods[i+1]} in @@ -1483,7 +1490,8 @@ subcommand_clear() { # Subcommands[search]='search' Options[search]='-o aH -l help -l no-header -l print-modulefiles ' -Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv' +Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv ' +Options[search]+='-l verbose' Help[search]=' USAGE: module search [switches] string... @@ -1509,26 +1517,21 @@ SWITCHES: module search --with=gcc/4.8.3 lists all modules in the hierarchy compiled with gcc 4.8.3. + + --verbose + vebose output ' subcommand_search() { local -r subcommand='search' local modules=() local with_modules='//' - local src_prefix='' + local src_prefix=() local opt_print_header='yes' local opt_print_modulefiles='no' local opt_print_csv='no' + local opt_print_verbose='no' local opt_use_releases=':' - local -r fmt="%-20s %-10s %-12s %-s\n" - - # no args - print_header() { - printf '\n' 1>&1 - printf "${fmt}" "Module" "Release" "Group" "Requires" 1>&2 - printf -- '-%.0s' {1..60} 1>&2 - printf '\n' 1>&2 - } #..................................................................... # @@ -1542,31 +1545,111 @@ subcommand_search() { # with_modules # print_result() { - local -r tmpfile=$1 - [[ "${opt_print_header}" == "yes" ]] && print_header - if [[ "${opt_print_modulefiles}" == "yes" ]]; then - while read -a line; do - # group first - local out="${line[2]}/" - # add directory of modulefiles - out+="${PMODULES_MODULEFILES_DIR}/" - for d in "${line[@]:3}"; do - out+="$d/" - done - out+="${line[0]}" - std::info "${out}" - done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}") - elif [[ "${opt_print_csv}" == "yes" ]]; then - while read -a toks; do - : - done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}") - else - "${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}" 1>&2 - fi + local func_print_header='' + local func_print_line='' + local fmt='' + + # no args + print_header_default() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Requires" + std::info '-%.0s' {1..60} + std::info '' + } + + print_line_default() { + local deps="${@:5}" + std::info "${fmt}" "$1" "$2" "$3" "${deps}" + } + + print_default() { + fmt="%-20s %-10s %-12s %-s" + if [[ ${opt_print_header} == 'yes' ]]; then + func_print_header='print_header_default' + else + func_print_header='print_header_none' + fi + func_print_line='print_line_default' + } + + print_header_verbose() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Overlay" "Requires" + std::info '-%.0s' {1..79} + std::info '' + } + + print_line_verbose() { + std::info "${fmt}" "$@" + } + + print_verbose() { + fmt="%-20s %-10s %-12s %-20s %-s" + func_print_header='print_header_verbose' + func_print_line='print_line_verbose' + } + + print_header_none() { + : + } + + print_line_modulefile() { + local line=( "$@" ) + # group first + local out="${line[2]}/" + # add directory of modulefiles + out+="${PMODULES_MODULEFILES_DIR}/" + for d in "${line[@]:3}"; do + out+="$d/" + done + out+="${line[0]}" + std::info "${out}" + } + + # print full modulefile names only + print_modulefiles() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_header_none' + } + + print_line_csv() { + : + } + + print_csv() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_line_csv' + } + + local -r tmpfile=$1 + + if [[ "${opt_print_modulefiles}" == 'yes' ]]; then + print_modulefiles + elif [[ "${opt_print_csv}" == 'yes' ]]; then + print_csv + elif [[ "${opt_print_verbose}" == 'yes' ]]; then + print_verbose + else + print_default + fi + + ${func_print_header} + while read -a toks; do + ${func_print_line} "${toks[@]}" + done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ + awk "${with_modules}") + } + + get_module_prefix() { + local "$1" + local modulefile="$2" + local -r _prefix=$("${modulecmd}" bash show "${modulefile}" 2>&1 | \ + awk '/_PREFIX |_HOME / {print $3; exit}') + std::upvar $1 "${_prefix}" } + #..................................................................... # # search modules @@ -1580,47 +1663,48 @@ subcommand_search() { local -r module=$1 # write results to a temporary file for later processing local -r tmpfile=$( "${mktemp}" /tmp/Pmodules.XXXXXX ) \ - || std::die 1 "Oops: unable to create tmp file!\n" + || std::die 1 "Oops: unable to create tmp file!" local group # loop over all groups for group in "${!GroupDepths[@]}"; do # loop over all directories which can be added to # MODULEPATH inside current group local depth=${GroupDepths[${group}]} - local mpaths=( $(find \ - "${src_prefix}/${group}/modulefiles" \ - -type d \ - -mindepth ${depth} -maxdepth ${depth} \ - 2>/dev/null)) - local mpath - for mpath in "${mpaths[@]}"; do - # get dependencies encoded in directory name - local p="${mpath/${src_prefix}}" - p=( ${p//\// } ) - local deps=() - local -i i - for ((i=2; i < ${#p[@]}; i+=2)); do - deps+=( ${p[i]}/${p[i+1]} ) - done - local requires=${deps[@]} + local modulepath=( $(find \ + "${src_prefix[@]/%//${group}/modulefiles}" \ + -mindepth ${depth} -maxdepth ${depth} \ + -type d \ + -printf "%p:" + 2>/dev/null)) - # get and print all available modules in $mpath - # with respect to the requested releases - # tmpfile: module/version release group group- - # dependencies... - local mods=( $( get_available_modules \ - "${mpath}" \ - "${module}" \ - "${opt_use_releases}" ) ) - [[ ${#mods[@]} == 0 ]] && continue - for (( i=0; i<${#mods[@]}; i+=2 )); do - printf "${fmt}" ${mods[i]} "${mods[i+1]}" \ - ${group} "${requires}" >> "${tmpfile}" - done + # get and print all available modules in $mpath + # with respect to the requested releases + # tmpfile: module/version release group group- + # dependencies... + local mods=( $( get_available_modules \ + "${modulepath}" \ + "${module}" \ + "${opt_use_releases}" ) ) + for (( i=0; i<${#mods[@]}; i+=3 )); do + local name=${mods[i]} + local release=${mods[i+1]} + local modulefile=${mods[i+2]} + local prefix='' + local requires='' + + get_module_prefix prefix "${modulefile}" + local dependencies_file="${prefix}/.dependencies" + if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then + requires=$(< "${dependencies_file}") + fi + + echo ${mods[i]} ${mods[i+1]} \ + ${group} ${mods[i+2]} \ + ${requires} >> "${tmpfile}" done done print_result "${tmpfile}" - rm -f "${tmpfile}" + #rm -f "${tmpfile}" } while (( $# > 0 )); do @@ -1647,7 +1731,7 @@ subcommand_search() { local arg=${1/--release=} fi is_release "${arg}" || \ - std::die 1 "%s %s: %s -- %s\n" \ + std::die 1 "%s %s: %s -- %s" \ "${CMD}" 'search' \ "illegal release name" \ "${arg}" @@ -1661,7 +1745,7 @@ subcommand_search() { local arg=${1/--with=} fi if [[ -z ${arg} ]] || [[ "${arg}" =~ "-*" ]]; then - std::die 1 "%s %s: %s -- %s\n" \ + std::die 1 "%s %s: %s -- %s" \ "${CMD}" 'search' \ "illegal value for --with option" \ "${arg}" @@ -1680,6 +1764,9 @@ subcommand_search() { pmodules::check_directories "${src_prefix}" shift ;; + -v | --verbose ) + opt_print_verbose='yes' + ;; -- ) ;; * ) @@ -1700,6 +1787,7 @@ subcommand_search() { modules+=( '' ) fi + # :FIXME: do we need this? if (( ${#GroupDepths[@]} == 0 )) || \ [[ ${src_prefix} != ${PMODULES_ROOT} ]]; then scan_groups "${src_prefix}" From 7001c86154e420b1b34625f5f2ead10dd959c24c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 9 Mar 2021 14:04:45 +0100 Subject: [PATCH 03/30] option to print all dependencies added to module search more changes - define and set a default IFS - use our own find - not find command installed on the system - the modulepath directories are now passed as arry to get_available_modules() - new option "--wrap" --- Pmodules/modulecmd.bash.in | 183 ++++++++++++++++++++++++------------- Tools/findutils/build | 4 + 2 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 Tools/findutils/build diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index d7dce81..b1bd0bd 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -18,6 +18,7 @@ declare -r base64="${sbindir}/base64" declare -r mktemp="${sbindir}/mktemp" declare -r sort="${sbindir}/sort" declare -r getopt="${sbindir}/getopt" +declare -r find="${sbindir}/find" source "${libdir}/libstd.bash" source "${libdir}/libpmodules.bash" @@ -35,6 +36,10 @@ fi declare verbosity_lvl=${PMODULES_VERBOSITY:-'verbose'} +# we use newline as internal field separator +IFS=$' \t\n' +declare -r __IFS=${IFS} # used to restore IFS + shopt -s nullglob declare -A GroupDepths='()' @@ -263,10 +268,9 @@ subcommand_load() { local prefix='' local m='' - local saved_IFS="${IFS}"; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} # # Test whether a given module is available. @@ -407,10 +411,9 @@ subcommand_load() { # - release:group:name or # - name:release - local save_ifs=${IFS} IFS=':' local -a toks=($m) - IFS=${save_ifs} + IFS=${__IFS} local group='' local release='' if is_group "${toks[0]}"; then @@ -715,12 +718,11 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local saved_IFS=${IFS}; - IFS=':' - local -a dirs=($1) - IFS=${saved_IFS} - local -r module="$2" - local -r use_releases="${3:-${UsedReleases}}" + local -r module="$1" + local -r use_releases="${2:-${UsedReleases}}" + shift 2 + local -a dirs=( "$@" ) + local -a mods=() local release local dir='' @@ -735,7 +737,7 @@ get_available_modules() { if [[ :${use_releases}: =~ :${release}: ]]; then mods+=( "${mod}" ${release} "${dir}/${mod}") fi - done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") + done < <(${find} * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") } done echo "${mods[@]}" @@ -914,16 +916,16 @@ subcommand_avail() { if (( ${#pattern[@]} == 0 )); then pattern+=( '' ) fi - local saved_IFS=${IFS}; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} local string for string in "${pattern[@]}"; do for dir in "${modulepath[@]}"; do mods=( $( get_available_modules \ - "${dir}" "${string}" \ - "${opt_use_releases}" ) ) + "${string}" \ + "${opt_use_releases}" \ + "${dir}" ) ) [[ ${#mods[@]} == 0 ]] && continue ${output_function} done @@ -942,7 +944,7 @@ compute_group_depth () { local group=${dir%/*} local group=${group##*/} [[ -n "${GroupDepths[${group}]}" ]] && return 0 - local -i depth=$(find "${dir}" -depth \( -type f -o -type l \) \ + local -i depth=$(${find} "${dir}" -depth \( -type f -o -type l \) \ -printf "%d" -quit 2>/dev/null) (( depth-=2 )) # if a group doesn't contain a modulefile, depth is negativ @@ -998,10 +1000,9 @@ SWITCHES: subcommand_use() { local -r subcommand='use' - local saved_IFS=${IFS}; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} local add2path_func='std::append_path' group_is_used() { @@ -1491,7 +1492,8 @@ subcommand_clear() { Subcommands[search]='search' Options[search]='-o aH -l help -l no-header -l print-modulefiles ' Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv ' -Options[search]+='-l verbose' +Options[search]+='-l verbose ' +Options[search]+='-l all-deps -l wrap' Help[search]=' USAGE: module search [switches] string... @@ -1520,18 +1522,25 @@ SWITCHES: --verbose vebose output + + --wrap + wrap output ' subcommand_search() { local -r subcommand='search' local modules=() local with_modules='//' + local -ir cols=$(tput cols) # get number of columns of terminal + local -i max_len_modulename=0 local src_prefix=() local opt_print_header='yes' local opt_print_modulefiles='no' local opt_print_csv='no' local opt_print_verbose='no' local opt_use_releases=':' + local opt_all_deps='no' + local opt_wrap='no' #..................................................................... # @@ -1549,21 +1558,8 @@ subcommand_search() { local func_print_line='' local fmt='' - # no args - print_header_default() { - std::info '' - std::info "${fmt}" "Module" "Release" "Group" "Requires" - std::info '-%.0s' {1..60} - std::info '' - } - - print_line_default() { - local deps="${@:5}" - std::info "${fmt}" "$1" "$2" "$3" "${deps}" - } - print_default() { - fmt="%-20s %-10s %-12s %-s" + fmt="%-${max_len_modulename}s %-10s %-12s %-s" if [[ ${opt_print_header} == 'yes' ]]; then func_print_header='print_header_default' else @@ -1572,23 +1568,64 @@ subcommand_search() { func_print_line='print_line_default' } - print_header_verbose() { - std::info '' - std::info "${fmt}" "Module" "Release" "Group" "Overlay" "Requires" - std::info '-%.0s' {1..79} + print_header_default() { std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Requires" + std::info '-%.0s' $(seq 1 ${cols}) } - print_line_verbose() { - std::info "${fmt}" "$@" + print_line_default() { + write_line() { + local str="$1" + if (( ${#str} >= cols )); then + str="${str:0:$((cols-1))}>" + fi + std::info "${str}" + } + if [[ "${opt_wrap}" == 'no' ]]; then + local deps="${@:5}" + local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[@]}") + write_line "${str}" + else + local deps=( "${@:5}" ) + local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[0]}") + for (( i = 1; i < ${#deps[@]}; i++ )); do + if (( ${#str} + ${#deps[i]} + 1 <= cols )); then + str+=" ${deps[i]}" + else + write_line "${str}" + str=$(printf "${fmt}" "" "" "" "> ${deps[i]}") + fi + done + write_line "${str}" + fi } print_verbose() { - fmt="%-20s %-10s %-12s %-20s %-s" + fmt="%-${max_len_modulename}s %-10s %-12s %-s" func_print_header='print_header_verbose' func_print_line='print_line_verbose' } + print_header_verbose() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Dependencies/Modulefile" + std::info '-%.0s' $(seq 1 ${cols}) + } + + print_line_verbose() { + local deps="${@:5}" + std::info "${fmt}" "$1" "$2" "$3" "dependencies: ${deps}" + std::info "${fmt}" "" "" "" "modulefile: $4" + } + + # print full modulefile names only + print_modulefiles() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_header_none' + } + print_header_none() { : } @@ -1606,13 +1643,6 @@ subcommand_search() { std::info "${out}" } - # print full modulefile names only - print_modulefiles() { - fmt='' - func_print_header='print_header_none' - func_print_line='print_header_none' - } - print_line_csv() { : } @@ -1670,37 +1700,54 @@ subcommand_search() { # loop over all directories which can be added to # MODULEPATH inside current group local depth=${GroupDepths[${group}]} - local modulepath=( $(find \ - "${src_prefix[@]/%//${group}/modulefiles}" \ - -mindepth ${depth} -maxdepth ${depth} \ - -type d \ - -printf "%p:" - 2>/dev/null)) - + local s='' + if (( depth > 0 )); then + s=$(printf '/*%.0s' $(seq 1 ${depth})) + fi + local modulepath=( ${src_prefix[@]/%//${group}/modulefiles$s} ) + # get and print all available modules in $mpath # with respect to the requested releases # tmpfile: module/version release group group- # dependencies... local mods=( $( get_available_modules \ - "${modulepath}" \ "${module}" \ - "${opt_use_releases}" ) ) + "${opt_use_releases}" \ + "${modulepath[@]}" \ + ) ) + for (( i=0; i<${#mods[@]}; i+=3 )); do local name=${mods[i]} local release=${mods[i+1]} local modulefile=${mods[i+2]} - local prefix='' - local requires='' - get_module_prefix prefix "${modulefile}" - local dependencies_file="${prefix}/.dependencies" - if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then - requires=$(< "${dependencies_file}") + if (( ${#name} > max_len_modulename)); then + max_len_modulename=${#name} fi - echo ${mods[i]} ${mods[i+1]} \ - ${group} ${mods[i+2]} \ - ${requires} >> "${tmpfile}" + if [[ "${opt_print_verbose}" == 'yes' ]] || [[ "${opt_all_deps}" == 'yes' ]]; then + local prefix='' + get_module_prefix prefix "${modulefile}" + local dependencies_file="${prefix}/.dependencies" + if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then + deps=($(< "${dependencies_file}")) + else + deps=() + fi + else + # get dependencies encoded in directory name + local deps=() + local -i j + IFS='/' + local toks=( ${modulefile} ) + for ((j = -depth-2; j < -2; j += 2)); do + deps+=( "${toks[*]: $j:2}" ); + done + IFS=${__IFS} + fi + + echo ${name} ${release} ${group} ${modulefile} \ + ${deps[@]} >> "${tmpfile}" done done print_result "${tmpfile}" @@ -1712,6 +1759,9 @@ subcommand_search() { -H | --help ) print_help "${subcommand}" ;; + --all-deps ) + opt_all_deps='yes' + ;; --no-header ) opt_print_header='no' ;; @@ -1767,6 +1817,9 @@ subcommand_search() { -v | --verbose ) opt_print_verbose='yes' ;; + --wrap ) + opt_wrap='yes' + ;; -- ) ;; * ) diff --git a/Tools/findutils/build b/Tools/findutils/build new file mode 100644 index 0000000..0f1fc2f --- /dev/null +++ b/Tools/findutils/build @@ -0,0 +1,4 @@ +#!/usr/bin/env modbuild + +pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.xz" +pbuild::add_configure_args "--bindir=${PREFIX}/sbin" From 0689b04259a8ff2ce0033c1509b2972c34b65b7d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 12 Mar 2021 11:51:49 +0100 Subject: [PATCH 04/30] do not add the Pmodules bin directory to PATH --- Pmodules/bash | 2 -- Pmodules/csh | 1 - Pmodules/zsh | 4 ---- 3 files changed, 7 deletions(-) diff --git a/Pmodules/bash b/Pmodules/bash index fa5da29..c7e9bf5 100644 --- a/Pmodules/bash +++ b/Pmodules/bash @@ -35,8 +35,6 @@ unset MODULE_VERSION_STACK unset MODULESHOME unset PMODULES_ENV -PATH="${PMODULES_HOME}/bin:$PATH" - ############################################################################# # implement module comand as shell function # diff --git a/Pmodules/csh b/Pmodules/csh index be69b42..854152a 100644 --- a/Pmodules/csh +++ b/Pmodules/csh @@ -47,4 +47,3 @@ if (! $?LOADEDMODULES ) then setenv LOADEDMODULES "" endif -setenv PATH "${PMODULES_HOME}/bin:${PATH}" diff --git a/Pmodules/zsh b/Pmodules/zsh index fc46c4d..80dc338 100644 --- a/Pmodules/zsh +++ b/Pmodules/zsh @@ -89,10 +89,6 @@ unset PMODULES_DEFAULT_RELEASES unset PMODULES_DEFAULT_GROUPS unset PMODULES_DEFINED_RELEASES -# setup PATH -dirs_to_remove="${PMODULES_HOME%/*}/*" -path=( ${(m)path:#${dirs_to_remove}} "${PMODULES_HOME}/bin" ) - # initialize MANPATH with output of `man --path` if not set [[ -z "${MANPATH}" ]] && manpath=$( man --path ) From 0007d5dc38a470da49b1056e0d88c997eb8c1ef1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 12 Mar 2021 13:39:41 +0100 Subject: [PATCH 05/30] Do not add Pmodule bin dir to PATH in initialisation --- Pmodules/modulecmd.bash.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index b1bd0bd..c7e6b8e 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1315,10 +1315,6 @@ reset_used_releases() { g_env_must_be_saved='yes' } -init_path() { - std::replace_path PATH "${PMODULES_HOME%/*}/.*" - std::prepend_path PATH "${PMODULES_HOME}/bin" -} init_manpath() { std::replace_path MANPATH "${PMODULES_HOME%/*}/.*" @@ -1350,13 +1346,11 @@ pmodules_init() { reset_modulepath reset_used_groups reset_used_releases - init_path init_manpath export_env \ LOADEDMODULES \ _LMFILES_ \ MODULEPATH \ - PATH \ MANPATH } From 2f79681ac643f43b8c4cfffd097f0c2da8c49ee7 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 17 Mar 2021 17:23:41 +0100 Subject: [PATCH 06/30] module search: some not really required sanity checks removed --- Pmodules/modulecmd.bash.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index c7e6b8e..99a9268 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -21,7 +21,6 @@ declare -r getopt="${sbindir}/getopt" declare -r find="${sbindir}/find" source "${libdir}/libstd.bash" -source "${libdir}/libpmodules.bash" : ${PMODULES_DEFINED_RELEASES:=':unstable:stable:deprecated:'} @@ -1804,8 +1803,8 @@ subcommand_search() { opt_use_releases="${PMODULES_DEFINED_RELEASES}" ;; --src ) + # :FIXME: do we have to add some sanity checks here? src_prefix=$2 - pmodules::check_directories "${src_prefix}" shift ;; -v | --verbose ) From 04d6047cd0b1e21a571f696a4cba3fc8789021b0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 17 Mar 2021 17:58:54 +0100 Subject: [PATCH 07/30] move helper binaries to from sbin to libexec - remove installed header files etc - add substitution for @BASH@, @MODULECMD@, @MODMANANGE@ --- Pmodules/modmanage | 6 ----- Pmodules/modmanage.in | 5 ++++ Pmodules/modulecmd | 6 ----- Pmodules/modulecmd.bash.in | 31 +++++++++++++++-------- Pmodules/modulecmd.in | 5 ++++ Tools/Tcl/build | 4 +-- Tools/bash/build | 2 +- Tools/coreutils/build | 2 +- Tools/findutils/build | 2 +- Tools/getopt/build | 22 ++++++++++------ Tools/gettext/build | 2 +- build | 51 +++++++++++++++++++++++++------------- config/versions.conf | 4 +-- 13 files changed, 88 insertions(+), 54 deletions(-) delete mode 100644 Pmodules/modmanage create mode 100644 Pmodules/modmanage.in delete mode 100644 Pmodules/modulecmd create mode 100644 Pmodules/modulecmd.in diff --git a/Pmodules/modmanage b/Pmodules/modmanage deleted file mode 100644 index c4c9d74..0000000 --- a/Pmodules/modmanage +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -unset BASH_ENV - -bindir=$( cd $(dirname "$0")/../sbin && pwd -P ) -"${bindir}/bash" --noprofile --norc "${bindir}/../libexec/modmanage.bash" "$@" diff --git a/Pmodules/modmanage.in b/Pmodules/modmanage.in new file mode 100644 index 0000000..10508ad --- /dev/null +++ b/Pmodules/modmanage.in @@ -0,0 +1,5 @@ +#!/bin/sh + +unset BASH_ENV + +"@BASH@" --noprofile --norc "@MODMANAGE@ "$@" diff --git a/Pmodules/modulecmd b/Pmodules/modulecmd deleted file mode 100644 index eb82f13..0000000 --- a/Pmodules/modulecmd +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -unset BASH_ENV - -bindir=$( cd $(dirname "$0")/../sbin && pwd -P ) -"${bindir}/bash" --noprofile --norc "${bindir}/../libexec/modulecmd.bash" "$@" diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 99a9268..8a70d54 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1,4 +1,4 @@ -#!@PMODULES_HOME@/sbin/bash --noprofile +#!@BASH@ --noprofile # #set -o nounset @@ -10,15 +10,26 @@ declare -r CMD='module' declare -r mydir=$(cd $(dirname "$0") && pwd) declare prefix=$(dirname "${mydir}") -declare -r sbindir="${prefix}/sbin" declare -r libdir="${prefix}/lib" declare -r libexecdir="${prefix}/libexec" -declare -r base64="${sbindir}/base64" -declare -r mktemp="${sbindir}/mktemp" -declare -r sort="${sbindir}/sort" -declare -r getopt="${sbindir}/getopt" -declare -r find="${sbindir}/find" +base64=$(PATH=/bin:/usr/bin /usr/bin/which base64) +declare -r base64 +mktemp=$(PATH=/bin:/usr/bin /usr/bin/which mktemp) +declare -r mktemp +sort=$(PATH=/bin:/usr/bin /usr/bin/which sort) +declare -r sort +awk=$(PATH=/bin:/usr/bin /usr/bin/which awk) +declare -r awk +if [[ $(uname -s) == 'Darwin' ]]; then + declare -r getopt="${libexecdir}/getopt" + declare -r find="${libexecdir}/find" +else + getopt=$(PATH=/bin:/usr/bin /usr/bin/which getopt) + declare -r getopt + find=$(PATH=/bin:/usr/bin /usr/bin/which find) + declare -r find +fi source "${libdir}/libstd.bash" @@ -317,7 +328,7 @@ subcommand_load() { # can be used. # mapfile -t array < <("${modulecmd}" 'bash' show "$m" 2>&1 | \ - awk 'NR == 2 {print substr($0, 1, length($0)-1)}; /_PREFIX |_HOME / {print $3; exit}') + ${awk} 'NR == 2 {print substr($0, 1, length($0)-1)}; /_PREFIX |_HOME / {print $3; exit}') current_modulefile="${array[0]}" prefix="${array[1]}" test -n "${current_modulefile}" || return 1 @@ -1662,14 +1673,14 @@ subcommand_search() { while read -a toks; do ${func_print_line} "${toks[@]}" done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}") + ${awk} "${with_modules}") } get_module_prefix() { local "$1" local modulefile="$2" local -r _prefix=$("${modulecmd}" bash show "${modulefile}" 2>&1 | \ - awk '/_PREFIX |_HOME / {print $3; exit}') + ${awk} '/_PREFIX |_HOME / {print $3; exit}') std::upvar $1 "${_prefix}" } diff --git a/Pmodules/modulecmd.in b/Pmodules/modulecmd.in new file mode 100644 index 0000000..bce0367 --- /dev/null +++ b/Pmodules/modulecmd.in @@ -0,0 +1,5 @@ +#!/bin/sh + +unset BASH_ENV + +"@BASH@" --noprofile --norc "@MODULECMD@" "$@" diff --git a/Tools/Tcl/build b/Tools/Tcl/build index ca7bfaf..47e53a8 100755 --- a/Tools/Tcl/build +++ b/Tools/Tcl/build @@ -14,12 +14,12 @@ pbuild::configure() { esac "${srcdir}"/configure \ --prefix="${PREFIX}" \ - --bindir="${PREFIX}/sbin" \ + --bindir="${PREFIX}/${UTILBIN_DIR}" \ --enable-shared=no \ || exit 1 } pbuild::post_install() { - { cd "${PREFIX}"/sbin && rm -f tclsh && ln -fs tclsh${V%.*} tclsh; }; + { cd "${PREFIX}/${UTILBIN_DIR}" && rm -f tclsh && ln -fs tclsh${V%.*} tclsh; }; } diff --git a/Tools/bash/build b/Tools/bash/build index 3319010..ff8489f 100755 --- a/Tools/bash/build +++ b/Tools/bash/build @@ -1,5 +1,5 @@ #!/usr/bin/env modbuild pbuild::set_download_url "https://ftp.gnu.org/gnu/$P/$P-$V.tar.gz" -pbuild::add_configure_args "--bindir=${PREFIX}/sbin" +pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" diff --git a/Tools/coreutils/build b/Tools/coreutils/build index 5d261e5..c561e2a 100755 --- a/Tools/coreutils/build +++ b/Tools/coreutils/build @@ -1,4 +1,4 @@ #!/usr/bin/env modbuild pbuild::set_download_url "http://ftp.gnu.org/gnu/coreutils/$P-$V.tar.xz" -pbuild::add_configure_args "--bindir=${PREFIX}/sbin" +pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" diff --git a/Tools/findutils/build b/Tools/findutils/build index 0f1fc2f..d63f114 100644 --- a/Tools/findutils/build +++ b/Tools/findutils/build @@ -1,4 +1,4 @@ #!/usr/bin/env modbuild pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.xz" -pbuild::add_configure_args "--bindir=${PREFIX}/sbin" +pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" diff --git a/Tools/getopt/build b/Tools/getopt/build index b5bd4c8..81d8433 100755 --- a/Tools/getopt/build +++ b/Tools/getopt/build @@ -17,22 +17,30 @@ pbuild::compile() { declare -x LIBRARY_PATH="${PREFIX}/lib" case ${OS} in Linux ) + declare -x C_INCLUDE_PATH="${PREFIX}/include" + declare -x LIBRARY_PATH="${PREFIX}/lib" declare -x LDFLAGS="-lintl -L${PREFIX}/lib" ;; Darwin ) - declare -x LDFLAGS="-lintl -liconv -framework CoreFoundation" + if [[ ! -d '/opt/local/bin' ]] || [[ ! -x '/opt/local/bin/msgfmt' ]]; then + std::die 1 "gettext port from Macports is required to build 'getopt'!" + fi + PATH+=':/opt/local/bin' + declare -x C_INCLUDE_PATH='/opt/local/include' + declare -x LDFLAGS="/opt/local/lib/libintl.a /opt/local/lib/libiconv.a -framework CoreFoundation" ;; esac - make -e all + make -e all || exit 1 + declare -x DESTDIR="${PREFIX}" + declare -x prefix='' || exit 1 + PATH="${PREFIX}/${UTILBIN_DIR}:${PATH}" + make -e install + mv "${PREFIX}/bin/getopt" "${PREFIX}/${UTILBIN_DIR}" } pbuild::install() { - declare -x DESTDIR="${PREFIX}" - declare -x prefix='' - make -e install - mkdir -p "${PREFIX}/sbin" - mv "${PREFIX}/bin/getopt" "${PREFIX}/sbin" + : } pbuild::cleanup_build() { diff --git a/Tools/gettext/build b/Tools/gettext/build index 9ed9b20..776185e 100755 --- a/Tools/gettext/build +++ b/Tools/gettext/build @@ -3,7 +3,7 @@ pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.gz" pbuild::add_configure_args "--prefix=${PREFIX}" -pbuild::add_configure_args "--bindir=${PREFIX}/sbin" +pbuild::add_configure_args "--bindir=${PREFIX}/${UTILBIN_DIR}" pbuild::add_configure_args "--disable-java" pbuild::add_configure_args "--disable-threads" pbuild::add_configure_args "--disable-shared" diff --git a/build b/build index 163a928..c1f2f57 100755 --- a/build +++ b/build @@ -25,6 +25,9 @@ declare -rx DEFAULT_DISTFILES_DIR="var/distfiles" declare -rx DEFAULT_VERSIONS_CONFIG="${CONFIG_DIR}/versions.conf" declare -rx DEFAULT_TMPDIR='var/tmp/${USER}' +# directory where the required tools will be installed (like bash, tclsh, etc) +declare -rx UTILBIN_DIR='libexec' + #----------------------------------------------------------------------------- # get_version() { @@ -228,7 +231,6 @@ Use the option --force to override.\nAborting..." install -d -m 0755 "${PMODULES_HOME}/init" install -d -m 0755 "${PMODULES_HOME}/lib" install -d -m 0755 "${PMODULES_HOME}/libexec" - install -d -m 0755 "${PMODULES_HOME}/sbin" echo "Configuration:" echo " root of Pmodules environment: ${prefix}" echo " Pmodule prefix: ${PMODULES_HOME}" @@ -324,38 +326,47 @@ pmodules::compile() { local config_file="${prefix}/${CONFIG_DIR}/${CONFIG_FILE}" read_config_file config_file + install -d -m 0755 "${PMODULES_HOME}/bin" + install -d -m 0755 "${PMODULES_HOME}/config" + install -d -m 0755 "${PMODULES_HOME}/init" + install -d -m 0755 "${PMODULES_HOME}/lib" + install -d -m 0755 "${PMODULES_HOME}/libexec" echo "Configuration:" echo " root of Pmodules environment: ${prefix}" echo " Pmodule prefix: ${PMODULES_HOME}" - if [[ ! -f "${PMODULES_HOME}/sbin/base64" ]] || [[ ${opt_force} == 'yes' ]]; then - build coreutils - fi + #if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/base64" ]] || [[ ${opt_force} == 'yes' ]]; then + # build coreutils + #fi - if [[ ! -f "${PMODULES_HOME}/sbin/xgettext" ]] || [[ ${opt_force} == 'yes' ]]; then - build gettext - fi - - if [[ ! -f "${PMODULES_HOME}/sbin/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then + if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then build getopt fi - if [[ ! -f "${PMODULES_HOME}/sbin/bash" ]] || [[ ${opt_force} == 'yes' ]]; then + if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/bash" ]] || [[ ${opt_force} == 'yes' ]]; then build bash fi - if [[ ! -f "${PMODULES_HOME}/sbin/find" ]] || [[ ${opt_force} == 'yes' ]]; then + if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/find" ]] || [[ ${opt_force} == 'yes' ]]; then build findutils fi - if [[ ! -e "${PMODULES_HOME}/sbin/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then + if [[ ! -e "${PMODULES_HOME}/${UTILBIN_DIR}/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then build Tcl fi if [[ ! -e "${PMODULES_HOME}/libexec/modulecmd.bin" ]] || [[ ${opt_force} == 'yes' ]]; then build modules fi + rm -rf "${PMODULES_HOME}/include" + rm -rf "${PMODULES_HOME}/lib/"*.a + rm -rf "${PMODULES_HOME}/lib/"*.la + rm -rf "${PMODULES_HOME}/lib/bash" + rm -rf "${PMODULES_HOME}/lib/pkginfo" + rm -rf "${PMODULES_HOME}/man" + rm -rf "${PMODULES_HOME}/share" + echo "Done..." } @@ -439,7 +450,7 @@ pmodules::install() { sed_cmd+="s:@MODULES_VERSION@:${MODULES_VERSION}:g;" sed_cmd+="s:@PMODULES_DISTFILESDIR@:${PMODULES_DISTFILESDIR}:g;" sed_cmd+="s:@PMODULES_TMPDIR@:${PMODULES_TMPDIR}:g;" - sed_cmd+="s:@TCLSHDIR@:${PMODULES_HOME}/sbin:g;" + sed_cmd+="s:@TCLSHDIR@:${PMODULES_HOME}/${UTILBIN_DIR}:g;" sed_cmd+="s:@pager@::g;" sed_cmd+="s:@pageropts@::g;" sed_cmd+="s:@etcdir@:${PMODULES_ROOT}/${CONFIG_DIR}:g;" @@ -447,6 +458,9 @@ pmodules::install() { sed_cmd+="s:@prefix@:${PMODULES_HOME}:g;" sed_cmd+="s:@initdir@:${PMODULES_HOME}/init:g;" sed_cmd+="s:@MODULES_RELEASE@:${PMODULES_VERSION}:g;" + sed_cmd+="s:@BASH@:${PMODULES_HOME}/${UTILBIN_DIR}/bash:g;" + sed_cmd+="s:@MODULECMD@:${PMODULES_HOME}/${UTILBIN_DIR}/modulecmd.bash:g;" + sed_cmd+="s:@MODMANAGE@:${PMODULES_HOME}/${UTILBIN_DIR}/modmanage.bash:g;" sed "${sed_cmd}" "${SRC_DIR}/profile.bash.in" > "${PMODULES_ROOT}/${CONFIG_DIR}/profile.bash-${PMODULES_VERSION}" sed "${sed_cmd}" "${SRC_DIR}/profile.csh.in" > "${PMODULES_ROOT}/${CONFIG_DIR}/profile.csh-${PMODULES_VERSION}" @@ -462,18 +476,21 @@ pmodules::install() { test -e "${PMODULES_ROOT}/${CONFIG_DIR}/profile.zsh" || \ install -m 0644 "$_-${PMODULES_VERSION}" "$_" + sed "${sed_cmd}" "${SRC_DIR}/modulecmd.in" > "${PMODULES_HOME}/bin/modulecmd" + chmod 0755 "${PMODULES_HOME}/bin/modulecmd" sed "${sed_cmd}" "${SRC_DIR}/modulecmd.bash.in" > "${PMODULES_HOME}/libexec/modulecmd.bash" chmod 0755 "${PMODULES_HOME}/libexec/modulecmd.bash" sed "${sed_cmd}" "${SRC_DIR}/modulecmd.tcl.in" > "${PMODULES_HOME}/libexec/modulecmd.tcl" chmod 0755 "${PMODULES_HOME}/libexec/modulecmd.tcl" + sed "${sed_cmd}" "${SRC_DIR}/modbuild.in" > "${PMODULES_HOME}/bin/modbuild" chmod 0755 "${PMODULES_HOME}/bin/modbuild" + + sed "${sed_cmd}" "${SRC_DIR}/modmanage.in" > "${PMODULES_HOME}/bin/modmanage" + chmod 0755 "${PMODULES_HOME}/bin/modmanage" sed "${sed_cmd}" "${SRC_DIR}/modmanage.bash.in" > "${PMODULES_HOME}/libexec/modmanage.bash" chmod 0755 "${PMODULES_HOME}/libexec/modmanage.bash" - install -m 0755 "${SRC_DIR}/modulecmd" "${PMODULES_HOME}/bin" - install -m 0755 "${SRC_DIR}/modmanage" "${PMODULES_HOME}/bin" - install -m 0644 "${SRC_DIR}/bash" "${PMODULES_HOME}/init" install -m 0644 "${SRC_DIR}/bash_completion" "${PMODULES_HOME}/init" install -m 0644 "${SRC_DIR}/csh" "${PMODULES_HOME}/init" @@ -487,7 +504,7 @@ pmodules::install() { install -m 0644 "${SRC_DIR}/libmodules.tcl" "${PMODULES_HOME}/lib/Pmodules" { - PATH="${PMODULES_HOME}/sbin:${PATH}" + PATH="${PMODULES_HOME}/${UTILBIN_DIR}:${PATH}" cd "${PMODULES_HOME}/lib/Pmodules" "${BOOTSTRAP_DIR}/mkindex.tcl" } diff --git a/config/versions.conf b/config/versions.conf index 00777ad..2311805 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -1,8 +1,8 @@ -bash 5.1-rc3 +bash 5.1 coreutils 8.31 findutils 4.7.0 getopt 1.1.6 gettext 0.21 modules 3.2.10.1 -Pmodules 1.0.0rc7 +Pmodules 1.0.0rc8 Tcl 8.6.10 From d8815d2a29d59a1b0feb02b5accdc69c32f3e6b3 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 18 Mar 2021 13:32:55 +0100 Subject: [PATCH 08/30] init files for unsupported shells removed --- Tools/modules/build | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Tools/modules/build b/Tools/modules/build index 8aaaeae..8a1f45f 100755 --- a/Tools/modules/build +++ b/Tools/modules/build @@ -29,18 +29,19 @@ pbuild::configure() { } pbuild::post_install() { - rm -v ${PREFIX}/Modules/bin/add.modules - rm -v ${PREFIX}/Modules/bin/mkroot - rm -rfv ${PREFIX}/Modules/modulefiles - mv -v ${PREFIX}/Modules/share/man/man1/module.1 ${PREFIX}/share/man/man1 - mv -v ${PREFIX}/Modules/share/man/man4/modulefile.4 ${PREFIX}/share/man/man4 - rmdir ${PREFIX}/Modules/bin - rmdir ${PREFIX}/Modules/share/man/man1 - rmdir ${PREFIX}/Modules/share/man/man4 - rmdir ${PREFIX}/Modules/share/man - rmdir ${PREFIX}/Modules/share - rmdir ${PREFIX}/Modules - cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 + rm -v "${PREFIX}/Modules/bin/add.modules" + rm -v "${PREFIX}/Modules/bin/mkroot" + rm -rfv "${PREFIX}/Modules/modulefiles" + mv -v "${PREFIX}/Modules/share/man/man1/module.1 ${PREFIX}/share/man/man1" + mv -v "${PREFIX}/Modules/share/man/man4/modulefile.4 ${PREFIX}/share/man/man4" + rmdir "${PREFIX}/Modules/bin" + rmdir "${PREFIX}/Modules/share/man/man1" + rmdir "${PREFIX}/Modules/share/man/man4" + rmdir "${PREFIX}/Modules/share/man" + rmdir "${PREFIX}/Modules/share" + rmdir "${PREFIX}/Modules" + rm -f "${PREIX}/init/{ksh,perl.pm,python.py,ruby.rb,cmake,.modulespath}" + cp -v "${BUILD_DIR}/modulecmd" "${PREFIX}/libexec/modulecmd.bin" || exit 1 } # fake module command From fcfbe187fdf23aefca196c7c4e0771cf63e00cee Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 18 Mar 2021 13:34:58 +0100 Subject: [PATCH 09/30] build and install findutils and getopt on macOS/Darwin only --- build | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/build b/build index c1f2f57..5f0b667 100755 --- a/build +++ b/build @@ -340,18 +340,20 @@ pmodules::compile() { # build coreutils #fi - if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then - build getopt + if [[ "${OS}" == 'Darwin' ]]; then + if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/getopt" ]] || [[ ${opt_force} == 'yes' ]]; then + build getopt + fi + + if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/find" ]] || [[ ${opt_force} == 'yes' ]]; then + build findutils + fi fi if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/bash" ]] || [[ ${opt_force} == 'yes' ]]; then build bash fi - if [[ ! -f "${PMODULES_HOME}/${UTILBIN_DIR}/find" ]] || [[ ${opt_force} == 'yes' ]]; then - build findutils - fi - if [[ ! -e "${PMODULES_HOME}/${UTILBIN_DIR}/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then build Tcl fi From af0f8e9e0c9f513f4017222ce2220394142b59ae Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 18 Mar 2021 13:36:34 +0100 Subject: [PATCH 10/30] build and install findutils and getopt only for macOS/Darwin --- build | 1 + 1 file changed, 1 insertion(+) diff --git a/build b/build index 5f0b667..d882edc 100755 --- a/build +++ b/build @@ -5,6 +5,7 @@ if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then exit 1 fi +declare -r OS=$(uname -s) declare BOOTSTRAP_DIR=$(dirname "$0") source "${BOOTSTRAP_DIR}/Pmodules/libstd.bash" || { echo "Oops!" 1>&2; exit 42; } From 7f5e6c75c36d2c5a47c1c7d05779336b19f81fcd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 18 Mar 2021 13:37:42 +0100 Subject: [PATCH 11/30] install modulefile for Pmodules --- Pmodules/modulefile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Pmodules/modulefile diff --git a/Pmodules/modulefile b/Pmodules/modulefile new file mode 100644 index 0000000..65aa4c1 --- /dev/null +++ b/Pmodules/modulefile @@ -0,0 +1,10 @@ +#%Module1.0 + +module-whatis "Pmodules environment" +module-url "https://gitlab.psi.ch/Pmodules/src" +module-license "GNU GPL v2" +module-maintainer "Achim Gsell " + +module-help " +Pmodules are a hierarchical module environment based on Environment Modules. +" From 7dcb8744e5fa4ed4fe7c0b2a761484b2bbda0220 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 18 Mar 2021 13:38:27 +0100 Subject: [PATCH 12/30] install modulefile for Pmodules --- build | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build b/build index d882edc..5e7febf 100755 --- a/build +++ b/build @@ -6,6 +6,7 @@ if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then fi declare -r OS=$(uname -s) + declare BOOTSTRAP_DIR=$(dirname "$0") source "${BOOTSTRAP_DIR}/Pmodules/libstd.bash" || { echo "Oops!" 1>&2; exit 42; } @@ -228,7 +229,6 @@ Use the option --force to override.\nAborting..." read_config_file 'config_file' install -d -m 0755 "${PMODULES_HOME}/bin" - install -d -m 0755 "${PMODULES_HOME}/config" install -d -m 0755 "${PMODULES_HOME}/init" install -d -m 0755 "${PMODULES_HOME}/lib" install -d -m 0755 "${PMODULES_HOME}/libexec" @@ -328,7 +328,6 @@ pmodules::compile() { local config_file="${prefix}/${CONFIG_DIR}/${CONFIG_FILE}" read_config_file config_file install -d -m 0755 "${PMODULES_HOME}/bin" - install -d -m 0755 "${PMODULES_HOME}/config" install -d -m 0755 "${PMODULES_HOME}/init" install -d -m 0755 "${PMODULES_HOME}/lib" install -d -m 0755 "${PMODULES_HOME}/libexec" @@ -512,9 +511,10 @@ pmodules::install() { "${BOOTSTRAP_DIR}/mkindex.tcl" } - mkdir -p "${PMODULES_ROOT}/Tools/modulefiles" + install -m 0755 -d "${PMODULES_ROOT}/Tools/modulefiles/Pmodules" + install -m 0644 "${SRC_DIR}/modulefile" "${PMODULES_ROOT}/Tools/modulefiles/Pmodules/${PMODULES_VERSION}" mkdir -p "${PMODULES_ROOT}/Libraries/modulefiles" - + echo "Done..." } From 6ba7525373c85afc634401743d076df52d949855 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 22 Mar 2021 11:50:24 +0100 Subject: [PATCH 13/30] follow symbolic link while searching for available modules --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 8a70d54..563a587 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -747,7 +747,7 @@ get_available_modules() { if [[ :${use_releases}: =~ :${release}: ]]; then mods+=( "${mod}" ${release} "${dir}/${mod}") fi - done < <(${find} * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") + done < <(${find} -L * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") } done echo "${mods[@]}" From 80ece1c8f64fbb5790aa17f40f1b76ccbc24a863 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 14:55:36 +0200 Subject: [PATCH 14/30] modbuild: extendable hierarchy definition via configuration file - the definition of module hierarchies can now be done in a configuration file. - The former settings are now defaults which can be overriden. --- Pmodules/libpbuild.bash | 198 +++++++++++++++++----------------------- 1 file changed, 85 insertions(+), 113 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 7b48df0..6ca95d2 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -183,9 +183,6 @@ pbuild.verbose() { verbose="$1" } -# module name including path in hierarchy and version -# (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) -declare -x fully_qualified_module_name='' # group this module is in (ex: 'Programming') declare -x GROUP='' @@ -245,76 +242,6 @@ pbuild::supported_os() { pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } -#...................................................................... -# -# compute full module name and installation prefix -# -# The following variables are expected to be set: -# GROUP module group -# P module name -# V module version -# variables defining the hierarchical environment like -# COMPILER and COMPILER_VERSION -# -# The following variables are set in this function -# fully_qualified_module_name -# PREFIX -# -set_full_module_name_and_prefix() { - join_by() { - local IFS="$1" - shift - echo "$*" - } - - [[ -n ${GROUP} ]] || std::die 1 \ - "${module_name}/${module_version}:" \ - "group not set." - - # build module name - # :FIXME: this should be read from a configuration file - local name=() - case ${GROUP} in - Compiler ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - MPI ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - HDF5 ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${HDF5}/${HDF5_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - OPAL ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${OPAL}/${OPAL_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - HDF5_serial ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "hdf5_serial/${HDF5_SERIAL_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - * ) - name+=("${module_name}/${module_version}" ) - ;; - esac - - # set full module name - fully_qualified_module_name=$( join_by '/' "${name[@]}" ) - # set PREFIX of module - PREFIX="${PMODULES_ROOT}/${GROUP}" - local -i i=0 - for ((i=${#name[@]}-1; i >= 0; i--)); do - PREFIX+="/${name[i]}" - done -} ############################################################################## # @@ -330,7 +257,6 @@ pbuild::add_to_group() { "${FUNCNAME}: missing group argument." fi GROUP="$1" - set_full_module_name_and_prefix } ############################################################################## @@ -722,6 +648,10 @@ pbuild::make_all() { set -e local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + # module name including path in hierarchy and version + # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) + local modulefile_dir='' + local modulefile_name='' # # To be able to set environment variables in one of the 'pbuild::TARGET' @@ -783,6 +713,74 @@ pbuild::make_all() { } #...................................................................... + # + # compute full module name and installation prefix + # + # The following variables are expected to be set: + # GROUP module group + # P module name + # V module version + # variables defining the hierarchical environment like + # COMPILER and COMPILER_VERSION + # + # The following variables are set in this function + # modulefile_dir + # modulefile_name + # PREFIX + # + set_full_module_name_and_prefix() { + join_by() { + local IFS="$1" + shift + echo "$*" + } + + [[ -n ${GROUP} ]] || std::die 1 \ + "${module_name}/${module_version}:" \ + "group not set." + + # define defaults if not set in configuration file + : ${Compiler_HIERARCHY:='${COMPILER}/${COMPILER_VERSION}'} + : ${CUDA_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} cuda/${CUDA_VERSION}'} + : ${MPI_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION}'} + : ${HDF5_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION} hdf5/${HDF5_VERSION}'} + : ${HDF5_SERIAL_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} hdf5_serial/${HDF5_SERIAL_VERSION}'} + + # evaluate + local names=() + local vname="${GROUP}_HIERARCHY" + if [[ -n ${!vname} ]]; then + names=( $(eval echo ${!vname}) ) + fi + + modulefile_dir=$(join_by '/' \ + "${PMODULES_ROOT}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${names[@]}" \ + "${module_name}") + modulefile_name="${modulefile_dir}/${module_version}" + PREFIX="${PMODULES_ROOT}/${GROUP}/${module_name}/${module_version}" + local -i i=0 + for ((i=${#names[@]}-1; i >= 0; i--)); do + PREFIX+="/${names[i]}" + done + } + + #...................................................................... + # Select the modulefile to install. Modulefiles can be versioned like + # modulefile-10.2.0 + # modulefile-10.2 + # modulefile-10 + # modulefile + # the most specific modulefile will be selected. Example: + # For a version 10.2.1 the file moduelfile-10.2 would be selected. + # + # Arguments: + # $1 upvar to return the filename + # + # Used gloabal variables: + # VERSIONS + # BUILDBLOCK_DIR + # find_modulefile() { local "$1" local fname='' @@ -924,9 +922,11 @@ pbuild::make_all() { } #...................................................................... - # Install modulefile + # Install modulefile in ${PMODULES_ROOT}/${GROUP}/modulefiles/... + # + # Arguments + # none install_modulefile() { - local src='' find_modulefile src if (( $? != 0 )); then @@ -936,34 +936,16 @@ pbuild::make_all() { "skipping modulefile installation ..." return fi - # assemble name of modulefile - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install modulefile - local -r dstdir=${dst%/*} - std::info \ "%s " \ "${module_name}/${module_version}:" \ - "installing modulefile in '${dstdir}' ..." - mkdir -p "${dstdir}" - install -m 0444 "${src}" "${dst}" + "installing modulefile '${modulefile_name}' ..." + mkdir -p "${modulefile_dir}" + install -m 0444 "${src}" "${modulefile_name}" } install_release_file() { - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install release file - local -r dstdir=${dst%/*} - mkdir -p "${dstdir}" - - local -r release_file="${dst%/*}/.release-${module_version}" + local -r release_file="${modulefile_dir}/.release-${module_version}" if [[ -r "${release_file}" ]]; then local release @@ -1121,24 +1103,14 @@ pbuild::make_all() { "removing all files in '${PREFIX}' ..." [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} fi - - # assemble name of modulefile - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install modulefile - local -r dstdir=${dst%/*} - - if [[ -e "${dst}" ]]; then + if [[ -e "${modulefile_name}" ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ - "removing modulefile '${dst}' ..." - [[ "${dry_run}" == 'no' ]] && rm -v "${dst}" + "removing modulefile '${modulefile_name}' ..." + [[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}" fi - local release_file="${dstdir}/.release-${module_version}" + local release_file="${modulefile_dir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then std::info \ "%s " \ @@ -1146,7 +1118,7 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p "${dstdir}" 2>/dev/null || : + rmdir -p "${modulefile_dir}" 2>/dev/null || : } ######################################################################## From aca3e2c8b597e8fdd2f4d1a1ce6e95c0530d2122 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:05:26 +0200 Subject: [PATCH 15/30] libstd: unused std::upvars() removed --- Pmodules/libstd.bash | 66 -------------------------------------------- 1 file changed, 66 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index be491b6..eb8525d 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -205,72 +205,6 @@ std::upvar() { fi } - -# Assign variables one scope above the caller -# Usage: local varname [varname ...] && -# upvars [-v varname value] | [-aN varname [value ...]] ... -# Available OPTIONS: -# -aN Assign next N values to varname as array -# -v Assign single value to varname -# Return: 1 if error occurs -# Example: -# -# f() { local a b; g a b; declare -p a b; } -# g() { -# local c=( foo bar ) -# local "$1" "$2" && upvars -v $1 A -a${#c[@]} $2 "${c[@]}" -# } -# f # Ok: a=A, b=(foo bar) -# -std::upvars() { - if ! (( $# )); then - echo "${FUNCNAME[0]}: usage: ${FUNCNAME[0]} [-v varname"\ - "value] | [-aN varname [value ...]] ..." 1>&2 - return 2 - fi - while (( $# )); do - case $1 in - -a*) - # Error checking - [[ ${1#-a} ]] || { echo "bash: ${FUNCNAME[0]}: \`$1': missing"\ - "number specifier" 1>&2; return 1; } - printf %d "${1#-a}" &> /dev/null || { echo "bash:"\ - "${FUNCNAME[0]}: \`$1': invalid number specifier" 1>&2 - return 1; } - # Assign array of -aN elements - [[ "$2" ]] && unset -v "$2" && eval $2=\(\"\${@:3:${1#-a}}\"\) && - shift $((${1#-a} + 2)) || { echo "bash: ${FUNCNAME[0]}:"\ - "\`$1${2+ }$2': missing argument(s)" 1>&2; return 1; } - ;; - -v) - # Assign single value - [[ "$2" ]] && unset -v "$2" && eval $2=\"\$3\" && - shift 3 || { echo "bash: ${FUNCNAME[0]}: $1: missing"\ - "argument(s)" 1>&2; return 1; } - ;; - --help) echo "\ -Usage: local varname [varname ...] && - ${FUNCNAME[0]} [-v varname value] | [-aN varname [value ...]] ... -Available OPTIONS: --aN VARNAME [value ...] assign next N values to varname as array --v VARNAME value assign single value to varname ---help display this help and exit ---version output version information and exit" - return 0 ;; - --version) echo "\ -${FUNCNAME[0]}-0.9.dev -Copyright (C) 2010 Freddy Vulto -License GPLv3+: GNU GPL version 3 or later -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law." - return 0 ;; - *) - echo "bash: ${FUNCNAME[0]}: $1: invalid option" 1>&2 - return 1 ;; - esac - done -} - std.get_os_release_linux() { local lsb_release=$(which lsb_release) local ID='' From 1497ca647de2c73e50c8377d9fb1f0e7b90f9483 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:06:06 +0200 Subject: [PATCH 16/30] version set to 1.0.0rc9 --- config/versions.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/versions.conf b/config/versions.conf index 2311805..442193e 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -4,5 +4,5 @@ findutils 4.7.0 getopt 1.1.6 gettext 0.21 modules 3.2.10.1 -Pmodules 1.0.0rc8 +Pmodules 1.0.0rc9 Tcl 8.6.10 From f24836958145f559aab41f0a80ff79d02c892ca4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:07:50 +0200 Subject: [PATCH 17/30] modulecmd: remove tmp file created in module search --- Pmodules/modulecmd.bash.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 563a587..3eaab5f 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -21,6 +21,7 @@ sort=$(PATH=/bin:/usr/bin /usr/bin/which sort) declare -r sort awk=$(PATH=/bin:/usr/bin /usr/bin/which awk) declare -r awk +rm=$(PATH=/bin:/usr/bin /usr/bin/which rm) if [[ $(uname -s) == 'Darwin' ]]; then declare -r getopt="${libexecdir}/getopt" declare -r find="${libexecdir}/find" @@ -1755,7 +1756,7 @@ subcommand_search() { done done print_result "${tmpfile}" - #rm -f "${tmpfile}" + ${rm} -f "${tmpfile}" } while (( $# > 0 )); do From 1cc1db14779f16ed46d4481bfba2089e1f174221 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:09:35 +0200 Subject: [PATCH 18/30] modulecmd: unloading/purging a Pmodules module bug fixed - the environment variable PMODULE_HOME must not be unset if a Pmodules module is unloaded or purged --- Pmodules/modulecmd.bash.in | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 3eaab5f..a7d220d 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -609,7 +609,15 @@ subcommand_unload() { "${CMD}" "${subcommand}" \ "missing argument" fi - + + # The module() function uses PMODULES_HOME to call modulecmd. + # If a Pmodules module is unloaded this evnvironment variable + # will be unset. In consequence the module() function would + # fail. Instead of comparing the name of the module to unload + # with 'Pmodules', we save the value and set it at the end of + # the loop again, if it has been unset. + local saved_home="${PMODULES_HOME}" + local arg for arg in "${args[@]}"; do local output=$("${modulecmd}" "${Shell}" 'unload' "${arg}") @@ -623,6 +631,10 @@ subcommand_unload() { ;; esac done + if [[ -z ${PMODULES_HOME} ]]; then + PMODULES_HOME=${saved_home} + export_env 'PMODULES_HOME' + fi } ############################################################################## @@ -1398,10 +1410,15 @@ subcommand_purge() { "${CMD}" "${subcommand}" \ "no arguments allowd" fi + # we cannot unset PMODULES_HOME, otherwise the module function + # would fail. + local saved_home="${PMODULES_HOME}" "${modulecmd}" "${Shell}" "${subcommand}" reset_modulepath reset_used_groups - export_env MODULEPATH + PMODULES_HOME="${saved_home}" + + export_env MODULEPATH PMODULES_HOME } ############################################################################## From 96a7ca406a3720c0bf20803b3bcc8af64dd91ceb Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:12:50 +0200 Subject: [PATCH 19/30] modulecmd: use upvar to return avail modules --- Pmodules/modulecmd.bash.in | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index a7d220d..ba2e4fc 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -741,17 +741,17 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local -r module="$1" - local -r use_releases="${2:-${UsedReleases}}" - shift 2 - local -a dirs=( "$@" ) + local var="$1" + local -r module="$2" + local -r use_releases="${3:-${UsedReleases}}" + shift 3 # in the for loop below we use $@ to loop over the directories local -a mods=() local release local dir='' - for dir in "${dirs[@]}"; do - test -d "${dir}" || return 0 + for dir in "$@"; do + test -d "${dir}" || continue { cd "${dir}" while read mod; do @@ -763,7 +763,7 @@ get_available_modules() { done < <(${find} -L * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") } done - echo "${mods[@]}" + std::upvar ${var} "${mods[@]}" } ############################################################################## @@ -945,10 +945,11 @@ subcommand_avail() { local string for string in "${pattern[@]}"; do for dir in "${modulepath[@]}"; do - mods=( $( get_available_modules \ - "${string}" \ - "${opt_use_releases}" \ - "${dir}" ) ) + get_available_modules \ + mods \ + "${string}" \ + "${opt_use_releases}" \ + "${dir}" [[ ${#mods[@]} == 0 ]] && continue ${output_function} done @@ -1732,11 +1733,12 @@ subcommand_search() { # with respect to the requested releases # tmpfile: module/version release group group- # dependencies... - local mods=( $( get_available_modules \ - "${module}" \ - "${opt_use_releases}" \ - "${modulepath[@]}" \ - ) ) + local mods + get_available_modules \ + mods \ + "${module}" \ + "${opt_use_releases}" \ + "${modulepath[@]}" \ for (( i=0; i<${#mods[@]}; i+=3 )); do local name=${mods[i]} From b2c51afe39494c8b10f6bcfffadd0367a41938ac Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:15:07 +0200 Subject: [PATCH 20/30] modulecmd: update usage output for sub-command search --- Pmodules/modulecmd.bash.in | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index ba2e4fc..8501996 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1524,6 +1524,12 @@ USAGE: for modules whose name match the argument. SWITCHES: + -a|--all-releases + Search within all releases. + + --all-deps + Show all dependecies + --no-header Suppress output of a header. @@ -1532,8 +1538,8 @@ SWITCHES: switch multiple times. Without this switch, the used releases will be searched. - -a|--all-releases - Search within all releases. + --verbose + vebose output --with=STRING Search for modules compiled with modules matching string. The @@ -1543,9 +1549,6 @@ SWITCHES: lists all modules in the hierarchy compiled with gcc 4.8.3. - --verbose - vebose output - --wrap wrap output ' From 85894a6f222caeb18064cb5b4366030a5a3424c4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:16:25 +0200 Subject: [PATCH 21/30] modbuild: add option to pass use-flags --- Pmodules/modbuild.in | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index fbff037..2d30689 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -229,6 +229,13 @@ parse_args() { --system=* ) opt_system="${1/*=}" ;; + --use-flags ) + USE_FLAGS="y:$2:" + shift + ;; + --use-flags=* ) + USE_FLAGS=":${1/--use-flags=}:" + ;; --with ) opt_with_modules+=( "$2" ) shift From 34af38d21687daf99bdd52f4b92a3e9517dc96c7 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:17:01 +0200 Subject: [PATCH 22/30] modbuild: libpbuild_dyn.bash is not required any more --- Pmodules/modbuild.in | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 2d30689..6aa9543 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -416,11 +416,6 @@ pbuild.update_modulefiles "${opt_update_modulefiles}" pbuild.system "${opt_system}" pbuild.verbose "${opt_verbose}" -source libpbuild_dyn.bash || \ - std::die 3 "Oops: cannot source library -- '$_'" - -# source build configuration, -# must be done before sourcing libpbuild! if [[ "${opt_bootstrap}" == 'yes' ]]; then test -d "${BUILDBLOCK_DIR}/../../config" && PATH+=":$_" else From e1bd5e8986e930c3ad56e6063f49764ac0351383 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:18:24 +0200 Subject: [PATCH 23/30] libpbuild: handle use-flags passed as argument to modbuild --- Pmodules/libpbuild.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 6ca95d2..78e085f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1182,13 +1182,13 @@ pbuild.init_env() { V_MINOR='' # second number in version string (or empty) V_PATCHLVL='' # third number in version string (or empty) V_RELEASE='' # module release (or empty) - USE_FLAGS='' # architectures (or empty) + : ${USE_FLAGS:=''} # architectures (or empty) local tmp='' if [[ "$v" =~ "_" ]]; then tmp="${v#*_}" - USE_FLAGS=":${tmp//_/:}:" + USE_FLAGS+=":${tmp//_/:}:" v="${v%%_*}" fi V_PKG="${v%%-*}" # version without the release number From 1068fafab1fc15301542c08991193474f204bdcb Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:19:17 +0200 Subject: [PATCH 24/30] libpbuild: handle error if tar fails in pbuild::prep() --- Pmodules/libpbuild.bash | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 78e085f..0ae9142 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -429,13 +429,13 @@ pbuild::prep() { unpack() { local -r file="$1" local -r dir="${2:-${SRC_DIR}}" - ( - if [[ -n "${dir}" ]]; then - mkdir -p "${dir}" - cd "${dir}" - fi - tar -xv --strip-components 1 -f "${file}" - ) + tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || { + rm -f "${file}" + std::die 4 \ + "%s " \ + "${module_name}/${module_version}:" \ + "cannot unpack sources!" + } } patch_sources() { From 36086fdd5724f1f73ae6b080f38eeb0d919ae406 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:37:58 +0200 Subject: [PATCH 25/30] libpbuild: handle error if appling a patch fails in pbuild::prep() --- Pmodules/libpbuild.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 0ae9142..07fee3c 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -447,7 +447,11 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" - patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" + patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ + std::die 4 \ + "%s " \ + "${module_name}/${module_version}:" \ + "error patching sources!" done } if [[ -z "${SOURCE_URLS}" ]]; then From c8d2e065aaf00aae4985931cfbca4c077b8aa065 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:41:58 +0200 Subject: [PATCH 26/30] libpbuild: handle error if make fails while compiling or installing --- Pmodules/libpbuild.bash | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 07fee3c..336b9f6 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -586,7 +586,10 @@ pbuild::pre_compile() { } pbuild::compile() { - make -j${JOBS} + make -j${JOBS} || \ + std::die 3 \ + "%s " "${module_name}/${module_version}:" \ + "compilation failed!" } pbuild::post_compile() { @@ -598,7 +601,10 @@ pbuild::pre_install() { } pbuild::install() { - make install + make install || \ + std::die 3 \ + "%s " "${module_name}/${module_version}:" \ + "compilation failed!" } pbuild::install_shared_libs() { From 837bf74522284385d203e5077043c9121cf22163 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:45:06 +0200 Subject: [PATCH 27/30] libpbuild: improve test whether a module exist or not --- Pmodules/libpbuild.bash | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 336b9f6..cc7b03e 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1142,8 +1142,9 @@ pbuild::make_all() { check_supported_os check_supported_compilers set_full_module_name_and_prefix - if module_exists "${module_name}/${module_version}" \ - && [[ ${forece_rebuild} != 'yes' ]]; then + if [[ -e "${modulefile_name}" ]] \ + && [[ -d ${PREFIX} ]] \ + && [[ ${force_rebuild} != 'yes' ]]; then if [[ "${module_release}" == 'removed' ]]; then remove_module else @@ -1260,18 +1261,6 @@ pbuild.init_env() { configure_with='undef' } -#.............................................................. -# -# Test whether a module with the given name already exists. -# -# Arguments: -# $1: module name/version -# -module_exists() { - [[ -n $("${MODULECMD}" bash avail -m "$1" \ - 2>&1 1>/dev/null) ]] -} - pbuild.build_module() { module_name="$1" module_version="$2" From 3c08327cdbcdb668d95bf59eece1e748b53679fe Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:45:57 +0200 Subject: [PATCH 28/30] libpbuild: add a bit more comments to post_install function --- Pmodules/libpbuild.bash | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index cc7b03e..424dbb9 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -806,11 +806,17 @@ pbuild::make_all() { } #...................................................................... - # non-redefinable post-install + # non-redefinable post-install. Install: + # - documentation files as defined in the build-script + # - modulefile and file with release + # . post_install() { #.............................................................. # install the doc-files specified in the build-script # + # Arguments: + # none + # install_doc() { if [[ -z "${MODULE_DOCFILES}" ]]; then for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do From 6dee9b74daf1e8a2ed826678dc521fb1ad0e544f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:47:10 +0200 Subject: [PATCH 29/30] libpbuild: don't print module name and all deps when we are done --- Pmodules/libpbuild.bash | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 424dbb9..0bc0820 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1459,11 +1459,6 @@ pbuild.build_module() { pbuild.init_env "${module_name}" "${module_version}" pbuild::make_all - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "done!" std::info "* * * * *\n" } From 7f089633c7401881ca7d82b0ab94a0dc5b801585 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 19 Apr 2021 15:48:18 +0200 Subject: [PATCH 30/30] libpbuild: better doc what installing build-block files does --- Pmodules/libpbuild.bash | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 0bc0820..3404b63 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -843,12 +843,14 @@ pbuild::make_all() { } #.............................................................. - # install build-block files + # install build-block files # - modulefile # - build-script - # - build dependencies + # - run-time and build dependencies + # in ${PREFIX}/share/${GROUP}/${module_name} # - # Skip installation if modulefile does not exist. + # Arguments: + # none # install_pmodules_files() { local modulefile=''