From 50c9b6d692d8ba6ae35520705856aeae632ed3e3 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 17 Sep 2021 15:32:56 +0200 Subject: [PATCH 01/28] modulecmd: fix output of module avail - for directories outside the hierarchy the header did not include the directory name. --- Pmodules/modulecmd.bash.in | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index fc51ad6..3382b36 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1042,8 +1042,12 @@ subcommand_avail() { local string for string in "${pattern[@]}"; do for dir in "${modulepath[@]}"; do - local group="${dir/${PMODULES_ROOT}\/}" - group="${group%%/*}" + if [[ ${dir} =~ ${PMODULES_ROOT} ]]; then + local group="${dir/${PMODULES_ROOT}\/}" + group="${group%%/*}" + else + local group="${dir}" + fi if (( ${#opt_groups[@]} > 0 )) && [[ ! -v opt_groups[${group}] ]]; then continue fi From c22422aca93d11d0addd9beae31995305d56f6ac Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 28 Oct 2021 18:20:58 +0200 Subject: [PATCH 02/28] fixes in module search sub-command - error fixed in printing the modulefile - sanity checks added if --src is specified --- Pmodules/modulecmd.bash.in | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index fc51ad6..95015f7 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1767,7 +1767,7 @@ subcommand_search() { print_modulefiles() { fmt='' func_print_header='print_header_none' - func_print_line='print_header_none' + func_print_line='print_line_modulefile' } print_header_none() { @@ -1775,16 +1775,7 @@ subcommand_search() { } print_line_modulefile() { - local line=( "$@" ) - # group first - local out="${line[2]}/" - # add directory of modulefiles - out+="${PMODULES_MODULEFILES_DIR}/" - for d in "${line[@]:3}"; do - out+="$d/" - done - out+="${line[0]}" - std::info "${out}" + std::info "$4" } print_line_csv() { @@ -1948,10 +1939,26 @@ subcommand_search() { -a | --all-releases ) opt_use_rel_stages+="${ReleaseStages}" ;; - --src ) - # :FIXME: do we have to add some sanity checks here? - src_prefix=$2 - shift + --src | --src=*) + if [[ "$1" == --src ]]; then + local src_prefix="$2" + shift + else + local src_prefix="${1/--src=}" + fi + if [[ ! -e "${src_prefix}" ]]; then + std::die 1 "%s %s: %s -- %s" \ + "${CMD}" 'search' \ + "illegal value for --src option" \ + "${src_prefix} does not exist" + fi + if [[ ! -d "${src_prefix}" ]]; then + std::die 1 "%s %s: %s -- %s" \ + "${CMD}" 'search' \ + "illegal value for --src option" \ + "${src_prefix} is not a directory" + fi + src_prefix=$(std::get_abspath "${src_prefix}") ;; -v | --verbose ) opt_print_verbose='yes' From b241031fc13aac99de07b5287e4c551070503818 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 28 Oct 2021 18:32:49 +0200 Subject: [PATCH 03/28] fix std::split_fname() in libstd.bash - if path start with a leading slash, remove it, otherwise the first component of the splitted path is empty --- Pmodules/libstd.bash | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index fedd0c4..bcec362 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -135,11 +135,26 @@ std::replace_path () { # # split file name # +# Args: +# $1 upvar +# $2 fname (=${@: -1}) +# or +# $1 upvar +# $2 number of components +# $3 fname (=${@: -1}) +# std::split_fname() { - local -r savedIFS="${IFS}" + local "$1" + local -r fname="${@: -1}" + if [[ "${fname:0:1}" == '/' ]]; then + local -r tmp="${fname:1}" + else + local -r tmp="${fname}" + fi + IFS='/' - local std__split_fname_result__=( $(echo "${@: -1}") ) - IFS=${savedIFS} + local std__split_fname_result__=( ${tmp} ) + unset IFS eval $1=\(\"\${std__split_fname_result__[@]}\"\) if (( $# >= 3 )); then eval $2=${#std__split_fname_result__[@]} From d4aaa4a1378bc8115bda9e44adb4a86e80f1d430 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 28 Oct 2021 18:35:48 +0200 Subject: [PATCH 04/28] bugfix in modmanage.in - missing dopple-quote added --- Pmodules/modmanage.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modmanage.in b/Pmodules/modmanage.in index 10508ad..4139b5d 100644 --- a/Pmodules/modmanage.in +++ b/Pmodules/modmanage.in @@ -2,4 +2,4 @@ unset BASH_ENV -"@BASH@" --noprofile --norc "@MODMANAGE@ "$@" +"@BASH@" --noprofile --norc "@MODMANAGE@" "$@" From 508380095838439e673c8e26b6c3cb783e4fd180 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 28 Oct 2021 18:36:46 +0200 Subject: [PATCH 05/28] modmanage.bash.in: fixes --- Pmodules/modmanage.bash.in | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index 6e11be2..abaea84 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -351,7 +351,7 @@ subcommand_init() { [[ -n "${PMODULES_HOME}" ]] && [[ -n "${PMODULES_VERSION}" ]] || \ std::die 1 " -Error: the module environment you are going to use as so urce has not been +Error: the module environment you are going to use as source has not been initialized properly!" [[ -d "${src_prefix}/${PMODULES_CONFIG_DIR}" ]] && @@ -360,7 +360,7 @@ initialized properly!" Error: the module environment '${src_prefix}' has not been initialized properly!" } - local src='' + local src_prefix='' local target_prefixes=() local user='' local opts='' @@ -373,7 +373,7 @@ Error: the module environment '${src_prefix}' has not been initialized properly! while (($# > 0)); do case $1 in --src ) - src=$2 + src_prefix=$2 shift ;; --user ) @@ -402,14 +402,14 @@ Error: the module environment '${src_prefix}' has not been initialized properly! std::die 1 "Error: no target directory specified!" # if source directory is not passed as argument, derive it from script name - if [[ -z "${src}" ]]; then - src=$(cd "${bindir}/../../../.." && pwd) + if [[ -z "${src_prefix}" ]]; then + src_prefix=$(cd "${bindir}/../../../.." && pwd) fi - [[ -d "${src}" ]] || \ - std::die 1 "Error: ${src}: source directory does not exist!" - [[ -r "${src}/config/profile.bash" ]] || \ - std::die 1 "Error: ${src}: shell profile does not exist or is not readable!" - source "${src}/config/profile.bash" + [[ -d "${src_prefix}" ]] || \ + std::die 1 "Error: ${src_prefix}: source directory does not exist!" + [[ -r "${src_prefix}/config/profile.bash" ]] || \ + std::die 1 "Error: ${src_prefix}: shell profile does not exist or is not readable!" + source "${src_prefix}/config/profile.bash" local -i euid=$(id -u) if (( euid == 0 )); then @@ -601,15 +601,15 @@ subcommand_install() { # compute filename with dependencies of given module local -i i=0 n=0 std::split_fname items n "${modulefile}" - local prefix="${src_prefix}/${items[0]}" + local _prefix="${src_prefix}/${items[3]}" for (( i = n-2; i >= 2; i-=2 )); do - prefix+="/${items[$i]}/${items[i+1]}" + _prefix+="/${items[$i]}/${items[i+1]}" done local tmpfile=$(mktemp /tmp/Pmodules_XXXXXX) - local fname_dependencies="${prefix}/.dependencies" - [[ -r "${prefix}/.dependencies" ]] && cat "$_" > "${tmpfile}" - [[ -r "${prefix}/.install_dependencies" ]] && cat "$_" >> "${tmpfile}" + local fname_dependencies="${_prefix}/.dependencies" + [[ -r "${_prefix}/.dependencies" ]] && cat "$_" > "${tmpfile}" + [[ -r "${_prefix}/.install_dependencies" ]] && cat "$_" >> "${tmpfile}" # loop over all dependecies local dep From 6d6609126f7cdc8b3c19e4e6365cd0319251a2cd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 28 Oct 2021 18:20:58 +0200 Subject: [PATCH 06/28] fixes in module search sub-command - error fixed in printing the modulefile - sanity checks added if --src is specified (cherry picked from commit c22422aca93d11d0addd9beae31995305d56f6ac) --- Pmodules/modulecmd.bash.in | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 3382b36..e009069 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1771,7 +1771,7 @@ subcommand_search() { print_modulefiles() { fmt='' func_print_header='print_header_none' - func_print_line='print_header_none' + func_print_line='print_line_modulefile' } print_header_none() { @@ -1779,16 +1779,7 @@ subcommand_search() { } print_line_modulefile() { - local line=( "$@" ) - # group first - local out="${line[2]}/" - # add directory of modulefiles - out+="${PMODULES_MODULEFILES_DIR}/" - for d in "${line[@]:3}"; do - out+="$d/" - done - out+="${line[0]}" - std::info "${out}" + std::info "$4" } print_line_csv() { @@ -1952,10 +1943,26 @@ subcommand_search() { -a | --all-releases ) opt_use_rel_stages+="${ReleaseStages}" ;; - --src ) - # :FIXME: do we have to add some sanity checks here? - src_prefix=$2 - shift + --src | --src=*) + if [[ "$1" == --src ]]; then + local src_prefix="$2" + shift + else + local src_prefix="${1/--src=}" + fi + if [[ ! -e "${src_prefix}" ]]; then + std::die 1 "%s %s: %s -- %s" \ + "${CMD}" 'search' \ + "illegal value for --src option" \ + "${src_prefix} does not exist" + fi + if [[ ! -d "${src_prefix}" ]]; then + std::die 1 "%s %s: %s -- %s" \ + "${CMD}" 'search' \ + "illegal value for --src option" \ + "${src_prefix} is not a directory" + fi + src_prefix=$(std::get_abspath "${src_prefix}") ;; -v | --verbose ) opt_print_verbose='yes' From 3d326a527aee094cbf069d5cc09d0873a85ab831 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 29 Oct 2021 12:05:41 +0200 Subject: [PATCH 07/28] modulecmd: sort output of sub-command avail numerically --- Pmodules/modulecmd.bash.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index e009069..02d22ab 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -972,6 +972,8 @@ subcommand_avail() { (( n > max_length )) && (( max_length=n )) available_modules+=("${mod}") done + IFS=$'\n' available_modules=($(sort --version-sort <<<"${available_modules[*]}")) + unset IFS local -i span=$(( max_length / 16 + 1 )) # compute column size local -i colsize=$(( span * 16 )) # as multiple of 16 local -i column=$cols # force a line-break From ef5dd9f761e716ea27b221ef3b55403fcf35ec23 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 29 Oct 2021 13:54:22 +0200 Subject: [PATCH 08/28] modulecmd: fixes in option and argument parsing --- Pmodules/modulecmd.bash.in | 62 ++++++++++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 02d22ab..1644675 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -258,7 +258,9 @@ subcommand_generic0() { print_help "${subcommand}" ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -284,7 +286,9 @@ subcommand_generic1() { print_help "${subcommand}" ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -314,7 +318,9 @@ subcommand_generic1plus() { print_help "${subcommand}" ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -477,6 +483,9 @@ subcommand_load() { verbosity_lvl='warn' ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( $1 ) @@ -678,6 +687,9 @@ subcommand_unload() { print_help "${subcommand}" ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -744,6 +756,9 @@ subcommand_swap() { print_help "${subcommand}" ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -798,6 +813,9 @@ subcommand_show() { print_help "${subcommand}" ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1027,7 +1045,12 @@ subcommand_avail() { fi opt_groups[${val}]=1 ;; - -- | '' ) + '' ) + ;; + -- ) + shift 1 + pattern+=( "$@" ) + break ;; * ) pattern+=( "$1" ) @@ -1219,6 +1242,9 @@ subcommand_use() { add2path_func='std::prepend_path' ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1324,6 +1350,9 @@ subcommand_unuse() { print_help "${subcommand}" ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1482,6 +1511,9 @@ subcommand_purge() { print_help "${subcommand}" ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1577,6 +1609,9 @@ subcommand_list() { opts+=( '-t' ) ;; -- ) + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1615,7 +1650,9 @@ subcommand_clear() { print_help "${subcommand}" ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -1973,6 +2010,9 @@ subcommand_search() { opt_wrap='yes' ;; -- ) + shift 1 + modules+=( "$@" ) + break ;; * ) modules+=( "$1" ) @@ -2054,7 +2094,9 @@ subcommand_help() { print_help 'version' ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -2209,7 +2251,9 @@ subcommand_initswitch() { print_help "${subcommand}" ;; -- ) - : + shift 1 + args+=( "$@" ) + break ;; * ) args+=( "$1" ) @@ -2288,7 +2332,7 @@ while (( $# > 0 )); do --debug ) set -x ;; - '' | -- ) + '' ) ;; -* ) opts+=( "$1" ) @@ -2379,7 +2423,7 @@ case ${subcommand} in esac declare options -options=$( "${getopt}" ${Options[${subcommand}]} -- -- "${opts[@]}" "$@" ) \ +options=$( "${getopt}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ || print_help "${subcommand}" eval set -- ${options} subcommand_${Subcommands[$subcommand]} "$@" From bdc9ddf8b5791c5019b7c7bb71188d12c39cb873 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 11:59:21 +0100 Subject: [PATCH 09/28] std:split_fname(): changed arguments and leading slash handling - a leading slash is now removed - the filename is now passed as second argument, the number of parts is returned in an optional third argument --- Pmodules/libstd.bash | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index fedd0c4..f7bee90 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -135,14 +135,27 @@ std::replace_path () { # # split file name # +# Args: +# $1 upvar +# $2 fname +# $3 opt upvar: number of components +# std::split_fname() { - local -r savedIFS="${IFS}" + local parts="$1" + local -r fname="$2" + if [[ "${fname:0:1}" == '/' ]]; then + local -r std__split_fname_tmp="${fname:1}" + else + local -r std__split_fname_tmp="${fname}" + fi + IFS='/' - local std__split_fname_result__=( $(echo "${@: -1}") ) - IFS=${savedIFS} - eval $1=\(\"\${std__split_fname_result__[@]}\"\) + local std__split_fname_result=( ${std__split_fname_tmp} ) + unset IFS + std::upvar ${parts} "${std__split_fname_result[@]}" if (( $# >= 3 )); then - eval $2=${#std__split_fname_result__[@]} + # return number of parts + std::upvar "$3" ${#std__split_fname_result[@]} fi } From 8112e8b6d4fd730e372eb371317c4b91f70514b1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 12:20:17 +0100 Subject: [PATCH 10/28] std::split_fname(): reviewed and renamed to std::split_abs_path() --- Pmodules/libstd.bash | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index f7bee90..6314872 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -133,29 +133,29 @@ std::replace_path () { } # -# split file name +# split an absolute path # # Args: # $1 upvar -# $2 fname +# $2 absolute path # $3 opt upvar: number of components # -std::split_fname() { +std::split_abs_path() { local parts="$1" - local -r fname="$2" - if [[ "${fname:0:1}" == '/' ]]; then - local -r std__split_fname_tmp="${fname:1}" + local -r path="$2" + if [[ "${path:0:1}" == '/' ]]; then + local -r std__split_path_tmp="${path:1}" else - local -r std__split_fname_tmp="${fname}" + std::die 255 "Oops: Internal error in '${FUNCNAME[0]}' called by '${FUNCNAME[1]}' }" fi IFS='/' - local std__split_fname_result=( ${std__split_fname_tmp} ) + local std__split_path_result=( ${std__split_path_tmp} ) unset IFS - std::upvar ${parts} "${std__split_fname_result[@]}" + std::upvar ${parts} "${std__split_path_result[@]}" if (( $# >= 3 )); then # return number of parts - std::upvar "$3" ${#std__split_fname_result[@]} + std::upvar "$3" ${#std__split_path_result[@]} fi } From 6d53069bedb6978a504cf078ed55b68bc92a43be Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 12:25:25 +0100 Subject: [PATCH 11/28] std::split_abs_path() renamed to std::split_abspath() --- Pmodules/libstd.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 6314872..f56ae57 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -140,7 +140,7 @@ std::replace_path () { # $2 absolute path # $3 opt upvar: number of components # -std::split_abs_path() { +std::split_abspath() { local parts="$1" local -r path="$2" if [[ "${path:0:1}" == '/' ]]; then From a67b3cdf5f1069fe39bdb27be2191dc3eadd2531 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 13:44:54 +0100 Subject: [PATCH 12/28] better solution to define used commands implemented --- Pmodules/libstd.bash | 11 +++++++++++ Pmodules/modulecmd.bash.in | 30 +++++++----------------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index f56ae57..5161915 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -35,6 +35,17 @@ std::die() { exit $ec } +std::def_cmds(){ + local path="$1" + shift + for cmd in "$@"; do + eval declare -g ${cmd}=$(PATH="${path}" which $cmd 2>/dev/null) + if [[ -z "${!cmd}" ]]; then + std::die 255 "${cmd} not found" + fi + done +} + # # get answer to yes/no question # diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 1644675..ccde9f6 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -13,30 +13,14 @@ declare prefix=$(dirname "${mydir}") declare -r libdir="${prefix}/lib" declare -r libexecdir="${prefix}/libexec" -base64=$(PATH=/bin:/usr/bin /usr/bin/which base64) -declare -r base64 -mktemp=$(PATH=/bin:/usr/bin /usr/bin/which mktemp) -declare -r mktemp -sort=$(PATH=/bin:/usr/bin /usr/bin/which sort) -declare -r sort -awk=$(PATH=/bin:/usr/bin /usr/bin/which awk) -declare -r awk -rm=$(PATH=/bin:/usr/bin /usr/bin/which rm) -declare -r rm -logger=$(PATH=/bin:/usr/bin /usr/bin/which logger) -declare -r logger - -if [[ $(uname -s) == 'Darwin' ]]; then - declare -r getopt="${libexecdir}/getopt" - declare -r find="${libexecdir}/find" -else - getopt=$(PATH=/bin:/usr/bin /usr/bin/which getopt) - declare -r getopt - find=$(PATH=/bin:/usr/bin /usr/bin/which find) - declare -r find -fi - source "${libdir}/libstd.bash" +source "${libdir}/libpmodules.bash" + +path="/bin:/usr/bin" +[[ $(uname -s) == 'Darwin' ]] && path+=":${libexecdir}" +std::def_cmds "${path}" \ + 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ + 'rm' 'sort' 'find' if [[ ${PMODULES_PURETCL} == yes ]]; then declare -r modulecmd="${libexecdir}/modulecmd.tcl" From d1a66d5e6f838bb5591c3d9a43c531cc309cae23 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 14:05:05 +0100 Subject: [PATCH 13/28] libpmodules.bash reviewed - cleanup code in lib - function we need in modmanage moved to lib - some cleanup in modulecmd --- .../{libpmodules.bash => libpmodules.bash.in} | 0 Pmodules/modulecmd.bash.in | 72 ++----------------- 2 files changed, 7 insertions(+), 65 deletions(-) rename Pmodules/{libpmodules.bash => libpmodules.bash.in} (100%) diff --git a/Pmodules/libpmodules.bash b/Pmodules/libpmodules.bash.in similarity index 100% rename from Pmodules/libpmodules.bash rename to Pmodules/libpmodules.bash.in diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index ccde9f6..5afa784 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1,9 +1,12 @@ #!@BASH@ --noprofile # -#set -o nounset -# we have to unset CDPATH, otherwise 'cd' prints the directoy! -unset CDPATH +PATH='/bin:/usr/bin' +case +unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy! +unset IFS # use default IFS + +shopt -s nullglob # used for some output only declare -r CMD='module' @@ -31,20 +34,11 @@ fi declare verbosity_lvl=${PMODULES_VERBOSITY:-'verbose'} -# use default IFS -unset IFS - -shopt -s nullglob - -declare -A GroupDepths='()' declare Shell='' -declare -A Subcommands -declare -A Options -declare -A Help declare -r pmodules_config_file="${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}/Pmodules.conf" -# the following settings are used if the Pmodules.conf is not available +# the following settings are used if the Pmodules.conf doesn't exist # set groups which should be available after initialization declare -- DefaultGroups='Tools Programming' @@ -55,22 +49,6 @@ declare -- ReleaseStages=':unstable:stable:deprecated:' # set releases which should be available after initialization declare -- DefaultReleaseStages='stable' -# In the dictionary Help we store the help text of each single command -# and for displaying the version. - -# initialize help text of 'module --version' -Help['version']=" -Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@ -Copyright GNU GPL v2 -" - -# -# display help text for command given in $1 -# -print_help() { - echo -e "${Help[$1]}" 1>&2 - std::die 1 -} export_env() { case "${Shell}" in @@ -170,42 +148,6 @@ is_release_stage() { [[ ${ReleaseStages} =~ :$1: ]] } -# -# compute depth of modulefile directory. -# -# Args: -# $1: absolute path of a modulefile directory -# -compute_group_depth () { - local -r dir=$1 - test -d "${dir}" || return 1 - local group=${dir%/*} - local group=${group##*/} - [[ -n "${GroupDepths[${group}]}" ]] && return 0 - local -i depth=$(${find} "${dir}" -depth \( -type f -o -type l \) \ - -printf "%d" -quit 2>/dev/null) - (( depth-=2 )) - # if a group doesn't contain a modulefile, depth is negativ - # :FIXME: better solution? - (( depth < 0 )) && (( depth = 0 )) - GroupDepths[$group]=${depth} - g_env_must_be_saved='yes' -} - -# -# (Re-)Scan available groups in given root and compute group depth's -# -# Args: -# $1: root of modulefile hierarchy -# -scan_groups () { - local -r root="$1" - local moduledir - for moduledir in ${root}/*/${PMODULES_MODULEFILES_DIR}; do - compute_group_depth "${moduledir}" - done -} - # # Check whether argument is a group # From 98ce3112eed6aacecf1df1054f6250514542d285 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 14:13:55 +0100 Subject: [PATCH 14/28] review libpmodules.bash, new functions added, cleanup --- Pmodules/libpmodules.bash.in | 74 +++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index cba80a7..7ad1140 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -1,38 +1,60 @@ #!/bin/bash -if [[ -z ${sbindir} ]]; then - local sbindir=$(dirname "${BASH_SOURCE}") - sbindir=$(cd "${sbindir}"/.. && pwd)"/sbin" -fi +declare PMODULES_MODULEFILES_DIR='modulefiles' +declare -A GroupDepths=() +declare -A Subcommands=() +declare -A Options=() +declare -A Help=() -pmodules::get_options() { - local "$1" - std::upvar $1 $("${sbindir}/getopt" "${@:2}") +# initialize help text of 'module --version' +Help['version']=" +Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@ +Copyright GNU GPL v2 +" + +# +# display help text for command given in $1 +# +print_help() { + echo -e "${Help[$1]}" 1>&2 + std::die 1 } -pmodules::check_env_vars() { - [[ -n "${PMODULES_ROOT}" ]] && - [[ -n "${PMODULES_HOME}" ]] && - [[ -n "${PMODULES_VERSION}" ]] || std::die 1 " -Error: the module environment you are going to use as source has not been -initialized properly!" +# +# compute depth of modulefile directory. +# +# Args: +# $1: absolute path of a modulefile directory +# +compute_group_depth () { + local -r dir=$1 + test -d "${dir}" || return 1 + local group=${dir%/*} + local group=${group##*/} + [[ -n "${GroupDepths[${group}]}" ]] && return 0 + local -i depth=$(${find} "${dir}" -depth \( -type f -o -type l \) \ + -printf "%d" -quit 2>/dev/null) + (( depth-=2 )) + # if a group doesn't contain a modulefile, depth is negativ + # :FIXME: better solution? + (( depth < 0 )) && (( depth = 0 )) + GroupDepths[$group]=${depth} } -pmodules::check_directories() { - local -r src_prefix="$1" - - [[ -d "${src_prefix}" ]] && - [[ -d "${src_prefix}/${PMODULES_CONFIG_DIR}" ]] && - [[ -d "${src_prefix}/Tools/Pmodules/${PMODULES_VERSION}" ]] || std::die 1 " -Error: the module environment '${src_prefix}' has not been initialized properly!" +# +# (Re-)Scan available groups in given root and compute group depth's +# +# Args: +# $1: root of modulefile hierarchy +# +scan_groups () { + local -r root="$1" + local moduledir + for moduledir in ${root}/*/${PMODULES_MODULEFILES_DIR}; do + compute_group_depth "${moduledir}" + done } -pmodules::check_env() { - pmodules::check_env_vars - pmodules::check_directories "${PMODULES_ROOT}" -} - - # Local Variables: # mode: sh # sh-basic-offset: 8 From 1204517a99be8702c7b30e527d7026ccb31f76f1 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 14:27:12 +0100 Subject: [PATCH 15/28] modulecmd: cmd-line arguments and handling reviewed --- Pmodules/modulecmd.bash.in | 97 ++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 1644675..c53db32 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -254,7 +254,7 @@ subcommand_generic0() { local -a args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -282,7 +282,7 @@ subcommand_generic1() { local -a args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -314,7 +314,7 @@ subcommand_generic1plus() { local args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -344,7 +344,7 @@ subcommand_generic1plus() { # Subcommands[add]='load' Subcommands[load]='load' -Options['load']='-l help -o Hfsvwi -l force -l silent -l verbose -l warn -l internal' +Options[load]='-l help -o \?Hfsvw -l force -l silent -l verbose -l warn' Help[load]=' USAGE: module add modulefile... @@ -467,7 +467,7 @@ subcommand_load() { opts=() while (($# > 0)); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -f | --force ) @@ -664,7 +664,7 @@ subcommand_load() { # Subcommands[rm]='unload' Subcommands[unload]='unload' -Options[unload]='-o H -l help' +Options[unload]='-o \?H -l help' Help[unload]=" USAGE: module rm modulefile... @@ -683,7 +683,7 @@ subcommand_unload() { local args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -737,7 +737,7 @@ subcommand_unload() { # Subcommands[switch]='swap' Subcommands[swap]='swap' -Options[swap]='-o H -l help' +Options[swap]='-o \?H -l help' Help[swap]=" USAGE: module switch [modulefile1] modulefile2 @@ -752,7 +752,7 @@ subcommand_swap() { local args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -792,7 +792,7 @@ subcommand_swap() { # Subcommands[display]='show' Subcommands[show]='show' -Options[show]='-o H -l help' +Options[show]='-o \?H -l help' Help[show]=' USAGE: module display modulefile... @@ -809,7 +809,7 @@ subcommand_show() { local args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -874,7 +874,8 @@ get_available_modules() { # avail [-hlt] [...] # Subcommands[avail]='avail' -Options[avail]='-l help -o Hahlmt -l all -l all-release-stages -l human -l long -l machine -l terse' +Options[avail]='-l help -o \?Hahlmtg: -l all -l all-release-stages -l group: ' +Options[avail]+='-l human -l long -l machine -l terse' Help[avail]=" USAGE: module avail [switches] string @@ -897,8 +898,12 @@ SWITCHES: -l|--long Output in long format. + -g|--group= + Output modules available in + -h|--human Output in human readable format. + -m|--machine Output in machine readable format " @@ -1018,7 +1023,7 @@ subcommand_avail() { local val='' while (($# > 0)); do case $1 in - -H | --help | -\? ) + -\? | -H | --help ) print_help "${subcommand}" ;; -a | --all | --all-release-stages ) @@ -1038,15 +1043,13 @@ subcommand_avail() { ;; -g | --group | --group=* ) if [[ $1 == --group=* ]]; then - val="${1/--group=}" + val="${1#--*=}" else val="$2" shift fi opt_groups[${val}]=1 ;; - '' ) - ;; -- ) shift 1 pattern+=( "$@" ) @@ -1092,7 +1095,7 @@ subcommand_avail() { # use [-a|--append|-p|--prepend] [directory|group|release_stage...] # Subcommands[use]='use' -Options[use]='-l help -o Hap -l append -l prepend' +Options[use]='-l help -o \?Hap -l append -l prepend' Help[use]=" USAGE: module use [-a|--append|-p|--prepend] [directory|group|release_stage|...] @@ -1232,7 +1235,7 @@ subcommand_use() { local -a args=() while (( $# > 0)); do case "$1" in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -a | --append ) @@ -1269,7 +1272,7 @@ subcommand_use() { # unuse directory|group|release_stage|... # Subcommands[unuse]='unuse' -Options[unuse]='-o H -l help' +Options[unuse]='-o \?H -l help' Help[unuse]=' unuse directory|group|release... Remove the given modulefiles directory, group, release stage, @@ -1346,7 +1349,7 @@ subcommand_unuse() { local -a args=() while (( $# > 0)); do case "$1" in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -1381,7 +1384,7 @@ subcommand_unuse() { # sub-command # Subcommands[update]='update' -Options[update]='-o H -l help' +Options[update]='-o \?H -l help' Help[update]=' USAGE: module update @@ -1397,7 +1400,7 @@ subcommand_update() { # refresh # Subcommands[refresh]='refresh' -Options[refresh]='-o H -l help' +Options[refresh]='-o \?H -l help' Help[refresh]=' USAGE: module refresh @@ -1488,7 +1491,7 @@ pmodules_init() { # purge # Subcommands[purge]='purge' -Options[purge]='-o H -l help' +Options[purge]='-o \?H -l help' Help[purge]=' USAGE: module purge @@ -1507,7 +1510,7 @@ subcommand_purge() { local -a args=() while (( $# > 0)); do case "$1" in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -1583,7 +1586,7 @@ subcommand_purge() { # list [-hlt] # Subcommands[list]='list' -Options[list]='-l help -o Hhlt -l human -l long -l terse' +Options[list]='-l help -o \?Hhlt -l human -l long -l terse' Help[list]=' USAGE: module list @@ -1596,7 +1599,7 @@ subcommand_list() { local args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -h | --human ) @@ -1633,7 +1636,7 @@ subcommand_list() { # clear # Subcommands[clear]='clear' -Options[clear]='-o H -l help' +Options[clear]='-o \?H -l help' Help[clear]=' USAGE: module clear @@ -1646,7 +1649,7 @@ subcommand_clear() { local -a args=() while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; -- ) @@ -1674,13 +1677,13 @@ subcommand_clear() { # search [switches] [STRING...] # Subcommands[search]='search' -Options[search]='-o aH -l help -l no-header -l print-modulefiles ' +Options[search]='-o a\?H -l help -l no-header -l print-modulefiles ' Options[search]+='-l release-stage: -l with: -l all-release-stages -l src: -l print-csv ' Options[search]+='-l verbose ' -Options[search]+='-l all-deps -l wrap' +Options[search]+='-l all-deps -l wrap ' Help[search]=' USAGE: - module search [switches] string... + module search [switches] STRING... Search installed modules. If an argument is given, search for modules whose name match the argument. @@ -2057,6 +2060,7 @@ USAGE: SWITCHES: -h|-H|-?|--help this usage info -V|--version modules version & configuration options + --debug enable debug output SUBCOMMANDS: + add|load [switches] modulefile [modulefile ...] @@ -2087,7 +2091,7 @@ subcommand_help() { local -a args=() while (( $# > 0 )); do case $1 in - -[hH] | --help ) + -\? | -h | -H | --help ) print_help "${subcommand}" ;; -V | --version ) @@ -2124,7 +2128,7 @@ subcommand_help() { # whatis # Subcommands[whatis]='whatis' -Options[whatis]='-o H -l help' +Options[whatis]='-o \?H -l help' Help[whatis]=' USAGE: module whatis [modulefile...] @@ -2147,7 +2151,7 @@ subcommand_whatis() { # Subcommands[apropos]='apropos' Subcommands[keyword]='apropos' -Options[apropos]='-o H -l help' +Options[apropos]='-o \?H -l help' Help[apropos]=' USAGE: module apropos string @@ -2165,7 +2169,7 @@ subcommand_apropos() { # initadd module... # Subcommands[initadd]='initadd' -Options[initadd]='-o H -l help' +Options[initadd]='-o \?H -l help' Help[initadd]=" USAGE: module initadd modulefile... @@ -2201,7 +2205,7 @@ subcommand_initadd() { # initprepend module... # Subcommands[initprepend]='initprepend' -Options[initprepend]='-o H -l help' +Options[initprepend]='-o \?H -l help' Help[initprepend]=" USAGE: module initprepend modulefile... @@ -2218,7 +2222,7 @@ subcommand_initprepend() { # initrm module... # Subcommands[initrm]='initrm' -Options[initrm]='-o H -l help' +Options[initrm]='-o \?H -l help' Help[initrm]=" USAGE: module initrm modulefile... @@ -2234,7 +2238,7 @@ subcommand_initrm() { # initswitch module1 module2 # Subcommands[initswitch]='initswitch' -Options[initswitch]='-o H -l help' +Options[initswitch]='-o \?H -l help' Help[initswitch]=" USAGE: module initswitch modulefile1 modulefile2 @@ -2247,7 +2251,7 @@ subcommand_initswitch() { local args=() while (( $# > 0 )); do case $1 in - -h | --help ) + -\? | --help ) print_help "${subcommand}" ;; -- ) @@ -2274,7 +2278,7 @@ subcommand_initswitch() { # initlist # Subcommands[initlist]='initlist' -Options[initlist]='-o H -l help' +Options[initlist]='-o \?H -l help' Help[initlist]=" USAGE: module initlist @@ -2291,7 +2295,7 @@ subcommand_initlist() { # initclear # Subcommands[initclear]='initclear' -Options[initclear]='-o H -l help' +Options[initclear]='-o \?H -l help' Help[initclear]=" USAGE: module initclear @@ -2323,7 +2327,7 @@ shift declare -a opts=() while (( $# > 0 )); do case $1 in - -H | -\? | --help | -help ) + -\? | -H | --help | -help ) print_help 'help' ;; -V | --version ) @@ -2422,11 +2426,12 @@ case ${subcommand} in ;; esac -declare options -options=$( "${getopt}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ + +tmp=$("${getopt}" --name="${CMD}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ || print_help "${subcommand}" -eval set -- ${options} -subcommand_${Subcommands[$subcommand]} "$@" +eval args=( "$tmp" ) +unset tmp +subcommand_${Subcommands[$subcommand]} "${args[@]}" # Local Variables: # mode: sh From 5f81fe32e93a56b496f87cd9774ef257a7f58746 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 14:40:35 +0100 Subject: [PATCH 16/28] modulecmd: bugfix in saving state to PMODULES_ENV --- Pmodules/modulecmd.bash.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 2bac638..b90b4a1 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -161,6 +161,7 @@ is_group () { local moduledir="${PMODULES_ROOT}/${group}/${PMODULES_MODULEFILES_DIR}" [[ -d "${moduledir}" ]] || return 1 compute_group_depth "${moduledir}" + g_env_must_be_saved='yes' } # @@ -1116,6 +1117,7 @@ subcommand_use() { fi if [[ -z ${GroupDepths[${arg}]} ]] && [[ -d "${PMODULES_ROOT}/${arg}" ]]; then scan_groups "${PMODULES_ROOT}" + g_env_must_be_saved='yes' fi if [[ -n ${GroupDepths[${arg}]} ]] && @@ -1965,6 +1967,7 @@ subcommand_search() { if (( ${#GroupDepths[@]} == 0 )) || \ [[ ${src_prefix} != ${PMODULES_ROOT} ]]; then scan_groups "${src_prefix}" + g_env_must_be_saved='yes' fi local module @@ -2340,6 +2343,7 @@ fi if (( ${#GroupDepths[@]} == 0 )); then scan_groups "${PMODULES_ROOT}" + g_env_must_be_saved='yes' fi case ${subcommand} in From 5815bc52017f8e330b8912534adef60fad37a83f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 14:43:46 +0100 Subject: [PATCH 17/28] modulecmd: shell glob pattern search implemented --- Pmodules/modulecmd.bash.in | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index b90b4a1..dc20928 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -789,7 +789,7 @@ get_available_modules() { fi done < <(${find} -L * \ \( -type f -o -type l \) -not -name ".*" \ - -ipath "${module}*" \ + -ipath "${module}" \ | ${sort} --version-sort) } done @@ -1008,7 +1008,7 @@ subcommand_avail() { fi get_available_modules \ mods \ - "${string}" \ + "${string}*" \ "${opt_use_rel_stages}" \ "${dir}" [[ ${#mods[@]} == 0 ]] && continue @@ -1609,6 +1609,7 @@ Options[search]='-o a\?H -l help -l no-header -l print-modulefiles ' Options[search]+='-l release-stage: -l with: -l all-release-stages -l src: -l print-csv ' Options[search]+='-l verbose ' Options[search]+='-l all-deps -l wrap ' +Options[search]+='-l glob' Help[search]=' USAGE: module search [switches] STRING... @@ -1622,6 +1623,9 @@ SWITCHES: --all-deps Show all dependecies + --glob + Interpret STRING as shell pattern. + --no-header Suppress output of a header. @@ -1659,6 +1663,7 @@ subcommand_search() { local opt_use_releases=':' local opt_all_deps='no' local opt_wrap='no' + local opt_glob='no' #..................................................................... # @@ -1797,7 +1802,12 @@ subcommand_search() { # :FIXME: # search () { - local -r module=$1 + if [[ ${opt_glob} == 'yes' ]]; then + local -r module="$1" + else + local -r module="${1}*" + fi + # write results to a temporary file for later processing local group # loop over all groups @@ -1860,7 +1870,7 @@ subcommand_search() { while (( $# > 0 )); do case $1 in - -H | --help ) + -\? | -H | --help ) print_help "${subcommand}" ;; --all-deps ) @@ -1940,6 +1950,9 @@ subcommand_search() { --wrap ) opt_wrap='yes' ;; + --glob ) + opt_glob='yes' + ;; -- ) shift 1 modules+=( "$@" ) From f6fe194e40f9bb60e1ae006ddbdd75d2219df559 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 8 Nov 2021 16:51:24 +0100 Subject: [PATCH 18/28] modmanage: review with lot of changes --- Pmodules/modmanage.bash.in | 896 ++++++++++++++----------------------- 1 file changed, 338 insertions(+), 558 deletions(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index abaea84..94e22b0 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -1,219 +1,120 @@ #!@BASH@ --noprofile -# we have to unset CDPATH, otherwise 'cd' prints the directoy! -unset CDPATH +PATH='/bin:/usr/bin' +unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy! +unset IFS # use default IFS + +shopt -s nullglob # used for some output only declare -r CMD='modmanage' -dirname=$(PATH=/bin:/usr/bin which dirname) -declare -r dirname -uname=$(PATH=/bin:/usr/bin which uname) -declare -r uname -mkdir=$(PATH=/bin:/usr/bin which mkdir) -declare -r mkdir -rsync=$(PATH=/bin:/usr/bin which rsync) -declare -r rsync -rm=$(PATH=/bin:/usr/bin /usr/bin/which rm) -declare -r rm - -if [[ $(${uname} -s) == 'Darwin' ]]; then - declare -r getopt="${libexecdir}/getopt" - declare -r find="${libexecdir}/find" -else - getopt=$(PATH=/bin:/usr/bin /usr/bin/which getopt) - declare -r getopt - find=$(PATH=/bin:/usr/bin /usr/bin/which find) - declare -r find -fi - - -declare -r mydir=$(cd $(${dirname} "$0") && pwd) -declare -r prefix=$(${dirname} "${mydir}") +declare mydir=$(cd $(dirname "$0") && pwd) +declare prefix=$(dirname "${mydir}") +declare libdir="${prefix}/lib" +declare libexecdir="${prefix}/libexec" declare -r bindir="${prefix}/bin" -declare -r libdir="${prefix}/lib" -declare -r libexecdir="${prefix}/libexec" - source "${libdir}/libstd.bash" - +source "${libdir}/libpmodules.bash" _exit () { - std::die 1 "Interrupted..." + std::die 1 "\nInterrupted..." } +trap '_exit' INT TERM _err () { - std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}" + std::info "\nOops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}" std::die 1 "Aborting ..." } - -trap '_exit' INT TERM trap '_err' ERR +path="/bin:/usr/bin:${bindir}" +[[ $(uname -s) == 'Darwin' ]] && path+=":${libexecdir}" +std::def_cmds "${path}" 'chown' 'dirname' 'mkdir' 'rsync' 'rm' 'getopt' 'find' 'modulecmd' + +unset mydir +unset prefix +unset libdir +unset libexecdir +# bindir we still need -# make sure that everything is used from this version declare PMODULES_VERSION='@PMODULES_VERSION@' -############################################################################## -# -# print version of program -# -# Arguments: -# none -# -print_version() { - echo " +# In the dictionary Help we store the help text of each single command +# and for displaying the version. + +# initialize help text of 'module --version' +Help['version']=" Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@ Copyright GNU GPL v2 -" 1>&2 -} - -############################################################################## -# -# print usage -# -# Arguments: -# none -# -usage() { - local -r prog=$(basename $0) - print_version - echo " -Usage: ${prog} [ switches ] [ subcommand ] [subcommand-args ] - -Switches: - --dry-run do nothing - --force force overwrite - -Available SubCommands and Args: - init [--src=] [--user=] - Initialize a new minimal Pmodule environment. - - install [--with=...] - Install matching modules - - sync [--delete] [--dst=] - Synchronize modules. " -} - -declare force='no' -declare dry_run='no' -declare DRY='' -declare subcommand='' -declare sargs=() ############################################################################## # -# help for subcommand 'init' +# help [module|sub-command] # -# Arguments: -# none -# -subcommand_help_init() { - echo " -init [--src=] [--user=] [--version=] - Initialize a new minimal Pmodule environment in directory - . The parameter must only be present if - ${prog} is executed as root. -" 1>&2 -} - -############################################################################## -# -# help for subcommand 'install' -# -# Arguments: -# none -# -subcommand_help_install() { - echo " -install ... [--with=...] [--release=...] [--src=] - Install matching modules -" 1>&2 -} - -############################################################################## -# -# help for subcommand 'search' -# -# Arguments: -# none -# -subcommand_help_search() { - echo " +Subcommands[help]='help' +Options[help]='-o hHV\? -l version -l help' +Help[help]=' USAGE: - module search [switches] string... - Search available modules. If an argument is given, search - for modules whose name match the argument. + modmanage [switches] subcommand [subcommand-args]... -SWITCHES: - --no-header - Suppress output of a header. +SWITCHES: + -h|-H|-?|--help this usage info + -V|--version modules version & configuration options + --debug enable debug output + --dry-run dry run - --with=STRING - Search for modules compiled with modules matching string. The - command - module search --with=gcc/4.8.3 +SUBCOMMANDS: + + init [switches] TARGET_DIR + + install [switches] module... + + help [subcommand] +' - lists all modules in the hierarchy compiled with gcc 4.8.3. -" 1>&2 -} - -############################################################################## -# -# help for subcommand 'sync' -# -# Arguments: -# none -# -subcommand_help_sync() { - echo " -sync [--delete] [--dst=] - Synchronize environment modules and configuration files - from Pmodule environment to Pmodule environment - (default: currently active Pmodule environment). - Not yet implemented: - If --delete is given, unmarked modules present in - will be deleted. -" 1>&2 -} - -############################################################################## -# -# print usage or help text for given sub-command -# -# Arguments: -# none or sub-command -# subcommand_help() { - if [[ $# == 0 ]]; then - usage - elif typeset -F subcommand_help_$1 > /dev/null 2>&1 ; then - # help for sub-command - subcommand_help_$1 - else - usage - fi + local -r subcommand='help' + local -a args=() + while (( $# > 0 )); do + case $1 in + -h | -\? | -H | --help ) + print_help "${subcommand}" + ;; + -V | --version ) + print_help 'version' + ;; + -- ) + shift 1 + args+=( "$@" ) + break + ;; + * ) + args+=( "$1" ) + ;; + esac + shift + done + for arg in "${args[@]}"; do + if [[ -n "${Help[${arg}]}" ]] ; then + print_help "${arg}" + else + std::die 1 "Unknown sub-command -- ${subcommand}" + fi + done } ############################################################################## # -# Derive the relative module installation path -# from the relative modulefile path -# +# Derive the module installation path from the modulefile path. +# The passed modulefile must be absolute. +# # Arguments: -# $1: relative module file path +# $1: absolute module file path # get_module_prefix() { - local -a comp=( ${1//\// } ) # split rel.path into components - local path="${comp[0]}" # result path - local -i i - for ((i=1; i<${#comp[@]}-1; i+=2)); do - path+="/${comp[$((-i-1))]}/${comp[$((-i))]}" - done - echo "${path}" + "${modulecmd}" bash show "$1" 2>&1 \ + |awk '/_HOME |_PREFIX / {print $3; exit}' } ############################################################################## @@ -235,66 +136,47 @@ get_releasefile_name() { # - sync modulefile # - sync release file # -# Note: -# We do not take care of files in $PMODULES_ROOT/$PMODULES_TEMPLATES_DIR. If -# the modulefile is a sym-link it is expected that the target exists. -# # Arguments: # $1: relative modulefile path (something like: Tools/gnuplot/5.0.0) # $2: source prefix of Pmodule environment # $3: target prefix of Pmodule environment # sync_module() { - local -r rel_modulefile=$1 - local -r src_prefix=$2 - local -r target_prefix=$3 + local -r rel_modulefile="$1" + local -r src_root="$2" + local -r target_root="$3" - local -r rel_module_prefix=$( get_module_prefix "${rel_modulefile}" ) - local -r rel_releasefile=$( get_releasefile_name "${rel_modulefile}" ) + local -r src_prefix=$( get_module_prefix "${src_root}/${rel_modulefile}" ) + local -r rel_prefix=${src_prefix#${src_root}} + local -r target_prefix="${target_root}/${rel_prefix}" # install/update module - if [[ ! -d "${target_prefix}/${rel_module_prefix}" ]] || [[ "${force}" == 'yes' ]]; then - $DRY ${mkdir} -p "${target_prefix}/${rel_module_prefix}" || return $? - $DRY ${rsync} --links --perms --recursive --delete \ - "${src_prefix}/${rel_module_prefix}/" \ - "${target_prefix}/${rel_module_prefix}/" || exit $? - fi - local -r src_modulefile="${src_prefix}/${rel_modulefile}" - local -r src_releasefile="${src_prefix}/${rel_releasefile}" - local -r target_modulefile="${target_prefix}/${rel_modulefile}" - local -r target_releasefile="${target_prefix}/${rel_releasefile}" - - # create target directory for module- and release-file - if [[ -e "${src_modulefile}" ]] || [[ -e "${src_releasefile}" ]]; then - local dir=$( ${dirname} "${target_modulefile}" ) - $DRY ${mkdir} -p "${dir}" || return $? + if [[ ! -d "${target_prefix}" ]] || [[ "${force}" == 'yes' ]]; then + ${mkdir} -p "${target_prefix}" || exit $? + ${rsync} --links --perms --recursive --delete \ + "${src_prefix}/" \ + "${target_prefix}/" || exit $? fi - # copy modulefile template - local -a rel_modulefile_splitted - std::split_fname rel_modulefile_splitted "${rel_modulefile}" - local -r module_group="${rel_modulefile_splitted[0]}" - local -r module_name="${rel_modulefile_splitted[-2]}" + # create modulefile direcrory and install modulefile + local -r src_modulefile="${src_root}/${rel_modulefile}" + local -r target_modulefile="${target_root}/${rel_modulefile}" + + ${mkdir} -p "$(${dirname} "${target_modulefile}")" || exit $? - local -r template="${module_group}/${PMODULES_TEMPLATES_DIR}/${module_name}/" - local -r src_template="${src_prefix}/${template}" - local -r target_template="${target_prefix}/${template}" - if [[ -e "${src_template}" ]]; then - $DRY ${mkdir} -p "${target_template}" - $DRY ${rsync} --links --perms --recursive \ - "${src_template}" "${target_template}" || exit $? - fi - - # copy modulefile if [[ -e "${src_modulefile}" ]]; then - $DRY ${rsync} --links --perms --recursive \ - "${src_modulefile}" "${target_modulefile}" || exit $? + ${rsync} --links --perms --recursive \ + "${src_modulefile}" "${target_modulefile}" || exit $? fi - # copy release-file + # install release-file + local -r rel_releasefile=$( get_releasefile_name "${rel_modulefile}" ) + local -r src_releasefile="${src_root}/${rel_releasefile}" + local -r target_releasefile="${target_root}/${rel_releasefile}" + if [[ -e "${src_releasefile}" ]]; then - $DRY ${rsync} --links --perms --recursive \ - "${src_releasefile}" "${target_releasefile}" || exit $? + ${rsync} --links --perms --recursive \ + "${src_releasefile}" "${target_releasefile}" || exit $? fi } @@ -310,8 +192,8 @@ sync_module() { sync_config() { src="$1/${PMODULES_CONFIG_DIR}/" dst="$2/${PMODULES_CONFIG_DIR}/" - $DRY ${rsync} --recursive --links --perms --delete \ - "${src}" "${dst}" 2>/dev/null || return $? + ${rsync} --recursive --links --perms --delete \ + "${src}" "${dst}" 2>/dev/null || return $? echo } @@ -331,19 +213,28 @@ delete_module() { # # initialize a new module environment # -# Arguments: -# [--src ] -# Module environment we are going to sync from. If not -# specified, the module environment this script is in -# will be used. -# [--user ] -# If this scripts runs with root privileges, a user name -# ore ID must be specified. -# [--version ] -# Set PMODULES_VERSION to -# TARGET_DIR -# Initialize a new module environment in this directory- -# +Subcommands[init]='init' +Options[init]='-o h -l src: -l user: -l help -l version:' +Help[init]=" +USAGE: + modmanage init [switches] TARGET_DIR + Initialize a new minimal Pmodule environment in TARGET_DIR. + A user must be specified with '--user=' if the + programm is executed as root. + + +SWITCHES: + --src + Module environment we are going to sync from. If not + specified, the module environment this script is in + will be used. + --user + If this scripts runs with root privileges, a user name + ore ID must be specified. + --force + re-initialise an already existing Pmodule environment. +" + subcommand_init() { check_env() { [[ -n "${PMODULES_ROOT}" ]] && @@ -354,62 +245,61 @@ subcommand_init() { Error: the module environment you are going to use as source has not been initialized properly!" - [[ -d "${src_prefix}/${PMODULES_CONFIG_DIR}" ]] && - [[ -d "${src_prefix}/Tools/Pmodules/${PMODULES_VERSION}" ]] || \ + [[ -d "${src_root}/${PMODULES_CONFIG_DIR}" ]] && + [[ -d "${src_root}/Tools/Pmodules/${PMODULES_VERSION}" ]] || \ std::die 1 " -Error: the module environment '${src_prefix}' has not been initialized properly!" +Error: the module environment '${src_root}' has not been initialized properly!" } - local src_prefix='' - local target_prefixes=() + local src_root='' + local target_root=() local user='' - local opts='' - opts=$(${getopt} -o h -l src: -l user: -l help -l version: -- "$@") - if [[ $? != 0 ]]; then - subcommand_help_init - exit 1 - fi - eval set -- "${opts}" while (($# > 0)); do case $1 in - --src ) - src_prefix=$2 - shift + -h | -H | -\? | --help | -help ) + print_help "${subcommand}" ;; - --user ) - user=$2 - shift + --force | -f ) + force='yes' ;; - --version ) - PMODULES_VERSION=$2 - shift + --src | --src=* ) + if [[ $1 == --src=* ]]; then + src_root="${1#--*=}" + else + src_root="$2" + shift + fi + ;; + --user | --user=* ) + if [[ $1 == --user=* ]]; then + user="${1#--*=}" + else + user="$2" + shift + fi ;; -- ) : ;; - -* | -h | --help ) - echo "$1: illegal option" 1>&2 - subcommand_help_init - exit 1 - ;; * ) - target_prefixes+=( "$1" ) + target_root="$1" ;; esac shift done - (( ${#target_prefixes[@]} != 0 )) || \ + if [[ -z ${target_root} ]]; then std::die 1 "Error: no target directory specified!" - - # if source directory is not passed as argument, derive it from script name - if [[ -z "${src_prefix}" ]]; then - src_prefix=$(cd "${bindir}/../../../.." && pwd) fi - [[ -d "${src_prefix}" ]] || \ - std::die 1 "Error: ${src_prefix}: source directory does not exist!" - [[ -r "${src_prefix}/config/profile.bash" ]] || \ - std::die 1 "Error: ${src_prefix}: shell profile does not exist or is not readable!" - source "${src_prefix}/config/profile.bash" + + # if source directory is not passed as argument, derive it from script name + if [[ -z "${src_root}" ]]; then + src_root=$(cd "${bindir}/../../../.." && pwd) + fi + [[ -d "${src_root}" ]] || \ + std::die 1 "Error: ${src_root}: source directory does not exist!" + [[ -r "${src_root}/config/profile.bash" ]] || \ + std::die 1 "Error: ${src_root}: shell profile does not exist or is not readable!" + source "${src_root}/config/profile.bash" local -i euid=$(id -u) if (( euid == 0 )); then @@ -437,204 +327,156 @@ environment at '${PMODULES_ROOT}' # $1 target directory # init_pmodules_environment() { - local -r src_prefix="${PMODULES_ROOT}" - local -r target_prefix=$1 + local -r src_root="${PMODULES_ROOT}" + local -r target_root=$1 local src='' local dst='' - echo "Initializing target directory '${target_prefix}' ..." + echo "Initializing target directory '${target_root}' ..." echo - if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then - echo "Warning: ${target_prefix} already exists." + if [[ -d "${target_root}" ]] && [[ ${force} == no ]]; then + echo "Warning: ${target_root} already exists." std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ std::die 1 "Abort ..." fi force='yes' - echo "Creating target directory '${target_prefix}'..." - $DRY ${mkdir} -p "${target_prefix}" || \ + echo "Creating target directory '${target_root}'..." + ${mkdir} -p "${target_root}" || \ std::die 1 "Error: make directory failed!" echo echo "Syncing configuration ..." - sync_config "${src_prefix}" \ - "${target_prefix}" || \ + sync_config "${src_root}" \ + "${target_root}" || \ std::die 1 "Error: configuration synchronization failed!" - echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..." + echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_root}' to '${target_root}'..." sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \ - "${src_prefix}" \ - "${target_prefix}" || \ + "${src_root}" \ + "${target_root}" || \ std::die 1 "Error: sync Pmodules failed!" - ${mkdir} -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}" + ${mkdir} -p "${target_root}/Tools/${PMODULES_MODULEFILES_DIR}" echo if [[ -n "${user}" ]]; then echo "Changing user of new module environment to '${user}'..." - $DRY chown -R "${user}" "${target_prefix}" || \ + ${chown} -R "${user}" "${target_root}" || \ std::die 1 "Error: changing owner failed!" echo fi - echo "New minimal module environment created at '${target_prefix}'." + echo "New minimal module environment created at '${target_root}'." echo "To use this environment, execute" - echo " sudo ln -fs ${target_prefix} /opt/psi" + echo " sudo ln -fs ${target_root} /opt/psi" echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash" } umask 022 - for target_prefix in "${target_prefixes[@]}"; do - init_pmodules_environment "${target_prefix}" - done - + init_pmodules_environment "${target_root}" } -declare -a Groups=() -declare -A GroupDepths - -############################################################################## -# -# Get available module groups. Found groups are added to the global array -# 'Groups'. -# -# Arguments: -# $1: root of module environment -# $2: relative directory with module files -# -get_groups () { - local -r root="$1" - local -r modulefiles_dir="$2" - { - cd "${root}" - # for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems - for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do - [[ -d ${f}/${modulefiles_dir} ]] || continue - Groups+=( $f ) - done - }; -} - -############################################################################## -# -# Compute hierarchy depth of all groups. Stores result in global array -# 'GroupDepths'. The group depth is defined as the hierarchy depth times 2. -# -# Examples: -# group depth of 'Programming' is 0 -# group depth of 'Compiler' is 2 -# group depth of 'MPI' is 4 -# -# Arguments: -# $1: root of module environment -# $2: relative directory with module files -# -get_group_depths () { - local -r root="$1" - local -r modulefiles_dir="$2" - local -ir off=4 - { - cd "${root}" - local group - for group in "${Groups[@]}"; do - local fname=$(${find} "${group}/${modulefiles_dir}" \ - -depth \( -type f -o -type l \) -print -quit) - [[ -n ${fname} ]] || continue - local -a tmp - std::split_fname tmp "${fname}" - (( GroupDepths[$group]=${#tmp[@]}-${off} )) - done - }; -} ############################################################################## # # sub-command 'install' # # Arguments: -# [--dry-run] -# Dry run -# -# [--force] | -f ] -# Install module even it already exists -# -# [--release ] -# Set release of module to -# -# [--src ] -# Install from module environment in -# -# [--with ] -# Install module(s) in this sub-group only -# -# -# Install modules matching given pattern -# +Subcommands[install]='install' +Options[install]='-o hf -l force -l with: -l help -l src: -l target:' +Help[install]=' +USAGE: + modmanage install [switches] ... + Install modules + +SWITCHES: + --force] | -f + Install module even it already exists + + --src + Install from module environment in + + --with + Install module(s) in this sub-group only + + + Install modules matching given pattern +' + subcommand_install() { + local -r subcommand='install' local opts='' local -a with=() - local -a releases=() local -a module_pattern=() - local src_prefix="${PMODULES_INSTALL_SOURCE}" - local -r target_prefix="${PMODULES_ROOT}" + local src_root="${PMODULES_INSTALL_SOURCE}" + local target_root="${PMODULES_ROOT}" local modulefile='' local -A modules_to_install local -A dependencies_to_install - local -A map_to_family - local -a initial_modulepath=() + local -A map_to_group + local -a modulepath=() + #...................................................................... + # + set_initial_modulepath() { + local group + for group in "${!GroupDepths[@]}"; do + (( ${GroupDepths[${group}]} == 0 )) || continue + modulepath+=( "${src_root}/${group}/${PMODULES_MODULEFILES_DIR}" ) + done + } + + #...................................................................... + # + create_groupheads_map() { + : + } + #...................................................................... # # Resolve dependencies to given module # # Arguments: - # $1 modulefile relativ to src prefix. Something like: - # MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14 + # $1 absolute module file name # # Notes: - # The variables - # initial_modulepath - # modules_to_install - # map_to_family - # from the calling function are used! + # Following variables from the enclosing function are used: + # modulepath (might be changed) + # map_to_group (read-only) # resolve_dependencies_of_module () { - local -r modulefile=$1 - local -a modulepath=( "${initial_modulepath[@]}" ) + local -r modulefile="$1" - # compute filename with dependencies of given module - local -i i=0 n=0 - std::split_fname items n "${modulefile}" - local _prefix="${src_prefix}/${items[3]}" - for (( i = n-2; i >= 2; i-=2 )); do - _prefix+="/${items[$i]}/${items[i+1]}" - done - local tmpfile=$(mktemp /tmp/Pmodules_XXXXXX) - - local fname_dependencies="${_prefix}/.dependencies" - [[ -r "${_prefix}/.dependencies" ]] && cat "$_" > "${tmpfile}" - [[ -r "${_prefix}/.install_dependencies" ]] && cat "$_" >> "${tmpfile}" + local -- prefix=$(get_module_prefix "${modulefile}") + local -a rdeps=() + local rdeps_file="${prefix}/.dependencies" + local -a ideps=() + local ideps_file="${prefix}/.install_dependencies" + + if [[ -r "${rdeps_file}" ]]; then + mapfile -t rdeps < <(grep -v '^ *#' "${rdeps_file}" ) + fi + if [[ -r "${ideps_file}" ]]; then + mapfile -t ideps < <(grep -v '^ *#' "${ideps_file}" ) + fi # loop over all dependecies local dep - while read dep; do - # skip empty lines - # :FIXME: skip comments?! - [[ -z ${dep} ]] && continue - - # search for module with current modulepath and remember + for dep in "${rdeps[@]}" "${ideps}"; do + [[ -n ${dep} ]] || continue + # search module with current modulepath local modulename=$(${find} "${modulepath[@]}" -path "*/${dep}" \ - 2>/dev/null | head -n 1 ) + -print -quit 2>/dev/null) [[ -n ${modulename} ]] || \ - std::die 3 "Oops: required module '${dep}' not found!" - modulename=${modulename/${src_prefix}\/} - dependencies_to_install[${modulename}]='.' - resolve_dependencies_of_module "${modulename}" - # append new node in hierarchy to modulepath - if [[ -n ${map_to_family[${dep}]} ]]; then - local path="${src_prefix}/${map_to_family[${dep}]}/" - path+="${PMODULES_MODULEFILES_DIR}/" + std::die 3 "Oops: required module '${dep}' not found!" + + dependencies_to_install[${modulename/${src_root}\/}]='.' + _resolve_dependencies "${modulename}" + if [[ -n ${map_to_group[${dep}]} ]]; then + # append hierarchical group to modulepath + local path="${src_root}/${map_to_group[${dep}]}/" + path+="${PMODULES_MODULEFILES_DIR}/" path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}" modulepath+=( "${path}" ) fi - done < "${tmpfile}" - ${rm} "${tmpfile}" + done } #...................................................................... @@ -646,29 +488,25 @@ subcommand_install() { # none # # Notes: - # The following variables of the enclosing function are used: + # Following variables from the enclosing function are used: + # target_root (read-only) # modules_to_install (read-only) - # target_prefix (read-only) # dependencies_to_install (read-only) - + # + print_modules_to_install() { local modulefile + local parts std::info "The following modules will be installed/updated:" for modulefile in "${!modules_to_install[@]}"; do - if [[ -e "${target_prefix}/${modulefile}" ]]; then - std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - else - std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - fi + std::split_fname parts "${modulefile}" + std::info " ${parts[-2]}/${parts[-1]}" done 2>&1 | sort if (( ${#dependencies_to_install[@]} > 0 )); then std::info "\nThe following dependencies will be installed/updated:" for modulefile in "${!dependencies_to_install[@]}"; do - if [[ -e "${target_prefix}/${modulefile}" ]]; then - std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - else - std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - fi + std::split_fname parts "${modulefile}" + std::info " ${parts[-2]}/${parts[-1]}" done 2>&1 | sort fi std::info "" @@ -677,44 +515,41 @@ subcommand_install() { std::info "" } - opts=$(${getopt} -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@") - if [[ $? != 0 ]]; then - subcommand_help_install - exit 1 - fi - eval set -- "${opts}" while (($# > 0)); do case $1 in - --dry-run ) - DRY='echo' + -h | -H | -\? | --help | -help ) + print_help "${subcommand}" ;; --force | -f ) force='yes' ;; - --release ) - releases+=( "$2" ) - shift + --src | --src=*) + if [[ $1 == --src=* ]]; then + src_root="${1#--*=}" + else + src_root="$2" + shift + fi ;; - --src ) - src_prefix="$2" - shift + --target | --target=*) + if [[ $1 == --target=* ]]; then + target_root="${1#--*=}" + else + target_root="$2" + shift + fi ;; - --with ) - with+=( "$2" ) - shift + --with | --with=* ) + if [[ "$1" == --with ]]; then + with+=( "$2" ) + shift + else + with+=( "${1/--with=}" ) + fi ;; -- ) : ;; - -h | --help ) - subcommand_help_install - exit 1 - ;; - -* ) - echo "$1: illegal option" 1>&2 - subcommand_help_init - exit 1 - ;; * ) module_pattern+=( "$1" ) ;; @@ -722,75 +557,56 @@ subcommand_install() { shift done - [[ -n ${src_prefix} ]] \ + [[ -n ${src_root} ]] \ || std::die 3 "Oops: no installation source given." - [[ -d ${src_prefix} ]] \ - || std::die 3 "Oops: '${src_prefix}' is not a valid installation source." + [[ -d ${src_root} ]] \ + || std::die 3 "Oops: '${src_root}' is not a valid installation source." - # scan available groups and their depth - get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" - get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" - - # set initial modulepath - local group - for group in "${!GroupDepths[@]}"; do - if (( ${GroupDepths[${group}]} == 0 )); then - initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" ) - fi - done + scan_groups "${src_root}" + set_initial_modulepath # - # create a mapping from module name to their family. + # create a mapping from module name to their group. # Examples: # gcc/5.2.0 -> Compiler # openmpi/1.8.4 -> MPI - local _fname='' - while read _fname; do - local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}" - local -a items - std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}" - local -i n=${#items[*]} - # We are only interested in families adding something to - # the modulepath. - if (( n >= 4 )); then - local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" ) - map_to_family[$_key]=${_family} + local fname='' + local -i n + local -a parts + while read fname; do + std::split_fname parts n "${fname}" + group="${parts[0]}" + # We are only interested in groups adding something to + # the modulepath. + if (( n >= 6 )); then + map_to_group[${parts[-4]}/${parts[-3]}]=${group} fi - done < <({ cd "${src_prefix}" && \ + done < <({ cd "${src_root}" && \ ${find} */"${PMODULES_MODULEFILES_DIR}" \ \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null ) - # # search for to be installed modules and their dependencies - # - local -i n=0 while read modulefile; do + modules_to_install["${modulefile/${src_root}}"]+='.' resolve_dependencies_of_module "${modulefile}" - modules_to_install["${modulefile}"]+='.' - let n+=1 - done < <(${PMODULES_HOME}/bin/modulecmd bash search \ + done < <("${modulecmd}" bash search \ "${module_pattern[@]}" \ "${with[@]/#/--with=}" \ - "${releases[@]/#/--release=}" \ + -a --glob \ --no-header --print-modulefiles \ - --src="${src_prefix}" 2>&1 1>/dev/null) - (( n == 0 )) && \ + --src="${src_root}" 2>&1 1>/dev/null) + (( ${#modules_to_install[@]} == 0 )) && \ std::die 0 "No matching modules found ..." print_modules_to_install - # install ... + # install/update ... for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do - if [[ -e "${target_prefix}/${modulefile}" ]]; then - std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - else - std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - fi - sync_module "${modulefile}" \ - "${src_prefix}" \ - "${target_prefix}" + std::split_fname parts "${modulefile}" + std::info " ${parts[-2]}/${parts[-1]}" + sync_module "${modulefile}" "${src_root}" "${target_root}" done std::info "\nDone!\n" -} +} # subcommand_install ############################################################################### # @@ -808,91 +624,55 @@ subcommand_cleanup() { : } -############################################################################### -# -# search modules in source -# :FIXME: this is still crap -# -subcommand_search() { - local src_prefix="${PMODULES_INSTALL_SOURCE}" - local -r target_prefix="${PMODULES_ROOT}" - local -A modules_found - - print_modules_found() { - std::info "The following modules are available:" - for modulefile in "${!modules_found[@]}"; do - std::info " ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" - done 2>&1 | sort - } - - [[ -n ${src_prefix} ]] \ - || std::die 3 "Oops: no installation source given." - [[ -d ${src_prefix} ]] \ - || std::die 3 "Oops: '${src_prefix}' is not a valid installation source." - - # scan available groups and their depth - get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" - get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" - - local -i n=0 - while read modulefile; do - modules_found["${modulefile}"]+='.' - let n+=1 - done < <(${PMODULES_HOME}/bin/modulecmd bash search \ - -a \ - --no-header --print-modulefiles \ - --src="${src_prefix}" \ - "$@" \ - 2>&1 1>/dev/null) - (( n == 0 )) && \ - std::die 0 "No matching modules found ..." - print_modules_found -} - +declare force='no' +declare subcommand='' +declare -a opts=() while (($# > 0)); do case $1 in -h | -H | -\? | --help | -help ) - usage - exit 1 + print_help 'help' ;; -V | --version ) - print_version - exit 1 - ;; - -f | --force ) - force='yes' + print_help 'version' ;; --debug ) set -x ;; --dry-run ) - dry_run='yes' - DRY='echo' + chown="echo ${chown}" + mkdir="echo ${mkdir}" + rsync="echo ${rsync}" ;; -* ) - echo "$1: unknown switch.\n" 1>&2 - exit 1 - ;; - init|install|sync|search|help ) - subcommand="subcommand_$1" - shift - sargs=( $* ) - shift $# + opts+=( "$1" ) ;; * ) - echo "$1: unknown sub-command" 1>&2 - exit 1 + subcommand="$1" + shift + break + ;; esac shift || : done -if [[ -z ${subcommand} ]]; then - usage - exit 1 +if [[ -z "${subcommand}" ]]; then + std::die 1 "${CMD}: no sub-command specified.\n" + print_help 'help' fi -[[ "${subcommand}" != "subcommand_init" ]] && [[ -z "${PMODULES_ROOT}" ]] && \ + +if [[ -z "${Subcommands[${subcommand}]}" ]]; then + std::die 1 "${CMD}: unknown sub-command -- ${subcommand}\n" +fi + +if [[ "${subcommand}" != "init" ]] && [[ -z "${PMODULES_ROOT}" ]]; then std::die 1 "Error: No current module environment configured!" -$subcommand "${sargs[@]}" +fi + +tmp=$("${getopt}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ + || print_help "${subcommand}" +eval args=( "$tmp" ) +unset tmp +subcommand_${Subcommands[$subcommand]} "${args[@]}" # Local Variables: # mode: sh From 26e7d0f24f49c7a2aa3785db6639df0a66c3cb46 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 17:51:46 +0100 Subject: [PATCH 19/28] libpmodules.bash: default value assigned to PMODULES_CONFIG_DIR --- Pmodules/libpmodules.bash.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index 7ad1140..99e09f5 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -1,6 +1,7 @@ #!/bin/bash declare PMODULES_MODULEFILES_DIR='modulefiles' +declare PMODULES_CONFIG_DIR='config' declare -A GroupDepths=() declare -A Subcommands=() declare -A Options=() From 9765a4c0ff1c8399228d72b9c5d730bd323aa624 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 17:52:48 +0100 Subject: [PATCH 20/28] libstd.bash: std::split_path() added --- Pmodules/libstd.bash | 48 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 5161915..1684727 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -144,13 +144,38 @@ std::replace_path () { } # -# split an absolute path +# Functions to split a path into its components. # # Args: # $1 upvar -# $2 absolute path +# $2 absolute or relative path (depends on the function) # $3 opt upvar: number of components # +# Notes: +# std::split_path() +# if the path is absolute, the first element of the returned array is empty. +# +# std::split_abspath() +# the path must begin with a slash, otherwise std::die() is called with +# an internal error message. +# +# std::split_relpath() +# analog to std::split_abspath() with a relative path. +# +std::split_path() { + local parts="$1" + local -r path="$2" + + IFS='/' + local std__split_path_result=( ${std__split_path_tmp} ) + unset IFS + std::upvar ${parts} "${std__split_path_result[@]}" + if (( $# >= 3 )); then + # return number of parts + std::upvar "$3" ${#std__split_path_result[@]} + fi +} + std::split_abspath() { local parts="$1" local -r path="$2" @@ -170,6 +195,25 @@ std::split_abspath() { fi } +std::split_relpath() { + local parts="$1" + local -r path="$2" + if [[ "${path:0:1}" == '/' ]]; then + std::die 255 "Oops: Internal error in '${FUNCNAME[0]}' called by '${FUNCNAME[1]}' }" + else + local -r std__split_path_tmp="${path}" + fi + + IFS='/' + local std__split_path_result=( ${std__split_path_tmp} ) + unset IFS + std::upvar ${parts} "${std__split_path_result[@]}" + if (( $# >= 3 )); then + # return number of parts + std::upvar "$3" ${#std__split_path_result[@]} + fi +} + std::read_versions() { local -r fname="$1" local varname='' From ea64c2bbf898f4d2f48814c0cf76f67cdcdd3516 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 17:53:56 +0100 Subject: [PATCH 21/28] modmange: execute modmanage.bash relative to own path --- Pmodules/modmanage.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Pmodules/modmanage.in b/Pmodules/modmanage.in index 4139b5d..6925f9f 100644 --- a/Pmodules/modmanage.in +++ b/Pmodules/modmanage.in @@ -2,4 +2,10 @@ unset BASH_ENV -"@BASH@" --noprofile --norc "@MODMANAGE@" "$@" +declare mydir=$(cd $(dirname "$0") && pwd) +declare libexecdir="$(dirname "${mydir}")/libexec" + +declare bash="${libexecdir}/bash" +declare modmanage="${libexecdir}/modmanage.bash" + +"${bash}" --noprofile --norc "${modmanage}" "$@" From 283167fad72481a11542158c7905f08502933e5f Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 17:55:58 +0100 Subject: [PATCH 22/28] build: install libpmodules.bash.in --- build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build b/build index 481d566..560b7c8 100755 --- a/build +++ b/build @@ -489,6 +489,9 @@ pmodules::install() { sed "${sed_cmd}" "${SRC_DIR}/modulecmd.tcl.in" > "${PMODULES_HOME}/libexec/modulecmd.tcl" chmod 0755 "${PMODULES_HOME}/libexec/modulecmd.tcl" + sed "${sed_cmd}" "${SRC_DIR}/libpmodules.bash.in" > "${PMODULES_HOME}/lib/libpmodules.bash" + chmod 0755 "${PMODULES_HOME}/lib/libpmodules.bash" + sed "${sed_cmd}" "${SRC_DIR}/modbuild.in" > "${PMODULES_HOME}/bin/modbuild" chmod 0755 "${PMODULES_HOME}/bin/modbuild" @@ -505,7 +508,6 @@ pmodules::install() { install -m 0644 "${SRC_DIR}/csh" "${PMODULES_HOME}/init" install -m 0644 "${SRC_DIR}/zsh" "${PMODULES_HOME}/init" - install -m 0644 "${SRC_DIR}/libpmodules.bash" "${PMODULES_HOME}/lib" install -m 0644 "${SRC_DIR}/libpbuild.bash" "${PMODULES_HOME}/lib" install -m 0644 "${SRC_DIR}/libpbuild_dyn.bash" "${PMODULES_HOME}/lib" install -m 0644 "${SRC_DIR}/libstd.bash" "${PMODULES_HOME}/lib" From a0b9dc7ba393b19131025bc2e1a6d57e784b8b0c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 17:58:06 +0100 Subject: [PATCH 23/28] modmanage.bash: reviewed, lot of modifications and fixes --- Pmodules/modmanage.bash.in | 264 ++++++++++++++++++------------------- 1 file changed, 127 insertions(+), 137 deletions(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index 94e22b0..52a9895 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -5,6 +5,7 @@ unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy! unset IFS # use default IFS shopt -s nullglob +shopt -s extglob # used for some output only declare -r CMD='modmanage' @@ -41,15 +42,6 @@ unset libexecdir declare PMODULES_VERSION='@PMODULES_VERSION@' -# In the dictionary Help we store the help text of each single command -# and for displaying the version. - -# initialize help text of 'module --version' -Help['version']=" -Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@ -Copyright GNU GPL v2 -" - ############################################################################## # # help [module|sub-command] @@ -66,7 +58,6 @@ SWITCHES: --debug enable debug output --dry-run dry run - SUBCOMMANDS: + init [switches] TARGET_DIR + install [switches] module... @@ -119,11 +110,10 @@ get_module_prefix() { ############################################################################## # -# Derive the relative module release file path -# from the relative module file path +# Derive the module release-file path from the module file-path. # # Arguments: -# $1: relative module file path +# $1: module file-path # get_releasefile_name() { echo "$(${dirname} "$1")/.release-$(basename "$1")" @@ -147,7 +137,7 @@ sync_module() { local -r target_root="$3" local -r src_prefix=$( get_module_prefix "${src_root}/${rel_modulefile}" ) - local -r rel_prefix=${src_prefix#${src_root}} + local -r rel_prefix=${src_prefix#${src_root}/} local -r target_prefix="${target_root}/${rel_prefix}" # install/update module @@ -192,8 +182,14 @@ sync_module() { sync_config() { src="$1/${PMODULES_CONFIG_DIR}/" dst="$2/${PMODULES_CONFIG_DIR}/" - ${rsync} --recursive --links --perms --delete \ - "${src}" "${dst}" 2>/dev/null || return $? + ${rsync} --links --perms \ + "${src}"/profile.{bash,csh,zsh} "${dst}" || return $? + ${rsync} --links --perms \ + "${src}"/profile.{bash,csh,zsh}-"${PMODULES_VERSION}" "${dst}" || return $? + ${rsync} --links --perms \ + "${src}/Pmodules.conf" "${dst}" || return $? + ${rsync} --links --perms \ + "${src}/modbuild.conf" "${dst}" || return $? echo } @@ -213,6 +209,8 @@ delete_module() { # # initialize a new module environment # +# +# Subcommands[init]='init' Options[init]='-o h -l src: -l user: -l help -l version:' Help[init]=" @@ -222,12 +220,7 @@ USAGE: A user must be specified with '--user=' if the programm is executed as root. - SWITCHES: - --src - Module environment we are going to sync from. If not - specified, the module environment this script is in - will be used. --user If this scripts runs with root privileges, a user name ore ID must be specified. @@ -244,14 +237,8 @@ subcommand_init() { std::die 1 " Error: the module environment you are going to use as source has not been initialized properly!" - - [[ -d "${src_root}/${PMODULES_CONFIG_DIR}" ]] && - [[ -d "${src_root}/Tools/Pmodules/${PMODULES_VERSION}" ]] || \ - std::die 1 " -Error: the module environment '${src_root}' has not been initialized properly!" } - local src_root='' local target_root=() local user='' while (($# > 0)); do @@ -262,27 +249,20 @@ Error: the module environment '${src_root}' has not been initialized properly!" --force | -f ) force='yes' ;; - --src | --src=* ) - if [[ $1 == --src=* ]]; then - src_root="${1#--*=}" - else - src_root="$2" - shift - fi - ;; --user | --user=* ) - if [[ $1 == --user=* ]]; then - user="${1#--*=}" - else + if [[ "$1" == '--user' ]]; then user="$2" shift + else + user="${1#--*=}" fi ;; -- ) : ;; * ) - target_root="$1" + # assign and remove trailing slashes + target_root="${1%%*([\/])}" ;; esac shift @@ -290,16 +270,6 @@ Error: the module environment '${src_root}' has not been initialized properly!" if [[ -z ${target_root} ]]; then std::die 1 "Error: no target directory specified!" fi - - # if source directory is not passed as argument, derive it from script name - if [[ -z "${src_root}" ]]; then - src_root=$(cd "${bindir}/../../../.." && pwd) - fi - [[ -d "${src_root}" ]] || \ - std::die 1 "Error: ${src_root}: source directory does not exist!" - [[ -r "${src_root}/config/profile.bash" ]] || \ - std::die 1 "Error: ${src_root}: shell profile does not exist or is not readable!" - source "${src_root}/config/profile.bash" local -i euid=$(id -u) if (( euid == 0 )); then @@ -312,6 +282,11 @@ Error: the module environment '${src_root}' has not been initialized properly!" std::die 1 "Error: --user option is only allowed if running as root!" fi + local src_root="$(std::get_abspath "${bindir}/../../../..")" + local config_file="${src_root}/${PMODULES_CONFIG_DIR}/profile.bash" + if [[ -r "${config_file}" ]]; then + source "${config_file}" + fi check_env || \ std::die 1 "Giving up..." @@ -319,58 +294,47 @@ Error: the module environment '${src_root}' has not been initialized properly!" Attempting to create a minimal module environment from the environment at '${PMODULES_ROOT}' " + echo "Initializing target directory '${target_root}' ..." + echo + if [[ -d "${target_root}" ]] && [[ ${force} == no ]]; then + echo "Warning: ${target_root} already exists." + std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ + std::die 1 "Abort ..." + fi + force='yes' + echo "Creating target directory '${target_root}'..." + ${mkdir} -p "${target_root}" || \ + std::die 1 "Error: make directory failed!" + echo - #..................................................................... - # initialize new module environment in given directory - # - # Arguments: - # $1 target directory - # - init_pmodules_environment() { - local -r src_root="${PMODULES_ROOT}" - local -r target_root=$1 - local src='' - local dst='' - echo "Initializing target directory '${target_root}' ..." - echo - if [[ -d "${target_root}" ]] && [[ ${force} == no ]]; then - echo "Warning: ${target_root} already exists." - std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ - std::die 1 "Abort ..." - fi - force='yes' - echo "Creating target directory '${target_root}'..." - ${mkdir} -p "${target_root}" || \ - std::die 1 "Error: make directory failed!" - echo + echo "Syncing configuration ..." + sync_config "${src_root}" \ + "${target_root}" || \ + std::die 1 "Error: configuration synchronization failed!" + + echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_root}' to '${target_root}'..." + sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \ + "${src_root}" \ + "${target_root}" || \ + std::die 1 "Error: sync Pmodules failed!" + echo - echo "Syncing configuration ..." - sync_config "${src_root}" \ - "${target_root}" || \ - std::die 1 "Error: configuration synchronization failed!" + for d in "${src_root}"/*/${PMODULES_MODULEFILES_DIR}; do + ${mkdir} -p "${target_root}/${d#${src_root}/}" + done + + if [[ -n "${user}" ]]; then + echo "Changing user of new module environment to '${user}'..." + ${chown} -R "${user}" "${target_root}" || \ + std::die 1 "Error: changing owner failed!" + echo + fi + echo "SourceRoot=${src_root}" > "${target_root}/${PMODULES_CONFIG_DIR}/modmanage.conf" + echo "New minimal module environment created at '${target_root}'." + echo "To use this environment, execute" + echo " sudo ln -fs ${target_root} /opt/psi" + echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash" - echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_root}' to '${target_root}'..." - sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \ - "${src_root}" \ - "${target_root}" || \ - std::die 1 "Error: sync Pmodules failed!" - ${mkdir} -p "${target_root}/Tools/${PMODULES_MODULEFILES_DIR}" - echo - - if [[ -n "${user}" ]]; then - echo "Changing user of new module environment to '${user}'..." - ${chown} -R "${user}" "${target_root}" || \ - std::die 1 "Error: changing owner failed!" - echo - fi - echo "New minimal module environment created at '${target_root}'." - echo "To use this environment, execute" - echo " sudo ln -fs ${target_root} /opt/psi" - echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash" - } - - umask 022 - init_pmodules_environment "${target_root}" } @@ -410,7 +374,7 @@ subcommand_install() { local modulefile='' local -A modules_to_install local -A dependencies_to_install - local -A map_to_group + local -A group_map local -a modulepath=() #...................................................................... @@ -425,8 +389,45 @@ subcommand_install() { #...................................................................... # - create_groupheads_map() { - : + create_group_map() { + # + # For the dependency resolution we need to know, whether a + # module - if loaded - adds a hierarchical group to MODULEPATH + # or not. + # + # Examples: + # Loading a compiler adds the hierarchical group for + # this compiler. The command + # module load gcc/10.3.0 + # prepends + # /Compiler/modulefiles/gcc/10.3.0 + # to MODULEPATH. + # + # The dependency files do not convey the information whether + # loading a module extends MODULEPATH or not. All we need to + # know is + # 1) does loading a specific module extends MODULEPATH? + # 2) if yes: what is the hierarchical group? + # + # Example: + # If we know that loading 'gcc/10.3.0' adds a directory + # in the hierarchical group 'Compiler' to MODULEPATH, we + # know that this directory is + # /Compiler/modulefiles/gcc/10.3.0 + local fname='' + local -i n + local -a parts + while read fname; do + std::split_relpath parts "${fname}" n + # We are only interested in groups adding something to + # the modulepath. + (( n >= 6 )) || continue + local key="${parts[-4]}/${parts[-3]}" + [[ -z "${group_map[${key}]}" ]] || continue + group_map[${key}]="${src_root}/${parts[0]}" + done < <({ cd "${src_root}" && \ + ${find} */"${PMODULES_MODULEFILES_DIR}" \ + \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null ) } #...................................................................... @@ -439,9 +440,9 @@ subcommand_install() { # Notes: # Following variables from the enclosing function are used: # modulepath (might be changed) - # map_to_group (read-only) + # group_map (read-only) # - resolve_dependencies_of_module () { + resolve_dependencies () { local -r modulefile="$1" local -- prefix=$(get_module_prefix "${modulefile}") @@ -466,15 +467,12 @@ subcommand_install() { -print -quit 2>/dev/null) [[ -n ${modulename} ]] || \ std::die 3 "Oops: required module '${dep}' not found!" - - dependencies_to_install[${modulename/${src_root}\/}]='.' - _resolve_dependencies "${modulename}" - if [[ -n ${map_to_group[${dep}]} ]]; then - # append hierarchical group to modulepath - local path="${src_root}/${map_to_group[${dep}]}/" - path+="${PMODULES_MODULEFILES_DIR}/" - path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}" - modulepath+=( "${path}" ) + + local rel_modulename="${modulename#${src_root}/}" + dependencies_to_install[${rel_modulename}]='.' + resolve_dependencies "${modulename}" + if [[ -n ${group_map[${dep}]} ]]; then + modulepath+=( "${group_map[${dep}]}/${rel_modulename##+([!/])/}" ) fi done } @@ -499,13 +497,13 @@ subcommand_install() { local parts std::info "The following modules will be installed/updated:" for modulefile in "${!modules_to_install[@]}"; do - std::split_fname parts "${modulefile}" + std::split_relpath parts "${modulefile}" std::info " ${parts[-2]}/${parts[-1]}" done 2>&1 | sort if (( ${#dependencies_to_install[@]} > 0 )); then std::info "\nThe following dependencies will be installed/updated:" for modulefile in "${!dependencies_to_install[@]}"; do - std::split_fname parts "${modulefile}" + std::split_relpath parts "${modulefile}" std::info " ${parts[-2]}/${parts[-1]}" done 2>&1 | sort fi @@ -557,38 +555,27 @@ subcommand_install() { shift done + if [[ -z ${src_root} ]]; then + local conf_file="${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}/modmanage.conf" + if [[ -r ${conf_file} ]]; then + source "${conf_file}" + src_root="${SourceRoot}" + fi + fi [[ -n ${src_root} ]] \ || std::die 3 "Oops: no installation source given." [[ -d ${src_root} ]] \ || std::die 3 "Oops: '${src_root}' is not a valid installation source." + source "${src_root}/${PMODULES_CONFIG_DIR}/profile.bash" scan_groups "${src_root}" set_initial_modulepath - - # - # create a mapping from module name to their group. - # Examples: - # gcc/5.2.0 -> Compiler - # openmpi/1.8.4 -> MPI - local fname='' - local -i n - local -a parts - while read fname; do - std::split_fname parts n "${fname}" - group="${parts[0]}" - # We are only interested in groups adding something to - # the modulepath. - if (( n >= 6 )); then - map_to_group[${parts[-4]}/${parts[-3]}]=${group} - fi - done < <({ cd "${src_root}" && \ - ${find} */"${PMODULES_MODULEFILES_DIR}" \ - \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null ) + create_group_map # search for to be installed modules and their dependencies while read modulefile; do - modules_to_install["${modulefile/${src_root}}"]+='.' - resolve_dependencies_of_module "${modulefile}" + modules_to_install["${modulefile#${src_root}/}"]+='.' + resolve_dependencies "${modulefile}" done < <("${modulecmd}" bash search \ "${module_pattern[@]}" \ "${with[@]/#/--with=}" \ @@ -601,7 +588,7 @@ subcommand_install() { # install/update ... for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do - std::split_fname parts "${modulefile}" + std::split_relpath parts "${modulefile}" std::info " ${parts[-2]}/${parts[-1]}" sync_module "${modulefile}" "${src_root}" "${target_root}" done @@ -672,6 +659,9 @@ tmp=$("${getopt}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ || print_help "${subcommand}" eval args=( "$tmp" ) unset tmp + +umask 022 + subcommand_${Subcommands[$subcommand]} "${args[@]}" # Local Variables: From 99d979677b787acba35ac8f970d4f97ec630bc60 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Wed, 10 Nov 2021 18:04:05 +0100 Subject: [PATCH 24/28] modulecmd.bash: case statement added by accident removed --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index dc20928..e79fbef 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -2,7 +2,7 @@ # PATH='/bin:/usr/bin' -case + unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy! unset IFS # use default IFS From 0a4d68d0150c985a1ac4c52bcaefd67b3d382c52 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 11 Nov 2021 19:32:00 +0100 Subject: [PATCH 25/28] modulecmd: handling od option --all-release-stages fixed --- Pmodules/modulecmd.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index e79fbef..cc53580 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -1920,7 +1920,7 @@ subcommand_search() { with_modules+=" && / ${module//\//\\/}/" done ;; - -a | --all-releases ) + -a | --all-releases-stages ) opt_use_rel_stages+="${ReleaseStages}" ;; --src | --src=*) From 0c6fbc635c63b577a9be0d60657e3a9e4d4008a4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 12 Nov 2021 10:55:29 +0100 Subject: [PATCH 26/28] modmanage: use 'modmanage' as name with getopt --- Pmodules/modmanage.bash.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index 52a9895..4a09067 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -655,7 +655,7 @@ if [[ "${subcommand}" != "init" ]] && [[ -z "${PMODULES_ROOT}" ]]; then std::die 1 "Error: No current module environment configured!" fi -tmp=$("${getopt}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ +tmp=$("${getopt}" --name="${CMD}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \ || print_help "${subcommand}" eval args=( "$tmp" ) unset tmp From 56b4a05b94f710bc5b4e56e78aa271d362ed3210 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 12 Nov 2021 11:09:19 +0100 Subject: [PATCH 27/28] modmanage: code cleanup, polishing and optimization - sync_config() moved into subcommand_init() - options '-y' and '--yes' added as synonym for '--force' - print environment source and target sub-cmd 'init' - building the group mapping optimized - improved readability of output in sub-cmd 'install' --- Pmodules/modmanage.bash.in | 187 ++++++++++++++++++++----------------- 1 file changed, 101 insertions(+), 86 deletions(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index 4a09067..4afced5 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -171,40 +171,6 @@ sync_module() { } -############################################################################## -# -# Sync the Pmodules configuration and templates -# -# Arguments: -# $1: source prefix of Pmodule environment -# $2: target prefix of Pmodule environment -# -sync_config() { - src="$1/${PMODULES_CONFIG_DIR}/" - dst="$2/${PMODULES_CONFIG_DIR}/" - ${rsync} --links --perms \ - "${src}"/profile.{bash,csh,zsh} "${dst}" || return $? - ${rsync} --links --perms \ - "${src}"/profile.{bash,csh,zsh}-"${PMODULES_VERSION}" "${dst}" || return $? - ${rsync} --links --perms \ - "${src}/Pmodules.conf" "${dst}" || return $? - ${rsync} --links --perms \ - "${src}/modbuild.conf" "${dst}" || return $? - echo -} - -############################################################################## -# -# Delete a module -# -# Arguments: -# $1: relative modulefile path -# $2: target prefix of Pmodule environment -# -delete_module() { - echo "Not implemented yet!" -} - ############################################################################## # # initialize a new module environment @@ -212,7 +178,7 @@ delete_module() { # # Subcommands[init]='init' -Options[init]='-o h -l src: -l user: -l help -l version:' +Options[init]='-o \?hfy -l src: -l user: -l help -l force -l yes' Help[init]=" USAGE: modmanage init [switches] TARGET_DIR @@ -224,7 +190,7 @@ SWITCHES: --user If this scripts runs with root privileges, a user name ore ID must be specified. - --force + --force|--yes|-f|-y re-initialise an already existing Pmodule environment. " @@ -238,7 +204,32 @@ subcommand_init() { Error: the module environment you are going to use as source has not been initialized properly!" } - + + #..................................................................... + # + # Sync the Pmodules configuration and templates + # + # Arguments: + # $1: source prefix of Pmodule environment + # $2: target prefix of Pmodule environment + # + sync_config() { + src="$1/${PMODULES_CONFIG_DIR}/" + dst="$2/${PMODULES_CONFIG_DIR}/" + ${rsync} --links --perms \ + "${src}"/profile.{bash,csh,zsh} "${dst}" \ + || return $? + ${rsync} --links --perms \ + "${src}"/profile.{bash,csh,zsh}-"${PMODULES_VERSION}" "${dst}" \ + || return $? + ${rsync} --links --perms \ + "${src}/Pmodules.conf" "${dst}" \ + || return $? + ${rsync} --links --perms \ + "${src}/modbuild.conf" "${dst}" \ + || return $? + } + local target_root=() local user='' while (($# > 0)); do @@ -246,7 +237,7 @@ initialized properly!" -h | -H | -\? | --help | -help ) print_help "${subcommand}" ;; - --force | -f ) + --force | --yes | -f | -y ) force='yes' ;; --user | --user=* ) @@ -290,22 +281,20 @@ initialized properly!" check_env || \ std::die 1 "Giving up..." - echo " -Attempting to create a minimal module environment from the -environment at '${PMODULES_ROOT}' -" - echo "Initializing target directory '${target_root}' ..." - echo + echo "Creating a minimal Pmodule environment from the environment at" + echo " ${PMODULES_ROOT}" + echo "in" + echo " ${target_root}" if [[ -d "${target_root}" ]] && [[ ${force} == no ]]; then echo "Warning: ${target_root} already exists." - std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ - std::die 1 "Abort ..." + std::get_YN_answer \ + "Do you really want to re-run the initialization? (y/N) " \ + || std::die 1 "Abort ..." fi force='yes' echo "Creating target directory '${target_root}'..." ${mkdir} -p "${target_root}" || \ std::die 1 "Error: make directory failed!" - echo echo "Syncing configuration ..." sync_config "${src_root}" \ @@ -317,7 +306,6 @@ environment at '${PMODULES_ROOT}' "${src_root}" \ "${target_root}" || \ std::die 1 "Error: sync Pmodules failed!" - echo for d in "${src_root}"/*/${PMODULES_MODULEFILES_DIR}; do ${mkdir} -p "${target_root}/${d#${src_root}/}" @@ -404,7 +392,7 @@ subcommand_install() { # to MODULEPATH. # # The dependency files do not convey the information whether - # loading a module extends MODULEPATH or not. All we need to + # loading a module extends MODULEPATH or not. What we need to # know is # 1) does loading a specific module extends MODULEPATH? # 2) if yes: what is the hierarchical group? @@ -414,20 +402,39 @@ subcommand_install() { # in the hierarchical group 'Compiler' to MODULEPATH, we # know that this directory is # /Compiler/modulefiles/gcc/10.3.0 - local fname='' - local -i n - local -a parts - while read fname; do - std::split_relpath parts "${fname}" n - # We are only interested in groups adding something to - # the modulepath. - (( n >= 6 )) || continue - local key="${parts[-4]}/${parts[-3]}" - [[ -z "${group_map[${key}]}" ]] || continue - group_map[${key}]="${src_root}/${parts[0]}" - done < <({ cd "${src_root}" && \ - ${find} */"${PMODULES_MODULEFILES_DIR}" \ - \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null ) + # + # This information we store in the dictionary 'group_map'. + # For concinience reasons we store the string 'src_root/group'. + # So, 'group_map' maps + # module/version -> src_root/group + # + # Example: + # group_map[gcc/10.3.0]="${src_root}/Compiler" + # + local group='' + for group in "${!GroupDepths[@]}"; do + (( ${GroupDepths[${group}]} > 0 )) || continue + local fname='' + while read fname; do + local -a parts=() + std::split_relpath parts "${fname}" + if (( ${#parts[@]} - 2 != ${GroupDepths[${group}]} )); then + std::warn "error in source group ${group}:" + std::warn "modulefile: ${fname}" + continue + fi + if [[ ${#parts[@]} < 4 ]]; then + echo "${group} ${parts[@]}" + fi + local key="${parts[-4]}/${parts[-3]}" + [[ -z "${group_map[${key}]}" ]] || continue + group_map[${key}]="${src_root}/${group}" + done < <(${find} -L "${src_root}/${group}/${PMODULES_MODULEFILES_DIR}" \ + \( -type l -o -type f \) \ + \! -name ".*" \ + -printf "%P\n" \ + ) + done } #...................................................................... @@ -439,7 +446,7 @@ subcommand_install() { # # Notes: # Following variables from the enclosing function are used: - # modulepath (might be changed) + # modulepath # group_map (read-only) # resolve_dependencies () { @@ -447,9 +454,9 @@ subcommand_install() { local -- prefix=$(get_module_prefix "${modulefile}") local -a rdeps=() - local rdeps_file="${prefix}/.dependencies" + local -- rdeps_file="${prefix}/.dependencies" local -a ideps=() - local ideps_file="${prefix}/.install_dependencies" + local -- ideps_file="${prefix}/.install_dependencies" if [[ -r "${rdeps_file}" ]]; then mapfile -t rdeps < <(grep -v '^ *#' "${rdeps_file}" ) @@ -471,9 +478,10 @@ subcommand_install() { local rel_modulename="${modulename#${src_root}/}" dependencies_to_install[${rel_modulename}]='.' resolve_dependencies "${modulename}" - if [[ -n ${group_map[${dep}]} ]]; then - modulepath+=( "${group_map[${dep}]}/${rel_modulename##+([!/])/}" ) - fi + [[ -v group_map[${dep}] ]] || continue + local dir="${group_map[${dep}]}" # = ${src_root}/ + dir+="/${rel_modulename##+([!/])/}" # += rel.name with group removed + modulepath+=( "${dir}" ) done } @@ -491,21 +499,28 @@ subcommand_install() { # modules_to_install (read-only) # dependencies_to_install (read-only) # - - print_modules_to_install() { + print_modules() { local modulefile - local parts - std::info "The following modules will be installed/updated:" - for modulefile in "${!modules_to_install[@]}"; do + for modulefile in "$@"; do + local -a parts std::split_relpath parts "${modulefile}" - std::info " ${parts[-2]}/${parts[-1]}" + local s='' + if (( ${#parts[@]} >= 6 )); then + s="(${parts[2]}/${parts[3]}" + for ((i = 4; i < ${#parts[@]}-2; i+=2)); do + s+=" ${parts[i]}/${parts[i+1]}" + done + s+=')' + fi + std::info "%-20s %s" "${parts[-2]}/${parts[-1]}" "$s" done 2>&1 | sort + } + print_modules_to_install() { + std::info "The following modules will be installed/updated:" + print_modules "${!modules_to_install[@]}" if (( ${#dependencies_to_install[@]} > 0 )); then std::info "\nThe following dependencies will be installed/updated:" - for modulefile in "${!dependencies_to_install[@]}"; do - std::split_relpath parts "${modulefile}" - std::info " ${parts[-2]}/${parts[-1]}" - done 2>&1 | sort + print_modules "${!dependencies_to_install[@]}" fi std::info "" std::get_YN_answer "Do you want to continue? [n] " || \ @@ -522,19 +537,19 @@ subcommand_install() { force='yes' ;; --src | --src=*) - if [[ $1 == --src=* ]]; then - src_root="${1#--*=}" - else + if [[ $1 == --src ]]; then src_root="$2" shift + else + src_root="${1#--*=}" fi ;; --target | --target=*) - if [[ $1 == --target=* ]]; then - target_root="${1#--*=}" - else + if [[ $1 == --target ]]; then target_root="$2" shift + else + target_root="${1#--*=}" fi ;; --with | --with=* ) @@ -542,7 +557,7 @@ subcommand_install() { with+=( "$2" ) shift else - with+=( "${1/--with=}" ) + with+=( "${1#--*=}" ) fi ;; -- ) From 58fcd79f9a5471d1ccf3c7ee80bec460cf6f0c90 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 12 Nov 2021 11:20:28 +0100 Subject: [PATCH 28/28] modmange: search sub-c,d implemented --- Pmodules/modmanage.bash.in | 85 ++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/Pmodules/modmanage.bash.in b/Pmodules/modmanage.bash.in index 4afced5..7748c11 100755 --- a/Pmodules/modmanage.bash.in +++ b/Pmodules/modmanage.bash.in @@ -61,6 +61,7 @@ SWITCHES: SUBCOMMANDS: + init [switches] TARGET_DIR + install [switches] module... + + search [switches] [string|pattern]... + help [subcommand] ' @@ -610,20 +611,80 @@ subcommand_install() { std::info "\nDone!\n" } # subcommand_install -############################################################################### +############################################################################## # -# delete specified module(s) -# -subcommand_delete() { - : -} +# sub-command 'search' +# +Subcommands[search]='search' +Options[search]='-o \?h -l with: -l help -l all-dep -l wrap -l glob -l src:' +Help[install]=' +USAGE: + modmanage search [switches] ... + search modules -############################################################################### -# -# remove modules which have been removed in our source -# -subcommand_cleanup() { - : +SWITCHES: + --src + Search modules in environment . + Default is the source defined in modmanage.conf. + + --with + Search module(s) in this sub-group. + + + Search modules matching given string. + + + Search modules matching given shell glob-pattern. +' + +subcommand_search() { + local -a args=() + while (($# > 0)); do + case $1 in + -h | -H | -\? | --help | -help ) + print_help "${subcommand}" + ;; + --src | --src=*) + if [[ $1 == --src ]]; then + src_root="$2" + shift + else + src_root="${1#--*=}" + fi + ;; + --with | --with=* ) + if [[ "$1" == --with ]]; then + args+=( '--with' "$2" ) + shift + else + args+=( "$1" ) + fi + ;; + --all-deps | --glob | --wrap ) + args+=( "$1" ) + ;; + -- ) + : + ;; + * ) + args+=( "$1" ) + ;; + esac + shift + done + if [[ -z ${src_root} ]]; then + local conf_file="${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}/modmanage.conf" + if [[ -r ${conf_file} ]]; then + source "${conf_file}" + src_root="${SourceRoot}" + fi + fi + [[ -n ${src_root} ]] \ + || std::die 3 "Oops: no installation source given." + [[ -d ${src_root} ]] \ + || std::die 3 "Oops: '${src_root}' is not a valid installation source." + ${modulecmd} bash search --src="${src_root}" --all-release-stages \ + "${args[@]}" 2>&1 1>/dev/null } declare force='no'