From ebdd93051b52537f4ab317e07d915583380390ce Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 5 Mar 2021 19:03:59 +0100 Subject: [PATCH 1/3] find added to required tools --- build | 4 ++++ config/versions.conf | 1 + 2 files changed, 5 insertions(+) diff --git a/build b/build index d85440e..163a928 100755 --- a/build +++ b/build @@ -345,6 +345,10 @@ pmodules::compile() { build bash fi + if [[ ! -f "${PMODULES_HOME}/sbin/find" ]] || [[ ${opt_force} == 'yes' ]]; then + build findutils + fi + if [[ ! -e "${PMODULES_HOME}/sbin/tclsh" ]] || [[ ${opt_force} == 'yes' ]]; then build Tcl fi diff --git a/config/versions.conf b/config/versions.conf index 9aa7e9d..00777ad 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -1,5 +1,6 @@ bash 5.1-rc3 coreutils 8.31 +findutils 4.7.0 getopt 1.1.6 gettext 0.21 modules 3.2.10.1 From 90d9cee20c39d462c1c2dbdc90efa9eca4787ad8 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 5 Mar 2021 19:04:54 +0100 Subject: [PATCH 2/3] module search: output all dependencies --- Pmodules/modulecmd.bash.in | 256 +++++++++++++++++++++++++------------ 1 file changed, 172 insertions(+), 84 deletions(-) diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 676e083..d7dce81 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -715,22 +715,29 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local -r dir="$1" + local saved_IFS=${IFS}; + IFS=':' + local -a dirs=($1) + IFS=${saved_IFS} local -r module="$2" local -r use_releases="${3:-${UsedReleases}}" local -a mods=() local release - test -d "${dir}" || return 0 - { - cd "${dir}" - while read mod; do - get_release release "${dir}/${mod}" - - if [[ :${use_releases}: =~ :${release}: ]]; then - mods+=( "${mod}" ${release} ) - fi - done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") - } + local dir='' + + for dir in "${dirs[@]}"; do + test -d "${dir}" || return 0 + { + cd "${dir}" + while read mod; do + get_release release "${dir}/${mod}" + + if [[ :${use_releases}: =~ :${release}: ]]; then + mods+=( "${mod}" ${release} "${dir}/${mod}") + fi + done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") + } + done echo "${mods[@]}" } @@ -790,7 +797,7 @@ subcommand_avail() { terse_output() { output_header - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do local mod=${mods[i]} local release=${mods[i+1]} case $release in @@ -807,7 +814,7 @@ subcommand_avail() { } machine_output() { - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do printf "%-20s\t%s\n" "${mods[i]}" "${mods[i+1]}" 1>&2 done } @@ -816,7 +823,7 @@ subcommand_avail() { # :FIXME: for the time being, this is the same as terse_output! long_output() { output_header - for (( i=0; i<${#mods[@]}; i+=2 )); do + for (( i=0; i<${#mods[@]}; i+=3 )); do local mod=${mods[i]} local release=${mods[i+1]} case $release in @@ -837,7 +844,7 @@ subcommand_avail() { local -i column=$cols local -i colsize=16 - for ((i=0; i<${#mods[@]}; i+=2)); do + for ((i=0; i<${#mods[@]}; i+=3 )); do if [[ ${verbosity_lvl} == 'verbose' ]]; then local release=${mods[i+1]} case ${mods[i+1]} in @@ -1483,7 +1490,8 @@ subcommand_clear() { # Subcommands[search]='search' Options[search]='-o aH -l help -l no-header -l print-modulefiles ' -Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv' +Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv ' +Options[search]+='-l verbose' Help[search]=' USAGE: module search [switches] string... @@ -1509,26 +1517,21 @@ SWITCHES: module search --with=gcc/4.8.3 lists all modules in the hierarchy compiled with gcc 4.8.3. + + --verbose + vebose output ' subcommand_search() { local -r subcommand='search' local modules=() local with_modules='//' - local src_prefix='' + local src_prefix=() local opt_print_header='yes' local opt_print_modulefiles='no' local opt_print_csv='no' + local opt_print_verbose='no' local opt_use_releases=':' - local -r fmt="%-20s %-10s %-12s %-s\n" - - # no args - print_header() { - printf '\n' 1>&1 - printf "${fmt}" "Module" "Release" "Group" "Requires" 1>&2 - printf -- '-%.0s' {1..60} 1>&2 - printf '\n' 1>&2 - } #..................................................................... # @@ -1542,31 +1545,111 @@ subcommand_search() { # with_modules # print_result() { - local -r tmpfile=$1 - [[ "${opt_print_header}" == "yes" ]] && print_header - if [[ "${opt_print_modulefiles}" == "yes" ]]; then - while read -a line; do - # group first - local out="${line[2]}/" - # add directory of modulefiles - out+="${PMODULES_MODULEFILES_DIR}/" - for d in "${line[@]:3}"; do - out+="$d/" - done - out+="${line[0]}" - std::info "${out}" - done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}") - elif [[ "${opt_print_csv}" == "yes" ]]; then - while read -a toks; do - : - done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}") - else - "${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ - awk "${with_modules}" 1>&2 - fi + local func_print_header='' + local func_print_line='' + local fmt='' + + # no args + print_header_default() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Requires" + std::info '-%.0s' {1..60} + std::info '' + } + + print_line_default() { + local deps="${@:5}" + std::info "${fmt}" "$1" "$2" "$3" "${deps}" + } + + print_default() { + fmt="%-20s %-10s %-12s %-s" + if [[ ${opt_print_header} == 'yes' ]]; then + func_print_header='print_header_default' + else + func_print_header='print_header_none' + fi + func_print_line='print_line_default' + } + + print_header_verbose() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Overlay" "Requires" + std::info '-%.0s' {1..79} + std::info '' + } + + print_line_verbose() { + std::info "${fmt}" "$@" + } + + print_verbose() { + fmt="%-20s %-10s %-12s %-20s %-s" + func_print_header='print_header_verbose' + func_print_line='print_line_verbose' + } + + print_header_none() { + : + } + + print_line_modulefile() { + local line=( "$@" ) + # group first + local out="${line[2]}/" + # add directory of modulefiles + out+="${PMODULES_MODULEFILES_DIR}/" + for d in "${line[@]:3}"; do + out+="$d/" + done + out+="${line[0]}" + std::info "${out}" + } + + # print full modulefile names only + print_modulefiles() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_header_none' + } + + print_line_csv() { + : + } + + print_csv() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_line_csv' + } + + local -r tmpfile=$1 + + if [[ "${opt_print_modulefiles}" == 'yes' ]]; then + print_modulefiles + elif [[ "${opt_print_csv}" == 'yes' ]]; then + print_csv + elif [[ "${opt_print_verbose}" == 'yes' ]]; then + print_verbose + else + print_default + fi + + ${func_print_header} + while read -a toks; do + ${func_print_line} "${toks[@]}" + done < <("${sort}" -k 1,1 -k 4,4 -k 5,5 "${tmpfile}" | \ + awk "${with_modules}") + } + + get_module_prefix() { + local "$1" + local modulefile="$2" + local -r _prefix=$("${modulecmd}" bash show "${modulefile}" 2>&1 | \ + awk '/_PREFIX |_HOME / {print $3; exit}') + std::upvar $1 "${_prefix}" } + #..................................................................... # # search modules @@ -1580,47 +1663,48 @@ subcommand_search() { local -r module=$1 # write results to a temporary file for later processing local -r tmpfile=$( "${mktemp}" /tmp/Pmodules.XXXXXX ) \ - || std::die 1 "Oops: unable to create tmp file!\n" + || std::die 1 "Oops: unable to create tmp file!" local group # loop over all groups for group in "${!GroupDepths[@]}"; do # loop over all directories which can be added to # MODULEPATH inside current group local depth=${GroupDepths[${group}]} - local mpaths=( $(find \ - "${src_prefix}/${group}/modulefiles" \ - -type d \ - -mindepth ${depth} -maxdepth ${depth} \ - 2>/dev/null)) - local mpath - for mpath in "${mpaths[@]}"; do - # get dependencies encoded in directory name - local p="${mpath/${src_prefix}}" - p=( ${p//\// } ) - local deps=() - local -i i - for ((i=2; i < ${#p[@]}; i+=2)); do - deps+=( ${p[i]}/${p[i+1]} ) - done - local requires=${deps[@]} + local modulepath=( $(find \ + "${src_prefix[@]/%//${group}/modulefiles}" \ + -mindepth ${depth} -maxdepth ${depth} \ + -type d \ + -printf "%p:" + 2>/dev/null)) - # get and print all available modules in $mpath - # with respect to the requested releases - # tmpfile: module/version release group group- - # dependencies... - local mods=( $( get_available_modules \ - "${mpath}" \ - "${module}" \ - "${opt_use_releases}" ) ) - [[ ${#mods[@]} == 0 ]] && continue - for (( i=0; i<${#mods[@]}; i+=2 )); do - printf "${fmt}" ${mods[i]} "${mods[i+1]}" \ - ${group} "${requires}" >> "${tmpfile}" - done + # get and print all available modules in $mpath + # with respect to the requested releases + # tmpfile: module/version release group group- + # dependencies... + local mods=( $( get_available_modules \ + "${modulepath}" \ + "${module}" \ + "${opt_use_releases}" ) ) + for (( i=0; i<${#mods[@]}; i+=3 )); do + local name=${mods[i]} + local release=${mods[i+1]} + local modulefile=${mods[i+2]} + local prefix='' + local requires='' + + get_module_prefix prefix "${modulefile}" + local dependencies_file="${prefix}/.dependencies" + if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then + requires=$(< "${dependencies_file}") + fi + + echo ${mods[i]} ${mods[i+1]} \ + ${group} ${mods[i+2]} \ + ${requires} >> "${tmpfile}" done done print_result "${tmpfile}" - rm -f "${tmpfile}" + #rm -f "${tmpfile}" } while (( $# > 0 )); do @@ -1647,7 +1731,7 @@ subcommand_search() { local arg=${1/--release=} fi is_release "${arg}" || \ - std::die 1 "%s %s: %s -- %s\n" \ + std::die 1 "%s %s: %s -- %s" \ "${CMD}" 'search' \ "illegal release name" \ "${arg}" @@ -1661,7 +1745,7 @@ subcommand_search() { local arg=${1/--with=} fi if [[ -z ${arg} ]] || [[ "${arg}" =~ "-*" ]]; then - std::die 1 "%s %s: %s -- %s\n" \ + std::die 1 "%s %s: %s -- %s" \ "${CMD}" 'search' \ "illegal value for --with option" \ "${arg}" @@ -1680,6 +1764,9 @@ subcommand_search() { pmodules::check_directories "${src_prefix}" shift ;; + -v | --verbose ) + opt_print_verbose='yes' + ;; -- ) ;; * ) @@ -1700,6 +1787,7 @@ subcommand_search() { modules+=( '' ) fi + # :FIXME: do we need this? if (( ${#GroupDepths[@]} == 0 )) || \ [[ ${src_prefix} != ${PMODULES_ROOT} ]]; then scan_groups "${src_prefix}" From 7001c86154e420b1b34625f5f2ead10dd959c24c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 9 Mar 2021 14:04:45 +0100 Subject: [PATCH 3/3] option to print all dependencies added to module search more changes - define and set a default IFS - use our own find - not find command installed on the system - the modulepath directories are now passed as arry to get_available_modules() - new option "--wrap" --- Pmodules/modulecmd.bash.in | 183 ++++++++++++++++++++++++------------- Tools/findutils/build | 4 + 2 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 Tools/findutils/build diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index d7dce81..b1bd0bd 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -18,6 +18,7 @@ declare -r base64="${sbindir}/base64" declare -r mktemp="${sbindir}/mktemp" declare -r sort="${sbindir}/sort" declare -r getopt="${sbindir}/getopt" +declare -r find="${sbindir}/find" source "${libdir}/libstd.bash" source "${libdir}/libpmodules.bash" @@ -35,6 +36,10 @@ fi declare verbosity_lvl=${PMODULES_VERBOSITY:-'verbose'} +# we use newline as internal field separator +IFS=$' \t\n' +declare -r __IFS=${IFS} # used to restore IFS + shopt -s nullglob declare -A GroupDepths='()' @@ -263,10 +268,9 @@ subcommand_load() { local prefix='' local m='' - local saved_IFS="${IFS}"; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} # # Test whether a given module is available. @@ -407,10 +411,9 @@ subcommand_load() { # - release:group:name or # - name:release - local save_ifs=${IFS} IFS=':' local -a toks=($m) - IFS=${save_ifs} + IFS=${__IFS} local group='' local release='' if is_group "${toks[0]}"; then @@ -715,12 +718,11 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local saved_IFS=${IFS}; - IFS=':' - local -a dirs=($1) - IFS=${saved_IFS} - local -r module="$2" - local -r use_releases="${3:-${UsedReleases}}" + local -r module="$1" + local -r use_releases="${2:-${UsedReleases}}" + shift 2 + local -a dirs=( "$@" ) + local -a mods=() local release local dir='' @@ -735,7 +737,7 @@ get_available_modules() { if [[ :${use_releases}: =~ :${release}: ]]; then mods+=( "${mod}" ${release} "${dir}/${mod}") fi - done < <(find * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") + done < <(${find} * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") } done echo "${mods[@]}" @@ -914,16 +916,16 @@ subcommand_avail() { if (( ${#pattern[@]} == 0 )); then pattern+=( '' ) fi - local saved_IFS=${IFS}; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} local string for string in "${pattern[@]}"; do for dir in "${modulepath[@]}"; do mods=( $( get_available_modules \ - "${dir}" "${string}" \ - "${opt_use_releases}" ) ) + "${string}" \ + "${opt_use_releases}" \ + "${dir}" ) ) [[ ${#mods[@]} == 0 ]] && continue ${output_function} done @@ -942,7 +944,7 @@ compute_group_depth () { local group=${dir%/*} local group=${group##*/} [[ -n "${GroupDepths[${group}]}" ]] && return 0 - local -i depth=$(find "${dir}" -depth \( -type f -o -type l \) \ + local -i depth=$(${find} "${dir}" -depth \( -type f -o -type l \) \ -printf "%d" -quit 2>/dev/null) (( depth-=2 )) # if a group doesn't contain a modulefile, depth is negativ @@ -998,10 +1000,9 @@ SWITCHES: subcommand_use() { local -r subcommand='use' - local saved_IFS=${IFS}; IFS=':' local -a modulepath=(${MODULEPATH}) - IFS=${saved_IFS} + IFS=${__IFS} local add2path_func='std::append_path' group_is_used() { @@ -1491,7 +1492,8 @@ subcommand_clear() { Subcommands[search]='search' Options[search]='-o aH -l help -l no-header -l print-modulefiles ' Options[search]+='-l release: -l with: -l all-releases -l src: -l print-csv ' -Options[search]+='-l verbose' +Options[search]+='-l verbose ' +Options[search]+='-l all-deps -l wrap' Help[search]=' USAGE: module search [switches] string... @@ -1520,18 +1522,25 @@ SWITCHES: --verbose vebose output + + --wrap + wrap output ' subcommand_search() { local -r subcommand='search' local modules=() local with_modules='//' + local -ir cols=$(tput cols) # get number of columns of terminal + local -i max_len_modulename=0 local src_prefix=() local opt_print_header='yes' local opt_print_modulefiles='no' local opt_print_csv='no' local opt_print_verbose='no' local opt_use_releases=':' + local opt_all_deps='no' + local opt_wrap='no' #..................................................................... # @@ -1549,21 +1558,8 @@ subcommand_search() { local func_print_line='' local fmt='' - # no args - print_header_default() { - std::info '' - std::info "${fmt}" "Module" "Release" "Group" "Requires" - std::info '-%.0s' {1..60} - std::info '' - } - - print_line_default() { - local deps="${@:5}" - std::info "${fmt}" "$1" "$2" "$3" "${deps}" - } - print_default() { - fmt="%-20s %-10s %-12s %-s" + fmt="%-${max_len_modulename}s %-10s %-12s %-s" if [[ ${opt_print_header} == 'yes' ]]; then func_print_header='print_header_default' else @@ -1572,23 +1568,64 @@ subcommand_search() { func_print_line='print_line_default' } - print_header_verbose() { - std::info '' - std::info "${fmt}" "Module" "Release" "Group" "Overlay" "Requires" - std::info '-%.0s' {1..79} + print_header_default() { std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Requires" + std::info '-%.0s' $(seq 1 ${cols}) } - print_line_verbose() { - std::info "${fmt}" "$@" + print_line_default() { + write_line() { + local str="$1" + if (( ${#str} >= cols )); then + str="${str:0:$((cols-1))}>" + fi + std::info "${str}" + } + if [[ "${opt_wrap}" == 'no' ]]; then + local deps="${@:5}" + local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[@]}") + write_line "${str}" + else + local deps=( "${@:5}" ) + local str=$(printf "${fmt}" "$1" "$2" "$3" "${deps[0]}") + for (( i = 1; i < ${#deps[@]}; i++ )); do + if (( ${#str} + ${#deps[i]} + 1 <= cols )); then + str+=" ${deps[i]}" + else + write_line "${str}" + str=$(printf "${fmt}" "" "" "" "> ${deps[i]}") + fi + done + write_line "${str}" + fi } print_verbose() { - fmt="%-20s %-10s %-12s %-20s %-s" + fmt="%-${max_len_modulename}s %-10s %-12s %-s" func_print_header='print_header_verbose' func_print_line='print_line_verbose' } + print_header_verbose() { + std::info '' + std::info "${fmt}" "Module" "Release" "Group" "Dependencies/Modulefile" + std::info '-%.0s' $(seq 1 ${cols}) + } + + print_line_verbose() { + local deps="${@:5}" + std::info "${fmt}" "$1" "$2" "$3" "dependencies: ${deps}" + std::info "${fmt}" "" "" "" "modulefile: $4" + } + + # print full modulefile names only + print_modulefiles() { + fmt='' + func_print_header='print_header_none' + func_print_line='print_header_none' + } + print_header_none() { : } @@ -1606,13 +1643,6 @@ subcommand_search() { std::info "${out}" } - # print full modulefile names only - print_modulefiles() { - fmt='' - func_print_header='print_header_none' - func_print_line='print_header_none' - } - print_line_csv() { : } @@ -1670,37 +1700,54 @@ subcommand_search() { # loop over all directories which can be added to # MODULEPATH inside current group local depth=${GroupDepths[${group}]} - local modulepath=( $(find \ - "${src_prefix[@]/%//${group}/modulefiles}" \ - -mindepth ${depth} -maxdepth ${depth} \ - -type d \ - -printf "%p:" - 2>/dev/null)) - + local s='' + if (( depth > 0 )); then + s=$(printf '/*%.0s' $(seq 1 ${depth})) + fi + local modulepath=( ${src_prefix[@]/%//${group}/modulefiles$s} ) + # get and print all available modules in $mpath # with respect to the requested releases # tmpfile: module/version release group group- # dependencies... local mods=( $( get_available_modules \ - "${modulepath}" \ "${module}" \ - "${opt_use_releases}" ) ) + "${opt_use_releases}" \ + "${modulepath[@]}" \ + ) ) + for (( i=0; i<${#mods[@]}; i+=3 )); do local name=${mods[i]} local release=${mods[i+1]} local modulefile=${mods[i+2]} - local prefix='' - local requires='' - get_module_prefix prefix "${modulefile}" - local dependencies_file="${prefix}/.dependencies" - if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then - requires=$(< "${dependencies_file}") + if (( ${#name} > max_len_modulename)); then + max_len_modulename=${#name} fi - echo ${mods[i]} ${mods[i+1]} \ - ${group} ${mods[i+2]} \ - ${requires} >> "${tmpfile}" + if [[ "${opt_print_verbose}" == 'yes' ]] || [[ "${opt_all_deps}" == 'yes' ]]; then + local prefix='' + get_module_prefix prefix "${modulefile}" + local dependencies_file="${prefix}/.dependencies" + if [[ -n ${prefix} ]] && [[ -r "${dependencies_file}" ]]; then + deps=($(< "${dependencies_file}")) + else + deps=() + fi + else + # get dependencies encoded in directory name + local deps=() + local -i j + IFS='/' + local toks=( ${modulefile} ) + for ((j = -depth-2; j < -2; j += 2)); do + deps+=( "${toks[*]: $j:2}" ); + done + IFS=${__IFS} + fi + + echo ${name} ${release} ${group} ${modulefile} \ + ${deps[@]} >> "${tmpfile}" done done print_result "${tmpfile}" @@ -1712,6 +1759,9 @@ subcommand_search() { -H | --help ) print_help "${subcommand}" ;; + --all-deps ) + opt_all_deps='yes' + ;; --no-header ) opt_print_header='no' ;; @@ -1767,6 +1817,9 @@ subcommand_search() { -v | --verbose ) opt_print_verbose='yes' ;; + --wrap ) + opt_wrap='yes' + ;; -- ) ;; * ) diff --git a/Tools/findutils/build b/Tools/findutils/build new file mode 100644 index 0000000..0f1fc2f --- /dev/null +++ b/Tools/findutils/build @@ -0,0 +1,4 @@ +#!/usr/bin/env modbuild + +pbuild::set_download_url "https://ftp.gnu.org/pub/gnu/$P/$P-$V.tar.xz" +pbuild::add_configure_args "--bindir=${PREFIX}/sbin"