From 56ab7553c9f44107f4e32250b261e35da466e7f7 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Thu, 6 Feb 2025 09:13:42 +0100 Subject: [PATCH] support for overlays without groups (e.g. for Spack modules) --- Pmodules/libmodules.tcl | 95 ++++----- Pmodules/libpmodules.bash.in | 228 +++++++++++++++------ Pmodules/modulecmd.bash.in | 380 +++++++++++++++++++++++++---------- 3 files changed, 490 insertions(+), 213 deletions(-) diff --git a/Pmodules/libmodules.tcl b/Pmodules/libmodules.tcl index 44cdea1..d286905 100644 --- a/Pmodules/libmodules.tcl +++ b/Pmodules/libmodules.tcl @@ -28,11 +28,12 @@ set ::MODULEFILES_DIR "modulefiles" set ::ol_replacing "r" # variable derived from PMODULES_ENV -set ::Dir2OverlayMap {} -set ::OverlayInfo {} +array set ::Dir2OverlayMap {} +array set ::OverlayInfo {} set ::UsedGroups {} set ::UsedOverlays {} +debug "_pmodules_parse_pmodules_env" proc _pmodules_parse_pmodules_env { } { # # In this library we need the value of some BASH variables @@ -41,6 +42,8 @@ proc _pmodules_parse_pmodules_env { } { # 'typeset -p VAR' - to Tcl. # foreach line [split [base64::decode $::env(PMODULES_ENV)] "\n"] { + # lines are something like "declare -a UsedGroups=value" + # assign variable name -> key and value -> value if { ![regexp -- {.* -[aAx]* (.*)=\((.*)\)} $line -> key value] } { continue } @@ -49,7 +52,8 @@ proc _pmodules_parse_pmodules_env { } { array set ::Dir2OverlayMap [regsub -all {[]=[]} $value " "] } OverlayInfo { - array set ::OverlayInfo [regsub -all {[]=[]} $value " "] + set tmp_olinfo [regsub -all {[]=[]} $value " "] + array set ::OverlayInfo $tmp_olinfo } UsedOverlays { array set tmp [regsub -all {[]=[]} $value " "] @@ -64,8 +68,10 @@ proc _pmodules_parse_pmodules_env { } { } } } + debug "return" } +debug "module-addgroup" proc module-addgroup { group } { global env global name @@ -289,88 +295,71 @@ proc ModulesHelp { } { # or # /group/modulefiles/X1/Y1//X2/Y2/name/version # -proc _find_overlay { modulefile_components } { +proc _find_overlay { modulefile } { debug "_find_overlay()" foreach ol $::UsedOverlays { debug "ol = $ol" set ol_modulefiles_root $::OverlayInfo(${ol}:modulefiles_root) - if { [string range $ol_modulefiles_root end end] == "/" } { - set ol_modulefiles_root [string range $ol_modulefiles_root 0 end-1] - } - debug "ol_modulefiles_root = $ol_modulefiles_root" - set ol_modulefiles_root_splitted [file split $ol_modulefiles_root] - set modulefile_root [file join \ - {*}[lrange \ - $modulefile_components \ - 0 [expr [llength $ol_modulefiles_root_splitted] - 1]]] - debug "modulefile_root = $modulefile_root" - if { [string compare $ol_modulefiles_root $modulefile_root] == 0 } { - debug "ol_modulefiles_root_splitted = $ol_modulefiles_root_splitted" - return $ol_modulefiles_root_splitted - } - } + if { [string match "$ol_modulefiles_root/*" modulefile] == 0 } { + return "$ol" + } + } debug "overlay not found" return {} } -proc _is_in_overlay { } { - debug "_is_in_overlay?" - set parts [_find_overlay [file split $::ModulesCurrentModulefile]] - debug "_is_in_overlay: $parts" - expr {[string compare $parts ""] == 0 } -} - proc _pmodules_init_global_vars { } { debug "_pmodules_init_global_vars() called" - global group global GROUP - global name - global P - global version - global V + global P # name of module (without version) + global V # full version of module global V_MAJOR global V_MINOR global V_PATCHLVL global V_RELEASE - global V_PKG - global variant + global V_PKG # version without release no. and/or suffix global PREFIX # prefix of package - set modulefile_splitted [file split $::ModulesCurrentModulefile] + set current_modulefile [file split $::ModulesCurrentModulefile] - set ol_modulefiles_root_splitted [_find_overlay ${modulefile_splitted}] - debug "init: ol_modulefiles_root_splitted=$ol_modulefiles_root_splitted" - set rel_modulefile [lrange $modulefile_splitted [llength $ol_modulefiles_root_splitted] end] - set group [lindex $rel_modulefile 0] - set GROUP "${group}" - set name [lindex $modulefile_splitted end-1] - set version [lindex $modulefile_splitted end] - set suffixes [lassign [split $version _] v] + set P [lindex $current_modulefile end-1] + set V [lindex $current_modulefile end] + set suffixes [lassign [split $V _] v] lassign [split $v -] V_PKG V_RELEASE lassign [split $V_PKG .] V_MAJOR V_MINOR V_PATCHLVL - set variant [lrange $rel_modulefile 2 end] - set modulefiles_root [file join {*}$ol_modulefiles_root_splitted] - set ol $::Dir2OverlayMap($modulefiles_root) + set ol [_find_overlay $::ModulesCurrentModulefile] + if { $::OverlayInfo(${ol}:has_groups) == "true" } { + set modulefiles_root [file split $::OverlayInfo(${ol}:modulefiles_root)] + set rel_modulefile [lrange $current_modulefile [llength $modulefiles_root] end] + set GROUP [lindex $rel_modulefile 0] + set install_prefix [file split $::OverlayInfo(${ol}:install_root)] + set ::variant [lrange $rel_modulefile 2 end] + set prefix "$install_prefix $GROUP [lreverse_n $::variant 2]" + set PREFIX [file join {*}$prefix] + } else { + set GROUP "None" + set ::variant {} + set PREFIX $::OverlayInfo(${ol}:install_root) + } - set install_prefix [file split $::OverlayInfo(${ol}:install_root)] - set prefix "$install_prefix $group [lreverse_n $variant 2]" - set PREFIX [file join {*}$prefix] - set P "${name}" - set V "${version}" + # :FIXME: the following vars are still used + set ::name $P + set ::version $V + set ::group $GROUP - debug "modulefiles_root=$modulefiles_root" debug "ol=$ol" debug "PREFIX=$PREFIX" - debug "group of module $name: $group" + debug "group of module $P: $GROUP" } if { [info exists ::whatis] } { module-whatis "$whatis" } +debug "init" _pmodules_parse_pmodules_env -if {[_is_in_overlay] == 0} { +if {[_find_overlay $::ModulesCurrentModulefile] != ""} { debug "setup env vars for module in overlay" _pmodules_init_global_vars conflict $name diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index cc6945c..c4164d7 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -8,20 +8,49 @@ declare -a Overlays=() declare -A OverlayInfo declare -A Dir2OverlayMap +# An overlay has a type defining the way modules in this overlay +# make modules in the other overlays unavailable. +# +# 'normal' +# Make modules in other overlay unavailable with the same full name. +# If the overlay doesn't support groups, the overlay should provide +# only modules with different names from the modules in the other +# overlays. Otherwise you modules as available which cannot be loaded. +# Example: +# A overlay providing modules with name gcc doesn't conceal the gcc +# modules in the base overlay, but they are listed as available. +# +# +# 'hiding' +# Make modules in other overlay unavailable with the same name. +# Examples: +# - If a module with name 'gcc' is in a overlay with this +# type, only the gcc modules in this overlay are available. This +# can for example be used to conceal old versions of gcc. +# - In same case we need special variants of a module for a system, +# for exampe openmpi and mpich. Variants of these software can be +# made available in an overlay. If such an overlay is used, modules +# which should not be used can be concealed. +# If the overlay doesn't support groups, a module in the overlay +# conceals all modules in other overlays independend from the group. +# Example: +# - In the Spack overlays - which doesn't support groups - are modules +# with name gcc. The gcc modules in the Spack overlays conceals the +# gcc modules in the group Programming of the base overlay. +# 'replacing' +# This type can be used to make hole groups unavailable. +# +# declare -r ol_normal='n' declare -r ol_hiding='h' declare -r ol_replacing='r' - -# -# compute depth of modulefile directory. -# -# Args: -# $1: absolute path of a modulefile directory -# compute_group_depth () { - local -n result="$1" - local -r dir="$2" + : " + Compute depth of modulefile directory. + " + local -n result="$1" # ref.var to return result + local -r dir="$2" # absolute path of a modulefile directory if [[ ! -d "${dir}" ]]; then ${mkdir} -p "${dir}" || \ std::die 1 "Cannot create directory -- ${dir}" @@ -36,27 +65,28 @@ compute_group_depth () { (( result < 0 )) && (( result = 0 )) || : } -# -# (Re-)Scan available groups in given overlays and compute group depth's -# -# Args: -# $@: overlay names -# scan_groups () { - local ol - local depth + : " + (Re-)Scan available groups in the overlays $@ and compute group depth's. + Set GroupDepths[group] and Dir2OverlayMap[ol:dir]. + " + local -- ol='' + local -i depth=0 for ol in "$@"; do + [[ "${OverlayInfo[${ol}:has_groups]}" == 'true' ]] || continue local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}" - local dir - for dir in "${modulefiles_root}"/*/"${__MODULEFILES_DIR__}"; do + local dir='' + for dir in "${modulefiles_root}"/*/"${__MODULEFILES_DIR__}"; do local group="${dir%/*}" group="${group##*/}" if [[ ! -v GroupDepths[${group}] ]]; then compute_group_depth depth "${dir}" GroupDepths[$group]=${depth} fi + # :FIXME: do we need this? + Dir2OverlayMap[${dir}]="${ol}" Dir2OverlayMap[${dir%/"${__MODULEFILES_DIR__}"*}]="${ol}" - done + done done GroupDepths['none']=0 } @@ -79,15 +109,21 @@ declare -A OverlayConfigKeys=( ['modulefiles_root']='' ['excludes']='' ['type']='n' - ['modulepath']='' - ['modulepath_unstable']='' - ['modulepath_stable']='' - ['modulepath_deprecated']='' + ['conflicts']='' + ['config']='' ['has_groups']='true' ['has_relstages']='true' ['default_relstage']='unstable' ['has_additional_modulepaths']='false' - ['conflicts']='' +) + +declare -A OverlayPathConfigKeys=( + ['target_cpus']='' + ['modulepath']='' + ['modulepath_unstable']='' + ['modulepath_stable']='' + ['modulepath_deprecated']='' + ) yml::die_parsing(){ @@ -185,33 +221,102 @@ yml::get_seq(){ return 1 } +yml::die_invalid_ol_install_root(){ + std::die 3 "Invalid installation root directory for overlay '$1' -- $2" +} + +yml::die_invalid_ol_modulefiles_root(){ + std::die 3 "Invalid modulefiles root directory for overlay '$1' -- $2" +} + +yml::die_invalid_ol_type(){ + std::die 3 "Invalid type for overlay '$1' -- $2" +} + +yml::die_invalid_ol_relstage(){ + std::die 3 "Invalid default release stage for overlay '$1' -- $2" +} + +yml::die_invalid_ol_key(){ + std::die 3 "%s -- %s\n%s" \ + "Invalid key in configuration" \ + "$1" "$2" +} + +parse_path_config(){ + local -n yaml="$1" + local -- ol_name="$2" + + local -- key='' + for key in "${!OverlayPathConfigKeys[@]}"; do + OverlayInfo[${ol_name}:${key}]="${OverlayPathConfigKeys[${key}]}" + done + local -i l=0 + yml::get_seq_length l yaml . + local -i i=0 + for ((i=0; i/dev/null [[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \ - std::die 3 \ - "Invalid installation root directory for overlay '${ol_name}' -- ${value}" + yml::die_invalid_ol_install_root "${ol_name}" "${value}" ;; modulefiles_root ) yml::get_value value yaml_input "${node}.${key}" '!!str' OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}") mkdir -p "${OverlayInfo[${ol_name}:modulefiles_root]}" 2>/dev/null [[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \ - std::die 3 \ - "Invalid modulefiles root directory for overlay '${ol_name}' -- ${value}" + yml::die_invalid_ol_modulefiles_root "${ol_name}" "${value}" ;; type ) yml::get_value value yaml_input "${node}.${key}" '!!str' @@ -240,31 +343,41 @@ pm::read_config(){ : ;; * ) - std::die 3 "Invalid type for overlay '${ol_name}' -- ${type}" + yml::die_invalid_ol_type "${ol_name}" "${value}" ;; esac OverlayInfo[${ol_name}:type]="${value}" ;; - excludes ) + default_relstage ) + yml::get_value value yaml_input "${node}.${key}" '!!str' + case ${value} in + 'unstable' | 'stable' | 'deprecated' ) + : + ;; + *) + yml::die_invalid_ol_relstage "${ol_name}" "${value}" + ;; + esac + OverlayInfo[${ol_name}:${key}]="${value}" + ;; + conflicts | excludes ) yml::get_seq value yaml_input "${node}.${key}" '!!seq' local -a tmp_array=() readarray -t tmp_array <<<${value} local excludes='' printf -v excludes "%s:" "${tmp_array[@]}" - OverlayInfo[${ol_name}:excludes]=$(${envsubst} <<<"${excludes%:}" ) + OverlayInfo[${ol_name}:${key}]=$(${envsubst} <<<"${excludes%:}" ) ;; - modulepath ) - yml::get_seq value yaml_input "${node}.${key}" '!!seq' - local -a tmp_array=() - readarray -t tmp_array <<<${value} - local modulepath='' - printf -v modulepath "%s:" "${tmp_array[@]}" - OverlayInfo[${ol_name}:modulepath]=$(${envsubst} <<< "${modulepath%:}") + path_config ) + yml::get_value value yaml_input "${node}.${key}" '!!seq' + parse_path_config value "${ol_name}" + ;; + has_groups | has_relstages) + yml::get_value value yaml_input "${node}.${key}" '!!bool' + OverlayInfo[${ol_name}:${key,,}]="${value}" ;; * ) - std::die 3 "%s -- %s\n%s" \ - "Invalid key in configuration" \ - "${key}" "${yaml_input}" + yml::die_invalid_ol_key "${key}" "${yaml_input}" ;; esac @@ -280,11 +393,9 @@ pm::read_config(){ get_config(){ : " Get Pmodules configuration. - - Args: - $1 Pmodules configuration file " - local -r config_file="$1" + local -r config_file="$1" # Pmodules configuration file + local -- yaml_input='' yml::read_file yaml_input "${config_file}" '.' @@ -344,6 +455,9 @@ pm::read_config(){ if [[ -r "${usr_config_file}" ]]; then get_config "${usr_config_file}" fi + OverlayInfo[none:type]='n' + OverlayInfo[none:has_group]='false' + OverlayInfo[none:has_relstages]='false' } # Local Variables: diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 978951a..24f30df 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -62,6 +62,9 @@ declare -g UsedGroups='' declare -g UsedReleaseStages='' declare -a UsedOverlays=() declare -- Version='' +declare -x OSRelease=$(std::get_os_release) +declare -x SystemCPU=$(uname -p) +declare -A MaskedGroups=() ############################################################################## declare -- Verbosity_lvl='verbose' @@ -143,6 +146,9 @@ save_env() { vars+=( 'PmFiles' ) vars+=( 'ModulePathAppend' ) vars+=( 'ModulePathPrepend' ) + vars+=( 'OSRelease' ) + vars+=( 'SystemCPU') + vars+=( 'MaskedGroups' ) local s='' s=$(typeset -p "${vars[@]}") declare -gx PMODULES_ENV=$( encode_base64 "$s" ) @@ -276,12 +282,6 @@ die_cannot_use_overlay(){ "overlay cannot be added since some modules are already loaded!" "$1" } -die_overlay_already_in_use(){ - std::die 3 "%s %s: %s -- %s" \ - "${CMD}" "${SubCommand}" \ - "overlay already in use" "$1" -} - die_not_a_modulefile(){ std::die 3 "%s %s: %s -- %s" \ "${CMD}" "${SubCommand}" \ @@ -318,6 +318,12 @@ die_removing_collection_failed(){ "cannot remove collection" "$1" } +die_ol_conflict(){ + std::die 3 "%s %s: %s -- %s" \ + "${CMD}" "${SubCommand}" \ + "Overlay '$1' conflicts with" "$2" +} + get_module_config(){ : " Read module configuration. @@ -398,8 +404,6 @@ is_available(){ " local -n ref_cfg="$1" local -- relstages="$2" - local -- os_release='' - os_release=$(std::get_os_release) check_relstage(){ [[ ":${relstages}:" == *:${ref_cfg['relstage']}:* ]] @@ -408,7 +412,7 @@ is_available(){ [[ -z ${ref_cfg['blocklist']} ]] && return 0 local -- s='' for s in ${ref_cfg['blocklist']}; do - if [[ "${os_release}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then + if [[ "${OSRelease}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then return 0 fi done @@ -418,7 +422,7 @@ is_available(){ [[ -z ${ref_cfg['systems']} ]] && return 0 local -- s='' for s in ${ref_cfg['systems']}; do - if [[ "${os_release}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then + if [[ "${OSRelease}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then return 0 fi done @@ -447,20 +451,46 @@ is_release_stage() { # $2 ref.var to return group # $3 moduledir to check # -find_overlay () { - local -n fo_ol="$1" - local -n fo_group="$2" - local path="${3//+(\/)/\/}" # replace multpile '/' with one - path="${path/%\/}" # remove trailing slash if exist - path="${path%/"${__MODULEFILES_DIR__}"*}" +find_overlay_old () { + local -n ref_ol="$1" + local -n ref_group="$2" + local -- path="${3%/"${__MODULEFILES_DIR__}"*}" # return if not in an overlay - [[ -v Dir2OverlayMap[${path}] ]] || return 1 + if [[ ! -v Dir2OverlayMap[${path}] ]]; then + ref_ol='None' + ref_group='None' + return 1 + fi - fo_ol="${Dir2OverlayMap[${path}]}" - fo_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}" + ref_ol="${Dir2OverlayMap[${path}]}" + if [[ "${OverlayInfo[${ref_ol}:has_groups]}" == 'true' ]]; then + ref_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}" + else + ref_group='None' + fi return 0 } +find_overlay () { + local -n ref_ol="$1" + local -n ref_group="$2" + local -- path="${3%/"${__MODULEFILES_DIR__}"*}" + for ol in "${UsedOverlays[@]}"; do + local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}" + if [[ "${path}" == ${modulefiles_root}/* ]]; then + ref_ol="${ol}" + if [[ "${OverlayInfo[${ref_ol}:has_groups]}" == 'true' ]]; then + ref_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}" + else + ref_group='None' + fi + return 0 + fi + done + ref_ol='None' + ref_group='None' + return 1 +} # # test whether the given file is a module file. @@ -482,7 +512,7 @@ find_overlay () { is_modulefile() { local -n im_interp="$1" local -r fname="$2" - + # is this a regular, readable file? [[ -f "${fname}" && -r "${fname}" ]] || return 2 @@ -741,7 +771,7 @@ subcommand_load() { # hierarchical depth of a group must always be the same. # is_group () { - find_overlay_with_group() { + find_overlay_with_group(){ local -n _ol="$1" local -r group="$2/${__MODULEFILES_DIR__}" local ol @@ -1194,28 +1224,22 @@ subcommand_show() { # Find all modules in a given modulepath matching a specific string. # The search can be restricted to certain release stages. # -# Args: -# $1 reference variable to return result -# $2 search pattern -# $3 release stages -# $4... module path (fully qualified directory names) -# # return list like # modulename_1 relstage_1 modulefile_1 ... # get_available_modules() { - local -n gam_mods="$1" - local -r module="$2" - local -r used_relstages="${3:-${UsedReleaseStages}}" - shift 3 # in the for loop below we use $@ to loop over module path - local relstage + local -n result="$1" # reference variable to return result + local -r pattern="$2" # search pattern + local -r relstages="$3" # excepted release stages + local -n ref_modules="$4" # dict. with available modules + local -n ref_modulenames="$5" # dict. with available module names + shift 5 + local -a dirs=("$@") # module path (absolute directory names) - local -A dict - local -A modulenames - local dir - gam_mods=() + local -- dir='' + result=() # loop over all entries in given module path - for dir in "$@"; do + for dir in "${dirs[@]}"; do test -d "${dir}" || continue cd "${dir}" || std::die 3 "Oops: cannot change to directory '${dir}'" # find overlay and group for this directory @@ -1227,7 +1251,6 @@ get_available_modules() { local -a dir_entries=(*) (( ${#dir_entries[@]} > 0 )) || continue - local sdirs="${dir##*/modulefiles}" # loop over all files (and sym-links) in this directory and # its sub-directories local mod='' # module_name/module_version @@ -1235,25 +1258,25 @@ get_available_modules() { [[ -n ${OverlayExcludes} && "${mod}" =~ ${OverlayExcludes} ]] && continue local name="${mod%/*}" local add='no' - if [[ -n "${ol}" && "${ol}" != 'none' ]]; then + if [[ -n "${ol}" && "${ol,,}" != 'none' ]]; then # module is in an overlay # # add to list of available modules, if # - first time found by name only # - in same overlay as first found # - new version and not hidden by overlay - if [[ ! -v modulenames[${name}] ]]; then + if [[ ! -v ref_modulenames[${name}] ]]; then # new entry if [[ "${OverlayInfo[${ol}:type]}" == "${ol_hiding}" ]]; then - modulenames[${name}]="${ol}" + ref_modulenames[${name}]="${ol}" else - modulenames[${name}]='0' + ref_modulenames[${name}]='0' fi add='yes' - elif [[ "${modulenames[${name}]}" == "${ol}" ]]; then + elif [[ "${ref_modulenames[${name}]}" == "${ol}" ]]; then add='yes' - elif [[ "${modulenames[${name}]}" == '0' ]] \ - && [[ ! -v dict[${sdirs}/${mod}] ]]; then + elif [[ "${ref_modulenames[${name}]}" == '0' ]] \ + && [[ ! -v ref_modules[${mod}] ]]; then add='yes' fi else @@ -1263,14 +1286,18 @@ get_available_modules() { [[ "${add}" == 'no' ]] && continue local -A cfg=() get_module_config cfg "${dir}" "${mod}" - is_available cfg "${used_relstages}" || continue - - gam_mods+=( "${mod}" "${cfg['relstage']}" "${dir}/${mod}" "${ol}" ) - dict[${sdirs}/${mod}]=1 + if [[ "${OverlayInfo[${ol}:has_relstages]}" == 'true' ]]; then + is_available cfg "${relstages}" || continue + result+=( "${mod}" "${cfg['relstage']}" "${dir}/${mod}" "${ol}" ) + else + is_available cfg "${ReleaseStages}" || continue + result+=( "${mod}" 'stable' "${dir}/${mod}" "${ol}" ) + fi + ref_modules[${mod}]=1 done < <(${find} -L "${dir_entries[@]}" \ \( -type f -o -type l \) \ -not -name ".*" \ - -ipath "${module}*" \ + -ipath "${pattern}*" \ | ${sort} --version-sort) done } # get_available_modules() @@ -1630,25 +1657,45 @@ subcommand_avail() { done groups+=( "${name}" ) done + local -A found_modules=() + local -A found_modulenames=() local string - for string in "${pattern[@]}"; do + for string in "${pattern[@]}"; do for group in "${groups[@]}"; do # limit output to certain groups if (( ${#opt_groups[@]} > 0 )) && [[ ! -v opt_groups[${group}] ]]; then continue fi - + # replace the characters '/', ' ', '.' and '-' with underscore local ref="${group//[\/ .-]/_}" + # continue if module path for this group is empty [[ -v modulepath_${ref} ]] || continue + typeset -n path"=modulepath_${ref}" + local -- header_text='' + local -- ol="${UsedOverlays[0]}" + if [[ "${OverlayInfo[${ol}:has_groups]}" == 'true' ]]; then + found_modules=() + found_modulenames=() + fi get_available_modules \ mods \ "${string}*" \ "${opt_use_relstages}" \ + found_modules \ + found_modulenames \ "${path[@]}" [[ ${#mods[@]} == 0 ]] && continue - ${output_function} "${group}" + if [[ "${group,,}" == 'none' ]]; then + # if we have no groups, the overlay is the + # same for all modules in ${mods[@]}. So we + # can use the overlay of the first module. + header_text="${mods[3]}" + else + header_text="${group}" + fi + ${output_function} "${header_text}" done done } # subcommand_avail() @@ -1683,6 +1730,74 @@ SWITCHES: searched releases. " +set_ol_modulepaths(){ + local -r ol_name="$1" + # if unstable is used: + # prepend modulepath_unstable if not empty + # otherwise prepend modulepath + # else if only stable is used + # prepend modulepath_stable if not empty + # otherwise prepend modulepath + # else if deprecated modules are available + # prepend modulepath_deprecated if not empty + # otherwise prepend modulepath + local -- path=''y + if [[ ":${UsedReleaseStages}:" == *:unstable:* ]]; then + path="${OverlayInfo[${ol_name}:modulepath_unstable]}" + if [[ -z "${path}" ]]; then + path="${OverlayInfo[${ol_name}:modulepath]}" + fi + elif [[ ":${UsedReleaseStages}:" == *:deprecated:* ]]; then + path="${OverlayInfo[${ol_name}:modulepath_deprecated]}" + if [[ -z "${path}" ]]; then + path="${OverlayInfo[${ol_name}:modulepath]}" + fi + else + path="${OverlayInfo[${ol_name}:modulepath_stable]}" + if [[ -z "${path}" ]]; then + path="${OverlayInfo[${ol_name}:modulepath]}" + fi + fi + if [[ -n "${path}" ]]; then + local -a modulepath=() + IFS=':' read -r -a modulepath <<<"${path}" + local -- dir='' + for dir in "${modulepath[@]}"; do + std::prepend_path MODULEPATH "${dir}" + std::prepend_path ModulePathAppend "${dir}" + std::prepend_path ModulePathPrepend "${dir}" + done + fi +} + +unset_ol_modulepaths(){ + # remove additional directories added by overlay + local -- path='' + if [[ -n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then + path+="${OverlayInfo[${ol_name}:modulepath]}:" + fi + if [[ -n "${OverlayInfo[${ol_name}:modulepath_unstable]}" ]]; then + path+="${OverlayInfo[${ol_name}:modulepath_unstable]}:" + fi + if [[ -n "${OverlayInfo[${ol_name}:modulepath_stable]}" ]]; then + path+="${OverlayInfo[${ol_name}:modulepath_stable]}:" + fi + if [[ -n "${OverlayInfo[${ol_name}:modulepath_deprecated]}" ]]; then + path+="${OverlayInfo[${ol_name}:modulepath_deprecated]}:" + fi + if [[ -n "${path}" ]]; then + path="${path%:}" + local -a modulepath=() + IFS=':' read -r -a modulepath <<<"${path}" + local -- dir='' + for dir in "${modulepath[@]}"; do + std::remove_path MODULEPATH "${dir}" + std::remove_path ModulePathAppend "${dir}" + std::remove_path ModulePathPrepend "${dir}" + done + fi +} + subcommand_use() { local -a modulepath=() IFS=':' read -r -a modulepath <<<"${MODULEPATH}" @@ -1771,6 +1886,21 @@ subcommand_use() { #...................................................................... use () { + #.............................................................. + use_relstage(){ + local -r relstage="$1" + [[ ":${UsedReleaseStages}" =~ :${relstage}: ]] && return 0 + std::append_path UsedReleaseStages "${relstage}" || rc=$? + + local -- ol_name='' + for ol_name in "${UsedOverlays[@]}"; do + [[ "${OverlayInfo[${ol_name}:has_additional_modulepaths]}" == 'false' ]] && \ + continue + unset_ol_modulepaths "${ol_name}" + set_ol_modulepaths "${ol_name}" + done + } + #.............................................................. use_overlay() { local ol_name="$1" @@ -1779,24 +1909,40 @@ subcommand_use() { die_cannot_use_overlay "${ol_name}" [[ ${OverlayInfo[${ol_name}:used]} == 'yes' ]] && return 0 - # die_overlay_already_in_use "${ol_name}" + + local -a conflicts=( "${OverlayInfo[${ol_name}:conflicts]//:/ }" ) + for ol in "${UsedOverlays[@]}"; do + for conflict in "${conflicts[@]}"; do + [[ "${ol}" == ${conflict} ]] && \ + die_ol_conflict "${ol_name}" "${ol}" + done + done if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then - # if this overlay replaces groups, we have - # to remove the modules made available by - # other overlays in these groups + # this overlay is masking groups. So, we have to remove + # the corresponding directories of the already used + # overlays from MODULEPATH. for group in ${UsedGroups//:/ }; do - # is this group in the to be added overlay? - local dir="${OverlayInfo[${ol_name}:modulefiles_root]}/" - dir+="${group}/${__MODULEFILES_DIR__}" - [[ -d "${dir}" ]] || continue # no - + MaskedGroups[${group}]='' + local -- dir='' + # if this overlay has groups, we have to check whether + # ${group} exists in this overlay. If not, we have to + # do nothing. + # if this overlay doesn't have groups, we remove all + # groups. + if [[ "${OverlayInfo[${ol_name}:has_groups]}" == 'true' ]]; then + dir="${OverlayInfo[${ol_name}:modulefiles_root]}/" + dir+="${group}/${__MODULEFILES_DIR__}" + # is this group in this overlay? + [[ -d "${dir}" ]] || continue # no + fi dir="/${group}/${__MODULEFILES_DIR__}" local -a dirs=() for ol in "${UsedOverlays[@]}"; do dirs+=( "${OverlayInfo[${ol}:modulefiles_root]}${dir}" ) done std::remove_path MODULEPATH "${dirs[@]}" + MaskedGroups[${group}]="${ol_name}:${MaskedGroups[${group}]}" done fi scan_groups "${ol_name}" @@ -1807,16 +1953,7 @@ subcommand_use() { std::prepend_path MODULEPATH "${dir}" fi done - - if [[ -v OverlayInfo[${ol_name}:modulepath] && \ - -n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then - local -a modulepath=() - IFS=':' read -r -a modulepath <<<"${OverlayInfo[${ol_name}:modulepath]}" - local -- dir='' - for dir in "${modulepath[@]}"; do - std::prepend_path MODULEPATH "${dir}" - done - fi + set_ol_modulepaths "${ol_name}" UsedOverlays=( "${ol_name}" "${UsedOverlays[@]}" ) OverlayInfo[${ol_name}:used]='yes' @@ -1855,8 +1992,7 @@ subcommand_use() { local -- arg="$1" local -i rc=0 if is_release_stage "${arg}"; then - # argument is release stage - std::append_path UsedReleaseStages "${arg}" || rc=$? + use_relstage "${arg}" || rc=$? return ${rc} fi if [[ -v OverlayInfo[${arg}:type] ]]; then @@ -1943,6 +2079,21 @@ subcommand_unuse() { #...................................................................... unuse() { + unuse_relstage() { + local -r relstage="$1" + + [[ ! ":${UsedReleaseStages}:" =~ :${relstage}: ]] && return 0 + std::remove_path UsedReleaseStages "${relstage}" + + local -- ol_name='' + for ol_name in "${UsedOverlays[@]}"; do + [[ "${OverlayInfo[${ol_name}:has_additional_modulepaths]}" == 'false' ]] && \ + continue + unset_ol_modulepaths "${ol_name}" + set_ol_modulepaths "${ol_name}" + done + } + #.............................................................. unuse_overlay() { local ol_name="$1" @@ -1974,23 +2125,46 @@ subcommand_unuse() { "it not on top of the stack" \ "${ol_name}" + OverlayInfo[${ol_name}:used]='no' + UsedOverlays=( "${UsedOverlays[@]:1}") + if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then - # if this overlay hides groups, we have to re-add - # the modules made available by other overlays - local modulefiles_root=${OverlayInfo[${ol_name}:modulefiles_root]} + # if this overlay masks groups, we have to re-add + # some directories to MODULEPATH. for group in ${UsedGroups//:/ }; do - # is this group in the to be removed overlay? - local dir="${modulefiles_root}/${group}/${__MODULEFILES_DIR__}" - [[ -d "${dir}" ]] || continue # no - - dir="/${group}/${__MODULEFILES_DIR__}" - std::prepend_path MODULEPATH "${UsedOverlays[@]/%/${dir}}" + local -a overlays=( ${MaskedGroups[${group}]//:/ } ) + (( ${#overlays[@]} == 0)) && continue + [[ "${ol_name}" != "${overlays[0]}" ]] && continue + # remove overlay "${ol_name}" + local -- s + printf -v s "%s:" "${overlays[@]:1}" + MaskedGroups[${group}]="$s" + + local -- ol='' + local -- dir='' + if (( ${#overlays[@]} == 1 )); then + # Only overlay ${ol_name} is masking this group. + # Add the group directories for all overlays. + for ol in "${UsedOverlays[@]}"; do + dir="${OverlayInfo[${ol}:modulefiles_root]}/" + dir+="${group}/${__MODULEFILES_DIR__}" + [[ -d "${dir}" ]] || continue + std::prepend_path MODULEPATH "${dir}" + done + continue + fi + # There is at least one more overlay masking this group. + # If this overlay has no groups, we have to do nothing. + ol="${overlays[1]}" + [[ "${OverlayInfo[${ol}:has_groups]}" == 'false' ]] && continue + # add group directory for this overlay, if exists + dir="${OverlayInfo[${ol}:modulefiles_root]}/" + dir+="${group}/${__MODULEFILES_DIR__}" + [[ -d "${dir}" ]] || continue + std::prepend_path MODULEPATH "${dir}" done fi - OverlayInfo[${ol_name}:used]='no' - UsedOverlays=( "${UsedOverlays[@]:1}") - # rebuild exclude list. # Note: # A module might be excluded in multiple overlays. So, we cannot @@ -2008,18 +2182,7 @@ subcommand_unuse() { if [[ -n "${OverlayExcludes}" ]]; then OverlayExcludes="${OverlayExcludes:0: -1}" fi - # remove additional directories added by overlay - if [[ -v OverlayInfo[${ol_name}:modulepath] && \ - -n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then - local -a modulepath=() - IFS=':' read -r -a modulepath <<<"${OverlayInfo[${ol_name}:modulepath]}" - local -- dir='' - for dir in "${modulepath[@]}"; do - std::remove_path MODULEPATH "${dir}" - std::remove_path ModulePathAppend "${dir}" - std::remove_path ModulePathPrepend "${dir}" - done - fi + unset_ol_modulepaths "${ol_name}" # remove root of overlay local dir @@ -2063,8 +2226,7 @@ subcommand_unuse() { local arg=$1 if is_release_stage "${arg}"; then - # argument is release stage - std::remove_path UsedReleaseStages "${arg}" + unuse_relstage "${arg}" return 0 fi if [[ -v OverlayInfo[${arg}:type] ]]; then @@ -2269,6 +2431,8 @@ pmodules_setup() { Version="${PMODULES_VERSION}" vars_to_be_exported[PMODULES_HOME]=1 fi + OSRelease=$(std::get_os_release) + SystemCPU=$(uname -p) save_env } @@ -2701,11 +2865,15 @@ subcommand_search() { # get and print all available modules in $mpath # with respect to the requested release stage # TmpFile: module/version relstage group dependencies... - local mods + local -- mods='' + local -A found_modules=() + local -A found_modulenames=() get_available_modules \ mods \ "${module}" \ "${opt_use_relstages}" \ + found_modules \ + found_modulenames \ "${modulepath[@]}" local i=0 for (( i=0; i<${#mods[@]}; i+=4 )); do @@ -3358,7 +3526,7 @@ subcommand_restore() { for item in "${tmp[@]}"; do (( ${GroupDepths["${item}"]} > 0 )) && continue std::is_member_of_array "${item}" default_grps && continue - items+=( "${grp}" ) + items+=( "${item}" ) done IFS=':' read -r -a tmp <<< "${UsedOverlays}" for item in "${tmp[@]}"; do @@ -3776,10 +3944,16 @@ else fi if [[ "${Version}" != "${PMODULES_VERSION}" ]]; then + pm::read_config + for ol_name in "${UsedOverlays[@]}"; do + OverlayInfo[${ol_name}:used]='yes' + done Version="${PMODULES_VERSION}" vars_to_be_exported['PMODULES_ENV']=1 fi - +#if [[ ! -v OverlayInfo[base:has_groups] ]]; then +# pm::read_config +#fi if [[ -z "${UsedGroups}" ]] || \ [[ -z "${UsedReleaseStages}" ]] || \ (( ${#UsedOverlays[@]} == 0 )); then