pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-06-25 00:57:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '388-new-feature-support-for-overlays-without-groups' into 'master'

Browse Source 
Resolve "new feature: support for overlays without groups"

Closes #388

See merge request Pmodules/src!416
This commit is contained in:
gsell
2025-02-06 09:15:35 +01:00
parent e0c3d33805 56ab7553c9
commit ec88509d30
3 changed files with 490 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pmodules/libmodules.tcl
+42 -53
View File
@@ -28,11 +28,12 @@ set ::MODULEFILES_DIR "modulefiles"
set ::ol_replacing "r"
# variable derived from PMODULES_ENV
set ::Dir2OverlayMap {}
set ::OverlayInfo {}
array set ::Dir2OverlayMap {}
array set ::OverlayInfo {}
set ::UsedGroups {}
set ::UsedOverlays {}
debug "_pmodules_parse_pmodules_env"
proc _pmodules_parse_pmodules_env { } {
#
# In this library we need the value of some BASH variables
@@ -41,6 +42,8 @@ proc _pmodules_parse_pmodules_env { } {
# 'typeset -p VAR' - to Tcl.
#
foreach line [split [base64::decode $::env(PMODULES_ENV)] "\n"] {
# lines are something like "declare -a UsedGroups=value"
# assign variable name -> key and value -> value
if { ![regexp -- {.* -[aAx]* (.*)=\((.*)\)} $line -> key value] } {
continue
}
@@ -49,7 +52,8 @@ proc _pmodules_parse_pmodules_env { } {
array set ::Dir2OverlayMap [regsub -all {[]=[]} $value " "]
}
OverlayInfo {
array set ::OverlayInfo [regsub -all {[]=[]} $value " "]
set tmp_olinfo [regsub -all {[]=[]} $value " "]
array set ::OverlayInfo $tmp_olinfo
}
UsedOverlays {
array set tmp [regsub -all {[]=[]} $value " "]
@@ -64,8 +68,10 @@ proc _pmodules_parse_pmodules_env { } {
}
}
}
debug "return"
}
debug "module-addgroup"
proc module-addgroup { group } {
global env
global name
@@ -289,88 +295,71 @@ proc ModulesHelp { } {
# or
# <root_dir>/group/modulefiles/X1/Y1//X2/Y2/name/version
#
proc _find_overlay { modulefile_components } {
proc _find_overlay { modulefile } {
debug "_find_overlay()"
foreach ol $::UsedOverlays {
debug "ol = $ol"
set ol_modulefiles_root $::OverlayInfo(${ol}:modulefiles_root)
if { [string range $ol_modulefiles_root end end] == "/" } {
set ol_modulefiles_root [string range $ol_modulefiles_root 0 end-1]
}
debug "ol_modulefiles_root = $ol_modulefiles_root"
set ol_modulefiles_root_splitted [file split $ol_modulefiles_root]
set modulefile_root [file join \
{*}[lrange \
$modulefile_components \
0 [expr [llength $ol_modulefiles_root_splitted] - 1]]]
debug "modulefile_root = $modulefile_root"
if { [string compare $ol_modulefiles_root $modulefile_root] == 0 } {
debug "ol_modulefiles_root_splitted = $ol_modulefiles_root_splitted"
return $ol_modulefiles_root_splitted
}
}
if { [string match "$ol_modulefiles_root/*" modulefile] == 0 } {
return "$ol"
}
}
debug "overlay not found"
return {}
}
proc _is_in_overlay { } {
debug "_is_in_overlay?"
set parts [_find_overlay [file split $::ModulesCurrentModulefile]]
debug "_is_in_overlay: $parts"
expr {[string compare $parts ""] == 0 }
}
proc _pmodules_init_global_vars { } {
debug "_pmodules_init_global_vars() called"
global group
global GROUP
global name
global P
global version
global V
global P # name of module (without version)
global V # full version of module
global V_MAJOR
global V_MINOR
global V_PATCHLVL
global V_RELEASE
global V_PKG
global variant
global V_PKG # version without release no. and/or suffix
global PREFIX # prefix of package
set modulefile_splitted [file split $::ModulesCurrentModulefile]
set current_modulefile [file split $::ModulesCurrentModulefile]
set ol_modulefiles_root_splitted [_find_overlay ${modulefile_splitted}]
debug "init: ol_modulefiles_root_splitted=$ol_modulefiles_root_splitted"
set rel_modulefile [lrange $modulefile_splitted [llength $ol_modulefiles_root_splitted] end]
set group [lindex $rel_modulefile 0]
set GROUP "${group}"
set name [lindex $modulefile_splitted end-1]
set version [lindex $modulefile_splitted end]
set suffixes [lassign [split $version _] v]
set P [lindex $current_modulefile end-1]
set V [lindex $current_modulefile end]
set suffixes [lassign [split $V _] v]
lassign [split $v -] V_PKG V_RELEASE
lassign [split $V_PKG .] V_MAJOR V_MINOR V_PATCHLVL
set variant [lrange $rel_modulefile 2 end]
set modulefiles_root [file join {*}$ol_modulefiles_root_splitted]
set ol $::Dir2OverlayMap($modulefiles_root)
set ol [_find_overlay $::ModulesCurrentModulefile]
if { $::OverlayInfo(${ol}:has_groups) == "true" } {
set modulefiles_root [file split $::OverlayInfo(${ol}:modulefiles_root)]
set rel_modulefile [lrange $current_modulefile [llength $modulefiles_root] end]
set GROUP [lindex $rel_modulefile 0]
set install_prefix [file split $::OverlayInfo(${ol}:install_root)]
set ::variant [lrange $rel_modulefile 2 end]
set prefix "$install_prefix $GROUP [lreverse_n $::variant 2]"
set PREFIX [file join {*}$prefix]
} else {
set GROUP "None"
set ::variant {}
set PREFIX $::OverlayInfo(${ol}:install_root)
}
set install_prefix [file split $::OverlayInfo(${ol}:install_root)]
set prefix "$install_prefix $group [lreverse_n $variant 2]"
set PREFIX [file join {*}$prefix]
set P "${name}"
set V "${version}"
# :FIXME: the following vars are still used
set ::name $P
set ::version $V
set ::group $GROUP
debug "modulefiles_root=$modulefiles_root"
debug "ol=$ol"
debug "PREFIX=$PREFIX"
debug "group of module $name: $group"
debug "group of module $P: $GROUP"
}
if { [info exists ::whatis] } {
module-whatis "$whatis"
}
debug "init"
_pmodules_parse_pmodules_env
if {[_is_in_overlay] == 0} {
if {[_find_overlay $::ModulesCurrentModulefile] != ""} {
debug "setup env vars for module in overlay"
_pmodules_init_global_vars
conflict $name
Pmodules/libpmodules.bash.in
+171 -57
View File
@@ -8,20 +8,49 @@ declare -a Overlays=()
declare -A OverlayInfo
declare -A Dir2OverlayMap
# An overlay has a type defining the way modules in this overlay
# make modules in the other overlays unavailable.
#
# 'normal'
# Make modules in other overlay unavailable with the same full name.
# If the overlay doesn't support groups, the overlay should provide
# only modules with different names from the modules in the other
# overlays. Otherwise you modules as available which cannot be loaded.
# Example:
# A overlay providing modules with name gcc doesn't conceal the gcc
# modules in the base overlay, but they are listed as available.
#
#
# 'hiding'
# Make modules in other overlay unavailable with the same name.
# Examples:
# - If a module with name 'gcc' is in a overlay with this
# type, only the gcc modules in this overlay are available. This
# can for example be used to conceal old versions of gcc.
# - In same case we need special variants of a module for a system,
# for exampe openmpi and mpich. Variants of these software can be
# made available in an overlay. If such an overlay is used, modules
# which should not be used can be concealed.
# If the overlay doesn't support groups, a module in the overlay
# conceals all modules in other overlays independend from the group.
# Example:
# - In the Spack overlays - which doesn't support groups - are modules
# with name gcc. The gcc modules in the Spack overlays conceals the
# gcc modules in the group Programming of the base overlay.
# 'replacing'
# This type can be used to make hole groups unavailable.
#
#
declare -r ol_normal='n'
declare -r ol_hiding='h'
declare -r ol_replacing='r'
#
# compute depth of modulefile directory.
#
# Args:
# $1: absolute path of a modulefile directory
#
compute_group_depth () {
local -n result="$1"
local -r dir="$2"
: "
Compute depth of modulefile directory.
"
local -n result="$1" # ref.var to return result
local -r dir="$2" # absolute path of a modulefile directory
if [[ ! -d "${dir}" ]]; then
${mkdir} -p "${dir}" || \
std::die 1 "Cannot create directory -- ${dir}"
@@ -36,27 +65,28 @@ compute_group_depth () {
(( result < 0 )) && (( result = 0 )) || :
}
#
# (Re-)Scan available groups in given overlays and compute group depth's
#
# Args:
# $@: overlay names
#
scan_groups () {
local ol
local depth
: "
(Re-)Scan available groups in the overlays $@ and compute group depth's.
Set GroupDepths[group] and Dir2OverlayMap[ol:dir].
"
local -- ol=''
local -i depth=0
for ol in "$@"; do
[[ "${OverlayInfo[${ol}:has_groups]}" == 'true' ]] || continue
local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}"
local dir
for dir in "${modulefiles_root}"/*/"${__MODULEFILES_DIR__}"; do
local dir=''
for dir in "${modulefiles_root}"/*/"${__MODULEFILES_DIR__}"; do
local group="${dir%/*}"
group="${group##*/}"
if [[ ! -v GroupDepths[${group}] ]]; then
compute_group_depth depth "${dir}"
GroupDepths[$group]=${depth}
fi
# :FIXME: do we need this?
Dir2OverlayMap[${dir}]="${ol}"
Dir2OverlayMap[${dir%/"${__MODULEFILES_DIR__}"*}]="${ol}"
done
done
done
GroupDepths['none']=0
}
@@ -79,15 +109,21 @@ declare -A OverlayConfigKeys=(
['modulefiles_root']=''
['excludes']=''
['type']='n'
['modulepath']=''
['modulepath_unstable']=''
['modulepath_stable']=''
['modulepath_deprecated']=''
['conflicts']=''
['config']=''
['has_groups']='true'
['has_relstages']='true'
['default_relstage']='unstable'
['has_additional_modulepaths']='false'
['conflicts']=''
)
declare -A OverlayPathConfigKeys=(
['target_cpus']=''
['modulepath']=''
['modulepath_unstable']=''
['modulepath_stable']=''
['modulepath_deprecated']=''
)
yml::die_parsing(){
@@ -185,33 +221,102 @@ yml::get_seq(){
return 1
}
yml::die_invalid_ol_install_root(){
std::die 3 "Invalid installation root directory for overlay '$1' -- $2"
}
yml::die_invalid_ol_modulefiles_root(){
std::die 3 "Invalid modulefiles root directory for overlay '$1' -- $2"
}
yml::die_invalid_ol_type(){
std::die 3 "Invalid type for overlay '$1' -- $2"
}
yml::die_invalid_ol_relstage(){
std::die 3 "Invalid default release stage for overlay '$1' -- $2"
}
yml::die_invalid_ol_key(){
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"$1" "$2"
}
parse_path_config(){
local -n yaml="$1"
local -- ol_name="$2"
local -- key=''
for key in "${!OverlayPathConfigKeys[@]}"; do
OverlayInfo[${ol_name}:${key}]="${OverlayPathConfigKeys[${key}]}"
done
local -i l=0
yml::get_seq_length l yaml .
local -i i=0
for ((i=0; i<l; i++)); do
local -- node=".[$i]"
local -a keys=()
yml::get_keys keys yaml "${node}"
for key in "${keys[@]}"; do
case ${key} in
target_cpus )
local -- str=''
yml::get_seq \
str \
yaml \
"${node}.${key}" || \
yml::die_parsing "${yaml}"
set -o noglob
local -a target_cpus=( "${str,,}" )
set +o noglob
local -- system_cpu=$(uname -p)
local -- cpu=''
for cpu in "${target_cpus[@]}"; do
if [[ ${cpu} != 'any' && ${cpu} != ${system_cpu} ]]; then
break 2
fi
done
;;
modulepath | modulepath_unstable | modulepath_stable | modulepath_deprecated)
local -- str=''
yml::get_seq str yaml "${node}.${key}" '!!seq'
local -a tmp_array=()
readarray -t tmp_array <<<${str}
local modulepath=''
printf -v modulepath "%s:" "${tmp_array[@]}"
OverlayInfo[${ol_name}:${key}]=$(${envsubst} <<< "${modulepath%:}")
OverlayInfo[${ol_name}:has_additional_modulepaths]='true'
;;
esac
done
done
}
pm::read_config(){
: "
Read Pmodules configuration files '${PMODULES_ROOT}/config/Pmodules.yaml'
and '${HOME}/.Pmodules/Pmodules.yaml'.
Args:
none
"
get_config_of_overlay(){
: "
Get configuration of an overlay.
Args:
$1 YAML config
$2 name of overlay
"
local -r yaml_input="$1"
local -r ol_name="$2"
local -r yaml_input="$1" # YAML formatted string
local -r ol_name="$2" # name of overlay
Overlays+=( "${ol_name}" )
# init overlay with defaults
local -- key=''
for key in "${!OverlayConfigKeys[@]}"; do
OverlayInfo[${ol_name}:${key}]="${OverlayConfigKeys[${key}]}"
done
for key in "${!OverlayPathConfigKeys[@]}"; do
OverlayInfo[${ol_name}:${key}]="${OverlayPathConfigKeys[${key}]}"
done
# get keys in YAML input
local -- node=".Overlays.${ol_name}"
local -- key=''
local -- node=".Overlays.\"${ol_name}\""
local -a keys=()
yml::get_keys keys yaml_input "${node}"
local -- value=''
@@ -222,16 +327,14 @@ pm::read_config(){
OverlayInfo[${ol_name}:install_root]=$(${envsubst} <<< "${value}")
mkdir -p "${OverlayInfo[${ol_name}:install_root]}" 2>/dev/null
[[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \
std::die 3 \
"Invalid installation root directory for overlay '${ol_name}' -- ${value}"
yml::die_invalid_ol_install_root "${ol_name}" "${value}"
;;
modulefiles_root )
yml::get_value value yaml_input "${node}.${key}" '!!str'
OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}")
mkdir -p "${OverlayInfo[${ol_name}:modulefiles_root]}" 2>/dev/null
[[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \
std::die 3 \
"Invalid modulefiles root directory for overlay '${ol_name}' -- ${value}"
yml::die_invalid_ol_modulefiles_root "${ol_name}" "${value}"
;;
type )
yml::get_value value yaml_input "${node}.${key}" '!!str'
@@ -240,31 +343,41 @@ pm::read_config(){
:
;;
* )
std::die 3 "Invalid type for overlay '${ol_name}' -- ${type}"
yml::die_invalid_ol_type "${ol_name}" "${value}"
;;
esac
OverlayInfo[${ol_name}:type]="${value}"
;;
excludes )
default_relstage )
yml::get_value value yaml_input "${node}.${key}" '!!str'
case ${value} in
'unstable' | 'stable' | 'deprecated' )
:
;;
*)
yml::die_invalid_ol_relstage "${ol_name}" "${value}"
;;
esac
OverlayInfo[${ol_name}:${key}]="${value}"
;;
conflicts | excludes )
yml::get_seq value yaml_input "${node}.${key}" '!!seq'
local -a tmp_array=()
readarray -t tmp_array <<<${value}
local excludes=''
printf -v excludes "%s:" "${tmp_array[@]}"
OverlayInfo[${ol_name}:excludes]=$(${envsubst} <<<"${excludes%:}" )
OverlayInfo[${ol_name}:${key}]=$(${envsubst} <<<"${excludes%:}" )
;;
modulepath )
yml::get_seq value yaml_input "${node}.${key}" '!!seq'
local -a tmp_array=()
readarray -t tmp_array <<<${value}
local modulepath=''
printf -v modulepath "%s:" "${tmp_array[@]}"
OverlayInfo[${ol_name}:modulepath]=$(${envsubst} <<< "${modulepath%:}")
path_config )
yml::get_value value yaml_input "${node}.${key}" '!!seq'
parse_path_config value "${ol_name}"
;;
has_groups | has_relstages)
yml::get_value value yaml_input "${node}.${key}" '!!bool'
OverlayInfo[${ol_name}:${key,,}]="${value}"
;;
* )
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
yml::die_invalid_ol_key "${key}" "${yaml_input}"
;;
esac
@@ -280,11 +393,9 @@ pm::read_config(){
get_config(){
: "
Get Pmodules configuration.
Args:
$1 Pmodules configuration file
"
local -r config_file="$1"
local -r config_file="$1" # Pmodules configuration file
local -- yaml_input=''
yml::read_file yaml_input "${config_file}" '.'
@@ -344,6 +455,9 @@ pm::read_config(){
if [[ -r "${usr_config_file}" ]]; then
get_config "${usr_config_file}"
fi
OverlayInfo[none:type]='n'
OverlayInfo[none:has_group]='false'
OverlayInfo[none:has_relstages]='false'
}
# Local Variables:
Pmodules/modulecmd.bash.in
+277 -103
View File
@@ -62,6 +62,9 @@ declare -g UsedGroups=''
declare -g UsedReleaseStages=''
declare -a UsedOverlays=()
declare -- Version=''
declare -x OSRelease=$(std::get_os_release)
declare -x SystemCPU=$(uname -p)
declare -A MaskedGroups=()
##############################################################################
declare -- Verbosity_lvl='verbose'
@@ -143,6 +146,9 @@ save_env() {
vars+=( 'PmFiles' )
vars+=( 'ModulePathAppend' )
vars+=( 'ModulePathPrepend' )
vars+=( 'OSRelease' )
vars+=( 'SystemCPU')
vars+=( 'MaskedGroups' )
local s=''
s=$(typeset -p "${vars[@]}")
declare -gx PMODULES_ENV=$( encode_base64 "$s" )
@@ -276,12 +282,6 @@ die_cannot_use_overlay(){
"overlay cannot be added since some modules are already loaded!" "$1"
}
die_overlay_already_in_use(){
std::die 3 "%s %s: %s -- %s" \
"${CMD}" "${SubCommand}" \
"overlay already in use" "$1"
}
die_not_a_modulefile(){
std::die 3 "%s %s: %s -- %s" \
"${CMD}" "${SubCommand}" \
@@ -318,6 +318,12 @@ die_removing_collection_failed(){
"cannot remove collection" "$1"
}
die_ol_conflict(){
std::die 3 "%s %s: %s -- %s" \
"${CMD}" "${SubCommand}" \
"Overlay '$1' conflicts with" "$2"
}
get_module_config(){
: "
Read module configuration.
@@ -398,8 +404,6 @@ is_available(){
"
local -n ref_cfg="$1"
local -- relstages="$2"
local -- os_release=''
os_release=$(std::get_os_release)
check_relstage(){
[[ ":${relstages}:" == *:${ref_cfg['relstage']}:* ]]
@@ -408,7 +412,7 @@ is_available(){
[[ -z ${ref_cfg['blocklist']} ]] && return 0
local -- s=''
for s in ${ref_cfg['blocklist']}; do
if [[ "${os_release}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then
if [[ "${OSRelease}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then
return 0
fi
done
@@ -418,7 +422,7 @@ is_available(){
[[ -z ${ref_cfg['systems']} ]] && return 0
local -- s=''
for s in ${ref_cfg['systems']}; do
if [[ "${os_release}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then
if [[ "${OSRelease}" =~ $s ]] || [[ "${HostName}" =~ $s ]]; then
return 0
fi
done
@@ -447,20 +451,46 @@ is_release_stage() {
# $2 ref.var to return group
# $3 moduledir to check
#
find_overlay () {
local -n fo_ol="$1"
local -n fo_group="$2"
local path="${3//+(\/)/\/}" # replace multpile '/' with one
path="${path/%\/}" # remove trailing slash if exist
path="${path%/"${__MODULEFILES_DIR__}"*}"
find_overlay_old () {
local -n ref_ol="$1"
local -n ref_group="$2"
local -- path="${3%/"${__MODULEFILES_DIR__}"*}"
# return if not in an overlay
[[ -v Dir2OverlayMap[${path}] ]] || return 1
if [[ ! -v Dir2OverlayMap[${path}] ]]; then
ref_ol='None'
ref_group='None'
return 1
fi
fo_ol="${Dir2OverlayMap[${path}]}"
fo_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}"
ref_ol="${Dir2OverlayMap[${path}]}"
if [[ "${OverlayInfo[${ref_ol}:has_groups]}" == 'true' ]]; then
ref_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}"
else
ref_group='None'
fi
return 0
}
find_overlay () {
local -n ref_ol="$1"
local -n ref_group="$2"
local -- path="${3%/"${__MODULEFILES_DIR__}"*}"
for ol in "${UsedOverlays[@]}"; do
local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}"
if [[ "${path}" == ${modulefiles_root}/* ]]; then
ref_ol="${ol}"
if [[ "${OverlayInfo[${ref_ol}:has_groups]}" == 'true' ]]; then
ref_group="${path#"${OverlayInfo[${ol}:modulefiles_root]}"/}"
else
ref_group='None'
fi
return 0
fi
done
ref_ol='None'
ref_group='None'
return 1
}
#
# test whether the given file is a module file.
@@ -482,7 +512,7 @@ find_overlay () {
is_modulefile() {
local -n im_interp="$1"
local -r fname="$2"
# is this a regular, readable file?
[[ -f "${fname}" && -r "${fname}" ]] || return 2
@@ -741,7 +771,7 @@ subcommand_load() {
# hierarchical depth of a group must always be the same.
#
is_group () {
find_overlay_with_group() {
find_overlay_with_group(){
local -n _ol="$1"
local -r group="$2/${__MODULEFILES_DIR__}"
local ol
@@ -1194,28 +1224,22 @@ subcommand_show() {
# Find all modules in a given modulepath matching a specific string.
# The search can be restricted to certain release stages.
#
# Args:
# $1 reference variable to return result
# $2 search pattern
# $3 release stages
# $4... module path (fully qualified directory names)
#
# return list like
# modulename_1 relstage_1 modulefile_1 ...
#
get_available_modules() {
local -n gam_mods="$1"
local -r module="$2"
local -r used_relstages="${3:-${UsedReleaseStages}}"
shift 3 # in the for loop below we use $@ to loop over module path
local relstage
local -n result="$1" # reference variable to return result
local -r pattern="$2" # search pattern
local -r relstages="$3" # excepted release stages
local -n ref_modules="$4" # dict. with available modules
local -n ref_modulenames="$5" # dict. with available module names
shift 5
local -a dirs=("$@") # module path (absolute directory names)
local -A dict
local -A modulenames
local dir
gam_mods=()
local -- dir=''
result=()
# loop over all entries in given module path
for dir in "$@"; do
for dir in "${dirs[@]}"; do
test -d "${dir}" || continue
cd "${dir}" || std::die 3 "Oops: cannot change to directory '${dir}'"
# find overlay and group for this directory
@@ -1227,7 +1251,6 @@ get_available_modules() {
local -a dir_entries=(*)
(( ${#dir_entries[@]} > 0 )) || continue
local sdirs="${dir##*/modulefiles}"
# loop over all files (and sym-links) in this directory and
# its sub-directories
local mod='' # module_name/module_version
@@ -1235,25 +1258,25 @@ get_available_modules() {
[[ -n ${OverlayExcludes} && "${mod}" =~ ${OverlayExcludes} ]] && continue
local name="${mod%/*}"
local add='no'
if [[ -n "${ol}" && "${ol}" != 'none' ]]; then
if [[ -n "${ol}" && "${ol,,}" != 'none' ]]; then
# module is in an overlay
#
# add to list of available modules, if
# - first time found by name only
# - in same overlay as first found
# - new version and not hidden by overlay
if [[ ! -v modulenames[${name}] ]]; then
if [[ ! -v ref_modulenames[${name}] ]]; then
# new entry
if [[ "${OverlayInfo[${ol}:type]}" == "${ol_hiding}" ]]; then
modulenames[${name}]="${ol}"
ref_modulenames[${name}]="${ol}"
else
modulenames[${name}]='0'
ref_modulenames[${name}]='0'
fi
add='yes'
elif [[ "${modulenames[${name}]}" == "${ol}" ]]; then
elif [[ "${ref_modulenames[${name}]}" == "${ol}" ]]; then
add='yes'
elif [[ "${modulenames[${name}]}" == '0' ]] \
&& [[ ! -v dict[${sdirs}/${mod}] ]]; then
elif [[ "${ref_modulenames[${name}]}" == '0' ]] \
&& [[ ! -v ref_modules[${mod}] ]]; then
add='yes'
fi
else
@@ -1263,14 +1286,18 @@ get_available_modules() {
[[ "${add}" == 'no' ]] && continue
local -A cfg=()
get_module_config cfg "${dir}" "${mod}"
is_available cfg "${used_relstages}" || continue
gam_mods+=( "${mod}" "${cfg['relstage']}" "${dir}/${mod}" "${ol}" )
dict[${sdirs}/${mod}]=1
if [[ "${OverlayInfo[${ol}:has_relstages]}" == 'true' ]]; then
is_available cfg "${relstages}" || continue
result+=( "${mod}" "${cfg['relstage']}" "${dir}/${mod}" "${ol}" )
else
is_available cfg "${ReleaseStages}" || continue
result+=( "${mod}" 'stable' "${dir}/${mod}" "${ol}" )
fi
ref_modules[${mod}]=1
done < <(${find} -L "${dir_entries[@]}" \
\( -type f -o -type l \) \
-not -name ".*" \
-ipath "${module}*" \
-ipath "${pattern}*" \
| ${sort} --version-sort)
done
} # get_available_modules()
@@ -1630,25 +1657,45 @@ subcommand_avail() {
done
groups+=( "${name}" )
done
local -A found_modules=()
local -A found_modulenames=()
local string
for string in "${pattern[@]}"; do
for string in "${pattern[@]}"; do
for group in "${groups[@]}"; do
# limit output to certain groups
if (( ${#opt_groups[@]} > 0 )) && [[ ! -v opt_groups[${group}] ]]; then
continue
fi
# replace the characters '/', ' ', '.' and '-' with underscore
local ref="${group//[\/ .-]/_}"
# continue if module path for this group is empty
[[ -v modulepath_${ref} ]] || continue
typeset -n path"=modulepath_${ref}"
local -- header_text=''
local -- ol="${UsedOverlays[0]}"
if [[ "${OverlayInfo[${ol}:has_groups]}" == 'true' ]]; then
found_modules=()
found_modulenames=()
fi
get_available_modules \
mods \
"${string}*" \
"${opt_use_relstages}" \
found_modules \
found_modulenames \
"${path[@]}"
[[ ${#mods[@]} == 0 ]] && continue
${output_function} "${group}"
if [[ "${group,,}" == 'none' ]]; then
# if we have no groups, the overlay is the
# same for all modules in ${mods[@]}. So we
# can use the overlay of the first module.
header_text="${mods[3]}"
else
header_text="${group}"
fi
${output_function} "${header_text}"
done
done
} # subcommand_avail()
@@ -1683,6 +1730,74 @@ SWITCHES:
searched releases.
"
set_ol_modulepaths(){
local -r ol_name="$1"
# if unstable is used:
# prepend modulepath_unstable if not empty
# otherwise prepend modulepath
# else if only stable is used
# prepend modulepath_stable if not empty
# otherwise prepend modulepath
# else if deprecated modules are available
# prepend modulepath_deprecated if not empty
# otherwise prepend modulepath
local -- path=''y
if [[ ":${UsedReleaseStages}:" == *:unstable:* ]]; then
path="${OverlayInfo[${ol_name}:modulepath_unstable]}"
if [[ -z "${path}" ]]; then
path="${OverlayInfo[${ol_name}:modulepath]}"
fi
elif [[ ":${UsedReleaseStages}:" == *:deprecated:* ]]; then
path="${OverlayInfo[${ol_name}:modulepath_deprecated]}"
if [[ -z "${path}" ]]; then
path="${OverlayInfo[${ol_name}:modulepath]}"
fi
else
path="${OverlayInfo[${ol_name}:modulepath_stable]}"
if [[ -z "${path}" ]]; then
path="${OverlayInfo[${ol_name}:modulepath]}"
fi
fi
if [[ -n "${path}" ]]; then
local -a modulepath=()
IFS=':' read -r -a modulepath <<<"${path}"
local -- dir=''
for dir in "${modulepath[@]}"; do
std::prepend_path MODULEPATH "${dir}"
std::prepend_path ModulePathAppend "${dir}"
std::prepend_path ModulePathPrepend "${dir}"
done
fi
}
unset_ol_modulepaths(){
# remove additional directories added by overlay
local -- path=''
if [[ -n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then
path+="${OverlayInfo[${ol_name}:modulepath]}:"
fi
if [[ -n "${OverlayInfo[${ol_name}:modulepath_unstable]}" ]]; then
path+="${OverlayInfo[${ol_name}:modulepath_unstable]}:"
fi
if [[ -n "${OverlayInfo[${ol_name}:modulepath_stable]}" ]]; then
path+="${OverlayInfo[${ol_name}:modulepath_stable]}:"
fi
if [[ -n "${OverlayInfo[${ol_name}:modulepath_deprecated]}" ]]; then
path+="${OverlayInfo[${ol_name}:modulepath_deprecated]}:"
fi
if [[ -n "${path}" ]]; then
path="${path%:}"
local -a modulepath=()
IFS=':' read -r -a modulepath <<<"${path}"
local -- dir=''
for dir in "${modulepath[@]}"; do
std::remove_path MODULEPATH "${dir}"
std::remove_path ModulePathAppend "${dir}"
std::remove_path ModulePathPrepend "${dir}"
done
fi
}
subcommand_use() {
local -a modulepath=()
IFS=':' read -r -a modulepath <<<"${MODULEPATH}"
@@ -1771,6 +1886,21 @@ subcommand_use() {
#......................................................................
use () {
#..............................................................
use_relstage(){
local -r relstage="$1"
[[ ":${UsedReleaseStages}" =~ :${relstage}: ]] && return 0
std::append_path UsedReleaseStages "${relstage}" || rc=$?
local -- ol_name=''
for ol_name in "${UsedOverlays[@]}"; do
[[ "${OverlayInfo[${ol_name}:has_additional_modulepaths]}" == 'false' ]] && \
continue
unset_ol_modulepaths "${ol_name}"
set_ol_modulepaths "${ol_name}"
done
}
#..............................................................
use_overlay() {
local ol_name="$1"
@@ -1779,24 +1909,40 @@ subcommand_use() {
die_cannot_use_overlay "${ol_name}"
[[ ${OverlayInfo[${ol_name}:used]} == 'yes' ]] && return 0
# die_overlay_already_in_use "${ol_name}"
local -a conflicts=( "${OverlayInfo[${ol_name}:conflicts]//:/ }" )
for ol in "${UsedOverlays[@]}"; do
for conflict in "${conflicts[@]}"; do
[[ "${ol}" == ${conflict} ]] && \
die_ol_conflict "${ol_name}" "${ol}"
done
done
if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then
# if this overlay replaces groups, we have
# to remove the modules made available by
# other overlays in these groups
# this overlay is masking groups. So, we have to remove
# the corresponding directories of the already used
# overlays from MODULEPATH.
for group in ${UsedGroups//:/ }; do
# is this group in the to be added overlay?
local dir="${OverlayInfo[${ol_name}:modulefiles_root]}/"
dir+="${group}/${__MODULEFILES_DIR__}"
[[ -d "${dir}" ]] || continue # no
MaskedGroups[${group}]=''
local -- dir=''
# if this overlay has groups, we have to check whether
# ${group} exists in this overlay. If not, we have to
# do nothing.
# if this overlay doesn't have groups, we remove all
# groups.
if [[ "${OverlayInfo[${ol_name}:has_groups]}" == 'true' ]]; then
dir="${OverlayInfo[${ol_name}:modulefiles_root]}/"
dir+="${group}/${__MODULEFILES_DIR__}"
# is this group in this overlay?
[[ -d "${dir}" ]] || continue # no
fi
dir="/${group}/${__MODULEFILES_DIR__}"
local -a dirs=()
for ol in "${UsedOverlays[@]}"; do
dirs+=( "${OverlayInfo[${ol}:modulefiles_root]}${dir}" )
done
std::remove_path MODULEPATH "${dirs[@]}"
MaskedGroups[${group}]="${ol_name}:${MaskedGroups[${group}]}"
done
fi
scan_groups "${ol_name}"
@@ -1807,16 +1953,7 @@ subcommand_use() {
std::prepend_path MODULEPATH "${dir}"
fi
done
if [[ -v OverlayInfo[${ol_name}:modulepath] && \
-n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then
local -a modulepath=()
IFS=':' read -r -a modulepath <<<"${OverlayInfo[${ol_name}:modulepath]}"
local -- dir=''
for dir in "${modulepath[@]}"; do
std::prepend_path MODULEPATH "${dir}"
done
fi
set_ol_modulepaths "${ol_name}"
UsedOverlays=( "${ol_name}" "${UsedOverlays[@]}" )
OverlayInfo[${ol_name}:used]='yes'
@@ -1855,8 +1992,7 @@ subcommand_use() {
local -- arg="$1"
local -i rc=0
if is_release_stage "${arg}"; then
# argument is release stage
std::append_path UsedReleaseStages "${arg}" || rc=$?
use_relstage "${arg}" || rc=$?
return ${rc}
fi
if [[ -v OverlayInfo[${arg}:type] ]]; then
@@ -1943,6 +2079,21 @@ subcommand_unuse() {
#......................................................................
unuse() {
unuse_relstage() {
local -r relstage="$1"
[[ ! ":${UsedReleaseStages}:" =~ :${relstage}: ]] && return 0
std::remove_path UsedReleaseStages "${relstage}"
local -- ol_name=''
for ol_name in "${UsedOverlays[@]}"; do
[[ "${OverlayInfo[${ol_name}:has_additional_modulepaths]}" == 'false' ]] && \
continue
unset_ol_modulepaths "${ol_name}"
set_ol_modulepaths "${ol_name}"
done
}
#..............................................................
unuse_overlay() {
local ol_name="$1"
@@ -1974,23 +2125,46 @@ subcommand_unuse() {
"it not on top of the stack" \
"${ol_name}"
OverlayInfo[${ol_name}:used]='no'
UsedOverlays=( "${UsedOverlays[@]:1}")
if [[ "${OverlayInfo[${ol_name}:type]}" == "${ol_replacing}" ]]; then
# if this overlay hides groups, we have to re-add
# the modules made available by other overlays
local modulefiles_root=${OverlayInfo[${ol_name}:modulefiles_root]}
# if this overlay masks groups, we have to re-add
# some directories to MODULEPATH.
for group in ${UsedGroups//:/ }; do
# is this group in the to be removed overlay?
local dir="${modulefiles_root}/${group}/${__MODULEFILES_DIR__}"
[[ -d "${dir}" ]] || continue # no
dir="/${group}/${__MODULEFILES_DIR__}"
std::prepend_path MODULEPATH "${UsedOverlays[@]/%/${dir}}"
local -a overlays=( ${MaskedGroups[${group}]//:/ } )
(( ${#overlays[@]} == 0)) && continue
[[ "${ol_name}" != "${overlays[0]}" ]] && continue
# remove overlay "${ol_name}"
local -- s
printf -v s "%s:" "${overlays[@]:1}"
MaskedGroups[${group}]="$s"
local -- ol=''
local -- dir=''
if (( ${#overlays[@]} == 1 )); then
# Only overlay ${ol_name} is masking this group.
# Add the group directories for all overlays.
for ol in "${UsedOverlays[@]}"; do
dir="${OverlayInfo[${ol}:modulefiles_root]}/"
dir+="${group}/${__MODULEFILES_DIR__}"
[[ -d "${dir}" ]] || continue
std::prepend_path MODULEPATH "${dir}"
done
continue
fi
# There is at least one more overlay masking this group.
# If this overlay has no groups, we have to do nothing.
ol="${overlays[1]}"
[[ "${OverlayInfo[${ol}:has_groups]}" == 'false' ]] && continue
# add group directory for this overlay, if exists
dir="${OverlayInfo[${ol}:modulefiles_root]}/"
dir+="${group}/${__MODULEFILES_DIR__}"
[[ -d "${dir}" ]] || continue
std::prepend_path MODULEPATH "${dir}"
done
fi
OverlayInfo[${ol_name}:used]='no'
UsedOverlays=( "${UsedOverlays[@]:1}")
# rebuild exclude list.
# Note:
# A module might be excluded in multiple overlays. So, we cannot
@@ -2008,18 +2182,7 @@ subcommand_unuse() {
if [[ -n "${OverlayExcludes}" ]]; then
OverlayExcludes="${OverlayExcludes:0: -1}"
fi
# remove additional directories added by overlay
if [[ -v OverlayInfo[${ol_name}:modulepath] && \
-n "${OverlayInfo[${ol_name}:modulepath]}" ]]; then
local -a modulepath=()
IFS=':' read -r -a modulepath <<<"${OverlayInfo[${ol_name}:modulepath]}"
local -- dir=''
for dir in "${modulepath[@]}"; do
std::remove_path MODULEPATH "${dir}"
std::remove_path ModulePathAppend "${dir}"
std::remove_path ModulePathPrepend "${dir}"
done
fi
unset_ol_modulepaths "${ol_name}"
# remove root of overlay
local dir
@@ -2063,8 +2226,7 @@ subcommand_unuse() {
local arg=$1
if is_release_stage "${arg}"; then
# argument is release stage
std::remove_path UsedReleaseStages "${arg}"
unuse_relstage "${arg}"
return 0
fi
if [[ -v OverlayInfo[${arg}:type] ]]; then
@@ -2269,6 +2431,8 @@ pmodules_setup() {
Version="${PMODULES_VERSION}"
vars_to_be_exported[PMODULES_HOME]=1
fi
OSRelease=$(std::get_os_release)
SystemCPU=$(uname -p)
save_env
}
@@ -2701,11 +2865,15 @@ subcommand_search() {
# get and print all available modules in $mpath
# with respect to the requested release stage
# TmpFile: module/version relstage group dependencies...
local mods
local -- mods=''
local -A found_modules=()
local -A found_modulenames=()
get_available_modules \
mods \
"${module}" \
"${opt_use_relstages}" \
found_modules \
found_modulenames \
"${modulepath[@]}"
local i=0
for (( i=0; i<${#mods[@]}; i+=4 )); do
@@ -3358,7 +3526,7 @@ subcommand_restore() {
for item in "${tmp[@]}"; do
(( ${GroupDepths["${item}"]} > 0 )) && continue
std::is_member_of_array "${item}" default_grps && continue
items+=( "${grp}" )
items+=( "${item}" )
done
IFS=':' read -r -a tmp <<< "${UsedOverlays}"
for item in "${tmp[@]}"; do
@@ -3776,10 +3944,16 @@ else
fi
if [[ "${Version}" != "${PMODULES_VERSION}" ]]; then
pm::read_config
for ol_name in "${UsedOverlays[@]}"; do
OverlayInfo[${ol_name}:used]='yes'
done
Version="${PMODULES_VERSION}"
vars_to_be_exported['PMODULES_ENV']=1
fi
#if [[ ! -v OverlayInfo[base:has_groups] ]]; then
# pm::read_config
#fi
if [[ -z "${UsedGroups}" ]] || \
[[ -z "${UsedReleaseStages}" ]] || \
(( ${#UsedOverlays[@]} == 0 )); then