pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-07-01 11:49:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

modulecmd & build-system: code review with shellcheck

Browse Source
This commit is contained in:
gsell
2024-07-24 16:53:13 +02:00
parent 43df5e48f3
commit 0a1d81946b
12 changed files with 1058 additions and 2056 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pmodules/Pmodules.py
-1
View File
@@ -2,7 +2,6 @@ import os, re, subprocess
def module(*args):
os.environ['PMODULES_MODULEFILES_DIR']='modulefiles'
os.environ['PMODULES_CONFIG_DIR']='config'
pm_home=os.environ['PMODULES_HOME']
os.environ['PMODULES_DIR']=pm_home
modulecmd=os.path.join(pm_home, 'bin', 'modulecmd')
Pmodules/libmodules.tcl
+18 -19
View File
@@ -4,7 +4,6 @@
# unload modules if parent removed
#
if {[info exists env(PMODULES_DEBUG)] && $env(PMODULES_DEBUG)} {
proc debug {msg} {
set level [expr [info level] -2]
@@ -92,7 +91,7 @@ proc module-addgroup { group } {
debug "group=$group"
debug "::variant=$::variant"
set dir [file join \
$::OverlayInfo($overlay:mod_root) \
$::OverlayInfo($overlay:modulefiles_root) \
$group \
$::MODULEFILES_DIR \
{*}$::variant]
@@ -126,7 +125,7 @@ proc module-addgroup { group } {
debug "mode=remove: $env(MODULEPATH)"
foreach overlay $::UsedOverlays {
set dir [file join \
$::OverlayInfo($overlay:mod_root) \
$::OverlayInfo($overlay:modulefiles_root) \
$group \
$::MODULEFILES_DIR \
{*}$::variant]
@@ -288,20 +287,20 @@ proc _find_overlay { modulefile_components } {
debug "_find_overlay()"
foreach ol $::UsedOverlays {
debug "ol = $ol"
set ol_mod_root $::OverlayInfo(${ol}:mod_root)
if { [string range $ol_mod_root end end] == "/" } {
set ol_mod_root [string range $ol_mod_root 0 end-1]
set ol_modulefiles_root $::OverlayInfo(${ol}:modulefiles_root)
if { [string range $ol_modulefiles_root end end] == "/" } {
set ol_modulefiles_root [string range $ol_modulefiles_root 0 end-1]
}
debug "ol_mod_root = $ol_mod_root"
set ol_mod_root_splitted [file split $ol_mod_root]
debug "ol_modulefiles_root = $ol_modulefiles_root"
set ol_modulefiles_root_splitted [file split $ol_modulefiles_root]
set modulefile_root [file join \
{*}[lrange \
$modulefile_components \
0 [expr [llength $ol_mod_root_splitted] - 1]]]
0 [expr [llength $ol_modulefiles_root_splitted] - 1]]]
debug "modulefile_root = $modulefile_root"
if { [string compare $ol_mod_root $modulefile_root] == 0 } {
debug "ol_mod_root_splitted = $ol_mod_root_splitted"
return $ol_mod_root_splitted
if { [string compare $ol_modulefiles_root $modulefile_root] == 0 } {
debug "ol_modulefiles_root_splitted = $ol_modulefiles_root_splitted"
return $ol_modulefiles_root_splitted
}
}
debug "overlay not found"
@@ -333,28 +332,28 @@ proc _pmodules_init_global_vars { } {
set modulefile_splitted [file split $::ModulesCurrentModulefile]
set ol_mod_root_splitted [_find_overlay ${modulefile_splitted}]
set rel_modulefile [lrange $modulefile_splitted [llength $ol_mod_root_splitted] end]
set ol_modulefiles_root_splitted [_find_overlay ${modulefile_splitted}]
debug "init: ol_modulefiles_root_splitted=$ol_modulefiles_root_splitted"
set rel_modulefile [lrange $modulefile_splitted [llength $ol_modulefiles_root_splitted] end]
set group [lindex $rel_modulefile 0]
set GROUP "${group}"
set name [lindex $modulefile_splitted end-1]
set version [lindex $modulefile_splitted end]
set suffixes [lassign [split $version _] v]
lassign [split $v -] V_PKG V_RELEASE
lassign [split $V_PKG .] V_MAJOR V_MINOR V_PATCHLVL
set variant [lrange $rel_modulefile 2 end]
set mod_root [file join {*}$ol_mod_root_splitted]
set ol $::Dir2OverlayMap($mod_root)
set modulefiles_root [file join {*}$ol_modulefiles_root_splitted]
set ol $::Dir2OverlayMap($modulefiles_root)
set install_prefix [file split $::OverlayInfo(${ol}:inst_root)]
set install_prefix [file split $::OverlayInfo(${ol}:install_root)]
set prefix "$install_prefix $group [lreverse_n $variant 2]"
set PREFIX [file join {*}$prefix]
set P "${name}"
set V "${version}"
debug "mod_root=$mod_root"
debug "modulefiles_root=$modulefiles_root"
debug "ol=$ol"
debug "PREFIX=$PREFIX"
debug "group of module $name: $group"
Pmodules/libpbuild.bash
+124 -156
View File
@@ -6,10 +6,8 @@ unset CDPATH
#.............................................................................
# define constants
declare -r BNAME_VARIANTS='variants'
declare -r FNAME_RDEPS='.dependencies'
declare -r FNAME_IDEPS='.install_dependencies'
declare -r FNAME_BDEPS='.build_dependencies'
# relative path of documentation
# abs. path is "${PREFIX}/${_docdir}/${module_name}"
@@ -24,7 +22,7 @@ declare -A SOURCE_UNPACK_DIRS=()
declare -ax CONFIGURE_ARGS=()
declare -a PATCH_FILES=()
declare -a PATCH_STRIPS=()
declare -a PATCH_STRIP_DEFAULT='1'
declare -- PATCH_STRIP_DEFAULT='1'
declare -- configure_with='auto'
declare -- SRC_DIR=''
declare -- BUILD_DIR=''
@@ -32,6 +30,9 @@ declare -- is_subpkg='no'
declare -i group_depth=0
declare -- COMPILER=''
declare -- MPI=''
#.............................................................................
#
# Exit script on errror.
@@ -54,7 +55,7 @@ trap "_error_handler" ERR
# write number of cores to stdout
#
_get_num_cores() {
case "${OS}" in
case "${KernelName}" in
Linux )
${grep} -c ^processor /proc/cpuinfo
;;
@@ -62,7 +63,7 @@ _get_num_cores() {
${sysctl} -n hw.ncpu
;;
* )
std::die 1 "OS ${OS} is not supported\n"
std::die 1 "OS ${KernelName} is not supported\n"
;;
esac
}
@@ -163,17 +164,15 @@ pbuild::add_to_group() {
if (( $# == 0 )); then
std::die 42 \
"%s " "${module_name}/${module_version}:" \
"${FUNCNAME}: missing group argument."
"${FUNCNAME[0]}: missing group argument."
fi
if (( $# > 1 )); then
std::die 42 \
"%s " "${module_name}/${module_version}:" \
"${FUNCNAME}: only one argument is allowed."
fi
if [[ ${opt_yaml} == 'yes' ]]; then
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
"${FUNCNAME[0]}: only one argument is allowed."
fi
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
pbuild.add_to_group "$@"
}
readonly -f pbuild::add_to_group
@@ -201,18 +200,17 @@ readonly -f pbuild.add_to_group
# 1 otherwise
#
pbuild::module_is_avail() {
local output=( $("${MODULECMD}" bash avail -a -m "$1" \
2>&1 1>/dev/null) )
local i
for (( i = 0; i < ${#output[@]}; i += 2 )); do
if [[ "${output[$i]}" == "$1" ]]; then
local -- name=''
local -- release=''
while read -r name release; do
if [[ "${name}" == "$1" ]]; then
if (( $# > 1 )); then
local -n _result="$2"
_result="${output[i+1]}"
_result="${release}"
fi
return 0
fi
done
done < <(${modulecmd} bash avail -a -m "$1" 2>&1 1>/dev/null)
return 1
}
readonly -f pbuild::module_is_avail
@@ -252,17 +250,19 @@ pbuild::version_compare () {
[[ $1 =~ ^[0-9]+$ ]]
}
[[ $1 == $2 ]] && return 0
local IFS=.
local i ver1=($1) ver2=($2)
[[ "$1" == "$2" ]] && return 0
local ver1 ver2
IFS='.' read -r -a ver1 <<<"$1"
IFS='.' read -r -a ver2 <<<"$2"
# fill empty fields in ver1 with zeros
local i
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++)); do
[[ -z ${ver2[i]} ]] && ver2[i]=0
if is_uint ${ver1[i]} && is_uint ${ver2[i]}; then
if is_uint "${ver1[i]}" && is_uint "${ver2[i]}"; then
((10#${ver1[i]} > 10#${ver2[i]})) && return 1
((10#${ver1[i]} < 10#${ver2[i]})) && return 2
else
@@ -352,9 +352,8 @@ readonly -f pbuild::version_eq
# Default is all.
#
pbuild::supported_compilers() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
pbuild.supported_compilers "$@"
}
readonly -f pbuild::supported_compilers
@@ -374,9 +373,8 @@ readonly -f pbuild.supported_compilers
# Default is all.
#
pbuild::supported_systems() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
pbuild.supported_systems "$@"
}
readonly -f pbuild::supported_systems
@@ -390,7 +388,7 @@ readonly -f pbuild.supported_systems
#..............................................................................
#
pbuild::use_flag() {
[[ "${USE_FLAGS}" =~ ":${1}:" ]]
[[ "${USE_FLAGS}" == *:${1}:* ]]
}
readonly -f pbuild::use_flag
@@ -406,16 +404,14 @@ readonly -f pbuild::use_flag
# $1 download URL
# $2 optional file-name (of)
pbuild::set_download_url() {
if [[ ${opt_yaml} == 'yes' ]]; then
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
fi
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
local -i _i=${#SOURCE_URLS[@]}
SOURCE_URLS[_i]=$1
SOURCE_URLS[_i]="$1"
if (( $# > 1 )); then
SOURCE_NAMES[$_i]=${2:-${1##*/}}
SOURCE_NAMES[_i]="${2:-${1##*/}}"
else
SOURCE_NAMES[$_i]="${1##*/}"
SOURCE_NAMES[_i]="${1##*/}"
fi
SOURCE_STRIP_DIRS[_i]='1'
}
@@ -424,7 +420,7 @@ readonly -f pbuild::set_download_url
pbuild.set_urls(){
local -i _i=${#SOURCE_URLS[@]}
SOURCE_URLS[_i]="$1"
SOURCE_NAMES[$_i]="$2"
SOURCE_NAMES[_i]="$2"
SOURCE_STRIP_DIRS[_i]="$3"
SOURCE_UNPACKER[_i]="$4"
}
@@ -440,9 +436,8 @@ pbuild.set_urls(){
# Maybe we should use a dictionary in the future.
#
pbuild::set_sha256sum() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
SOURCE_SHA256_SUMS+=("$1")
}
readonly -f pbuild::set_sha256sum
@@ -463,13 +458,12 @@ readonly -f pbuild::set_unpack_dir
#..............................................................................
#
pbuild::add_patch() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
[[ -z "$1" ]] && \
std::die 1 \
"%s " "${module_name}/${module_version}:" \
"${FUNCNAME}: missing argument!"
"${FUNCNAME[0]}: missing argument!"
PATCH_FILES+=( "$1" )
if (( $# >= 2 )); then
PATCH_STRIPS+=( "$2" )
@@ -480,9 +474,8 @@ pbuild::add_patch() {
readonly -f pbuild::add_patch
pbuild.add_patch_files(){
local -a args="$@"
local -- arg=''
for arg in "${args[@]}"; do
for arg in "$@"; do
[[ -z "${arg}" ]] && continue
if [[ ${arg} == *:* ]]; then
PATCH_FILES+=( "${arg%%:*}" )
@@ -498,13 +491,12 @@ readonly -f pbuild.add_patch_files
#..............................................................................
#
pbuild::set_default_patch_strip() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
[[ -n "$1" ]] || \
std::die 1 \
"%s " "${module_name}/${module_version}:" \
"${FUNCNAME}: missing argument!"
"${FUNCNAME[0]}: missing argument!"
PATCH_STRIP_DEFAULT="$1"
}
@@ -606,8 +598,7 @@ pbuild::prep() {
done
if [[ "${dir}" == 'not found' ]]; then
dir="${dirs[0]}"
download_with_curl "${dir}/${fname}" "${url}"
(( $? == 0 )) || \
download_with_curl "${dir}/${fname}" "${url}" || \
std::die 42 \
"%s " \
"${module_name}/${module_version}:" \
@@ -675,12 +666,6 @@ pbuild::prep() {
"error patching sources!"
done
}
for fname in ${VERSIONS[@]/#/pbuild::set_download_url_}; do
if typeset -F ${fname} 2>/dev/null; then
$f
break
fi
done
(( ${#SOURCE_URLS[@]} == 0 )) && return 0
${mkdir} -p "${PMODULES_DISTFILESDIR}"
local i=0
@@ -718,9 +703,8 @@ pbuild::prep() {
#..............................................................................
#
pbuild::add_configure_args() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
CONFIGURE_ARGS+=( "$@" )
}
readonly -f pbuild::add_configure_args
@@ -733,9 +717,8 @@ readonly -f pbuild.add_configure_args
#..............................................................................
#
pbuild::use_autotools() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
configure_with='autotools'
}
readonly -f pbuild::use_autotools
@@ -743,9 +726,8 @@ readonly -f pbuild::use_autotools
#..............................................................................
#
pbuild::use_cmake() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
configure_with='cmake'
}
readonly -f pbuild::use_cmake
@@ -766,7 +748,7 @@ pbuild::use_cc() {
"%s " "${module_name}/${module_version}:" \
"Error in setting CC:" \
"'$1' is not an executable!"
CC="$1"
export CC="$1"
}
readonly -f pbuild::use_cc
@@ -780,9 +762,8 @@ readonly -f pbuild::use_cc
declare -- compile_in_sourcetree='no'
pbuild::compile_in_sourcetree() {
[[ ${opt_yaml} == 'yes' ]] && \
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
compile_in_sourcetree='yes'
}
readonly -f pbuild::compile_in_sourcetree
@@ -826,7 +807,7 @@ pbuild::configure() {
if [[ -r "${SRC_DIR}/configure" ]] && \
[[ "${configure_with}" == 'auto' ]] || \
[[ "${configure_with}" == 'autotools' ]]; then
${SRC_DIR}/configure \
"${SRC_DIR}/configure" \
--prefix="${PREFIX}" \
"${config_args[@]}" || \
std::die 3 \
@@ -890,10 +871,8 @@ pbuild::compile() {
# $@: documentation files relative to source
#
pbuild::install_docfiles() {
if [[ ${opt_yaml} == 'yes' ]]; then
std::info \
"Using ${FUNCNAME} is deprecated with YAML module configuration files."
fi
std::info \
"Using ${FUNCNAME[0]} is deprecated with YAML module configuration files."
MODULE_DOCFILES+=("$@")
}
readonly -f pbuild::install_docfiles
@@ -920,8 +899,9 @@ pbuild::install_shared_libs() {
local -r pattern="${3//\//\\/}" # escape slash
install_shared_libs_Linux() {
local libs=( $(ldd "${binary}" | \
${awk} "/ => \// && /${pattern}/ {print \$3}") )
local -a libs=()
mapfile -t libs < <(${ldd} "${binary}" | \
${awk} "/ => \// && /${pattern}/ {print \$3}")
if (( ${#libs[@]} > 0 )); then
${cp} -vL "${libs[@]}" "${dstdir}" || return $?
fi
@@ -930,8 +910,9 @@ pbuild::install_shared_libs() {
install_shared_libs_Darwin() {
# https://stackoverflow.com/questions/33991581/install-name-tool-to-update-a-executable-to-search-for-dylib-in-mac-os-x
local libs=( $(${otool} -L "${binary}" | \
${awk} "/${pattern}/ {print \$1}"))
local -a libs=()
mapfile -t libs < <(${otool} -L "${binary}" | \
${awk} "/${pattern}/ {print \$1}")
if (( ${#libs[@]} > 0 )); then
${cp} -vL "${libs[@]}" "${dstdir}" || return $?
fi
@@ -943,7 +924,7 @@ pbuild::install_shared_libs() {
"%s " "${module_name}/${module_version}:" \
"${binary}: does not exist or is not executable!"
${mkdir} -p "${dstdir}"
case "${OS}" in
case "${KernelName}" in
Linux )
install_shared_libs_Linux
;;
@@ -1028,8 +1009,9 @@ _build_module() {
build_dependency() {
find_build_script(){
local p=$1
local script=$(${find} "${BUILDBLOCK_DIR}/../.." \
-path "*/$p/build")
local script=''
script=$(${find} "${BUILDBLOCK_DIR}/../.." \
-path "*/$p/build")
std::get_abspath "${script}"
}
@@ -1044,7 +1026,7 @@ _build_module() {
std::info "%s " \
"$m: module does not exist, trying to build it..."
local args=( '' )
set -- ${ARGS[@]}
set -- "${ARGS[@]}"
while (( $# > 0 )); do
case $1 in
-j )
@@ -1052,23 +1034,24 @@ _build_module() {
shift
;;
--jobs=[0-9]* )
args+=( $1 )
args+=( "$1" )
;;
-v | --verbose)
args+=( $1 )
args+=( "$1" )
;;
--with=*/* )
args+=( $1 )
args+=( "$1" )
;;
esac
shift
done
local buildscript=$(find_build_script "${m%/*}")
local buildscript=''
buildscript=$(find_build_script "${m%/*}")
[[ -x "${buildscript}" ]] || \
std::die 1 \
"$m: build-block not found!"
if ! "${buildscript}" "${m#*/}" ${args[@]}; then
if ! "${buildscript}" "${m#*/}" "${args[@]}"; then
std::die 1 \
"$m: oops: build failed..."
fi
@@ -1111,9 +1094,6 @@ _build_module() {
pbuild::module_is_avail "$m" release_of_dependency || \
std::die 6 "Oops"
fi
# should be set, just in case it is not...
: ${release_of_dependency:='unstable'}
# for a stable module all dependencies must be stable
if [[ "${module_release}" == 'stable' ]] \
&& [[ "${release_of_dependency}" != 'stable' ]]; then
@@ -1131,7 +1111,7 @@ _build_module() {
fi
std::info "Loading module: ${m}"
eval $( "${MODULECMD}" bash load "${m}" )
eval "$( "${modulecmd}" bash load "${m}" )"
done
}
@@ -1140,13 +1120,11 @@ _build_module() {
if [[ "${opt_yaml,,}" == 'no' ]]; then
(( ${#SUPPORTED_SYSTEMS[@]} == 0 )) && return 0
for sys in "${SUPPORTED_SYSTEMS[@]}"; do
[[ ${sys,,} == ${system,,} ]] && return 0
[[ "${sys,,}" == "${system,,}" ]] && return 0
done
std::die 1 \
"%s " "${module_name}/${module_version}:" \
"Not available for ${system}."
else
: debug "Systems: $Systems"
fi
}
@@ -1154,7 +1132,7 @@ _build_module() {
check_supported_compilers() {
(( ${#SUPPORTED_COMPILERS[@]} == 0 )) && return 0
for compiler in "${SUPPORTED_COMPILERS[@]}"; do
[[ ${compiler,,} == ${COMPILER,,} ]] && return 0
[[ "${compiler,,}" == "${COMPILER,,}" ]] && return 0
done
std::die 1 \
"%s " "${module_name}/${module_version}:" \
@@ -1197,8 +1175,8 @@ _build_module() {
"module is in group '${GROUP}' but no HDF5 module loaded!"
}
modulefile_dir="${ol_mod_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}/"
PREFIX="${ol_inst_root}/${GROUP}/${module_name}/${module_version}/"
modulefile_dir="${ol_modulefiles_root}/${GROUP}/${PMODULES_MODULEFILES_DIR}/"
PREFIX="${ol_install_root}/${GROUP}/${module_name}/${module_version}/"
case "${GROUP}" in
Compiler )
[[ -v COMPILER_VERSION ]] || die_no_compiler
@@ -1221,8 +1199,8 @@ _build_module() {
[[ -v HDF5_VERSION ]] || die_no_hdf5
modulefile_dir+="${COMPILER}/${COMPILER_VERSION}/"
modulefile_dir+="${MPI}/${MPI_VERSION}/"
modulefile_dir+="${HDF5}/${HDF5_VERSION}/"
PREFIX+="${HDF5}/${HDF5_VERSION}/"
modulefile_dir+="hdf5/${HDF5_VERSION}/"
PREFIX+="hdf5/${HDF5_VERSION}/"
PREFIX+="${MPI}/${MPI_VERSION}/"
PREFIX+="${COMPILER}/${COMPILER_VERSION}/"
group_depth=6
@@ -1231,8 +1209,8 @@ _build_module() {
[[ -v COMPILER_VERSION ]] || die_no_compiler
[[ -v HDF5_SERIAL_VERSION ]] || die_no_hdf5
modulefile_dir+="${COMPILER}/${COMPILER_VERSION}/"
modulefile_dir+="${hdf5_serial}/${HDF5_SERIAL_VERSION}/"
PREFIX+="${hdf5_serial}/${HDF5_SERIAL_VERSION}/"
modulefile_dir+="hdf5_serial/${HDF5_SERIAL_VERSION}/"
PREFIX+="hdf5_serial/${HDF5_SERIAL_VERSION}/"
PREFIX+="${COMPILER}/${COMPILER_VERSION}/"
group_depth=4
;;
@@ -1270,20 +1248,11 @@ _build_module() {
"${module_name}/${module_version}:" \
"Installing documentation to ${docdir}"
${install} -m 0755 -d "${docdir}"
${install} -m0644 "${BUILD_SCRIPT}" "${docdir}"
"${MODULECMD}" bash list -t 2>&1 1>/dev/null | \
${install} -m 0644 "${BUILD_SCRIPT}" "${docdir}"
"${modulecmd}" bash list -t 2>&1 1>/dev/null | \
${grep} -v "Currently Loaded" > \
"${docdir}/dependencies" || :
# loop over version specific functions. In these function
# more MODULE_DOCFILES can be defined.
# :FIXME: maybe we find a better solution.
for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do
if typeset -F "$f" 2>/dev/null; then
$f
break
fi
done
(( ${#MODULE_DOCFILES[@]} == 0 )) && return 0
${install} -m0644 \
"${MODULE_DOCFILES[@]/#/${SRC_DIR}/}" \
@@ -1307,28 +1276,29 @@ _build_module() {
if [[ ! $dep == */* ]]; then
# no version given: derive the version
# from the currently loaded module
dep=$( "${MODULECMD}" bash list -t 2>&1 1>/dev/null \
dep=$( "${modulecmd}" bash list -t 2>&1 1>/dev/null \
| grep "^${dep}/" )
fi
echo "${dep}" >> "${fname}"
done
}
patch_elf_exe_and_libs(){
local -- libdir="${OverlayInfo[${ol_name}:inst_root]}/lib64"
local -- libdir="${OverlayInfo[${ol_name}:install_root]}/lib64"
[[ -d "${libdir}" ]] || return 0
local -a bin_objects=( $(std::find_executables '.' ) )
local -a bin_objects=()
mapfile -t bin_objects < <(std::find_executables '.')
local -- fname=''
local -- rpath=''
local -i depth=0
for fname in "${bin_objects[@]}"; do
local -i depth=$(std::get_dir_depth "${fname}")
(( depth+=group_depth+3 ))
local -- rpath='$ORIGIN/'$(printf "../%.0s" $(${seq} 1 ${depth}))lib64
(( depth=$(std::get_dir_depth "${fname}") + group_depth + 3 ))
rpath='$ORIGIN/'$(printf "../%.0s" $(${seq} 1 ${depth}))lib64
${patchelf} --force-rpath --set-rpath "${rpath}" "${fname}"
done
local -a bin_objects=( $(std::find_shared_objects '.' ) )
mapfile -t bin_objects < <(std::find_shared_objects '.')
for fname in "${bin_objects[@]}"; do
local -i depth=$(std::get_dir_depth "${fname}")
(( depth+=group_depth+3 ))
local -- rpath='$ORIGIN/'$(printf "../%.0s" $(${seq} 1 ${depth}))lib64
(( depth=$(std::get_dir_depth "${fname}") + group_depth + 3 ))
rpath='$ORIGIN/'$(printf "../%.0s" $(${seq} 1 ${depth}))lib64
${patchelf} --force-rpath --set-rpath "${rpath}" "${fname}"
done
}
@@ -1340,7 +1310,7 @@ _build_module() {
std::info \
"%s " \
"${module_name}/${module_version}:" \
"running post-installation for ${OS} ..."
"running post-installation for ${KernelName} ..."
cd "${PREFIX}"
[[ -d "lib" ]] && [[ ! -d "lib64" ]] && ln -s lib lib64
patch_elf_exe_and_libs
@@ -1350,7 +1320,7 @@ _build_module() {
#..............................................................
# post-install
cd "${BUILD_DIR}"
[[ "${OS}" == "Linux" ]] && post_install_linux
[[ "${KernelName}" == "Linux" ]] && post_install_linux
install_doc
if (( ${#runtime_dependencies[@]} > 0 )); then
write_runtime_dependencies \
@@ -1374,7 +1344,7 @@ _build_module() {
} # post_install
#......................................................................
# Install modulefile in ${ol_mod_root}/${GROUP}/modulefiles/...
# Install modulefile in ${ol_modulefiles_root}/${GROUP}/modulefiles/...
# The modulefiles in the build-block can be
# versioned like
# modulefile-10.2.0
@@ -1413,8 +1383,7 @@ _build_module() {
}
[[ "${is_subpkg}" == 'yes' ]] && return 0
local src=''
find_modulefile src
if (( $? != 0 )); then
if ! find_modulefile src; then
std::info \
"%s " \
"${module_name}/${module_version}:" \
@@ -1434,8 +1403,8 @@ _build_module() {
local ol=''
for ol in "${Overlays[@]}"; do
[[ "${ol}" == "${ol_name}" ]] && continue
local mod_root="${OverlayInfo[${ol}:mod_root]}"
local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}"
local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}"
local dir="${modulefile_dir/${ol_modulefiles_root}/${modulefiles_root}}"
local fname="${dir}/${module_version}"
if [[ -e "${fname}" ]]; then
std::info "%s "\
@@ -1459,12 +1428,9 @@ _build_module() {
local -r legacy_config_file="${modulefile_dir}/.release-${module_version}"
local -- status_legay_config_file='unchanged'
local -r yaml_config_file="${modulefile_dir}/.config-${module_version}"
local -- status_yaml_config_file='unchanged'
local -- relstage_legacy=''
if [[ -r "${legacy_config_file}" ]]; then
local relstage_legacy
read relstage_legacy < "${legacy_config_file}"
read -r relstage_legacy < "${legacy_config_file}"
if [[ "${relstage_legacy}" != "${module_release}" ]]; then
status_legay_config_file='changed'
fi
@@ -1476,8 +1442,10 @@ _build_module() {
echo "${module_release}" > "${legacy_config_file}"
fi
local -r yaml_config_file="${modulefile_dir}/.config-${module_version}"
local -- status_yaml_config_file='unchanged'
if [[ -r "${yaml_config_file}" ]]; then
while read key value; do
while read -r key value; do
local -n ref="${key:0:-1}"
ref="${value}"
done < "${yaml_config_file}"
@@ -1583,26 +1551,26 @@ _build_module() {
return 0
fi
local targets=()
targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} )
targets+=( pbuild::pre_${target}_${system} )
targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} )
targets+=( pbuild::pre_${target}_${OS} )
targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} )
targets+=( pbuild::pre_${target} )
targets+=( "${VERSIONS[@]/#/pbuild::pre_${target}_${system}_}" )
targets+=( "pbuild::pre_${target}_${system}" )
targets+=( "${VERSIONS[@]/#/pbuild::pre_${target}_${KernelName}_}" )
targets+=( "pbuild::pre_${target}_${KernelName}" )
targets+=( "${VERSIONS[@]/#/pbuild::pre_${target}_}" )
targets+=( "pbuild::pre_${target}" )
targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} )
targets+=( pbuild::${target}_${system} )
targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} )
targets+=( pbuild::${target}_${OS} )
targets+=( ${VERSIONS[@]/#/pbuild::${target}_} )
targets+=( pbuild::${target} )
targets+=( "${VERSIONS[@]/#/pbuild::${target}_${system}_}" )
targets+=( "pbuild::${target}_${system}" )
targets+=( "${VERSIONS[@]/#/pbuild::${target}_${KernelName}_}" )
targets+=( "pbuild::${target}_${KernelName}" )
targets+=( "${VERSIONS[@]/#/pbuild::${target}_}" )
targets+=( "pbuild::${target}" )
targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} )
targets+=( pbuild::post_${target}_${system} )
targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} )
targets+=( pbuild::post_${target}_${OS} )
targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} )
targets+=( pbuild::post_${target} )
targets+=( "${VERSIONS[@]/#/pbuild::post_${target}_${system}_}" )
targets+=( "pbuild::post_${target}_${system}" )
targets+=( "${VERSIONS[@]/#/pbuild::post_${target}_${KernelName}_}" )
targets+=( "pbuild::post_${target}_${KernelName}" )
targets+=( "${VERSIONS[@]/#/pbuild::post_${target}_}" )
targets+=( "pbuild::post_${target}" )
for t in "${targets[@]}"; do
# We cd into the dir before calling the function -
@@ -1661,7 +1629,7 @@ _build_module() {
"%s " \
"${module_name}/${module_version}:" \
"removing all files in '${PREFIX}' ..."
[[ "${dry_run}" == 'no' ]] && ${rm} -rf ${PREFIX}
[[ "${dry_run}" == 'no' ]] && ${rm} -rf "${PREFIX}"
fi
if [[ -e "${modulefile_name}" ]]; then
std::info \
Pmodules/libpmodules.bash.in
+214 -171
View File
@@ -1,16 +1,11 @@
#!/bin/bash
declare PMODULES_MODULEFILES_DIR='modulefiles'
declare PMODULES_CONFIG_DIR='config'
declare PMODULES_VERSION='@PMODULES_VERSION@'
declare -A GroupDepths=()
declare -A Subcommands=()
declare -A Options=()
declare -A Help=()
declare -a Overlays=()
declare -A OverlayInfo
declare -a UsedOverlays
declare -A Dir2OverlayMap
declare -r ol_normal='n'
@@ -18,21 +13,6 @@ declare -r ol_hiding='h'
declare -r ol_replacing='r'
# initialize help text of 'module --version'
Help['version']="
Pmodules @PMODULES_VERSION@
using Tcl Environment Modules
VERSION = @MODULES_VERSION@
"
#
# display help text for command given in $1
#
print_help() {
echo -e "${Help[$1]}" 1>&2
std::die 1
}
#
# compute depth of modulefile directory.
#
@@ -50,10 +30,10 @@ compute_group_depth () {
local group=${group##*/}
result=$(${find} "${dir}" -depth \( -type f -o -type l \) \
-printf "%d" -quit 2>/dev/null)
(( result-=2 ))
(( result-=2 )) || :
# if a group doesn't contain a modulefile, depth is negativ
# :FIXME: better solution?
(( result < 0 )) && (( result = 0 ))
(( result < 0 )) && (( result = 0 )) || :
}
#
@@ -63,173 +43,236 @@ compute_group_depth () {
# $@: overlay names
#
scan_groups () {
local ol
local ol
local depth
for ol in "$@"; do
local mod_root="${OverlayInfo[${ol}:mod_root]}"
local modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}"
local dir
for dir in ${mod_root}/*/${PMODULES_MODULEFILES_DIR}; do
for dir in "${modulefiles_root}"/*/"${PMODULES_MODULEFILES_DIR}"; do
local group="${dir%/*}"
group="${group##*/}"
if [[ ! -v GroupDepths[${group}] ]]; then
compute_group_depth depth "${dir}"
GroupDepths[$group]=${depth}
fi
Dir2OverlayMap[${dir%/${PMODULES_MODULEFILES_DIR}*}]="${ol}"
Dir2OverlayMap[${dir%/"${PMODULES_MODULEFILES_DIR}"*}]="${ol}"
done
done
}
pm::read_config(){
local -a config_files=()
declare -A DefaultPmodulesConfig=(
['defaultgroups']='Tools:Programming'
['default_groups']='Tools:Programming'
['defaultreleasestages']='stable'
['default_reltages']='stable'
['tmpdir']="/opt/psi/var/tmp/${USER}"
['tmp_dir']="/opt/psi/var/tmp/${USER}"
['distfilesdir']='/opt/psi/var/distfiles'
['distfiles_dir']='/opt/psi/var/distfiles'
['download_dir']='/opt/psi/var/distfiles'
['overlays']=''
)
declare -A OverlayConfigKeys=(
['install_root']='/opt/psi'
['modulefiles_root']=''
['excludes']=''
['type']='n'
)
pm::get_value(){
local -- yaml_input="$1"
local -n val="$2"
local -- key="$3"
local -- expected_type="$4"
_get_config_files(){
#
# return array with overlay configuration files
#
# Args:
# $1 [upvar] result
local -n fnames="$1"
local -- type=''
type=$( ${yq} -e ".${key} | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type}" != "${expected_type}" ]]; then
std::die 3 "%s" \
"Value of '${key}' must be of type '${expected_type:2}', but is '${type:2}'!"
fi
val=$( ${yq} -e ".${key}" \
2>/dev/null <<<"${yaml_input}" ) || val=''
}
pm::get_seq(){
local -- yaml_input="$1"
local -n val="$2"
local -- key="$3"
# user defined via environment variable
if [[ -v PMODULES_OVERLAYS_DEF ]]; then
test -r "${PMODULES_OVERLAYS_DEF}" || \
std::die 3 \
"%s -- %s" \
"overlay definition file is not readable" \
"$_"
fnames+=("${PMODULES_OVERLAYS_DEF}")
fi
# user defined
if [[ -r "${HOME}/.Pmodules/Pmodules.yaml" ]]; then
fnames+=("${HOME}/.Pmodules/Pmodules.yaml")
fi
# system config file
test -r "${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml" || \
std::die 3 \
"%s %s -- %s" \
"base overlay definition file" \
"does not exist or is not readable" \
"$_"
fnames+=("${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml")
}
_get_ol_names(){
#
# get the names of all overlays
#
local -n fnames="$1"
${yq} -Ne eval-all '. as $item ireduce ({}; . *+ $item) |.Overlays|keys()' \
"${fnames[@]}" | awk '{print $2}'
}
_get_install_root(){
local -n fnames="$1"
local -- _ol="$2"
local -- _tmp
_tmp=$(${yq} -Ne eval-all ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${_ol}.install_root" \
"${fnames[@]}" 2>/dev/null) || return 1
echo "${_tmp}" | envsubst
}
_get_modulefiles_root(){
local -n fnames="$1"
local -- _ol="$2"
local -- _tmp
_tmp=$(${yq} -Ne eval-all ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${_ol}.modulefiles_root" \
"${fnames[@]}" 2>/dev/null) || return 1
echo "${_tmp}" | envsubst
}
_get_excludes(){
local -n fnames="$1"
local -- _ol="$2"
${yq} -Ne eval-all ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${_ol}.excludes[]" \
"${fnames[@]}" 2>/dev/null
}
_get_type(){
local -n fnames="$1"
local -- _ol="$2"
${yq} -Ne eval-all ". as \$item ireduce ({}; . *+ \$item) |.Overlays.${_ol}.type" \
"${fnames[@]}" 2>/dev/null
}
_join_array() {
local IFS="$1"
shift
echo -n "$*"
}
_get_config_files config_files
eval $(std::parse_yaml "${config_files[-1]}" 'cfg_')
[[ -v cfg_DefaultGroups ]] && DefaultGroups="${cfg_DefaultGroups}"
[[ -v cfg_DefaultReleaseStages ]] && DefaultReleaseStages="${cfg_DefaultReleaseStages}"
[[ -v cfg_ReleaseStages ]] && ReleaseStages="${cfg_ReleaseStages}"
[[ -v cfg_SysCollectionsDir ]] && SysCollectionsDir="${cfg_SysCollectionsDir}"
unset ${!cfg_*}
#
# loop over all overlays and save config in OverlayInfo and Dir2OverlayMap
#
# - at least install_root must be specified
# - modulefiles_root default so install_root
# - the type defaults to ${ol_normal}
#
Overlays=( $(_get_ol_names config_files) )
local ol=''
local install_root=''
local modulefiles_root=''
local type=''
local excludes=()
for ol in "${Overlays[@]}"; do
# install_root
install_root=$(_get_install_root config_files "${ol}") || \
std::die 3 \
"Install_root missing for overlay -- ${ol}"
install_root=$(envsubst <<< "${install_root}")
[[ -d ${install_root} ]] || \
std::die 3 \
"Invalid installation root directory for overlay '${ol}' -- ${install_root}"
OverlayInfo[${ol}:inst_root]="${install_root}"
# modulefiles_root
modulefiles_root=$(_get_modulefiles_root config_files "${ol}") || \
modulefiles_root=${install_root}
[[ -d ${modulefiles_root} ]] || \
std::die 3 \
"Invalid modulefiles root directory for overlay '${ol}' -- ${modulefiles_root}"
OverlayInfo[${ol}:mod_root]="${modulefiles_root}"
Dir2OverlayMap[${modulefiles_root}]="${ol}"
# type
type=$(_get_type config_files "${ol}") || \
type=${ol_normal}
case ${type} in
${ol_normal} | ${ol_replacing} | ${ol_hiding} )
:
;;
* )
std::die 3 "Invalid type for overlay '${ol}' -- ${type}"
;;
esac
OverlayInfo[${ol}:type]="${type}"
# excludes
excludes=$(_get_excludes config_files "${ol}") || \
excludes=()
OverlayInfo[${ol}:excludes]=$(_join_array ':' "${excludes[@]}")
# mark as unused
OverlayInfo[${ol}:used]='no'
done
local -- type=''
type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type:2}" == 'null' ]]; then
val=''
return 0
fi
if [[ "${type}" != '!!seq' ]]; then
std::die 3 "%s" \
"Value of '${key}' must be of type 'seq', but is of type '${type:2}'!"
fi
val=$( ${yq} -e ".${key}[]" \
2>/dev/null <<<"${yaml_input}" ) || val=''
}
pm::read_config(){
: "
Read Pmodules configuration files '${PMODULES_HOME}/config/Pmodules.yaml'
and '${HOME}/.Pmodules/Pmodules.yaml'.
Args:
none
"
get_config_of_overlay(){
: "
Get configuration of an overlay.
Args:
$1 YAML config
$2 name of overlay
"
local -r yaml_input="$1"
local -r ol_name="$2"
local -- key=''
local -a keys=()
local -- value=''
# init overlay with defaults
for key in "${!OverlayConfigKeys[@]}"; do
OverlayInfo[${ol_name}:${key}]="${OverlayConfigKeys[${key}]}"
done
# get keys in YAML input
readarray -t keys < <( ${yq} -e ".|keys|.[]" <<<"${yaml_input}" 2>/dev/null ) || \
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
for key in "${keys[@]}"; do
[[ -v OverlayConfigKeys[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key,,} in
install_root )
pm::get_value "${yaml_input}" value "${key}" '!!str'
OverlayInfo[${ol_name}:install_root]=$(${envsubst} <<< "${value}")
[[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \
std::die 3 \
"Invalid installation root directory for overlay '${ol_name}' -- ${value}"
;;
modulefiles_root )
pm::get_value "${yaml_input}" value "${key}" '!!str'
OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}")
[[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \
std::die 3 \
"Invalid modulefiles root directory for overlay '${ol_name}' -- ${value}"
;;
type )
pm::get_value "${yaml_input}" value "${key}" '!!str'
case ${value} in
"${ol_normal}" | "${ol_replacing}" | "${ol_hiding}" )
:
;;
* )
std::die 3 "Invalid type for overlay '${ol_name}' -- ${type}"
;;
esac
OverlayInfo[${ol_name}:type]="${value}"
;;
excludes )
pm::get_seq "${yaml_input}" value "${key}" '!!seq'
local -a tmp_array=()
read -r -a tmp_array <<<"${value}"
local excludes=''
printf -v excludes "%s:" "${tmp_array[@]}"
OverlayInfo[${ol_name}:excludes]="${excludes%:}"
;;
esac
done
if [[ -z "${OverlayInfo[${ol_name}:modulefiles_root]}" ]]; then
OverlayInfo[${ol_name}:modulefiles_root]=${OverlayInfo[${ol_name}:install_root]}
fi
local modulefiles_root=${OverlayInfo[${ol_name}:modulefiles_root]}
Dir2OverlayMap[${modulefiles_root}]="${ol_name}"
}
get_config_of_overlays(){
: "
Get configuration of overlays in YAML input.
Args:
$1 YAML input
"
local -r yaml_input="$1"
local -- ol_name=''
readarray -t Overlays < <( ${yq} -e '.|keys|.[]' <<<"${yaml_input}" )
for ol_name in "${Overlays[@]}"; do
local yaml_tmp
pm::get_value "${yaml_input}" yaml_tmp "${ol_name}" '!!map'
get_config_of_overlay "${yaml_tmp}" "${ol_name}"
done
}
get_config(){
: "
Get Pmodules configuration.
Args:
$1 Pmodules configuration file
"
local -r config_file="$1"
local -- yaml_input=''
yaml_input=$( ${yq} -Ne e "." "${config_file}" 2>/dev/null ) || \
std::die 3 "Cannot read configuration file '${config_file}'!"
local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null ) || \
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
for key in "${keys[@]}"; do
[[ -v DefaultPmodulesConfig[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key,,} in
defaultgroups | default_groups )
pm::get_value "${yaml_input}" DefaultGroups "${key}" '!!str'
;;
defaultreleasestages | default_reltages )
pm::get_value "${yaml_input}" DefaultReleaseStages "${key}" '!!str'
;;
tmpdir | tmp_dir )
pm::get_value "${yaml_input}" TmpDir "${key}" '!!str'
;;
distfilesdir | download_dir )
pm::get_value "${yaml_input}" DistfilesDir "${key}" '!!str'
;;
overlays )
local tmp_str=''
pm::get_value "${yaml_input}" tmp_str "${key}" '!!map'
get_config_of_overlays "${tmp_str}"
;;
esac
done
}
# system config file
local -r sys_config_file="${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml"
test -r "${sys_config_file}" || \
std::die 3 \
"%s %s -- %s" \
"base overlay definition file" \
"does not exist or is not readable" \
"$_"
DefaultGroups="${DefaultPmodulesConfig['default_groups']}"
DefaultReleaseStages="${DefaultPmodulesConfig['default_reltages']}"
TmpDir="${DefaultPmodulesConfig['tmp_dir']}"
DistfilesDir="${DefaultPmodulesConfig['download_dir']}"
get_config "${sys_config_file}"
local -r usr_config_file="${HOME}/.Pmodules/Pmodules.yaml"
if [[ -r "${usr_config_file}" ]]; then
get_config "${usr_config_file}"
fi
}
# Local Variables:
# mode: sh
# sh-basic-offset: 8
Pmodules/libstd.bash
+61 -162
View File
@@ -27,7 +27,7 @@ std::debug() {
std::die() {
local -ri ec=$1
shift
if [[ -n $@ ]]; then
if (( ${#@} > 0 )); then
local -r fmt=$1
shift
std::log 2 "$fmt" "$@"
@@ -39,16 +39,51 @@ std::def_cmd(){
which "$1" 2>/dev/null || std::die 255 "'$1' not found!"
}
std::def_cmds(){
local path="$1"
shift
for cmd in "$@"; do
eval declare -gr ${cmd}=$(PATH="${path}" /usr/bin/which $cmd 2>/dev/null)
if [[ -z "${!cmd}" ]]; then
std::die 255 "${cmd} not found"
fi
done
}
awk=$(std::def_cmd 'awk'); declare -r awk
base64=$(std::def_cmd 'base64'); declare -r base64
bash=$(std::def_cmd 'bash'); declare -r bash
cat=$(std::def_cmd 'cat'); declare -r cat
cp=$(std::def_cmd 'cp'); declare -r cp
curl=$(std::def_cmd 'curl'); declare -r curl
envsubst=$(std::def_cmd 'envsubst'); declare -r envsubst
dirname=$(std::def_cmd 'dirname'); declare -r dirname
file=$(std::def_cmd 'file'); declare -r file
find=$(std::def_cmd 'find'); declare -r find
getopt=$(std::def_cmd 'getopt'); declare -r getopt
grep=$(std::def_cmd 'grep'); declare -r grep
install=$(std::def_cmd 'install'); declare -r install
logger=$(std::def_cmd 'logger'); declare -r logger
make=$(std::def_cmd 'make'); declare -r make
mkdir=$(std::def_cmd 'mkdir'); declare -r mkdir
mktemp=$(std::def_cmd 'mktemp'); declare -r mktemp
modulecmd=$(std::def_cmd 'modulecmd'); declare -- modulecmd
patch=$(std::def_cmd 'patch'); declare -r patch
pwd=$(std::def_cmd 'pwd'); declare -r pwd
rm=$(std::def_cmd 'rm'); declare -r rm
rmdir=$(std::def_cmd 'rmdir'); declare -r rmdir
sed=$(std::def_cmd 'sed'); declare -r sed
seq=$(std::def_cmd 'seq'); declare -r seq
sevenz=$(std::def_cmd 'sevenz'); declare -r sevenz
sort=$(std::def_cmd 'sort'); declare -r sort
tar=$(std::def_cmd 'tar'); declare -r tar
tee=$(std::def_cmd 'tee'); declare -r tee
tput=$(std::def_cmd 'tput'); declare -r tput
uname=$(std::def_cmd 'uname'); declare -r uname
yq=$(std::def_cmd 'yq'); declare -r yq
KernelName=$(${uname} -s); declare -r KernelName
if [[ ${KernelName} == 'Darwin' ]]; then
PATH+=':/opt/local/bin'
otool=$(std::def_cmd 'otool'); declare -r otool
shasum=$(std::def_cmd 'shasum');declare -r shasum
sysctl=$(std::def_cmd 'sysctl');declare -r sysctl
declare -r sha256sum="${shasum -a 256}"
else
ldd=$(std::def_cmd 'ldd'); declare -r ldd
patchelf=$(std::def_cmd 'patchelf'); declare -r patchelf
sha256sum=$(std::def_cmd 'sha256sum');
declare -r sha256sum
fi
#
# get answer to yes/no question
@@ -58,7 +93,7 @@ std::def_cmds(){
std::get_YN_answer() {
local -r prompt="$1"
local ans
read -p "${prompt}" ans
read -r -p "${prompt}" ans
case ${ans} in
y|Y )
return 0;;
@@ -71,14 +106,16 @@ std::get_YN_answer() {
# return normalized abolute pathname
# $1: filename
std::get_abspath() {
local -r fname=$1
local -r fname="$1"
local -- abspath=''
#[[ -r "${fname}" ]] || return 1
if [[ -d ${fname} ]]; then
echo $(cd "${fname}" && pwd)
abspath=$(cd "${fname}" && pwd -L)
else
local -r dname=$(dirname "${fname}")
echo $(cd "${dname}" && pwd)/$(basename "${fname}")
abspath=$(cd "${dname}" && pwd -L)/$(basename "${fname}")
fi
echo "${abspath}"
}
std::append_path () {
@@ -117,135 +154,20 @@ std::prepend_path () {
}
std::remove_path() {
local -nr P="$1"
local -nr path="$1"
shift 1
local -ar dirs="$@"
local -ar remove_dirs=("$@")
local new_path=''
local -r _P=( ${P//:/ } )
local -a _path=()
IFS=':' read -r -a _path <<<"${path}"
local dir=''
for dir in "${dirs[@]}"; do
for dir in "${remove_dirs[@]}"; do
# loop over all entries in path
for entry in "${_P[@]}"; do
for entry in "${_path[@]}"; do
[[ "${entry}" != "${dir}" ]] && new_path+=":${entry}"
done
done
P="${new_path:1}" # remove leading ':'
}
#
# Replace or remove a directory in a path variable.
#
# To remove a dir:
# std::replace_path PATH <pattern>
#
# To replace a dir:
# std::replace_path PATH <pattern> /replacement/path
#
# Args:
# $1 name of the shell variable to set (e.g. PATH)
# $2 a grep pattern identifying the element to be removed/replaced
# $3 the replacement string (use "" for removal)
#
# Based on solution published here:
# https://stackoverflow.com/questions/273909/how-do-i-manipulate-path-elements-in-shell-scripts
#
std::replace_path () {
local -r path="$1"
local -r removepat="$2"
local -r replacestr="${3:-''}"
local -r removestr=$(echo "${!path}" | tr ":" "\n" | grep -m 1 "^$removepat\$")
export $path="$(echo "${!path}" | tr ":" "\n" | sed "s:^${removestr}\$:${replacestr}:" |
sed '/^\s*$/d' | tr "\n" ":" | sed -e 's|^:||' -e 's|:$||')"
}
#
# Functions to split a path into its components.
#
# Args:
# $1 upvar
# $2 absolute or relative path (depends on the function)
# $3 opt upvar: number of components
#
# Notes:
# std::split_path()
# if the path is absolute, the first element of the returned array is empty.
#
# std::split_abspath()
# the path must begin with a slash, otherwise std::die() is called with
# an internal error message.
#
# std::split_relpath()
# analog to std::split_abspath() with a relative path.
#
std::split_path() {
local -n parts="$1"
local -r path="$2"
IFS='/'
local std__split_path_result=( ${std__split_path_tmp} )
unset IFS
parts="${std__split_path_result[@]}"
if (( $# >= 3 )); then
# return number of parts
local -n num="$3"
num="${#std__split_path_result[@]}"
fi
}
std::split_abspath() {
local -n parts="$1"
local -r path="$2"
if [[ "${path:0:1}" == '/' ]]; then
local -r std__split_path_tmp="${path:1}"
else
std::die 255 "Oops: Internal error in '${FUNCNAME[0]}' called by '${FUNCNAME[1]}' }"
fi
IFS='/'
local std__split_path_result=( ${std__split_path_tmp} )
unset IFS
parts="${std__split_path_result[@]}"
if (( $# >= 3 )); then
# return number of parts
local -n num="$3"
num="${#std__split_path_result[@]}"
fi
}
std::split_relpath() {
local -n parts="$1"
local -r path="$2"
if [[ "${path:0:1}" == '/' ]]; then
std::die 255 "Oops: Internal error in '${FUNCNAME[0]}' called by '${FUNCNAME[1]}' }"
else
local -r std__split_path_tmp="${path}"
fi
IFS='/'
local std__split_path_result=( ${std__split_path_tmp} )
unset IFS
parts="${std__split_path_result[@]}"
if (( $# >= 3 )); then
# return number of parts
local -n num="$3"
num="${#std__split_path_result[@]}"
fi
}
std::read_versions() {
local -r fname="$1"
local varname=''
while read _name _version; do
[[ -z ${_name} ]] && continue
[[ -z ${_version} ]] && continue
[[ "${_name:0:1}" == '#' ]] && continue
var_name=$(echo ${_name} | tr [:lower:] [:upper:])_VERSION
# don't set version, if already set
if [[ -z ${!var_name} ]]; then
eval ${var_name}="${_version}"
fi
done < "${fname}"
path="${new_path:1}" # remove leading ':'
}
std.get_os_release_linux() {
@@ -288,7 +210,8 @@ std::get_os_release() {
}
std::get_type() {
local -a signature=( $(typeset -p "$1") )
local -a signature=()
read -r -a signature <(typeset -p "$1")
case ${signature[1]} in
-Ai* )
echo 'int dict'
@@ -314,30 +237,6 @@ std::get_type() {
esac
}
std::parse_yaml() {
#
# parse a YAML file
# See: https://gist.github.com/pkuczynski/8665367
#
local -r fname="$1"
local -r prefix="$2"
local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" "${fname}" |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {
if (i > indent) {delete vname[i]}
}
if (length($3) > 0) {
vn="";
for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
std::is_member_of_array(){
local -- item="$1"
local -n array="$2"
Pmodules/modbuild.in
+113 -324
View File
@@ -7,58 +7,32 @@
#
#.............................................................................
declare VERSION='@PMODULES_VERSION@'
declare -r VERSION='@PMODULES_VERSION@'
unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy!
unset IFS # use default IFS
#set -e # exit on error
set -o pipefail
set -o nounset
shopt -s nocaseglob
shopt -s extglob
shopt -s nullglob
# get absolute path of script
declare mydir=$(dirname "$0")
declare -r mydir=$(cd ${mydir} && pwd -P)
mydir=$(cd "$(/usr/bin/dirname "$0")" && pwd -P)
prefix=$(/usr/bin/dirname "${mydir}")
source "${mydir}/../lib/libstd.bash" || {
PATH="${prefix}/bin:${prefix}/libexec:/bin:/usr/bin:/sbin:/usr/sbin"
source "${prefix}/lib/libstd.bash" || {
echo "Oops: cannot source library -- '$_'" 1>&2; exit 3;
}
##############################################################################
#
# check availability of used commands and set environment variables
# cmd=$(which cmd)
# in the following we use these environment variable to call binaries.
#
declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd"
[[ -x ${MODULECMD} ]] || \
std::die 1 "Oops: modulecmd binary not available!"
std::def_cmds "${mydir}/../libexec" \
'patchelf' 'sevenz' 'yq'
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'awk' 'base64' 'cat' 'cp' 'envsubst' 'file' 'find' 'getopt' 'grep' \
'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \
'rm' 'rmdir' 'seq' 'sort' 'tar' 'tee' 'uname'
declare -r OS="$(${uname} -s)"
if [[ ${OS} == 'Darwin' ]]; then
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \
'curl' 'otool' 'shasum' 'sysctl'
declare -r sha256sum="${shasum -a 256}"
else
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'ldd' 'curl' 'sha256sum'
fi
# for the time being, we still set PATH. Just in case we forgot a binary
PATH="${PMODULES_HOME}/bin:/usr/bin:/bin:/usr/sbin:/sbin"
##############################################################################
source "${mydir}/../lib/libpbuild.bash" || \
source "${prefix}/lib/libpmodules.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
source "${mydir}/../lib/libpmodules.bash" || \
source "${prefix}/lib/libpbuild.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
##############################################################################
set -o nounset
shopt -s nocaseglob
shopt -s extglob
shopt -s nullglob
unset mydir
unset prefix
##############################################################################
#
@@ -143,13 +117,6 @@ MISCELLANEOUS OPTIONS:
--tmpdir
Directory used for building a module.
--legacy
Use legacy configuration files.
--overlay
Install in this overlay. Defaults to the base overlay. This
option can only be used with legacy configuration file.
DOCUMENTATION:
Full documentation is available at
http://pmodules.gitpages.psi.ch
@@ -166,7 +133,7 @@ DOCUMENTATION:
# last default
# save arguments, required for building dependencies
declare -r ARGS="$@"
declare -ra ARGS=( "$@" )
# versions to be build, '.*' or none means all
declare versions_to_build=''
@@ -178,14 +145,12 @@ declare opt_force_rebuild='no'
declare -i opt_jobs=0
declare opt_update_modulefiles='no'
declare opt_system=''
declare opt_overlay=''
declare opt_verbose='no'
# array collecting all modules specified on the command line via '--with=module'
declare -a opt_with_modules=() # :FIXME: legacy build should also use the dict
declare -a opt_with_modules=()
declare -A opt_with_dict=()
declare -- opt_config_file=''
declare -- opt_debug='no'
declare -- opt_yaml='yes'
declare -- opt_check_mode='no'
declare -- opt_variant=''
declare -- opt_clean_install='no'
@@ -193,7 +158,6 @@ declare -- opt_parent_prefix=''
declare -- BUILD_SCRIPT=''
declare -- yaml_config_file=''
declare -a legacy_config_files=()
declare -- module_name=''
declare -- module_type='module'
declare -- echo=':'
@@ -272,23 +236,9 @@ parse_args() {
shift
fi
;;
--overlay | --overlay=* )
if [[ $1 == *=* ]]; then
opt_overlay="${1#--*=}"
else
opt_overlay="$2"
shift
fi
;;
--yaml )
opt_yaml='yes'
;;
--check-mode )
opt_check_mode='yes'
;;
--legacy )
opt_yaml='no'
;;
--use-flags | --use-flags=* )
if [[ $1 == *=* ]]; then
USE_FLAGS=":${1#--*=}:"
@@ -334,9 +284,9 @@ parse_args() {
;;
--parent-prefix | --parent-prefix=* )
if [[ $1 == *=* ]]; then
opt_parent_prefix=( "${1#--*=}" )
opt_parent_prefix="${1#--*=}"
else
opt_parent_prefix=( "$2" )
opt_parent_prefix="$2"
shift
fi
module_type='sub_package'
@@ -375,152 +325,13 @@ parse_args() {
opt_system="${opt_system:-$(std::get_os_release)}"
# set config file
if [[ ${opt_yaml,,} == 'no' ]]; then
# look for legacy config files
if [[ -n ${opt_config_file} ]]; then
legacy_config_files=( "${opt_config_file}" )
else
shopt -q nullglob || :
local -i nullglob_set=$?
legacy_config_files=( "${BUILDBLOCK_DIR}"/*/variants.${opt_system} )
legacy_config_files+=( "${BUILDBLOCK_DIR}"/*/variants."$(uname -s)" )
legacy_config_files+=( "${BUILDBLOCK_DIR}"/*/variants )
(( nullglob_set == 1 )) && shopt -u nullglob
fi
(( ${#legacy_config_files[@]} == 0 )) && \
std::die 1 "No legacy configuration file found!"
std::info "Using legacy variants files."
opt_overlay="${opt_overlay:-'base'}"
else
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
[[ -r "${yaml_config_file}" ]] || \
std::die 2 \
"%s -- %s" \
"YAML config file doesn't exist or is not readable" \
"${yaml_config_file}"
[[ "${opt_overlay}" == 'yes' ]] && \
std::die 2 \
"opt '--overlay' can only be used together with legacy config files!"
std::info "Using YAML configuration file - ${yaml_config_file}"
fi
}
#
# bash brace expansion of given args. Input args like:
#
# "text" "gcc/{9.3.0,10.3.0}" "openmpi/{4.0.5,4.1.0}"
#
# will be expanded to the following four lines:
#
# "text gcc/9.3.0 openmpi/4.0.5"
# "text gcc/9.3.0 openmpi/4.1.0"
# "text gcc/10.3.0 openmpi/4.0.5"
# "text gcc/10.3.0 openmpi/4.1.0"
#
bash_expand(){
local text="$1"
shift
local to_expand=( "${@}" )
if (( ${#to_expand[@]} == 0 )); then
echo ${text}
else
local list
eval list=( ${to_expand[0]} )
local s
for s in ${list[*]}; do
bash_expand "${text} ${s}" "${to_expand[@]:1}"
done;
fi;
}
build_modules_legacy() {
local -a files=( "${legacy_config_files[@]}" )
local -A mod_overlays=()
expand_variants_file(){
local -r input="$1"
local -a toks=()
while read -a toks; do
# skip empty and comment lines
(( ${#toks[@]} == 0 )) && continue
[[ ${toks[0]:0:1} == '#' ]] && continue
local -a deps=( ${toks[*]:2} )
bash_expand "${toks[0]} ${toks[1]}" "${deps[@]}"
done < "${input}"
}
local name="$1"
local version="$2"
local exact_match='no'
if [[ "${version:0:1}" == "=" ]]; then
exact_match='yes'
version="${version:1}"
fi
shift 2
local with_modules=( $* )
# if we have to build a dependency, we might have less dependencies
# on it. Or in other words: the list of "with modules" might be
# overdetermined. In the loop below we check, which dependencies
# specified with '--with' are required.
local m
local pattern="/^${name}\/${version}[[:blank:]]/"
for m in "${with_modules[@]}"; do
if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then
pattern+=" && /${m//\//\\/}/"
fi
done
local variants=()
for f in "${files[@]}"; do
local line=''
while read line; do
variants+=( "${line}" )
done < <(expand_variants_file "${f}" | ${awk} "${pattern}")
done
if (( ${#variants[@]} == 0 )); then
std::info "%s " \
"${name}/${version}:" \
"no suitable variant found!"
std::die 10 "Aborting..."
elif (( ${#variants[@]} > 1 )) && [[ ${exact_match} == 'yes' ]]; then
std::info "%s " \
"Multiple variants found:"
for variant in "${variants[@]}"; do
std::info "${variant}"
done
std::die 10 "Aborting..."
fi
[[ -v OverlayInfo[${opt_overlay}:inst_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${opt_overlay}"
local ol_name="${opt_overlay}"
declare ol_inst_root="${OverlayInfo[${opt_overlay}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${opt_overlay}:mod_root]}"
local -i i=0
local -i num_variants=${#variants[@]}
for ((i = 0; i < num_variants; i++)); do
local tokens=( ${variants[i]} )
local name="${tokens[0]%/*}"
version="${tokens[0]#*/}"
release="${tokens[1]}"
case ${release} in
unstable|stable|deprecated|remove|removed)
:
;;
* )
std::info "%s " \
"${name}/${version}:" \
"invalid release stage '${release}'!"
std::die 10 "Aborting..."
;;
esac
with_modules=( "${tokens[@]:2}" )
pbuild.build_module_legacy \
"${name}" "${version}" \
"${release}" "${with_modules[@]}"
done
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
[[ -r "${yaml_config_file}" ]] || \
std::die 2 \
"%s -- %s" \
"YAML config file doesn't exist or is not readable" \
"${yaml_config_file}"
std::info "Using YAML configuration file - ${yaml_config_file}"
}
get_yaml_file_fmt(){
@@ -560,23 +371,23 @@ read_yaml_config_file() {
echo "${yaml}"
}
build_modules_yaml(){
build_modules(){
local -- name="$1"
local -- version="$2"
shift 2
local -a with_modules=( $* )
local -a with_modules=( "$*" )
if [[ "${opt_check_mode}" == 'yes' ]]; then
local -- yamllint
if ! which yamllint > /dev/null 2>&1; then
eval $( "${MODULECMD}" bash load yamllint/1.28.0 )
eval $( "${modulecmd}" bash load yamllint/1.28.0 )
fi
which yamllint > /dev/null 2>&1 || \
std::die 3 "yamllint not found"
yamllint "${yaml_config_file}"
fi
local -- file_fmt=$(get_yaml_file_fmt "${yaml_config_file}")
local -- file_fmt=''
file_fmt=$(get_yaml_file_fmt "${yaml_config_file}")
local -- yaml_mod_config=''
yaml_mod_config=$(read_yaml_config_file "${yaml_config_file}" "${name}")
@@ -601,17 +412,17 @@ build_modules_yaml(){
# none
#
init_module_environment(){
eval $( "${MODULECMD}" bash use unstable )
eval $( "${MODULECMD}" bash use deprecated )
eval $( "${MODULECMD}" bash purge )
eval $( "${modulecmd}" bash use unstable )
eval $( "${modulecmd}" bash use deprecated )
eval $( "${modulecmd}" bash purge )
# :FIXME: this is a hack!!!
# shouldn't this be set in the build-script?
if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then
eval $( "${MODULECMD}" bash use Libraries )
eval $( "${modulecmd}" bash use Libraries )
fi
if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then
eval $( "${MODULECMD}" bash use System )
eval $( "${modulecmd}" bash use System )
fi
unset C_INCLUDE_PATH
unset CPLUS_INCLUDE_PATH
@@ -636,7 +447,7 @@ init_module_environment(){
parse_version() {
local v="$1"
V="$1"
: ${USE_FLAGS:=''} # architectures (or empty)
USE_FLAGS=${USE_FLAGS:-''}
local tmp=''
@@ -667,16 +478,17 @@ parse_version() {
VERSIONS=()
if [[ -n ${V_RELEASE} ]]; then
VERSIONS+=( ${V_PKG}-${V_RELEASE} )
VERSIONS+=( "${V_PKG}-${V_RELEASE}" )
fi
if [[ -n ${V_PATCHLVL} ]]; then
VERSIONS+=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} )
VERSIONS+=( "${V_MAJOR}.${V_MINOR}.${V_PATCHLVL}" )
fi
if [[ -n ${V_MINOR} ]]; then
VERSIONS+=( ${V_MAJOR}.${V_MINOR} )
VERSIONS+=( "${V_MAJOR}.${V_MINOR}" )
fi
VERSIONS+=( ${V_MAJOR} )
VERSIONS+=( "${V_MAJOR}" )
}
# these variables must be export for envsubst(1)
declare -x P=''
declare -x V=''
@@ -763,7 +575,7 @@ build_modules_yaml_v1(){
local -- name="$2"
local -- version="$3"
shift 3
local -a with_modules=( $* )
local -a with_modules=( "$@" )
check_yaml_keys(){
local -n valid_yaml_keys="$1"
@@ -771,8 +583,8 @@ build_modules_yaml_v1(){
used_yaml_keys=()
local -- key=''
local -a keys=()
keys=( $(${yq} '.|keys().[]' 2>/dev/null) )
debug "top-level keys: ${keys[@]}"
readarray -t keys < <( ${yq} '.|keys().[]' 2>/dev/null)
debug "top-level keys: ${keys[*]}"
for key in "${keys[@]}"; do
[[ -v valid_yaml_keys[${key}] ]] || \
std::die 3 "Invalid key in YAML configuration file -- ${key}"
@@ -780,37 +592,6 @@ build_modules_yaml_v1(){
done
}
get_value(){
local -- yaml_input="$1"
local -n val="$2"
local -- key="$3"
local -- expected_type="$4"
local -- type=''
type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type}" != "${expected_type}" ]]; then
std::die 3 "%s" \
"Value of '${key}' must be of type '${expected_type:2}', but is '${type:2}'!"
fi
val=$( ${yq} -e ".${key}" \
2>/dev/null <<<"${yaml_input}" ) || val=''
}
get_seq(){
local -- yaml_input="$1"
local -n val="$2"
local -- key="$3"
local -- type=''
type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type}" != '!!seq' ]]; then
std::die 3 "%s" \
"Value of '${key}' must be of type 'seq', but is of type '${type:2}'!"
fi
val=$( ${yq} -e ".${key}[]" \
2>/dev/null <<<"${yaml_input}" ) || val=''
}
get_config(){
local -- yaml_input="$1"
local -n cfg="$2" # ref. to return configuration
@@ -827,9 +608,9 @@ build_modules_yaml_v1(){
fi
local -a keys=()
keys=( $( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null )) || \
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null ) || \
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
debug "config keys: ${keys[@]}"
debug "config keys: ${keys[*]}"
for key in "${keys[@]}"; do
[[ -v dfl[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
@@ -837,7 +618,7 @@ build_modules_yaml_v1(){
"${key}" "${yaml_input}"
case ${key} in
compile_in_sourcetree )
get_value "${yaml_input}" value "${key}" '!!bool'
pm::get_value "${yaml_input}" value "${key}" '!!bool'
case ${value,,} in
true )
cfg[${key,,}]='yes'
@@ -848,13 +629,13 @@ build_modules_yaml_v1(){
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'${key}' \
"${key}" \
"${value}"
;;
esac
;;
configure_with )
get_value "${yaml_input}" value "${key}" '!!str'
pm::get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
auto | cmake | autotools )
cfg[${key,,}]="${value,,}"
@@ -868,15 +649,15 @@ build_modules_yaml_v1(){
esac
;;
default_variant | download_dir | group | overlay | script | suffix )
get_value "${yaml_input}" value "${key}" '!!str'
pm::get_value "${yaml_input}" value "${key}" '!!str'
cfg[${key,,}]="${value}"
;;
group_deps )
get_value "${yaml_input}" value "${key}" '!!map'
pm::get_value "${yaml_input}" value "${key}" '!!map'
cfg[${key,,}]="${value}"
;;
relstage )
get_value "${yaml_input}" value "${key}" '!!str'
pm::get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
unstable | stable | deprecated )
cfg[${key,,}]="${value,,}"
@@ -893,15 +674,15 @@ build_modules_yaml_v1(){
esac
;;
urls | sub_packages )
get_value "${yaml_input}" value "${key}" '!!seq'
pm::get_value "${yaml_input}" value "${key}" '!!seq'
cfg[${key,,}]="${value}"
;;
build_requires|configure_args|docfiles|patch_files|runtime_deps|systems|target_cpus|variant )
get_seq "${yaml_input}" value "${key}"
build_requires|configure_args|docfiles|patch_files|runtime_deps|systems|variant )
pm::get_seq "${yaml_input}" value "${key}"
cfg[${key,,}]="${value}"
;;
kernels )
get_seq "${yaml_input}" value "${key}"
pm::get_seq "${yaml_input}" value "${key}"
set -o noglob
local -a items=( "${value,,}" )
set +o noglob
@@ -917,7 +698,7 @@ build_modules_yaml_v1(){
cfg[${key,,}]="${value}"
;;
target_cpus )
get_seq "${yaml_input}" value "${key}"
pm::get_seq "${yaml_input}" value "${key}"
set -o noglob
local -a items=( "${value,,}" )
set +o noglob
@@ -933,7 +714,7 @@ build_modules_yaml_v1(){
cfg[${key,,}]="${value}"
;;
'configure_args+' | 'docfiles+' | 'patch_files+' )
get_seq "${yaml_input}" value "${key}"
pm::get_seq "${yaml_input}" value "${key}"
key="${key:0:-1}"
if [[ -z "${cfg[${key,,}]}" ]]; then
cfg[${key,,}]="${value}"
@@ -946,7 +727,7 @@ build_modules_yaml_v1(){
std::die 3 "%s '%s' in %s" \
"Oops unhandled key" \
"${key}" \
"${FUNCNAME}"
"${FUNCNAME[0]}"
esac
done
}
@@ -959,7 +740,7 @@ build_modules_yaml_v1(){
local version="$2"
local -a keys=()
keys=( $( ${yq} -e '.versions|keys().[]' 2>/dev/null) ) || \
readarray -t keys < <( ${yq} -e '.versions|keys().[]' 2>/dev/null ) || \
std::die 3 "No version keys in configuration file!"
refvar=()
@@ -969,7 +750,7 @@ build_modules_yaml_v1(){
for k in ${key//;/ }; do
# brace expansion of key
local list=()
eval list=( $k )
list=( $(${bash} -c "echo $k") )
if [[ ${list[@]} =~ ${version} ]]; then
refvar+=("${key}")
break
@@ -1000,7 +781,8 @@ build_modules_yaml_v1(){
}
get_num_variants(){
local -i n=$(${yq} '.|length' 2>/dev/null)
local -i n=0
n=$(${yq} '.|length' 2>/dev/null)
echo "$n"
}
@@ -1027,7 +809,7 @@ build_modules_yaml_v1(){
# query all specified group dependencies
local -a keys=()
keys=( $(${yq} ".|keys|.[]" <<<"${yaml}" 2>/dev/null) )
readarray -t keys < <( ${yq} ".|keys|.[]" <<<"${yaml}" 2>/dev/null )
local -- key=''
for key in "${keys[@]}"; do
@@ -1039,7 +821,7 @@ build_modules_yaml_v1(){
die_invalid_group_dep "${name}" "${version}" "${group}"
done
# are all required group dependencies defined?
for key in "${hierarchical_groups[${group,,}]}"; do
for key in ${hierarchical_groups[${group,,}]}; do
is_in_array "${key,,}" "${keys[@]}" || \
die_missing_group_dep "${name}" "${version}" "${group}"
done
@@ -1052,11 +834,11 @@ build_modules_yaml_v1(){
local -a modules=()
local keys=()
keys=( $(${yq} ".${group,,}|keys|.[]" <<<"${yaml}" 2>/dev/null) )
readarray -t keys < <( ${yq} ".${group,,}|keys|.[]" <<<"${yaml}" 2>/dev/null )
local key
for key in "${keys[@]}"; do
local versions=()
versions=( $( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml}" 2>/dev/null) )
readarray -t versions < <( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml}" 2>/dev/null )
local version
for version in "${versions[@]}"; do
if [[ -v opt_with_dict[${key}/${version}] ]]; then
@@ -1165,8 +947,8 @@ build_modules_yaml_v1(){
&& continue
debug "build $module_name/$module_version with $compiler and $hdf5"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
@@ -1201,8 +983,8 @@ build_modules_yaml_v1(){
&& continue
debug "build $module_name/$module_version with $compiler and $mpi"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
@@ -1236,8 +1018,8 @@ build_modules_yaml_v1(){
for mpi in "${with_mpi[@]}"; do
for hdf5 in "${with_hdf5[@]}"; do
debug "build $module_name/$module_version with $compiler, $mpi and $hdf5"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
# build if opt_with_modules is empty or compiler is in this array
(( ${#opt_with_modules[@]} != 0 )) \
@@ -1300,7 +1082,7 @@ build_modules_yaml_v1(){
local unpacker=''
local key=''
local value=''
while read key value; do
while read -r key value; do
key=${key:0:-1}
case "${key}" in
url )
@@ -1359,6 +1141,12 @@ build_modules_yaml_v1(){
pbuild.add_patch_files "${args[@]}"
}
die_sub_package_name_missing(){
std::die 3 "Name of sub-package not specified in \n===\n$1\n===\n"
}
die_sub_package_version_missing(){
std::die 3 "Version of sub-package not specified in \n===\n$1\n===\n"
}
build_sub_packages(){
local -- yaml="$1"
local -i l=0
@@ -1374,20 +1162,20 @@ build_modules_yaml_v1(){
local -- pkg_version=''
local -a pkg_build_args=()
local -a keys=()
keys=( $( ${yq} -e ".|keys().[]" <<<"${pkgs_yaml}" 2>/dev/null )) || \
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${pkgs_yaml}" 2>/dev/null ) || \
die_parsing "${pkgs_yaml}"
local -- key=''
for key in "${keys[@]}"; do
case ${key,,} in
'name' )
get_value "${pkgs_yaml}" pkg_name "${key}" '!!str'
pm::get_value "${pkgs_yaml}" pkg_name "${key}" '!!str'
;;
'version' )
get_value "${pkgs_yaml}" pkg_version "${key}" '!!str'
pm::get_value "${pkgs_yaml}" pkg_version "${key}" '!!str'
;;
'build_args' )
local -- value=''
get_seq "${pkgs_yaml}" value "${key}"
pm::get_seq "${pkgs_yaml}" value "${key}"
readarray -t pkg_build_args <<< "${value}"
;;
* )
@@ -1398,6 +1186,11 @@ build_modules_yaml_v1(){
;;
esac
done
[[ -n "${pkg_name}" ]] || \
die_sub_package_name_missing "${pkgs_yaml}"
[[ -n "${pkg_version}" ]] || \
die_sub_package_version_missing "${pkgs_yaml}"
[[ "${opt_verbose}" == 'yes' ]] && \
pkg_build_args+=( '--verbose' )
[[ "${opt_debug}" == 'yes' ]] && \
@@ -1444,10 +1237,10 @@ build_modules_yaml_v1(){
local -- kernel=''
for kernel in "${kernels[@]}"; do
[[ ${kernel} == 'any' ]] && return 0
[[ ${kernel} == ${OS,,} ]] & return 0
[[ ${kernel} == ${KernelName,,} ]] & return 0
done
std::info "Skipping variant '${module_version}':"
std::info " The kernel of this systems is: ${OS}"
std::info " The kernel of this systems is: ${KernelName}"
std::info " But the variant is for the following kernels: ${module_config['kernels']}"
return 1
}
@@ -1489,11 +1282,11 @@ build_modules_yaml_v1(){
debug "build variant ${module_name}/${module_version}"
local ol_name="${module_config['overlay']}"
[[ -v OverlayInfo[${ol_name}:inst_root] ]] || \
[[ -v OverlayInfo[${ol_name}:install_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${ol_name}"
declare ol_inst_root="${OverlayInfo[${ol_name}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${ol_name}:mod_root]}"
declare ol_install_root="${OverlayInfo[${ol_name}:install_root]}"
declare ol_modulefiles_root="${OverlayInfo[${ol_name}:modulefiles_root]}"
module_version+="${module_config['suffix']}"
@@ -1554,7 +1347,7 @@ build_modules_yaml_v1(){
# do curly brackets expansion {}
local l
local list=()
eval list=( $k )
list=( $(${bash} -c "echo $k" ) )
for l in "${list[@]}"; do
if [[ $l =~ ${version} ]]; then
ev_result+=("${l}")
@@ -1580,15 +1373,16 @@ build_modules_yaml_v1(){
get_config "${yaml_input}" default_config Yaml_default_config
if [[ -v used_keys['shasums'] ]]; then
local yaml_input=$(${yq} '.shasums' <<<"${yaml_mod_config}" 2>/dev/null)
while read key value; do
local yaml_input=''
yaml_input=$(${yq} '.shasums' <<<"${yaml_mod_config}" 2>/dev/null)
while read -r key value; do
[[ -z ${key} ]] && continue
SHASUMS[${key//:}]="${value}"
done <<<"${yaml_input}"
fi
if [[ -v used_keys['type'] ]]; then
local -- value=''
get_value "${yaml_mod_config}" value 'type' '!!str'
pm::get_value "${yaml_mod_config}" value 'type' '!!str'
case "${value,,}" in
'module' )
[[ "${module_type}" == 'sub_package' ]] && \
@@ -1605,7 +1399,8 @@ build_modules_yaml_v1(){
fi
get_matching_version_keys version_keys "${version}" <<<"${yaml_mod_config}"
for version_key in "${version_keys[@]}"; do
local yaml_vk_config=$(get_yaml_vk_config "${version_key}" <<<"${yaml_mod_config}")
local yaml_vk_config=''
yaml_vk_config=$(get_yaml_vk_config "${version_key}" <<<"${yaml_mod_config}")
# check keys: allowed are 'config' and 'variants'
used_keys=()
@@ -1621,8 +1416,10 @@ build_modules_yaml_v1(){
# reminder: if YAML input is empty, next line copies defaults to 'vk_config'
get_config "${yaml_input}" vk_config default_config
local -- yaml_variants=$(get_variants <<<"${yaml_vk_config}")
local -i num_variants=$(get_num_variants <<<"${yaml_variants}")
local -- yaml_variants=''
yaml_variants=$(get_variants <<<"${yaml_vk_config}")
local -i num_variants=0
num_variants=$(get_num_variants <<<"${yaml_variants}")
local versions=()
expand_version_key versions "${version_key}" "${version}"
local v=''
@@ -1647,15 +1444,7 @@ build_modules_yaml_v1(){
fi
done
done
} # build_modules_yaml()
build_modules() {
if [[ "${opt_yaml}" == 'yes' ]]; then
build_modules_yaml "$@"
else
build_modules_legacy "$@"
fi
}
} # build_modules()
debug(){
${echo} "INFO: " "$@" 1>&2
@@ -1686,8 +1475,8 @@ pm::read_config
# or
# ${overlay}/var/distfiles
# ?
: ${PMODULES_DISTFILESDIR:="${PMODULES_HOME%%/Tools*}/var/distfiles"}
: ${PMODULES_TMPDIR:=/var/tmp/${USER}}
PMODULES_DISTFILESDIR=${PMODULES_DISTFILESDIR:-"${PMODULES_HOME%%/Tools*}/var/distfiles"}
PMODULES_TMPDIR="${PMODULES_TMPDIR:-/var/tmp/${USER}}"
export PMODULES_DISTFILESDIR PMODULES_TMPDIR
declare -r BUILD_SCRIPT
Pmodules/modmanage.bash.in
-747
View File
@@ -1,747 +0,0 @@
#!@BASH@ --noprofile
PATH='/bin:/usr/bin'
unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy!
unset IFS # use default IFS
shopt -s nullglob
shopt -s extglob
# used for some output only
declare -r CMD='modmanage'
declare mydir=$(cd $(dirname "$0") && pwd)
declare prefix=$(dirname "${mydir}")
declare libdir="${prefix}/lib"
declare libexecdir="${prefix}/libexec"
declare -r bindir="${prefix}/bin"
source "${libdir}/libstd.bash"
source "${libdir}/libpmodules.bash"
_exit () {
std::die 1 "\nInterrupted..."
}
trap '_exit' INT TERM
_err () {
std::info "\nOops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
std::die 1 "Aborting ..."
}
trap '_err' ERR
path="/bin:/usr/bin:${bindir}"
[[ $(uname -s) == 'Darwin' ]] && path+=":${libexecdir}"
std::def_cmds "${path}" 'chown' 'dirname' 'mkdir' 'rsync' 'rm' 'getopt' 'find' 'modulecmd'
unset mydir
unset prefix
unset libdir
unset libexecdir
# bindir we still need
declare PMODULES_VERSION='@PMODULES_VERSION@'
##############################################################################
#
# help [module|sub-command]
#
Subcommands[help]='help'
Options[help]='-o hHV\? -l version -l help'
Help[help]='
USAGE:
modmanage [switches] subcommand [subcommand-args]...
SWITCHES:
-h|-H|-?|--help this usage info
-V|--version modules version & configuration options
--debug enable debug output
--dry-run dry run
SUBCOMMANDS:
+ init [switches] TARGET_DIR
+ install [switches] module...
+ search [switches] [string|pattern]...
+ help [subcommand]
'
subcommand_help() {
local -r subcommand='help'
local -a args=()
while (( $# > 0 )); do
case $1 in
-h | -\? | -H | --help )
print_help "${subcommand}"
;;
-V | --version )
print_help 'version'
;;
-- )
shift 1
args+=( "$@" )
break
;;
* )
args+=( "$1" )
;;
esac
shift
done
for arg in "${args[@]}"; do
if [[ -n "${Help[${arg}]}" ]] ; then
print_help "${arg}"
else
std::die 1 "Unknown sub-command -- ${subcommand}"
fi
done
}
##############################################################################
#
# Derive the module installation path from the modulefile path.
# The passed modulefile must be absolute.
#
# Arguments:
# $1: absolute module file path
#
get_module_prefix() {
"${modulecmd}" bash show "$1" 2>&1 \
|awk '/_HOME |_PREFIX / {print $3; exit}'
}
##############################################################################
#
# Derive the module release-file path from the module file-path.
#
# Arguments:
# $1: module file-path
#
get_releasefile_name() {
echo "$(${dirname} "$1")/.release-$(basename "$1")"
}
##############################################################################
#
# Sync a module from one Pmodules environment to another:
# - sync module installation
# - sync modulefile
# - sync release file
#
# Arguments:
# $1: relative modulefile path (something like: Tools/gnuplot/5.0.0)
# $2: source prefix of Pmodule environment
# $3: target prefix of Pmodule environment
#
sync_module() {
local -r rel_modulefile="$1"
local -r src_root="$2"
local -r target_root="$3"
local -r src_prefix=$( get_module_prefix "${src_root}/${rel_modulefile}" )
local -r rel_prefix=${src_prefix#${src_root}/}
local -r target_prefix="${target_root}/${rel_prefix}"
# install/update module
if [[ ! -d "${target_prefix}" ]] || [[ "${force}" == 'yes' ]]; then
${mkdir} -p "${target_prefix}" || exit $?
${rsync} --links --perms --recursive --delete \
"${src_prefix}/" \
"${target_prefix}/" || exit $?
fi
# create modulefile direcrory and install modulefile
local -r src_modulefile="${src_root}/${rel_modulefile}"
local -r target_modulefile="${target_root}/${rel_modulefile}"
${mkdir} -p "$(${dirname} "${target_modulefile}")" || exit $?
if [[ -e "${src_modulefile}" ]]; then
${rsync} --links --perms --recursive \
"${src_modulefile}" "${target_modulefile}" || exit $?
fi
# install release-file
local -r rel_releasefile=$( get_releasefile_name "${rel_modulefile}" )
local -r src_releasefile="${src_root}/${rel_releasefile}"
local -r target_releasefile="${target_root}/${rel_releasefile}"
if [[ -e "${src_releasefile}" ]]; then
${rsync} --links --perms --recursive \
"${src_releasefile}" "${target_releasefile}" || exit $?
fi
}
##############################################################################
#
# initialize a new module environment
#
#
#
Subcommands[init]='init'
Options[init]='-o \?hfy -l src: -l user: -l help -l force -l yes'
Help[init]="
USAGE:
modmanage init [switches] TARGET_DIR
Initialize a new minimal Pmodule environment in TARGET_DIR.
A user must be specified with '--user=<USER>' if the
programm is executed as root.
SWITCHES:
--user <USER>
If this scripts runs with root privileges, a user name
ore ID must be specified.
--force|--yes|-f|-y
re-initialise an already existing Pmodule environment.
"
subcommand_init() {
check_env() {
[[ -n "${PMODULES_ROOT}" ]] &&
[[ -d "${PMODULES_ROOT}" ]] &&
[[ -n "${PMODULES_HOME}" ]] &&
[[ -n "${PMODULES_VERSION}" ]] || \
std::die 1 "
Error: the module environment you are going to use as source has not been
initialized properly!"
}
#.....................................................................
#
# Sync the Pmodules configuration and templates
#
# Arguments:
# $1: source prefix of Pmodule environment
# $2: target prefix of Pmodule environment
#
sync_config() {
src="$1/${PMODULES_CONFIG_DIR}/"
dst="$2/${PMODULES_CONFIG_DIR}/"
${rsync} --links --perms \
"${src}"/profile.{bash,csh,zsh} "${dst}" \
|| return $?
${rsync} --links --perms \
"${src}"/profile.{bash,csh,zsh}-"${PMODULES_VERSION}" "${dst}" \
|| return $?
${rsync} --links --perms \
"${src}/Pmodules.conf" "${dst}" \
|| return $?
${rsync} --links --perms \
"${src}/modbuild.conf" "${dst}" \
|| return $?
}
local target_root=()
local user=''
while (($# > 0)); do
case $1 in
-h | -H | -\? | --help | -help )
print_help "${subcommand}"
;;
--force | --yes | -f | -y )
force='yes'
;;
--user | --user=* )
if [[ "$1" == '--user' ]]; then
user="$2"
shift
else
user="${1#--*=}"
fi
;;
-- )
:
;;
* )
# assign and remove trailing slashes
target_root="${1%%*([\/])}"
;;
esac
shift
done
if [[ -z ${target_root} ]]; then
std::die 1 "Error: no target directory specified!"
fi
local -i euid=$(id -u)
if (( euid == 0 )); then
[[ -n "${user}" ]] || \
std::die 1 "Error: --user parameter is required!"
id -u "${user}" > /dev/null 2>&1 || \
std::die 1 "Error: Unable to retrieve user id of user '${user}'"
else
[[ -z "${user}" ]] || \
std::die 1 "Error: --user option is only allowed if running as root!"
fi
local src_root="$(std::get_abspath "${bindir}/../../../..")"
local config_file="${src_root}/${PMODULES_CONFIG_DIR}/profile.bash"
if [[ -r "${config_file}" ]]; then
source "${config_file}"
fi
check_env || \
std::die 1 "Giving up..."
echo "Creating a minimal Pmodule environment from the environment at"
echo " ${PMODULES_ROOT}"
echo "in"
echo " ${target_root}"
if [[ -d "${target_root}" ]] && [[ ${force} == no ]]; then
echo "Warning: ${target_root} already exists."
std::get_YN_answer \
"Do you really want to re-run the initialization? (y/N) " \
|| std::die 1 "Abort ..."
fi
force='yes'
echo "Creating target directory '${target_root}'..."
${mkdir} -p "${target_root}" || \
std::die 1 "Error: make directory failed!"
echo "Syncing configuration ..."
sync_config "${src_root}" \
"${target_root}" || \
std::die 1 "Error: configuration synchronization failed!"
echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_root}' to '${target_root}'..."
sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \
"${src_root}" \
"${target_root}" || \
std::die 1 "Error: sync Pmodules failed!"
for d in "${src_root}"/*/${PMODULES_MODULEFILES_DIR}; do
${mkdir} -p "${target_root}/${d#${src_root}/}"
done
if [[ -n "${user}" ]]; then
echo "Changing user of new module environment to '${user}'..."
${chown} -R "${user}" "${target_root}" || \
std::die 1 "Error: changing owner failed!"
echo
fi
echo "SourceRoot=${src_root}" > "${target_root}/${PMODULES_CONFIG_DIR}/modmanage.conf"
echo "New minimal module environment created at '${target_root}'."
echo "To use this environment, execute"
echo " sudo ln -fs ${target_root} /opt/psi"
echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash"
}
##############################################################################
#
# sub-command 'install'
#
# Arguments:
Subcommands[install]='install'
Options[install]='-o hf -l force -l with: -l help -l src: -l target:'
Help[install]='
USAGE:
modmanage install [switches] <module>...
Install modules
SWITCHES:
--force] | -f
Install module even it already exists
--src <src>
Install from module environment in <src>
--with <module>
Install module(s) in this sub-group only
<module_pattern>
Install modules matching given pattern
'
subcommand_install() {
local -r subcommand='install'
local opts=''
local -a with=()
local -a module_pattern=()
local src_root="${PMODULES_INSTALL_SOURCE}"
local target_root="${PMODULES_ROOT}"
local modulefile=''
local -A modules_to_install
local -A dependencies_to_install
local -A group_map
local -a modulepath=()
#......................................................................
#
set_initial_modulepath() {
local group
for group in "${!GroupDepths[@]}"; do
(( ${GroupDepths[${group}]} == 0 )) || continue
modulepath+=( "${src_root}/${group}/${PMODULES_MODULEFILES_DIR}" )
done
}
#......................................................................
#
create_group_map() {
#
# For the dependency resolution we need to know, whether a
# module - if loaded - adds a hierarchical group to MODULEPATH
# or not.
#
# Examples:
# Loading a compiler adds the hierarchical group for
# this compiler. The command
# module load gcc/10.3.0
# prepends
# <pmodules_root>/Compiler/modulefiles/gcc/10.3.0
# to MODULEPATH.
#
# The dependency files do not convey the information whether
# loading a module extends MODULEPATH or not. What we need to
# know is
# 1) does loading a specific module extends MODULEPATH?
# 2) if yes: what is the hierarchical group?
#
# Example:
# If we know that loading 'gcc/10.3.0' adds a directory
# in the hierarchical group 'Compiler' to MODULEPATH, we
# know that this directory is
# <pmodules_root>/Compiler/modulefiles/gcc/10.3.0
#
# This information we store in the dictionary 'group_map'.
# For concinience reasons we store the string 'src_root/group'.
# So, 'group_map' maps
# module/version -> src_root/group
#
# Example:
# group_map[gcc/10.3.0]="${src_root}/Compiler"
#
local group=''
for group in "${!GroupDepths[@]}"; do
(( ${GroupDepths[${group}]} > 0 )) || continue
local fname=''
while read fname; do
local -a parts=()
std::split_relpath parts "${fname}"
if (( ${#parts[@]} - 2 != ${GroupDepths[${group}]} )); then
std::warn "error in source group ${group}:"
std::warn "modulefile: ${fname}"
continue
fi
if [[ ${#parts[@]} < 4 ]]; then
echo "${group} ${parts[@]}"
fi
local key="${parts[-4]}/${parts[-3]}"
[[ -z "${group_map[${key}]}" ]] || continue
group_map[${key}]="${src_root}/${group}"
done < <(${find} -L "${src_root}/${group}/${PMODULES_MODULEFILES_DIR}" \
\( -type l -o -type f \) \
\! -name ".*" \
-printf "%P\n" \
)
done
}
#......................................................................
#
# Resolve dependencies to given module
#
# Arguments:
# $1 absolute module file name
#
# Notes:
# Following variables from the enclosing function are used:
# modulepath
# group_map (read-only)
#
resolve_dependencies () {
local -r modulefile="$1"
local -- prefix=$(get_module_prefix "${modulefile}")
local -a rdeps=()
local -- rdeps_file="${prefix}/.dependencies"
local -a ideps=()
local -- ideps_file="${prefix}/.install_dependencies"
if [[ -r "${rdeps_file}" ]]; then
mapfile -t rdeps < <(grep -v '^ *#' "${rdeps_file}" )
fi
if [[ -r "${ideps_file}" ]]; then
mapfile -t ideps < <(grep -v '^ *#' "${ideps_file}" )
fi
# loop over all dependecies
local dep
for dep in "${rdeps[@]}" "${ideps}"; do
[[ -n ${dep} ]] || continue
# search module with current modulepath
local modulename=$(${find} "${modulepath[@]}" -path "*/${dep}" \
-print -quit 2>/dev/null)
[[ -n ${modulename} ]] || \
std::die 3 "Oops: required module '${dep}' not found!"
local rel_modulename="${modulename#${src_root}/}"
dependencies_to_install[${rel_modulename}]='.'
resolve_dependencies "${modulename}"
[[ -v group_map[${dep}] ]] || continue
local dir="${group_map[${dep}]}" # = ${src_root}/<group>
dir+="/${rel_modulename##+([!/])/}" # += rel.name with group removed
modulepath+=( "${dir}" )
done
}
#......................................................................
#
# Print list of modules which will be installed and ask user wheter
# he wants to continue or abort.
#
# Arguments:
# none
#
# Notes:
# Following variables from the enclosing function are used:
# target_root (read-only)
# modules_to_install (read-only)
# dependencies_to_install (read-only)
#
print_modules() {
local modulefile
for modulefile in "$@"; do
local -a parts
std::split_relpath parts "${modulefile}"
local s=''
if (( ${#parts[@]} >= 6 )); then
s="(${parts[2]}/${parts[3]}"
for ((i = 4; i < ${#parts[@]}-2; i+=2)); do
s+=" ${parts[i]}/${parts[i+1]}"
done
s+=')'
fi
std::info "%-20s %s" "${parts[-2]}/${parts[-1]}" "$s"
done 2>&1 | sort
}
print_modules_to_install() {
std::info "The following modules will be installed/updated:"
print_modules "${!modules_to_install[@]}"
if (( ${#dependencies_to_install[@]} > 0 )); then
std::info "\nThe following dependencies will be installed/updated:"
print_modules "${!dependencies_to_install[@]}"
fi
std::info ""
std::get_YN_answer "Do you want to continue? [n] " || \
std::die 1 "Aborting..."
std::info ""
}
while (($# > 0)); do
case $1 in
-h | -H | -\? | --help | -help )
print_help "${subcommand}"
;;
--force | -f )
force='yes'
;;
--src | --src=*)
if [[ $1 == --src ]]; then
src_root="$2"
shift
else
src_root="${1#--*=}"
fi
;;
--target | --target=*)
if [[ $1 == --target ]]; then
target_root="$2"
shift
else
target_root="${1#--*=}"
fi
;;
--with | --with=* )
if [[ "$1" == --with ]]; then
with+=( "$2" )
shift
else
with+=( "${1#--*=}" )
fi
;;
-- )
:
;;
* )
module_pattern+=( "$1" )
;;
esac
shift
done
if [[ -z ${src_root} ]]; then
local conf_file="${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}/modmanage.conf"
if [[ -r ${conf_file} ]]; then
source "${conf_file}"
src_root="${SourceRoot}"
fi
fi
[[ -n ${src_root} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_root} ]] \
|| std::die 3 "Oops: '${src_root}' is not a valid installation source."
source "${src_root}/${PMODULES_CONFIG_DIR}/profile.bash"
scan_groups "${src_root}"
set_initial_modulepath
create_group_map
# search for to be installed modules and their dependencies
while read modulefile; do
modules_to_install["${modulefile#${src_root}/}"]+='.'
resolve_dependencies "${modulefile}"
done < <("${modulecmd}" bash search \
"${module_pattern[@]}" \
"${with[@]/#/--with=}" \
-a --glob \
--no-header --print-modulefiles \
--src="${src_root}" 2>&1 1>/dev/null)
(( ${#modules_to_install[@]} == 0 )) && \
std::die 0 "No matching modules found ..."
print_modules_to_install
# install/update ...
for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do
std::split_relpath parts "${modulefile}"
std::info " ${parts[-2]}/${parts[-1]}"
sync_module "${modulefile}" "${src_root}" "${target_root}"
done
std::info "\nDone!\n"
} # subcommand_install
##############################################################################
#
# sub-command 'search'
#
Subcommands[search]='search'
Options[search]='-o \?h -l with: -l help -l all-dep -l wrap -l glob -l src:'
Help[install]='
USAGE:
modmanage search [switches] <string>...
search modules
SWITCHES:
--src <src>
Search modules in environment <src>.
Default is the source defined in modmanage.conf.
--with <module>
Search module(s) in this sub-group.
<string>
Search modules matching given string.
<pattern>
Search modules matching given shell glob-pattern.
'
subcommand_search() {
local -a args=()
while (($# > 0)); do
case $1 in
-h | -H | -\? | --help | -help )
print_help "${subcommand}"
;;
--src | --src=*)
if [[ $1 == --src ]]; then
src_root="$2"
shift
else
src_root="${1#--*=}"
fi
;;
--with | --with=* )
if [[ "$1" == --with ]]; then
args+=( '--with' "$2" )
shift
else
args+=( "$1" )
fi
;;
--all-deps | --glob | --wrap )
args+=( "$1" )
;;
-- )
:
;;
* )
args+=( "$1" )
;;
esac
shift
done
if [[ -z ${src_root} ]]; then
local conf_file="${PMODULES_ROOT}/${PMODULES_CONFIG_DIR}/modmanage.conf"
if [[ -r ${conf_file} ]]; then
source "${conf_file}"
src_root="${SourceRoot}"
fi
fi
[[ -n ${src_root} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_root} ]] \
|| std::die 3 "Oops: '${src_root}' is not a valid installation source."
${modulecmd} bash search --src="${src_root}" --all-release-stages \
"${args[@]}" 2>&1 1>/dev/null
}
declare force='no'
declare subcommand=''
declare -a opts=()
while (($# > 0)); do
case $1 in
-h | -H | -\? | --help | -help )
print_help 'help'
;;
-V | --version )
print_help 'version'
;;
--debug )
set -x
;;
--dry-run )
chown="echo ${chown}"
mkdir="echo ${mkdir}"
rsync="echo ${rsync}"
;;
-* )
opts+=( "$1" )
;;
* )
subcommand="$1"
shift
break
;;
esac
shift || :
done
if [[ -z "${subcommand}" ]]; then
std::die 1 "${CMD}: no sub-command specified.\n"
print_help 'help'
fi
if [[ -z "${Subcommands[${subcommand}]}" ]]; then
std::die 1 "${CMD}: unknown sub-command -- ${subcommand}\n"
fi
if [[ "${subcommand}" != "init" ]] && [[ -z "${PMODULES_ROOT}" ]]; then
std::die 1 "Error: No current module environment configured!"
fi
tmp=$("${getopt}" --name="${CMD}" ${Options[${subcommand}]} -- "${opts[@]}" "$@" ) \
|| print_help "${subcommand}"
eval args=( "$tmp" )
unset tmp
umask 022
subcommand_${Subcommands[$subcommand]} "${args[@]}"
# Local Variables:
# mode: sh
# sh-basic-offset: 8
# tab-width: 8
# End:
Pmodules/modmanage.in
-11
View File
@@ -1,11 +0,0 @@
#!/bin/sh
unset BASH_ENV
declare mydir=$(cd $(dirname "$0") && pwd)
declare libexecdir="$(dirname "${mydir}")/libexec"
declare bash="${libexecdir}/bash"
declare modmanage="${libexecdir}/modmanage.bash"
"${bash}" --noprofile --norc "${modmanage}" "$@"
Pmodules/modulecmd.bash.in
+472 -453
View File
File diff suppressed because it is too large Load Diff
build
+55 -9
View File
@@ -25,8 +25,6 @@ shopt -s nullglob
declare -r BOOTSTRAP_DIR="$(cd "$(dirname "$0")" && pwd -P)"
declare -r SRC_DIR="${BOOTSTRAP_DIR}/Pmodules"
source "${SRC_DIR}/libstd.bash" || { echo "Oops!" 1>&2; exit 42; }
declare -r PMOD_DIR="Tools/Pmodules/${VERSION}"
# config directory and file relative to install root
declare -rx CONFIG_DIR='config'
@@ -40,6 +38,61 @@ declare -rx DEFAULT_INSTALL_ROOT='/opt/psi'
declare -rx DEFAULT_DISTFILES_DIR='var/distfiles'
declare -rx DEFAULT_TMP_DIR='var/tmp/${USER}'
std::log() {
local -ri fd=$1
local -r fmt="$2"
shift 2
printf -- "${fmt}" "$@" 1>&$fd
printf -- "\n" 1>&$fd
}
std::info() {
std::log 2 "$1" "${@:2}"
}
std::error() {
std::log 2 "$1" "${@:2}"
}
std::debug() {
[[ -v PMODULES_DEBUG ]] || return 0
std::log 2 "$@"
}
std::die() {
local -ri ec=$1
shift
if [[ -n $@ ]]; then
local -r fmt=$1
shift
std::log 2 "$fmt" "$@"
fi
exit $ec
}
std::parse_yaml() {
#
# parse a YAML file
# See: https://gist.github.com/pkuczynski/8665367
#
local -r fname="$1"
local -r prefix="$2"
local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" "${fname}" |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {
if (i > indent) {delete vname[i]}
}
if (length($3) > 0) {
vn="";
for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
#-----------------------------------------------------------------------------
#
read_config_file() {
@@ -477,13 +530,6 @@ build::install() {
> "${PREFIX}/bin/modbuild"
chmod 0755 "${PREFIX}/bin/modbuild"
sed "${sed_cmd}" "${SRC_DIR}/modmanage.in" \
> "${PREFIX}/bin/modmanage"
chmod 0755 "${PREFIX}/bin/modmanage"
sed "${sed_cmd}" "${SRC_DIR}/modmanage.bash.in" \
> "${PREFIX}/libexec/modmanage.bash"
chmod 0755 "${PREFIX}/libexec/modmanage.bash"
test -e "${INSTALL_ROOT}/${CONFIG_FILE}" || \
install -m 0644 "$_" "${INSTALL_ROOT}/${CONFIG_DIR}"
config/Pmodules.yaml.in
-2
View File
@@ -1,9 +1,7 @@
DefaultGroups: Tools:Programming
DefaultReleaseStages: stable
ReleaseStages: unstable:stable:deprecated
TmpDir: @PMODULES_TMPDIR@
DistfilesDir: @PMODULES_DISTFILESDIR@
SysCollectionsDir: @INSTALL_ROOT@/collections
Overlays:
base:
install_root: @INSTALL_ROOT@
recipes/120-7z
+1 -1
View File
@@ -19,7 +19,7 @@ cd 'CPP/7zip/Bundles/Alone2'
#---
# compile & install
make -j -f ../../cmpl_gcc.mak
cp b/g/7zz "${PREFIX}/${UTILBIN_DIR}"
cp b/g/7zz "${PREFIX}/${UTILBIN_DIR}/sevenz"
#---
# post-install