pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-06-27 10:03:08 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
66b5dcd99d17ed6b1648e08fead4e477f244b712
Pmodules/Pmodules/modbuild.in
T

1289 lines
32 KiB
Plaintext
Executable File
Raw Blame History

#!@BASH@
#
# The following build specific variables are set and used in libpbuild.bash:
# ARGS
# BUILD_SCRIPT
# BUILDBLOCK_DIR
#
#.............................................................................
echo "$0 $@"
declare VERSION='@PMODULES_VERSION@'
# get absolute path of script
declare mydir=$(dirname "$0")
declare -r mydir=$(cd ${mydir} && pwd -P)
source "${mydir}/../lib/libstd.bash" || {
echo "Oops: cannot source library -- '$_'" 1>&2; exit 3;
}
##############################################################################
#
# check availability of used commands and set environment variables
# cmd=$(which cmd)
# in the following we use these environment variable to call binaries.
#
declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd"
[[ -x ${MODULECMD} ]] || \
std::die 1 "Oops: modulecmd binary not available!"
std::def_cmds "${mydir}/../libexec" \
'yq'
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'awk' 'base64' 'cat' 'cp' 'find' 'getopt' 'grep' \
'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \
'rm' 'rmdir' 'sort' 'tar' 'tee' 'uname'
declare -r OS="$(${uname} -s)"
if [[ ${OS} == 'Darwin' ]]; then
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \
'curl' 'otool' 'shasum' 'sysctl'
declare -r sha256sum="${shasum -a 256}"
else
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'ldd' 'curl' 'sha256sum'
fi
# for the time being, we still set PATH. Just in case we forgot a binary
PATH='/usr/bin:/bin:/usr/sbin:/sbin'
##############################################################################
source "${mydir}/../lib/libpbuild.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
source "${mydir}/../lib/libpmodules.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
##############################################################################
set -o nounset
shopt -s nocaseglob
shopt -s extglob
shopt -s nullglob
##############################################################################
#
usage() {
std::error "
USAGE:
$0 [options..] [build_script] version
MANDATORY ARGUMENTS:
version
Version of module to build.
SELECT VARIANT TO BUILD:
--system
Specify the system for selecting a variants. Defaults to the
OS version and release like 'rhel6'.
--with=P/V
Select variant to compile. Use multiple '--with' arguments
to make the selected variant unique.
BUILD-STEPS OPTIONS:
--prep
Prepare sources: unpack sources and apply patches only.
--configure
Prepare and configure sources.
--compile
Prepare, configure and compile everything.
--install
Prepare, configure and compile everything. Finally run install
step. Do not cleanup build and source directory.
--all
Run throu all steps including cleanup.
-update-modulefiles
Only install the modulefile and set the release.
MISCELLANEOUS OPTIONS:
-? | -h | --help
Print usage.
-V | --version )
Print version.
-v | --verbose )
Verbose output.
--debug )
Run in debug mode.
-j N | --jobs=N
Run N parallel make jobs.
-f | --force-rebuild
Force rebuild of module.
--dry-run
Dry run.
--disable-cleanup-build
--enable-cleanup-build
Cleanup files in the build directory. Default is to remove.
all files in the build-directory.
--disable-cleanup-src
--enable-cleanup-src
Cleanup files in the source directory. Default is to
remove all files in the source directory.
--disable-cleanup
--enable-cleanup
Cleanup all files in temporary directory. Default is to
remove all files created during building.
--distdir
Directory where to store and lookup downloaded files.
--tmpdir
Directory used for building a module.
--overlay
Install in this overlay. Defaults to '${PMODULES_HOME%%/Tools*}'.
"
exit 1
}
##############################################################################
#
# parse options and arguments
#
# command line arguments are taken first
# then configuration file
# last default
# save arguments, required for building dependencies
declare -r ARGS="$@"
# versions to be build, '.*' or none means all
declare -a versions=()
declare opt_build_target='all'
declare opt_dry_run='no'
declare opt_enable_cleanup_build='yes'
declare opt_enable_cleanup_src='yes'
declare opt_force_rebuild='no'
declare -i opt_jobs=0
declare opt_update_modulefiles='no'
declare opt_system=''
declare opt_overlay=''
declare opt_verbose='no'
# array collecting all modules specified on the command line via '--with=module'
declare -a opt_with_modules=() # :FIXME: legacy build should also use the dict
declare -A opt_with_dict=()
declare -- opt_config_file=''
declare -- opt_debug='no'
declare -- opt_yaml='yes'
declare -- opt_variant=''
declare -- BUILD_SCRIPT=''
declare -- yaml_config_file=''
declare -a legacy_config_files=()
declare -- module_name=''
parse_args() {
#
# The first argument ($1) is the build-script, if called in the
# usual way:
# ./<build-script> <version> [options]
# If called via:
# modbuild <build-script> [options]
# the build-script MUST be passed as first argument.
#
BUILD_SCRIPT=$(std::get_abspath "$1")
test -r ${BUILD_SCRIPT} || \
std::die 1 "%s " \
"Build script does not exist" \
"or is not readable -- '$_'"
BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}")
shift 1
while (( $# > 0 )); do
case $1 in
-j )
opt_jobs="$2"
shift
;;
--jobs=[0-9]* )
opt_jobs="${1/--jobs=}"
;;
-v | --verbose )
trap 'echo "$BASH_COMMAND"' DEBUG
opt_verbose='yes'
set -x
;;
--debug )
opt_debug='yes'
;;
-f | --force-rebuild )
opt_force_rebuild='yes'
;;
-\? | -h | --help )
usage
;;
-V | --version )
std::die 0 "\nPmodules version ${VERSION}\nCopyright GNU GPL v2\n"
;;
--dry-run )
opt_dry_run='yes'
;;
--enable-cleanup )
opt_enable_cleanup_build='yes'
opt_enable_cleanup_src='yes'
;;
--disable-cleanup )
opt_enable_cleanup_build='no'
opt_enable_cleanup_src='no'
;;
--enable-cleanup-build )
opt_enable_cleanup_build='yes'
;;
--disable-cleanup-build )
opt_enable_cleanup_build='no'
;;
--enable-cleanup-src )
opt_enable_cleanup_src='yes'
;;
--disable-cleanup-src )
opt_enable_cleanup_src='no'
;;
--distdir | --distdir=* )
if [[ $1 == *=* ]]; then
PMODULES_DISTFILESDIR="${1/--distdir=}"
else
PMODULES_DISTFILESDIR="$2"
shift
fi
;;
--tmpdir | --tmpdir=* )
if [[ $1 == *=* ]]; then
PMODULES_TMPDIR="${1#--*=}"
else
PMODULES_TMPDIR="$2"
shift
fi
;;
--system | --system=* )
if [[ $1 == *=* ]]; then
opt_system="${1#--*=}"
else
opt_system="$2"
shift
fi
;;
--overlay | --overlay=* )
if [[ $1 == *=* ]]; then
opt_overlay="${1#--*=}"
else
opt_overlay="$2"
shift
fi
;;
--yaml )
opt_yaml='yes'
;;
--legacy )
opt_yaml='no'
;;
--use-flags | --use-flags=* )
if [[ $1 == *=* ]]; then
USE_FLAGS=":${1#--*=}:"
else
USE_FLAGS=":$2:"
shift
fi
;;
--with | --with=*/* )
if [[ $1 == *=* ]]; then
opt_with_modules+=( "${1#--*=}" )
opt_with_dict[${1#--*=}]=0
else
opt_with_modules+=( "$2" )
opt_with_dict[$2]=0
shift
fi
;;
--prep | --configure | --compile | --install | --all )
opt_build_target=${1:2}
;;
--update-modulefiles )
opt_update_modulefiles='yes'
;;
--config-file | --config-file=* )
if [[ $1 == *=* ]]; then
opt_config_file=( "${1#--*=}" )
else
opt_config_file=( "$2" )
shift
fi
;;
--variant | --variant=* )
if [[ $1 == *=* ]]; then
opt_variant=( "${1#--*=}" )
else
opt_variant=( "$2" )
shift
fi
;;
-- | '' )
:
;;
-* )
std::die 1 "Invalid option -- '$1'"
;;
* )
local -- arg="$1"
if [[ "${arg}" == */* ]]; then
module_name="${arg%/*}"
versions+=( {arg#*/} )
else
versions+=( "$1" )
fi
;;
esac
shift
done
# if no version is specified on the cmd-line, build all versions
(( ${#versions[@]} > 0)) || versions+=( '.*' )
# set system if not set on the cmd-line
opt_system="${opt_system:-$(std::get_os_release)}"
# set config file
if [[ ${opt_yaml,,} != 'yes' ]]; then
# look for legacy config files
if [[ -n ${opt_config_file} ]]; then
legacy_config_files=( "${opt_config_file}" )
else
shopt -q nullglob || :
local -i nullglob_set=$?
legacy_config_files=( "${BUILDBLOCK_DIR}"/*/variants.${opt_system} )
legacy_config_files+=( "${BUILDBLOCK_DIR}"/*/variants."$(uname -s)" )
legacy_config_files+=( "${BUILDBLOCK_DIR}"/*/variants )
(( nullglob_set == 1 )) && shopt -u nullglob
fi
(( ${#legacy_config_files[@]} == 0 )) && \
std::die 1 "No legacy configuration file found!"
std::info "Using legacy variants files."
opt_overlay="${opt_overlay:-'base'}"
else
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
[[ -r "${yaml_config_file}" ]] || \
std::die 2 \
"%s -- %s" \
"YAML config file doesn't exist or is not readable" \
"${yaml_config_file}"
std::info "Using YAML configuration file - ${yaml_config_file}"
fi
}
#
# bash brace expansion of given args. Input args like:
#
# "text" "gcc/{9.3.0,10.3.0}" "openmpi/{4.0.5,4.1.0}"
#
# will be expanded to the following four lines:
#
# "text gcc/9.3.0 openmpi/4.0.5"
# "text gcc/9.3.0 openmpi/4.1.0"
# "text gcc/10.3.0 openmpi/4.0.5"
# "text gcc/10.3.0 openmpi/4.1.0"
#
bash_expand(){
local text="$1"
shift
local to_expand=( "${@}" )
if (( ${#to_expand[@]} == 0 )); then
echo ${text}
else
local list
eval list=( ${to_expand[0]} )
local s
for s in ${list[*]}; do
bash_expand "${text} ${s}" "${to_expand[@]:1}"
done;
fi;
}
build_modules_legacy() {
local -a files=( "${legacy_config_files[@]}" )
local -A mod_overlays=()
expand_variants_file(){
local -r input="$1"
local -a toks=()
while read -a toks; do
# skip empty and comment lines
(( ${#toks[@]} == 0 )) && continue
[[ ${toks[0]:0:1} == '#' ]] && continue
local -a deps=( ${toks[*]:2} )
bash_expand "${toks[0]} ${toks[1]}" "${deps[@]}"
done < "${input}"
}
local name="$1"
local version="$2"
local exact_match='no'
if [[ "${version:0:1}" == "=" ]]; then
exact_match='yes'
version="${version:1}"
fi
shift 2
local with_modules=( $* )
# if we have to build a dependency, we might have less dependencies
# on it. Or in other words: the list of "with modules" might be
# overdetermined. In the loop below we check, which dependencies
# specified with '--with' are required.
local m
local pattern="/^${name}\/${version}[[:blank:]]/"
for m in "${with_modules[@]}"; do
if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then
pattern+=" && /${m//\//\\/}/"
fi
done
local variants=()
for f in "${files[@]}"; do
local line=''
while read line; do
variants+=( "${line}" )
done < <(expand_variants_file "${f}" | ${awk} "${pattern}")
done
if (( ${#variants[@]} == 0 )); then
std::info "%s " \
"${name}/${version}:" \
"no suitable variant found!"
std::die 10 "Aborting..."
elif (( ${#variants[@]} > 1 )) && [[ ${exact_match} == 'yes' ]]; then
std::info "%s " \
"Multiple variants found:"
for variant in "${variants[@]}"; do
std::info "${variant}"
done
std::die 10 "Aborting..."
fi
[[ -v OverlayInfo[${opt_overlay}:inst_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${opt_overlay}"
local ol_name="${opt_overlay}"
declare ol_inst_root="${OverlayInfo[${opt_overlay}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${opt_overlay}:mod_root]}"
local -i i=0
local -i num_variants=${#variants[@]}
for ((i = 0; i < num_variants; i++)); do
local tokens=( ${variants[i]} )
local name="${tokens[0]%/*}"
version="${tokens[0]#*/}"
release="${tokens[1]}"
case ${release} in
unstable|stable|deprecated|remove|removed)
:
;;
* )
std::info "%s " \
"${name}/${version}:" \
"invalid release stage '${release}'!"
std::die 10 "Aborting..."
;;
esac
with_modules=( "${tokens[@]:2}" )
pbuild.build_module_legacy \
"${name}" "${version}" \
"${release}" "${with_modules[@]}"
done
}
get_yaml_file_fmt(){
: "
Get format version of configuration file. Print the version number
to stdout if it is valid.
"
${yq} -e '.' "$1" &>/dev/null || \
std::die 3 "%s -- %s" \
"Error in YAML config file, please check with linter" \
"$1"
local -- fmt=''
fmt=$(${yq} -e ".format" "$1") || \
std::die 3 "Error reading config file format -- $1"
case "${fmt}" in
1 )
:
;;
* )
std::die 3 "Unknown YAML Pmodules config file format -- $1"
;;
esac
echo "${fmt}"
}
read_yaml_config_file() {
: "
Test whether the configuration file '$1' provides configurations
for the module '$2'. If yes, the YAML block with the configuration
is printed to stdout.
"
local -- file_name="$1"
local -- module_name="$2"
local -- yaml=''
yaml=$( ${yq} -Ne e ".${module_name}" "${file_name}" 2>/dev/null ) || \
std::die 3 "Configuration for '${module_name}' missing -- ${file_name}"
echo "${yaml}"
}
build_modules_yaml(){
local -- name="$1"
local -- version="$2"
shift 2
local -a with_modules=( $* )
local -- file_fmt=$(get_yaml_file_fmt "${yaml_config_file}")
local -- yaml_mod_config=''
yaml_mod_config=$(read_yaml_config_file "${yaml_config_file}" "${name}")
case "${file_fmt}" in
1 )
build_modules_yaml_v1 \
"${yaml_mod_config}" \
"${name}" "${version}" \
"${with_modules[@]}"
;;
* )
std::die 255 "Oops!" # we should never be here!
;;
esac
}
# these variables must be export for envsubst(1)
declare -x P=''
declare -x V=''
declare -x V_PKG=''
declare -x V_MAJOR='' # first number in version string
declare -x V_MINOR='' # second number in version string (or empty)
declare -x V_PATCHLVL='' # third number in version string (or empty)
declare -x V_RELEASE='' # module release (or empty)
declare -A SHASUMS=()
declare -a MODULE_DOCFILES=()
parse_version() {
local v="$1"
V="$1"
: ${USE_FLAGS:=''} # architectures (or empty)
local tmp=''
if [[ "$v" =~ "_" ]]; then
tmp="${v#*_}"
USE_FLAGS+=":${tmp//_/:}:"
v="${v%%_*}"
fi
V_PKG="${v%%-*}" # version without the release number
if [[ $v == *-* ]]; then
V_RELEASE="${v#*-}" # release number
fi
case "${V_PKG}" in
*.*.* )
V_MAJOR="${V_PKG%%.*}"
tmp="${V_PKG#*.}"
V_MINOR="${tmp%%.*}"
V_PATCHLVL="${tmp#*.}"
;;
*.* )
V_MAJOR="${V_PKG%.*}"
V_MINOR="${V_PKG#*.}"
;;
* )
V_MAJOR="${V_PKG}"
;;
esac
VERSIONS=()
if [[ -n ${V_RELEASE} ]]; then
VERSIONS+=( ${V_PKG}-${V_RELEASE} )
fi
if [[ -n ${V_PATCHLVL} ]]; then
VERSIONS+=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} )
fi
if [[ -n ${V_MINOR} ]]; then
VERSIONS+=( ${V_MAJOR}.${V_MINOR} )
fi
VERSIONS+=( ${V_MAJOR} )
}
declare -A Yaml_valid_keys_for_module=(
['defaults']=1 # !!map
['shasums']=1 # !!map
['versions']=1 # !!int
)
declare -A Yaml_default_config=(
["build_requires"]='' # !!seq of strings
["compile_in_sourcetree"]='No' # !!str
["configure_with"]='auto' # !!str
["default_variant"]='' # !!str
["docfiles"]='' # !!seq of strings
["group"]='Tools' # !!str
["group_deps"]='' # !!map
["overlay"]='base' # !!str
["relstage"]='unstable' # !!str
["runtime_deps"]='' # !!seq of strings
["script"]='build' # !!str
["suffix"]='' # !!str
["systems"]='' # !!seq of strings
["urls"]='' # !!map
["variant"]='' # !!str
)
declare -A Yaml_valid_vk_keys=(
['config']=1 # !!map
['variants']=1 # !!map
)
build_modules_yaml_v1(){
: "
"
local -- yaml_mod_config="$1"
local -- name="$2"
local -- version="$3"
shift 3
local -a with_modules=( $* )
check_yaml_keys(){
local -n valid_yaml_keys="$1"
local -n used_yaml_keys="$2"
used_yaml_keys=()
local -- key=''
local -a keys=()
keys=( $(${yq} '.|keys().[]' 2>/dev/null) )
debug "top-level keys: ${keys[@]}"
for key in "${keys[@]}"; do
[[ -v valid_yaml_keys[${key}] ]] || \
std::die 3 "Invalid key in YAML configuration file -- ${key}"
used_yaml_keys[${key}]=1
done
}
get_value(){
local -- yaml_input="$1"
local -n val="$2"
local -- key="$3"
local -- expected_type="$4"
local -- type=''
type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type}" != "${expected_type}" ]]; then
std::die 3 "%s" \
"Value of '${key}' must be of type '${expected_type:2}', but is '${type:2}'!"
fi
val=$( ${yq} -e ".${key}" \
2>/dev/null <<<"${yaml_input}" ) || val=''
}
get_config(){
local -- yaml_input="$1"
local -n cfg="$2" # ref. to return configuration
local -n dfl="$3" # ref. to defaults
local -- key
local -- value
for key in "${!dfl[@]}"; do
cfg[${key}]="${dfl[${key}]}"
done
if [[ -z "${yaml_input}" ]]; then
return 0
fi
local -a keys=()
keys=( $( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null )) || \
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
debug "config keys: ${keys[@]}"
local -- type;
for key in "${keys[@]}"; do
[[ -v dfl[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key} in
compile_in_sourcetree )
get_value "${yaml_input}" value "${key}" '!!bool'
case ${value,,} in
true )
cfg[${key,,}]='Yes'
;;
false )
cfg[${key,,}]='No'
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'${key}' \
"${value}"
;;
esac
;;
configure_with )
get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
auto | cmake | autotools )
cfg[${key,,}]="${value,,}"
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'configure_with' \
"${value}"
;;
esac
;;
relstage )
get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
unstable | stable | deprecated )
cfg[${key,,}]="${value,,}"
;;
remove | removed )
cfg[${key,,}]='remove'
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'relstage' \
"${value}"
;;
esac
;;
urls )
get_value "${yaml_input}" value "${key}" '!!seq'
cfg[${key,,}]="${value}"
;;
group_deps )
get_value "${yaml_input}" value "${key}" '!!map'
cfg[${key,,}]="${value}"
;;
systems )
get_value "${yaml_input}" value "${key}" '!!seq'
readarray -t tmp <<<"${value}"
cfg[${key,,}]="${tmp[@]}"
;;
variant )
get_value "${yaml_input}" value "${key}" '!!seq'
readarray -t tmp <<<"${value}"
printf -v tmp2 ":%s:" "${tmp[@]}"
cfg[${key,,}]="${tmp2}"
:
;;
docfiles | runtime_deps | build_requires )
get_value "${yaml_input}" value "${key}" '!!seq'
cfg[${key,,}]="${value}"
;;
* )
value=$( ${yq} -e ".${key}" \
2>/dev/null <<<"${yaml_input}" ) || \
value=''
cfg[${key,,}]="${value}"
esac
done
}
get_matching_version_keys(){
: "
return list of version keys in refvar $1 matching version specified in $2.
"
local -n refvar="$1"
local version="$2"
local -a keys=()
keys=( $( ${yq} -e '.versions|keys().[]' 2>/dev/null) ) || \
std::die 3 "No version keys in configuration file!"
refvar=()
for key in "${keys[@]}"; do
l=()
# loop over semicolon separated list of keys
for k in ${key//;/ }; do
# brace expansion of key
local list=()
eval list=( $k )
if [[ ${list[@]} =~ ${version} ]]; then
refvar+=("${key}")
break
fi
done
done
(( ${#refvar[@]} == 0 )) && \
std::die 3 "No configuration for version -- ${version}"
return 0
}
get_yaml_vk_config(){
: "
Get configuration for version key $1.
Please note: this can be the empty string.
"
local -- key="$1"
result=$( ${yq} ".versions.\"${key}\"" 2>/dev/null ) || \
result=""
echo "${result}"
}
get_variants(){
local yaml
yaml=$(${yq} '.variants' 2>/dev/null)
echo "${yaml}"
}
get_num_variants(){
local -i n=$(${yq} '.|length' 2>/dev/null)
echo "$n"
}
get_variant(){
local n="$1"
local yaml
yaml=$(${yq} -e ".[$n]" 2>/dev/null)
[[ "${yaml}" == 'null' ]] && _result=''
echo "${yaml}"
}
get_group_deps(){
local -- yaml="$1" # yaml formatted string with group config
local -- group="$2" # compiler|mpi|hdf5|hdf5_serial
local -n with_modules="$3" # refvar to return a list of modules
local -a modules=()
local keys=()
keys=( $(${yq} ".${group}|keys|.[]" <<<"${yaml}" 2>/dev/null) )
local key
for key in "${keys[@]}"; do
local versions=()
versions=( $( ${yq} -e ".${group}.${key}[]" <<<"${yaml}" 2>/dev/null) )
local version
for version in "${versions[@]}"; do
if [[ -v opt_with_dict[${key}/${version}] ]]; then
with_modules+=( "${key}/${version}" )
fi
modules+=( "${key}/${version}" )
done
done
if (( ${#with_modules[@]} == 0 )); then
with_modules=( "${modules[@]}" )
fi
}
build_modules_compiler(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_cfg="$3"
local -a with_compiler=()
get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler
debug "${with_compiler[@]}"
local compiler=''
for compiler in "${with_compiler[@]}"; do
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${@:4}"
done
}
build_modules_hdf5_serial(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_cfg="$3"
local -a with_compiler=()
local -a with_hdf5=()
get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler
get_group_deps "${module_cfg['group_deps']}" 'hdf5_serial' with_hdf5
debug "${with_compiler[@]}"
debug "${with_hdf5[@]}"
local -- compiler
local -- hdf5
for compiler in "${with_compiler[@]}"; do
for hdf5 in "${with_hdf5[@]}"; do
debug "build $module_name/$module_version with $compiler and $hdf5"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${hdf5}" \
"${@:4}"
done
done
}
build_modules_mpi(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_cfg="$3"
local -a with_compiler=()
local -a with_mpi=()
get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler
get_group_deps "${module_cfg['group_deps']}" 'mpi' with_mpi
debug "${with_compiler[@]}"
debug "${with_mpi[@]}"
local -- compiler
local -- mpi
for compiler in "${with_compiler[@]}"; do
for mpi in "${with_mpi[@]}"; do
debug "build $module_name/$module_version with $compiler and $mpi"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${mpi}" \
"${@:4}"
done
done
}
build_modules_hdf5(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_cfg="$3"
local -a with_compiler=()
local -a with_mpi=()
local -a with_hdf5=()
get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler
get_group_deps "${module_cfg['group_deps']}" 'mpi' with_mpi
get_group_deps "${module_cfg['group_deps']}" 'hdf5' with_hdf5
debug "${with_compiler[@]}"
debug "${with_mpi[@]}"
debug "${with_hdf5[@]}"
local -- compiler
local -- mpi
local -- hdf5
for compiler in "${with_compiler[@]}"; do
for mpi in "${with_mpi[@]}"; do
for hdf5 in "${with_hdf5[@]}"; do
debug "build $module_name/$module_version with $compiler, $mpi and $hdf5"
debug " runtime deps: ${runtime_deps[@]}"
debug " build requires: ${build_requires[@]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${mpi}" \
"${hdf5}" \
"${@:4}"
done
done
done
}
build_modules_other(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_cfg="$3"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${@:4}"
}
set_urls() {
local -- yaml="$1"
l=$( ${yq} -Ne e '.|length' <<<"${yaml}" 2>/dev/null) || \
std::die 3 "error parsing YAML configuration"
local -i i=0
local -- fname=''
local -- url_yaml=''
for ((i=0; i<l; i++)); do
url_yaml=$(${yq} -Ne e ".[$i]" <<<"${yaml}" 2>/dev/null) || \
std::die 3 "error parsing YAML configuration"
local url=''
local fname=''
while read key value; do
key=${key:0:-1}
case "${key}" in
url )
url=$(envsubst <<<"${value}")
;;
name )
fname=$(envsubst <<<"${value}")
;;
* )
std::die 3 "Invalid key in list of URLs -- ${key}"
;;
esac
if [[ -z ${url} ]]; then
std::die 3 "URL missing!"
fi
done <<<"${url_yaml}"
pbuild.set_urls "${url}" "${fname}"
done
}
build_modules_variant(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_config="$3"
check_system(){
[[ -z ${module_config['systems']} ]] && return 0
set -o noglob
local -a systems=( ${module_config['systems']} )
set +o noglob
local -- system
for system in "${systems[@]}"; do
[[ "${opt_system}" == ${system} ]] && return 0
[[ "${HOSTNAME}" == ${system} ]] && return 0
done
std::info "Skipping variant '${module_version}', neither OS nor hostname match:"
std::info " This system: ${opt_system}; hostname: ${HOSTNAME}"
std::info " Systems to build on: ${systems[@]}"
return 1
}
P="${module_name}"
parse_version "${module_version}"
local build_variant="${opt_variant:-${module_config['default_variant']}}"
# build this variant?
if [[ ":${module_config['variant']}:" != *:${build_variant}:* ]]; then
debug "don't build this variant: ${module_config['variant']} != *:${build_variant}:*"
return 0
fi
# build for this system?
check_system || return 0
debug "build variant ${module_name}/${module_version}"
local ol_name="${module_config['overlay']}"
[[ -v OverlayInfo[${ol_name}:inst_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${ol_name}"
declare ol_inst_root="${OverlayInfo[${ol_name}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${ol_name}:mod_root]}"
module_version+="${module_config['suffix']}"
pbuild.compile_in_sourcetree "${module_config['compile_in_sourcetree']}"
pbuild.configure_with "${module_config['configure_with']}"
pbuild.add_to_group "${module_config['group']}"
set_urls "${module_config['urls']}"
local -a runtime_deps=()
if [[ -n ${module_config['runtime_deps']} ]]; then
readarray -t runtime_deps <<<"${module_config['runtime_deps']}"
debug "runtime_deps=${runtime_deps[@]} length: ${#runtime_deps[@]}"
fi
local -a build_requires=()
if [[ -n ${module_config['build_requires']} ]]; then
readarray -t build_requires <<<"${module_config['build_requires']}"
build_requires=( "${build_requires[@]/#/b:}" )
debug "build_requires=${build_requires[@]} length: ${#build_requires[@]}"
fi
if [[ -n ${module_config['docfiles']} ]]; then
readarray -t MODULE_DOCFILES <<<"${module_config['docfiles']}"
fi
local -A build_functions=(
['Compiler']=build_modules_compiler
[HDF5_serial]=build_modules_hdf5_serial
[MPI]=build_modules_mpi
[HDF5]=build_modules_hdf5
)
local func=build_modules_other
if [[ -v build_functions[${module_config['group']}] ]]; then
func=${build_functions[${module_config['group']}]}
fi
${func} \
"${module_name}" \
"${module_version}" \
module_config \
"${runtime_deps[@]}" "${build_requires[@]}"
}
expand_version_key(){
local -n ev_result="$1"
local -- key="$2"
local -- version="$3"
ev_result=()
# loop over comma separated list of keys
for k in ${key//;/ }; do
# do curly brackets expansion {}
local l
local list=()
eval list=( $k )
for l in "${list[@]}"; do
if [[ $l =~ ${version} ]]; then
ev_result+=("${l}")
fi
done
done
}
local -A used_keys=()
local -- yaml_input=''
local -A default_config=()
local -- version_key=''
local -a version_keys=()
local -A shasums=()
check_yaml_keys Yaml_valid_keys_for_module used_keys <<<"${yaml_mod_config}"
[[ -v used_keys['versions'] ]] || \
std::die 3 "No version(s) specified in YAML configuration file."
if [[ -v used_keys['defaults'] ]]; then
yaml_input=$(${yq} '.defaults' <<<"${yaml_mod_config}" 2>/dev/null)
fi
get_config "${yaml_input}" default_config Yaml_default_config
if [[ -v used_keys['shasums'] ]]; then
local yaml_input=$(${yq} '.shasums' <<<"${yaml_mod_config}" 2>/dev/null)
while read key value; do
SHASUMS[${key//:}]="${value}"
done <<<"${yaml_input}"
fi
get_matching_version_keys version_keys "${version}" <<<"${yaml_mod_config}"
for version_key in "${version_keys[@]}"; do
local yaml_vk_config=$(get_yaml_vk_config "${version_key}" <<<"${yaml_mod_config}")
# check keys: allowed are 'config' and 'variants'
used_keys=()
check_yaml_keys Yaml_valid_vk_keys used_keys <<<"${yaml_vk_config}"
# read config if set
local -A vk_config=()
yaml_input=''
if [[ -v used_keys['config'] ]]; then
yaml_input=$(${yq} '.config' <<<"${yaml_vk_config}" 2>/dev/null)
debug "vk input: ${yaml_input}"
fi
# reminder: if YAML input is empty, next line copies defaults to 'vk_config'
get_config "${yaml_input}" vk_config default_config
local -- yaml_variants=$(get_variants <<<"${yaml_vk_config}")
local -i num_variants=$(get_num_variants <<<"${yaml_variants}")
local versions=()
expand_version_key versions "${version_key}" "${version}"
local v=''
for v in "${versions[@]}"; do
debug "version: $v"
if (( num_variants == 0 )); then
build_modules_variant \
"${name}" "${v}" \
vk_config
else
local -i n=0
local -- yaml_variant=''
for ((n=0; n<num_variants; n++)); do
yaml_variant=$(get_variant "${n}" <<<"${yaml_variants}")
local -A mod_config=()
get_config "${yaml_variant}" mod_config vk_config
build_modules_variant \
"${name}" "${v}" \
mod_config
done
fi
done
done
} # build_modules_yaml()
build_modules() {
if [[ "${opt_yaml}" == 'yes' ]]; then
build_modules_yaml "$@"
else
build_modules_legacy "$@"
fi
}
#.............................................................................
# main
parse_args "$@"
if [[ "${opt_debug,,}" == 'no' ]]; then
debug(){
:
}
else
debug(){
echo "DEBUG: " "$@"
}
fi
pbuild.jobs "${opt_jobs}"
pbuild.force_rebuild "${opt_force_rebuild}"
pbuild.build_target "${opt_build_target}"
pbuild.dry_run "${opt_dry_run}"
pbuild.enable_cleanup_build "${opt_enable_cleanup_build}"
pbuild.enable_cleanup_src "${opt_enable_cleanup_src}"
pbuild.update_modulefiles "${opt_update_modulefiles}"
pbuild.system "${opt_system}"
pbuild.verbose "${opt_verbose}"
#
# read configuration for modbuild
#
pm::read_config
# :FIXME: should dist files go to
# ${pm_root}/var/distfiles
# or
# ${overlay}/var/distfiles
# ?
: ${PMODULES_DISTFILESDIR:="${PMODULES_HOME%%/Tools*}/var/distfiles"}
: ${PMODULES_TMPDIR:=/var/tmp/${USER}}
export PMODULES_DISTFILESDIR PMODULES_TMPDIR
declare -r BUILD_SCRIPT
declare -r BUILDBLOCK_DIR
if [[ -z ${module_name} ]]; then
# the module name is defined by the directory the build script is in
IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}"
module_name=${fname[${#fname[@]}-2]}
fi
for version in "${versions[@]}"; do
build_modules "${module_name}" "${version}" "${opt_with_modules[@]}"
done
# Local Variables:
# mode: sh
# sh-basic-offset: 8
# tab-width: 8
# End:
Reference in New Issue View Git Blame Copy Permalink