pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-06-25 00:57:58 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
e853159fc7bae95d85ef5bb13a4ec15257537bbf
Pmodules/Pmodules/modbuild.in
T
gsell e30338240a Merge branch '350-build-system-handle-empty-variants-blocks' into 'master' 
Resolve "build-system: handle empty variants blocks"

Closes #350

See merge request Pmodules/src!347

(cherry picked from commit ffbc6292eb)

047d55d8 build-system: bugfix: handle empty variants

Co-authored-by: gsell <achim.gsell@psi.ch>
2024-09-02 18:25:20 +02:00

1585 lines
40 KiB
Plaintext
Raw Blame History

#!@BASH@
#
# The following build specific variables are set and used in libpbuild.bash:
# ARGS
# BUILD_SCRIPT
# BUILDBLOCK_DIR
#
#.............................................................................
declare -r VERSION='@PMODULES_VERSION@'
unset CDPATH # unset CDPATH, otherwise 'cd' prints the directoy!
unset IFS # use default IFS
#set -e # exit on error
set -o pipefail
set -o nounset
shopt -s nocaseglob
shopt -s extglob
shopt -s nullglob
# get absolute path of script
mydir=$(cd "$(/usr/bin/dirname "$0")" && pwd -P)
prefix=$(/usr/bin/dirname "${mydir}")
PATH="${prefix}/bin:${prefix}/libexec:/bin:/usr/bin:/sbin:/usr/sbin"
source "${prefix}/lib/libstd.bash" || {
echo "Oops: cannot source library -- '$_'" 1>&2; exit 3;
}
source "${prefix}/lib/libpmodules.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
source "${prefix}/lib/libpbuild.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
unset mydir
unset prefix
##############################################################################
#
usage() {
std::error "
USAGE:
$0 [options..] [build_script] version
MANDATORY ARGUMENTS:
version
Version of module to build.
SELECT VARIANT TO BUILD:
--system
Specify the system for selecting a variants. Defaults to the
OS version and release like 'rhel6'.
--with=P/V
Select variant to compile. Use multiple '--with' arguments
to make the selected variant unique.
BUILD-STEPS OPTIONS:
--clean-install
Remove module if already exist before building.
--prep
Prepare sources: unpack sources and apply patches only.
--configure
Prepare and configure sources.
--compile
Prepare, configure and compile everything.
--install
Prepare, configure and compile everything. Finally run install
step. Do not cleanup build and source directory.
--all
Run throu all steps including cleanup.
--update-modulefiles
Only install the modulefile and set the release.
MISCELLANEOUS OPTIONS:
-? | -h | --help
Print usage.
-V | --version )
Print version.
-v | --verbose )
Verbose output.
--debug )
Run in debug mode.
-j N | --jobs=N
Run N parallel make jobs.
-f | --force-rebuild
Force rebuild of module.
--dry-run
Dry run.
--disable-cleanup-build
--disable-cleanup-src
--disable-cleanup
Disable the cleanup of files in the build and/or source
directory. Default is to remove all files in the build
respective source directory at the beginning of a build
and after a successful build.
--distdir
Directory where to store and lookup downloaded files.
--tmpdir
Directory used for building a module.
DOCUMENTATION:
Full documentation is available at
http://pmodules.gitpages.psi.ch
"
exit 1
}
##############################################################################
#
# parse options and arguments
#
# command line arguments are taken first
# then configuration file
# last default
# save arguments, required for building dependencies
declare -ra ARGS=( "$@" )
# versions to be build, '.*' or none means all
declare -a versions_to_build=()
declare opt_build_target='all'
declare opt_dry_run='no'
declare opt_enable_cleanup_build='yes'
declare opt_enable_cleanup_src='yes'
declare opt_force_rebuild='no'
declare -i opt_jobs=0
declare opt_update_modulefiles='no'
declare opt_system=''
declare opt_verbose='no'
# array collecting all modules specified on the command line via '--with=module'
declare -a opt_with_modules=()
declare -A opt_with_dict=()
declare -- opt_config_file=''
declare -- opt_debug='no'
declare -- opt_check_mode='no'
declare -- opt_variant=''
declare -- opt_clean_install='no'
declare -- opt_parent_prefix=''
declare -- BUILD_SCRIPT=''
declare -- yaml_config_file=''
declare -- module_name=''
declare -- module_type='module'
declare -- echo=':'
parse_args() {
#
# The first argument ($1) is the build-script, if called in the
# usual way:
# ./<build-script> <version> [options]
# If called via:
# modbuild <build-script> [options]
# the build-script MUST be passed as first argument.
#
(( $# == 0 )) && usage
while (( $# > 0 )); do
case $1 in
-j )
opt_jobs="$2"
shift
;;
--jobs=[0-9]* )
opt_jobs="${1/--jobs=}"
;;
-v | --verbose )
trap 'echo "$BASH_COMMAND"' DEBUG
opt_verbose='yes'
echo='echo'
;;
--debug )
opt_debug='yes'
set -x
;;
-f | --force-rebuild )
opt_force_rebuild='yes'
;;
-\? | -h | --help )
usage
;;
-V | --version )
std::die 0 "\nPmodules version ${VERSION}\nCopyright GNU GPL v2\n"
;;
--dry-run )
opt_dry_run='yes'
;;
--disable-cleanup )
opt_enable_cleanup_build='no'
opt_enable_cleanup_src='no'
;;
--disable-cleanup-build )
opt_enable_cleanup_build='no'
;;
--disable-cleanup-src )
opt_enable_cleanup_src='no'
;;
--distdir | --distdir=* )
if [[ $1 == *=* ]]; then
PMODULES_DISTFILESDIR="${1/--distdir=}"
else
PMODULES_DISTFILESDIR="$2"
shift
fi
;;
--tmpdir | --tmpdir=* )
if [[ $1 == *=* ]]; then
PMODULES_TMPDIR="${1#--*=}"
else
PMODULES_TMPDIR="$2"
shift
fi
;;
--system | --system=* )
if [[ $1 == *=* ]]; then
opt_system="${1#--*=}"
else
opt_system="$2"
shift
fi
;;
--check-mode )
opt_check_mode='yes'
;;
--use-flags | --use-flags=* )
if [[ $1 == *=* ]]; then
USE_FLAGS=":${1#--*=}:"
else
USE_FLAGS=":$2:"
shift
fi
;;
--with | --with=*/* )
if [[ $1 == *=* ]]; then
opt_with_modules+=( "${1#--*=}" )
opt_with_dict[${1#--*=}]=0
else
opt_with_modules+=( "$2" )
opt_with_dict[$2]=0
shift
fi
;;
--prep | --configure | --compile | --install | --all )
opt_build_target=${1:2}
;;
--clean-install )
opt_force_rebuild='yes'
opt_clean_install='yes'
;;
--update-modulefiles )
opt_update_modulefiles='yes'
;;
--config-file | --config-file=* )
if [[ $1 == *=* ]]; then
opt_config_file=( "${1#--*=}" )
else
opt_config_file=( "$2" )
shift
fi
;;
--variant | --variant=* )
if [[ $1 == *=* ]]; then
opt_variant=( "${1#--*=}" )
else
opt_variant=( "$2" )
shift
fi
;;
--parent-prefix | --parent-prefix=* )
if [[ $1 == *=* ]]; then
opt_parent_prefix="${1#--*=}"
else
opt_parent_prefix="$2"
shift
fi
module_type='sub_package'
pbuild.set_prefix "${opt_parent_prefix}"
;;
-- | '' )
:
;;
-* )
std::die 1 "Invalid option -- '$1'"
;;
* )
local -- arg="$1"
if [[ -f "${arg}" && -x "${arg}" ]]; then
BUILD_SCRIPT="$(std::get_abspath "${arg}")"
BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}")
elif [[ "${arg}" == */* ]]; then
module_name="${arg%/*}"
versions_to_build+=( "${arg#*/}" )
else
versions_to_build+=( "$1" )
fi
;;
esac
shift
done
[[ -z "${BUILD_SCRIPT}" ]] && \
std::die 1 "%s " \
"Build script argument missing?"
# if no version is specified on the cmd-line, build all versions
(( ${#versions_to_build[@]} > 0)) || versions_to_build+=( '.*' )
# set system if not set on the cmd-line
opt_system="${opt_system:-$(std::get_os_release)}"
# set config file
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
[[ -f "${yaml_config_file}" && -r "${yaml_config_file}" ]] || \
std::die 2 \
"%s -- %s" \
"YAML config file doesn't exist or is not readable" \
"${yaml_config_file}"
std::info "Using YAML configuration file - ${yaml_config_file}"
}
get_yaml_file_fmt(){
: "
Get format version of configuration file. Print the version number
to stdout if it is valid.
"
local -n result="$1"
local -r fname="$2"
${yq} -e '.' "${fname}" &>/dev/null || \
std::die 3 "%s -- %s" \
"Error in YAML config file, please check with linter" \
"${fname}"
local -- fmt=''
result=$(${yq} -e ".format" "${fname}") || \
std::die 3 "Error reading config file format -- ${fname}"
case "${result}" in
1 )
:
;;
* )
std::die 3 "Unknown YAML Pmodules config file format -- ${fname}"
;;
esac
}
read_yaml_config_file() {
: "
Test whether the configuration file '$1' provides configurations
for the module '$2'. If yes, the YAML block with the configuration
is printed to stdout.
"
local -n result="$1"
local -- file_name="$2"
local -- module_name="$3"
result=$( ${yq} -Ne e ".${module_name}" "${file_name}" 2>/dev/null ) || \
std::die 3 "Configuration for '${module_name}' missing -- ${file_name}"
}
build_modules(){
local -- name="$1"
local -- version="$2"
shift 2
local -a with_modules=( "$*" )
if [[ "${opt_check_mode}" == 'yes' ]]; then
if ! which yamllint > /dev/null 2>&1; then
eval $( "${modulecmd}" bash load yamllint/1.28.0 )
fi
which yamllint > /dev/null 2>&1 || \
std::die 3 "yamllint not found"
yamllint "${yaml_config_file}"
fi
local -- file_fmt=''
get_yaml_file_fmt \
file_fmt \
"${yaml_config_file}"
local -- module_config=''
read_yaml_config_file \
yaml_module_config \
"${yaml_config_file}" \
"${name}"
case "${file_fmt}" in
1 )
build_modules_yaml_v1 \
"${yaml_module_config}" \
"${name}" "${version}" \
"${with_modules[@]}"
;;
* )
std::die 255 "Oops!" # we should never be here!
;;
esac
}
#......................................................................
#
# Initialise environment modules.
#
# Arguments:
# none
#
init_module_environment(){
eval $( "${modulecmd}" bash use unstable )
eval $( "${modulecmd}" bash use deprecated )
eval $( "${modulecmd}" bash purge )
# :FIXME: this is a hack!!!
# shouldn't this be set in the build-script?
if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then
eval $( "${modulecmd}" bash use Libraries )
fi
if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then
eval $( "${modulecmd}" bash use System )
fi
unset C_INCLUDE_PATH
unset CPLUS_INCLUDE_PATH
unset CPP_INCLUDE_PATH
unset LIBRARY_PATH
unset LD_LIBRARY_PATH
unset DYLD_LIBRARY_PATH
unset CFLAGS
unset CPPFLAGS
unset CXXFLAGS
unset LIBS
unset LDFLAGS
unset CC
unset CXX
unset FC
unset F77
unset F90
}
parse_version() {
local v="$1"
V="$1"
USE_FLAGS=${USE_FLAGS:-''}
local tmp=''
if [[ "$v" =~ "_" ]]; then
tmp="${v#*_}"
USE_FLAGS+=":${tmp//_/:}:"
v="${v%%_*}"
fi
V_PKG="${v%%-*}" # version without the release number
if [[ $v == *-* ]]; then
V_RELEASE="${v#*-}" # release number
fi
case "${V_PKG}" in
*.*.* )
V_MAJOR="${V_PKG%%.*}"
tmp="${V_PKG#*.}"
V_MINOR="${tmp%%.*}"
V_PATCHLVL="${tmp#*.}"
;;
*.* )
V_MAJOR="${V_PKG%.*}"
V_MINOR="${V_PKG#*.}"
;;
* )
V_MAJOR="${V_PKG}"
;;
esac
VERSIONS=()
if [[ -n ${V_RELEASE} ]]; then
VERSIONS+=( "${V_PKG}-${V_RELEASE}" )
fi
if [[ -n ${V_PATCHLVL} ]]; then
VERSIONS+=( "${V_MAJOR}.${V_MINOR}.${V_PATCHLVL}" )
fi
if [[ -n ${V_MINOR} ]]; then
VERSIONS+=( "${V_MAJOR}.${V_MINOR}" )
fi
VERSIONS+=( "${V_MAJOR}" )
}
# these variables must be export for envsubst(1)
declare -x P=''
declare -x V=''
declare -x V_PKG=''
declare -x V_MAJOR='' # first number in version string
declare -x V_MINOR='' # second number in version string (or empty)
declare -x V_PATCHLVL='' # third number in version string (or empty)
declare -x V_RELEASE='' # module release (or empty)
declare -x PREFIX=''
declare -A SHASUMS=()
declare -a MODULE_DOCFILES=()
declare -A Yaml_valid_keys_for_module=(
['defaults']=1
['shasums']=1
['type']=1
['versions']=1
)
declare -A Yaml_default_config=(
['build_requires']='' # !!seq of strings
['compile_in_sourcetree']='no' # !!str
['configure_with']='auto' # !!str
['configure_args']='' # !!seq of strings
['configure_args+']='' # !!seq of strings
['default_variant']='' # !!str
['docfiles']='' # !!seq of strings
['docfiles+']='' # !!seq of strings
['download_dir']='' # !!str
['group']='Tools' # !!str
['kernels']='' # !!seq of strings
['group_deps']='' # !!map
['overlay']='base' # !!str
['patch_files']='' # !!seq
['patch_files+']='' # !!seq
['relstage']='unstable' # !!str
['runtime_deps']='' # !!seq of strings
['script']='build' # !!str
['suffix']='' # !!str
['systems']='' # !!seq of strings
['sub_packages']='' # !!map
['target_cpus']='' # !!seq of strings
['urls']='' # !!map
['use_overlays']='' # !!seq
['variant']='' # !!str
)
declare -A Yaml_valid_vk_keys=(
['config']='1' # !!map
['variants']='1' # !!map
)
declare -A Unpackers=(
['tar']='tar'
['7z']='7z'
['none']='none'
)
declare -A KernelNames=(
['linux']='1'
['darwin']='1'
['any']='1'
)
declare -A TargetCPUs=(
['x86_64']='x86_64'
['arm64']='arm64'
['aarch64']='arm64'
['any']='1'
)
declare -A hierarchical_groups=(
['compiler']='compiler'
['mpi']='compiler mpi'
['hdf5']='compiler mpi hdf5'
['hdf5_serial']='compiler hdf5_serial'
)
build_modules_yaml_v1(){
: "
"
local -- yaml_module_config="$1"
local -- name="$2"
local -- version="$3"
shift 3
local -a with_modules=( "$@" )
die_missing_group_dep(){
std::die 3 "%s/%s: %s" \
"${1}" "${2}" \
"is in group '$3', but the group dependency for this group is missing!"
}
die_invalid_group_dep(){
std::die 3 "%s/%s: %s" \
"${1}" "${2}" \
"invalid group dependency '$3' for module in group '$4'!"
}
die_illegal_group_dep(){
std::die 3 "%s/%s: %s" \
"${1}" "${2}" \
"illegal group dependency '$4' for module in group '$3'!"
}
die_invalid_variants_block(){
std::die 3 "%s/%s: %s" \
"${1}" "${2}" \
"invalid type of variants block: must be '!!seq' but is '$3'!"
}
yml::check_keys(){
local -n yaml_input="$1"
local -n valid_yaml_keys="$2"
local -n used_yaml_keys="$3"
used_yaml_keys=()
local -- key=''
local -a keys=()
readarray -t keys < <( ${yq} -e '.|keys|.[]' 2>/dev/null <<<"${yaml_input}")
debug "top-level keys: ${keys[*]}"
for key in "${keys[@]}"; do
[[ -v valid_yaml_keys[${key}] ]] || \
std::die 3 "Invalid key in YAML configuration file -- ${key}"
used_yaml_keys[${key}]=1
done
}
yml::get_config(){
local -n yaml_input="$1"
local -n cfg="$2" # ref. to return configuration
local -n dfl="$3" # ref. to defaults
local -- key=''
local -- value=''
for key in "${!dfl[@]}"; do
cfg[${key}]="${dfl[${key}]}"
done
if [[ -z "${yaml_input}" ]]; then
return 0
fi
local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null ) || \
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
debug "config keys: ${keys[*]}"
for key in "${keys[@]}"; do
[[ -v dfl[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key} in
compile_in_sourcetree )
pm::get_value "${yaml_input}" value "${key}" '!!bool'
case ${value,,} in
true )
cfg[${key,,}]='yes'
;;
false )
cfg[${key,,}]='no'
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
"${key}" \
"${value}"
;;
esac
;;
configure_with )
pm::get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
auto | cmake | autotools )
cfg[${key,,}]="${value,,}"
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'configure_with' \
"${value}"
;;
esac
;;
default_variant | download_dir | group | overlay | script | suffix )
pm::get_value "${yaml_input}" value "${key}" '!!str'
cfg[${key,,}]="${value}"
;;
group_deps )
pm::get_value "${yaml_input}" value "${key}" '!!map'
cfg[${key,,}]="${value}"
;;
relstage )
pm::get_value "${yaml_input}" value "${key}" '!!str'
case ${value,,} in
unstable | stable | deprecated )
cfg[${key,,}]="${value,,}"
;;
remove | removed )
cfg[${key,,}]='remove'
;;
* )
std::die 3 "%s '%s' -- %s" \
"Invalid value for" \
'relstage' \
"${value}"
;;
esac
;;
urls | sub_packages )
pm::get_value "${yaml_input}" value "${key}" '!!seq'
cfg[${key,,}]="${value}"
;;
build_requires|configure_args|docfiles|patch_files|runtime_deps|systems|use_overlays|variant )
pm::get_seq "${yaml_input}" value "${key}"
cfg[${key,,}]="${value}"
;;
kernels )
pm::get_seq "${yaml_input}" value "${key}"
set -o noglob
local -a items=( "${value,,}" )
set +o noglob
local -- item=''
for item in "${items[@]}"; do
[[ -v KernelNames[${item}] ]] || \
die_invalid_value \
"${yaml_input}" \
'config section' \
'kernel' \
"${item}"
done
cfg[${key,,}]="${value}"
;;
target_cpus )
pm::get_seq "${yaml_input}" value "${key}"
set -o noglob
local -a items=( "${value,,}" )
set +o noglob
local -- item=''
for item in "${items[@]}"; do
[[ -v TargetCPUs[${item}] ]] || \
die_invalid_value \
"${yaml_input}" \
'config section'
'CPU' \
"${item}"
done
cfg[${key,,}]="${value}"
;;
'configure_args+' | 'docfiles+' | 'patch_files+' )
pm::get_seq "${yaml_input}" value "${key}"
key="${key:0:-1}"
if [[ -z "${cfg[${key,,}]}" ]]; then
cfg[${key,,}]="${value}"
else
cfg[${key,,}]+=$'\n'"${value}"
fi
;;
* )
std::die 3 "%s '%s' in %s" \
"Oops unhandled key" \
"${key}" \
"${FUNCNAME[0]}"
esac
done
}
yml::get_matching_version_keys(){
#
# return list of versions matching a specific version.
#
local -n yaml_input="$1" # [in] YAML input
local -n result="$2" # [out] list of versions
local -- version="$3" # [in] version to match
local -a keys=()
readarray -t keys < <( ${yq} -e '.versions|keys().[]' 2>/dev/null <<<"${yaml_input}" ) || \
std::die 3 "No version keys in configuration file!"
result=()
for key in "${keys[@]}"; do
l=()
# loop over semicolon separated list of keys
for k in ${key//;/ }; do
# brace expansion of key
local list=()
list=( $(${bash} -c "echo $k") )
if [[ ${list[@]} =~ ${version} ]]; then
result+=("${key}")
break
fi
done
done
(( ${#result[@]} == 0 )) && \
std::die 3 "No configuration for version -- ${version}"
return 0
}
yml::get_version_block(){
#
# Get configuration for specific version.
#
# Please note: this can be an empty string.
#
local -n yaml_input="$1" # [in] YAML input
local -n result="$2" # [out] result in YAML format
local -- version="$3" # [in] return config for this version
result=$( ${yq} -e ".versions.\"${version}\"" 2>/dev/null <<<"${yaml_input}" ) || \
result=""
}
yml::get_variants(){
#
# get variants of a specific version
#
local -n yaml_input="$1" # [in] YAML input with the variants
local -n result="$2" # [out] variants in YAML format
local -n n="$3" # [out] number of variants
local -- type_of_key=''
type_of_key=$( ${yq} -e ".variants | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type_of_key}" == '!!null' ]]; then
result=''
n=0
return 0
fi
if [[ "${type_of_key}" != '!!seq' ]]; then
die_invalid_variants_block "${name}" "${version}" \
"${type_of_key}"
fi
result=$(${yq} -e '.variants' 2>/dev/null <<<"${yaml_input}") || \
result=''
n=$(${yq} '.|length' 2>/dev/null <<<"${result}")
}
yml::get_nth_variant(){
local -n yaml_input="$1" # [in] YAML input
local -n result="$2" # [out] nth variant in YAML format
local -i n="$3" # [in] index of variant to return
result=$(${yq} -e ".[$n]" 2>/dev/null <<<"${yaml_input}") || result=''
}
yml::chk_group_deps(){
#
# Check the group dependencies:
# 1. are all keys valid?
# 2. all required group deps defined?
# 3. more group deps defined as required?
#
# Die if check fails.
#
local -- yaml_input="$1" # [in] value of key group_deps
local -- group="$2" # [in] compiler|mpi|hdf5|hdf5_serial
local -- name="$3" # [in] module name
local -- version="$4" # [in] module version
# query all specified group dependencies
local -a keys=()
readarray -t keys < <( ${yq} ".|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local -- key=''
for key in "${keys[@]}"; do
# is this a name of a hierarchical group?
[[ -v hierarchical_groups[${key}] ]] || \
die_illegal_group_dep "${name}" "${version}" "${group}" "${key}"
# is this in the list of required group dependencies?
is_in_array "${key}" "${hierarchical_groups[${key}]}" || \
die_invalid_group_dep "${name}" "${version}" "${group}"
done
# are all required group dependencies defined?
for key in ${hierarchical_groups[${group,,}]}; do
is_in_array "${key,,}" "${keys[@]}" || \
die_missing_group_dep "${name}" "${version}" "${group}"
done
}
yml::get_group_deps(){
local -- yaml_input="$1" # [in] value of key group_deps
local -- group="$2" # [in] compiler|mpi|hdf5|hdf5_serial
local -n with_modules="$3" # [out] list of required modules
local -a modules=()
local keys=()
readarray -t keys < <( ${yq} ".${group,,}|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local key
for key in "${keys[@]}"; do
local versions=()
readarray -t versions < <( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml_input}" 2>/dev/null )
local version
for version in "${versions[@]}"; do
if [[ -v opt_with_dict[${key}/${version}] ]]; then
with_modules+=( "${key}/${version}" )
fi
modules+=( "${key}/${version}" )
done
done
if (( ${#with_modules[@]} == 0 )); then
with_modules=( "${modules[@]}" )
fi
}
is_in_array(){
local -r key="$1"
shift 1
[[ $* =~ (^|[[:space:]])"${key}"($|[[:space:]]) ]]
}
is_subset(){
local -n subset="$1"
shift 1
local el=''
for el in "${subset[@]}"; do
is_in_array "${el}" "$@" || return 1
done
return 0
}
#
# To compile a module with a certain compiler||mpi||hdf5 dependency
# the '--with' option can be used. Depending on the hierarchical
# group the modules specified with the option '--with' must be a
# subset of
# - compiler: ( compiler)
# - mpi: ( compiler mpi )
# - hdf5: ( compiler mpi hdf5 )
# - hdf5_serial: ( compiler hdf5_serial )
#
build_modules_compiler(){
#
# build a module in hierarchical group 'Compiler'
#
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'Compiler' \
"${module_name}" "${module_version}"
local -a with_compiler=()
yml::get_group_deps \
"${module_cfg['group_deps']}" \
'Compiler' with_compiler
debug "${with_compiler[@]}"
local compiler=''
for compiler in "${with_compiler[@]}"; do
# build if opt_with_modules is empty or compiler is in this array
(( ${#opt_with_modules[@]} != 0 )) \
&& [[ "${compiler}" != "${opt_with_modules[0]}" ]] \
&& continue
[[ "${opt_check_mode}" == 'yes' ]] && continue
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${@:4}"
done
}
build_modules_hdf5_serial(){
#
# build a module in hierarchical group 'HDF5_serial'
#
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'HDF5_serial' \
"${module_name}" "${module_version}"
local -a with_compiler=()
yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_hdf5=()
yml::get_group_deps "${module_cfg['group_deps']}" 'HDF5_serial' with_hdf5
local -- compiler
local -- hdf5
for compiler in "${with_compiler[@]}"; do
for hdf5 in "${with_hdf5[@]}"; do
# build if opt_with_modules is empty or compiler is in this array
(( ${#opt_with_modules[@]} != 0 )) \
&& ! is_subset opt_with_modules "${compiler}" "${hdf5}" \
&& continue
debug "build $module_name/$module_version with $compiler and $hdf5"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${hdf5}" \
"${@:4}"
done
done
}
build_modules_mpi(){
#
# build a module in hierarchical group 'MPI'
#
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'MPI' \
"${module_name}" "${module_version}"
local -a with_compiler=()
yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_mpi=()
yml::get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi
local -- compiler
local -- mpi
for compiler in "${with_compiler[@]}"; do
for mpi in "${with_mpi[@]}"; do
# build if opt_with_modules is empty or compiler is in this array
(( ${#opt_with_modules[@]} != 0 )) \
&& ! is_subset opt_with_modules "${compiler}" "${mpi}" \
&& continue
debug "build $module_name/$module_version with $compiler and $mpi"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${mpi}" \
"${@:4}"
done
done
}
build_modules_hdf5(){
#
# build a module in hierarchical group 'HDF5'
#
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'HDF5' \
"${module_name}" "${module_version}"
local -a with_compiler=()
yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_mpi=()
yml::get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi
local -a with_hdf5=()
yml::get_group_deps "${module_cfg['group_deps']}" 'HDF5' with_hdf5
local -- compiler
local -- mpi
local -- hdf5
for compiler in "${with_compiler[@]}"; do
for mpi in "${with_mpi[@]}"; do
for hdf5 in "${with_hdf5[@]}"; do
debug "build $module_name/$module_version with $compiler, $mpi and $hdf5"
debug " runtime deps: ${runtime_deps[*]}"
debug " build requires: ${build_requires[*]}"
# build if opt_with_modules is empty or compiler is in this array
(( ${#opt_with_modules[@]} != 0 )) \
&& ! is_subset opt_with_modules \
"${compiler}" "${mpi}" "${hdf5}" \
&& continue
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${compiler}" \
"${mpi}" \
"${hdf5}" \
"${@:4}"
done
done
done
}
build_modules_other(){
#
# build a module in a non-hierarchical group
#
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
pbuild.build_module_yaml \
"${module_name}" "${module_version}" \
"$3" \
"${@:4}"
}
die_parsing(){
std::die 3 "error parsing YAML:\n----\n$1\n----"
}
die_invalid_value(){
std::die 3 "Invalid value for key '$3' in $2 -- '$4'\n----\n$1\n----"
}
die_invalid_key(){
std::die 3 "Invalid key '$3' in $2\n----\n$1\n----"
}
die_missing_key(){
std::die 3 "Key '$3' missing in $2\n----\n$1\n----"
}
set_urls() {
local -- yaml="$1"
local -i l=0
l=$( ${yq} -Ne e '.|length' <<<"${yaml}" 2>/dev/null) || \
die_parsing "{yaml}"
local -i i=0
local -- url_yaml=''
for ((i=0; i<l; i++)); do
url_yaml=$(${yq} -Ne e ".[$i]" <<<"${yaml}" 2>/dev/null) || \
die_parsing "{yaml}"
local url=''
local fname=''
local strip_dirs=''
local unpacker=''
local key=''
local value=''
while read -r key value; do
key=${key:0:-1}
case "${key}" in
url )
url=$(${envsubst} <<<"${value}")
;;
name )
fname=$(${envsubst} <<<"${value}")
;;
strip_dirs )
[[ ${value} =~ ^[0-9]+$ ]] || \
die_invalid_value \
"${url_yaml}" \
'list of URLS' \
'strip_dirs' \
"${value}"
strip_dirs="${value}"
;;
unpacker )
[[ -v Unpackers[${value}] ]] || \
die_invalid_value \
"${url_yaml}" \
'list of URLs' \
'unpacker' \
"${value}"
unpacker="${value}"
;;
* )
die_invalid_key \
"${url_yaml}" \
'list of URLs' \
"${key}"
;;
esac
done <<<"${url_yaml}"
[[ -z "${url}" ]] && \
die_missing_key \
"${url_yaml}" \
'list of URLs' \
'url'
[[ -z "${fname}" ]] && fname="${url##*/}"
[[ -z "${strip_dirs}" ]] && strip_dirs=1
[[ -z "${unpacker}" ]] && unpacker='tar'
pbuild.set_urls "${url}" "${fname}" "${strip_dirs}" "${unpacker}"
done
}
set_configure_args(){
local -a args=()
readarray -t args <<< "$1"
pbuild.add_configure_args "${args[@]}"
}
set_patch_files(){
local -a args=()
readarray -t args <<< "$1"
pbuild.add_patch_files "${args[@]}"
}
die_sub_package_name_missing(){
std::die 3 "Name of sub-package not specified in \n===\n$1\n===\n"
}
die_sub_package_version_missing(){
std::die 3 "Version of sub-package not specified in \n===\n$1\n===\n"
}
build_sub_packages(){
local -- yaml="$1"
local -i l=0
l=$( ${yq} -Ne e '.|length' <<<"${yaml}" 2>/dev/null) || \
die_parsing "${yaml}"
local -i i=0
local -- fname=''
local -- pkgs_yaml=''
for ((i=0; i<l; i++)); do
pkgs_yaml=$(${yq} -Ne e ".[$i]" <<<"${yaml}" 2>/dev/null) || \
die_parsing "${yaml}"
local -- pkg_name=''
local -- pkg_version=''
local -a pkg_build_args=()
local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${pkgs_yaml}" 2>/dev/null ) || \
die_parsing "${pkgs_yaml}"
local -- key=''
for key in "${keys[@]}"; do
case ${key,,} in
'name' )
pm::get_value "${pkgs_yaml}" pkg_name "${key}" '!!str'
;;
'version' )
pm::get_value "${pkgs_yaml}" pkg_version "${key}" '!!str'
;;
'build_args' )
local -- value=''
pm::get_seq "${pkgs_yaml}" value "${key}"
readarray -t pkg_build_args <<< "${value}"
;;
* )
die_invalid_key \
"${pkgs_yaml}" \
"in subpackage '$i'" \
"${key}"
;;
esac
done
[[ -n "${pkg_name}" ]] || \
die_sub_package_name_missing "${pkgs_yaml}"
[[ -n "${pkg_version}" ]] || \
die_sub_package_version_missing "${pkgs_yaml}"
[[ "${opt_verbose}" == 'yes' ]] && \
pkg_build_args+=( '--verbose' )
[[ "${opt_debug}" == 'yes' ]] && \
pkg_build_args+=( '--debug' )
[[ "${opt_force_rebuild}" == 'yes' ]] && \
pkg_build_args+=( '-f' )
pkg_build_args+=( "--parent-prefix=${PREFIX}" )
"$BUILDBLOCK_DIR/build-${pkg_name}" \
"${pkg_name}/${pkg_version}" \
"${pkg_build_args[@]}"
done
}
build_modules_variant(){
local -- module_name="$1"
local -- module_version="$2"
local -n module_config="$3"
check_system(){
[[ -z ${module_config['systems']} ]] && return 0
set -o noglob
local -a systems=( ${module_config['systems']} )
set +o noglob
local -- system
for system in "${systems[@]}"; do
[[ "${opt_system}" =~ ${system} ]] && return 0
[[ "${HOSTNAME}" =~ ${system} ]] && return 0
done
std::info "Skipping variant '${module_version}', neither OS nor hostname match:"
std::info " This system: ${opt_system}; hostname: ${HOSTNAME}"
std::info " Systems to build on: ${systems[@]}"
return 1
}
die_invalid_kernel_name(){
std::die 3 "Invalid kernel name in configuration!"
}
check_kernel(){
[[ -z ${module_config['kernels']} ]] && return 0
set -o noglob
local -a kernels=( "${module_config['kernels'],,}" )
set +o noglob
local -- kernel=''
for kernel in "${kernels[@]}"; do
[[ ${kernel} == 'any' ]] && return 0
[[ ${kernel} == ${KernelName,,} ]] & return 0
done
std::info "Skipping variant '${module_version}':"
std::info " The kernel of this systems is: ${KernelName}"
std::info " But the variant is for the following kernels: ${module_config['kernels']}"
return 1
}
check_target_cpu(){
[[ -z ${module_config['target_cpus']} ]] && return 0
set -o noglob
local -a target_cpus=( "${module_config['target_cpus'],,}" )
set +o noglob
local -- system_cpu=$(uname -p)
local -- cpu=''
for cpu in "${target_cpus[@]}"; do
[[ ${cpu} == 'any' ]] && return 0
[[ ${cpu} == ${system_cpu} ]] && return 0
done
std::info "Skipping variant '${module_version}':"
std::info " The CPU of this systems is: ${system_cpu}"
std::info " But this variant is for the following CPUs: ${module_config['target_cpus']}"
return 1
}
P="${module_name}"
parse_version "${module_version}"
local build_variant="${opt_variant:-${module_config['default_variant']}}"
# build this variant?
if [[ ":${module_config['variant']}:" != *:${build_variant}:* ]]; then
debug "don't build this variant: ${module_config['variant']} != *:${build_variant}:*"
return 0
fi
# build for this system, kernel and target_cpu?
check_system || return 0
check_kernel || return 0
check_target_cpu || return 0
debug "build variant ${module_name}/${module_version}"
local ol_name="${module_config['overlay']}"
[[ -v OverlayInfo[${ol_name}:install_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${ol_name}"
declare ol_install_root="${OverlayInfo[${ol_name}:install_root]}"
declare ol_modulefiles_root="${OverlayInfo[${ol_name}:modulefiles_root]}"
module_version+="${module_config['suffix']}"
pbuild.compile_in_sourcetree "${module_config['compile_in_sourcetree']}"
pbuild.configure_with "${module_config['configure_with']}"
pbuild.add_to_group "${module_config['group']}"
set_urls "${module_config['urls']}"
set_configure_args "${module_config['configure_args']}"
set_patch_files "${module_config['patch_files']}"
if [[ -n "${module_config['download_dir']}" ]]; then
PMODULES_DISTFILESDIR="${module_config['download_dir']}"
fi
local -a runtime_deps=()
if [[ -n ${module_config['runtime_deps']} ]]; then
readarray -t runtime_deps <<<"${module_config['runtime_deps']}"
debug "runtime_deps=${runtime_deps[@]} length: ${#runtime_deps[@]}"
fi
local -a build_requires=()
if [[ -n ${module_config['build_requires']} ]]; then
readarray -t build_requires <<<"${module_config['build_requires']}"
build_requires=( "${build_requires[@]/#/b:}" )
debug "build_requires=${build_requires[@]} length: ${#build_requires[@]}"
fi
if [[ -n ${module_config['docfiles']} ]]; then
readarray -t MODULE_DOCFILES <<<"${module_config['docfiles']}"
fi
local -A build_functions=(
['Compiler']=build_modules_compiler
[HDF5_serial]=build_modules_hdf5_serial
[MPI]=build_modules_mpi
[HDF5]=build_modules_hdf5
)
local func=build_modules_other
if [[ -v build_functions[${module_config['group']}] ]]; then
func=${build_functions[${module_config['group']}]}
fi
${func} \
"${module_name}" \
"${module_version}" \
module_config \
"${runtime_deps[@]}" "${build_requires[@]}"
build_sub_packages "${module_config['sub_packages']}"
}
expand_version_key(){
local -n ev_result="$1"
local -- key="$2"
local -- version="$3"
ev_result=()
# loop over comma separated list of keys
for k in ${key//;/ }; do
# do curly brackets expansion {}
local l
local list=()
list=( $(${bash} -c "echo $k" ) )
for l in "${list[@]}"; do
if [[ $l =~ ${version} ]]; then
ev_result+=("${l}")
fi
done
done
}
local -A used_keys=()
local -A default_config=()
local -- version_key=''
local -a version_keys=()
local -A shasums=()
yml::check_keys \
yaml_module_config \
Yaml_valid_keys_for_module \
used_keys
[[ -v used_keys['versions'] ]] || \
std::die 3 "No version(s) specified in YAML configuration file."
local -- yaml_default_config=''
if [[ -v used_keys['defaults'] ]]; then
yaml_default_config=$(${yq} '.defaults' <<<"${yaml_module_config}" 2>/dev/null)
fi
yml::get_config \
yaml_default_config \
default_config \
Yaml_default_config
if [[ -v used_keys['shasums'] ]]; then
local yaml_shasums=''
yaml_shasums=$(${yq} '.shasums' <<<"${yaml_module_config}" 2>/dev/null)
while read -r key value; do
[[ -z ${key} ]] && continue
SHASUMS[${key//:}]="${value}"
done <<<"${yaml_shasums}"
fi
if [[ -v used_keys['type'] ]]; then
local -- value=''
pm::get_value "${yaml_module_config}" value 'type' '!!str'
case "${value,,}" in
'module' )
[[ "${module_type}" == 'sub_package' ]] && \
std::die 3 "Module type is 'module' but was called as 'sub_package'!"
;;
'sub_package' )
[[ "${module_type}" == 'module' ]] && \
std::die 3 "Module type is 'sub_package' but was called as 'module'!"
;;
* )
std::die 3 "Invalid module type -- '${value}'!"
;;
esac
fi
yml::get_matching_version_keys \
yaml_module_config \
version_keys \
"${version}"
for version_key in "${version_keys[@]}"; do
local -- yaml_version_block=''
yml::get_version_block \
yaml_module_config \
yaml_version_block \
"${version_key}"
# check keys: allowed are 'config' and 'variants'
used_keys=()
yml::check_keys \
yaml_version_block \
Yaml_valid_vk_keys \
used_keys
# read (default) config of version if set
local -- yaml_version_config=''
if [[ -v used_keys['config'] ]]; then
yaml_version_config=$(${yq} '.config' <<<"${yaml_version_block}" 2>/dev/null)
debug "vk input: ${yaml_version_config}"
fi
# reminder: if YAML input is empty, next line copies defaults to 'vk_config'
local -A vk_config=()
yml::get_config \
yaml_version_config \
vk_config \
default_config
local -- yaml_variants=''
local -i num_variants=0
yml::get_variants \
yaml_version_block \
yaml_variants \
num_variants
local versions=()
expand_version_key versions "${version_key}" "${version}"
local v=''
for v in "${versions[@]}"; do
debug "version: $v"
if (( num_variants == 0 )); then
build_modules_variant \
"${name}" "${v}" \
vk_config
else
local -i n=0
local -- yaml_variant_config=''
for ((n=0; n<num_variants; n++)); do
yml::get_nth_variant \
yaml_variants \
yaml_variant_config \
"${n}"
local -A mod_config=()
yml::get_config \
yaml_variant_config \
mod_config \
vk_config
build_modules_variant \
"${name}" "${v}" \
mod_config
done
fi
done
done
} # build_modules()
debug(){
${echo} "INFO: " "$@" 1>&2
}
#.............................................................................
# main
init_module_environment
parse_args "$@"
pbuild.jobs "${opt_jobs}"
pbuild.force_rebuild "${opt_force_rebuild}"
pbuild.build_target "${opt_build_target}"
pbuild.dry_run "${opt_dry_run}"
pbuild.enable_cleanup_build "${opt_enable_cleanup_build}"
pbuild.enable_cleanup_src "${opt_enable_cleanup_src}"
pbuild.update_modulefiles "${opt_update_modulefiles}"
pbuild.system "${opt_system}"
pbuild.verbose "${opt_verbose}"
#
# read configuration for modbuild
#
pm::read_config
# :FIXME: should dist files go to
# ${pm_root}/var/distfiles
# or
# ${overlay}/var/distfiles
# ?
PMODULES_DISTFILESDIR=${PMODULES_DISTFILESDIR:-"${PMODULES_HOME%%/Tools*}/var/distfiles"}
PMODULES_TMPDIR="${PMODULES_TMPDIR:-/var/tmp/${USER}}"
export PMODULES_DISTFILESDIR PMODULES_TMPDIR
declare -r BUILD_SCRIPT
declare -r BUILDBLOCK_DIR
if [[ -z ${module_name} ]]; then
# the module name is defined by the directory the build script is in
IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}"
module_name=${fname[${#fname[@]}-2]}
fi
declare version=''
for version in "${versions_to_build[@]}"; do
build_modules "${module_name}" "${version}" "${opt_with_modules[@]}"
done
# Local Variables:
# mode: sh
# sh-basic-offset: 8
# tab-width: 8
# End:
Reference in New Issue View Git Blame Copy Permalink