pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-06-27 10:03:08 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
746e8419dfc7a748874fd17415a475f22c8ee29c
Pmodules/Pmodules/modbuild.in
T
gsell 5d6f53b7d4 build in overlays fixed
2023-04-27 16:54:28 +02:00

761 lines
19 KiB
Plaintext
Executable File
Raw Blame History

#!@BASH@
#
# The following build specific variables are set and used in libpbuild.bash:
# ARGS
# BUILD_SCRIPT
# BUILDBLOCK_DIR
#
#.............................................................................
set -x
declare VERSION='@PMODULES_VERSION@'
# get absolute path of script
declare mydir=$(dirname "$0")
declare -r mydir=$(cd ${mydir} && pwd -P)
source "${mydir}/../lib/libstd.bash" || {
echo "Oops: cannot source library -- '$_'" 1>&2; exit 3;
}
##############################################################################
#
# check availability of used commands and set environment variables
# cmd=$(which cmd)
# in the following we use these environment variable to call binaries.
#
declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd"
[[ -x ${MODULECMD} ]] || \
std::die 1 "Oops: modulecmd binary not available!"
std::def_cmds "${mydir}/../libexec" \
'yq'
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'awk' 'base64' 'cat' 'cp' 'find' 'getopt' 'grep' \
'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \
'rm' 'rmdir' 'sort' 'tar' 'tee' 'uname'
declare -r OS="$(${uname} -s)"
if [[ ${OS} == 'Darwin' ]]; then
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \
'curl' 'otool' 'shasum' 'sysctl'
declare -r sha256sum="${shasum -a 256}"
else
std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \
'ldd' 'curl' 'sha256sum'
fi
# for the time being, we still set PATH. Just in case we forgot a binary
PATH='/usr/bin:/bin:/usr/sbin:/sbin'
##############################################################################
source "${mydir}/../lib/libpbuild.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
source "${mydir}/../lib/libpmodules.bash" || \
std::die 3 "Oops: cannot source library -- '$_'"
##############################################################################
set -o nounset
shopt -s nocaseglob
shopt -s extglob
shopt -s nullglob
##############################################################################
#
usage() {
std::error "
USAGE:
$0 [options..] [build_script] version
MANDATORY ARGUMENTS:
version
Variant of module to build.
SELECT VARIANT TO BUILD:
--system
Specify the system for selecting a variants. Defaults to the
OS version and release like 'rhel6'.
--with=P/V
Select variant to compile. Use multiple '--with' arguments
to make the selected variant unique.
BUILD-STEPS OPTIONS:
--prep
Prepare sources: unpack sources and apply patches only.
--configure
Prepare and configure sources.
--compile
Prepare, configure and compile everything.
--install
Prepare, configure and compile everything. Finally run install
step. Do not cleanup build and source directory.
--all
Run throu all steps including cleanup.
-update-modulefiles
Only install the modulefile and set the release.
MISCELLANEOUS OPTIONS:
-? | -h | --help
Print usage.
-V | --version )
Print version.
-v | --verbose )
Verbose output.
--debug )
Run in debug mode.
-j N | --jobs=N
Run N parallel make jobs.
-f | --force-rebuild
Force rebuild of module.
--dry-run
Dry run.
--disable-cleanup-build
--enable-cleanup-build
Cleanup files in the build directory. Default is to remove.
all files in the build-directory.
--disable-cleanup-src
--enable-cleanup-src
Cleanup files in the source directory. Default is to
remove all files in the source directory.
--disable-cleanup
--enable-cleanup
Cleanup all files in temporary directory. Default is to
remove all files created during building.
--distdir
Directory where to store and lookup downloaded files.
--tmpdir
Directory used for building a module.
--overlay
Install in this overlay. Defaults to '${PMODULES_HOME%%/Tools*}'.
"
exit 1
}
##############################################################################
#
# parse options and arguments
#
# command line arguments are taken first
# then configuration file
# last default
# save arguments, required for building dependencies
declare -r ARGS="$@"
declare PMODULES_DISTFILESDIR=''
declare PMODULES_TMPDIR=''
# versions to be build, '.*' or none means all
declare -a versions=()
declare opt_build_target='all'
declare opt_dry_run='no'
declare opt_enable_cleanup_build='yes'
declare opt_enable_cleanup_src='yes'
declare opt_force_rebuild='no'
declare -i opt_jobs=0
declare opt_update_modulefiles='no'
declare opt_system=''
declare opt_overlay=''
declare opt_verbose='no'
# array collecting all modules specified on the command line via '--with=module'
declare -a opt_with_modules=()
declare -- opt_config_file=''
declare -- opt_yaml='no'
declare BUILD_SCRIPT=''
declare -- yaml_config_file=''
declare -a legacy_config_files=()
parse_args() {
while (( $# > 0 )); do
case $1 in
-j )
opt_jobs="$2"
shift
;;
--jobs=[0-9]* )
opt_jobs="${1/--jobs=}"
;;
-v | --verbose )
trap 'echo "$BASH_COMMAND"' DEBUG
opt_verbose='yes'
;;
--debug )
set -x
;;
-f | --force-rebuild )
opt_force_rebuild='yes'
;;
-\? | -h | --help )
usage
;;
-V | --version )
std::die 0 "\nPmodules version ${VERSION}\nCopyright GNU GPL v2\n"
;;
--dry-run )
opt_dry_run='yes'
;;
--enable-cleanup )
opt_enable_cleanup_build='yes'
opt_enable_cleanup_src='yes'
;;
--disable-cleanup )
opt_enable_cleanup_build='no'
opt_enable_cleanup_src='no'
;;
--enable-cleanup-build )
opt_enable_cleanup_build='yes'
;;
--disable-cleanup-build )
opt_enable_cleanup_build='no'
;;
--enable-cleanup-src )
opt_enable_cleanup_src='yes'
;;
--disable-cleanup-src )
opt_enable_cleanup_src='no'
;;
--distdir | --distdir=* )
if [[ $1 == *=* ]]; then
PMODULES_DISTFILESDIR="${1/--distdir=}"
else
PMODULES_DISTFILESDIR="$2"
shift
fi
;;
--tmpdir | --tmpdir=* )
if [[ $1 == *=* ]]; then
PMODULES_TMPDIR="${1#--*=}"
else
PMODULES_TMPDIR="$2"
shift
fi
;;
--system | --system=* )
if [[ $1 == *=* ]]; then
opt_system="${1#--*=}"
else
opt_system="$2"
shift
fi
;;
--overlay | --overlay=* )
if [[ $1 == *=* ]]; then
opt_overlay="${1#--*=}"
else
opt_overlay="$2"
shift
fi
;;
--yaml )
opt_yaml='yes'
;;
--use-flags | --use-flags=* )
if [[ $1 == *=* ]]; then
USE_FLAGS=":${1#--*=}:"
else
USE_FLAGS=":$2:"
shift
fi
;;
--with | --with=*/* )
if [[ $1 == *=* ]]; then
opt_with_modules+=( "${1#--*=}" )
else
opt_with_modules+=( "$2" )
shift
fi
;;
--prep | --configure | --compile | --install | --all )
opt_build_target=${1:2}
;;
--update-modulefiles )
opt_update_modulefiles='yes'
;;
--config-file | --config-file=* )
if [[ $1 == *=* ]]; then
opt_config_file=( "${1#--*=}" )
else
opt_config_file=( "$2" )
shift
fi
;;
-- )
:
;;
-* )
std::die 1 "Invalid option -- '$1'"
;;
[=0-9]* | '.*' )
versions+=( "$1" )
;;
'')
:
;;
* )
[[ -z "${BUILD_SCRIPT}" ]] || \
std::die 1 "%s "\
"Build script already set to" \
"'${BUILD_SCRIPT}' -- '$1'"
BUILD_SCRIPT=$(std::get_abspath "$1")
test -r ${BUILD_SCRIPT} || \
std::die 1 "%s " \
"Build script does not exist" \
"or is not readable -- '$_'"
BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}")
;;
esac
shift
done
if [[ -z ${BUILD_SCRIPT} ]]; then
if [[ -r "${PWD}/build" ]]; then
if grep -q '#!.* modbuild' "${PWD}/build"; then
BUILD_SCRIPT="${PWD}/build"
BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}")
fi
fi
if [[ -z ${BUILD_SCRIPT} ]]; then
std::die 1 "Don't know what to build!"
fi
fi
(( ${#versions[@]} > 0)) || versions+=( '.*' )
if [[ ${opt_yaml} == 'no' ]]; then
# look for legacy config files
# if nothing found, try with YAML
if [[ -n ${opt_config_file} ]]; then
legacy_config_files=( "${opt_config_file}" )
else
shopt -q nullglob || :
local -i nullglob_set=$?
legacy_config_files=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}".${opt_system} )
legacy_config_files+=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}.$(uname -s)" )
local f
for f in "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}"; do
[[ -e "${f}.${opt_system}" ]] \
|| [[ -e "${f}.$(uname -s)" ]] \
|| legacy_config_files+=( "$f" )
done
(( nullglob_set == 1 )) && shopt -u nullglob
fi
(( ${#legacy_config_files[@]} == 0 )) && opt_yaml='check'
fi
if [[ ${opt_yaml} != 'no' ]]; then
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
fi
if [[ "${opt_yaml}" == 'yes' ]]; then
[[ -r "${yaml_config_file}" ]] || \
std::die 2 \
"YAML config file doesn't exist or is not readable - ${yaml_config_file}"
else
if [[ "${opt_yaml}" == 'check' ]]; then
[[ -r "${yaml_config_file}" ]] && \
std::die 2 \
"No suitable variants or YAML config file found!"
opt_yaml='yes'
else
(( ${#legacy_config_files[@]} == 0 )) && \
std::die 2 \
"No suitable variants file found!"
fi
fi
if [[ "${opt_yaml}" == 'yes' ]]; then
std::info "Using YAML configuration file - ${yaml_config_file}"
else
std::info "Using legacy variants files."
[[ -n "${opt_overlay}" ]] || opt_overlay='base'
fi
}
#
# bash brace expansion of given args. Input args like:
#
# "text" "gcc/{9.3.0,10.3.0}" "openmpi/{4.0.5,4.1.0}"
#
# will be expanded to the following four lines:
#
# "text gcc/9.3.0 openmpi/4.0.5"
# "text gcc/9.3.0 openmpi/4.1.0"
# "text gcc/10.3.0 openmpi/4.0.5"
# "text gcc/10.3.0 openmpi/4.1.0"
#
bash_expand(){
local text="$1"
shift
local to_expand=( "${@}" )
if (( ${#to_expand[@]} == 0 )); then
echo ${text}
else
local list
eval list=( ${to_expand[0]} )
local s
for s in ${list[*]}; do
bash_expand "${text} ${s}" "${to_expand[@]:1}"
done;
fi;
}
build_modules_legacy() {
local -a files=( "${legacy_config_files[@]}" )
local -A mod_overlays=()
expand_variants_file(){
local -r input="$1"
local -a toks=()
while read -a toks; do
# skip empty and comment lines
[[ -z ${toks} ]] && continue
[[ ${toks:0:1} == '#' ]] && continue
local -a deps=( ${toks[*]:2} )
bash_expand "${toks[0]} ${toks[1]}" "${deps[@]}"
done < "${input}"
}
local name="$1"
local version="$2"
local exact_match='no'
if [[ "${version:0:1}" == "=" ]]; then
exact_match='yes'
version="${version:1}"
fi
shift 2
local with_modules=( $* )
# if we have to build a dependency, we might have less dependencies
# on it. Or in other words: the list of "with modules" might be
# overdetermined. In the loop below we check, which dependencies
# specified with '--with' are required.
local m
local pattern="/^${name}\/${version}[[:blank:]]/"
for m in "${with_modules[@]}"; do
if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then
pattern+=" && /${m//\//\\/}/"
fi
done
local variants=()
for f in "${files[@]}"; do
local line=''
while read line; do
variants+=( "${line}" )
done < <(expand_variants_file "${f}" | ${awk} "${pattern}")
done
if (( ${#variants[@]} == 0 )); then
std::info "%s " \
"${name}/${version}:" \
"no suitable variant found!"
std::die 10 "Aborting..."
elif (( ${#variants[@]} > 1 )) && [[ ${exact_match} == 'yes' ]]; then
std::info "%s " \
"Multiple variants found:"
for variant in "${variants[@]}"; do
std::info "${variant}"
done
std::die 10 "Aborting..."
fi
[[ -v OverlayInfo[${opt_overlay}:inst_root] ]] || \
std::die 2 "%s" \
"Overlay doesn't exist - ${opt_overlay}"
local ol_name="${opt_overlay}"
declare ol_inst_root="${OverlayInfo[${opt_overlay}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${opt_overlay}:mod_root]}"
local -i i=0
local -i num_variants=${#variants[@]}
for ((i = 0; i < num_variants; i++)); do
local tokens=( ${variants[i]} )
local name="${tokens[0]%/*}"
version="${tokens[0]#*/}"
release="${tokens[1]}"
case ${release} in
unstable|stable|deprecated|remove|removed)
:
;;
* )
std::info "%s " \
"${name}/${version}:" \
"invalid release stage '${release}'!"
std::die 10 "Aborting..."
;;
esac
with_modules=( "${tokens[@]:2}" )
pbuild.build_module_legacy \
"${name}" "${version}" \
"${release}" "${with_modules[@]}"
done
}
build_modules_yaml(){
local -- fname="${yaml_config_file}"
local -A mod_overlays=()
yaml_get_versions(){
local -n _result="$1"
local fname="$2"
local version="$3"
_result=( $(${yq} -Ne e \
"with_entries(select(.key | test(\"^${version}\$\")))|keys" \
"${fname}" 2>/dev/null | ${awk} '{print $2}') )
}
yaml_get_num_variants(){
local -n _result="$1"
local fname="$2"
local version="$3"
_result=$(${yq} -Ne e ".\"${version}\"|length" \
"${fname}" 2>/dev/null)
if (( $? != 0 )); then
_result=0
fi
}
yaml_get_relstage(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=$(${yq} -Ne e ".\"${version}\"[${idx}].relstage" \
"${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result=$(${yq} -Ne e ".relstage" "${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result='unstable'
}
yaml_get_overlay(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=$(${yq} -Ne e ".\"${version}\"[${idx}].overlay" \
"${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result=$(${yq} -Ne e ".overlay" "${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result='base'
}
yaml_get_group(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=$(${yq} -Ne e ".\"${version}\"[${idx}].group" \
"${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result=$(${yq} -Ne e ".group" "${fname}" 2>/dev/null)
(( $? == 0 )) && return
_result=''
}
yaml_get_systems(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=( $(${yq} -Ne e ".\"${version}\"[${idx}].systems" \
"${fname}" 2>/dev/null) )
(( $? == 0 )) && return
_result=( $(${yq} -Ne e ".systems" "${fname}" 2>/dev/null) )
(( $? == 0 )) && return
_result=()
}
yaml_get_compilers(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=( $(${yq} -Ne e ".\"${version}\"[${idx}].compilers" \
"${fname}" 2>/dev/null) )
(( $? == 0 )) && return
_result=( $(${yq} -Ne e ".compilers" "${fname}" 2>/dev/null) )
(( $? == 0 )) && return
_result=()
}
yaml_get_dependencies(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=( $(${yq} -Ne e ".\"${version}\"[${idx}]|(.with, .dependencies)" \
"${fname}" 2>/dev/null) )
if (( $? != 0 )); then
# neither .with nor .dependencies are set
_result=()
return
fi
# if one of .with, .dependencies is not set, the vaulue is
# returned as 'null'.
local -i i
for i in "${!_result[@]}"; do
[[ ${_result[$i]} == 'null' ]] && unset -v '_result[$i]'
done
_result=("${_result[@]}")
}
yaml_get_build_requirements(){
local -n _result="$1"
local fname="$2"
local version="$3"
local idx="$4"
_result=( $(${yq} -Ne e ".\"${version}\"[${idx}].build_requires" \
"${fname}" 2>/dev/null) )
if (( $? != 0 )); then
_result=()
return
fi
}
local name="$1"
local version="$2"
shift 2
local with_modules=( $* )
local m
local pattern="//"
for m in "${with_modules[@]}"; do
pattern+=" && /${m//\//\\/}/"
done
local -a versions
yaml_get_versions versions "${fname}" "${name}/${version}"
for v in "${versions[@]}"; do
local -i n_variants
yaml_get_num_variants n_variants "${fname}" "${v}"
(( n_variants == 0 )) && (( n_variants = 1 ))
local -i i
local -a deps=()
local -a build_requires=()
local relstage
local group
local ol_name
for (( i=0; i<n_variants; i++)); do
if [[ -z ${opt_overlay} ]]; then
yaml_get_overlay ol_name "${fname}" "${v}" $i
else
ol_name="${opt_overlay}"
fi
if [[ ! -v OverlayInfo[${ol_name}:inst_root] ]]; then
std::die 3 "Overlay is not defined -- ${ol_name}"
fi
mod_overlays[$i]="${ol_name}"
done
for (( i=0; i<n_variants; i++)); do
ol_name="${mod_overlays[$i]}"
yaml_get_relstage relstage "${fname}" "${v}" $i
yaml_get_dependencies deps "${fname}" "${v}" $i
yaml_get_build_requirements build_requires "${fname}" "${v}" $i
yaml_get_group group "${fname}" "${v}" $i
pbuild.add_to_group "${group}"
# :FIXME:
# for the time being we prefix the build requirements
# with 'b:' and append them to the other dependencies.
# With this solution we don't have to change anything
# else in the code but have better readable variant
# files.
deps+=("${build_requires[@]/#/b:}")
declare ol_inst_root="${OverlayInfo[${ol_name}:inst_root]}"
declare ol_mod_root="${OverlayInfo[${ol_name}:mod_root]}"
local -a systems=()
local -a compilers=()
yaml_get_systems systems "${fname}" "${v}" $i
yaml_get_compilers compilers "${fname}" "${v}" $i
pbuild.supported_systems "${systems[@]}"
pbuild.supported_compilers "${compilers[@]}"
if (( ${#deps[@]} > 0 )); then
while read -a with_modules; do
pbuild.build_module_yaml \
"${name}" "${v##*/}" \
"${relstage}" "${with_modules[@]}"
done < <(bash_expand "" ${deps[@]}|${awk} "${pattern}")
else
pbuild.build_module_yaml \
"${name}" "${v##*/}" \
"${relstage}"
fi
done
done
}
build_modules() {
if [[ "${opt_yaml}" == 'yes' ]]; then
build_modules_yaml "$@"
else
build_modules_legacy "$@"
fi
}
#.............................................................................
# main
parse_args "$@"
opt_system="${opt_system:-$(std::get_os_release)}"
pbuild.jobs "${opt_jobs}"
pbuild.force_rebuild "${opt_force_rebuild}"
pbuild.build_target "${opt_build_target}"
pbuild.dry_run "${opt_dry_run}"
pbuild.enable_cleanup_build "${opt_enable_cleanup_build}"
pbuild.enable_cleanup_src "${opt_enable_cleanup_src}"
pbuild.update_modulefiles "${opt_update_modulefiles}"
pbuild.system "${opt_system}"
pbuild.verbose "${opt_verbose}"
#
# read configuration for modbuild
#
pm::read_config
# :FIXME: should dist files go to
# ${pm_root}/var/distfiles
# or
# ${overlay}/var/distfiles
# ?
: ${PMODULES_DISTFILESDIR:="${PMODULES_HOME%%/Tools*}/var/distfiles"}
: ${PMODULES_TMPDIR:=/var/tmp/${USER}}
declare -r BUILD_SCRIPT
declare -r BUILDBLOCK_DIR
# the module name is defined by the directory the build script is in
IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}"
module_name=${fname[${#fname[@]}-2]}
for version in "${versions[@]}"; do
build_modules "${module_name}" "${version}" "${opt_with_modules[@]}"
done
# Local Variables:
# mode: sh
# sh-basic-offset: 8
# tab-width: 8
# End:
Reference in New Issue View Git Blame Copy Permalink