diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 4feab50..72f3fd3 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1,22 +1,5 @@ #!/bin/bash -#............................................................................. -# -# We need GNU versions of the following utilities. This code works -# well on Linux and Mac OS X with MacPorts. -# :FIXME: implement a smarter, portable solution. -# -shopt -s expand_aliases -unalias -a - -__path=$(which gsed 2>/dev/null || : ) -if [[ $__path ]]; then - alias sed=$__path -else - alias sed=$(which sed 2>/dev/null) -fi -unset __path - #............................................................................. # disable auto-echo feature of 'cd' unset CDPATH @@ -28,8 +11,9 @@ declare -r FNAME_RDEPS='.dependencies' declare -r FNAME_IDEPS='.install_dependencies' declare -r FNAME_BDEPS='.build_dependencies' -#............................................................................. -declare -A SOURCE_UNPACK_DIRS +# relative path of documentation +# abs. path is "${PREFIX}/${_docdir}/${module_name}" +declare -r _DOCDIR='share/doc' #............................................................................. # @@ -39,15 +23,104 @@ declare -A SOURCE_UNPACK_DIRS # #set -o errexit -error_handler() { +_error_handler() { local -i ec=$? std::die ${ec} "Oops" } +readonly -f _error_handler -trap "error_handler" ERR +trap "_error_handler" ERR -declare configure_with='undef' +#.............................................................................. +# +# write number of cores to stdout +# +_get_num_cores() { + case "${OS}" in + Linux ) + ${grep} -c ^processor /proc/cpuinfo + ;; + Darwin ) + ${sysctl} -n hw.ncpu + ;; + * ) + std::die 1 "OS ${OS} is not supported\n" + ;; + esac +} +readonly -f _get_num_cores + +#.............................................................................. +# global variables which can be set/overwritten by command line args +# and their corresponding functions +# +declare force_rebuild='' +pbuild.force_rebuild() { + force_rebuild="$1" +} +readonly -f pbuild.force_rebuild + +declare dry_run='' +pbuild.dry_run() { + dry_run="$1" +} +readonly -f pbuild.dry_run + +declare enable_cleanup_build='' +pbuild.enable_cleanup_build() { + enable_cleanup_build="$1" +} +readonly -f pbuild.enable_cleanup_build + +declare enable_cleanup_src='' +pbuild.enable_cleanup_src() { + enable_cleanup_src="$1" +} +readonly -f pbuild.enable_cleanup_src + +declare build_target='' +pbuild.build_target() { + build_target="$1" +} +readonly -f pbuild.build_target + +declare opt_update_modulefiles='' +pbuild.update_modulefiles() { + opt_update_modulefiles="$1" +} +readonly -f pbuild.update_modulefiles + +# number of parallel make jobs +declare -i JOBS=0 +pbuild.jobs() { + if (( $1 == 0 )); then + JOBS=$(_get_num_cores) + (( JOBS > 10 )) && JOBS=10 || : + else + JOBS="$1" + fi +} +readonly -f pbuild.jobs + +declare system='' +pbuild.system() { + system="$1" +} +readonly -f pbuild.system + +declare verbose='' +pbuild.verbose() { + verbose="$1" +} +readonly -f pbuild.verbose + + +############################################################################### +# +# function in the "namespace" (with prefix) 'pbuild::' can be used in +# build-scripts +# #.............................................................................. # @@ -81,120 +154,53 @@ pbuild::version_compare () { done return 0 } +readonly -f pbuild::version_compare +#.............................................................................. +# version less than +# +# return 0 if version passed in $1 is older then $2 +# pbuild::version_lt() { pbuild::version_compare "$1" "$2" (( $? == 2 )) } +readonly -f pbuild::version_lt +#.............................................................................. +# version less than or equal +# +# return 0 if version passed in $1 is older or equal then $2 +# pbuild::version_le() { pbuild::version_compare "$1" "$2" local -i exit_code=$? (( exit_code == 0 || exit_code == 2 )) } +readonly -f pbuild::version_le - +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 is newer then $2 +# pbuild::version_gt() { pbuild::version_compare "$1" "$2" (( $? == 1 )) local -i exit_code=$? (( exit_code == 0 || exit_code == 1 )) } +readonly -f pbuild::version_gt +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 and $2 are equal +# pbuild::version_eq() { pbuild::version_compare "$1" "$2" } - -#.............................................................................. -# -# The following variables are available in build-blocks and set read-only -# :FIXME: do we have to export them? - -# install prefix of module. -declare -x PREFIX='' - -declare -r OS=$(uname -s) - -pbuild::get_num_cores() { - case "${OS}" in - Linux ) - echo $(grep -c ^processor /proc/cpuinfo) - ;; - Darwin ) - echo $(sysctl -n hw.ncpu) - ;; - * ) - std::die 1 "OS ${OS} is not supported\n" - ;; - esac -} - -#.............................................................................. -# global variables which can be set/overwritten by command line args - -declare force_rebuild='' -pbuild.force_rebuild() { - force_rebuild="$1" -} - -declare dry_run='' -pbuild.dry_run() { - dry_run="$1" -} - -declare enable_cleanup_build='' -pbuild.enable_cleanup_build() { - enable_cleanup_build="$1" -} - -declare enable_cleanup_src='' -pbuild.enable_cleanup_src() { - enable_cleanup_src="$1" -} - -declare build_target='' -pbuild.build_target() { - build_target="$1" -} - -declare opt_update_modulefiles='' -pbuild.update_modulefiles() { - opt_update_modulefiles="$1" -} - -# number of parallel make jobs -declare -i JOBS=$(pbuild::get_num_cores) -pbuild.jobs() { - if (( $1 == 0 )); then - JOBS=$(pbuild::get_num_cores) - (( JOBS > 10 )) && JOBS=10 || : - else - JOBS="$1" - fi -} - -declare system='' -pbuild.system() { - system="$1" -} - -declare verbose='' -pbuild.verbose() { - verbose="$1" -} - - -# group this module is in (ex: 'Programming') -declare -x GROUP='' - -# name, version and release of module -declare -x module_name='' -declare -x module_version='' -declare -x module_release='' - -# relative path of documentation -# abs. path is "${PREFIX}/${_docdir}/${module_name}" -declare -r _DOCDIR='share/doc' +readonly -f pbuild::version_eq ############################################################################## # @@ -206,6 +212,7 @@ declare -r _DOCDIR='share/doc' pbuild::compile_in_sourcetree() { BUILD_DIR="${SRC_DIR}" } +readonly -f pbuild::compile_in_sourcetree ############################################################################## # @@ -218,6 +225,7 @@ pbuild::compile_in_sourcetree() { pbuild::supported_systems() { SUPPORTED_SYSTEMS+=( "$@" ) } +readonly -f pbuild::supported_systems ############################################################################## # @@ -230,6 +238,7 @@ pbuild::supported_systems() { pbuild::supported_os() { SUPPORTED_OS+=( "$@" ) } +readonly -f pbuild::supported_os ############################################################################## # @@ -242,6 +251,7 @@ pbuild::supported_os() { pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } +readonly -f pbuild::supported_compilers ############################################################################## # @@ -258,6 +268,7 @@ pbuild::add_to_group() { fi GROUP="$1" } +readonly -f pbuild::add_to_group ############################################################################## # @@ -269,6 +280,7 @@ pbuild::add_to_group() { pbuild::install_docfiles() { MODULE_DOCFILES+=("$@") } +readonly -f pbuild::install_docfiles ############################################################################## # @@ -277,45 +289,85 @@ pbuild::install_docfiles() { # # Arguments: # $1: module name -# $2: optional variable name to return release via upvar +# $2: optional variable name to return release # # Notes: -# The passed module name must be NAME/VERSION! +# The passed module name must be module/version! +# +# Exit codes: +# 0 if module/version is available +# 1 otherwise # pbuild::module_is_avail() { - local "$2" - local uvar="$2" - [[ -n "${uvar}" ]] || uvar="__unused__" local output=( $("${MODULECMD}" bash avail -a -m "$1" \ 2>&1 1>/dev/null) ) local i for (( i = 0; i < ${#output[@]}; i += 2 )); do if [[ "${output[$i]}" == "$1" ]]; then - std::upvar "${uvar}" "${output[i+1]}" + if (( $# > 1 )); then + local -n _result="$2" + _result="${output[i+1]}" + fi return 0 fi done return 1 } +readonly -f pbuild::module_is_avail +############################################################################## +# +# Set the download URL and name of downloaded file. +# +# Arguments: +# $1 download URL +# $2 optional file-name (of) pbuild::set_download_url() { local -i _i=${#SOURCE_URLS[@]} SOURCE_URLS[_i]="$1" - if (( $# == 2 )); then - SOURCE_NAMES[$_i]="$2" + if (( $# > 1 )); then + SOURCE_NAMES[$_i]="${2:-${1##*/}}" else - SOURCE_NAMES[$_i]='' + SOURCE_NAMES[$_i]="${1##*/}" fi } +readonly -f pbuild::set_download_url +############################################################################## +# +# Set hash sum for file. +# +# Arguments: +# $1 filen-name:hash-sum +# +# :FIXME: +# Maybe we should use a dictionary in the future. +# pbuild::set_sha256sum() { SOURCE_SHA256_SUMS+=("$1") } +readonly -f pbuild::set_sha256sum +############################################################################## +# +# Unpack file $1 in directory $2 +# +# Arguments: +# $1 file-name +# $2 directory +# pbuild::set_unpack_dir() { SOURCE_UNPACK_DIRS[$1]=$2 } +readonly -f pbuild::set_unpack_dir +############################################################################## +# +# Use this C-compiler +# +# Arguments: +# $1 C-compiler to use. +# pbuild::use_cc() { [[ -x "$1" ]] || std::die 3 \ "%s " "${module_name}/${module_version}:" \ @@ -323,14 +375,59 @@ pbuild::use_cc() { "'$1' is not an executable!" CC="$1" } +readonly -f pbuild::use_cc -pbuild::pre_prep() { - : +############################################################################### +# +pbuild::add_patch() { + [[ -z "$1" ]] && \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + PATCH_FILES+=( "$1" ) + PATCH_STRIPS+=( "$2" ) } +readonly -f pbuild::add_patch -pbuild::post_prep() { - : +############################################################################### +# +pbuild::set_default_patch_strip() { + [[ -n "$1" ]] || \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + + PATCH_STRIP_DEFAULT="$1" } +readonly -f pbuild::set_default_patch_strip + +############################################################################### +# +pbuild::use_flag() { + [[ "${USE_FLAGS}" =~ ":${1}:" ]] +} +readonly -f pbuild::use_flag + +############################################################################### +# +pbuild::add_configure_args() { + CONFIGURE_ARGS+=( "$@" ) +} +readonly -f pbuild::add_configure_args + +############################################################################### +# +pbuild::use_autotools() { + configure_with='autotools' +} +readonly -f pbuild::use_autotools + +############################################################################### +# +pbuild::use_cmake() { + configure_with='cmake' +} +readonly -f pbuild::use_cmake ############################################################################### # @@ -339,63 +436,58 @@ pbuild::post_prep() { pbuild::prep() { #...................................................................... # - # Find/download tarball for given module. + # download the source file if not already downloaded and validate + # checksum (if known). + # Abort on any error! # # Arguments: - # $1: store file name with upvar here - # $2: download URL - # $3: output filename (can be empty string) - # $4...: download directories + # $1 reference varibale to return result + # $2 download URL + # $3 save downloaded file with this name. If the empty + # string is passed, derive file name from URL + # $4... directories the source file might be already in. If the + # file does not exist in one of these directories, it + # is downloaded and stored in the first given directory. # - # Returns: - # 0 on success otherwise a value > 0 - # - download_with_curl() { - local -r output="$1" - local -r url="$2" - curl \ - -L \ - --output "${output}" \ - "${url}" - if (( $? != 0 )); then - curl \ - --insecure \ + download_source_file() { + download_with_curl() { + local -r output="$1" + local -r url="$2" + ${curl} \ + --location \ + --fail \ + --remove-on-error \ --output "${output}" \ "${url}" - fi - } + # :FIXME: How to handle insecure downloads? + #if (( $? != 0 )); then + # curl \ + # --insecure \ + # --output "${output}" \ + # "${url}" + #fi + } - check_hash_sum() { - local -r fname="$1" - local -r expected_hash_sum="$2" - local hash_sum='' + check_hash_sum() { + local -r fname="$1" + local -r expected_hash_sum="$2" + local hash_sum='' - if which 'sha256sum' 1>/dev/null; then - hash_sum=$(sha256sum "${fname}" | awk '{print $1}') - elif which 'shasum' 1>/dev/null; then - hash_sum=$(shasum -a 256 "${fname}" | awk '{print $1}') - else - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "Binary to compute SHA256 sum missing!" - fi - test "${hash_sum}" == "${expected_hash_sum}" || \ - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "hash-sum missmatch for file '%s'" "${fname}" - } + hash_sum=$(${sha256sum} "${fname}" | awk '{print $1}') + test "${hash_sum}" == "${expected_hash_sum}" || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "hash-sum missmatch for file '${fname}'!" + } - download_source_file() { - local "$1" - local var="$1" + local -n _result="$1" local -r url="$2" local fname="$3" shift 3 dirs+=( "$@" ) [[ -n "${fname}" ]] || fname="${url##*/}" - local expr='s/.*\(.tar.bz2\|.tbz2\|.tar.gz\|.tgz\|.tar.xz\|.zip\)/\1/' - local -r extension=$(echo ${fname} | sed "${expr}") local dir='' dirs+=( 'not found' ) for dir in "${dirs[@]}"; do @@ -403,19 +495,20 @@ pbuild::prep() { done if [[ "${dir}" == 'not found' ]]; then dir="${dirs[0]}" - local -r method="${url%:*}" - case "${method}" in - http | https | ftp ) - download_with_curl "${dir}/${fname}" "${url}" - ;; - * ) - std::die 4 \ - "%s " "${module_name}/${module_version}:" \ - "Error in download URL:" \ - "unknown download method '${method}'!" - ;; - esac + download_with_curl "${dir}/${fname}" "${url}" + (( $? == 0 )) || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "downloading source file '${fname}' failed!" fi + _result="${dir}/${fname}" + [[ -r "${_result}" ]] || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "source file '${_result}' is not readable!" + local sha256_sum='' local hash='' for hash in "${SOURCE_SHA256_SUMS[@]}"; do @@ -426,15 +519,13 @@ pbuild::prep() { if [[ -n "${sha256_sum}" ]]; then check_hash_sum "${dir}/${fname}" "${sha256_sum}" fi - std::upvar "${var}" "${dir}/${fname}" - [[ -r "${dir}/${fname}" ]] } unpack() { local -r file="$1" local -r dir="${2:-${SRC_DIR}}" - tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || { - rm -f "${file}" + ${tar} --directory="${dir}" -xv --strip-components 1 -f "${file}" || { + ${rm} -f "${file}" std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -451,7 +542,7 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" - patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ + ${patch} -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -470,7 +561,7 @@ pbuild::prep() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "Download source not set!" - mkdir -p "${PMODULES_DISTFILESDIR}" + ${mkdir} -p "${PMODULES_DISTFILESDIR}" local i=0 local source_fname for ((i = 0; i < ${#SOURCE_URLS[@]}; i++)); do @@ -483,59 +574,29 @@ pbuild::prep() { std::die 4 \ "%s " "${module_name}/${module_version}:" \ "sources for not found." - local dir=${SOURCE_UNPACK_DIRS[${source_fname##*/}]:-''} + local dir='' + local key="${SOURCE_URLS[i]##*/}" + if [[ -v SOURCE_UNPACK_DIRS[${key}] ]]; then + echo "dir specified" + dir="${SOURCE_UNPACK_DIRS[${SOURCE_URLS[i]##*/}]}" + else + echo "use SRC_DIR" + dir="${SRC_DIR}" + fi unpack "${source_fname}" "${dir}" done patch_sources # create build directory - mkdir -p "${BUILD_DIR}" -} - -pbuild::add_patch() { - [[ -z "$1" ]] && \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - PATCH_FILES+=( "$1" ) - PATCH_STRIPS+=( "$2" ) -} - -pbuild::set_default_patch_strip() { - [[ -n "$1" ]] || \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - - PATCH_STRIP_DEFAULT="$1" -} - -pbuild::use_flag() { - [[ "${USE_FLAGS}" =~ ":${1}:" ]] + ${mkdir} -p "${BUILD_DIR}" } ############################################################################### # +# Configure the software to be compiled. +# +# Arguments: +# none # -pbuild::pre_configure() { - : -} - -pbuild::set_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::add_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::use_autotools() { - configure_with='autotools' -} - -pbuild::use_cmake() { - configure_with='cmake' -} - pbuild::configure() { case "${configure_with}" in autotools ) @@ -567,6 +628,7 @@ pbuild::configure() { elif [[ -r "${SRC_DIR}/CMakeLists.txt" ]] && \ [[ "${configure_with}" == 'undef' ]] || \ [[ "${configure_with}" == "cmake" ]]; then + # note: in most/many cases a cmake module is used! cmake \ -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ "${CONFIGURE_ARGS[@]}" \ @@ -582,36 +644,38 @@ pbuild::configure() { fi } -pbuild::post_configure() { - : -} - -pbuild::pre_compile() { - : -} +############################################################################### +# +# Default compile function. +# +# Arguments: +# none +# pbuild::compile() { - make -j${JOBS} || \ + (( JOBS == 0 )) && JOBS=$(_get_num_cores) + ${make} -j${JOBS} || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } -pbuild::post_compile() { - : -} - -pbuild::pre_install() { - : -} - +############################################################################### +# +# Default install function. +# +# Arguments: +# none +# pbuild::install() { - make install || \ + ${make} install || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } +############################################################################### +# pbuild::install_shared_libs() { local -r binary="$1" local -r dstdir="$2" @@ -619,19 +683,19 @@ pbuild::install_shared_libs() { install_shared_libs_Linux() { local libs=( $(ldd "${binary}" | \ - awk "/ => \// && /${pattern}/ {print \$3}") ) + ${awk} "/ => \// && /${pattern}/ {print \$3}") ) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } install_shared_libs_Darwin() { # https://stackoverflow.com/questions/33991581/install-name-tool-to-update-a-executable-to-search-for-dylib-in-mac-os-x - local libs=( $(otool -L "${binary}" | \ - awk "/${pattern}/ {print \$1}")) + local libs=( $(${otool} -L "${binary}" | \ + ${awk} "/${pattern}/ {print \$1}")) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } @@ -640,7 +704,7 @@ pbuild::install_shared_libs() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "${binary}: does not exist or is not executable!" - mkdir -p "${dstdir}" + ${mkdir} -p "${dstdir}" case "${OS}" in Linux ) install_shared_libs_Linux @@ -651,48 +715,292 @@ pbuild::install_shared_libs() { esac } -pbuild::post_install() { - : -} - +############################################################################### # -# The 'do it all' function. +# This is the main entry function called by modbuild! # -pbuild::make_all() { - source "${BUILD_SCRIPT}" +pbuild.build_module() { + declare -gx module_name="$1" + declare -gx module_version="$2" + declare -gx module_release="$3" + shift 3 + with_modules=( "$@" ) - #set -o errexit - local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" - # module name including path in hierarchy and version - # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) - local modulefile_dir='' - local modulefile_name='' + # used in _make_all + declare -a runtime_dependencies=() + declare -a install_dependencies=() + #...................................................................... # - # To be able to set environment variables in one of the 'pbuild::TARGET' - # function we cannot use PIPE's like - # pbuild::configure | tee -a ... + # test whether a module is loaded or not # - rm -f "${logfile}" - if [[ "${verbose}" == 'yes' ]]; then - exec > >(tee -a "${logfile}") - else - exec > >(cat >> "${logfile}") - fi - exec 2> >(tee -a "${logfile}" >&2) + # Arguments: + # $1 module name + # + is_loaded() { + [[ :${LOADEDMODULES}: =~ :$1: ]] + } + #...................................................................... # - # everything set up? + # Initialise environment modules. # - [[ -n ${GROUP} ]] || \ - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "Module group not set! Aborting ..." + # Arguments: + # none + # + init_module_environment(){ + eval $( "${MODULECMD}" bash use unstable ) + eval $( "${MODULECMD}" bash use deprecated ) + eval $( "${MODULECMD}" bash purge ) + # :FIXME: this is a hack!!! + # shouldn't this be set in the build-script? + if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then + eval $( "${MODULECMD}" bash use Libraries ) + fi + if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then + eval $( "${MODULECMD}" bash use System ) + fi + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + unset CPP_INCLUDE_PATH + unset LIBRARY_PATH + unset LD_LIBRARY_PATH + unset DYLD_LIBRARY_PATH + + unset CFLAGS + unset CPPFLAGS + unset CXXFLAGS + unset LIBS + unset LDFLAGS + unset CC + unset CXX + unset FC + unset F77 + unset F90 + } + + #...................................................................... # - # helper functions + # Load build- and run-time dependencies. # + # Arguments: + # none + # + # Variables + # module_release set if defined in a variants file + # runtime_dependencies runtime dependencies from variants added + # + load_build_dependencies() { + + #.............................................................. + # + # build a dependency + # + # $1: name of module to build + # + # :FIXME: needs testing + # + build_dependency() { + find_build_script(){ + local p=$1 + local script=$(${find} "${BUILDBLOCK_DIR}/../.." \ + -path "*/$p/build") + std::get_abspath "${script}" + } + + local -r m=$1 + std::debug "${m}: module not available" + [[ ${dry_run} == yes ]] && \ + std::die 1 \ + "%s " \ + "${m}: module does not exist," \ + "cannot continue with dry run..." + + std::info "%s " \ + "$m: module does not exist, trying to build it..." + local args=( '' ) + set -- ${ARGS[@]} + while (( $# > 0 )); do + case $1 in + -j ) + args+=( "-j $2" ) + shift + ;; + --jobs=[0-9]* ) + args+=( $1 ) + ;; + -v | --verbose) + args+=( $1 ) + ;; + --with=*/* ) + args+=( $1 ) + ;; + esac + shift + done + + local buildscript=$(find_build_script "${m%/*}") + [[ -x "${buildscript}" ]] || \ + std::die 1 \ + "$m: build-block not found!" + if ! "${buildscript}" "${m#*/}" ${args[@]}; then + std::die 1 \ + "$m: oops: build failed..." + fi + } + + local m='' + for m in "${with_modules[@]}"; do + + # module name prefixes in dependency declarations: + # 'b:' this is a build dependency + # 'r:' this a run-time dependency, *not* required for + # building + # without prefix: this is a build and + # run-time dependency + if [[ "${m:0:2}" == "b:" ]]; then + m=${m#*:} # remove 'b:' + elif [[ "${m:0:2}" == "r:" ]]; then + m=${m#*:} # remove 'r:' + runtime_dependencies+=( "$m" ) + elif [[ "${m:0:2}" == "R:" ]]; then + m=${m#*:} # remove 'R:' + install_dependencies+=( "$m" ) + continue + else + runtime_dependencies+=( "$m" ) + fi + is_loaded "$m" && continue + + # 'module avail' might output multiple matches if module + # name and version are not fully specified or in case + # modules with and without a release number exist. + # Example: + # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list + # from 'module avail' and the full version should be set + # in the variants file, we look for the first exact + # match. + local release_of_dependency='' + if ! pbuild::module_is_avail "$m" release_of_dependency; then + build_dependency "$m" + pbuild::module_is_avail "$m" release_of_dependency || \ + std::die 6 "Oops" + fi + # should be set, just in case it is not... + : ${release_of_dependency:='unstable'} + + # for a stable module all dependencies must be stable + if [[ "${module_release}" == 'stable' ]] \ + && [[ "${release_of_dependency}" != 'stable' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + # for a unstable module no dependency must be deprecated + elif [[ "${module_release}" == 'unstable' ]] \ + && [[ "${release_of_dependency}" == 'deprecated' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + fi + + std::info "Loading module: ${m}" + eval $( "${MODULECMD}" bash load "${m}" ) + done + } + + init_build_environment() { + #...................................................................... + # + # parse the passed version string + # + # the following global variables will be set in this function: + # V_MAJOR + # V_MINOR + # V_PATCHLVL + # V_RELEASE + # USE_FLAGS + # + parse_version() { + local v="$1" + V_MAJOR='' # first number in version string + V_MINOR='' # second number in version string (or empty) + V_PATCHLVL='' # third number in version string (or empty) + V_RELEASE='' # module release (or empty) + : ${USE_FLAGS:=''} # architectures (or empty) + + local tmp='' + + if [[ "$v" =~ "_" ]]; then + tmp="${v#*_}" + USE_FLAGS+=":${tmp//_/:}:" + v="${v%%_*}" + fi + V_PKG="${v%%-*}" # version without the release number + if [[ $v == *-* ]]; then + V_RELEASE="${v#*-}" # release number + else + V_RELEASE='' + fi + case "${V_PKG}" in + *.*.* ) + V_MAJOR="${V_PKG%%.*}" + tmp="${V_PKG#*.}" + V_MINOR="${tmp%%.*}" + V_PATCHLVL="${tmp#*.}" + ;; + *.* ) + V_MAJOR="${V_PKG%.*}" + V_MINOR="${V_PKG#*.}" + ;; + * ) + V_MAJOR="${V_PKG}" + ;; + esac + + VERSIONS=() + if [[ -n ${V_RELEASE} ]]; then + VERSIONS+=( ${V_PKG}-${V_RELEASE} ) + fi + if [[ -n ${V_PATCHLVL} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ) + fi + if [[ -n ${V_MINOR} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR} ) + fi + VERSIONS+=( ${V_MAJOR} ) + } + + local -r module_name="$1" + local -r module_version="$2" + + SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" + BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" + + # P and V can be used in the build-script, so we have to set them here + P="${module_name}" + V="${module_version}" + parse_version "${module_version}" + declare -gx GROUP='' + declare -g PREFIX='' + + SOURCE_URLS=() + SOURCE_SHA256_SUMS=() + SOURCE_NAMES=() + declare -Ag SOURCE_UNPACK_DIRS=() + CONFIGURE_ARGS=() + SUPPORTED_SYSTEMS=() + SUPPORTED_OS=() + SUPPORTED_COMPILERS=() + PATCH_FILES=() + PATCH_STRIPS=() + PATCH_STRIP_DEFAULT='1' + MODULE_DOCFILES=() + configure_with='undef' + } # init_build_environment() #...................................................................... check_supported_systems() { @@ -762,19 +1070,23 @@ pbuild::make_all() { else unset Compiler_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v HDF5_SERIAL_VERSION ]]; then HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' else unset HDF5_serial_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]]; then MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' else unset MPI_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]] && \ + [[ HDF5_VERSION ]]; then HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' @@ -821,110 +1133,57 @@ pbuild::make_all() { else do_hierarchical_group fi - } + } # set_full_module_name_and_prefix #...................................................................... - # Select the modulefile to install. Modulefiles can be versioned like - # modulefile-10.2.0 - # modulefile-10.2 - # modulefile-10 - # modulefile - # the most specific modulefile will be selected. Example: - # For a version 10.2.1 the file moduelfile-10.2 would be selected. + # post-install. # # Arguments: - # $1 upvar to return the filename - # - # Used gloabal variables: - # VERSIONS - # BUILDBLOCK_DIR - # - find_modulefile() { - local "$1" - local fname='' - local modulefile='' - for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do - if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then - modulefile="${BUILDBLOCK_DIR}/${fname}" - break; - fi - done - std::upvar $1 "${modulefile}" - [[ -n "${modulefile}" ]] - } - - #...................................................................... - # non-redefinable post-install. Install: - # - documentation files as defined in the build-script - # - modulefile and file with release - # . + # none post_install() { #.............................................................. - # install the doc-files specified in the build-script + # post-install: + # - build-script + # - list of loaded modules while building + # - doc-files specified in the build-script # # Arguments: # none # install_doc() { - if [[ -z "${MODULE_DOCFILES}" ]]; then + local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "Installing documentation to ${docdir}" + ${install} -m 0755 -d "${docdir}" + ${install} -m0644 "${BUILD_SCRIPT}" "${docdir}" + "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ + ${grep} -v "Currently Loaded" > \ + "${docdir}/dependencies" || : + + if [[ ! -v MODULE_DOCFILES[0] ]]; then + # loop over version specific functions. In these function + # more MODULE_DOCFILES can be defined. + # :FIXME: maybe we find a better solution. for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do if typeset -F "$f" 2>/dev/null; then $f break fi done - fi - [[ -n "${MODULE_DOCFILES}" ]] || return 0 - local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "Installing documentation to ${docdir}" - install -m 0755 -d \ - "${docdir}" - install -m0644 \ + if [[ ! -v MODULE_DOCFILES[0] ]]; then + return 0 + fi + ${install} -m0644 \ "${MODULE_DOCFILES[@]/#/${SRC_DIR}/}" \ "${docdir}" return 0 } #.............................................................. - # install build-block files - # - modulefile - # - build-script - # - run-time and build dependencies - # in ${PREFIX}/share/${GROUP}/${module_name} - # - # Arguments: - # none - # - install_pmodules_files() { - local modulefile='' - find_modulefile modulefile || return 0 - - local -r target_dir="${PREFIX}/share/$GROUP/${module_name}" - mkdir -p "${target_dir}" - install -m0644 \ - "${BUILD_SCRIPT}" \ - "${target_dir}" - install -m0644 \ - "${modulefile}" \ - "${target_dir}" - #install -m 0755 \ - # -d "${target_dir}/files" - #install -m0644 \ - # "${variants_file}" \ - # "${target_dir}/files" - - local -r fname="${target_dir}/dependencies" - "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ - grep -v "Currently Loaded" > "${fname}" || : - } - - #.............................................................. - # write run time dependencies to file + # post-install: write file with required modules write_runtime_dependencies() { local -r fname="$1" shift @@ -947,8 +1206,8 @@ pbuild::make_all() { } #.............................................................. - # for Linux we need a special post-install to solve the - # multilib problem with LIBRARY_PATH on 64-bit systems + # post-install: for Linux we need a special post-install to + # solve the multilib problem with LIBRARY_PATH on 64-bit systems post_install_linux() { std::info \ "%s " \ @@ -960,10 +1219,10 @@ pbuild::make_all() { } #.............................................................. + # post-install cd "${BUILD_DIR}" [[ "${OS}" == "Linux" ]] && post_install_linux install_doc - install_pmodules_files if [[ -v runtime_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_RDEPS}" \ @@ -976,7 +1235,6 @@ pbuild::make_all() { fi install_modulefile install_release_file - cleanup_modulefiles cleanup_build cleanup_src std::info \ @@ -984,14 +1242,47 @@ pbuild::make_all() { "${module_name}/${module_version}:" \ "Done ..." return 0 - } + } # post_install #...................................................................... # Install modulefile in ${pm_root}/${GROUP}/modulefiles/... + # The modulefiles in the build-block can be + # versioned like + # modulefile-10.2.0 + # modulefile-10.2 + # modulefile-10 + # modulefile + # the most specific modulefile will be selected. Example: + # For a version 10.2.1 the file moduelfile-10.2 would be + # selected. # # Arguments # none + # + # Used gloabal variables: + # VERSIONS + # BUILDBLOCK_DIR + # modulefile_name + # install_modulefile() { + #.............................................................. + # Select the modulefile to install. + # + # Arguments: + # $1 upvar to return the filename + # + find_modulefile() { + local -n _modulefile="$1" + local fname='' + for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do + if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then + _modulefile="${BUILDBLOCK_DIR}/${fname}" + break; + fi + done + [[ -n "${_modulefile}" ]] + } + local src='' find_modulefile src if (( $? != 0 )); then @@ -1005,8 +1296,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "adding modulefile to overlay '${ol_name}' ..." - mkdir -p "${modulefile_dir}" - install -m 0644 "${src}" "${modulefile_name}" + ${mkdir} -p "${modulefile_dir}" + ${install} -m 0644 "${src}" "${modulefile_name}" } cleanup_modulefiles(){ @@ -1024,7 +1315,7 @@ pbuild::make_all() { std::info "%s "\ "${module_name}/${module_version}:" \ "removing modulefile from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi fname="${dir}/.release-${module_version}" if [[ -e "${fname}" ]]; then @@ -1032,7 +1323,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing release file from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi done } @@ -1065,7 +1356,7 @@ pbuild::make_all() { [[ "${BUILD_DIR}" == "${SRC_DIR}" ]] && return 0 { cd "/${BUILD_DIR}/.." || std::die 42 "Internal error" - [[ "$(pwd)" == "/" ]] && \ + [[ "$(${pwd})" == "/" ]] && \ std::die 1 \ "%s " "${module_name}/${module_version}:" \ "Oops: internal error:" \ @@ -1075,7 +1366,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "Cleaning up '${BUILD_DIR}'..." - rm -rf "${BUILD_DIR##*/}" + ${rm} -rf "${BUILD_DIR##*/}" }; return 0 } @@ -1098,67 +1389,66 @@ pbuild::make_all() { return 0 } - build_target() { - local dir="$1" # src or build directory, depends on target - local target="$2" # prep, configure, compile or install - - if [[ -e "${BUILD_DIR}/.${target}" ]] && \ - [[ ${force_rebuild} != 'yes' ]]; then - return 0 - fi - local targets=() - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) - targets+=( pbuild::pre_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) - targets+=( pbuild::pre_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) - targets+=( pbuild::pre_${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) - targets+=( pbuild::${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) - targets+=( pbuild::${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) - targets+=( pbuild::${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) - targets+=( pbuild::post_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) - targets+=( pbuild::post_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) - targets+=( pbuild::post_${target} ) - - for t in "${targets[@]}"; do - # We cd into the dir before calling the function - - # just to be sure we are in the right directory. - # - # Executing the function in a sub-process doesn't - # work because in some function global variables - # might/need to be set. - # - cd "${dir}" - if typeset -F "$t" 2>/dev/null; then - "$t" || \ - std::die 10 "Aborting..." - fi - done - touch "${BUILD_DIR}/.${target}" - } - #...................................................................... # build module ${module_name}/${module_version} - build_module() { + compile_and_install() { + build_target() { + local dir="$1" # src or build directory, depends on target + local target="$2" # prep, configure, compile or install + + if [[ -e "${BUILD_DIR}/.${target}" ]] && \ + [[ ${force_rebuild} != 'yes' ]]; then + return 0 + fi + local targets=() + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) + targets+=( pbuild::pre_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) + targets+=( pbuild::pre_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) + targets+=( pbuild::pre_${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) + targets+=( pbuild::${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) + targets+=( pbuild::${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) + targets+=( pbuild::${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) + targets+=( pbuild::post_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) + targets+=( pbuild::post_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) + targets+=( pbuild::post_${target} ) + + for t in "${targets[@]}"; do + # We cd into the dir before calling the function - + # just to be sure we are in the right directory. + # + # Executing the function in a sub-process doesn't + # work because in some function global variables + # might/need to be set. + # + cd "${dir}" + if typeset -F "$t" 2>/dev/null; then + "$t" || \ + std::die 10 "Aborting..." + fi + done + touch "${BUILD_DIR}/.${target}" + } # compile_and_install():build_target() + [[ ${dry_run} == yes ]] && std::die 0 "" - mkdir -p "${SRC_DIR}" - mkdir -p "${BUILD_DIR}" + ${mkdir} -p "${SRC_DIR}" + ${mkdir} -p "${BUILD_DIR}" + ${mkdir} -p "${PREFIX}" std::info \ "%s " \ "${module_name}/${module_version}:" \ "preparing sources ..." - # write stdout and stderr to logfile, stderr to terminal - # write all to logfile and terminal build_target "${SRC_DIR}" prep [[ "${build_target}" == "prep" ]] && return 0 @@ -1180,10 +1470,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "installing ..." - mkdir -p "${PREFIX}" build_target "${BUILD_DIR}" install - post_install - } + } # compile_and_install() remove_module() { if [[ -d "${PREFIX}" ]]; then @@ -1191,14 +1479,14 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing all files in '${PREFIX}' ..." - [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} + [[ "${dry_run}" == 'no' ]] && ${rm} -rf ${PREFIX} fi if [[ -e "${modulefile_name}" ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ "removing modulefile '${modulefile_name}' ..." - [[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}" + [[ "${dry_run}" == 'no' ]] && ${rm} -v "${modulefile_name}" fi local release_file="${modulefile_dir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then @@ -1208,34 +1496,70 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p "${modulefile_dir}" 2>/dev/null || : + ${rmdir} -p "${modulefile_dir}" 2>/dev/null || : } - ######################################################################## - # - # here we really start with make_all() - # + deprecate_module(){ + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "is deprecated, skiping!" + install_release_file + } - # setup module specific environment + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + ${with_modules:+with ${with_modules[@]}} \ + "building ..." + + init_module_environment + load_build_dependencies + init_build_environment "${module_name}" "${module_version}" + + source "${BUILD_SCRIPT}" + + # module name including path in hierarchy and version + # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) + local modulefile_dir='' + local modulefile_name='' + + # + # :FIXME: add comments what and why we are doing this. + # + local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + rm -f "${logfile}" + if [[ "${verbose}" == 'yes' ]]; then + exec > >(${tee} -a "${logfile}") + else + exec > >(${cat} >> "${logfile}") + fi + exec 2> >(${tee} -a "${logfile}" >&2) + + # the group must have been defined - otherwise we cannot continue + [[ -n ${GROUP} ]] || \ + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "Module group not set! Aborting ..." + + # check whether this module is supported check_supported_systems check_supported_os check_supported_compilers + # setup module name and prefix set_full_module_name_and_prefix + + # ok, finally we can start ... std::info \ "%s " \ "${module_name}/${module_version}:" \ ${with_modules:+build with ${with_modules[@]}} - if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then - # don't (re-)build the module, but - # - if the release stage has been changed to 'removed', - # remove the module. - # - if requested, update the modulefile. - # - if modulefile does not exist, install it. - # - update release stage. - if [[ "${module_release}" == 'removed' ]]; then - remove_module - return $? - fi + + if [[ "${module_release}" == 'removed' ]]; then + remove_module + elif [[ "${module_release}" == 'deprecated' ]]; then + deprecate_module + elif [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ @@ -1245,308 +1569,18 @@ pbuild::make_all() { install_modulefile fi install_release_file - cleanup_modulefiles - return $? - fi - if [[ "${module_release}" == 'deprecated' ]]; then + else std::info \ "%s " \ "${module_name}/${module_version}:" \ - "is deprecated, skiping!" - install_release_file - return $? + "start building ..." + compile_and_install + post_install fi - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "start building ..." - - build_module - return $? -} - -pbuild.init_env() { - #...................................................................... - # - # parse the passed version string - # - # the following global variables will be set in this function: - # V_MAJOR - # V_MINOR - # V_PATCHLVL - # V_RELEASE - # USE_FLAGS - # - parse_version() { - local v="$1" - V_MAJOR='' # first number in version string - V_MINOR='' # second number in version string (or empty) - V_PATCHLVL='' # third number in version string (or empty) - V_RELEASE='' # module release (or empty) - : ${USE_FLAGS:=''} # architectures (or empty) - - local tmp='' - - if [[ "$v" =~ "_" ]]; then - tmp="${v#*_}" - USE_FLAGS+=":${tmp//_/:}:" - v="${v%%_*}" - fi - V_PKG="${v%%-*}" # version without the release number - if [[ $v == *-* ]]; then - V_RELEASE="${v#*-}" # release number - else - V_RELEASE='' - fi - case "${V_PKG}" in - *.*.* ) - V_MAJOR="${V_PKG%%.*}" - tmp="${V_PKG#*.}" - V_MINOR="${tmp%%.*}" - V_PATCHLVL="${tmp#*.}" - ;; - *.* ) - V_MAJOR="${V_PKG%.*}" - V_MINOR="${V_PKG#*.}" - ;; - * ) - V_MAJOR="${V_PKG}" - ;; - esac - - VERSIONS=( ${V_MAJOR} ) - if [[ -n ${V_MINOR} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR} ${VERSIONS[@]} ) - fi - if [[ -n ${V_PATCHLVL} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ${VERSIONS[@]} ) - fi - if [[ -n ${V_RELEASE} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL}-${V_RELEASE} ${VERSIONS[@]} ) - fi - } - - local -r module_name="$1" - local -r module_version="$2" - - SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" - BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" - - # P and V can be used in the build-script, so we have to set them here - P="${module_name}" - V="${module_version}" - parse_version "${module_version}" - - SOURCE_URLS=() - SOURCE_SHA256_SUMS=() - SOURCE_NAMES=() - CONFIGURE_ARGS=() - SUPPORTED_SYSTEMS=() - SUPPORTED_OS=() - SUPPORTED_COMPILERS=() - PATCH_FILES=() - PATCH_STRIPS=() - PATCH_STRIP_DEFAULT='1' - MODULE_DOCFILES=() - configure_with='undef' -} - -pbuild.build_module() { - module_name="$1" - module_version="$2" - module_release="$3" - shift 3 - with_modules=( "$@" ) - - # used in pbuild::make_all - declare -a runtime_dependencies=() - declare -a install_dependencies=() - - #...................................................................... - # - # test whether a module is loaded or not - # - # $1: module name - # - is_loaded() { - [[ :${LOADEDMODULES}: =~ :$1: ]] - } - - #...................................................................... - # - # build a dependency - # - # $1: name of module to build - # - # :FIXME: needs testing - # - build_dependency() { - local -r m=$1 - std::debug "${m}: module not available" - local rels=( ${ReleaseStages//:/ } ) - [[ ${dry_run} == yes ]] && \ - std::die 1 \ - "%s " \ - "${m}: module does not exist," \ - "cannot continue with dry run..." - - std::info "%s " \ - "$m: module does not exist, trying to build it..." - local args=( '' ) - set -- ${ARGS[@]} - while (( $# > 0 )); do - case $1 in - -j ) - args+=( "-j $2" ) - shift - ;; - --jobs=[0-9]* ) - args+=( $1 ) - ;; - -v | --verbose) - args+=( $1 ) - ;; - --with=*/* ) - args+=( $1 ) - ;; - esac - shift - done - - find_build_script(){ - local p=$1 - local script=$(find "${BUILDBLOCK_DIR}/../.." -path "*/$p/build") - std::get_abspath "${script}" - } - local buildscript=$(find_build_script "${m%/*}") - [[ -x "${buildscript}" ]] || \ - std::die 1 \ - "$m: build-block not found!" - if ! "${buildscript}" "${m#*/}" ${args[@]}; then - std::die 1 \ - "$m: oops: build failed..." - fi - } - - #...................................................................... - # - # Load build- and run-time dependencies. - # - # Arguments: - # none - # - # Variables - # [r] module_release set if defined in a variants file - # runtime_dependencies runtime dependencies from variants added - # - load_build_dependencies() { - local m='' - for m in "${with_modules[@]}"; do - - # module name prefixes in dependency declarations: - # 'b:' this is a build dependency - # 'r:' this a run-time dependency, *not* required for - # building - # without prefix: this is a build and - # run-time dependency - if [[ "${m:0:2}" == "b:" ]]; then - m=${m#*:} # remove 'b:' - elif [[ "${m:0:2}" == "r:" ]]; then - m=${m#*:} # remove 'r:' - runtime_dependencies+=( "$m" ) - elif [[ "${m:0:2}" == "R:" ]]; then - m=${m#*:} # remove 'R:' - install_dependencies+=( "$m" ) - continue - else - runtime_dependencies+=( "$m" ) - fi - is_loaded "$m" && continue - - # 'module avail' might output multiple matches if module - # name and version are not fully specified or in case - # modules with and without a release number exist. - # Example: - # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list - # from 'module avail' and the full version should be set - # in the variants file, we look for the first exact - # match. - local release_of_dependency='' - if ! pbuild::module_is_avail "$m" release_of_dependency; then - build_dependency "$m" - pbuild::module_is_avail "$m" release_of_dependency || \ - std::die 6 "Oops" - fi - # should be set, just in case it is not... - : ${release_of_dependency:='unstable'} - - # for a stable module all dependencies must be stable - if [[ "${module_release}" == 'stable' ]] \ - && [[ "${release_of_dependency}" != 'stable' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - # for a unstable module no dependency must be deprecated - elif [[ "${module_release}" == 'unstable' ]] \ - && [[ "${release_of_dependency}" == 'deprecated' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - fi - - std::info "Loading module: ${m}" - eval $( "${MODULECMD}" bash load "${m}" ) - done - } - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "building ..." - MODULECMD="${PMODULES_HOME}/bin/modulecmd" - [[ -x ${MODULECMD} ]] || \ - std::die 2 "No such file or executable -- '${MODULECMD}'" - - eval $( "${MODULECMD}" bash use unstable ) - eval $( "${MODULECMD}" bash use deprecated ) - eval $( "${MODULECMD}" bash purge ) - - # :FIXME: this is a hack!!! - # shouldn't this be set in the build-script? - if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then - eval $( "${MODULECMD}" bash use Libraries ) - fi - if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then - eval $( "${MODULECMD}" bash use System ) - fi - unset C_INCLUDE_PATH - unset CPLUS_INCLUDE_PATH - unset CPP_INCLUDE_PATH - unset LIBRARY_PATH - unset LD_LIBRARY_PATH - unset DYLD_LIBRARY_PATH - - unset CFLAGS - unset CPPFLAGS - unset CXXFLAGS - unset LIBS - unset LDFLAGS - - unset CC - unset CXX - unset FC - unset F77 - unset F90 - - load_build_dependencies - - pbuild.init_env "${module_name}" "${module_version}" - pbuild::make_all + cleanup_modulefiles std::info "* * * * *\n" } +readonly -f pbuild.build_module # Local Variables: # mode: sh diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 6a52073..eb3b6ba 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -39,7 +39,7 @@ std::def_cmds(){ local path="$1" shift for cmd in "$@"; do - eval declare -g ${cmd}=$(PATH="${path}" which $cmd 2>/dev/null) + eval declare -gr ${cmd}=$(PATH="${path}" /usr/bin/which $cmd 2>/dev/null) if [[ -z "${!cmd}" ]]; then std::die 255 "${cmd} not found" fi @@ -324,7 +324,7 @@ std::get_os_release() { local -A func_map; func_map['Linux']=std.get_os_release_linux func_map['Darwin']=std.get_os_release_macos - ${func_map[${OS}]} + ${func_map[$(uname -s)]} } std::get_type() { diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c5813d1..7f92148 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -7,58 +7,69 @@ # #............................................................................. -#set -x +set -x declare VERSION='@PMODULES_VERSION@' # get absolute path of script declare mydir=$(dirname "$0") declare -r mydir=$(cd ${mydir} && pwd -P) -# initialize PATH, -# add library installation directories to the PATH, -# so 'source' is able find them - -if [[ $(uname -s) == 'Darwin' ]]; then - PATH='/opt/local/bin:' -else - PATH='' -fi -PATH+='/usr/bin:/bin:/usr/sbin:/sbin' -PATH+=":${mydir}" -PATH+=":${mydir}/../lib:${mydir}/../config" -PATH+=":${mydir}/../libexec" - -path=$PATH - -source libstd.bash || { +source "${mydir}/../lib/libstd.bash" || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } -std::def_cmds "${path}" \ - 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' +############################################################################## +# +# check availability of used commands and set environment variables +# cmd=$(which cmd) +# in the following we use these environment variable to call binaries. +# +declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd" +[[ -x ${MODULECMD} ]] || \ + std::die 1 "Oops: modulecmd binary not available!" -# can be set in the configuration file +std::def_cmds "${mydir}/../libexec" \ + 'yq' + +std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'awk' 'base64' 'cat' 'cp' 'find' 'getopt' 'grep' \ + 'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \ + 'rm' 'rmdir' 'sort' 'tar' 'tee' 'uname' + +declare -r OS="$(${uname} -s)" +if [[ ${OS} == 'Darwin' ]]; then + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \ + 'curl' 'otool' 'shasum' 'sysctl' + declare -r sha256sum="${shasum -a 256}" +else + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'ldd' 'curl' 'sha256sum' +fi + +# for the time being, we still set PATH. Just in case we forgot a binary +PATH='/usr/bin:/bin:/usr/sbin:/sbin' + +############################################################################## +# save arguments, required for building dependencies +declare -r ARGS="$@" declare PMODULES_DISTFILESDIR='' declare PMODULES_TMPDIR='' declare pm_root="${PMODULES_HOME%%/Tools*}" +declare ol_mod_root='' +declare ol_inst_root='' -source libpbuild.bash || \ +source "${mydir}/../lib/libpbuild.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -source libpmodules.bash || \ +source "${mydir}/../lib/libpmodules.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -# save arguments, (still) required for building dependencies -declare -r ARGS="$@" - +############################################################################## +# set some shell options set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob -declare ol_mod_root -declare ol_inst_root - ############################################################################## # usage() { @@ -350,9 +361,6 @@ bash_expand(){ build_modules_legacy() { find_variants_files(){ - shopt -q nullglob || : - local -i nullglob_set=$? - shopt -s nullglob local files=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}"\.${opt_system} ) files+=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}.$(uname -s)" ) local f @@ -361,7 +369,6 @@ build_modules_legacy() { || [[ -e "${f}.$(uname -s)" ]] \ || files+=( "$f" ) done - (( nullglob_set == 1 )) && shopt -u nullglob std::upvar "$1" "${files[@]}" } @@ -395,7 +402,7 @@ build_modules_legacy() { local m local pattern="/^${name}\/${version}[[:blank:]]/" for m in "${with_modules[@]}"; do - if [[ -n $(awk "/${m%/*}[\/ ]/" "${files[@]}") ]]; then + if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then pattern+=" && /${m//\//\\/}/" fi done @@ -405,7 +412,7 @@ build_modules_legacy() { local line='' while read line; do variants+=( "${line}" ) - done < <(expand_variants_file "${f}" | awk "${pattern}") + done < <(expand_variants_file "${f}" | ${awk} "${pattern}") done if (( ${#variants[@]} == 0 )); then std::info "%s " \ @@ -457,7 +464,7 @@ build_modules_yaml(){ local version="$3" _result=( $(${yq} -Ne e \ "with_entries(select(.key | test(\"^${version}\$\")))|keys" \ - "${fname}" 2>/dev/null | awk '{print $2}') ) + "${fname}" 2>/dev/null | ${awk} '{print $2}') ) } yaml_get_num_variants(){ @@ -593,7 +600,7 @@ build_modules_yaml(){ pbuild.build_module \ "${name}" "${v##*/}" \ "${relstage}" "${with_modules[@]}" - done < <(bash_expand "" ${deps[@]}|awk "${pattern}") + done < <(bash_expand "" ${deps[@]}|${awk} "${pattern}") else pbuild.build_module \ "${name}" "${v##*/}" \ @@ -634,7 +641,7 @@ pbuild.verbose "${opt_verbose}" # pm::read_config -# :FIXME: should go dist files to +# :FIXME: should dist files go to # ${pm_root}/var/distfiles # or # ${overlay}/var/distfiles diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 24f7dcb..b4a2035 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -30,7 +30,7 @@ declare -r os_name="$(uname -s)" path="${libexecdir}:/bin:/usr/bin" std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' + 'rm' 'sort' 'yq' if [[ ${PMODULES_PURETCL} == yes ]]; then declare -r modulecmd="${libexecdir}/modulecmd.tcl"