From dca348c4e3ce93ef881d3c1adda37f0ab85dbc8c Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:03:19 +0200 Subject: [PATCH 01/15] accessing unset variables fixed --- Pmodules/libpbuild.bash | 12 ++++++++---- Pmodules/libstd.bash | 2 +- Pmodules/modbuild.in | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 4770e9e..f104874 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -301,7 +301,11 @@ pbuild::module_is_avail() { pbuild::set_download_url() { local -i _i=${#SOURCE_URLS[@]} SOURCE_URLS[_i]="$1" - SOURCE_NAMES[_i]="$2" + if (( $# == 2 )); then + SOURCE_NAMES[$_i]="$2" + else + SOURCE_NAMES[$_i]='' + fi } pbuild::set_sha256sum() { @@ -691,7 +695,7 @@ pbuild::make_all() { #...................................................................... check_supported_systems() { - [[ -z "${SUPPORTED_SYSTEMS}" ]] && return 0 + (( ${#SUPPORTED_SYSTEMS[@]} == 0 )) && return 0 for sys in "${SUPPORTED_SYSTEMS[@]}"; do [[ ${sys,,} == ${system,,} ]] && return 0 done @@ -702,7 +706,7 @@ pbuild::make_all() { #...................................................................... check_supported_os() { - [[ -z "${SUPPORTED_OS}" ]] && return 0 + (( ${#SUPPORTED_OS[@]} == 0 )) && return 0 for os in "${SUPPORTED_OS[@]}"; do [[ ${os,,} == ${OS,,} ]] && return 0 done @@ -713,7 +717,7 @@ pbuild::make_all() { #...................................................................... check_supported_compilers() { - [[ -z "${SUPPORTED_COMPILERS}" ]] && return 0 + (( ${#SUPPORTED_COMPILERS[@]} == 0 )) && return 0 for compiler in "${SUPPORTED_COMPILERS[@]}"; do [[ ${compiler,,} == ${COMPILER,,} ]] && return 0 done diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 95e70c7..6a52073 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -20,7 +20,7 @@ std::error() { } std::debug() { - [[ ${PMODULES_DEBUG} ]] || return 0 + [[ -v PMODULES_DEBUG ]] || return 0 std::log 2 "$@" } diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index a36ef48..819b521 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -171,6 +171,8 @@ declare opt_verbose='no' # array collecting all modules specified on the command line via '--with=module' declare -a opt_with_modules=() +declare BUILD_SCRIPT='' + declare opt_ol_name_or_dir='' parse_args() { From cd1788704d0dc195442ced887bfee24105fcd82a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:07:24 +0200 Subject: [PATCH 02/15] modbuild: issue with cleaning up modulefiles in overlays fixed --- Pmodules/libpbuild.bash | 29 +++++++++++++++++++++++++++++ Pmodules/modbuild.in | 7 +++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index f104874..bdaed40 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -967,6 +967,7 @@ pbuild::make_all() { fi install_modulefile install_release_file + cleanup_modulefiles cleanup_build cleanup_src std::info \ @@ -999,6 +1000,33 @@ pbuild::make_all() { install -m 0644 "${src}" "${modulefile_name}" } + cleanup_modulefiles(){ + local ol='' + for ol in "${Overlays[@]}"; do + local i + for ((i=0; i<${#mod_overlays}; i++ )); do + [[ "${ol}" == "{mod_overlays[i]}" ]] && continue 2 + done + [[ "${ol}" == "${ol_name}" ]] && continue + local mod_root="${OverlayInfo[${ol}:mod_root]}" + local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" + local fname="${dir}/${module_version}" + if [[ -e "${fname}" ]]; then + std::info "%s "\ + "${module_name}/${module_version}:" \ + "removing modulefile from overlay '${ol}' ..." + rm "${fname}" + fi + fname="${dir}/.release-${module_version}" + if [[ -e "${fname}" ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "removing release file from overlay '${ol}' ..." + rm "${fname}" + fi + done + } install_release_file() { local -r release_file="${modulefile_dir}/.release-${module_version}" @@ -1208,6 +1236,7 @@ pbuild::make_all() { install_modulefile fi install_release_file + cleanup_modulefiles return $? fi if [[ "${module_release}" == 'deprecated' ]]; then diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 819b521..98570c9 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -578,10 +578,14 @@ build_modules_yaml(){ else ol_name="${opt_overlay}" fi - mod_overlays[${ol_name}]=1 if [[ ! -v OverlayInfo[${ol_name}:inst_root] ]]; then std::die 3 "Overlay is not defined -- ${ol_name}" fi + mod_overlays[$i]="${ol_name}" + done + + for (( i=0; i Date: Mon, 4 Jul 2022 19:08:10 +0200 Subject: [PATCH 03/15] modbuild: handling of empty variant array fixed an entry /: without further information is legal and must be handled properly. --- Pmodules/modbuild.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 98570c9..b80afb1 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -568,6 +568,7 @@ build_modules_yaml(){ for v in "${versions[@]}"; do local -i n_variants yaml_get_num_variants n_variants "${fname}" "${v}" + (( n_variants == 0 )) && (( n_variants = 1 )) local -i i local -a deps=() local relstage From e9cfc1fee438d771548066910a1a2a2dc0c6f28d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:13:59 +0200 Subject: [PATCH 04/15] modbuild: a default release stage can now be set in the header --- Pmodules/modbuild.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b80afb1..0e39831 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -477,7 +477,10 @@ build_modules_yaml(){ local idx="$4" _result=$(${yq} -Ne e ".\"${version}\"[${idx}].relstage" \ "${fname}" 2>/dev/null) - (( $? != 0 )) && relstage='unstable' || : + (( $? == 0 )) && return + _result=$(${yq} -Ne e ".relstage" "${fname}" 2>/dev/null) + (( $? == 0 )) && return + _result='unstable' } yaml_get_overlay(){ From 9e63b4571bc821de0fdd4a900f90b35baaf50074 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:14:51 +0200 Subject: [PATCH 05/15] modbuild: querying dependencies fixed and old cleanup_modulefiles() removed --- Pmodules/modbuild.in | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 0e39831..ddec29a 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -524,35 +524,10 @@ build_modules_yaml(){ # if one of .with, .dependencies is not set, the vaulue is # returned as 'null'. local -i i - for ((i=0; i<${#_result[@]}; i++)); do - if [[ ${_result[$i]} == 'null' ]]; then - unset _result[$i] - fi - done - } - - cleanup_modulefiles(){ - local ol='' - for ol in "${Overlays[@]}"; do - [[ -v mod_overlays[${ol}] ]] && continue - local mod_root="${OverlayInfo[${ol}:mod_root]}" - local dir="${modulefile_dir/${ol_mod_root}/${mod_root}}" - local fname="${dir}/${module_version}" - if [[ -e "${fname}" ]]; then - std::info "%s "\ - "${module_name}/${module_version}:" \ - "removing modulefile from overlay '${ol}' ..." - rm "${fname}" - fi - fname="${dir}/.release-${module_version}" - if [[ -e "${fname}" ]]; then - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "removing release file from overlay '${ol}' ..." - rm "${fname}" - fi + for i in "${!_result[@]}"; do + [[ ${_result[$i]} == 'null' ]] && unset -v '_result[$i]' done + _result=("${_result[@]}") } local name="$1" From 6eced654ebf77d78b7df648c3f55b68dfec46b1a Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Mon, 4 Jul 2022 19:15:43 +0200 Subject: [PATCH 06/15] modbuild: for the time being we don't set 'nounset' --- Pmodules/modbuild.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index ddec29a..b8c050c 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -51,6 +51,7 @@ source libpmodules.bash || \ # save arguments, (still) required for building dependencies declare -r ARGS="$@" +#set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob From 48ea630332d5d898800be5a0f6c2af6ec9f050b0 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 8 Jul 2022 11:43:08 +0200 Subject: [PATCH 07/15] modbuild: fixed some issues if nounset is set --- Pmodules/libpbuild.bash | 21 +++++++++++++++------ Pmodules/modbuild.in | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index bdaed40..4feab50 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -483,7 +483,8 @@ pbuild::prep() { std::die 4 \ "%s " "${module_name}/${module_version}:" \ "sources for not found." - unpack "${source_fname}" "${SOURCE_UNPACK_DIRS[${source_fname##*/}]}" + local dir=${SOURCE_UNPACK_DIRS[${source_fname##*/}]:-''} + unpack "${source_fname}" "${dir}" done patch_sources # create build directory @@ -758,26 +759,34 @@ pbuild::make_all() { # define hierarchies if [[ -v COMPILER_VERSION ]]; then Compiler_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' + else + unset Compiler_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' + else + unset HDF5_serial_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' + else + unset MPI_HIERARCHY fi if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' + else + unset HDF5_HIERARCHY fi # evaluate local names=() - local vname="${GROUP}_HIERARCHY" - if [[ -n ${!vname} ]]; then - names=( $(eval echo ${!vname}) ) + local -n vname="${GROUP}"_HIERARCHY + if [[ -v vname ]]; then + names=( $(eval echo ${vname}) ) else std::die 1 \ "%s: %s" \ @@ -955,12 +964,12 @@ pbuild::make_all() { [[ "${OS}" == "Linux" ]] && post_install_linux install_doc install_pmodules_files - if [[ -n "${runtime_dependencies}" ]]; then + if [[ -v runtime_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_RDEPS}" \ "${runtime_dependencies[@]}" fi - if [[ -n "${install_dependencies}" ]]; then + if [[ -v install_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_IDEPS}" \ "${install_dependencies[@]}" diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index b8c050c..c5813d1 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -51,7 +51,7 @@ source libpmodules.bash || \ # save arguments, (still) required for building dependencies declare -r ARGS="$@" -#set -o nounset +set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob From a7c50a94598658ac1621b7fbb746fa64473b49e4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Tue, 12 Jul 2022 16:21:21 +0200 Subject: [PATCH 08/15] cleanup, re-factoring, minor fixes, comments added --- Pmodules/libpbuild.bash | 1530 ++++++++++++++++++------------------ Pmodules/libstd.bash | 4 +- Pmodules/modbuild.in | 85 +- Pmodules/modulecmd.bash.in | 2 +- 4 files changed, 831 insertions(+), 790 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 4feab50..72f3fd3 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1,22 +1,5 @@ #!/bin/bash -#............................................................................. -# -# We need GNU versions of the following utilities. This code works -# well on Linux and Mac OS X with MacPorts. -# :FIXME: implement a smarter, portable solution. -# -shopt -s expand_aliases -unalias -a - -__path=$(which gsed 2>/dev/null || : ) -if [[ $__path ]]; then - alias sed=$__path -else - alias sed=$(which sed 2>/dev/null) -fi -unset __path - #............................................................................. # disable auto-echo feature of 'cd' unset CDPATH @@ -28,8 +11,9 @@ declare -r FNAME_RDEPS='.dependencies' declare -r FNAME_IDEPS='.install_dependencies' declare -r FNAME_BDEPS='.build_dependencies' -#............................................................................. -declare -A SOURCE_UNPACK_DIRS +# relative path of documentation +# abs. path is "${PREFIX}/${_docdir}/${module_name}" +declare -r _DOCDIR='share/doc' #............................................................................. # @@ -39,15 +23,104 @@ declare -A SOURCE_UNPACK_DIRS # #set -o errexit -error_handler() { +_error_handler() { local -i ec=$? std::die ${ec} "Oops" } +readonly -f _error_handler -trap "error_handler" ERR +trap "_error_handler" ERR -declare configure_with='undef' +#.............................................................................. +# +# write number of cores to stdout +# +_get_num_cores() { + case "${OS}" in + Linux ) + ${grep} -c ^processor /proc/cpuinfo + ;; + Darwin ) + ${sysctl} -n hw.ncpu + ;; + * ) + std::die 1 "OS ${OS} is not supported\n" + ;; + esac +} +readonly -f _get_num_cores + +#.............................................................................. +# global variables which can be set/overwritten by command line args +# and their corresponding functions +# +declare force_rebuild='' +pbuild.force_rebuild() { + force_rebuild="$1" +} +readonly -f pbuild.force_rebuild + +declare dry_run='' +pbuild.dry_run() { + dry_run="$1" +} +readonly -f pbuild.dry_run + +declare enable_cleanup_build='' +pbuild.enable_cleanup_build() { + enable_cleanup_build="$1" +} +readonly -f pbuild.enable_cleanup_build + +declare enable_cleanup_src='' +pbuild.enable_cleanup_src() { + enable_cleanup_src="$1" +} +readonly -f pbuild.enable_cleanup_src + +declare build_target='' +pbuild.build_target() { + build_target="$1" +} +readonly -f pbuild.build_target + +declare opt_update_modulefiles='' +pbuild.update_modulefiles() { + opt_update_modulefiles="$1" +} +readonly -f pbuild.update_modulefiles + +# number of parallel make jobs +declare -i JOBS=0 +pbuild.jobs() { + if (( $1 == 0 )); then + JOBS=$(_get_num_cores) + (( JOBS > 10 )) && JOBS=10 || : + else + JOBS="$1" + fi +} +readonly -f pbuild.jobs + +declare system='' +pbuild.system() { + system="$1" +} +readonly -f pbuild.system + +declare verbose='' +pbuild.verbose() { + verbose="$1" +} +readonly -f pbuild.verbose + + +############################################################################### +# +# function in the "namespace" (with prefix) 'pbuild::' can be used in +# build-scripts +# #.............................................................................. # @@ -81,120 +154,53 @@ pbuild::version_compare () { done return 0 } +readonly -f pbuild::version_compare +#.............................................................................. +# version less than +# +# return 0 if version passed in $1 is older then $2 +# pbuild::version_lt() { pbuild::version_compare "$1" "$2" (( $? == 2 )) } +readonly -f pbuild::version_lt +#.............................................................................. +# version less than or equal +# +# return 0 if version passed in $1 is older or equal then $2 +# pbuild::version_le() { pbuild::version_compare "$1" "$2" local -i exit_code=$? (( exit_code == 0 || exit_code == 2 )) } +readonly -f pbuild::version_le - +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 is newer then $2 +# pbuild::version_gt() { pbuild::version_compare "$1" "$2" (( $? == 1 )) local -i exit_code=$? (( exit_code == 0 || exit_code == 1 )) } +readonly -f pbuild::version_gt +#.............................................................................. +# version greater than +# +# return 0 if version passed in $1 and $2 are equal +# pbuild::version_eq() { pbuild::version_compare "$1" "$2" } - -#.............................................................................. -# -# The following variables are available in build-blocks and set read-only -# :FIXME: do we have to export them? - -# install prefix of module. -declare -x PREFIX='' - -declare -r OS=$(uname -s) - -pbuild::get_num_cores() { - case "${OS}" in - Linux ) - echo $(grep -c ^processor /proc/cpuinfo) - ;; - Darwin ) - echo $(sysctl -n hw.ncpu) - ;; - * ) - std::die 1 "OS ${OS} is not supported\n" - ;; - esac -} - -#.............................................................................. -# global variables which can be set/overwritten by command line args - -declare force_rebuild='' -pbuild.force_rebuild() { - force_rebuild="$1" -} - -declare dry_run='' -pbuild.dry_run() { - dry_run="$1" -} - -declare enable_cleanup_build='' -pbuild.enable_cleanup_build() { - enable_cleanup_build="$1" -} - -declare enable_cleanup_src='' -pbuild.enable_cleanup_src() { - enable_cleanup_src="$1" -} - -declare build_target='' -pbuild.build_target() { - build_target="$1" -} - -declare opt_update_modulefiles='' -pbuild.update_modulefiles() { - opt_update_modulefiles="$1" -} - -# number of parallel make jobs -declare -i JOBS=$(pbuild::get_num_cores) -pbuild.jobs() { - if (( $1 == 0 )); then - JOBS=$(pbuild::get_num_cores) - (( JOBS > 10 )) && JOBS=10 || : - else - JOBS="$1" - fi -} - -declare system='' -pbuild.system() { - system="$1" -} - -declare verbose='' -pbuild.verbose() { - verbose="$1" -} - - -# group this module is in (ex: 'Programming') -declare -x GROUP='' - -# name, version and release of module -declare -x module_name='' -declare -x module_version='' -declare -x module_release='' - -# relative path of documentation -# abs. path is "${PREFIX}/${_docdir}/${module_name}" -declare -r _DOCDIR='share/doc' +readonly -f pbuild::version_eq ############################################################################## # @@ -206,6 +212,7 @@ declare -r _DOCDIR='share/doc' pbuild::compile_in_sourcetree() { BUILD_DIR="${SRC_DIR}" } +readonly -f pbuild::compile_in_sourcetree ############################################################################## # @@ -218,6 +225,7 @@ pbuild::compile_in_sourcetree() { pbuild::supported_systems() { SUPPORTED_SYSTEMS+=( "$@" ) } +readonly -f pbuild::supported_systems ############################################################################## # @@ -230,6 +238,7 @@ pbuild::supported_systems() { pbuild::supported_os() { SUPPORTED_OS+=( "$@" ) } +readonly -f pbuild::supported_os ############################################################################## # @@ -242,6 +251,7 @@ pbuild::supported_os() { pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } +readonly -f pbuild::supported_compilers ############################################################################## # @@ -258,6 +268,7 @@ pbuild::add_to_group() { fi GROUP="$1" } +readonly -f pbuild::add_to_group ############################################################################## # @@ -269,6 +280,7 @@ pbuild::add_to_group() { pbuild::install_docfiles() { MODULE_DOCFILES+=("$@") } +readonly -f pbuild::install_docfiles ############################################################################## # @@ -277,45 +289,85 @@ pbuild::install_docfiles() { # # Arguments: # $1: module name -# $2: optional variable name to return release via upvar +# $2: optional variable name to return release # # Notes: -# The passed module name must be NAME/VERSION! +# The passed module name must be module/version! +# +# Exit codes: +# 0 if module/version is available +# 1 otherwise # pbuild::module_is_avail() { - local "$2" - local uvar="$2" - [[ -n "${uvar}" ]] || uvar="__unused__" local output=( $("${MODULECMD}" bash avail -a -m "$1" \ 2>&1 1>/dev/null) ) local i for (( i = 0; i < ${#output[@]}; i += 2 )); do if [[ "${output[$i]}" == "$1" ]]; then - std::upvar "${uvar}" "${output[i+1]}" + if (( $# > 1 )); then + local -n _result="$2" + _result="${output[i+1]}" + fi return 0 fi done return 1 } +readonly -f pbuild::module_is_avail +############################################################################## +# +# Set the download URL and name of downloaded file. +# +# Arguments: +# $1 download URL +# $2 optional file-name (of) pbuild::set_download_url() { local -i _i=${#SOURCE_URLS[@]} SOURCE_URLS[_i]="$1" - if (( $# == 2 )); then - SOURCE_NAMES[$_i]="$2" + if (( $# > 1 )); then + SOURCE_NAMES[$_i]="${2:-${1##*/}}" else - SOURCE_NAMES[$_i]='' + SOURCE_NAMES[$_i]="${1##*/}" fi } +readonly -f pbuild::set_download_url +############################################################################## +# +# Set hash sum for file. +# +# Arguments: +# $1 filen-name:hash-sum +# +# :FIXME: +# Maybe we should use a dictionary in the future. +# pbuild::set_sha256sum() { SOURCE_SHA256_SUMS+=("$1") } +readonly -f pbuild::set_sha256sum +############################################################################## +# +# Unpack file $1 in directory $2 +# +# Arguments: +# $1 file-name +# $2 directory +# pbuild::set_unpack_dir() { SOURCE_UNPACK_DIRS[$1]=$2 } +readonly -f pbuild::set_unpack_dir +############################################################################## +# +# Use this C-compiler +# +# Arguments: +# $1 C-compiler to use. +# pbuild::use_cc() { [[ -x "$1" ]] || std::die 3 \ "%s " "${module_name}/${module_version}:" \ @@ -323,14 +375,59 @@ pbuild::use_cc() { "'$1' is not an executable!" CC="$1" } +readonly -f pbuild::use_cc -pbuild::pre_prep() { - : +############################################################################### +# +pbuild::add_patch() { + [[ -z "$1" ]] && \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + PATCH_FILES+=( "$1" ) + PATCH_STRIPS+=( "$2" ) } +readonly -f pbuild::add_patch -pbuild::post_prep() { - : +############################################################################### +# +pbuild::set_default_patch_strip() { + [[ -n "$1" ]] || \ + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "${FUNCNAME}: missing argument!" + + PATCH_STRIP_DEFAULT="$1" } +readonly -f pbuild::set_default_patch_strip + +############################################################################### +# +pbuild::use_flag() { + [[ "${USE_FLAGS}" =~ ":${1}:" ]] +} +readonly -f pbuild::use_flag + +############################################################################### +# +pbuild::add_configure_args() { + CONFIGURE_ARGS+=( "$@" ) +} +readonly -f pbuild::add_configure_args + +############################################################################### +# +pbuild::use_autotools() { + configure_with='autotools' +} +readonly -f pbuild::use_autotools + +############################################################################### +# +pbuild::use_cmake() { + configure_with='cmake' +} +readonly -f pbuild::use_cmake ############################################################################### # @@ -339,63 +436,58 @@ pbuild::post_prep() { pbuild::prep() { #...................................................................... # - # Find/download tarball for given module. + # download the source file if not already downloaded and validate + # checksum (if known). + # Abort on any error! # # Arguments: - # $1: store file name with upvar here - # $2: download URL - # $3: output filename (can be empty string) - # $4...: download directories + # $1 reference varibale to return result + # $2 download URL + # $3 save downloaded file with this name. If the empty + # string is passed, derive file name from URL + # $4... directories the source file might be already in. If the + # file does not exist in one of these directories, it + # is downloaded and stored in the first given directory. # - # Returns: - # 0 on success otherwise a value > 0 - # - download_with_curl() { - local -r output="$1" - local -r url="$2" - curl \ - -L \ - --output "${output}" \ - "${url}" - if (( $? != 0 )); then - curl \ - --insecure \ + download_source_file() { + download_with_curl() { + local -r output="$1" + local -r url="$2" + ${curl} \ + --location \ + --fail \ + --remove-on-error \ --output "${output}" \ "${url}" - fi - } + # :FIXME: How to handle insecure downloads? + #if (( $? != 0 )); then + # curl \ + # --insecure \ + # --output "${output}" \ + # "${url}" + #fi + } - check_hash_sum() { - local -r fname="$1" - local -r expected_hash_sum="$2" - local hash_sum='' + check_hash_sum() { + local -r fname="$1" + local -r expected_hash_sum="$2" + local hash_sum='' - if which 'sha256sum' 1>/dev/null; then - hash_sum=$(sha256sum "${fname}" | awk '{print $1}') - elif which 'shasum' 1>/dev/null; then - hash_sum=$(shasum -a 256 "${fname}" | awk '{print $1}') - else - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "Binary to compute SHA256 sum missing!" - fi - test "${hash_sum}" == "${expected_hash_sum}" || \ - std::die 42 \ - "%s " "${module_name}/${module_version}:" \ - "hash-sum missmatch for file '%s'" "${fname}" - } + hash_sum=$(${sha256sum} "${fname}" | awk '{print $1}') + test "${hash_sum}" == "${expected_hash_sum}" || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "hash-sum missmatch for file '${fname}'!" + } - download_source_file() { - local "$1" - local var="$1" + local -n _result="$1" local -r url="$2" local fname="$3" shift 3 dirs+=( "$@" ) [[ -n "${fname}" ]] || fname="${url##*/}" - local expr='s/.*\(.tar.bz2\|.tbz2\|.tar.gz\|.tgz\|.tar.xz\|.zip\)/\1/' - local -r extension=$(echo ${fname} | sed "${expr}") local dir='' dirs+=( 'not found' ) for dir in "${dirs[@]}"; do @@ -403,19 +495,20 @@ pbuild::prep() { done if [[ "${dir}" == 'not found' ]]; then dir="${dirs[0]}" - local -r method="${url%:*}" - case "${method}" in - http | https | ftp ) - download_with_curl "${dir}/${fname}" "${url}" - ;; - * ) - std::die 4 \ - "%s " "${module_name}/${module_version}:" \ - "Error in download URL:" \ - "unknown download method '${method}'!" - ;; - esac + download_with_curl "${dir}/${fname}" "${url}" + (( $? == 0 )) || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "downloading source file '${fname}' failed!" fi + _result="${dir}/${fname}" + [[ -r "${_result}" ]] || \ + std::die 42 \ + "%s " \ + "${module_name}/${module_version}:" \ + "source file '${_result}' is not readable!" + local sha256_sum='' local hash='' for hash in "${SOURCE_SHA256_SUMS[@]}"; do @@ -426,15 +519,13 @@ pbuild::prep() { if [[ -n "${sha256_sum}" ]]; then check_hash_sum "${dir}/${fname}" "${sha256_sum}" fi - std::upvar "${var}" "${dir}/${fname}" - [[ -r "${dir}/${fname}" ]] } unpack() { local -r file="$1" local -r dir="${2:-${SRC_DIR}}" - tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || { - rm -f "${file}" + ${tar} --directory="${dir}" -xv --strip-components 1 -f "${file}" || { + ${rm} -f "${file}" std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -451,7 +542,7 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" - patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ + ${patch} -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ std::die 4 \ "%s " \ "${module_name}/${module_version}:" \ @@ -470,7 +561,7 @@ pbuild::prep() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "Download source not set!" - mkdir -p "${PMODULES_DISTFILESDIR}" + ${mkdir} -p "${PMODULES_DISTFILESDIR}" local i=0 local source_fname for ((i = 0; i < ${#SOURCE_URLS[@]}; i++)); do @@ -483,59 +574,29 @@ pbuild::prep() { std::die 4 \ "%s " "${module_name}/${module_version}:" \ "sources for not found." - local dir=${SOURCE_UNPACK_DIRS[${source_fname##*/}]:-''} + local dir='' + local key="${SOURCE_URLS[i]##*/}" + if [[ -v SOURCE_UNPACK_DIRS[${key}] ]]; then + echo "dir specified" + dir="${SOURCE_UNPACK_DIRS[${SOURCE_URLS[i]##*/}]}" + else + echo "use SRC_DIR" + dir="${SRC_DIR}" + fi unpack "${source_fname}" "${dir}" done patch_sources # create build directory - mkdir -p "${BUILD_DIR}" -} - -pbuild::add_patch() { - [[ -z "$1" ]] && \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - PATCH_FILES+=( "$1" ) - PATCH_STRIPS+=( "$2" ) -} - -pbuild::set_default_patch_strip() { - [[ -n "$1" ]] || \ - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "${FUNCNAME}: missing argument!" - - PATCH_STRIP_DEFAULT="$1" -} - -pbuild::use_flag() { - [[ "${USE_FLAGS}" =~ ":${1}:" ]] + ${mkdir} -p "${BUILD_DIR}" } ############################################################################### # +# Configure the software to be compiled. +# +# Arguments: +# none # -pbuild::pre_configure() { - : -} - -pbuild::set_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::add_configure_args() { - CONFIGURE_ARGS+=( "$@" ) -} - -pbuild::use_autotools() { - configure_with='autotools' -} - -pbuild::use_cmake() { - configure_with='cmake' -} - pbuild::configure() { case "${configure_with}" in autotools ) @@ -567,6 +628,7 @@ pbuild::configure() { elif [[ -r "${SRC_DIR}/CMakeLists.txt" ]] && \ [[ "${configure_with}" == 'undef' ]] || \ [[ "${configure_with}" == "cmake" ]]; then + # note: in most/many cases a cmake module is used! cmake \ -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ "${CONFIGURE_ARGS[@]}" \ @@ -582,36 +644,38 @@ pbuild::configure() { fi } -pbuild::post_configure() { - : -} - -pbuild::pre_compile() { - : -} +############################################################################### +# +# Default compile function. +# +# Arguments: +# none +# pbuild::compile() { - make -j${JOBS} || \ + (( JOBS == 0 )) && JOBS=$(_get_num_cores) + ${make} -j${JOBS} || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } -pbuild::post_compile() { - : -} - -pbuild::pre_install() { - : -} - +############################################################################### +# +# Default install function. +# +# Arguments: +# none +# pbuild::install() { - make install || \ + ${make} install || \ std::die 3 \ "%s " "${module_name}/${module_version}:" \ "compilation failed!" } +############################################################################### +# pbuild::install_shared_libs() { local -r binary="$1" local -r dstdir="$2" @@ -619,19 +683,19 @@ pbuild::install_shared_libs() { install_shared_libs_Linux() { local libs=( $(ldd "${binary}" | \ - awk "/ => \// && /${pattern}/ {print \$3}") ) + ${awk} "/ => \// && /${pattern}/ {print \$3}") ) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } install_shared_libs_Darwin() { # https://stackoverflow.com/questions/33991581/install-name-tool-to-update-a-executable-to-search-for-dylib-in-mac-os-x - local libs=( $(otool -L "${binary}" | \ - awk "/${pattern}/ {print \$1}")) + local libs=( $(${otool} -L "${binary}" | \ + ${awk} "/${pattern}/ {print \$1}")) if [[ -n "${libs}" ]]; then - cp -vL "${libs[@]}" "${dstdir}" || return $? + ${cp} -vL "${libs[@]}" "${dstdir}" || return $? fi return 0 } @@ -640,7 +704,7 @@ pbuild::install_shared_libs() { std::die 3 \ "%s " "${module_name}/${module_version}:" \ "${binary}: does not exist or is not executable!" - mkdir -p "${dstdir}" + ${mkdir} -p "${dstdir}" case "${OS}" in Linux ) install_shared_libs_Linux @@ -651,48 +715,292 @@ pbuild::install_shared_libs() { esac } -pbuild::post_install() { - : -} - +############################################################################### # -# The 'do it all' function. +# This is the main entry function called by modbuild! # -pbuild::make_all() { - source "${BUILD_SCRIPT}" +pbuild.build_module() { + declare -gx module_name="$1" + declare -gx module_version="$2" + declare -gx module_release="$3" + shift 3 + with_modules=( "$@" ) - #set -o errexit - local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" - # module name including path in hierarchy and version - # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) - local modulefile_dir='' - local modulefile_name='' + # used in _make_all + declare -a runtime_dependencies=() + declare -a install_dependencies=() + #...................................................................... # - # To be able to set environment variables in one of the 'pbuild::TARGET' - # function we cannot use PIPE's like - # pbuild::configure | tee -a ... + # test whether a module is loaded or not # - rm -f "${logfile}" - if [[ "${verbose}" == 'yes' ]]; then - exec > >(tee -a "${logfile}") - else - exec > >(cat >> "${logfile}") - fi - exec 2> >(tee -a "${logfile}" >&2) + # Arguments: + # $1 module name + # + is_loaded() { + [[ :${LOADEDMODULES}: =~ :$1: ]] + } + #...................................................................... # - # everything set up? + # Initialise environment modules. # - [[ -n ${GROUP} ]] || \ - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "Module group not set! Aborting ..." + # Arguments: + # none + # + init_module_environment(){ + eval $( "${MODULECMD}" bash use unstable ) + eval $( "${MODULECMD}" bash use deprecated ) + eval $( "${MODULECMD}" bash purge ) + # :FIXME: this is a hack!!! + # shouldn't this be set in the build-script? + if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then + eval $( "${MODULECMD}" bash use Libraries ) + fi + if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then + eval $( "${MODULECMD}" bash use System ) + fi + unset C_INCLUDE_PATH + unset CPLUS_INCLUDE_PATH + unset CPP_INCLUDE_PATH + unset LIBRARY_PATH + unset LD_LIBRARY_PATH + unset DYLD_LIBRARY_PATH + + unset CFLAGS + unset CPPFLAGS + unset CXXFLAGS + unset LIBS + unset LDFLAGS + unset CC + unset CXX + unset FC + unset F77 + unset F90 + } + + #...................................................................... # - # helper functions + # Load build- and run-time dependencies. # + # Arguments: + # none + # + # Variables + # module_release set if defined in a variants file + # runtime_dependencies runtime dependencies from variants added + # + load_build_dependencies() { + + #.............................................................. + # + # build a dependency + # + # $1: name of module to build + # + # :FIXME: needs testing + # + build_dependency() { + find_build_script(){ + local p=$1 + local script=$(${find} "${BUILDBLOCK_DIR}/../.." \ + -path "*/$p/build") + std::get_abspath "${script}" + } + + local -r m=$1 + std::debug "${m}: module not available" + [[ ${dry_run} == yes ]] && \ + std::die 1 \ + "%s " \ + "${m}: module does not exist," \ + "cannot continue with dry run..." + + std::info "%s " \ + "$m: module does not exist, trying to build it..." + local args=( '' ) + set -- ${ARGS[@]} + while (( $# > 0 )); do + case $1 in + -j ) + args+=( "-j $2" ) + shift + ;; + --jobs=[0-9]* ) + args+=( $1 ) + ;; + -v | --verbose) + args+=( $1 ) + ;; + --with=*/* ) + args+=( $1 ) + ;; + esac + shift + done + + local buildscript=$(find_build_script "${m%/*}") + [[ -x "${buildscript}" ]] || \ + std::die 1 \ + "$m: build-block not found!" + if ! "${buildscript}" "${m#*/}" ${args[@]}; then + std::die 1 \ + "$m: oops: build failed..." + fi + } + + local m='' + for m in "${with_modules[@]}"; do + + # module name prefixes in dependency declarations: + # 'b:' this is a build dependency + # 'r:' this a run-time dependency, *not* required for + # building + # without prefix: this is a build and + # run-time dependency + if [[ "${m:0:2}" == "b:" ]]; then + m=${m#*:} # remove 'b:' + elif [[ "${m:0:2}" == "r:" ]]; then + m=${m#*:} # remove 'r:' + runtime_dependencies+=( "$m" ) + elif [[ "${m:0:2}" == "R:" ]]; then + m=${m#*:} # remove 'R:' + install_dependencies+=( "$m" ) + continue + else + runtime_dependencies+=( "$m" ) + fi + is_loaded "$m" && continue + + # 'module avail' might output multiple matches if module + # name and version are not fully specified or in case + # modules with and without a release number exist. + # Example: + # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list + # from 'module avail' and the full version should be set + # in the variants file, we look for the first exact + # match. + local release_of_dependency='' + if ! pbuild::module_is_avail "$m" release_of_dependency; then + build_dependency "$m" + pbuild::module_is_avail "$m" release_of_dependency || \ + std::die 6 "Oops" + fi + # should be set, just in case it is not... + : ${release_of_dependency:='unstable'} + + # for a stable module all dependencies must be stable + if [[ "${module_release}" == 'stable' ]] \ + && [[ "${release_of_dependency}" != 'stable' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + # for a unstable module no dependency must be deprecated + elif [[ "${module_release}" == 'unstable' ]] \ + && [[ "${release_of_dependency}" == 'deprecated' ]]; then + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "release cannot be set to '${module_release}'" \ + "since the dependency '$m' is ${release_of_dependency}" + fi + + std::info "Loading module: ${m}" + eval $( "${MODULECMD}" bash load "${m}" ) + done + } + + init_build_environment() { + #...................................................................... + # + # parse the passed version string + # + # the following global variables will be set in this function: + # V_MAJOR + # V_MINOR + # V_PATCHLVL + # V_RELEASE + # USE_FLAGS + # + parse_version() { + local v="$1" + V_MAJOR='' # first number in version string + V_MINOR='' # second number in version string (or empty) + V_PATCHLVL='' # third number in version string (or empty) + V_RELEASE='' # module release (or empty) + : ${USE_FLAGS:=''} # architectures (or empty) + + local tmp='' + + if [[ "$v" =~ "_" ]]; then + tmp="${v#*_}" + USE_FLAGS+=":${tmp//_/:}:" + v="${v%%_*}" + fi + V_PKG="${v%%-*}" # version without the release number + if [[ $v == *-* ]]; then + V_RELEASE="${v#*-}" # release number + else + V_RELEASE='' + fi + case "${V_PKG}" in + *.*.* ) + V_MAJOR="${V_PKG%%.*}" + tmp="${V_PKG#*.}" + V_MINOR="${tmp%%.*}" + V_PATCHLVL="${tmp#*.}" + ;; + *.* ) + V_MAJOR="${V_PKG%.*}" + V_MINOR="${V_PKG#*.}" + ;; + * ) + V_MAJOR="${V_PKG}" + ;; + esac + + VERSIONS=() + if [[ -n ${V_RELEASE} ]]; then + VERSIONS+=( ${V_PKG}-${V_RELEASE} ) + fi + if [[ -n ${V_PATCHLVL} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ) + fi + if [[ -n ${V_MINOR} ]]; then + VERSIONS+=( ${V_MAJOR}.${V_MINOR} ) + fi + VERSIONS+=( ${V_MAJOR} ) + } + + local -r module_name="$1" + local -r module_version="$2" + + SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" + BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" + + # P and V can be used in the build-script, so we have to set them here + P="${module_name}" + V="${module_version}" + parse_version "${module_version}" + declare -gx GROUP='' + declare -g PREFIX='' + + SOURCE_URLS=() + SOURCE_SHA256_SUMS=() + SOURCE_NAMES=() + declare -Ag SOURCE_UNPACK_DIRS=() + CONFIGURE_ARGS=() + SUPPORTED_SYSTEMS=() + SUPPORTED_OS=() + SUPPORTED_COMPILERS=() + PATCH_FILES=() + PATCH_STRIPS=() + PATCH_STRIP_DEFAULT='1' + MODULE_DOCFILES=() + configure_with='undef' + } # init_build_environment() #...................................................................... check_supported_systems() { @@ -762,19 +1070,23 @@ pbuild::make_all() { else unset Compiler_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v HDF5_SERIAL_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v HDF5_SERIAL_VERSION ]]; then HDF5_serial_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_serial_HIERARCHY+=' hdf5_serial/${HDF5_SERIAL_VERSION}' else unset HDF5_serial_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]]; then MPI_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' MPI_HIERARCHY+=' ${MPI}/${MPI_VERSION}' else unset MPI_HIERARCHY fi - if [[ -v COMPILER_VERSION ]] && [[ -v MPI_VERSION ]] && [[ HDF5_VERSION ]]; then + if [[ -v COMPILER_VERSION ]] && \ + [[ -v MPI_VERSION ]] && \ + [[ HDF5_VERSION ]]; then HDF5_HIERARCHY='${COMPILER}/${COMPILER_VERSION}' HDF5_HIERARCHY+=' ${MPI}/${MPI_VERSION}' HDF5_HIERARCHY+=' hdf5/${HDF5_VERSION}' @@ -821,110 +1133,57 @@ pbuild::make_all() { else do_hierarchical_group fi - } + } # set_full_module_name_and_prefix #...................................................................... - # Select the modulefile to install. Modulefiles can be versioned like - # modulefile-10.2.0 - # modulefile-10.2 - # modulefile-10 - # modulefile - # the most specific modulefile will be selected. Example: - # For a version 10.2.1 the file moduelfile-10.2 would be selected. + # post-install. # # Arguments: - # $1 upvar to return the filename - # - # Used gloabal variables: - # VERSIONS - # BUILDBLOCK_DIR - # - find_modulefile() { - local "$1" - local fname='' - local modulefile='' - for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do - if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then - modulefile="${BUILDBLOCK_DIR}/${fname}" - break; - fi - done - std::upvar $1 "${modulefile}" - [[ -n "${modulefile}" ]] - } - - #...................................................................... - # non-redefinable post-install. Install: - # - documentation files as defined in the build-script - # - modulefile and file with release - # . + # none post_install() { #.............................................................. - # install the doc-files specified in the build-script + # post-install: + # - build-script + # - list of loaded modules while building + # - doc-files specified in the build-script # # Arguments: # none # install_doc() { - if [[ -z "${MODULE_DOCFILES}" ]]; then + local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "Installing documentation to ${docdir}" + ${install} -m 0755 -d "${docdir}" + ${install} -m0644 "${BUILD_SCRIPT}" "${docdir}" + "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ + ${grep} -v "Currently Loaded" > \ + "${docdir}/dependencies" || : + + if [[ ! -v MODULE_DOCFILES[0] ]]; then + # loop over version specific functions. In these function + # more MODULE_DOCFILES can be defined. + # :FIXME: maybe we find a better solution. for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do if typeset -F "$f" 2>/dev/null; then $f break fi done - fi - [[ -n "${MODULE_DOCFILES}" ]] || return 0 - local -r docdir="${PREFIX}/${_DOCDIR}/${module_name}" - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "Installing documentation to ${docdir}" - install -m 0755 -d \ - "${docdir}" - install -m0644 \ + if [[ ! -v MODULE_DOCFILES[0] ]]; then + return 0 + fi + ${install} -m0644 \ "${MODULE_DOCFILES[@]/#/${SRC_DIR}/}" \ "${docdir}" return 0 } #.............................................................. - # install build-block files - # - modulefile - # - build-script - # - run-time and build dependencies - # in ${PREFIX}/share/${GROUP}/${module_name} - # - # Arguments: - # none - # - install_pmodules_files() { - local modulefile='' - find_modulefile modulefile || return 0 - - local -r target_dir="${PREFIX}/share/$GROUP/${module_name}" - mkdir -p "${target_dir}" - install -m0644 \ - "${BUILD_SCRIPT}" \ - "${target_dir}" - install -m0644 \ - "${modulefile}" \ - "${target_dir}" - #install -m 0755 \ - # -d "${target_dir}/files" - #install -m0644 \ - # "${variants_file}" \ - # "${target_dir}/files" - - local -r fname="${target_dir}/dependencies" - "${MODULECMD}" bash list -t 2>&1 1>/dev/null | \ - grep -v "Currently Loaded" > "${fname}" || : - } - - #.............................................................. - # write run time dependencies to file + # post-install: write file with required modules write_runtime_dependencies() { local -r fname="$1" shift @@ -947,8 +1206,8 @@ pbuild::make_all() { } #.............................................................. - # for Linux we need a special post-install to solve the - # multilib problem with LIBRARY_PATH on 64-bit systems + # post-install: for Linux we need a special post-install to + # solve the multilib problem with LIBRARY_PATH on 64-bit systems post_install_linux() { std::info \ "%s " \ @@ -960,10 +1219,10 @@ pbuild::make_all() { } #.............................................................. + # post-install cd "${BUILD_DIR}" [[ "${OS}" == "Linux" ]] && post_install_linux install_doc - install_pmodules_files if [[ -v runtime_dependencies[0] ]]; then write_runtime_dependencies \ "${PREFIX}/${FNAME_RDEPS}" \ @@ -976,7 +1235,6 @@ pbuild::make_all() { fi install_modulefile install_release_file - cleanup_modulefiles cleanup_build cleanup_src std::info \ @@ -984,14 +1242,47 @@ pbuild::make_all() { "${module_name}/${module_version}:" \ "Done ..." return 0 - } + } # post_install #...................................................................... # Install modulefile in ${pm_root}/${GROUP}/modulefiles/... + # The modulefiles in the build-block can be + # versioned like + # modulefile-10.2.0 + # modulefile-10.2 + # modulefile-10 + # modulefile + # the most specific modulefile will be selected. Example: + # For a version 10.2.1 the file moduelfile-10.2 would be + # selected. # # Arguments # none + # + # Used gloabal variables: + # VERSIONS + # BUILDBLOCK_DIR + # modulefile_name + # install_modulefile() { + #.............................................................. + # Select the modulefile to install. + # + # Arguments: + # $1 upvar to return the filename + # + find_modulefile() { + local -n _modulefile="$1" + local fname='' + for fname in "${VERSIONS[@]/#/modulefile-}" 'modulefile'; do + if [[ -r "${BUILDBLOCK_DIR}/${fname}" ]]; then + _modulefile="${BUILDBLOCK_DIR}/${fname}" + break; + fi + done + [[ -n "${_modulefile}" ]] + } + local src='' find_modulefile src if (( $? != 0 )); then @@ -1005,8 +1296,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "adding modulefile to overlay '${ol_name}' ..." - mkdir -p "${modulefile_dir}" - install -m 0644 "${src}" "${modulefile_name}" + ${mkdir} -p "${modulefile_dir}" + ${install} -m 0644 "${src}" "${modulefile_name}" } cleanup_modulefiles(){ @@ -1024,7 +1315,7 @@ pbuild::make_all() { std::info "%s "\ "${module_name}/${module_version}:" \ "removing modulefile from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi fname="${dir}/.release-${module_version}" if [[ -e "${fname}" ]]; then @@ -1032,7 +1323,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing release file from overlay '${ol}' ..." - rm "${fname}" + ${rm} "${fname}" fi done } @@ -1065,7 +1356,7 @@ pbuild::make_all() { [[ "${BUILD_DIR}" == "${SRC_DIR}" ]] && return 0 { cd "/${BUILD_DIR}/.." || std::die 42 "Internal error" - [[ "$(pwd)" == "/" ]] && \ + [[ "$(${pwd})" == "/" ]] && \ std::die 1 \ "%s " "${module_name}/${module_version}:" \ "Oops: internal error:" \ @@ -1075,7 +1366,7 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "Cleaning up '${BUILD_DIR}'..." - rm -rf "${BUILD_DIR##*/}" + ${rm} -rf "${BUILD_DIR##*/}" }; return 0 } @@ -1098,67 +1389,66 @@ pbuild::make_all() { return 0 } - build_target() { - local dir="$1" # src or build directory, depends on target - local target="$2" # prep, configure, compile or install - - if [[ -e "${BUILD_DIR}/.${target}" ]] && \ - [[ ${force_rebuild} != 'yes' ]]; then - return 0 - fi - local targets=() - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) - targets+=( pbuild::pre_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) - targets+=( pbuild::pre_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) - targets+=( pbuild::pre_${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) - targets+=( pbuild::${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) - targets+=( pbuild::${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) - targets+=( pbuild::${target} ) - - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) - targets+=( pbuild::post_${target}_${system} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) - targets+=( pbuild::post_${target}_${OS} ) - targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) - targets+=( pbuild::post_${target} ) - - for t in "${targets[@]}"; do - # We cd into the dir before calling the function - - # just to be sure we are in the right directory. - # - # Executing the function in a sub-process doesn't - # work because in some function global variables - # might/need to be set. - # - cd "${dir}" - if typeset -F "$t" 2>/dev/null; then - "$t" || \ - std::die 10 "Aborting..." - fi - done - touch "${BUILD_DIR}/.${target}" - } - #...................................................................... # build module ${module_name}/${module_version} - build_module() { + compile_and_install() { + build_target() { + local dir="$1" # src or build directory, depends on target + local target="$2" # prep, configure, compile or install + + if [[ -e "${BUILD_DIR}/.${target}" ]] && \ + [[ ${force_rebuild} != 'yes' ]]; then + return 0 + fi + local targets=() + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${system}_} ) + targets+=( pbuild::pre_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_${OS}_} ) + targets+=( pbuild::pre_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::pre_${target}_} ) + targets+=( pbuild::pre_${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${system}_} ) + targets+=( pbuild::${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_${OS}_} ) + targets+=( pbuild::${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::${target}_} ) + targets+=( pbuild::${target} ) + + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${system}_} ) + targets+=( pbuild::post_${target}_${system} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_${OS}_} ) + targets+=( pbuild::post_${target}_${OS} ) + targets+=( ${VERSIONS[@]/#/pbuild::post_${target}_} ) + targets+=( pbuild::post_${target} ) + + for t in "${targets[@]}"; do + # We cd into the dir before calling the function - + # just to be sure we are in the right directory. + # + # Executing the function in a sub-process doesn't + # work because in some function global variables + # might/need to be set. + # + cd "${dir}" + if typeset -F "$t" 2>/dev/null; then + "$t" || \ + std::die 10 "Aborting..." + fi + done + touch "${BUILD_DIR}/.${target}" + } # compile_and_install():build_target() + [[ ${dry_run} == yes ]] && std::die 0 "" - mkdir -p "${SRC_DIR}" - mkdir -p "${BUILD_DIR}" + ${mkdir} -p "${SRC_DIR}" + ${mkdir} -p "${BUILD_DIR}" + ${mkdir} -p "${PREFIX}" std::info \ "%s " \ "${module_name}/${module_version}:" \ "preparing sources ..." - # write stdout and stderr to logfile, stderr to terminal - # write all to logfile and terminal build_target "${SRC_DIR}" prep [[ "${build_target}" == "prep" ]] && return 0 @@ -1180,10 +1470,8 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "installing ..." - mkdir -p "${PREFIX}" build_target "${BUILD_DIR}" install - post_install - } + } # compile_and_install() remove_module() { if [[ -d "${PREFIX}" ]]; then @@ -1191,14 +1479,14 @@ pbuild::make_all() { "%s " \ "${module_name}/${module_version}:" \ "removing all files in '${PREFIX}' ..." - [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} + [[ "${dry_run}" == 'no' ]] && ${rm} -rf ${PREFIX} fi if [[ -e "${modulefile_name}" ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ "removing modulefile '${modulefile_name}' ..." - [[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}" + [[ "${dry_run}" == 'no' ]] && ${rm} -v "${modulefile_name}" fi local release_file="${modulefile_dir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then @@ -1208,34 +1496,70 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p "${modulefile_dir}" 2>/dev/null || : + ${rmdir} -p "${modulefile_dir}" 2>/dev/null || : } - ######################################################################## - # - # here we really start with make_all() - # + deprecate_module(){ + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "is deprecated, skiping!" + install_release_file + } - # setup module specific environment + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + ${with_modules:+with ${with_modules[@]}} \ + "building ..." + + init_module_environment + load_build_dependencies + init_build_environment "${module_name}" "${module_version}" + + source "${BUILD_SCRIPT}" + + # module name including path in hierarchy and version + # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) + local modulefile_dir='' + local modulefile_name='' + + # + # :FIXME: add comments what and why we are doing this. + # + local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + rm -f "${logfile}" + if [[ "${verbose}" == 'yes' ]]; then + exec > >(${tee} -a "${logfile}") + else + exec > >(${cat} >> "${logfile}") + fi + exec 2> >(${tee} -a "${logfile}" >&2) + + # the group must have been defined - otherwise we cannot continue + [[ -n ${GROUP} ]] || \ + std::die 5 \ + "%s " "${module_name}/${module_version}:" \ + "Module group not set! Aborting ..." + + # check whether this module is supported check_supported_systems check_supported_os check_supported_compilers + # setup module name and prefix set_full_module_name_and_prefix + + # ok, finally we can start ... std::info \ "%s " \ "${module_name}/${module_version}:" \ ${with_modules:+build with ${with_modules[@]}} - if [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then - # don't (re-)build the module, but - # - if the release stage has been changed to 'removed', - # remove the module. - # - if requested, update the modulefile. - # - if modulefile does not exist, install it. - # - update release stage. - if [[ "${module_release}" == 'removed' ]]; then - remove_module - return $? - fi + + if [[ "${module_release}" == 'removed' ]]; then + remove_module + elif [[ "${module_release}" == 'deprecated' ]]; then + deprecate_module + elif [[ -d ${PREFIX} ]] && [[ ${force_rebuild} != 'yes' ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ @@ -1245,308 +1569,18 @@ pbuild::make_all() { install_modulefile fi install_release_file - cleanup_modulefiles - return $? - fi - if [[ "${module_release}" == 'deprecated' ]]; then + else std::info \ "%s " \ "${module_name}/${module_version}:" \ - "is deprecated, skiping!" - install_release_file - return $? + "start building ..." + compile_and_install + post_install fi - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "start building ..." - - build_module - return $? -} - -pbuild.init_env() { - #...................................................................... - # - # parse the passed version string - # - # the following global variables will be set in this function: - # V_MAJOR - # V_MINOR - # V_PATCHLVL - # V_RELEASE - # USE_FLAGS - # - parse_version() { - local v="$1" - V_MAJOR='' # first number in version string - V_MINOR='' # second number in version string (or empty) - V_PATCHLVL='' # third number in version string (or empty) - V_RELEASE='' # module release (or empty) - : ${USE_FLAGS:=''} # architectures (or empty) - - local tmp='' - - if [[ "$v" =~ "_" ]]; then - tmp="${v#*_}" - USE_FLAGS+=":${tmp//_/:}:" - v="${v%%_*}" - fi - V_PKG="${v%%-*}" # version without the release number - if [[ $v == *-* ]]; then - V_RELEASE="${v#*-}" # release number - else - V_RELEASE='' - fi - case "${V_PKG}" in - *.*.* ) - V_MAJOR="${V_PKG%%.*}" - tmp="${V_PKG#*.}" - V_MINOR="${tmp%%.*}" - V_PATCHLVL="${tmp#*.}" - ;; - *.* ) - V_MAJOR="${V_PKG%.*}" - V_MINOR="${V_PKG#*.}" - ;; - * ) - V_MAJOR="${V_PKG}" - ;; - esac - - VERSIONS=( ${V_MAJOR} ) - if [[ -n ${V_MINOR} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR} ${VERSIONS[@]} ) - fi - if [[ -n ${V_PATCHLVL} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL} ${VERSIONS[@]} ) - fi - if [[ -n ${V_RELEASE} ]]; then - VERSIONS=( ${V_MAJOR}.${V_MINOR}.${V_PATCHLVL}-${V_RELEASE} ${VERSIONS[@]} ) - fi - } - - local -r module_name="$1" - local -r module_version="$2" - - SRC_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/src" - BUILD_DIR="${PMODULES_TMPDIR}/${module_name}-${module_version}/build" - - # P and V can be used in the build-script, so we have to set them here - P="${module_name}" - V="${module_version}" - parse_version "${module_version}" - - SOURCE_URLS=() - SOURCE_SHA256_SUMS=() - SOURCE_NAMES=() - CONFIGURE_ARGS=() - SUPPORTED_SYSTEMS=() - SUPPORTED_OS=() - SUPPORTED_COMPILERS=() - PATCH_FILES=() - PATCH_STRIPS=() - PATCH_STRIP_DEFAULT='1' - MODULE_DOCFILES=() - configure_with='undef' -} - -pbuild.build_module() { - module_name="$1" - module_version="$2" - module_release="$3" - shift 3 - with_modules=( "$@" ) - - # used in pbuild::make_all - declare -a runtime_dependencies=() - declare -a install_dependencies=() - - #...................................................................... - # - # test whether a module is loaded or not - # - # $1: module name - # - is_loaded() { - [[ :${LOADEDMODULES}: =~ :$1: ]] - } - - #...................................................................... - # - # build a dependency - # - # $1: name of module to build - # - # :FIXME: needs testing - # - build_dependency() { - local -r m=$1 - std::debug "${m}: module not available" - local rels=( ${ReleaseStages//:/ } ) - [[ ${dry_run} == yes ]] && \ - std::die 1 \ - "%s " \ - "${m}: module does not exist," \ - "cannot continue with dry run..." - - std::info "%s " \ - "$m: module does not exist, trying to build it..." - local args=( '' ) - set -- ${ARGS[@]} - while (( $# > 0 )); do - case $1 in - -j ) - args+=( "-j $2" ) - shift - ;; - --jobs=[0-9]* ) - args+=( $1 ) - ;; - -v | --verbose) - args+=( $1 ) - ;; - --with=*/* ) - args+=( $1 ) - ;; - esac - shift - done - - find_build_script(){ - local p=$1 - local script=$(find "${BUILDBLOCK_DIR}/../.." -path "*/$p/build") - std::get_abspath "${script}" - } - local buildscript=$(find_build_script "${m%/*}") - [[ -x "${buildscript}" ]] || \ - std::die 1 \ - "$m: build-block not found!" - if ! "${buildscript}" "${m#*/}" ${args[@]}; then - std::die 1 \ - "$m: oops: build failed..." - fi - } - - #...................................................................... - # - # Load build- and run-time dependencies. - # - # Arguments: - # none - # - # Variables - # [r] module_release set if defined in a variants file - # runtime_dependencies runtime dependencies from variants added - # - load_build_dependencies() { - local m='' - for m in "${with_modules[@]}"; do - - # module name prefixes in dependency declarations: - # 'b:' this is a build dependency - # 'r:' this a run-time dependency, *not* required for - # building - # without prefix: this is a build and - # run-time dependency - if [[ "${m:0:2}" == "b:" ]]; then - m=${m#*:} # remove 'b:' - elif [[ "${m:0:2}" == "r:" ]]; then - m=${m#*:} # remove 'r:' - runtime_dependencies+=( "$m" ) - elif [[ "${m:0:2}" == "R:" ]]; then - m=${m#*:} # remove 'R:' - install_dependencies+=( "$m" ) - continue - else - runtime_dependencies+=( "$m" ) - fi - is_loaded "$m" && continue - - # 'module avail' might output multiple matches if module - # name and version are not fully specified or in case - # modules with and without a release number exist. - # Example: - # mpc/1.1.0 and mpc/1.1.0-1. Since we get a sorted list - # from 'module avail' and the full version should be set - # in the variants file, we look for the first exact - # match. - local release_of_dependency='' - if ! pbuild::module_is_avail "$m" release_of_dependency; then - build_dependency "$m" - pbuild::module_is_avail "$m" release_of_dependency || \ - std::die 6 "Oops" - fi - # should be set, just in case it is not... - : ${release_of_dependency:='unstable'} - - # for a stable module all dependencies must be stable - if [[ "${module_release}" == 'stable' ]] \ - && [[ "${release_of_dependency}" != 'stable' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - # for a unstable module no dependency must be deprecated - elif [[ "${module_release}" == 'unstable' ]] \ - && [[ "${release_of_dependency}" == 'deprecated' ]]; then - std::die 5 \ - "%s " "${module_name}/${module_version}:" \ - "release cannot be set to '${module_release}'" \ - "since the dependency '$m' is ${release_of_dependency}" - fi - - std::info "Loading module: ${m}" - eval $( "${MODULECMD}" bash load "${m}" ) - done - } - - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "building ..." - MODULECMD="${PMODULES_HOME}/bin/modulecmd" - [[ -x ${MODULECMD} ]] || \ - std::die 2 "No such file or executable -- '${MODULECMD}'" - - eval $( "${MODULECMD}" bash use unstable ) - eval $( "${MODULECMD}" bash use deprecated ) - eval $( "${MODULECMD}" bash purge ) - - # :FIXME: this is a hack!!! - # shouldn't this be set in the build-script? - if [[ -e "${PMODULES_HOME%%/Tools*}/Libraries" ]]; then - eval $( "${MODULECMD}" bash use Libraries ) - fi - if [[ -e "${PMODULES_HOME%%/Tools*}/System" ]]; then - eval $( "${MODULECMD}" bash use System ) - fi - unset C_INCLUDE_PATH - unset CPLUS_INCLUDE_PATH - unset CPP_INCLUDE_PATH - unset LIBRARY_PATH - unset LD_LIBRARY_PATH - unset DYLD_LIBRARY_PATH - - unset CFLAGS - unset CPPFLAGS - unset CXXFLAGS - unset LIBS - unset LDFLAGS - - unset CC - unset CXX - unset FC - unset F77 - unset F90 - - load_build_dependencies - - pbuild.init_env "${module_name}" "${module_version}" - pbuild::make_all + cleanup_modulefiles std::info "* * * * *\n" } +readonly -f pbuild.build_module # Local Variables: # mode: sh diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index 6a52073..eb3b6ba 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -39,7 +39,7 @@ std::def_cmds(){ local path="$1" shift for cmd in "$@"; do - eval declare -g ${cmd}=$(PATH="${path}" which $cmd 2>/dev/null) + eval declare -gr ${cmd}=$(PATH="${path}" /usr/bin/which $cmd 2>/dev/null) if [[ -z "${!cmd}" ]]; then std::die 255 "${cmd} not found" fi @@ -324,7 +324,7 @@ std::get_os_release() { local -A func_map; func_map['Linux']=std.get_os_release_linux func_map['Darwin']=std.get_os_release_macos - ${func_map[${OS}]} + ${func_map[$(uname -s)]} } std::get_type() { diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c5813d1..7f92148 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -7,58 +7,69 @@ # #............................................................................. -#set -x +set -x declare VERSION='@PMODULES_VERSION@' # get absolute path of script declare mydir=$(dirname "$0") declare -r mydir=$(cd ${mydir} && pwd -P) -# initialize PATH, -# add library installation directories to the PATH, -# so 'source' is able find them - -if [[ $(uname -s) == 'Darwin' ]]; then - PATH='/opt/local/bin:' -else - PATH='' -fi -PATH+='/usr/bin:/bin:/usr/sbin:/sbin' -PATH+=":${mydir}" -PATH+=":${mydir}/../lib:${mydir}/../config" -PATH+=":${mydir}/../libexec" - -path=$PATH - -source libstd.bash || { +source "${mydir}/../lib/libstd.bash" || { echo "Oops: cannot source library -- '$_'" 1>&2; exit 3; } -std::def_cmds "${path}" \ - 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' +############################################################################## +# +# check availability of used commands and set environment variables +# cmd=$(which cmd) +# in the following we use these environment variable to call binaries. +# +declare -r MODULECMD="${PMODULES_HOME}/bin/modulecmd" +[[ -x ${MODULECMD} ]] || \ + std::die 1 "Oops: modulecmd binary not available!" -# can be set in the configuration file +std::def_cmds "${mydir}/../libexec" \ + 'yq' + +std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'awk' 'base64' 'cat' 'cp' 'find' 'getopt' 'grep' \ + 'install' 'logger' 'make' 'mkdir' 'mktemp' 'patch' 'pwd' \ + 'rm' 'rmdir' 'sort' 'tar' 'tee' 'uname' + +declare -r OS="$(${uname} -s)" +if [[ ${OS} == 'Darwin' ]]; then + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin' \ + 'curl' 'otool' 'shasum' 'sysctl' + declare -r sha256sum="${shasum -a 256}" +else + std::def_cmds '/usr/bin:/bin:/usr/sbin:/sbin' \ + 'ldd' 'curl' 'sha256sum' +fi + +# for the time being, we still set PATH. Just in case we forgot a binary +PATH='/usr/bin:/bin:/usr/sbin:/sbin' + +############################################################################## +# save arguments, required for building dependencies +declare -r ARGS="$@" declare PMODULES_DISTFILESDIR='' declare PMODULES_TMPDIR='' declare pm_root="${PMODULES_HOME%%/Tools*}" +declare ol_mod_root='' +declare ol_inst_root='' -source libpbuild.bash || \ +source "${mydir}/../lib/libpbuild.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -source libpmodules.bash || \ +source "${mydir}/../lib/libpmodules.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" -# save arguments, (still) required for building dependencies -declare -r ARGS="$@" - +############################################################################## +# set some shell options set -o nounset shopt -s nocaseglob shopt -s extglob shopt -s nullglob -declare ol_mod_root -declare ol_inst_root - ############################################################################## # usage() { @@ -350,9 +361,6 @@ bash_expand(){ build_modules_legacy() { find_variants_files(){ - shopt -q nullglob || : - local -i nullglob_set=$? - shopt -s nullglob local files=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}"\.${opt_system} ) files+=( "${BUILDBLOCK_DIR}"/*/"${BNAME_VARIANTS}.$(uname -s)" ) local f @@ -361,7 +369,6 @@ build_modules_legacy() { || [[ -e "${f}.$(uname -s)" ]] \ || files+=( "$f" ) done - (( nullglob_set == 1 )) && shopt -u nullglob std::upvar "$1" "${files[@]}" } @@ -395,7 +402,7 @@ build_modules_legacy() { local m local pattern="/^${name}\/${version}[[:blank:]]/" for m in "${with_modules[@]}"; do - if [[ -n $(awk "/${m%/*}[\/ ]/" "${files[@]}") ]]; then + if [[ -n $(${awk} "/${m%/*}[\/ ]/" "${files[@]}") ]]; then pattern+=" && /${m//\//\\/}/" fi done @@ -405,7 +412,7 @@ build_modules_legacy() { local line='' while read line; do variants+=( "${line}" ) - done < <(expand_variants_file "${f}" | awk "${pattern}") + done < <(expand_variants_file "${f}" | ${awk} "${pattern}") done if (( ${#variants[@]} == 0 )); then std::info "%s " \ @@ -457,7 +464,7 @@ build_modules_yaml(){ local version="$3" _result=( $(${yq} -Ne e \ "with_entries(select(.key | test(\"^${version}\$\")))|keys" \ - "${fname}" 2>/dev/null | awk '{print $2}') ) + "${fname}" 2>/dev/null | ${awk} '{print $2}') ) } yaml_get_num_variants(){ @@ -593,7 +600,7 @@ build_modules_yaml(){ pbuild.build_module \ "${name}" "${v##*/}" \ "${relstage}" "${with_modules[@]}" - done < <(bash_expand "" ${deps[@]}|awk "${pattern}") + done < <(bash_expand "" ${deps[@]}|${awk} "${pattern}") else pbuild.build_module \ "${name}" "${v##*/}" \ @@ -634,7 +641,7 @@ pbuild.verbose "${opt_verbose}" # pm::read_config -# :FIXME: should go dist files to +# :FIXME: should dist files go to # ${pm_root}/var/distfiles # or # ${overlay}/var/distfiles diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 24f7dcb..b4a2035 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -30,7 +30,7 @@ declare -r os_name="$(uname -s)" path="${libexecdir}:/bin:/usr/bin" std::def_cmds "${path}" \ 'awk' 'base64' 'find' 'getopt' 'logger' 'mktemp' \ - 'rm' 'sort' 'find' 'yq' + 'rm' 'sort' 'yq' if [[ ${PMODULES_PURETCL} == yes ]]; then declare -r modulecmd="${libexecdir}/modulecmd.tcl" From 2112a05dea3c2292b9dc47481aaaa86ad67bb1fe Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:07:05 +0200 Subject: [PATCH 09/15] modbuild: issues with nounset fixed --- Pmodules/libpbuild.bash | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 72f3fd3..87e1dc3 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -261,7 +261,7 @@ readonly -f pbuild::supported_compilers # $1: group # pbuild::add_to_group() { - if [[ -z ${1} ]]; then + if (( $# == 0 )); then std::die 42 \ "%s " "${module_name}/${module_version}:" \ "${FUNCNAME}: missing group argument." @@ -385,7 +385,11 @@ pbuild::add_patch() { "%s " "${module_name}/${module_version}:" \ "${FUNCNAME}: missing argument!" PATCH_FILES+=( "$1" ) - PATCH_STRIPS+=( "$2" ) + if (( $# >= 2 )); then + PATCH_STRIPS+=( "$2" ) + else + PATCH_STRIPS+=( "${PATCH_STRIP_DEFAULT}" ) + fi } readonly -f pbuild::add_patch From 4e2150b6d733bc4cefad5700c34f20bd67ee4a72 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:10:58 +0200 Subject: [PATCH 10/15] libpmodules: create directory in compute_group_depth() if missing --- Pmodules/libpmodules.bash.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Pmodules/libpmodules.bash.in b/Pmodules/libpmodules.bash.in index f394df5..30d2f25 100644 --- a/Pmodules/libpmodules.bash.in +++ b/Pmodules/libpmodules.bash.in @@ -40,7 +40,10 @@ print_help() { compute_group_depth () { local -n result="$1" local -r dir="$2" - test -d "${dir}" || return 1 + if [[ ! -d "${dir}" ]]; then + ${mkdir} -p "${dir}" || \ + std::die 1 "Cannot create directory -- ${dir}" + fi local group=${dir%/*} local group=${group##*/} result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ From 18284822f881021b4fa9ecba74c06c37b2c41136 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:14:26 +0200 Subject: [PATCH 11/15] libpbuild.bash: curl on RHEL7 doesn't support the opt '--remove-on-error' --- Pmodules/libpbuild.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 87e1dc3..b111021 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -460,7 +460,6 @@ pbuild::prep() { ${curl} \ --location \ --fail \ - --remove-on-error \ --output "${output}" \ "${url}" # :FIXME: How to handle insecure downloads? From 3b0ed98524733bbae9d3953d22dc207319834f8e Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:15:34 +0200 Subject: [PATCH 12/15] libpbuild.bash: create $PREFIX dir before install target The directory $PREFIX should not created to early. If $PREFIX is created at the beginning and a build target fails, the next build attempt will skip all build targets. --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index b111021..d775a5f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1446,7 +1446,6 @@ pbuild.build_module() { ${mkdir} -p "${SRC_DIR}" ${mkdir} -p "${BUILD_DIR}" - ${mkdir} -p "${PREFIX}" std::info \ "%s " \ @@ -1469,6 +1468,7 @@ pbuild.build_module() { build_target "${BUILD_DIR}" compile [[ "${build_target}" == "compile" ]] && return 0 + ${mkdir} -p "${PREFIX}" std::info \ "%s " \ "${module_name}/${module_version}:" \ From fa84196a04bf80a54d1d9a646d00fe76a516d22d Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:20:53 +0200 Subject: [PATCH 13/15] libpbuild.bash: fix in comment --- Pmodules/libpbuild.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index d775a5f..ddb954f 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -1248,7 +1248,7 @@ pbuild.build_module() { } # post_install #...................................................................... - # Install modulefile in ${pm_root}/${GROUP}/modulefiles/... + # Install modulefile in ${ol_mod_root}/${GROUP}/modulefiles/... # The modulefiles in the build-block can be # versioned like # modulefile-10.2.0 From 3a7746805d13a4092aa36caea175daccedf94cd4 Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:21:55 +0200 Subject: [PATCH 14/15] modbuild: cleanup --- Pmodules/modbuild.in | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index 7f92148..c7597b4 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -50,21 +50,12 @@ fi PATH='/usr/bin:/bin:/usr/sbin:/sbin' ############################################################################## -# save arguments, required for building dependencies -declare -r ARGS="$@" -declare PMODULES_DISTFILESDIR='' -declare PMODULES_TMPDIR='' -declare pm_root="${PMODULES_HOME%%/Tools*}" -declare ol_mod_root='' -declare ol_inst_root='' - source "${mydir}/../lib/libpbuild.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" source "${mydir}/../lib/libpmodules.bash" || \ std::die 3 "Oops: cannot source library -- '$_'" ############################################################################## -# set some shell options set -o nounset shopt -s nocaseglob shopt -s extglob @@ -168,6 +159,11 @@ MISCELLANEOUS OPTIONS: # then configuration file # last default +# save arguments, required for building dependencies +declare -r ARGS="$@" +declare PMODULES_DISTFILESDIR='' +declare PMODULES_TMPDIR='' + # versions to be build, '.*' or none means all declare -a versions=() declare opt_build_target='all' @@ -185,8 +181,6 @@ declare -a opt_with_modules=() declare BUILD_SCRIPT='' -declare opt_ol_name_or_dir='' - parse_args() { while (( $# > 0 )); do case $1 in @@ -429,8 +423,8 @@ build_modules_legacy() { fi declare ol_name='base' declare ol_type='n' - declare ol_mod_root="${pm_root}" - declare ol_inst_root="${pm_root}" + declare ol_mod_root="${PMODULES_HOME%%/Tools*}" + declare ol_inst_root="${PMODULES_HOME%%/Tools*}" local -i i=0 local -i num_variants=${#variants[@]} for ((i = 0; i < num_variants; i++)); do @@ -646,7 +640,7 @@ pm::read_config # or # ${overlay}/var/distfiles # ? -: ${PMODULES_DISTFILESDIR:=${pm_root}/var/distfiles} +: ${PMODULES_DISTFILESDIR:="${PMODULES_HOME%%/Tools*}/var/distfiles"} : ${PMODULES_TMPDIR:=/var/tmp/${USER}} declare -r BUILD_SCRIPT From cddc91865271ba85e06876a01504c4d756d9eccd Mon Sep 17 00:00:00 2001 From: Achim Gsell Date: Fri, 15 Jul 2022 18:23:11 +0200 Subject: [PATCH 15/15] modbuild: new key 'build_requires' in YAML variant files --- Pmodules/modbuild.in | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index c7597b4..6620c9b 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -532,6 +532,19 @@ build_modules_yaml(){ _result=("${_result[@]}") } + yaml_get_build_requirements(){ + local -n _result="$1" + local fname="$2" + local version="$3" + local idx="$4" + _result=( $(${yq} -Ne e ".\"${version}\"[${idx}].build_requires" \ + "${fname}" 2>/dev/null) ) + if (( $? != 0 )); then + _result=() + return + fi + } + local name="$1" local version="$2" shift 2 @@ -551,6 +564,7 @@ build_modules_yaml(){ (( n_variants == 0 )) && (( n_variants = 1 )) local -i i local -a deps=() + local -a build_requires=() local relstage local ol_name for (( i=0; i