diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index 7b48df0..3404b63 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -183,9 +183,6 @@ pbuild.verbose() { verbose="$1" } -# module name including path in hierarchy and version -# (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) -declare -x fully_qualified_module_name='' # group this module is in (ex: 'Programming') declare -x GROUP='' @@ -245,76 +242,6 @@ pbuild::supported_os() { pbuild::supported_compilers() { SUPPORTED_COMPILERS+=( "$@" ) } -#...................................................................... -# -# compute full module name and installation prefix -# -# The following variables are expected to be set: -# GROUP module group -# P module name -# V module version -# variables defining the hierarchical environment like -# COMPILER and COMPILER_VERSION -# -# The following variables are set in this function -# fully_qualified_module_name -# PREFIX -# -set_full_module_name_and_prefix() { - join_by() { - local IFS="$1" - shift - echo "$*" - } - - [[ -n ${GROUP} ]] || std::die 1 \ - "${module_name}/${module_version}:" \ - "group not set." - - # build module name - # :FIXME: this should be read from a configuration file - local name=() - case ${GROUP} in - Compiler ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - MPI ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - HDF5 ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${HDF5}/${HDF5_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - OPAL ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "${MPI}/${MPI_VERSION}" ) - name+=( "${OPAL}/${OPAL_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - HDF5_serial ) - name+=( "${COMPILER}/${COMPILER_VERSION}" ) - name+=( "hdf5_serial/${HDF5_SERIAL_VERSION}" ) - name+=( "${module_name}/${module_version}" ) - ;; - * ) - name+=("${module_name}/${module_version}" ) - ;; - esac - - # set full module name - fully_qualified_module_name=$( join_by '/' "${name[@]}" ) - # set PREFIX of module - PREFIX="${PMODULES_ROOT}/${GROUP}" - local -i i=0 - for ((i=${#name[@]}-1; i >= 0; i--)); do - PREFIX+="/${name[i]}" - done -} ############################################################################## # @@ -330,7 +257,6 @@ pbuild::add_to_group() { "${FUNCNAME}: missing group argument." fi GROUP="$1" - set_full_module_name_and_prefix } ############################################################################## @@ -503,13 +429,13 @@ pbuild::prep() { unpack() { local -r file="$1" local -r dir="${2:-${SRC_DIR}}" - ( - if [[ -n "${dir}" ]]; then - mkdir -p "${dir}" - cd "${dir}" - fi - tar -xv --strip-components 1 -f "${file}" - ) + tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || { + rm -f "${file}" + std::die 4 \ + "%s " \ + "${module_name}/${module_version}:" \ + "cannot unpack sources!" + } } patch_sources() { @@ -521,7 +447,11 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "Appling patch '${PATCH_FILES[_i]}' ..." local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}" - patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" + patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \ + std::die 4 \ + "%s " \ + "${module_name}/${module_version}:" \ + "error patching sources!" done } if [[ -z "${SOURCE_URLS}" ]]; then @@ -656,7 +586,10 @@ pbuild::pre_compile() { } pbuild::compile() { - make -j${JOBS} + make -j${JOBS} || \ + std::die 3 \ + "%s " "${module_name}/${module_version}:" \ + "compilation failed!" } pbuild::post_compile() { @@ -668,7 +601,10 @@ pbuild::pre_install() { } pbuild::install() { - make install + make install || \ + std::die 3 \ + "%s " "${module_name}/${module_version}:" \ + "compilation failed!" } pbuild::install_shared_libs() { @@ -722,6 +658,10 @@ pbuild::make_all() { set -e local -r logfile="${BUILDBLOCK_DIR}/pbuild.log" + # module name including path in hierarchy and version + # (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0) + local modulefile_dir='' + local modulefile_name='' # # To be able to set environment variables in one of the 'pbuild::TARGET' @@ -783,6 +723,74 @@ pbuild::make_all() { } #...................................................................... + # + # compute full module name and installation prefix + # + # The following variables are expected to be set: + # GROUP module group + # P module name + # V module version + # variables defining the hierarchical environment like + # COMPILER and COMPILER_VERSION + # + # The following variables are set in this function + # modulefile_dir + # modulefile_name + # PREFIX + # + set_full_module_name_and_prefix() { + join_by() { + local IFS="$1" + shift + echo "$*" + } + + [[ -n ${GROUP} ]] || std::die 1 \ + "${module_name}/${module_version}:" \ + "group not set." + + # define defaults if not set in configuration file + : ${Compiler_HIERARCHY:='${COMPILER}/${COMPILER_VERSION}'} + : ${CUDA_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} cuda/${CUDA_VERSION}'} + : ${MPI_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION}'} + : ${HDF5_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION} hdf5/${HDF5_VERSION}'} + : ${HDF5_SERIAL_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} hdf5_serial/${HDF5_SERIAL_VERSION}'} + + # evaluate + local names=() + local vname="${GROUP}_HIERARCHY" + if [[ -n ${!vname} ]]; then + names=( $(eval echo ${!vname}) ) + fi + + modulefile_dir=$(join_by '/' \ + "${PMODULES_ROOT}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \ + "${names[@]}" \ + "${module_name}") + modulefile_name="${modulefile_dir}/${module_version}" + PREFIX="${PMODULES_ROOT}/${GROUP}/${module_name}/${module_version}" + local -i i=0 + for ((i=${#names[@]}-1; i >= 0; i--)); do + PREFIX+="/${names[i]}" + done + } + + #...................................................................... + # Select the modulefile to install. Modulefiles can be versioned like + # modulefile-10.2.0 + # modulefile-10.2 + # modulefile-10 + # modulefile + # the most specific modulefile will be selected. Example: + # For a version 10.2.1 the file moduelfile-10.2 would be selected. + # + # Arguments: + # $1 upvar to return the filename + # + # Used gloabal variables: + # VERSIONS + # BUILDBLOCK_DIR + # find_modulefile() { local "$1" local fname='' @@ -798,11 +806,17 @@ pbuild::make_all() { } #...................................................................... - # non-redefinable post-install + # non-redefinable post-install. Install: + # - documentation files as defined in the build-script + # - modulefile and file with release + # . post_install() { #.............................................................. # install the doc-files specified in the build-script # + # Arguments: + # none + # install_doc() { if [[ -z "${MODULE_DOCFILES}" ]]; then for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do @@ -829,12 +843,14 @@ pbuild::make_all() { } #.............................................................. - # install build-block files + # install build-block files # - modulefile # - build-script - # - build dependencies + # - run-time and build dependencies + # in ${PREFIX}/share/${GROUP}/${module_name} # - # Skip installation if modulefile does not exist. + # Arguments: + # none # install_pmodules_files() { local modulefile='' @@ -924,9 +940,11 @@ pbuild::make_all() { } #...................................................................... - # Install modulefile + # Install modulefile in ${PMODULES_ROOT}/${GROUP}/modulefiles/... + # + # Arguments + # none install_modulefile() { - local src='' find_modulefile src if (( $? != 0 )); then @@ -936,34 +954,16 @@ pbuild::make_all() { "skipping modulefile installation ..." return fi - # assemble name of modulefile - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install modulefile - local -r dstdir=${dst%/*} - std::info \ "%s " \ "${module_name}/${module_version}:" \ - "installing modulefile in '${dstdir}' ..." - mkdir -p "${dstdir}" - install -m 0444 "${src}" "${dst}" + "installing modulefile '${modulefile_name}' ..." + mkdir -p "${modulefile_dir}" + install -m 0444 "${src}" "${modulefile_name}" } install_release_file() { - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install release file - local -r dstdir=${dst%/*} - mkdir -p "${dstdir}" - - local -r release_file="${dst%/*}/.release-${module_version}" + local -r release_file="${modulefile_dir}/.release-${module_version}" if [[ -r "${release_file}" ]]; then local release @@ -1121,24 +1121,14 @@ pbuild::make_all() { "removing all files in '${PREFIX}' ..." [[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX} fi - - # assemble name of modulefile - local dst="${PMODULES_ROOT}/" - dst+="${GROUP}/" - dst+="${PMODULES_MODULEFILES_DIR}/" - dst+="${fully_qualified_module_name}" - - # directory where to install modulefile - local -r dstdir=${dst%/*} - - if [[ -e "${dst}" ]]; then + if [[ -e "${modulefile_name}" ]]; then std::info \ "%s " \ "${module_name}/${module_version}:" \ - "removing modulefile '${dst}' ..." - [[ "${dry_run}" == 'no' ]] && rm -v "${dst}" + "removing modulefile '${modulefile_name}' ..." + [[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}" fi - local release_file="${dstdir}/.release-${module_version}" + local release_file="${modulefile_dir}/.release-${module_version}" if [[ -e "${release_file}" ]]; then std::info \ "%s " \ @@ -1146,7 +1136,7 @@ pbuild::make_all() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && rm -v "${release_file}" fi - rmdir -p "${dstdir}" 2>/dev/null || : + rmdir -p "${modulefile_dir}" 2>/dev/null || : } ######################################################################## @@ -1160,8 +1150,9 @@ pbuild::make_all() { check_supported_os check_supported_compilers set_full_module_name_and_prefix - if module_exists "${module_name}/${module_version}" \ - && [[ ${forece_rebuild} != 'yes' ]]; then + if [[ -e "${modulefile_name}" ]] \ + && [[ -d ${PREFIX} ]] \ + && [[ ${force_rebuild} != 'yes' ]]; then if [[ "${module_release}" == 'removed' ]]; then remove_module else @@ -1210,13 +1201,13 @@ pbuild.init_env() { V_MINOR='' # second number in version string (or empty) V_PATCHLVL='' # third number in version string (or empty) V_RELEASE='' # module release (or empty) - USE_FLAGS='' # architectures (or empty) + : ${USE_FLAGS:=''} # architectures (or empty) local tmp='' if [[ "$v" =~ "_" ]]; then tmp="${v#*_}" - USE_FLAGS=":${tmp//_/:}:" + USE_FLAGS+=":${tmp//_/:}:" v="${v%%_*}" fi V_PKG="${v%%-*}" # version without the release number @@ -1278,18 +1269,6 @@ pbuild.init_env() { configure_with='undef' } -#.............................................................. -# -# Test whether a module with the given name already exists. -# -# Arguments: -# $1: module name/version -# -module_exists() { - [[ -n $("${MODULECMD}" bash avail -m "$1" \ - 2>&1 1>/dev/null) ]] -} - pbuild.build_module() { module_name="$1" module_version="$2" @@ -1482,11 +1461,6 @@ pbuild.build_module() { pbuild.init_env "${module_name}" "${module_version}" pbuild::make_all - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - ${with_modules:+with ${with_modules[@]}} \ - "done!" std::info "* * * * *\n" } diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index be491b6..eb8525d 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -205,72 +205,6 @@ std::upvar() { fi } - -# Assign variables one scope above the caller -# Usage: local varname [varname ...] && -# upvars [-v varname value] | [-aN varname [value ...]] ... -# Available OPTIONS: -# -aN Assign next N values to varname as array -# -v Assign single value to varname -# Return: 1 if error occurs -# Example: -# -# f() { local a b; g a b; declare -p a b; } -# g() { -# local c=( foo bar ) -# local "$1" "$2" && upvars -v $1 A -a${#c[@]} $2 "${c[@]}" -# } -# f # Ok: a=A, b=(foo bar) -# -std::upvars() { - if ! (( $# )); then - echo "${FUNCNAME[0]}: usage: ${FUNCNAME[0]} [-v varname"\ - "value] | [-aN varname [value ...]] ..." 1>&2 - return 2 - fi - while (( $# )); do - case $1 in - -a*) - # Error checking - [[ ${1#-a} ]] || { echo "bash: ${FUNCNAME[0]}: \`$1': missing"\ - "number specifier" 1>&2; return 1; } - printf %d "${1#-a}" &> /dev/null || { echo "bash:"\ - "${FUNCNAME[0]}: \`$1': invalid number specifier" 1>&2 - return 1; } - # Assign array of -aN elements - [[ "$2" ]] && unset -v "$2" && eval $2=\(\"\${@:3:${1#-a}}\"\) && - shift $((${1#-a} + 2)) || { echo "bash: ${FUNCNAME[0]}:"\ - "\`$1${2+ }$2': missing argument(s)" 1>&2; return 1; } - ;; - -v) - # Assign single value - [[ "$2" ]] && unset -v "$2" && eval $2=\"\$3\" && - shift 3 || { echo "bash: ${FUNCNAME[0]}: $1: missing"\ - "argument(s)" 1>&2; return 1; } - ;; - --help) echo "\ -Usage: local varname [varname ...] && - ${FUNCNAME[0]} [-v varname value] | [-aN varname [value ...]] ... -Available OPTIONS: --aN VARNAME [value ...] assign next N values to varname as array --v VARNAME value assign single value to varname ---help display this help and exit ---version output version information and exit" - return 0 ;; - --version) echo "\ -${FUNCNAME[0]}-0.9.dev -Copyright (C) 2010 Freddy Vulto -License GPLv3+: GNU GPL version 3 or later -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law." - return 0 ;; - *) - echo "bash: ${FUNCNAME[0]}: $1: invalid option" 1>&2 - return 1 ;; - esac - done -} - std.get_os_release_linux() { local lsb_release=$(which lsb_release) local ID='' diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index fbff037..6aa9543 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -229,6 +229,13 @@ parse_args() { --system=* ) opt_system="${1/*=}" ;; + --use-flags ) + USE_FLAGS="y:$2:" + shift + ;; + --use-flags=* ) + USE_FLAGS=":${1/--use-flags=}:" + ;; --with ) opt_with_modules+=( "$2" ) shift @@ -409,11 +416,6 @@ pbuild.update_modulefiles "${opt_update_modulefiles}" pbuild.system "${opt_system}" pbuild.verbose "${opt_verbose}" -source libpbuild_dyn.bash || \ - std::die 3 "Oops: cannot source library -- '$_'" - -# source build configuration, -# must be done before sourcing libpbuild! if [[ "${opt_bootstrap}" == 'yes' ]]; then test -d "${BUILDBLOCK_DIR}/../../config" && PATH+=":$_" else diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 563a587..8501996 100755 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -21,6 +21,7 @@ sort=$(PATH=/bin:/usr/bin /usr/bin/which sort) declare -r sort awk=$(PATH=/bin:/usr/bin /usr/bin/which awk) declare -r awk +rm=$(PATH=/bin:/usr/bin /usr/bin/which rm) if [[ $(uname -s) == 'Darwin' ]]; then declare -r getopt="${libexecdir}/getopt" declare -r find="${libexecdir}/find" @@ -608,7 +609,15 @@ subcommand_unload() { "${CMD}" "${subcommand}" \ "missing argument" fi - + + # The module() function uses PMODULES_HOME to call modulecmd. + # If a Pmodules module is unloaded this evnvironment variable + # will be unset. In consequence the module() function would + # fail. Instead of comparing the name of the module to unload + # with 'Pmodules', we save the value and set it at the end of + # the loop again, if it has been unset. + local saved_home="${PMODULES_HOME}" + local arg for arg in "${args[@]}"; do local output=$("${modulecmd}" "${Shell}" 'unload' "${arg}") @@ -622,6 +631,10 @@ subcommand_unload() { ;; esac done + if [[ -z ${PMODULES_HOME} ]]; then + PMODULES_HOME=${saved_home} + export_env 'PMODULES_HOME' + fi } ############################################################################## @@ -728,17 +741,17 @@ subcommand_show() { # modulename1 release1 modulename2 release2 ... # get_available_modules() { - local -r module="$1" - local -r use_releases="${2:-${UsedReleases}}" - shift 2 - local -a dirs=( "$@" ) + local var="$1" + local -r module="$2" + local -r use_releases="${3:-${UsedReleases}}" + shift 3 # in the for loop below we use $@ to loop over the directories local -a mods=() local release local dir='' - for dir in "${dirs[@]}"; do - test -d "${dir}" || return 0 + for dir in "$@"; do + test -d "${dir}" || continue { cd "${dir}" while read mod; do @@ -750,7 +763,7 @@ get_available_modules() { done < <(${find} -L * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*") } done - echo "${mods[@]}" + std::upvar ${var} "${mods[@]}" } ############################################################################## @@ -932,10 +945,11 @@ subcommand_avail() { local string for string in "${pattern[@]}"; do for dir in "${modulepath[@]}"; do - mods=( $( get_available_modules \ - "${string}" \ - "${opt_use_releases}" \ - "${dir}" ) ) + get_available_modules \ + mods \ + "${string}" \ + "${opt_use_releases}" \ + "${dir}" [[ ${#mods[@]} == 0 ]] && continue ${output_function} done @@ -1397,10 +1411,15 @@ subcommand_purge() { "${CMD}" "${subcommand}" \ "no arguments allowd" fi + # we cannot unset PMODULES_HOME, otherwise the module function + # would fail. + local saved_home="${PMODULES_HOME}" "${modulecmd}" "${Shell}" "${subcommand}" reset_modulepath reset_used_groups - export_env MODULEPATH + PMODULES_HOME="${saved_home}" + + export_env MODULEPATH PMODULES_HOME } ############################################################################## @@ -1505,6 +1524,12 @@ USAGE: for modules whose name match the argument. SWITCHES: + -a|--all-releases + Search within all releases. + + --all-deps + Show all dependecies + --no-header Suppress output of a header. @@ -1513,8 +1538,8 @@ SWITCHES: switch multiple times. Without this switch, the used releases will be searched. - -a|--all-releases - Search within all releases. + --verbose + vebose output --with=STRING Search for modules compiled with modules matching string. The @@ -1524,9 +1549,6 @@ SWITCHES: lists all modules in the hierarchy compiled with gcc 4.8.3. - --verbose - vebose output - --wrap wrap output ' @@ -1714,11 +1736,12 @@ subcommand_search() { # with respect to the requested releases # tmpfile: module/version release group group- # dependencies... - local mods=( $( get_available_modules \ - "${module}" \ - "${opt_use_releases}" \ - "${modulepath[@]}" \ - ) ) + local mods + get_available_modules \ + mods \ + "${module}" \ + "${opt_use_releases}" \ + "${modulepath[@]}" \ for (( i=0; i<${#mods[@]}; i+=3 )); do local name=${mods[i]} @@ -1755,7 +1778,7 @@ subcommand_search() { done done print_result "${tmpfile}" - #rm -f "${tmpfile}" + ${rm} -f "${tmpfile}" } while (( $# > 0 )); do diff --git a/config/versions.conf b/config/versions.conf index 2311805..442193e 100644 --- a/config/versions.conf +++ b/config/versions.conf @@ -4,5 +4,5 @@ findutils 4.7.0 getopt 1.1.6 gettext 0.21 modules 3.2.10.1 -Pmodules 1.0.0rc8 +Pmodules 1.0.0rc9 Tcl 8.6.10