Merge branch '109-add-cuda-as-hierarchical-group-to-build-system' into 'master'

Resolve "add CUDA as hierarchical group to build system"

Closes #109

See merge request Pmodules/src!77

Pmodules/libpbuild.bash

@@ -183,9 +183,6 @@ pbuild.verbose() {
         verbose="$1"
 }
 
-# module name including path in hierarchy and version
-# (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0)
-declare -x  fully_qualified_module_name=''
 
 # group this module is in (ex: 'Programming')
 declare -x  GROUP=''
@@ -245,76 +242,6 @@ pbuild::supported_os() {
 pbuild::supported_compilers() {
 	SUPPORTED_COMPILERS+=( "$@" )
 }
-#......................................................................
-#
-# compute full module name and installation prefix
-#
-# The following variables are expected to be set:
-#	GROUP	    module group
-#	P		    module name
-#	V		    module version
-#       variables defining the hierarchical environment like
-#	COMPILER and COMPILER_VERSION
-#
-# The following variables are set in this function
-#	fully_qualified_module_name
-#	PREFIX
-#
-set_full_module_name_and_prefix() {
-	join_by() {
-		local IFS="$1"
-		shift
-		echo "$*"
-	}
-
-	[[ -n ${GROUP} ]] || std::die 1 \
-                                      "${module_name}/${module_version}:" \
-                                      "group not set."
-
-	# build module name
-	# :FIXME: this should be read from a configuration file
-	local name=()
-	case ${GROUP} in
-	Compiler )
-		name+=( "${COMPILER}/${COMPILER_VERSION}" )
-	        name+=( "${module_name}/${module_version}" )
-		;;
-	MPI )
-		name+=( "${COMPILER}/${COMPILER_VERSION}" )
-		name+=( "${MPI}/${MPI_VERSION}" )
-		name+=( "${module_name}/${module_version}" )
-		;;
-	HDF5 )
-		name+=( "${COMPILER}/${COMPILER_VERSION}" )
-		name+=( "${MPI}/${MPI_VERSION}" )
-		name+=( "${HDF5}/${HDF5_VERSION}" )
-		name+=( "${module_name}/${module_version}" )
-		;;
-	OPAL )
-		name+=( "${COMPILER}/${COMPILER_VERSION}" )
-		name+=( "${MPI}/${MPI_VERSION}" )
-		name+=( "${OPAL}/${OPAL_VERSION}" )
-		name+=( "${module_name}/${module_version}" )
-		;;
-	HDF5_serial )
-		name+=( "${COMPILER}/${COMPILER_VERSION}" )
-		name+=( "hdf5_serial/${HDF5_SERIAL_VERSION}" )
-		name+=( "${module_name}/${module_version}" )
-		;;
-	* )
-		name+=("${module_name}/${module_version}" )
-		;;
-	esac
-
-	# set full module name
-	fully_qualified_module_name=$( join_by '/' "${name[@]}" )
-	# set PREFIX of module
-	PREFIX="${PMODULES_ROOT}/${GROUP}"
-        local -i i=0
-	for ((i=${#name[@]}-1; i >= 0; i--)); do
-		PREFIX+="/${name[i]}"
-	done
-}
 
 ##############################################################################
 #
@@ -330,7 +257,6 @@ pbuild::add_to_group() {
                          "${FUNCNAME}: missing group argument."
 	fi
 	GROUP="$1"
-	set_full_module_name_and_prefix
 }
 
 ##############################################################################
@@ -503,13 +429,13 @@ pbuild::prep() {
 	unpack() {
 		local -r file="$1"
 		local -r dir="${2:-${SRC_DIR}}"
-		(
-			if [[ -n "${dir}" ]]; then
-				mkdir -p "${dir}"
-				cd "${dir}"
-			fi
-			tar -xv --strip-components 1 -f "${file}"
-		)
+		tar --directory="${dir}" -xv --strip-components 1 -f "${file}" || {
+			rm -f "${file}"
+			std::die 4 \
+				 "%s " \
+				 "${module_name}/${module_version}:" \
+				 "cannot unpack sources!"
+		}
 	}
 
 	patch_sources() {
@@ -521,7 +447,11 @@ pbuild::prep() {
 				"${module_name}/${module_version}:" \
 				"Appling patch '${PATCH_FILES[_i]}' ..."
 			local -i strip_val="${PATCH_STRIPS[_i]:-${PATCH_STRIP_DEFAULT}}"
-			patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}"
+			patch -p${strip_val} < "${BUILDBLOCK_DIR}/${PATCH_FILES[_i]}" || \
+				std::die 4 \
+					 "%s " \
+					 "${module_name}/${module_version}:" \
+					 "error patching sources!"
 		done
 	}
 	if [[ -z "${SOURCE_URLS}" ]]; then
@@ -656,7 +586,10 @@ pbuild::pre_compile() {
 }
 
 pbuild::compile() {
-	make -j${JOBS}
+	make -j${JOBS} || \
+		std::die 3 \
+			 "%s " "${module_name}/${module_version}:" \
+			 "compilation failed!"
 }
 
 pbuild::post_compile() {
@@ -668,7 +601,10 @@ pbuild::pre_install() {
 }
 
 pbuild::install() {
-	make install
+	make install || \
+		std::die 3 \
+			 "%s " "${module_name}/${module_version}:" \
+			 "compilation failed!"
 }
 
 pbuild::install_shared_libs() {
@@ -722,6 +658,10 @@ pbuild::make_all() {
 
 	set -e
 	local -r logfile="${BUILDBLOCK_DIR}/pbuild.log"
+	# module name including path in hierarchy and version
+	# (ex: 'gcc/6.1.0/openmpi/1.10.2' for openmpi compiled with gcc 6.1.0)
+	local    modulefile_dir=''
+	local    modulefile_name=''
 
 	#
 	# To be able to set environment variables in one of the 'pbuild::TARGET'
@@ -783,6 +723,74 @@ pbuild::make_all() {
 	}
 
 	#......................................................................
+	#
+	# compute full module name and installation prefix
+	#
+	# The following variables are expected to be set:
+	#	GROUP	    module group
+	#	P		    module name
+	#	V		    module version
+	#       variables defining the hierarchical environment like
+	#	COMPILER and COMPILER_VERSION
+	#
+	# The following variables are set in this function
+	#	modulefile_dir
+	#	modulefile_name
+	#	PREFIX
+	#
+	set_full_module_name_and_prefix() {
+		join_by() {
+			local IFS="$1"
+			shift
+			echo "$*"
+		}
+
+		[[ -n ${GROUP} ]] || std::die 1 \
+					      "${module_name}/${module_version}:" \
+					      "group not set."
+
+		# define defaults if not set in configuration file
+		: ${Compiler_HIERARCHY:='${COMPILER}/${COMPILER_VERSION}'}
+		: ${CUDA_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} cuda/${CUDA_VERSION}'}
+		: ${MPI_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION}'}
+		: ${HDF5_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} ${MPI}/${MPI_VERSION} hdf5/${HDF5_VERSION}'}
+		: ${HDF5_SERIAL_HIERARCHY:='${COMPILER}/${COMPILER_VERSION} hdf5_serial/${HDF5_SERIAL_VERSION}'}
+
+		# evaluate
+		local names=()
+		local vname="${GROUP}_HIERARCHY"
+		if [[ -n ${!vname} ]]; then
+			names=( $(eval echo ${!vname}) )
+		fi
+
+		modulefile_dir=$(join_by '/' \
+					 "${PMODULES_ROOT}/${GROUP}/${PMODULES_MODULEFILES_DIR}" \
+					 "${names[@]}" \
+					 "${module_name}")
+		modulefile_name="${modulefile_dir}/${module_version}"
+		PREFIX="${PMODULES_ROOT}/${GROUP}/${module_name}/${module_version}"
+		local -i i=0
+		for ((i=${#names[@]}-1; i >= 0; i--)); do
+			PREFIX+="/${names[i]}"
+		done
+	}
+	
+	#......................................................................
+	# Select the modulefile to install. Modulefiles can be versioned like
+	#     modulefile-10.2.0
+	#     modulefile-10.2
+	#     modulefile-10
+	#     modulefile
+	# the most specific modulefile will be selected. Example:
+	# For a version 10.2.1 the file moduelfile-10.2 would be selected.
+	#
+	# Arguments:
+	#     $1  upvar to return the filename
+	#
+	# Used gloabal variables:
+	#     VERSIONS
+	#     BUILDBLOCK_DIR
+	#
 	find_modulefile() {
 		local "$1"
 		local fname=''
@@ -798,11 +806,17 @@ pbuild::make_all() {
 	}
 
 	#......................................................................
-	# non-redefinable post-install
+	# non-redefinable post-install. Install:
+	# - documentation files as defined in the build-script
+	# - modulefile and file with release
+	# .
 	post_install() {
 		#..............................................................
 		# install the doc-files specified in the build-script
 		#
+		# Arguments:
+		#     none
+		#
 		install_doc() {
 			if [[ -z "${MODULE_DOCFILES}" ]]; then
 				for f in ${VERSIONS[@]/#/pbuild::install_docfiles_}; do
@@ -829,12 +843,14 @@ pbuild::make_all() {
 		}
 
 		#..............................................................
-		# install build-block files
+		# install build-block files 
 		# - modulefile
 		# - build-script
-		# - build dependencies
+		# - run-time and build dependencies
+		# in ${PREFIX}/share/${GROUP}/${module_name}
 		#
-		# Skip installation if modulefile does not exist.
+		# Arguments:
+		#     none
 		#
 		install_pmodules_files() {
 			local modulefile=''
@@ -924,9 +940,11 @@ pbuild::make_all() {
 	}
 
  	#......................................................................
-	# Install modulefile
+	# Install modulefile in ${PMODULES_ROOT}/${GROUP}/modulefiles/...
+	#
+	# Arguments
+	#     none
 	install_modulefile() {
-
 		local src=''
 		find_modulefile src
 		if (( $? != 0 )); then
@@ -936,34 +954,16 @@ pbuild::make_all() {
 				"skipping modulefile installation ..."
 			return
 		fi
-		# assemble name of modulefile
-		local dst="${PMODULES_ROOT}/"
-		dst+="${GROUP}/"
-		dst+="${PMODULES_MODULEFILES_DIR}/"
-		dst+="${fully_qualified_module_name}"
-
-		# directory where to install modulefile
- 		local -r dstdir=${dst%/*}
-
 		std::info \
 			"%s " \
 			"${module_name}/${module_version}:" \
-			"installing modulefile in '${dstdir}' ..."
-		mkdir -p "${dstdir}"
-		install -m 0444 "${src}" "${dst}"
+			"installing modulefile '${modulefile_name}' ..."
+		mkdir -p "${modulefile_dir}"
+		install -m 0444 "${src}" "${modulefile_name}"
 	}
 
 	install_release_file() {
-		local dst="${PMODULES_ROOT}/"
-		dst+="${GROUP}/"
-		dst+="${PMODULES_MODULEFILES_DIR}/"
-		dst+="${fully_qualified_module_name}"
-
-		# directory where to install release file
-		local -r dstdir=${dst%/*}
-		mkdir -p "${dstdir}"
-
- 		local -r release_file="${dst%/*}/.release-${module_version}"
+ 		local -r release_file="${modulefile_dir}/.release-${module_version}"
 
 		if [[ -r "${release_file}" ]]; then
 			local release
@@ -1121,24 +1121,14 @@ pbuild::make_all() {
 				"removing all files in '${PREFIX}' ..."
 			[[ "${dry_run}" == 'no' ]] && rm -rf ${PREFIX}
 		fi
-
-		# assemble name of modulefile
-		local dst="${PMODULES_ROOT}/"
-		dst+="${GROUP}/"
-		dst+="${PMODULES_MODULEFILES_DIR}/"
-		dst+="${fully_qualified_module_name}"
-
-		# directory where to install modulefile
- 		local -r dstdir=${dst%/*}
-
-		if [[ -e "${dst}" ]]; then
+		if [[ -e "${modulefile_name}" ]]; then
 			std::info \
 				"%s " \
 				"${module_name}/${module_version}:" \
-				"removing modulefile '${dst}' ..."
-			[[ "${dry_run}" == 'no' ]] && rm -v "${dst}"
+				"removing modulefile '${modulefile_name}' ..."
+			[[ "${dry_run}" == 'no' ]] && rm -v "${modulefile_name}"
 		fi
-		local release_file="${dstdir}/.release-${module_version}"
+		local release_file="${modulefile_dir}/.release-${module_version}"
 		if [[ -e "${release_file}" ]]; then
 			std::info \
 				"%s " \
@@ -1146,7 +1136,7 @@ pbuild::make_all() {
 				"removing release file '${release_file}' ..."
 			[[ "${dry_run}" == 'no' ]] && rm -v "${release_file}"
 		fi
-		rmdir -p "${dstdir}" 2>/dev/null || :
+		rmdir -p "${modulefile_dir}" 2>/dev/null || :
 	}
 
 	########################################################################
@@ -1160,8 +1150,9 @@ pbuild::make_all() {
 		check_supported_os
 		check_supported_compilers
 		set_full_module_name_and_prefix
-		if module_exists "${module_name}/${module_version}" \
-			&& [[ ${forece_rebuild} != 'yes' ]]; then
+		if [[ -e "${modulefile_name}" ]] \
+			   && [[ -d ${PREFIX} ]] \
+			   && [[ ${force_rebuild} != 'yes' ]]; then
 			if [[ "${module_release}" == 'removed' ]]; then
 				remove_module
 			else
@@ -1210,13 +1201,13 @@ pbuild.init_env() {
 		V_MINOR=''		# second number in version string (or empty)
 		V_PATCHLVL=''		# third number in version string (or empty)
 		V_RELEASE=''		# module release (or empty)
-		USE_FLAGS=''		# architectures (or empty)
+		: ${USE_FLAGS:=''}	# architectures (or empty)
 
 		local tmp=''
 
 		if [[ "$v" =~ "_" ]]; then
 			tmp="${v#*_}"
-			USE_FLAGS=":${tmp//_/:}:"
+			USE_FLAGS+=":${tmp//_/:}:"
 			v="${v%%_*}"
 		fi
 		V_PKG="${v%%-*}"	# version without the release number
@@ -1278,18 +1269,6 @@ pbuild.init_env() {
 	configure_with='undef'
 }
 
-#..............................................................
-#
-# Test whether a module with the given name already exists.
-#
-# Arguments:
-#   $1: module name/version
-#
-module_exists() {
-	[[ -n $("${MODULECMD}" bash avail -m "$1" \
-			       2>&1 1>/dev/null) ]]
-}
-
 pbuild.build_module() {
 	module_name="$1"
 	module_version="$2"
@@ -1482,11 +1461,6 @@ pbuild.build_module() {
 
 	pbuild.init_env "${module_name}" "${module_version}"
 	pbuild::make_all
-	std::info \
-		"%s " \
-		"${module_name}/${module_version}:" \
-		${with_modules:+with ${with_modules[@]}} \
-		"done!"
 	std::info "* * * * *\n"
 }
 

Pmodules/libstd.bash

@@ -205,72 +205,6 @@ std::upvar() {
     fi
 }
 
-
-# Assign variables one scope above the caller
-# Usage: local varname [varname ...] && 
-#        upvars [-v varname value] | [-aN varname [value ...]] ...
-# Available OPTIONS:
-#     -aN  Assign next N values to varname as array
-#     -v   Assign single value to varname
-# Return: 1 if error occurs
-# Example:
-#
-#    f() { local a b; g a b; declare -p a b; }
-#    g() {
-#        local c=( foo bar )
-#        local "$1" "$2" && upvars -v $1 A -a${#c[@]} $2 "${c[@]}"
-#    }
-#    f  # Ok: a=A, b=(foo bar)
-#
-std::upvars() {
-    if ! (( $# )); then
-        echo "${FUNCNAME[0]}: usage: ${FUNCNAME[0]} [-v varname"\
-            "value] | [-aN varname [value ...]] ..." 1>&2
-        return 2
-    fi
-    while (( $# )); do
-        case $1 in
-            -a*)
-                # Error checking
-                [[ ${1#-a} ]] || { echo "bash: ${FUNCNAME[0]}: \`$1': missing"\
-                    "number specifier" 1>&2; return 1; }
-                printf %d "${1#-a}" &> /dev/null || { echo "bash:"\
-                    "${FUNCNAME[0]}: \`$1': invalid number specifier" 1>&2
-                    return 1; }
-                # Assign array of -aN elements
-                [[ "$2" ]] && unset -v "$2" && eval $2=\(\"\${@:3:${1#-a}}\"\) && 
-                shift $((${1#-a} + 2)) || { echo "bash: ${FUNCNAME[0]}:"\
-                    "\`$1${2+ }$2': missing argument(s)" 1>&2; return 1; }
-                ;;
-            -v)
-                # Assign single value
-                [[ "$2" ]] && unset -v "$2" && eval $2=\"\$3\" &&
-                shift 3 || { echo "bash: ${FUNCNAME[0]}: $1: missing"\
-                "argument(s)" 1>&2; return 1; }
-                ;;
-            --help) echo "\
-Usage: local varname [varname ...] &&
-   ${FUNCNAME[0]} [-v varname value] | [-aN varname [value ...]] ...
-Available OPTIONS:
--aN VARNAME [value ...]   assign next N values to varname as array
--v VARNAME value          assign single value to varname
---help                    display this help and exit
---version                 output version information and exit"
-                return 0 ;;
-            --version) echo "\
-${FUNCNAME[0]}-0.9.dev
-Copyright (C) 2010 Freddy Vulto
-License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
-This is free software: you are free to change and redistribute it.
-There is NO WARRANTY, to the extent permitted by law."
-                return 0 ;;
-            *)
-                echo "bash: ${FUNCNAME[0]}: $1: invalid option" 1>&2
-                return 1 ;;
-        esac
-    done
-}
-
 std.get_os_release_linux() {
         local lsb_release=$(which lsb_release)
         local ID=''

Pmodules/modbuild.in

@@ -229,6 +229,13 @@ parse_args() {
 		--system=* )
 			opt_system="${1/*=}"
 			;;
+		--use-flags )
+			USE_FLAGS="y:$2:"
+			shift
+			;;
+		--use-flags=* )
+			USE_FLAGS=":${1/--use-flags=}:"
+			;;
 		--with )
 			opt_with_modules+=( "$2" )
 			shift
@@ -409,11 +416,6 @@ pbuild.update_modulefiles "${opt_update_modulefiles}"
 pbuild.system "${opt_system}"
 pbuild.verbose "${opt_verbose}"
 
-source libpbuild_dyn.bash    || \
-        std::die 3 "Oops: cannot source library -- '$_'"
-
-# source build configuration,
-# must be done before sourcing libpbuild!
 if [[ "${opt_bootstrap}" == 'yes' ]]; then
         test -d "${BUILDBLOCK_DIR}/../../config" && PATH+=":$_"
 else

Pmodules/modulecmd.bash.in

@@ -21,6 +21,7 @@ sort=$(PATH=/bin:/usr/bin /usr/bin/which sort)
 declare -r sort
 awk=$(PATH=/bin:/usr/bin /usr/bin/which awk)
 declare -r awk
+rm=$(PATH=/bin:/usr/bin /usr/bin/which rm)
 if [[ $(uname -s) == 'Darwin' ]]; then
 	declare -r getopt="${libexecdir}/getopt"
 	declare -r find="${libexecdir}/find"
@@ -608,7 +609,15 @@ subcommand_unload() {
                          "${CMD}" "${subcommand}" \
                          "missing argument"
 	fi
-
+	
+	# The module() function uses PMODULES_HOME to call modulecmd.
+	# If a Pmodules module is unloaded this evnvironment variable
+	# will be unset. In consequence the module() function would
+	# fail. Instead of comparing the name of the module to unload
+	# with 'Pmodules', we save the value and set it at the end of
+	# the loop again, if it has been unset.
+	local saved_home="${PMODULES_HOME}"
+	
         local arg
 	for arg in "${args[@]}"; do
 	        local output=$("${modulecmd}" "${Shell}" 'unload' "${arg}")
@@ -622,6 +631,10 @@ subcommand_unload() {
                 	;;
 		esac
 	done
+	if [[ -z ${PMODULES_HOME} ]]; then
+		PMODULES_HOME=${saved_home}
+		export_env 'PMODULES_HOME'
+	fi
 }
 
 ##############################################################################
@@ -728,17 +741,17 @@ subcommand_show() {
 #	modulename1 release1 modulename2 release2 ...
 #
 get_available_modules() {
-	local -r module="$1"
-	local -r use_releases="${2:-${UsedReleases}}"
-	shift 2
-	local -a dirs=( "$@" )
+	local var="$1"
+	local -r module="$2"
+	local -r use_releases="${3:-${UsedReleases}}"
+	shift 3 # in the for loop below we use $@ to loop over the directories
 
 	local -a mods=()
 	local    release
 	local dir=''
 	
-	for dir in "${dirs[@]}"; do
-		test -d "${dir}" || return 0
+	for dir in "$@"; do
+		test -d "${dir}" || continue
 		{
 			cd "${dir}"
 			while read mod; do
@@ -750,7 +763,7 @@ get_available_modules() {
 			done < <(${find} -L * \( -type f -o -type l \) -not -name ".*" -ipath "${module}*")
 		}
 	done
-	echo "${mods[@]}"
+	std::upvar ${var} "${mods[@]}"
 }
 
 ##############################################################################
@@ -932,10 +945,11 @@ subcommand_avail() {
         local string
 	for string in "${pattern[@]}"; do
 		for dir in "${modulepath[@]}"; do
-			mods=( $( get_available_modules \
-                                          "${string}" \
-                                          "${opt_use_releases}" \
-					  "${dir}" ) )
+			get_available_modules \
+				mods \
+				"${string}" \
+                                "${opt_use_releases}" \
+				"${dir}"
 			[[ ${#mods[@]} == 0 ]] && continue
 			${output_function}
 		done
@@ -1397,10 +1411,15 @@ subcommand_purge() {
                          "${CMD}" "${subcommand}" \
                          "no arguments allowd"
         fi
+	# we cannot unset PMODULES_HOME, otherwise the module function
+	# would fail.
+	local saved_home="${PMODULES_HOME}"
 	"${modulecmd}" "${Shell}" "${subcommand}"
         reset_modulepath
         reset_used_groups
-	export_env MODULEPATH
+	PMODULES_HOME="${saved_home}"
+	
+	export_env MODULEPATH PMODULES_HOME
 }
 
 ##############################################################################
@@ -1505,6 +1524,12 @@ USAGE:
 		for modules whose name match the argument.
 
 SWITCHES:
+        -a|--all-releases
+		Search within all releases.
+
+	--all-deps
+                Show all dependecies
+
         --no-header
 		Suppress output of a header.
 
@@ -1513,8 +1538,8 @@ SWITCHES:
 		switch multiple times.  Without this switch, the used releases
 		will be searched.
 
-        -a|--all-releases
-		Search within all releases.
+        --verbose
+                vebose output
 
         --with=STRING
 		Search for modules compiled with modules matching string. The
@@ -1524,9 +1549,6 @@ SWITCHES:
 
 		lists all modules in the hierarchy compiled with gcc 4.8.3.
 
-        --verbose
-                vebose output
-
         --wrap
                 wrap output
 '
@@ -1714,11 +1736,12 @@ subcommand_search() {
 			# with respect to the requested releases
 			# tmpfile: module/version release group group-
                         # dependencies...
-			local mods=( $( get_available_modules \
-						"${module}" \
-						"${opt_use_releases}" \
-				     		"${modulepath[@]}" \
-				     ) )
+			local mods
+			get_available_modules \
+				mods \
+				"${module}" \
+				"${opt_use_releases}" \
+				"${modulepath[@]}" \
 
 			for (( i=0; i<${#mods[@]}; i+=3 )); do
 				local name=${mods[i]}
@@ -1755,7 +1778,7 @@ subcommand_search() {
 			done
 		done
 		print_result "${tmpfile}"
-		#rm -f "${tmpfile}"
+		${rm} -f "${tmpfile}"
 	}
 
 	while (( $# > 0 )); do

config/versions.conf

@@ -4,5 +4,5 @@ findutils	4.7.0
 getopt		1.1.6
 gettext		0.21
 modules		3.2.10.1
-Pmodules	1.0.0rc8
+Pmodules	1.0.0rc9
 Tcl		8.6.10