From 8eb34d76f8a3394cec4a0b2082c56a3105a25d83 Mon Sep 17 00:00:00 2001
From: Achim Gsell <achim.gsell@psi.ch>
Date: Thu, 17 Sep 2015 11:12:44 +0200
Subject: [PATCH] Pmodules/modmanage.bash.in: - refactor function names with
 namespace - dependency resolution implemented for 'install' sub-command

---
 scripts/Bootstrap/Pmodules/modmanage.bash.in | 340 +++++++++++++------
 1 file changed, 232 insertions(+), 108 deletions(-)

diff --git a/scripts/Bootstrap/Pmodules/modmanage.bash.in b/scripts/Bootstrap/Pmodules/modmanage.bash.in
index 708b0ec..87ed047 100755
--- a/scripts/Bootstrap/Pmodules/modmanage.bash.in
+++ b/scripts/Bootstrap/Pmodules/modmanage.bash.in
@@ -17,12 +17,12 @@ source "${libdir}/libpmodules.bash"
 PATH="${bindir}:${PATH}"
 
 _exit () {
-	die 1 "Interrupted..."
+	std::die 1 "Interrupted..."
 }
 
 _err () {
-	info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
-	die 1 "Aborting ..."
+	std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
+	std::die 1 "Aborting ..."
 }
 
 trap '_exit' INT TERM
@@ -221,7 +221,7 @@ subcommand_init() {
 	local target_prefixes=()
 	local user=''
 	local opts=''
-	opts=$(get_options -o h -l src: -l user: -l help -l version: -- "$@")
+	opts=$(pmodules::get_options -o h -l src: -l user: -l help -l version: -- "$@")
 	if [[ $? != 0 ]]; then
 		subcommand_help_init
 		exit 1
@@ -255,30 +255,32 @@ subcommand_init() {
 		esac
 		shift
 	done
-	(( ${#target_prefixes[@]} != 0 )) || die 1 "Error: no target directory specified!"
+	(( ${#target_prefixes[@]} != 0 )) || \
+		std::die 1 "Error: no target directory specified!"
 
 	# if source directory is not passed as argument, derive it from script name
 	if [[ -z "${src}" ]]; then
 	    src=$(cd "${bindir}/../../../.." && pwd) 
 	fi
 	[[ -d "${src}" ]] || \
-	    die 1 "Error: ${src}: source directory does not exist!"
+		std::die 1 "Error: ${src}: source directory does not exist!"
 	[[ -r "${src}/config/profile.bash" ]] || \
-	    die 1 "Error: ${src}: shell profile does not exist or is not readable!"
+		std::die 1 "Error: ${src}: shell profile does not exist or is not readable!"
 	source "${src}/config/profile.bash"
 
 	local -i euid=$(id -u)
 	if (( euid == 0 )); then
 	        [[ -n "${user}" ]] || \
-		    die 1 "Error: --user parameter is required!"
+		    std::die 1 "Error: --user parameter is required!"
 		id -u "${user}" > /dev/null 2>&1 || \
-		    die 1 "Error: Unable to retrieve user id of user '${user}'"
+		    std::die 1 "Error: Unable to retrieve user id of user '${user}'"
 	else
 		[[ -z "${user}" ]] || \
-		    die 1 "Error: --user option is only allowed if running as root!"
+		    std::die 1 "Error: --user option is only allowed if running as root!"
 	fi
 
-	check_pmodules_env || die 1 "Giving up..."
+	pmodules::check_env || \
+		std::die 1 "Giving up..."
 
 	echo "
 Attempting to create a minimal module environment from the
@@ -294,22 +296,25 @@ environment at '${PMODULES_ROOT}'
 		echo
 		if [[ -d "${target_prefix}" ]]  && [[ ${force} == no ]]; then
 			echo "Warning: ${target_prefix} already exists."
-			get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
-				      die 1 "Abort ..."
+			std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
+			        std::die 1 "Abort ..."
 		fi
 		force='yes'
 		echo "Creating target directory '${target_prefix}'..."
-		$DRY mkdir -p "${target_prefix}" || die 1 "Error: make directory failed!"
+		$DRY mkdir -p "${target_prefix}" || \
+		        std::die 1 "Error: make directory failed!"
 		echo
 
 		echo "Syncing configuration ..."
 		sync_config "${PMODULES_ROOT}" \
-		            "${target_prefix}" || die 1 "Error: configuration synchronization failed!"
+		            "${target_prefix}" || \
+		        std::die 1 "Error: configuration synchronization failed!"
 
 		echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..."
 		sync_module "Tools/Pmodules/${PMODULES_VERSION}" \
 			    "${src_prefix}" \
-			    "${target_prefix}" || die 1 "Error: sync Pmodules failed!"
+			    "${target_prefix}" || \
+		        std::die 1 "Error: sync Pmodules failed!"
 		mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}"
 		echo
 
@@ -321,7 +326,8 @@ environment at '${PMODULES_ROOT}'
 
 		if [[ -n "${user}" ]]; then
 	                echo "Changing user of new module environment to '${user}'..."
-                        $DRY chown -R "${user}" "${target_prefix}" || die 1 "Error: changing owner failed!"
+                        $DRY chown -R "${user}" "${target_prefix}" || \
+			        std::die 1 "Error: changing owner failed!"
                         echo
 		fi
 		echo "New minimal module environment created at '${target_prefix}'."
@@ -337,70 +343,133 @@ environment at '${PMODULES_ROOT}'
 
 }
 
+declare -a Groups=()
+declare -A HierarchyDepths
+
+get_groups () {
+	local -r root="$1"
+	{
+		cd "${root}" 
+		# for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems
+		for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do
+			[[  -d ${f}/${PMODULES_MODULEFILES_DIR} ]] || continue
+			Groups+=( $f )
+		done
+	};
+}
+
+#
+# $1: root of modulefile hierarchy
+get_hierarchy_depth () {
+	local -r root="$1"
+	local -a modulefiles_dir
+	std::split_fname modulefiles_dir "${PMODULES_MODULEFILES_DIR}"
+	local -ir off=$(( ${#modulefiles_dir[@]} + 3 ))
+	{
+		cd "${root}"
+		local group
+		for group in "${Groups[@]}"; do
+			local  fname=$(find "${group}/${PMODULES_MODULEFILES_DIR}" \
+					    -depth \( -type f -o -type l \) -print -quit)
+			[[ -n ${fname} ]] || continue
+			#local -a tmp2=( ${fname//\// } )
+			local -a tmp
+			std::split_fname tmp "${fname}"
+			(( HierarchyDepths[$group]=${#tmp[@]}-off ))
+		done
+	};
+}
+
 subcommand_install() {
-	local opts=''
-	local -a with=()
-	local -a releases=()
-	local -a module_pattern=()
-	local -r src_prefix="${PMODULES_INSTALL_SOURCE}"
-	local -r target_prefix="${PMODULES_ROOT}"
-	local    rel_modulefile=''
-	local -A modules_to_install
-	local -A _modules_to_group
-	local -A _hierarchy_nodes
-	while read output; do
-		local _group=${output%/modulefiles/*}
-		local _abs_modulename="${output#*/modulefiles/}"
-		_modules_to_group[${_abs_modulename}]=${_group}
-		IFS='/'
-		local -a _items=(${_abs_modulename})
-		IFS=${_saved_IFS}
-		local -i _n=${#_items[*]}
-		if (( _n >= 4 )); then
-		    local _key=$( IFS='/'; echo "${_items[*]:$_n-4:2}" )
-		    _hierarchy_nodes[$_key]=${_group}
-		fi
-	done < <({ cd "${src_prefix}" && find */modulefiles -type l -o -type f  \! -name ".*"; } )
-	
+	local	    opts=''
+	local -a    with=()
+	local -a    releases=()
+	local -a    module_pattern=()
+	local	    src_prefix="${PMODULES_INSTALL_SOURCE}"
+	local -r    target_prefix="${PMODULES_ROOT}"
+	local	    modulefile=''
+	local -A    modules_to_install
+	local -A    dependencies_to_install
+	local -A    map_to_family
+	local -a    initial_modulepath=()
+
+	#
+	# Resolve dependencies to given module
+	#
+	# $1: modulefile relativ to src prefix. Something like:
+	#     MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14
+	#
+	# Notes:
+	#   The variables
+	#	initial_modulepath
+	#	modules_to_install
+	#	map_to_family
+	#   from the calling function are used!
+	#
 	resolve_dependencies_of_module () {
-	 	local -r __rel_modulefile=$1
-		# :FIXME: This must be initialized with all "root" groups
-		local -a __modulepath=("${src_prefix}/Tools/modulefiles" "${src_prefix}/Programming/modulefiles")
-	 	IFS='/'
-	 	local -a _items=(${__rel_modulefile})
-	 	IFS=${_saved_IFS}
-		local -i _n=${#_items[@]}
-		local -i _i
-		local _prefix="${src_prefix}/${_items[0]}"
-		for (( _i = _n-2; _i >= 2; _i-=2 )); do
-			_prefix+="/${_items[$_i]}/${_items[_i+1]}"
+	 	local -r modulefile=$1
+		local -a modulepath=( "${initial_modulepath[@]}" )
+
+		# compute filename with dependencies of given module
+		local -i i=0 n=0
+		std::split_fname items n "${modulefile}"
+		local fname_dependencies="${src_prefix}/${items[0]}"
+		for (( i = n-2; i >= 2; i-=2 )); do
+			fname_dependencies+="/${items[$i]}/${items[i+1]}"
 		done
-		local __dep
-		while read __dep; do
-			[[ -z ${__dep} ]] && continue
-			local __long_module_name=$(find "${__modulepath[@]}" -path "*/${__dep}" | head -n 1 )
-			__long_module_name=${__long_module_name/${src_prefix}\/}
-			modules_to_install[${__long_module_name/}]='.'
-			if [[ -n ${_hierarchy_nodes[${__dep}]} ]]; then
-				local __path="${src_prefix}/${_hierarchy_nodes[${__dep}]}/modulefiles/"
-				__path+="${__long_module_name/*\/modulefiles\/}"
-				__modulepath+=( "${__path}" )
+		fname_dependencies+='/.dependencies'
+		[[ -r ${fname_dependencies} ]] || return 0
+
+		# loop over all dependecies
+		local dep
+		while read dep; do
+			# skip empty lines
+			# :FIXME: skip comments?!
+			[[ -z ${dep} ]] && continue
+
+			# search for module with current modulepath and remember
+			local modulename=$(find "${modulepath[@]}" -path "*/${dep}" 2>/dev/null | head -n 1 )
+			[[ -n ${modulename} ]] || \
+			        std::die 3 "Oops: required module '${dep}' not found!"
+			modulename=${modulename/${src_prefix}\/}
+			dependencies_to_install[${modulename}]='.'
+			resolve_dependencies_of_module "${modulename}"
+			# append new node in hierarchy to modulepath
+			if [[ -n ${map_to_family[${dep}]} ]]; then
+				local path="${src_prefix}/${map_to_family[${dep}]}/${PMODULES_MODULEFILES_DIR}/"
+				path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}"
+				modulepath+=( "${path}" )
 			fi
-		done < "${_prefix}/.dependencies"
+		done < "${fname_dependencies}"
+	}
+
+	print_modules_to_install() {
+		local modulefile
+		std::info "The following modules will be installed/updated:"
+		for modulefile in "${!modules_to_install[@]}"; do
+			if [[ -e "${target_prefix}/${modulefile}" ]]; then
+				std::info "    Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+			else
+				std::info "    Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+			fi
+		done 2>&1 | sort
+		if (( ${#dependencies_to_install[@]} > 0 )); then
+			std::info "\nThe following dependencies will be installed/updated:"
+			for modulefile in "${!dependencies_to_install[@]}"; do
+				if [[ -e "${target_prefix}/${modulefile}" ]]; then
+					std::info "    Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+				else
+					std::info "    Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+				fi
+			done 2>&1 | sort
+		fi
+		std::info ""
+		std::get_YN_answer "Do you want to continue? [n] " || \
+			std::die 1 "Aborting..."
+		std::info ""
 	}
 	
-	resolve_dependencies () {
-		local _saved_IFS=${IFS}; 
-
-		# a relative modulefile is something like:
-		#     MPI/modulefiles/gcc/4.8.4/openmpi/1.8.4/hdf5/1.8.14
-		local _rel_modulefile
-		for _rel_modulefile in "${!modules_to_install[@]}"; do
-			resolve_dependencies_of_module "${_rel_modulefile}"
-		done
-	}
-
-	opts=$(get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
+	opts=$(pmodules::get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
 	if [[ $? != 0 ]]; then
 		subcommand_help_install
 		exit 1
@@ -444,9 +513,52 @@ subcommand_install() {
 		esac
 		shift
 	done
+
+	[[ -n ${src_prefix} ]] \
+		|| std::die 3 "Oops: no installation source given."
+	[[ -d ${src_prefix} ]] \
+		|| std::die 3 "Oops: '${src_prefix}' is not a valid installation source."
+
+	# scan available groups and their depth
+	get_groups "${src_prefix}"
+	get_hierarchy_depth "${src_prefix}"
+
+	# set initial modulepath
+	local group
+	for  group in "${!HierarchyDepths[@]}"; do
+		if (( ${HierarchyDepths[${group}]} == 0 )); then
+			initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" )
+		fi
+	done
+
+	#
+	# create a mapping from module name to their family.
+	# Examples:
+	#     gcc/5.2.0 -> Compiler
+	#     openmpi/1.8.4 -> MPI
+	local    _fname=''	
+	while read _fname; do
+		local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}"
+		local -a items
+		std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}"
+		local -i n=${#items[*]}
+		# We are only interested in families adding something to
+		# the modulepath. 
+		if (( n >= 4 )); then
+			local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" )
+			map_to_family[$_key]=${_family}
+		fi
+	done < <({ cd "${src_prefix}" && \
+				 find */"${PMODULES_MODULEFILES_DIR}" \
+				      \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null )
+
+	#
+	# search for to be installed modules and their dependencies
+	#
 	local -i n=0
-	while read rel_modulefile; do
-		modules_to_install["${rel_modulefile}"]+='.'
+	while read modulefile; do
+		resolve_dependencies_of_module "${modulefile}"
+		modules_to_install["${modulefile}"]+='.'
 		let n+=1
 	done < <(${PMODULES_HOME}/bin/modulecmd bash search \
 				 "${module_pattern[@]}" \
@@ -454,37 +566,45 @@ subcommand_install() {
 				 "${releases[@]/#/--release=}" \
 				 --no-header --print-modulefiles \
 				 --src="${src_prefix}"  2>&1 1>/dev/null)
-	(( n == 0 )) && die 0 "Nothing to install..."
-	resolve_dependencies
-	info "The following modules will be installed/updated:\n"
-	for rel_modulefile in "${!modules_to_install[@]}"; do
-		info "    ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
-	done
-	info ""
-	get_YN_answer "Do you want to continue? [n] " || die 1 "Aborting..."
-	info ""
-	for rel_modulefile in "${!modules_to_install[@]}"; do
-		if [[ -e "${target_prefix}/${rel_modulefile}" ]]; then
-			info "    Updating: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+	(( n == 0 )) && \
+		std::die 0 "No matching modules found ..."
+	print_modules_to_install
+
+	# install ...
+	for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do
+		if [[ -e "${target_prefix}/${modulefile}" ]]; then
+			std::info "    Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
 		else
-			info "    Installing: ${rel_modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
+			std::info "    Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
 		fi
-		sync_module "${rel_modulefile}" \
+		sync_module "${modulefile}" \
 			    "${src_prefix}" \
 			    "${target_prefix}"
 	done
-	info "\nDone!\n"
+	std::info "\nDone!\n"
+}
+
+#
+# delete specified module(s)
+#
+subcommand_delete() {
+	:
+}
+
+#
+# remove modules which have been removed in our source
+#
+subcommand_cleanup() {
+	:
 }
 
 subcommand_sync() {
-        [[ -z "${PMODULES_ROOT}" ]] && die 1 "Error: No current module environment is configured!"
-
 	local delete=false
 	local opts=''
 	local dst_prefix=''
 	local src_prefix=''
 
-	opts=$(get_options -o h -l dst: -l delete -l help -- "$@")
+	opts=$(pmodules::get_options -o h -l dst: -l delete -l help -- "$@")
 	if [[ $? != 0 ]]; then
 		subcommand_help_sync
 		exit 1
@@ -509,7 +629,7 @@ subcommand_sync() {
 				;;
 			* )
 				[[ -n "${src_prefix}" ]] && \
-				    die 1 "Error: Only one source is allowed!"
+				        std::die 1 "Error: Only one source is allowed!"
 				src_prefix="$1"
 				;;
 		esac
@@ -521,29 +641,29 @@ subcommand_sync() {
 		dst_prefix="${PMODULES_ROOT}"
 	fi
 	(
-		PMODULES_ROOT="${dst_prefix}" check_pmodules_env || \
-		    die 1 "Error: invalid destination modules environment!"
-	) || die 1 "Giving up..."
+		PMODULES_ROOT="${dst_prefix}" pmodules::check_env || \
+		        std::die 1 "Error: invalid destination modules environment!"
+	) || std::die 1 "Giving up..."
 	
 	: ${src_prefix:=${PMODULES_INSTALL_SOURCE}} 
 	if [[ -z "${src_prefix}" ]]; then
-            die 1 "Error: no source module environment was specified!"
+            std::die 1 "Error: no source module environment was specified!"
         fi
 	(
-		PMODULES_ROOT="${src_prefix}" check_pmodules_env || \
-		    die 1 "Error: invalid source modules environment!"
-	) || die 1 "Giving up..."
+		PMODULES_ROOT="${src_prefix}" pmodules::check_env || \
+		    std::die 1 "Error: invalid source modules environment!"
+	) || std::die 1 "Giving up..."
 	[[ "$( cd "$src_prefix"; pwd -P )" == "$( cd "$dst_prefix"; pwd -P )" ]] && \
-	    die 1 "Error: source and destination are equal!"
+	    std::die 1 "Error: source and destination are equal!"
 	local modbin=${PMODULES_HOME#"${PMODULES_ROOT}/"}/bin/modulecmd.tcl
 	local file_type_src=$( file -b "${src_prefix}/${modbin}" 2>&1 || echo err1 )
 	local file_type_dst=$( file -b "${dst_prefix}/${modbin}" 2>&1 || echo err2 )
 	[[ "${file_type_src}" == "${file_type_dst}" ]] || \
-	    die 1 "Error: The file signatures in the source and destination installation do not match!"
+	    std::die 1 "Error: The file signatures in the source and destination installation do not match!"
 	unset -v file_type_src file_type_dst
 	local dialog_script="${PMODULES_HOME}/bin/dialog.bash"
 	[[ -r "$dialog_script" ]] || \
-	    die 1 "Error: Unable to find dialog script of installation $dialog_script";
+		std::die 1 "Error: Unable to find dialog script of installation $dialog_script";
 
 	DIALOG_LIB=1                # use dialog script as a library
 	source "$dialog_script"     # dialog functions
@@ -595,11 +715,13 @@ subcommand_sync() {
 	local -a modules_copy=( $(set_difference "${selected_modules[*]}" "${destination_modules[*]}") )
 	if [[ -n $modules_copy ]]; then
 		echo "Syncing configuration ..."
-		sync_config "$src_prefix" "$dst_prefix" || die 1 "Error: syncing the configuration failed"
+		sync_config "$src_prefix" "$dst_prefix" || \
+		        std::die 1 "Error: syncing the configuration failed"
         fi
         for m in "${modules_copy[@]}"; do
                 echo "Copying module $m ..."
-		sync_module "$m" "$src_prefix" "$dst_prefix" || die 1 "Error: syncing of module $m failed!"
+		sync_module "$m" "$src_prefix" "$dst_prefix" || \
+		        std::die 1 "Error: syncing of module $m failed!"
         done
 	unset modules_copy
 }
@@ -642,6 +764,8 @@ if [[ -z ${subcommand} ]]; then
         usage
 	exit 1
 fi
+[[ -z "${PMODULES_ROOT}" ]] && \
+        std::die 1 "Error: No current module environment is configured!"
 $subcommand "${sargs[@]}"
 
 # Local Variables: