Files
Pmodules/Pmodules/modmanage.bash.in
T

868 lines
22 KiB
Bash
Executable File

#!@PMODULES_HOME@/sbin/bash
# we have to unset CDPATH, otherwise 'cd' prints the directoy!
unset CDPATH
# used for some output only
declare -r CMD=$(basename "$0")
declare -r mydir=$(cd $(dirname "$0") && pwd)
declare -r prefix=$(dirname "${mydir}")
declare -r bindir="${prefix}/bin"
declare -r sbindir="${prefix}/sbin"
declare -r libdir="${prefix}/lib"
declare -r libexecdir="${prefix}/libexec"
source "${libdir}/libstd.bash"
source "${libdir}/libpmodules.bash"
PATH="${sbindir}:${bindir}:${PATH}"
_exit () {
std::die 1 "Interrupted..."
}
_err () {
std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}"
std::die 1 "Aborting ..."
}
trap '_exit' INT TERM
trap '_err' ERR
# make sure that everything is used from this version
declare PMODULES_VERSION='@PMODULES_VERSION@'
##############################################################################
#
# print version of program
#
# Arguments:
# none
#
print_version() {
echo "
Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@
Copyright GNU GPL v2
" 1>&2
}
##############################################################################
#
# print usage
#
# Arguments:
# none
#
usage() {
local -r prog=$(basename $0)
print_version
echo "
Usage: ${prog} [ switches ] [ subcommand ] [subcommand-args ]
Switches:
--dry-run do nothing
--force force overwrite
Available SubCommands and Args:
init [--src=<src>] [--user=<user>] <dst>
Initialize a new minimal Pmodule environment.
install <module> [--with=<dep>...]
Install matching modules
sync [--delete] [--dst=<dst>] <src>
Synchronize modules.
"
}
declare force='no'
declare dry_run='no'
declare DRY=''
declare subcommand=''
declare sargs=()
##############################################################################
#
# help for subcommand 'init'
#
# Arguments:
# none
#
subcommand_help_init() {
echo "
init [--src=<src>] [--user=<user>] [--version=<version>] <dst>
Initialize a new minimal Pmodule environment in directory
<dst>. The <user> parameter must only be present if
${prog} is executed as root.
" 1>&2
}
##############################################################################
#
# help for subcommand 'install'
#
# Arguments:
# none
#
subcommand_help_install() {
echo "
install <module>... [--with=<dep>...] [--release=<release>...] [--src=<src>]
Install matching modules
" 1>&2
}
##############################################################################
#
# help for subcommand 'search'
#
# Arguments:
# none
#
subcommand_help_search() {
echo "
USAGE:
module search [switches] string...
Search available modules. If an argument is given, search
for modules whose name match the argument.
SWITCHES:
--no-header
Suppress output of a header.
--with=STRING
Search for modules compiled with modules matching string. The
command
module search --with=gcc/4.8.3
lists all modules in the hierarchy compiled with gcc 4.8.3.
" 1>&2
}
##############################################################################
#
# help for subcommand 'sync'
#
# Arguments:
# none
#
subcommand_help_sync() {
echo "
sync [--delete] [--dst=<dst>] <src>
Synchronize environment modules and configuration files
from Pmodule environment <src> to Pmodule environment <dst>
(default: currently active Pmodule environment).
Not yet implemented:
If --delete is given, unmarked modules present in <dst>
will be deleted.
" 1>&2
}
##############################################################################
#
# print usage or help text for given sub-command
#
# Arguments:
# none or sub-command
#
subcommand_help() {
if [[ $# == 0 ]]; then
usage
elif typeset -F subcommand_help_$1 > /dev/null 2>&1 ; then
# help for sub-command
subcommand_help_$1
else
usage
fi
}
##############################################################################
#
# Derive the relative module installation path
# from the relative modulefile path
#
# Arguments:
# $1: relative module file path
#
get_module_prefix() {
local -a comp=( ${1//\// } ) # split rel.path into components
local path="${comp[0]}" # result path
local -i i
for ((i=1; i<${#comp[@]}-1; i+=2)); do
path+="/${comp[$((-i-1))]}/${comp[$((-i))]}"
done
echo "${path}"
}
##############################################################################
#
# Derive the relative module release file path
# from the relative module file path
#
# Arguments:
# $1: relative module file path
#
get_releasefile_name() {
echo "$(dirname "$1")/.release-$(basename "$1")"
}
##############################################################################
#
# Sync a module from one Pmodules environment to another:
# - sync module installation
# - sync modulefile
# - sync release file
#
# Note:
# We do not take care of files in $PMODULES_ROOT/$PMODULES_TEMPLATES_DIR. If
# the modulefile is a sym-link it is expected that the target exists.
#
# Arguments:
# $1: relative modulefile path (something like: Tools/gnuplot/5.0.0)
# $2: source prefix of Pmodule environment
# $3: target prefix of Pmodule environment
#
sync_module() {
local -r rel_modulefile=$1
local -r src_prefix=$2
local -r target_prefix=$3
local -r rel_module_prefix=$( get_module_prefix "${rel_modulefile}" )
local -r rel_releasefile=$( get_releasefile_name "${rel_modulefile}" )
# install/update module
if [[ ! -d "${target_prefix}/${rel_module_prefix}" ]] || [[ "${force}" == 'yes' ]]; then
$DRY mkdir -p "${target_prefix}/${rel_module_prefix}" || return $?
$DRY rsync --links --perms --recursive --delete \
"${src_prefix}/${rel_module_prefix}/" \
"${target_prefix}/${rel_module_prefix}/" || exit $?
fi
local -r src_modulefile="${src_prefix}/${rel_modulefile}"
local -r src_releasefile="${src_prefix}/${rel_releasefile}"
local -r target_modulefile="${target_prefix}/${rel_modulefile}"
local -r target_releasefile="${target_prefix}/${rel_releasefile}"
# create target directory for module- and release-file
if [[ -e "${src_modulefile}" ]] || [[ -e "${src_releasefile}" ]]; then
local dir=$( dirname "${target_modulefile}" )
$DRY mkdir -p "${dir}" || return $?
fi
# copy modulefile template
local -a rel_modulefile_splitted
std::split_fname rel_modulefile_splitted "${rel_modulefile}"
local -r module_group="${rel_modulefile_splitted[0]}"
local -r module_name="${rel_modulefile_splitted[-2]}"
local -r template="${module_group}/${PMODULES_TEMPLATES_DIR}/${module_name}/"
local -r src_template="${src_prefix}/${template}"
local -r target_template="${target_prefix}/${template}"
if [[ -e "${src_template}" ]]; then
$DRY mkdir -p "${target_template}"
$DRY rsync --links --perms --recursive \
"${src_template}" "${target_template}" || exit $?
fi
# copy modulefile
if [[ -e "${src_modulefile}" ]]; then
$DRY rsync --links --perms --recursive \
"${src_modulefile}" "${target_modulefile}" || exit $?
fi
# copy release-file
if [[ -e "${src_releasefile}" ]]; then
$DRY rsync --links --perms --recursive \
"${src_releasefile}" "${target_releasefile}" || exit $?
fi
}
##############################################################################
#
# Sync the Pmodules configuration and templates
#
# Arguments:
# $1: source prefix of Pmodule environment
# $2: target prefix of Pmodule environment
#
sync_config() {
src="$1/${PMODULES_CONFIG_DIR}/"
dst="$2/${PMODULES_CONFIG_DIR}/"
$DRY rsync --recursive --links --perms --delete \
"${src}" "${dst}" 2>/dev/null || return $?
sed -i.bak "s/PMODULES_VERSION=\(.*\)/PMODULES_VERSION=${PMODULES_VERSION}/" "${dst}/environment.bash"
echo
}
##############################################################################
#
# Delete a module
#
# Arguments:
# $1: relative modulefile path
# $2: target prefix of Pmodule environment
#
delete_module() {
echo "Not implemented yet!"
}
##############################################################################
#
# initialize a new module environment
#
# Arguments:
# [--src <SRCDIR>]
# Module environment we are going to sync from. If not
# specified, the module environment this script is in
# will be used.
# [--user <USER>]
# If this scripts runs with root privileges, a user name
# ore ID must be specified.
# [--version <VERSION>]
# Set PMODULES_VERSION to <version>
# TARGET_DIR
# Initialize a new module environment in this directory-
#
subcommand_init() {
local src=''
local target_prefixes=()
local user=''
local opts=''
opts=$(pmodules::get_options -o h -l src: -l user: -l help -l version: -- "$@")
if [[ $? != 0 ]]; then
subcommand_help_init
exit 1
fi
eval set -- "${opts}"
while (($# > 0)); do
case $1 in
--src )
src=$2
shift
;;
--user )
user=$2
shift
;;
--version )
PMODULES_VERSION=$2
shift
;;
-- )
:
;;
-* | -h | --help )
echo "$1: illegal option" 1>&2
subcommand_help_init
exit 1
;;
* )
target_prefixes+=( "$1" )
;;
esac
shift
done
(( ${#target_prefixes[@]} != 0 )) || \
std::die 1 "Error: no target directory specified!"
# if source directory is not passed as argument, derive it from script name
if [[ -z "${src}" ]]; then
src=$(cd "${bindir}/../../../.." && pwd)
fi
[[ -d "${src}" ]] || \
std::die 1 "Error: ${src}: source directory does not exist!"
[[ -r "${src}/config/profile.bash" ]] || \
std::die 1 "Error: ${src}: shell profile does not exist or is not readable!"
source "${src}/config/profile.bash"
local -i euid=$(id -u)
if (( euid == 0 )); then
[[ -n "${user}" ]] || \
std::die 1 "Error: --user parameter is required!"
id -u "${user}" > /dev/null 2>&1 || \
std::die 1 "Error: Unable to retrieve user id of user '${user}'"
else
[[ -z "${user}" ]] || \
std::die 1 "Error: --user option is only allowed if running as root!"
fi
pmodules::check_env || \
std::die 1 "Giving up..."
echo "
Attempting to create a minimal module environment from the
environment at '${PMODULES_ROOT}'
"
#.....................................................................
# initialize new module environment in given directory
#
# Arguments:
# $1 target directory
#
init_pmodules_environment() {
local -r src_prefix="${PMODULES_ROOT}"
local -r target_prefix=$1
local src=''
local dst=''
echo "Initializing target directory '${target_prefix}' ..."
echo
if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then
echo "Warning: ${target_prefix} already exists."
std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \
std::die 1 "Abort ..."
fi
force='yes'
echo "Creating target directory '${target_prefix}'..."
$DRY mkdir -p "${target_prefix}" || \
std::die 1 "Error: make directory failed!"
echo
echo "Syncing configuration ..."
sync_config "${src_prefix}" \
"${target_prefix}" || \
std::die 1 "Error: configuration synchronization failed!"
echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..."
sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \
"${src_prefix}" \
"${target_prefix}" || \
std::die 1 "Error: sync Pmodules failed!"
mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}"
echo
dst="${target_prefix}/${PMODULES_CONFIG_DIR}/environment.bash"
echo "Adding installation source '${src_prefix}' to '${dst}'..."
sed -i.bak '/PMODULES_INSTALL_SOURCE/d' "${dst}"
echo "declare -x PMODULES_INSTALL_SOURCE=\"${src_prefix}\"" >> "${dst}"
echo
if [[ -n "${user}" ]]; then
echo "Changing user of new module environment to '${user}'..."
$DRY chown -R "${user}" "${target_prefix}" || \
std::die 1 "Error: changing owner failed!"
echo
fi
echo "New minimal module environment created at '${target_prefix}'."
echo "To use this environment, execute"
echo " sudo ln -fs ${target_prefix} /opt/psi"
echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash"
}
umask 022
for target_prefix in "${target_prefixes[@]}"; do
init_pmodules_environment "${target_prefix}"
done
}
declare -a Groups=()
declare -A GroupDepths
##############################################################################
#
# Get available module groups. Found groups are added to the global array
# 'Groups'.
#
# Arguments:
# $1: root of module environment
# $2: relative directory with module files
#
get_groups () {
local -r root="$1"
local -r modulefiles_dir="$2"
{
cd "${root}"
# for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems
for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do
[[ -d ${f}/${modulefiles_dir} ]] || continue
Groups+=( $f )
done
};
}
##############################################################################
#
# Compute hierarchy depth of all groups. Stores result in global array
# 'GroupDepths'. The group depth is defined as the hierarchy depth times 2.
#
# Examples:
# group depth of 'Programming' is 0
# group depth of 'Compiler' is 2
# group depth of 'MPI' is 4
#
# Arguments:
# $1: root of module environment
# $2: relative directory with module files
#
get_group_depths () {
local -r root="$1"
local -r modulefiles_dir="$2"
local -ir off=4
{
cd "${root}"
local group
for group in "${Groups[@]}"; do
local fname=$(find "${group}/${modulefiles_dir}" \
-depth \( -type f -o -type l \) -print -quit)
[[ -n ${fname} ]] || continue
local -a tmp
std::split_fname tmp "${fname}"
(( GroupDepths[$group]=${#tmp[@]}-${off} ))
done
};
}
##############################################################################
#
# sub-command 'install'
#
# Arguments:
# [--dry-run]
# Dry run
#
# [--force] | -f ]
# Install module even it already exists
#
# [--release <rel>]
# Set release of module to <rel>
#
# [--src <src>]
# Install from module environment in <src>
#
# [--with <module>]
# Install module(s) in this sub-group only
#
# <module_pattern>
# Install modules matching given pattern
#
subcommand_install() {
local opts=''
local -a with=()
local -a releases=()
local -a module_pattern=()
local src_prefix="${PMODULES_INSTALL_SOURCE}"
local -r target_prefix="${PMODULES_ROOT}"
local modulefile=''
local -A modules_to_install
local -A dependencies_to_install
local -A map_to_family
local -a initial_modulepath=()
#......................................................................
#
# Resolve dependencies to given module
#
# Arguments:
# $1 modulefile relativ to src prefix. Something like:
# MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14
#
# Notes:
# The variables
# initial_modulepath
# modules_to_install
# map_to_family
# from the calling function are used!
#
resolve_dependencies_of_module () {
local -r modulefile=$1
local -a modulepath=( "${initial_modulepath[@]}" )
# compute filename with dependencies of given module
local -i i=0 n=0
std::split_fname items n "${modulefile}"
local fname_dependencies="${src_prefix}/${items[0]}"
for (( i = n-2; i >= 2; i-=2 )); do
fname_dependencies+="/${items[$i]}/${items[i+1]}"
done
fname_dependencies+='/.dependencies'
[[ -r ${fname_dependencies} ]] || return 0
# loop over all dependecies
local dep
while read dep; do
# skip empty lines
# :FIXME: skip comments?!
[[ -z ${dep} ]] && continue
# search for module with current modulepath and remember
local modulename=$(find "${modulepath[@]}" -path "*/${dep}" 2>/dev/null | head -n 1 )
[[ -n ${modulename} ]] || \
std::die 3 "Oops: required module '${dep}' not found!"
modulename=${modulename/${src_prefix}\/}
dependencies_to_install[${modulename}]='.'
resolve_dependencies_of_module "${modulename}"
# append new node in hierarchy to modulepath
if [[ -n ${map_to_family[${dep}]} ]]; then
local path="${src_prefix}/${map_to_family[${dep}]}/${PMODULES_MODULEFILES_DIR}/"
path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}"
modulepath+=( "${path}" )
fi
done < "${fname_dependencies}"
}
#......................................................................
#
# Print list of modules which will be installed and ask user wheter
# he wants to continue or abort.
#
# Arguments:
# none
#
# Notes:
# The following variables of the enclosing function are used:
# modules_to_install (read-only)
# target_prefix (read-only)
# dependencies_to_install (read-only)
print_modules_to_install() {
local modulefile
std::info "The following modules will be installed/updated:"
for modulefile in "${!modules_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
if (( ${#dependencies_to_install[@]} > 0 )); then
std::info "\nThe following dependencies will be installed/updated:"
for modulefile in "${!dependencies_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
done 2>&1 | sort
fi
std::info ""
std::get_YN_answer "Do you want to continue? [n] " || \
std::die 1 "Aborting..."
std::info ""
}
opts=$(pmodules::get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@")
if [[ $? != 0 ]]; then
subcommand_help_install
exit 1
fi
eval set -- "${opts}"
while (($# > 0)); do
case $1 in
--dry-run )
DRY='echo'
;;
--force | -f )
force='yes'
;;
--release )
releases+=( "$2" )
shift
;;
--src )
src_prefix="$2"
shift
;;
--with )
with+=( "$2" )
shift
;;
-- )
:
;;
-h | --help )
subcommand_help_install
exit 1
;;
-* )
echo "$1: illegal option" 1>&2
subcommand_help_init
exit 1
;;
* )
module_pattern+=( "$1" )
;;
esac
shift
done
[[ -n ${src_prefix} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_prefix} ]] \
|| std::die 3 "Oops: '${src_prefix}' is not a valid installation source."
# scan available groups and their depth
get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}"
get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}"
# set initial modulepath
local group
for group in "${!GroupDepths[@]}"; do
if (( ${GroupDepths[${group}]} == 0 )); then
initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" )
fi
done
#
# create a mapping from module name to their family.
# Examples:
# gcc/5.2.0 -> Compiler
# openmpi/1.8.4 -> MPI
local _fname=''
while read _fname; do
local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}"
local -a items
std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}"
local -i n=${#items[*]}
# We are only interested in families adding something to
# the modulepath.
if (( n >= 4 )); then
local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" )
map_to_family[$_key]=${_family}
fi
done < <({ cd "${src_prefix}" && \
find */"${PMODULES_MODULEFILES_DIR}" \
\( -type l -o -type f \) \! -name ".*"; } 2>/dev/null )
#
# search for to be installed modules and their dependencies
#
local -i n=0
while read modulefile; do
resolve_dependencies_of_module "${modulefile}"
modules_to_install["${modulefile}"]+='.'
let n+=1
done < <(${PMODULES_HOME}/bin/modulecmd bash search \
"${module_pattern[@]}" \
"${with[@]/#/--with=}" \
"${releases[@]/#/--release=}" \
--no-header --print-modulefiles \
--src="${src_prefix}" 2>&1 1>/dev/null)
(( n == 0 )) && \
std::die 0 "No matching modules found ..."
print_modules_to_install
# install ...
for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do
if [[ -e "${target_prefix}/${modulefile}" ]]; then
std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
else
std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
fi
sync_module "${modulefile}" \
"${src_prefix}" \
"${target_prefix}"
done
std::info "\nDone!\n"
}
###############################################################################
#
# delete specified module(s)
#
subcommand_delete() {
:
}
###############################################################################
#
# remove modules which have been removed in our source
#
subcommand_cleanup() {
:
}
###############################################################################
#
# search modules in source
# :FIXME: this is still crap
#
subcommand_search() {
local src_prefix="${PMODULES_INSTALL_SOURCE}"
local -r target_prefix="${PMODULES_ROOT}"
local -A modules_found
print_modules_found() {
std::info "The following modules are available:"
for modulefile in "${!modules_found[@]}"; do
std::info " ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }"
done 2>&1 | sort
}
[[ -n ${src_prefix} ]] \
|| std::die 3 "Oops: no installation source given."
[[ -d ${src_prefix} ]] \
|| std::die 3 "Oops: '${src_prefix}' is not a valid installation source."
# scan available groups and their depth
get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}"
get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}"
local -i n=0
while read modulefile; do
modules_found["${modulefile}"]+='.'
let n+=1
done < <(${PMODULES_HOME}/bin/modulecmd bash search \
-a \
--no-header --print-modulefiles \
--src="${src_prefix}" \
"$@" \
2>&1 1>/dev/null)
(( n == 0 )) && \
std::die 0 "No matching modules found ..."
print_modules_found
}
while (($# > 0)); do
case $1 in
-h | -H | -\? | --help | -help )
usage
exit 1
;;
-V | --version )
print_version
exit 1
;;
-f | --force )
force='yes'
;;
--debug )
set -x
;;
--dry-run )
dry_run='yes'
DRY='echo'
;;
-* )
echo "$1: unknown switch.\n" 1>&2
exit 1
;;
init|install|sync|search|help )
subcommand="subcommand_$1"
shift
sargs=( $* )
shift $#
;;
* )
echo "$1: unknown sub-command" 1>&2
exit 1
esac
shift || :
done
if [[ -z ${subcommand} ]]; then
usage
exit 1
fi
[[ "${subcommand}" != "subcommand_init" ]] && [[ -z "${PMODULES_ROOT}" ]] && \
std::die 1 "Error: No current module environment configured!"
$subcommand "${sargs[@]}"
# Local Variables:
# mode: sh
# sh-basic-offset: 8
# tab-width: 8
# End: