#!@PMODULES_HOME@/sbin/bash # we have to unset CDPATH, otherwise 'cd' prints the directoy! unset CDPATH # used for some output only declare -r CMD=$(basename "$0") declare -r mydir=$(cd $(dirname "$0") && pwd) declare -r prefix=$(dirname "${mydir}") declare -r bindir="${prefix}/bin" declare -r sbindir="${prefix}/sbin" declare -r libdir="${prefix}/lib" declare -r libexecdir="${prefix}/libexec" source "${libdir}/libstd.bash" source "${libdir}/libpmodules.bash" PATH="${sbindir}:${bindir}:${PATH}" _exit () { std::die 1 "Interrupted..." } _err () { std::info "Oops: got an error in function '${FUNCNAME[1]}', line ${BASH_LINENO[0]}" std::die 1 "Aborting ..." } trap '_exit' INT TERM trap '_err' ERR # make sure that everything is used from this version declare PMODULES_VERSION='@PMODULES_VERSION@' ############################################################################## # # print version of program # # Arguments: # none # print_version() { echo " Pmodules @PMODULES_VERSION@ using Tcl Environment Modules @MODULES_VERSION@ Copyright GNU GPL v2 " 1>&2 } ############################################################################## # # print usage # # Arguments: # none # usage() { local -r prog=$(basename $0) print_version echo " Usage: ${prog} [ switches ] [ subcommand ] [subcommand-args ] Switches: --dry-run do nothing --force force overwrite Available SubCommands and Args: init [--src=] [--user=] Initialize a new minimal Pmodule environment. install [--with=...] Install matching modules sync [--delete] [--dst=] Synchronize modules. " } declare force='no' declare dry_run='no' declare DRY='' declare subcommand='' declare sargs=() ############################################################################## # # help for subcommand 'init' # # Arguments: # none # subcommand_help_init() { echo " init [--src=] [--user=] [--version=] Initialize a new minimal Pmodule environment in directory . The parameter must only be present if ${prog} is executed as root. " 1>&2 } ############################################################################## # # help for subcommand 'install' # # Arguments: # none # subcommand_help_install() { echo " install ... [--with=...] [--release=...] [--src=] Install matching modules " 1>&2 } ############################################################################## # # help for subcommand 'search' # # Arguments: # none # subcommand_help_search() { echo " USAGE: module search [switches] string... Search available modules. If an argument is given, search for modules whose name match the argument. SWITCHES: --no-header Suppress output of a header. --with=STRING Search for modules compiled with modules matching string. The command module search --with=gcc/4.8.3 lists all modules in the hierarchy compiled with gcc 4.8.3. " 1>&2 } ############################################################################## # # help for subcommand 'sync' # # Arguments: # none # subcommand_help_sync() { echo " sync [--delete] [--dst=] Synchronize environment modules and configuration files from Pmodule environment to Pmodule environment (default: currently active Pmodule environment). Not yet implemented: If --delete is given, unmarked modules present in will be deleted. " 1>&2 } ############################################################################## # # print usage or help text for given sub-command # # Arguments: # none or sub-command # subcommand_help() { if [[ $# == 0 ]]; then usage elif typeset -F subcommand_help_$1 > /dev/null 2>&1 ; then # help for sub-command subcommand_help_$1 else usage fi } ############################################################################## # # Derive the relative module installation path # from the relative modulefile path # # Arguments: # $1: relative module file path # get_module_prefix() { local -a comp=( ${1//\// } ) # split rel.path into components local path="${comp[0]}" # result path local -i i for ((i=1; i<${#comp[@]}-1; i+=2)); do path+="/${comp[$((-i-1))]}/${comp[$((-i))]}" done echo "${path}" } ############################################################################## # # Derive the relative module release file path # from the relative module file path # # Arguments: # $1: relative module file path # get_releasefile_name() { echo "$(dirname "$1")/.release-$(basename "$1")" } ############################################################################## # # Sync a module from one Pmodules environment to another: # - sync module installation # - sync modulefile # - sync release file # # Note: # We do not take care of files in $PMODULES_ROOT/$PMODULES_TEMPLATES_DIR. If # the modulefile is a sym-link it is expected that the target exists. # # Arguments: # $1: relative modulefile path (something like: Tools/gnuplot/5.0.0) # $2: source prefix of Pmodule environment # $3: target prefix of Pmodule environment # sync_module() { local -r rel_modulefile=$1 local -r src_prefix=$2 local -r target_prefix=$3 local -r rel_module_prefix=$( get_module_prefix "${rel_modulefile}" ) local -r rel_releasefile=$( get_releasefile_name "${rel_modulefile}" ) # install/update module if [[ ! -d "${target_prefix}/${rel_module_prefix}" ]] || [[ "${force}" == 'yes' ]]; then $DRY mkdir -p "${target_prefix}/${rel_module_prefix}" || return $? $DRY rsync --links --perms --recursive --delete \ "${src_prefix}/${rel_module_prefix}/" \ "${target_prefix}/${rel_module_prefix}/" || exit $? fi local -r src_modulefile="${src_prefix}/${rel_modulefile}" local -r src_releasefile="${src_prefix}/${rel_releasefile}" local -r target_modulefile="${target_prefix}/${rel_modulefile}" local -r target_releasefile="${target_prefix}/${rel_releasefile}" # create target directory for module- and release-file if [[ -e "${src_modulefile}" ]] || [[ -e "${src_releasefile}" ]]; then local dir=$( dirname "${target_modulefile}" ) $DRY mkdir -p "${dir}" || return $? fi # copy modulefile template local -a rel_modulefile_splitted std::split_fname rel_modulefile_splitted "${rel_modulefile}" local -r module_group="${rel_modulefile_splitted[0]}" local -r module_name="${rel_modulefile_splitted[-2]}" local -r template="${module_group}/${PMODULES_TEMPLATES_DIR}/${module_name}/" local -r src_template="${src_prefix}/${template}" local -r target_template="${target_prefix}/${template}" if [[ -e "${src_template}" ]]; then $DRY mkdir -p "${target_template}" $DRY rsync --links --perms --recursive \ "${src_template}" "${target_template}" || exit $? fi # copy modulefile if [[ -e "${src_modulefile}" ]]; then $DRY rsync --links --perms --recursive \ "${src_modulefile}" "${target_modulefile}" || exit $? fi # copy release-file if [[ -e "${src_releasefile}" ]]; then $DRY rsync --links --perms --recursive \ "${src_releasefile}" "${target_releasefile}" || exit $? fi } ############################################################################## # # Sync the Pmodules configuration and templates # # Arguments: # $1: source prefix of Pmodule environment # $2: target prefix of Pmodule environment # sync_config() { src="$1/${PMODULES_CONFIG_DIR}/" dst="$2/${PMODULES_CONFIG_DIR}/" $DRY rsync --recursive --links --perms --delete \ "${src}" "${dst}" 2>/dev/null || return $? sed -i.bak "s/PMODULES_VERSION=\(.*\)/PMODULES_VERSION=${PMODULES_VERSION}/" "${dst}/environment.bash" echo } ############################################################################## # # Delete a module # # Arguments: # $1: relative modulefile path # $2: target prefix of Pmodule environment # delete_module() { echo "Not implemented yet!" } ############################################################################## # # initialize a new module environment # # Arguments: # [--src ] # Module environment we are going to sync from. If not # specified, the module environment this script is in # will be used. # [--user ] # If this scripts runs with root privileges, a user name # ore ID must be specified. # [--version ] # Set PMODULES_VERSION to # TARGET_DIR # Initialize a new module environment in this directory- # subcommand_init() { local src='' local target_prefixes=() local user='' local opts='' opts=$(pmodules::get_options -o h -l src: -l user: -l help -l version: -- "$@") if [[ $? != 0 ]]; then subcommand_help_init exit 1 fi eval set -- "${opts}" while (($# > 0)); do case $1 in --src ) src=$2 shift ;; --user ) user=$2 shift ;; --version ) PMODULES_VERSION=$2 shift ;; -- ) : ;; -* | -h | --help ) echo "$1: illegal option" 1>&2 subcommand_help_init exit 1 ;; * ) target_prefixes+=( "$1" ) ;; esac shift done (( ${#target_prefixes[@]} != 0 )) || \ std::die 1 "Error: no target directory specified!" # if source directory is not passed as argument, derive it from script name if [[ -z "${src}" ]]; then src=$(cd "${bindir}/../../../.." && pwd) fi [[ -d "${src}" ]] || \ std::die 1 "Error: ${src}: source directory does not exist!" [[ -r "${src}/config/profile.bash" ]] || \ std::die 1 "Error: ${src}: shell profile does not exist or is not readable!" source "${src}/config/profile.bash" local -i euid=$(id -u) if (( euid == 0 )); then [[ -n "${user}" ]] || \ std::die 1 "Error: --user parameter is required!" id -u "${user}" > /dev/null 2>&1 || \ std::die 1 "Error: Unable to retrieve user id of user '${user}'" else [[ -z "${user}" ]] || \ std::die 1 "Error: --user option is only allowed if running as root!" fi pmodules::check_env || \ std::die 1 "Giving up..." echo " Attempting to create a minimal module environment from the environment at '${PMODULES_ROOT}' " #..................................................................... # initialize new module environment in given directory # # Arguments: # $1 target directory # init_pmodules_environment() { local -r src_prefix="${PMODULES_ROOT}" local -r target_prefix=$1 local src='' local dst='' echo "Initializing target directory '${target_prefix}' ..." echo if [[ -d "${target_prefix}" ]] && [[ ${force} == no ]]; then echo "Warning: ${target_prefix} already exists." std::get_YN_answer "Do you really want to re-run the initialization? (y/N) " || \ std::die 1 "Abort ..." fi force='yes' echo "Creating target directory '${target_prefix}'..." $DRY mkdir -p "${target_prefix}" || \ std::die 1 "Error: make directory failed!" echo echo "Syncing configuration ..." sync_config "${src_prefix}" \ "${target_prefix}" || \ std::die 1 "Error: configuration synchronization failed!" echo "Syncing Pmodules ${PMODULES_VERSION} from '${src_prefix}' to '${target_prefix}'..." sync_module "Tools/${PMODULES_MODULEFILES_DIR}/Pmodules/${PMODULES_VERSION}" \ "${src_prefix}" \ "${target_prefix}" || \ std::die 1 "Error: sync Pmodules failed!" mkdir -p "${target_prefix}/Tools/${PMODULES_MODULEFILES_DIR}" echo dst="${target_prefix}/${PMODULES_CONFIG_DIR}/environment.bash" echo "Adding installation source '${src_prefix}' to '${dst}'..." sed -i.bak '/PMODULES_INSTALL_SOURCE/d' "${dst}" echo "declare -x PMODULES_INSTALL_SOURCE=\"${src_prefix}\"" >> "${dst}" echo if [[ -n "${user}" ]]; then echo "Changing user of new module environment to '${user}'..." $DRY chown -R "${user}" "${target_prefix}" || \ std::die 1 "Error: changing owner failed!" echo fi echo "New minimal module environment created at '${target_prefix}'." echo "To use this environment, execute" echo " sudo ln -fs ${target_prefix} /opt/psi" echo " source /opt/psi/${PMODULES_CONFIG_DIR}/profile.bash" } umask 022 for target_prefix in "${target_prefixes[@]}"; do init_pmodules_environment "${target_prefix}" done } declare -a Groups=() declare -A GroupDepths ############################################################################## # # Get available module groups. Found groups are added to the global array # 'Groups'. # # Arguments: # $1: root of module environment # $2: relative directory with module files # get_groups () { local -r root="$1" local -r modulefiles_dir="$2" { cd "${root}" # for some unknown reason [A-Z]* doesn't work on (some?) SL6 systems for f in [ABCDEFGHIJKLMNOPQRSTUVWXYZ]*; do [[ -d ${f}/${modulefiles_dir} ]] || continue Groups+=( $f ) done }; } ############################################################################## # # Compute hierarchy depth of all groups. Stores result in global array # 'GroupDepths'. The group depth is defined as the hierarchy depth times 2. # # Examples: # group depth of 'Programming' is 0 # group depth of 'Compiler' is 2 # group depth of 'MPI' is 4 # # Arguments: # $1: root of module environment # $2: relative directory with module files # get_group_depths () { local -r root="$1" local -r modulefiles_dir="$2" local -ir off=4 { cd "${root}" local group for group in "${Groups[@]}"; do local fname=$(find "${group}/${modulefiles_dir}" \ -depth \( -type f -o -type l \) -print -quit) [[ -n ${fname} ]] || continue local -a tmp std::split_fname tmp "${fname}" (( GroupDepths[$group]=${#tmp[@]}-${off} )) done }; } ############################################################################## # # sub-command 'install' # # Arguments: # [--dry-run] # Dry run # # [--force] | -f ] # Install module even it already exists # # [--release ] # Set release of module to # # [--src ] # Install from module environment in # # [--with ] # Install module(s) in this sub-group only # # # Install modules matching given pattern # subcommand_install() { local opts='' local -a with=() local -a releases=() local -a module_pattern=() local src_prefix="${PMODULES_INSTALL_SOURCE}" local -r target_prefix="${PMODULES_ROOT}" local modulefile='' local -A modules_to_install local -A dependencies_to_install local -A map_to_family local -a initial_modulepath=() #...................................................................... # # Resolve dependencies to given module # # Arguments: # $1 modulefile relativ to src prefix. Something like: # MPI/modulefiles/gcc/4.9.2/openmpi/1.8.4/hdf5/1.8.14 # # Notes: # The variables # initial_modulepath # modules_to_install # map_to_family # from the calling function are used! # resolve_dependencies_of_module () { local -r modulefile=$1 local -a modulepath=( "${initial_modulepath[@]}" ) # compute filename with dependencies of given module local -i i=0 n=0 std::split_fname items n "${modulefile}" local fname_dependencies="${src_prefix}/${items[0]}" for (( i = n-2; i >= 2; i-=2 )); do fname_dependencies+="/${items[$i]}/${items[i+1]}" done fname_dependencies+='/.dependencies' [[ -r ${fname_dependencies} ]] || return 0 # loop over all dependecies local dep while read dep; do # skip empty lines # :FIXME: skip comments?! [[ -z ${dep} ]] && continue # search for module with current modulepath and remember local modulename=$(find "${modulepath[@]}" -path "*/${dep}" 2>/dev/null | head -n 1 ) [[ -n ${modulename} ]] || \ std::die 3 "Oops: required module '${dep}' not found!" modulename=${modulename/${src_prefix}\/} dependencies_to_install[${modulename}]='.' resolve_dependencies_of_module "${modulename}" # append new node in hierarchy to modulepath if [[ -n ${map_to_family[${dep}]} ]]; then local path="${src_prefix}/${map_to_family[${dep}]}/${PMODULES_MODULEFILES_DIR}/" path+="${modulename/*\/${PMODULES_MODULEFILES_DIR}\/}" modulepath+=( "${path}" ) fi done < "${fname_dependencies}" } #...................................................................... # # Print list of modules which will be installed and ask user wheter # he wants to continue or abort. # # Arguments: # none # # Notes: # The following variables of the enclosing function are used: # modules_to_install (read-only) # target_prefix (read-only) # dependencies_to_install (read-only) print_modules_to_install() { local modulefile std::info "The following modules will be installed/updated:" for modulefile in "${!modules_to_install[@]}"; do if [[ -e "${target_prefix}/${modulefile}" ]]; then std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" else std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" fi done 2>&1 | sort if (( ${#dependencies_to_install[@]} > 0 )); then std::info "\nThe following dependencies will be installed/updated:" for modulefile in "${!dependencies_to_install[@]}"; do if [[ -e "${target_prefix}/${modulefile}" ]]; then std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" else std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" fi done 2>&1 | sort fi std::info "" std::get_YN_answer "Do you want to continue? [n] " || \ std::die 1 "Aborting..." std::info "" } opts=$(pmodules::get_options -o hf -l dry-run -l force -l with: -l release: -l help -l src: -- "$@") if [[ $? != 0 ]]; then subcommand_help_install exit 1 fi eval set -- "${opts}" while (($# > 0)); do case $1 in --dry-run ) DRY='echo' ;; --force | -f ) force='yes' ;; --release ) releases+=( "$2" ) shift ;; --src ) src_prefix="$2" shift ;; --with ) with+=( "$2" ) shift ;; -- ) : ;; -h | --help ) subcommand_help_install exit 1 ;; -* ) echo "$1: illegal option" 1>&2 subcommand_help_init exit 1 ;; * ) module_pattern+=( "$1" ) ;; esac shift done [[ -n ${src_prefix} ]] \ || std::die 3 "Oops: no installation source given." [[ -d ${src_prefix} ]] \ || std::die 3 "Oops: '${src_prefix}' is not a valid installation source." # scan available groups and their depth get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" # set initial modulepath local group for group in "${!GroupDepths[@]}"; do if (( ${GroupDepths[${group}]} == 0 )); then initial_modulepath+=( "${src_prefix}/${group}/${PMODULES_MODULEFILES_DIR}" ) fi done # # create a mapping from module name to their family. # Examples: # gcc/5.2.0 -> Compiler # openmpi/1.8.4 -> MPI local _fname='' while read _fname; do local _family="${_fname%/${PMODULES_MODULEFILES_DIR}/*}" local -a items std::split_fname items "${_fname#*/${PMODULES_MODULEFILES_DIR}/}" local -i n=${#items[*]} # We are only interested in families adding something to # the modulepath. if (( n >= 4 )); then local _key=$( IFS='/'; echo "${items[*]:$n-4:2}" ) map_to_family[$_key]=${_family} fi done < <({ cd "${src_prefix}" && \ find */"${PMODULES_MODULEFILES_DIR}" \ \( -type l -o -type f \) \! -name ".*"; } 2>/dev/null ) # # search for to be installed modules and their dependencies # local -i n=0 while read modulefile; do resolve_dependencies_of_module "${modulefile}" modules_to_install["${modulefile}"]+='.' let n+=1 done < <(${PMODULES_HOME}/bin/modulecmd bash search \ "${module_pattern[@]}" \ "${with[@]/#/--with=}" \ "${releases[@]/#/--release=}" \ --no-header --print-modulefiles \ --src="${src_prefix}" 2>&1 1>/dev/null) (( n == 0 )) && \ std::die 0 "No matching modules found ..." print_modules_to_install # install ... for modulefile in "${!modules_to_install[@]}" "${!dependencies_to_install[@]}"; do if [[ -e "${target_prefix}/${modulefile}" ]]; then std::info " Updating: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" else std::info " Installing: ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" fi sync_module "${modulefile}" \ "${src_prefix}" \ "${target_prefix}" done std::info "\nDone!\n" } ############################################################################### # # delete specified module(s) # subcommand_delete() { : } ############################################################################### # # remove modules which have been removed in our source # subcommand_cleanup() { : } ############################################################################### # # search modules in source # :FIXME: this is still crap # subcommand_search() { local src_prefix="${PMODULES_INSTALL_SOURCE}" local -r target_prefix="${PMODULES_ROOT}" local -A modules_found print_modules_found() { std::info "The following modules are available:" for modulefile in "${!modules_found[@]}"; do std::info " ${modulefile/\/${PMODULES_MODULEFILES_DIR}\//: }" done 2>&1 | sort } [[ -n ${src_prefix} ]] \ || std::die 3 "Oops: no installation source given." [[ -d ${src_prefix} ]] \ || std::die 3 "Oops: '${src_prefix}' is not a valid installation source." # scan available groups and their depth get_groups "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" get_group_depths "${src_prefix}" "${PMODULES_MODULEFILES_DIR}" local -i n=0 while read modulefile; do modules_found["${modulefile}"]+='.' let n+=1 done < <(${PMODULES_HOME}/bin/modulecmd bash search \ -a \ --no-header --print-modulefiles \ --src="${src_prefix}" \ "$@" \ 2>&1 1>/dev/null) (( n == 0 )) && \ std::die 0 "No matching modules found ..." print_modules_found } while (($# > 0)); do case $1 in -h | -H | -\? | --help | -help ) usage exit 1 ;; -V | --version ) print_version exit 1 ;; -f | --force ) force='yes' ;; --debug ) set -x ;; --dry-run ) dry_run='yes' DRY='echo' ;; -* ) echo "$1: unknown switch.\n" 1>&2 exit 1 ;; init|install|sync|search|help ) subcommand="subcommand_$1" shift sargs=( $* ) shift $# ;; * ) echo "$1: unknown sub-command" 1>&2 exit 1 esac shift || : done if [[ -z ${subcommand} ]]; then usage exit 1 fi [[ "${subcommand}" != "subcommand_init" ]] && [[ -z "${PMODULES_ROOT}" ]] && \ std::die 1 "Error: No current module environment configured!" $subcommand "${sargs[@]}" # Local Variables: # mode: sh # sh-basic-offset: 8 # tab-width: 8 # End: