#!/bin/bash declare -r __MODULEFILES_DIR__='modulefiles' declare PMODULES_VERSION='@PMODULES_VERSION@' declare -A GroupDepths=(['none']=0) declare -a Overlays=() declare -A OverlayInfo declare -A Dir2OverlayMap # An overlay has a type defining the way modules in this overlay # make modules in the other overlays unavailable. # # 'normal' # Make modules in other overlay unavailable with the same full name. # If the overlay doesn't support groups, the overlay should provide # only modules with different names from the modules in the other # overlays. Otherwise you modules as available which cannot be loaded. # Example: # A overlay providing modules with name gcc doesn't conceal the gcc # modules in the base overlay, but they are listed as available. # # # 'hiding' # Make modules in other overlay unavailable with the same name. # Examples: # - If a module with name 'gcc' is in a overlay with this # type, only the gcc modules in this overlay are available. This # can for example be used to conceal old versions of gcc. # - In same case we need special variants of a module for a system, # for exampe openmpi and mpich. Variants of these software can be # made available in an overlay. If such an overlay is used, modules # which should not be used can be concealed. # If the overlay doesn't support groups, a module in the overlay # conceals all modules in other overlays independend from the group. # Example: # - In the Spack overlays - which doesn't support groups - are modules # with name gcc. The gcc modules in the Spack overlays conceals the # gcc modules in the group Programming of the base overlay. # 'replacing' # This type can be used to make hole groups unavailable. # # declare -r ol_normal='n' declare -r ol_hiding='h' declare -r ol_replacing='r' compute_group_depth () { : " Compute depth of modulefile directory. " local -n result="$1" # ref.var to return result local -r dir="$2" # absolute path of a modulefile directory if [[ ! -d "${dir}" ]]; then ${mkdir} -p "${dir}" || \ std::die 1 "Cannot create directory -- ${dir}" fi local -- group=${dir%/*} local -- group=${group##*/} result=$(${find} "${dir}" -depth \( -type f -o -type l \) \ -printf "%d" -quit 2>/dev/null) (( result-=2 )) || : # if a group doesn't contain a modulefile, depth is negativ # :FIXME: better solution? (( result < 0 )) && (( result = 0 )) || : } scan_groups () { : " (Re-)Scan available groups in the overlays $@ and compute group depth's. Set GroupDepths[group] and Dir2OverlayMap[ol:dir]. " local -- ol='' local -i depth=0 for ol in "$@"; do [[ "${OverlayInfo[${ol}:layout]}" == 'Pmodules' ]] || continue local -- modulefiles_root="${OverlayInfo[${ol}:modulefiles_root]}" local -- dir='' for dir in "${modulefiles_root}"/*/"${__MODULEFILES_DIR__}"; do local -- group="${dir%/*}" group="${group##*/}" if [[ ! -v GroupDepths[${group}] ]]; then compute_group_depth depth "${dir}" GroupDepths[$group]=${depth} fi # :FIXME: do we need this? Dir2OverlayMap[${dir}]="${ol}" Dir2OverlayMap[${dir%/"${__MODULEFILES_DIR__}"*}]="${ol}" done done GroupDepths['none']=0 } declare -A DefaultPmodulesConfig=( ['defaultgroups']='Tools:Programming' ['default_groups']='Tools:Programming' ['defaultreleasestages']='stable' ['default_reltages']='stable' ['tmpdir']="/opt/psi/var/tmp/${USER}" ['tmp_dir']="/opt/psi/var/tmp/${USER}" ['distfilesdir']='/opt/psi/var/distfiles' ['distfiles_dir']='/opt/psi/var/distfiles' ['download_dir']='/opt/psi/var/distfiles' ['overlays']='' ) declare -A OverlayConfigKeys=( ['install_root']='/opt/psi' ['modulefiles_root']='' ['excludes']='' ['type']='n' ['conflicts']='' ['path_config']='' ['default_relstage']='unstable' ['layout']='Pmodules' ['has_additional_modulepaths']='false' ['groups']='' ) declare -A OverlayPathConfigKeys=( ['target_cpus']='' ['modulepath']='' ['modulepath_unstable']='' ['modulepath_stable']='' ['modulepath_deprecated']='' ) yml::die_parsing(){ std::die 3 "error parsing YAML:\n----\n$1\n----" } yml::die_type_error(){ std::die 3 "%s" \ "Value of '$1' must be of type '$2', but is '$3'!" } yml::die_invalid_key(){ std::die 3 "%s -- %s\n%s" \ "Invalid key in configuration" \ "$1" "$2" } yml::die_read_file(){ std::die 3 "Cannot read file '$1'. Please check with yamllint!" } yml::read_file(){ local -n yml_content="$1" local -- yml_fname="$2" local -- yml_node="$3" yml_content=$( ${yq} -Ne e "${yml_node}|explode(.)" "${yml_fname}" 2>/dev/null ) || \ yml::die_read_file "${yml_fname}" } yml::get_keys(){ local -n yml_keys="$1" local -n yml_input="$2" local -- yml_node="$3" local -- str='' str="$( ${yq} -e "${yml_node}|keys|.[]" 2>/dev/null <<<"${yml_input}")" || \ { yml_keys=(); return 0; }; readarray -t yml_keys <<<"${str}" } yml::get_type(){ local -n yml_type="$1" local -n yml_input="$2" local -- yml_node="$3" yml_type="$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}")" || \ yml::die_parsing "${yml_input}" } yml::get_value(){ local -n yml_val="$1" local -n yml_input="$2" local -- yml_node="$3" local -- yml_expected_type="$4" local -- type='' type="$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}")" || \ yml::die_parsing "${yml_input}" [[ "${type}" == "${yml_expected_type}" ]] || \ yml::die_type_error "${yml_node}" "${yml_expected_type:2}" "${type:2}" yml_val=$( ${yq} -e "${yml_node}" 2>/dev/null <<<"${yml_input}" ) || \ return 1 } yml::get_seq_length(){ local -n yml_seq_length="$1" # [out] number of variants local -n yml_input="$2" # [in] YAML input local -- yml_node="$3" # [in] node yml_seq_length=$(${yq} -e "${yml_node}|length" 2>/dev/null <<<"${yml_input}") || \ yml::die_parsing "${yml_input}" } yml::get_seq(){ local -n yml_val="$1" local -n yml_input="$2" local -- yml_node="$3" local -- type='' type=$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}") if [[ "${type:2}" == 'null' ]]; then yml_val='' return 0 fi [[ "${type}" == '!!seq' ]] || \ yml::die_type_error "${yml_node}" 'seq' "${type:2}" local -i length=0 length=$(${yq} -e "${yml_node}|length" 2>/dev/null <<<"${yml_input}") if (( length == 0 )); then yml_val='' return 0 fi yml_val=$( ${yq} -e "${yml_node}[]" 2>/dev/null <<<"${yml_input}" ) || \ return 1 } yml::die_invalid_ol_install_root(){ std::die 3 "Invalid installation root directory for overlay '$1' -- $2" } yml::die_invalid_ol_modulefiles_root(){ std::die 3 "Invalid modulefiles root directory for overlay '$1' -- $2" } yml::die_invalid_ol_type(){ std::die 3 "Invalid type for overlay '$1' -- $2" } yml::die_invalid_ol_layout(){ std::die 3 "Invalid layout for overlay '$1' -- $2\nAllowed values are 'Pmodules', 'Spack' and 'flat'." } yml::die_invalid_ol_relstage(){ std::die 3 "Invalid default release stage for overlay '$1' -- $2" } yml::die_invalid_ol_key(){ std::die 3 "%s -- %s\n%s" \ "Invalid key in configuration" \ "$1" "$2" } parse_path_config(){ local -n yaml="$1" local -- ol_name="$2" local -- key='' for key in "${!OverlayPathConfigKeys[@]}"; do OverlayInfo[${ol_name}:${key}]="${OverlayPathConfigKeys[${key}]}" done local -i l=0 yml::get_seq_length l yaml . local -i i=0 for ((i=0; i/dev/null [[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \ yml::die_invalid_ol_install_root "${ol_name}" "${value}" ;; modulefiles_root ) yml::get_value value yaml_input "${node}.${key}" '!!str' OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}") mkdir -p "${OverlayInfo[${ol_name}:modulefiles_root]}" 2>/dev/null [[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \ yml::die_invalid_ol_modulefiles_root "${ol_name}" "${value}" ;; type ) yml::get_value value yaml_input "${node}.${key}" '!!str' case ${value} in "${ol_normal}" | "${ol_replacing}" | "${ol_hiding}" ) : ;; * ) yml::die_invalid_ol_type "${ol_name}" "${value}" ;; esac OverlayInfo[${ol_name}:type]="${value}" ;; layout ) yml::get_value value yaml_input "${node}.${key}" '!!str' case ${value} in 'Pmodules' | 'Spack' | 'flat' ) : ;; * ) yml::die_invalid_ol_layout "${ol_name}" "${value}" ;; esac OverlayInfo[${ol_name}:${key,,}]="${value}" ;; default_relstage ) yml::get_value value yaml_input "${node}.${key}" '!!str' case ${value} in 'unstable' | 'stable' | 'deprecated' ) : ;; *) yml::die_invalid_ol_relstage "${ol_name}" "${value}" ;; esac OverlayInfo[${ol_name}:${key}]="${value}" ;; conflicts | excludes | groups) yml::get_seq value yaml_input "${node}.${key}" '!!seq' local -a tmp_array=() readarray -t tmp_array <<<${value} local -- tmp_str='' printf -v tmp_str "%s:" "${tmp_array[@]}" OverlayInfo[${ol_name}:${key}]=$(${envsubst} <<<"${tmp_str%:}" ) ;; path_config ) yml::get_value value yaml_input "${node}.${key}" '!!seq' parse_path_config value "${ol_name}" ;; * ) yml::die_invalid_ol_key "${key}" "${yaml_input}" ;; esac done OverlayInfo[${ol_name}:used]='no' if [[ -z "${OverlayInfo[${ol_name}:modulefiles_root]}" ]]; then OverlayInfo[${ol_name}:modulefiles_root]=${OverlayInfo[${ol_name}:install_root]} fi local -- modulefiles_root=${OverlayInfo[${ol_name}:modulefiles_root]} Dir2OverlayMap[${modulefiles_root}]="${ol_name}" } get_config(){ : " Get Pmodules configuration. " local -r config_file="$1" # Pmodules configuration file local -- yaml_input='' yml::read_file yaml_input "${config_file}" '.' local -- key='' local -a keys=() yml::get_keys keys yaml_input '.' for key in "${keys[@]}"; do case ${key,,} in defaultgroups | default_groups ) yml::get_value DefaultGroups yaml_input ".${key}" '!!str' ;; defaultreleasestages | default_reltages ) yml::get_value DefaultReleaseStages yaml_input ".${key}" '!!str' ;; tmpdir | tmp_dir ) yml::get_value TmpDir yaml_input ".${key}" '!!str' ;; distfilesdir | download_dir ) yml::get_value DistfilesDir yaml_input ".${key}" '!!str' ;; overlays ) local -- overlay='' local -a overlays=() yml::get_keys overlays yaml_input ".${key}" for overlay in "${overlays[@]}"; do get_config_of_overlay "${yaml_input}" "${overlay}" done ;; * ) yml::die_invalid_key "${key}" "${yaml_input}" ;; esac done } Overlays=() # system config file local -- sys_config_file="${PMODULES_HOME%%/Tools*}/config/Pmodules.yaml" if [[ -v PMODULES_CONFIG_FILE && -n "${PMODULES_CONFIG_FILE}" ]]; then sys_config_file="${PMODULES_HOME%%/Tools*}/config/${PMODULES_CONFIG_FILE}" fi test -r "${sys_config_file}" || \ std::die 3 \ "%s %s -- %s" \ "base overlay definition file" \ "does not exist or is not readable" \ "$_" DefaultGroups="${DefaultPmodulesConfig['default_groups']}" DefaultReleaseStages="${DefaultPmodulesConfig['default_reltages']}" TmpDir="${DefaultPmodulesConfig['tmp_dir']}" DistfilesDir="${DefaultPmodulesConfig['download_dir']}" get_config "${sys_config_file}" local -r usr_config_file="${HOME}/.Pmodules/Pmodules.yaml" if [[ -r "${usr_config_file}" ]]; then get_config "${usr_config_file}" fi OverlayInfo[none:type]='n' OverlayInfo[none:layout]='flat' } # Local Variables: # mode: sh # sh-basic-offset: 8 # tab-width: 8 # End: