diff --git a/Pmodules/libpbuild.bash b/Pmodules/libpbuild.bash index bccf179..42dd0e6 100644 --- a/Pmodules/libpbuild.bash +++ b/Pmodules/libpbuild.bash @@ -23,7 +23,6 @@ declare -a CONFIGURE_ARGS=() declare -a PATCH_FILES=() declare -a PATCH_STRIPS=() declare -a PATCH_STRIP_DEFAULT='1' -declare -a MODULE_DOCFILES=() declare -- configure_with='auto' #............................................................................. @@ -425,6 +424,10 @@ pbuild.set_urls(){ # Maybe we should use a dictionary in the future. # pbuild::set_sha256sum() { + if [[ ${opt_yaml} == 'yes' ]]; then + std::info \ + "Using ${FUNCNAME} is deprecated with YAML module configuration files." + fi SOURCE_SHA256_SUMS+=("$1") } readonly -f pbuild::set_sha256sum @@ -557,15 +560,25 @@ pbuild::prep() { "${module_name}/${module_version}:" \ "source file '${_result}' is not readable!" - local sha256_sum='' - local hash='' - for hash in "${SOURCE_SHA256_SUMS[@]}"; do - if [[ ${hash} =~ $fname: ]]; then - sha256_sum="${hash#*:}" + local -- sha256_sum='' + if [[ "${opt_yaml}" == 'yes' ]]; then + if [[ -v SHASUMS[${fname}] ]]; then + sha256_sum="${SHASUMS[${fname}]}" fi - done + else + local hash='' + for hash in "${SOURCE_SHA256_SUMS[@]}"; do + if [[ ${hash} =~ $fname: ]]; then + sha256_sum="${hash#*:}" + break + fi + done + fi if [[ -n "${sha256_sum}" ]]; then check_hash_sum "${dir}/${fname}" "${sha256_sum}" + std::info "${module_name}/${module_version}: SHA256 hash sum is OK ..." + else + std::info "${module_name}/${module_version}: SHA256 hash sum missing NOK ..." fi } @@ -648,6 +661,11 @@ readonly -f pbuild::add_configure_args #.............................................................................. # pbuild::use_autotools() { + if [[ ${opt_yaml} == 'yes' ]]; then + std::info \ + "Using ${FUNCNAME} is deprecated with YAML module configuration files." + fi + configure_with='autotools' } readonly -f pbuild::use_autotools @@ -655,6 +673,10 @@ readonly -f pbuild::use_autotools #.............................................................................. # pbuild::use_cmake() { + if [[ ${opt_yaml} == 'yes' ]]; then + std::info \ + "Using ${FUNCNAME} is deprecated with YAML module configuration files." + fi configure_with='cmake' } readonly -f pbuild::use_cmake @@ -689,6 +711,10 @@ readonly -f pbuild::use_cc declare -- compile_in_sourcetree='No' pbuild::compile_in_sourcetree() { + if [[ ${opt_yaml} == 'yes' ]]; then + std::info \ + "Using ${FUNCNAME} is deprecated with YAML module configuration files." + fi compile_in_sourcetree='Yes' } readonly -f pbuild::compile_in_sourcetree @@ -791,6 +817,10 @@ pbuild::compile() { # $@: documentation files relative to source # pbuild::install_docfiles() { + if [[ ${opt_yaml} == 'yes' ]]; then + std::info \ + "Using ${FUNCNAME} is deprecated with YAML module configuration files." + fi MODULE_DOCFILES+=("$@") } readonly -f pbuild::install_docfiles @@ -862,8 +892,17 @@ pbuild.build_module_legacy(){ } readonly -f pbuild.build_module_legacy +declare -n Config +declare -- Systems pbuild.build_module_yaml(){ - _build_module "$@" + local -- module_name="$1" + local -- module_version="$2" + Config=$3 + local -- module_relstage="${Config['relstage']}" + local -- tmp="${Config['systems']//::/, }" + Systems="${tmp:1:-1}" + shift 3 + _build_module "${module_name}" "${module_version}" "${module_relstage}" "$@" } readonly -f pbuild.build_module_yaml @@ -1145,13 +1184,17 @@ _build_module() { #...................................................................... check_supported_systems() { - (( ${#SUPPORTED_SYSTEMS[@]} == 0 )) && return 0 - for sys in "${SUPPORTED_SYSTEMS[@]}"; do - [[ ${sys,,} == ${system,,} ]] && return 0 - done - std::die 1 \ - "%s " "${module_name}/${module_version}:" \ - "Not available for ${system}." + if [[ "${opt_yaml,,}" == 'no' ]]; then + (( ${#SUPPORTED_SYSTEMS[@]} == 0 )) && return 0 + for sys in "${SUPPORTED_SYSTEMS[@]}"; do + [[ ${sys,,} == ${system,,} ]] && return 0 + done + std::die 1 \ + "%s " "${module_name}/${module_version}:" \ + "Not available for ${system}." + else + : debug "Systems: $Systems" + fi } #...................................................................... @@ -1454,26 +1497,71 @@ _build_module() { } install_release_file() { - local -r release_file="${modulefile_dir}/.release-${module_version}" + local -r legacy_config_file="${modulefile_dir}/.release-${module_version}" + local -- status_legay_config_file='unchanged' + local -r yaml_config_file="${modulefile_dir}/.config-${module_version}" + local -- status_yaml_config_file='unchanged' - if [[ -r "${release_file}" ]]; then - local release - read release < "${release_file}" - if [[ "${release}" != "${module_release}" ]]; then + if [[ -r "${legacy_config_file}" ]]; then + local relstage_legacy + read relstage_legacy < "${legacy_config_file}" + if [[ "${relstage_legacy}" != "${module_release}" ]]; then + status_legay_config_file='changed' + fi + else + status_legay_config_file='new' + fi + if [[ "${status_legay_config_file}" != 'unchanged' ]]; then + echo "${module_release}" > "${legacy_config_file}" + fi + + if [[ -r "${yaml_config_file}" ]]; then + while read key value; do + local -n ref="${key:0:-1}" + ref="${value}" + done < "${yaml_config_file}" + if [[ "${relstage}" != "${module_release}" ]]; then + status_yaml_config_file='changed' + fi + else + status_yaml_config_file='new' + fi + if [[ "${status_yaml_config_file}" != 'unchanged' ]]; then + echo "relstage: ${module_release}" > "${yaml_config_file}" + echo "Systems: [${Systems}]" >> "${yaml_config_file}" + fi + + case ${status_yaml_config_file},${status_legay_config_file} in + unchanged,unchanged | new,unchanged) + : + ;; + unchanged,changed ) std::info \ "%s " \ "${module_name}/${module_version}:" \ - "changing release from" \ - "'${release}' to '${module_release}' ..." - echo "${module_release}" > "${release_file}" - fi - else - std::info \ - "%s " \ - "${module_name}/${module_version}:" \ - "setting release to '${module_release}' ..." - echo "${module_release}" > "${release_file}" - fi + "changing release stage from" \ + "'${relstage_legacy}' to '${module_release}' in legacy config file ..." + ;; + unchanged,new ) + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "setting release stage to '${module_release}' in legacy config file ..." + ;; + changed,unchanged | changed,changed | changed,new | new,changed ) + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "changing release stage from" \ + "'${relstage_legacy}' to '${module_release}' ..." + ;; + new,new ) + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "setting release stage to '${module_release}' ..." + ;; + esac } cleanup_build() { @@ -1621,6 +1709,14 @@ _build_module() { "removing release file '${release_file}' ..." [[ "${dry_run}" == 'no' ]] && ${rm} -vf "${release_file}" fi + release_file="${modulefile_dir}/.config-${module_version}" + if [[ -e "${release_file}" ]]; then + std::info \ + "%s " \ + "${module_name}/${module_version}:" \ + "removing release file '${release_file}' ..." + [[ "${dry_run}" == 'no' ]] && ${rm} -vf "${release_file}" + fi ${rmdir} -p "${modulefile_dir}" 2>/dev/null || : } @@ -1679,7 +1775,7 @@ _build_module() { "${module_name}/${module_version}:" \ ${with_modules:+build with ${with_modules[@]}} - if [[ "${module_release}" == 'removed' ]]; then + if [[ "${module_release}" == 'remove' ]]; then remove_module elif [[ "${module_release}" == 'deprecated' ]]; then deprecate_module diff --git a/Pmodules/libstd.bash b/Pmodules/libstd.bash index e9eda18..dbb3a3f 100644 --- a/Pmodules/libstd.bash +++ b/Pmodules/libstd.bash @@ -284,7 +284,7 @@ std::get_os_release() { } std::get_type() { - local -a signature=$(typeset -p "$1") + local -a signature=( $(typeset -p "$1") ) case ${signature[1]} in -Ai* ) echo 'int dict' diff --git a/Pmodules/modbuild.in b/Pmodules/modbuild.in index e9c27f7..0baf4be 100755 --- a/Pmodules/modbuild.in +++ b/Pmodules/modbuild.in @@ -7,6 +7,8 @@ # #............................................................................. +echo "$0 $@" + declare VERSION='@PMODULES_VERSION@' # get absolute path of script @@ -70,7 +72,7 @@ USAGE: MANDATORY ARGUMENTS: version - Variant of module to build. + Version of module to build. SELECT VARIANT TO BUILD: @@ -180,14 +182,32 @@ declare opt_verbose='no' declare -a opt_with_modules=() # :FIXME: legacy build should also use the dict declare -A opt_with_dict=() declare -- opt_config_file='' +declare -- opt_debug='no' declare -- opt_yaml='yes' declare -- opt_variant='' -declare BUILD_SCRIPT='' +declare -- BUILD_SCRIPT='' declare -- yaml_config_file='' declare -a legacy_config_files=() +declare -- module_name='' parse_args() { + # + # The first argument ($1) is the build-script, if called in the + # usual way: + # ./ [options] + # If called via: + # modbuild [options] + # the build-script MUST be passed as first argument. + # + BUILD_SCRIPT=$(std::get_abspath "$1") + test -r ${BUILD_SCRIPT} || \ + std::die 1 "%s " \ + "Build script does not exist" \ + "or is not readable -- '$_'" + BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}") + shift 1 + while (( $# > 0 )); do case $1 in -j ) @@ -200,9 +220,10 @@ parse_args() { -v | --verbose ) trap 'echo "$BASH_COMMAND"' DEBUG opt_verbose='yes' + set -x ;; --debug ) - set -x + opt_debug='yes' ;; -f | --force-rebuild ) opt_force_rebuild='yes' @@ -314,46 +335,25 @@ parse_args() { shift fi ;; - -- ) + -- | '' ) : ;; -* ) std::die 1 "Invalid option -- '$1'" ;; - [=0-9]* | '.*' ) - versions+=( "$1" ) - ;; - '') - : - ;; * ) - [[ -z "${BUILD_SCRIPT}" ]] || \ - std::die 1 "%s "\ - "Build script already set to" \ - "'${BUILD_SCRIPT}' -- '$1'" - BUILD_SCRIPT=$(std::get_abspath "$1") - test -r ${BUILD_SCRIPT} || \ - std::die 1 "%s " \ - "Build script does not exist" \ - "or is not readable -- '$_'" + local -- arg="$1" + if [[ "${arg}" == */* ]]; then + module_name="${arg%/*}" + versions+=( {arg#*/} ) + else + versions+=( "$1" ) + fi ;; esac shift done - # BUILD_SCRIPT is not set if called with `modbuild [options]` - if [[ -z ${BUILD_SCRIPT} ]]; then - if [[ -r "${PWD}/build" ]]; then - if grep -q '#!.* modbuild' "${PWD}/build"; then - BUILD_SCRIPT="${PWD}/build" - fi - fi - if [[ -z ${BUILD_SCRIPT} ]]; then - std::die 1 "Don't know what to build!" - fi - fi - BUILDBLOCK_DIR=$(dirname "${BUILD_SCRIPT}") - # if no version is specified on the cmd-line, build all versions (( ${#versions[@]} > 0)) || versions+=( '.*' ) @@ -511,7 +511,7 @@ get_yaml_file_fmt(){ Get format version of configuration file. Print the version number to stdout if it is valid. " - ${yq} -e '.' "$1" &>- || \ + ${yq} -e '.' "$1" &>/dev/null || \ std::die 3 "%s -- %s" \ "Error in YAML config file, please check with linter" \ "$1" @@ -531,9 +531,9 @@ get_yaml_file_fmt(){ read_yaml_config_file() { : " - Test whether the configuration file passed with $1 provides - configuration for the module passed with $2. If yes, the - configuration for this module is printed. + Test whether the configuration file '$1' provides configurations + for the module '$2'. If yes, the YAML block with the configuration + is printed to stdout. " local -- file_name="$1" local -- module_name="$2" @@ -575,6 +575,9 @@ declare -x V_MINOR='' # second number in version string (or empty) declare -x V_PATCHLVL='' # third number in version string (or empty) declare -x V_RELEASE='' # module release (or empty) +declare -A SHASUMS=() +declare -a MODULE_DOCFILES=() + parse_version() { local v="$1" V="$1" @@ -621,28 +624,34 @@ parse_version() { } declare -A Yaml_valid_keys_for_module=( - ['defaults']=1 - ['shasums']=1 - ['versions']=1 + ['defaults']=1 # !!map + ['shasums']=1 # !!map + ['versions']=1 # !!int ) + declare -A Yaml_default_config=( - ["build_requires"]='' - ["compile_in_sourcetree"]='No' - ["configure_with"]='auto' - ["default_variant"]='' - ["group"]='Tools' - ["group_deps"]='' - ["overlay"]='base' - ["relstage"]='unstable' - ["runtime_deps"]='' - ["script"]='build' - ["suffix"]='' - ["systems"]='' - ["urls"]='' - ["variant"]='' + ["build_requires"]='' # !!seq of strings + ["compile_in_sourcetree"]='No' # !!str + ["configure_with"]='auto' # !!str + ["default_variant"]='' # !!str + ["docfiles"]='' # !!seq of strings + ["group"]='Tools' # !!str + ["group_deps"]='' # !!map + ["overlay"]='base' # !!str + ["relstage"]='unstable' # !!str + ["runtime_deps"]='' # !!seq of strings + ["script"]='build' # !!str + ["suffix"]='' # !!str + ["systems"]='' # !!seq of strings + ["urls"]='' # !!map + ["variant"]='' # !!str ) - +declare -A Yaml_valid_vk_keys=( + ['config']=1 # !!map + ['variants']=1 # !!map +) + build_modules_yaml_v1(){ : " @@ -655,32 +664,35 @@ build_modules_yaml_v1(){ local -a with_modules=( $* ) check_yaml_keys(){ - local -n valid_keys="$1" - local -n used_keys="$2" - used_keys=() + local -n valid_yaml_keys="$1" + local -n used_yaml_keys="$2" + used_yaml_keys=() local -- key='' local -a keys=() - keys=( $(${yq} '.|keys().[]' <<<"${yaml_mod_config}" 2>/dev/null) ) - echo "DEBUG: top-level keys: ${keys[@]}" + keys=( $(${yq} '.|keys().[]' 2>/dev/null) ) + debug "top-level keys: ${keys[@]}" for key in "${keys[@]}"; do - [[ -v valid_keys[${key}] ]] || \ + [[ -v valid_yaml_keys[${key}] ]] || \ std::die 3 "Invalid key in YAML configuration file -- ${key}" - used_keys[${key}]=1 + used_yaml_keys[${key}]=1 done } - local -A used_yaml_keys=() - #check_yaml_keys Yaml_valid_keys_for_module used_yaml_keys <<<"${yaml_mod_config}" - local -a keys=( $(${yq} '.|keys().[]' <<<"${yaml_mod_config}" 2>/dev/null) ) - local -- key='' - echo "DEBUG: top-level keys: ${keys[@]}" - for key in "${keys[@]}"; do - [[ -v Yaml_valid_keys_for_module[${key}] ]] || \ - std::die 3 "Invalid key in YAML configuration file -- ${key}" - used_yaml_keys[${key}]=1 - done - [[ -v used_yaml_keys['versions'] ]] || \ - std::die 3 "No version(s) specified in YAML configuration file." + get_value(){ + local -- yaml_input="$1" + local -n val="$2" + local -- key="$3" + local -- expected_type="$4" + + local -- type='' + type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}") + if [[ "${type}" != "${expected_type}" ]]; then + std::die 3 "%s" \ + "Value of '${key}' must be of type '${expected_type:2}', but is '${type:2}'!" + fi + val=$( ${yq} -e ".${key}" \ + 2>/dev/null <<<"${yaml_input}" ) || val='' + } get_config(){ local -- yaml_input="$1" @@ -700,23 +712,22 @@ build_modules_yaml_v1(){ local -a keys=() keys=( $( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null )) || \ std::die 3 "Oops: retrieving keys from:\n${yaml_input}" - echo "DEBUG: config keys: ${keys[@]}" + debug "config keys: ${keys[@]}" + local -- type; for key in "${keys[@]}"; do - [[ -v dfl[$key] ]] || \ + [[ -v dfl[${key,,}] ]] || \ std::die 3 "%s -- %s\n%s" \ - "Ivalid key in configuration" \ + "Invalid key in configuration" \ "${key}" "${yaml_input}" case ${key} in compile_in_sourcetree ) - value=$( ${yq} -e ".${key}" \ - 2>/dev/null <<<"${yaml_input}" ) || \ - value='' + get_value "${yaml_input}" value "${key}" '!!bool' case ${value,,} in - true | on | yes ) - cfg[${key}]='Yes' + true ) + cfg[${key,,}]='Yes' ;; - false | off | no ) - cfg[${key}]='No' + false ) + cfg[${key,,}]='No' ;; * ) std::die 3 "%s '%s' -- %s" \ @@ -727,12 +738,10 @@ build_modules_yaml_v1(){ esac ;; configure_with ) - value=$( ${yq} ".${key}" \ - 2>/dev/null <<<"${yaml_input}" ) || \ - value='' + get_value "${yaml_input}" value "${key}" '!!str' case ${value,,} in auto | cmake | autotools ) - cfg[${key}]="${value,,}" + cfg[${key,,}]="${value,,}" ;; * ) std::die 3 "%s '%s' -- %s" \ @@ -742,32 +751,52 @@ build_modules_yaml_v1(){ ;; esac ;; - urls | group_deps ) - value=$( ${yq} -e ".${key}" \ - 2>/dev/null <<<"${yaml_input}" ) || \ - value='' - cfg[${key}]="${value}" + relstage ) + get_value "${yaml_input}" value "${key}" '!!str' + case ${value,,} in + unstable | stable | deprecated ) + cfg[${key,,}]="${value,,}" + ;; + remove | removed ) + cfg[${key,,}]='remove' + ;; + * ) + std::die 3 "%s '%s' -- %s" \ + "Invalid value for" \ + 'relstage' \ + "${value}" + ;; + esac ;; - systems | variant ) - value=$( ${yq} -e ".${key}.[]" \ - 2>/dev/null <<<"${yaml_input}" ) || \ - value='' + urls ) + get_value "${yaml_input}" value "${key}" '!!seq' + cfg[${key,,}]="${value}" + ;; + group_deps ) + get_value "${yaml_input}" value "${key}" '!!map' + cfg[${key,,}]="${value}" + ;; + systems ) + get_value "${yaml_input}" value "${key}" '!!seq' + readarray -t tmp <<<"${value}" + cfg[${key,,}]="${tmp[@]}" + ;; + variant ) + get_value "${yaml_input}" value "${key}" '!!seq' readarray -t tmp <<<"${value}" printf -v tmp2 ":%s:" "${tmp[@]}" - cfg[${key}]="${tmp2}" + cfg[${key,,}]="${tmp2}" : ;; - runtime_deps | build_requires ) - value=$( ${yq} -e ".${key}[]" \ - 2>/dev/null <<<"${yaml_input}" ) || \ - value='' - cfg[${key}]="${value}" + docfiles | runtime_deps | build_requires ) + get_value "${yaml_input}" value "${key}" '!!seq' + cfg[${key,,}]="${value}" ;; * ) value=$( ${yq} -e ".${key}" \ 2>/dev/null <<<"${yaml_input}" ) || \ value='' - cfg[${key}]="${value}" + cfg[${key,,}]="${value}" esac done } @@ -788,7 +817,7 @@ build_modules_yaml_v1(){ l=() # loop over semicolon separated list of keys for k in ${key//;/ }; do - # do curly brackets expansion {} + # brace expansion of key local list=() eval list=( $k ) if [[ ${list[@]} =~ ${version} ]]; then @@ -798,7 +827,7 @@ build_modules_yaml_v1(){ done done (( ${#refvar[@]} == 0 )) && \ - std::die 3 "No configuration for version -- $1" + std::die 3 "No configuration for version -- ${version}" return 0 } @@ -858,39 +887,21 @@ build_modules_yaml_v1(){ fi } - get_runtime_deps(){ - local config="$1" - local -n with_modules="$2" - - with_modules=( $(${yq} -e '.runtime_deps.[]' <<<"${config}" 2>/dev/null) ) - } - - get_build_requires(){ - local -- config="$1" - local -n with_modules="$2" - - with_modules=( $(${yq} -e '.build_requires.[]' <<<"${config}" 2>/dev/null) ) - with_modules=( "${with_modules[@]/#/b:}" ) - } - build_modules_compiler(){ local -- module_name="$1" local -- module_version="$2" local -n module_cfg="$3" - shift 3 - local -a module_deps=( "$@" ) - local -a with_compiler=() get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler - echo "${with_compiler[@]}" + debug "${with_compiler[@]}" local compiler='' for compiler in "${with_compiler[@]}"; do pbuild.build_module_yaml \ "${module_name}" "${module_version}" \ - "${module_cfg['relstage']}" \ + "$3" \ "${compiler}" \ - "${module_deps[@]}" + "${@:4}" done } @@ -898,28 +909,25 @@ build_modules_yaml_v1(){ local -- module_name="$1" local -- module_version="$2" local -n module_cfg="$3" - shift 3 - local -a module_deps=( "$@" ) - local -a with_compiler=() local -a with_hdf5=() get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler get_group_deps "${module_cfg['group_deps']}" 'hdf5_serial' with_hdf5 - echo "${with_compiler[@]}" - echo "${with_hdf5[@]}" + debug "${with_compiler[@]}" + debug "${with_hdf5[@]}" local -- compiler local -- hdf5 for compiler in "${with_compiler[@]}"; do for hdf5 in "${with_hdf5[@]}"; do - echo "build $module_name/$module_version with $compiler and $hdf5" - echo " runtime deps: ${runtime_deps[@]}" - echo " build requires: ${build_requires[@]}" + debug "build $module_name/$module_version with $compiler and $hdf5" + debug " runtime deps: ${runtime_deps[@]}" + debug " build requires: ${build_requires[@]}" pbuild.build_module_yaml \ "${module_name}" "${module_version}" \ - "${module_cfg['relstage']}" \ + "$3" \ "${compiler}" \ "${hdf5}" \ - "${module_deps[@]}" + "${@:4}" done done } @@ -929,30 +937,27 @@ build_modules_yaml_v1(){ local -- module_name="$1" local -- module_version="$2" local -n module_cfg="$3" - shift 3 - local -a module_deps=( "$@" ) - local -a with_compiler=() local -a with_mpi=() get_group_deps "${module_cfg['group_deps']}" 'compiler' with_compiler get_group_deps "${module_cfg['group_deps']}" 'mpi' with_mpi - echo "${with_compiler[@]}" - echo "${with_mpi[@]}" + debug "${with_compiler[@]}" + debug "${with_mpi[@]}" local -- compiler local -- mpi for compiler in "${with_compiler[@]}"; do for mpi in "${with_mpi[@]}"; do - echo "build $module_name/$module_version with $compiler and $mpi" - echo " runtime deps: ${runtime_deps[@]}" - echo " build requires: ${build_requires[@]}" + debug "build $module_name/$module_version with $compiler and $mpi" + debug " runtime deps: ${runtime_deps[@]}" + debug " build requires: ${build_requires[@]}" pbuild.build_module_yaml \ "${module_name}" "${module_version}" \ - "${module_cfg['relstage']}" \ + "$3" \ "${compiler}" \ "${mpi}" \ - "${module_deps[@]}" + "${@:4}" done done } @@ -961,9 +966,6 @@ build_modules_yaml_v1(){ local -- module_name="$1" local -- module_version="$2" local -n module_cfg="$3" - shift 3 - local -a module_deps=( "$@" ) - local -a with_compiler=() local -a with_mpi=() local -a with_hdf5=() @@ -971,9 +973,9 @@ build_modules_yaml_v1(){ get_group_deps "${module_cfg['group_deps']}" 'mpi' with_mpi get_group_deps "${module_cfg['group_deps']}" 'hdf5' with_hdf5 - echo "${with_compiler[@]}" - echo "${with_mpi[@]}" - echo "${with_hdf5[@]}" + debug "${with_compiler[@]}" + debug "${with_mpi[@]}" + debug "${with_hdf5[@]}" local -- compiler local -- mpi @@ -981,16 +983,16 @@ build_modules_yaml_v1(){ for compiler in "${with_compiler[@]}"; do for mpi in "${with_mpi[@]}"; do for hdf5 in "${with_hdf5[@]}"; do - echo "build $module_name/$module_version with $compiler, $mpi and $hdf5" - echo " runtime deps: ${runtime_deps[@]}" - echo " build requires: ${build_requires[@]}" + debug "build $module_name/$module_version with $compiler, $mpi and $hdf5" + debug " runtime deps: ${runtime_deps[@]}" + debug " build requires: ${build_requires[@]}" pbuild.build_module_yaml \ "${module_name}" "${module_version}" \ - "${module_cfg['relstage']}" \ + "$3" \ "${compiler}" \ "${mpi}" \ "${hdf5}" \ - "${module_deps[@]}" + "${@:4}" done done done @@ -1001,12 +1003,10 @@ build_modules_yaml_v1(){ local -- module_name="$1" local -- module_version="$2" local -n module_cfg="$3" - shift 3 - local -a module_deps=( "$@" ) pbuild.build_module_yaml \ "${module_name}" "${module_version}" \ - "${module_cfg['relstage']}" \ - "${module_deps[@]}" + "$3" \ + "${@:4}" } set_urls() { @@ -1047,6 +1047,24 @@ build_modules_yaml_v1(){ local -- module_name="$1" local -- module_version="$2" local -n module_config="$3" + + check_system(){ + [[ -z ${module_config['systems']} ]] && return 0 + + set -o noglob + local -a systems=( ${module_config['systems']} ) + set +o noglob + + local -- system + for system in "${systems[@]}"; do + [[ "${opt_system}" == ${system} ]] && return 0 + [[ "${HOSTNAME}" == ${system} ]] && return 0 + done + std::info "Skipping variant '${module_version}', neither OS nor hostname match:" + std::info " This system: ${opt_system}; hostname: ${HOSTNAME}" + std::info " Systems to build on: ${systems[@]}" + return 1 + } P="${module_name}" parse_version "${module_version}" @@ -1055,18 +1073,14 @@ build_modules_yaml_v1(){ # build this variant? if [[ ":${module_config['variant']}:" != *:${build_variant}:* ]]; then - echo "DEBUG: don't build this variant: ${module_config['variant']} != *:${build_variant}:*" + debug "don't build this variant: ${module_config['variant']} != *:${build_variant}:*" return 0 fi # build for this system? - if [[ -n ${module_config['systems']} ]] && \ - [[ ":${module_config['systems']}:" != *:${opt_system}:* ]]; then - echo "DEBUG: don't build for this system: ${module_config['systems']} != *:${opt_system}:*" - return 0 - fi + check_system || return 0 - echo "build variant ${module_name}/${module_version}" + debug "build variant ${module_name}/${module_version}" local ol_name="${module_config['overlay']}" [[ -v OverlayInfo[${ol_name}:inst_root] ]] || \ @@ -1086,14 +1100,19 @@ build_modules_yaml_v1(){ local -a runtime_deps=() if [[ -n ${module_config['runtime_deps']} ]]; then readarray -t runtime_deps <<<"${module_config['runtime_deps']}" + debug "runtime_deps=${runtime_deps[@]} length: ${#runtime_deps[@]}" fi - echo "runtime_deps=${runtime_deps[@]} length: ${#runtime_deps[@]}" + local -a build_requires=() if [[ -n ${module_config['build_requires']} ]]; then readarray -t build_requires <<<"${module_config['build_requires']}" + build_requires=( "${build_requires[@]/#/b:}" ) + debug "build_requires=${build_requires[@]} length: ${#build_requires[@]}" + fi + + if [[ -n ${module_config['docfiles']} ]]; then + readarray -t MODULE_DOCFILES <<<"${module_config['docfiles']}" fi - build_requires=( "${build_requires[@]/#/b:}" ) - echo "build_requires=${build_requires[@]} length: ${#build_requires[@]}" local -A build_functions=( ['Compiler']=build_modules_compiler @@ -1131,42 +1150,47 @@ build_modules_yaml_v1(){ done done } - + + local -A used_keys=() local -- yaml_input='' - if [[ -v used_yaml_keys['defaults'] ]]; then + local -A default_config=() + local -- version_key='' + local -a version_keys=() + local -A shasums=() + + check_yaml_keys Yaml_valid_keys_for_module used_keys <<<"${yaml_mod_config}" + [[ -v used_keys['versions'] ]] || \ + std::die 3 "No version(s) specified in YAML configuration file." + + if [[ -v used_keys['defaults'] ]]; then yaml_input=$(${yq} '.defaults' <<<"${yaml_mod_config}" 2>/dev/null) fi - local -A default_config=() get_config "${yaml_input}" default_config Yaml_default_config - local -a version_keys - get_matching_version_keys version_keys "${version}" <<<"${yaml_mod_config}" - - local -A valid_vk_yaml_keys=( ['config']=1 ['variants']=1 ) + if [[ -v used_keys['shasums'] ]]; then + local yaml_input=$(${yq} '.shasums' <<<"${yaml_mod_config}" 2>/dev/null) + while read key value; do + SHASUMS[${key//:}]="${value}" + done <<<"${yaml_input}" + fi - local -- version_key='' + + get_matching_version_keys version_keys "${version}" <<<"${yaml_mod_config}" for version_key in "${version_keys[@]}"; do local yaml_vk_config=$(get_yaml_vk_config "${version_key}" <<<"${yaml_mod_config}") # check keys: allowed are 'config' and 'variants' - local -A used_vk_yaml_keys=() - #check_yaml_keys valid_vk_yaml_keys used_vk_yaml_keys <<<"${yaml_vk_config}" - local -a keys=( $(${yq} '.|keys().[]' <<<"${yaml_vk_config}" 2>/dev/null ) ) - local -- key='' - echo "DEBUG: "vk keys: ${keys[@]} - for key in "${keys[@]}"; do - [[ -v valid_vk_yaml_keys[${key}] ]] || \ - std::die 3 "Invalid key in YAML configuration file -- ${key}" - used_vk_yaml_keys[${key}]=1 - done + used_keys=() + check_yaml_keys Yaml_valid_vk_keys used_keys <<<"${yaml_vk_config}" # read config if set local -A vk_config=() yaml_input='' - if [[ -v used_vk_yaml_keys['config'] ]]; then - yaml_input=$(${yq} '.config.[]' <<<"${yaml_vk_config}" 2>/dev/null) + if [[ -v used_keys['config'] ]]; then + yaml_input=$(${yq} '.config' <<<"${yaml_vk_config}" 2>/dev/null) + debug "vk input: ${yaml_input}" fi - echo "DEBUG: vk input: ${yaml_input}" + # reminder: if YAML input is empty, next line copies defaults to 'vk_config' get_config "${yaml_input}" vk_config default_config local -- yaml_variants=$(get_variants <<<"${yaml_vk_config}") @@ -1175,7 +1199,7 @@ build_modules_yaml_v1(){ expand_version_key versions "${version_key}" "${version}" local v='' for v in "${versions[@]}"; do - echo "DEBUG: version: $v" + debug "version: $v" if (( num_variants == 0 )); then build_modules_variant \ "${name}" "${v}" \ @@ -1210,6 +1234,15 @@ build_modules() { parse_args "$@" +if [[ "${opt_debug,,}" == 'no' ]]; then + debug(){ + : + } +else + debug(){ + echo "DEBUG: " "$@" + } +fi pbuild.jobs "${opt_jobs}" pbuild.force_rebuild "${opt_force_rebuild}" @@ -1238,9 +1271,11 @@ export PMODULES_DISTFILESDIR PMODULES_TMPDIR declare -r BUILD_SCRIPT declare -r BUILDBLOCK_DIR -# the module name is defined by the directory the build script is in -IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}" -module_name=${fname[${#fname[@]}-2]} +if [[ -z ${module_name} ]]; then + # the module name is defined by the directory the build script is in + IFS=/ read -r -a fname <<< "${BUILD_SCRIPT:1}" + module_name=${fname[${#fname[@]}-2]} +fi for version in "${versions[@]}"; do build_modules "${module_name}" "${version}" "${opt_with_modules[@]}" diff --git a/Pmodules/modulecmd.bash.in b/Pmodules/modulecmd.bash.in index 0aeaf8a..04cac2a 100644 --- a/Pmodules/modulecmd.bash.in +++ b/Pmodules/modulecmd.bash.in @@ -187,7 +187,7 @@ die_illegal_group(){ "${CMD}" "${subcommand}" "invalid group name" "$1" } -die_illegal_rel_stage(){ +die_illegal_relstage(){ std::die 3 "%s %s: %s -- %s\n" \ "${CMD}" "${subcommand}" "invalid release stage" "$1" } @@ -273,12 +273,6 @@ die_removing_collection_failed(){ "cannot remove collection" "$1" } -die_is_blocked(){ - std::die 3 "%s %s: %s -- %s" \ - "${CMD}" "${subcommand}" \ - "module is blocked on this system" "$1" -} - get_module_config(){ : " Read module configuration. @@ -305,19 +299,21 @@ get_module_config(){ local -n ref_cfg="$1" local -r dir="$2" local -r modulefile="${dir}/$3" + debug "modulefile: $modulefile" - ref_cfg['rel_stage']='unstable' + ref_cfg['relstage']='unstable' ref_cfg['systems']='' ref_cfg['blocklist']='' if [[ ! -v Dir2OverlayMap[${dir%/${PMODULES_MODULEFILES_DIR}*}] ]]; then # this module is not in an overlay - ref_cfg['rel_stage']='stable' + ref_cfg['relstage']='stable' return fi local -r config_file="${modulefile%/*}/.config-${modulefile##*/}" - local -r rel_stage_file="${modulefile%/*}/.release-${modulefile##*/}" + local -r relstage_file="${modulefile%/*}/.release-${modulefile##*/}" if [[ -r ${config_file} ]]; then - local -- yaml=$(${yq} -e '.') + local -- yaml=$(${yq} -e '.' < "${config_file}") + debug "yaml: ${yaml}" local -- key='' for key in ${!ref_cfg[@]}; do case "${key,,}" in @@ -333,14 +329,14 @@ get_module_config(){ ;; esac if [[ "${value}" != 'null' ]]; then - ref_cfg[${key}]="${value}" + ref_cfg[${key,,}]="${value}" fi done return 0 - elif [[ -r ${rel_stage_file} ]]; then - ref_cfg['rel_stage']=$( < "${rel_stage_file}" ) + elif [[ -r ${relstage_file} ]]; then + ref_cfg['relstage']=$( < "${relstage_file}" ) else - ref_cfg['rel_stage']='unstable' + ref_cfg['relstage']='unstable' fi } @@ -352,17 +348,39 @@ is_available(){ - the blocklist is empty or the hostname is NOT in the list " local -n ref_cfg="$1" - local -- rel_stages="$2" + local -- relstages="$2" local -- system="$3" local -- hostname="$4" - [[ ":${rel_stages}:" =~ ":${ref_cfg['rel_stage']}:" ]] || return 1 - if [[ "${ref_cfg['systems']}" != "" ]]; then - [[ "${ref_cfg['systems']//$'\n'/:}:" =~ "${system}" ]] || return 1 - fi - if [[ "${ref_cfg['blocklist']}" != "" ]]; then - [[ "${ref_cfg['blocklist']//$'\n'/:}:" =~ "${hostname}" ]] && return 1 - fi + check_relstage(){ + [[ ":${relstages}:" =~ ":${ref_cfg['relstage']}:" ]] + } + check_blocklist(){ + [[ -z ${ref_cfg['blocklist']} ]] && return 0 + local -- s='' + for s in ${ref_cfg['blocklist']}; do + if [[ "${system}" == $s ]] || [[ "${HOSTNAME}" == $s ]]; then + return 0 + fi + done + return 1 + } + check_systems(){ + [[ -z ${ref_cfg['systems']} ]] && return 0 + local -- s='' + for s in ${ref_cfg['systems']}; do + if [[ "${system}" == $s ]] || [[ "${HOSTNAME}" == $s ]]; then + return 0 + fi + done + return 1 + } + set -o noglob + check_relstage && check_blocklist && check_systems + local -i ec=$? + set +o noglob + return ${ec} + } # @@ -557,7 +575,7 @@ USAGE: ' subcommand_load() { - local rel_stage='undef' + local relstage='undef' local current_modulefile='' local prefix='' local m='' @@ -573,13 +591,13 @@ subcommand_load() { # none get_load_hints() { local -n output="$1" - local rel_stage='' + local relstage='' output='' while read -a line; do [[ -z ${line} ]] && continue - rel_stage=${line[1]} - if [[ ! ":${UsedReleaseStages}:" =~ "${rel_stage}" ]]; then - output+="module use ${rel_stage}; " + relstage=${line[1]} + if [[ ! ":${UsedReleaseStages}:" =~ "${relstage}" ]]; then + output+="module use ${relstage}; " fi local group=${line[2]} if [[ ! ":${UsedGroups}:" =~ ":${group}:" ]] && \ @@ -656,30 +674,6 @@ subcommand_load() { g_env_must_be_saved='yes' } - is_blocked() { - : " - test whether a module is blocked or not. - - args: - $1: prefix of module - - exit codes: - 0: if blocked - 1: otherwise - " - local _prefix="$1" - test -e "${_prefix}/.blocklist" || return 1 # not blocked! - local _pattern='//' - while read line; do - _pattern+=' && !'"/${line}/" - done < "${_prefix}/.blocklist" - local out=$(awk "${_pattern}"' {print $0}' <<< "${HOSTNAME}") - [[ -z "${out}" ]] || return 1 # not blocked! - [[ -r "${_prefix}/.blockmsg" ]] || return 0 - cat "${_prefix}/.blockmsg" 1>&2 - return 0 - } - #...................................................................... local args=() local opts=() @@ -723,23 +717,23 @@ subcommand_load() { # extendet module name is either # - group:name or - # - group:name:rel_stage or - # - rel_stage:name or - # - rel_stage:group:name or - # - name:rel_stage + # - group:name:relstage or + # - relstage:name or + # - relstage:group:name or + # - name:stage IFS=':' local -a toks=($m) unset IFS local group='' - local rel_stage='' + local relstage='' if is_group "${toks[0]}"; then group=${toks[0]} m=${toks[1]} - rel_stage=${toks[2]} + relstage=${toks[2]} elif is_release_stage "${toks[0]}"; then - rel_stage=${toks[0]} + relstage=${toks[0]} if is_group "${toks[1]}"; then group=${toks[1]} m=${toks[2]} @@ -751,9 +745,9 @@ subcommand_load() { m=${toks[0]} if is_group "${toks[1]}"; then group=${toks[1]} - rel_stage=${toks[2]} + relstage=${toks[2]} else - rel_stage=${toks[1]} + relstage=${toks[1]} group=${toks[2]} fi fi @@ -769,20 +763,20 @@ subcommand_load() { modulepath=( "${mod_root}/${group}" "${modulepath[@]}" ) done fi - if [[ -n ${rel_stage} ]]; then - is_release_stage "${rel_stage}" || \ - die_illegal_rel_stage "${rel_stage}" - std::append_path UsedReleaseStages "${rel_stage}" + if [[ -n ${relstage} ]]; then + is_release_stage "${relstage}" || \ + die_illegal_relstage "${relstage}" + std::append_path UsedReleaseStages "${relstage}" g_env_must_be_saved='yes' fi fi # handle extended module names local moduledir='' - find_modulefile current_modulefile rel_stage moduledir "${m}" "${modulepath[@]}" + find_modulefile current_modulefile relstage moduledir "${m}" "${modulepath[@]}" if [[ -z ${current_modulefile} ]]; then local fname=$(std::get_abspath "${m}") if [[ -r "${fname}" ]]; then current_modulefile="${fname}" - rel_stage='stable' + relstage='stable' else local hints='' get_load_hints hints @@ -803,7 +797,6 @@ subcommand_load() { die_not_a_modulefile "${m}" local prefix='' get_module_prefix prefix "${current_modulefile}" - is_blocked "${prefix}" && die_is_blocked "${m}" if [[ -n ${prefix} ]]; then test -r "${prefix}/.info" && cat "$_" 1>&2 test -r "${prefix}/.dependencies" && load_dependencies "$_" @@ -846,17 +839,17 @@ subcommand_load() { fi local msg='' if [[ ${verbosity_lvl} != silent ]] && \ - [[ ${rel_stage} != stable ]]; then + [[ ${relstage} != stable ]]; then msg=$(printf "%s %s: %s -- %s" \ "${CMD}" 'load' \ - "${rel_stage} module has been loaded" \ + "${relstage} module has been loaded" \ "${m}") std::info "%s" "${msg}" fi msg=$(printf "%s: %s %s %s" \ 'load' \ "modulefile=${current_modulefile}" \ - "rel-stage=${rel_stage}" \ + "rel-stage=${relstage}" \ "user=${USER}") ${logger} -t Pmodules "${msg}" done @@ -1072,14 +1065,14 @@ subcommand_show() { # $4... module path (fully qualified directory names) # # return list like -# modulename_1 rel_stage_1 modulefile_1 ... +# modulename_1 relstage_1 modulefile_1 ... # get_available_modules() { local -n gam_mods="$1" local -r module="$2" - local -r used_rel_stages="${3:-${UsedReleaseStages}}" + local -r used_relstages="${3:-${UsedReleaseStages}}" shift 3 # in the for loop below we use $@ to loop over module path - local rel_stage + local relstage local -A dict local -A modulenames @@ -1133,9 +1126,9 @@ get_available_modules() { [[ "${add}" == 'no' ]] && continue local -A cfg=() get_module_config cfg "${dir}" "${mod}" - is_available cfg "${used_rel_stages}" "${os_release}" "${HOSTNAME}" || continue + is_available cfg "${used_relstages}" "${os_release}" "${HOSTNAME}" || continue - gam_mods+=( "${mod}" "${cfg['rel_stage']}" "${dir}/${mod}" "${ol}" ) + gam_mods+=( "${mod}" "${cfg['relstage']}" "${dir}/${mod}" "${ol}" ) dict[${sdirs}/${mod}]=1 done < <(${find} -L "${dir_entries[@]}" \ \( -type f -o -type l \) \ @@ -1164,7 +1157,7 @@ get_available_modules() { # find_modulefile() { local -n fm_modulefile="$1" - local -n fm_rel_stage="$2" + local -n fm_relstage="$2" local -n fm_dir="$3" local -r module="$4" local -a dirs=("${@:5}") @@ -1194,7 +1187,7 @@ find_modulefile() { is_available cfg "${ReleaseStages}" "${os_release}" "${HOSTNAME}" || continue fm_modulefile="${dir}/${mod}" - fm_rel_stage="${cfg['rel_stage']}" + fm_relstage="${cfg['relstage']}" fm_dir="${dir}" return 0 done @@ -1224,7 +1217,7 @@ find_modulefile() { get_module_config cfg "${dir}" "${mod}" is_available cfg "${ReleaseStages}" "${os_release}" "${HOSTNAME}" || continue fm_modulefile="${dir}/${mod}" - fm_rel_stage="${cfg['rel_stage']}" + fm_relstage="${cfg['relstage']}" fm_dir="${dir}" return 0 done @@ -1296,13 +1289,13 @@ subcommand_avail() { local -i i=0 for (( i=0; i<${#mods[@]}; i+=4 )); do local mod=${mods[i]} - local rel_stage=${mods[i+1]} - case ${rel_stage} in + local relstage=${mods[i+1]} + case ${relstage} in stable ) out='' ;; * ) - out="${rel_stage}" + out="${relstage}" ;; esac printf "%-20s\t%s\n" "${mod}" "${out}" 1>&2 @@ -1323,13 +1316,13 @@ subcommand_avail() { output_header "$1" for (( i=0; i<${#mods[@]}; i+=4 )); do local mod=${mods[i]} - local rel_stage=${mods[i+1]} - case ${rel_stage} in + local relstage=${mods[i+1]} + case ${relstage} in stable ) out='' ;; * ) - out=${rel_stage} + out=${relstage} ;; esac printf "%-20s\t%s\n" "${mod}" "${out}" 1>&2 @@ -1346,13 +1339,13 @@ subcommand_avail() { local -i max_length=1 for ((i=0; i<${#mods[@]}; i+=4)); do if [[ ${verbosity_lvl} == 'verbose' ]]; then - local rel_stage=${mods[i+1]} - case ${rel_stage} in + local relstage=${mods[i+1]} + case ${relstage} in stable ) mod="${mods[i]}" ;; * ) - mod="${mods[i]}(${rel_stage:0:1})" + mod="${mods[i]}(${relstage:0:1})" ;; esac else @@ -1387,7 +1380,7 @@ subcommand_avail() { #...................................................................... local pattern=() local output_function='human_readable_output' - local opt_use_rel_stages="${UsedReleaseStages}" + local opt_use_relstages="${UsedReleaseStages}" local -A opt_groups=() local val='' while (($# > 0)); do @@ -1396,7 +1389,7 @@ subcommand_avail() { print_help "${subcommand}" ;; -a | --all | --all-release-stages ) - opt_use_rel_stages="${ReleaseStages}" + opt_use_relstages="${ReleaseStages}" ;; -h | --human ) output_function='human_readable_output' @@ -1484,7 +1477,7 @@ subcommand_avail() { get_available_modules \ mods \ "${string}*" \ - "${opt_use_rel_stages}" \ + "${opt_use_relstages}" \ "${path[@]}" [[ ${#mods[@]} == 0 ]] && continue @@ -2471,17 +2464,17 @@ subcommand_search() { # get and print all available modules in $mpath # with respect to the requested release stage - # tmpfile: module/version rel_stage group dependencies... + # tmpfile: module/version relstage group dependencies... local mods get_available_modules \ mods \ "${module}" \ - "${opt_use_rel_stages}" \ + "${opt_use_relstages}" \ "${modulepath[@]}" local i=0 for (( i=0; i<${#mods[@]}; i+=4 )); do local name=${mods[i]} - local rel_stage=${mods[i+1]} + local relstage=${mods[i+1]} local modulefile=${mods[i+2]} local ol=${mods[i+3]} @@ -2511,7 +2504,7 @@ subcommand_search() { unset IFS fi - echo ${name} ${rel_stage} ${group} ${modulefile} \ + echo ${name} ${relstage} ${group} ${modulefile} \ ${ol} \ ${deps[@]} >> "${tmpfile}" done @@ -2548,7 +2541,7 @@ subcommand_search() { "${CMD}" 'search' \ "illegal release stage" \ "${arg}" - opt_use_rel_stages+="${arg}:" + opt_use_relstages+="${arg}:" ;; --with | --with=* ) if [[ "$1" == --with ]]; then @@ -2570,7 +2563,7 @@ subcommand_search() { done ;; -a | --all-releases-stages ) - opt_use_rel_stages+="${ReleaseStages}" + opt_use_relstages+="${ReleaseStages}" ;; --src | --src=*) if [[ "$1" == --src ]]; then @@ -2632,8 +2625,8 @@ subcommand_search() { done fi - if [[ "${opt_use_rel_stages}" == ":" ]]; then - opt_use_rel_stages=":${UsedReleaseStages}:" + if [[ "${opt_use_relstages}" == ":" ]]; then + opt_use_relstages=":${UsedReleaseStages}:" fi if [[ ${#modules[@]} == 0 ]]; then @@ -3399,6 +3392,10 @@ case "$1" in esac shift +debug(){ + : +} + # parse agruments till and including the sub-command declare -a opts=() while (( $# > 0 )); do @@ -3410,6 +3407,11 @@ while (( $# > 0 )); do print_help 'version' ;; --debug ) + debug(){ + echo "DEBUG: " "$@" 1>&2 + } + ;; + --verbose ) set -x ;; '' ) diff --git a/Pmodules/yq.x86_64_Linux b/Pmodules/yq.x86_64_Linux index 9e2a58f..1d884df 100755 Binary files a/Pmodules/yq.x86_64_Linux and b/Pmodules/yq.x86_64_Linux differ