pmodules/Pmodules
1
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-06-30 11:19:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

build-system: use ref variable to pass YAML input

Browse Source
This commit is contained in:
gsell
2024-08-30 11:12:50 +02:00
parent 98f7d6b06e
commit b046a0c029
1 changed files with 209 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pmodules/modbuild.in
+209 -147
View File
@@ -327,7 +327,7 @@ parse_args() {
# set config file # set config file
yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}" yaml_config_file="${opt_config_file:-${BUILDBLOCK_DIR}/files/config.yaml}"
[[ -r "${yaml_config_file}" ]] || \ [[ -f "${yaml_config_file}" && -r "${yaml_config_file}" ]] || \
std::die 2 \ std::die 2 \
"%s -- %s" \ "%s -- %s" \
"YAML config file doesn't exist or is not readable" \ "YAML config file doesn't exist or is not readable" \
@@ -340,22 +340,24 @@ get_yaml_file_fmt(){
Get format version of configuration file. Print the version number Get format version of configuration file. Print the version number
to stdout if it is valid. to stdout if it is valid.
" "
${yq} -e '.' "$1" &>/dev/null || \ local -n result="$1"
local -r fname="$2"
${yq} -e '.' "${fname}" &>/dev/null || \
std::die 3 "%s -- %s" \ std::die 3 "%s -- %s" \
"Error in YAML config file, please check with linter" \ "Error in YAML config file, please check with linter" \
"$1" "${fname}"
local -- fmt='' local -- fmt=''
fmt=$(${yq} -e ".format" "$1") || \ result=$(${yq} -e ".format" "${fname}") || \
std::die 3 "Error reading config file format -- $1" std::die 3 "Error reading config file format -- ${fname}"
case "${fmt}" in case "${result}" in
1 ) 1 )
: :
;; ;;
* ) * )
std::die 3 "Unknown YAML Pmodules config file format -- $1" std::die 3 "Unknown YAML Pmodules config file format -- ${fname}"
;; ;;
esac esac
echo "${fmt}"
} }
read_yaml_config_file() { read_yaml_config_file() {
@@ -364,12 +366,11 @@ read_yaml_config_file() {
for the module '$2'. If yes, the YAML block with the configuration for the module '$2'. If yes, the YAML block with the configuration
is printed to stdout. is printed to stdout.
" "
local -- file_name="$1" local -n result="$1"
local -- module_name="$2" local -- file_name="$2"
local -- yaml='' local -- module_name="$3"
yaml=$( ${yq} -Ne e ".${module_name}" "${file_name}" 2>/dev/null ) || \ result=$( ${yq} -Ne e ".${module_name}" "${file_name}" 2>/dev/null ) || \
std::die 3 "Configuration for '${module_name}' missing -- ${file_name}" std::die 3 "Configuration for '${module_name}' missing -- ${file_name}"
echo "${yaml}"
} }
build_modules(){ build_modules(){
@@ -388,14 +389,19 @@ build_modules(){
yamllint "${yaml_config_file}" yamllint "${yaml_config_file}"
fi fi
local -- file_fmt='' local -- file_fmt=''
file_fmt=$(get_yaml_file_fmt "${yaml_config_file}") get_yaml_file_fmt \
file_fmt \
"${yaml_config_file}"
local -- yaml_mod_config='' local -- module_config=''
yaml_mod_config=$(read_yaml_config_file "${yaml_config_file}" "${name}") read_yaml_config_file \
yaml_module_config \
"${yaml_config_file}" \
"${name}"
case "${file_fmt}" in case "${file_fmt}" in
1 ) 1 )
build_modules_yaml_v1 \ build_modules_yaml_v1 \
"${yaml_mod_config}" \ "${yaml_module_config}" \
"${name}" "${version}" \ "${name}" "${version}" \
"${with_modules[@]}" "${with_modules[@]}"
;; ;;
@@ -573,7 +579,7 @@ build_modules_yaml_v1(){
: " : "
" "
local -- yaml_mod_config="$1" local -- yaml_module_config="$1"
local -- name="$2" local -- name="$2"
local -- version="$3" local -- version="$3"
shift 3 shift 3
@@ -602,13 +608,14 @@ build_modules_yaml_v1(){
"invalid type of variants block: must be '!!seq' but is '$3'!" "invalid type of variants block: must be '!!seq' but is '$3'!"
} }
check_yaml_keys(){ yml::check_keys(){
local -n valid_yaml_keys="$1" local -n yaml_input="$1"
local -n used_yaml_keys="$2" local -n valid_yaml_keys="$2"
local -n used_yaml_keys="$3"
used_yaml_keys=() used_yaml_keys=()
local -- key='' local -- key=''
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} '.|keys().[]' 2>/dev/null) readarray -t keys < <( ${yq} -e '.|keys|.[]' 2>/dev/null <<<"${yaml_input}")
debug "top-level keys: ${keys[*]}" debug "top-level keys: ${keys[*]}"
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
[[ -v valid_yaml_keys[${key}] ]] || \ [[ -v valid_yaml_keys[${key}] ]] || \
@@ -617,8 +624,8 @@ build_modules_yaml_v1(){
done done
} }
get_config(){ yml::get_config(){
local -- yaml_input="$1" local -n yaml_input="$1"
local -n cfg="$2" # ref. to return configuration local -n cfg="$2" # ref. to return configuration
local -n dfl="$3" # ref. to defaults local -n dfl="$3" # ref. to defaults
local -- key='' local -- key=''
@@ -757,18 +764,19 @@ build_modules_yaml_v1(){
done done
} }
get_matching_version_keys(){ yml::get_matching_version_keys(){
: " #
return list of version keys in refvar $1 matching version specified in $2. # return list of versions matching a specific version.
" #
local -n refvar="$1" local -n yaml_input="$1" # [in] YAML input
local version="$2" local -n result="$2" # [out] list of versions
local -- version="$3" # [in] version to match
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} -e '.versions|keys().[]' 2>/dev/null ) || \ readarray -t keys < <( ${yq} -e '.versions|keys().[]' 2>/dev/null <<<"${yaml_input}" ) || \
std::die 3 "No version keys in configuration file!" std::die 3 "No version keys in configuration file!"
refvar=() result=()
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
l=() l=()
# loop over semicolon separated list of keys # loop over semicolon separated list of keys
@@ -777,73 +785,72 @@ build_modules_yaml_v1(){
local list=() local list=()
list=( $(${bash} -c "echo $k") ) list=( $(${bash} -c "echo $k") )
if [[ ${list[@]} =~ ${version} ]]; then if [[ ${list[@]} =~ ${version} ]]; then
refvar+=("${key}") result+=("${key}")
break break
fi fi
done done
done done
(( ${#refvar[@]} == 0 )) && \ (( ${#result[@]} == 0 )) && \
std::die 3 "No configuration for version -- ${version}" std::die 3 "No configuration for version -- ${version}"
return 0 return 0
} }
get_yaml_vk_config(){ yml::get_version_block(){
: " #
Get configuration for version key $1. # Get configuration for specific version.
#
Please note: this can be the empty string. # Please note: this can be an empty string.
" #
local -- key="$1" local -n yaml_input="$1" # [in] YAML input
result=$( ${yq} ".versions.\"${key}\"" 2>/dev/null ) || \ local -n result="$2" # [out] result in YAML format
local -- version="$3" # [in] return config for this version
result=$( ${yq} -e ".versions.\"${version}\"" 2>/dev/null <<<"${yaml_input}" ) || \
result="" result=""
echo "${result}"
} }
get_variants(){ yml::get_variants(){
local yaml #
if yaml=$(${yq} -e '.variants' 2>/dev/null); then # get variants of a specific version
local -- type_of_key='' #
type_of_key=$( ${yq} -e ".variants | type" 2>/dev/null <<<"${yaml}") local -n yaml_input="$1" # [in] YAML input with the variants
if [[ "${type_of_key}" != 'seq' ]]; then local -n result="$2" # [out] variants in YAML format
die_invalid_variants_block "${name}" "${version}" \ local -n n="$3" # [out] number of variants
"${type_of_key}"
fi local -- type_of_key=''
else type_of_key=$( ${yq} -e ".variants | type" 2>/dev/null <<<"${yaml_input}")
yaml='' if [[ "${type_of_key}" != '!!seq' ]]; then
die_invalid_variants_block "${name}" "${version}" \
"${type_of_key}"
fi fi
echo "${yaml}" result=$(${yq} -e '.variants' 2>/dev/null <<<"${yaml_input}") || \
result=''
n=$(${yq} '.|length' 2>/dev/null <<<"${result}")
} }
get_num_variants(){ yml::get_nth_variant(){
local -i n=0 local -n yaml_input="$1" # [in] YAML input
n=$(${yq} '.|length' 2>/dev/null) local -n result="$2" # [out] nth variant in YAML format
echo "$n" local -i n="$3" # [in] index of variant to return
result=$(${yq} -e ".[$n]" 2>/dev/null <<<"${yaml_input}") || result=''
} }
get_variant(){ yml::chk_group_deps(){
local n="$1" #
local yaml # Check the group dependencies:
yaml=$(${yq} -e ".[$n]" 2>/dev/null) # 1. are all keys valid?
[[ "${yaml}" == 'null' ]] && _result='' # 2. all required group deps defined?
echo "${yaml}" # 3. more group deps defined as required?
} #
# Die if check fails.
chk_group_deps(){ #
: " local -- yaml_input="$1" # [in] value of key group_deps
Check the group dependencies: local -- group="$2" # [in] compiler|mpi|hdf5|hdf5_serial
1. are all keys valid? local -- name="$3" # [in] module name
2. all required group deps defined? local -- version="$4" # [in] module version
3. more group deps defined as required?
"
:
local -- yaml="$1" # yaml formatted string: value of group_deps
local -- group="$2" # compiler|mpi|hdf5|hdf5_serial
local -- name="$3" # module name
local -- version="$4" # module version
# query all specified group dependencies # query all specified group dependencies
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} ".|keys|.[]" <<<"${yaml}" 2>/dev/null ) readarray -t keys < <( ${yq} ".|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local -- key='' local -- key=''
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
@@ -861,18 +868,18 @@ build_modules_yaml_v1(){
done done
} }
get_group_deps(){ yml::get_group_deps(){
local -- yaml="$1" # yaml formatted string with group config local -- yaml_input="$1" # [in] value of key group_deps
local -- group="$2" # compiler|mpi|hdf5|hdf5_serial local -- group="$2" # [in] compiler|mpi|hdf5|hdf5_serial
local -n with_modules="$3" # refvar to return a list of modules local -n with_modules="$3" # [out] list of required modules
local -a modules=() local -a modules=()
local keys=() local keys=()
readarray -t keys < <( ${yq} ".${group,,}|keys|.[]" <<<"${yaml}" 2>/dev/null ) readarray -t keys < <( ${yq} ".${group,,}|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local key local key
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
local versions=() local versions=()
readarray -t versions < <( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml}" 2>/dev/null ) readarray -t versions < <( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml_input}" 2>/dev/null )
local version local version
for version in "${versions[@]}"; do for version in "${versions[@]}"; do
if [[ -v opt_with_dict[${key}/${version}] ]]; then if [[ -v opt_with_dict[${key}/${version}] ]]; then
@@ -902,27 +909,34 @@ build_modules_yaml_v1(){
return 0 return 0
} }
: " #
To compile a module with a certain compiler||mpi||hdf5 dependency # To compile a module with a certain compiler||mpi||hdf5 dependency
the '--with' option can be used. Depending on the hierarchical # the '--with' option can be used. Depending on the hierarchical
group the modules specified with the option '--with' must be a # group the modules specified with the option '--with' must be a
subset of # subset of
- compiler: ( compiler) # - compiler: ( compiler)
- mpi: ( compiler mpi ) # - mpi: ( compiler mpi )
- hdf5: ( compiler mpi hdf5 ) # - hdf5: ( compiler mpi hdf5 )
- hdf5_serial: ( compiler hdf5_serial ) # - hdf5_serial: ( compiler hdf5_serial )
" #
build_modules_compiler(){ build_modules_compiler(){
local -- module_name="$1" #
local -- module_version="$2" # build a module in hierarchical group 'Compiler'
local -n module_cfg="$3" #
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
chk_group_deps "${module_cfg['group_deps']}" 'Compiler' \ yml::chk_group_deps \
"${module_name}" "${module_version}" "${module_cfg['group_deps']}" \
'Compiler' \
"${module_name}" "${module_version}"
local -a with_compiler=() local -a with_compiler=()
get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler yml::get_group_deps \
"${module_cfg['group_deps']}" \
'Compiler' with_compiler
debug "${with_compiler[@]}" debug "${with_compiler[@]}"
local compiler='' local compiler=''
@@ -941,18 +955,23 @@ build_modules_yaml_v1(){
} }
build_modules_hdf5_serial(){ build_modules_hdf5_serial(){
local -- module_name="$1" #
local -- module_version="$2" # build a module in hierarchical group 'HDF5_serial'
local -n module_cfg="$3" #
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
chk_group_deps "${module_cfg['group_deps']}" 'HDF5_serial' \ yml::chk_group_deps \
"${module_name}" "${module_version}" "${module_cfg['group_deps']}" \
'HDF5_serial' \
"${module_name}" "${module_version}"
local -a with_compiler=() local -a with_compiler=()
get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_hdf5=() local -a with_hdf5=()
get_group_deps "${module_cfg['group_deps']}" 'HDF5_serial' with_hdf5 yml::get_group_deps "${module_cfg['group_deps']}" 'HDF5_serial' with_hdf5
local -- compiler local -- compiler
local -- hdf5 local -- hdf5
@@ -978,17 +997,23 @@ build_modules_yaml_v1(){
build_modules_mpi(){ build_modules_mpi(){
local -- module_name="$1" #
local -- module_version="$2" # build a module in hierarchical group 'MPI'
local -n module_cfg="$3" #
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
chk_group_deps "${module_cfg['group_deps']}" 'MPI' "${module_name}" "${module_version}" yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'MPI' \
"${module_name}" "${module_version}"
local -a with_compiler=() local -a with_compiler=()
get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_mpi=() local -a with_mpi=()
get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi yml::get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi
local -- compiler local -- compiler
local -- mpi local -- mpi
@@ -1013,20 +1038,26 @@ build_modules_yaml_v1(){
} }
build_modules_hdf5(){ build_modules_hdf5(){
local -- module_name="$1" #
local -- module_version="$2" # build a module in hierarchical group 'HDF5'
local -n module_cfg="$3" #
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
chk_group_deps "${module_cfg['group_deps']}" 'HDF5' "${module_name}" "${module_version}" yml::chk_group_deps \
"${module_cfg['group_deps']}" \
'HDF5' \
"${module_name}" "${module_version}"
local -a with_compiler=() local -a with_compiler=()
get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler yml::get_group_deps "${module_cfg['group_deps']}" 'Compiler' with_compiler
local -a with_mpi=() local -a with_mpi=()
get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi yml::get_group_deps "${module_cfg['group_deps']}" 'MPI' with_mpi
local -a with_hdf5=() local -a with_hdf5=()
get_group_deps "${module_cfg['group_deps']}" 'HDF5' with_hdf5 yml::get_group_deps "${module_cfg['group_deps']}" 'HDF5' with_hdf5
local -- compiler local -- compiler
local -- mpi local -- mpi
@@ -1058,9 +1089,13 @@ build_modules_yaml_v1(){
} }
build_modules_other(){ build_modules_other(){
local -- module_name="$1" #
local -- module_version="$2" # build a module in a non-hierarchical group
local -n module_cfg="$3" #
local -- module_name="$1" # [in] module name
local -- module_version="$2" # [in] module version
local -n module_cfg="$3" # [in] ref to module config
pbuild.build_module_yaml \ pbuild.build_module_yaml \
"${module_name}" "${module_version}" \ "${module_name}" "${module_version}" \
"$3" \ "$3" \
@@ -1374,32 +1409,38 @@ build_modules_yaml_v1(){
} }
local -A used_keys=() local -A used_keys=()
local -- yaml_input=''
local -A default_config=() local -A default_config=()
local -- version_key='' local -- version_key=''
local -a version_keys=() local -a version_keys=()
local -A shasums=() local -A shasums=()
check_yaml_keys Yaml_valid_keys_for_module used_keys <<<"${yaml_mod_config}" yml::check_keys \
yaml_module_config \
Yaml_valid_keys_for_module \
used_keys
[[ -v used_keys['versions'] ]] || \ [[ -v used_keys['versions'] ]] || \
std::die 3 "No version(s) specified in YAML configuration file." std::die 3 "No version(s) specified in YAML configuration file."
local -- yaml_default_config=''
if [[ -v used_keys['defaults'] ]]; then if [[ -v used_keys['defaults'] ]]; then
yaml_input=$(${yq} '.defaults' <<<"${yaml_mod_config}" 2>/dev/null) yaml_default_config=$(${yq} '.defaults' <<<"${yaml_module_config}" 2>/dev/null)
fi fi
get_config "${yaml_input}" default_config Yaml_default_config yml::get_config \
yaml_default_config \
default_config \
Yaml_default_config
if [[ -v used_keys['shasums'] ]]; then if [[ -v used_keys['shasums'] ]]; then
local yaml_input='' local yaml_shasums=''
yaml_input=$(${yq} '.shasums' <<<"${yaml_mod_config}" 2>/dev/null) yaml_shasums=$(${yq} '.shasums' <<<"${yaml_module_config}" 2>/dev/null)
while read -r key value; do while read -r key value; do
[[ -z ${key} ]] && continue [[ -z ${key} ]] && continue
SHASUMS[${key//:}]="${value}" SHASUMS[${key//:}]="${value}"
done <<<"${yaml_input}" done <<<"${yaml_shasums}"
fi fi
if [[ -v used_keys['type'] ]]; then if [[ -v used_keys['type'] ]]; then
local -- value='' local -- value=''
pm::get_value "${yaml_mod_config}" value 'type' '!!str' pm::get_value "${yaml_module_config}" value 'type' '!!str'
case "${value,,}" in case "${value,,}" in
'module' ) 'module' )
[[ "${module_type}" == 'sub_package' ]] && \ [[ "${module_type}" == 'sub_package' ]] && \
@@ -1414,29 +1455,44 @@ build_modules_yaml_v1(){
;; ;;
esac esac
fi fi
get_matching_version_keys version_keys "${version}" <<<"${yaml_mod_config}" yml::get_matching_version_keys \
yaml_module_config \
version_keys \
"${version}"
for version_key in "${version_keys[@]}"; do for version_key in "${version_keys[@]}"; do
local yaml_vk_config='' local -- yaml_version_block=''
yaml_vk_config=$(get_yaml_vk_config "${version_key}" <<<"${yaml_mod_config}") yml::get_version_block \
yaml_module_config \
yaml_version_block \
"${version_key}"
# check keys: allowed are 'config' and 'variants' # check keys: allowed are 'config' and 'variants'
used_keys=() used_keys=()
check_yaml_keys Yaml_valid_vk_keys used_keys <<<"${yaml_vk_config}" yml::check_keys \
yaml_version_block \
Yaml_valid_vk_keys \
used_keys
# read config if set # read (default) config of version if set
local -A vk_config=() local -A vk_config=()
yaml_input='' local -- yaml_version_config=''
if [[ -v used_keys['config'] ]]; then if [[ -v used_keys['config'] ]]; then
yaml_input=$(${yq} '.config' <<<"${yaml_vk_config}" 2>/dev/null) yaml_version_config=$(${yq} '.config' <<<"${yaml_vk_config}" 2>/dev/null)
debug "vk input: ${yaml_input}" debug "vk input: ${yaml_version_config}"
fi fi
# reminder: if YAML input is empty, next line copies defaults to 'vk_config' # reminder: if YAML input is empty, next line copies defaults to 'vk_config'
get_config "${yaml_input}" vk_config default_config yml::get_config \
yaml_version_config \
vk_config \
default_config
local -- yaml_variants='' local -- yaml_variants=''
yaml_variants=$(get_variants <<<"${yaml_vk_config}")
local -i num_variants=0 local -i num_variants=0
num_variants=$(get_num_variants <<<"${yaml_variants}") yml::get_variants \
yaml_version_block \
yaml_variants \
num_variants
local versions=() local versions=()
expand_version_key versions "${version_key}" "${version}" expand_version_key versions "${version_key}" "${version}"
local v='' local v=''
@@ -1448,12 +1504,18 @@ build_modules_yaml_v1(){
vk_config vk_config
else else
local -i n=0 local -i n=0
local -- yaml_variant='' local -- yaml_variant_config=''
for ((n=0; n<num_variants; n++)); do for ((n=0; n<num_variants; n++)); do
yaml_variant=$(get_variant "${n}" <<<"${yaml_variants}") yml::get_nth_variant \
yaml_variants \
yaml_variant_config \
"${n}"
local -A mod_config=() local -A mod_config=()
get_config "${yaml_variant}" mod_config vk_config yml::get_config \
yaml_variant_config \
mod_config \
vk_config
build_modules_variant \ build_modules_variant \
"${name}" "${v}" \ "${name}" "${v}" \
mod_config mod_config