pmodules/Pmodules
2
0
Fork 0
mirror of https://github.com/Pmodules/Pmodules.git synced 2026-08-06 19:32:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch '374-build-system-support-for-yam-anchors' into 'master'

Browse Source 
Resolve "build-system: support for YAM anchors"

Closes #374

See merge request Pmodules/src!387
This commit is contained in:
gsell
2024-11-28 14:02:30 +01:00
parent 2bd6e76b33 ac33def277
commit 2a475efbcd
2 changed files with 350 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pmodules/libpmodules.bash.in
+116 -72
View File
@@ -82,41 +82,101 @@ declare -A OverlayConfigKeys=(
['modulepath']='' ['modulepath']=''
) )
pm::get_value(){ yml::die_parsing(){
local -- yaml_input="$1" std::die 3 "error parsing YAML:\n----\n$1\n----"
local -n val="$2" }
local -- key="$3"
local -- expected_type="$4" yml::die_type_error(){
std::die 3 "%s" \
"Value of '$1' must be of type '$2', but is '$3'!"
}
yml::die_invalid_key(){
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"$1" "$2"
}
yml::die_read_file(){
std::die 3 "Cannot read file '$1'. Please check with yamllint!"
}
yml::read_file(){
local -n yml_content="$1"
local -- yml_fname="$2"
local -- yml_node="$3"
yml_content=$( ${yq} -Ne e "${yml_node}|explode(.)" "${yml_fname}" 2>/dev/null ) || \
yml::die_read_file "${yml_fname}"
}
yml::get_keys(){
local -n yml_keys="$1"
local -n yml_input="$2"
local -- yml_node="$3"
local -- str=''
str="$( ${yq} -e "${yml_node}|keys|.[]" 2>/dev/null <<<"${yml_input}")" || \
{ yml_keys=(); return 0; };
readarray -t yml_keys <<<"${str}"
}
yml::get_type(){
local -n yml_type="$1"
local -n yml_input="$2"
local -- yml_node="$3"
yml_type="$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}")" || \
yml::die_parsing "${yml_input}"
}
yml::get_value(){
local -n yml_val="$1"
local -n yml_input="$2"
local -- yml_node="$3"
local -- yml_expected_type="$4"
local -- type='' local -- type=''
type=$( ${yq} -e ".${key} | type" 2>/dev/null <<<"${yaml_input}") type="$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}")" || \
if [[ "${type}" != "${expected_type}" ]]; then yml::die_parsing "${yml_input}"
std::die 3 "%s" \ [[ "${type}" == "${yml_expected_type}" ]] || \
"Value of '${key}' must be of type '${expected_type:2}', but is '${type:2}'!" yml::die_type_error "${yml_node}" "${yml_expected_type:2}" "${type:2}"
fi yml_val=$( ${yq} -e "${yml_node}" 2>/dev/null <<<"${yml_input}" ) || \
val=$( ${yq} -e ".${key}" \ return 1
2>/dev/null <<<"${yaml_input}" ) || val=''
} }
pm::get_seq(){
local -- yaml_input="$1" yml::get_seq_length(){
local -n val="$2"
local -- key="$3" local -n yml_seq_length="$1" # [out] number of variants
local -n yml_input="$2" # [in] YAML input
local -- yml_node="$3" # [in] node
yml_seq_length=$(${yq} -e "${yml_node}|length" 2>/dev/null <<<"${yml_input}") || \
yml::die_parsing "${yml_input}"
}
yml::get_seq(){
local -n yml_val="$1"
local -n yml_input="$2"
local -- yml_node="$3"
local -- type='' local -- type=''
type=$( ${yq} ".${key} | type" 2>/dev/null <<<"${yaml_input}") type=$( ${yq} -e "${yml_node}|type" 2>/dev/null <<<"${yml_input}")
if [[ "${type:2}" == 'null' ]]; then if [[ "${type:2}" == 'null' ]]; then
val='' yml_val=''
return 0 return 0
fi fi
if [[ "${type}" != '!!seq' ]]; then [[ "${type}" == '!!seq' ]] || \
std::die 3 "%s" \ yml::die_type_error "${yml_node}" 'seq' "${type:2}"
"Value of '${key}' must be of type 'seq', but is of type '${type:2}'!" local -i length=0
length=$(${yq} -e "${yml_node}|length" 2>/dev/null <<<"${yml_input}")
if (( length == 0 )); then
yml_val=''
return 0
fi fi
val=$( ${yq} -e ".${key}[]" \ yml_val=$( ${yq} -e "${yml_node}[]" 2>/dev/null <<<"${yml_input}" ) || \
2>/dev/null <<<"${yaml_input}" ) || val='' return 1
} }
pm::read_config(){ pm::read_config(){
: " : "
Read Pmodules configuration files '${PMODULES_ROOT}/config/Pmodules.yaml' Read Pmodules configuration files '${PMODULES_ROOT}/config/Pmodules.yaml'
@@ -135,9 +195,6 @@ pm::read_config(){
" "
local -r yaml_input="$1" local -r yaml_input="$1"
local -r ol_name="$2" local -r ol_name="$2"
local -- key=''
local -a keys=()
local -- value=''
Overlays+=( "${ol_name}" ) Overlays+=( "${ol_name}" )
# init overlay with defaults # init overlay with defaults
@@ -145,16 +202,15 @@ pm::read_config(){
OverlayInfo[${ol_name}:${key}]="${OverlayConfigKeys[${key}]}" OverlayInfo[${ol_name}:${key}]="${OverlayConfigKeys[${key}]}"
done done
# get keys in YAML input # get keys in YAML input
readarray -t keys < <( ${yq} -e ".|keys|.[]" <<<"${yaml_input}" 2>/dev/null ) || \ local -- node=".Overlays.${ol_name}"
std::die 3 "Oops: retrieving keys from:\n${yaml_input}" local -- key=''
local -a keys=()
yml::get_keys keys yaml_input "${node}"
local -- value=''
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
[[ -v OverlayConfigKeys[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key,,} in case ${key,,} in
install_root ) install_root )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value value yaml_input "${node}.${key}" '!!str'
OverlayInfo[${ol_name}:install_root]=$(${envsubst} <<< "${value}") OverlayInfo[${ol_name}:install_root]=$(${envsubst} <<< "${value}")
mkdir -p "${OverlayInfo[${ol_name}:install_root]}" 2>/dev/null mkdir -p "${OverlayInfo[${ol_name}:install_root]}" 2>/dev/null
[[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \ [[ -d ${OverlayInfo[${ol_name}:install_root]} ]] || \
@@ -162,7 +218,7 @@ pm::read_config(){
"Invalid installation root directory for overlay '${ol_name}' -- ${value}" "Invalid installation root directory for overlay '${ol_name}' -- ${value}"
;; ;;
modulefiles_root ) modulefiles_root )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value value yaml_input "${node}.${key}" '!!str'
OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}") OverlayInfo[${ol_name}:modulefiles_root]=$(${envsubst} <<< "${value}")
mkdir -p "${OverlayInfo[${ol_name}:modulefiles_root]}" 2>/dev/null mkdir -p "${OverlayInfo[${ol_name}:modulefiles_root]}" 2>/dev/null
[[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \ [[ -d ${OverlayInfo[${ol_name}:modulefiles_root]} ]] || \
@@ -170,7 +226,7 @@ pm::read_config(){
"Invalid modulefiles root directory for overlay '${ol_name}' -- ${value}" "Invalid modulefiles root directory for overlay '${ol_name}' -- ${value}"
;; ;;
type ) type )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value value yaml_input "${node}.${key}" '!!str'
case ${value} in case ${value} in
"${ol_normal}" | "${ol_replacing}" | "${ol_hiding}" ) "${ol_normal}" | "${ol_replacing}" | "${ol_hiding}" )
: :
@@ -182,7 +238,7 @@ pm::read_config(){
OverlayInfo[${ol_name}:type]="${value}" OverlayInfo[${ol_name}:type]="${value}"
;; ;;
excludes ) excludes )
pm::get_seq "${yaml_input}" value "${key}" '!!seq' yml::get_seq value yaml_input "${node}.${key}" '!!seq'
local -a tmp_array=() local -a tmp_array=()
readarray -t tmp_array <<<${value} readarray -t tmp_array <<<${value}
local excludes='' local excludes=''
@@ -190,13 +246,19 @@ pm::read_config(){
OverlayInfo[${ol_name}:excludes]=$(${envsubst} <<<"${excludes%:}" ) OverlayInfo[${ol_name}:excludes]=$(${envsubst} <<<"${excludes%:}" )
;; ;;
modulepath ) modulepath )
pm::get_seq "${yaml_input}" value "${key}" '!!seq' yml::get_seq value yaml_input "${node}.${key}" '!!seq'
local -a tmp_array=() local -a tmp_array=()
readarray -t tmp_array <<<${value} readarray -t tmp_array <<<${value}
local modulepath='' local modulepath=''
printf -v modulepath "%s:" "${tmp_array[@]}" printf -v modulepath "%s:" "${tmp_array[@]}"
OverlayInfo[${ol_name}:modulepath]=$(${envsubst} <<< "${modulepath%:}") OverlayInfo[${ol_name}:modulepath]=$(${envsubst} <<< "${modulepath%:}")
;; ;;
* )
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
;;
esac esac
done done
OverlayInfo[${ol_name}:used]='no' OverlayInfo[${ol_name}:used]='no'
@@ -207,25 +269,6 @@ pm::read_config(){
Dir2OverlayMap[${modulefiles_root}]="${ol_name}" Dir2OverlayMap[${modulefiles_root}]="${ol_name}"
} }
get_config_of_overlays(){
: "
Get configuration of overlays in YAML input.
Args:
$1 YAML input
"
local -r yaml_input="$1"
local -- overlay=''
local -a overlays=()
readarray -t overlays < <( ${yq} -e '.|keys|.[]' <<<"${yaml_input}" )
for overlay in "${overlays[@]}"; do
local yaml_tmp
pm::get_value "${yaml_input}" yaml_tmp "${overlay}" '!!map'
get_config_of_overlay "${yaml_tmp}" "${overlay}"
done
}
get_config(){ get_config(){
: " : "
Get Pmodules configuration. Get Pmodules configuration.
@@ -235,34 +278,35 @@ pm::read_config(){
" "
local -r config_file="$1" local -r config_file="$1"
local -- yaml_input='' local -- yaml_input=''
yaml_input=$( ${yq} -Ne e "." "${config_file}" 2>/dev/null ) || \ yml::read_file yaml_input "${config_file}" '.'
std::die 3 "Cannot read configuration file '${config_file}'!"
local -- key=''
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null ) || \ yml::get_keys keys yaml_input '.'
std::die 3 "Oops: retrieving keys from:\n${yaml_input}"
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
[[ -v DefaultPmodulesConfig[${key,,}] ]] || \
std::die 3 "%s -- %s\n%s" \
"Invalid key in configuration" \
"${key}" "${yaml_input}"
case ${key,,} in case ${key,,} in
defaultgroups | default_groups ) defaultgroups | default_groups )
pm::get_value "${yaml_input}" DefaultGroups "${key}" '!!str' yml::get_value DefaultGroups yaml_input ".${key}" '!!str'
;; ;;
defaultreleasestages | default_reltages ) defaultreleasestages | default_reltages )
pm::get_value "${yaml_input}" DefaultReleaseStages "${key}" '!!str' yml::get_value DefaultReleaseStages yaml_input ".${key}" '!!str'
;; ;;
tmpdir | tmp_dir ) tmpdir | tmp_dir )
pm::get_value "${yaml_input}" TmpDir "${key}" '!!str' yml::get_value TmpDir yaml_input ".${key}" '!!str'
;; ;;
distfilesdir | download_dir ) distfilesdir | download_dir )
pm::get_value "${yaml_input}" DistfilesDir "${key}" '!!str' yml::get_value DistfilesDir yaml_input ".${key}" '!!str'
;; ;;
overlays ) overlays )
local tmp_str='' local -- overlay=''
pm::get_value "${yaml_input}" tmp_str "${key}" '!!map' local -a overlays=()
get_config_of_overlays "${tmp_str}" yml::get_keys overlays yaml_input ".${key}"
for overlay in "${overlays[@]}"; do
get_config_of_overlay "${yaml_input}" "${overlay}"
done
;;
* )
yml::die_invalid_key "${key}" "${yaml_input}"
;; ;;
esac esac
done done
Pmodules/modbuild.in
+234 -261
View File
@@ -336,38 +336,21 @@ yml::get_file_fmt(){
Get format version of configuration file. Print the version number Get format version of configuration file. Print the version number
to stdout if it is valid. to stdout if it is valid.
" "
local -n result="$1" local -n yml_fmt="$1"
local -r fname="$2" local -n yml_content="$2"
${yq} -e '.' "${fname}" &>/dev/null || \ yml::get_value yml_fmt yml_content '.format' '!!int' || \
std::die 3 "%s -- %s" \
"Error in YAML config file, please check with linter" \
"${fname}"
local -- fmt=''
result=$(${yq} -e ".format" "${fname}") || \
std::die 3 "Error reading config file format -- ${fname}" std::die 3 "Error reading config file format -- ${fname}"
case "${result}" in case "${yml_fmt}" in
1 ) 1 )
: :
;; ;;
* ) * )
std::die 3 "Unknown YAML Pmodules config file format -- ${fname}" std::die 3 "Unsupported YAML Pmodules config file format -- ${yml_fmt}"
;; ;;
esac esac
} }
yml::read_config_file() {
#
# Test whether the configuration file '$1' provides configurations
# for the module '$2'.
#
local -n result="$1" # [out] return configuration for module
local -- file_name="$2" # [in] name of configuration file
local -- module_name="$3" # [in] module name (without version)
result=$( ${yq} -Ne e ".${module_name}" "${file_name}" 2>/dev/null ) || \
std::die 3 "Configuration for '${module_name}' missing -- ${file_name}"
}
build_modules(){ build_modules(){
local -- name="$1" local -- name="$1"
local -- version="$2" local -- version="$2"
@@ -383,18 +366,18 @@ build_modules(){
std::die 3 "yamllint not found" std::die 3 "yamllint not found"
yamllint "${yaml_config_file}" yamllint "${yaml_config_file}"
fi fi
local -- file_fmt=''
yml::get_file_fmt \ local -- yaml_config=''
file_fmt \ local -i file_fmt=0
"${yaml_config_file}" yml::read_file yaml_config "${yaml_config_file}" '.'
yml::get_file_fmt file_fmt yaml_config
echo "file_fmt=${file_fmt}"
local -- yaml_module_config='' local -- yaml_module_config=''
yml::read_config_file \
yaml_module_config \
"${yaml_config_file}" \
"${name}"
case "${file_fmt}" in case "${file_fmt}" in
1 ) 1 )
yml::get_value yaml_module_config yaml_config ".${name}" '!!map' || \
std::die 3 "Configuration for '${name}' missing in config file -- ${file_name}"
build_modules_yaml_v1 \ build_modules_yaml_v1 \
"${yaml_module_config}" \ "${yaml_module_config}" \
"${name}" "${version}" \ "${name}" "${version}" \
@@ -589,10 +572,6 @@ build_modules_yaml_v1(){
local -a with_modules=( "$@" ) local -a with_modules=( "$@" )
die_parsing(){
std::die 3 "error parsing YAML:\n----\n$1\n----"
}
die_missing_group_dep(){ die_missing_group_dep(){
std::die 3 "%s/%s: %s" \ std::die 3 "%s/%s: %s" \
"${1}" "${2}" \ "${1}" "${2}" \
@@ -645,47 +624,33 @@ build_modules_yaml_v1(){
std::die 3 "Invalid module type -- '$1'!" std::die 3 "Invalid module type -- '$1'!"
} }
yml::check_keys(){
local -n yaml_input="$1"
local -n valid_yaml_keys="$2"
local -n used_yaml_keys="$3"
used_yaml_keys=()
local -- key=''
local -a keys=()
readarray -t keys < <( ${yq} -e '.|keys|.[]' 2>/dev/null <<<"${yaml_input}")
debug "top-level keys: ${keys[*]}"
for key in "${keys[@]}"; do
[[ -v valid_yaml_keys[${key}] ]] || \
die_invalid_key \
"${yaml_input}" \
'configuration file' \
"${key}"
used_yaml_keys[${key}]=1
done
}
yml::get_config(){ yml::get_config(){
local -n yaml_input="$1" local -n yaml_input="$1" # [in] YAML formated string
local -n cfg="$2" # ref. to return configuration local -- node="$2" # [in] config node
local -n dfl="$3" # ref. to defaults local -n cfg="$3" # [in/out] module configuration
local -- key='' local -- key=''
local -- value='' local -- value=''
get_build_functions(){ get_build_functions(){
local -n yaml_in="$1"
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" \ yml::get_keys \
<<<"${yaml_in}" \ keys \
2>/dev/null ) || \ yaml_input \
die_error_reading_keys "${yaml_input}" "${node}.build_functions"
local -- key='' local -- key=''
local -- _val='' local -- value=''
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
case ${key} in case ${key} in
prep | configure | compile | install) prep | configure | compile | install)
pm::get_seq "${yaml_in}" _val "${key}" yml::get_seq \
debug "${key} functions: ${_val}" value \
cfg[target_funcs:${key}]="${_val}" yaml_input \
"${node}.build_functions.${key}" || \
yml::die_parsing "${yaml_input}"
debug "${key} functions: ${value}"
cfg[target_funcs:${key}]="${value}"
;; ;;
* ) * )
die_invalid_key \ die_invalid_key \
@@ -697,31 +662,24 @@ build_modules_yaml_v1(){
done done
} }
for key in "${!dfl[@]}"; do
cfg[${key}]="${dfl[${key}]}"
done
if [[ -z "${yaml_input}" ]]; then if [[ -z "${yaml_input}" ]]; then
return 0 return 0
fi fi
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" <<<"${yaml_input}" 2>/dev/null ) || \ yml::get_keys keys yaml_input "${node}"
die_error_reading_keys "${yaml_input}"
debug "config keys: ${keys[*]}" debug "config keys: ${keys[*]}"
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
[[ -v dfl[${key,,}] ]] || \
die_invalid_key \
"${yaml_input}" \
'configuration' \
"${key}"
case ${key} in case ${key} in
build_functions ) build_functions )
pm::get_value "${yaml_input}" value "${key}" '!!map'
get_build_functions value get_build_functions value
;; ;;
build_variants ) build_variants )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!str' || \
yml::die_parsing "${yaml_input}"
case ${value,,} in case ${value,,} in
all ) : ;; all ) : ;;
first_match ) : ;; first_match ) : ;;
@@ -736,7 +694,11 @@ build_modules_yaml_v1(){
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
compile_in_sourcetree ) compile_in_sourcetree )
pm::get_value "${yaml_input}" value "${key}" '!!bool' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!bool' || \
yml::die_parsing "${yaml_input}"
case ${value,,} in case ${value,,} in
true ) true )
cfg[${key,,}]='yes' cfg[${key,,}]='yes'
@@ -754,7 +716,11 @@ build_modules_yaml_v1(){
esac esac
;; ;;
configure_with ) configure_with )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!str' || \
yml::die_parsing "${yaml_input}"
case ${value,,} in case ${value,,} in
auto | cmake | autotools ) auto | cmake | autotools )
cfg[${key,,}]="${value,,}" cfg[${key,,}]="${value,,}"
@@ -769,21 +735,37 @@ build_modules_yaml_v1(){
esac esac
;; ;;
default_variant | download_dir | group | modulefile | overlay | script | suffix ) default_variant | download_dir | group | modulefile | overlay | script | suffix )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!str' || \
yml::die_parsing "${yaml_input}"
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
group_deps ) group_deps )
pm::get_value "${yaml_input}" value "${key}" '!!map' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!map' || \
yml::die_parsing "${yaml_input}"
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
use_flags ) use_flags )
pm::get_seq "${yaml_input}" value "${key}" yml::get_seq \
value \
yaml_input \
"${node}.${key}" || \
yml::die_parsing "${yaml_input}"
local -a tmp=() local -a tmp=()
readarray -t tmp <<<"${value}" readarray -t tmp <<<"${value}"
cfg[${key,,}]=" ${tmp[*]} " cfg[${key,,}]=" ${tmp[*]} "
;; ;;
relstage ) relstage )
pm::get_value "${yaml_input}" value "${key}" '!!str' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!str' || \
yml::die_parsing "${yaml_input}"
case ${value,,} in case ${value,,} in
unstable | stable | deprecated ) unstable | stable | deprecated )
cfg[${key,,}]="${value,,}" cfg[${key,,}]="${value,,}"
@@ -801,15 +783,27 @@ build_modules_yaml_v1(){
esac esac
;; ;;
urls | sub_packages ) urls | sub_packages )
pm::get_value "${yaml_input}" value "${key}" '!!seq' yml::get_value \
value \
yaml_input \
"${node}.${key}" '!!seq' || \
yml::die_parsing "${yaml_input}"
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
build_requires|configure_args|docfiles|patch_files|runtime_deps|systems|use_overlays|variant ) build_requires|configure_args|docfiles|patch_files|runtime_deps|systems|use_overlays|variant )
pm::get_seq "${yaml_input}" value "${key}" yml::get_seq \
value \
yaml_input \
"${node}.${key}" || \
yml::die_parsing "${yaml_input}"
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
kernels ) kernels )
pm::get_seq "${yaml_input}" value "${key}" yml::get_seq \
value \
yaml_input \
"${node}.${key}" || \
yml::die_parsing "${yaml_input}"
set -o noglob set -o noglob
local -a items=( "${value,,}" ) local -a items=( "${value,,}" )
set +o noglob set +o noglob
@@ -825,7 +819,11 @@ build_modules_yaml_v1(){
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
target_cpus ) target_cpus )
pm::get_seq "${yaml_input}" value "${key}" yml::get_seq \
value \
yaml_input \
"${node}.${key}" || \
yml::die_parsing "${yaml_input}"
set -o noglob set -o noglob
local -a items=( "${value,,}" ) local -a items=( "${value,,}" )
set +o noglob set +o noglob
@@ -834,14 +832,18 @@ build_modules_yaml_v1(){
[[ -v TargetCPUs[${item}] ]] || \ [[ -v TargetCPUs[${item}] ]] || \
die_invalid_value \ die_invalid_value \
"${yaml_input}" \ "${yaml_input}" \
'config section' 'config section' \
'CPU' \ 'CPU' \
"${item}" "${item}"
done done
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
;; ;;
'configure_args+' | 'docfiles+' | 'patch_files+' ) 'configure_args+' | 'docfiles+' | 'patch_files+' )
pm::get_seq "${yaml_input}" value "${key}" yml::get_seq \
value \
yaml_input \
"${node}.${key}" || \
yml::die_parsing "${yaml_input}"
key="${key:0:-1}" key="${key:0:-1}"
if [[ -z "${cfg[${key,,}]}" ]]; then if [[ -z "${cfg[${key,,}]}" ]]; then
cfg[${key,,}]="${value}" cfg[${key,,}]="${value}"
@@ -851,10 +853,11 @@ build_modules_yaml_v1(){
;; ;;
* ) * )
std::die 3 "%s '%s' in %s" \ die_invalid_key \
"Oops unhandled key" \ "${yaml_input}" \
"${key}" \ 'configuration' \
"${FUNCNAME[0]}" "${key}"
;;
esac esac
done done
} }
@@ -868,9 +871,7 @@ build_modules_yaml_v1(){
local -- version="$3" # [in] version to match local -- version="$3" # [in] version to match
local -a keys=() local -a keys=()
readarray -t keys < <( ${yq} -e '.versions|keys().[]' 2>/dev/null <<<"${yaml_input}" ) || \ yml::get_keys keys yaml_input '.versions'
std::die 3 "No version keys in configuration file!"
result=() result=()
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
l=() l=()
@@ -890,50 +891,6 @@ build_modules_yaml_v1(){
return 0 return 0
} }
yml::get_version_block(){
#
# Get configuration for specific version.
#
# Please note: this can be an empty string.
#
local -n yaml_input="$1" # [in] YAML input
local -n result="$2" # [out] result in YAML format
local -- version="$3" # [in] return config for this version
result=$( ${yq} -e ".versions.\"${version}\"" 2>/dev/null <<<"${yaml_input}" ) || \
result=""
}
yml::get_variants(){
#
# get variants of a specific version
#
local -n yaml_input="$1" # [in] YAML input with the variants
local -n result="$2" # [out] variants in YAML format
local -n n="$3" # [out] number of variants
local -- type_of_key=''
type_of_key=$( ${yq} -e ".variants | type" 2>/dev/null <<<"${yaml_input}")
if [[ "${type_of_key}" == '!!null' ]]; then
result=''
n=0
return 0
fi
if [[ "${type_of_key}" != '!!seq' ]]; then
die_invalid_variants_block "${name}" "${version}" \
"${type_of_key}"
fi
result=$(${yq} -e '.variants' 2>/dev/null <<<"${yaml_input}") || \
result=''
n=$(${yq} '.|length' 2>/dev/null <<<"${result}")
}
yml::get_nth_variant(){
local -n yaml_input="$1" # [in] YAML input
local -n result="$2" # [out] nth variant in YAML format
local -i n="$3" # [in] index of variant to return
result=$(${yq} -e ".[$n]" 2>/dev/null <<<"${yaml_input}") || result=''
}
yml::chk_group_deps(){ yml::chk_group_deps(){
# #
# Check the group dependencies: # Check the group dependencies:
@@ -949,10 +906,9 @@ build_modules_yaml_v1(){
local -- version="$4" # [in] module version local -- version="$4" # [in] module version
# query all specified group dependencies # query all specified group dependencies
local -a keys=()
readarray -t keys < <( ${yq} ".|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local -- key='' local -- key=''
local -a keys=()
yml::get_keys keys yaml_input '.'
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
# is this a name of a hierarchical group? # is this a name of a hierarchical group?
[[ -v hierarchical_groups[${key}] ]] || \ [[ -v hierarchical_groups[${key}] ]] || \
@@ -974,13 +930,14 @@ build_modules_yaml_v1(){
local -n with_modules="$3" # [out] list of required modules local -n with_modules="$3" # [out] list of required modules
local -a modules=() local -a modules=()
local keys=()
readarray -t keys < <( ${yq} ".${group,,}|keys|.[]" <<<"${yaml_input}" 2>/dev/null )
local key local key
local keys=()
yml::get_keys keys yaml_input ".${group,,}"
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
local versions=()
readarray -t versions < <( ${yq} -e ".${group,,}.${key}[]" <<<"${yaml_input}" 2>/dev/null )
local version local version
local versions=()
yml::get_seq versions yaml_input ".${group,,}.${key}" || \
yml::die_parsing "${yaml_input}"
for version in "${versions[@]}"; do for version in "${versions[@]}"; do
if [[ -v opt_with_dict[${key}/${version}] ]]; then if [[ -v opt_with_dict[${key}/${version}] ]]; then
with_modules+=( "${key}/${version}" ) with_modules+=( "${key}/${version}" )
@@ -1205,8 +1162,7 @@ build_modules_yaml_v1(){
set_urls() { set_urls() {
local -- yaml="$1" local -- yaml="$1"
local -i l=0 local -i l=0
l=$( ${yq} -Ne e '.|length' <<<"${yaml}" 2>/dev/null) || \ yml::get_seq_length l yaml .
die_parsing "{yaml}"
local -i i=0 local -i i=0
local -- url_yaml='' local -- url_yaml=''
for ((i=0; i<l; i++)); do for ((i=0; i<l; i++)); do
@@ -1285,53 +1241,62 @@ build_modules_yaml_v1(){
} }
build_sub_packages(){ build_sub_packages(){
local -- yaml="$1" local -- yaml="$1"
echo "sub_packages: ${yaml}" 1>&2
local -i l=0 local -i l=0
l=$( ${yq} -Ne e '.|length' <<<"${yaml}" 2>/dev/null) || \ yml::get_seq_length l yaml .
die_parsing "${yaml}"
if (( l == 0 )); then if (( l == 0 )); then
debug "No sub-packages to built" debug "No sub-packages to built"
fi fi
local -i i=0 local -i i=0
local -- fname='' local -- fname=''
local -- pkgs_yaml=''
for ((i=0; i<l; i++)); do for ((i=0; i<l; i++)); do
pkgs_yaml=$(${yq} -Ne e ".[$i]" <<<"${yaml}" 2>/dev/null) || \ local -- node=".[$i]"
die_parsing "${yaml}"
local -- pkg_name='' local -- pkg_name=''
local -- pkg_version='' local -- pkg_version=''
local -a pkg_build_args=() local -a pkg_build_args=()
local -a keys=()
readarray -t keys < <( ${yq} -e ".|keys().[]" \
<<<"${pkgs_yaml}" 2>/dev/null ) || \
die_parsing "${pkgs_yaml}"
local -- key='' local -- key=''
local -a keys=()
yml::get_keys keys yaml "${node}"
for key in "${keys[@]}"; do for key in "${keys[@]}"; do
case ${key,,} in case ${key,,} in
'name' ) 'name' )
pm::get_value "${pkgs_yaml}" \ yml::get_value \
pkg_name "${key}" '!!str' pkg_name \
yaml \
"${node}.${key}" \
'!!str' || \
yml::die_parsing "${yaml}"
;; ;;
'version' ) 'version' )
pm::get_value "${pkgs_yaml}" \ yml::get_value \
pkg_version "${key}" '!!str' pkg_version \
yaml \
"${node}.${key}" \
'!!str' || \
yml::die_parsing "${yaml}"
;; ;;
'build_args' ) 'build_args' )
local -- value='' local -- value=''
pm::get_seq "${pkgs_yaml}" value "${key}" yml::get_seq \
value \
yaml \
"${node}.${key}" || \
yml::die_parsing "${yaml}"
readarray -t pkg_build_args <<< "${value}" readarray -t pkg_build_args <<< "${value}"
;; ;;
* ) * )
die_invalid_key \ die_invalid_key \
"${pkgs_yaml}" \ "${yaml}" \
"in subpackage '$i'" \ "in subpackage '$i'" \
"${key}" "${key}"
;; ;;
esac esac
done done
[[ -n "${pkg_name}" ]] || \ [[ -n "${pkg_name}" ]] || \
die_sub_package_name_missing "${pkgs_yaml}" die_sub_package_name_missing "${yaml}"
[[ -n "${pkg_version}" ]] || \ [[ -n "${pkg_version}" ]] || \
die_sub_package_version_missing "${pkgs_yaml}" die_sub_package_version_missing "${yaml}"
[[ "${opt_verbose}" == 'yes' ]] && \ [[ "${opt_verbose}" == 'yes' ]] && \
pkg_build_args+=( '--verbose' ) pkg_build_args+=( '--verbose' )
@@ -1348,7 +1313,7 @@ build_modules_yaml_v1(){
} }
is_variant_to_be_built(){ is_variant_to_build(){
local -- name="$1" local -- name="$1"
local -- version="$2" local -- version="$2"
local -n config="$3" local -n config="$3"
@@ -1496,90 +1461,101 @@ build_modules_yaml_v1(){
done done
} }
local -A used_keys=() local -a found_keys=()
local -A default_config=()
local -- version_key=''
local -a version_keys=() local -a version_keys=()
local -A shasums=()
yml::check_keys \ yml::get_keys \
found_keys \
yaml_module_config \ yaml_module_config \
Yaml_valid_keys_for_module \ '.'
used_keys
[[ -v used_keys['versions'] ]] || \
std::die 3 "No version(s) specified in YAML configuration file."
local -- yaml_default_config='' local -A mod_config=()
if [[ -v used_keys['defaults'] ]]; then local key=''
yaml_default_config=$(${yq} '.defaults' <<<"${yaml_module_config}" 2>/dev/null) for key in "${!Yaml_default_config[@]}"; do
fi mod_config[${key}]="${Yaml_default_config[${key}]}"
yml::get_config \ done
yaml_default_config \ for key in "${found_keys[@]}"; do
default_config \ case "${key}" in
Yaml_default_config 'defaults' )
yml::get_config \
if [[ -v used_keys['shasums'] ]]; then yaml_module_config \
local yaml_shasums='' ".${key}" \
yaml_shasums=$(${yq} '.shasums' <<<"${yaml_module_config}" 2>/dev/null) mod_config \
while read -r key value; do
[[ -z ${key} ]] && continue
SHASUMS[${key//:}]="${value}"
done <<<"${yaml_shasums}"
fi
if [[ -v used_keys['type'] ]]; then
local -- value=''
pm::get_value "${yaml_module_config}" value 'type' '!!str'
case "${value,,}" in
'module' )
[[ "${module_type}" == 'sub_package' ]] && \
die_wrong_module_type 'module' 'sub_package'
;; ;;
'sub_package' ) 'shasums' )
[[ "${module_type}" == 'module' ]] && \ local yaml_shasums=''
die_wrong_module_type 'sub_package' 'module' yaml_shasums=$(${yq} ".${key}" <<<"${yaml_module_config}" 2>/dev/null)
while read -r key value; do
[[ -z ${key} ]] && continue
SHASUMS[${key//:}]="${value}"
done <<<"${yaml_shasums}"
;; ;;
* ) 'type' )
die_invalid_module_type "${value}" local -- value=''
yml::get_value \
value \
yaml_module_config \
".${key}" '!!str' || \
yml::die_parsing "${yaml_module_config}"
case "${value,,}" in
'module' )
[[ "${module_type}" == 'sub_package' ]] && \
die_wrong_module_type 'module' 'sub_package'
;;
'sub_package' )
[[ "${module_type}" == 'module' ]] && \
die_wrong_module_type 'sub_package' 'module'
;;
* )
die_invalid_module_type "${value}"
;;
esac
;;
'versions' )
yml::get_matching_version_keys \
yaml_module_config \
version_keys \
"${version}"
;; ;;
esac esac
fi done
yml::get_matching_version_keys \ (( ${#version_keys[@]} > 0 )) || \
yaml_module_config \ std::die 3 "No version(s) specified in YAML configuration file."
version_keys \
"${version}"
local -- version_key=''
local -i num_variants=0
for version_key in "${version_keys[@]}"; do for version_key in "${version_keys[@]}"; do
local -- yaml_version_block='' local node=".versions.\"${version_key}\""
yml::get_version_block \ yml::get_keys \
found_keys \
yaml_module_config \ yaml_module_config \
yaml_version_block \ "${node}"
"${version_key}" for key in "${found_keys[@]}"; do
case "${key}" in
# check keys: allowed are 'config' and 'variants' 'config' )
used_keys=() yml::get_config \
yml::check_keys \ yaml_module_config \
yaml_version_block \ "${node}.config" \
Yaml_valid_vk_keys \ mod_config
used_keys ;;
'variants' )
# read (default) config of version if set local -- type_of_node=''
local -- yaml_version_config='' yml::get_type type_of_node yaml_module_config "${node}.variants"
if [[ -v used_keys['config'] ]]; then if [[ "${type_of_node}" == '!!null' ]]; then
yaml_version_config=$(${yq} '.config' <<<"${yaml_version_block}" 2>/dev/null) num_variants=0
debug "vk input: ${yaml_version_config}" elif [[ "${type_of_node}" != '!!seq' ]]; then
fi die_invalid_variants_block "${name}" "${version}" \
# reminder: if YAML input is empty, next line copies defaults to 'vk_config' "${type_of_key}"
local -A vk_config=() else
yml::get_config \ yml::get_seq_length \
yaml_version_config \ num_variants \
vk_config \ yaml_module_config \
default_config "${node}.variants"
fi
local -- yaml_variants='' ;;
local -i num_variants=0 esac
yml::get_variants \ done
yaml_version_block \
yaml_variants \
num_variants
local versions=() local versions=()
expand_version_key versions "${version_key}" "${version}" expand_version_key versions "${version_key}" "${version}"
@@ -1590,33 +1566,30 @@ build_modules_yaml_v1(){
parse_version "${version}" parse_version "${version}"
if (( num_variants == 0 )); then if (( num_variants == 0 )); then
is_variant_to_be_built \ is_variant_to_build \
"${name}" \ "${name}" "${version}" \
"${version}" \ mod_config || continue
vk_config || continue
build_modules_variant \ build_modules_variant \
"${name}" "${version}" \ "${name}" "${version}" \
vk_config mod_config
else else
local -i n=0 local -i n=0
local -- yaml_variant_config=''
for ((n=0; n<num_variants; n++)); do for ((n=0; n<num_variants; n++)); do
yml::get_nth_variant \ local -A variant_config=()
yaml_variants \ local -- key=''
yaml_variant_config \ for key in "${!mod_config[@]}"; do
"${n}" variant_config[${key}]="${mod_config[${key}]}"
done
local -A mod_config=()
yml::get_config \ yml::get_config \
yaml_variant_config \ yaml_module_config \
mod_config \ "${node}.variants[$n]" \
vk_config variant_config
is_variant_to_be_built \ is_variant_to_build \
"${name}" "${version}" \ "${name}" "${version}" \
mod_config || continue variant_config || continue
build_modules_variant \ build_modules_variant \
"${name}" "${version}" \ "${name}" "${version}" \
mod_config variant_config
if [[ "${mod_config['build_variants']}" == 'first_match' ]]; then if [[ "${mod_config['build_variants']}" == 'first_match' ]]; then
break break
fi fi