From 5e4246508b55577a16251d23e6c27883cf569784 Mon Sep 17 00:00:00 2001 From: Benjamin Labrecque Date: Thu, 20 Aug 2026 12:04:57 +0200 Subject: [PATCH] refactor: route deploy consumers through the central config (step 3) Both deploy paths now take their token values from config/environments.yml instead of defining their own copies. CI / shell path: * scripts/ci/render_subs.py renders deployed files via agebd.subs. * deploy-and-restart-service.sh drops its dev/prod if-else and its four sed calls for one render_subs.py invocation, and aborts the deploy if rendering fails rather than restarting a service whose files still contain placeholders. Ansible path: * ansible/tasks/utils/load-substitutions.yml reads the central config on the controller and exposes the token table for this environment. * ioc-install and service-deploy-and-restart now loop over that table instead of listing tokens inline, so a new token needs no ansible edit. * env_suffix_upper/lower are gone from both inventories. The inventory says which environment it is; the central config says what that environment means. The facts are renamed to agebd_env_suffix_upper / _lower to match the token names, removing a rename trap. Ansible keeps using ansible.builtin.replace rather than calling agebd.subs, because ioc-install runs on hosts with no uv or Python environment for this repo. Only the mechanism differs now; the values and the token list are shared. Verified: both playbook paths run locally against dev and prod and produce the expected output, an undeclared environment fails loudly in both paths, and a misspelled token exits non-zero instead of shipping the placeholder. Not addressed here, still step 5: ansible/tasks/utils/get-ioc-port.yml matches CamelCase registry names against a lowercase input and returns nothing, so the ansible ioc-install path cannot resolve a port. This is pre-existing and unchanged by this commit. Co-Authored-By: Claude Opus 5 --- .gitea/scripts/deploy-and-restart-service.sh | 29 ++---- ansible/inventories/dev/hosts.yml | 4 +- ansible/inventories/prod/hosts.yml | 4 +- ansible/playbooks/ioc-install.yml | 2 + ansible/playbooks/ioc-restart.yml | 2 + ansible/playbooks/ioc-start.yml | 2 + ansible/tasks/ioc/ioc-install.yml | 37 ++++---- ansible/tasks/ioc/ioc-restart.yml | 5 +- .../service/service-deploy-and-restart.yml | 14 ++- ansible/tasks/utils/load-substitutions.yml | 32 +++++++ scripts/ci/render_subs.py | 71 ++++++++++++++ scripts/tests/test_render_subs.py | 93 +++++++++++++++++++ 12 files changed, 250 insertions(+), 45 deletions(-) create mode 100644 ansible/tasks/utils/load-substitutions.yml create mode 100644 scripts/ci/render_subs.py create mode 100644 scripts/tests/test_render_subs.py diff --git a/.gitea/scripts/deploy-and-restart-service.sh b/.gitea/scripts/deploy-and-restart-service.sh index ccbbb8f..95834c3 100755 --- a/.gitea/scripts/deploy-and-restart-service.sh +++ b/.gitea/scripts/deploy-and-restart-service.sh @@ -13,15 +13,6 @@ SERVICE_DIR=/sls/bd/hla/${AGEBD_ENV}/services/${SERVICE_NAME_LOWER}/current SYSTEMD_UNIT_NAME=AGEBD-SERVICE-${SERVICE_NAME_UPPER}.service SYSTEMD_UNIT_FILE_PATH=${SERVICE_DIR}/systemd/${SYSTEMD_UNIT_NAME} IOC_DIR=${SERVICE_DIR}/ioc -IOC_PORT=$(uv run --directory scripts ci/get_ioc_port.py ${SERVICE_NAME_LOWER}) - -if [ "${AGEBD_ENV}" = "dev" ]; then - AGEBD_ENV_SUFFIX_UPPER="-DEV" - AGEBD_ENV_SUFFIX_LOWER="-dev" -else - AGEBD_ENV_SUFFIX_UPPER="" - AGEBD_ENV_SUFFIX_LOWER="" -fi # TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50 # rm -rf "${SERVICE_DIR}" @@ -38,17 +29,15 @@ rsync -avz --delete \ ./services/${SERVICE_NAME_LOWER}/current/* ${SERVICE_DIR}/ ### SUBSTITUTIONS -# Deploy-time tokens are '@@token@@'. Their values are declared in -# config/environments.yml -- keep the two in sync until this is replaced by a -# call to agebd.subs (see the refactor plan, step 3). -### IOC -sed -i "s/@@[[:space:]]*agebd_env_suffix_upper[[:space:]]*@@/${AGEBD_ENV_SUFFIX_UPPER}/g" "${IOC_DIR}/"* -sed -i "s/@@[[:space:]]*agebd_env_suffix_lower[[:space:]]*@@/${AGEBD_ENV_SUFFIX_LOWER}/g" "${IOC_DIR}/"* -sed -i "s/@@[[:space:]]*ioc_port[[:space:]]*@@/${IOC_PORT}/g" "${IOC_DIR}/"* - -### SYSTEMD -if [ -f "${SYSTEMD_UNIT_FILE_PATH}" ]; then - sed -i "s/@@[[:space:]]*agebd_env[[:space:]]*@@/${AGEBD_ENV}/g" "${SYSTEMD_UNIT_FILE_PATH}" +# Every @@token@@ is resolved from config/environments.yml and +# config/services_registry.yml. An unknown or misspelled token aborts the +# deploy instead of leaving the placeholder in a file the IOC will read. +if ! uv run --directory scripts ci/render_subs.py \ + --env "${AGEBD_ENV}" \ + --service "${SERVICE_NAME_LOWER}" \ + "${IOC_DIR}" "${SERVICE_DIR}/systemd"; then + echo "Substitution failed -- aborting deploy of ${SERVICE_NAME}" >&2 + exit 1 fi diff --git a/ansible/inventories/dev/hosts.yml b/ansible/inventories/dev/hosts.yml index 3ac0e24..fe426e5 100644 --- a/ansible/inventories/dev/hosts.yml +++ b/ansible/inventories/dev/hosts.yml @@ -1,8 +1,8 @@ all: vars: + # Suffix values for this environment live in config/environments.yml -- + # do not reintroduce them here. agebd_env: "dev" - env_suffix_upper: "-DEV" - env_suffix_lower: "-dev" children: lc: diff --git a/ansible/inventories/prod/hosts.yml b/ansible/inventories/prod/hosts.yml index 20ea709..ec25a8e 100644 --- a/ansible/inventories/prod/hosts.yml +++ b/ansible/inventories/prod/hosts.yml @@ -1,8 +1,8 @@ all: vars: + # Suffix values for this environment live in config/environments.yml -- + # do not reintroduce them here. agebd_env: "prod" - env_suffix_upper: "" - env_suffix_lower: "" children: lc: diff --git a/ansible/playbooks/ioc-install.yml b/ansible/playbooks/ioc-install.yml index f337812..c78d698 100644 --- a/ansible/playbooks/ioc-install.yml +++ b/ansible/playbooks/ioc-install.yml @@ -5,6 +5,8 @@ vars: workspace_dir: "/tmp/hla_framework_bd" + # Controller-side checkout, where config/environments.yml is read from + repo_root: "{{ playbook_dir }}/../.." tasks: - name: Run ioc install tasks diff --git a/ansible/playbooks/ioc-restart.yml b/ansible/playbooks/ioc-restart.yml index a36728e..7c54986 100644 --- a/ansible/playbooks/ioc-restart.yml +++ b/ansible/playbooks/ioc-restart.yml @@ -5,6 +5,8 @@ vars: workspace_dir: "/tmp/hla_framework_bd" + # Controller-side checkout, where config/environments.yml is read from + repo_root: "{{ playbook_dir }}/../.." tasks: - name: Run ioc restart tasks diff --git a/ansible/playbooks/ioc-start.yml b/ansible/playbooks/ioc-start.yml index 02a6386..b50b4bc 100644 --- a/ansible/playbooks/ioc-start.yml +++ b/ansible/playbooks/ioc-start.yml @@ -8,6 +8,8 @@ vars: workspace_dir: "/tmp/hla_framework_bd" + # Controller-side checkout, where config/environments.yml is read from + repo_root: "{{ playbook_dir }}/../.." tasks: - name: Run ioc start tasks diff --git a/ansible/tasks/ioc/ioc-install.yml b/ansible/tasks/ioc/ioc-install.yml index d807ee4..d991c1c 100644 --- a/ansible/tasks/ioc/ioc-install.yml +++ b/ansible/tasks/ioc/ioc-install.yml @@ -8,6 +8,9 @@ - name: Get and set ioc_port ansible.builtin.import_tasks: ../utils/get-ioc-port.yml +- name: Load deploy-time substitutions + ansible.builtin.import_tasks: ../utils/load-substitutions.yml + - name: "[{{ target_service_name_lower }}] Establish environment naming conventions" ansible.builtin.set_fact: service_name_upper: "{{ target_service_name_lower | upper }}" @@ -16,9 +19,9 @@ ansible.builtin.set_fact: ioc_base_dir: "{{ workspace_dir }}/services/{{ target_service_name_lower }}/current/ioc" orig_subs_name: "AGEBD-CPCL-{{ service_name_upper }}_main.subs" - target_subs_name: "AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}_main.subs" + target_subs_name: "AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}_main.subs" orig_params_name: "AGEBD-CPCL-{{ service_name_upper }}_parameters.yaml" - target_params_name: "AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}_parameters.yaml" + target_params_name: "AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}_parameters.yaml" # Rename the subs file if we are in DEV - name: "[{{ target_service_name_lower }}] Rename substitution file for environment" @@ -32,26 +35,26 @@ cmd: "mv {{ ioc_base_dir }}/{{ orig_params_name }} {{ ioc_base_dir }}/{{ target_params_name }}" removes: "{{ ioc_base_dir }}/{{ orig_params_name }}" # Only runs if the original file exists -# Replace the @@agebd_env_suffix_upper@@ token inside the newly renamed subs file -- name: "[{{ target_service_name_lower }}] Substitute environment tokens inside subs file" - ansible.builtin.replace: - path: "{{ ioc_base_dir }}/{{ target_subs_name }}" - regexp: '@@\s*agebd_env_suffix_upper\s*@@' - replace: "{{ env_suffix_upper }}" +# Every @@token@@ is resolved from the central token table -- the token list +# is not repeated here, so a new token needs no change to this file. +- name: "[{{ target_service_name_lower }}] Collect deploy-time token values" + ansible.builtin.set_fact: + deploy_tokens: "{{ env_tokens | combine({'ioc_port': ioc_port}) }}" -# Replace the @@agebd_env_suffix_lower@@ and @@ioc_port@@ tokens inside the newly renamed params file -- name: "[{{ target_service_name_lower }}] Substitute environment token inside params file" +- name: "[{{ target_service_name_lower }}] Substitute deploy-time tokens in the IOC files" ansible.builtin.replace: - path: "{{ ioc_base_dir }}/{{ target_params_name }}" - regexp: "{{ item.regexp }}" - replace: "{{ item.replace | string }}" - with_items: - - { regexp: '@@\s*agebd_env_suffix_lower\s*@@', replace: "{{ env_suffix_lower }}" } - - { regexp: '@@\s*ioc_port\s*@@', replace: "{{ ioc_port }}" } + path: "{{ item.0 }}" + regexp: '@@\s*{{ item.1.key }}\s*@@' + replace: "{{ item.1.value | string }}" + loop: >- + {{ [ioc_base_dir ~ '/' ~ target_subs_name, ioc_base_dir ~ '/' ~ target_params_name] + | product(deploy_tokens | dict2items) | list }} + loop_control: + label: "{{ item.0 | basename }} :: {{ item.1.key }}" - name: "[{{ target_service_name_lower }}] Run ioc install" ansible.builtin.shell: - cmd: "ioc install -V --facility sls --ioc AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }} --clean" + cmd: "ioc install -V --facility sls --ioc AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }} --clean" chdir: "{{ ioc_base_dir }}" changed_when: true register: ioc_install_result diff --git a/ansible/tasks/ioc/ioc-restart.yml b/ansible/tasks/ioc/ioc-restart.yml index ad637ac..8d846c9 100644 --- a/ansible/tasks/ioc/ioc-restart.yml +++ b/ansible/tasks/ioc/ioc-restart.yml @@ -5,6 +5,9 @@ - name: Clone repo ansible.builtin.import_tasks: ../utils/clone-repo.yml +- name: Load deploy-time substitutions + ansible.builtin.import_tasks: ../utils/load-substitutions.yml + - name: "[{{ target_service_name_lower }}] Establish environment naming conventions" ansible.builtin.set_fact: service_name_upper: "{{ target_service_name_lower | upper }}" @@ -15,6 +18,6 @@ - name: "[{{ target_service_name_lower }}] Restart IOC" ansible.builtin.shell: - cmd: "ioc restart AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}" + cmd: "ioc restart AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}" chdir: "{{ ioc_base_dir }}" changed_when: true diff --git a/ansible/tasks/service/service-deploy-and-restart.yml b/ansible/tasks/service/service-deploy-and-restart.yml index 2380c7c..7509cda 100644 --- a/ansible/tasks/service/service-deploy-and-restart.yml +++ b/ansible/tasks/service/service-deploy-and-restart.yml @@ -4,6 +4,9 @@ # Uses preferred_service_name_lower if defined in vars, otherwise falls back to the CLI extra-vars target_service_name_lower: "{{ preferred_service_name_lower | default(service_name_lower) }}" +- name: Load deploy-time substitutions + ansible.builtin.import_tasks: ../utils/load-substitutions.yml + - name: "[{{ target_service_name_lower }}] Define service specific paths and names" ansible.builtin.set_fact: svc_current_dir: "/sls/bd/hla/{{ agebd_env }}/services/{{ target_service_name_lower }}/current" @@ -24,11 +27,16 @@ delete: yes # This removes stale files on the target that no longer exist locally recursive: yes -- name: "[{{ target_service_name_lower }}] Substitute environment variable inline" +# Every @@token@@ is resolved from the central token table -- the token list +# is not repeated here, so a new token needs no change to this file. +- name: "[{{ target_service_name_lower }}] Substitute deploy-time tokens in the systemd unit" ansible.builtin.replace: path: "{{ svc_current_dir }}/systemd/{{ service_unit_name }}" - regexp: '@@\s*agebd_env\s*@@' - replace: "{{ agebd_env }}" + regexp: '@@\s*{{ item.key }}\s*@@' + replace: "{{ item.value | string }}" + loop: "{{ env_tokens | dict2items }}" + loop_control: + label: "{{ item.key }}" # systemctl --user link --force "./{{ service_unit_name }}" - name: "[{{ target_service_name_lower }}] Symlink systemd unit file (Absolute Path)" diff --git a/ansible/tasks/utils/load-substitutions.yml b/ansible/tasks/utils/load-substitutions.yml new file mode 100644 index 0000000..f63a26a --- /dev/null +++ b/ansible/tasks/utils/load-substitutions.yml @@ -0,0 +1,32 @@ +--- +# Load the deploy-time token values for {{ agebd_env }} from the single source +# of truth, config/environments.yml. +# +# The inventory says which environment it is; this file says what that +# environment means. Do not reintroduce suffix values into the inventories. +# +# include_vars reads on the controller, where the repo checkout lives, so this +# works for both local-connection plays and plays targeting remote hosts. +# +# Ansible substitutes tokens with ansible.builtin.replace rather than calling +# agebd.subs, because these tasks also run on hosts that have no uv or Python +# environment for this repo. The values and the token list still come from the +# central config, so there is nothing to keep in sync by hand. + +- name: Load the central substitution config + ansible.builtin.include_vars: + file: "{{ repo_root }}/config/environments.yml" + name: substitutions + +- name: Fail if this environment is not declared centrally + ansible.builtin.fail: + msg: >- + Environment '{{ agebd_env }}' is not declared in config/environments.yml. + Declared: {{ substitutions.environments.keys() | list | join(', ') }} + when: agebd_env not in substitutions.environments + +- name: Select the deploy-time token values for this environment + ansible.builtin.set_fact: + env_tokens: "{{ substitutions.environments[agebd_env] }}" + agebd_env_suffix_upper: "{{ substitutions.environments[agebd_env].agebd_env_suffix_upper }}" + agebd_env_suffix_lower: "{{ substitutions.environments[agebd_env].agebd_env_suffix_lower }}" diff --git a/scripts/ci/render_subs.py b/scripts/ci/render_subs.py new file mode 100644 index 0000000..9deb165 --- /dev/null +++ b/scripts/ci/render_subs.py @@ -0,0 +1,71 @@ +""" +Resolve deploy-time tokens in deployed files. + +Values come from config/environments.yml and config/services_registry.yml -- +the single source of truth. Nothing here knows what '-DEV' is. + + render_subs.py --env dev --service tune /path/to/ioc /path/to/systemd + render_subs.py --env prod /sls/bd/hla/prod/qt + +Without --service only the environment tokens are available, so a file that +needs a per-service token (ioc_port) fails loudly rather than shipping the +placeholder. A path that does not exist is reported and skipped, which is how +services without a systemd unit are handled. +""" + +import argparse +import sys +from pathlib import Path + +from agebd.subs import SubstitutionError, Substitutions + + +def render(subs, paths): + """ + Render every path in place. Returns the files that changed. + """ + changed = [] + for path in paths: + if not path.exists(): + print(f"skipped (does not exist): {path}") + continue + if path.is_dir(): + changed.extend(subs.render_tree(path)) + elif subs.render_file(path): + changed.append(path) + return changed + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--env", required=True, help="Deployment environment, e.g. dev") + parser.add_argument( + "--service", + default=None, + help="Service name. Required for files holding per-service tokens.", + ) + parser.add_argument("paths", nargs="+", type=Path, help="Files or directories to render") + args = parser.parse_args(argv) + + if args.service: + subs = Substitutions.for_service(args.env, args.service) + else: + subs = Substitutions.for_env(args.env) + + changed = render(subs, args.paths) + + for path in changed: + print(f"rendered: {path}") + print(f"{len(changed)} file(s) rendered for env '{args.env}'") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except SubstitutionError as e: + print(f"error: {e}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/tests/test_render_subs.py b/scripts/tests/test_render_subs.py new file mode 100644 index 0000000..72b0a23 --- /dev/null +++ b/scripts/tests/test_render_subs.py @@ -0,0 +1,93 @@ +import pytest +from agebd.subs import SubstitutionError, Substitutions, UnknownTokenError +from agebd.utils import get_git_root + +from ci.render_subs import main, render + +REPO_ROOT = get_git_root(__file__) +CONFIG_DIR = REPO_ROOT / "config" + + +@pytest.fixture +def service_tree(tmp_path): + ioc = tmp_path / "ioc" + ioc.mkdir() + (ioc / "AGEBD-CPCL-TUNE_main.subs").write_text("{ AGEBD-TUNE@@agebd_env_suffix_upper@@ }\n") + (ioc / "AGEBD-CPCL-TUNE_parameters.yaml").write_text( + "ioc_host: sls-vserv-bd-01@@agebd_env_suffix_lower@@\nioc_port: @@ioc_port@@\n" + ) + systemd = tmp_path / "systemd" + systemd.mkdir() + (systemd / "AGEBD-SERVICE-TUNE.service").write_text("Environment=AGEBD_ENV=@@agebd_env@@\n") + return tmp_path + + +def test_renders_a_whole_service_for_dev(service_tree): + subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR) + + changed = render(subs, [service_tree / "ioc", service_tree / "systemd"]) + + assert len(changed) == 3 + assert "AGEBD-TUNE-DEV" in (service_tree / "ioc/AGEBD-CPCL-TUNE_main.subs").read_text() + assert (service_tree / "ioc/AGEBD-CPCL-TUNE_parameters.yaml").read_text() == ( + "ioc_host: sls-vserv-bd-01-dev\nioc_port: 50007\n" + ) + assert (service_tree / "systemd/AGEBD-SERVICE-TUNE.service").read_text() == ( + "Environment=AGEBD_ENV=dev\n" + ) + + +def test_renders_a_whole_service_for_prod(service_tree): + subs = Substitutions.for_service("prod", "tune", config_dir=CONFIG_DIR) + + render(subs, [service_tree / "ioc", service_tree / "systemd"]) + + assert "{ AGEBD-TUNE }" in (service_tree / "ioc/AGEBD-CPCL-TUNE_main.subs").read_text() + assert "sls-vserv-bd-01\n" in (service_tree / "ioc/AGEBD-CPCL-TUNE_parameters.yaml").read_text() + + +def test_missing_path_is_skipped_not_fatal(service_tree, capsys): + subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR) + + render(subs, [service_tree / "systemd", service_tree / "does-not-exist"]) + + assert "skipped (does not exist)" in capsys.readouterr().out + + +def test_leaves_nothing_unrendered(service_tree): + subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR) + + render(subs, [service_tree]) + + for path in service_tree.rglob("*"): + if path.is_file(): + assert "@@" not in path.read_text(), path + + +def test_rendering_is_idempotent(service_tree): + subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR) + + assert len(render(subs, [service_tree])) == 3 + assert render(subs, [service_tree]) == [] + + +def test_per_service_token_without_service_fails_loudly(service_tree): + """Without --service there is no ioc_port, and that must not pass silently.""" + subs = Substitutions.for_env("dev", config_dir=CONFIG_DIR) + + with pytest.raises(UnknownTokenError, match="ioc_port"): + render(subs, [service_tree / "ioc"]) + + +def test_main_renders_via_the_command_line(service_tree): + exit_code = main(["--env", "dev", "--service", "tune", str(service_tree / "systemd")]) + + assert exit_code == 0 + assert (service_tree / "systemd/AGEBD-SERVICE-TUNE.service").read_text() == ( + "Environment=AGEBD_ENV=dev\n" + ) + + +def test_main_rejects_an_undeclared_environment(service_tree): + with pytest.raises(SubstitutionError, match="staging"): + main(["--env", "staging", "--service", "tune", str(service_tree / "systemd")])