refactor: route deploy consumers through the central config (step 3)
Both deploy paths now take their token values from
config/environments.yml instead of defining their own copies.
CI / shell path:
* scripts/ci/render_subs.py renders deployed files via agebd.subs.
* deploy-and-restart-service.sh drops its dev/prod if-else and its four
sed calls for one render_subs.py invocation, and aborts the deploy if
rendering fails rather than restarting a service whose files still
contain placeholders.
Ansible path:
* ansible/tasks/utils/load-substitutions.yml reads the central config on
the controller and exposes the token table for this environment.
* ioc-install and service-deploy-and-restart now loop over that table
instead of listing tokens inline, so a new token needs no ansible
edit.
* env_suffix_upper/lower are gone from both inventories. The inventory
says which environment it is; the central config says what that
environment means. The facts are renamed to agebd_env_suffix_upper /
_lower to match the token names, removing a rename trap.
Ansible keeps using ansible.builtin.replace rather than calling
agebd.subs, because ioc-install runs on hosts with no uv or Python
environment for this repo. Only the mechanism differs now; the values and
the token list are shared.
Verified: both playbook paths run locally against dev and prod and
produce the expected output, an undeclared environment fails loudly in
both paths, and a misspelled token exits non-zero instead of shipping the
placeholder.
Not addressed here, still step 5: ansible/tasks/utils/get-ioc-port.yml
matches CamelCase registry names against a lowercase input and returns
nothing, so the ansible ioc-install path cannot resolve a port. This is
pre-existing and unchanged by this commit.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cfd3ecad6c
commit
5e4246508b
12 files changed
+250
-45
No files matched your search
@@ -13,15 +13,6 @@ SERVICE_DIR=/sls/bd/hla/${AGEBD_ENV}/services/${SERVICE_NAME_LOWER}/current
|
||||
SYSTEMD_UNIT_NAME=AGEBD-SERVICE-${SERVICE_NAME_UPPER}.service
|
||||
SYSTEMD_UNIT_FILE_PATH=${SERVICE_DIR}/systemd/${SYSTEMD_UNIT_NAME}
|
||||
IOC_DIR=${SERVICE_DIR}/ioc
|
||||
IOC_PORT=$(uv run --directory scripts ci/get_ioc_port.py ${SERVICE_NAME_LOWER})
|
||||
|
||||
if [ "${AGEBD_ENV}" = "dev" ]; then
|
||||
AGEBD_ENV_SUFFIX_UPPER="-DEV"
|
||||
AGEBD_ENV_SUFFIX_LOWER="-dev"
|
||||
else
|
||||
AGEBD_ENV_SUFFIX_UPPER=""
|
||||
AGEBD_ENV_SUFFIX_LOWER=""
|
||||
fi
|
||||
|
||||
# TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50
|
||||
# rm -rf "${SERVICE_DIR}"
|
||||
@@ -38,17 +29,15 @@ rsync -avz --delete \
|
||||
./services/${SERVICE_NAME_LOWER}/current/* ${SERVICE_DIR}/
|
||||
|
||||
### SUBSTITUTIONS
|
||||
# Deploy-time tokens are '@@token@@'. Their values are declared in
|
||||
# config/environments.yml -- keep the two in sync until this is replaced by a
|
||||
# call to agebd.subs (see the refactor plan, step 3).
|
||||
### IOC
|
||||
sed -i "s/@@[[:space:]]*agebd_env_suffix_upper[[:space:]]*@@/${AGEBD_ENV_SUFFIX_UPPER}/g" "${IOC_DIR}/"*
|
||||
sed -i "s/@@[[:space:]]*agebd_env_suffix_lower[[:space:]]*@@/${AGEBD_ENV_SUFFIX_LOWER}/g" "${IOC_DIR}/"*
|
||||
sed -i "s/@@[[:space:]]*ioc_port[[:space:]]*@@/${IOC_PORT}/g" "${IOC_DIR}/"*
|
||||
|
||||
### SYSTEMD
|
||||
if [ -f "${SYSTEMD_UNIT_FILE_PATH}" ]; then
|
||||
sed -i "s/@@[[:space:]]*agebd_env[[:space:]]*@@/${AGEBD_ENV}/g" "${SYSTEMD_UNIT_FILE_PATH}"
|
||||
# Every @@token@@ is resolved from config/environments.yml and
|
||||
# config/services_registry.yml. An unknown or misspelled token aborts the
|
||||
# deploy instead of leaving the placeholder in a file the IOC will read.
|
||||
if ! uv run --directory scripts ci/render_subs.py \
|
||||
--env "${AGEBD_ENV}" \
|
||||
--service "${SERVICE_NAME_LOWER}" \
|
||||
"${IOC_DIR}" "${SERVICE_DIR}/systemd"; then
|
||||
echo "Substitution failed -- aborting deploy of ${SERVICE_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
all:
|
||||
vars:
|
||||
# Suffix values for this environment live in config/environments.yml --
|
||||
# do not reintroduce them here.
|
||||
agebd_env: "dev"
|
||||
env_suffix_upper: "-DEV"
|
||||
env_suffix_lower: "-dev"
|
||||
|
||||
children:
|
||||
lc:
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
all:
|
||||
vars:
|
||||
# Suffix values for this environment live in config/environments.yml --
|
||||
# do not reintroduce them here.
|
||||
agebd_env: "prod"
|
||||
env_suffix_upper: ""
|
||||
env_suffix_lower: ""
|
||||
|
||||
children:
|
||||
lc:
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
vars:
|
||||
workspace_dir: "/tmp/hla_framework_bd"
|
||||
# Controller-side checkout, where config/environments.yml is read from
|
||||
repo_root: "{{ playbook_dir }}/../.."
|
||||
|
||||
tasks:
|
||||
- name: Run ioc install tasks
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
vars:
|
||||
workspace_dir: "/tmp/hla_framework_bd"
|
||||
# Controller-side checkout, where config/environments.yml is read from
|
||||
repo_root: "{{ playbook_dir }}/../.."
|
||||
|
||||
tasks:
|
||||
- name: Run ioc restart tasks
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
|
||||
vars:
|
||||
workspace_dir: "/tmp/hla_framework_bd"
|
||||
# Controller-side checkout, where config/environments.yml is read from
|
||||
repo_root: "{{ playbook_dir }}/../.."
|
||||
|
||||
tasks:
|
||||
- name: Run ioc start tasks
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
- name: Get and set ioc_port
|
||||
ansible.builtin.import_tasks: ../utils/get-ioc-port.yml
|
||||
|
||||
- name: Load deploy-time substitutions
|
||||
ansible.builtin.import_tasks: ../utils/load-substitutions.yml
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Establish environment naming conventions"
|
||||
ansible.builtin.set_fact:
|
||||
service_name_upper: "{{ target_service_name_lower | upper }}"
|
||||
@@ -16,9 +19,9 @@
|
||||
ansible.builtin.set_fact:
|
||||
ioc_base_dir: "{{ workspace_dir }}/services/{{ target_service_name_lower }}/current/ioc"
|
||||
orig_subs_name: "AGEBD-CPCL-{{ service_name_upper }}_main.subs"
|
||||
target_subs_name: "AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}_main.subs"
|
||||
target_subs_name: "AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}_main.subs"
|
||||
orig_params_name: "AGEBD-CPCL-{{ service_name_upper }}_parameters.yaml"
|
||||
target_params_name: "AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}_parameters.yaml"
|
||||
target_params_name: "AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}_parameters.yaml"
|
||||
|
||||
# Rename the subs file if we are in DEV
|
||||
- name: "[{{ target_service_name_lower }}] Rename substitution file for environment"
|
||||
@@ -32,26 +35,26 @@
|
||||
cmd: "mv {{ ioc_base_dir }}/{{ orig_params_name }} {{ ioc_base_dir }}/{{ target_params_name }}"
|
||||
removes: "{{ ioc_base_dir }}/{{ orig_params_name }}" # Only runs if the original file exists
|
||||
|
||||
# Replace the @@agebd_env_suffix_upper@@ token inside the newly renamed subs file
|
||||
- name: "[{{ target_service_name_lower }}] Substitute environment tokens inside subs file"
|
||||
ansible.builtin.replace:
|
||||
path: "{{ ioc_base_dir }}/{{ target_subs_name }}"
|
||||
regexp: '@@\s*agebd_env_suffix_upper\s*@@'
|
||||
replace: "{{ env_suffix_upper }}"
|
||||
# Every @@token@@ is resolved from the central token table -- the token list
|
||||
# is not repeated here, so a new token needs no change to this file.
|
||||
- name: "[{{ target_service_name_lower }}] Collect deploy-time token values"
|
||||
ansible.builtin.set_fact:
|
||||
deploy_tokens: "{{ env_tokens | combine({'ioc_port': ioc_port}) }}"
|
||||
|
||||
# Replace the @@agebd_env_suffix_lower@@ and @@ioc_port@@ tokens inside the newly renamed params file
|
||||
- name: "[{{ target_service_name_lower }}] Substitute environment token inside params file"
|
||||
- name: "[{{ target_service_name_lower }}] Substitute deploy-time tokens in the IOC files"
|
||||
ansible.builtin.replace:
|
||||
path: "{{ ioc_base_dir }}/{{ target_params_name }}"
|
||||
regexp: "{{ item.regexp }}"
|
||||
replace: "{{ item.replace | string }}"
|
||||
with_items:
|
||||
- { regexp: '@@\s*agebd_env_suffix_lower\s*@@', replace: "{{ env_suffix_lower }}" }
|
||||
- { regexp: '@@\s*ioc_port\s*@@', replace: "{{ ioc_port }}" }
|
||||
path: "{{ item.0 }}"
|
||||
regexp: '@@\s*{{ item.1.key }}\s*@@'
|
||||
replace: "{{ item.1.value | string }}"
|
||||
loop: >-
|
||||
{{ [ioc_base_dir ~ '/' ~ target_subs_name, ioc_base_dir ~ '/' ~ target_params_name]
|
||||
| product(deploy_tokens | dict2items) | list }}
|
||||
loop_control:
|
||||
label: "{{ item.0 | basename }} :: {{ item.1.key }}"
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Run ioc install"
|
||||
ansible.builtin.shell:
|
||||
cmd: "ioc install -V --facility sls --ioc AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }} --clean"
|
||||
cmd: "ioc install -V --facility sls --ioc AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }} --clean"
|
||||
chdir: "{{ ioc_base_dir }}"
|
||||
changed_when: true
|
||||
register: ioc_install_result
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
- name: Clone repo
|
||||
ansible.builtin.import_tasks: ../utils/clone-repo.yml
|
||||
|
||||
- name: Load deploy-time substitutions
|
||||
ansible.builtin.import_tasks: ../utils/load-substitutions.yml
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Establish environment naming conventions"
|
||||
ansible.builtin.set_fact:
|
||||
service_name_upper: "{{ target_service_name_lower | upper }}"
|
||||
@@ -15,6 +18,6 @@
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Restart IOC"
|
||||
ansible.builtin.shell:
|
||||
cmd: "ioc restart AGEBD-CPCL-{{ service_name_upper }}{{ env_suffix_upper }}"
|
||||
cmd: "ioc restart AGEBD-CPCL-{{ service_name_upper }}{{ agebd_env_suffix_upper }}"
|
||||
chdir: "{{ ioc_base_dir }}"
|
||||
changed_when: true
|
||||
@@ -4,6 +4,9 @@
|
||||
# Uses preferred_service_name_lower if defined in vars, otherwise falls back to the CLI extra-vars
|
||||
target_service_name_lower: "{{ preferred_service_name_lower | default(service_name_lower) }}"
|
||||
|
||||
- name: Load deploy-time substitutions
|
||||
ansible.builtin.import_tasks: ../utils/load-substitutions.yml
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Define service specific paths and names"
|
||||
ansible.builtin.set_fact:
|
||||
svc_current_dir: "/sls/bd/hla/{{ agebd_env }}/services/{{ target_service_name_lower }}/current"
|
||||
@@ -24,11 +27,16 @@
|
||||
delete: yes # This removes stale files on the target that no longer exist locally
|
||||
recursive: yes
|
||||
|
||||
- name: "[{{ target_service_name_lower }}] Substitute environment variable inline"
|
||||
# Every @@token@@ is resolved from the central token table -- the token list
|
||||
# is not repeated here, so a new token needs no change to this file.
|
||||
- name: "[{{ target_service_name_lower }}] Substitute deploy-time tokens in the systemd unit"
|
||||
ansible.builtin.replace:
|
||||
path: "{{ svc_current_dir }}/systemd/{{ service_unit_name }}"
|
||||
regexp: '@@\s*agebd_env\s*@@'
|
||||
replace: "{{ agebd_env }}"
|
||||
regexp: '@@\s*{{ item.key }}\s*@@'
|
||||
replace: "{{ item.value | string }}"
|
||||
loop: "{{ env_tokens | dict2items }}"
|
||||
loop_control:
|
||||
label: "{{ item.key }}"
|
||||
|
||||
# systemctl --user link --force "./{{ service_unit_name }}"
|
||||
- name: "[{{ target_service_name_lower }}] Symlink systemd unit file (Absolute Path)"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
---
|
||||
# Load the deploy-time token values for {{ agebd_env }} from the single source
|
||||
# of truth, config/environments.yml.
|
||||
#
|
||||
# The inventory says which environment it is; this file says what that
|
||||
# environment means. Do not reintroduce suffix values into the inventories.
|
||||
#
|
||||
# include_vars reads on the controller, where the repo checkout lives, so this
|
||||
# works for both local-connection plays and plays targeting remote hosts.
|
||||
#
|
||||
# Ansible substitutes tokens with ansible.builtin.replace rather than calling
|
||||
# agebd.subs, because these tasks also run on hosts that have no uv or Python
|
||||
# environment for this repo. The values and the token list still come from the
|
||||
# central config, so there is nothing to keep in sync by hand.
|
||||
|
||||
- name: Load the central substitution config
|
||||
ansible.builtin.include_vars:
|
||||
file: "{{ repo_root }}/config/environments.yml"
|
||||
name: substitutions
|
||||
|
||||
- name: Fail if this environment is not declared centrally
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
Environment '{{ agebd_env }}' is not declared in config/environments.yml.
|
||||
Declared: {{ substitutions.environments.keys() | list | join(', ') }}
|
||||
when: agebd_env not in substitutions.environments
|
||||
|
||||
- name: Select the deploy-time token values for this environment
|
||||
ansible.builtin.set_fact:
|
||||
env_tokens: "{{ substitutions.environments[agebd_env] }}"
|
||||
agebd_env_suffix_upper: "{{ substitutions.environments[agebd_env].agebd_env_suffix_upper }}"
|
||||
agebd_env_suffix_lower: "{{ substitutions.environments[agebd_env].agebd_env_suffix_lower }}"
|
||||
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Resolve deploy-time tokens in deployed files.
|
||||
|
||||
Values come from config/environments.yml and config/services_registry.yml --
|
||||
the single source of truth. Nothing here knows what '-DEV' is.
|
||||
|
||||
render_subs.py --env dev --service tune /path/to/ioc /path/to/systemd
|
||||
render_subs.py --env prod /sls/bd/hla/prod/qt
|
||||
|
||||
Without --service only the environment tokens are available, so a file that
|
||||
needs a per-service token (ioc_port) fails loudly rather than shipping the
|
||||
placeholder. A path that does not exist is reported and skipped, which is how
|
||||
services without a systemd unit are handled.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from agebd.subs import SubstitutionError, Substitutions
|
||||
|
||||
|
||||
def render(subs, paths):
|
||||
"""
|
||||
Render every path in place. Returns the files that changed.
|
||||
"""
|
||||
changed = []
|
||||
for path in paths:
|
||||
if not path.exists():
|
||||
print(f"skipped (does not exist): {path}")
|
||||
continue
|
||||
if path.is_dir():
|
||||
changed.extend(subs.render_tree(path))
|
||||
elif subs.render_file(path):
|
||||
changed.append(path)
|
||||
return changed
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--env", required=True, help="Deployment environment, e.g. dev")
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
default=None,
|
||||
help="Service name. Required for files holding per-service tokens.",
|
||||
)
|
||||
parser.add_argument("paths", nargs="+", type=Path, help="Files or directories to render")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.service:
|
||||
subs = Substitutions.for_service(args.env, args.service)
|
||||
else:
|
||||
subs = Substitutions.for_env(args.env)
|
||||
|
||||
changed = render(subs, args.paths)
|
||||
|
||||
for path in changed:
|
||||
print(f"rendered: {path}")
|
||||
print(f"{len(changed)} file(s) rendered for env '{args.env}'")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except SubstitutionError as e:
|
||||
print(f"error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,93 @@
|
||||
import pytest
|
||||
from agebd.subs import SubstitutionError, Substitutions, UnknownTokenError
|
||||
from agebd.utils import get_git_root
|
||||
|
||||
from ci.render_subs import main, render
|
||||
|
||||
REPO_ROOT = get_git_root(__file__)
|
||||
CONFIG_DIR = REPO_ROOT / "config"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service_tree(tmp_path):
|
||||
ioc = tmp_path / "ioc"
|
||||
ioc.mkdir()
|
||||
(ioc / "AGEBD-CPCL-TUNE_main.subs").write_text("{ AGEBD-TUNE@@agebd_env_suffix_upper@@ }\n")
|
||||
(ioc / "AGEBD-CPCL-TUNE_parameters.yaml").write_text(
|
||||
"ioc_host: sls-vserv-bd-01@@agebd_env_suffix_lower@@\nioc_port: @@ioc_port@@\n"
|
||||
)
|
||||
systemd = tmp_path / "systemd"
|
||||
systemd.mkdir()
|
||||
(systemd / "AGEBD-SERVICE-TUNE.service").write_text("Environment=AGEBD_ENV=@@agebd_env@@\n")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_renders_a_whole_service_for_dev(service_tree):
|
||||
subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR)
|
||||
|
||||
changed = render(subs, [service_tree / "ioc", service_tree / "systemd"])
|
||||
|
||||
assert len(changed) == 3
|
||||
assert "AGEBD-TUNE-DEV" in (service_tree / "ioc/AGEBD-CPCL-TUNE_main.subs").read_text()
|
||||
assert (service_tree / "ioc/AGEBD-CPCL-TUNE_parameters.yaml").read_text() == (
|
||||
"ioc_host: sls-vserv-bd-01-dev\nioc_port: 50007\n"
|
||||
)
|
||||
assert (service_tree / "systemd/AGEBD-SERVICE-TUNE.service").read_text() == (
|
||||
"Environment=AGEBD_ENV=dev\n"
|
||||
)
|
||||
|
||||
|
||||
def test_renders_a_whole_service_for_prod(service_tree):
|
||||
subs = Substitutions.for_service("prod", "tune", config_dir=CONFIG_DIR)
|
||||
|
||||
render(subs, [service_tree / "ioc", service_tree / "systemd"])
|
||||
|
||||
assert "{ AGEBD-TUNE }" in (service_tree / "ioc/AGEBD-CPCL-TUNE_main.subs").read_text()
|
||||
assert "sls-vserv-bd-01\n" in (service_tree / "ioc/AGEBD-CPCL-TUNE_parameters.yaml").read_text()
|
||||
|
||||
|
||||
def test_missing_path_is_skipped_not_fatal(service_tree, capsys):
|
||||
subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR)
|
||||
|
||||
render(subs, [service_tree / "systemd", service_tree / "does-not-exist"])
|
||||
|
||||
assert "skipped (does not exist)" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_leaves_nothing_unrendered(service_tree):
|
||||
subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR)
|
||||
|
||||
render(subs, [service_tree])
|
||||
|
||||
for path in service_tree.rglob("*"):
|
||||
if path.is_file():
|
||||
assert "@@" not in path.read_text(), path
|
||||
|
||||
|
||||
def test_rendering_is_idempotent(service_tree):
|
||||
subs = Substitutions.for_service("dev", "tune", config_dir=CONFIG_DIR)
|
||||
|
||||
assert len(render(subs, [service_tree])) == 3
|
||||
assert render(subs, [service_tree]) == []
|
||||
|
||||
|
||||
def test_per_service_token_without_service_fails_loudly(service_tree):
|
||||
"""Without --service there is no ioc_port, and that must not pass silently."""
|
||||
subs = Substitutions.for_env("dev", config_dir=CONFIG_DIR)
|
||||
|
||||
with pytest.raises(UnknownTokenError, match="ioc_port"):
|
||||
render(subs, [service_tree / "ioc"])
|
||||
|
||||
|
||||
def test_main_renders_via_the_command_line(service_tree):
|
||||
exit_code = main(["--env", "dev", "--service", "tune", str(service_tree / "systemd")])
|
||||
|
||||
assert exit_code == 0
|
||||
assert (service_tree / "systemd/AGEBD-SERVICE-TUNE.service").read_text() == (
|
||||
"Environment=AGEBD_ENV=dev\n"
|
||||
)
|
||||
|
||||
|
||||
def test_main_rejects_an_undeclared_environment(service_tree):
|
||||
with pytest.raises(SubstitutionError, match="staging"):
|
||||
main(["--env", "staging", "--service", "tune", str(service_tree / "systemd")])
|
||||
Reference in new issue
Block a user