diff --git a/.gitea/scripts/deploy-and-restart-service.sh b/.gitea/scripts/deploy-and-restart-service.sh index 4bc01d5..9feeed3 100755 --- a/.gitea/scripts/deploy-and-restart-service.sh +++ b/.gitea/scripts/deploy-and-restart-service.sh @@ -13,11 +13,19 @@ SERVICE_DIR=/sls/bd/hla/${AGEBD_ENV}/services/${SERVICE_NAME_LOWER}/current SYSTEMD_UNIT_NAME=AGEBD-SERVICE-${SERVICE_NAME_UPPER}.service SYSTEMD_UNIT_FILE_PATH=${SERVICE_DIR}/systemd/${SYSTEMD_UNIT_NAME} -mkdir -p ${SERVICE_DIR} +# TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50 +# rm -rf "${SERVICE_DIR}" +# mkdir -p ${SERVICE_DIR} +# cp -r ./services/${SERVICE_NAME_LOWER}/current/* ${SERVICE_DIR}/ -rsync -avz --delete --exclude='.venv' \ - ./services/${SERVICE_NAME_LOWER}/current/ \ - ${SERVICE_DIR}/ +rsync -avz --delete \ + --exclude='.nfs*' \ + --exclude='.git/' \ + --exclude='.venv/' \ + --exclude='__pycache__/' \ + --exclude='*.pyc' \ + --exclude='.pytest_cache/' \ + ./services/${SERVICE_NAME_LOWER}/current/* ${SERVICE_DIR}/ if [ -f "${SYSTEMD_UNIT_FILE_PATH}" ]; then diff --git a/.gitea/workflows/add-new-service.yml b/.gitea/workflows/add-new-service.yml index dc6ae77..0d465d6 100644 --- a/.gitea/workflows/add-new-service.yml +++ b/.gitea/workflows/add-new-service.yml @@ -5,6 +5,10 @@ on: branches: - 'feature/add-service-*' +env: + UV_CACHE_DIR: /var/cache/uv + UV_LINK_MODE: hardlink + # TODO: when to deploy to prod? Manual (i.e. cli command)? jobs: deploy: @@ -14,18 +18,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Install and setup uv - uses: actions/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - version: "latest" - enable-cache: true # Speeds up runs by caching python dependencies # TODO: does this work? - - - name: Install Python 3.10 - run: uv python install 3.10 - - - name: Run tests + - name: Run cli tests run: | - cd cli && uv run pytest + echo $UV_CACHE_DIR + uv run --directory cli pytest - name: Parse Service Directory Name id: parse_branch diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 42b933e..ef7e9ff 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -21,6 +21,8 @@ concurrency: env: # commit tagged ? yes->prod : no->dev AGEBD_ENV: ${{ github.ref_type == 'tag' && 'prod' || 'dev' }} + UV_CACHE_DIR: /var/cache/uv + UV_LINK_MODE: hardlink jobs: deploy: @@ -46,15 +48,9 @@ jobs: services: - 'services/**' - - name: Install uv - uses: actions/setup-uv@v5 - - - name: Install Python 3.10 - run: uv python install 3.10 - - - name: Run tests + - name: Run core tests run: | - UV_CACHE_DIR="/sls/bd/hla/.cache/uv" + echo $UV_CACHE_DIR uv run --directory cli pytest uv run --directory packages/agebd pytest uv run --directory scripts pytest @@ -67,46 +63,111 @@ jobs: env: DEST_DIR: /sls/bd/hla/${{ env.AGEBD_ENV }}/bin run: | - mkdir -p "${DEST_DIR}" - rsync -av --delete --exclude='.nfs*' ./bin/ "${DEST_DIR}/" + # TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50 + # rm -rf "${DEST_DIR}" + # mkdir -p "${DEST_DIR}" + # cp -r ./bin/* "${DEST_DIR}/" + + rsync -avz --delete \ + --exclude='.nfs*' \ + --exclude='.git/' \ + --exclude='.venv/' \ + --exclude='__pycache__/' \ + --exclude='*.pyc' \ + --exclude='.pytest_cache/' \ + ./bin/* "${DEST_DIR}/" + - name: Agebd - deploy if: steps.filter.outputs.agebd == 'true' env: DEST_DIR: /sls/bd/hla/${{ env.AGEBD_ENV }}/packages/agebd run: | - mkdir -p "${DEST_DIR}" - rsync -av --delete --exclude='.nfs*' ./packages/agebd/ "${DEST_DIR}/" + # TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50 + # rm -rf "${DEST_DIR}" + # mkdir -p "${DEST_DIR}" + # cp -r ./packages/agebd/* "${DEST_DIR}/" + + rsync -avz --delete \ + --exclude='.nfs*' \ + --exclude='.git/' \ + --exclude='.venv/' \ + --exclude='__pycache__/' \ + --exclude='*.pyc' \ + --exclude='.pytest_cache/' \ + ./packages/agebd/* "${DEST_DIR}/" - name: Qt - deploy if: steps.filter.outputs.qt == 'true' env: DEST_DIR: /sls/bd/hla/${{ env.AGEBD_ENV }}/qt run: | - mkdir -p "${DEST_DIR}" - rsync -av --delete --exclude='.nfs*' ./qt/ "${DEST_DIR}/" + # TODO: see https://gitea.psi.ch/sls/hla_framework_bd/issues/50 + # rm -rf "${DEST_DIR}" + # mkdir -p "${DEST_DIR}" + # cp -r ./qt/* "${DEST_DIR}/" + + rsync -avz --delete \ + --exclude='.nfs*' \ + --exclude='.git/' \ + --exclude='.venv/' \ + --exclude='__pycache__/' \ + --exclude='*.pyc' \ + --exclude='.pytest_cache/' \ + ./qt/* "${DEST_DIR}/" # ----------------------------------------------------------------- - # Services Deployment + # Services Detection, Testing, and Deployment # ----------------------------------------------------------------- + - name: Detect Services to Test + # Run tests for a service if the service changed OR if core package agebd changed + if: steps.filter.outputs.services == 'true' || steps.filter.outputs.agebd == 'true' + id: detect-services + run: | + # If agebd changed, test ALL services to prevent breaking downstream dependencies + if [ "${{ steps.filter.outputs.agebd }}" = "true" ]; then + echo ":packages/agebd changed. Triggering tests for ALL services." + SERVICES_TO_TEST=$(ls -d services/*/ | xargs -n1 basename | tr '\n' ' ') + else + BEFORE="${{ github.event.before }}" + if [ -z "$BEFORE" ] || [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then + BEFORE="HEAD~1" + fi + + # Get space-separated list of modified service folder names + SERVICES_TO_TEST=$(git diff --name-only "$BEFORE" "${{ github.sha }}" | awk -F/ '$1=="services" && $2!="" {print $2}' | sort -u | tr '\n' ' ') + fi + + echo "services=$SERVICES_TO_TEST" >> $GITHUB_OUTPUT + echo "Services queued for testing: $SERVICES_TO_TEST" + + - name: Test Services + if: steps.detect-services.outputs.services != '' + run: | + SERVICES="${{ steps.detect-services.outputs.services }}" + + for SERVICE in $SERVICES; do + SERVICE_APP_DIR="services/$SERVICE/current/app" + if [ -d "$SERVICE_APP_DIR" ]; then + echo "==========================================" + echo "Running tests for: $SERVICE_APP_DIR" + echo "==========================================" + + uv run --directory "$SERVICE_APP_DIR" pytest + fi + done + - name: Deploy Changed Services + # ONLY deploy services that explicitly had file changes (prevents redeploying all services if only agebd changed) if: steps.filter.outputs.services == 'true' run: | - # 1. Determine git comparison point BEFORE="${{ github.event.before }}" if [ -z "$BEFORE" ] || [ "$BEFORE" = "0000000000000000000000000000000000000000" ]; then BEFORE="HEAD~1" fi - # 2. Extract list of modified service folder names CHANGED_SERVICES=$(git diff --name-only "$BEFORE" "${{ github.sha }}" | awk -F/ '$1=="services" && $2!="" {print $2}' | sort -u) - - if [ -z "$CHANGED_SERVICES" ]; then - echo "No service directory changes detected in diff." - exit 0 - fi - - # 3. Loop through changed services and invoke deployment script - echo "$CHANGED_SERVICES" | while read -r SERVICE; do + + for SERVICE in $CHANGED_SERVICES; do ./.gitea/scripts/deploy-and-restart-service.sh "${SERVICE}" "${AGEBD_ENV}" done diff --git a/bin/start_service.sh b/bin/start_service.sh index ce12f1a..d5d269a 100755 --- a/bin/start_service.sh +++ b/bin/start_service.sh @@ -42,4 +42,4 @@ source ${APP_DIR}/.venv/bin/activate # TODO: use `exec python3 ...` here? python -u -B -m agebd_${SVC_NAME_LOWER_UNDERSCORES}.main -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 diff --git a/docs/developer/cicd_runner.md b/docs/developer/cicd_runner.md index a43584d..1efa972 100644 --- a/docs/developer/cicd_runner.md +++ b/docs/developer/cicd_runner.md @@ -22,6 +22,9 @@ cd ~/gitea-runner /usr/local/bin/act_runner register --instance https://gitea.psi.ch/ --labels hla-dev --token --no-interactive ``` +Note, if you use a systemd service with a config file as in the example below, +the labels will be read from that file. + This generates a `.runner` file. @@ -42,6 +45,7 @@ tar -xf node-v20.11.1-linux-x64.tar.xz --strip-components=2 -C ~/.local/bin/ nod rm node-v20.11.1-linux-x64.tar.xz ``` + ## Manually run the runner ```shell @@ -105,6 +109,38 @@ SocketMode=0660 SocketGroup=podman ``` +Created uv cache dir, for `UV_CACHE_DIR`: + +``` +sudo mkdir -m 777 /var/cache/uv +``` + +Change the location of the act cache and working directory (uv .venvs are built here, take up lots of space) + +``` +sudo mkdir -m 777 /var/lib/act_runner/cache +sudo mkdir -m 777 /var/lib/act_runner/work +``` + +and edit `/etc/act_runner/config.yaml` + +``` +cache: + dir: "/var/lib/act_runner/cache" +host: + # The parent directory of a job's working directory. + # If it's empty, $HOME/.cache/act/ will be used. + workdir_parent: "/var/lib/act_runner/work" +``` + +Configure the runner labels: (edit `/etc/act_runner/config.yaml`) + +``` +runner: + labels: + - "hla-dev" +``` + Enable and restart: ``` diff --git a/packages/agebd/src/agebd/service/pvs.py b/packages/agebd/src/agebd/service/pvs.py index 88db81f..dd098e2 100644 --- a/packages/agebd/src/agebd/service/pvs.py +++ b/packages/agebd/src/agebd/service/pvs.py @@ -1,6 +1,6 @@ from agebd.pv import PVLink -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 class BasePVs: diff --git a/services/dbpm3curr/current/app/src/agebd_dbpm3curr/main.py b/services/dbpm3curr/current/app/src/agebd_dbpm3curr/main.py index 8976d38..011d64c 100644 --- a/services/dbpm3curr/current/app/src/agebd_dbpm3curr/main.py +++ b/services/dbpm3curr/current/app/src/agebd_dbpm3curr/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_dbpm3curr import PVs, Service from agebd_dbpm3curr.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/injectionguard/current/app/src/agebd_injectionguard/main.py b/services/injectionguard/current/app/src/agebd_injectionguard/main.py index 4be6b36..ad8395f 100644 --- a/services/injectionguard/current/app/src/agebd_injectionguard/main.py +++ b/services/injectionguard/current/app/src/agebd_injectionguard/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_injectionguard import PVs, Service from agebd_injectionguard.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/master/current/app/src/agebd_master/main.py b/services/master/current/app/src/agebd_master/main.py index 562c65d..44f31e9 100644 --- a/services/master/current/app/src/agebd_master/main.py +++ b/services/master/current/app/src/agebd_master/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_master import PVs, Service from agebd_master.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/plots/current/app/src/agebd_plots/main.py b/services/plots/current/app/src/agebd_plots/main.py index 34f4e6e..0d1ed2f 100644 --- a/services/plots/current/app/src/agebd_plots/main.py +++ b/services/plots/current/app/src/agebd_plots/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_plots import PVs, Service from agebd_plots.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/postmortemlog/current/app/src/agebd_postmortemlog/main.py b/services/postmortemlog/current/app/src/agebd_postmortemlog/main.py index 74b1b1b..22b0c6e 100644 --- a/services/postmortemlog/current/app/src/agebd_postmortemlog/main.py +++ b/services/postmortemlog/current/app/src/agebd_postmortemlog/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_postmortemlog import PVs, Service from agebd_postmortemlog.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/scrubbing/current/app/src/agebd_scrubbing/main.py b/services/scrubbing/current/app/src/agebd_scrubbing/main.py index 7a81b9d..2fcbe1a 100644 --- a/services/scrubbing/current/app/src/agebd_scrubbing/main.py +++ b/services/scrubbing/current/app/src/agebd_scrubbing/main.py @@ -6,7 +6,7 @@ from agebd_scrubbing import PVs, Service from agebd_scrubbing.runner import Runner from agebd_scrubbing.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/shifttool/current/app/src/agebd_shifttool/main.py b/services/shifttool/current/app/src/agebd_shifttool/main.py index cba0182..193909c 100644 --- a/services/shifttool/current/app/src/agebd_shifttool/main.py +++ b/services/shifttool/current/app/src/agebd_shifttool/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_shifttool import PVs, Service from agebd_shifttool.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/taubpm/current/app/src/agebd_taubpm/main.py b/services/taubpm/current/app/src/agebd_taubpm/main.py index 41a617f..f7cd48f 100644 --- a/services/taubpm/current/app/src/agebd_taubpm/main.py +++ b/services/taubpm/current/app/src/agebd_taubpm/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_taubpm import PVs, Service from agebd_taubpm.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/taupct/current/app/src/agebd_taupct/main.py b/services/taupct/current/app/src/agebd_taupct/main.py index bae3362..a05c57a 100644 --- a/services/taupct/current/app/src/agebd_taupct/main.py +++ b/services/taupct/current/app/src/agebd_taupct/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_taupct import PVs, Service from agebd_taupct.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/timing/current/app/src/agebd_timing/main.py b/services/timing/current/app/src/agebd_timing/main.py index 108a8ad..cda08a0 100644 --- a/services/timing/current/app/src/agebd_timing/main.py +++ b/services/timing/current/app/src/agebd_timing/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_timing import PVs, Service from agebd_timing.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/topuptool/current/app/src/agebd_topuptool/main.py b/services/topuptool/current/app/src/agebd_topuptool/main.py index 9411ce4..bac7316 100644 --- a/services/topuptool/current/app/src/agebd_topuptool/main.py +++ b/services/topuptool/current/app/src/agebd_topuptool/main.py @@ -6,7 +6,7 @@ from agebd_topuptool import PVs, Service from agebd_topuptool.runner import Runner from agebd_topuptool.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/tune/current/app/src/agebd_tune/main.py b/services/tune/current/app/src/agebd_tune/main.py index 4ac9a8c..11d68f8 100644 --- a/services/tune/current/app/src/agebd_tune/main.py +++ b/services/tune/current/app/src/agebd_tune/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_tune import PVs, Service from agebd_tune.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/tunebump/current/app/src/agebd_tunebump/main.py b/services/tunebump/current/app/src/agebd_tunebump/main.py index 0796a71..c2c4dae 100644 --- a/services/tunebump/current/app/src/agebd_tunebump/main.py +++ b/services/tunebump/current/app/src/agebd_tunebump/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_tunebump import PVs, Service from agebd_tunebump.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/tunefbx/current/app/src/agebd_tunefbx/main.py b/services/tunefbx/current/app/src/agebd_tunefbx/main.py index 6fb3e8a..5b947e6 100644 --- a/services/tunefbx/current/app/src/agebd_tunefbx/main.py +++ b/services/tunefbx/current/app/src/agebd_tunefbx/main.py @@ -6,7 +6,7 @@ from agebd.utils import init_logging from agebd_tunefbx import PVs, Service from agebd_tunefbx.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( diff --git a/services/tunefby/current/app/src/agebd_tunefby/main.py b/services/tunefby/current/app/src/agebd_tunefby/main.py index 42f00cc..08665aa 100644 --- a/services/tunefby/current/app/src/agebd_tunefby/main.py +++ b/services/tunefby/current/app/src/agebd_tunefby/main.py @@ -7,7 +7,7 @@ from agebd_tunefby import PVs, Service from agebd_tunefby.service import SERVICE_NAME -# TODO: remove: cicd test 26 +# TODO: remove: cicd test 51 def main( log_level: LogLevel = typer.Option(