From 4477cc11be1ac8ff5069cbb931d334bca074b92e Mon Sep 17 00:00:00 2001 From: wakonig_k Date: Wed, 28 Oct 2026 17:35:49 +0100 Subject: [PATCH] feat!: package bec-slurm-worker for Linux and RPM deployment --- .github/workflows/ci.yml | 20 ++++ .github/workflows/go_worker.yml | 32 ++++- examples/submit_job.py | 2 +- slurm_manager/slurm_manager.py | 4 +- test/test_review_regressions.py | 2 +- test/test_worker_packages.py | 112 ++++++++++++++++++ worker/.goreleaser.yaml | 51 ++++++++ worker/Makefile | 9 +- worker/README.md | 34 ++++++ .../main.go | 4 +- .../main_test.go | 4 +- worker/internal/worker/worker.go | 4 +- worker/scripts/upload-packages.sh | 45 +++++++ 13 files changed, 307 insertions(+), 16 deletions(-) create mode 100644 test/test_worker_packages.py create mode 100644 worker/.goreleaser.yaml create mode 100644 worker/README.md rename worker/cmd/{slurm-worker => bec-slurm-worker}/main.go (90%) rename worker/cmd/{slurm-worker => bec-slurm-worker}/main_test.go (91%) create mode 100755 worker/scripts/upload-packages.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3c3131..804746f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,3 +29,23 @@ jobs: needs: check_pr_status if: needs.check_pr_status.outputs.branch-pr == '' uses: ./.github/workflows/go_worker.yml + + publish_worker: + needs: [formatter, unit_tests, go_worker] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: bec-slurm-worker-packages + path: worker/dist + - name: Publish Linux archives and RPMs to Gitea + env: + GITEA_REGISTRY_TOKEN: ${{ secrets.GITEA_REGISTRY_TOKEN }} + run: bash worker/scripts/upload-packages.sh "${GITHUB_REF_NAME#v}" diff --git a/.github/workflows/go_worker.yml b/.github/workflows/go_worker.yml index 63847a7..3943bef 100644 --- a/.github/workflows/go_worker.yml +++ b/.github/workflows/go_worker.yml @@ -8,6 +8,8 @@ jobs: working-directory: worker steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-go@v6 with: go-version-file: worker/go.mod @@ -16,9 +18,31 @@ jobs: run: test -z "$(gofmt -l .)" - name: Test and vet run: make test - - name: Build static Linux binary - run: make build + - uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: v2.18.0 + install-only: true + - name: Select package version + shell: bash + run: | + if [[ "$GITHUB_REF" == refs/tags/v* ]]; then + version="${GITHUB_REF_NAME#v}" + else + version="0.0.0-dev.${GITHUB_RUN_NUMBER}.g${GITHUB_SHA:0:12}" + fi + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo 'Expected a semantic version after the v tag prefix.' >&2 + exit 1 + fi + printf 'BUILD_VERSION=%s\n' "$version" >> "$GITHUB_ENV" + - name: Build Linux archives and RPMs + run: make package VERSION="$BUILD_VERSION" - uses: actions/upload-artifact@v4 with: - name: slurm-worker-linux-amd64 - path: dist/slurm-worker-linux-amd64 + name: bec-slurm-worker-packages + if-no-files-found: error + path: | + worker/dist/*.tar.gz + worker/dist/*.rpm + worker/dist/checksums.txt diff --git a/examples/submit_job.py b/examples/submit_job.py index 58919ac..f546b47 100644 --- a/examples/submit_job.py +++ b/examples/submit_job.py @@ -72,5 +72,5 @@ if __name__ == "__main__": script_path=Path(__file__).with_name("print_for_10s.sh"), working_directory="/das/home/your_username", log_directory="/das/home/your_username/slurm_logs", - worker_path="/das/home/your_username/slurm-worker", + worker_path="/das/home/your_username/bec-slurm-worker", ) diff --git a/slurm_manager/slurm_manager.py b/slurm_manager/slurm_manager.py index 87fc523..f081558 100644 --- a/slurm_manager/slurm_manager.py +++ b/slurm_manager/slurm_manager.py @@ -53,7 +53,7 @@ class SlurmManager: *, env_file: str | Path = ".env", rest_client: SlurmRestClient | None = None, - worker_path: str = "/usr/local/bin/slurm-worker", + worker_path: str = "/usr/bin/bec-slurm-worker", ) -> None: """Create REST/Redis clients and start background job monitoring. @@ -342,7 +342,7 @@ class SlurmManager: env_path: str | Path | None, job_id: str, *, - worker_path: str = "/usr/local/bin/slurm-worker", + worker_path: str = "/usr/bin/bec-slurm-worker", redis_address: str = "localhost:6379", ) -> str: """Package local Bash file contents for the compute-node Go worker. diff --git a/test/test_review_regressions.py b/test/test_review_regressions.py index bc02081..4cc6cef 100644 --- a/test/test_review_regressions.py +++ b/test/test_review_regressions.py @@ -92,7 +92,7 @@ def test_job_is_held_until_redis_acknowledges_subscriptions(manager): assert job["current_working_directory"] == "/remote/node/work" assert '"activation": ""' in script assert "stdout-test %02d/10" in script - assert "/usr/local/bin/slurm-worker" in script + assert "/usr/bin/bec-slurm-worker" in script future = next(iter(manager._job_tracker._future_registry.values())) topics = TopicInfo(future._job_id) assert manager.sub_client._subscription_ready[topics.event].is_set() diff --git a/test/test_worker_packages.py b/test/test_worker_packages.py new file mode 100644 index 0000000..9635825 --- /dev/null +++ b/test/test_worker_packages.py @@ -0,0 +1,112 @@ +"""Check Gitea package publishing without contacting external registries.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +PUBLISH_SCRIPT = Path(__file__).parents[1] / "worker/scripts/upload-packages.sh" + + +@pytest.fixture +def publishing_environment(tmp_path): + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + names = [ + "bec-slurm-worker_0.2.0_linux_amd64.tar.gz", + "bec-slurm-worker_0.2.0_linux_arm64.tar.gz", + "bec-slurm-worker-0.2.0-1.x86_64.rpm", + "bec-slurm-worker-0.2.0-1.aarch64.rpm", + "checksums.txt", + ] + for name in names: + (artifacts / name).write_text("test artifact") + calls = tmp_path / "calls.jsonl" + curl = tmp_path / "curl" + curl.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "path = Path(os.environ['MOCK_CURL_CALLS'])\n" + "with path.open('a') as output:\n" + " output.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "sys.exit(int(os.environ.get('MOCK_CURL_EXIT', '0')))\n" + ) + curl.chmod(0o700) + env = { + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.defpath}", + "GITEA_REGISTRY_TOKEN": "example:test-token", + "MOCK_CURL_CALLS": str(calls), + } + return artifacts, calls, env + + +def publish(artifacts, env, version="0.2.0"): + return subprocess.run( + ["bash", str(PUBLISH_SCRIPT), version, str(artifacts)], + env=env, + capture_output=True, + text=True, + check=False, + ) + + +def test_uploads_complete_bundle_and_indexes_rpms(publishing_environment): + artifacts, calls, env = publishing_environment + result = publish(artifacts, env) + assert result.returncode == 0, result.stderr + requests = [json.loads(line) for line in calls.read_text().splitlines()] + assert len(requests) == 7 + generic = "https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/0.2.0/" + assert {request[-1] for request in requests[:5]} == { + generic + path.name for path in artifacts.iterdir() + } + assert all( + request[-1] == "https://gitea.psi.ch/api/packages/bec/rpm/upload" + for request in requests[5:] + ) + for request in requests: + assert request[request.index("--user") + 1] == "example:test-token" + assert "--fail-with-body" in request + assert "--max-time" in request + assert "test-token" not in result.stdout + result.stderr + + +@pytest.mark.parametrize("missing", ["*.rpm", "*.tar.gz", "checksums.txt"]) +def test_incomplete_packages_are_rejected_before_upload(publishing_environment, missing): + artifacts, calls, env = publishing_environment + for path in artifacts.glob(missing): + path.unlink() + result = publish(artifacts, env) + assert result.returncode != 0 + assert "refusing an incomplete upload" in result.stderr + assert not calls.exists() + + +def test_upload_failure_stops_publication(publishing_environment): + artifacts, calls, env = publishing_environment + result = publish(artifacts, {**env, "MOCK_CURL_EXIT": "22"}) + assert result.returncode == 22 + assert len(calls.read_text().splitlines()) == 1 + + +@pytest.mark.parametrize("version", ["../other-package", "v0.2.0", "0.2.0/extra"]) +def test_invalid_version_cannot_change_upload_path(publishing_environment, version): + artifacts, calls, env = publishing_environment + result = publish(artifacts, env, version) + assert result.returncode != 0 + assert "semantic version" in result.stderr + assert not calls.exists() + + +def test_missing_credentials_are_rejected_before_upload(publishing_environment): + artifacts, calls, env = publishing_environment + env.pop("GITEA_REGISTRY_TOKEN") + result = publish(artifacts, env) + assert result.returncode != 0 + assert "GITEA_REGISTRY_TOKEN" in result.stderr + assert not calls.exists() diff --git a/worker/.goreleaser.yaml b/worker/.goreleaser.yaml new file mode 100644 index 0000000..7ae2c42 --- /dev/null +++ b/worker/.goreleaser.yaml @@ -0,0 +1,51 @@ +version: 2 +project_name: bec-slurm-worker + +builds: + - id: bec-slurm-worker + main: ./cmd/bec-slurm-worker + binary: bec-slurm-worker + env: + - CGO_ENABLED=0 + goos: [linux] + goarch: [amd64, arm64] + flags: [-trimpath] + ldflags: [-s -w] + +archives: + - formats: [tar.gz] + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' + files: + - README.md + +nfpms: + - id: bec-slurm-worker-rpm + ids: [bec-slurm-worker] + package_name: bec-slurm-worker + # Gitea generic package filenames do not allow RPM's prerelease tilde. + file_name_template: '{{ replace .ConventionalFileName "~" "_" }}' + homepage: https://github.com/bec-project/slurm_manager + maintainer: BEC Project + description: Execute Slurm batch workloads and forward logs, status, and heartbeats to Redis. + formats: [rpm] + bindir: /usr/bin + dependencies: + - bash + - /usr/bin/getent + contents: + - src: README.md + dst: /usr/share/doc/bec-slurm-worker/README.md + +checksum: + name_template: checksums.txt + +# CI supplies the release-tag version or a unique development version. +# Publishing to Gitea is a separate job after all CI checks have passed. +snapshot: + version_template: '{{ .Env.BUILD_VERSION }}' + +changelog: + disable: true + +release: + disable: true diff --git a/worker/Makefile b/worker/Makefile index 95a207c..1a0279a 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -1,8 +1,13 @@ TARGET_ARCH ?= amd64 +VERSION ?= 0.0.0-dev -.PHONY: build test +.PHONY: build package test build: - CGO_ENABLED=0 GOOS=linux GOARCH=$(TARGET_ARCH) go build -trimpath -ldflags='-s -w' -o ../dist/slurm-worker-linux-$(TARGET_ARCH) ./cmd/slurm-worker + CGO_ENABLED=0 GOOS=linux GOARCH=$(TARGET_ARCH) go build -trimpath -ldflags='-s -w' -o ../dist/bec-slurm-worker-linux-$(TARGET_ARCH) ./cmd/bec-slurm-worker + +package: export BUILD_VERSION = $(VERSION) +package: + goreleaser release --snapshot --clean test: go test -race -timeout 60s ./... diff --git a/worker/README.md b/worker/README.md new file mode 100644 index 0000000..21d266b --- /dev/null +++ b/worker/README.md @@ -0,0 +1,34 @@ +# bec-slurm-worker + +Compute-node executable for Slurm Manager. It runs one uploaded Bash workload +and forwards stdout, stderr, lifecycle status, and heartbeats to Redis. +It is launched by Slurm Manager, not as a system service. + +## Builds + +With Go installed, `make build` creates `../dist/bec-slurm-worker-linux-amd64`. +Use `TARGET_ARCH=arm64` for ARM64. + +With GoReleaser 2.18.0 installed, `make package VERSION=0.2.0` creates static Linux +archives, RPMs for x86_64 and aarch64, and SHA-256 checksums in `dist/`. +The RPM installs `/usr/bin/bec-slurm-worker` and requires Bash and `getent` for +compute-node account lookup. No Python or Go runtime is required on the node. + +## CI and Gitea + +Every CI run tests the worker and builds both architectures and RPMs. GitHub Actions +retains them in the `bec-slurm-worker-packages` artifact. Branch and PR builds use a +development version; version tags use the matching release version. + +After all CI jobs pass for a pushed `v*` tag, CI publishes the archives, RPMs, +and checksums to the Gitea generic package `bec/bec-slurm-worker`, and indexes +the RPMs in the BEC RPM registry, following CAGO's packaging approach. +`GITEA_REGISTRY_TOKEN` must be available to this repository as an Actions secret, +with the value `username:token` and permission to publish BEC packages. + +Generic downloads: +`https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/VERSION/FILENAME` + +The publisher refuses incomplete artifact sets and propagates upload failures. +Existing package files are not overwritten. No packages are uploaded for PRs or +untagged branch builds. diff --git a/worker/cmd/slurm-worker/main.go b/worker/cmd/bec-slurm-worker/main.go similarity index 90% rename from worker/cmd/slurm-worker/main.go rename to worker/cmd/bec-slurm-worker/main.go index 6912d51..1ff3382 100644 --- a/worker/cmd/slurm-worker/main.go +++ b/worker/cmd/bec-slurm-worker/main.go @@ -1,4 +1,4 @@ -// slurm-worker executes one uploaded Bash workload and forwards feedback to Redis. +// bec-slurm-worker executes one uploaded Bash workload and forwards feedback to Redis. package main import ( @@ -38,7 +38,7 @@ func main() { stop() _ = publisher.Close() if err != nil { - fmt.Fprintln(os.Stderr, "slurm-worker:", err) + fmt.Fprintln(os.Stderr, "bec-slurm-worker:", err) } os.Exit(code) } diff --git a/worker/cmd/slurm-worker/main_test.go b/worker/cmd/bec-slurm-worker/main_test.go similarity index 91% rename from worker/cmd/slurm-worker/main_test.go rename to worker/cmd/bec-slurm-worker/main_test.go index f8325ed..30ef0b7 100644 --- a/worker/cmd/slurm-worker/main_test.go +++ b/worker/cmd/bec-slurm-worker/main_test.go @@ -18,7 +18,7 @@ func TestCLIHelper(t *testing.T) { if os.Getenv("SLURM_WORKER_TEST_HELPER") != "1" { return } - os.Args = []string{"slurm-worker", "--job-id", "cli-job", "--redis-address", os.Getenv("SLURM_WORKER_TEST_REDIS")} + os.Args = []string{"bec-slurm-worker", "--job-id", "cli-job", "--redis-address", os.Getenv("BEC_SLURM_WORKER_TEST_REDIS")} main() } @@ -38,7 +38,7 @@ func TestCLIExecutesUploadedPayload(t *testing.T) { t.Fatal(err) } command := exec.CommandContext(ctx, executable, "-test.run=^TestCLIHelper$") - command.Env = append(os.Environ(), "SLURM_WORKER_TEST_HELPER=1", "SLURM_WORKER_TEST_REDIS="+server.Addr()) + command.Env = append(os.Environ(), "SLURM_WORKER_TEST_HELPER=1", "BEC_SLURM_WORKER_TEST_REDIS="+server.Addr()) command.Stdin = strings.NewReader(`{"script":"printf 'hello from CLI\\n'","activation":"export WORKLOAD_TEST=1"}`) output, err := command.CombinedOutput() if err != nil { diff --git a/worker/internal/worker/worker.go b/worker/internal/worker/worker.go index ce0ad13..cfea971 100644 --- a/worker/internal/worker/worker.go +++ b/worker/internal/worker/worker.go @@ -100,7 +100,7 @@ func Run(ctx context.Context, config Config, payload Payload, publisher Publishe if config.Stderr == nil { config.Stderr = io.Discard } - directory, err := os.MkdirTemp("", "slurm-worker-") + directory, err := os.MkdirTemp("", "bec-slurm-worker-") if err != nil { return 1, err } @@ -162,7 +162,7 @@ func Run(ctx context.Context, config Config, payload Payload, publisher Publishe stdout := &lineWriter{local: config.Stdout, stream: "stdout", publish: publish, onError: cancelWork} stderr := &lineWriter{local: config.Stderr, stream: "stderr", publish: publish, onError: cancelWork} - command := exec.Command("/bin/bash", "-c", shellSetup, "slurm-worker", activation, script) + command := exec.Command("/bin/bash", "-c", shellSetup, "bec-slurm-worker", activation, script) command.Env = append(os.Environ(), "HOME="+home) command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} command.Stdout, command.Stderr = stdout, stderr diff --git a/worker/scripts/upload-packages.sh b/worker/scripts/upload-packages.sh new file mode 100755 index 0000000..6fd05a5 --- /dev/null +++ b/worker/scripts/upload-packages.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Match CAGO's Gitea authentication: username:token in GITEA_REGISTRY_TOKEN. +set -euo pipefail + +if (( $# < 1 || $# > 2 )); then + echo 'Usage: upload-packages.sh VERSION [ARTIFACT_DIRECTORY]' >&2 + exit 1 +fi +: "${GITEA_REGISTRY_TOKEN:?Set GITEA_REGISTRY_TOKEN to username:token}" +version=$1 +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo 'Expected a semantic version without a leading v.' >&2 + exit 1 +fi +artifact_directory=${2:-worker/dist} +shopt -s nullglob +archives=("$artifact_directory"/bec-slurm-worker_*.tar.gz) +rpms=("$artifact_directory"/bec-slurm-worker-*.rpm) +checksums="$artifact_directory/checksums.txt" +if (( ${#archives[@]} == 0 || ${#rpms[@]} == 0 )) || [[ ! -s "$checksums" ]]; then + echo 'Linux archives, RPMs, and checksums are required; refusing an incomplete upload.' >&2 + exit 1 +fi + +# Keep a downloadable release bundle in the generic registry, including RPMs so +# every entry in checksums.txt can be downloaded from the same package version. +for package in "${archives[@]}" "${rpms[@]}" "$checksums"; do + file_name=$(basename "$package") + echo "Uploading $file_name to the BEC generic registry" + curl --fail-with-body --silent --show-error \ + --connect-timeout 15 --max-time 300 \ + --user "$GITEA_REGISTRY_TOKEN" \ + --upload-file "$package" \ + "https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/$version/$file_name" +done + +# Also index RPMs in the existing BEC RPM repository for Puppet/dnf deployment. +for package in "${rpms[@]}"; do + echo "Uploading $(basename "$package") to the BEC RPM registry" + curl --fail-with-body --silent --show-error \ + --connect-timeout 15 --max-time 300 \ + --user "$GITEA_REGISTRY_TOKEN" \ + --upload-file "$package" \ + 'https://gitea.psi.ch/api/packages/bec/rpm/upload' +done