diff --git a/test/fixtures/curl_stub.py b/test/fixtures/curl_stub.py new file mode 100644 index 0000000..d700e5d --- /dev/null +++ b/test/fixtures/curl_stub.py @@ -0,0 +1,50 @@ +import json +import os +import sys +from pathlib import Path + + +def main() -> None: + """Simulate immutable generic and RPM registries without any network access.""" + args = sys.argv[1:] + calls = Path(os.environ["MOCK_CURL_CALLS"]) + with calls.open("a") as output: + output.write(json.dumps(args) + "\n") + if os.environ.get("MOCK_CURL_EXIT"): + sys.exit(int(os.environ["MOCK_CURL_EXIT"])) + state_path = Path(os.environ["MOCK_CURL_STATE"]) + state = json.loads(state_path.read_text()) if state_path.exists() else {"files": {}} + url = args[-1] + body = b"" + if "--upload-file" in args: + source = Path(args[args.index("--upload-file") + 1]) + target = url + if url.endswith("/rpm/upload"): + name = source.name.removesuffix(".rpm").removeprefix("bec-slurm-worker-") + version, architecture = name.rsplit(".", 1) + target = f"https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/{version.replace('_', '~')}/{architecture}" + fail_at = int(os.environ.get("MOCK_CURL_FAIL_AT", "0")) + request_count = len(calls.read_text().splitlines()) + if not state.get("failed") and request_count == fail_at: + state["failed"] = True + status = 503 + elif target in state["files"]: + status = 409 + else: + state["files"][target] = source.read_bytes().hex() + status = 201 + else: + status = 200 if url in state["files"] else 404 + if status == 200: + body = bytes.fromhex(state["files"][url]) + state_path.write_text(json.dumps(state)) + if "--output" in args: + Path(args[args.index("--output") + 1]).write_bytes(body) + if "--write-out" in args: + print(status, end="") + if status >= 400 and "--fail-with-body" in args: + sys.exit(22) + + +if __name__ == "__main__": + main() diff --git a/test/test_worker_packages.py b/test/test_worker_packages.py index 9635825..874f69b 100644 --- a/test/test_worker_packages.py +++ b/test/test_worker_packages.py @@ -1,5 +1,6 @@ """Check Gitea package publishing without contacting external registries.""" +import hashlib import json import os import subprocess @@ -8,6 +9,16 @@ from pathlib import Path import pytest + +def write_checksums(artifacts): + entries = [] + for path in sorted(artifacts.iterdir()): + if path.name == "checksums.txt": + continue + entries.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n") + (artifacts / "checksums.txt").write_text("".join(entries)) + + PUBLISH_SCRIPT = Path(__file__).parents[1] / "worker/scripts/upload-packages.sh" @@ -20,27 +31,21 @@ def publishing_environment(tmp_path): "bec-slurm-worker_0.2.0_linux_arm64.tar.gz", "bec-slurm-worker-0.2.0-1.x86_64.rpm", "bec-slurm-worker-0.2.0-1.aarch64.rpm", - "checksums.txt", ] for name in names: - (artifacts / name).write_text("test artifact") + (artifacts / name).write_text(f"test artifact {name}") + write_checksums(artifacts) calls = tmp_path / "calls.jsonl" curl = tmp_path / "curl" - curl.write_text( - f"#!{sys.executable}\n" - "import json, os, sys\n" - "from pathlib import Path\n" - "path = Path(os.environ['MOCK_CURL_CALLS'])\n" - "with path.open('a') as output:\n" - " output.write(json.dumps(sys.argv[1:]) + '\\n')\n" - "sys.exit(int(os.environ.get('MOCK_CURL_EXIT', '0')))\n" - ) + stub = Path(__file__).with_name("fixtures") / "curl_stub.py" + curl.write_text(f"#!{sys.executable}\n" + stub.read_text()) curl.chmod(0o700) env = { **os.environ, "PATH": f"{tmp_path}{os.pathsep}{os.defpath}", "GITEA_REGISTRY_TOKEN": "example:test-token", "MOCK_CURL_CALLS": str(calls), + "MOCK_CURL_STATE": str(tmp_path / "registry.json"), } return artifacts, calls, env @@ -71,12 +76,14 @@ def test_uploads_complete_bundle_and_indexes_rpms(publishing_environment): ) for request in requests: assert request[request.index("--user") + 1] == "example:test-token" - assert "--fail-with-body" in request + assert "--write-out" in request assert "--max-time" in request assert "test-token" not in result.stdout + result.stderr -@pytest.mark.parametrize("missing", ["*.rpm", "*.tar.gz", "checksums.txt"]) +@pytest.mark.parametrize( + "missing", ["*.rpm", "*.tar.gz", "checksums.txt", "*.aarch64.rpm", "*linux_arm64.tar.gz"] +) def test_incomplete_packages_are_rejected_before_upload(publishing_environment, missing): artifacts, calls, env = publishing_environment for path in artifacts.glob(missing): @@ -110,3 +117,84 @@ def test_missing_credentials_are_rejected_before_upload(publishing_environment): assert result.returncode != 0 assert "GITEA_REGISTRY_TOKEN" in result.stderr assert not calls.exists() + + +@pytest.mark.parametrize("fail_at", [2, 6, 7]) +def test_retry_resumes_partial_generic_and_rpm_uploads(publishing_environment, fail_at): + artifacts, calls, env = publishing_environment + env["MOCK_CURL_FAIL_AT"] = str(fail_at) + first = publish(artifacts, env) + assert first.returncode != 0 + assert "HTTP 503" in first.stderr + + second = publish(artifacts, env) + assert second.returncode == 0, second.stderr + assert "Identical package already published" in second.stdout + state = json.loads(Path(env["MOCK_CURL_STATE"]).read_text()) + assert len(state["files"]) == 7 + assert any("--upload-file" not in json.loads(line) for line in calls.read_text().splitlines()) + + +def test_successful_release_can_be_retried_without_overwriting(publishing_environment): + artifacts, _, env = publishing_environment + assert publish(artifacts, env).returncode == 0 + before = json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"] + result = publish(artifacts, env) + assert result.returncode == 0, result.stderr + assert json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"] == before + + +@pytest.mark.parametrize("registry", ["generic", "rpm"]) +def test_conflicting_remote_bytes_are_rejected(publishing_environment, registry): + artifacts, _, env = publishing_environment + assert publish(artifacts, env).returncode == 0 + state_path = Path(env["MOCK_CURL_STATE"]) + state = json.loads(state_path.read_text()) + url = next(url for url in state["files"] if f"/{registry}/" in url) + state["files"][url] = b"different remote contents".hex() + state_path.write_text(json.dumps(state)) + + result = publish(artifacts, env) + assert result.returncode != 0 + assert "Conflicting remote package" in result.stderr + assert json.loads(state_path.read_text())["files"][url] == b"different remote contents".hex() + + +@pytest.mark.parametrize( + "damage", ["missing_entry", "duplicate_entry", "unknown_entry", "changed_bytes"] +) +def test_invalid_manifest_or_local_checksum_stops_all_uploads(publishing_environment, damage): + artifacts, calls, env = publishing_environment + manifest = artifacts / "checksums.txt" + lines = manifest.read_text().splitlines(keepends=True) + if damage == "missing_entry": + manifest.write_text("".join(lines[1:])) + elif damage == "duplicate_entry": + manifest.write_text("".join(lines + [lines[0]])) + elif damage == "unknown_entry": + manifest.write_text("".join(lines) + f"{'0' * 64} unexpected.rpm\n") + else: + next(artifacts.glob("*.rpm")).write_text("corrupt package") + + result = publish(artifacts, env) + assert result.returncode != 0 + assert not calls.exists() + + +def test_prerelease_rpm_retry_uses_rpm_metadata_version(publishing_environment): + artifacts, _, env = publishing_environment + for path in list(artifacts.iterdir()): + if path.name == "checksums.txt": + continue + version = "0.2.0_dev.1" if path.suffix == ".rpm" else "0.2.0-dev.1" + path.rename(path.with_name(path.name.replace("0.2.0", version))) + write_checksums(artifacts) + + for _ in range(2): + result = publish(artifacts, env, version="0.2.0-dev.1") + assert result.returncode == 0, result.stderr + files = json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"] + assert ( + "https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/0.2.0~dev.1-1/x86_64" + in files + ) diff --git a/worker/.goreleaser.yaml b/worker/.goreleaser.yaml index 7ae2c42..4277c92 100644 --- a/worker/.goreleaser.yaml +++ b/worker/.goreleaser.yaml @@ -10,13 +10,23 @@ builds: goos: [linux] goarch: [amd64, arm64] flags: [-trimpath] + mod_timestamp: '{{ .CommitTimestamp }}' ldflags: [-s -w] archives: - formats: [tar.gz] name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' + builds_info: + owner: root + group: root + mtime: '{{ .CommitDate }}' files: - - README.md + - src: README.md + info: + owner: root + group: root + mode: 0644 + mtime: '{{ .CommitDate }}' nfpms: - id: bec-slurm-worker-rpm @@ -29,12 +39,16 @@ nfpms: description: Execute Slurm batch workloads and forward logs, status, and heartbeats to Redis. formats: [rpm] bindir: /usr/bin + mtime: '{{ .CommitDate }}' dependencies: - bash - /usr/bin/getent contents: - src: README.md dst: /usr/share/doc/bec-slurm-worker/README.md + file_info: + mode: 0644 + mtime: '{{ .CommitDate }}' checksum: name_template: checksums.txt diff --git a/worker/README.md b/worker/README.md index 21d266b..7e118c5 100644 --- a/worker/README.md +++ b/worker/README.md @@ -29,6 +29,8 @@ with the value `username:token` and permission to publish BEC packages. Generic downloads: `https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/VERSION/FILENAME` -The publisher refuses incomplete artifact sets and propagates upload failures. -Existing package files are not overwritten. No packages are uploaded for PRs or -untagged branch builds. +The publisher requires both architectures and verifies the complete checksum manifest +before uploading. Reruns compare existing remote files byte-for-byte and continue +only when they match; conflicting files are never overwritten. Upload failures are +reported, and a later run can resume missing generic files and RPM indexing. +No packages are uploaded for PRs or untagged branch builds. diff --git a/worker/scripts/upload-packages.sh b/worker/scripts/upload-packages.sh index 6fd05a5..649038e 100755 --- a/worker/scripts/upload-packages.sh +++ b/worker/scripts/upload-packages.sh @@ -2,44 +2,97 @@ # Match CAGO's Gitea authentication: username:token in GITEA_REGISTRY_TOKEN. set -euo pipefail -if (( $# < 1 || $# > 2 )); then - echo 'Usage: upload-packages.sh VERSION [ARTIFACT_DIRECTORY]' >&2 +fail() { + echo "$*" >&2 exit 1 +} + +if (( $# < 1 || $# > 2 )); then + fail 'Usage: upload-packages.sh VERSION [ARTIFACT_DIRECTORY]' fi : "${GITEA_REGISTRY_TOKEN:?Set GITEA_REGISTRY_TOKEN to username:token}" version=$1 if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then - echo 'Expected a semantic version without a leading v.' >&2 - exit 1 + fail 'Expected a semantic version without a leading v.' fi artifact_directory=${2:-worker/dist} shopt -s nullglob archives=("$artifact_directory"/bec-slurm-worker_*.tar.gz) rpms=("$artifact_directory"/bec-slurm-worker-*.rpm) +x86_rpms=("$artifact_directory"/bec-slurm-worker-*.x86_64.rpm) +arm_rpms=("$artifact_directory"/bec-slurm-worker-*.aarch64.rpm) checksums="$artifact_directory/checksums.txt" -if (( ${#archives[@]} == 0 || ${#rpms[@]} == 0 )) || [[ ! -s "$checksums" ]]; then - echo 'Linux archives, RPMs, and checksums are required; refusing an incomplete upload.' >&2 - exit 1 +if (( ${#archives[@]} != 2 || ${#rpms[@]} != 2 || ${#x86_rpms[@]} != 1 || ${#arm_rpms[@]} != 1 )) || + [[ ! -s "$checksums" || + ! -s "$artifact_directory/bec-slurm-worker_${version}_linux_amd64.tar.gz" || + ! -s "$artifact_directory/bec-slurm-worker_${version}_linux_arm64.tar.gz" ]]; then + fail 'Both Linux architectures, their RPMs, and checksums are required; refusing an incomplete upload.' fi -# Keep a downloadable release bundle in the generic registry, including RPMs so -# every entry in checksums.txt can be downloaded from the same package version. +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT +: > "$temporary/manifest-names" +: > "$temporary/expected-names" +for package in "${archives[@]}" "${rpms[@]}"; do + file_name=$(basename "$package") + [[ "$file_name" =~ ^[A-Za-z0-9._+-]+$ ]] || fail "Invalid package filename: $file_name" + printf '%s\n' "$file_name" >> "$temporary/expected-names" +done +while read -r digest file_name extra || [[ -n "$digest$file_name$extra" ]]; do + [[ "$digest" =~ ^[[:xdigit:]]{64}$ && "$file_name" =~ ^[A-Za-z0-9._+-]+$ && -z "$extra" ]] || + fail 'Invalid checksum manifest entry.' + printf '%s\n' "$file_name" >> "$temporary/manifest-names" +done < "$checksums" +sort "$temporary/expected-names" > "$temporary/expected-sorted" +sort "$temporary/manifest-names" > "$temporary/manifest-sorted" +cmp -s "$temporary/expected-sorted" "$temporary/manifest-sorted" || + fail 'Checksum manifest must list each of the four packages exactly once.' +# Validate all local bytes before making any registry requests. +if command -v sha256sum >/dev/null 2>&1; then + (cd "$artifact_directory" && sha256sum --check checksums.txt) +else + (cd "$artifact_directory" && shasum -a 256 --check checksums.txt) +fi + +upload_package() { + local package=$1 upload_url=$2 download_url=$3 status + echo "Publishing $(basename "$package")" + # Inspect HTTP status explicitly: 409 is safe only after comparing existing bytes. + status=$(curl --silent --show-error \ + --connect-timeout 15 --max-time 300 \ + --user "$GITEA_REGISTRY_TOKEN" \ + --output "$temporary/response" --write-out '%{http_code}' \ + --upload-file "$package" "$upload_url") + case "$status" in + 201) return ;; + 409) + curl --fail-with-body --silent --show-error \ + --connect-timeout 15 --max-time 300 \ + --user "$GITEA_REGISTRY_TOKEN" \ + --output "$temporary/existing" "$download_url" + cmp -s "$package" "$temporary/existing" || + fail "Conflicting remote package: $(basename "$package"); refusing to overwrite." + echo 'Identical package already published; continuing.' + ;; + *) fail "Package upload failed (HTTP $status): $(basename "$package")" ;; + esac +} + +# Keep the complete downloadable bundle together in the generic registry. for package in "${archives[@]}" "${rpms[@]}" "$checksums"; do file_name=$(basename "$package") - echo "Uploading $file_name to the BEC generic registry" - curl --fail-with-body --silent --show-error \ - --connect-timeout 15 --max-time 300 \ - --user "$GITEA_REGISTRY_TOKEN" \ - --upload-file "$package" \ - "https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/$version/$file_name" + url="https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/$version/$file_name" + upload_package "$package" "$url" "$url" done -# Also index RPMs in the existing BEC RPM repository for Puppet/dnf deployment. +# Gitea indexes RPMs by name, version-release, and architecture. Undo the filename's +# tilde substitution to recover the RPM version used by Gitea's download endpoint. for package in "${rpms[@]}"; do - echo "Uploading $(basename "$package") to the BEC RPM registry" - curl --fail-with-body --silent --show-error \ - --connect-timeout 15 --max-time 300 \ - --user "$GITEA_REGISTRY_TOKEN" \ - --upload-file "$package" \ - 'https://gitea.psi.ch/api/packages/bec/rpm/upload' + rpm_name=$(basename "$package" .rpm) + architecture=${rpm_name##*.} + rpm_version=${rpm_name%.*} + rpm_version=${rpm_version#bec-slurm-worker-} + rpm_version=${rpm_version//_/\~} + download_url="https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/$rpm_version/$architecture" + upload_package "$package" 'https://gitea.psi.ch/api/packages/bec/rpm/upload' "$download_url" done