fix(ci): validate worker bundles and resume package uploads

test/fixtures/curl_stub.py

@@ -0,0 +1,50 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+
+def main() -> None:
+    """Simulate immutable generic and RPM registries without any network access."""
+    args = sys.argv[1:]
+    calls = Path(os.environ["MOCK_CURL_CALLS"])
+    with calls.open("a") as output:
+        output.write(json.dumps(args) + "\n")
+    if os.environ.get("MOCK_CURL_EXIT"):
+        sys.exit(int(os.environ["MOCK_CURL_EXIT"]))
+    state_path = Path(os.environ["MOCK_CURL_STATE"])
+    state = json.loads(state_path.read_text()) if state_path.exists() else {"files": {}}
+    url = args[-1]
+    body = b""
+    if "--upload-file" in args:
+        source = Path(args[args.index("--upload-file") + 1])
+        target = url
+        if url.endswith("/rpm/upload"):
+            name = source.name.removesuffix(".rpm").removeprefix("bec-slurm-worker-")
+            version, architecture = name.rsplit(".", 1)
+            target = f"https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/{version.replace('_', '~')}/{architecture}"
+        fail_at = int(os.environ.get("MOCK_CURL_FAIL_AT", "0"))
+        request_count = len(calls.read_text().splitlines())
+        if not state.get("failed") and request_count == fail_at:
+            state["failed"] = True
+            status = 503
+        elif target in state["files"]:
+            status = 409
+        else:
+            state["files"][target] = source.read_bytes().hex()
+            status = 201
+    else:
+        status = 200 if url in state["files"] else 404
+        if status == 200:
+            body = bytes.fromhex(state["files"][url])
+    state_path.write_text(json.dumps(state))
+    if "--output" in args:
+        Path(args[args.index("--output") + 1]).write_bytes(body)
+    if "--write-out" in args:
+        print(status, end="")
+    if status >= 400 and "--fail-with-body" in args:
+        sys.exit(22)
+
+
+if __name__ == "__main__":
+    main()

test/test_worker_packages.py

@@ -1,5 +1,6 @@
 """Check Gitea package publishing without contacting external registries."""
 
+import hashlib
 import json
 import os
 import subprocess
@@ -8,6 +9,16 @@ from pathlib import Path
 
 import pytest
 
+
+def write_checksums(artifacts):
+    entries = []
+    for path in sorted(artifacts.iterdir()):
+        if path.name == "checksums.txt":
+            continue
+        entries.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()}  {path.name}\n")
+    (artifacts / "checksums.txt").write_text("".join(entries))
+
+
 PUBLISH_SCRIPT = Path(__file__).parents[1] / "worker/scripts/upload-packages.sh"
 
 
@@ -20,27 +31,21 @@ def publishing_environment(tmp_path):
         "bec-slurm-worker_0.2.0_linux_arm64.tar.gz",
         "bec-slurm-worker-0.2.0-1.x86_64.rpm",
         "bec-slurm-worker-0.2.0-1.aarch64.rpm",
-        "checksums.txt",
     ]
     for name in names:
-        (artifacts / name).write_text("test artifact")
+        (artifacts / name).write_text(f"test artifact {name}")
+    write_checksums(artifacts)
     calls = tmp_path / "calls.jsonl"
     curl = tmp_path / "curl"
-    curl.write_text(
-        f"#!{sys.executable}\n"
-        "import json, os, sys\n"
-        "from pathlib import Path\n"
-        "path = Path(os.environ['MOCK_CURL_CALLS'])\n"
-        "with path.open('a') as output:\n"
-        "    output.write(json.dumps(sys.argv[1:]) + '\\n')\n"
-        "sys.exit(int(os.environ.get('MOCK_CURL_EXIT', '0')))\n"
-    )
+    stub = Path(__file__).with_name("fixtures") / "curl_stub.py"
+    curl.write_text(f"#!{sys.executable}\n" + stub.read_text())
     curl.chmod(0o700)
     env = {
         **os.environ,
         "PATH": f"{tmp_path}{os.pathsep}{os.defpath}",
         "GITEA_REGISTRY_TOKEN": "example:test-token",
         "MOCK_CURL_CALLS": str(calls),
+        "MOCK_CURL_STATE": str(tmp_path / "registry.json"),
     }
     return artifacts, calls, env
 
@@ -71,12 +76,14 @@ def test_uploads_complete_bundle_and_indexes_rpms(publishing_environment):
     )
     for request in requests:
         assert request[request.index("--user") + 1] == "example:test-token"
-        assert "--fail-with-body" in request
+        assert "--write-out" in request
         assert "--max-time" in request
     assert "test-token" not in result.stdout + result.stderr
 
 
-@pytest.mark.parametrize("missing", ["*.rpm", "*.tar.gz", "checksums.txt"])
+@pytest.mark.parametrize(
+    "missing", ["*.rpm", "*.tar.gz", "checksums.txt", "*.aarch64.rpm", "*linux_arm64.tar.gz"]
+)
 def test_incomplete_packages_are_rejected_before_upload(publishing_environment, missing):
     artifacts, calls, env = publishing_environment
     for path in artifacts.glob(missing):
@@ -110,3 +117,84 @@ def test_missing_credentials_are_rejected_before_upload(publishing_environment):
     assert result.returncode != 0
     assert "GITEA_REGISTRY_TOKEN" in result.stderr
     assert not calls.exists()
+
+
+@pytest.mark.parametrize("fail_at", [2, 6, 7])
+def test_retry_resumes_partial_generic_and_rpm_uploads(publishing_environment, fail_at):
+    artifacts, calls, env = publishing_environment
+    env["MOCK_CURL_FAIL_AT"] = str(fail_at)
+    first = publish(artifacts, env)
+    assert first.returncode != 0
+    assert "HTTP 503" in first.stderr
+
+    second = publish(artifacts, env)
+    assert second.returncode == 0, second.stderr
+    assert "Identical package already published" in second.stdout
+    state = json.loads(Path(env["MOCK_CURL_STATE"]).read_text())
+    assert len(state["files"]) == 7
+    assert any("--upload-file" not in json.loads(line) for line in calls.read_text().splitlines())
+
+
+def test_successful_release_can_be_retried_without_overwriting(publishing_environment):
+    artifacts, _, env = publishing_environment
+    assert publish(artifacts, env).returncode == 0
+    before = json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"]
+    result = publish(artifacts, env)
+    assert result.returncode == 0, result.stderr
+    assert json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"] == before
+
+
+@pytest.mark.parametrize("registry", ["generic", "rpm"])
+def test_conflicting_remote_bytes_are_rejected(publishing_environment, registry):
+    artifacts, _, env = publishing_environment
+    assert publish(artifacts, env).returncode == 0
+    state_path = Path(env["MOCK_CURL_STATE"])
+    state = json.loads(state_path.read_text())
+    url = next(url for url in state["files"] if f"/{registry}/" in url)
+    state["files"][url] = b"different remote contents".hex()
+    state_path.write_text(json.dumps(state))
+
+    result = publish(artifacts, env)
+    assert result.returncode != 0
+    assert "Conflicting remote package" in result.stderr
+    assert json.loads(state_path.read_text())["files"][url] == b"different remote contents".hex()
+
+
+@pytest.mark.parametrize(
+    "damage", ["missing_entry", "duplicate_entry", "unknown_entry", "changed_bytes"]
+)
+def test_invalid_manifest_or_local_checksum_stops_all_uploads(publishing_environment, damage):
+    artifacts, calls, env = publishing_environment
+    manifest = artifacts / "checksums.txt"
+    lines = manifest.read_text().splitlines(keepends=True)
+    if damage == "missing_entry":
+        manifest.write_text("".join(lines[1:]))
+    elif damage == "duplicate_entry":
+        manifest.write_text("".join(lines + [lines[0]]))
+    elif damage == "unknown_entry":
+        manifest.write_text("".join(lines) + f"{'0' * 64}  unexpected.rpm\n")
+    else:
+        next(artifacts.glob("*.rpm")).write_text("corrupt package")
+
+    result = publish(artifacts, env)
+    assert result.returncode != 0
+    assert not calls.exists()
+
+
+def test_prerelease_rpm_retry_uses_rpm_metadata_version(publishing_environment):
+    artifacts, _, env = publishing_environment
+    for path in list(artifacts.iterdir()):
+        if path.name == "checksums.txt":
+            continue
+        version = "0.2.0_dev.1" if path.suffix == ".rpm" else "0.2.0-dev.1"
+        path.rename(path.with_name(path.name.replace("0.2.0", version)))
+    write_checksums(artifacts)
+
+    for _ in range(2):
+        result = publish(artifacts, env, version="0.2.0-dev.1")
+        assert result.returncode == 0, result.stderr
+    files = json.loads(Path(env["MOCK_CURL_STATE"]).read_text())["files"]
+    assert (
+        "https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/0.2.0~dev.1-1/x86_64"
+        in files
+    )

worker/.goreleaser.yaml

@@ -10,13 +10,23 @@ builds:
     goos: [linux]
     goarch: [amd64, arm64]
     flags: [-trimpath]
+    mod_timestamp: '{{ .CommitTimestamp }}'
     ldflags: [-s -w]
 
 archives:
   - formats: [tar.gz]
     name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}'
+    builds_info:
+      owner: root
+      group: root
+      mtime: '{{ .CommitDate }}'
     files:
-      - README.md
+      - src: README.md
+        info:
+          owner: root
+          group: root
+          mode: 0644
+          mtime: '{{ .CommitDate }}'
 
 nfpms:
   - id: bec-slurm-worker-rpm
@@ -29,12 +39,16 @@ nfpms:
     description: Execute Slurm batch workloads and forward logs, status, and heartbeats to Redis.
     formats: [rpm]
     bindir: /usr/bin
+    mtime: '{{ .CommitDate }}'
     dependencies:
       - bash
       - /usr/bin/getent
     contents:
       - src: README.md
         dst: /usr/share/doc/bec-slurm-worker/README.md
+        file_info:
+          mode: 0644
+          mtime: '{{ .CommitDate }}'
 
 checksum:
   name_template: checksums.txt

worker/README.md

@@ -29,6 +29,8 @@ with the value `username:token` and permission to publish BEC packages.
 Generic downloads:
 `https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/VERSION/FILENAME`
 
-The publisher refuses incomplete artifact sets and propagates upload failures.
-Existing package files are not overwritten. No packages are uploaded for PRs or
-untagged branch builds.
+The publisher requires both architectures and verifies the complete checksum manifest
+before uploading. Reruns compare existing remote files byte-for-byte and continue
+only when they match; conflicting files are never overwritten. Upload failures are
+reported, and a later run can resume missing generic files and RPM indexing.
+No packages are uploaded for PRs or untagged branch builds.

worker/scripts/upload-packages.sh

@@ -2,44 +2,97 @@
 # Match CAGO's Gitea authentication: username:token in GITEA_REGISTRY_TOKEN.
 set -euo pipefail
 
-if (( $# < 1 || $# > 2 )); then
-  echo 'Usage: upload-packages.sh VERSION [ARTIFACT_DIRECTORY]' >&2
+fail() {
+  echo "$*" >&2
   exit 1
+}
+
+if (( $# < 1 || $# > 2 )); then
+  fail 'Usage: upload-packages.sh VERSION [ARTIFACT_DIRECTORY]'
 fi
 : "${GITEA_REGISTRY_TOKEN:?Set GITEA_REGISTRY_TOKEN to username:token}"
 version=$1
 if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then
-  echo 'Expected a semantic version without a leading v.' >&2
-  exit 1
+  fail 'Expected a semantic version without a leading v.'
 fi
 artifact_directory=${2:-worker/dist}
 shopt -s nullglob
 archives=("$artifact_directory"/bec-slurm-worker_*.tar.gz)
 rpms=("$artifact_directory"/bec-slurm-worker-*.rpm)
+x86_rpms=("$artifact_directory"/bec-slurm-worker-*.x86_64.rpm)
+arm_rpms=("$artifact_directory"/bec-slurm-worker-*.aarch64.rpm)
 checksums="$artifact_directory/checksums.txt"
-if (( ${#archives[@]} == 0 || ${#rpms[@]} == 0 )) || [[ ! -s "$checksums" ]]; then
-  echo 'Linux archives, RPMs, and checksums are required; refusing an incomplete upload.' >&2
-  exit 1
+if (( ${#archives[@]} != 2 || ${#rpms[@]} != 2 || ${#x86_rpms[@]} != 1 || ${#arm_rpms[@]} != 1 )) ||
+   [[ ! -s "$checksums" ||
+      ! -s "$artifact_directory/bec-slurm-worker_${version}_linux_amd64.tar.gz" ||
+      ! -s "$artifact_directory/bec-slurm-worker_${version}_linux_arm64.tar.gz" ]]; then
+  fail 'Both Linux architectures, their RPMs, and checksums are required; refusing an incomplete upload.'
 fi
 
-# Keep a downloadable release bundle in the generic registry, including RPMs so
-# every entry in checksums.txt can be downloaded from the same package version.
+temporary=$(mktemp -d)
+trap 'rm -rf "$temporary"' EXIT
+: > "$temporary/manifest-names"
+: > "$temporary/expected-names"
+for package in "${archives[@]}" "${rpms[@]}"; do
+  file_name=$(basename "$package")
+  [[ "$file_name" =~ ^[A-Za-z0-9._+-]+$ ]] || fail "Invalid package filename: $file_name"
+  printf '%s\n' "$file_name" >> "$temporary/expected-names"
+done
+while read -r digest file_name extra || [[ -n "$digest$file_name$extra" ]]; do
+  [[ "$digest" =~ ^[[:xdigit:]]{64}$ && "$file_name" =~ ^[A-Za-z0-9._+-]+$ && -z "$extra" ]] ||
+    fail 'Invalid checksum manifest entry.'
+  printf '%s\n' "$file_name" >> "$temporary/manifest-names"
+done < "$checksums"
+sort "$temporary/expected-names" > "$temporary/expected-sorted"
+sort "$temporary/manifest-names" > "$temporary/manifest-sorted"
+cmp -s "$temporary/expected-sorted" "$temporary/manifest-sorted" ||
+  fail 'Checksum manifest must list each of the four packages exactly once.'
+# Validate all local bytes before making any registry requests.
+if command -v sha256sum >/dev/null 2>&1; then
+  (cd "$artifact_directory" && sha256sum --check checksums.txt)
+else
+  (cd "$artifact_directory" && shasum -a 256 --check checksums.txt)
+fi
+
+upload_package() {
+  local package=$1 upload_url=$2 download_url=$3 status
+  echo "Publishing $(basename "$package")"
+  # Inspect HTTP status explicitly: 409 is safe only after comparing existing bytes.
+  status=$(curl --silent --show-error \
+    --connect-timeout 15 --max-time 300 \
+    --user "$GITEA_REGISTRY_TOKEN" \
+    --output "$temporary/response" --write-out '%{http_code}' \
+    --upload-file "$package" "$upload_url")
+  case "$status" in
+    201) return ;;
+    409)
+      curl --fail-with-body --silent --show-error \
+        --connect-timeout 15 --max-time 300 \
+        --user "$GITEA_REGISTRY_TOKEN" \
+        --output "$temporary/existing" "$download_url"
+      cmp -s "$package" "$temporary/existing" ||
+        fail "Conflicting remote package: $(basename "$package"); refusing to overwrite."
+      echo 'Identical package already published; continuing.'
+      ;;
+    *) fail "Package upload failed (HTTP $status): $(basename "$package")" ;;
+  esac
+}
+
+# Keep the complete downloadable bundle together in the generic registry.
 for package in "${archives[@]}" "${rpms[@]}" "$checksums"; do
   file_name=$(basename "$package")
-  echo "Uploading $file_name to the BEC generic registry"
-  curl --fail-with-body --silent --show-error \
-    --connect-timeout 15 --max-time 300 \
-    --user "$GITEA_REGISTRY_TOKEN" \
-    --upload-file "$package" \
-    "https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/$version/$file_name"
+  url="https://gitea.psi.ch/api/packages/bec/generic/bec-slurm-worker/$version/$file_name"
+  upload_package "$package" "$url" "$url"
 done
 
-# Also index RPMs in the existing BEC RPM repository for Puppet/dnf deployment.
+# Gitea indexes RPMs by name, version-release, and architecture. Undo the filename's
+# tilde substitution to recover the RPM version used by Gitea's download endpoint.
 for package in "${rpms[@]}"; do
-  echo "Uploading $(basename "$package") to the BEC RPM registry"
-  curl --fail-with-body --silent --show-error \
-    --connect-timeout 15 --max-time 300 \
-    --user "$GITEA_REGISTRY_TOKEN" \
-    --upload-file "$package" \
-    'https://gitea.psi.ch/api/packages/bec/rpm/upload'
+  rpm_name=$(basename "$package" .rpm)
+  architecture=${rpm_name##*.}
+  rpm_version=${rpm_name%.*}
+  rpm_version=${rpm_version#bec-slurm-worker-}
+  rpm_version=${rpm_version//_/\~}
+  download_url="https://gitea.psi.ch/api/packages/bec/rpm/package/bec-slurm-worker/$rpm_version/$architecture"
+  upload_package "$package" 'https://gitea.psi.ch/api/packages/bec/rpm/upload' "$download_url"
 done