Publish existing Linux builds to linked Gitea packages

This commit is contained in:
salman committed 2026-10-29 21:33:29 +01:00
1 parent 30a8b0ce80
commit 8bcc89d195
7 files changed
+331 -2

No files matched your search

+2
View File
@@ -30,8 +30,10 @@ jobs:
- name: Check runtime selection
env:
RUNTIME_RUN_ID: ${{ inputs.runtime_run_id }}
PACKAGE_RUN_ID: ${{ gitea.run_id }}
run: |
[[ "$RUNTIME_RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo 'A numeric runtime workflow run ID is required' >&2; exit 1; }
echo "Packaging run ID for Publish packages: $PACKAGE_RUN_ID"
- name: Download selected runtime
uses: https://github.com/actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+70
View File
@@ -0,0 +1,70 @@
name: Publish packages
on:
workflow_dispatch:
inputs:
package_run_id:
description: 'Workflow run ID of a successful Build and package run (not the runtime run ID)'
required: true
permissions:
contents: read
actions: read
packages: write
jobs:
publish:
runs-on: ubuntu-latest-full
timeout-minutes: 120
defaults:
run:
shell: bash
env:
GITEA_SERVER_URL: ${{ gitea.server_url }}
GITEA_REPOSITORY: ${{ gitea.repository }}
PACKAGE_RUN_ID: ${{ inputs.package_run_id }}
steps:
- name: Check out publisher
uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
github-server-url: ${{ gitea.server_url }}
persist-credentials: false
- name: Set up Node.js
uses: https://github.com/actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22.23.2'
- name: Check selected build succeeded
env:
ACTIONS_TOKEN: ${{ secrets.GITEA_TOKEN }}
run: |
node <<'JS'
(async () => {
const { GITEA_SERVER_URL, GITEA_REPOSITORY, PACKAGE_RUN_ID, ACTIONS_TOKEN } = process.env;
if (!/^[1-9][0-9]*$/.test(PACKAGE_RUN_ID)) throw new Error('A numeric packaging workflow run ID is required');
const response = await fetch(`${GITEA_SERVER_URL}/api/v1/repos/${GITEA_REPOSITORY}/actions/runs/${PACKAGE_RUN_ID}`, {
headers: { Authorization: `token ${ACTIONS_TOKEN}` },
redirect: 'error', signal: AbortSignal.timeout(60000)
});
if (!response.ok) throw new Error(`Cannot inspect packaging run: HTTP ${response.status}`);
const run = await response.json();
if (run.conclusion !== 'success') throw new Error('The selected packaging run did not finish successfully');
console.log(`Publishing package artifact from workflow run ${PACKAGE_RUN_ID}`);
})().catch(error => { console.error(error.message); process.exitCode = 1; });
JS
- name: Download existing packages
uses: https://github.com/actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: musrflow-linux-packages
path: artifacts
run-id: ${{ inputs.package_run_id }}
repository: ${{ gitea.repository }}
github-token: ${{ secrets.GITEA_TOKEN }}
- name: Publish and link packages to repository
env:
# Optional fallback for servers that restrict registry access by job tokens.
PACKAGES_TOKEN: ${{ secrets.PACKAGES_TOKEN || secrets.GITEA_TOKEN }}
run: node scripts/ci-publish-packages.js artifacts
+53 -1
View File
@@ -122,7 +122,8 @@ supply a runtime on a fresh runner. The two workflow files are
[build-runtime.yml](../.gitea/workflows/build-runtime.yml) and
[build-and-package.yml](../.gitea/workflows/build-and-package.yml).
The latter currently builds **Linux packages only**. Windows/WSL packaging and
release publication are separate next steps.
release-page publication are separate next steps. Package-registry publishing
is available separately as described below.
Both workflows use pinned artifact actions. PSI's runner must support their
Gitea adaptation and artifact storage. An upload preflight failure must be
@@ -130,6 +131,57 @@ resolved before compilation; a download failure should be checked for an
incorrect run ID, expired artifact, or Actions-token permissions. No personal
token is required by the normal same-repository workflow.
### Publish to the repository Packages section
Publishing is an explicit step, separate from compilation. This can publish an
already successful **Build and package** run without rebuilding ROOT, musrfit,
or the application.
1. Push the workflow changes, then open **Actions -> Publish packages -> Run
workflow** on the updated branch.
2. Enter **package_run_id**, the workflow run ID of the successful **Build and
package** run. This is **not** the runtime run ID or a job/task ID. New
packaging runs print it in **Check runtime selection** as `Packaging run ID
for Publish packages`. For older runs, open
`https://gitea.psi.ch/api/v1/repos/salman/muSRFlow/actions/runs` while signed
in, find the successful packaging entry by its `html_url`/`head_sha`, and use
its `id` field. Do not confuse that with `run_number` or assume that the
browser URL's number is the API run ID. Private repositories may require
authenticated API access rather than a browser session.
3. The publisher downloads the existing **musrflow-linux-packages** artifact,
checks every RPM/AppImage against its SHA-256 and MD5 sidecars, uploads the
files, and links the packages to this repository.
4. Open the repository's **Packages** tab. There are two Generic packages:
**musrflow** (application RPM/AppImage) and **musrflow-runtime** (runtime
RPM/AppImage). Each includes checksums and build-source information; the
runtime also includes its manifest.
Registry versions use `<app-version>-build.<packaging-run-ID>`, for example
`1.0.0-rc.5-build.12345`. The application version and downloaded filenames stay
unchanged. The version is read from the selected artifact, not from the newer
publisher checkout. Separate builds cannot silently replace each other.
Retrying the same publication checks existing files byte-for-byte by SHA-256
and skips identical uploads; different contents fail without deleting files.
Publication is not atomic: a failed upload may leave a partial package. Rerun
**Publish packages** with the same ID to complete it.
The workflow requests `packages: write` for its built-in job token. If PSI's
token policy refuses uploads or repository linking (HTTP 401/403), add a
repository **Actions secret** named **PACKAGES_TOKEN** with a token owned by
the package owner and scoped to **write:package**. Do not put tokens in files
or workflow inputs. Also ensure the repository's Packages feature is enabled.
The optional secret is used only for registry upload/linking; artifact download
still uses the job token.
This uses Gitea's [Generic Package Registry](https://docs.gitea.com/usage/packages/generic/),
not an RPM installation repository, and explicitly
[links packages to the repository](https://docs.gitea.com/usage/packages/overview/).
It creates neither a Git tag nor a Release page. Registry files are independent
of the 30-day Actions artifact retention, but remain subject to server package
cleanup policies. The offline runtime `.tar.gz`/`runtime-download.json` bundle
remains in the original Actions artifact; this first publisher exposes the
Linux RPM/AppImage installers only.
Do not run the current WSL builder concurrently against a shared container
engine: it still uses a fixed staging-container name. The native runtime CI
build now uses unique image/container names and scoped cleanup.
+1 -1
View File
@@ -16,7 +16,7 @@
"checksums:md5": "sh -c 'd=dist; [ -d \"$d\" ] || exit 0; for f in \"$d\"/*; do [ -f \"$f\" ] || continue; case \"$f\" in *.rpm|*.AppImage) base=$(basename \"$f\"); (cd \"$d\" && md5sum \"$base\" > \"$base.md5\");; esac; done'",
"checksums:sha256": "sh -c 'd=dist; [ -d \"$d\" ] || exit 0; for f in \"$d\"/*; do [ -f \"$f\" ] || continue; case \"$f\" in *.rpm|*.AppImage) base=$(basename \"$f\"); (cd \"$d\" && sha256sum \"$base\" > \"$base.sha256\");; esac; done'",
"checksums": "npm run checksums:md5 && npm run checksums:sha256",
"test:electron": "node tests/log-highlighting.test.js && node tests/action-state.test.js && node tests/electron-missing-data.test.js && node tests/electron-run-data.test.js && node tests/electron-msr-title.test.js && node tests/electron-musrfit-threads.test.js && node tests/electron-t0-prompt-peak.test.js && node tests/window-layering.test.js && node tests/msr-generation.test.js && node tests/root-plot-clean.test.js && node tests/parameter-plot.test.js && node tests/help-window.test.js && node tests/release-packaging.test.js && node tests/runtime-validation.test.js && node tests/local-runtime.test.js && node tests/addrun.test.js && node tests/fit-statistic.test.js && node tests/series-fit-planner.test.js && node tests/series-fit-result.test.js && node tests/series-fit-inventory.test.js && node tests/series-fit-msr-preparation.test.js && node tests/series-artifact-contract.test.js && node tests/series-global-merge.test.js && node tests/series-fit-staging.test.js && node tests/series-fit-runner.test.js && node tests/series-runner-adapters.test.js && node tests/series-fit-routing.test.js && node tests/series-action-chips.test.js && node tests/singlehist-parity.test.js && node tests/url-parameters.test.js && node tests/run-naming.test.js && node tests/electron-portable-naming.test.js && node tests/shared-configuration.test.js && node tests/session-persistence.test.js && node tests/reference-fixtures.test.js && node tests/electron-bridge.test.js && node tests/electron-bundled-runtime.test.js && node tests/wsl-runtime.test.js"
"test:electron": "node tests/log-highlighting.test.js && node tests/action-state.test.js && node tests/electron-missing-data.test.js && node tests/electron-run-data.test.js && node tests/electron-msr-title.test.js && node tests/electron-musrfit-threads.test.js && node tests/electron-t0-prompt-peak.test.js && node tests/window-layering.test.js && node tests/msr-generation.test.js && node tests/root-plot-clean.test.js && node tests/parameter-plot.test.js && node tests/help-window.test.js && node tests/release-packaging.test.js && node tests/ci-publish-packages.test.js && node tests/runtime-validation.test.js && node tests/local-runtime.test.js && node tests/addrun.test.js && node tests/fit-statistic.test.js && node tests/series-fit-planner.test.js && node tests/series-fit-result.test.js && node tests/series-fit-inventory.test.js && node tests/series-fit-msr-preparation.test.js && node tests/series-artifact-contract.test.js && node tests/series-global-merge.test.js && node tests/series-fit-staging.test.js && node tests/series-fit-runner.test.js && node tests/series-runner-adapters.test.js && node tests/series-fit-routing.test.js && node tests/series-action-chips.test.js && node tests/singlehist-parity.test.js && node tests/url-parameters.test.js && node tests/run-naming.test.js && node tests/electron-portable-naming.test.js && node tests/shared-configuration.test.js && node tests/session-persistence.test.js && node tests/reference-fixtures.test.js && node tests/electron-bridge.test.js && node tests/electron-bundled-runtime.test.js && node tests/wsl-runtime.test.js"
},
"repository": {
"type": "git",
+107
View File
@@ -0,0 +1,107 @@
// Copyright (C) 2026 Zaher Salman
// SPDX-License-Identifier: GPL-3.0-or-later
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
async function publishPackages(directory, env = process.env, request = fetch) {
const { GITEA_SERVER_URL, GITEA_REPOSITORY, PACKAGES_TOKEN, PACKAGE_RUN_ID } = env;
if (!/^[1-9][0-9]*$/.test(PACKAGE_RUN_ID || '')) throw new Error('A packaging workflow run ID is required');
if (!PACKAGES_TOKEN) throw new Error('A package write token is required');
if (!/^[\w.-]+\/[\w.-]+$/.test(GITEA_REPOSITORY || '')) throw new Error('Expected owner/repository');
const server = new URL(GITEA_SERVER_URL);
if (server.protocol !== 'https:' || server.username || server.password || server.search || server.hash) {
throw new Error('Gitea must use an HTTPS server URL without credentials, query, or fragment');
}
const base = server.href.replace(/\/$/, '');
const [owner, repository] = GITEA_REPOSITORY.split('/').map(encodeURIComponent);
const files = fs.readdirSync(directory);
const application = files.filter(name => /^musrflow-.+-x86_64\.rpm$/.test(name) && !name.startsWith('musrflow-runtime-'));
if (application.length !== 1) throw new Error('Expected exactly one application RPM');
// Read the version from the selected artifact, not the publisher checkout.
const version = application[0].slice('musrflow-'.length, -'-x86_64.rpm'.length);
if (!/^[\w.+-]+$/.test(version)) throw new Error('Invalid package version');
const registryVersion = `${version}-build.${PACKAGE_RUN_ID}`;
const groups = {
musrflow: [application[0], `muSRFlow-${version}-x86_64.AppImage`],
'musrflow-runtime': [`musrflow-runtime-${version}-x86_64.rpm`, `musrflow-runtime-${version}-x86_64.AppImage`]
};
// Validate the complete set before the first upload. Stream large runtime files.
for (const names of Object.values(groups)) {
for (const name of [...names]) {
const file = path.join(directory, name);
if (!fs.lstatSync(file).isFile() || fs.statSync(file).size === 0) throw new Error(`Missing or empty package: ${name}`);
const hashes = { sha256: crypto.createHash('sha256'), md5: crypto.createHash('md5') };
for await (const chunk of fs.createReadStream(file)) {
for (const hash of Object.values(hashes)) hash.update(chunk);
}
for (const [algorithm, hash] of Object.entries(hashes)) {
const sidecar = `${name}.${algorithm}`;
const checksum = fs.readFileSync(path.join(directory, sidecar), 'utf8').trim();
const expected = `${hash.digest('hex')} ${name}`;
if (checksum !== expected) throw new Error(`Checksum mismatch: ${sidecar}`);
names.push(sidecar);
}
}
names.push('build-source.txt');
}
groups['musrflow-runtime'].push('runtime-manifest.json');
for (const names of Object.values(groups)) {
for (const name of names) {
const stat = fs.lstatSync(path.join(directory, name));
if (!stat.isFile() || !stat.size) throw new Error(`Missing or empty artifact: ${name}`);
}
}
const headers = { Authorization: `token ${PACKAGES_TOKEN}` };
for (const [name, names] of Object.entries(groups)) {
for (const file of names) {
const url = `${base}/api/packages/${owner}/generic/${name}/${encodeURIComponent(registryVersion)}/${encodeURIComponent(file)}`;
const body = fs.createReadStream(path.join(directory, file));
let response;
try {
response = await request(url, {
method: 'PUT', headers: { ...headers, 'Content-Type': 'application/octet-stream',
'Content-Length': String(fs.statSync(path.join(directory, file)).size) },
body, duplex: 'half', redirect: 'error', signal: AbortSignal.timeout(30 * 60 * 1000)
});
} finally {
body.destroy();
}
// A retried publish can safely skip identical files, but never replace them.
if (response.status === 409) {
const existing = await request(url, {
headers, redirect: 'error', signal: AbortSignal.timeout(30 * 60 * 1000)
});
if (existing.status !== 200) throw new Error(`Cannot verify existing ${file}: HTTP ${existing.status}`);
const remoteHash = crypto.createHash('sha256');
for await (const chunk of existing.body) remoteHash.update(chunk);
const localHash = crypto.createHash('sha256');
for await (const chunk of fs.createReadStream(path.join(directory, file))) localHash.update(chunk);
if (remoteHash.digest('hex') !== localHash.digest('hex')) throw new Error(`Existing package differs: ${file}; nothing was overwritten`);
} else if (response.status !== 201) {
throw new Error(`Upload ${file}: HTTP ${response.status}. Check package write permissions and server upload limits`);
}
if (response.body) await response.body.cancel();
console.log(`Published ${name}/${registryVersion}/${file}`);
}
const linked = await request(`${base}/api/v1/packages/${owner}/generic/${name}/-/link/${repository}`, {
method: 'POST', headers, redirect: 'error', signal: AbortSignal.timeout(60000)
});
if (linked.status !== 201) throw new Error(`Link ${name} to repository: HTTP ${linked.status}`);
if (linked.body) await linked.body.cancel();
}
console.log(`Packages: ${base}/${owner}/${repository}/packages (version ${registryVersion})`);
return registryVersion;
}
module.exports = { publishPackages };
if (require.main === module) {
publishPackages(process.argv[2] || 'artifacts').catch(error => {
console.error(error.message);
process.exitCode = 1;
});
}
+88
View File
@@ -0,0 +1,88 @@
// Copyright (C) 2026 Zaher Salman
// SPDX-License-Identifier: GPL-3.0-or-later
const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const crypto = require('crypto');
const { publishPackages } = require('../scripts/ci-publish-packages');
async function main() {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'musrflow-publish-'));
const env = {
GITEA_SERVER_URL: 'https://gitea.example.test', GITEA_REPOSITORY: 'owner/muSRFlow',
PACKAGE_RUN_ID: '123', PACKAGES_TOKEN: 'test-token'
};
const stored = new Map();
const calls = [];
async function registry(url, options) {
calls.push({ url, method: options.method || 'GET' });
assert.strictEqual(options.headers.Authorization, 'token test-token');
assert.strictEqual(options.redirect, 'error');
if (options.method === 'PUT') {
const chunks = [];
for await (const chunk of options.body) chunks.push(chunk);
if (stored.has(url)) return new Response(null, { status: 409 });
stored.set(url, Buffer.concat(chunks));
return new Response(null, { status: 201 });
}
if (options.method === 'POST') {
assert(url.endsWith('/-/link/muSRFlow'));
return new Response(null, { status: 201 });
}
assert(stored.has(url));
return new Response(stored.get(url));
}
try {
const names = ['musrflow-9.8.7-x86_64.rpm', 'muSRFlow-9.8.7-x86_64.AppImage',
'musrflow-runtime-9.8.7-x86_64.rpm', 'musrflow-runtime-9.8.7-x86_64.AppImage'];
for (const name of names) {
fs.writeFileSync(path.join(directory, name), name);
for (const algorithm of ['md5', 'sha256']) {
const sum = crypto.createHash(algorithm).update(name).digest('hex');
fs.writeFileSync(path.join(directory, `${name}.${algorithm}`), `${sum} ${name}\n`);
}
}
fs.writeFileSync(path.join(directory, 'runtime-manifest.json'), '{}\n');
fs.writeFileSync(path.join(directory, 'build-source.txt'), 'muSRFlow=test\n');
// Unrelated artifact files must not be accidentally published.
fs.writeFileSync(path.join(directory, 'private.txt'), 'do not publish');
const missing = path.join(directory, names[3]);
fs.renameSync(missing, missing + '.backup');
await assert.rejects(publishPackages(directory, env, registry), /ENOENT/);
assert.strictEqual(calls.length, 0);
fs.renameSync(missing + '.backup', missing);
fs.appendFileSync(missing, 'corruption');
await assert.rejects(publishPackages(directory, env, registry), /Checksum mismatch/);
assert.strictEqual(calls.length, 0);
fs.writeFileSync(missing, names[3]);
await assert.rejects(publishPackages(directory, { ...env, PACKAGE_RUN_ID: '../123' }, registry), /run ID/);
await assert.rejects(publishPackages(directory, { ...env, PACKAGES_TOKEN: '' }, registry), /write token/);
await assert.rejects(publishPackages(directory, { ...env, GITEA_SERVER_URL: 'http://example.test' }, registry), /HTTPS/);
assert.strictEqual(calls.length, 0);
assert.strictEqual(await publishPackages(directory, env, registry), '9.8.7-build.123');
assert.strictEqual(stored.size, 15);
assert.strictEqual(calls.filter(call => call.method === 'POST').length, 2);
assert([...stored.keys()].every(url => url.includes('/9.8.7-build.123/')));
assert(![...stored.keys()].some(url => url.includes('private.txt')));
// An interrupted publish may be retried without deleting existing packages.
await publishPackages(directory, env, registry);
assert.strictEqual(calls.filter(call => call.method === 'GET').length, 15);
stored.set(stored.keys().next().value, Buffer.from('different published build'));
await assert.rejects(publishPackages(directory, env, registry), /Existing package differs/);
await assert.rejects(publishPackages(directory, env, async () => new Response(null, { status: 403 })), /HTTP 403/);
await assert.rejects(publishPackages(directory, env, async (url, options) => {
for await (const chunk of options.body || []) { /* consume upload */ }
return new Response(null, { status: options.method === 'POST' ? 403 : 201 });
}), /Link musrflow to repository: HTTP 403/);
assert(!calls.some(call => call.method === 'DELETE'));
console.log('Gitea package publishing tests passed');
} finally {
fs.rmSync(directory, { recursive: true, force: true });
}
}
main().catch(error => { console.error(error); process.exitCode = 1; });
+10
View File
@@ -95,4 +95,14 @@ assert(packageWorkflow.indexOf('node scripts/ci-restore-runtime.js') < packageWo
assert(packageWorkflow.includes('cp "dist/$file" "dist/$file.md5" "dist/$file.sha256" artifacts/'));
assert(!packageWorkflow.includes('|| true'));
const publishWorkflow = fs.readFileSync(path.join(root, '.gitea/workflows/publish-packages.yml'), 'utf8');
assert(publishWorkflow.includes('package_run_id:'));
assert(publishWorkflow.includes('packages: write'));
assert(publishWorkflow.includes("run.conclusion !== 'success'"));
assert(publishWorkflow.includes('name: musrflow-linux-packages'));
assert(publishWorkflow.includes('node scripts/ci-publish-packages.js artifacts'));
assert(!publishWorkflow.includes('electron-builder'));
assert(!publishWorkflow.includes('npm ci'));
assert(packageWorkflow.includes('Packaging run ID for Publish packages:'));
console.log('release packaging tests passed');