diff --git a/.gitea/workflows/build-and-package.yml b/.gitea/workflows/build-and-package.yml index 3aad01a..ca2ebb5 100644 --- a/.gitea/workflows/build-and-package.yml +++ b/.gitea/workflows/build-and-package.yml @@ -30,8 +30,10 @@ jobs: - name: Check runtime selection env: RUNTIME_RUN_ID: ${{ inputs.runtime_run_id }} + PACKAGE_RUN_ID: ${{ gitea.run_id }} run: | [[ "$RUNTIME_RUN_ID" =~ ^[1-9][0-9]*$ ]] || { echo 'A numeric runtime workflow run ID is required' >&2; exit 1; } + echo "Packaging run ID for Publish packages: $PACKAGE_RUN_ID" - name: Download selected runtime uses: https://github.com/actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 diff --git a/.gitea/workflows/publish-packages.yml b/.gitea/workflows/publish-packages.yml new file mode 100644 index 0000000..5e8e9d3 --- /dev/null +++ b/.gitea/workflows/publish-packages.yml @@ -0,0 +1,70 @@ +name: Publish packages + +on: + workflow_dispatch: + inputs: + package_run_id: + description: 'Workflow run ID of a successful Build and package run (not the runtime run ID)' + required: true + +permissions: + contents: read + actions: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest-full + timeout-minutes: 120 + defaults: + run: + shell: bash + env: + GITEA_SERVER_URL: ${{ gitea.server_url }} + GITEA_REPOSITORY: ${{ gitea.repository }} + PACKAGE_RUN_ID: ${{ inputs.package_run_id }} + steps: + - name: Check out publisher + uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + github-server-url: ${{ gitea.server_url }} + persist-credentials: false + + - name: Set up Node.js + uses: https://github.com/actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.2' + + - name: Check selected build succeeded + env: + ACTIONS_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: | + node <<'JS' + (async () => { + const { GITEA_SERVER_URL, GITEA_REPOSITORY, PACKAGE_RUN_ID, ACTIONS_TOKEN } = process.env; + if (!/^[1-9][0-9]*$/.test(PACKAGE_RUN_ID)) throw new Error('A numeric packaging workflow run ID is required'); + const response = await fetch(`${GITEA_SERVER_URL}/api/v1/repos/${GITEA_REPOSITORY}/actions/runs/${PACKAGE_RUN_ID}`, { + headers: { Authorization: `token ${ACTIONS_TOKEN}` }, + redirect: 'error', signal: AbortSignal.timeout(60000) + }); + if (!response.ok) throw new Error(`Cannot inspect packaging run: HTTP ${response.status}`); + const run = await response.json(); + if (run.conclusion !== 'success') throw new Error('The selected packaging run did not finish successfully'); + console.log(`Publishing package artifact from workflow run ${PACKAGE_RUN_ID}`); + })().catch(error => { console.error(error.message); process.exitCode = 1; }); + JS + + - name: Download existing packages + uses: https://github.com/actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: musrflow-linux-packages + path: artifacts + run-id: ${{ inputs.package_run_id }} + repository: ${{ gitea.repository }} + github-token: ${{ secrets.GITEA_TOKEN }} + + - name: Publish and link packages to repository + env: + # Optional fallback for servers that restrict registry access by job tokens. + PACKAGES_TOKEN: ${{ secrets.PACKAGES_TOKEN || secrets.GITEA_TOKEN }} + run: node scripts/ci-publish-packages.js artifacts diff --git a/docs/DESKTOP_DEPLOYMENT.md b/docs/DESKTOP_DEPLOYMENT.md index 5fd2d7a..50689b2 100644 --- a/docs/DESKTOP_DEPLOYMENT.md +++ b/docs/DESKTOP_DEPLOYMENT.md @@ -122,7 +122,8 @@ supply a runtime on a fresh runner. The two workflow files are [build-runtime.yml](../.gitea/workflows/build-runtime.yml) and [build-and-package.yml](../.gitea/workflows/build-and-package.yml). The latter currently builds **Linux packages only**. Windows/WSL packaging and -release publication are separate next steps. +release-page publication are separate next steps. Package-registry publishing +is available separately as described below. Both workflows use pinned artifact actions. PSI's runner must support their Gitea adaptation and artifact storage. An upload preflight failure must be @@ -130,6 +131,57 @@ resolved before compilation; a download failure should be checked for an incorrect run ID, expired artifact, or Actions-token permissions. No personal token is required by the normal same-repository workflow. +### Publish to the repository Packages section + +Publishing is an explicit step, separate from compilation. This can publish an +already successful **Build and package** run without rebuilding ROOT, musrfit, +or the application. + +1. Push the workflow changes, then open **Actions -> Publish packages -> Run + workflow** on the updated branch. +2. Enter **package_run_id**, the workflow run ID of the successful **Build and + package** run. This is **not** the runtime run ID or a job/task ID. New + packaging runs print it in **Check runtime selection** as `Packaging run ID + for Publish packages`. For older runs, open + `https://gitea.psi.ch/api/v1/repos/salman/muSRFlow/actions/runs` while signed + in, find the successful packaging entry by its `html_url`/`head_sha`, and use + its `id` field. Do not confuse that with `run_number` or assume that the + browser URL's number is the API run ID. Private repositories may require + authenticated API access rather than a browser session. +3. The publisher downloads the existing **musrflow-linux-packages** artifact, + checks every RPM/AppImage against its SHA-256 and MD5 sidecars, uploads the + files, and links the packages to this repository. +4. Open the repository's **Packages** tab. There are two Generic packages: + **musrflow** (application RPM/AppImage) and **musrflow-runtime** (runtime + RPM/AppImage). Each includes checksums and build-source information; the + runtime also includes its manifest. + +Registry versions use `-build.`, for example +`1.0.0-rc.5-build.12345`. The application version and downloaded filenames stay +unchanged. The version is read from the selected artifact, not from the newer +publisher checkout. Separate builds cannot silently replace each other. +Retrying the same publication checks existing files byte-for-byte by SHA-256 +and skips identical uploads; different contents fail without deleting files. +Publication is not atomic: a failed upload may leave a partial package. Rerun +**Publish packages** with the same ID to complete it. + +The workflow requests `packages: write` for its built-in job token. If PSI's +token policy refuses uploads or repository linking (HTTP 401/403), add a +repository **Actions secret** named **PACKAGES_TOKEN** with a token owned by +the package owner and scoped to **write:package**. Do not put tokens in files +or workflow inputs. Also ensure the repository's Packages feature is enabled. +The optional secret is used only for registry upload/linking; artifact download +still uses the job token. + +This uses Gitea's [Generic Package Registry](https://docs.gitea.com/usage/packages/generic/), +not an RPM installation repository, and explicitly +[links packages to the repository](https://docs.gitea.com/usage/packages/overview/). +It creates neither a Git tag nor a Release page. Registry files are independent +of the 30-day Actions artifact retention, but remain subject to server package +cleanup policies. The offline runtime `.tar.gz`/`runtime-download.json` bundle +remains in the original Actions artifact; this first publisher exposes the +Linux RPM/AppImage installers only. + Do not run the current WSL builder concurrently against a shared container engine: it still uses a fixed staging-container name. The native runtime CI build now uses unique image/container names and scoped cleanup. diff --git a/package.json b/package.json index 9a8cfa1..6dcc517 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "checksums:md5": "sh -c 'd=dist; [ -d \"$d\" ] || exit 0; for f in \"$d\"/*; do [ -f \"$f\" ] || continue; case \"$f\" in *.rpm|*.AppImage) base=$(basename \"$f\"); (cd \"$d\" && md5sum \"$base\" > \"$base.md5\");; esac; done'", "checksums:sha256": "sh -c 'd=dist; [ -d \"$d\" ] || exit 0; for f in \"$d\"/*; do [ -f \"$f\" ] || continue; case \"$f\" in *.rpm|*.AppImage) base=$(basename \"$f\"); (cd \"$d\" && sha256sum \"$base\" > \"$base.sha256\");; esac; done'", "checksums": "npm run checksums:md5 && npm run checksums:sha256", - "test:electron": "node tests/log-highlighting.test.js && node tests/action-state.test.js && node tests/electron-missing-data.test.js && node tests/electron-run-data.test.js && node tests/electron-msr-title.test.js && node tests/electron-musrfit-threads.test.js && node tests/electron-t0-prompt-peak.test.js && node tests/window-layering.test.js && node tests/msr-generation.test.js && node tests/root-plot-clean.test.js && node tests/parameter-plot.test.js && node tests/help-window.test.js && node tests/release-packaging.test.js && node tests/runtime-validation.test.js && node tests/local-runtime.test.js && node tests/addrun.test.js && node tests/fit-statistic.test.js && node tests/series-fit-planner.test.js && node tests/series-fit-result.test.js && node tests/series-fit-inventory.test.js && node tests/series-fit-msr-preparation.test.js && node tests/series-artifact-contract.test.js && node tests/series-global-merge.test.js && node tests/series-fit-staging.test.js && node tests/series-fit-runner.test.js && node tests/series-runner-adapters.test.js && node tests/series-fit-routing.test.js && node tests/series-action-chips.test.js && node tests/singlehist-parity.test.js && node tests/url-parameters.test.js && node tests/run-naming.test.js && node tests/electron-portable-naming.test.js && node tests/shared-configuration.test.js && node tests/session-persistence.test.js && node tests/reference-fixtures.test.js && node tests/electron-bridge.test.js && node tests/electron-bundled-runtime.test.js && node tests/wsl-runtime.test.js" + "test:electron": "node tests/log-highlighting.test.js && node tests/action-state.test.js && node tests/electron-missing-data.test.js && node tests/electron-run-data.test.js && node tests/electron-msr-title.test.js && node tests/electron-musrfit-threads.test.js && node tests/electron-t0-prompt-peak.test.js && node tests/window-layering.test.js && node tests/msr-generation.test.js && node tests/root-plot-clean.test.js && node tests/parameter-plot.test.js && node tests/help-window.test.js && node tests/release-packaging.test.js && node tests/ci-publish-packages.test.js && node tests/runtime-validation.test.js && node tests/local-runtime.test.js && node tests/addrun.test.js && node tests/fit-statistic.test.js && node tests/series-fit-planner.test.js && node tests/series-fit-result.test.js && node tests/series-fit-inventory.test.js && node tests/series-fit-msr-preparation.test.js && node tests/series-artifact-contract.test.js && node tests/series-global-merge.test.js && node tests/series-fit-staging.test.js && node tests/series-fit-runner.test.js && node tests/series-runner-adapters.test.js && node tests/series-fit-routing.test.js && node tests/series-action-chips.test.js && node tests/singlehist-parity.test.js && node tests/url-parameters.test.js && node tests/run-naming.test.js && node tests/electron-portable-naming.test.js && node tests/shared-configuration.test.js && node tests/session-persistence.test.js && node tests/reference-fixtures.test.js && node tests/electron-bridge.test.js && node tests/electron-bundled-runtime.test.js && node tests/wsl-runtime.test.js" }, "repository": { "type": "git", diff --git a/scripts/ci-publish-packages.js b/scripts/ci-publish-packages.js new file mode 100644 index 0000000..b6d9d00 --- /dev/null +++ b/scripts/ci-publish-packages.js @@ -0,0 +1,107 @@ +// Copyright (C) 2026 Zaher Salman +// SPDX-License-Identifier: GPL-3.0-or-later + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +async function publishPackages(directory, env = process.env, request = fetch) { + const { GITEA_SERVER_URL, GITEA_REPOSITORY, PACKAGES_TOKEN, PACKAGE_RUN_ID } = env; + if (!/^[1-9][0-9]*$/.test(PACKAGE_RUN_ID || '')) throw new Error('A packaging workflow run ID is required'); + if (!PACKAGES_TOKEN) throw new Error('A package write token is required'); + if (!/^[\w.-]+\/[\w.-]+$/.test(GITEA_REPOSITORY || '')) throw new Error('Expected owner/repository'); + const server = new URL(GITEA_SERVER_URL); + if (server.protocol !== 'https:' || server.username || server.password || server.search || server.hash) { + throw new Error('Gitea must use an HTTPS server URL without credentials, query, or fragment'); + } + const base = server.href.replace(/\/$/, ''); + const [owner, repository] = GITEA_REPOSITORY.split('/').map(encodeURIComponent); + const files = fs.readdirSync(directory); + const application = files.filter(name => /^musrflow-.+-x86_64\.rpm$/.test(name) && !name.startsWith('musrflow-runtime-')); + if (application.length !== 1) throw new Error('Expected exactly one application RPM'); + // Read the version from the selected artifact, not the publisher checkout. + const version = application[0].slice('musrflow-'.length, -'-x86_64.rpm'.length); + if (!/^[\w.+-]+$/.test(version)) throw new Error('Invalid package version'); + const registryVersion = `${version}-build.${PACKAGE_RUN_ID}`; + const groups = { + musrflow: [application[0], `muSRFlow-${version}-x86_64.AppImage`], + 'musrflow-runtime': [`musrflow-runtime-${version}-x86_64.rpm`, `musrflow-runtime-${version}-x86_64.AppImage`] + }; + + // Validate the complete set before the first upload. Stream large runtime files. + for (const names of Object.values(groups)) { + for (const name of [...names]) { + const file = path.join(directory, name); + if (!fs.lstatSync(file).isFile() || fs.statSync(file).size === 0) throw new Error(`Missing or empty package: ${name}`); + const hashes = { sha256: crypto.createHash('sha256'), md5: crypto.createHash('md5') }; + for await (const chunk of fs.createReadStream(file)) { + for (const hash of Object.values(hashes)) hash.update(chunk); + } + for (const [algorithm, hash] of Object.entries(hashes)) { + const sidecar = `${name}.${algorithm}`; + const checksum = fs.readFileSync(path.join(directory, sidecar), 'utf8').trim(); + const expected = `${hash.digest('hex')} ${name}`; + if (checksum !== expected) throw new Error(`Checksum mismatch: ${sidecar}`); + names.push(sidecar); + } + } + names.push('build-source.txt'); + } + groups['musrflow-runtime'].push('runtime-manifest.json'); + for (const names of Object.values(groups)) { + for (const name of names) { + const stat = fs.lstatSync(path.join(directory, name)); + if (!stat.isFile() || !stat.size) throw new Error(`Missing or empty artifact: ${name}`); + } + } + + const headers = { Authorization: `token ${PACKAGES_TOKEN}` }; + for (const [name, names] of Object.entries(groups)) { + for (const file of names) { + const url = `${base}/api/packages/${owner}/generic/${name}/${encodeURIComponent(registryVersion)}/${encodeURIComponent(file)}`; + const body = fs.createReadStream(path.join(directory, file)); + let response; + try { + response = await request(url, { + method: 'PUT', headers: { ...headers, 'Content-Type': 'application/octet-stream', + 'Content-Length': String(fs.statSync(path.join(directory, file)).size) }, + body, duplex: 'half', redirect: 'error', signal: AbortSignal.timeout(30 * 60 * 1000) + }); + } finally { + body.destroy(); + } + // A retried publish can safely skip identical files, but never replace them. + if (response.status === 409) { + const existing = await request(url, { + headers, redirect: 'error', signal: AbortSignal.timeout(30 * 60 * 1000) + }); + if (existing.status !== 200) throw new Error(`Cannot verify existing ${file}: HTTP ${existing.status}`); + const remoteHash = crypto.createHash('sha256'); + for await (const chunk of existing.body) remoteHash.update(chunk); + const localHash = crypto.createHash('sha256'); + for await (const chunk of fs.createReadStream(path.join(directory, file))) localHash.update(chunk); + if (remoteHash.digest('hex') !== localHash.digest('hex')) throw new Error(`Existing package differs: ${file}; nothing was overwritten`); + } else if (response.status !== 201) { + throw new Error(`Upload ${file}: HTTP ${response.status}. Check package write permissions and server upload limits`); + } + if (response.body) await response.body.cancel(); + console.log(`Published ${name}/${registryVersion}/${file}`); + } + const linked = await request(`${base}/api/v1/packages/${owner}/generic/${name}/-/link/${repository}`, { + method: 'POST', headers, redirect: 'error', signal: AbortSignal.timeout(60000) + }); + if (linked.status !== 201) throw new Error(`Link ${name} to repository: HTTP ${linked.status}`); + if (linked.body) await linked.body.cancel(); + } + console.log(`Packages: ${base}/${owner}/${repository}/packages (version ${registryVersion})`); + return registryVersion; +} + +module.exports = { publishPackages }; + +if (require.main === module) { + publishPackages(process.argv[2] || 'artifacts').catch(error => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/tests/ci-publish-packages.test.js b/tests/ci-publish-packages.test.js new file mode 100644 index 0000000..c4e33cb --- /dev/null +++ b/tests/ci-publish-packages.test.js @@ -0,0 +1,88 @@ +// Copyright (C) 2026 Zaher Salman +// SPDX-License-Identifier: GPL-3.0-or-later + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { publishPackages } = require('../scripts/ci-publish-packages'); + +async function main() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'musrflow-publish-')); + const env = { + GITEA_SERVER_URL: 'https://gitea.example.test', GITEA_REPOSITORY: 'owner/muSRFlow', + PACKAGE_RUN_ID: '123', PACKAGES_TOKEN: 'test-token' + }; + const stored = new Map(); + const calls = []; + async function registry(url, options) { + calls.push({ url, method: options.method || 'GET' }); + assert.strictEqual(options.headers.Authorization, 'token test-token'); + assert.strictEqual(options.redirect, 'error'); + if (options.method === 'PUT') { + const chunks = []; + for await (const chunk of options.body) chunks.push(chunk); + if (stored.has(url)) return new Response(null, { status: 409 }); + stored.set(url, Buffer.concat(chunks)); + return new Response(null, { status: 201 }); + } + if (options.method === 'POST') { + assert(url.endsWith('/-/link/muSRFlow')); + return new Response(null, { status: 201 }); + } + assert(stored.has(url)); + return new Response(stored.get(url)); + } + try { + const names = ['musrflow-9.8.7-x86_64.rpm', 'muSRFlow-9.8.7-x86_64.AppImage', + 'musrflow-runtime-9.8.7-x86_64.rpm', 'musrflow-runtime-9.8.7-x86_64.AppImage']; + for (const name of names) { + fs.writeFileSync(path.join(directory, name), name); + for (const algorithm of ['md5', 'sha256']) { + const sum = crypto.createHash(algorithm).update(name).digest('hex'); + fs.writeFileSync(path.join(directory, `${name}.${algorithm}`), `${sum} ${name}\n`); + } + } + fs.writeFileSync(path.join(directory, 'runtime-manifest.json'), '{}\n'); + fs.writeFileSync(path.join(directory, 'build-source.txt'), 'muSRFlow=test\n'); + // Unrelated artifact files must not be accidentally published. + fs.writeFileSync(path.join(directory, 'private.txt'), 'do not publish'); + + const missing = path.join(directory, names[3]); + fs.renameSync(missing, missing + '.backup'); + await assert.rejects(publishPackages(directory, env, registry), /ENOENT/); + assert.strictEqual(calls.length, 0); + fs.renameSync(missing + '.backup', missing); + fs.appendFileSync(missing, 'corruption'); + await assert.rejects(publishPackages(directory, env, registry), /Checksum mismatch/); + assert.strictEqual(calls.length, 0); + fs.writeFileSync(missing, names[3]); + await assert.rejects(publishPackages(directory, { ...env, PACKAGE_RUN_ID: '../123' }, registry), /run ID/); + await assert.rejects(publishPackages(directory, { ...env, PACKAGES_TOKEN: '' }, registry), /write token/); + await assert.rejects(publishPackages(directory, { ...env, GITEA_SERVER_URL: 'http://example.test' }, registry), /HTTPS/); + assert.strictEqual(calls.length, 0); + + assert.strictEqual(await publishPackages(directory, env, registry), '9.8.7-build.123'); + assert.strictEqual(stored.size, 15); + assert.strictEqual(calls.filter(call => call.method === 'POST').length, 2); + assert([...stored.keys()].every(url => url.includes('/9.8.7-build.123/'))); + assert(![...stored.keys()].some(url => url.includes('private.txt'))); + // An interrupted publish may be retried without deleting existing packages. + await publishPackages(directory, env, registry); + assert.strictEqual(calls.filter(call => call.method === 'GET').length, 15); + stored.set(stored.keys().next().value, Buffer.from('different published build')); + await assert.rejects(publishPackages(directory, env, registry), /Existing package differs/); + await assert.rejects(publishPackages(directory, env, async () => new Response(null, { status: 403 })), /HTTP 403/); + await assert.rejects(publishPackages(directory, env, async (url, options) => { + for await (const chunk of options.body || []) { /* consume upload */ } + return new Response(null, { status: options.method === 'POST' ? 403 : 201 }); + }), /Link musrflow to repository: HTTP 403/); + assert(!calls.some(call => call.method === 'DELETE')); + console.log('Gitea package publishing tests passed'); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/tests/release-packaging.test.js b/tests/release-packaging.test.js index 6ab8452..fa0c65b 100644 --- a/tests/release-packaging.test.js +++ b/tests/release-packaging.test.js @@ -95,4 +95,14 @@ assert(packageWorkflow.indexOf('node scripts/ci-restore-runtime.js') < packageWo assert(packageWorkflow.includes('cp "dist/$file" "dist/$file.md5" "dist/$file.sha256" artifacts/')); assert(!packageWorkflow.includes('|| true')); +const publishWorkflow = fs.readFileSync(path.join(root, '.gitea/workflows/publish-packages.yml'), 'utf8'); +assert(publishWorkflow.includes('package_run_id:')); +assert(publishWorkflow.includes('packages: write')); +assert(publishWorkflow.includes("run.conclusion !== 'success'")); +assert(publishWorkflow.includes('name: musrflow-linux-packages')); +assert(publishWorkflow.includes('node scripts/ci-publish-packages.js artifacts')); +assert(!publishWorkflow.includes('electron-builder')); +assert(!publishWorkflow.includes('npm ci')); +assert(packageWorkflow.includes('Packaging run ID for Publish packages:')); + console.log('release packaging tests passed');