From 8832bc9fb91d04a8e3b6bfedd3d108a987fdc949 Mon Sep 17 00:00:00 2001 From: salman Date: Sun, 30 Aug 2026 12:43:31 +0200 Subject: [PATCH] Transaction finalizatin for series fits --- cgi/README.md | 24 +++ cgi/single_musrfit.cgi | 269 +++++++++++++++++++++++++++ muSRFitelec.js | 56 +++++- muSRFitlib.js | 108 +++++++++++ muSRFitweb.js | 25 +++ tests/cgi-series-inventory.test.js | 127 +++++++++++++ tests/cgi-series-transaction.test.js | 241 ++++++++++++++++++++++++ tests/series-fit-staging.test.js | 66 ++++++- tests/web-runner.test.js | 30 +++ 9 files changed, 938 insertions(+), 8 deletions(-) create mode 100644 tests/cgi-series-inventory.test.js create mode 100644 tests/cgi-series-transaction.test.js diff --git a/cgi/README.md b/cgi/README.md index b8d7384..2c75671 100644 --- a/cgi/README.md +++ b/cgi/README.md @@ -104,6 +104,30 @@ exclusive recoverable backup. A failed promotion rolls earlier targets back in reverse order. Successful retry/refit backups are deliberately retained for the later sequential-runner commit/rollback step. +After promotion, the server seals the batch in a private transaction manifest. +The manifest records the server-derived target and backup names, content hashes, +and the canonical `RUN`/`ADDRUN` directives. It is not supplied or editable by +the browser. + +### `action=series_transaction_finalize` + +Commit or roll back one previously promoted independent-run transaction without +running an executable. The request contains: + +- `fn`: one `/tmp//.msr` path used only to identify the job folder +- `finalization`: JSON with the server transaction `id` and a `commit|rollback` + decision + +Commit first verifies the complete batch. Every target must be a valid, +converged fitted MSR and must retain the exact `RUN`/`ADDRUN` identity recorded +at promotion. Only then are replacement backups deleted. Rollback operates in +reverse order: newly created targets are removed and replacement backups are +restored. The manifest is removed only after every item is finalized, allowing +a failed partial operation to be retried safely. + +This endpoint only finalizes filesystem state. It does not call `musrfit`, +`msr2data`, or `musrview` and is not yet connected to the Fit button. + ### `action=series_inventory` Inspect the selected independent-run files without creating a job or running an diff --git a/cgi/single_musrfit.cgi b/cgi/single_musrfit.cgi index 220e1c6..326f063 100755 --- a/cgi/single_musrfit.cgi +++ b/cgi/single_musrfit.cgi @@ -11,6 +11,7 @@ use CGI; use JSON::PP qw(encode_json decode_json); use File::Path qw(make_path); use Fcntl qw(:DEFAULT :flock); +use Digest::SHA qw(sha256_hex); my $WEBROOT = $ENV{MUSRFIT_WEBROOT} || "/var/www/html"; $WEBROOT =~ s{/+$}{} if $WEBROOT ne "/"; @@ -193,6 +194,17 @@ sub rollback_series_receipts { return \@errors; } +sub series_run_directive_signature { + my ($content) = @_; + my @directives = (); + foreach my $line (split(/\r?\n/, $content || "")) { + next if $line !~ /^\s*(?:RUN|ADDRUN)\b/i; + my @parts = grep { $_ ne "" } split(/\s+/, $line); + push(@directives, join(" ", @parts)); + } + return sha256_hex(join("\n", @directives)); +} + sub perform_series_transaction { my ($work_path, $job_name, $transaction_text) = @_; my $failure = sub { @@ -218,6 +230,8 @@ sub perform_series_transaction { my $transaction_id = defined($decoded->{id}) ? "$decoded->{id}" : ""; return $failure->("invalid-series-transaction", "Invalid series transaction identifier") if $transaction_id !~ /^[A-Za-z0-9-]{1,64}$/; + my $manifest_name = ".musrflow-series-transaction-$transaction_id.json"; + my $manifest_path = "$work_path/$manifest_name"; my $candidates = $decoded->{candidates}; return $failure->("invalid-series-transaction", "Invalid series candidate list") if ref($candidates) ne "ARRAY" || !@{$candidates} || @{$candidates} > 500; @@ -279,6 +293,10 @@ sub perform_series_transaction { if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600); return $failure->("series-transaction-lock-failed", "Another series transaction is active") if !flock($lock_fh, LOCK_EX | LOCK_NB); + if (-e $manifest_path) { + close($lock_fh); + return $failure->("transaction-path-collision", "Series transaction manifest already exists"); + } foreach my $item (@items) { my $target_exists = -f $item->{target_path} ? 1 : 0; @@ -393,6 +411,39 @@ sub perform_series_transaction { }); } + my $manifest_text = encode_json({ + version => 1, + id => $transaction_id, + job_name => $job_name, + candidates => [map { + { + run => $_->{run}, + action => $_->{action}, + target_name => $_->{target_name}, + backup_name => $_->{backup_name}, + replaced => $_->{replaced} ? JSON::PP::true : JSON::PP::false, + promoted_sha256 => sha256_hex($_->{promoted_text}), + run_directives_sha256 => series_run_directive_signature($_->{promoted_text}), + original_sha256 => $_->{replaced} + ? sha256_hex($_->{original_text}) : "" + } + } @receipts] + }); + my $manifest_created = write_text_file_exclusive($manifest_path, $manifest_text); + if (!$manifest_created || read_text_file($manifest_path) ne $manifest_text) { + unlink($manifest_path) if $manifest_created && -e $manifest_path; + my $rollback_errors = rollback_series_receipts(\@receipts); + my $cleanup_errors = cleanup_series_stages(\@staged_items); + push(@{$rollback_errors}, @{$cleanup_errors}); + close($lock_fh); + return $failure->(@{$rollback_errors} + ? "promotion-rollback-failed" : "transaction-manifest-failed", + "Could not seal series transaction", [{ + run => 0, operation => "seal-transaction", + path => $manifest_name, message => "Could not create verified transaction manifest" + }], $rollback_errors); + } + close($lock_fh); my @public_receipts = map { { @@ -427,6 +478,212 @@ sub perform_series_transaction { }; } +sub series_msr_is_converged { + my ($content) = @_; + return 0 if !is_valid_generated_global_msr($content); + return 0 if $content !~ /^\s*STATISTIC\b/im; + return 0 if $content =~ /FIT\s+DID\s+NOT\s+CONVERGE/i; + return 1 if $content =~ /^\s*chisq\s*=\s*[^,\r\n]+,\s*NDF\s*=\s*\d+\s*,\s*chisq\/NDF\s*=/im; + return 1 if $content =~ /^\s*maxLH\s*=\s*[^,\r\n]+,\s*NDF\s*=\s*\d+\s*,\s*maxLH\/NDF\s*=/im; + return 0; +} + +sub perform_series_transaction_finalization { + my ($work_path, $job_name, $finalization_text) = @_; + my $failure = sub { + my ($reason, $message, $errors, $finalized) = @_; + return { + ok => JSON::PP::false, + code => 409, + reason => $reason, + message => $message, + finalized => $finalized || [], + errors => $errors || [] + }; + }; + return $failure->("invalid-series-finalization", "Series finalization request is too large") + if !defined($finalization_text) || length($finalization_text) > 100_000; + my $request = eval { decode_json($finalization_text) }; + return $failure->("invalid-series-finalization", "Invalid series finalization JSON") + if $@ || ref($request) ne "HASH"; + my $transaction_id = defined($request->{id}) ? "$request->{id}" : ""; + my $decision = defined($request->{decision}) ? "$request->{decision}" : ""; + return $failure->("invalid-series-finalization", "Invalid series transaction identifier") + if $transaction_id !~ /^[A-Za-z0-9-]{1,64}$/; + return $failure->("invalid-series-finalization", "Invalid finalization decision") + if $decision !~ /^(?:commit|rollback)$/; + + my $manifest_name = ".musrflow-series-transaction-$transaction_id.json"; + my $manifest_path = "$work_path/$manifest_name"; + return $failure->("series-transaction-manifest-not-found", "Series transaction manifest not found") + if !-f $manifest_path || -l $manifest_path || (-s $manifest_path || 0) > 1_000_000; + my $manifest = eval { decode_json(read_text_file($manifest_path)) }; + return $failure->("invalid-series-transaction-manifest", "Invalid series transaction manifest") + if $@ || ref($manifest) ne "HASH" + || ($manifest->{version} || 0) != 1 + || ($manifest->{id} || "") ne $transaction_id + || ($manifest->{job_name} || "") ne $job_name + || ref($manifest->{candidates}) ne "ARRAY" + || !@{$manifest->{candidates}} || @{$manifest->{candidates}} > 500; + + my @items = (); + my %seen_targets = (); + foreach my $candidate (@{$manifest->{candidates}}) { + return $failure->("invalid-series-transaction-manifest", "Invalid manifest candidate") + if ref($candidate) ne "HASH"; + my $run = defined($candidate->{run}) ? "$candidate->{run}" : ""; + my $action = defined($candidate->{action}) ? "$candidate->{action}" : ""; + my $target_name = defined($candidate->{target_name}) ? "$candidate->{target_name}" : ""; + my $backup_name = defined($candidate->{backup_name}) ? "$candidate->{backup_name}" : ""; + my $promoted_hash = defined($candidate->{promoted_sha256}) + ? "$candidate->{promoted_sha256}" : ""; + my $original_hash = defined($candidate->{original_sha256}) + ? "$candidate->{original_sha256}" : ""; + my $run_directives_hash = defined($candidate->{run_directives_sha256}) + ? "$candidate->{run_directives_sha256}" : ""; + my $replaced = $candidate->{replaced} ? 1 : 0; + return $failure->("invalid-series-transaction-manifest", "Invalid manifest run") + if $run !~ /^\d+$/ || $run <= 0; + return $failure->("invalid-series-transaction-manifest", "Invalid manifest action") + if $action !~ /^(?:fit|retry|refit)$/ || $replaced != ($action eq "fit" ? 0 : 1); + return $failure->("invalid-series-transaction-manifest", "Invalid manifest target") + if $target_name !~ /^\Q$run\E(?:[._+\-][A-Za-z0-9._+\-]*)?\.msr$/ + || $seen_targets{$target_name}++; + return $failure->("invalid-series-transaction-manifest", "Invalid manifest hash") + if $promoted_hash !~ /^[a-f0-9]{64}$/ + || $run_directives_hash !~ /^[a-f0-9]{64}$/ + || ($replaced && $original_hash !~ /^[a-f0-9]{64}$/); + my $expected_backup = $replaced + ? series_transaction_file_name($target_name, "backup", $transaction_id) : ""; + return $failure->("invalid-series-transaction-manifest", "Invalid manifest backup") + if $backup_name ne $expected_backup; + push(@items, { + run => 0 + $run, + action => $action, + target_name => $target_name, + target_path => "$work_path/$target_name", + backup_name => $backup_name, + backup_path => $replaced ? "$work_path/$backup_name" : "", + replaced => $replaced, + promoted_hash => $promoted_hash, + original_hash => $original_hash, + run_directives_hash => $run_directives_hash + }); + } + + my $lock_path = "$work_path/.musrflow-series.lock"; + return $failure->("series-finalization-lock-failed", "Could not open series transaction lock") + if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600); + return $failure->("series-finalization-lock-failed", "Another series transaction is active") + if !flock($lock_fh, LOCK_EX | LOCK_NB); + + # Preflight the complete manifest before mutating anything. Missing files + # are accepted only when they exactly represent an earlier partial retry of + # the same idempotent finalization decision. + foreach my $item (@items) { + my $target_present = -e $item->{target_path} || -l $item->{target_path}; + my $target_exists = -f $item->{target_path} && !-l $item->{target_path}; + my $target_hash = $target_exists + ? sha256_hex(read_text_file($item->{target_path})) : ""; + if ($target_present && !$target_exists) { + close($lock_fh); + return $failure->("series-finalization-preflight-failed", + "Series target is not a regular file", [{ + run => $item->{run}, operation => "$decision-preflight", + path => $item->{target_name}, message => "Regular target required" + }]); + } + if ($decision eq "commit") { + if (!$target_exists + || !series_msr_is_converged(read_text_file($item->{target_path})) + || series_run_directive_signature(read_text_file($item->{target_path})) + ne $item->{run_directives_hash}) { + close($lock_fh); + return $failure->("series-finalization-preflight-failed", + "Committed series target is not converged", [{ + run => $item->{run}, operation => "commit-preflight", + path => $item->{target_name}, message => "Converged fitted MSR required" + }]); + } + if ($item->{replaced} && -e $item->{backup_path} + && (!-f $item->{backup_path} || -l $item->{backup_path} + || sha256_hex(read_text_file($item->{backup_path})) ne $item->{original_hash})) { + close($lock_fh); + return $failure->("series-finalization-preflight-failed", + "Series backup is invalid", [{ + run => $item->{run}, operation => "commit-preflight", + path => $item->{backup_name}, message => "Verified backup required" + }]); + } + } elsif ($item->{replaced}) { + my $backup_exists = -f $item->{backup_path} && !-l $item->{backup_path}; + if (!$backup_exists && $target_hash ne $item->{original_hash}) { + close($lock_fh); + return $failure->("series-finalization-preflight-failed", + "Rollback backup is unavailable", [{ + run => $item->{run}, operation => "rollback-preflight", + path => $item->{backup_name}, message => "Verified backup required" + }]); + } + if ($backup_exists + && sha256_hex(read_text_file($item->{backup_path})) ne $item->{original_hash}) { + close($lock_fh); + return $failure->("series-finalization-preflight-failed", + "Rollback backup changed", [{ + run => $item->{run}, operation => "rollback-preflight", + path => $item->{backup_name}, message => "Backup hash mismatch" + }]); + } + } + } + + my @ordered = $decision eq "rollback" ? reverse(@items) : @items; + my @finalized = (); + foreach my $item (@ordered) { + my $ok = 1; + if ($decision eq "commit") { + $ok = unlink($item->{backup_path}) if $item->{replaced} && -e $item->{backup_path}; + } elsif ($item->{replaced}) { + $ok = rename($item->{backup_path}, $item->{target_path}) + if -e $item->{backup_path}; + } else { + $ok = unlink($item->{target_path}) if -e $item->{target_path}; + } + if (!$ok) { + close($lock_fh); + return $failure->("series-finalization-partial", + "Series transaction was only partly finalized", [{ + run => $item->{run}, operation => $decision, + path => $item->{target_name}, message => "Filesystem finalization failed" + }], \@finalized); + } + push(@finalized, { + run => $item->{run}, action => $item->{action}, + targetPath => "$TMP_URL/$job_name/$item->{target_name}", + decision => $decision + }); + } + if (!unlink($manifest_path)) { + close($lock_fh); + return $failure->("series-finalization-partial", + "Series transaction manifest could not be removed", [{ + run => 0, operation => $decision, + path => $manifest_name, message => "Could not remove transaction manifest" + }], \@finalized); + } + close($lock_fh); + return { + ok => JSON::PP::true, + code => 0, + reason => $decision eq "commit" + ? "series-transaction-committed" : "series-transaction-rolled-back", + message => $decision eq "commit" + ? "Series transaction committed" : "Series transaction rolled back", + finalized => \@finalized, + errors => [] + }; +} + # msr2data represents run-qualified parameters such as Asy_2274 as Asy_ in # the exported data header. Remove exactly one trailing underscore from header # fields while preserving embedded underscores and labels such as Asy_PosErr. @@ -916,6 +1173,18 @@ my $file_name = $2; my $work_path = "$TMPROOT/$job_name"; my $full_path = "$work_path/$file_name"; +if ($action eq "series_transaction_finalize") { + respond_json({ + ok => JSON::PP::false, code => 404, + reason => "series-transaction-manifest-not-found", + message => "Series web job not found", finalized => [], errors => [] + }) if !-d $work_path; + my $finalization_text = $in->param("finalization") || ""; + respond_json(perform_series_transaction_finalization( + $work_path, $job_name, $finalization_text + )); +} + if ($action eq "series_inventory") { my $inventory_text = $in->param("inventory") || ""; # This branch intentionally precedes make_path(): inspecting a selection diff --git a/muSRFitelec.js b/muSRFitelec.js index 6ec5b5a..6304385 100644 --- a/muSRFitelec.js +++ b/muSRFitelec.js @@ -20,6 +20,20 @@ function seriesFilesystemError(code, filePath) { return error; } +function isSafeSeriesRegularFile(filePath) { + if (!filePath || !nodeFs.existsSync(filePath)) return false; + let stat = nodeFs.lstatSync(filePath); + return stat.isFile() && !stat.isSymbolicLink(); +} + +function seriesRunDirectiveIdentity(msrText) { + return String(msrText || "").split(/\r?\n/).filter(function(line) { + return /^\s*(?:RUN|ADDRUN)\b/i.test(line); + }).map(function(line) { + return line.trim().split(/\s+/).join(" "); + }).join("\n"); +} + // Electron's independent-series transaction adapter. The shared coordinator // decides when a whole batch may be promoted; this adapter supplies exclusive // filesystem operations for one item at a time. It is exposed now for focused @@ -124,27 +138,59 @@ function createSeriesMSRTransactionAdapter() { }; }, - rollback: function(receipt) { - if (!receipt || !receipt.targetPath || !nodeFs.existsSync(receipt.targetPath)) { + verifyFinalize: function(receipt, decision) { + if (!receipt || !receipt.targetPath || !isSafeSeriesRegularFile(receipt.targetPath)) { throw seriesFilesystemError("promoted-target-not-found", receipt ? receipt.targetPath : ""); } let currentText = nodeFs.readFileSync(receipt.targetPath, "utf8"); - if (currentText !== receipt.promotedText) { + if (decision == "commit" + && (!isValidGeneratedGlobalMSR(currentText) + || parseMSRFitStatistic(currentText).converged !== true + || seriesRunDirectiveIdentity(currentText) + !== seriesRunDirectiveIdentity(receipt.promotedText))) { + throw seriesFilesystemError("committed-target-not-converged", receipt.targetPath); + } + if (receipt.replaced) { + if (!receipt.backupPath || !isSafeSeriesRegularFile(receipt.backupPath)) { + throw seriesFilesystemError("finalization-backup-not-found", receipt.backupPath || ""); + } + let backupText = nodeFs.readFileSync(receipt.backupPath, "utf8"); + if (backupText !== receipt.originalText) { + throw seriesFilesystemError("finalization-backup-changed", receipt.backupPath); + } + } + return true; + }, + + commit: function(receipt) { + this.verifyFinalize(receipt, "commit"); + if (receipt.replaced && nodeFs.existsSync(receipt.backupPath)) { + nodeFs.unlinkSync(receipt.backupPath); + } + return true; + }, + + rollback: function(receipt, options = {}) { + if (!receipt || !receipt.targetPath || !isSafeSeriesRegularFile(receipt.targetPath)) { + throw seriesFilesystemError("promoted-target-not-found", + receipt ? receipt.targetPath : ""); + } + let currentText = nodeFs.readFileSync(receipt.targetPath, "utf8"); + if (!options.allowChangedTarget && currentText !== receipt.promotedText) { throw seriesFilesystemError("promoted-target-changed", receipt.targetPath); } if (!receipt.replaced) { nodeFs.unlinkSync(receipt.targetPath); return true; } - if (!receipt.backupPath || !nodeFs.existsSync(receipt.backupPath)) { + if (!receipt.backupPath || !isSafeSeriesRegularFile(receipt.backupPath)) { throw seriesFilesystemError("rollback-backup-not-found", receipt.backupPath || ""); } let backupText = nodeFs.readFileSync(receipt.backupPath, "utf8"); if (backupText !== receipt.originalText) { throw seriesFilesystemError("rollback-backup-changed", receipt.backupPath); } - nodeFs.unlinkSync(receipt.targetPath); nodeFs.renameSync(receipt.backupPath, receipt.targetPath); return true; } diff --git a/muSRFitlib.js b/muSRFitlib.js index a871e69..b8730e0 100644 --- a/muSRFitlib.js +++ b/muSRFitlib.js @@ -1771,6 +1771,114 @@ async function stageAndPromoteSeriesMSRCandidates(prepared, adapter, options = { return result; } +// Complete or undo a promoted batch. A commit is intentionally stricter than +// promotion: every target must now contain a converged fit before recoverable +// backups are deleted. Rollback is allowed after musrfit has changed a target, +// but platform adapters must first verify the transaction receipt/manifest and +// every required backup. This helper remains disconnected from startFit(). +async function finalizeSeriesMSRTransaction(transaction, adapter, decision) { + let result = { + ok: false, + reason: "invalid-series-transaction", + decision: decision, + transactionId: transaction && transaction.transactionId + ? transaction.transactionId : "", + finalized: [], + errors: [] + }; + if (!transaction || transaction.ok !== true + || !Array.isArray(transaction.receipts) + || (decision !== "commit" && decision !== "rollback")) { + return result; + } + if (transaction.receipts.length === 0) { + result.ok = true; + result.reason = "nothing-to-finalize"; + return result; + } + + if (adapter && typeof adapter.finalizeBatch === "function") { + try { + let remote = await adapter.finalizeBatch(transaction, decision); + if (!remote || typeof remote !== "object") { + result.reason = "remote-finalization-response-invalid"; + return result; + } + result.ok = remote.ok === true; + result.reason = remote.reason || (result.ok + ? (decision == "commit" + ? "series-transaction-committed" : "series-transaction-rolled-back") + : "remote-series-finalization-failed"); + result.finalized = Array.isArray(remote.finalized) ? remote.finalized : []; + result.errors = Array.isArray(remote.errors) ? remote.errors : []; + return result; + } catch (error) { + result.reason = "remote-series-finalization-failed"; + result.errors.push({ + run: 0, + operation: decision, + path: "", + message: seriesTransactionError(error, "series-finalization-request-failed") + }); + return result; + } + } + + if (!adapter || typeof adapter.verifyFinalize !== "function" + || typeof adapter.commit !== "function" || typeof adapter.rollback !== "function") { + result.reason = "finalization-adapter-unavailable"; + return result; + } + + // Verify the whole batch before removing any backup or restoring any file. + for (let receipt of transaction.receipts) { + try { + await adapter.verifyFinalize(receipt, decision); + } catch (error) { + result.reason = "series-finalization-preflight-failed"; + result.errors.push({ + run: receipt.run || 0, + operation: "finalize-preflight", + path: receipt.targetPath || "", + message: seriesTransactionError(error, "could-not-verify-finalization") + }); + return result; + } + } + + let ordered = decision == "rollback" + ? transaction.receipts.slice().reverse() : transaction.receipts.slice(); + for (let receipt of ordered) { + try { + if (decision == "commit") { + await adapter.commit(receipt); + } else { + await adapter.rollback(receipt, { allowChangedTarget: true }); + } + result.finalized.push({ + run: receipt.run || 0, + action: receipt.action || "", + targetPath: receipt.targetPath || "", + decision: decision + }); + } catch (error) { + result.reason = "series-finalization-partial"; + result.errors.push({ + run: receipt.run || 0, + operation: decision, + path: receipt.targetPath || "", + message: seriesTransactionError(error, "could-not-finalize-target") + }); + return result; + } + } + + result.ok = true; + result.reason = decision == "commit" + ? "series-transaction-committed" : "series-transaction-rolled-back"; + return result; +} + // Store only presentation-ready fit quality in the job state. Both web and // Electron call this after reading the fitted MSR, keeping parsing independent // of the transport used to run musrfit. diff --git a/muSRFitweb.js b/muSRFitweb.js index fe574a3..fe17ee4 100644 --- a/muSRFitweb.js +++ b/muSRFitweb.js @@ -673,6 +673,31 @@ function createSeriesMSRWebTransactionAdapter() { || "Web series transaction failed."); } return request.payload; + }, + + finalizeBatch: async function(transaction, decision) { + if (!transaction || !transaction.transactionId + || !Array.isArray(transaction.receipts) || transaction.receipts.length === 0 + || (decision !== "commit" && decision !== "rollback")) { + throw new Error("Invalid web series finalization request."); + } + let targetPath = String(transaction.receipts[0].targetPath || ""); + if (targetPath.lastIndexOf("/") <= 0) { + throw new Error("Invalid web series transaction target."); + } + let request = await postToMusrfitCGI({ + action: "series_transaction_finalize", + fn: targetPath, + finalization: JSON.stringify({ + id: transaction.transactionId, + decision: decision + }) + }); + if (!request.ok || !request.payload) { + throw new Error(request.error || request.statusText + || "Web series finalization failed."); + } + return request.payload; } }; } diff --git a/tests/cgi-series-inventory.test.js b/tests/cgi-series-inventory.test.js new file mode 100644 index 0000000..34f6adb --- /dev/null +++ b/tests/cgi-series-inventory.test.js @@ -0,0 +1,127 @@ +// Copyright (C) 2026 Zaher Salman +// SPDX-License-Identifier: GPL-3.0-or-later + +const assert = require("assert"); +const childProcess = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repoRoot = path.resolve(__dirname, ".."); +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "musrflow-cgi-inventory-")); + +function runInventory(tmpRoot, dataRoot, jobName, anchorName, inventory) { + const body = new URLSearchParams({ + action: "series_inventory", + fn: "/tmp/" + jobName + "/" + anchorName, + inventory: JSON.stringify(inventory) + }).toString(); + const result = childProcess.spawnSync( + "perl", + [path.join(repoRoot, "cgi", "single_musrfit.cgi")], + { + cwd: repoRoot, + encoding: "utf8", + input: body, + env: Object.assign({}, process.env, { + REQUEST_METHOD: "POST", + CONTENT_TYPE: "application/x-www-form-urlencoded", + CONTENT_LENGTH: String(Buffer.byteLength(body)), + PERL5LIB: [path.join(repoRoot, "tests", "perl"), process.env.PERL5LIB] + .filter(Boolean).join(path.delimiter), + MUSRFIT_WEBROOT: path.dirname(tmpRoot), + MUSRFIT_TMPROOT: tmpRoot, + MUSRFIT_TMP_URL: "/tmp", + MUSRFIT_DATA_ROOT: dataRoot + }) + } + ); + assert.strictEqual(result.status, 0, result.stderr); + const responseBody = result.stdout.split(/\r?\n\r?\n/).slice(1).join("\n\n"); + return JSON.parse(responseBody); +} + +try { + const webRoot = path.join(testRoot, "web"); + const tmpRoot = path.join(webRoot, "tmp"); + const dataRoot = path.join(testRoot, "data"); + const dataDirectory = path.join(dataRoot, "lem", "his", "2026"); + const jobName = "inventoryjob"; + const workPath = path.join(tmpRoot, jobName); + fs.mkdirSync(dataDirectory, { recursive: true }); + fs.mkdirSync(workPath, { recursive: true }); + + const data514 = path.join(dataDirectory, "lem26_his_0514.root"); + const data515 = path.join(dataDirectory, "lem26_his_0515.root"); + const outsideFile = path.join(testRoot, "outside.txt"); + fs.writeFileSync(data514, "data", "utf8"); + fs.writeFileSync(outsideFile, "private", "utf8"); + fs.symlinkSync(outsideFile, data515); + + const msr514 = "FITPARAMETER\n1 A 1 0.1\nTHEORY\nasymmetry 1\n"; + fs.writeFileSync(path.join(workPath, "514_asy.msr"), msr514, "utf8"); + fs.symlinkSync(outsideFile, path.join(workPath, "515_asy.msr")); + + const response = runInventory(tmpRoot, dataRoot, jobName, "514_asy.msr", { + msr_files: [ + { run: 514, name: "514_asy.msr" }, + { run: 515, name: "515_asy.msr" }, + { run: 516, name: "516_asy.msr" } + ], + data_files: [ + { run: 514, path: data514 }, + { run: 515, path: data515 }, + { run: 516, path: path.join(dataDirectory, "lem26_his_0516.root") } + ] + }); + assert.strictEqual(response.ok, true); + assert.strictEqual(response.reason, "series-inventory-ready"); + assert.strictEqual(response.job_exists, true); + assert.deepStrictEqual(response.msr_files.map(function(item) { + return [item.run, item.exists, item.readable]; + }), [ + [514, true, true], + [515, false, false], + [516, false, false] + ]); + assert.strictEqual(response.msr_files[0].text, msr514); + assert.strictEqual(response.msr_files[1].text, ""); + assert.deepStrictEqual(response.data_files.map(function(item) { + return [item.run, item.exists]; + }), [ + [514, true], + [515, false], + [516, false] + ]); + + const missingJob = "does-not-exist"; + const missingWorkPath = path.join(tmpRoot, missingJob); + const missing = runInventory(tmpRoot, dataRoot, missingJob, "514_asy.msr", { + msr_files: [{ run: 514, name: "514_asy.msr" }], + data_files: [{ run: 514, path: data514 }] + }); + assert.strictEqual(missing.ok, true); + assert.strictEqual(missing.job_exists, false); + assert.strictEqual(missing.msr_files[0].exists, false); + assert.strictEqual(missing.data_files[0].exists, true); + assert.strictEqual(fs.existsSync(missingWorkPath), false, + "read-only inventory must not create a web job directory"); + + const outside = runInventory(tmpRoot, dataRoot, jobName, "514_asy.msr", { + msr_files: [{ run: 514, name: "514_asy.msr" }], + data_files: [{ run: 999, path: outsideFile }] + }); + assert.strictEqual(outside.ok, false); + assert.strictEqual(outside.reason, "invalid-series-inventory"); + + const traversal = runInventory(tmpRoot, dataRoot, jobName, "514_asy.msr", { + msr_files: [{ run: 514, name: "../514_asy.msr" }], + data_files: [] + }); + assert.strictEqual(traversal.ok, false); + assert.strictEqual(traversal.reason, "invalid-series-inventory"); + + console.log("CGI series inventory tests passed"); +} finally { + fs.rmSync(testRoot, { recursive: true, force: true }); +} diff --git a/tests/cgi-series-transaction.test.js b/tests/cgi-series-transaction.test.js new file mode 100644 index 0000000..b7abb32 --- /dev/null +++ b/tests/cgi-series-transaction.test.js @@ -0,0 +1,241 @@ +// Copyright (C) 2026 Zaher Salman +// SPDX-License-Identifier: GPL-3.0-or-later + +const assert = require("assert"); +const childProcess = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repoRoot = path.resolve(__dirname, ".."); +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "musrflow-cgi-series-")); + +function msr(run) { + return [ + "FITPARAMETER", + "1 Asy_" + String(run).padStart(4, "0") + " 0.2 0.01", + "THEORY", + "asymmetry 1", + "GLOBAL", + "fittype 2", + "RUN /data/run" + run + " MUE4 PSI MUSR-ROOT", + "COMMANDS", + "MIGRAD", + "PLOT 2", + "runs 1", + "" + ].join("\n"); +} + +function runTransaction(tmpRoot, jobName, anchorName, transaction) { + const body = new URLSearchParams({ + action: "series_transaction", + fn: "/tmp/" + jobName + "/" + anchorName, + transaction: JSON.stringify(transaction) + }).toString(); + const result = childProcess.spawnSync( + "perl", + [path.join(repoRoot, "cgi", "single_musrfit.cgi")], + { + cwd: repoRoot, + encoding: "utf8", + input: body, + env: Object.assign({}, process.env, { + REQUEST_METHOD: "POST", + CONTENT_TYPE: "application/x-www-form-urlencoded", + CONTENT_LENGTH: String(Buffer.byteLength(body)), + PERL5LIB: [path.join(repoRoot, "tests", "perl"), process.env.PERL5LIB] + .filter(Boolean).join(path.delimiter), + MUSRFIT_WEBROOT: path.dirname(tmpRoot), + MUSRFIT_TMPROOT: tmpRoot, + MUSRFIT_TMP_URL: "/tmp" + }) + } + ); + assert.strictEqual(result.status, 0, result.stderr); + const responseBody = result.stdout.split(/\r?\n\r?\n/).slice(1).join("\n\n"); + return JSON.parse(responseBody); +} + +function runFinalization(tmpRoot, jobName, anchorName, id, decision) { + const body = new URLSearchParams({ + action: "series_transaction_finalize", + fn: "/tmp/" + jobName + "/" + anchorName, + finalization: JSON.stringify({ id: id, decision: decision }) + }).toString(); + const result = childProcess.spawnSync( + "perl", + [path.join(repoRoot, "cgi", "single_musrfit.cgi")], + { + cwd: repoRoot, + encoding: "utf8", + input: body, + env: Object.assign({}, process.env, { + REQUEST_METHOD: "POST", + CONTENT_TYPE: "application/x-www-form-urlencoded", + CONTENT_LENGTH: String(Buffer.byteLength(body)), + PERL5LIB: [path.join(repoRoot, "tests", "perl"), process.env.PERL5LIB] + .filter(Boolean).join(path.delimiter), + MUSRFIT_WEBROOT: path.dirname(tmpRoot), + MUSRFIT_TMPROOT: tmpRoot, + MUSRFIT_TMP_URL: "/tmp" + }) + } + ); + assert.strictEqual(result.status, 0, result.stderr); + const responseBody = result.stdout.split(/\r?\n\r?\n/).slice(1).join("\n\n"); + return JSON.parse(responseBody); +} + +function converged(msrText) { + return msrText + + "STATISTIC --- 2026-08-29 12:00:00\n" + + "chisq = 101.2, NDF = 100, chisq/NDF = 1.012\n"; +} + +try { + const webRoot = path.join(testRoot, "web"); + const tmpRoot = path.join(webRoot, "tmp"); + const jobName = "seriesjob"; + const workPath = path.join(tmpRoot, jobName); + fs.mkdirSync(workPath, { recursive: true }); + + const old516 = msr(516) + "# old result\n"; + const new515 = msr(515); + const new516 = msr(516); + fs.writeFileSync(path.join(workPath, "516_asy.msr"), old516, "utf8"); + + const success = runTransaction(tmpRoot, jobName, "515_asy.msr", { + id: "batch-1", + candidates: [ + { run: 515, action: "fit", target_name: "515_asy.msr", expected_text: "", text: new515 }, + { run: 516, action: "retry", target_name: "516_asy.msr", + expected_text: old516, text: new516 } + ] + }); + + assert.strictEqual(success.ok, true); + assert.strictEqual(success.reason, "series-candidates-promoted"); + assert.strictEqual(success.receipts.length, 2); + assert.strictEqual(success.backups.length, 1); + assert.strictEqual(fs.readFileSync(path.join(workPath, "515_asy.msr"), "utf8"), new515); + assert.strictEqual(fs.readFileSync(path.join(workPath, "516_asy.msr"), "utf8"), new516); + const backupPath = path.join(workPath, "516_asy.musrflow-backup-batch-1.msr"); + const manifestPath = path.join(workPath, ".musrflow-series-transaction-batch-1.json"); + assert.strictEqual(fs.readFileSync(backupPath, "utf8"), old516); + assert.strictEqual(fs.existsSync(manifestPath), true); + assert.strictEqual(fs.existsSync(path.join(workPath, + "515_asy.musrflow-stage-batch-1.msr")), false); + assert.strictEqual(fs.existsSync(path.join(workPath, + "516_asy.musrflow-stage-batch-1.msr")), false); + + // A replacement changed since browser inventory must fail before staging. + const stale = runTransaction(tmpRoot, jobName, "516_asy.msr", { + id: "stale", + candidates: [ + { run: 516, action: "refit", target_name: "516_asy.msr", + expected_text: old516, text: msr(516) + "# another attempt\n" } + ] + }); + assert.strictEqual(stale.ok, false); + assert.strictEqual(stale.reason, "series-target-state-changed"); + assert.strictEqual(fs.readFileSync(path.join(workPath, "516_asy.msr"), "utf8"), new516); + assert.strictEqual(fs.existsSync(path.join(workPath, + "516_asy.musrflow-stage-stale.msr")), false); + + // Transaction-owned path collisions are never reused or removed. + const collisionStage = path.join(workPath, "517_asy.musrflow-stage-collision.msr"); + fs.writeFileSync(collisionStage, "unrelated", "utf8"); + const collision = runTransaction(tmpRoot, jobName, "517_asy.msr", { + id: "collision", + candidates: [ + { run: 517, action: "fit", target_name: "517_asy.msr", + expected_text: "", text: msr(517) } + ] + }); + assert.strictEqual(collision.ok, false); + assert.strictEqual(collision.reason, "transaction-path-collision"); + assert.strictEqual(fs.readFileSync(collisionStage, "utf8"), "unrelated"); + assert.strictEqual(fs.existsSync(path.join(workPath, "517_asy.msr")), false); + + const outsidePath = path.join(tmpRoot, "518_asy.msr"); + const traversal = runTransaction(tmpRoot, jobName, "518_asy.msr", { + id: "traversal", + candidates: [ + { run: 518, action: "fit", target_name: "../518_asy.msr", + expected_text: "", text: msr(518) } + ] + }); + assert.strictEqual(traversal.ok, false); + assert.strictEqual(traversal.reason, "invalid-series-candidate"); + assert.strictEqual(fs.existsSync(outsidePath), false); + + const rolledBack = runFinalization(tmpRoot, jobName, "515_asy.msr", "batch-1", "rollback"); + assert.strictEqual(rolledBack.ok, true); + assert.strictEqual(rolledBack.reason, "series-transaction-rolled-back"); + assert.strictEqual(fs.existsSync(path.join(workPath, "515_asy.msr")), false); + assert.strictEqual(fs.readFileSync(path.join(workPath, "516_asy.msr"), "utf8"), old516); + assert.strictEqual(fs.existsSync(backupPath), false); + assert.strictEqual(fs.existsSync(manifestPath), false); + + const forCommit = runTransaction(tmpRoot, jobName, "515_asy.msr", { + id: "commit", + candidates: [ + { run: 515, action: "fit", target_name: "515_asy.msr", expected_text: "", text: new515 }, + { run: 516, action: "refit", target_name: "516_asy.msr", + expected_text: old516, text: new516 } + ] + }); + assert.strictEqual(forCommit.ok, true); + fs.writeFileSync(path.join(workPath, "515_asy.msr"), converged(new515), "utf8"); + fs.writeFileSync(path.join(workPath, "516_asy.msr"), converged(new516), "utf8"); + const committed = runFinalization(tmpRoot, jobName, "515_asy.msr", "commit", "commit"); + assert.strictEqual(committed.ok, true); + assert.strictEqual(committed.reason, "series-transaction-committed"); + assert.strictEqual(fs.readFileSync(path.join(workPath, "515_asy.msr"), "utf8"), converged(new515)); + assert.strictEqual(fs.readFileSync(path.join(workPath, "516_asy.msr"), "utf8"), converged(new516)); + assert.strictEqual(fs.existsSync(path.join(workPath, + "516_asy.musrflow-backup-commit.msr")), false); + assert.strictEqual(fs.existsSync(path.join(workPath, + ".musrflow-series-transaction-commit.json")), false); + + const pendingJob = "pendingjob"; + const pendingPath = path.join(tmpRoot, pendingJob); + fs.mkdirSync(pendingPath, { recursive: true }); + const pending = runTransaction(tmpRoot, pendingJob, "520_asy.msr", { + id: "pending", + candidates: [ + { run: 520, action: "fit", target_name: "520_asy.msr", + expected_text: "", text: msr(520) } + ] + }); + assert.strictEqual(pending.ok, true); + const rejectedCommit = runFinalization( + tmpRoot, pendingJob, "520_asy.msr", "pending", "commit" + ); + assert.strictEqual(rejectedCommit.ok, false); + assert.strictEqual(rejectedCommit.reason, "series-finalization-preflight-failed"); + assert.strictEqual(fs.existsSync(path.join(pendingPath, "520_asy.msr")), true); + assert.strictEqual(fs.existsSync(path.join(pendingPath, + ".musrflow-series-transaction-pending.json")), true); + + // A converged file for another run is not a valid transaction result. + fs.writeFileSync(path.join(pendingPath, "520_asy.msr"), converged(msr(521)), "utf8"); + const wrongRunCommit = runFinalization( + tmpRoot, pendingJob, "520_asy.msr", "pending", "commit" + ); + assert.strictEqual(wrongRunCommit.ok, false); + assert.strictEqual(wrongRunCommit.reason, "series-finalization-preflight-failed"); + assert.strictEqual(fs.existsSync(path.join(pendingPath, + ".musrflow-series-transaction-pending.json")), true); + + const pendingRollback = runFinalization( + tmpRoot, pendingJob, "520_asy.msr", "pending", "rollback" + ); + assert.strictEqual(pendingRollback.ok, true); + assert.strictEqual(fs.existsSync(path.join(pendingPath, "520_asy.msr")), false); + + console.log("CGI series transaction tests passed"); +} finally { + fs.rmSync(testRoot, { recursive: true, force: true }); +} diff --git a/tests/series-fit-staging.test.js b/tests/series-fit-staging.test.js index 0bf33df..f4c8d30 100644 --- a/tests/series-fit-staging.test.js +++ b/tests/series-fit-staging.test.js @@ -47,6 +47,27 @@ function candidate(run, action, text, expectedTargetText) { return result; } +function runnableMSR(run, converged) { + const lines = [ + "FITPARAMETER", + "1 Asy_" + String(run).padStart(4, "0") + " 0.2 0.01", + "THEORY", + "asymmetry 1", + "GLOBAL", + "fittype 2", + "RUN /data/run" + run + " MUE4 PSI MUSR-ROOT", + "COMMANDS", + "MIGRAD", + "PLOT 2", + "runs 1" + ]; + if (converged) { + lines.push("STATISTIC --- 2026-08-30 12:00:00"); + lines.push("chisq = 101.2, NDF = 100, chisq/NDF = 1.012"); + } + return lines.join("\n") + "\n"; +} + function memoryAdapter(initialFiles, options = {}) { const files = Object.assign({}, initialFiles); const events = []; @@ -267,12 +288,51 @@ async function testElectronFilesystemAdapter(lib) { assert.strictEqual(fs.readFileSync(replaceTarget, "utf8"), "new-516"); assert.strictEqual(fs.readFileSync(result.backups[0].path, "utf8"), "old-516"); - for (const receipt of result.receipts.slice().reverse()) { - adapter.rollback(receipt); - } + // Rollback remains available after a runner has changed both targets. + fs.writeFileSync(newTarget, "changed-by-runner", "utf8"); + fs.writeFileSync(replaceTarget, "changed-by-runner", "utf8"); + const rolledBack = await lib.finalizeSeriesMSRTransaction(result, adapter, "rollback"); + assert.strictEqual(rolledBack.ok, true); + assert.strictEqual(rolledBack.reason, "series-transaction-rolled-back"); assert.strictEqual(fs.existsSync(newTarget), false); assert.strictEqual(fs.readFileSync(replaceTarget, "utf8"), "old-516"); assert.strictEqual(fs.existsSync(result.backups[0].path), false); + + const commitTarget = path.join(tempDirectory, "517_asy.msr"); + const commitBatch = await lib.stageAndPromoteSeriesMSRCandidates( + prepared([ + { run: 517, action: "fit", targetPath: commitTarget, + text: runnableMSR(517, false) }, + { run: 516, action: "refit", targetPath: replaceTarget, + expectedTargetText: "old-516", text: runnableMSR(516, false) } + ]), + adapter, + { transactionId: "commit" } + ); + assert.strictEqual(commitBatch.ok, true); + const prematureCommit = await lib.finalizeSeriesMSRTransaction( + commitBatch, adapter, "commit" + ); + assert.strictEqual(prematureCommit.ok, false); + assert.strictEqual(prematureCommit.reason, "series-finalization-preflight-failed"); + assert.strictEqual(fs.existsSync(commitBatch.backups[0].path), true); + + fs.writeFileSync(commitTarget, runnableMSR(518, true), "utf8"); + fs.writeFileSync(replaceTarget, runnableMSR(516, true), "utf8"); + const wrongRunCommit = await lib.finalizeSeriesMSRTransaction( + commitBatch, adapter, "commit" + ); + assert.strictEqual(wrongRunCommit.ok, false); + assert.strictEqual(wrongRunCommit.reason, "series-finalization-preflight-failed"); + assert.strictEqual(fs.existsSync(commitBatch.backups[0].path), true); + + fs.writeFileSync(commitTarget, runnableMSR(517, true), "utf8"); + const committed = await lib.finalizeSeriesMSRTransaction(commitBatch, adapter, "commit"); + assert.strictEqual(committed.ok, true); + assert.strictEqual(committed.reason, "series-transaction-committed"); + assert.strictEqual(fs.readFileSync(commitTarget, "utf8"), runnableMSR(517, true)); + assert.strictEqual(fs.readFileSync(replaceTarget, "utf8"), runnableMSR(516, true)); + assert.strictEqual(fs.existsSync(commitBatch.backups[0].path), false); } finally { fs.rmSync(tempDirectory, { recursive: true, force: true }); } diff --git a/tests/web-runner.test.js b/tests/web-runner.test.js index dd2f01a..00452e7 100644 --- a/tests/web-runner.test.js +++ b/tests/web-runner.test.js @@ -119,6 +119,21 @@ async function testSeriesTransactionUsesOneServerRequest() { const calls = []; const lib = loadWebLibrary(async function(url, options) { calls.push({ url: url, options: options }); + const body = new URLSearchParams(options.body); + if (body.get("action") === "series_transaction_finalize") { + return createResponse(JSON.stringify({ + ok: true, + code: 0, + reason: "series-transaction-rolled-back", + finalized: [ + { run: 516, action: "retry", targetPath: "/tmp/job/516_asy.msr", + decision: "rollback" }, + { run: 515, action: "fit", targetPath: "/tmp/job/515_asy.msr", + decision: "rollback" } + ], + errors: [] + })); + } return createResponse(JSON.stringify({ ok: true, code: 0, @@ -171,6 +186,21 @@ async function testSeriesTransactionUsesOneServerRequest() { return candidate.target_name; }), ["515_asy.msr", "516_asy.msr"]); assert.strictEqual(transaction.candidates[1].expected_text, "old 516"); + + const finalization = await lib.finalizeSeriesMSRTransaction( + result, lib.window.MusrfitSeriesTransactionAdapter, "rollback" + ); + assert.strictEqual(finalization.ok, true); + assert.strictEqual(finalization.reason, "series-transaction-rolled-back"); + assert.strictEqual(finalization.finalized.length, 2); + assert.strictEqual(calls.length, 2); + const finalizeBody = new URLSearchParams(calls[1].options.body); + assert.strictEqual(finalizeBody.get("action"), "series_transaction_finalize"); + assert.strictEqual(finalizeBody.get("fn"), "/tmp/job/515_asy.msr"); + assert.deepStrictEqual(JSON.parse(finalizeBody.get("finalization")), { + id: "web-batch", + decision: "rollback" + }); } async function testSeriesTransactionRejectsMixedJobDirectories() {