From 4ec6c7eb02766c917b81e2d4d977cf79add6cdb0 Mon Sep 17 00:00:00 2001 From: salman Date: Sun, 30 Aug 2026 20:22:14 +0200 Subject: [PATCH] Use msr2data after series fit. --- cgi/single_musrfit.cgi | 191 ++++++++++++++++++++++++++++ muSRFitweb.js | 77 +++++++++++ package.json | 2 +- tests/cgi-series-collection.test.js | 117 +++++++++++++++++ tests/web-runner.test.js | 56 ++++++++ 5 files changed, 442 insertions(+), 1 deletion(-) create mode 100644 tests/cgi-series-collection.test.js diff --git a/cgi/single_musrfit.cgi b/cgi/single_musrfit.cgi index 5bae593..1632ad0 100755 --- a/cgi/single_musrfit.cgi +++ b/cgi/single_musrfit.cgi @@ -995,6 +995,185 @@ sub normalize_plot_runs_line { return $content; } +# Collect a completed independent series without running musrfit again. The +# requested non-empty data table and structurally valid combined MSR are the +# success artifacts; msr2data's process status is retained only as a warning. +sub perform_series_artifact_collection { + my ($work_path, $job_name, $anchor_name, $collection_text) = @_; + my $failure = sub { + my ($reason, $message, $warnings) = @_; + return { + ok => JSON::PP::false, + code => 1, + reason => $reason, + message => $message, + warnings => $warnings || [], + job_name => $job_name, + work_url => "$TMP_URL/$job_name", + msr_url => "", out_url => "", parameter_url => "", + root_url => "", root_urls => [], root_items => [], command_log => "" + }; + }; + return $failure->("invalid-series-collection", "Invalid series collection request") + if !defined($collection_text) || length($collection_text) > 100_000; + my $collection = eval { decode_json($collection_text) }; + return $failure->("invalid-series-collection", "Invalid series collection JSON") + if $@ || ref($collection) ne "HASH"; + my $runs = $collection->{runs}; + my $suffix = defined($collection->{suffix}) ? "$collection->{suffix}" : ""; + my $fftv = defined($collection->{fftv}) ? "$collection->{fftv}" : "n"; + return $failure->("invalid-series-collection", "Invalid series collection fields") + if ref($runs) ne "ARRAY" || @{$runs} < 2 || @{$runs} > 500 + || $suffix !~ /^_[A-Za-z0-9._+\-]+$/ || $fftv !~ /^(?:y|n)$/; + + my @run_numbers = (); + my %seen = (); + foreach my $value (@{$runs}) { + my $run = defined($value) && !ref($value) ? "$value" : ""; + return $failure->("invalid-series-collection", "Invalid series run number") + if $run !~ /^\d+$/ || $run <= 0 || $seen{$run}++; + my $source_name = $run . $suffix . ".msr"; + my $source_path = "$work_path/$source_name"; + return $failure->("series-artifact-source-invalid", + "A fitted series MSR is missing or unsafe") + if !-f $source_path || -l $source_path + || !series_msr_is_converged(read_text_file($source_path)); + push(@run_numbers, $run); + } + return $failure->("invalid-series-collection", "Invalid series anchor") + if $anchor_name ne $run_numbers[0] . $suffix . ".msr"; + return $failure->("series-msr2data-unavailable", "msr2data is not executable") + if !-x $MSR2DATA_BIN; + + my $base_run = $run_numbers[0]; + my $generated_name = $base_run . "+global" . $suffix . ".msr"; + my $generated_path = "$work_path/$generated_name"; + my $combined_name = $base_run . "+series" . $suffix . ".msr"; + my $combined_path = "$work_path/$combined_name"; + my $parameter_name = $base_run . "+series" . $suffix . "_par.dat"; + my $parameter_path = "$work_path/$parameter_name"; + my $out_name = $base_run . "+series" . $suffix . ".out"; + my $out_path = "$work_path/$out_name"; + foreach my $stale_path ($generated_path, $combined_path, $parameter_path) { + return $failure->("series-artifact-output-invalid", "A series output path is unsafe") + if -l $stale_path || (-e $stale_path && !-f $stale_path); + return $failure->("series-artifact-output-remove-failed", + "Could not remove a stale series artifact") + if -e $stale_path && !unlink($stale_path); + } + return $failure->("series-artifact-log-failed", "Could not initialize collection log") + if -l $out_path || (-e $out_path && !-f $out_path) + || !write_text_file($out_path, + "# muSRFlow independent series collection\n# Working directory: $work_path\n"); + + my @warnings = (); + my $run_argument = "[" . join(" ", @run_numbers) . "]"; + my $parameter_cmd = "cd " . shell_quote($work_path) + . " && " . shell_quote($MSR2DATA_BIN) + . " " . shell_quote($run_argument) + . " " . shell_quote($suffix) + . " new global data -o " . shell_quote($parameter_name); + my $parameter_result = run_logged_command( + $out_path, "msr2data independent series collection", $parameter_cmd + ); + + my $parameter_url = ""; + if (-s $parameter_path) { + if (!clean_parameter_data_header($parameter_path)) { + push(@warnings, "Could not normalize the parameter-table header"); + } + # Keep the non-empty file available even when header cleanup fails. + $parameter_url = "$TMP_URL/$job_name/$parameter_name"; + } else { + push(@warnings, "msr2data did not produce parameter data"); + } + + my $combined_url = ""; + if (-f $generated_path && !-l $generated_path) { + my $generated_text = read_text_file($generated_path); + if (is_valid_generated_global_msr($generated_text)) { + my $combined_text = normalize_plot_runs_line( + $generated_text, join(" ", @run_numbers) + ); + if (write_text_file($combined_path, $combined_text)) { + $combined_url = "$TMP_URL/$job_name/$combined_name"; + unlink($generated_path); + } else { + push(@warnings, "Could not write the combined display MSR"); + } + } else { + push(@warnings, "msr2data produced an invalid combined display MSR"); + } + } else { + push(@warnings, "msr2data did not produce a combined display MSR"); + } + if ($parameter_result->{raw_status} != 0 + && ($parameter_url ne "" || $combined_url ne "")) { + push(@warnings, "msr2data returned code " . $parameter_result->{exit_code} + . " but produced usable series artifacts"); + } + + my @root_urls = (); + my @root_items = (); + if ($combined_url ne "") { + my $plot_base = $combined_name; + $plot_base =~ s/\.msr$//i; + if (opendir(my $old_plot_dir, $work_path)) { + my @old_plots = grep { + /^\Q$plot_base\E_\d+(?:_F)?\.root$/ && -f "$work_path/$_" + } readdir($old_plot_dir); + closedir($old_plot_dir); + unlink("$work_path/$_") foreach @old_plots; + } + if (-x $MUSRVIEW_BIN) { + my $view_cmd = "cd " . shell_quote($work_path) + . " && " . shell_quote($MUSRVIEW_BIN) . " --timeout 300 --root" + . ($fftv eq "y" ? " -f" : "") + . " " . shell_quote($combined_name); + my $view_result = run_logged_command($out_path, "musrview", $view_cmd); + push(@warnings, "musrview returned code " . $view_result->{exit_code}) + if $view_result->{raw_status} != 0; + } else { + push(@warnings, "musrview is not executable"); + } + if (opendir(my $plot_dir, $work_path)) { + my @root_files = sort grep { + my $matches = $fftv eq "y" + ? /^\Q$plot_base\E_\d+_F\.root$/ + : /^\Q$plot_base\E_\d+\.root$/; + $matches && -f "$work_path/$_" + } readdir($plot_dir); + closedir($plot_dir); + @root_urls = map { "$TMP_URL/$job_name/$_" } @root_files; + foreach my $root_file (@root_files) { + my ($canvas) = $root_file =~ /_(\d+)(?:_F)?\.root$/; + push(@root_items, "fMainCanvas" . ($canvas || 0) . ";1"); + } + } + push(@warnings, "musrview did not produce a ROOT plot") if !@root_urls; + } + + my $command_log = read_text_file($out_path); + my $usable = $parameter_url ne "" || $combined_url ne ""; + return { + ok => $usable ? JSON::PP::true : JSON::PP::false, + code => $usable ? 0 : 1, + reason => $usable ? "series-artifacts-collected" : "series-artifacts-not-produced", + message => $usable ? "Independent series artifacts collected" + : "Independent series artifacts were not produced", + warnings => \@warnings, + job_name => $job_name, + work_url => "$TMP_URL/$job_name", + msr_url => $combined_url, + out_url => "$TMP_URL/$job_name/$out_name", + parameter_url => $parameter_url, + root_url => @root_urls ? $root_urls[0] : "", + root_urls => \@root_urls, + root_items => \@root_items, + command_log => $command_log + }; +} + sub path_is_under_data_root { my ($path) = @_; return 0 if !defined($path) || $path eq "" || $path =~ /\s/; @@ -1356,6 +1535,18 @@ if ($action eq "series_run_candidate") { )); } +if ($action eq "series_collect") { + respond_json({ + ok => JSON::PP::false, code => 404, + reason => "series-web-job-not-found", + message => "Series web job not found", warnings => [] + }) if !-d $work_path; + my $collection_text = $in->param("collection") || ""; + respond_json(perform_series_artifact_collection( + $work_path, $job_name, $file_name, $collection_text + )); +} + if ($action eq "series_transaction_finalize") { respond_json({ ok => JSON::PP::false, code => 404, diff --git a/muSRFitweb.js b/muSRFitweb.js index 9cd816e..8adb4a4 100644 --- a/muSRFitweb.js +++ b/muSRFitweb.js @@ -749,6 +749,83 @@ async function runSeriesMSRCandidateWeb(receipt, candidate, context = {}) { window.runSeriesMSRCandidateWeb = runSeriesMSRCandidateWeb; +// Ask the server to collect already fitted independent-run MSRs. This endpoint +// never invokes musrfit; it only creates the parameter table, a combined display +// MSR, and the ROOT canvas used by the browser viewer. +async function collectIndependentSeriesArtifactsWeb(spec) { + let result = { + ok: false, + reason: "invalid-series-artifact-spec", + parameterDataPath: "", + combinedMSRPath: "", + logPath: "", + plotPath: "", + plotStarted: false, + warnings: [] + }; + if (!spec || spec.ok !== true || !Array.isArray(spec.runs) + || spec.runs.length < 2 || !Array.isArray(spec.msrPaths) + || spec.msrPaths.length !== spec.runs.length + || !/^_[A-Za-z0-9._+\-]+$/.test(spec.fileNameSuffix || "")) { + return result; + } + let workUrl = String(spec.workPath || "").replace(/\/+$/, ""); + for (let index = 0; index < spec.runs.length; index++) { + if (!/^\d+$/.test(spec.runs[index]) + || spec.msrPaths[index] + !== workUrl + "/" + spec.runs[index] + spec.fileNameSuffix + ".msr") { + result.reason = "series-artifact-source-invalid"; + return result; + } + } + + let request = await postToMusrfitCGI({ + action: "series_collect", + fn: spec.msrPaths[0], + collection: JSON.stringify({ + runs: spec.runs, + suffix: spec.fileNameSuffix, + fftv: spec.fftv == "y" ? "y" : "n" + }) + }); + if (!request.ok || !request.payload) { + result.reason = "web-series-artifact-request-failed"; + result.warnings.push(request.error || request.statusText + || "Series collection request failed."); + return result; + } + + let response = request.payload; + result.ok = response.ok === true || response.ok === 1; + result.reason = response.reason || (result.ok + ? "series-artifacts-collected" : "series-artifacts-not-produced"); + result.parameterDataPath = response.parameter_url || ""; + result.combinedMSRPath = response.msr_url || ""; + result.logPath = response.out_url || ""; + result.warnings = Array.isArray(response.warnings) + ? response.warnings.filter(Boolean) : []; + if (response.command_log && typeof appendJobLog === "function") { + appendJobLog(response.command_log.trim()); + } + let rootUrls = typeof musrfitRootUrlsFromResponse === "function" + ? musrfitRootUrlsFromResponse(response) + : (Array.isArray(response.root_urls) ? response.root_urls.filter(Boolean) : []); + if (rootUrls.length > 0) { + result.plotPath = rootUrls[0]; + if (typeof showWebRootPlotWindow === "function") { + showWebRootPlotWindow(rootUrls, response.job_name || "", { + msrPath: result.combinedMSRPath, + runs: spec.runs.join(" "), + rootItems: Array.isArray(response.root_items) ? response.root_items : [] + }); + result.plotStarted = true; + } + } + return result; +} + +window.collectIndependentSeriesArtifactsWeb = collectIndependentSeriesArtifactsWeb; + function webSeriesInventoryFailure(config, execution, reason, message) { let all = config.all || {}; return { diff --git a/package.json b/package.json index 5b528b9..bafdd35 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "main": "main.js", "scripts": { "start": "electron .", - "test": "npm run test:electron && node tests/cgi-series-inventory.test.js && node tests/cgi-series-transaction.test.js && node tests/cgi-runner.test.js", + "test": "npm run test:electron && node tests/cgi-series-inventory.test.js && node tests/cgi-series-transaction.test.js && node tests/cgi-series-collection.test.js && node tests/cgi-runner.test.js", "dist:linux": "npm run test:electron && npx --yes electron-builder@26.15.3 --linux rpm AppImage --x64", "dist:rpm": "npm run test:electron && npx --yes electron-builder@26.15.3 --linux rpm --x64", "dist:appimage": "npm run test:electron && npx --yes electron-builder@26.15.3 --linux AppImage --x64", diff --git a/tests/cgi-series-collection.test.js b/tests/cgi-series-collection.test.js new file mode 100644 index 0000000..05cc7d6 --- /dev/null +++ b/tests/cgi-series-collection.test.js @@ -0,0 +1,117 @@ +// Copyright (C) 2026 Zaher Salman +// SPDX-License-Identifier: GPL-3.0-or-later + +const assert = require("assert"); +const childProcess = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const repoRoot = path.resolve(__dirname, ".."); +const testRoot = fs.mkdtempSync(path.join(os.tmpdir(), "musrflow-cgi-series-collect-")); +const perlLibraryPath = [path.join(repoRoot, "tests", "perl"), process.env.PERL5LIB] + .filter(Boolean).join(path.delimiter); + +function fittedMSR(run) { + return [ + "FITPARAMETER", + "1 Asy_" + run + " 0.2 0.01", + "THEORY", + "asymmetry 1", + "GLOBAL", + "fittype 2", + "RUN /data/run" + run + " MUE4 PSI MUSR-ROOT", + "COMMANDS", + "MIGRAD", + "PLOT 2", + "runs 1", + "STATISTIC --- 2026-08-30 12:00:00", + "chisq = 101.2, NDF = 100, chisq/NDF = 1.012", + "" + ].join("\n"); +} + +try { + const webRoot = path.join(testRoot, "web"); + const tmpRoot = path.join(webRoot, "tmp"); + const workPath = path.join(tmpRoot, "seriesjob"); + fs.mkdirSync(workPath, { recursive: true }); + fs.writeFileSync(path.join(workPath, "514_asy.msr"), fittedMSR(514)); + fs.writeFileSync(path.join(workPath, "515_asy.msr"), fittedMSR(515)); + + const m2dArgsPath = path.join(testRoot, "msr2data-args.txt"); + const viewArgsPath = path.join(testRoot, "musrview-args.txt"); + const fakeMsr2data = path.join(testRoot, "msr2data"); + const fakeMusrview = path.join(testRoot, "musrview"); + fs.writeFileSync(fakeMsr2data, [ + "#!/bin/sh", + "printf '%s\\n' \"$*\" > \"$MUSRFLOW_TEST_M2D_ARGS\"", + "printf 'RUN Asy_ Asy_PosErr\\n514 0.2 0.01\\n515 0.21 0.01\\n' > 514+series_asy_par.dat", + "cp 514_asy.msr 514+global_asy.msr", + "printf 'RUN /data/run515 MUE4 PSI MUSR-ROOT\\n' >> 514+global_asy.msr", + "exit 9", + "" + ].join("\n")); + fs.chmodSync(fakeMsr2data, 0o755); + fs.writeFileSync(fakeMusrview, [ + "#!/bin/sh", + "printf '%s\\n' \"$*\" > \"$MUSRFLOW_TEST_VIEW_ARGS\"", + ": > 514+series_asy_0.root", + "exit 0", + "" + ].join("\n")); + fs.chmodSync(fakeMusrview, 0o755); + + const body = new URLSearchParams({ + action: "series_collect", + fn: "/tmp/seriesjob/514_asy.msr", + collection: JSON.stringify({ runs: ["514", "515"], suffix: "_asy", fftv: "n" }) + }).toString(); + const execution = childProcess.spawnSync( + "perl", [path.join(repoRoot, "cgi", "single_musrfit.cgi")], { + cwd: repoRoot, + encoding: "utf8", + input: body, + env: Object.assign({}, process.env, { + REQUEST_METHOD: "POST", + PERL5LIB: perlLibraryPath, + CONTENT_TYPE: "application/x-www-form-urlencoded", + CONTENT_LENGTH: String(Buffer.byteLength(body)), + MUSRFIT_WEBROOT: webRoot, + MUSRFIT_TMPROOT: tmpRoot, + MUSRFIT_TMP_URL: "/tmp", + MSR2DATA_BIN: fakeMsr2data, + MUSRVIEW_BIN: fakeMusrview, + MUSRFIT_RUNTIME_HOME: tmpRoot, + MUSRFLOW_TEST_M2D_ARGS: m2dArgsPath, + MUSRFLOW_TEST_VIEW_ARGS: viewArgsPath + }) + } + ); + + assert.strictEqual(execution.status, 0, execution.stderr); + const responseText = execution.stdout.split(/\r?\n\r?\n/).slice(1).join("\n\n"); + const response = JSON.parse(responseText); + assert.strictEqual(response.ok, true); + assert.strictEqual(response.parameter_url, "/tmp/seriesjob/514+series_asy_par.dat"); + assert.strictEqual(response.msr_url, "/tmp/seriesjob/514+series_asy.msr"); + assert.deepStrictEqual(response.root_urls, + ["/tmp/seriesjob/514+series_asy_0.root"]); + assert.match(response.warnings.join("\n"), /returned code 9/); + assert.strictEqual(fs.readFileSync(m2dArgsPath, "utf8").trim(), + "[514 515] _asy new global data -o 514+series_asy_par.dat"); + assert.strictEqual(fs.readFileSync(viewArgsPath, "utf8").trim(), + "--timeout 300 --root 514+series_asy.msr"); + assert.strictEqual(fs.readFileSync( + path.join(workPath, "514+series_asy_par.dat"), "utf8" + ).split(/\r?\n/)[0], "RUN Asy Asy_PosErr"); + assert.match(fs.readFileSync(path.join(workPath, "514+series_asy.msr"), "utf8"), + /^runs\s+1\s+2$/m); + assert.strictEqual(fs.existsSync(path.join(workPath, "514+global_asy.msr")), false); + assert.match(response.command_log, /msr2data independent series collection/); + assert.match(response.command_log, /\[exit code 9\]/); +} finally { + fs.rmSync(testRoot, { recursive: true, force: true }); +} + +console.log("CGI series collection tests passed"); diff --git a/tests/web-runner.test.js b/tests/web-runner.test.js index 4e3ea25..ad07f69 100644 --- a/tests/web-runner.test.js +++ b/tests/web-runner.test.js @@ -266,6 +266,61 @@ async function testWebSeriesCandidateUsesDedicatedEndpoint() { assert.strictEqual(body.get("plot_only"), null); } +async function testWebSeriesCollectionRegistersArtifactsAndPlot() { + const calls = []; + const plots = []; + const logs = []; + const lib = loadWebLibrary(async function(url, options) { + calls.push({ url: url, options: options }); + return createResponse(JSON.stringify({ + ok: true, + code: 0, + reason: "series-artifacts-collected", + job_name: "job", + msr_url: "/tmp/job/514+series_asy.msr", + out_url: "/tmp/job/514+series_asy.out", + parameter_url: "/tmp/job/514+series_asy_par.dat", + root_urls: ["/tmp/job/514+series_asy_0.root"], + root_items: ["fMainCanvas0;1"], + command_log: "$ msr2data [514 515] _asy new global data\n", + warnings: ["msr2data returned code 9 but produced usable series artifacts"] + })); + }); + lib.appendJobLog = function(text) { logs.push(text); }; + lib.musrfitRootUrlsFromResponse = function(response) { return response.root_urls; }; + lib.showWebRootPlotWindow = function(urls, jobName, options) { + plots.push({ urls: urls, jobName: jobName, options: options }); + }; + + const result = await lib.window.collectIndependentSeriesArtifactsWeb({ + ok: true, + workPath: "/tmp/job", + fileNameSuffix: "_asy", + runs: ["514", "515"], + msrPaths: ["/tmp/job/514_asy.msr", "/tmp/job/515_asy.msr"], + fftv: "n" + }); + + assert.strictEqual(result.ok, true); + assert.strictEqual(result.parameterDataPath, "/tmp/job/514+series_asy_par.dat"); + assert.strictEqual(result.combinedMSRPath, "/tmp/job/514+series_asy.msr"); + assert.strictEqual(result.plotPath, "/tmp/job/514+series_asy_0.root"); + assert.strictEqual(result.plotStarted, true); + assert.match(result.warnings[0], /returned code 9/); + assert.strictEqual(calls.length, 1); + const body = new URLSearchParams(calls[0].options.body); + assert.strictEqual(body.get("action"), "series_collect"); + assert.strictEqual(body.get("fn"), "/tmp/job/514_asy.msr"); + assert.deepStrictEqual(JSON.parse(body.get("collection")), { + runs: ["514", "515"], suffix: "_asy", fftv: "n" + }); + assert.strictEqual(plots.length, 1); + assert.deepStrictEqual(Array.from(plots[0].urls), + ["/tmp/job/514+series_asy_0.root"]); + assert.strictEqual(plots[0].options.msrPath, "/tmp/job/514+series_asy.msr"); + assert.match(logs[0], /msr2data/); +} + function convergedWebMSR(run) { const suffix = String(run).padStart(4, "0"); return [ @@ -559,6 +614,7 @@ async function runTests() { await testSeriesTransactionUsesOneServerRequest(); await testSeriesTransactionRejectsMixedJobDirectories(); await testWebSeriesCandidateUsesDedicatedEndpoint(); + await testWebSeriesCollectionRegistersArtifactsAndPlot(); await testWebSeriesInventoryFeedsSharedClassifier(); await testIncompleteWebSeriesInventoryFailsClosed(); await testWebRunLoadsFittedMSRAndArtifacts();