Add server-side safeguards so heavy web workflows cannot crash Apache: - Run-concurrency gate: whole run/series_run_candidate requests lease flock slots under TMPROOT/.musrflow-run-slots (MUSRFIT_MAX_RUNS=2 default, MUSRFIT_MAX_RUNS_PER_IP optional per-client pool); saturated servers answer 503 server-busy and leases are released when the CGI process exits. - Per-child resource wrapper: every external command runs with RLIMIT_AS (MUSRFIT_CHILD_MEM_KB, 16 GiB), optional RLIMIT_CPU, and exec nice (MUSRFIT_CHILD_NICE=10) so a runaway child cannot starve Apache workers. - Retune MUSRFIT_CMD_TIMEOUT default 400 -> 360 (deploy knob). - Size/job backstops: POST body cap before CGI->new (413), write MSR cap (413), and MUSRFIT_MAX_JOBS refusal of new job folders (never deletes). - Exclude the .musrflow-run-slots dir from prune_stale_jobs and job counting. - Document the new knobs in cgi/README.md and add CGI hardening tests.
2626 lines
112 KiB
Perl
Executable File
2626 lines
112 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
# Copyright (C) 2026 Zaher Salman
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
#
|
|
# single_musrfit.cgi
|
|
# Template/write/run endpoint for muSRFlow web workflows.
|
|
|
|
use strict;
|
|
use warnings;
|
|
use CGI;
|
|
use JSON::PP qw(encode_json decode_json);
|
|
use File::Path qw(make_path);
|
|
use Fcntl qw(:DEFAULT :flock);
|
|
use Digest::SHA qw(sha256_hex);
|
|
|
|
my $WEBROOT = $ENV{MUSRFIT_WEBROOT} || "/var/www/html";
|
|
$WEBROOT =~ s{/+$}{} if $WEBROOT ne "/";
|
|
my $TMPROOT = $ENV{MUSRFIT_TMPROOT} || "$WEBROOT/tmp";
|
|
$TMPROOT =~ s{/+$}{} if $TMPROOT ne "/";
|
|
my $TMP_URL = $ENV{MUSRFIT_TMP_URL} || "/tmp";
|
|
$TMP_URL =~ s{/+$}{} if $TMP_URL ne "/";
|
|
my $MUSRFIT_BIN = $ENV{MUSRFIT_BIN} || "/usr/local/bin/musrfit";
|
|
my $MUSRVIEW_BIN = $ENV{MUSRVIEW_BIN} || "/usr/local/bin/musrview";
|
|
my $MSR2DATA_BIN = $ENV{MSR2DATA_BIN} || "/usr/local/bin/msr2data";
|
|
my $DUMP_HEADER_BIN = $ENV{DUMP_HEADER_BIN} || "/usr/local/bin/dump_header";
|
|
my $DATA_ROOT = $ENV{MUSRFIT_DATA_ROOT} || "/psi.ch/group/lmu/public/data";
|
|
$DATA_ROOT =~ s{/+$}{} if $DATA_ROOT ne "/";
|
|
|
|
# Per-command watchdog. Every external executable (musrfit, musrview, msr2data,
|
|
# dump_header, tar) is bounded by an alarm() watchdog in run_logged_command /
|
|
# run_backtick_command regardless of whether the tool implements its own
|
|
# --timeout flag. Default is comfortably above the 300s --timeout the tools are
|
|
# launched with so a well-behaved run is never cut off, while a hung msr2data,
|
|
# dump_header, or tar cannot strand an Apache worker indefinitely.
|
|
my $CMD_TIMEOUT = (defined $ENV{MUSRFIT_CMD_TIMEOUT} && $ENV{MUSRFIT_CMD_TIMEOUT} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_CMD_TIMEOUT}) : 360;
|
|
|
|
# Per-child resource hooks applied before every external executable runs (see
|
|
# child_resource_prefix). Limiting virtual memory and CPU-seconds, and lowering
|
|
# the scheduling priority, keeps a single musrfit/musrview child from starving
|
|
# the web server or exhausting RAM under simultaneous users. CPU-seconds stays
|
|
# unlimited by default because the alarm() watchdog above already bounds wall
|
|
# time; any of the three can be tuned per deployment.
|
|
my $CHILD_MEM_KB = (defined $ENV{MUSRFIT_CHILD_MEM_KB} && $ENV{MUSRFIT_CHILD_MEM_KB} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_CHILD_MEM_KB}) : 16777216; # RLIMIT_AS (ulimit -v): 16 GiB
|
|
my $CHILD_CPU_SEC = (defined $ENV{MUSRFIT_CHILD_CPU_SEC} && $ENV{MUSRFIT_CHILD_CPU_SEC} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_CHILD_CPU_SEC}) : 0; # RLIMIT_CPU (ulimit -t): unlimited
|
|
my $CHILD_NICE = (defined $ENV{MUSRFIT_CHILD_NICE} && $ENV{MUSRFIT_CHILD_NICE} =~ /^-?\d+$/)
|
|
? int($ENV{MUSRFIT_CHILD_NICE}) : 10; # priority (nice): background
|
|
|
|
# Heavy-run admission control bounds how many fits/plots may execute at once
|
|
# across all web users, plus an optional per-client pool so one busy tab cannot
|
|
# monopolize the global slots. Slots are advisory flocks on small regular files;
|
|
# they are released automatically when the CGI process exits, so a crashed
|
|
# server can never leave a stale lock behind.
|
|
my $RUN_SLOT_BASENAME = ".musrflow-run-slots";
|
|
my $RUN_SLOT_DIR = "$TMPROOT/$RUN_SLOT_BASENAME";
|
|
my $MAX_RUNS = (defined $ENV{MUSRFIT_MAX_RUNS} && $ENV{MUSRFIT_MAX_RUNS} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_MAX_RUNS}) : 2;
|
|
my $MAX_RUNS_PER_IP = (defined $ENV{MUSRFIT_MAX_RUNS_PER_IP} && $ENV{MUSRFIT_MAX_RUNS_PER_IP} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_MAX_RUNS_PER_IP}) : 0;
|
|
|
|
# Request and artifact size backstops. Apache may also limit the request body;
|
|
# these keep a single bundled deployment self-protecting on other servers.
|
|
my $MAX_REQUEST_BYTES = (defined $ENV{MUSRFIT_MAX_REQUEST_BYTES} && $ENV{MUSRFIT_MAX_REQUEST_BYTES} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_MAX_REQUEST_BYTES}) : 8_000_000;
|
|
my $MAX_MSR_BYTES = (defined $ENV{MUSRFIT_MAX_MSR_BYTES} && $ENV{MUSRFIT_MAX_MSR_BYTES} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_MAX_MSR_BYTES}) : 2_000_000;
|
|
my $MAX_JOBS = (defined $ENV{MUSRFIT_MAX_JOBS} && $ENV{MUSRFIT_MAX_JOBS} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_MAX_JOBS}) : 0; # 0 = unlimited sessions
|
|
|
|
# Job-directory lifetime in seconds. prune_stale_jobs() removes web job
|
|
# directories under TMPROOT that are older than this, bounding disk usage from
|
|
# long-running or abandoned analysis sessions. Jobs newer than this are left
|
|
# in place so an active browser session is never lost.
|
|
my $JOB_MAXAGE = (defined $ENV{MUSRFIT_JOB_MAXAGE} && $ENV{MUSRFIT_JOB_MAXAGE} =~ /^\d+$/)
|
|
? int($ENV{MUSRFIT_JOB_MAXAGE}) : 86400;
|
|
|
|
# Server-owned musrfit_startup.xml used to build portable RUN names. Defaults to
|
|
# the file the deploy script installs beside this CGI (outside the document
|
|
# root). Never is it read from a browser-supplied path.
|
|
my $STARTUP_XML = $ENV{MUSRFIT_STARTUP_XML} || "/var/www/cgi-bin/musrfit_startup.xml";
|
|
|
|
# Apache starts CGI processes with a minimal environment. Keep all writable
|
|
# runtime configuration under the web temporary root and identify installed
|
|
# musrfit programs explicitly instead of falling back to the ROOT bin folder.
|
|
$ENV{ROOTSYS} = $ENV{ROOTSYS} || "/usr/lib64/root";
|
|
$ENV{MUSRFITPATH} = $ENV{MUSRFITPATH} || "/usr/local/bin";
|
|
# MUSRFULLDATAPATH tells musrfit/musrview where to resolve relative RUN paths
|
|
# such as "2026/lem26_his_0514.root". An explicit admin value always wins; when
|
|
# unset, ensure_full_data_path() later derives it from the first on-disk
|
|
# <data_path> of the startup XML (falling back to $DATA_ROOT). That keeps the
|
|
# search root consistent with the portable-RUN resolver and, unlike the shared
|
|
# $DATA_ROOT, points at the instrument root where relative paths actually live.
|
|
my $MUSRFULLDATAPATH_EXPLICIT = (defined $ENV{MUSRFULLDATAPATH} && length($ENV{MUSRFULLDATAPATH}) > 0);
|
|
$ENV{MUSRFULLDATAPATH} = $ENV{MUSRFULLDATAPATH} || $DATA_ROOT;
|
|
$ENV{HOME} = $ENV{MUSRFIT_RUNTIME_HOME} || $TMPROOT;
|
|
|
|
# Reject oversized POST bodies before CGI->new buffers them for parsing, so a
|
|
# single huge request cannot consume a disproportionate share of the worker's
|
|
# memory. Apache's own LimitRequestBody remains an outer bound; this is the
|
|
# CGI-side backstop.
|
|
if (defined($ENV{CONTENT_LENGTH}) && $ENV{CONTENT_LENGTH} =~ /^\d+$/
|
|
&& int($ENV{CONTENT_LENGTH}) > $MAX_REQUEST_BYTES) {
|
|
print "Content-type: application/json\n";
|
|
print "Cache-Control: no-store\n\n";
|
|
print encode_json({
|
|
ok => JSON::PP::false, code => 413, reason => "request-too-large",
|
|
message => "Request body exceeds the configured limit of $MAX_REQUEST_BYTES bytes"
|
|
});
|
|
exit(0);
|
|
}
|
|
|
|
my $in = CGI->new;
|
|
my $action = $in->param("action") || "write";
|
|
my $fn = $in->param("fn") || "";
|
|
|
|
# JSON response and health-check helpers.
|
|
sub respond_json {
|
|
my ($payload_ref) = @_;
|
|
print "Content-type: application/json\n";
|
|
print "Cache-Control: no-store\n\n";
|
|
print encode_json($payload_ref);
|
|
exit(0);
|
|
}
|
|
|
|
# Prefix that bounds every external executable (musrfit, musrview, msr2data,
|
|
# dump_header, tar). It is applied inside the forked child, before the real
|
|
# command: set the virtual-memory and optional CPU-seconds limits with the
|
|
# shell's ulimit builtin, then exec a de-prioritized copy of the command. Each
|
|
# limit degrades gracefully (stderr suppressed, `;` separators) so an
|
|
# environment that cannot honor a particular limit still runs the job.
|
|
sub child_resource_prefix {
|
|
my ($command) = @_;
|
|
my @parts;
|
|
push @parts, "ulimit -v $CHILD_MEM_KB 2>/dev/null" if $CHILD_MEM_KB > 0;
|
|
push @parts, "ulimit -t $CHILD_CPU_SEC 2>/dev/null" if $CHILD_CPU_SEC > 0;
|
|
if ($CHILD_NICE != 0) {
|
|
push @parts, "exec nice -n $CHILD_NICE sh -c " . shell_quote($command);
|
|
}
|
|
return $command if !@parts;
|
|
return join(" ; ", @parts);
|
|
}
|
|
|
|
# Heavy-run admission control: lease one of a small set of flock-gated slot
|
|
# files. A slot is held by keeping the file handle open for the lifetime of the
|
|
# request; every handle closes (and every flock is released) when the CGI
|
|
# process exits, so no stale counter can be left behind by a dead worker.
|
|
sub ensure_run_slot_dir {
|
|
my ($dir) = @_;
|
|
return 1 if -d $dir;
|
|
return 0 if !make_path($dir, { mode => 0755 });
|
|
return 1;
|
|
}
|
|
|
|
sub acquire_run_slot {
|
|
my ($pool) = @_; # undef/"" = global pool, otherwise a client pool tag
|
|
my $max = $pool ? $MAX_RUNS_PER_IP : $MAX_RUNS;
|
|
my $dir = $pool ? "$RUN_SLOT_DIR/$pool" : $RUN_SLOT_DIR;
|
|
return undef if $max < 1 || !ensure_run_slot_dir($dir);
|
|
for my $i (0 .. $max - 1) {
|
|
my $slot_path = "$dir/slot.$i";
|
|
next if !sysopen(my $slot_fh, $slot_path, O_RDWR | O_CREAT, 0600);
|
|
if (flock($slot_fh, LOCK_EX | LOCK_NB)) {
|
|
truncate($slot_fh, 0);
|
|
return $slot_fh;
|
|
}
|
|
close($slot_fh);
|
|
}
|
|
return undef;
|
|
}
|
|
|
|
sub acquire_run_admission {
|
|
# Admission succeeds only when the client (per-IP) and global pools both
|
|
# have capacity. Returns the list of slot handles the caller must keep open
|
|
# for the request, or () when the server is saturated (any client slot that
|
|
# was already taken is released before returning).
|
|
my @held;
|
|
if ($MAX_RUNS_PER_IP > 0) {
|
|
my $client_key = defined($ENV{REMOTE_ADDR}) ? $ENV{REMOTE_ADDR} : "unknown";
|
|
$client_key =~ s/[^A-Za-z0-9._-]/_/g;
|
|
my $slot = acquire_run_slot("ip.$client_key");
|
|
return () if !$slot;
|
|
push @held, $slot;
|
|
}
|
|
my $slot = acquire_run_slot("");
|
|
if (!$slot) {
|
|
close($_) for @held;
|
|
return ();
|
|
}
|
|
push @held, $slot;
|
|
return @held;
|
|
}
|
|
|
|
sub count_job_dirs {
|
|
# Number of live web job directories under TMPROOT, mirroring the safe-name
|
|
# and symlink rules of prune_stale_jobs. The hidden slot directory is not a
|
|
# job and is excluded so it does not exhaust a configured job limit.
|
|
return 0 if !-d $TMPROOT;
|
|
opendir(my $dh, $TMPROOT) or return 0;
|
|
my $count = 0;
|
|
while (my $entry = readdir($dh)) {
|
|
next if $entry eq "." || $entry eq "..";
|
|
next if $entry eq $RUN_SLOT_BASENAME;
|
|
next if $entry !~ /^[A-Za-z0-9._-]+$/;
|
|
next if !-d "$TMPROOT/$entry";
|
|
next if -l "$TMPROOT/$entry";
|
|
$count++;
|
|
}
|
|
closedir($dh);
|
|
return $count;
|
|
}
|
|
|
|
sub executable_status {
|
|
my ($path) = @_;
|
|
my $exists = -f $path ? 1 : 0;
|
|
my $executable = -x $path ? 1 : 0;
|
|
my $runnable = 0;
|
|
my $version = "";
|
|
my $diagnostic = "";
|
|
my $version_code = undef;
|
|
|
|
if ($executable) {
|
|
# Capture loader errors as health diagnostics. Letting stderr escape here
|
|
# would put text before the CGI headers and invalidate the JSON response.
|
|
my $version_cmd = shell_quote($path) . " --version 2>&1";
|
|
my $version_output = `$version_cmd`;
|
|
$version_code = $? == -1 ? 127 : $? >> 8;
|
|
$version_output = "" if !defined($version_output);
|
|
$version_output =~ s/\r//g;
|
|
my @output_lines = grep { /\S/ } split(/\n/, $version_output);
|
|
$diagnostic = @output_lines ? $output_lines[0] : "";
|
|
if ($version_code == 0) {
|
|
$runnable = 1;
|
|
$version = $diagnostic;
|
|
}
|
|
}
|
|
|
|
return {
|
|
path => $path,
|
|
exists => $exists ? JSON::PP::true : JSON::PP::false,
|
|
executable => $executable ? JSON::PP::true : JSON::PP::false,
|
|
runnable => $runnable ? JSON::PP::true : JSON::PP::false,
|
|
version_code => $version_code,
|
|
version => $version,
|
|
diagnostic => $diagnostic
|
|
};
|
|
}
|
|
|
|
# Shell/file helpers used only with server-controlled executable and job paths.
|
|
sub shell_quote {
|
|
my ($s) = @_;
|
|
$s =~ s/'/'\\''/g;
|
|
return "'" . $s . "'";
|
|
}
|
|
|
|
sub read_text_file {
|
|
my ($path) = @_;
|
|
open(my $fh, "<", $path) or return "";
|
|
local $/;
|
|
my $content = <$fh>;
|
|
close($fh);
|
|
return defined($content) ? $content : "";
|
|
}
|
|
|
|
# Return true when the MSR carries an explicit first-line title that musrfit's
|
|
# -t (--title-from-data-file) would otherwise replace. The first non-blank,
|
|
# non-comment line of a real MSR is its run title; a GUI-generated MSR starts
|
|
# with the '# Run Numbers:' comment (or a blank line) and has no title to
|
|
# preserve. Mirrors hasExplicitMSRTitle() in muSRFitelec.js.
|
|
sub has_explicit_msr_title {
|
|
my ($path) = @_;
|
|
my $content = read_text_file($path);
|
|
return 0 if $content eq "";
|
|
for my $raw_line (split(/\r?\n/, $content)) {
|
|
my $line = $raw_line;
|
|
$line =~ s/^\s+|\s+$//g;
|
|
next if $line eq "";
|
|
return $line !~ /^#/ ? 1 : 0;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
sub write_text_file {
|
|
my ($path, $content) = @_;
|
|
open(my $fh, ">", $path) or return 0;
|
|
print {$fh} $content;
|
|
close($fh);
|
|
return 1;
|
|
}
|
|
|
|
sub write_text_file_exclusive {
|
|
my ($path, $content) = @_;
|
|
return 0 if !sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_EXCL, 0644);
|
|
my $ok = print {$fh} $content;
|
|
$ok = 0 if !close($fh);
|
|
if (!$ok) {
|
|
unlink($path);
|
|
return 0;
|
|
}
|
|
return 1;
|
|
}
|
|
|
|
sub series_transaction_file_name {
|
|
my ($target_name, $kind, $transaction_id) = @_;
|
|
my $base = $target_name;
|
|
$base =~ s/\.msr$//i;
|
|
return $base . ".musrflow-" . $kind . "-" . $transaction_id . ".msr";
|
|
}
|
|
|
|
sub cleanup_series_stages {
|
|
my ($items) = @_;
|
|
my @errors = ();
|
|
foreach my $item (reverse(@{$items || []})) {
|
|
next if !-e $item->{stage_path};
|
|
if (!unlink($item->{stage_path})) {
|
|
push(@errors, {
|
|
run => $item->{run},
|
|
operation => "discard-stage",
|
|
path => $item->{stage_name},
|
|
message => "Could not remove staged MSR"
|
|
});
|
|
}
|
|
}
|
|
return \@errors;
|
|
}
|
|
|
|
sub rollback_series_receipts {
|
|
my ($receipts) = @_;
|
|
my @errors = ();
|
|
foreach my $receipt (reverse(@{$receipts || []})) {
|
|
my $current = -f $receipt->{target_path}
|
|
? read_text_file($receipt->{target_path}) : undef;
|
|
if (!defined($current) || $current ne $receipt->{promoted_text}) {
|
|
push(@errors, {
|
|
run => $receipt->{run},
|
|
operation => "rollback",
|
|
path => $receipt->{target_name},
|
|
message => "Promoted target changed before rollback"
|
|
});
|
|
next;
|
|
}
|
|
|
|
if (!$receipt->{replaced}) {
|
|
if (!unlink($receipt->{target_path})) {
|
|
push(@errors, {
|
|
run => $receipt->{run},
|
|
operation => "rollback",
|
|
path => $receipt->{target_name},
|
|
message => "Could not remove promoted target"
|
|
});
|
|
}
|
|
next;
|
|
}
|
|
|
|
my $backup = -f $receipt->{backup_path}
|
|
? read_text_file($receipt->{backup_path}) : undef;
|
|
if (!defined($backup) || $backup ne $receipt->{original_text}) {
|
|
push(@errors, {
|
|
run => $receipt->{run},
|
|
operation => "rollback",
|
|
path => $receipt->{backup_name},
|
|
message => "Rollback backup is missing or changed"
|
|
});
|
|
next;
|
|
}
|
|
if (!rename($receipt->{backup_path}, $receipt->{target_path})) {
|
|
push(@errors, {
|
|
run => $receipt->{run},
|
|
operation => "rollback",
|
|
path => $receipt->{target_name},
|
|
message => "Could not restore replacement backup"
|
|
});
|
|
}
|
|
}
|
|
return \@errors;
|
|
}
|
|
|
|
sub series_run_directives {
|
|
my ($content) = @_;
|
|
my @directives = ();
|
|
foreach my $line (split(/\r?\n/, $content || "")) {
|
|
next if $line !~ /^\s*(?:RUN|ADDRUN)\b/i;
|
|
my @parts = grep { $_ ne "" } split(/\s+/, $line);
|
|
if (@parts < 5) {
|
|
push(@directives, "INVALID " . join(" ", @parts));
|
|
next;
|
|
}
|
|
# Only the first five fields define the data-set identity. musrfit may
|
|
# append a parenthetical field description when writing a fitted MSR.
|
|
$parts[0] = uc($parts[0]);
|
|
push(@directives, join(" ", @parts[0 .. 4]));
|
|
}
|
|
return \@directives;
|
|
}
|
|
|
|
sub series_run_directive_signature {
|
|
my ($content) = @_;
|
|
return sha256_hex(join("\n", @{series_run_directives($content)}));
|
|
}
|
|
|
|
sub perform_series_transaction {
|
|
my ($work_path, $job_name, $transaction_text) = @_;
|
|
my $failure = sub {
|
|
my ($reason, $message, $errors, $rollback_errors) = @_;
|
|
return {
|
|
ok => JSON::PP::false,
|
|
code => 409,
|
|
reason => $reason,
|
|
message => $message,
|
|
staged => [],
|
|
receipts => [],
|
|
backups => [],
|
|
errors => $errors || [],
|
|
rollbackErrors => $rollback_errors || []
|
|
};
|
|
};
|
|
|
|
return $failure->("invalid-series-transaction", "Series transaction is too large")
|
|
if !defined($transaction_text) || length($transaction_text) > 20_000_000;
|
|
my $decoded = eval { decode_json($transaction_text) };
|
|
return $failure->("invalid-series-transaction", "Invalid series transaction JSON")
|
|
if $@ || ref($decoded) ne "HASH";
|
|
my $transaction_id = defined($decoded->{id}) ? "$decoded->{id}" : "";
|
|
return $failure->("invalid-series-transaction", "Invalid series transaction identifier")
|
|
if $transaction_id !~ /^[A-Za-z0-9-]{1,64}$/;
|
|
my $manifest_name = ".musrflow-series-transaction-$transaction_id.json";
|
|
my $manifest_path = "$work_path/$manifest_name";
|
|
my $candidates = $decoded->{candidates};
|
|
return $failure->("invalid-series-transaction", "Invalid series candidate list")
|
|
if ref($candidates) ne "ARRAY" || !@{$candidates} || @{$candidates} > 500;
|
|
|
|
my @items = ();
|
|
my %seen_targets = ();
|
|
my $total_text = 0;
|
|
foreach my $candidate (@{$candidates}) {
|
|
return $failure->("invalid-series-candidate", "Invalid series candidate")
|
|
if ref($candidate) ne "HASH";
|
|
my $run = defined($candidate->{run}) ? "$candidate->{run}" : "";
|
|
my $action = defined($candidate->{action}) ? "$candidate->{action}" : "";
|
|
my $target_name = defined($candidate->{target_name}) ? "$candidate->{target_name}" : "";
|
|
my $text = defined($candidate->{text}) && !ref($candidate->{text})
|
|
? "$candidate->{text}" : "";
|
|
my $expected_text = defined($candidate->{expected_text}) && !ref($candidate->{expected_text})
|
|
? "$candidate->{expected_text}" : "";
|
|
return $failure->("invalid-series-candidate", "Invalid series run number")
|
|
if $run !~ /^\d+$/ || $run <= 0;
|
|
return $failure->("invalid-series-candidate", "Invalid series action")
|
|
if $action !~ /^(?:fit|retry|refit)$/;
|
|
return $failure->("invalid-series-candidate", "Invalid series target filename")
|
|
if $target_name !~ /^\Q$run\E(?:[._+\-][A-Za-z0-9._+\-]*)?\.msr$/;
|
|
return $failure->("duplicate-series-target", "Duplicate series target filename")
|
|
if $seen_targets{$target_name}++;
|
|
return $failure->("invalid-series-candidate", "Series MSR is empty or too large")
|
|
if $text eq "" || length($text) > 2_000_000;
|
|
return $failure->("invalid-series-candidate", "Existing target content is required")
|
|
if $action ne "fit" && $expected_text eq "";
|
|
return $failure->("invalid-series-candidate", "Series MSR structure is invalid")
|
|
if !is_valid_generated_global_msr($text)
|
|
|| $text =~ /^\s*STATISTIC\b/im || $text =~ /\bundefined\b/i;
|
|
$total_text += length($text) + length($expected_text);
|
|
return $failure->("invalid-series-transaction", "Series transaction is too large")
|
|
if $total_text > 20_000_000;
|
|
|
|
my $stage_name = series_transaction_file_name($target_name, "stage", $transaction_id);
|
|
my $backup_name = $action eq "fit" ? ""
|
|
: series_transaction_file_name($target_name, "backup", $transaction_id);
|
|
push(@items, {
|
|
run => 0 + $run,
|
|
action => $action,
|
|
target_name => $target_name,
|
|
target_path => "$work_path/$target_name",
|
|
stage_name => $stage_name,
|
|
stage_path => "$work_path/$stage_name",
|
|
backup_name => $backup_name,
|
|
backup_path => $backup_name eq "" ? "" : "$work_path/$backup_name",
|
|
text => $text,
|
|
expected_text => $expected_text,
|
|
replace => $action eq "fit" ? 0 : 1
|
|
});
|
|
}
|
|
|
|
# Serialize transaction endpoints within one web job. This lock does not
|
|
# widen path access and will also be used by the later series runner patch.
|
|
my $lock_path = "$work_path/.musrflow-series.lock";
|
|
return $failure->("series-transaction-lock-failed", "Could not open series transaction lock")
|
|
if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600);
|
|
return $failure->("series-transaction-lock-failed", "Another series transaction is active")
|
|
if !flock($lock_fh, LOCK_EX | LOCK_NB);
|
|
if (-e $manifest_path) {
|
|
close($lock_fh);
|
|
return $failure->("transaction-path-collision", "Series transaction manifest already exists");
|
|
}
|
|
|
|
foreach my $item (@items) {
|
|
my $target_exists = -f $item->{target_path} ? 1 : 0;
|
|
if ((!$item->{replace} && $target_exists) || ($item->{replace} && !$target_exists)) {
|
|
close($lock_fh);
|
|
return $failure->("series-target-state-changed", "Series target state changed", [{
|
|
run => $item->{run},
|
|
operation => "preflight",
|
|
path => $item->{target_name},
|
|
message => $item->{replace}
|
|
? "Replacement target was not found" : "New target already exists"
|
|
}]);
|
|
}
|
|
if ($item->{replace} && read_text_file($item->{target_path}) ne $item->{expected_text}) {
|
|
close($lock_fh);
|
|
return $failure->("series-target-state-changed", "Series target content changed", [{
|
|
run => $item->{run},
|
|
operation => "preflight",
|
|
path => $item->{target_name},
|
|
message => "Replacement target content changed"
|
|
}]);
|
|
}
|
|
if (-e $item->{stage_path} || ($item->{backup_path} ne "" && -e $item->{backup_path})) {
|
|
close($lock_fh);
|
|
return $failure->("transaction-path-collision", "Series transaction path already exists", [{
|
|
run => $item->{run},
|
|
operation => "preflight",
|
|
path => $item->{stage_name},
|
|
message => "Transaction-owned path already exists"
|
|
}]);
|
|
}
|
|
}
|
|
|
|
my @staged_items = ();
|
|
foreach my $item (@items) {
|
|
my $stage_created = write_text_file_exclusive($item->{stage_path}, $item->{text});
|
|
if (!$stage_created || read_text_file($item->{stage_path}) ne $item->{text}) {
|
|
my $cleanup_errors = cleanup_series_stages(\@staged_items);
|
|
# Include the current path if exclusive creation succeeded but its
|
|
# readback failed before it entered the verified list.
|
|
if ($stage_created && -e $item->{stage_path} && !unlink($item->{stage_path})) {
|
|
push(@{$cleanup_errors}, {
|
|
run => $item->{run}, operation => "discard-stage",
|
|
path => $item->{stage_name}, message => "Could not remove staged MSR"
|
|
});
|
|
}
|
|
close($lock_fh);
|
|
return $failure->(@{$cleanup_errors} ? "stage-cleanup-failed" : "series-staging-failed",
|
|
"Could not stage series MSR", [{
|
|
run => $item->{run}, operation => "stage",
|
|
path => $item->{stage_name}, message => "Exclusive write or readback failed"
|
|
}], $cleanup_errors);
|
|
}
|
|
push(@staged_items, $item);
|
|
}
|
|
|
|
my @receipts = ();
|
|
foreach my $item (@items) {
|
|
my $promotion_error = "";
|
|
my $original_text = "";
|
|
if (!$item->{replace}) {
|
|
if (!link($item->{stage_path}, $item->{target_path})) {
|
|
$promotion_error = "Could not publish new target exclusively";
|
|
} elsif (!unlink($item->{stage_path})) {
|
|
unlink($item->{target_path})
|
|
if read_text_file($item->{target_path}) eq $item->{text};
|
|
$promotion_error = "Could not remove published stage link";
|
|
}
|
|
} else {
|
|
$original_text = read_text_file($item->{target_path});
|
|
if ($original_text ne $item->{expected_text}) {
|
|
$promotion_error = "Replacement target changed during promotion";
|
|
} else {
|
|
my $backup_created = write_text_file_exclusive(
|
|
$item->{backup_path}, $original_text
|
|
);
|
|
if (!$backup_created) {
|
|
$promotion_error = "Could not create replacement backup exclusively";
|
|
} elsif (read_text_file($item->{backup_path}) ne $original_text) {
|
|
unlink($item->{backup_path});
|
|
$promotion_error = "Could not verify replacement backup";
|
|
} elsif (!rename($item->{stage_path}, $item->{target_path})) {
|
|
unlink($item->{backup_path});
|
|
$promotion_error = "Could not promote replacement target";
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($promotion_error ne "") {
|
|
my $rollback_errors = rollback_series_receipts(\@receipts);
|
|
my $cleanup_errors = cleanup_series_stages(\@staged_items);
|
|
push(@{$rollback_errors}, @{$cleanup_errors});
|
|
close($lock_fh);
|
|
return $failure->(@{$rollback_errors}
|
|
? "promotion-rollback-failed" : "series-promotion-failed",
|
|
"Could not promote series MSR", [{
|
|
run => $item->{run}, operation => "promote",
|
|
path => $item->{target_name}, message => $promotion_error
|
|
}], $rollback_errors);
|
|
}
|
|
|
|
push(@receipts, {
|
|
run => $item->{run},
|
|
action => $item->{action},
|
|
target_name => $item->{target_name},
|
|
target_path => $item->{target_path},
|
|
backup_name => $item->{backup_name},
|
|
backup_path => $item->{backup_path},
|
|
replaced => $item->{replace},
|
|
promoted_text => $item->{text},
|
|
original_text => $original_text
|
|
});
|
|
}
|
|
|
|
my $manifest_text = encode_json({
|
|
version => 1,
|
|
id => $transaction_id,
|
|
job_name => $job_name,
|
|
candidates => [map {
|
|
{
|
|
run => $_->{run},
|
|
action => $_->{action},
|
|
target_name => $_->{target_name},
|
|
backup_name => $_->{backup_name},
|
|
replaced => $_->{replaced} ? JSON::PP::true : JSON::PP::false,
|
|
promoted_sha256 => sha256_hex($_->{promoted_text}),
|
|
run_directives_sha256 => series_run_directive_signature($_->{promoted_text}),
|
|
original_sha256 => $_->{replaced}
|
|
? sha256_hex($_->{original_text}) : ""
|
|
}
|
|
} @receipts]
|
|
});
|
|
my $manifest_created = write_text_file_exclusive($manifest_path, $manifest_text);
|
|
if (!$manifest_created || read_text_file($manifest_path) ne $manifest_text) {
|
|
unlink($manifest_path) if $manifest_created && -e $manifest_path;
|
|
my $rollback_errors = rollback_series_receipts(\@receipts);
|
|
my $cleanup_errors = cleanup_series_stages(\@staged_items);
|
|
push(@{$rollback_errors}, @{$cleanup_errors});
|
|
close($lock_fh);
|
|
return $failure->(@{$rollback_errors}
|
|
? "promotion-rollback-failed" : "transaction-manifest-failed",
|
|
"Could not seal series transaction", [{
|
|
run => 0, operation => "seal-transaction",
|
|
path => $manifest_name, message => "Could not create verified transaction manifest"
|
|
}], $rollback_errors);
|
|
}
|
|
|
|
close($lock_fh);
|
|
my @public_receipts = map {
|
|
{
|
|
run => $_->{run}, action => $_->{action},
|
|
targetPath => "$TMP_URL/$job_name/$_->{target_name}",
|
|
backupPath => $_->{replaced}
|
|
? "$TMP_URL/$job_name/$_->{backup_name}" : "",
|
|
replaced => $_->{replaced} ? JSON::PP::true : JSON::PP::false
|
|
}
|
|
} @receipts;
|
|
my @backups = map {
|
|
{ run => $_->{run}, path => "$TMP_URL/$job_name/$_->{backup_name}" }
|
|
} grep { $_->{replaced} } @receipts;
|
|
my @staged = map {
|
|
{
|
|
run => $_->{run},
|
|
stagePath => "$TMP_URL/$job_name/$_->{stage_name}",
|
|
targetPath => "$TMP_URL/$job_name/$_->{target_name}"
|
|
}
|
|
} @items;
|
|
return {
|
|
ok => JSON::PP::true,
|
|
code => 0,
|
|
reason => "series-candidates-promoted",
|
|
message => "Series MSR candidates promoted",
|
|
transaction_id => $transaction_id,
|
|
staged => \@staged,
|
|
receipts => \@public_receipts,
|
|
backups => \@backups,
|
|
errors => [],
|
|
rollbackErrors => []
|
|
};
|
|
}
|
|
|
|
sub series_msr_is_converged {
|
|
my ($content) = @_;
|
|
return 0 if !is_valid_generated_global_msr($content);
|
|
return 0 if $content !~ /^\s*STATISTIC\b/im;
|
|
return 0 if $content =~ /FIT\s+DID\s+NOT\s+CONVERGE/i;
|
|
return 1 if $content =~ /^\s*chisq\s*=\s*[^,\r\n]+,\s*NDF\s*=\s*\d+\s*,\s*chisq\/NDF\s*=/im;
|
|
return 1 if $content =~ /^\s*maxLH\s*=\s*[^,\r\n]+,\s*NDF\s*=\s*\d+\s*,\s*maxLH\/NDF\s*=/im;
|
|
return 0;
|
|
}
|
|
|
|
sub perform_series_candidate_run {
|
|
my ($work_path, $job_name, $target_name, $transaction_id) = @_;
|
|
my $failure = sub {
|
|
my ($reason, $message, $code, $outcome, $missing_files, $out_name) = @_;
|
|
return {
|
|
ok => JSON::PP::false,
|
|
code => defined($code) ? $code : 1,
|
|
reason => $reason,
|
|
message => $message,
|
|
outcome => $outcome || "",
|
|
missing_files => $missing_files || [],
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_name",
|
|
out_url => $out_name ? "$TMP_URL/$job_name/$out_name" : ""
|
|
};
|
|
};
|
|
return $failure->("invalid-series-transaction", "Invalid series transaction identifier")
|
|
if !defined($transaction_id) || $transaction_id !~ /^[A-Za-z0-9-]{1,64}$/;
|
|
|
|
my $lock_path = "$work_path/.musrflow-series.lock";
|
|
return $failure->("series-run-lock-failed", "Could not open series transaction lock")
|
|
if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600);
|
|
if (!flock($lock_fh, LOCK_EX | LOCK_NB)) {
|
|
close($lock_fh);
|
|
return $failure->("series-run-lock-failed", "Another series operation is active");
|
|
}
|
|
my $locked_failure = sub {
|
|
my $result = $failure->(@_);
|
|
close($lock_fh);
|
|
return $result;
|
|
};
|
|
|
|
my $manifest_name = ".musrflow-series-transaction-$transaction_id.json";
|
|
my $manifest_path = "$work_path/$manifest_name";
|
|
return $locked_failure->(
|
|
"series-transaction-manifest-not-found", "Series transaction manifest not found"
|
|
) if !-f $manifest_path || -l $manifest_path || (-s $manifest_path || 0) > 1_000_000;
|
|
my $manifest = eval { decode_json(read_text_file($manifest_path)) };
|
|
return $locked_failure->(
|
|
"invalid-series-transaction-manifest", "Invalid series transaction manifest"
|
|
) if $@ || ref($manifest) ne "HASH"
|
|
|| ($manifest->{version} || 0) != 1
|
|
|| ($manifest->{id} || "") ne $transaction_id
|
|
|| ($manifest->{job_name} || "") ne $job_name
|
|
|| ref($manifest->{candidates}) ne "ARRAY"
|
|
|| !@{$manifest->{candidates}} || @{$manifest->{candidates}} > 500;
|
|
|
|
my @matches = grep {
|
|
ref($_) eq "HASH" && ($_->{target_name} || "") eq $target_name
|
|
} @{$manifest->{candidates}};
|
|
return $locked_failure->(
|
|
"series-candidate-not-in-transaction", "Series candidate is not in this transaction"
|
|
) if @matches != 1;
|
|
my $candidate = $matches[0];
|
|
my $run = defined($candidate->{run}) ? "$candidate->{run}" : "";
|
|
my $action = defined($candidate->{action}) ? "$candidate->{action}" : "";
|
|
my $promoted_hash = defined($candidate->{promoted_sha256})
|
|
? "$candidate->{promoted_sha256}" : "";
|
|
my $run_directives_hash = defined($candidate->{run_directives_sha256})
|
|
? "$candidate->{run_directives_sha256}" : "";
|
|
return $locked_failure->(
|
|
"invalid-series-transaction-manifest", "Invalid series transaction candidate"
|
|
) if $run !~ /^\d+$/ || $run <= 0
|
|
|| $action !~ /^(?:fit|retry|refit)$/
|
|
|| $target_name !~ /^\Q$run\E(?:[._+\-][A-Za-z0-9._+\-]*)?\.msr$/
|
|
|| $promoted_hash !~ /^[a-f0-9]{64}$/
|
|
|| $run_directives_hash !~ /^[a-f0-9]{64}$/;
|
|
|
|
my $target_path = "$work_path/$target_name";
|
|
return $locked_failure->(
|
|
"series-candidate-target-invalid", "Series candidate target is not a regular file"
|
|
) if !-f $target_path || -l $target_path;
|
|
my $promoted_text = read_text_file($target_path);
|
|
return $locked_failure->(
|
|
"series-candidate-target-changed", "Series candidate changed before execution"
|
|
) if sha256_hex($promoted_text) ne $promoted_hash
|
|
|| series_run_directive_signature($promoted_text) ne $run_directives_hash;
|
|
return $locked_failure->(
|
|
"series-musrfit-unavailable", "musrfit is not executable", 127
|
|
) if !-x $MUSRFIT_BIN;
|
|
|
|
my $out_name = $target_name;
|
|
$out_name =~ s/\.msr$/.out/i;
|
|
my $out_path = "$work_path/$out_name";
|
|
return $locked_failure->(
|
|
"series-log-path-invalid", "Series log path is not a regular file", 1, "", [], $out_name
|
|
) if -l $out_path || (-e $out_path && !-f $out_path);
|
|
return $locked_failure->(
|
|
"series-log-initialization-failed", "Could not initialize series fit log", 1,
|
|
"", [], $out_name
|
|
) if !write_text_file(
|
|
$out_path,
|
|
"# muSRFlow independent series fit\n# Run: $run\n# Working directory: $work_path\n"
|
|
);
|
|
|
|
my $fit_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MUSRFIT_BIN)
|
|
. (has_explicit_msr_title($target_path) ? " -e" : " -t -e")
|
|
. " --timeout 300 " . shell_quote($target_name);
|
|
my $fit_result = run_logged_command($out_path, "musrfit run $run", $fit_cmd);
|
|
my $missing_data = classify_missing_data_output($fit_result->{output});
|
|
my $fitted_text = (-f $target_path && !-l $target_path)
|
|
? read_text_file($target_path) : "";
|
|
my $identity_matches = $fitted_text ne ""
|
|
&& series_run_directive_signature($fitted_text) eq $run_directives_hash;
|
|
my $converged = $identity_matches && series_msr_is_converged($fitted_text);
|
|
my $ok = $fit_result->{raw_status} == 0
|
|
&& !$missing_data->{detected} && $converged;
|
|
my $reason = $ok ? "series-candidate-fitted"
|
|
: ($missing_data->{detected} ? "missing-data"
|
|
: ($fit_result->{raw_status} != 0 ? "series-musrfit-failed"
|
|
: (!$identity_matches ? "series-candidate-identity-changed"
|
|
: "series-candidate-not-converged")));
|
|
my $message = $ok ? "Run $run fitted"
|
|
: ($missing_data->{detected} ? $missing_data->{message}
|
|
: ($fit_result->{raw_status} != 0
|
|
? "musrfit failed for run $run" : "Run $run did not produce a converged MSR"));
|
|
close($lock_fh);
|
|
return {
|
|
ok => $ok ? JSON::PP::true : JSON::PP::false,
|
|
code => $fit_result->{exit_code},
|
|
reason => $reason,
|
|
message => $message,
|
|
outcome => $missing_data->{detected} ? "missing-data" : "",
|
|
missing_files => $missing_data->{missing_files},
|
|
converged => $converged ? JSON::PP::true : JSON::PP::false,
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_name",
|
|
out_url => "$TMP_URL/$job_name/$out_name"
|
|
};
|
|
}
|
|
|
|
sub perform_series_transaction_finalization {
|
|
my ($work_path, $job_name, $finalization_text) = @_;
|
|
my $failure = sub {
|
|
my ($reason, $message, $errors, $finalized) = @_;
|
|
return {
|
|
ok => JSON::PP::false,
|
|
code => 409,
|
|
reason => $reason,
|
|
message => $message,
|
|
finalized => $finalized || [],
|
|
errors => $errors || []
|
|
};
|
|
};
|
|
return $failure->("invalid-series-finalization", "Series finalization request is too large")
|
|
if !defined($finalization_text) || length($finalization_text) > 100_000;
|
|
my $request = eval { decode_json($finalization_text) };
|
|
return $failure->("invalid-series-finalization", "Invalid series finalization JSON")
|
|
if $@ || ref($request) ne "HASH";
|
|
my $transaction_id = defined($request->{id}) ? "$request->{id}" : "";
|
|
my $decision = defined($request->{decision}) ? "$request->{decision}" : "";
|
|
return $failure->("invalid-series-finalization", "Invalid series transaction identifier")
|
|
if $transaction_id !~ /^[A-Za-z0-9-]{1,64}$/;
|
|
return $failure->("invalid-series-finalization", "Invalid finalization decision")
|
|
if $decision !~ /^(?:commit|rollback|retain)$/;
|
|
|
|
my $manifest_name = ".musrflow-series-transaction-$transaction_id.json";
|
|
my $manifest_path = "$work_path/$manifest_name";
|
|
return $failure->("series-transaction-manifest-not-found", "Series transaction manifest not found")
|
|
if !-f $manifest_path || -l $manifest_path || (-s $manifest_path || 0) > 1_000_000;
|
|
my $manifest = eval { decode_json(read_text_file($manifest_path)) };
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid series transaction manifest")
|
|
if $@ || ref($manifest) ne "HASH"
|
|
|| ($manifest->{version} || 0) != 1
|
|
|| ($manifest->{id} || "") ne $transaction_id
|
|
|| ($manifest->{job_name} || "") ne $job_name
|
|
|| ref($manifest->{candidates}) ne "ARRAY"
|
|
|| !@{$manifest->{candidates}} || @{$manifest->{candidates}} > 500;
|
|
|
|
my @items = ();
|
|
my %seen_targets = ();
|
|
foreach my $candidate (@{$manifest->{candidates}}) {
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest candidate")
|
|
if ref($candidate) ne "HASH";
|
|
my $run = defined($candidate->{run}) ? "$candidate->{run}" : "";
|
|
my $action = defined($candidate->{action}) ? "$candidate->{action}" : "";
|
|
my $target_name = defined($candidate->{target_name}) ? "$candidate->{target_name}" : "";
|
|
my $backup_name = defined($candidate->{backup_name}) ? "$candidate->{backup_name}" : "";
|
|
my $promoted_hash = defined($candidate->{promoted_sha256})
|
|
? "$candidate->{promoted_sha256}" : "";
|
|
my $original_hash = defined($candidate->{original_sha256})
|
|
? "$candidate->{original_sha256}" : "";
|
|
my $run_directives_hash = defined($candidate->{run_directives_sha256})
|
|
? "$candidate->{run_directives_sha256}" : "";
|
|
my $replaced = $candidate->{replaced} ? 1 : 0;
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest run")
|
|
if $run !~ /^\d+$/ || $run <= 0;
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest action")
|
|
if $action !~ /^(?:fit|retry|refit)$/ || $replaced != ($action eq "fit" ? 0 : 1);
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest target")
|
|
if $target_name !~ /^\Q$run\E(?:[._+\-][A-Za-z0-9._+\-]*)?\.msr$/
|
|
|| $seen_targets{$target_name}++;
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest hash")
|
|
if $promoted_hash !~ /^[a-f0-9]{64}$/
|
|
|| $run_directives_hash !~ /^[a-f0-9]{64}$/
|
|
|| ($replaced && $original_hash !~ /^[a-f0-9]{64}$/);
|
|
my $expected_backup = $replaced
|
|
? series_transaction_file_name($target_name, "backup", $transaction_id) : "";
|
|
return $failure->("invalid-series-transaction-manifest", "Invalid manifest backup")
|
|
if $backup_name ne $expected_backup;
|
|
push(@items, {
|
|
run => 0 + $run,
|
|
action => $action,
|
|
target_name => $target_name,
|
|
target_path => "$work_path/$target_name",
|
|
backup_name => $backup_name,
|
|
backup_path => $replaced ? "$work_path/$backup_name" : "",
|
|
replaced => $replaced,
|
|
promoted_hash => $promoted_hash,
|
|
original_hash => $original_hash,
|
|
run_directives_hash => $run_directives_hash
|
|
});
|
|
}
|
|
|
|
my $lock_path = "$work_path/.musrflow-series.lock";
|
|
return $failure->("series-finalization-lock-failed", "Could not open series transaction lock")
|
|
if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600);
|
|
return $failure->("series-finalization-lock-failed", "Another series transaction is active")
|
|
if !flock($lock_fh, LOCK_EX | LOCK_NB);
|
|
|
|
# Preflight the complete manifest before mutating anything. Missing files
|
|
# are accepted only when they exactly represent an earlier partial retry of
|
|
# the same idempotent finalization decision.
|
|
foreach my $item (@items) {
|
|
my $target_present = -e $item->{target_path} || -l $item->{target_path};
|
|
my $target_exists = -f $item->{target_path} && !-l $item->{target_path};
|
|
my $target_hash = $target_exists
|
|
? sha256_hex(read_text_file($item->{target_path})) : "";
|
|
if ($target_present && !$target_exists) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Series target is not a regular file", [{
|
|
run => $item->{run}, operation => "$decision-preflight",
|
|
path => $item->{target_name}, message => "Regular target required"
|
|
}]);
|
|
}
|
|
if ($decision eq "commit") {
|
|
if (!$target_exists) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Committed series target is missing", [{
|
|
run => $item->{run}, operation => "commit-preflight",
|
|
path => $item->{target_name}, message => "Regular fitted MSR required"
|
|
}]);
|
|
}
|
|
my $target_text = read_text_file($item->{target_path});
|
|
if (!series_msr_is_converged($target_text)) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Committed series target is not converged", [{
|
|
run => $item->{run}, operation => "commit-preflight",
|
|
path => $item->{target_name}, message => "Converged STATISTIC required"
|
|
}]);
|
|
}
|
|
if (series_run_directive_signature($target_text)
|
|
ne $item->{run_directives_hash}) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Committed series target changed run identity", [{
|
|
run => $item->{run}, operation => "commit-preflight",
|
|
path => $item->{target_name}, message => "Original RUN/ADDRUN identity required"
|
|
}]);
|
|
}
|
|
if ($item->{replaced} && -e $item->{backup_path}
|
|
&& (!-f $item->{backup_path} || -l $item->{backup_path}
|
|
|| sha256_hex(read_text_file($item->{backup_path})) ne $item->{original_hash})) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Series backup is invalid", [{
|
|
run => $item->{run}, operation => "commit-preflight",
|
|
path => $item->{backup_name}, message => "Verified backup required"
|
|
}]);
|
|
}
|
|
} elsif ($decision eq "retain") {
|
|
if ($item->{replaced} || !$target_exists) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Only a newly created regular target can be retained", [{
|
|
run => $item->{run}, operation => "retain-preflight",
|
|
path => $item->{target_name}, message => "New regular target required"
|
|
}]);
|
|
}
|
|
} elsif ($item->{replaced}) {
|
|
my $backup_exists = -f $item->{backup_path} && !-l $item->{backup_path};
|
|
if (!$backup_exists && $target_hash ne $item->{original_hash}) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Rollback backup is unavailable", [{
|
|
run => $item->{run}, operation => "rollback-preflight",
|
|
path => $item->{backup_name}, message => "Verified backup required"
|
|
}]);
|
|
}
|
|
if ($backup_exists
|
|
&& sha256_hex(read_text_file($item->{backup_path})) ne $item->{original_hash}) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-preflight-failed",
|
|
"Rollback backup changed", [{
|
|
run => $item->{run}, operation => "rollback-preflight",
|
|
path => $item->{backup_name}, message => "Backup hash mismatch"
|
|
}]);
|
|
}
|
|
}
|
|
}
|
|
|
|
my @ordered = $decision eq "rollback" ? reverse(@items) : @items;
|
|
my @finalized = ();
|
|
foreach my $item (@ordered) {
|
|
my $ok = 1;
|
|
if ($decision eq "commit") {
|
|
$ok = unlink($item->{backup_path}) if $item->{replaced} && -e $item->{backup_path};
|
|
} elsif ($decision eq "retain") {
|
|
# Keep the newly created target exactly where musrfit left it.
|
|
$ok = 1;
|
|
} elsif ($item->{replaced}) {
|
|
$ok = rename($item->{backup_path}, $item->{target_path})
|
|
if -e $item->{backup_path};
|
|
} else {
|
|
$ok = unlink($item->{target_path}) if -e $item->{target_path};
|
|
}
|
|
if (!$ok) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-partial",
|
|
"Series transaction was only partly finalized", [{
|
|
run => $item->{run}, operation => $decision,
|
|
path => $item->{target_name}, message => "Filesystem finalization failed"
|
|
}], \@finalized);
|
|
}
|
|
push(@finalized, {
|
|
run => $item->{run}, action => $item->{action},
|
|
targetPath => "$TMP_URL/$job_name/$item->{target_name}",
|
|
decision => $decision
|
|
});
|
|
}
|
|
if (!unlink($manifest_path)) {
|
|
close($lock_fh);
|
|
return $failure->("series-finalization-partial",
|
|
"Series transaction manifest could not be removed", [{
|
|
run => 0, operation => $decision,
|
|
path => $manifest_name, message => "Could not remove transaction manifest"
|
|
}], \@finalized);
|
|
}
|
|
close($lock_fh);
|
|
return {
|
|
ok => JSON::PP::true,
|
|
code => 0,
|
|
reason => $decision eq "commit"
|
|
? "series-transaction-committed"
|
|
: ($decision eq "rollback"
|
|
? "series-transaction-rolled-back" : "series-transaction-retained"),
|
|
message => $decision eq "commit"
|
|
? "Series transaction committed"
|
|
: ($decision eq "rollback"
|
|
? "Series transaction rolled back" : "Series target retained"),
|
|
finalized => \@finalized,
|
|
errors => []
|
|
};
|
|
}
|
|
|
|
# Global msr2data exports represent a run-qualified parameter such as Asy_2274
|
|
# as Asy_, while ordinary per-run exports retain the first run suffix as
|
|
# Asy_2274, Asy_2274PosErr, and Asy_2274NegErr. Normalize only suffixes that
|
|
# match this collection's run list while preserving embedded underscores.
|
|
sub clean_parameter_data_header {
|
|
my ($path, $run_numbers) = @_;
|
|
my $content = read_text_file($path);
|
|
return 0 if $content eq "";
|
|
my %runs = map { (0 + $_) => 1 }
|
|
grep { defined($_) && /^\d+$/ } @{ref($run_numbers) eq "ARRAY" ? $run_numbers : []};
|
|
|
|
my @lines = split(/\n/, $content, -1);
|
|
foreach my $line (@lines) {
|
|
my $probe = $line;
|
|
$probe =~ s/\r$//;
|
|
next if $probe =~ /^\s*$/ || $probe =~ /^\s*#/;
|
|
$line =~ s{(\S+?)_([0-9]+)((?:_)?(?:PosErr|NegErr))?(?=\s|$)}{
|
|
exists($runs{0 + $2}) ? $1 . (defined($3) ? $3 : "") : $&
|
|
}gex;
|
|
$line =~ s/(\S+)_(?=\s|$)/$1/g;
|
|
last;
|
|
}
|
|
|
|
return write_text_file($path, join("\n", @lines));
|
|
}
|
|
|
|
# Run an arbitrary shell command with an alarm() watchdog, isolating the child
|
|
# in its own process group so a timeout can kill the whole pipeline (for example
|
|
# `cd work && msr2data ...` or a tar archive) without touching this CGI process.
|
|
# Returns a value in system()'s encoding: Waited-status bits for exit/signal, -1
|
|
# when the child could not be started, and -2 when the watchdog fired.
|
|
sub run_bounded_command {
|
|
my ($command) = @_;
|
|
+ ensure_full_data_path();
|
|
my $pid = fork();
|
|
return -1 if !defined($pid);
|
|
if ($pid == 0) {
|
|
setpgrp(0, 0);
|
|
umask(022);
|
|
open(STDIN, "<", "/dev/null") or _exit(127);
|
|
exec { "/bin/sh" } "/bin/sh", "-c", child_resource_prefix($command);
|
|
_exit(127);
|
|
}
|
|
|
|
my $timed_out = 0;
|
|
local $SIG{ALRM} = sub {
|
|
$timed_out = 1;
|
|
kill "KILL", -$pid;
|
|
};
|
|
my $status = -1;
|
|
eval {
|
|
alarm($CMD_TIMEOUT);
|
|
waitpid($pid, 0);
|
|
alarm(0);
|
|
$status = $?;
|
|
};
|
|
my $wait_error = $@;
|
|
alarm(0);
|
|
if ($timed_out) {
|
|
# waitpid may either be interrupted by SIGALRM or return after the killed
|
|
# child is reaped. A second wait is harmless in the latter case and avoids
|
|
# leaving a zombie in the former case.
|
|
waitpid($pid, 0);
|
|
return -2;
|
|
}
|
|
if ($wait_error) {
|
|
kill "KILL", -$pid;
|
|
waitpid($pid, 0);
|
|
return -1;
|
|
}
|
|
return $status;
|
|
}
|
|
|
|
# Bounded variant of a backtick capture used by dump_header. The child runs in
|
|
# its own process group (so the watchdog can kill the whole pipeline) and writes
|
|
# to a pipe the parent reads after the child exits. Returns the captured output,
|
|
# the wait status, and a timed_out flag.
|
|
sub run_backtick_command {
|
|
my ($command) = @_;
|
|
ensure_full_data_path();
|
|
my $pid = open(my $child_stdout, "-|");
|
|
return (undef, -1, 0) if !defined($pid);
|
|
if ($pid == 0) {
|
|
setpgrp(0, 0);
|
|
umask(022);
|
|
open(STDIN, "<", "/dev/null") or _exit(127);
|
|
exec { "/bin/sh" } "/bin/sh", "-c", child_resource_prefix($command);
|
|
_exit(127);
|
|
}
|
|
|
|
my $timed_out = 0;
|
|
local $SIG{ALRM} = sub {
|
|
$timed_out = 1;
|
|
kill "KILL", -$pid;
|
|
};
|
|
my $status = -1;
|
|
local $/;
|
|
my $output = "";
|
|
eval {
|
|
alarm($CMD_TIMEOUT);
|
|
$output = <$child_stdout>;
|
|
$output = "" if !defined($output);
|
|
# Closing the pipe reaps the child and makes its wait status available
|
|
# in $?, so capture it directly instead of a second waitpid (which would
|
|
# fail with ECHILD because the process is already reaped).
|
|
my $closed = close($child_stdout);
|
|
alarm(0);
|
|
$status = $closed ? ($? // 0) : -1;
|
|
};
|
|
if ($@) {
|
|
kill "KILL", -$pid;
|
|
close($child_stdout);
|
|
waitpid($pid, 0);
|
|
$output = "";
|
|
return ("", -2, 1);
|
|
}
|
|
return ($output, $timed_out ? -2 : $status, $timed_out);
|
|
}
|
|
|
|
sub run_logged_command {
|
|
# Keep the downloadable .out file and the in-app command log useful on their
|
|
# own: show the exact shell command first, then its combined output and status.
|
|
# Return this command's output separately so expected conditions such as a
|
|
# missing RUN file can be distinguished from executable failures.
|
|
my ($log_path, $label, $command) = @_;
|
|
if (open(my $log_fh, ">>", $log_path)) {
|
|
print {$log_fh} "\n=== $label ===\n\$ $command\n";
|
|
close($log_fh);
|
|
}
|
|
|
|
my $output_start = -s $log_path || 0;
|
|
my $result = run_bounded_command($command . " >> " . shell_quote($log_path) . " 2>&1");
|
|
my $completion = "";
|
|
my $exit_code = 1;
|
|
if ($result == -2) {
|
|
$completion = "timed out after ${CMD_TIMEOUT}s";
|
|
$exit_code = 124;
|
|
} elsif ($result == -1) {
|
|
$completion = "failed to start: $!";
|
|
$exit_code = 127;
|
|
} elsif ($result & 127) {
|
|
$completion = "terminated by signal " . ($result & 127);
|
|
$exit_code = 128 + ($result & 127);
|
|
} else {
|
|
$exit_code = $result >> 8;
|
|
$completion = "exit code " . $exit_code;
|
|
}
|
|
|
|
my $command_output = "";
|
|
if (open(my $output_fh, "<", $log_path)) {
|
|
seek($output_fh, $output_start, 0);
|
|
local $/;
|
|
$command_output = <$output_fh>;
|
|
$command_output = "" if !defined($command_output);
|
|
close($output_fh);
|
|
}
|
|
|
|
if (open(my $log_fh, ">>", $log_path)) {
|
|
print {$log_fh} "\n[$completion]\n";
|
|
close($log_fh);
|
|
}
|
|
return {
|
|
raw_status => $result,
|
|
exit_code => $exit_code,
|
|
output => $command_output
|
|
};
|
|
}
|
|
|
|
sub classify_missing_data_output {
|
|
# Both musrfit and musrview report unresolved RUN files through the same data
|
|
# handler. Restrict matching to those diagnostics so unrelated missing
|
|
# configuration files remain genuine execution errors.
|
|
my ($output) = @_;
|
|
$output = "" if !defined($output);
|
|
my @missing_files = ();
|
|
my %seen_files = ();
|
|
while ($output =~ /Couldn't find\s+['"]([^'"\r\n]+)['"]\s+in any standard path/gi) {
|
|
my $path = $1;
|
|
if (!$seen_files{$path}) {
|
|
$seen_files{$path} = 1;
|
|
push(@missing_files, $path);
|
|
}
|
|
}
|
|
|
|
my $detected = @missing_files
|
|
|| ($output =~ /PRunDataHandler::FileExistsCheck\(\)/i
|
|
&& $output =~ /Couldn't read all data files/i);
|
|
my $message = "Required data files were not found";
|
|
if (@missing_files == 1) {
|
|
my $file_name = $missing_files[0];
|
|
$file_name =~ s{.*[\\/]}{};
|
|
$message = "Data file not found: " . $file_name;
|
|
} elsif (@missing_files > 1) {
|
|
$message = scalar(@missing_files) . " data files were not found";
|
|
}
|
|
|
|
return {
|
|
detected => $detected ? 1 : 0,
|
|
message => $message,
|
|
missing_files => \@missing_files
|
|
};
|
|
}
|
|
|
|
# Validate and normalize files produced by msr2data before they become runnable.
|
|
sub is_valid_generated_global_msr {
|
|
my ($content) = @_;
|
|
return 0 if !defined($content);
|
|
return 0 if $content !~ /\S/;
|
|
|
|
my @required_blocks = qw(FITPARAMETER THEORY GLOBAL RUN COMMANDS PLOT);
|
|
foreach my $block_name (@required_blocks) {
|
|
return 0 if $content !~ /^\s*\Q$block_name\E\b/m;
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
|
|
sub normalize_plot_runs_line {
|
|
# PLOT uses MSR RUN-block indices, not PSI run numbers. Count the RUN blocks
|
|
# that are actually present in the file because one physical run can expand
|
|
# to several RUN blocks (for example, one per detector in a SingleHist fit).
|
|
# The physical run list is only a fallback for incomplete/legacy files.
|
|
my ($content, $runs) = @_;
|
|
return $content if !defined($content);
|
|
return $content if $content !~ /^PLOT\b/m;
|
|
|
|
my $num_runs = () = $content =~ /^\s*RUN\b/gm;
|
|
if ($num_runs == 0) {
|
|
my @run_numbers = grep { $_ ne "" } split(/\s+/, $runs || "");
|
|
$num_runs = @run_numbers ? scalar(@run_numbers) : 1;
|
|
}
|
|
|
|
my @plot_runs = ();
|
|
for (my $i = 1; $i <= $num_runs; $i++) {
|
|
push(@plot_runs, $i);
|
|
}
|
|
|
|
my $runs_line = "runs " . join(" ", @plot_runs);
|
|
$content =~ s/^(runs\s+).*$/$runs_line/m;
|
|
return $content;
|
|
}
|
|
|
|
# Display-MSR parsing and merging live in muSRFitlib.js and run in the browser.
|
|
|
|
# Collect a completed independent series without running musrfit again. The
|
|
# requested non-empty data table and structurally valid combined MSR are the
|
|
# success artifacts; msr2data's process status is retained only as a warning.
|
|
sub perform_series_artifact_collection {
|
|
my ($work_path, $job_name, $anchor_name, $collection_text) = @_;
|
|
my $failure = sub {
|
|
my ($reason, $message, $warnings) = @_;
|
|
return {
|
|
ok => JSON::PP::false,
|
|
code => 1,
|
|
reason => $reason,
|
|
message => $message,
|
|
warnings => $warnings || [],
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "", out_url => "", parameter_url => "",
|
|
root_url => "", root_urls => [], root_items => [], command_log => ""
|
|
};
|
|
};
|
|
return $failure->("invalid-series-collection", "Invalid series collection request")
|
|
if !defined($collection_text) || length($collection_text) > 5_000_000;
|
|
my $collection = eval { decode_json($collection_text) };
|
|
return $failure->("invalid-series-collection", "Invalid series collection JSON")
|
|
if $@ || ref($collection) ne "HASH";
|
|
my $runs = $collection->{runs};
|
|
my $suffix = defined($collection->{suffix}) ? "$collection->{suffix}" : "";
|
|
my $fftv = defined($collection->{fftv}) ? "$collection->{fftv}" : "n";
|
|
my $contract_version = defined($collection->{version})
|
|
&& !ref($collection->{version}) ? 0 + $collection->{version} : 0;
|
|
my $client_combined = exists($collection->{combinedMSR})
|
|
&& !ref($collection->{combinedMSR}) ? "$collection->{combinedMSR}" : undef;
|
|
return $failure->("invalid-series-collection", "Invalid series collection fields")
|
|
if ref($runs) ne "ARRAY" || @{$runs} < 2 || @{$runs} > 500
|
|
|| $suffix !~ /^_[A-Za-z0-9._+\-]+$/ || $fftv !~ /^(?:y|n)$/
|
|
|| $contract_version != 1 || !defined($client_combined);
|
|
|
|
my @run_numbers = ();
|
|
my %seen = ();
|
|
my %source_texts = ();
|
|
foreach my $value (@{$runs}) {
|
|
my $run = defined($value) && !ref($value) ? "$value" : "";
|
|
return $failure->("invalid-series-collection", "Invalid series run number")
|
|
if $run !~ /^\d+$/ || $run <= 0 || $seen{$run}++;
|
|
my $source_name = $run . $suffix . ".msr";
|
|
my $source_path = "$work_path/$source_name";
|
|
return $failure->("series-artifact-source-invalid",
|
|
"A fitted series MSR is missing or unsafe")
|
|
if !-f $source_path || -l $source_path
|
|
|| !series_msr_is_converged($source_texts{$run} = read_text_file($source_path));
|
|
push(@run_numbers, $run);
|
|
}
|
|
return $failure->("invalid-series-collection", "Invalid series anchor")
|
|
if $anchor_name ne $run_numbers[0] . $suffix . ".msr";
|
|
return $failure->("series-msr2data-unavailable", "msr2data is not executable")
|
|
if !-x $MSR2DATA_BIN;
|
|
|
|
my $base_run = $run_numbers[0];
|
|
# Keep display-only independent-series artifacts separate from genuine
|
|
# global-fit results that may already exist in the same HTTP job folder.
|
|
my $combined_name = $base_run . "+series" . $suffix . ".msr";
|
|
my $combined_path = "$work_path/$combined_name";
|
|
my $parameter_name = $base_run . "+series" . $suffix . "_par.dat";
|
|
my $parameter_path = "$work_path/$parameter_name";
|
|
my $out_name = $base_run . "+series" . $suffix . ".out";
|
|
my $out_path = "$work_path/$out_name";
|
|
my $artifacts = $collection->{artifacts};
|
|
return $failure->("invalid-series-collection", "Invalid series collection contract")
|
|
if ref($artifacts) ne "HASH"
|
|
|| ($artifacts->{combined} || "") ne $combined_name
|
|
|| ($artifacts->{parameters} || "") ne $parameter_name
|
|
|| ($artifacts->{log} || "") ne $out_name
|
|
|| length($client_combined) > 4_000_000;
|
|
|
|
# Collection mutates the job's generated artifacts and starts external
|
|
# programs. Serialize it with fitting/promotion operations for this job.
|
|
my $lock_path = "$work_path/.musrflow-series.lock";
|
|
return $failure->("series-collection-lock-failed", "Could not open series job lock")
|
|
if !sysopen(my $lock_fh, $lock_path, O_RDWR | O_CREAT, 0600);
|
|
if (!flock($lock_fh, LOCK_EX | LOCK_NB)) {
|
|
close($lock_fh);
|
|
return $failure->("series-collection-lock-failed", "Another series operation is active");
|
|
}
|
|
foreach my $stale_path ($combined_path, $parameter_path) {
|
|
return $failure->("series-artifact-output-invalid", "A series output path is unsafe")
|
|
if -l $stale_path || (-e $stale_path && !-f $stale_path);
|
|
return $failure->("series-artifact-output-remove-failed",
|
|
"Could not remove a stale series artifact")
|
|
if -e $stale_path && !unlink($stale_path);
|
|
}
|
|
return $failure->("series-artifact-log-failed", "Could not initialize collection log")
|
|
if -l $out_path || (-e $out_path && !-f $out_path)
|
|
|| !write_text_file($out_path,
|
|
"# muSRFlow independent series collection\n# Working directory: $work_path\n");
|
|
|
|
my @warnings = ();
|
|
my $run_argument = "[" . join(" ", @run_numbers) . "]";
|
|
my @source_texts = map { $source_texts{$_} } @run_numbers;
|
|
|
|
# Collect fitted per-run values directly. This is independent of display
|
|
# merging, so models that cannot yet be represented in one display MSR
|
|
# still produce the table consumed by Plot parameters.
|
|
my $parameter_url = "";
|
|
my $parameter_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MSR2DATA_BIN)
|
|
. " " . shell_quote($run_argument)
|
|
. " " . shell_quote($suffix)
|
|
. " new data -o " . shell_quote($parameter_name);
|
|
my $parameter_result = run_logged_command(
|
|
$out_path, "msr2data independent series collection", $parameter_cmd
|
|
);
|
|
if (-s $parameter_path) {
|
|
if (!clean_parameter_data_header($parameter_path, \@run_numbers)) {
|
|
push(@warnings, "Could not normalize the parameter-table header");
|
|
}
|
|
$parameter_url = "$TMP_URL/$job_name/$parameter_name";
|
|
if ($parameter_result->{raw_status} != 0) {
|
|
push(@warnings, "msr2data returned code " . $parameter_result->{exit_code}
|
|
. " but produced the parameter table");
|
|
}
|
|
} else {
|
|
push(@warnings, "msr2data did not produce parameter data");
|
|
}
|
|
|
|
# The browser carries the display MSR produced by the shared JavaScript
|
|
# merger. Verify that it retains exactly the fitted sources' RUN/ADDRUN
|
|
# identities before publishing or passing it to musrview.
|
|
my $combined_url = "";
|
|
my $combined_text = $client_combined || "";
|
|
if ($combined_text ne "") {
|
|
my @expected_directives = map {
|
|
@{series_run_directives($_)}
|
|
} @source_texts;
|
|
my @actual_directives = @{series_run_directives($combined_text)};
|
|
if (!is_valid_generated_global_msr($combined_text)) {
|
|
push(@warnings, "Submitted combined display MSR failed validation");
|
|
$combined_text = "";
|
|
} elsif (join("\n", @actual_directives) ne join("\n", @expected_directives)) {
|
|
push(@warnings, "Submitted combined display MSR changed RUN identity");
|
|
$combined_text = "";
|
|
}
|
|
}
|
|
if ($combined_text ne "") {
|
|
my $combined_stage = "$work_path/.musrflow-series-display-$$.tmp";
|
|
if (!-e $combined_stage
|
|
&& write_text_file($combined_stage, $combined_text)
|
|
&& rename($combined_stage, $combined_path)) {
|
|
$combined_url = "$TMP_URL/$job_name/$combined_name";
|
|
} else {
|
|
unlink($combined_stage) if -f $combined_stage && !-l $combined_stage;
|
|
push(@warnings, "Could not write the combined display MSR");
|
|
}
|
|
}
|
|
|
|
my @root_urls = ();
|
|
my @root_items = ();
|
|
if ($combined_url ne "") {
|
|
my $plot_base = $combined_name;
|
|
$plot_base =~ s/\.msr$//i;
|
|
if (opendir(my $old_plot_dir, $work_path)) {
|
|
my @old_plots = grep {
|
|
/^\Q$plot_base\E_\d+(?:_F)?\.root$/ && -f "$work_path/$_"
|
|
} readdir($old_plot_dir);
|
|
closedir($old_plot_dir);
|
|
unlink("$work_path/$_") foreach @old_plots;
|
|
}
|
|
if (-x $MUSRVIEW_BIN) {
|
|
my $view_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MUSRVIEW_BIN) . " --timeout 300 --root"
|
|
. ($fftv eq "y" ? " -f" : "")
|
|
. " " . shell_quote($combined_name);
|
|
my $view_result = run_logged_command($out_path, "musrview", $view_cmd);
|
|
push(@warnings, "musrview returned code " . $view_result->{exit_code})
|
|
if $view_result->{raw_status} != 0;
|
|
} else {
|
|
push(@warnings, "musrview is not executable");
|
|
}
|
|
if (opendir(my $plot_dir, $work_path)) {
|
|
my @root_files = sort grep {
|
|
my $matches = $fftv eq "y"
|
|
? /^\Q$plot_base\E_\d+_F\.root$/
|
|
: /^\Q$plot_base\E_\d+\.root$/;
|
|
$matches && -f "$work_path/$_"
|
|
} readdir($plot_dir);
|
|
closedir($plot_dir);
|
|
@root_urls = map { "$TMP_URL/$job_name/$_" } @root_files;
|
|
foreach my $root_file (@root_files) {
|
|
my ($canvas) = $root_file =~ /_(\d+)(?:_F)?\.root$/;
|
|
push(@root_items, "fMainCanvas" . ($canvas || 0) . ";1");
|
|
}
|
|
}
|
|
push(@warnings, "musrview did not produce a ROOT plot") if !@root_urls;
|
|
}
|
|
|
|
# Keep the archive in this job's directory. A first-run-based name in the
|
|
# shared TMPROOT allowed concurrent browser sessions to overwrite each
|
|
# other's download even though their working directories were independent.
|
|
my $package_name = $base_run . "+series" . $suffix . ".tgz";
|
|
my $package_path = "$work_path/$package_name";
|
|
if (-e $package_path || -l $package_path) {
|
|
if (-l $package_path || !unlink($package_path)) {
|
|
push(@warnings, "Could not replace the series results archive");
|
|
$package_path = "";
|
|
}
|
|
}
|
|
if ($package_path ne "") {
|
|
my $package_cmd = "cd " . shell_quote($work_path)
|
|
. " && tar -czf " . shell_quote($package_path) . " *";
|
|
my $package_result = run_logged_command(
|
|
$out_path, "results archive", $package_cmd
|
|
);
|
|
if ($package_result->{raw_status} != 0 || !-s $package_path) {
|
|
my $archive_warning = $package_result->{raw_status} == -2
|
|
? "Series results archive timed out after ${CMD_TIMEOUT}s"
|
|
: "Could not create the series results archive";
|
|
push(@warnings, $archive_warning);
|
|
unlink($package_path) if -f $package_path && !-l $package_path;
|
|
$package_path = "";
|
|
}
|
|
}
|
|
|
|
my $command_log = read_text_file($out_path);
|
|
my $usable = $parameter_url ne "" || $combined_url ne "";
|
|
close($lock_fh);
|
|
return {
|
|
ok => $usable ? JSON::PP::true : JSON::PP::false,
|
|
code => $usable ? 0 : 1,
|
|
reason => $usable ? "series-artifacts-collected" : "series-artifacts-not-produced",
|
|
message => $usable ? "Independent series artifacts collected"
|
|
: "Independent series artifacts were not produced",
|
|
warnings => \@warnings,
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => $combined_url,
|
|
out_url => "$TMP_URL/$job_name/$out_name",
|
|
parameter_url => $parameter_url,
|
|
# The first fitted run's own MSR is the per-run template shown by the
|
|
# Results "Template MSR" entry; the full set is inside the archive.
|
|
template_url => "$TMP_URL/$job_name/$run_numbers[0]$suffix.msr",
|
|
package_url => $package_path ne ""
|
|
? "$TMP_URL/$job_name/$package_name" : "",
|
|
root_url => @root_urls ? $root_urls[0] : "",
|
|
root_urls => \@root_urls,
|
|
root_items => \@root_items,
|
|
command_log => $command_log
|
|
};
|
|
}
|
|
|
|
my $STARTUP_CFG;
|
|
|
|
sub read_startup_config {
|
|
return $STARTUP_CFG if defined($STARTUP_CFG);
|
|
my $cfg = { data_paths => [], templates => {}, file => $STARTUP_XML, ok => 0 };
|
|
if (open(my $fh, "<", $STARTUP_XML)) {
|
|
local $/;
|
|
my $xml = <$fh>;
|
|
close($fh);
|
|
# Strip XML comments so prose inside <!-- ... --> cannot be mistaken for
|
|
# a <data_path> or <run_name_template> element.
|
|
$xml =~ s{<!--.*?-->}{}gs;
|
|
while ($xml =~ m{<data_path>(.*?)</data_path>}sg) {
|
|
my $p = $1;
|
|
$p =~ s/^\s+|\s+$//g;
|
|
next if $p eq "";
|
|
unless ($cfg->{_seen_path}{$p}++) {
|
|
push @{$cfg->{data_paths}}, $p;
|
|
}
|
|
}
|
|
while ($xml =~ m{<run_name_template\s+inst=["']([^"']+)["'][^>]*>(.*?)</run_name_template>}sg) {
|
|
my ($inst, $tmpl) = ($1, $2);
|
|
$tmpl =~ s/^\s+|\s+$//g;
|
|
next if $tmpl eq "";
|
|
push @{$cfg->{templates}{$inst}}, $tmpl;
|
|
}
|
|
$cfg->{ok} = scalar(@{$cfg->{data_paths}}) > 0 || scalar(keys %{$cfg->{templates}}) > 0;
|
|
}
|
|
$STARTUP_CFG = $cfg;
|
|
return $cfg;
|
|
}
|
|
|
|
# Resolve the single MUSRFULLDATAPATH musrfit/musrview uses to find relative
|
|
# RUN data files. An admin-supplied value wins. Otherwise prefer a <data_path>
|
|
# from the startup XML that exists on disk (matching what the portable-RUN
|
|
# resolver accepts) over the shared $DATA_ROOT, whose <instrument>/his level is
|
|
# missing. Memoized so it is computed once per request.
|
|
my $FULL_DATA_PATH_SET = 0;
|
|
sub ensure_full_data_path {
|
|
if (!$FULL_DATA_PATH_SET) {
|
|
if (!$MUSRFULLDATAPATH_EXPLICIT) {
|
|
my $cfg = read_startup_config();
|
|
if ($cfg->{ok}) {
|
|
for my $root (@{$cfg->{data_paths}}) {
|
|
if (-d $root) {
|
|
$ENV{MUSRFULLDATAPATH} = $root;
|
|
last;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
$FULL_DATA_PATH_SET = 1;
|
|
}
|
|
return $ENV{MUSRFULLDATAPATH};
|
|
}
|
|
|
|
sub path_is_under_data_root {
|
|
my ($path) = @_;
|
|
return 0 if !defined($path) || $path eq "" || $path =~ /\s/;
|
|
|
|
# ADDRUN specifications arrive from the browser. Keep them inside the
|
|
# server-owned data root and reject lexical traversal before musrfit opens
|
|
# any referenced data file.
|
|
return 0 if $path =~ m{(?:^|/)\.\.(?:/|$)};
|
|
return $path =~ m{^/} if $DATA_ROOT eq "/";
|
|
return index($path, $DATA_ROOT . "/") == 0;
|
|
}
|
|
|
|
# Returns the canonical absolute path of a series data file, or undef when the
|
|
# path is not a valid, in-scope series data file. Accepts:
|
|
# - a portable path that resolves under a configured <data_path> root from the
|
|
# startup XML (an absolute path there, or a bare sub-path relative to it —
|
|
# the form the browser emits when portable RUN naming is enabled), or
|
|
# - a legacy absolute path under $DATA_ROOT of the form
|
|
# <instrument>/his/<4-digit-year>/<file>.
|
|
# Browser input is never trusted for traversal or symlink resolution.
|
|
sub series_data_file_root {
|
|
my ($path) = @_;
|
|
return undef if !defined($path) || $path eq "" || $path =~ /\s/;
|
|
return undef if $path =~ m{(?:^|/)\.\.(?:/|$)};
|
|
|
|
my $cfg = read_startup_config();
|
|
my @roots = defined($cfg->{ok}) && $cfg->{ok}
|
|
? @{$cfg->{data_paths}} : ();
|
|
|
|
# Legacy absolute layout under $DATA_ROOT.
|
|
if (path_is_under_data_root($path)) {
|
|
my $relative = $DATA_ROOT eq "/"
|
|
? substr($path, 1) : substr($path, length($DATA_ROOT) + 1);
|
|
return $path if $relative =~ m{^[A-Za-z0-9_-]+/his/\d{4}/[A-Za-z0-9._+\-]+$};
|
|
}
|
|
|
|
# Portable layout: $path is either an absolute path under one of the
|
|
# configured <data_path> roots, or a bare relative sub-path that resolves
|
|
# under one of those roots. An absolute path only needs a valid shape (its
|
|
# existence is checked by the caller); a bare relative sub-path is
|
|
# ambiguous across roots, so it is accepted only where the file actually
|
|
# exists, mirroring how the portable-RUN resolver and MUSRFULLDATAPATH pick
|
|
# that same root.
|
|
for my $root (@roots) {
|
|
my $joined;
|
|
if ($path =~ m{^/}) {
|
|
next unless index($path, $root . "/") == 0;
|
|
$joined = $path;
|
|
} else {
|
|
$joined = "$root/$path";
|
|
next unless -f $joined && !-l $joined;
|
|
}
|
|
# A valid data file has a year/…/file (or d<year>/…/file) tail and no
|
|
# further path traversal.
|
|
my $tail = substr($joined, length($root) + 1);
|
|
return $joined if $tail =~ m{^(?:d?\d{4}/)?(?:[A-Za-z0-9._+\-]+/)*[A-Za-z0-9._+\-]+$};
|
|
}
|
|
return undef;
|
|
}
|
|
|
|
# Age out abandoned web job directories under TMPROOT so a long-lived or
|
|
# abandoned browser session cannot exhaust the disk. Only directories whose
|
|
# name matches the job-name rule are candidates, and symlinks are skipped, so
|
|
# this never follows a crafted entry out of the temporary root. Returns the
|
|
# number of directories removed.
|
|
sub prune_stale_jobs {
|
|
return 0 if !-d $TMPROOT;
|
|
my $now = time();
|
|
my $removed = 0;
|
|
opendir(my $dh, $TMPROOT) or return 0;
|
|
while (my $entry = readdir($dh)) {
|
|
next if $entry eq "." || $entry eq "..";
|
|
next if $entry eq $RUN_SLOT_BASENAME;
|
|
next if $entry !~ /^[A-Za-z0-9._-]+$/;
|
|
my $candidate = "$TMPROOT/$entry";
|
|
next if -l $candidate || !-d $candidate;
|
|
my @stats = stat($candidate);
|
|
next if @stats < 9; # race: directory vanished
|
|
next if $stats[9] > $now - $JOB_MAXAGE; # still fresh
|
|
File::Path::remove_tree($candidate, { safe => 1, error => \my $err });
|
|
$removed++ if !@{$err};
|
|
}
|
|
closedir($dh);
|
|
return $removed;
|
|
}
|
|
|
|
# Ensure a requested job directory is a real directory we control, rejecting
|
|
# symlinks or anything that is not a plain writable directory before any file
|
|
# is written into it. Returns (ok, message).
|
|
sub ensure_safe_work_dir {
|
|
my ($work_path) = @_;
|
|
if (-l $work_path || (-e $work_path && !-d $work_path)) {
|
|
return (0, "Job directory is not a plain directory");
|
|
}
|
|
if (!-d $work_path) {
|
|
# Refusing to create a new session when the configured job ceiling is
|
|
# reached bounds disk growth at the source without ever deleting an
|
|
# active job mid-run. Existing sessions (an already-created directory)
|
|
# keep working; only brand-new job folders are refused.
|
|
return (0, "Server job limit reached; retry when an existing session ends")
|
|
if $MAX_JOBS > 0 && count_job_dirs() >= $MAX_JOBS;
|
|
return (0, "Job directory does not exist") if !-d $TMPROOT;
|
|
return (0, "Job directory creation failed")
|
|
if !make_path($work_path, { mode => 0755 });
|
|
}
|
|
# A freshly created directory is mode 0755; an existing job dir should never
|
|
# be world-writable, which would let another system user plant artifacts.
|
|
chmod(0755, $work_path) if -w $work_path;
|
|
return (1, "");
|
|
}
|
|
|
|
sub perform_series_inventory {
|
|
my ($work_path, $job_name, $inventory_text) = @_;
|
|
my $failure = sub {
|
|
my ($reason, $message) = @_;
|
|
return {
|
|
ok => JSON::PP::false,
|
|
code => 400,
|
|
reason => $reason,
|
|
message => $message,
|
|
msr_files => [],
|
|
data_files => []
|
|
};
|
|
};
|
|
|
|
return $failure->("invalid-series-inventory", "Series inventory request is too large")
|
|
if !defined($inventory_text) || length($inventory_text) > 1_000_000;
|
|
my $decoded = eval { decode_json($inventory_text) };
|
|
return $failure->("invalid-series-inventory", "Invalid series inventory JSON")
|
|
if $@ || ref($decoded) ne "HASH";
|
|
my $msr_requests = $decoded->{msr_files};
|
|
my $data_requests = $decoded->{data_files};
|
|
return $failure->("invalid-series-inventory", "Invalid MSR inventory list")
|
|
if ref($msr_requests) ne "ARRAY" || !@{$msr_requests} || @{$msr_requests} > 500;
|
|
return $failure->("invalid-series-inventory", "Invalid data-file inventory list")
|
|
if ref($data_requests) ne "ARRAY" || @{$data_requests} > 2000;
|
|
|
|
my @safe_msr = ();
|
|
my %seen_msr = ();
|
|
foreach my $request (@{$msr_requests}) {
|
|
return $failure->("invalid-series-inventory", "Invalid MSR inventory entry")
|
|
if ref($request) ne "HASH";
|
|
my $run = defined($request->{run}) ? "$request->{run}" : "";
|
|
my $name = defined($request->{name}) ? "$request->{name}" : "";
|
|
return $failure->("invalid-series-inventory", "Invalid MSR inventory run")
|
|
if $run !~ /^\d+$/ || $run <= 0;
|
|
return $failure->("invalid-series-inventory", "Invalid MSR inventory filename")
|
|
if $name !~ /^\Q$run\E(?:[._+\-][A-Za-z0-9._+\-]*)?\.msr$/;
|
|
return $failure->("invalid-series-inventory", "Duplicate MSR inventory filename")
|
|
if $seen_msr{$name}++;
|
|
push(@safe_msr, { run => 0 + $run, name => $name });
|
|
}
|
|
|
|
my @safe_data = ();
|
|
my %seen_data = ();
|
|
foreach my $request (@{$data_requests}) {
|
|
return $failure->("invalid-series-inventory", "Invalid data-file inventory entry")
|
|
if ref($request) ne "HASH";
|
|
my $run = defined($request->{run}) ? "$request->{run}" : "";
|
|
my $path = defined($request->{path}) ? "$request->{path}" : "";
|
|
return $failure->("invalid-series-inventory", "Invalid data-file inventory run")
|
|
if $run !~ /^\d+$/ || $run <= 0;
|
|
# Resolve to the canonical absolute path (legacy $DATA_ROOT layout or a
|
|
# startup-XML <data_path> root for the portable layout); undef means the
|
|
# path is outside the server's data scope.
|
|
my $resolved = series_data_file_root($path);
|
|
return $failure->("invalid-series-inventory", "Invalid series data-file path")
|
|
if !defined($resolved);
|
|
return $failure->("invalid-series-inventory", "Duplicate data-file inventory path")
|
|
if $seen_data{$resolved}++;
|
|
push(@safe_data, { run => 0 + $run, path => $resolved });
|
|
}
|
|
|
|
my @msr_results = ();
|
|
foreach my $request (@safe_msr) {
|
|
my $path = "$work_path/$request->{name}";
|
|
# Never follow a symlink from a browser-owned job folder. An oversized
|
|
# or unreadable existing file remains `exists=true` but `readable=false`
|
|
# so the shared classifier blocks rather than treating it as missing.
|
|
my $exists = -f $path && !-l $path ? 1 : 0;
|
|
my $size = $exists ? (-s $path || 0) : 0;
|
|
my $readable = $exists && $size <= 2_000_000 && -r $path ? 1 : 0;
|
|
my $text = $readable ? read_text_file($path) : "";
|
|
push(@msr_results, {
|
|
run => $request->{run},
|
|
name => $request->{name},
|
|
path => "$TMP_URL/$job_name/$request->{name}",
|
|
exists => $exists ? JSON::PP::true : JSON::PP::false,
|
|
readable => $readable ? JSON::PP::true : JSON::PP::false,
|
|
size => 0 + $size,
|
|
text => $text
|
|
});
|
|
}
|
|
|
|
my @data_results = map {
|
|
my $exists = -f $_->{path} && !-l $_->{path} ? 1 : 0;
|
|
{
|
|
run => $_->{run},
|
|
path => $_->{path},
|
|
exists => $exists ? JSON::PP::true : JSON::PP::false
|
|
}
|
|
} @safe_data;
|
|
|
|
return {
|
|
ok => JSON::PP::true,
|
|
code => 0,
|
|
reason => "series-inventory-ready",
|
|
message => "Series inventory inspected",
|
|
job_exists => -d $work_path ? JSON::PP::true : JSON::PP::false,
|
|
msr_files => \@msr_results,
|
|
data_files => \@data_results
|
|
};
|
|
}
|
|
|
|
# Validate browser-provided ADDRUN and parameter synchronization specifications.
|
|
sub parse_addrun_spec {
|
|
my ($text) = @_;
|
|
return ([], "") if !defined($text) || $text eq "";
|
|
return (undef, "ADDRUN specification is too large") if length($text) > 100000;
|
|
|
|
my $decoded = eval { decode_json($text) };
|
|
return (undef, "Invalid ADDRUN specification") if $@ || ref($decoded) ne "ARRAY";
|
|
return (undef, "Too many ADDRUN data sets") if scalar(@{$decoded}) > 500;
|
|
|
|
my @validated = ();
|
|
my %seen_main = ();
|
|
foreach my $entry (@{$decoded}) {
|
|
return (undef, "Invalid ADDRUN specification entry") if ref($entry) ne "HASH";
|
|
my $main = defined($entry->{main}) ? "$entry->{main}" : "";
|
|
my $main_file = $entry->{mainFile} || "";
|
|
my $add_lines = $entry->{addLines};
|
|
return (undef, "Invalid ADDRUN main run") if $main !~ /^\d+$/;
|
|
return (undef, "Invalid ADDRUN main file")
|
|
if !path_is_under_data_root($main_file);
|
|
return (undef, "Duplicate ADDRUN main file") if $seen_main{$main_file}++;
|
|
return (undef, "Invalid ADDRUN line list") if ref($add_lines) ne "ARRAY" || scalar(@{$add_lines}) > 1000;
|
|
|
|
my @safe_lines = ();
|
|
foreach my $line (@{$add_lines}) {
|
|
return (undef, "Invalid ADDRUN directive") if !defined($line) || ref($line);
|
|
return (undef, "Invalid ADDRUN directive")
|
|
if $line !~ /^ADDRUN\s+(\S+)\s+([A-Za-z0-9._-]+)\s+PSI\s+([A-Za-z0-9._-]+)$/;
|
|
my $add_file = $1;
|
|
return (undef, "ADDRUN path is outside the configured data root")
|
|
if !path_is_under_data_root($add_file);
|
|
push(@safe_lines, $line);
|
|
}
|
|
push(@validated, { main => $main, mainFile => $main_file, addLines => \@safe_lines });
|
|
}
|
|
|
|
return (\@validated, "");
|
|
}
|
|
|
|
sub parse_fit_parameter_spec {
|
|
my ($text) = @_;
|
|
return ({}, "") if !defined($text) || $text eq "";
|
|
return (undef, "Fit-parameter specification is too large") if length($text) > 200000;
|
|
my $decoded = eval { decode_json($text) };
|
|
return (undef, "Invalid fit-parameter specification") if $@ || ref($decoded) ne "HASH";
|
|
my $parameters = $decoded->{parameters};
|
|
return (undef, "Invalid fit-parameter list") if ref($parameters) ne "ARRAY" || @{$parameters} > 2000;
|
|
|
|
my %safe = ();
|
|
foreach my $entry (@{$parameters}) {
|
|
return (undef, "Invalid fit-parameter entry") if ref($entry) ne "HASH";
|
|
my $name = defined($entry->{name}) ? "$entry->{name}" : "";
|
|
return (undef, "Invalid fit-parameter name") if $name !~ /^[A-Za-z_][A-Za-z0-9_.+-]*$/;
|
|
my %copy = ( name => $name );
|
|
foreach my $field (qw(value step min max)) {
|
|
my $value = defined($entry->{$field}) ? "$entry->{$field}" : "";
|
|
return (undef, "Invalid fit-parameter $field") if length($value) > 80 || $value =~ /[\r\n]/;
|
|
$copy{$field} = $value;
|
|
}
|
|
$safe{$name} = \%copy;
|
|
}
|
|
return (\%safe, "");
|
|
}
|
|
|
|
sub fit_parameter_line {
|
|
my ($order, $name, $entry) = @_;
|
|
my $line = " $order $name $entry->{value} $entry->{step}";
|
|
my $min = defined($entry->{min}) ? $entry->{min} : "";
|
|
my $max = defined($entry->{max}) ? $entry->{max} : "";
|
|
if ($min ne "" || $max ne "") {
|
|
# Bounded FITPARAMETER lines require the PosError slot between Step
|
|
# and the limits. It is not known before fitting, so write "none".
|
|
$line .= " none"
|
|
. " " . ($min eq "" ? "none" : $min)
|
|
. " " . ($max eq "" ? "none" : $max);
|
|
}
|
|
return $line;
|
|
}
|
|
|
|
# Apply the legacy synchronization rule: replace a FITPARAMETER line only when
|
|
# its exact parameter name exists in the GUI-provided parameter map.
|
|
sub apply_fit_parameters_to_msr {
|
|
my ($content, $parameters) = @_;
|
|
return $content if !defined($content) || ref($parameters) ne "HASH" || !keys(%{$parameters});
|
|
my @lines = split(/\r?\n/, $content, -1);
|
|
my $in_fit = 0;
|
|
for (my $i = 0; $i < @lines; $i++) {
|
|
my $trimmed = $lines[$i];
|
|
$trimmed =~ s/^\s+|\s+$//g;
|
|
if ($trimmed =~ /^FITPARAMETER\b/i) {
|
|
$in_fit = 1;
|
|
next;
|
|
}
|
|
next if !$in_fit;
|
|
last if $trimmed =~ /^(THEORY|FUNCTIONS|GLOBAL|RUN|COMMANDS|PLOT|FOURIER|STATISTIC)\b/i;
|
|
next if $trimmed eq "" || $trimmed =~ /^#/;
|
|
my @parts = split(/\s+/, $trimmed);
|
|
next if @parts < 4 || $parts[0] !~ /^\d+$/;
|
|
my $entry = $parameters->{$parts[1]};
|
|
next if !defined($entry);
|
|
$lines[$i] = fit_parameter_line($parts[0], $parts[1], $entry);
|
|
}
|
|
return join("\n", @lines);
|
|
}
|
|
|
|
sub apply_add_runs_to_msr {
|
|
my ($content, $spec) = @_;
|
|
return ($content, []) if !defined($content) || ref($spec) ne "ARRAY" || !@{$spec};
|
|
|
|
my %by_main = map { $_->{mainFile} => $_->{addLines} } @{$spec};
|
|
my %matched = ();
|
|
my @lines = split(/\r?\n/, $content, -1);
|
|
my @output = ();
|
|
my $replace_existing = 0;
|
|
|
|
foreach my $line (@lines) {
|
|
if ($line =~ /^\s*RUN\s+(\S+)/) {
|
|
my $main_file = $1;
|
|
push(@output, $line);
|
|
if (exists($by_main{$main_file})) {
|
|
push(@output, @{$by_main{$main_file}});
|
|
$matched{$main_file} = 1;
|
|
$replace_existing = 1;
|
|
} else {
|
|
$replace_existing = 0;
|
|
}
|
|
next;
|
|
}
|
|
|
|
# Make the operation idempotent: if a generated file is decorated twice,
|
|
# replace the old ADDRUNs following a matched RUN rather than duplicating them.
|
|
if ($replace_existing && $line =~ /^\s*ADDRUN\b/) {
|
|
next;
|
|
}
|
|
$replace_existing = 0;
|
|
push(@output, $line);
|
|
}
|
|
|
|
my @missing = grep { !$matched{$_->{mainFile}} } @{$spec};
|
|
return (join("\n", @output), \@missing);
|
|
}
|
|
|
|
if ($action eq "health") {
|
|
my $musrfit_status = executable_status($MUSRFIT_BIN);
|
|
my $musrview_status = executable_status($MUSRVIEW_BIN);
|
|
my $msr2data_status = executable_status($MSR2DATA_BIN);
|
|
my $dump_header_status = executable_status($DUMP_HEADER_BIN);
|
|
my $tmp_exists = -d $TMPROOT ? 1 : 0;
|
|
my $tmp_writable = $tmp_exists && -w $TMPROOT ? 1 : 0;
|
|
my $webroot_exists = -d $WEBROOT ? 1 : 0;
|
|
ensure_full_data_path();
|
|
my $ready = $musrfit_status->{runnable}
|
|
&& $musrview_status->{runnable}
|
|
&& $msr2data_status->{runnable}
|
|
&& $tmp_writable;
|
|
|
|
respond_json({
|
|
ok => $ready ? JSON::PP::true : JSON::PP::false,
|
|
code => $ready ? 0 : 1,
|
|
message => $ready ? "Web runner is ready" : "Web runner configuration is incomplete",
|
|
executables => {
|
|
musrfit => $musrfit_status,
|
|
musrview => $musrview_status,
|
|
msr2data => $msr2data_status,
|
|
dump_header => $dump_header_status
|
|
},
|
|
storage => {
|
|
web_root => {
|
|
path => $WEBROOT,
|
|
exists => $webroot_exists ? JSON::PP::true : JSON::PP::false
|
|
},
|
|
tmp_root => {
|
|
path => $TMPROOT,
|
|
url => $TMP_URL,
|
|
exists => $tmp_exists ? JSON::PP::true : JSON::PP::false,
|
|
writable => $tmp_writable ? JSON::PP::true : JSON::PP::false,
|
|
job_maxage_seconds => $JOB_MAXAGE,
|
|
command_timeout_seconds => $CMD_TIMEOUT
|
|
}
|
|
},
|
|
environment => {
|
|
data_root => $DATA_ROOT,
|
|
startup_xml => {
|
|
file => $STARTUP_CFG ? $STARTUP_CFG->{file} : $STARTUP_XML,
|
|
ok => read_startup_config()->{ok} ? JSON::PP::true : JSON::PP::false,
|
|
data_paths => read_startup_config()->{data_paths},
|
|
templates => { map { $_ => read_startup_config()->{templates}{$_} } keys %{ read_startup_config()->{templates} } }
|
|
},
|
|
musr_full_data_path => $ENV{MUSRFULLDATAPATH} || "",
|
|
musrfit_path => $ENV{MUSRFITPATH} || "",
|
|
root_sys => $ENV{ROOTSYS} || "",
|
|
home => $ENV{HOME} || "",
|
|
ld_library_path => $ENV{LD_LIBRARY_PATH} || ""
|
|
}
|
|
});
|
|
}
|
|
|
|
# Confirm that a candidate portable RUN file exists. This is the authoritative,
|
|
# server-backed existence check backing the browser's portable-naming resolver.
|
|
# It is tightly scoped: the data path must be one the admin configured in the
|
|
# startup XML, and the sub-path must be a plain relative path (no traversal).
|
|
# It never acts as a general server file-existence API.
|
|
if ($action eq "resolve_run_name") {
|
|
my $sub_path = $in->param("sub_path") || "";
|
|
my $data_path = $in->param("data_path") || "";
|
|
my $cfg = read_startup_config();
|
|
my @roots = @{$cfg->{data_paths}};
|
|
@roots = ($DATA_ROOT) if !@roots;
|
|
|
|
my $root_ok = 0;
|
|
for my $root (@roots) {
|
|
if ($data_path eq $root) { $root_ok = 1; last; }
|
|
}
|
|
if (!$root_ok) {
|
|
respond_json({ ok => JSON::PP::false, code => 403, message => "Data path not configured" });
|
|
}
|
|
if ($sub_path eq "" || $sub_path =~ /^\// || $sub_path =~ m{(?:^|/)\.\.(?:/|$)} || $sub_path =~ /\s/) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid sub path" });
|
|
}
|
|
my $full = "$data_path/$sub_path";
|
|
if (!-f $full) {
|
|
respond_json({ ok => JSON::PP::false, code => 404, message => "Portable RUN file not found" });
|
|
}
|
|
respond_json({
|
|
ok => JSON::PP::true,
|
|
code => 0,
|
|
sub_path => $sub_path,
|
|
data_path => $data_path,
|
|
run_name => $sub_path
|
|
});
|
|
}
|
|
|
|
if ($action eq "dump_header") {
|
|
my $data_file = $fn;
|
|
my $file_format = $in->param("file_format") || "";
|
|
if ($file_format !~ /^(?:MusrRoot|NeXus|ROOT|PSI-BIN|PSI-MDU|MUD|WKM)$/) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid data-file format" });
|
|
}
|
|
if (!path_is_under_data_root($data_file)) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid data-file path" });
|
|
}
|
|
if (!-f $data_file) {
|
|
respond_json({ ok => JSON::PP::false, code => 404, message => "Data file not found" });
|
|
}
|
|
if (!-x $DUMP_HEADER_BIN) {
|
|
respond_json({ ok => JSON::PP::false, code => 500, message => "dump_header is not executable" });
|
|
}
|
|
|
|
my $command = shell_quote($DUMP_HEADER_BIN)
|
|
. " -fn " . shell_quote($data_file)
|
|
. " -ff " . shell_quote($file_format) . " 2>&1";
|
|
my ($header, $dump_status, $dump_timed_out) = run_backtick_command($command);
|
|
my $exit_code = $dump_timed_out ? 124 : ($dump_status == -1 ? 127 : $dump_status >> 8);
|
|
respond_json({
|
|
ok => $exit_code == 0 ? JSON::PP::true : JSON::PP::false,
|
|
code => $exit_code,
|
|
message => $dump_timed_out ? "dump_header timed out" : ($exit_code == 0 ? "Header read" : "dump_header failed"),
|
|
header => $header
|
|
});
|
|
}
|
|
|
|
my $tmp_url_pattern = quotemeta($TMP_URL);
|
|
if ($fn !~ m{^$tmp_url_pattern/([A-Za-z0-9._-]+)/([A-Za-z0-9._+\-]+\.msr)$}) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid fn path" });
|
|
}
|
|
|
|
my $job_name = $1;
|
|
my $file_name = $2;
|
|
my $work_path = "$TMPROOT/$job_name";
|
|
my $full_path = "$work_path/$file_name";
|
|
|
|
if ($action eq "series_run_candidate") {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 404,
|
|
reason => "series-transaction-manifest-not-found",
|
|
message => "Series web job not found", outcome => "", missing_files => []
|
|
}) if !-d $work_path;
|
|
my @run_slots = acquire_run_admission();
|
|
if (!@run_slots) {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 503, reason => "server-busy",
|
|
message => "muSRFlow is busy running other fits. Please retry in a moment.",
|
|
outcome => "", missing_files => []
|
|
});
|
|
}
|
|
my $transaction_id = $in->param("transaction_id") || "";
|
|
respond_json(perform_series_candidate_run(
|
|
$work_path, $job_name, $file_name, $transaction_id
|
|
));
|
|
}
|
|
|
|
if ($action eq "series_collect") {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 404,
|
|
reason => "series-web-job-not-found",
|
|
message => "Series web job not found", warnings => []
|
|
}) if !-d $work_path;
|
|
my $collection_text = $in->param("collection") || "";
|
|
respond_json(perform_series_artifact_collection(
|
|
$work_path, $job_name, $file_name, $collection_text
|
|
));
|
|
}
|
|
|
|
if ($action eq "series_transaction_finalize") {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 404,
|
|
reason => "series-transaction-manifest-not-found",
|
|
message => "Series web job not found", finalized => [], errors => []
|
|
}) if !-d $work_path;
|
|
my $finalization_text = $in->param("finalization") || "";
|
|
respond_json(perform_series_transaction_finalization(
|
|
$work_path, $job_name, $finalization_text
|
|
));
|
|
}
|
|
|
|
if ($action eq "series_inventory") {
|
|
my $inventory_text = $in->param("inventory") || "";
|
|
# This branch intentionally precedes make_path(): inspecting a selection
|
|
# must not create a web job directory or any other filesystem object.
|
|
respond_json(perform_series_inventory(
|
|
$work_path, $job_name, $inventory_text
|
|
));
|
|
}
|
|
|
|
# Opportunistically age out abandoned job directories before acting on a new
|
|
# request. Bounded, symlink-safe, and cheap; keeps the temp root from growing
|
|
# without bound across long-running browser sessions.
|
|
prune_stale_jobs();
|
|
|
|
my ($work_ok, $work_message) = ensure_safe_work_dir($work_path);
|
|
if (!$work_ok) {
|
|
respond_json({ ok => JSON::PP::false, code => 500, message => $work_message });
|
|
}
|
|
|
|
if ($action eq "series_transaction") {
|
|
my $transaction_text = $in->param("transaction") || "";
|
|
respond_json(perform_series_transaction(
|
|
$work_path, $job_name, $transaction_text
|
|
));
|
|
}
|
|
|
|
if ($action eq "write") {
|
|
my @line_keys = grep { /^line\d+$/ } $in->param;
|
|
@line_keys = sort {
|
|
my ($ai) = $a =~ /^line(\d+)$/;
|
|
my ($bi) = $b =~ /^line(\d+)$/;
|
|
$ai <=> $bi;
|
|
} @line_keys;
|
|
|
|
my $content = "";
|
|
foreach my $key (@line_keys) {
|
|
my $line = $in->param($key);
|
|
$line = "" if !defined($line);
|
|
$content .= $line . "\n";
|
|
}
|
|
|
|
if (length($content) > $MAX_MSR_BYTES) {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 413, reason => "msr-too-large",
|
|
message => "MSR content exceeds the configured limit of $MAX_MSR_BYTES bytes"
|
|
});
|
|
}
|
|
|
|
open(my $outf, ">", $full_path)
|
|
or respond_json({ ok => JSON::PP::false, code => 500, message => "Failed to write file" });
|
|
print {$outf} $content;
|
|
close($outf);
|
|
|
|
respond_json({
|
|
ok => JSON::PP::true,
|
|
code => 0,
|
|
message => "File written",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$file_name"
|
|
});
|
|
}
|
|
|
|
if ($action eq "run") {
|
|
my @run_slots = acquire_run_admission();
|
|
if (!@run_slots) {
|
|
respond_json({
|
|
ok => JSON::PP::false, code => 503, reason => "server-busy",
|
|
message => "muSRFlow is busy running other fits. Please retry in a moment.",
|
|
job_name => $job_name
|
|
});
|
|
}
|
|
my $use_m2d = ($in->param("use_msr2data") || "0") eq "1" ? 1 : 0;
|
|
my $runs = $in->param("runs") || "";
|
|
my $template_name = $in->param("template_name") || "";
|
|
my $base_run = $in->param("base_run") || "";
|
|
my $addrun_spec_text = $in->param("addrun_spec") || "";
|
|
my ($addrun_spec, $addrun_spec_error) = parse_addrun_spec($addrun_spec_text);
|
|
if (!defined($addrun_spec)) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => $addrun_spec_error });
|
|
}
|
|
my $fit_parameter_spec_text = $in->param("fit_parameter_spec") || "";
|
|
my ($fit_parameters, $fit_parameter_error) = parse_fit_parameter_spec($fit_parameter_spec_text);
|
|
if (!defined($fit_parameters)) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => $fit_parameter_error });
|
|
}
|
|
my $fftv = $in->param("fftv") || "n";
|
|
my $plot_only = ($in->param("plot_only") || "0") eq "1" ? 1 : 0;
|
|
my $plot_refresh = ($in->param("plot_refresh") || "0") eq "1" ? 1 : 0;
|
|
|
|
if ($runs ne "" && $runs !~ /^[0-9 ]+$/) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid runs list" });
|
|
}
|
|
if ($base_run ne "" && $base_run !~ /^\d+$/) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid base_run" });
|
|
}
|
|
if ($template_name ne "" && $template_name !~ /^[A-Za-z0-9._+\-]+\.msr$/) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid template_name" });
|
|
}
|
|
if ($fftv ne "y" && $fftv ne "n") {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Invalid fftv value" });
|
|
}
|
|
if ($plot_refresh && !$plot_only) {
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "plot_refresh requires plot_only" });
|
|
}
|
|
if (!-x $MUSRFIT_BIN) {
|
|
respond_json({
|
|
ok => JSON::PP::false,
|
|
code => 500,
|
|
message => "muSRFit is not executable",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$file_name",
|
|
out_url => "$TMP_URL/$job_name/" . ($file_name =~ s/\.msr$/.out/r)
|
|
});
|
|
}
|
|
if ($use_m2d && !-x $MSR2DATA_BIN) {
|
|
respond_json({
|
|
ok => JSON::PP::false,
|
|
code => 500,
|
|
message => "msr2data is not executable",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$file_name",
|
|
out_url => "$TMP_URL/$job_name/" . ($file_name =~ s/\.msr$/.out/r)
|
|
});
|
|
}
|
|
my $target_file = $file_name;
|
|
my $status_code = 0;
|
|
my $stderr = "";
|
|
my $outcome = "";
|
|
my @missing_files = ();
|
|
my $message = $plot_refresh && $fftv eq "y"
|
|
? "FFT plot completed" : ($plot_only ? "Plot completed" : "Fit completed");
|
|
my $out_file = $target_file;
|
|
$out_file =~ s/\.msr$/.out/;
|
|
my $abort_after_m2d = 0;
|
|
my $fit_succeeded = 0;
|
|
my $plot_base = $target_file;
|
|
$plot_base =~ s/\.msr$//;
|
|
# The web UI uses JSROOT as its plot viewer, so the CGI always asks
|
|
# musrview for batch ROOT output rather than generating PNG previews.
|
|
my $plot_extension = "root";
|
|
my $plot_kind = "ROOT";
|
|
my $parameter_file = $plot_base . "_par.dat";
|
|
my $parameter_path = "$work_path/$parameter_file";
|
|
my $out_path = "$work_path/$out_file";
|
|
my $log_start = 0;
|
|
# msr2data always derives its output name from the `_tmpl` input suffix. The
|
|
# browser-provided target name carries the user prefix and becomes runnable.
|
|
my $msr2data_global_name = $base_run ne "" ? $base_run . "+global_tmpl.msr" : "";
|
|
my $msr2data_global_path = $msr2data_global_name ne ""
|
|
? "$work_path/$msr2data_global_name" : "";
|
|
|
|
# A normal Fit/Plot starts a fresh command log. A transform requested by an
|
|
# existing plot window appends to that job so the original fit remains traceable.
|
|
if ($plot_refresh) {
|
|
if (!-e $out_path && !write_text_file($out_path, "")) {
|
|
respond_json({ ok => JSON::PP::false, code => 500, message => "Failed to initialize job output" });
|
|
}
|
|
$log_start = -s $out_path || 0;
|
|
} elsif (!write_text_file($out_path, "")) {
|
|
respond_json({
|
|
ok => JSON::PP::false,
|
|
code => 500,
|
|
message => "Failed to initialize job output",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_file",
|
|
out_url => "$TMP_URL/$job_name/$out_file"
|
|
});
|
|
}
|
|
if (!$plot_refresh && -e $parameter_path && !unlink($parameter_path)) {
|
|
respond_json({
|
|
ok => JSON::PP::false,
|
|
code => 500,
|
|
message => "Failed to remove stale parameter data",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_file",
|
|
out_url => "$TMP_URL/$job_name/$out_file"
|
|
});
|
|
}
|
|
if (opendir(my $old_plot_dir, $work_path)) {
|
|
my @old_plot_files = grep {
|
|
my $matches_output = $plot_refresh
|
|
? ($fftv eq "y"
|
|
? /^\Q$plot_base\E_\d+_F\.root$/
|
|
: /^\Q$plot_base\E_\d+\.root$/)
|
|
: /^\Q$plot_base\E_\d+(?:_F)?\.root$/;
|
|
$matches_output && -f "$work_path/$_"
|
|
} readdir($old_plot_dir);
|
|
closedir($old_plot_dir);
|
|
foreach my $old_plot_file (@old_plot_files) {
|
|
unlink("$work_path/$old_plot_file");
|
|
}
|
|
}
|
|
if (!$plot_refresh && $msr2data_global_path ne "" && -e $msr2data_global_path
|
|
&& !unlink($msr2data_global_path)) {
|
|
respond_json({
|
|
ok => JSON::PP::false,
|
|
code => 500,
|
|
message => "Failed to remove stale global MSR",
|
|
stderr => "Could not remove $msr2data_global_name\n",
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_file",
|
|
out_url => "$TMP_URL/$job_name/$out_file",
|
|
tgz_url => "$TMP_URL/$job_name.tgz"
|
|
});
|
|
}
|
|
|
|
if ($use_m2d && $runs ne "" && $template_name ne "" && $base_run ne "") {
|
|
my $runs_quoted = "[$runs]";
|
|
my $m2d_mode = "global";
|
|
my $m2d_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MSR2DATA_BIN)
|
|
. " " . shell_quote($runs_quoted)
|
|
. " _tmpl msr-" . shell_quote($base_run) . " " . $m2d_mode;
|
|
my $m2d_result = run_logged_command(
|
|
$out_path, "msr2data global generation", $m2d_cmd
|
|
);
|
|
my $global_content = (!$abort_after_m2d && -f $msr2data_global_path)
|
|
? read_text_file($msr2data_global_path) : "";
|
|
if (!$abort_after_m2d && is_valid_generated_global_msr($global_content)) {
|
|
# Normalize here, before musrview. Browser-side fixes are too late
|
|
# for Plot because Plot skips musrfit and opens the file directly.
|
|
my $normalized_global = normalize_plot_runs_line($global_content, $runs);
|
|
# msr2data intentionally receives a template without ADDRUN lines. Its
|
|
# documented behavior is to copy ADDRUN unchanged, which would attach
|
|
# the first data set's additions to every generated RUN block. Decorate
|
|
# the expanded global file here, before either musrfit or musrview.
|
|
my ($decorated_global, $missing_addrun_targets) = apply_add_runs_to_msr(
|
|
$normalized_global, $addrun_spec
|
|
);
|
|
if (@{$missing_addrun_targets}) {
|
|
$stderr .= "Could not match all ADDRUN data sets in msr2data output\n";
|
|
$status_code = 1;
|
|
$message = "Global MSR ADDRUN generation failed";
|
|
$abort_after_m2d = 1;
|
|
}
|
|
if (!$abort_after_m2d) {
|
|
my $target_written = write_text_file($full_path, $decorated_global);
|
|
if ($target_written) {
|
|
if ($msr2data_global_path ne $full_path && -e $msr2data_global_path
|
|
&& !unlink($msr2data_global_path)) {
|
|
$stderr .= "Could not remove temporary $msr2data_global_name\n";
|
|
}
|
|
if ($m2d_result->{raw_status} != 0) {
|
|
my $m2d_exit = $m2d_result->{exit_code};
|
|
$stderr .= "msr2data returned code $m2d_exit but produced a valid global MSR\n";
|
|
}
|
|
} else {
|
|
$stderr .= "Failed to write normalized global MSR before plotting\n";
|
|
$status_code = 1;
|
|
$message = "Global MSR generation failed";
|
|
$abort_after_m2d = 1;
|
|
}
|
|
}
|
|
} elsif (!$abort_after_m2d) {
|
|
$stderr .= "msr2data did not produce a valid global MSR\n";
|
|
$status_code = 1;
|
|
$message = "Global MSR generation failed";
|
|
$abort_after_m2d = 1;
|
|
}
|
|
}
|
|
|
|
# Proven legacy behavior: the current GUI parameter table wins immediately
|
|
# before execution. Walk the MSR's FITPARAMETER lines and replace only exact
|
|
# name matches; extra GUI parameters are deliberately ignored.
|
|
if (!$abort_after_m2d && keys(%{$fit_parameters})) {
|
|
my $prepared_content = read_text_file($full_path);
|
|
my $synchronized_content = apply_fit_parameters_to_msr($prepared_content, $fit_parameters);
|
|
if (!write_text_file($full_path, $synchronized_content)) {
|
|
$stderr .= "Failed to write synchronized fit parameters\n";
|
|
$status_code = 1;
|
|
$message = "Fit-parameter synchronization failed";
|
|
$abort_after_m2d = 1;
|
|
}
|
|
}
|
|
|
|
if (!$abort_after_m2d && !$plot_only) {
|
|
# Match the original CGI invocation. The generated placeholder title is
|
|
# replaced from the first data file, N0 is estimated for histogram fits,
|
|
# and the timeout prevents a web fit from running indefinitely. When the
|
|
# MSR carries an explicit editor title, -t is omitted so musrfit does not
|
|
# silently replace it (mirrors runMSRFileLocal in muSRFitelec.js).
|
|
my $fit_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MUSRFIT_BIN)
|
|
. (has_explicit_msr_title($full_path) ? " -e" : " -t -e")
|
|
. " --timeout 300 " . shell_quote($target_file);
|
|
my $fit_result = run_logged_command($out_path, "musrfit", $fit_cmd);
|
|
my $missing_data = classify_missing_data_output($fit_result->{output});
|
|
if ($missing_data->{detected}) {
|
|
# A fit cannot proceed without its RUN data, and launching musrview
|
|
# would only repeat the same diagnostic. Preserve the MSR and log.
|
|
$status_code = 1;
|
|
$outcome = "missing-data";
|
|
@missing_files = @{$missing_data->{missing_files}};
|
|
$message = $missing_data->{message};
|
|
$abort_after_m2d = 1;
|
|
} elsif ($fit_result->{raw_status} != 0) {
|
|
$status_code = 1;
|
|
$stderr .= "musrfit failed\n";
|
|
$message = "Fit failed";
|
|
} else {
|
|
$fit_succeeded = 1;
|
|
}
|
|
}
|
|
|
|
if ($fit_succeeded) {
|
|
my @run_numbers = grep { $_ ne "" } split(/\s+/, $runs);
|
|
my $parameter_base_run = $base_run ne "" ? $base_run
|
|
: (@run_numbers ? $run_numbers[0] : "");
|
|
|
|
if (@run_numbers && $parameter_base_run ne "") {
|
|
my $parameter_runs = @run_numbers > 1
|
|
? "[" . join(" ", @run_numbers) . "]"
|
|
: $run_numbers[0];
|
|
|
|
# msr2data identifies its input by run number and suffix. Stage the
|
|
# fitted result under the legacy `_tmpl` name, then restore the
|
|
# user's real template afterwards.
|
|
my $parameter_source_name = @run_numbers > 1
|
|
? $parameter_base_run . "+global_tmpl.msr"
|
|
: $parameter_base_run . "_tmpl.msr";
|
|
my $parameter_source_path = "$work_path/$parameter_source_name";
|
|
my $source_existed = -f $parameter_source_path ? 1 : 0;
|
|
my $source_backup = $source_existed ? read_text_file($parameter_source_path) : "";
|
|
my $fitted_content = read_text_file($full_path);
|
|
my $staged = $fitted_content ne ""
|
|
&& write_text_file($parameter_source_path, $fitted_content);
|
|
|
|
if ($staged) {
|
|
my $parameter_cmd = "cd " . shell_quote($work_path)
|
|
. " && " . shell_quote($MSR2DATA_BIN)
|
|
. " " . shell_quote($parameter_runs)
|
|
. " _tmpl new"
|
|
. (@run_numbers > 1 ? " global" : "")
|
|
. " data -o " . shell_quote($parameter_file);
|
|
my $parameter_result = run_logged_command(
|
|
$out_path, "msr2data parameter export", $parameter_cmd
|
|
);
|
|
|
|
# Some msr2data versions have unreliable exit codes. The
|
|
# requested non-empty output file is the authoritative result.
|
|
if (-s $parameter_path) {
|
|
if (!clean_parameter_data_header($parameter_path)) {
|
|
$status_code = 1;
|
|
$stderr .= "Failed to clean parameter-data header\n";
|
|
$message = "Fit finished with parameter export errors";
|
|
} elsif ($parameter_result->{raw_status} != 0) {
|
|
my $parameter_exit = $parameter_result->{exit_code};
|
|
$stderr .= "msr2data parameter export returned code $parameter_exit but produced $parameter_file\n";
|
|
}
|
|
} else {
|
|
$status_code = 1;
|
|
$stderr .= "msr2data did not produce parameter data\n";
|
|
$message = "Fit finished with parameter export errors";
|
|
}
|
|
} else {
|
|
$status_code = 1;
|
|
$stderr .= "Failed to stage fitted MSR for parameter export\n";
|
|
$message = "Fit finished with parameter export errors";
|
|
}
|
|
|
|
my $source_restored = $source_existed
|
|
? write_text_file($parameter_source_path, $source_backup)
|
|
: (!-e $parameter_source_path || unlink($parameter_source_path));
|
|
if (!$source_restored) {
|
|
$status_code = 1;
|
|
$stderr .= "Failed to restore $parameter_source_name after parameter export\n";
|
|
$message = "Fit finished with parameter export errors";
|
|
}
|
|
} else {
|
|
$status_code = 1;
|
|
$stderr .= "Run metadata is missing; parameter data was not generated\n";
|
|
$message = "Fit finished with parameter export errors";
|
|
}
|
|
}
|
|
|
|
if (!$abort_after_m2d) {
|
|
# musrfit can reset the PLOT runs line. Repair once more before
|
|
# visualization so Fit and Plot open the same selected-run set.
|
|
my $target_content = -f $full_path ? read_text_file($full_path) : "";
|
|
my $normalized_target = normalize_plot_runs_line($target_content, $runs);
|
|
if ($normalized_target ne $target_content) {
|
|
if (!write_text_file($full_path, $normalized_target)) {
|
|
$status_code = 1;
|
|
$stderr .= "Failed to update PLOT runs line before musrview\n";
|
|
$message = $plot_only ? "Plot failed" : "Fit finished with visualization errors";
|
|
$abort_after_m2d = 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!$abort_after_m2d) {
|
|
my $view_cmd = "cd " . shell_quote($work_path) . " && " . shell_quote($MUSRVIEW_BIN)
|
|
. " --timeout 300 --" . $plot_extension;
|
|
if ($fftv eq "y") {
|
|
$view_cmd .= " -f";
|
|
}
|
|
# Apache has no interactive display. musrview's graphics option runs in
|
|
# batch mode and writes plot files without starting a ROOT GUI session.
|
|
$view_cmd .= " " . shell_quote($target_file);
|
|
my $view_result = run_logged_command($out_path, "musrview", $view_cmd);
|
|
my $missing_data = classify_missing_data_output($view_result->{output});
|
|
if ($missing_data->{detected}) {
|
|
$status_code = 1;
|
|
$outcome = "missing-data";
|
|
@missing_files = @{$missing_data->{missing_files}};
|
|
$message = $missing_data->{message};
|
|
} elsif ($view_result->{raw_status} != 0) {
|
|
$status_code = 1;
|
|
$stderr .= "musrview failed\n";
|
|
$message = $plot_only ? "Plot failed" : "Fit finished with visualization errors";
|
|
}
|
|
}
|
|
|
|
# A single PLOT block normally creates one ROOT file, but return every
|
|
# numbered output so the client also handles MSR files with multiple PLOT
|
|
# blocks. The item list mirrors musrview's fMainCanvas<N>;1 convention.
|
|
my @root_urls = ();
|
|
my @root_items = ();
|
|
if (opendir(my $plot_dir, $work_path)) {
|
|
my @root_files = sort grep {
|
|
my $matches_output = $fftv eq "y"
|
|
? /^\Q$plot_base\E_\d+_F\.\Q$plot_extension\E$/
|
|
: /^\Q$plot_base\E_\d+\.\Q$plot_extension\E$/;
|
|
$matches_output && -f "$work_path/$_"
|
|
} readdir($plot_dir);
|
|
closedir($plot_dir);
|
|
@root_urls = map { "$TMP_URL/$job_name/$_" } @root_files;
|
|
foreach my $root_file (@root_files) {
|
|
my $canvas_index = 0;
|
|
if ($root_file =~ /_(\d+)(?:_F)?\.root$/) {
|
|
$canvas_index = $1;
|
|
}
|
|
push(@root_items, "fMainCanvas" . $canvas_index . ";1");
|
|
}
|
|
}
|
|
my $root_url = @root_urls ? $root_urls[0] : "";
|
|
my $parameter_url = -s $parameter_path
|
|
? "$TMP_URL/$job_name/$parameter_file" : "";
|
|
if (!$abort_after_m2d && $outcome ne "missing-data" && !@root_urls) {
|
|
$status_code = 1;
|
|
$stderr .= "musrview did not produce a $plot_kind plot\n";
|
|
$message = $plot_only ? "Plot failed" : "Fit finished with visualization errors";
|
|
}
|
|
|
|
# Archiving is secondary to fitting and plotting: report a failure and omit
|
|
# the download URL, but do not relabel a successful scientific operation as
|
|
# failed. The same watchdog used for executables prevents tar from holding a
|
|
# web request indefinitely.
|
|
my $tgz_path = "$TMPROOT/$job_name.tgz";
|
|
my $tgz_url = "";
|
|
my $archive_ready = 1;
|
|
if (-e $tgz_path || -l $tgz_path) {
|
|
if (-l $tgz_path || !unlink($tgz_path)) {
|
|
$stderr .= "Could not replace the results archive\n";
|
|
$archive_ready = 0;
|
|
}
|
|
}
|
|
if ($archive_ready) {
|
|
my $tgz_cmd = "cd " . shell_quote($work_path)
|
|
. " && tar -czf " . shell_quote($tgz_path) . " *";
|
|
my $tgz_result = run_logged_command($out_path, "results archive", $tgz_cmd);
|
|
if ($tgz_result->{raw_status} == 0 && -s $tgz_path) {
|
|
$tgz_url = "$TMP_URL/$job_name.tgz";
|
|
} else {
|
|
$stderr .= $tgz_result->{raw_status} == -2
|
|
? "Results archive timed out after ${CMD_TIMEOUT}s\n"
|
|
: "Could not create the results archive\n";
|
|
unlink($tgz_path) if -f $tgz_path && !-l $tgz_path;
|
|
}
|
|
}
|
|
|
|
my $command_log = "";
|
|
if ($plot_refresh && -f $out_path) {
|
|
my $complete_log = read_text_file($out_path);
|
|
$command_log = substr($complete_log, $log_start);
|
|
}
|
|
|
|
respond_json({
|
|
ok => $status_code == 0 ? JSON::PP::true : JSON::PP::false,
|
|
code => $status_code,
|
|
message => $message,
|
|
outcome => $outcome,
|
|
missing_files => \@missing_files,
|
|
stderr => $stderr,
|
|
job_name => $job_name,
|
|
work_url => "$TMP_URL/$job_name",
|
|
msr_url => "$TMP_URL/$job_name/$target_file",
|
|
out_url => "$TMP_URL/$job_name/$out_file",
|
|
parameter_url => $parameter_url,
|
|
root_url => $root_url,
|
|
root_urls => \@root_urls,
|
|
root_items => \@root_items,
|
|
command_log => $command_log,
|
|
tgz_url => $tgz_url
|
|
});
|
|
}
|
|
|
|
respond_json({ ok => JSON::PP::false, code => 400, message => "Unsupported action" });
|