Offer HTTPS downloads for missing Fit/Plot inputs and independent-series runs that need fitting. Preserve reused and skipped results without downloading raw data. Prefer the configured Data directory and fall back to persistent per-user storage when it is read-only. Keep user XML unchanged and provide native/WSL search paths through MUSRFULLDATAPATH. Stream cancellable downloads into temporary files, validate lengths, publish without overwriting existing data, and remember exact filenames for offline reuse. Add regression coverage and document the desktop workflow.
222 lines
11 KiB
JavaScript
222 lines
11 KiB
JavaScript
// Copyright (C) 2026 Zaher Salman
|
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
// Main-process run retrieval. Raw bytes stay out of the renderer, and the
|
|
// index remembers exact server filenames across sessions, including old runs
|
|
// whose campaign names differ from the desktop's startup templates.
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { randomUUID } = require('crypto');
|
|
const { Readable, Transform } = require('stream');
|
|
const { pipeline } = require('stream/promises');
|
|
|
|
const ENDPOINT = 'https://musruser.psi.ch/cgi-bin/single_musrfit.cgi';
|
|
|
|
class RunDataDownloader {
|
|
constructor(options) {
|
|
this.directory = path.resolve(options.directory);
|
|
this.indexPath = path.join(this.directory, 'index.json');
|
|
this.fetch = options.fetch;
|
|
}
|
|
|
|
async localFiles() {
|
|
let records;
|
|
try {
|
|
records = JSON.parse(await fs.promises.readFile(this.indexPath, 'utf8'));
|
|
} catch (error) {
|
|
if (error.code === 'ENOENT') return [];
|
|
throw new Error('Could not read the downloaded-run index: ' + error.message);
|
|
}
|
|
if (!Array.isArray(records)) throw new Error('The downloaded-run index is invalid.');
|
|
const files = [];
|
|
for (const record of records) {
|
|
if (!record || typeof record.filePath !== 'string') continue;
|
|
try {
|
|
const stat = await fs.promises.stat(record.filePath);
|
|
await fs.promises.access(record.filePath, fs.constants.R_OK);
|
|
if (stat.isFile() && stat.size > 0 && stat.size === record.size) files.push(record);
|
|
} catch (error) {
|
|
// A disconnected mount or a removed download can be retrieved
|
|
// again; neither should prevent other saved runs from opening.
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
async request(action, identity, signal) {
|
|
const response = await this.fetch(ENDPOINT, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
body: new URLSearchParams(Object.assign({ action: action }, identity)).toString(),
|
|
redirect: 'error',
|
|
signal: AbortSignal.any([signal, AbortSignal.timeout(
|
|
action === 'run_file_info' ? 30000 : 300000)])
|
|
});
|
|
if (!response.ok) {
|
|
let diagnostic = {};
|
|
try {
|
|
diagnostic = await response.json();
|
|
} catch (error) {
|
|
// Preserve the HTTP failure when a proxy returns an HTML page.
|
|
}
|
|
const error = new Error(diagnostic.message || 'Data server returned HTTP ' + response.status);
|
|
error.status = response.status;
|
|
error.reason = diagnostic.reason || '';
|
|
throw error;
|
|
}
|
|
return response;
|
|
}
|
|
|
|
async destination(dataRoot, relativePath) {
|
|
let lastError;
|
|
// Test the actual destination, not just the top-level folder: an
|
|
// instrument/year directory can be read-only within a writable root.
|
|
for (const root of Array.from(new Set([path.resolve(dataRoot), this.directory]))) {
|
|
const filePath = path.join(root, ...relativePath.split('/'));
|
|
const parent = path.dirname(filePath);
|
|
const probePath = path.join(parent, '.musrflow-write-' + randomUUID());
|
|
try {
|
|
await fs.promises.mkdir(parent, { recursive: true });
|
|
const probe = await fs.promises.open(probePath, 'wx', 0o600);
|
|
await probe.close();
|
|
await fs.promises.unlink(probePath);
|
|
return { filePath: filePath, storageRoot: root };
|
|
} catch (error) {
|
|
lastError = error;
|
|
}
|
|
}
|
|
throw new Error('Neither the Data directory nor the download fallback is writable: '
|
|
+ lastError.message);
|
|
}
|
|
|
|
async prepare(requests, dataRoot, signal, onProgress) {
|
|
if (!Array.isArray(requests) || requests.length > 2000 || typeof dataRoot !== 'string' || !dataRoot) {
|
|
throw new Error('Invalid run download request.');
|
|
}
|
|
const aliases = { lem: 'LEM', gps: 'GPS', flame: 'FLAME', ltf: 'LTF', dolly: 'Dolly',
|
|
vms: 'VMS', gpd: 'GPD', hal: 'HAL', hifi: 'HAL', alc: 'ALC', misc: 'MISC' };
|
|
const identities = [];
|
|
const seen = new Set();
|
|
for (const input of requests) {
|
|
const instrument = input && aliases[String(input.instrument).toLowerCase()];
|
|
const year = input && Number(input.year);
|
|
const run = input && Number(input.run);
|
|
if (!instrument || !Number.isInteger(year) || year < 1900 || year > 9999
|
|
|| !Number.isInteger(run) || run <= 0 || run > 999999999) {
|
|
throw new Error('Invalid instrument, year, or run number.');
|
|
}
|
|
const key = instrument + '/' + year + '/' + run;
|
|
if (!seen.has(key)) identities.push({ instrument: instrument, year: year, run: run });
|
|
seen.add(key);
|
|
}
|
|
const index = await this.localFiles();
|
|
const plan = { files: [], downloads: [], missing: [], index: index, totalBytes: 0 };
|
|
for (const identity of identities) {
|
|
signal.throwIfAborted();
|
|
const cached = index.find(function(record) {
|
|
return record.instrument === identity.instrument && record.year === identity.year
|
|
&& record.run === identity.run;
|
|
});
|
|
if (cached) {
|
|
plan.files.push(cached);
|
|
continue;
|
|
}
|
|
onProgress({ phase: 'checking', run: identity.run, total: identities.length });
|
|
let metadata;
|
|
try {
|
|
const response = await this.request('run_file_info', identity, signal);
|
|
metadata = await response.json();
|
|
} catch (error) {
|
|
if (error.status !== 404 && error.status !== 409) throw error;
|
|
plan.missing.push(Object.assign({ message: error.message }, identity));
|
|
continue;
|
|
}
|
|
const relative = metadata.relative_path;
|
|
const folder = identity.instrument.toLowerCase();
|
|
if (metadata.ok !== true || metadata.instrument !== identity.instrument
|
|
|| metadata.year !== identity.year || metadata.run !== identity.run
|
|
|| !Number.isSafeInteger(metadata.size) || metadata.size <= 0
|
|
|| !/^[0-9a-f]{64}$/.test(metadata.version)
|
|
|| typeof relative !== 'string' || !relative.startsWith(folder + '/his/')
|
|
|| !/^[A-Za-z0-9_+.-]+(?:\/[A-Za-z0-9_+.-]+)*$/.test(relative)
|
|
|| relative.split('/').some(function(part) { return part === '.' || part === '..'; })
|
|
|| relative.split('/').pop() !== metadata.filename
|
|
|| !['MUSR-ROOT', 'PSI-BIN', 'PSI-MDU'].includes(metadata.format)) {
|
|
throw new Error('The data server returned invalid run metadata.');
|
|
}
|
|
const target = await this.destination(dataRoot, relative);
|
|
plan.downloads.push(Object.assign({}, metadata, target));
|
|
plan.totalBytes += metadata.size;
|
|
}
|
|
return plan;
|
|
}
|
|
|
|
async download(plan, signal, onProgress) {
|
|
let completedBytes = 0;
|
|
for (const file of plan.downloads) {
|
|
signal.throwIfAborted();
|
|
const partialPath = file.filePath + '.' + randomUUID() + '.part';
|
|
let received = 0;
|
|
let lastProgress = 0;
|
|
try {
|
|
const response = await this.request('download_run_file', {
|
|
instrument: file.instrument, year: file.year, run: file.run, version: file.version
|
|
}, signal);
|
|
const length = response.headers.get('content-length');
|
|
if (response.headers.get('content-type') !== 'application/octet-stream'
|
|
|| (length !== null && Number(length) !== file.size)) {
|
|
await response.body.cancel();
|
|
throw new Error('The data server returned an unexpected file response.');
|
|
}
|
|
const progress = new Transform({
|
|
transform: function(chunk, encoding, callback) {
|
|
received += chunk.length;
|
|
if (received > file.size) {
|
|
callback(new Error('Downloaded file is larger than the advertised size.'));
|
|
return;
|
|
}
|
|
if (Date.now() - lastProgress > 100 || received === file.size) {
|
|
lastProgress = Date.now();
|
|
onProgress({ phase: 'downloading', filename: file.filename,
|
|
received: completedBytes + received, totalBytes: plan.totalBytes });
|
|
}
|
|
callback(null, chunk);
|
|
}
|
|
});
|
|
await pipeline(Readable.fromWeb(response.body), progress,
|
|
fs.createWriteStream(partialPath, { flags: 'wx' }), { signal: signal });
|
|
if (received !== file.size) throw new Error('Downloaded file is incomplete.');
|
|
signal.throwIfAborted();
|
|
// Publish only a complete file and never overwrite a run that
|
|
// appeared locally while the transfer was in progress.
|
|
await fs.promises.link(partialPath, file.filePath);
|
|
const previous = plan.index.findIndex(function(record) {
|
|
return record.instrument === file.instrument && record.year === file.year
|
|
&& record.run === file.run;
|
|
});
|
|
if (previous !== -1) plan.index.splice(previous, 1);
|
|
plan.index.push(file);
|
|
await fs.promises.mkdir(this.directory, { recursive: true });
|
|
const stage = this.indexPath + '.' + randomUUID() + '.tmp';
|
|
try {
|
|
await fs.promises.writeFile(stage, JSON.stringify(plan.index), { flag: 'wx', mode: 0o600 });
|
|
await fs.promises.rename(stage, this.indexPath);
|
|
} finally {
|
|
await fs.promises.unlink(stage).catch(function(error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
});
|
|
}
|
|
plan.files.push(file);
|
|
completedBytes += received;
|
|
} finally {
|
|
await fs.promises.unlink(partialPath).catch(function(error) {
|
|
if (error.code !== 'ENOENT') throw error;
|
|
});
|
|
}
|
|
}
|
|
return { ok: true, files: plan.files, missing: plan.missing };
|
|
}
|
|
}
|
|
|
|
module.exports = { RunDataDownloader, ENDPOINT };
|