From 5584460de031f8cf918f11891dfea40bf4810f55 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 9 Oct 2026 17:52:43 +0200 Subject: [PATCH] Reflection files record their provenance: commit, command line, input file The merged .mtz and the unmerged .mtz carry, after the existing "From Rugnux " record, HISTORY records with the git commit and date, the command line, the input file (absolute path) and a cheap identity of it - size and modification time, the master file's for HDF5, nothing hashed - and the collection date. MTZ allows 30 records of 80 characters; a longer value runs on over records starting with '+', cut so that no record ends in a blank a reader would strip. The mmCIF names the commit in _software.description and the command line, input file and identity under _jfjoch_provenance (the local prefix the other private items use; no PDBx item holds them), the first two as text fields so any character survives. OutputProvenance is filled once in the CLI and travels in ProcessConfig to every writer of a reflection file. Reflection data are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01SVmAWnzCmRKAXVUCdc4iNi --- docs/RUGNUX_ADVANCED.md | 15 ++++++++ image_analysis/WriteReflections.cpp | 60 ++++++++++++++++++++++++++--- image_analysis/WriteReflections.h | 11 +++++- rugnux/Rugnux.cpp | 4 +- rugnux/RugnuxScaleMerge.cpp | 4 +- rugnux/rugnux_cli.cpp | 31 ++++++++++++++- tests/WriteReflectionsTest.cpp | 58 ++++++++++++++++++++++++++-- 7 files changed, 166 insertions(+), 17 deletions(-) diff --git a/docs/RUGNUX_ADVANCED.md b/docs/RUGNUX_ADVANCED.md index 6abcea11e..75e2a69a2 100644 --- a/docs/RUGNUX_ADVANCED.md +++ b/docs/RUGNUX_ADVANCED.md @@ -269,6 +269,21 @@ Items PDBx does not define — ISa, the error model, the twinning and anisotropy refined distance and beam centre — are written under the `jfjoch_` local prefix, which a dictionary check reports as unknown and a deposition ignores. +### Provenance + +Every reflection file says where it came from, so that it can be traced back without the report +beside it: + +- the **`.cif`** names the Rugnux version in `_software.version` and the commit it was built from in + `_software.description`, and the command line, the input file (absolute path) and the input's + identity under `_jfjoch_provenance.command_line`, `.input_file` and `.input_identity`; +- the **`.mtz`** and the **`_unmerged.mtz`** carry the same in their `HISTORY` records (`mtzdump` + prints them), each run on over further records where it is longer than the 80 characters of one. + +The identity is the input file's size and modification time — for an HDF5 data set, the master +file's — which tells two files of one name apart without reading the data. A collection followed from +the broker is not a file here, and has none. + ## Command-line options General: diff --git a/image_analysis/WriteReflections.cpp b/image_analysis/WriteReflections.cpp index da0fbe3dd..179c137e5 100644 --- a/image_analysis/WriteReflections.cpp +++ b/image_analysis/WriteReflections.cpp @@ -110,6 +110,36 @@ std::string IsoDate(const std::string ×tamp) { return timestamp.substr(0, 10); } +// The commit this build was made from, with its date; empty when the build does not know it. +std::string GitDescription() { + return jfjoch_git_sha1().empty() ? std::string() : jfjoch_git_sha1() + " " + jfjoch_git_date(); +} + +// The MTZ HISTORY records saying where a file came from, after the one naming the program: the +// commit, the command line and the input file. MTZ holds at most 30 records of 80 characters +// (mtzformat.html), so a long value runs on over as many records as it needs, up to that limit, each +// continuation starting with '+'. Readers strip the blanks at both ends of a record, so a record is +// never cut just after a space: the space starts the next one, behind its '+'. +void AddProvenanceHistory(gemmi::Mtz &mtz, const OutputProvenance &provenance) { + const auto add = [&mtz](const std::string &label, const std::string &value) { + if (value.empty()) + return; + std::string rest = label + value; + while (!rest.empty() && mtz.history.size() < 30) { + size_t n = std::min(80, rest.size()); + while (n > 1 && n < rest.size() && rest[n - 1] == ' ') + --n; + mtz.history.push_back(rest.substr(0, n)); + rest = n < rest.size() ? "+" + rest.substr(n) : std::string(); + } + }; + add("Rugnux git ", GitDescription()); + add("Command: ", provenance.command_line); + add("Input: ", provenance.input_file); + add("Input file: ", provenance.input_identity); + add("Collected: ", provenance.collection_date); +} + // _entry, _cell and _symmetry: what every data block of the file restates, so each one stands alone. void WriteBlockHeader(std::ostream &out, const std::string &entry_id, const UnitCell &unitCell, const gemmi::SpaceGroup &sg) { @@ -253,14 +283,29 @@ void WriteMmcifReflections(const std::vector &reflections, // ---------- _software ---------- // Rugnux integrated the images (data reduction) and scaled and merged them (data scaling), the - // two rows the wwPDB asks a data-processing program to be named under. + // two rows the wwPDB asks a data-processing program to be named under; the description is the + // commit the build was made from. + const std::string software_description = CifStr(GitDescription().empty() ? "" : "git " + GitDescription()); out << "loop_\n"; out << "_software.pdbx_ordinal\n"; out << "_software.name\n"; out << "_software.version\n"; out << "_software.classification\n"; - out << "1 Rugnux " << CifStr(jfjoch_version()) << " 'data reduction'\n"; - out << "2 Rugnux " << CifStr(jfjoch_version()) << " 'data scaling'\n"; + out << "_software.description\n"; + out << "1 Rugnux " << CifStr(jfjoch_version()) << " 'data reduction' " << software_description << "\n"; + out << "2 Rugnux " << CifStr(jfjoch_version()) << " 'data scaling' " << software_description << "\n"; + out << "#\n"; + + // ---------- provenance ---------- + // How this file was made and from what: the command line and the input file, which no PDBx item + // holds, under the jfjoch local prefix as the other private items here. The two that can carry + // any character are text fields. + if (!provenance.command_line.empty()) + out << "_jfjoch_provenance.command_line\n;" << provenance.command_line << "\n;\n"; + if (!provenance.input_file.empty()) + out << "_jfjoch_provenance.input_file\n;" << provenance.input_file << "\n;\n"; + if (!provenance.input_identity.empty()) + out << "_jfjoch_provenance.input_identity " << CifStr(provenance.input_identity) << "\n"; out << "#\n"; // ---------- _exptl_crystal / _diffrn ---------- @@ -624,6 +669,7 @@ void WriteMmcifReflections(const std::vector &reflections, void WriteMtzReflections(const std::vector &reflections, const UnitCell &unitCell, const DiffractionExperiment &experiment, + const OutputProvenance &provenance, const std::string &filename) { gemmi::Mtz mtz; @@ -634,6 +680,7 @@ void WriteMtzReflections(const std::vector &reflections, // Producing-software provenance in the MTZ header (title + HISTORY, the CCP4 convention). mtz.title = "Rugnux merged reflections"; mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); + AddProvenanceHistory(mtz, provenance); // Add dataset. HKL_base first, so the data dataset is id 1: a dataset written as id 0 is the // reserved base dataset as far as CCP4's mtzlib is concerned, and the wavelength on it is @@ -954,12 +1001,14 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, const UnitCell &unitCell, const DiffractionExperiment &experiment, bool sum_partials, + const OutputProvenance &provenance, size_t nthreads) { gemmi::Mtz mtz; mtz.spacegroup = &experiment.GetSpaceGroupOrP1(); mtz.set_cell_for_all(unitCell); mtz.title = "Rugnux unmerged observations"; mtz.history.push_back("From Rugnux " + jfjoch_version() + ", data reduction"); + AddProvenanceHistory(mtz, provenance); mtz.add_base(); // the HKL_base dataset and the H K L columns gemmi::Mtz::Dataset &ds = mtz.add_dataset("unmerged"); ds.crystal_name = experiment.GetSampleName(); @@ -1223,9 +1272,10 @@ void WriteUnmergedMtzReflections(const std::vector &outcomes const UnitCell &unitCell, const DiffractionExperiment &experiment, bool sum_partials, + const OutputProvenance &provenance, const std::string &filename, size_t nthreads) { - gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, nthreads); + gemmi::Mtz mtz = UnmergedMtz(outcomes, unitCell, experiment, sum_partials, provenance, nthreads); SetUnmergedMtzMosaicity(mtz, outcomes); mtz.write_to_file(filename); } @@ -1244,7 +1294,7 @@ void WriteReflections(const std::vector &reflections, // phenix reflection tools, mmCIF for deposition and as the self-describing native format, HKLF-4 for // SHELXL refinement and SHELXC / ANODE substructure solution). The .hkl holds the unmerged scaled // fulls where the merge provides them (rotation), the merged reflections otherwise (stills). - WriteMtzReflections(reflections, unitCell, experiment, filename + ".mtz"); + WriteMtzReflections(reflections, unitCell, experiment, provenance, filename + ".mtz"); WriteMmcifReflections(reflections, unitCell, experiment, statistics, error_model, twinning, scaled_fulls ? *scaled_fulls : std::vector{}, provenance, filename + ".cif", nthreads); diff --git a/image_analysis/WriteReflections.h b/image_analysis/WriteReflections.h index bdac24f65..6fc02f042 100644 --- a/image_analysis/WriteReflections.h +++ b/image_analysis/WriteReflections.h @@ -27,9 +27,13 @@ struct ErrorModelReport { std::string b; }; -// What the reflection files record about where they came from, beyond the experiment itself. -// Empty strings are left out. +// What the reflection files record about where they came from, beyond the experiment itself and the +// Rugnux version and commit: the MTZ HISTORY records and the mmCIF's _software and +// _jfjoch_provenance. Empty strings are left out. struct OutputProvenance { + std::string command_line; + std::string input_file; // the input as found on disk, absolute + std::string input_identity; // a cheap identifier of it: size and modification time std::string collection_date; // as the input states it (NXmx /entry/start_time), ISO-8601 }; @@ -51,6 +55,7 @@ void WriteMmcifReflections(const std::vector &reflections, void WriteMtzReflections(const std::vector &reflections, const UnitCell &unitCell, const DiffractionExperiment &experiment, + const OutputProvenance &provenance, const std::string &filename); // SHELX HKLF-4 text file, FORMAT(3I4,2F8.2) h k l I sigma(I), scaled so the largest value fits the @@ -83,6 +88,7 @@ void WriteUnmergedMtzReflections(const std::vector &outcomes const UnitCell &unitCell, const DiffractionExperiment &experiment, bool sum_partials, + const OutputProvenance &provenance, const std::string &filename, size_t nthreads = 1); @@ -94,6 +100,7 @@ gemmi::Mtz UnmergedMtz(const std::vector &outcomes, const UnitCell &unitCell, const DiffractionExperiment &experiment, bool sum_partials, + const OutputProvenance &provenance, size_t nthreads = 1); void SetUnmergedMtzMosaicity(gemmi::Mtz &mtz, const std::vector &outcomes); diff --git a/rugnux/Rugnux.cpp b/rugnux/Rugnux.cpp index 804f78bdd..269dee347 100644 --- a/rugnux/Rugnux.cpp +++ b/rugnux/Rugnux.cpp @@ -952,14 +952,14 @@ ProcessResult Rugnux::RunPipeline(RugnuxObserver *observer, bool write_output, b mtz.write_to_file(path); } else WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell, - experiment_, true, path, config_.nthreads); + experiment_, true, config_.provenance, path, config_.nthreads); logger.Info("Unmerged observations written to {}", path); } if (config_.export_unmerged_partials) { if (observer) observer->OnPhase("Writing unmerged reflections"); const std::string path = config_.output_prefix + "_unmerged_partials.mtz"; WriteUnmergedMtzReflections(indexer->GetIntegrationOutcome(), *result.consensus_cell, - experiment_, false, path, config_.nthreads); + experiment_, false, config_.provenance, path, config_.nthreads); logger.Info("Unmerged partials written to {}", path); } } diff --git a/rugnux/RugnuxScaleMerge.cpp b/rugnux/RugnuxScaleMerge.cpp index e00a0f95a..03e9aee32 100644 --- a/rugnux/RugnuxScaleMerge.cpp +++ b/rugnux/RugnuxScaleMerge.cpp @@ -3446,8 +3446,8 @@ bool Rugnux::ScaleMergeAndSymmetry(PipelineLocals &p) { unmerged_mtz = std::async(std::launch::async, [&outcomes = indexer->GetIntegrationOutcome(), cell = *result.consensus_cell, &x = *unmerged_experiment, - nthreads = config_.nthreads] { - return UnmergedMtz(outcomes, cell, x, true, nthreads); + &provenance = config_.provenance, nthreads = config_.nthreads] { + return UnmergedMtz(outcomes, cell, x, true, provenance, nthreads); }); } diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index 003f59bde..f370f5be5 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -39,6 +39,7 @@ #include "../common/Logger.h" #include "../common/ThreadAffinity.h" #include "../common/GitInfo.h" +#include "../common/time_utc.h" #include "../common/Definitions.h" #include "../common/DiffractionExperiment.h" #include "../common/PixelMask.h" @@ -767,6 +768,28 @@ namespace { return out; } + // The input as found on disk, for the reflection files to name: its absolute path, and its size and + // modification time - an identifier that tells two files of one name apart without reading the + // data, which can run to terabytes. For an HDF5 data set it is the master file's. Nothing is + // recorded about an input that is not a file here (a collection followed from the broker). + void DescribeInput(const std::string &input_file, OutputProvenance &out) { + std::error_code ec; + const std::filesystem::path path = std::filesystem::absolute(input_file, ec); + if (ec || !std::filesystem::exists(path, ec)) + return; + out.input_file = path.lexically_normal().string(); + const auto size = std::filesystem::is_regular_file(path, ec) ? std::filesystem::file_size(path, ec) + : std::uintmax_t{0}; + const auto modified = std::filesystem::last_write_time(path, ec); + if (ec) + return; + // file_clock to system_clock through "now" on both, to the second. + const auto sys = std::chrono::floor( + modified - std::filesystem::file_time_type::clock::now() + std::chrono::system_clock::now()); + out.input_identity = (size > 0 ? fmt::format("{} bytes, ", size) : std::string()) + + "modified " + time_UTC(sys); + } + // Running out of host memory ends a run with an exception that names only what failed in C++ terms - // std::bad_alloc, a vector past its maximum size, a thread that could not get its stack - which // does not tell the user what to do. The data set is not at fault; the machine is too small for it. @@ -1666,6 +1689,8 @@ static int RunRugnux(int argc, char **argv) { logger.Info("Loaded dataset from {}", input_file); // Recorded in the reflection files. OutputProvenance output_provenance; + output_provenance.command_line = provenance.command_line; + DescribeInput(input_file, output_provenance); output_provenance.collection_date = dataset->arm_date; { // The header as the file states it, before anything is measured: what a question about a run @@ -2214,12 +2239,14 @@ static int RunRugnux(int argc, char **argv) { // come out as a bare "_unmerged.mtz". if (export_unmerged && !output_prefix.empty()) { const std::string path = output_prefix + "_unmerged.mtz"; - WriteUnmergedMtzReflections(reflections, *experiment.GetUnitCell(), experiment, true, path); + WriteUnmergedMtzReflections(reflections, *experiment.GetUnitCell(), experiment, true, + output_provenance, path); logger.Info("Unmerged observations written to {}", path); } if (export_unmerged_partials && !output_prefix.empty()) { const std::string path = output_prefix + "_unmerged_partials.mtz"; - WriteUnmergedMtzReflections(reflections, *experiment.GetUnitCell(), experiment, false, path); + WriteUnmergedMtzReflections(reflections, *experiment.GetUnitCell(), experiment, false, + output_provenance, path); logger.Info("Unmerged partials written to {}", path); } diff --git a/tests/WriteReflectionsTest.cpp b/tests/WriteReflectionsTest.cpp index 9c95b7c07..c66b4a33f 100644 --- a/tests/WriteReflectionsTest.cpp +++ b/tests/WriteReflectionsTest.cpp @@ -47,7 +47,7 @@ TEST_CASE("Merged MTZ: the data dataset is id 1 and carries the wavelength", "[w const auto experiment = TestExperiment(); const auto path = (std::filesystem::temp_directory_path() / "rugnux_merged_wavelength.mtz").string(); - WriteMtzReflections(reflections, TETRAGONAL_CELL, experiment, path); + WriteMtzReflections(reflections, TETRAGONAL_CELL, experiment, OutputProvenance{}, path); const gemmi::Mtz mtz = gemmi::read_mtz_file(path); std::filesystem::remove(path); @@ -108,7 +108,7 @@ TEST_CASE("Unmerged MTZ: LP is Lorentz-polarization, QE the sensor efficiency, F } const auto path = (std::filesystem::temp_directory_path() / "rugnux_unmerged_qe.mtz").string(); - WriteUnmergedMtzReflections({outcome}, TETRAGONAL_CELL, experiment, false, path); + WriteUnmergedMtzReflections({outcome}, TETRAGONAL_CELL, experiment, false, OutputProvenance{}, path); const gemmi::Mtz mtz = gemmi::read_mtz_file(path); std::filesystem::remove(path); @@ -180,8 +180,10 @@ TEST_CASE("Unmerged MTZ: built on several workers, the same rows as on one", "[w for (bool sum_partials : {true, false}) { INFO("sum_partials " << sum_partials); - const gemmi::Mtz serial = UnmergedMtz(outcomes, TETRAGONAL_CELL, experiment, sum_partials, 1); - const gemmi::Mtz parallel = UnmergedMtz(outcomes, TETRAGONAL_CELL, experiment, sum_partials, 8); + const gemmi::Mtz serial = UnmergedMtz(outcomes, TETRAGONAL_CELL, experiment, sum_partials, + OutputProvenance{}, 1); + const gemmi::Mtz parallel = UnmergedMtz(outcomes, TETRAGONAL_CELL, experiment, sum_partials, + OutputProvenance{}, 8); REQUIRE(serial.nreflections > 100000); REQUIRE(parallel.nreflections == serial.nreflections); CHECK(std::memcmp(parallel.data.data(), serial.data.data(), serial.data.size() * sizeof(float)) == 0); @@ -260,6 +262,9 @@ TEST_CASE("Merged mmCIF: PDBx keys, merging R-factors and the scaled unmerged bl }; OutputProvenance provenance; provenance.collection_date = "2026-07-13T13:48:52.413Z"; + provenance.command_line = "rugnux -o 'my run' /data/x_master.h5"; + provenance.input_file = "/data/x_master.h5"; + provenance.input_identity = "1234 bytes, modified 2026-07-13T14:00:00.000Z"; const auto path = (std::filesystem::temp_directory_path() / "rugnux_merged_test.cif").string(); WriteMmcifReflections(reflections, TETRAGONAL_CELL, experiment, statistics, ErrorModelReport{}, @@ -278,6 +283,12 @@ TEST_CASE("Merged mmCIF: PDBx keys, merging R-factors and the scaled unmerged bl == Catch::Approx(std::sqrt(2 * 0.98 / 1.98)).margin(1e-4)); CHECK(*merged.find_value("_diffrn.ambient_temp") == "100.0"); CHECK(*merged.find_value("_diffrn_detector.pdbx_collection_date") == "2026-07-13"); + // Provenance: the command line survives its own quotes, as a text field. + CHECK(gemmi::cif::as_string(*merged.find_value("_jfjoch_provenance.command_line")) + == provenance.command_line); + CHECK(gemmi::cif::as_string(*merged.find_value("_jfjoch_provenance.input_file")) == provenance.input_file); + CHECK(gemmi::cif::as_string(*merged.find_value("_jfjoch_provenance.input_identity")) + == provenance.input_identity); gemmi::cif::Table software = merged.find("_software.", {"pdbx_ordinal", "classification"}); REQUIRE(software.length() == 2); @@ -303,3 +314,42 @@ TEST_CASE("Merged mmCIF: PDBx keys, merging R-factors and the scaled unmerged bl CHECK(gemmi::cif::as_number(obs[0][3]) == Catch::Approx(200.0)); CHECK(gemmi::cif::as_number(obs[1][5]) == Catch::Approx(0.8)); } + +TEST_CASE("MTZ history: command line and input, in records of at most 80 characters", + "[write_reflections][portable]") { + jfjoch_test::SyntheticMergeParams params; + params.true_space_group = "P 43 21 2"; + params.twin_supergroup = "P 43 21 2"; + params.d_min_A = 5.0; + const auto reflections = jfjoch_test::GenerateSyntheticMerged(params); + + OutputProvenance provenance; + // A space falls exactly on the first record's last character, which a reader would strip. + provenance.command_line = "rugnux -o out " + std::string(56, 'x') + " " + std::string(150, 'x') + "_master.h5"; + provenance.input_file = "/data/" + std::string(150, 'x') + "_master.h5"; + provenance.input_identity = "1234 bytes, modified 2026-07-13T14:00:00.000Z"; + + const auto path = (std::filesystem::temp_directory_path() / "rugnux_history.mtz").string(); + WriteMtzReflections(reflections, TETRAGONAL_CELL, TestExperiment(), provenance, path); + const gemmi::Mtz mtz = gemmi::read_mtz_file(path); + std::filesystem::remove(path); + + REQUIRE(!mtz.history.empty()); + CHECK(mtz.history.front().rfind("From Rugnux ", 0) == 0); + std::string joined; + for (const auto &line : mtz.history) { + CHECK(line.size() <= 80); + joined += line; + } + // Each value is all there, run on over records that start with '+'. + std::string command; + for (size_t i = 0; i < mtz.history.size(); ++i) + if (mtz.history[i].rfind("Command: ", 0) == 0) { + command = mtz.history[i].substr(9); + for (size_t j = i + 1; j < mtz.history.size() && mtz.history[j].rfind("+", 0) == 0; ++j) + command += mtz.history[j].substr(1); + } + CHECK(command == provenance.command_line); + CHECK(joined.find("Input file: " + provenance.input_identity) != std::string::npos); + CHECK(mtz.history.size() <= 30); +}