From a78bfe951d8d6659c336e98f41175fc9353f2e8b Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Mon, 26 Oct 2026 00:40:00 +0100 Subject: [PATCH] rugnux: a model and a reference among the inputs, in any order, told by their contents `rugnux ` in any order, without --model or -z. Each positional file is classified by its contents (RugnuxInputKind.h, in JFJochRugnuxOutput - the library still reads no files): an MTZ (magic) or an mmCIF with a _refln / _diffrn_refln loop is reference data (-z; the input itself in --mode refine), a PDB file (a PDB record first, then ATOM/HETATM) or an mmCIF with an _atom_site loop is a model (--model), anything else is images for the readers. CBF (###CBF or _array_data) counts as images; a file whose first line is neither data_ nor a PDB record is decided there, so image files are not read through. An mmCIF holding both atoms and reflections is refused as ambiguous. At most one model and one reference, --model/--mr and -z included. Gzip read through. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu (cherry picked from commit 8f844b0bf86f338d20e8aed426a387722dd8d7cc) --- docs/RUGNUX_MODEL_AND_REFERENCE.md | 9 ++ rugnux/CMakeLists.txt | 4 +- rugnux/RugnuxInputKind.cpp | 137 +++++++++++++++++++++++++++++ rugnux/RugnuxInputKind.h | 37 ++++++++ rugnux/rugnux_cli.cpp | 32 +++++++ tests/CMakeLists.txt | 1 + tests/RugnuxInputKindTest.cpp | 79 +++++++++++++++++ 7 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 rugnux/RugnuxInputKind.cpp create mode 100644 rugnux/RugnuxInputKind.h create mode 100644 tests/RugnuxInputKindTest.cpp diff --git a/docs/RUGNUX_MODEL_AND_REFERENCE.md b/docs/RUGNUX_MODEL_AND_REFERENCE.md index 0a389aade..9f2c89f1c 100644 --- a/docs/RUGNUX_MODEL_AND_REFERENCE.md +++ b/docs/RUGNUX_MODEL_AND_REFERENCE.md @@ -58,6 +58,15 @@ the rest walks through a run step by step. | `--force-rotation-lattice` | Nine numbers: a lattice | Skips the first-pass indexing and uses this lattice | **Yes**: the lattice | | `--report-resolution [,]` | Another program's resolution range | A second statistics table at that range (`REFRES_*`) | No — report only | +**Without the options.** A model and a reference reflection file can also be given among the inputs, in any +order: `rugnux data_master.h5 model.cif`, `rugnux model.pdb.gz data_master.h5 ref.mtz`. Each file is told by +its contents, never its name: an MTZ (its `MTZ ` magic) or an mmCIF with a `_refln` or `_diffrn_refln` loop +(a PDB `-sf.cif`) is reference data, taken as `-z` - in `--mode refine` it is the input itself; a PDB file (a +PDB record first, with an `ATOM` or `HETATM` record) or an mmCIF with an `_atom_site` loop is a model, taken +as `--model`; anything else is images, for the readers. An mmCIF holding both coordinates and reflections is +refused as ambiguous and has to be given with `--model` or `-z`. At most one model and one reference, counting +`--model`/`--mr` and `-z` as well; gzipped files are read through. + `-S`, `-C`, `-z` and `--force-rotation-lattice` are *assertions*: Rugnux takes them as true and does not test them against a null. Use them when you know; leave them out to let the data decide (a de-novo run is the recommended default — see [Running Rugnux](RUGNUX_TUTORIAL.md#rotation-data)). diff --git a/rugnux/CMakeLists.txt b/rugnux/CMakeLists.txt index ff903aef3..396d610f5 100644 --- a/rugnux/CMakeLists.txt +++ b/rugnux/CMakeLists.txt @@ -40,8 +40,8 @@ TARGET_LINK_LIBRARIES(JFJochRugnux JFJochReader JFJochImageAnalysis JFJochPrevie # The files of a run, written by the programs that run one (rugnux, the viewer) from what JFJochRugnux # returns: the library itself writes none. -ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h) -TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter) +ADD_LIBRARY(JFJochRugnuxOutput STATIC RugnuxOutputFiles.cpp RugnuxOutputFiles.h RugnuxInputKind.cpp RugnuxInputKind.h) +TARGET_LINK_LIBRARIES(JFJochRugnuxOutput JFJochRugnux JFJochWriter JFJochModelWriter ZLIB::ZLIB) # rugnux is the single offline analysis CLI; --mode picks what it does: mx (the full pipeline, the # default), azint, scale (re-scale/merge stored reflections) or calibration (powder-ring geometry). diff --git a/rugnux/RugnuxInputKind.cpp b/rugnux/RugnuxInputKind.cpp new file mode 100644 index 000000000..565ead494 --- /dev/null +++ b/rugnux/RugnuxInputKind.cpp @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "RugnuxInputKind.h" + +#include +#include +#include +#include + +#include + +const char *RugnuxInputKindName(RugnuxInputKind kind) { + switch (kind) { + case RugnuxInputKind::Images: return "images"; + case RugnuxInputKind::Model: return "model"; + case RugnuxInputKind::Reflections: return "reflections"; + case RugnuxInputKind::Ambiguous: return "ambiguous"; + } + return "?"; +} + +namespace { + // The record names a PDB file starts with (its first line that is not blank) + constexpr const char *PDB_RECORDS[] = {"HEADER", "TITLE ", "COMPND", "SOURCE", "KEYWDS", "EXPDTA", "AUTHOR", + "REVDAT", "REMARK", "SEQRES", "CRYST1", "MODEL ", "ATOM ", "HETATM"}; + + // The rule, fed one line at a time; done() once the answer cannot change. + class Sniffer { + enum class Text { Unknown, Cif, Pdb } text = Text::Unknown; + bool atoms = false, reflections = false; + std::optional decided; + public: + explicit Sniffer(const std::string &head) { + if (head.starts_with("MTZ ")) + decided = RugnuxInputKind::Reflections; + else if (head.starts_with("\x89HDF\r\n\x1a\n") || head.starts_with("###CBF") + || head.find('\0') != std::string::npos) + decided = RugnuxInputKind::Images; + } + bool done() const { return decided.has_value(); } + void line(const std::string &l) { + if (decided) + return; + if (text == Text::Unknown) { + const size_t i = l.find_first_not_of(" \t\r"); + if (i == std::string::npos || l[i] == '#') + return; + if (l.compare(i, 5, "data_") == 0) + text = Text::Cif; + else if (std::any_of(std::begin(PDB_RECORDS), std::end(PDB_RECORDS), + [&](const char *r) { return l.starts_with(r); })) + text = Text::Pdb; + else + decided = RugnuxInputKind::Images; // not a model nor reflections: for the image readers + } + if (text == Text::Pdb && (l.starts_with("ATOM ") || l.starts_with("HETATM"))) + decided = RugnuxInputKind::Model; + if (text == Text::Cif) { + if (l.starts_with("_array_data")) + decided = RugnuxInputKind::Images; // a CBF without its ###CBF line + atoms = atoms || l.starts_with("_atom_site."); + reflections = reflections || l.starts_with("_refln.") || l.starts_with("_diffrn_refln."); + } + } + RugnuxInputKind kind() const { + if (decided) + return *decided; + if (atoms && reflections) + return RugnuxInputKind::Ambiguous; + if (atoms) + return RugnuxInputKind::Model; + if (reflections) + return RugnuxInputKind::Reflections; + return RugnuxInputKind::Images; + } + }; + + // A text line is shorter than this + constexpr int MAX_LINE = 4096; + constexpr int HEAD = 16; +} + +RugnuxInputKind ClassifyRugnuxInputContent(const std::string &head, const std::vector &lines) { + Sniffer s(head); + for (const auto &l : lines) + s.line(l); + return s.kind(); +} + +RugnuxInputKind ClassifyRugnuxInput(const std::string &path) { + gzFile f = gzopen(path.c_str(), "rb"); + if (f == nullptr) + throw std::runtime_error("cannot open " + path); + std::string head(HEAD, '\0'); + head.resize(std::max(gzread(f, head.data(), HEAD), 0)); + Sniffer s(head); + gzrewind(f); + std::string buf(MAX_LINE, '\0'); + while (!s.done() && gzgets(f, buf.data(), MAX_LINE) != nullptr) { + std::string l(buf.c_str()); + if (!l.empty() && l.back() == '\n') + l.pop_back(); + s.line(l); + } + gzclose(f); + return s.kind(); +} + +RugnuxPositionalInputs SplitRugnuxInputs(const std::vector &paths) { + RugnuxPositionalInputs out; + for (const std::string &p : paths) { + // what is not a file (a missing path, a broker URL) is left to the image readers to report + const RugnuxInputKind kind = std::filesystem::is_regular_file(p) ? ClassifyRugnuxInput(p) + : RugnuxInputKind::Images; + switch (kind) { + case RugnuxInputKind::Images: + out.images.push_back(p); + break; + case RugnuxInputKind::Model: + if (!out.model.empty()) + throw std::runtime_error("two atomic models given, " + out.model + " and " + p + "; give one"); + out.model = p; + break; + case RugnuxInputKind::Reflections: + if (!out.reflections.empty()) + throw std::runtime_error("two reflection files given, " + out.reflections + " and " + p + + "; give one"); + out.reflections = p; + break; + case RugnuxInputKind::Ambiguous: + throw std::runtime_error(p + " holds both atomic coordinates (_atom_site) and reflections (_refln); " + "give it with --model or -z"); + } + } + return out; +} diff --git a/rugnux/RugnuxInputKind.h b/rugnux/RugnuxInputKind.h new file mode 100644 index 000000000..a34d23809 --- /dev/null +++ b/rugnux/RugnuxInputKind.h @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +#include +#include + +// What a file given to rugnux as a positional argument holds, read from its contents, never its name, so that +// `rugnux ` works in any order without --model or -z: +// Reflections MTZ (the "MTZ " magic), or an mmCIF with a _refln or _diffrn_refln loop (a PDB -sf.cif) +// Model PDB (a file starting with a PDB record, with an ATOM or HETATM record), or an mmCIF with an +// _atom_site loop +// Images anything else - HDF5, CBF (an mmCIF-like file starting ###CBF, or with _array_data), and every +// other format the readers take, which then decide. A file whose first line is neither "data_" +// nor a PDB record is decided on that line, so an image file is not read through. +// An mmCIF holding both atoms and reflections is Ambiguous: it has to be given with --model or -z. Gzipped files +// are read through. +enum class RugnuxInputKind { Images, Model, Reflections, Ambiguous }; + +const char *RugnuxInputKindName(RugnuxInputKind kind); + +// Classifies a file (reads it, gunzipping where it is gzipped) +RugnuxInputKind ClassifyRugnuxInput(const std::string &path); + +// The rule itself, on decompressed content: `head` is the first 16 bytes, `lines` the lines from the start. +RugnuxInputKind ClassifyRugnuxInputContent(const std::string &head, const std::vector &lines); + +// The positional inputs of a rugnux command line, split by what they hold +struct RugnuxPositionalInputs { + std::vector images; + std::string model; // empty: none + std::string reflections; // empty: none +}; + +// Throws std::runtime_error (with a message for the user) on two models, two reflection files or an ambiguous file. +RugnuxPositionalInputs SplitRugnuxInputs(const std::vector &paths); diff --git a/rugnux/rugnux_cli.cpp b/rugnux/rugnux_cli.cpp index d1b66382a..b2c7e0719 100644 --- a/rugnux/rugnux_cli.cpp +++ b/rugnux/rugnux_cli.cpp @@ -84,6 +84,7 @@ #include "ResultReport.h" #include "RugnuxOutputFiles.h" #include "RugnuxConsole.h" +#include "RugnuxInputKind.h" #ifdef __APPLE__ #include "../image_analysis/bragg_integration/BraggIntegrationEngineMetal.h" #endif @@ -115,6 +116,12 @@ void print_usage() { "together, the indexing rate reported per master as well; these must share the detector, " "wavelength, distance and beam centre. Several inputs must be one format; rotation sweeps (name " "one frame of a CBF/marCCD/SMV sweep), --mode other than mx and following a broker take one" << std::endl; + std::cout << " An atomic model and a reference reflection file can be given among the inputs, in any order, instead of " + "--model and -z: each file is told by its contents, not its name - an MTZ (the MTZ magic) or an mmCIF " + "with a _refln or _diffrn_refln loop is reflections (-z; the input itself in --mode refine), a PDB file " + "or an mmCIF with an _atom_site loop is a model (--model), anything else is images. An mmCIF holding " + "both coordinates and reflections has to be given with --model or -z. At most one of each, also " + "counting --model/--mr and -z; gzipped or not. E.g. rugnux data_master.h5 model.cif" << std::endl; std::cout << " With a broker URL (http://host:port/) and no on disk yet, follows the broker's current " "collection while it is written: must be the path its master file will have, the data " "files are read next to it as the broker reports them closed, and the run ends with the images the " @@ -1746,6 +1753,31 @@ static int RunRugnux(int argc, char **argv, bool no_gpu) { broker_url = input_files.back(); input_files.pop_back(); } + // The inputs in any order: an atomic model and a reflection file are told from the images by their contents + // (RugnuxInputKind.h), and taken as --model and -z are - as the input of --mode refine, its reflections. + try { + RugnuxPositionalInputs split = SplitRugnuxInputs(input_files); + if (!split.model.empty()) { + if (!model_pdb.empty() || !mr_model.empty()) { + logger.Error("Two models: {} and {}; give one", model_pdb.empty() ? mr_model : model_pdb, split.model); + exit(EXIT_FAILURE); + } + model_pdb = split.model; + } + if (!split.reflections.empty()) { + if (mode == RugnuxMode::Refine) + split.images.insert(split.images.begin(), split.reflections); + else if (!ref_mtz.empty()) { + logger.Error("Two reference reflection files: {} and {}; give one", ref_mtz, split.reflections); + exit(EXIT_FAILURE); + } else + ref_mtz = split.reflections; + } + input_files = std::move(split.images); + } catch (const std::exception &e) { + logger.Error("{}", e.what()); + exit(EXIT_FAILURE); + } if (input_files.empty()) { logger.Error("Input file not specified"); print_usage(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f94e6495b..e1e3288bb 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -208,6 +208,7 @@ ADD_EXECUTABLE(jfjoch_test LoadReferenceSfCifTest.cpp WriteReflectionsTest.cpp RugnuxResultFilesTest.cpp + RugnuxInputKindTest.cpp UnitCellTest.cpp CCTest.cpp MultiLatticeSearchTest.cpp diff --git a/tests/RugnuxInputKindTest.cpp b/tests/RugnuxInputKindTest.cpp new file mode 100644 index 000000000..7eb42fc4f --- /dev/null +++ b/tests/RugnuxInputKindTest.cpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include +#include +#include +#include + +#include "../rugnux/RugnuxInputKind.h" + +namespace { + const std::string MODEL_CIF = "data_test\n#\n_cell.length_a 50.0\nloop_\n_atom_site.group_PDB\n_atom_site.id\n" + "ATOM 1\n"; + const std::string SF_CIF = "data_r1abcsf\n_cell.length_a 50.0\nloop_\n_refln.index_h\n_refln.index_k\n" + "_refln.index_l\n_refln.intensity_meas\n1 0 0 10.0\n"; + const std::string PDB = "HEADER TEST\nREMARK 1 SYNTHETIC\nCRYST1 50.000 50.000 50.000 90.00 90.00 90.00 P 1\n" + "ATOM 1 CA ALA A 1 1.000 2.000 3.000 1.00 10.00 C\nEND\n"; + + std::string Write(const std::string &name, const std::string &content) { + std::ofstream(name, std::ios::binary) << content; + return name; + } + + std::string WriteGz(const std::string &name, const std::string &content) { + gzFile f = gzopen(name.c_str(), "wb"); + gzwrite(f, content.data(), static_cast(content.size())); + gzclose(f); + return name; + } +} + +TEST_CASE("RugnuxInputKind_Classify", "[rugnux_inputs]") { + using K = RugnuxInputKind; + CHECK(ClassifyRugnuxInput(Write("ik_model.cif", MODEL_CIF)) == K::Model); + CHECK(ClassifyRugnuxInput(Write("ik_sf.cif", SF_CIF)) == K::Reflections); + CHECK(ClassifyRugnuxInput(Write("ik_diffrn.cif", "data_x\nloop_\n_diffrn_refln.index_h\n1\n")) == K::Reflections); + CHECK(ClassifyRugnuxInput(Write("ik_model.pdb", PDB)) == K::Model); + CHECK(ClassifyRugnuxInput(WriteGz("ik_model.pdb.gz", PDB)) == K::Model); + CHECK(ClassifyRugnuxInput(WriteGz("ik_sf.cif.gz", SF_CIF)) == K::Reflections); + // the name says nothing: a model in a file called .mtz, reflections in one called .pdb + CHECK(ClassifyRugnuxInput(Write("ik_model_named.mtz", MODEL_CIF)) == K::Model); + CHECK(ClassifyRugnuxInput(Write("ik_mtz_named.pdb", std::string("MTZ \0\0\0\0", 8) + "binary")) == K::Reflections); + CHECK(ClassifyRugnuxInput(Write("ik_image.h5", std::string("\x89HDF\r\n\x1a\n\0\0\0\0", 12))) == K::Images); + CHECK(ClassifyRugnuxInput(Write("ik_image.cbf", "###CBF: VERSION 1.5\ndata_image\n_array_data.data\n")) == K::Images); + CHECK(ClassifyRugnuxInput(Write("ik_image2.cbf", "data_image\n_array_data.data\n_atom_site.x\n")) == K::Images); + CHECK(ClassifyRugnuxInput(Write("ik_image.img", "{\nHEADER_BYTES= 512;\nDIM=2;\n}")) == K::Images); + CHECK(ClassifyRugnuxInput(Write("ik_binary", std::string("\0\1\2\3\4\5\6\7\0\0\0\0\0\0\0\0x", 17))) == K::Images); + // A PDB file without an atom is not a model + CHECK(ClassifyRugnuxInput(Write("ik_noatom.pdb", "HEADER TEST\nEND\n")) == K::Images); + // Both coordinates and reflections: the user has to say + CHECK(ClassifyRugnuxInput(Write("ik_both.cif", MODEL_CIF + "loop_\n_refln.index_h\n1\n")) == K::Ambiguous); + for (const char *f: {"ik_model.cif", "ik_sf.cif", "ik_diffrn.cif", "ik_model.pdb", "ik_model.pdb.gz", "ik_sf.cif.gz", + "ik_model_named.mtz", "ik_mtz_named.pdb", "ik_image.h5", "ik_image.cbf", "ik_image2.cbf", + "ik_image.img", "ik_binary", "ik_noatom.pdb", "ik_both.cif"}) + std::filesystem::remove(f); +} + +TEST_CASE("RugnuxInputKind_SplitAnyOrder", "[rugnux_inputs]") { + const std::string img = Write("ik_s_image.h5", std::string("\x89HDF\r\n\x1a\n\0\0\0\0", 12)); + const std::string model = Write("ik_s_model.pdb", PDB); + const std::string sf = Write("ik_s_sf.cif", SF_CIF); + for (const auto &order: std::vector>{{img, model, sf}, {model, sf, img}, {sf, img, model}}) { + const RugnuxPositionalInputs s = SplitRugnuxInputs(order); + CHECK(s.images == std::vector{img}); + CHECK(s.model == model); + CHECK(s.reflections == sf); + } + // image files keep their order; a path that is no file (yet) is left to the image readers + const RugnuxPositionalInputs s = SplitRugnuxInputs({"later_master.h5", model, img}); + CHECK(s.images == std::vector{"later_master.h5", img}); + + const std::string model2 = Write("ik_s_model2.cif", MODEL_CIF); + CHECK_THROWS(SplitRugnuxInputs({img, model, model2})); + CHECK_THROWS(SplitRugnuxInputs({sf, img, sf})); + const std::string both = Write("ik_s_both.cif", MODEL_CIF + "loop_\n_refln.index_h\n1\n"); + CHECK_THROWS(SplitRugnuxInputs({img, both})); + for (const auto &f: {img, model, sf, model2, both}) + std::filesystem::remove(f); +}