mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 84 Wiki Activity

rugnux: the model is parsed by the caller and handed in as RugnuxModel

Browse Source 
The analysis read the --model file itself, four to five times per run
(the model kind, every validation, molecular replacement, the stills
reference intensities) and located the PAE sidecar from the file name.
The library is to do no file I/O, so the model now arrives in memory:

- RugnuxModel (image_analysis/structure_refinement/RugnuxModel.h): the
  gemmi structure as read, whether it is ModelCIF, the PAE matrix where a
  prediction came with one, and the name the report gives it.
- ParseRugnuxModel / ReadRugnuxModelFile (writer/RugnuxModelFile.h, in
  JFJochRugnuxFiles): bytes in, gzip by magic, PDB/mmCIF by content,
  _ma_qa_metric detection as before; the file reader takes the PAE JSON
  from the same place PaePathFor named.
- ProcessConfig::model replaces model_path; PrepareSearchModel,
  RunMolecularReplacement, ValidateAgainstModel and
  ModelReferenceIntensities take the model; result fields name it by
  model_name.
- rugnux reads the model before the run starts and stops with an error
  where the file holds no model (it used to run and report the model as
  not validated). The viewer reads it when the job is set up.
- The broker parses an uploaded model at once and answers 400 where it is
  not one; it keeps the parsed model in memory, with no temporary
  directory or file. It takes no PAE.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-17 16:11:58 +02:00
1 parent f215300a07
commit 4bf41983ed
32 files changed
+352 -242
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/ModelValidationTest.cpp
+42 -37
View File
@@ -31,6 +31,7 @@
#endif
#include "../image_analysis/structure_refinement/SigmaA.h"
#include "../writer/WriteModel.h"
#include "../writer/RugnuxModelFile.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
namespace {
@@ -107,6 +108,10 @@ ATOM 2 C . CB GLY A 1 12.000 14.000 16.000 1.00 20.00
f.close();
return name;
}
RugnuxModel ReadModel(const std::string &path) {
return ReadRugnuxModelFile(path);
}
}
// --model used to call gemmi::read_pdb unconditionally, so a deposited model handed over as mmCIF -
@@ -212,8 +217,8 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") {
const auto pdb = WriteTemp("model_validation_test.pdb", kPdb);
const auto cif = WriteTemp("model_validation_test.cif", kMmcif);
const auto from_pdb = ModelReferenceIntensities(pdb, {}, {}, 4.0, logger);
const auto from_cif = ModelReferenceIntensities(cif, {}, {}, 4.0, logger);
const auto from_pdb = ModelReferenceIntensities(ReadModel(pdb), {}, {}, 4.0, logger);
const auto from_cif = ModelReferenceIntensities(ReadModel(cif), {}, {}, 4.0, logger);
REQUIRE_FALSE(from_pdb.empty());
REQUIRE_FALSE(from_cif.empty());
@@ -222,7 +227,7 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") {
// The extension is not what decides: the same mmCIF under a .pdb name still reads.
const auto misnamed = WriteTemp("model_validation_test_misnamed.pdb", kMmcif);
CHECK_FALSE(ModelReferenceIntensities(misnamed, {}, {}, 4.0, logger).empty());
CHECK_FALSE(ModelReferenceIntensities(ReadModel(misnamed), {}, {}, 4.0, logger).empty());
std::filesystem::remove(pdb);
std::filesystem::remove(cif);
@@ -256,8 +261,8 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") {
const auto ncs_path = WriteTemp("model_validation_test_ncs.pdb", with_ncs);
const auto full_path = WriteTemp("model_validation_test_ncs_full.pdb", expanded);
const auto from_ncs = ModelReferenceIntensities(ncs_path, {}, {}, 4.0, logger);
const auto from_full = ModelReferenceIntensities(full_path, {}, {}, 4.0, logger);
const auto from_ncs = ModelReferenceIntensities(ReadModel(ncs_path), {}, {}, 4.0, logger);
const auto from_full = ModelReferenceIntensities(ReadModel(full_path), {}, {}, 4.0, logger);
REQUIRE(from_ncs.size() == from_full.size());
REQUIRE_FALSE(from_ncs.empty());
for (size_t i = 0; i < from_ncs.size(); i++) {
@@ -275,22 +280,22 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") {
TEST_CASE("ModelValidation_UnusableModelGivesAReason", "[ModelValidation]") {
Logger logger("ModelValidation_UnusableModelGivesAReason");
// A file that holds no model is refused where it is read, before anything runs.
const auto empty_file = WriteTemp("model_validation_test_bogus.pdb", "not a coordinate file\n");
CHECK_THROWS(ReadModel(empty_file));
CHECK_THROWS(ReadModel("model_validation_test_does_not_exist.pdb"));
std::filesystem::remove(empty_file);
// A model with atoms but no crystal cell to put them in.
RugnuxModel no_cell;
no_cell.name = "no_cell.pdb";
no_cell.structure.models.emplace_back(1);
const auto result = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60,
.alpha = 90, .beta = 90, .gamma = 90},
empty_file, "", logger);
no_cell, "", logger);
CHECK_FALSE(result.ok);
CHECK_FALSE(result.failure_reason.empty());
CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring(empty_file));
const auto missing = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60,
.alpha = 90, .beta = 90, .gamma = 90},
"model_validation_test_does_not_exist.pdb", "", logger);
CHECK_FALSE(missing.ok);
CHECK_FALSE(missing.failure_reason.empty());
std::filesystem::remove(empty_file);
CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring("no_cell.pdb"));
}
// A rigid-body step is only worth taking if it can find a shift it was not told about. The check is
@@ -306,7 +311,7 @@ TEST_CASE("ModelValidation_RigidBodyRecoversASmallShift", "[ModelValidation]") {
// "Observed" amplitudes: the model's own structure factors, so the target's minimum is exactly
// where the model started.
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -353,7 +358,7 @@ TEST_CASE("ModelValidation_RigidBodyLeavesThePolarDirectionAlone", "[ModelValida
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -400,7 +405,7 @@ TEST_CASE("ModelValidation_RigidBodySolventStaysPhysical", "[ModelValidation]")
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -487,7 +492,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
// The data: the cluster's own |F|^2 in its own frame, 30 x 34 x 38, P 21 21 21.
const auto data_model = WriteTemp("cob_data_test.pdb", ClusterPdb().c_str());
auto obs = ModelReferenceIntensities(data_model, {}, {}, 2.5, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_model), {}, {}, 2.5, logger);
REQUIRE(obs.size() > 500);
for (size_t i = 0; i < obs.size(); i++) {
obs[i].F = std::sqrt(std::max(0.0f, obs[i].I));
@@ -518,7 +523,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
}
const auto other_setting = WriteTemp("cob_model_test.pdb", permuted.c_str());
const auto moved = ValidateAgainstModel(obs, data_cell, other_setting, prefix, logger, data_sg,
const auto moved = ValidateAgainstModel(obs, data_cell, ReadModel(other_setting), prefix, logger, data_sg,
/*probe_indexing_ambiguity=*/false, 1, 1.0);
REQUIRE(moved.ok);
logger.Info("Change of basis test: took {} out of {}, R-work {:.4f}",
@@ -529,7 +534,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
// The isomorphous case, which is what nearly every run is: the model already describes the
// lattice the way the data do, the only operators that map its cell are its own symmetry, and
// nothing is scored or moved.
const auto same = ValidateAgainstModel(obs, data_cell, data_model, prefix, logger, data_sg,
const auto same = ValidateAgainstModel(obs, data_cell, ReadModel(data_model), prefix, logger, data_sg,
false, 1, 1.0);
REQUIRE(same.ok);
CHECK(same.change_of_basis_op == gemmi::Op::identity());
@@ -551,7 +556,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
// Point group 4 on a tetragonal lattice (4/mmm): one alternative indexing.
const auto model = WriteTemp("reidx_model_test.pdb",
ClusterPdb("CRYST1 34.000 34.000 38.000 90.00 90.00 90.00 P 4 4\n").c_str());
auto own = ModelReferenceIntensities(model, {}, {}, 2.5, logger);
auto own = ModelReferenceIntensities(ReadModel(model), {}, {}, 2.5, logger);
REQUIRE(own.size() > 500);
for (size_t i = 0; i < own.size(); i++) {
own[i].F = std::sqrt(std::max(0.0f, own[i].I));
@@ -566,7 +571,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
// The data as a run that picked the other indexing would have merged them.
const auto obs = ReindexReflections(own, laws.front());
const auto to_model = ValidateAgainstModel(obs, cell, model, prefix, logger, p4,
const auto to_model = ValidateAgainstModel(obs, cell, ReadModel(model), prefix, logger, p4,
/*probe_indexing_ambiguity=*/true, 1, 1.0);
REQUIRE(to_model.ok);
CHECK(to_model.change_of_basis_op == gemmi::Op::identity());
@@ -575,7 +580,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
CHECK(to_model.r_work < 0.15);
// A reference already fixed the data's indexing: the data stay, and the model is moved.
const auto fixed = ValidateAgainstModel(obs, cell, model, prefix, logger, p4, false, 1, 1.0);
const auto fixed = ValidateAgainstModel(obs, cell, ReadModel(model), prefix, logger, p4, false, 1, 1.0);
REQUIRE(fixed.ok);
CHECK_FALSE(fixed.change_of_basis_op == gemmi::Op::identity());
CHECK(fixed.indexing_op == gemmi::Op::identity());
@@ -599,7 +604,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
r.F = std::sqrt(std::max(0.0f, r.I));
}
}
const auto twin = ValidateAgainstModel(twinned, cell, model, prefix, logger, p4, true, 1, 1.0);
const auto twin = ValidateAgainstModel(twinned, cell, ReadModel(model), prefix, logger, p4, true, 1, 1.0);
REQUIRE(twin.ok);
CHECK_FALSE(twin.indexing_decided);
CHECK(twin.indexing_op == gemmi::Op::identity());
@@ -607,7 +612,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
// Data already in the model's indexing: nothing moves. The null is run all the same, as on every
// validation that judges its model (ModelValidationSchedule::fit_test), and the model fits.
const auto same = ValidateAgainstModel(own, cell, model, prefix, logger, p4, true, 1, 1.0);
const auto same = ValidateAgainstModel(own, cell, ReadModel(model), prefix, logger, p4, true, 1, 1.0);
REQUIRE(same.ok);
CHECK(same.change_of_basis_op == gemmi::Op::identity());
CHECK(same.indexing_op == gemmi::Op::identity());
@@ -736,7 +741,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
Logger logger("ModelValidation_CCModelFollowsTheSignalByShell");
const auto path = WriteTemp("cc_model_test.pdb", ClusterPdb().c_str());
auto obs = ModelReferenceIntensities(path, {}, {}, 2.5, logger);
auto obs = ModelReferenceIntensities(ReadModel(path), {}, {}, 2.5, logger);
REQUIRE(obs.size() > 1000);
// The shells the correlation is reported on, coarse to fine, and the noise each one gets as a
@@ -777,7 +782,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38,
.alpha = 90, .beta = 90, .gamma = 90},
path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
/*probe_indexing_ambiguity=*/false, 1, 1.0, shells);
REQUIRE(result.ok);
REQUIRE(result.cc_model_shells.size() == shells.size());
@@ -805,7 +810,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
const auto no_shells =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38,
.alpha = 90, .beta = 90, .gamma = 90},
path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
false, 1, 1.0);
CHECK(no_shells.ok);
CHECK(no_shells.cc_model_shells.empty());
@@ -827,7 +832,7 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio
const auto data_path = WriteTemp("off_metric_data.pdb", ClusterPdb().c_str());
const auto wrong_path = WriteTemp("off_metric_wrong.pdb",
ClusterPdb("CRYST1 40.000 40.000 40.000 90.00 90.00 90.00 I 21 3 24\n", 7).c_str());
auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger);
REQUIRE(obs.size() > 1000);
for (size_t i = 0; i < obs.size(); i++) {
obs[i].sigma = std::max(1.0f, 0.05f * obs[i].I);
@@ -837,7 +842,7 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio
const std::string prefix = (std::filesystem::temp_directory_path() / "off_metric_test").string();
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"));
ReadModel(wrong_path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"));
CHECK(result.ok);
CHECK(!result.adopted_model_enantiomorph);
@@ -854,7 +859,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat
Logger logger("ModelValidation_CCModelUnrelatedModelConfirmsNothing");
const auto data_path = WriteTemp("cc_null_data.pdb", ClusterPdb().c_str());
const auto wrong_path = WriteTemp("cc_null_wrong.pdb", ClusterPdb(kCryst, 7).c_str());
auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger);
std::erase_if(obs, [](const MergedReflection &r) { return r.d > 8.0f; });
REQUIRE(obs.size() > 1000);
for (size_t i = 0; i < obs.size(); i++) {
@@ -870,7 +875,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat
const std::string prefix = (std::filesystem::temp_directory_path() / "cc_null_test").string();
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(wrong_path), prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
/*probe_indexing_ambiguity=*/false, 1, 1.0, shells);
REQUIRE(result.ok);
REQUIRE(result.cc_model_shells.size() == shells.size());
@@ -1003,7 +1008,7 @@ TEST_CASE("ModelValidation_RigidBodySameOnAnyNumberOfThreads", "[ModelValidation
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -1298,7 +1303,7 @@ namespace {
gemmi::AsuData<gemmi::ValueSigma<float>> OwnAmplitudes(const char *cryst, const gemmi::Structure &st,
double d_min, Logger &logger) {
const auto path = WriteTemp("rigid_body_own_amplitudes.pdb", ClusterPdb(cryst).c_str());
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -1327,7 +1332,7 @@ namespace {
const gemmi::SpaceGroup *sg = st.find_spacegroup();
// "Observed" amplitudes: the model's own, as placed at q = 0.
const auto path = WriteTemp("rigid_body_jacobian_test.pdb", ClusterPdb(cryst).c_str());
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
REQUIRE_FALSE(ref.empty());
@@ -1455,7 +1460,7 @@ TEST_CASE("ModelValidation_RigidBodyReportsTheRotationItApplied", "[ModelValidat
const gemmi::SpaceGroup *sg = st.find_spacegroup();
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;