mx/Jungfraujoch
3
2 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 84 Wiki Activity

Merge broker-rugnux-live into rc176: rugnux in jfjoch_broker as a live thread during a DECTRIS rotation collection (off by default), the rugnux library without file I/O (results in ProcessResult::outputs, the CLI writes them through JFJochRugnuxOutput), END carrying the merged table, map coefficients and placed model

Browse Source 
Branch reviewed at e16db07ff (review: owner's decisions accepted - _maps.mtz instead of .ccp4 in the
viewer, -o "" computes everything incl. MR/validation, Ctrl+C/error paths may leave fewer files, the
writer links gemmi/zlib) and merged at 55184eb09 with its review fixes (END size guard, frame drop as
in Lite, outputs released before re-runs, pool priority, docs, frontend gating).

Conflicts: rugnux/Rugnux.h (kept rc176's host_memory_budget_bytes, output_prefix removed as on the
branch), rugnux/RugnuxPipeline.h (MtzTable future + rc176's GPUMerge pointer), MolecularReplacement.h/.cpp
(rc176's GPU cost model, packed transform and split functions with the branch's RugnuxModel signature;
search_model with rc176's other_budgets, result.model_name). tests/RugnuxLargeTest.cpp: rc176's
host-budget test writes its files through WriteRugnuxOutputs now that the library writes none.
Generated code (broker/gen, frontend client, Python client docs, Redoc) regenerated from the merged
jfjoch_api.yaml with update_version.sh (version unchanged).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4
This commit is contained in:
leonarski_fandClaude Opus 5.5 committed 2026-10-17 21:01:38 +02:00
2 parents 3e36fe9468 55184eb096
commit 78c2a69267
130 files changed
+7612 -1010
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/ModelValidationTest.cpp
+46 -52
View File
@@ -31,6 +31,7 @@
#endif
#include "../image_analysis/structure_refinement/SigmaA.h"
#include "../writer/WriteModel.h"
#include "../writer/RugnuxModelFile.h"
#include "../image_analysis/scale_merge/ReindexAmbiguity.h"
namespace {
@@ -107,6 +108,10 @@ ATOM 2 C . CB GLY A 1 12.000 14.000 16.000 1.00 20.00
f.close();
return name;
}
RugnuxModel ReadModel(const std::string &path) {
return ReadRugnuxModelFile(path);
}
}
// --model used to call gemmi::read_pdb unconditionally, so a deposited model handed over as mmCIF -
@@ -261,8 +266,8 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") {
const auto pdb = WriteTemp("model_validation_test.pdb", kPdb);
const auto cif = WriteTemp("model_validation_test.cif", kMmcif);
const auto from_pdb = ModelReferenceIntensities(pdb, {}, {}, 4.0, logger);
const auto from_cif = ModelReferenceIntensities(cif, {}, {}, 4.0, logger);
const auto from_pdb = ModelReferenceIntensities(ReadModel(pdb), {}, {}, 4.0, logger);
const auto from_cif = ModelReferenceIntensities(ReadModel(cif), {}, {}, 4.0, logger);
REQUIRE_FALSE(from_pdb.empty());
REQUIRE_FALSE(from_cif.empty());
@@ -271,7 +276,7 @@ TEST_CASE("ModelValidation_ReadsPdbAndMmcif", "[ModelValidation]") {
// The extension is not what decides: the same mmCIF under a .pdb name still reads.
const auto misnamed = WriteTemp("model_validation_test_misnamed.pdb", kMmcif);
CHECK_FALSE(ModelReferenceIntensities(misnamed, {}, {}, 4.0, logger).empty());
CHECK_FALSE(ModelReferenceIntensities(ReadModel(misnamed), {}, {}, 4.0, logger).empty());
std::filesystem::remove(pdb);
std::filesystem::remove(cif);
@@ -305,8 +310,8 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") {
const auto ncs_path = WriteTemp("model_validation_test_ncs.pdb", with_ncs);
const auto full_path = WriteTemp("model_validation_test_ncs_full.pdb", expanded);
const auto from_ncs = ModelReferenceIntensities(ncs_path, {}, {}, 4.0, logger);
const auto from_full = ModelReferenceIntensities(full_path, {}, {}, 4.0, logger);
const auto from_ncs = ModelReferenceIntensities(ReadModel(ncs_path), {}, {}, 4.0, logger);
const auto from_full = ModelReferenceIntensities(ReadModel(full_path), {}, {}, 4.0, logger);
REQUIRE(from_ncs.size() == from_full.size());
REQUIRE_FALSE(from_ncs.empty());
for (size_t i = 0; i < from_ncs.size(); i++) {
@@ -324,22 +329,22 @@ TEST_CASE("ModelValidation_ExpandsStrictNcs", "[ModelValidation]") {
TEST_CASE("ModelValidation_UnusableModelGivesAReason", "[ModelValidation]") {
Logger logger("ModelValidation_UnusableModelGivesAReason");
// A file that holds no model is refused where it is read, before anything runs.
const auto empty_file = WriteTemp("model_validation_test_bogus.pdb", "not a coordinate file\n");
CHECK_THROWS(ReadModel(empty_file));
CHECK_THROWS(ReadModel("model_validation_test_does_not_exist.pdb"));
std::filesystem::remove(empty_file);
// A model with atoms but no crystal cell to put them in.
RugnuxModel no_cell;
no_cell.name = "no_cell.pdb";
no_cell.structure.models.emplace_back(1);
const auto result = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60,
.alpha = 90, .beta = 90, .gamma = 90},
empty_file, "", logger);
no_cell, logger);
CHECK_FALSE(result.ok);
CHECK_FALSE(result.failure_reason.empty());
CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring(empty_file));
const auto missing = ValidateAgainstModel({}, UnitCell{.a = 40, .b = 50, .c = 60,
.alpha = 90, .beta = 90, .gamma = 90},
"model_validation_test_does_not_exist.pdb", "", logger);
CHECK_FALSE(missing.ok);
CHECK_FALSE(missing.failure_reason.empty());
std::filesystem::remove(empty_file);
CHECK_THAT(result.failure_reason, Catch::Matchers::ContainsSubstring("no_cell.pdb"));
}
// A rigid-body step is only worth taking if it can find a shift it was not told about. The check is
@@ -355,7 +360,7 @@ TEST_CASE("ModelValidation_RigidBodyRecoversASmallShift", "[ModelValidation]") {
// "Observed" amplitudes: the model's own structure factors, so the target's minimum is exactly
// where the model started.
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -402,7 +407,7 @@ TEST_CASE("ModelValidation_RigidBodyLeavesThePolarDirectionAlone", "[ModelValida
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -449,7 +454,7 @@ TEST_CASE("ModelValidation_RigidBodySolventStaysPhysical", "[ModelValidation]")
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -536,7 +541,7 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
// The data: the cluster's own |F|^2 in its own frame, 30 x 34 x 38, P 21 21 21.
const auto data_model = WriteTemp("cob_data_test.pdb", ClusterPdb().c_str());
auto obs = ModelReferenceIntensities(data_model, {}, {}, 2.5, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_model), {}, {}, 2.5, logger);
REQUIRE(obs.size() > 500);
for (size_t i = 0; i < obs.size(); i++) {
obs[i].F = std::sqrt(std::max(0.0f, obs[i].I));
@@ -544,7 +549,6 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
}
const UnitCell data_cell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90};
const gemmi::SpaceGroup *data_sg = gemmi::find_spacegroup_by_name("P 21 21 21");
const std::string prefix = (std::filesystem::temp_directory_path() / "cob_test").string();
// The same structure as a deposition on permuted axes, a' = b, b' = c, c' = a. The cell is
// orthogonal, so the Cartesian coordinates permute with the axes.
@@ -567,9 +571,13 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
}
const auto other_setting = WriteTemp("cob_model_test.pdb", permuted.c_str());
const auto moved = ValidateAgainstModel(obs, data_cell, other_setting, prefix, logger, data_sg,
const auto moved = ValidateAgainstModel(obs, data_cell, ReadModel(other_setting), logger, data_sg,
/*probe_indexing_ambiguity=*/false, 1, 1.0);
REQUIRE(moved.ok);
// The maps come back in memory: their coefficients always, the maps themselves only when asked for.
REQUIRE(moved.maps);
CHECK(moved.maps->coefficients.columns.size() == 12);
CHECK_FALSE(moved.maps->map_2fofc);
logger.Info("Change of basis test: took {} out of {}, R-work {:.4f}",
moved.change_of_basis_op.triplet(), moved.setting_as_read, moved.r_work);
CHECK_FALSE(moved.change_of_basis_op == gemmi::Op::identity());
@@ -578,15 +586,13 @@ TEST_CASE("ModelValidation_FindsTheDatasDescriptionOfTheLattice", "[ModelValidat
// The isomorphous case, which is what nearly every run is: the model already describes the
// lattice the way the data do, the only operators that map its cell are its own symmetry, and
// nothing is scored or moved.
const auto same = ValidateAgainstModel(obs, data_cell, data_model, prefix, logger, data_sg,
const auto same = ValidateAgainstModel(obs, data_cell, ReadModel(data_model), logger, data_sg,
false, 1, 1.0);
REQUIRE(same.ok);
CHECK(same.change_of_basis_op == gemmi::Op::identity());
CHECK(same.setting_as_read.empty());
CHECK(same.r_work < 0.15);
for (const char *suffix : {"_2fofc.ccp4", "_fofc.ccp4", "_anom.ccp4", "_maps.mtz"})
std::filesystem::remove(prefix + suffix);
std::filesystem::remove(data_model);
std::filesystem::remove(other_setting);
}
@@ -600,7 +606,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
// Point group 4 on a tetragonal lattice (4/mmm): one alternative indexing.
const auto model = WriteTemp("reidx_model_test.pdb",
ClusterPdb("CRYST1 34.000 34.000 38.000 90.00 90.00 90.00 P 4 4\n").c_str());
auto own = ModelReferenceIntensities(model, {}, {}, 2.5, logger);
auto own = ModelReferenceIntensities(ReadModel(model), {}, {}, 2.5, logger);
REQUIRE(own.size() > 500);
for (size_t i = 0; i < own.size(); i++) {
own[i].F = std::sqrt(std::max(0.0f, own[i].I));
@@ -610,12 +616,11 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
const gemmi::SpaceGroup *p4 = gemmi::find_spacegroup_by_name("P 4");
const auto laws = ReindexAmbiguityOperators(cell, *p4);
REQUIRE(laws.size() == 1);
const std::string prefix = (std::filesystem::temp_directory_path() / "reidx_test").string();
// The data as a run that picked the other indexing would have merged them.
const auto obs = ReindexReflections(own, laws.front());
const auto to_model = ValidateAgainstModel(obs, cell, model, prefix, logger, p4,
const auto to_model = ValidateAgainstModel(obs, cell, ReadModel(model), logger, p4,
/*probe_indexing_ambiguity=*/true, 1, 1.0);
REQUIRE(to_model.ok);
CHECK(to_model.change_of_basis_op == gemmi::Op::identity());
@@ -624,7 +629,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
CHECK(to_model.r_work < 0.15);
// A reference already fixed the data's indexing: the data stay, and the model is moved.
const auto fixed = ValidateAgainstModel(obs, cell, model, prefix, logger, p4, false, 1, 1.0);
const auto fixed = ValidateAgainstModel(obs, cell, ReadModel(model), logger, p4, false, 1, 1.0);
REQUIRE(fixed.ok);
CHECK_FALSE(fixed.change_of_basis_op == gemmi::Op::identity());
CHECK(fixed.indexing_op == gemmi::Op::identity());
@@ -648,7 +653,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
r.F = std::sqrt(std::max(0.0f, r.I));
}
}
const auto twin = ValidateAgainstModel(twinned, cell, model, prefix, logger, p4, true, 1, 1.0);
const auto twin = ValidateAgainstModel(twinned, cell, ReadModel(model), logger, p4, true, 1, 1.0);
REQUIRE(twin.ok);
CHECK_FALSE(twin.indexing_decided);
CHECK(twin.indexing_op == gemmi::Op::identity());
@@ -656,7 +661,7 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
// Data already in the model's indexing: nothing moves. The null is run all the same, as on every
// validation that judges its model (ModelValidationSchedule::fit_test), and the model fits.
const auto same = ValidateAgainstModel(own, cell, model, prefix, logger, p4, true, 1, 1.0);
const auto same = ValidateAgainstModel(own, cell, ReadModel(model), logger, p4, true, 1, 1.0);
REQUIRE(same.ok);
CHECK(same.change_of_basis_op == gemmi::Op::identity());
CHECK(same.indexing_op == gemmi::Op::identity());
@@ -664,8 +669,6 @@ TEST_CASE("ModelValidation_ReindexesTheDataIntoTheModelsIndexing", "[ModelValida
CHECK(same.model_fits);
CHECK(same.r_work < 0.15);
for (const char *suffix : {"_2fofc.ccp4", "_fofc.ccp4", "_anom.ccp4", "_maps.mtz"})
std::filesystem::remove(prefix + suffix);
std::filesystem::remove(model);
}
@@ -785,7 +788,7 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
Logger logger("ModelValidation_CCModelFollowsTheSignalByShell");
const auto path = WriteTemp("cc_model_test.pdb", ClusterPdb().c_str());
auto obs = ModelReferenceIntensities(path, {}, {}, 2.5, logger);
auto obs = ModelReferenceIntensities(ReadModel(path), {}, {}, 2.5, logger);
REQUIRE(obs.size() > 1000);
// The shells the correlation is reported on, coarse to fine, and the noise each one gets as a
@@ -822,11 +825,10 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
obs[i].rfree_flag = (i % 20) == 0;
}
const std::string prefix = (std::filesystem::temp_directory_path() / "cc_model_test").string();
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38,
.alpha = 90, .beta = 90, .gamma = 90},
path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(path), logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
/*probe_indexing_ambiguity=*/false, 1, 1.0, shells);
REQUIRE(result.ok);
REQUIRE(result.cc_model_shells.size() == shells.size());
@@ -854,13 +856,11 @@ TEST_CASE("ModelValidation_CCModelFollowsTheSignalByShell", "[ModelValidation]")
const auto no_shells =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38,
.alpha = 90, .beta = 90, .gamma = 90},
path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(path), logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
false, 1, 1.0);
CHECK(no_shells.ok);
CHECK(no_shells.cc_model_shells.empty());
for (const char *suffix : {"_2fofc.ccp4", "_fofc.ccp4", "_anom.ccp4", "_maps.mtz"})
std::filesystem::remove(prefix + suffix);
std::filesystem::remove(path);
}
@@ -876,22 +876,19 @@ TEST_CASE("ModelValidation_WrongModelInAGroupOffTheDataMetric", "[ModelValidatio
const auto data_path = WriteTemp("off_metric_data.pdb", ClusterPdb().c_str());
const auto wrong_path = WriteTemp("off_metric_wrong.pdb",
ClusterPdb("CRYST1 40.000 40.000 40.000 90.00 90.00 90.00 I 21 3 24\n", 7).c_str());
auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger);
REQUIRE(obs.size() > 1000);
for (size_t i = 0; i < obs.size(); i++) {
obs[i].sigma = std::max(1.0f, 0.05f * obs[i].I);
obs[i].F = std::sqrt(std::max(0.0f, obs[i].I));
obs[i].rfree_flag = (i % 20) == 0;
}
const std::string prefix = (std::filesystem::temp_directory_path() / "off_metric_test").string();
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"));
ReadModel(wrong_path), logger, gemmi::find_spacegroup_by_name("P 21 21 21"));
CHECK(result.ok);
CHECK(!result.adopted_model_enantiomorph);
for (const char *suffix : {"_2fofc.ccp4", "_fofc.ccp4", "_anom.ccp4", "_maps.mtz"})
std::filesystem::remove(prefix + suffix);
std::filesystem::remove(data_path);
std::filesystem::remove(wrong_path);
}
@@ -903,7 +900,7 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat
Logger logger("ModelValidation_CCModelUnrelatedModelConfirmsNothing");
const auto data_path = WriteTemp("cc_null_data.pdb", ClusterPdb().c_str());
const auto wrong_path = WriteTemp("cc_null_wrong.pdb", ClusterPdb(kCryst, 7).c_str());
auto obs = ModelReferenceIntensities(data_path, {}, {}, 2.0, logger);
auto obs = ModelReferenceIntensities(ReadModel(data_path), {}, {}, 2.0, logger);
std::erase_if(obs, [](const MergedReflection &r) { return r.d > 8.0f; });
REQUIRE(obs.size() > 1000);
for (size_t i = 0; i < obs.size(); i++) {
@@ -916,10 +913,9 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat
obs[i].rfree_flag = (i % 20) == 0;
}
const std::vector<float> shells{4.0f, 3.0f, 2.5f, 2.2f, 2.0f};
const std::string prefix = (std::filesystem::temp_directory_path() / "cc_null_test").string();
const auto result =
ValidateAgainstModel(obs, UnitCell{.a = 30, .b = 34, .c = 38, .alpha = 90, .beta = 90, .gamma = 90},
wrong_path, prefix, logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
ReadModel(wrong_path), logger, gemmi::find_spacegroup_by_name("P 21 21 21"),
/*probe_indexing_ambiguity=*/false, 1, 1.0, shells);
REQUIRE(result.ok);
REQUIRE(result.cc_model_shells.size() == shells.size());
@@ -934,8 +930,6 @@ TEST_CASE("ModelValidation_CCModelUnrelatedModelConfirmsNothing", "[ModelValidat
// The trap is there: on independent reflections some shell would have read as significant.
CHECK(fisher_max > 3.0);
for (const char *suffix : {"_2fofc.ccp4", "_fofc.ccp4", "_anom.ccp4", "_maps.mtz"})
std::filesystem::remove(prefix + suffix);
std::filesystem::remove(data_path);
std::filesystem::remove(wrong_path);
}
@@ -1052,7 +1046,7 @@ TEST_CASE("ModelValidation_RigidBodySameOnAnyNumberOfThreads", "[ModelValidation
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -1347,7 +1341,7 @@ namespace {
gemmi::AsuData<gemmi::ValueSigma<float>> OwnAmplitudes(const char *cryst, const gemmi::Structure &st,
double d_min, Logger &logger) {
const auto path = WriteTemp("rigid_body_own_amplitudes.pdb", ClusterPdb(cryst).c_str());
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;
fobs.unit_cell_ = st.cell;
@@ -1376,7 +1370,7 @@ namespace {
const gemmi::SpaceGroup *sg = st.find_spacegroup();
// "Observed" amplitudes: the model's own, as placed at q = 0.
const auto path = WriteTemp("rigid_body_jacobian_test.pdb", ClusterPdb(cryst).c_str());
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
REQUIRE_FALSE(ref.empty());
@@ -1504,7 +1498,7 @@ TEST_CASE("ModelValidation_RigidBodyReportsTheRotationItApplied", "[ModelValidat
const gemmi::SpaceGroup *sg = st.find_spacegroup();
REQUIRE(sg != nullptr);
st.setup_cell_images();
const auto ref = ModelReferenceIntensities(path, {}, {}, 3.0, logger);
const auto ref = ModelReferenceIntensities(ReadModel(path), {}, {}, 3.0, logger);
std::filesystem::remove(path);
REQUIRE_FALSE(ref.empty());
gemmi::AsuData<gemmi::ValueSigma<float>> fobs;