// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute // SPDX-License-Identifier: GPL-3.0-only #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../common/CUDAWrapper.h" #include "../common/Logger.h" #include "../common/UnitCell.h" #include "../image_analysis/scale_merge/RfreeFlags.h" #include "../image_analysis/structure_refinement/ClashRelief.h" #include "../image_analysis/structure_refinement/ModelGrid.h" #include "../image_analysis/structure_refinement/MonomerLibrary.h" #include "../image_analysis/structure_refinement/PortableRandom.h" #include "../image_analysis/structure_refinement/Refinement.h" #include "../image_analysis/structure_refinement/RefinementADP.h" #include "../image_analysis/structure_refinement/RefinementMath.h" #include "../image_analysis/structure_refinement/RefinementMinimiser.h" #include "../image_analysis/structure_refinement/RefinementNcs.h" #include "../image_analysis/structure_refinement/RefinementRestraints.h" #include "../image_analysis/structure_refinement/RefinementXray.h" #ifdef JFJOCH_USE_CUDA #include "../image_analysis/structure_refinement/RefinementGPU.h" #include "../image_analysis/structure_refinement/RefinementNewtonGPU.h" #endif namespace { // Copies of residue `code` at the embedded dictionary's ideal coordinates (hydrogens left out), each turned and // moved to a place of its own in the cell. gemmi::Structure residue_crystal(const std::vector &codes, const gemmi::UnitCell &cell, const char *group, unsigned seed) { std::mt19937 rng(seed); std::uniform_real_distribution frac(0.0, 1.0), angle(-3.1, 3.1); gemmi::Structure st; st.cell = cell; st.spacegroup_hm = group; st.setup_cell_images(); gemmi::Model model(1); gemmi::Chain chain("A"); int seq = 1; for (const std::string &code : codes) { const gemmi::ChemComp &cc = EmbeddedMonomerLibrary().monomers.at(code); gemmi::Residue res; res.name = code; res.seqid = gemmi::SeqId(seq++, ' '); res.het_flag = 'H'; res.entity_type = gemmi::EntityType::NonPolymer; const gemmi::Position centre = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng))); const double ax = angle(rng), ay = angle(rng), az = angle(rng); const gemmi::Mat33 rot = gemmi::Mat33(1, 0, 0, 0, std::cos(ax), -std::sin(ax), 0, std::sin(ax), std::cos(ax)) .multiply(gemmi::Mat33(std::cos(ay), 0, std::sin(ay), 0, 1, 0, -std::sin(ay), 0, std::cos(ay))) .multiply(gemmi::Mat33(std::cos(az), -std::sin(az), 0, std::sin(az), std::cos(az), 0, 0, 0, 1)); for (const gemmi::ChemComp::Atom &ca : cc.atoms) { if (ca.is_hydrogen()) continue; gemmi::Atom a; a.name = ca.id; a.element = ca.el; a.pos = gemmi::Position(rot.multiply(ca.xyz)) + centre; a.b_iso = 20.0f; a.occ = 1.0f; res.atoms.push_back(a); } chain.residues.push_back(res); } model.chains.push_back(chain); st.models.push_back(model); return st; } // Atoms of a few elements scattered over the cell. gemmi::Structure random_crystal(const gemmi::UnitCell &cell, const char *group, int n, unsigned seed) { std::mt19937 rng(seed); std::uniform_real_distribution frac(0.0, 1.0), b(10.0, 40.0); const gemmi::El elements[] = {gemmi::El::C, gemmi::El::N, gemmi::El::O, gemmi::El::S}; gemmi::Structure st; st.cell = cell; st.spacegroup_hm = group; st.setup_cell_images(); gemmi::Model model(1); gemmi::Chain chain("A"); gemmi::Residue res; res.name = "UNK"; res.seqid = gemmi::SeqId(1, ' '); for (int i = 0; i < n; i++) { gemmi::Atom a; a.name = "X"; a.element = elements[i % 4]; a.pos = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng))); a.b_iso = static_cast(b(rng)); a.occ = 1.0f; res.atoms.push_back(a); } chain.residues.push_back(res); model.chains.push_back(chain); st.models.push_back(model); return st; } // Every reflection of the group's ASU to d_min, amplitude 1, one in twenty free. std::vector asu_reflections(const gemmi::UnitCell &cell, const gemmi::SpaceGroup &sg, double d_min) { std::vector out; const gemmi::ReciprocalAsu asu(&sg); const gemmi::GroupOps gops = sg.operations(); const gemmi::Miller lim = cell.get_hkl_limits(d_min); int i = 0; for (int h = -lim[0]; h <= lim[0]; h++) for (int k = -lim[1]; k <= lim[1]; k++) for (int l = -lim[2]; l <= lim[2]; l++) { const gemmi::Miller m{{h, k, l}}; if ((h == 0 && k == 0 && l == 0) || !asu.is_in(m) || gops.is_systematically_absent(m)) continue; const double d = cell.calculate_d(m); if (d < d_min) continue; MergedReflection r; r.h = h; r.k = k; r.l = l; r.d = static_cast(d); r.F = 1.0f; r.sigmaF = 0.05f; r.rfree_flag = (i++ % 20) == 0; out.push_back(r); } return out; } // The observations set to |F| of `truth`, and a row model with no scale, no solvent and the likelihood's // parameters estimated against `model`. void set_observations(RefinementEngine &engine, RefinementReflections &refl, const gemmi::Model &truth, const gemmi::Model &model) { std::vector> f; engine.Fcalc(truth, f); for (size_t m = 0; m < refl.Size(); m++) refl.fo[m] = std::abs(f[m]); engine.Fcalc(model, f); XrayRowModel rows; rows.k_total.assign(refl.Size(), 1.0f); rows.fmask.assign(refl.Size(), {0, 0}); EstimateLikelihoodParameters(refl, f, rows); engine.SetRowModel(rows); } gemmi::Model shaken(const gemmi::Model &model, double sigma, double b_sigma, unsigned seed) { std::mt19937 rng(seed); std::normal_distribution d(0, sigma), db(0, b_sigma); gemmi::Model out = model; for (gemmi::Chain &ch : out.chains) for (gemmi::Residue &r : ch.residues) for (gemmi::Atom &a : r.atoms) { a.pos += gemmi::Position(d(rng), d(rng), d(rng)); a.b_iso = static_cast(std::max(5.0, a.b_iso + db(rng))); } return out; } std::vector atoms_of(gemmi::Model &model) { std::vector out; for (gemmi::Chain &ch : model.chains) for (gemmi::Residue &r : ch.residues) for (gemmi::Atom &a : r.atoms) out.push_back(&a); return out; } // The engine's gradient of a few atoms against central differences of its own target. void check_xray_gradient(RefinementEngine &engine, gemmi::Model model, double tolerance) { std::vector> grad; engine.Evaluate(model, &grad); std::vector atoms = atoms_of(model); const double hx = 2e-3, hb = 2e-2; double max_g = 0; for (const auto &g : grad) for (int c = 0; c < 4; c++) max_g = std::max(max_g, std::fabs(g[c])); for (size_t i = 0; i < atoms.size(); i += atoms.size() / 7) { for (int c = 0; c < 4; c++) { const gemmi::Position p0 = atoms[i]->pos; const float b0 = atoms[i]->b_iso; const double h = c < 3 ? hx : hb; auto set = [&](double s) { atoms[i]->pos = p0; atoms[i]->b_iso = b0; if (c < 3) atoms[i]->pos.at(c) += s; else atoms[i]->b_iso = static_cast(b0 + s); }; set(h); const double fp = engine.Evaluate(model, nullptr); set(-h); const double fm = engine.Evaluate(model, nullptr); set(0); const double fd = (fp - fm) / (2 * h); CAPTURE(i, c, fd, grad[i][c], max_g); CHECK(std::fabs(fd - grad[i][c]) < tolerance * std::max(std::fabs(fd), 0.05 * max_g)); } } } } // namespace // The draws are pinned to values computed apart from the C++ standard library (Python's random module on a // MT19937 seeded as std::mt19937 seeds it), so every library (libstdc++, libc++, MSVC) draws the same numbers. TEST_CASE("RefinementPortableRandom_pinned", "[Refinement]") { std::mt19937 rng(1); CHECK(PortableUniform(rng) == 0.417022004702574); CHECK(PortableUniform(rng) == 0.7203244934421581); CHECK(PortableUniform(rng) == 0.00011437481734488664); std::mt19937 rng_n(1); CHECK(PortableNormal(rng_n, 0.0, 1.0) == Catch::Approx(-0.1925803402108228).epsilon(1e-14)); CHECK(PortableNormal(rng_n, 0.0, 1.0) == Catch::Approx(-0.0048841638912387455).epsilon(1e-12)); CHECK(PortableNormal(rng_n, 0.0, 1.0) == Catch::Approx(0.4712070437631509).epsilon(1e-14)); std::mt19937 rng_k(7); CHECK(PortableUniform(rng_k, -0.5, 0.5) == -0.42369171062604283); CHECK(PortableUniform(rng_k, -0.5, 0.5) == 0.2799187922401146); CHECK(PortableUniform(rng_k, -0.5, 0.5) == -0.0615907685591065); } TEST_CASE("RefinementLikelihood_derivative", "[Refinement]") { for (bool centric : {false, true}) for (double fc : {0.3, 2.0, 9.0}) { double dfc = 0, dummy = 0; const double f0 = refine_math::AmplitudeNegLogLikelihood(4.0, fc, 0.8, 2.5, centric, dfc); const double h = 1e-5; const double fp = refine_math::AmplitudeNegLogLikelihood(4.0, fc + h, 0.8, 2.5, centric, dummy); const double fm = refine_math::AmplitudeNegLogLikelihood(4.0, fc - h, 0.8, 2.5, centric, dummy); CAPTURE(centric, fc, f0); CHECK(dfc == Catch::Approx((fp - fm) / (2 * h)).epsilon(1e-5)); } // the Bessel approximation on both sides of its switch double l0, r0, l1, r1; refine_math::BesselI0I1(3.75 * (1 - 1e-12), l0, r0); refine_math::BesselI0I1(3.75 * (1 + 1e-12), l1, r1); CHECK(l0 == Catch::Approx(l1).epsilon(1e-5)); CHECK(r0 == Catch::Approx(r1).epsilon(1e-5)); } TEST_CASE("RefinementMonomerLibrary_embedded", "[Refinement]") { const gemmi::MonLib &monlib = EmbeddedMonomerLibrary(); for (const char *code : {"ALA", "PRO", "MSE", "DA", "U", "ATP", "GOL", "SO4", "HOH", "ZN", "NAG", "NDG"}) CHECK(InEmbeddedMonomerLibrary(code)); CHECK_FALSE(InEmbeddedMonomerLibrary("XYZ")); for (const char *link : {"TRANS", "CIS", "PTRANS", "p", "SS", "BETA1-4", "ALPHA1-4", "pyr-ASN"}) CHECK(monlib.links.count(link) == 1); CHECK(monlib.modifications.count("NH3") == 1); CHECK(monlib.ener_lib.atoms.count("CH1") == 1); } namespace { // A restraint dictionary of a compound the embedded one lacks, as the monomer library, AceDRG and grade write it const char *RESTRAINT_XYZ = R"(data_comp_list loop_ _chem_comp.id _chem_comp.three_letter_code _chem_comp.name _chem_comp.group XYZ XYZ "test compound" non-polymer data_comp_XYZ loop_ _chem_comp_atom.comp_id _chem_comp_atom.atom_id _chem_comp_atom.type_symbol _chem_comp_atom.type_energy _chem_comp_atom.x _chem_comp_atom.y _chem_comp_atom.z XYZ C1 C CH3 0.0 0.0 0.0 XYZ C2 C CH2 1.5 0.0 0.0 XYZ O1 O OH1 2.0 1.4 0.0 loop_ _chem_comp_bond.comp_id _chem_comp_bond.atom_id_1 _chem_comp_bond.atom_id_2 _chem_comp_bond.type _chem_comp_bond.value_dist _chem_comp_bond.value_dist_esd XYZ C1 C2 single 1.513 0.014 XYZ C2 O1 single 1.426 0.012 loop_ _chem_comp_angle.comp_id _chem_comp_angle.atom_id_1 _chem_comp_angle.atom_id_2 _chem_comp_angle.atom_id_3 _chem_comp_angle.value_angle _chem_comp_angle.value_angle_esd XYZ C1 C2 O1 109.5 1.5 data_link_list loop_ _chem_link.id _chem_link.comp_id_1 _chem_link.mod_id_1 _chem_link.group_comp_1 _chem_link.comp_id_2 _chem_link.mod_id_2 _chem_link.group_comp_2 _chem_link.name XYZ-SER SER . . XYZ . . XYZ-SER data_link_XYZ-SER loop_ _chem_link_bond.link_id _chem_link_bond.atom_1_comp_id _chem_link_bond.atom_id_1 _chem_link_bond.atom_2_comp_id _chem_link_bond.atom_id_2 _chem_link_bond.type _chem_link_bond.value_dist _chem_link_bond.value_dist_esd XYZ-SER 1 OG 2 C1 single 1.43 0.02 )"; // The same compound as a PDB Chemical Component Dictionary entry: ideal coordinates, bonds without lengths, a // planar carbonyl carbon. The model coordinates are not the ideal ones and must not be taken. const char *CCD_XYZ = R"(data_XYZ _chem_comp.id XYZ _chem_comp.type NON-POLYMER loop_ _chem_comp_atom.comp_id _chem_comp_atom.atom_id _chem_comp_atom.type_symbol _chem_comp_atom.model_Cartn_x _chem_comp_atom.model_Cartn_y _chem_comp_atom.model_Cartn_z _chem_comp_atom.pdbx_model_Cartn_x_ideal _chem_comp_atom.pdbx_model_Cartn_y_ideal _chem_comp_atom.pdbx_model_Cartn_z_ideal XYZ C1 C 9.0 9.0 9.0 0.000 0.000 0.000 XYZ C2 C 9.0 9.0 9.0 1.500 0.000 0.000 XYZ O1 O 9.0 9.0 9.0 2.110 1.057 0.000 XYZ C3 C 9.0 9.0 9.0 2.250 -1.299 0.000 loop_ _chem_comp_bond.comp_id _chem_comp_bond.atom_id_1 _chem_comp_bond.atom_id_2 _chem_comp_bond.value_order _chem_comp_bond.pdbx_aromatic_flag XYZ C1 C2 SING N XYZ C2 O1 DOUB N XYZ C2 C3 SING N )"; // One residue `code` with the atoms and positions of `cc` gemmi::Structure one_residue(const gemmi::ChemComp &cc) { gemmi::Structure st; st.cell = gemmi::UnitCell(30, 30, 30, 90, 90, 90); st.spacegroup_hm = "P 1"; st.setup_cell_images(); gemmi::Residue res; res.name = cc.name; res.seqid = gemmi::SeqId(1, ' '); res.het_flag = 'H'; res.entity_type = gemmi::EntityType::NonPolymer; for (const gemmi::ChemComp::Atom &ca : cc.atoms) { gemmi::Atom a; a.name = ca.id; a.element = ca.el; a.pos = ca.xyz + gemmi::Position(10, 10, 10); a.occ = 1; a.b_iso = 20; res.atoms.push_back(a); } gemmi::Model model(1); gemmi::Chain chain("A"); chain.residues.push_back(res); model.chains.push_back(chain); st.models.push_back(model); return st; } } // namespace TEST_CASE("RefinementRestraints_user_dictionary", "[Refinement]") { RestraintDictionary dict; ReadRestraintDictionary(RESTRAINT_XYZ, "xyz.cif", dict); REQUIRE(dict.monomers.size() == 1); const gemmi::ChemComp &xyz = dict.monomers[0]; CHECK(xyz.name == "XYZ"); CHECK(xyz.group == gemmi::ChemComp::Group::NonPolymer); CHECK(xyz.rt.bonds.size() == 2); CHECK(xyz.rt.bonds[0].value == Catch::Approx(1.513)); CHECK(xyz.rt.angles.size() == 1); CHECK(dict.from_coordinates.empty()); REQUIRE(dict.links.size() == 1); CHECK(dict.links[0].id == "XYZ-SER"); CHECK_FALSE(InEmbeddedMonomerLibrary("XYZ")); // Held fixed without the dictionary, restrained with it gemmi::Structure st = one_residue(xyz); const RefinementTopology without = BuildRefinementTopology(st); CHECK(without.fixed_residues == 1); CHECK(without.fixed_names == std::vector{"XYZ"}); const RefinementTopology with = BuildRefinementTopology(st, false, dict); CHECK(with.fixed_residues == 0); CHECK(with.restrained_residues == 1); CHECK(with.bonds == 2); CHECK(with.angles == 1); CHECK(with.monlib->links.count("XYZ-SER") == 1); // An entry of the user's replaces the embedded one of the same code; the embedded library is unchanged const std::string gol = R"(data_comp_list loop_ _chem_comp.id _chem_comp.group GOL non-polymer data_comp_GOL loop_ _chem_comp_atom.comp_id _chem_comp_atom.atom_id _chem_comp_atom.type_symbol _chem_comp_atom.type_energy GOL C1 C CH2 GOL O1 O OH1 loop_ _chem_comp_bond.comp_id _chem_comp_bond.atom_id_1 _chem_comp_bond.atom_id_2 _chem_comp_bond.type _chem_comp_bond.value_dist _chem_comp_bond.value_dist_esd GOL C1 O1 single 1.600 0.010 )"; ReadRestraintDictionary(gol, "gol.cif", dict); const gemmi::MonLib lib = MonomerLibraryWith(dict); CHECK(lib.monomers.at("GOL").atoms.size() == 2); CHECK(lib.monomers.at("GOL").rt.bonds.at(0).value == Catch::Approx(1.6)); CHECK(EmbeddedMonomerLibrary().monomers.at("GOL").atoms.size() > 2); CHECK(lib.monomers.count("XYZ") == 1); CHECK(lib.monomers.count("ALA") == 1); // A file with nothing to restrain is refused RestraintDictionary empty; CHECK_THROWS(ReadRestraintDictionary("data_x\n_cell.length_a 50\n", "x.cif", empty)); } TEST_CASE("RefinementRestraints_user_dictionary_ccd", "[Refinement]") { RestraintDictionary dict; ReadRestraintDictionary(CCD_XYZ, "XYZ.cif", dict); REQUIRE(dict.monomers.size() == 1); const gemmi::ChemComp &cc = dict.monomers[0]; CHECK(dict.from_coordinates.count("XYZ") == 1); CHECK(cc.group == gemmi::ChemComp::Group::NonPolymer); // the ideal coordinates' geometry: bonds, the three angles at C2 and the carbonyl's plane REQUIRE(cc.rt.bonds.size() == 3); CHECK(cc.rt.bonds[0].value == Catch::Approx(1.5)); CHECK(cc.rt.bonds[0].esd == Catch::Approx(DERIVED_BOND_SIGMA)); CHECK(cc.rt.angles.size() == 3); REQUIRE(cc.rt.planes.size() == 1); CHECK(cc.rt.planes[0].ids.size() == 4); CHECK(cc.rt.chirs.empty()); CHECK(cc.rt.torsions.empty()); gemmi::Structure st = one_residue(cc); const RefinementTopology topo = BuildRefinementTopology(st, false, dict); CHECK(topo.restrained_residues == 1); CHECK(topo.bonds == 3); CHECK(topo.angles == 3); CHECK(topo.planes == 1); std::vector> xyzb; for (gemmi::Atom *a : atoms_of(st.models[0])) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); const GeometryStatistics g = MeasureGeometry(topo.geometry, {}, xyzb); CHECK(g.rmsd_bonds < 1e-3); CHECK(g.rmsd_angles < 0.1); // a CCD entry without ideal coordinates restrains nothing it could stand by std::string no_ideal = CCD_XYZ; for (const char *from : {"0.000 0.000 0.000", "1.500 0.000 0.000"}) no_ideal.replace(no_ideal.find(from), std::strlen(from), "? ? ?"); RestraintDictionary bad; CHECK_THROWS(ReadRestraintDictionary(no_ideal, "XYZ.cif", bad)); } TEST_CASE("RefinementRestraints_ideal_residues", "[Refinement]") { const gemmi::UnitCell cell(60, 65, 70, 90, 90, 90); gemmi::Structure st = residue_crystal({"PHE", "DA", "ATP", "HOH", "ZN"}, cell, "P 21 21 21", 3); const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.restrained_residues == 5); CHECK(topo.fixed_residues == 0); CHECK(topo.bonds > 60); CHECK(topo.planes >= 3); CHECK(topo.chirals >= 4); std::vector> xyzb; for (gemmi::Atom *a : atoms_of(st.models[0])) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); const GeometryStatistics g = MeasureGeometry(topo.geometry, {}, xyzb); CHECK(g.rmsd_bonds < 0.02); CHECK(g.rmsd_angles < 2.5); CHECK(g.rmsd_chirals < 0.2); } // N-acetylglucosamine, beta and alpha, restrained at its dictionary geometry; and two of them joined beta-1,4 // (chitobiose) with no LINK record: the glycosidic bond is found and restrained by the library's link. TEST_CASE("RefinementRestraints_glcnac", "[Refinement]") { const gemmi::UnitCell cell(50, 55, 60, 90, 90, 90); { gemmi::Structure st = residue_crystal({"NAG", "NDG"}, cell, "P 21 21 21", 7); const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.restrained_residues == 2); CHECK(topo.fixed_residues == 0); CHECK(topo.bonds == 30); CHECK(topo.chirals == 10); // C1-C5 of each ring std::vector> xyzb; for (gemmi::Atom *a : atoms_of(st.models[0])) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); const GeometryStatistics g = MeasureGeometry(topo.geometry, {}, xyzb); CHECK(g.rmsd_bonds < 0.02); CHECK(g.rmsd_chirals < 0.2); } gemmi::Structure st = residue_crystal({"NAG", "NAG"}, cell, "P 1", 3); std::vector &res = st.models[0].chains[0].residues; // the second ring moved for its O1 to be on the first's O4, and its O1 removed: C1-O4 at 1.39 A const gemmi::Position shift = res[0].find_atom("O4", '*')->pos - res[1].find_atom("O1", '*')->pos; for (gemmi::Atom &a : res[1].atoms) a.pos += shift; std::erase_if(res[1].atoms, [](const gemmi::Atom &a) { return a.name == "O1"; }); const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.sugar_links == 1); CHECK(topo.restrained_residues == 2); std::vector atoms = atoms_of(st.models[0]); auto index = [&](const gemmi::Atom *a) { return static_cast(std::find(atoms.begin(), atoms.end(), a) - atoms.begin()); }; const int o4 = index(res[0].find_atom("O4", '*')), c1 = index(res[1].find_atom("C1", '*')); int glycosidic = 0; for (const RestraintRecord &r : topo.geometry.records) if (r.kind == static_cast(RestraintKind::Bond) && ((r.atom[0] == o4 && r.atom[1] == c1) || (r.atom[0] == c1 && r.atom[1] == o4))) { glycosidic++; CHECK(r.ideal == Catch::Approx(1.393).margin(0.01)); // the library's BETA1-4 } CHECK(glycosidic == 1); } // A GlcNAc on an Asn, both in two conformers A and B with no LINK record: the sugar links join A to A and B to B, // never the C1 of one conformer to the ND2 of the other, though that pair is within bonding distance too. TEST_CASE("RefinementRestraints_sugar_links_altloc", "[Refinement]") { const gemmi::UnitCell cell(50, 55, 60, 90, 90, 90); gemmi::Structure st = residue_crystal({"ASN", "NAG"}, cell, "P 1", 5); std::vector &res = st.models[0].chains[0].residues; // the ring moved for its C1 to be 1.43 A from ND2, and its O1 removed const gemmi::Position shift = res[0].find_atom("ND2", '*')->pos + gemmi::Position(1.43, 0, 0) - res[1].find_atom("C1", '*')->pos; for (gemmi::Atom &a : res[1].atoms) a.pos += shift; std::erase_if(res[1].atoms, [](const gemmi::Atom &a) { return a.name == "O1"; }); // conformer B of both residues 0.6 A off A: C1(A)-ND2(B) is 1.55 A for (gemmi::Residue &r : res) { std::vector atoms; for (gemmi::Atom a : r.atoms) { a.occ = 0.5f; a.altloc = 'A'; atoms.push_back(a); a.altloc = 'B'; a.pos += gemmi::Position(0, 0.6, 0); atoms.push_back(a); } r.atoms = atoms; } const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.sugar_links == 2); int same = 0, mixed = 0; for (const gemmi::Connection &c : st.connections) (c.partner1.altloc == c.partner2.altloc ? same : mixed)++; CHECK(same == 2); CHECK(mixed == 0); } // A zinc bound by four cysteines (the dictionary's own ZN-CYS link) and a magnesium by six waters (links made from // the library's metal-site statistics), both from LINK records: the bonds come out at the expected distances, // and a metal set off its site returns there under them alone. TEST_CASE("RefinementRestraints_metal_links", "[Refinement]") { const gemmi::UnitCell cell(40, 40, 40, 90, 90, 90); gemmi::Structure st = residue_crystal({"CYS", "CYS", "CYS", "CYS", "ZN", "HOH", "HOH", "HOH", "HOH", "HOH", "HOH", "MG"}, cell, "P 1", 11); double mg_o = 0, mg_sigma = 0; REQUIRE(EmbeddedMetalDistance(gemmi::El::Mg, gemmi::El::O, 6, mg_o, mg_sigma)); CHECK(mg_o == Catch::Approx(2.07).margin(0.02)); // Mg-O of an octahedral site constexpr double ZN_S = 2.34; // the ZN-CYS link std::vector &res = st.models[0].chains[0].residues; // the donors on a tetrahedron and an octahedron around their sites, each residue moved as a whole const gemmi::Position zn_site(10, 10, 10), mg_site(25, 25, 25); const double t = 1.0 / std::sqrt(3.0); const gemmi::Position tetra[4] = {{t, t, t}, {t, -t, -t}, {-t, t, -t}, {-t, -t, t}}; const gemmi::Position octa[6] = {{1, 0, 0}, {-1, 0, 0}, {0, 1, 0}, {0, -1, 0}, {0, 0, 1}, {0, 0, -1}}; auto donor_to = [](gemmi::Residue &r, const std::string &atom, const gemmi::Position &target) { const gemmi::Position shift = target - r.find_atom(atom, '*')->pos; for (gemmi::Atom &a : r.atoms) a.pos += shift; }; for (int k = 0; k < 4; k++) donor_to(res[k], "SG", zn_site + tetra[k] * ZN_S); for (int k = 0; k < 6; k++) donor_to(res[5 + k], "O", mg_site + octa[k] * mg_o); // the metals off their sites by 0.3 A res[4].atoms[0].pos = zn_site + gemmi::Position(0.2, -0.15, 0.17); res[11].atoms[0].pos = mg_site + gemmi::Position(-0.1, 0.25, 0.12); auto link = [&](int metal, int donor, const char *atom) { gemmi::Connection c; c.type = gemmi::Connection::MetalC; c.asu = gemmi::Asu::Same; c.partner1 = gemmi::AtomAddress("A", res[donor].seqid, res[donor].name, atom); c.partner2 = gemmi::AtomAddress("A", res[metal].seqid, res[metal].name, res[metal].atoms[0].name); st.connections.push_back(c); }; for (int k = 0; k < 4; k++) link(4, k, "SG"); for (int k = 0; k < 6; k++) link(11, 5 + k, "O"); const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.metal_links == 6); // the Mg-O links; Zn-S is the dictionary's std::vector atoms = atoms_of(st.models[0]); const int zn = static_cast(std::find(atoms.begin(), atoms.end(), &res[4].atoms[0]) - atoms.begin()); const int mg = static_cast(std::find(atoms.begin(), atoms.end(), &res[11].atoms[0]) - atoms.begin()); // the metals' bonds kept apart from the covalent ones, none left among them for (const RestraintRecord &r : topo.geometry.records) CHECK_FALSE((r.kind == static_cast(RestraintKind::Bond) && (r.atom[0] == zn || r.atom[1] == zn || r.atom[0] == mg || r.atom[1] == mg))); const RestraintSet &metal_bonds = topo.metal_bonds; REQUIRE(metal_bonds.records.size() == 10); for (const RestraintRecord &r : metal_bonds.records) { const bool is_zn = r.atom[0] == zn || r.atom[1] == zn; CHECK((is_zn || r.atom[0] == mg || r.atom[1] == mg)); CHECK(r.ideal == Catch::Approx(is_zn ? ZN_S : mg_o).margin(1e-4)); } // the two metals moved down the gradient of their bonds, everything else held std::vector> xyzb; for (gemmi::Atom *a : atoms) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); double e = EvaluateRestraints(metal_bonds, xyzb, nullptr), step = 1e-4; for (int it = 0; it < 2000 && step > 1e-12; it++) { std::vector> grad(xyzb.size(), {0, 0, 0, 0}), trial = xyzb; EvaluateRestraints(metal_bonds, xyzb, &grad); for (int a : {zn, mg}) for (int k = 0; k < 3; k++) trial[a][k] -= step * grad[a][k]; const double e_trial = EvaluateRestraints(metal_bonds, trial, nullptr); if (e_trial < e) { xyzb = trial; e = e_trial; step *= 1.5; } else step *= 0.5; } for (const RestraintRecord &r : metal_bonds.records) CHECK(RestraintValue(metal_bonds, r, xyzb) == Catch::Approx(r.ideal).margin(0.005)); } // A magnesium with six waters at its inner-sphere distance and a seventh at 2.9 A, all seven from LINK records: the // six become bonds, the seventh - a record at an outer-sphere distance - stays a contact, and a second topology of // the same model (a second round of refinement) finds the same, the records keeping the names they were read with. TEST_CASE("RefinementRestraints_metal_links_outer_sphere", "[Refinement]") { const gemmi::UnitCell cell(40, 40, 40, 90, 90, 90); gemmi::Structure st = residue_crystal({"HOH", "HOH", "HOH", "HOH", "HOH", "HOH", "HOH", "MG"}, cell, "P 1", 7); std::vector &res = st.models[0].chains[0].residues; const gemmi::Position site(20, 20, 20); const gemmi::Position octa[6] = {{1, 0, 0}, {-1, 0, 0}, {0, 1, 0}, {0, -1, 0}, {0, 0, 1}, {0, 0, -1}}; for (int k = 0; k < 6; k++) res[k].atoms[0].pos = site + octa[k] * 2.07; res[6].atoms[0].pos = site + gemmi::Position(1, 1, 1) * (2.9 / std::sqrt(3.0)); res[7].atoms[0].pos = site; for (int k = 0; k < 7; k++) { gemmi::Connection c; c.type = gemmi::Connection::MetalC; c.asu = gemmi::Asu::Same; c.partner1 = gemmi::AtomAddress("A", res[k].seqid, "HOH", "O"); c.partner2 = gemmi::AtomAddress("A", res[7].seqid, "MG", res[7].atoms[0].name); st.connections.push_back(c); } std::vector atoms = atoms_of(st.models[0]); auto index = [&](const gemmi::Atom &a) { return static_cast(std::find(atoms.begin(), atoms.end(), &a) - atoms.begin()); }; const int mg = index(res[7].atoms[0]), outer = index(res[6].atoms[0]); for (int round = 0; round < 2; round++) { const RefinementTopology topo = BuildRefinementTopology(st); CHECK(topo.metal_links == 6); CHECK(topo.metal_links_refused == 1); REQUIRE(topo.metal_bonds.records.size() == 6); for (const RestraintRecord &r : topo.metal_bonds.records) CHECK((r.atom[0] != outer && r.atom[1] != outer)); for (const gemmi::Connection &c : st.connections) CHECK(c.link_id.empty()); // the outer water is a contact of the metal's (listed with a margin that reaches it) const RestraintSet contacts = BuildRepulsions(topo, st.models[0], st.cell, *st.find_spacegroup(), 1.5); int found = 0; for (const RestraintRecord &r : contacts.records) found += (r.atom[0] == mg && r.atom[1] == outer) || (r.atom[0] == outer && r.atom[1] == mg); CHECK(found == 1); } } // Deposited models cut down to a ligand and the residues within 4.2 A of it (test_data/ligands, from the PDB entries // their names start with; the tRNA as its modified nucleotides and their neighbours in the chain): every ligand is // restrained, its own restraints included, the deposited geometry agrees with the dictionary, and the links the // ligands need are made - the RNA backbone to and from each modified nucleotide, the iron of a heme to its histidine // (and a heme C's to its methionine) from the LINK records, and a heme C's two thioethers to its cysteines. TEST_CASE("RefinementRestraints_deposited_ligands", "[Refinement]") { const std::vector modified_nucleotides = {"PSU", "5MC", "1MA", "OMG", "OMC", "H2U", "M2G", "7MG", "2MG", "5MU"}; const std::vector>> cases = { {"1ehz_psu", modified_nucleotides}, {"8bkh_hem", {"HEM"}}, {"6k9i_hec", {"HEC"}}, {"2w3v_ndp", {"NDP"}}, {"5vml_nap", {"NAP"}}, {"6f3p_nad", {"NAD"}}, {"8con_nai", {"NAI"}}, {"3tk0_fad", {"FAD"}}, {"7tcd_fmn", {"FMN"}}, {"6w4h_sam", {"SAM"}}, {"5epe_sah", {"SAH"}}}; for (const auto &[name, ligands] : cases) { DYNAMIC_SECTION(name) { gemmi::Structure st = gemmi::read_structure_gz("../../tests/test_data/ligands/" + name + ".pdb.gz"); const RefinementTopology topo = BuildRefinementTopology(st); std::vector atoms = atoms_of(st.models[0]); std::vector residue_of; for (const gemmi::Chain &ch : st.models[0].chains) for (const gemmi::Residue &r : ch.residues) for (size_t k = 0; k < r.atoms.size(); k++) residue_of.push_back(&r); auto index_of = [&](const gemmi::Residue &r, const char *atom) { const gemmi::Atom *a = r.find_atom(atom, '*'); return a == nullptr ? -1 : static_cast(std::find(atoms.begin(), atoms.end(), a) - atoms.begin()); }; auto bonded = [&](const RestraintSet &set, int i, int j) { for (const RestraintRecord &r : set.records) if (r.kind == static_cast(RestraintKind::Bond) && ((r.atom[0] == i && r.atom[1] == j) || (r.atom[0] == j && r.atom[1] == i))) return true; return false; }; std::map restraints, found; for (const RestraintRecord &r : topo.geometry.records) if (r.kind != static_cast(RestraintKind::Plane)) restraints[residue_of[r.atom[0]]->name]++; for (size_t i = 0; i < atoms.size(); i++) if (std::find(ligands.begin(), ligands.end(), residue_of[i]->name) != ligands.end()) { found[residue_of[i]->name]++; CHECK(topo.refinable[i]); } for (const std::string &code : ligands) { INFO(code); CHECK(found[code] > 0); CHECK(restraints[code] > 0); CHECK(std::find(topo.fixed_names.begin(), topo.fixed_names.end(), code) == topo.fixed_names.end()); } std::vector> xyzb; for (gemmi::Atom *a : atoms) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); const GeometryStatistics g = MeasureGeometry(topo.geometry, {}, xyzb, &topo.metal_bonds); CHECK(g.rmsd_bonds < 0.03); CHECK(g.rmsd_angles < 3.0); CHECK(g.rmsd_metal < 0.1); if (name == "1ehz_psu") { // each modified nucleotide bonded to the residue before it in the chain, where that one is there int links = 0; for (const gemmi::Residue &r : st.models[0].chains[0].residues) { const gemmi::Residue *prev = &r == &st.models[0].chains[0].residues.front() ? nullptr : &r - 1; if (prev == nullptr || prev->seqid.num != r.seqid.num - 1 || std::find(ligands.begin(), ligands.end(), r.name) == ligands.end()) continue; INFO(r.name << " " << *r.seqid.num); CHECK(bonded(topo.geometry, index_of(*prev, "O3'"), index_of(r, "P"))); links++; } CHECK(links >= 10); } if (name == "8bkh_hem" || name == "6k9i_hec") { const gemmi::Residue *heme = nullptr, *his = nullptr; for (const gemmi::Residue &r : st.models[0].chains[0].residues) { if (r.name == ligands[0]) heme = &r; if (r.name == "HIS" && r.seqid.num == (name == "8bkh_hem" ? 93 : 18)) his = &r; } REQUIRE(heme != nullptr); REQUIRE(his != nullptr); const int fe = index_of(*heme, "FE"); CHECK(topo.metal[fe]); CHECK(topo.metal_links == (name == "8bkh_hem" ? 1 : 2)); // the histidine; and the methionine CHECK(bonded(topo.metal_bonds, fe, index_of(*his, "NE2"))); CHECK(bonded(topo.metal_bonds, fe, index_of(*heme, "NA"))); // the porphyrin's own, among the metal's if (name == "6k9i_hec") for (auto [cys, carbon] : {std::pair{14, "CAB"}, std::pair{17, "CAC"}}) for (const gemmi::Residue &r : st.models[0].chains[0].residues) if (r.name == "CYS" && r.seqid.num == cys) CHECK(bonded(topo.geometry, index_of(r, "SG"), index_of(*heme, carbon))); } } } } TEST_CASE("RefinementRestraints_gauss_newton", "[Refinement]") { // near the ideal geometry the restraints' Hessian is their Gauss-Newton J^T J (the residuals' own curvature times // residuals that are small): v.J^T J v against central differences of the gradient along v, for random v const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); st.models[0] = shaken(st.models[0], 0.005, 0.0, 2); const RefinementTopology topo = BuildRefinementTopology(st); RestraintSet set = topo.geometry; set.Finalize(atoms_of(st.models[0]).size()); std::vector> xyzb; for (gemmi::Atom *a : atoms_of(st.models[0])) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); REQUIRE(jac.start.size() > 10); std::mt19937 rng(4); std::normal_distribution normal(0.0, 1.0); for (int trial = 0; trial < 5; trial++) { std::vector> v(xyzb.size(), {0, 0, 0, 0}); for (auto &e : v) for (int c = 0; c < 3; c++) e[c] = normal(rng); double gn = 0; for (size_t r = 0; r + 1 < jac.start.size(); r++) { double jv = 0; for (int e = jac.start[r]; e < jac.start[r + 1]; e++) for (int c = 0; c < 3; c++) jv += jac.j[e][c] * v[jac.atom[e]][c]; gn += jv * jv; } const double h = 1e-5; std::array>, 2> g; for (int side = 0; side < 2; side++) { auto x = xyzb; for (size_t i = 0; i < x.size(); i++) for (int c = 0; c < 3; c++) x[i][c] += (side == 0 ? h : -h) * v[i][c]; g[side].assign(x.size(), {0, 0, 0, 0}); EvaluateRestraints(set, x, &g[side]); } double exact = 0; for (size_t i = 0; i < xyzb.size(); i++) for (int c = 0; c < 3; c++) exact += v[i][c] * (g[0][i][c] - g[1][i][c]) / (2 * h); CAPTURE(trial, gn, exact); CHECK(gn > 0); CHECK(gn == Catch::Approx(exact).epsilon(0.1)); } } TEST_CASE("RefinementRestraints_gradient", "[Refinement]") { const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); const RefinementTopology topo = BuildRefinementTopology(st); // the geometry, the B similarity and the repulsions (symmetry mates included, at a contact distance that // makes every pair within reach count) RestraintSet set = topo.geometry; set.Append(topo.adp); RefinementTopology wide = topo; for (float &r : wide.vdw_radius) r = 2.5f; std::fill(wide.hbond.begin(), wide.hbond.end(), 0); const RestraintSet rep = BuildRepulsions(wide, st.models[0], cell, *st.find_spacegroup(), 0.0); REQUIRE(rep.records.size() > 10); bool image = false; for (const RestraintRecord &r : rep.records) image = image || r.rot[0] != 1.0f || r.tran[0] != 0.0f || r.tran[1] != 0.0f || r.tran[2] != 0.0f; CAPTURE(rep.records.size()); CHECK(image); set.Append(rep); set.Finalize(atoms_of(st.models[0]).size()); std::vector> xyzb; for (gemmi::Atom *a : atoms_of(st.models[0])) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); // each kind of restraint on its own, against central differences for (int kind = 0; kind <= static_cast(RestraintKind::BSimilarity); kind++) { RestraintSet one; for (const RestraintRecord &r : set.records) if (r.kind == kind) { RestraintRecord copy = r; if (kind == static_cast(RestraintKind::Plane)) { copy.atom[0] = static_cast(one.plane_atoms.size()); for (int k = 0; k < r.atom[1]; k++) one.plane_atoms.push_back(set.plane_atoms[r.atom[0] + k]); } one.Add(copy, kind == static_cast(RestraintKind::Plane) ? r.atom[1] : 4); } CAPTURE(kind, one.records.size()); std::vector> grad(xyzb.size(), {0, 0, 0, 0}); EvaluateRestraints(one, xyzb, &grad); const double h = 1e-6; for (size_t i = 0; i < xyzb.size(); i++) for (int c = 0; c < 4; c++) { auto x = xyzb; x[i][c] += h; const double fp = EvaluateRestraints(one, x, nullptr); x[i][c] -= 2 * h; const double fm = EvaluateRestraints(one, x, nullptr); CAPTURE(i, c); CHECK(grad[i][c] == Catch::Approx((fp - fm) / (2 * h)).epsilon(1e-4).margin(1e-3)); } } #ifdef JFJOCH_USE_CUDA if (get_gpu_count() > 0) { // the GPU's float evaluation of the same set // (the engine is made for a cell wide enough for its gather; the restraints do not read the cell) const gemmi::UnitCell wide_cell(40, 40, 40, 90, 90, 90); std::vector merged = asu_reflections(wide_cell, *st.find_spacegroup(), 3.0); RefinementReflections refl = PrepareRefinementReflections(merged, wide_cell, *st.find_spacegroup(), 3.0); auto gpu = MakeRefinementEngineGPU(0, wide_cell, *st.find_spacegroup(), 3.0, refl, st.models[0], 100.0, SIZE_MAX); REQUIRE(gpu); gpu->SetRestraints(0, set); std::vector> ggpu; const double egpu = gpu->EvaluateRestraints(0, xyzb, &ggpu); std::vector> gcpu(xyzb.size(), {0, 0, 0, 0}); const double ecpu = EvaluateRestraints(set, xyzb, &gcpu); CHECK(egpu == Catch::Approx(ecpu).epsilon(1e-4)); for (size_t i = 0; i < xyzb.size(); i++) for (int c = 0; c < 4; c++) CHECK(ggpu[i][c] == Catch::Approx(gcpu[i][c]).epsilon(2e-3).margin(2e-2)); } #endif } // Contacts with other copies of the molecule: in P1 two waters 2.4 A apart across the a face (17.6 A apart within // the cell) repel each other once, and a water 1.4 A from a 3-fold axis, 2.4 A from its images x -> T x and // x -> T^-1 x (one contact per asymmetric unit), repels its image once. TEST_CASE("RefinementRestraints_lattice_and_symmetry_contacts", "[Refinement]") { auto oxygens = [](gemmi::Structure &st, const std::vector &where) { std::vector atoms = atoms_of(st.models[0]); REQUIRE(atoms.size() == where.size()); for (size_t i = 0; i < where.size(); i++) atoms[i]->pos = st.cell.orthogonalize(where[i]); }; { const gemmi::UnitCell cell(20, 21, 22, 90, 90, 90); gemmi::Structure st = residue_crystal({"HOH", "HOH"}, cell, "P 1", 1); oxygens(st, {gemmi::Fractional(0.01, 0.5, 0.5), gemmi::Fractional(0.89, 0.5, 0.5)}); const RefinementTopology topo = BuildRefinementTopology(st); const RestraintSet rep = BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0); REQUIRE(rep.records.size() == 1); CHECK(rep.records[0].tran[0] == Catch::Approx(-20.0f)); CHECK(BuildBContacts(topo, st.models[0], cell, *st.find_spacegroup()).records.size() == 1); } { const gemmi::UnitCell cell(20, 20, 22, 90, 90, 120); gemmi::Structure st = residue_crystal({"HOH"}, cell, "P 3", 1); oxygens(st, {cell.fractionalize(gemmi::Position(1.4, 0.0, 11.0))}); const RefinementTopology topo = BuildRefinementTopology(st); const RestraintSet rep = BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0); CHECK(rep.records.size() == 1); } } // A complete model whose every atom is off by a Gaussian error of sigma_x per axis: D(s) = = // exp(-2 pi^2 sigma_x^2 s^2) (Luzzati). With a negligible measurement error D comes out at that; with an error of // half the rms amplitude of the outermost shell it falls below it there, where the error is a large part of the // spread and is counted twice, and stays where the amplitudes are strong. TEST_CASE("RefinementLikelihood_D_coordinate_error", "[Refinement]") { const char *group = "P 1"; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell(40, 45, 50, 90, 90, 90); const double d_min = 1.5, sigma_x = 0.25; gemmi::Structure truth = random_crystal(cell, group, 1500, 3); gemmi::Model model = truth.models[0]; std::mt19937 rng(4); std::normal_distribution dx(0, sigma_x); for (gemmi::Atom &a : model.chains[0].residues[0].atoms) a.pos += gemmi::Position(dx(rng), dx(rng), dx(rng)); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, d_min), cell, sg, d_min); auto cpu = MakeRefinementEngineCPU(cell, sg, d_min, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, d_min, 5.0)); std::vector> f; cpu->Fcalc(truth.models[0], f); for (size_t m = 0; m < refl.Size(); m++) refl.fo[m] = std::abs(f[m]); cpu->Fcalc(model, f); // mean D and mean Luzzati D in five shells of 1/d^2 constexpr int SHELLS = 5; const double s2_max = 1.0 / (d_min * d_min); auto shells = [&](const XrayRowModel &rows, std::vector &d, std::vector &luzzati) { std::vector n(SHELLS, 0); d.assign(SHELLS, 0); luzzati.assign(SHELLS, 0); for (size_t m = 0; m < refl.Size(); m++) { const int k = std::min(SHELLS - 1, static_cast(refl.inv_d2[m] / s2_max * SHELLS)); d[k] += rows.d[m]; luzzati[k] += std::exp(-2 * M_PI * M_PI * sigma_x * sigma_x * refl.inv_d2[m]); n[k]++; } for (int k = 0; k < SHELLS; k++) { d[k] /= n[k]; luzzati[k] /= n[k]; } }; XrayRowModel rows; rows.k_total.assign(refl.Size(), 1.0f); rows.fmask.assign(refl.Size(), {0, 0}); for (float &s : refl.sigma) s = 0.01f; EstimateLikelihoodParameters(refl, f, rows); std::vector d_clean, luzzati; shells(rows, d_clean, luzzati); for (int k = 0; k < SHELLS; k++) { CAPTURE(k, d_clean[k], luzzati[k]); CHECK(std::fabs(d_clean[k] - luzzati[k]) < 0.05); } CHECK(luzzati[SHELLS - 1] < 0.65); // a measurement error of half the rms amplitude of the outermost shell double fo2 = 0; int n = 0; for (size_t m = 0; m < refl.Size(); m++) if (refl.inv_d2[m] > s2_max * (SHELLS - 1) / SHELLS) { fo2 += double(refl.fo[m]) * refl.fo[m]; n++; } for (float &s : refl.sigma) s = static_cast(0.5 * std::sqrt(fo2 / n)); EstimateLikelihoodParameters(refl, f, rows); std::vector d_noisy; shells(rows, d_noisy, luzzati); CHECK(d_noisy[0] > d_clean[0] - 0.03); CHECK(d_noisy[SHELLS - 1] < d_clean[SHELLS - 1] - 0.1); } TEST_CASE("RefinementXray_gradient_CPU", "[Refinement]") { for (const char *group : {"P 1", "P 21 21 21", "C 1 2 1", "P 61 2 2"}) { CAPTURE(group); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell = sg.crystal_system() == gemmi::CrystalSystem::Hexagonal ? gemmi::UnitCell(30, 30, 45, 90, 90, 120) : gemmi::UnitCell(28, 31, 34, 90, sg.number == 5 ? 104 : 90, 90); gemmi::Structure truth = random_crystal(cell, group, 240, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, 2.2), cell, sg, 2.2); REQUIRE(refl.Size() > 500); auto cpu = MakeRefinementEngineCPU(cell, sg, 2.2, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 2.2, 5.0)); set_observations(*cpu, refl, truth.models[0], model); check_xray_gradient(*cpu, model, 0.03); } } #ifdef JFJOCH_USE_CUDA TEST_CASE("RefinementNewton_product_GPU_vs_CPU", "[Refinement]") { if (get_gpu_count() == 0) return; const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); const RefinementTopology topo = BuildRefinementTopology(st); RestraintSet set = topo.geometry; set.Append(topo.adp); set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0)); const std::vector atoms = atoms_of(st.models[0]); set.Finalize(atoms.size()); std::vector> xyzb; for (gemmi::Atom *a : atoms) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); // every atom but each fifth a parameter: the rest drop out std::vector params, param_of(atoms.size(), -1); for (int a = 0; a < static_cast(atoms.size()); a++) if (a % 5 != 4) { param_of[a] = static_cast(params.size()); params.push_back(a); } const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); REQUIRE(jac.start.size() > 100); std::mt19937 gen(3); std::uniform_real_distribution uniform(-1, 1); // a random v, and a smooth one - the same shift of every atom and a thousandth of it at random, J v then the small // difference of large terms for (const auto [xyz, smooth] : {std::pair{true, false}, {true, true}, {false, false}, {false, true}}) { CAPTURE(xyz, smooth); const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3; const size_t n = per * params.size(); std::vector v(n); for (size_t i = 0; i < n; i++) v[i] = smooth ? 0.3 + 0.2 * (i % per) + 1e-3 * uniform(gen) : uniform(gen); // J^T J v and the diagonal on the CPU, each entry in turn std::vector hv(n, 0.0), diagonal(n, 0.0); for (size_t r = 0; r + 1 < jac.start.size(); r++) { double t = 0; for (int e = jac.start[r]; e < jac.start[r + 1]; e++) if (const int k = param_of[jac.atom[e]]; k >= 0) for (int c = 0; c < per; c++) t += jac.j[e][c0 + c] * v[per * k + c]; for (int e = jac.start[r]; e < jac.start[r + 1]; e++) if (const int k = param_of[jac.atom[e]]; k >= 0) for (int c = 0; c < per; c++) hv[per * k + c] += t * jac.j[e][c0 + c]; } for (size_t e = 0; e < jac.atom.size(); e++) if (const int k = param_of[jac.atom[e]]; k >= 0) { const std::array &j = jac.j[e]; for (int c = 0; c < per; c++) diagonal[per * k + c] += j[c0 + c] * j[c0 + c]; } RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); gpu.At(set, xyzb); std::vector hv_gpu; gpu.Multiply(v, hv_gpu); const std::vector diagonal_gpu = gpu.Diagonal(); REQUIRE(hv_gpu.size() == n); REQUIRE(diagonal_gpu.size() == n); // In float, from float positions: within a few float epsilons of the double product in norm (in refinement // its error is 2e-7 on average, 7e-6 at most), each element of the diagonal relative to itself. double num = 0, den = 0, worst_diagonal = 0; for (size_t i = 0; i < n; i++) { num += (hv_gpu[i] - hv[i]) * (hv_gpu[i] - hv[i]); den += hv[i] * hv[i]; if (diagonal[i] > 0) worst_diagonal = std::max(worst_diagonal, std::fabs(diagonal_gpu[i] - diagonal[i]) / diagonal[i]); } CAPTURE(std::sqrt(num / den), worst_diagonal); CHECK(std::sqrt(num / den) < 1e-5); CHECK(worst_diagonal < 1e-5); } } // The conjugate gradients on the card against ConjugateGradients on the CPU, for H = J^T J + diag(h) preconditioned by // blocks of H: two atoms' x y z per block for the positions, each B alone. TEST_CASE("RefinementNewton_solve_GPU_vs_CPU", "[Refinement]") { if (get_gpu_count() == 0) return; const gemmi::UnitCell cell(14, 15, 16, 90, 90, 90); gemmi::Structure st = residue_crystal({"TYR", "DG"}, cell, "P 21 21 21", 5); st.models[0] = shaken(st.models[0], 0.15, 3.0, 2); const RefinementTopology topo = BuildRefinementTopology(st); RestraintSet set = topo.geometry; set.Append(topo.adp); set.Append(BuildRepulsions(topo, st.models[0], cell, *st.find_spacegroup(), 0.0)); const std::vector atoms = atoms_of(st.models[0]); set.Finalize(atoms.size()); std::vector> xyzb; for (gemmi::Atom *a : atoms) xyzb.push_back({a->pos.x, a->pos.y, a->pos.z, a->b_iso}); std::vector params, param_of(atoms.size(), -1); for (int a = 0; a < static_cast(atoms.size()); a++) if (a % 5 != 4) { param_of[a] = static_cast(params.size()); params.push_back(a); } const RestraintJacobian jac = RestraintGaussNewton(set, xyzb); std::mt19937 gen(5); std::uniform_real_distribution uniform(-1, 1); for (const bool xyz : {true, false}) { CAPTURE(xyz); const int per = xyz ? 3 : 1, c0 = xyz ? 0 : 3; const size_t n = per * params.size(); // H in full, the X-ray diagonal small beside the stiffest restraints Eigen::MatrixXd hm = Eigen::MatrixXd::Zero(n, n); for (size_t r = 0; r + 1 < jac.start.size(); r++) for (int e1 = jac.start[r]; e1 < jac.start[r + 1]; e1++) for (int e2 = jac.start[r]; e2 < jac.start[r + 1]; e2++) { const int k1 = param_of[jac.atom[e1]], k2 = param_of[jac.atom[e2]]; if (k1 >= 0 && k2 >= 0) for (int a = 0; a < per; a++) for (int c = 0; c < per; c++) hm(per * k1 + a, per * k2 + c) += jac.j[e1][c0 + a] * jac.j[e2][c0 + c]; } std::vector h(n), b(n); for (size_t i = 0; i < n; i++) { h[i] = xyz ? 10 + 5 * uniform(gen) : 0.01 + 0.005 * uniform(gen); hm(i, i) += h[i]; b[i] = uniform(gen); } const int block = xyz ? 6 : 1; std::vector start{0}, element; std::vector inverse; std::vector> llt; for (size_t first = 0; first < n; first += block) { const int m = static_cast(std::min(block, n - first)); llt.emplace_back(hm.block(first, first, m, m)); const Eigen::MatrixXd w = llt.back().solve(Eigen::MatrixXd::Identity(m, m)); for (int i = 0; i < m; i++) { element.push_back(static_cast(first) + i); for (int k = 0; k <= i; k++) inverse.push_back(w(i, k)); } start.push_back(static_cast(element.size())); } auto product = [&](const std::vector &v, std::vector &hv) { const Eigen::VectorXd out = hm * Eigen::Map(v.data(), n); hv.assign(out.data(), out.data() + n); }; auto precondition = [&](const std::vector &r, std::vector &z) { z.resize(n); for (size_t g = 0; g < llt.size(); g++) { const int m = start[g + 1] - start[g]; const Eigen::VectorXd zg = llt[g].solve(Eigen::Map(r.data() + start[g], m)); std::copy(zg.data(), zg.data() + m, z.begin() + start[g]); } }; std::vector x; ConjugateGradients(product, precondition, b, x, 20, 0.0); RestraintNormalMatrixGPU gpu(0, params, atoms.size(), xyz); gpu.At(set, xyzb); gpu.SetSolve(h, start, element, inverse); std::vector x_gpu, x_again; gpu.Solve(b, x_gpu, 20); gpu.Solve(b, x_again, 20); REQUIRE(x_gpu.size() == n); CHECK(x_gpu == x_again); // a fixed order: the same bits every time double num = 0, den = 0; for (size_t i = 0; i < n; i++) { num += (x_gpu[i] - x[i]) * (x_gpu[i] - x[i]); den += x[i] * x[i]; } CAPTURE(std::sqrt(num / den)); CHECK(std::sqrt(num / den) < 1e-5); } } TEST_CASE("RefinementXray_gradient_GPU", "[Refinement]") { if (get_gpu_count() == 0) return; for (const char *group : {"P 1", "P 21 21 21", "C 1 2 1", "P 61 2 2"}) { CAPTURE(group); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell = sg.crystal_system() == gemmi::CrystalSystem::Hexagonal ? gemmi::UnitCell(30, 30, 45, 90, 90, 120) : gemmi::UnitCell(28, 31, 34, 90, sg.number == 5 ? 104 : 90, 90); gemmi::Structure truth = random_crystal(cell, group, 240, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, 2.2), cell, sg, 2.2); auto cpu = MakeRefinementEngineCPU(cell, sg, 2.2, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 2.2, 5.0)); set_observations(*cpu, refl, truth.models[0], model); // the GPU engine sees the same observations: it is made after they are set auto gpu = MakeRefinementEngineGPU(0, cell, sg, 2.2, refl, model, 300.0, SIZE_MAX); REQUIRE(gpu); gpu->SetBlur(RefinementBlur(cell, sg, 2.2, 5.0)); std::vector> f; cpu->Fcalc(model, f); XrayRowModel rows; rows.k_total.assign(refl.Size(), 1.0f); rows.fmask.assign(refl.Size(), {0, 0}); EstimateLikelihoodParameters(refl, f, rows); cpu->SetRowModel(rows); gpu->SetRowModel(rows); std::vector> fg; gpu->Fcalc(model, fg); double num = 0, den = 0; for (size_t m = 0; m < f.size(); m++) { num += std::abs(f[m] - fg[m]); den += std::abs(f[m]); } CHECK(num / den < 1e-3); std::vector> gc, gg; const double tc = cpu->Evaluate(model, &gc); const double tg = gpu->Evaluate(model, &gg); CHECK(tg == Catch::Approx(tc).epsilon(1e-4)); double max_g = 0; for (const auto &g : gc) for (int c = 0; c < 4; c++) max_g = std::max(max_g, std::fabs(g[c])); for (size_t i = 0; i < gc.size(); i++) for (int c = 0; c < 4; c++) { CAPTURE(i, c); CHECK(std::fabs(gg[i][c] - gc[i][c]) < 0.01 * max_g); } check_xray_gradient(*gpu, model, 0.05); } } #endif TEST_CASE("RefinementMinimiser_quadratic", "[Refinement]") { // f = sum a_k (x_k - 1)^2 with curvatures spread over two decades const size_t n = 50; std::vector a(n); for (size_t k = 0; k < n; k++) a[k] = std::pow(100.0, static_cast(k) / (n - 1)); const RefinementMinimiser::Target target = [&](const std::vector &x, double &f, std::vector &g) { f = 0; for (size_t k = 0; k < n; k++) { f += a[k] * (x[k] - 1) * (x[k] - 1); g[k] = 2 * a[k] * (x[k] - 1); } return true; }; auto distance = [&](const std::vector &x) { double d = 0; for (double v : x) d = std::max(d, std::fabs(v - 1)); return d; }; // the exact inverse Hessian: one step RefinementMinimiser exact; exact.SetInverseHessian([&](const std::vector &, std::vector &q) { for (size_t k = 0; k < n; k++) q[k] /= 2 * a[k]; }); std::vector x(n, 0.0); exact.Minimise(target, x, 1, 0.0, 10.0); CHECK(distance(x) < 1e-9); // with the identity, the memory carried from one call to the next does what one long call does RefinementMinimiser split, whole; const RefinementMinimiser::InverseHessian identity = [](const std::vector &, std::vector &) {}; split.SetInverseHessian(identity); whole.SetInverseHessian(identity); std::vector xs(n, 0.0), xw(n, 0.0); for (int call = 0; call < 6; call++) split.Minimise(target, xs, 10, 0.0, 10.0); whole.Minimise(target, xw, 60, 0.0, 10.0); CHECK(distance(xw) < 1e-3); for (size_t k = 0; k < n; k++) CHECK(xs[k] == Catch::Approx(xw[k]).margin(1e-12)); } TEST_CASE("RefinementMinimiser_conjugate_gradients", "[Refinement]") { // a tridiagonal positive definite system, solved to the tolerance with and without a (Jacobi) preconditioner const size_t n = 200; auto product = [&](const std::vector &v, std::vector &hv) { hv.assign(n, 0.0); for (size_t k = 0; k < n; k++) { hv[k] = (2.0 + 0.01 * k) * v[k]; if (k > 0) hv[k] -= v[k - 1]; if (k + 1 < n) hv[k] -= v[k + 1]; } }; std::vector b(n); for (size_t k = 0; k < n; k++) b[k] = std::sin(0.1 * k); auto residual = [&](const std::vector &x) { std::vector hx; product(x, hx); double r = 0, b2 = 0; for (size_t k = 0; k < n; k++) { r += (hx[k] - b[k]) * (hx[k] - b[k]); b2 += b[k] * b[k]; } return std::sqrt(r / b2); }; std::vector x; ConjugateGradients(product, [](const std::vector &r, std::vector &z) { z = r; }, b, x, 500, 1e-8); CHECK(residual(x) < 1e-8); const int jacobi = ConjugateGradients(product, [&](const std::vector &r, std::vector &z) { z.resize(n); for (size_t k = 0; k < n; k++) z[k] = r[k] / (2.0 + 0.01 * k); }, b, x, 500, 0.1); CHECK(residual(x) < 0.1); CHECK(jacobi < 50); } TEST_CASE("Refinement_restrained_recovers_model", "[Refinement]") { const gemmi::UnitCell cell(36, 40, 44, 90, 90, 90); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 21 21 21"); gemmi::Structure truth = residue_crystal({"PHE", "TRP", "DA", "DG", "TYR", "HOH", "HOH", "HOH"}, cell, "P 21 21 21", 11); gemmi::Structure start = truth; start.models[0] = shaken(truth.models[0], 0.25, 5.0, 12); std::vector merged = asu_reflections(cell, sg, 1.8); { RefinementReflections refl = PrepareRefinementReflections(merged, cell, sg, 1.8 * 0.999); auto cpu = MakeRefinementEngineCPU(cell, sg, 1.8 * 0.999, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 1.8, 5.0)); std::vector> f; cpu->Fcalc(truth.models[0], f); std::map amplitude; for (size_t m = 0; m < refl.Size(); m++) amplitude[refl.hkl[m]] = std::abs(f[m]); std::mt19937 rng(3); std::normal_distribution noise(0.0, 0.03); for (MergedReflection &r : merged) { const auto it = amplitude.find(gemmi::Miller{{r.h, r.k, r.l}}); r.F = it == amplitude.end() ? NAN : static_cast(it->second * (1 + noise(rng))); r.sigmaF = 0.03f * r.F + 0.01f; } } Logger logger("RefinementTest"); for (bool gpu : {false, true}) { if (gpu && get_gpu_count() == 0) continue; CAPTURE(gpu); RefinementSettings settings; settings.gpu = gpu; settings.nthreads = 4; settings.weight_ladder = {0.0625}; settings.adp_ladder = {4.0}; settings.max_steps = 2; settings.final_ladder = {1.0}; settings.riding_hydrogens = false; // the data were made without them settings.solvent = true; const RefinementResult r = RefineModel(start, merged, cell, sg, 1.0, logger, settings); REQUIRE(r.ok); CHECK(r.on_gpu == gpu); CHECK(r.restrained_residues == 8); CHECK(r.r_free < 0.6 * r.r_free_start); CHECK(r.geometry.rmsd_bonds < r.geometry_start.rmsd_bonds); double rms = 0; std::vector refined = atoms_of(r.model->models[0]); std::erase_if(refined, [](const gemmi::Atom *a) { return a->is_hydrogen(); }); // the riding hydrogens const std::vector ref = atoms_of(truth.models[0]); REQUIRE(refined.size() == ref.size()); // the solvent build's placeholders not among them for (const gemmi::Chain &ch : r.model->models[0].chains) CHECK(ch.name != "~"); for (size_t i = 0; i < ref.size(); i++) rms += refined[i]->pos.dist_sq(ref[i]->pos); rms = std::sqrt(rms / ref.size()); CHECK(rms < 0.15); // the map coefficients as amplitude and phase: FWT, FWT_no_fill and DELFWT non-negative, their phases in // [0, 360); FWT filled where there is no FP, the other two missing there REQUIRE(r.maps); const auto &columns = r.maps->coefficients.columns; REQUIRE(columns[6].label == "FWT"); REQUIRE(columns[8].label == "FWT_no_fill"); REQUIRE(columns[10].label == "DELFWT"); for (size_t c : {6, 8, 10}) for (size_t i = 0; i < columns[c].values.size(); i++) { if (c != 6 && std::isnan(columns[3].values[i])) { CHECK(std::isnan(columns[c].values[i])); continue; } CHECK(columns[c].values[i] >= 0.0f); CHECK(columns[c + 1].values[i] >= 0.0f); CHECK(columns[c + 1].values[i] < 360.0f); } // one trial at a time instead of three side by side (sharing a card's tables, mask and scale fit): the same // model, bit for bit RefinementSettings one = settings; one.max_trials_at_once = 1; const RefinementResult s = RefineModel(start, merged, cell, sg, 1.0, logger, one); REQUIRE(s.ok); CHECK(s.r_free == r.r_free); // the same trials kept, those run ahead where the slots take them left out CHECK(s.trials.size() == r.trials.size()); CHECK(s.adp_trials.size() == r.adp_trials.size()); CHECK(s.adp_model == r.adp_model); const std::vector a_side = atoms_of(r.model->models[0]), a_one = atoms_of(s.model->models[0]); REQUIRE(a_one.size() == a_side.size()); size_t same = 0; for (size_t i = 0; i < a_one.size(); i++) same += a_one[i]->pos.x == a_side[i]->pos.x && a_one[i]->pos.y == a_side[i]->pos.y && a_one[i]->pos.z == a_side[i]->pos.z && a_one[i]->b_iso == a_side[i]->b_iso; CHECK(same == a_one.size()); // and the maps and what is read off them after it CHECK(s.mean_fom == r.mean_fom); CHECK(s.difference_map_rms == r.difference_map_rms); REQUIRE(s.blobs.size() == r.blobs.size()); for (size_t i = 0; i < s.blobs.size(); i++) { CHECK(s.blobs[i].centroid.dist(r.blobs[i].centroid) == 0.0); CHECK(s.blobs[i].score == r.blobs[i].score); } REQUIRE(s.solvent); REQUIRE(r.solvent); CHECK(s.solvent->size() == r.solvent->size()); // cut short before the first weight trial: the model as placed comes back, no trial chosen const std::atomic cancel{true}; settings.cancel = &cancel; const RefinementResult c = RefineModel(start, merged, cell, sg, 1.0, logger, settings); REQUIRE(c.ok); CHECK(c.cancelled); CHECK_FALSE(c.restrained); CHECK(c.trials.empty()); CHECK(c.model != nullptr); CHECK_FALSE(r.cancelled); } } // Two copies of a tRNA: with a slot for every ladder trial, the NCS rungs run ahead at both B weights and the // intermediate TLS cuts with the others, those not needed left out; one trial at a time runs only what is needed. // The same trials kept and the same model, bit for bit. TEST_CASE("Refinement_trials_run_ahead", "[Refinement]") { gemmi::Structure truth = gemmi::read_structure_gz("../../tests/test_data/ligands/1ehz_psu.pdb.gz"); truth.models.resize(1); gemmi::remove_hydrogens(truth.models[0]); truth.cell = gemmi::UnitCell(120, 120, 120, 90, 90, 90); truth.spacegroup_hm = "P 1"; truth.setup_cell_images(); gemmi::Chain copy = truth.models[0].chains.at(0); copy.name = "B"; const double a = 0.7; const gemmi::Mat33 rot(std::cos(a), -std::sin(a), 0, std::sin(a), std::cos(a), 0, 0, 0, 1); for (gemmi::Residue &r : copy.residues) for (gemmi::Atom &at : r.atoms) at.pos = gemmi::Position(rot.multiply(at.pos)) + gemmi::Position(40, 10, -5); truth.models[0].chains.push_back(copy); truth.connections.clear(); // the LINK records name chain A only gemmi::Structure start = truth; start.models[0] = shaken(truth.models[0], 0.3, 5.0, 21); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 1"); const double d_min = 5.0; std::vector merged = asu_reflections(truth.cell, sg, d_min); { RefinementReflections refl = PrepareRefinementReflections(merged, truth.cell, sg, d_min * 0.999); auto cpu = MakeRefinementEngineCPU(truth.cell, sg, d_min * 0.999, refl, 8); cpu->SetBlur(RefinementBlur(truth.cell, sg, d_min, 5.0)); std::vector> f; cpu->Fcalc(truth.models[0], f); std::map amplitude; for (size_t m = 0; m < refl.Size(); m++) amplitude[refl.hkl[m]] = std::abs(f[m]); for (MergedReflection &r : merged) { const auto it = amplitude.find(gemmi::Miller{{r.h, r.k, r.l}}); r.F = it == amplitude.end() ? NAN : static_cast(it->second); r.sigmaF = 0.03f * r.F + 0.01f; } } Logger logger("RefinementTest"); RefinementSettings settings; settings.gpu = false; settings.nthreads = 8; settings.weight_ladder = {1.0 / 128, 1.0 / 16}; settings.macro_cycles = 1; settings.max_steps = 1; settings.final_ladder = {1.0}; settings.riding_hydrogens = false; settings.solvent = false; const RefinementResult ahead = RefineModel(start, merged, truth.cell, sg, 1.0, logger, settings); RefinementSettings one = settings; one.max_trials_at_once = 1; const RefinementResult needed = RefineModel(start, merged, truth.cell, sg, 1.0, logger, one); REQUIRE(ahead.ok); REQUIRE(needed.ok); CHECK(ahead.ncs_copies == 2); CHECK(ahead.trials.size() == needed.trials.size()); CHECK(ahead.adp_trials.size() == needed.adp_trials.size()); CHECK(ahead.adp_model == needed.adp_model); CHECK(ahead.r_free == needed.r_free); std::vector x = atoms_of(ahead.model->models[0]), y = atoms_of(needed.model->models[0]); REQUIRE(x.size() == y.size()); size_t same = 0; for (size_t i = 0; i < x.size(); i++) same += x[i]->pos.x == y[i]->pos.x && x[i]->pos.y == y[i]->pos.y && x[i]->pos.z == y[i]->pos.z && x[i]->b_iso == y[i]->b_iso; CHECK(same == x.size()); } // A model whose group is a proper subgroup of the data's: a P2(1) crystal whose two residues are pseudo-symmetric by // the 2-fold along a, its data merged in P2(1)2(1)2(1). Refined in its own group against the data expanded to it, the // model scores as it does in the validation; refined in the data's group its contents are overlaid and it does not. TEST_CASE("Refinement_model_in_a_subgroup_of_the_data", "[Refinement]") { const gemmi::UnitCell cell(36, 40, 44, 90, 90, 90); const gemmi::SpaceGroup &sub = *gemmi::find_spacegroup_by_name("P 1 21 1"); const gemmi::SpaceGroup &super = *gemmi::find_spacegroup_by_name("P 21 21 21"); CHECK(ProperSubgroup(sub.operations(), super.operations())); CHECK(!ProperSubgroup(super.operations(), sub.operations())); CHECK(!ProperSubgroup(sub.operations(), sub.operations())); CHECK(!ProperSubgroup(gemmi::find_spacegroup_by_name("C 1 2 1")->operations(), super.operations())); // the translations count, on any origin of the subgroup const gemmi::GroupOps p21212 = gemmi::find_spacegroup_by_name("P 21 21 2")->operations(); CHECK(ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 21 1")->operations(), p21212)); CHECK(ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 1 2")->operations(), p21212)); CHECK(!ProperSubgroup(gemmi::find_spacegroup_by_name("P 1 2 1")->operations(), p21212)); gemmi::Structure truth = residue_crystal({"TRP"}, cell, "P 1 21 1", 5); { // the residue's mate by the 2-fold along a, exact: the crystal is then P 2 2(1) 1 (pseudo-P2(1)2(1)2(1)) gemmi::Residue mate = truth.models[0].chains[0].residues[0]; mate.seqid = gemmi::SeqId(2, ' '); for (gemmi::Atom &a : mate.atoms) { const gemmi::Fractional f = cell.fractionalize(a.pos); a.pos = cell.orthogonalize(gemmi::Fractional(f.x, -f.y, -f.z)); } truth.models[0].chains[0].residues.push_back(mate); } const double d_min = 2.0; // the data: the amplitudes of the truth in its own group, merged in the supergroup (each of its reflections the // root mean square of the amplitudes it makes equivalent) std::map amplitude; { std::vector own = asu_reflections(cell, sub, d_min); RefinementReflections refl = PrepareRefinementReflections(own, cell, sub, d_min * 0.999); auto cpu = MakeRefinementEngineCPU(cell, sub, d_min * 0.999, refl, 4); cpu->SetBlur(RefinementBlur(cell, sub, d_min, 5.0)); std::vector> f; cpu->Fcalc(truth.models[0], f); for (size_t m = 0; m < refl.Size(); m++) amplitude[refl.hkl[m]] = std::abs(f[m]); } const gemmi::GroupOps super_ops = super.operations(), sub_ops = sub.operations(); const gemmi::ReciprocalAsu sub_asu(&sub); std::vector merged = asu_reflections(cell, super, d_min); for (MergedReflection &r : merged) { double sum = 0; int n = 0; for (const gemmi::Op &op : super_ops.sym_ops) { const gemmi::Miller h = sub_asu.to_asu(op.apply_to_hkl({{r.h, r.k, r.l}}), sub_ops).first; if (const auto it = amplitude.find(h); it != amplitude.end()) { sum += it->second * it->second; n++; } } r.F = n > 0 ? static_cast(std::sqrt(sum / n)) : NAN; r.sigmaF = 0.03f * r.F + 0.01f; } std::erase_if(merged, [](const MergedReflection &r) { return !std::isfinite(r.F); }); // the expansion: every reflection of the subgroup's set comes from one of the supergroup's, with its amplitude // and its flag, and the supergroup's set is reached from it const std::vector expanded = ExpandToSubgroup(merged, super, sub); CHECK(expanded.size() > 1.5 * merged.size()); CHECK(expanded.size() <= 2 * merged.size()); const gemmi::ReciprocalAsu super_asu(&super); std::map source; for (const MergedReflection &r : merged) source[{{r.h, r.k, r.l}}] = &r; std::set distinct; size_t consistent = 0; for (const MergedReflection &e : expanded) { distinct.insert({{e.h, e.k, e.l}}); const auto it = source.find(super_asu.to_asu({{e.h, e.k, e.l}}, super_ops).first); consistent += it != source.end() && it->second->F == e.F && it->second->rfree_flag == e.rfree_flag; } CHECK(distinct.size() == expanded.size()); CHECK(consistent == expanded.size()); Logger logger("RefinementTest"); RefinementSettings settings; settings.gpu = false; settings.nthreads = 4; settings.score_only = true; settings.riding_hydrogens = false; const RefinementResult own_group = RefineModel(truth, merged, cell, super, 1.0, logger, settings); REQUIRE(own_group.ok); CHECK(own_group.space_group == "P 1 21 1"); CHECK(own_group.n_work + own_group.n_free == static_cast(expanded.size())); CHECK(own_group.r_free_start < 0.1); gemmi::Structure as_data = truth; as_data.spacegroup_hm = "P 21 21 21"; const RefinementResult data_group = RefineModel(as_data, merged, cell, super, 1.0, logger, settings); REQUIRE(data_group.ok); CHECK(data_group.space_group == "P 21 21 21"); CHECK(data_group.r_free_start > 0.3); } // A K+ on the 4-fold axis of P4 at occupancy 0.25 (its four coinciding copies one ion), 0.2 A off the axis in the // model as given, with a water 2.8 A from it: the topology puts the ion on the axis, its contact with the water is // listed once (not once per copy) and none with itself; refinement keeps it on the axis at the occupancy given, // while the rest of the model moves. TEST_CASE("Refinement_special_position", "[Refinement]") { const gemmi::UnitCell cell(30, 30, 40, 90, 90, 90); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 4"); gemmi::Structure truth = residue_crystal({"PHE", "TRP", "TYR", "HOH", "HOH", "K"}, cell, "P 4", 5); std::vector &res = truth.models[0].chains[0].residues; const gemmi::Position axis(0, 0, 20); res[5].atoms[0].pos = axis; res[5].atoms[0].occ = 0.25f; res[4].atoms[0].pos = axis + gemmi::Position(2.8, 0, 0.3); gemmi::Structure start = truth; start.models[0] = shaken(truth.models[0], 0.25, 5.0, 6); gemmi::Atom &k_start = start.models[0].chains[0].residues[5].atoms[0]; k_start.pos = axis + gemmi::Position(0.2, 0, 0.1); { gemmi::Structure st = start; const std::vector sp = FindSpecialPositions(st); REQUIRE(sp.size() == 1); CHECK(sp[0].multiplicity == 4); const RefinementTopology topo = BuildRefinementTopology(st); const std::vector atoms = atoms_of(st.models[0]); const int k = static_cast(atoms.size()) - 1, water = k - 1; REQUIRE(topo.special.size() == 1); CHECK(topo.special[0].atom == k); CHECK(atoms[k]->pos.x == Catch::Approx(0).margin(1e-9)); CHECK(atoms[k]->pos.y == Catch::Approx(0).margin(1e-9)); CHECK(atoms[k]->pos.z == Catch::Approx(20.1)); const RestraintSet rep = BuildRepulsions(topo, st.models[0], cell, sg, 1.5); int with_water = 0, with_itself = 0; for (const RestraintRecord &r : rep.records) { with_water += (r.atom[0] == k && r.atom[1] == water) || (r.atom[0] == water && r.atom[1] == k); with_itself += r.atom[0] == k && r.atom[1] == k; } CHECK(with_water == 1); CHECK(with_itself == 0); // the water's neighbours: the ion once, not four times gemmi::UnitCell images = cell; images.set_cell_images_from_spacegroup(&sg); gemmi::NeighborSearch ns(st.models[0], images, 5.0); ns.populate(false); int ion = 0; for (const ModelNeighbour &n : AtomsNear(ns, st.models[0], atoms[water]->pos, 4.0)) ion += n.residue == 5; CHECK(ion == 1); } std::vector merged = asu_reflections(cell, sg, 1.8); { RefinementReflections refl = PrepareRefinementReflections(merged, cell, sg, 1.8 * 0.999); auto cpu = MakeRefinementEngineCPU(cell, sg, 1.8 * 0.999, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 1.8, 5.0)); std::vector> f; cpu->Fcalc(truth.models[0], f); std::map amplitude; for (size_t m = 0; m < refl.Size(); m++) amplitude[refl.hkl[m]] = std::abs(f[m]); for (MergedReflection &r : merged) { const auto it = amplitude.find(gemmi::Miller{{r.h, r.k, r.l}}); r.F = it == amplitude.end() ? NAN : static_cast(it->second); r.sigmaF = 0.03f * r.F + 0.01f; } } Logger logger("RefinementTest"); RefinementSettings settings; settings.gpu = false; settings.nthreads = 4; settings.weight_ladder = {0.0625}; settings.adp_ladder = {4.0}; settings.max_steps = 2; settings.final_ladder = {1.0}; settings.riding_hydrogens = false; settings.solvent = false; const RefinementResult r = RefineModel(start, merged, cell, sg, 1.0, logger, settings); REQUIRE(r.ok); REQUIRE(r.restrained); const gemmi::Atom &k = r.model->models[0].chains[0].residues[5].atoms[0]; CHECK(std::hypot(k.pos.x, k.pos.y) < 1e-6); CHECK(k.pos.z == Catch::Approx(20).margin(0.15)); CHECK(k.occ == 0.25f); CHECK(r.r_free < 0.6 * r.r_free_start); } // Single-atom ions scatter as the ions they are (10 electrons for Mg2+, 18 for Cl-), every other atom as the // neutral atom whatever charge the file gives it; the CPU and the GPU engines score such a model alike. TEST_CASE("Refinement_ion_form_factors", "[Refinement]") { CHECK(IonFormFactor(gemmi::El::Mg, 2).calculate_sf(0.0f) == Catch::Approx(10.0).margin(0.02)); CHECK(IonFormFactor(gemmi::El::Mg, 0).calculate_sf(0.0f) == Catch::Approx(12.0).margin(0.02)); CHECK(IonFormFactor(gemmi::El::Cl, -1).calculate_sf(0.0) == Catch::Approx(18.0).margin(0.02)); CHECK(IonFormFactor(gemmi::El::N, 1).calculate_sf(0.0f) == Catch::Approx(7.0).margin(0.02)); // none tabulated const gemmi::UnitCell cell(30, 32, 34, 90, 90, 90); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 21 21 21"); gemmi::Structure st = residue_crystal({"LYS", "TRP", "MG", "CL", "HOH"}, cell, "P 21 21 21", 3); for (gemmi::Atom &a : st.models[0].chains[0].residues[0].atoms) if (a.name == "NZ") a.charge = 1; // as a file may give it std::vector merged = asu_reflections(cell, sg, 2.0); std::mt19937 rng(4); std::uniform_real_distribution f(10.0f, 100.0f); for (MergedReflection &r : merged) { r.F = f(rng); r.sigmaF = 1.0f; } Logger logger("RefinementTest"); RefinementSettings settings; settings.score_only = true; settings.nthreads = 4; settings.gpu = false; const RefinementResult cpu = RefineModel(st, merged, cell, sg, 1.0, logger, settings); REQUIRE(cpu.ok); for (const gemmi::Residue &r : cpu.model->models[0].chains[0].residues) for (const gemmi::Atom &a : r.atoms) CHECK(static_cast(a.charge) == (r.name == "MG" ? 2 : r.name == "CL" ? -1 : 0)); if (get_gpu_count() > 0) { settings.gpu = true; const RefinementResult gpu = RefineModel(st, merged, cell, sg, 1.0, logger, settings); REQUIRE(gpu.ok); CHECK(gpu.on_gpu); CHECK(gpu.r_work_start == Catch::Approx(cpu.r_work_start).margin(1e-4)); } } // A model missing one anomalous scatterer, a zinc, and one water: the zinc's blob in the difference map reads a strong // peak in the anomalous difference map (|F+| - |F-| on the model's phases less 90 degrees), the water's none. TEST_CASE("Refinement_blob_anomalous_peak", "[Refinement]") { const gemmi::UnitCell cell(30, 32, 34, 90, 90, 90); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 21 21 21"); gemmi::Structure truth = residue_crystal({"TRP", "LYS", "ARG", "TYR"}, cell, "P 21 21 21", 5); const gemmi::Structure start = truth; // a site 4 A from every atom and every symmetry mate, the new atom's own among them std::mt19937 rng(6); std::uniform_real_distribution frac(0.0, 1.0); auto add_free_site = [&](const char *name, gemmi::El el) { for (;;) { const gemmi::Position p = cell.orthogonalize(gemmi::Fractional(frac(rng), frac(rng), frac(rng))); gemmi::NeighborSearch ns(truth.models[0], truth.cell, 5.0); ns.populate(); if (ns.find_nearest_atom(p, 4.0) != nullptr || truth.cell.find_nearest_image(p, p, gemmi::Asu::Different).dist_sq < 16.0) continue; gemmi::Residue res; res.name = name; res.seqid = gemmi::SeqId(static_cast(truth.models[0].chains[0].residues.size()) + 1, ' '); res.het_flag = 'H'; gemmi::Atom a; a.name = name; a.element = el; a.pos = p; a.b_iso = 20.0f; a.occ = 1.0f; res.atoms.push_back(a); truth.models[0].chains[0].residues.push_back(res); return a; } }; const gemmi::Atom zn = add_free_site("ZN", gemmi::El::Zn); const gemmi::Atom water = add_free_site("O", gemmi::El::O); // F(h) of the truth with the zinc's f'' (3 e) added, i f'' times its structure factor, and F(-h) std::vector merged = asu_reflections(cell, sg, 2.0); gemmi::UnitCell images = cell; images.set_cell_images_from_spacegroup(&sg); gemmi::StructureFactorCalculator> calc(images); for (MergedReflection &r : merged) { const gemmi::Miller h{{r.h, r.k, r.l}}; const std::complex f = calc.calculate_sf_from_model(truth.models[0], h); const std::complex fpp = calc.calculate_sf_from_atom_sf(images.fractionalize(zn.pos), zn, h, 3.0); const std::complex i(0, 1); r.F_plus = static_cast(std::abs(f + i * fpp)); r.F_minus = static_cast(std::abs(std::conj(f) + i * std::conj(fpp))); r.F = 0.5f * (r.F_plus + r.F_minus); r.sigmaF = 0.05f; } Logger logger("Refinement_blob_anomalous_peak"); RefinementSettings settings; settings.score_only = true; settings.nthreads = 4; settings.gpu = false; const RefinementResult r = RefineModel(start, merged, cell, sg, 1.0, logger, settings); REQUIRE(r.ok); REQUIRE(r.anomalous_map); const DensityBlob *at_zn = nullptr, *at_water = nullptr; for (const DensityBlob &b : r.blobs) { const double to_zn = images.find_nearest_image(zn.pos, b.centroid, gemmi::Asu::Any).dist(); const double to_water = images.find_nearest_image(water.pos, b.centroid, gemmi::Asu::Any).dist(); if (to_zn < 1.0 && at_zn == nullptr) at_zn = &b; if (to_water < 1.0 && at_water == nullptr) at_water = &b; } REQUIRE(at_zn != nullptr); REQUIRE(at_water != nullptr); INFO(fmt::format("zinc: anomalous {:.1f} sigma, water: {:.1f} sigma", at_zn->anomalous_sigma, at_water->anomalous_sigma)); CHECK(at_zn->anomalous_sigma > 8.0); CHECK(at_zn->kind == DensityBlob::Kind::Ion); CHECK(std::abs(at_water->anomalous_sigma) < 3.0); } // Two copies of one RNA chain in the asymmetric unit, the second turned and moved, a fragment of it and the chain in // another conformation: the two found as copies of one molecule, the fragment and the other conformation not, held // to each other by local distance restraints whose gradient is right on the CPU and the GPU, and let go where the // copies really differ. A single copy has none. TEST_CASE("RefinementNcs_two_copies", "[Refinement]") { gemmi::Structure st = gemmi::read_structure_gz("../../tests/test_data/ligands/1ehz_psu.pdb.gz"); st.models.resize(1); gemmi::remove_hydrogens(st.models[0]); st.cell = gemmi::UnitCell(120, 120, 120, 90, 90, 90); st.spacegroup_hm = "P 1"; st.setup_cell_images(); { gemmi::Structure single = st; const RefinementTopology topo = BuildRefinementTopology(single); CHECK(FindNcsCopies(single.models[0], topo).Empty()); } gemmi::Chain copy = st.models[0].chains.at(0); copy.name = "B"; const double a = 0.7; const gemmi::Mat33 rot(std::cos(a), -std::sin(a), 0, std::sin(a), std::cos(a), 0, 0, 0, 1); for (gemmi::Residue &r : copy.residues) for (gemmi::Atom &at : r.atoms) at.pos = gemmi::Position(rot.multiply(at.pos)) + gemmi::Position(40, 10, -5); st.models[0].chains.push_back(copy); // and a fragment of the molecule, its first 18 residues: not a copy of the whole gemmi::Chain fragment = st.models[0].chains.at(0); fragment.name = "C"; fragment.residues.resize(18); for (gemmi::Residue &r : fragment.residues) for (gemmi::Atom &at : r.atoms) at.pos += gemmi::Position(-30, 40, 20); st.models[0].chains.push_back(fragment); // and the whole molecule in another conformation, each residue moved 3 A its own way: the same sequence, not a copy gemmi::Chain other = st.models[0].chains.at(0); other.name = "D"; std::mt19937 shift_rng(11); std::uniform_real_distribution shift(-1.0, 1.0); for (gemmi::Residue &r : other.residues) { const gemmi::Vec3 v = gemmi::Vec3(shift(shift_rng), shift(shift_rng), shift(shift_rng)).normalized() * 3.0; for (gemmi::Atom &at : r.atoms) at.pos += gemmi::Position(v) + gemmi::Position(30, -40, 25); } st.models[0].chains.push_back(other); st.connections.clear(); // the LINK records name chain A only const RefinementTopology topo = BuildRefinementTopology(st); const NcsCopies ncs = FindNcsCopies(st.models[0], topo); REQUIRE_FALSE(ncs.Empty()); CHECK(ncs.groups == 1); CHECK(ncs.copies == 2); CHECK(ncs.description == "A=B"); CHECK(ncs.anchors.size() == 1); CHECK(ncs.anchors[0].size() > 25); CAPTURE(ncs.distances.records.size()); CHECK(ncs.distances.records.size() > 1000); std::vector atoms = atoms_of(st.models[0]); std::vector> xyzb; for (gemmi::Atom *at : atoms) xyzb.push_back({at->pos.x, at->pos.y, at->pos.z, at->b_iso}); RestraintSet set = NcsRestraintsAt(ncs, 0.1); set.Finalize(xyzb.size()); // the same molecule twice: nothing to restrain CHECK(EvaluateRestraints(set, xyzb, nullptr) == Catch::Approx(0.0).margin(1e-6)); CHECK(NcsRmsd(ncs, xyzb) < 1e-3); CHECK(NcsReleasedResidues(ncs, xyzb) == 0); // the second copy shaken: the gradient against central differences std::mt19937 rng(7); std::normal_distribution d(0, 0.1); const size_t half = gemmi::count_atom_sites(st.models[0].chains[0]); for (size_t i = half; i < 2 * half; i++) for (int c = 0; c < 3; c++) xyzb[i][c] += d(rng); std::vector> grad(xyzb.size(), {0, 0, 0, 0}); const double e = EvaluateRestraints(set, xyzb, &grad); CHECK(e > 0); const double h = 1e-6; for (size_t i = 0; i < xyzb.size(); i += 17) for (int c = 0; c < 3; c++) { auto x = xyzb; x[i][c] += h; const double fp = EvaluateRestraints(set, x, nullptr); x[i][c] -= 2 * h; const double fm = EvaluateRestraints(set, x, nullptr); CAPTURE(i, c); CHECK(grad[i][c] == Catch::Approx((fp - fm) / (2 * h)).epsilon(1e-4).margin(1e-3)); } #ifdef JFJOCH_USE_CUDA if (get_gpu_count() > 0) { const gemmi::SpaceGroup &p1 = *gemmi::find_spacegroup_by_name("P 1"); const gemmi::UnitCell wide_cell(40, 40, 40, 90, 90, 90); std::vector merged = asu_reflections(wide_cell, p1, 3.0); RefinementReflections refl = PrepareRefinementReflections(merged, wide_cell, p1, 3.0); auto gpu = MakeRefinementEngineGPU(0, wide_cell, p1, 3.0, refl, st.models[0], 100.0, SIZE_MAX); REQUIRE(gpu); gpu->SetRestraints(0, set); std::vector> ggpu; const double egpu = gpu->EvaluateRestraints(0, xyzb, &ggpu); CHECK(egpu == Catch::Approx(e).epsilon(1e-3)); for (size_t i = 0; i < xyzb.size(); i++) for (int c = 0; c < 3; c++) CHECK(ggpu[i][c] == Catch::Approx(grad[i][c]).epsilon(2e-3).margin(2e-2)); } #endif // one residue of the second copy (its 16th) distorted, each atom moved by 1.5 A in a direction of its own: its // restraints let go, the others hold size_t first = gemmi::count_atom_sites(st.models[0].chains[0]); for (int k = 0; k < 15; k++) first += st.models[0].chains[1].residues[k].atoms.size(); std::vector> moved_xyzb = xyzb; std::uniform_real_distribution u(-1.0, 1.0); for (size_t i = first; i < first + st.models[0].chains[1].residues[15].atoms.size(); i++) { const gemmi::Vec3 v = gemmi::Vec3(u(rng), u(rng), u(rng)).normalized() * 1.5; moved_xyzb[i][0] += v.x; moved_xyzb[i][1] += v.y; moved_xyzb[i][2] += v.z; } CHECK(NcsReleasedResidues(ncs, xyzb) == 0); CHECK(NcsReleasedResidues(ncs, moved_xyzb) >= 1); CHECK(NcsReleasedResidues(ncs, moved_xyzb) <= 3); } namespace { // Every other atom anisotropic, its dev random but traceless, up to a third of its B. std::vector random_aniso(const gemmi::Model &model, unsigned seed) { std::mt19937 rng(seed); std::uniform_real_distribution d(-1.0, 1.0); std::vector out; int i = 0; for (const gemmi::Chain &ch : model.chains) for (const gemmi::Residue &r : ch.residues) for (const gemmi::Atom &a : r.atoms) { AtomAniso an; an.on = (i++ % 2) == 0; if (an.on) { const double u = a.b_iso / gemmi::u_to_b() / 3.0; double v[6]; for (double &x : v) x = u * d(rng); const double t = (v[0] + v[1] + v[2]) / 3.0; an.dev = {static_cast(v[0] - t), static_cast(v[1] - t), static_cast(v[2] - t), static_cast(v[3] / 2), static_cast(v[4] / 2), static_cast(v[5] / 2)}; } out.push_back(an); } return out; } // The engine's U gradient of a few anisotropic atoms against central differences of its own target. void check_u_gradient(RefinementEngine &engine, const gemmi::Model &model, std::vector &aniso, double tolerance) { std::vector> grad; engine.SetAniso(&aniso); engine.Evaluate(model, &grad); const std::vector> gu = engine.UGradient(); REQUIRE(gu.size() == aniso.size()); double max_g = 0; for (size_t i = 0; i < gu.size(); i++) for (int c = 0; c < 6; c++) max_g = std::max(max_g, std::fabs(gu[i][c])); const double h = 2e-3; // A^2 for (size_t i = 0; i < aniso.size(); i += 2 * (aniso.size() / 14)) { REQUIRE(aniso[i].on); for (int c = 0; c < 6; c++) { const float v0 = aniso[i].dev[c]; aniso[i].dev[c] = static_cast(v0 + h); const double fp = engine.Evaluate(model, nullptr); aniso[i].dev[c] = static_cast(v0 - h); const double fm = engine.Evaluate(model, nullptr); aniso[i].dev[c] = v0; const double fd = (fp - fm) / (2 * h); CAPTURE(i, c, fd, gu[i][c], max_g); CHECK(std::fabs(fd - gu[i][c]) < tolerance * std::max(std::fabs(fd), 0.05 * max_g)); } } } } // namespace TEST_CASE("RefinementXray_aniso_gradient_CPU", "[Refinement]") { for (const char *group : {"P 1", "P 21 21 21"}) { CAPTURE(group); const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell(28, 31, 34, 90, 90, 90); gemmi::Structure truth = random_crystal(cell, group, 240, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, 2.0), cell, sg, 2.0); auto cpu = MakeRefinementEngineCPU(cell, sg, 2.0, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 2.0, 5.0)); set_observations(*cpu, refl, truth.models[0], model); std::vector aniso = random_aniso(model, 9); check_u_gradient(*cpu, model, aniso, 0.06); // the positions and the B along the isotropic direction, the shapes held check_xray_gradient(*cpu, model, 0.03); } } #ifdef JFJOCH_USE_CUDA TEST_CASE("RefinementXray_aniso_gradient_GPU", "[Refinement]") { if (get_gpu_count() == 0) return; const char *group = "P 21 21 21"; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell(28, 31, 34, 90, 90, 90); gemmi::Structure truth = random_crystal(cell, group, 240, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, 2.0), cell, sg, 2.0); auto cpu = MakeRefinementEngineCPU(cell, sg, 2.0, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 2.0, 5.0)); set_observations(*cpu, refl, truth.models[0], model); auto gpu = MakeRefinementEngineGPU(0, cell, sg, 2.0, refl, model, 300.0, SIZE_MAX); REQUIRE(gpu); gpu->SetBlur(RefinementBlur(cell, sg, 2.0, 5.0)); std::vector> f; std::vector aniso = random_aniso(model, 9); cpu->SetAniso(&aniso); gpu->SetAniso(&aniso); cpu->Fcalc(model, f); XrayRowModel rows; rows.k_total.assign(refl.Size(), 1.0f); rows.fmask.assign(refl.Size(), {0, 0}); EstimateLikelihoodParameters(refl, f, rows); cpu->SetRowModel(rows); gpu->SetRowModel(rows); std::vector> fg; gpu->Fcalc(model, fg); double num = 0, den = 0; for (size_t m = 0; m < f.size(); m++) { num += std::abs(f[m] - fg[m]); den += std::abs(f[m]); } CHECK(num / den < 1e-3); std::vector> gc, gg; cpu->Evaluate(model, &gc); const std::vector> uc = cpu->UGradient(); gpu->Evaluate(model, &gg); const std::vector> ug = gpu->UGradient(); REQUIRE(ug.size() == uc.size()); double max_g = 0, max_u = 0; for (size_t i = 0; i < gc.size(); i++) { for (int c = 0; c < 4; c++) max_g = std::max(max_g, std::fabs(gc[i][c])); for (int c = 0; c < 6; c++) max_u = std::max(max_u, std::fabs(uc[i][c])); } for (size_t i = 0; i < gc.size(); i++) { CAPTURE(i); for (int c = 0; c < 4; c++) CHECK(std::fabs(gg[i][c] - gc[i][c]) < 0.01 * max_g); for (int c = 0; c < 6; c++) CHECK(std::fabs(ug[i][c] - uc[i][c]) < 0.01 * max_u); } check_u_gradient(*gpu, model, aniso, 0.08); } // The GPU engine's evaluation with the gradient, timed on a protein-sized case: about 5500 atoms in the asymmetric // unit of a P 21 21 21 cell at 1.6 A, the density of a protein's atoms with their hydrogens. Run by hand, under a // profiler for the time of each kernel. TEST_CASE("RefinementGPU_Benchmark", "[.][refine_bench]") { if (get_gpu_count() == 0) return; const char *group = "P 21 21 21"; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell(60, 64, 69, 90, 90, 90); const double d_min = 1.6; gemmi::Structure truth = random_crystal(cell, group, 5500, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, d_min), cell, sg, d_min); auto gpu = MakeRefinementEngineGPU(0, cell, sg, d_min, refl, model, 300.0, SIZE_MAX); REQUIRE(gpu); gpu->SetBlur(RefinementBlur(cell, sg, d_min, 5.0)); set_observations(*gpu, refl, truth.models[0], model); std::vector> g; gpu->Evaluate(model, &g); const int n = 200; const auto start = std::chrono::steady_clock::now(); double t = 0; for (int i = 0; i < n; i++) t += gpu->Evaluate(model, &g); const double ms = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); printf("RefinementGPU_Benchmark: %zu atoms, %zu reflections, %.3f ms per evaluation (target %.6g)\n", atoms_of(model).size(), refl.Size(), ms / n, t / n); } #endif // RefinementGPU_Benchmark's crystal on the CPU, with every atom isotropic and then with half of them anisotropic: // the time of an evaluation with the gradient, and the target and the sum of the gradient, to compare builds by. // Run by hand: jfjoch_test "[refine_bench]". TEST_CASE("RefinementCPU_Benchmark", "[.][refine_bench]") { const char *group = "P 21 21 21"; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name(group); const gemmi::UnitCell cell(60, 64, 69, 90, 90, 90); const double d_min = 1.6; gemmi::Structure truth = random_crystal(cell, group, 5500, 7); gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 8); RefinementReflections refl = PrepareRefinementReflections(asu_reflections(cell, sg, d_min), cell, sg, d_min); const std::vector aniso = random_aniso(model, 9); for (const size_t threads : {12, 48}) for (const bool with_aniso : {false, true}) { auto cpu = MakeRefinementEngineCPU(cell, sg, d_min, refl, threads); cpu->SetBlur(RefinementBlur(cell, sg, d_min, 5.0)); set_observations(*cpu, refl, truth.models[0], model); cpu->SetAniso(with_aniso ? &aniso : nullptr); std::vector> g; cpu->Evaluate(model, &g); const int n = 20; const auto start = std::chrono::steady_clock::now(); double t = 0; for (int i = 0; i < n; i++) t += cpu->Evaluate(model, &g); const double ms = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); double sum = 0; for (const std::array &a : g) sum += a[0] + a[1] + a[2] + a[3]; printf("RefinementCPU_Benchmark: %zu threads, %s, %.1f ms per evaluation (target %.12g, gradient sum %.12g)\n", threads, with_aniso ? "half anisotropic" : "isotropic", ms / n, t / n, sum); } } TEST_CASE("RefinementADP_tls_gradient", "[Refinement]") { std::mt19937 rng(4); std::uniform_real_distribution d(-1.0, 1.0); gemmi::Model model(1); gemmi::Chain chain("A"); gemmi::Residue res; for (int i = 0; i < 30; i++) { gemmi::Atom a; a.pos = gemmi::Position(10 + 8 * d(rng), -3 + 8 * d(rng), 5 + 8 * d(rng)); res.atoms.push_back(a); } chain.residues.push_back(res); model.chains.push_back(chain); std::vector atoms = atoms_of(model); TlsBody body; body.origin = gemmi::Position(9, -2, 4); std::array p{}; for (double &x : p) x = 0.01 * d(rng); body.Set(p); for (int i = 0; i < 30; i++) body.atoms.push_back(i); // U is linear in the parameters: U(p) is sum p_k dU/dp_k for (gemmi::Atom *a : atoms) { const gemmi::SMat33 u = body.U(a->pos); const auto dk = body.Derivatives(a->pos); const double uu[6] = {u.u11, u.u22, u.u33, u.u12, u.u13, u.u23}; for (int c = 0; c < 6; c++) { double sum = 0; for (int k = 0; k < TlsBody::PARAMETERS; k++) sum += p[k] * dk[k][c]; CHECK(sum == Catch::Approx(uu[c]).margin(1e-12)); } } // the chain rule in matrix form against the sum over dU/dp std::vector> gu(atoms.size()); for (auto &g : gu) for (double &x : g) x = d(rng); const std::array g = body.Gradient(atoms, gu); for (int k = 0; k < TlsBody::PARAMETERS; k++) { double sum = 0; for (size_t i = 0; i < atoms.size(); i++) { const auto dk = body.Derivatives(atoms[i]->pos); for (int c = 0; c < 6; c++) sum += gu[i][c] * dk[k][c]; } CAPTURE(k); CHECK(g[k] == Catch::Approx(sum).margin(1e-9)); } // and through the metadata in degrees and back gemmi::Model m2 = model; const std::vector groups = TlsBodiesToGroups(m2, {body}); REQUIRE(groups.size() == 1); const std::vector back = TlsBodiesFromModel(m2, groups); REQUIRE(back.size() == 1); CHECK(back[0].atoms.size() == 30); const gemmi::SMat33 u0 = body.U(atoms[3]->pos), u1 = back[0].U(atoms[3]->pos); CHECK(u1.u12 == Catch::Approx(u0.u12).margin(1e-12)); CHECK(u1.u33 == Catch::Approx(u0.u33).margin(1e-12)); } TEST_CASE("RefinementADP_place_tls_groups", "[Refinement]") { std::mt19937 rng(6); std::uniform_real_distribution d(-1.0, 1.0); // A model of two chains, three residues of ten atoms each, a TLS group on each chain. gemmi::Structure model; model.models.emplace_back(1); for (const char *name : {"A", "B"}) { gemmi::Chain chain(name); for (int r = 1; r <= 3; r++) { gemmi::Residue res; res.name = "ALA"; res.seqid = gemmi::SeqId(r, ' '); for (int i = 0; i < 10; i++) { gemmi::Atom a; a.name = "C" + std::to_string(i); a.pos = gemmi::Position(10 + 8 * d(rng), -3 + 8 * d(rng), 5 + 8 * d(rng)); res.atoms.push_back(a); } chain.residues.push_back(res); } model.models[0].chains.push_back(chain); } std::vector bodies(2); for (int b = 0; b < 2; b++) { bodies[b].id = std::to_string(b + 1); bodies[b].origin = gemmi::Position(9 + b, -2, 4); std::array p{}; for (double &x : p) x = 0.01 * d(rng); bodies[b].Set(p); for (int i = 0; i < 30; i++) bodies[b].atoms.push_back(30 * b + i); } model.meta.refinement.emplace_back(); model.meta.refinement[0].tls_groups = TlsBodiesToGroups(model.models[0], bodies); // Placed as molecular replacement places one chain: chain A twice, each copy turned and moved, named A and B. const gemmi::Mat33 r0 = gemmi::Mat33::from_columns(gemmi::Vec3(0, 1, 0), gemmi::Vec3(-1, 0, 0), gemmi::Vec3(0, 0, 1)); const gemmi::Mat33 r1 = gemmi::Mat33::from_columns(gemmi::Vec3(1, 0, 0), gemmi::Vec3(0, 0, 1), gemmi::Vec3(0, -1, 0)); const gemmi::Transform move[2] = {{r0, gemmi::Vec3(40, 5, -7)}, {r1, gemmi::Vec3(-12, 30, 8)}}; gemmi::Structure placed = model; placed.models[0].chains.resize(1); placed.models[0].chains.push_back(placed.models[0].chains[0]); placed.models[0].chains[1].name = "B"; for (int k = 0; k < 2; k++) for (gemmi::Residue &r : placed.models[0].chains[k].residues) for (gemmi::Atom &a : r.atoms) a.pos = gemmi::Position(move[k].apply(a.pos)); PlaceTlsGroups(model, placed); const std::vector moved = TlsBodiesFromModel(placed.models[0], *placed.meta.get_tls_groups()); // the group of chain A once on each copy, that of chain B on none; U turned with its copy REQUIRE(moved.size() == 2); for (int k = 0; k < 2; k++) { REQUIRE(moved[k].atoms.size() == 30); for (int i = 0; i < 30; i++) { const gemmi::Position &x = model.models[0].chains[0].residues[i / 10].atoms[i % 10].pos; const gemmi::Position &y = placed.models[0].chains[k].residues[i / 10].atoms[i % 10].pos; CHECK(moved[k].atoms[i] == 30 * k + i); const gemmi::SMat33 want = bodies[0].U(x).transformed_by(move[k].mat), got = moved[k].U(y); CHECK(got.u11 == Catch::Approx(want.u11).margin(1e-9)); CHECK(got.u22 == Catch::Approx(want.u22).margin(1e-9)); CHECK(got.u33 == Catch::Approx(want.u33).margin(1e-9)); CHECK(got.u12 == Catch::Approx(want.u12).margin(1e-9)); CHECK(got.u13 == Catch::Approx(want.u13).margin(1e-9)); CHECK(got.u23 == Catch::Approx(want.u23).margin(1e-9)); } } } TEST_CASE("RefinementADP_u_restraints_gradient", "[Refinement]") { std::mt19937 rng(5); std::uniform_real_distribution d(-1.0, 1.0); const int n = 6; std::vector pos; std::vector> u(n); for (int i = 0; i < n; i++) { pos.emplace_back(1.5 * i + 0.2 * d(rng), 0.3 * d(rng), 0.3 * d(rng)); u[i] = {0.2 + 0.05 * d(rng), 0.2 + 0.05 * d(rng), 0.2 + 0.05 * d(rng), 0.03 * d(rng), 0.03 * d(rng), 0.03 * d(rng)}; } const URestraints r = MakeURestraints({{0, 1}, {1, 2}, {0, 2}, {3, 4}}, {1, 1, 1, 1, 1, 1}); CHECK(r.pairs.size() == 4); REQUIRE(r.isotropic == std::vector{5}); std::vector> g(n, {0, 0, 0, 0, 0, 0}); EvaluateURestraints(r, pos, u, &g); const double h = 1e-6; for (int i = 0; i < n; i++) for (int c = 0; c < 6; c++) { std::vector> up = u, um = u; up[i][c] += h; um[i][c] -= h; const double fd = (EvaluateURestraints(r, pos, up, nullptr) - EvaluateURestraints(r, pos, um, nullptr)) / (2 * h); CAPTURE(i, c); CHECK(g[i][c] == Catch::Approx(fd).epsilon(1e-5).margin(1e-3)); } // a rigid pair: the same U along the bond, however different across it, costs no rigid-bond term const URestraints one = MakeURestraints({{0, 1}}, {1, 1}); const std::vector line = {{0, 0, 0}, {0, 0, 1.5}}; const std::vector> across = {{0.1, 0.3, 0.2, 0, 0, 0}, {0.3, 0.1, 0.2, 0.05, 0, 0}}; const double sim = (0.2 * 0.2 + 0.2 * 0.2 + 2 * 0.05 * 0.05) / (URestraints::SIGMA_SIMILAR * URestraints::SIGMA_SIMILAR); CHECK(EvaluateURestraints(one, line, across, nullptr) == Catch::Approx(sim)); } TEST_CASE("RefinementADP_tls_segments", "[Refinement]") { // a chain of 60 residues along x whose B follows one quadratic on the first 25 residues and another after std::vector> residues; std::vector pos; std::vector b; for (int r = 0; r < 60; r++) { std::vector ra; for (int a = 0; a < 5; a++) { const gemmi::Position p(3.8 * r, 2.0 * std::sin(a + r), 2.0 * std::cos(a * r)); ra.push_back(static_cast(pos.size())); pos.push_back(p); const double y = p.y, z = p.z; b.push_back(r < 25 ? 20 + 0.5 * (y * y + z * z) : 60 + 3.0 * (y * y) - 0.02 * p.x); } residues.push_back(ra); } CHECK(TlsSegments(residues, pos, b, 1, 10) == std::vector{0}); CHECK(TlsSegments(residues, pos, b, 2, 10) == std::vector{0, 25}); // too short for three segments of 25 CHECK(TlsSegments(residues, pos, b, 3, 25).size() == 2); } namespace { // A twinned test case in P 4, whose lattice adds the twin law: the reflections of the group's ASU to 2.2 A with // every fourth of the zone l = 1 left out, so that some twin mates are rows of their own, and the free set drawn // on the lattice's holohedry (AssignRfreeFlags) so that no twin pair is split. struct TwinCase { gemmi::UnitCell cell{30, 30, 40, 90, 90, 90}; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 4"); gemmi::Op law; std::vector merged; RefinementReflections refl; TwinCase() { const std::vector laws = RefinementTwinLaws(cell, sg); REQUIRE(laws.size() == 1); law = laws[0]; merged = asu_reflections(cell, sg, 2.2); int i = 0; std::erase_if(merged, [&](const MergedReflection &r) { return r.l == 1 && i++ % 4 == 0; }); AssignRfreeFlags(merged, sg, 0.05, 0, UnitCell{static_cast(cell.a), static_cast(cell.b), static_cast(cell.c), 90, 90, 90}); refl = PrepareRefinementReflections(merged, cell, sg, 2.2); CHECK(SetTwinLaw(refl, cell, sg, 2.2, law) == 0); } }; } // namespace TEST_CASE("RefinementTwin_laws_and_mates", "[Refinement]") { TwinCase t; CHECK(RefinementTwinLaws(t.cell, *gemmi::find_spacegroup_by_name("P 4 2 2")).empty()); REQUIRE(t.refl.Twinned()); CHECK(!t.refl.mate_hkl.empty()); const gemmi::GroupOps gops = t.sg.operations(); const gemmi::ReciprocalAsu asu(&t.sg); const std::vector rows = t.refl.RowHkl(); REQUIRE(t.refl.Laws() == 1); for (size_t m = 0; m < t.refl.Size(); m++) { const int j = t.refl.mates[0][m]; CAPTURE(m, j); REQUIRE(j >= 0); if (j != static_cast(m)) { CHECK(rows[j] == asu.to_asu(t.law.apply_to_hkl(rows[m]), gops).first); CHECK(t.refl.mates_from[0][j] == static_cast(m)); if (j < static_cast(t.refl.Size())) CHECK(t.refl.mates[0][j] == static_cast(m)); // an involution } } // a free set drawn on the merging group alone splits twin pairs RefinementReflections split = t.refl; for (size_t m = 0; m < split.Size(); m++) split.free[m] = m % 20 == 0; CHECK(SetTwinLaw(split, t.cell, t.sg, 2.2, t.law) > 0); } // The free set closed under the lattice's domains: one drawn on the holohedry is left as it is; one drawn on // another cell's (P 2 2 2 on a tetragonal metric, its flags drawn on an orthorhombic one: twin pairs split) ends with // every pair in one set and about the free fraction it had - where a majority of each pair would empty it, and a // draw that repeats the flag of one member would inflate it. TEST_CASE("RefinementTwin_free_set_closed", "[Refinement]") { const gemmi::UnitCell cell{30, 40, 30, 90, 90, 90}; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 2 2 2"); std::vector merged = asu_reflections(cell, sg, 1.5); AssignRfreeFlags(merged, sg, 0.05, 0, UnitCell{30, 40, 30, 90, 90, 90}); CHECK(CloseFreeSet(merged, cell, sg) == 0); AssignRfreeFlags(merged, sg, 0.05, 0, UnitCell{30, 40, 36, 90, 90, 90}); const auto free_count = [&] { return std::count_if(merged.begin(), merged.end(), [](const MergedReflection &r) { return r.rfree_flag; }); }; const auto before = free_count(); CHECK(CloseFreeSet(merged, cell, sg) > 0); CHECK(free_count() == Catch::Approx(before).epsilon(0.1)); std::map flag; const gemmi::GroupOps gops = sg.operations(); const gemmi::ReciprocalAsu asu(&sg); for (const MergedReflection &r : merged) flag[asu.to_asu(gemmi::Miller{{r.h, r.k, r.l}}, gops).first] = r.rfree_flag; int split = 0; for (const MergedReflection &r : merged) for (const gemmi::Op &law : RefinementTwinDomains(cell, sg)) { const auto it = flag.find(asu.to_asu(law.apply_to_hkl({{r.h, r.k, r.l}}), gops).first); split += it != flag.end() && it->second != r.rfree_flag; } CHECK(split == 0); CHECK(CloseFreeSet(merged, cell, sg) == 0); } TEST_CASE("RefinementTwin_rule_with_data_evidence", "[Refinement]") { // the null alone: 3 standard errors CHECK(AdoptTwinLaw(0.1, 3.0, false)); CHECK_FALSE(AdoptTwinLaw(0.1, 2.9, false)); // the data read twinned as well: 2 standard errors CHECK(AdoptTwinLaw(0.1, 2.0, true)); CHECK(AdoptTwinLaw(0.1, 2.5, true)); CHECK_FALSE(AdoptTwinLaw(0.1, 1.9, true)); // no fraction, no twin, whatever the evidence CHECK_FALSE(AdoptTwinLaw(0.0, 10.0, true)); } TEST_CASE("RefinementTwin_gradient", "[Refinement]") { TwinCase t; gemmi::Structure truth = random_crystal(t.cell, "P 4", 240, 17); const gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 18); auto cpu = MakeRefinementEngineCPU(t.cell, t.sg, 2.2, t.refl, 4); cpu->SetBlur(RefinementBlur(t.cell, t.sg, 2.2, 5.0)); XrayRowModel rows; rows.k_total.assign(t.refl.Size(), 1.0f); rows.fmask.assign(t.refl.Rows(), {0, 0}); rows.twin_fractions = {0.3f}; std::vector> f; cpu->Fcalc(truth.models[0], f); REQUIRE(f.size() == t.refl.Rows()); const std::vector> twinned = TwinnedModel(t.refl, rows, f); for (size_t m = 0; m < t.refl.Size(); m++) t.refl.fo[m] = std::abs(twinned[m]); // the fraction comes back from the data CHECK(FitTwinFraction(t.refl, rows, f) == Catch::Approx(0.3).margin(0.01)); cpu->Fcalc(model, f); EstimateLikelihoodParameters(t.refl, TwinnedModel(t.refl, rows, f), rows); cpu->SetRowModel(rows); // the CPU engine kept a reference to t.refl and reads the observations set above check_xray_gradient(*cpu, model, 0.03); #ifdef JFJOCH_USE_CUDA if (get_gpu_count() > 0) { auto gpu = MakeRefinementEngineGPU(0, t.cell, t.sg, 2.2, t.refl, model, 300.0, SIZE_MAX); REQUIRE(gpu); gpu->SetBlur(RefinementBlur(t.cell, t.sg, 2.2, 5.0)); gpu->SetRowModel(rows); std::vector> gc, gg; const double tc = cpu->Evaluate(model, &gc); const double tg = gpu->Evaluate(model, &gg); CHECK(tg == Catch::Approx(tc).epsilon(1e-4)); double max_g = 0; for (const auto &g : gc) for (int c = 0; c < 4; c++) max_g = std::max(max_g, std::fabs(g[c])); for (size_t i = 0; i < gc.size(); i++) for (int c = 0; c < 4; c++) { CAPTURE(i, c); CHECK(std::fabs(gg[i][c] - gc[i][c]) < 0.01 * max_g); } } #endif } TEST_CASE("RefinementTwin_domains_gradient", "[Refinement]") { // P 2 on a tetragonal metric (a = c, beta = 90): three domain operators beside the identity, a two-fold whose // square is in the group and two four-fold cosets const gemmi::UnitCell cell{30, 40, 30, 90, 90, 90}; const gemmi::SpaceGroup &sg = *gemmi::find_spacegroup_by_name("P 1 2 1"); const std::vector laws = RefinementTwinDomains(cell, sg); REQUIRE(laws.size() == 3); std::vector merged = asu_reflections(cell, sg, 2.4); int i = 0; std::erase_if(merged, [&](const MergedReflection &r) { return r.k == 1 && i++ % 4 == 0; }); AssignRfreeFlags(merged, sg, 0.05, 0, UnitCell{30, 40, 30, 90, 90, 90}); RefinementReflections refl = PrepareRefinementReflections(merged, cell, sg, 2.4); CHECK(SetTwinLaws(refl, cell, sg, 2.4, laws) == 0); REQUIRE(refl.Laws() == 3); CHECK(!refl.mate_hkl.empty()); gemmi::Structure truth = random_crystal(cell, "P 1 2 1", 240, 27); const gemmi::Model model = shaken(truth.models[0], 0.3, 4.0, 28); auto cpu = MakeRefinementEngineCPU(cell, sg, 2.4, refl, 4); cpu->SetBlur(RefinementBlur(cell, sg, 2.4, 5.0)); XrayRowModel rows; rows.k_total.assign(refl.Size(), 1.0f); rows.fmask.assign(refl.Rows(), {0, 0}); rows.twin_fractions = {0.2f, 0.1f, 0.15f}; std::vector> f; cpu->Fcalc(truth.models[0], f); const std::vector> twinned = TwinnedModel(refl, rows, f); for (size_t m = 0; m < refl.Size(); m++) refl.fo[m] = std::abs(twinned[m]); const std::vector fitted = FitTwinFractions(refl, rows, f); REQUIRE(fitted.size() == 3); for (size_t d = 0; d < 3; d++) CHECK(fitted[d] == Catch::Approx(rows.twin_fractions[d]).margin(0.01)); cpu->Fcalc(model, f); EstimateLikelihoodParameters(refl, TwinnedModel(refl, rows, f), rows); cpu->SetRowModel(rows); check_xray_gradient(*cpu, model, 0.03); #ifdef JFJOCH_USE_CUDA if (get_gpu_count() > 0) { auto gpu = MakeRefinementEngineGPU(0, cell, sg, 2.4, refl, model, 300.0, SIZE_MAX); REQUIRE(gpu); gpu->SetBlur(RefinementBlur(cell, sg, 2.4, 5.0)); gpu->SetRowModel(rows); std::vector> gc, gg; const double tc = cpu->Evaluate(model, &gc); const double tg = gpu->Evaluate(model, &gg); CHECK(tg == Catch::Approx(tc).epsilon(1e-4)); double max_g = 0; for (const auto &g : gc) for (int c = 0; c < 4; c++) max_g = std::max(max_g, std::fabs(g[c])); for (size_t a = 0; a < gc.size(); a++) for (int c = 0; c < 4; c++) { CAPTURE(a, c); CHECK(std::fabs(gg[a][c] - gc[a][c]) < 0.01 * max_g); } } #endif } TEST_CASE("Refinement_twin_decided", "[Refinement]") { TwinCase t; gemmi::Structure truth = residue_crystal({"PHE", "TRP", "DA", "DG", "TYR", "HOH", "HOH", "HOH"}, t.cell, "P 4", 21); gemmi::Structure start = truth; start.models[0] = shaken(truth.models[0], 0.2, 4.0, 22); std::vector> f; { RefinementReflections refl = t.refl; auto cpu = MakeRefinementEngineCPU(t.cell, t.sg, 2.2, refl, 4); cpu->SetBlur(RefinementBlur(t.cell, t.sg, 2.2, 5.0)); cpu->Fcalc(truth.models[0], f); } Logger logger("RefinementTest"); RefinementSettings settings; settings.gpu = false; settings.nthreads = 4; settings.weight_ladder = {0.0625}; settings.adp_ladder = {0.0}; settings.max_steps = 1; settings.final_ladder = {1.0}; settings.riding_hydrogens = false; settings.solvent = false; for (double fraction : {0.0, 0.3}) { CAPTURE(fraction); // the observations as twinned at `fraction`, with 3% noise XrayRowModel rows; rows.k_total.assign(t.refl.Size(), 1.0f); rows.fmask.assign(t.refl.Rows(), {0, 0}); rows.twin_fractions = {static_cast(fraction)}; const std::vector> twinned = TwinnedModel(t.refl, rows, f); std::map amplitude; for (size_t m = 0; m < t.refl.Size(); m++) amplitude[t.refl.hkl[m]] = std::abs(twinned[m]); std::vector merged = t.merged; std::mt19937 rng(5); std::normal_distribution noise(0.0, 0.03); for (MergedReflection &r : merged) { const auto it = amplitude.find(gemmi::Miller{{r.h, r.k, r.l}}); r.F = it == amplitude.end() ? NAN : static_cast(it->second * (1 + noise(rng))); r.sigmaF = 0.03f * r.F + 0.01f; } const RefinementResult r = RefineModel(start, merged, t.cell, t.sg, 1.0, logger, settings); REQUIRE(r.ok); CHECK(r.twin_law == t.law.as_hkl().triplet('h')); if (fraction > 0) { CHECK(r.twin_decision == "TWINNED"); CHECK(r.twinned); CHECK(r.twin_fraction == Catch::Approx(fraction).margin(0.03)); CHECK(r.r_free < r.untwinned_r_free - 0.02); CHECK(r.twin_r_free < r.twin_r_free_null - 0.02); CHECK(r.twin_null_z >= 3.0); } else { CHECK((r.twin_decision == "ZERO_FRACTION" || r.twin_null_z < 3.0)); CHECK(r.twin_decision != "TWINNED"); CHECK_FALSE(r.twinned); // the untwinned refinement, as it is without the twin hypothesis RefinementSettings plain = settings; plain.twin = false; const RefinementResult u = RefineModel(start, merged, t.cell, t.sg, 1.0, logger, plain); CHECK(u.r_free == r.r_free); CHECK(u.r_work == r.r_work); } // scored only, the model as given is tested against the twin law by the same rule RefinementSettings score = settings; score.score_only = true; const RefinementResult s = RefineModel(truth, merged, t.cell, t.sg, 1.0, logger, score); REQUIRE(s.ok); CHECK((s.twin_decision == "TWINNED") == (fraction > 0)); if (fraction > 0) CHECK(s.r_free_start < s.untwinned_r_free - 0.02); } } // A glutamine's amide placed through the main chain of another chain, as a predicted model's rotamer lands when // molecular replacement packs the copies: turned to a rotamer clear of it, the main chain and the beta carbon // untouched, the side chain's bonds as they were. The other residue stands as a copy in the asymmetric unit, // and as a symmetry mate. TEST_CASE("ClashRelief_side_chain_through_another_residue", "[Refinement]") { struct AtomXYZ { const char *name; gemmi::El el; double x, y, z; }; const std::vector gln = {{"N", gemmi::El::N, -14.511, 66.525, 29.018}, {"CA", gemmi::El::C, -13.327, 66.868, 28.222}, {"C", gemmi::El::C, -13.127, 65.890, 27.053}, {"O", gemmi::El::O, -12.773, 66.307, 25.952}, {"CB", gemmi::El::C, -12.097, 66.881, 29.138}, {"CG", gemmi::El::C, -10.814, 67.262, 28.380}, {"CD", gemmi::El::C, -9.588, 67.307, 29.281}, {"OE1", gemmi::El::O, -9.635, 67.049, 30.468}, {"NE2", gemmi::El::N, -8.430, 67.627, 28.746}}; // the main chain the amide runs through: an alanine and the next residue's N and CA const std::vector ala = {{"N", gemmi::El::N, -6.547, 68.057, 30.231}, {"CA", gemmi::El::C, -7.057, 67.212, 29.150}, {"C", gemmi::El::C, -8.108, 67.895, 28.263}, {"O", gemmi::El::O, -8.5, 68.9, 28.5}, {"CB", gemmi::El::C, -6.3, 66.8, 28.1}}; const std::vector next = {{"N", gemmi::El::N, -8.762, 68.964, 28.726}, {"CA", gemmi::El::C, -9.632, 69.779, 27.871}}; auto add = [](gemmi::Chain &ch, const char *resname, int num, const std::vector &atoms, const gemmi::Transform &t) { gemmi::Residue r; r.name = resname; r.seqid.num = num; for (const AtomXYZ &a : atoms) { gemmi::Atom at; at.name = a.name; at.element = gemmi::Element(a.el); at.pos = gemmi::Position(t.apply(gemmi::Vec3(a.x, a.y, a.z))); at.occ = 1; at.b_iso = 20; r.atoms.push_back(at); } ch.residues.push_back(r); }; const bool as_symmetry_mate = GENERATE(false, true); gemmi::Structure st; st.cell = gemmi::UnitCell(100, 100, 100, 90, 90, 90); st.spacegroup_hm = as_symmetry_mate ? "P 1 2 1" : "P 1"; st.models.emplace_back(1); gemmi::Chain a("A"), b("B"); const gemmi::Transform identity; // in P 1 2 1 the other chain is put where the 2-fold (-x, y, -z) brings it back onto the glutamine gemmi::Transform two_fold; two_fold.mat = gemmi::Mat33(-1, 0, 0, 0, 1, 0, 0, 0, -1); const gemmi::Transform &place = as_symmetry_mate ? two_fold : identity; add(a, "GLN", 17, gln, identity); add(b, "ALA", 16, ala, place); add(b, "GLN", 17, next, place); st.models[0].chains = {a, b}; st.setup_cell_images(); const gemmi::Structure given = st; const std::vector turned = RelieveSideChainClashes(st); REQUIRE(turned.size() == 1); CHECK(turned[0].name == "GLN A17"); CHECK(turned[0].overlap_before > 2.0); // NE2 0.64 A from the alanine's C CHECK(turned[0].overlap_after < CLASH_RELIEF_OVERLAP); CHECK(turned[0].chi_before.size() == 3); const gemmi::Residue &was = given.models[0].chains[0].residues[0], &now = st.models[0].chains[0].residues[0]; REQUIRE(now.atoms.size() == was.atoms.size()); for (size_t i = 0; i < was.atoms.size(); i++) { if (was.atoms[i].name == "N" || was.atoms[i].name == "CA" || was.atoms[i].name == "C" || was.atoms[i].name == "O" || was.atoms[i].name == "CB") CHECK(now.atoms[i].pos.dist(was.atoms[i].pos) < 1e-9); CHECK(now.atoms[i].b_iso == was.atoms[i].b_iso); } auto bond = [](const gemmi::Residue &r, const char *p, const char *q) { return r.find_atom(p, '*')->pos.dist(r.find_atom(q, '*')->pos); }; for (const auto &[p, q] : {std::pair{"CB", "CG"}, std::pair{"CG", "CD"}, std::pair{"CD", "OE1"}, std::pair{"CD", "NE2"}}) CHECK(bond(now, p, q) == Catch::Approx(bond(was, p, q)).margin(1e-6)); CHECK(now.find_atom("NE2", '*')->pos.dist(was.find_atom("NE2", '*')->pos) > 2.0); // the other chain as it was, and no atom of the side chain nearer to it than a contact gemmi::UnitCell cell = st.cell; cell.set_cell_images_from_spacegroup(st.find_spacegroup()); for (size_t r = 0; r < 2; r++) for (size_t i = 0; i < given.models[0].chains[1].residues[r].atoms.size(); i++) { const gemmi::Atom &other = st.models[0].chains[1].residues[r].atoms[i]; CHECK(other.pos.dist(given.models[0].chains[1].residues[r].atoms[i].pos) < 1e-9); for (const gemmi::Atom &sc : now.atoms) if (sc.name != "N" && sc.name != "CA" && sc.name != "C" && sc.name != "O") CHECK(cell.find_nearest_image(sc.pos, other.pos, gemmi::Asu::Any).dist() > 2.5); } // a model without the clash is left as it is gemmi::Structure clear = given; clear.models[0].chains.pop_back(); CHECK(RelieveSideChainClashes(clear).empty()); }