From 058ba80fc73860f2ef28c0aee0d00cd613e85500 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 16:28:08 +0100 Subject: [PATCH 1/7] Refinement: B ceiling from the data's Wilson B, weights balanced at the refined level Two start-conditioned defects of the first round, found on a flat-B start of the crystal model and on a placed prediction (pLDDT-derived B of ~6 A^2), both taken from the B column as given: - The ceiling of B was 2 x b_max of the model, i.e. 100 A^2 for a flat or pLDDT B column, and it bit through the TLS tensors (BoundedB scales them down to it) long before atoms reached it: on the flat start one round gave R-free 0.4040 at ceiling 100, 0.3989 at 140, 0.3870 at 200, 0.3848 at 300, while the refined max B was 123. The ceiling is now set from at least 2.5 x the data's Wilson B as well (WilsonB, fitted against the model's own atoms from 4 A), which leaves a deposited model's ceiling where it was. - The ADP weight was balanced before the overall B of the scale was moved into the atoms, which every trial does: the B similarity's gradient goes as 1/B, so a column at 6 A^2 balanced a weight ~100 x too stiff (2350 vs ~20 once refined). The move is now made once after the placement, before the weights are balanced. Together these give in round 1 most of what the second and third rounds were adding: flat start 0.4040 -> 0.3822 (base with rounds 0.3798 at 3 x the time), placed prediction 0.4162 -> 0.4038 (0.3982 with rounds), crystal model with its own B unchanged (0.3790). 13 AlphaFold starts from images with the rounds as they are: six lower by 0.005-0.021, none higher beyond 0.0025, median -0.002, refinement time unchanged. Measured and not adopted: elastic-network and packing-density B starts (within noise once the ceiling is right), and a coarse-to-fine B solve along chain / network-mode / residue shifts before the per-atom one (same optimum to 0.0002 at +25 % evaluations). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/Refinement.cpp | 64 ++++++++++++++++++- .../structure_refinement/Refinement.h | 1 + 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 8997505b9..bef8159eb 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -53,6 +53,8 @@ namespace { constexpr double B_FLOOR = 2.0; // A^2: no atom's B goes below it // nor above twice the model's largest B, within these bounds (the GPU's gather is reserved for it) constexpr double B_CEILING_MIN = 100.0, B_CEILING_MAX = 300.0; +// The B the ceiling is set from is at least this times the data's Wilson B (WilsonB). +constexpr double B_CEILING_WILSON = 2.5; // The ceiling of B for a model whose highest B is `b_max`: twice it, within the bounds - the bounds lifted for a // model that comes with B above them (a structure refined at 4-5 A, B 400-600 A^2), which the ceiling would // otherwise flatten (the gather is reserved for it). @@ -2288,6 +2290,54 @@ double RefinementDMin(double max_inv_d2) { return 0.999 / std::sqrt(max_inv_d2); } +// The Wilson B of the data against the model's atoms: the slope of ln(<|F|^2> / sum_j f_j^2) in s^2 / 2 over shells +// from 4 A to the finest amplitude; 0 where the data do not reach 4 A. The B factors refine around it whatever the +// model's say, so the ceiling of B is set from it as well as from them (BCeiling): a model may come with B factors +// that say nothing of where they will refine - a flat B, a prediction's pLDDT. +// Following Wilson (1949) Acta Cryst. 2, 318-321 +double WilsonB(const std::vector &merged, const std::vector &atoms) { + constexpr int SHELLS = 10; + constexpr double S2_MIN = 1.0 / 16.0; // 4 A + const double s2_max = MaxInvD2(merged); + if (s2_max <= S2_MIN) + return 0; + std::map, int> composition; + for (const gemmi::Atom *a : atoms) + if (!a->is_hydrogen()) + composition[{static_cast(a->element.elem), a->charge}]++; + std::vector f2_sum(SHELLS, 0.0), s2_sum(SHELLS, 0.0); + std::vector n(SHELLS, 0); + for (const MergedReflection &r : merged) { + if (!(std::isfinite(r.F) && r.d > 0)) + continue; + const double s2 = 1.0 / (double(r.d) * r.d); + if (s2 < S2_MIN) + continue; + const int k = std::min(SHELLS - 1, static_cast((s2 - S2_MIN) / (s2_max - S2_MIN) * SHELLS)); + f2_sum[k] += double(r.F) * r.F; + s2_sum[k] += s2; + n[k]++; + } + // the line y = ln k - B x through the shells (x = s^2 / 2), each weighted by its reflections + double sw = 0, sx = 0, sy = 0, sxx = 0, sxy = 0; + for (int k = 0; k < SHELLS; k++) { + if (n[k] < 20) + continue; + const double s2 = s2_sum[k] / n[k]; + double f2 = 0; + for (const auto &[ion, count] : composition) + f2 += count * std::pow(IonFormFactor(static_cast(ion.first), ion.second).calculate_sf(s2 / 4.0), 2); + const double x = s2 / 2.0, y = std::log(f2_sum[k] / n[k] / f2), w = n[k]; + sw += w; + sx += w * x; + sy += w * y; + sxx += w * x * x; + sxy += w * x * y; + } + const double det = sw * sxx - sx * sx; + return det > 0 ? std::max(0.0, -(sw * sxy - sx * sy) / det) : 0.0; +} + // One refinement of a model, with the solvent build in it where settings.solvent asks for it. // `twin_laws`: against the data as twinned by them (one domain each beside the model's own), empty for untwinned; // `null_first`: the first of them the null (TwinNull) replaces the mates of. @@ -2326,7 +2376,8 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vectorb_iso + gemmi::u_to_b() * body.U(ctx.atoms[i]->pos).trace() / 3.0); - ctx.b_ceiling = BCeiling(b_max); + result.wilson_b = WilsonB(merged, ctx.atoms); + ctx.b_ceiling = BCeiling(std::max(b_max, B_CEILING_WILSON * result.wilson_b)); if (ctx.atoms.empty()) { result.failure_reason = "the model has no atoms"; return result; @@ -2432,14 +2483,15 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector> solvent; // the solvent audit (settings.solvent) + double wilson_b = 0; // the data's Wilson B against the model's atoms (A^2), 0 where the data do not reach 4 A double b_ceiling = 0; // the highest B the refinement allowed SolventBuildResult solvent_build; // the solvent build's rounds (settings.solvent) From 2676b22312d7b71014daf09e210d4ff15fc00fa1 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 15:33:10 +0100 Subject: [PATCH 2/7] Refinement: the side-chain rotamer helpers of ClashRelief in a header of their own MakeSideChain, SideChain::Build, the staggered rotamer list and the overlap against the rest of the model move from ClashRelief.cpp to SideChainRotamers.{h,cpp}, unchanged, so that the side-chain field can turn side chains and score their contacts the same way. RelieveSideChainClashes is as it was. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/ClashRelief.cpp | 203 +----------------- .../SideChainRotamers.cpp | 187 ++++++++++++++++ .../structure_refinement/SideChainRotamers.h | 50 +++++ 3 files changed, 244 insertions(+), 196 deletions(-) create mode 100644 image_analysis/structure_refinement/SideChainRotamers.cpp create mode 100644 image_analysis/structure_refinement/SideChainRotamers.h diff --git a/image_analysis/structure_refinement/ClashRelief.cpp b/image_analysis/structure_refinement/ClashRelief.cpp index 67979e4f2..2100f886b 100644 --- a/image_analysis/structure_refinement/ClashRelief.cpp +++ b/image_analysis/structure_refinement/ClashRelief.cpp @@ -2,204 +2,17 @@ // SPDX-License-Identifier: GPL-3.0-only #include "ClashRelief.h" +#include "SideChainRotamers.h" #include -#include -#include #include -#include -#include #include #include namespace { -constexpr double PI = 3.14159265358979323846; -constexpr double HBOND_CONTACT = 2.6; // A: between N and O, as the refinement's repulsions have it -constexpr double REACH = 4.0; // A: the farthest pair that can overlap - -using Chi = std::array; - -// The chi angles of each amino acid, by the atoms that define them. -const std::map> &ChiAtoms() { - static const std::map> chis = { - {"ARG", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "NE"}, {"CG", "CD", "NE", "CZ"}}}, - {"ASN", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "OD1"}}}, - {"ASP", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "OD1"}}}, - {"GLN", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "OE1"}}}, - {"GLU", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "OE1"}}}, - {"HIS", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "ND1"}}}, - {"ILE", {{"N", "CA", "CB", "CG1"}, {"CA", "CB", "CG1", "CD1"}}}, - {"LEU", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, - {"LYS", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "CE"}, {"CG", "CD", "CE", "NZ"}}}, - {"MET", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "SD"}, {"CB", "CG", "SD", "CE"}}}, - {"PHE", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, - {"SER", {{"N", "CA", "CB", "OG"}}}, - {"THR", {{"N", "CA", "CB", "OG1"}}}, - {"TRP", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, - {"TYR", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, - {"VAL", {{"N", "CA", "CB", "CG1"}}}, - }; - return chis; -} - -// The last chi turns a planar group (an amide, a carboxylate, a ring, a guanidinium): any angle, not only the -// staggered ones. -bool PlanarEnd(const std::string &res) { - return res == "ASP" || res == "ASN" || res == "HIS" || res == "PHE" || res == "TYR" || res == "TRP" || - res == "GLU" || res == "GLN" || res == "ARG"; -} - -// How far along the side chain an atom is, hydrogens by their parent's letter: CB 1, gamma 2, delta 3, epsilon 4, -// zeta 5, eta 6; the main chain and its hydrogens 0. -int Depth(const std::string &name) { - if (name.size() < 2 || name == "CA" || name == "OXT") - return 0; - switch (name[1]) { - case 'B': return 1; - case 'G': return 2; - case 'D': return 3; - case 'E': return 4; - case 'Z': return 5; - case 'H': return 6; - default: return 0; - } -} - -double VdwRadius(gemmi::El el) { - switch (el) { - case gemmi::El::N: return 1.55; - case gemmi::El::O: return 1.52; - case gemmi::El::S: - case gemmi::El::P: return 1.80; - default: return 1.70; - } -} - -bool Polar(gemmi::El el) { - return el == gemmi::El::N || el == gemmi::El::O; -} - -gemmi::Position Rotate(const gemmi::Position &p, const gemmi::Position &a, const gemmi::Position &b, double angle) { - const gemmi::Vec3 k = (b - a).normalized(); - const gemmi::Vec3 v = p - b; - const double c = std::cos(angle), s = std::sin(angle); - return gemmi::Position(gemmi::Vec3(b) + v * c + k.cross(v) * s + k * (k.dot(v) * (1 - c))); -} - -struct SideChain { - std::vector pos; // the residue's atoms, as given - std::vector depth; - std::vector> chi; // atom indices of each chi - - double Dihedral(const std::vector &p, size_t k) const { - const auto &c = chi[k]; - return gemmi::calculate_dihedral(p[c[0]], p[c[1]], p[c[2]], p[c[3]]) * 180.0 / PI; - } - - // The residue's atoms with the side chain at these chi angles (degrees), turned from the given ones. - std::vector Build(const std::vector &chis) const { - std::vector p = pos; - for (size_t k = 0; k < chi.size(); k++) { - const double angle = (chis[k] - Dihedral(p, k)) * PI / 180.0; - const int pivot = depth[chi[k][2]]; - for (size_t i = 0; i < p.size(); i++) - if (depth[i] > pivot) - p[i] = Rotate(p[i], p[chi[k][1]], p[chi[k][2]], angle); - } - return p; - } -}; - -// The residue's chi atoms, found by name; none where any is missing (an incomplete side chain stays as it is) or -// the residue has alternative conformations (its atoms would be found by name once, for both). -std::optional MakeSideChain(const gemmi::Residue &res) { - auto it = ChiAtoms().find(res.name); - if (it == ChiAtoms().end()) - return std::nullopt; - if (std::any_of(res.atoms.begin(), res.atoms.end(), [](const gemmi::Atom &a) { return a.altloc != '\0'; })) - return std::nullopt; - SideChain sc; - for (const gemmi::Atom &a : res.atoms) { - sc.pos.push_back(a.pos); - sc.depth.push_back(Depth(a.name)); - } - for (const Chi &c : it->second) { - std::array idx{}; - for (int k = 0; k < 4; k++) { - const auto found = std::find_if(res.atoms.begin(), res.atoms.end(), - [&](const gemmi::Atom &a) { return a.name == c[k]; }); - if (found == res.atoms.end()) - return std::nullopt; - idx[k] = static_cast(found - res.atoms.begin()); - } - sc.chi.push_back(idx); - } - return sc; -} - -// The overlaps of a residue's turned atoms (depth 2 and beyond, heavy) with the rest of the model: the sum of -// squared overlaps and the deepest one. Its own atoms, hydrogens, absent atoms (occupancy 0), metals and the -// other conformations of an alternative are no contact; nor is a covalent partner (`linked`: a LINK's, or a -// disulfide's). -struct Overlap { - double sum = 0, deepest = 0; -}; - -Overlap OverlapOf(gemmi::NeighborSearch &ns, const gemmi::Model &model, const gemmi::UnitCell &cell, - const gemmi::Residue &res, int chain_idx, int residue_idx, const SideChain &sc, - const std::vector &p, const std::vector &linked) { - Overlap out; - for (size_t i = 0; i < p.size(); i++) { - const gemmi::Atom &a = res.atoms[i]; - if (sc.depth[i] < 2 || a.is_hydrogen()) - continue; - ns.for_each(p[i], '\0', REACH, [&](const gemmi::NeighborSearch::Mark &m, double) { - if (m.image_idx == 0 && m.chain_idx == chain_idx && m.residue_idx == residue_idx) - return; // its own atoms; its own symmetry copy is another residue to it - const gemmi::Atom &b = model.chains[m.chain_idx].residues[m.residue_idx].atoms[m.atom_idx]; - if (b.is_hydrogen() || !(b.occ > 0) || b.element.is_metal()) - return; - if (std::find(linked.begin(), linked.end(), &b) != linked.end()) - return; - if (a.altloc && b.altloc && a.altloc != b.altloc) - return; - if (a.element == gemmi::El::S && b.element == gemmi::El::S) - return; // a disulfide, where it is one, is no clash - const double d = cell.find_nearest_pbc_image(p[i], b.pos, m.image_idx).dist(); - const double d0 = Polar(a.element.elem) && Polar(b.element.elem) - ? HBOND_CONTACT - : VdwRadius(a.element.elem) + VdwRadius(b.element.elem); - if (d < d0) { - out.sum += (d0 - d) * (d0 - d); - out.deepest = std::max(out.deepest, d0 - d); - } - }); - } - return out; -} - -// The rotamers tried: each chi at the staggered angles, the last one of a planar group every 30 degrees. -std::vector> Rotamers(const std::string &resname, size_t n_chi) { - std::vector> out = {{}}; - for (size_t k = 0; k < n_chi; k++) { - std::vector values = {-60, 60, 180}; - if (k + 1 == n_chi && PlanarEnd(resname)) { - values.clear(); - for (int deg = -150; deg <= 180; deg += 30) - values.push_back(deg); - } - std::vector> next; - for (const std::vector &head : out) - for (double v : values) { - next.push_back(head); - next.back().push_back(v); - } - out = std::move(next); - } - return out; -} +constexpr double REACH = 4.0; // A: the farthest pair that can overlap (OverlapOf) } // namespace @@ -244,17 +57,15 @@ std::vector RelieveSideChainClashes(gemmi::Structure &st) { auto overlap = [&](const std::vector &p) { return OverlapOf(ns, model, cell, res, static_cast(ci), static_cast(ri), *sc, p, linked); }; - const Overlap given = overlap(sc->pos); + const SideChainOverlap given = overlap(sc->pos); const auto earlier = turned_index.find({ci, ri}); if (given.deepest < CLASH_RELIEF_OVERLAP && earlier == turned_index.end()) continue; - std::vector chi; - for (size_t k = 0; k < sc->chi.size(); k++) - chi.push_back(sc->Dihedral(sc->pos, k)); - Overlap best = given; + const std::vector chi = sc->Chis(); + SideChainOverlap best = given; std::vector best_chi = chi; - for (const std::vector &chis : Rotamers(res.name, sc->chi.size())) { - const Overlap o = overlap(sc->Build(chis)); + for (const std::vector &chis : SideChainRotamers(res.name, sc->chi.size())) { + const SideChainOverlap o = overlap(sc->Build(chis)); if (o.sum < best.sum) { best = o; best_chi = chis; diff --git a/image_analysis/structure_refinement/SideChainRotamers.cpp b/image_analysis/structure_refinement/SideChainRotamers.cpp new file mode 100644 index 000000000..9117910cb --- /dev/null +++ b/image_analysis/structure_refinement/SideChainRotamers.cpp @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "SideChainRotamers.h" + +#include +#include +#include + +#include + +namespace { + +constexpr double PI = 3.14159265358979323846; +constexpr double HBOND_CONTACT = 2.6; // A: between N and O, as the refinement's repulsions have it +constexpr double REACH = 4.0; // A: the farthest pair that can overlap + +using Chi = std::array; + +// The chi angles of each amino acid, by the atoms that define them. +const std::map> &ChiAtoms() { + static const std::map> chis = { + {"ARG", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "NE"}, {"CG", "CD", "NE", "CZ"}}}, + {"ASN", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "OD1"}}}, + {"ASP", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "OD1"}}}, + {"GLN", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "OE1"}}}, + {"GLU", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "OE1"}}}, + {"HIS", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "ND1"}}}, + {"ILE", {{"N", "CA", "CB", "CG1"}, {"CA", "CB", "CG1", "CD1"}}}, + {"LEU", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, + {"LYS", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD"}, {"CB", "CG", "CD", "CE"}, {"CG", "CD", "CE", "NZ"}}}, + {"MET", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "SD"}, {"CB", "CG", "SD", "CE"}}}, + {"PHE", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, + {"SER", {{"N", "CA", "CB", "OG"}}}, + {"THR", {{"N", "CA", "CB", "OG1"}}}, + {"TRP", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, + {"TYR", {{"N", "CA", "CB", "CG"}, {"CA", "CB", "CG", "CD1"}}}, + {"VAL", {{"N", "CA", "CB", "CG1"}}}, + }; + return chis; +} + +// The last chi turns a planar group (an amide, a carboxylate, a ring, a guanidinium): any angle, not only the +// staggered ones. +bool PlanarEnd(const std::string &res) { + return res == "ASP" || res == "ASN" || res == "HIS" || res == "PHE" || res == "TYR" || res == "TRP" || + res == "GLU" || res == "GLN" || res == "ARG"; +} + +double VdwRadius(gemmi::El el) { + switch (el) { + case gemmi::El::N: return 1.55; + case gemmi::El::O: return 1.52; + case gemmi::El::S: + case gemmi::El::P: return 1.80; + default: return 1.70; + } +} + +bool Polar(gemmi::El el) { + return el == gemmi::El::N || el == gemmi::El::O; +} + +gemmi::Position Rotate(const gemmi::Position &p, const gemmi::Position &a, const gemmi::Position &b, double angle) { + const gemmi::Vec3 k = (b - a).normalized(); + const gemmi::Vec3 v = p - b; + const double c = std::cos(angle), s = std::sin(angle); + return gemmi::Position(gemmi::Vec3(b) + v * c + k.cross(v) * s + k * (k.dot(v) * (1 - c))); +} + +} // namespace + +int SideChainDepth(const std::string &name) { + if (name.size() < 2 || name == "CA" || name == "OXT") + return 0; + switch (name[1]) { + case 'B': return 1; + case 'G': return 2; + case 'D': return 3; + case 'E': return 4; + case 'Z': return 5; + case 'H': return 6; + default: return 0; + } +} + +double SideChain::Dihedral(const std::vector &p, size_t k) const { + const auto &c = chi[k]; + return gemmi::calculate_dihedral(p[c[0]], p[c[1]], p[c[2]], p[c[3]]) * 180.0 / PI; +} + +std::vector SideChain::Build(const std::vector &chis) const { + std::vector p = pos; + for (size_t k = 0; k < chi.size(); k++) { + const double angle = (chis[k] - Dihedral(p, k)) * PI / 180.0; + const int pivot = depth[chi[k][2]]; + for (size_t i = 0; i < p.size(); i++) + if (depth[i] > pivot) + p[i] = Rotate(p[i], p[chi[k][1]], p[chi[k][2]], angle); + } + return p; +} + +std::vector SideChain::Chis() const { + std::vector out; + for (size_t k = 0; k < chi.size(); k++) + out.push_back(Dihedral(pos, k)); + return out; +} + +std::optional MakeSideChain(const gemmi::Residue &res) { + auto it = ChiAtoms().find(res.name); + if (it == ChiAtoms().end()) + return std::nullopt; + if (std::any_of(res.atoms.begin(), res.atoms.end(), [](const gemmi::Atom &a) { return a.altloc != '\0'; })) + return std::nullopt; + SideChain sc; + for (const gemmi::Atom &a : res.atoms) { + sc.pos.push_back(a.pos); + sc.depth.push_back(SideChainDepth(a.name)); + } + for (const Chi &c : it->second) { + std::array idx{}; + for (int k = 0; k < 4; k++) { + const auto found = std::find_if(res.atoms.begin(), res.atoms.end(), + [&](const gemmi::Atom &a) { return a.name == c[k]; }); + if (found == res.atoms.end()) + return std::nullopt; + idx[k] = static_cast(found - res.atoms.begin()); + } + sc.chi.push_back(idx); + } + return sc; +} + +std::vector> SideChainRotamers(const std::string &resname, size_t n_chi) { + std::vector> out = {{}}; + for (size_t k = 0; k < n_chi; k++) { + std::vector values = {-60, 60, 180}; + if (k + 1 == n_chi && PlanarEnd(resname)) { + values.clear(); + for (int deg = -150; deg <= 180; deg += 30) + values.push_back(deg); + } + std::vector> next; + for (const std::vector &head : out) + for (double v : values) { + next.push_back(head); + next.back().push_back(v); + } + out = std::move(next); + } + return out; +} + +SideChainOverlap OverlapOf(gemmi::NeighborSearch &ns, const gemmi::Model &model, const gemmi::UnitCell &cell, + const gemmi::Residue &res, int chain_idx, int residue_idx, const SideChain &sc, + const std::vector &p, const std::vector &linked) { + SideChainOverlap out; + for (size_t i = 0; i < p.size(); i++) { + const gemmi::Atom &a = res.atoms[i]; + if (sc.depth[i] < 2 || a.is_hydrogen()) + continue; + ns.for_each(p[i], '\0', REACH, [&](const gemmi::NeighborSearch::Mark &m, double) { + if (m.image_idx == 0 && m.chain_idx == chain_idx && m.residue_idx == residue_idx) + return; // its own atoms; its own symmetry copy is another residue to it + const gemmi::Atom &b = model.chains[m.chain_idx].residues[m.residue_idx].atoms[m.atom_idx]; + if (b.is_hydrogen() || !(b.occ > 0) || b.element.is_metal()) + return; + if (std::find(linked.begin(), linked.end(), &b) != linked.end()) + return; + if (a.altloc && b.altloc && a.altloc != b.altloc) + return; + if (a.element == gemmi::El::S && b.element == gemmi::El::S) + return; // a disulfide, where it is one, is no clash + const double d = cell.find_nearest_pbc_image(p[i], b.pos, m.image_idx).dist(); + const double d0 = Polar(a.element.elem) && Polar(b.element.elem) + ? HBOND_CONTACT + : VdwRadius(a.element.elem) + VdwRadius(b.element.elem); + if (d < d0) { + out.sum += (d0 - d) * (d0 - d); + out.deepest = std::max(out.deepest, d0 - d); + } + }); + } + return out; +} diff --git a/image_analysis/structure_refinement/SideChainRotamers.h b/image_analysis/structure_refinement/SideChainRotamers.h new file mode 100644 index 000000000..c86490b75 --- /dev/null +++ b/image_analysis/structure_refinement/SideChainRotamers.h @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// A residue's side chain turned about its chi bonds, and its overlap with the rest of the model: what ClashRelief +// (the side chains turned clear of another residue before refining) and SideChainField (the side chains searched +// against the likelihood) share. + +#include +#include +#include +#include + +#include +#include +#include + +// How far along the side chain an atom is, hydrogens by their parent's letter: CB 1, gamma 2, delta 3, epsilon 4, +// zeta 5, eta 6; the main chain and its hydrogens 0. +int SideChainDepth(const std::string &name); + +struct SideChain { + std::vector pos; // the residue's atoms, as given + std::vector depth; + std::vector> chi; // atom indices of each chi + double Dihedral(const std::vector &p, size_t k) const; + // The residue's atoms with the side chain at these chi angles (degrees), turned from the given ones. + std::vector Build(const std::vector &chis) const; + std::vector Chis() const; +}; + +// The residue's chi atoms, found by name; none for a residue without chi angles (Gly, Ala, Pro, Cys and anything +// but an amino acid), where any is missing (an incomplete side chain stays as it is) or the residue has alternative +// conformations (its atoms would be found by name once, for both). +std::optional MakeSideChain(const gemmi::Residue &res); + +// The rotamers tried: each chi at the staggered angles, the last one of a planar group every 30 degrees. +std::vector> SideChainRotamers(const std::string &resname, size_t n_chi); + +// The overlaps of a residue's turned atoms (depth 2 and beyond, heavy) with the rest of the model: the sum of +// squared overlaps and the deepest one. Its own atoms, hydrogens, absent atoms (occupancy 0), metals and the +// other conformations of an alternative are no contact; nor is a covalent partner (`linked`: a LINK's, or a +// disulfide's). Polar pairs may come to the hydrogen-bond distance, others to the sum of their van der Waals radii. +struct SideChainOverlap { + double sum = 0, deepest = 0; +}; +SideChainOverlap OverlapOf(gemmi::NeighborSearch &ns, const gemmi::Model &model, const gemmi::UnitCell &cell, + const gemmi::Residue &res, int chain_idx, int residue_idx, const SideChain &sc, + const std::vector &p, const std::vector &linked); From 181e764e973fe53b7b88490636496a9e84b77b85 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 15:33:10 +0100 Subject: [PATCH 3/7] Refinement: the side chains as a discrete field searched against the likelihood in reciprocal space A new stage after the weight trajectory and the solvent build (SideChainField.h): every searchable side chain is a variable whose states are its staggered rotamers (planar ends every 30 degrees), the conformation it has, and absent (pruned to CB). The energy of a state is the refinement's own target with the rest held: -ln L over the working reflections at the fit as it stands, plus the repulsion term over the X-ray weight, plus, for Asn and Gln, the hydrogen bonds the amide can and cannot make (the two orientations fit the density alike). The structure factors are linear in the atoms, so a state's contribution is computed by direct summation (a dozen atoms over the group's operators, a CUDA kernel; a CPU evaluator for the other builds and the test) and the target is re-evaluated at every reflection for every state in one pass, with no map and no gradient. The copies of a residue the NCS relates are one variable, searched together in each rotamer (the NCS restraints of the step after pull a copy moved alone back) and each alone. The field is minimised by iterated conditional modes, small side chains first and arginine last; a move must beat the state as given by the log of the number of alternatives. One step at the trajectory's weights from the field is judged against the same step from the model as it was and kept where both its R-free and its -ln L of the free reflections are lower; the field never sees the free set. On the live cytochrome C placement (three copies, no haem) the field moves 3-12 side chains per run, 15 of 18 of them toward the crystal structure (Asn104, Glu22, Ile82 in all copies; the amide of Asn104 the right way round only with the hydrogen-bond term), the step from it lower on the free set in every run; the final R-free moves with the path (+0.001, -0.008, -0.022 on three jittered starts). The field costs ~0.8 s on the GPU plus the two steps; on a model whose side chains the data hold it moves nothing. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 12 + docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 21 + docs/RUGNUX_REPORT.md | 2 + .../structure_refinement/CMakeLists.txt | 3 + .../structure_refinement/Refinement.cpp | 161 +++++++ .../structure_refinement/Refinement.h | 8 + .../structure_refinement/SideChainField.cpp | 451 ++++++++++++++++++ .../structure_refinement/SideChainField.h | 146 ++++++ .../structure_refinement/SideChainFieldGPU.cu | 284 +++++++++++ rugnux/ResultReport.cpp | 2 + tests/CMakeLists.txt | 1 + tests/SideChainFieldTest.cpp | 199 ++++++++ 12 files changed, 1290 insertions(+) create mode 100644 image_analysis/structure_refinement/SideChainField.cpp create mode 100644 image_analysis/structure_refinement/SideChainField.h create mode 100644 image_analysis/structure_refinement/SideChainFieldGPU.cu create mode 100644 tests/SideChainFieldTest.cpp diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 0a8ddab27..824ebffc3 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -712,6 +712,18 @@ subspace its site symmetry leaves unchanged. R. W. Grosse-Kunstleve and P. D. Ad deriving crystallographic space-group information. II. Treatment of special positions" (2002), Acta Cryst. A58, 60-65 [doi:10.1107/S0108767301016658](https://doi.org/10.1107/S0108767301016658). +**Side chains as a discrete field** — the rotamers of the side chains searched as a combinatorial problem rather +than by a gradient, with the likelihood of the working reflections as the energy and iterated conditional modes +(each residue in turn put in its best state given the others) as the solver; small residues first and arginine +last, as PDB-REDO rebuilds. J. Desmet, M. De Maeyer, B. Hazes and I. Lasters, "The dead-end elimination theorem +and its use in protein side-chain positioning" (1992), Nature 356, 539-542 +[doi:10.1038/356539a0](https://doi.org/10.1038/356539a0); J. Besag, "On the statistical analysis of dirty +pictures" (1986), J. R. Statist. Soc. B48, 259-279 +[doi:10.1111/j.2517-6161.1986.tb01412.x](https://doi.org/10.1111/j.2517-6161.1986.tb01412.x); R. P. Joosten, +K. Joosten, S. X. Cohen, G. Vriend and A. Perrakis, "Automatic rebuilding and optimization of crystallographic +structures in the Protein Data Bank" (2011), Bioinformatics 27, 3392-3398 +[doi:10.1093/bioinformatics/btr590](https://doi.org/10.1093/bioinformatics/btr590). + ### Solvent, ions and unmodelled density **Unmodelled density** — the connected regions of the difference map above a cutoff, found by flood diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 827f38946..612f4cdf6 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -331,6 +331,27 @@ are: stays within 0.001 of the best so far, and the trajectory ends at its best step. On a deposited model, whose free set the depositor's refinement has already fitted, R-work falls while R-free rises, and the trajectory ends where it began. + Then, from where the trajectory (and the solvent build, §15.10) ended, **the side-chain field**: every side + chain the search can take - a complete amino-acid side chain at full occupancy, refinable, without alternative + conformations and not bound by a LINK - is a discrete variable whose states are its staggered rotamers (the + terminal planar group every 30°), the conformation it has, and *absent* (pruned to CB, its atoms at occupancy 0: + the honest answer where no rotamer fits). The energy of a state is the refinement's own target with the rest of + the model held: $-\ln L$ over the working reflections at the scale, the solvent and the likelihood parameters + as they stand, plus the repulsion term against the rest of the model (symmetry mates and the other copies + included) over the X-ray weight. The structure factors are linear in the atoms, so with the side chain's own + contribution computed by direct summation - a dozen atoms over the group's operators - a change of state + moves F by $dF_{new} - dF_{old}$ and the target is re-evaluated at every reflection for every state in one pass + on the GPU, microseconds per state, no map and no gradient. The field is minimised by iterated conditional + modes: the residues in turn, each put in its state of least energy given the others, small side chains first + and arginine last, until a sweep changes nothing. An FFT gradient cannot carry a side chain 1.5 Å at 1.7 Å + resolution; the field, a search, can, and it is scored by the likelihood rather than by a map, which on an + incomplete model rewards whatever fills unmodelled density. The free reflections are never seen by the field: + one step at the trajectory's weights is then taken from it and judged against the same step from the model as + it was, kept only where both its R-free and its $-\ln L$ of the free reflections are lower and the geometry + within bounds, undone otherwise (a model whose side chains the data already hold moves nothing and costs the + sweep alone). The log names each side chain moved with its chi angles before and after and the energy the move + took off (`REFINE_SIDE_CHAIN_FIELD_MOVED`, `REFINE_SIDE_CHAIN_FIELD_PRUNED`). Twinned data and a model with + anisotropic atoms are refined without the field. 3. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved was never kept on 55 refinements, and taking it away changed no choice). 4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U diff --git a/docs/RUGNUX_REPORT.md b/docs/RUGNUX_REPORT.md index fd07ac934..95b351548 100644 --- a/docs/RUGNUX_REPORT.md +++ b/docs/RUGNUX_REPORT.md @@ -1549,6 +1549,8 @@ Where a model was placed and refined ([Refinement](RUGNUX_MODEL_AND_REFERENCE.md | `REFINE_RMSD_BONDS_START`, `REFINE_RMSD_ANGLES_START` | *dev*. The same for the model as given | | `REFINE_CLASHES` | Pairs of atoms in the refined model more than 0.4 Å closer than their contact distance (symmetry mates included) | | `REFINE_SIDE_CHAINS_TURNED` | Side chains turned to another rotamer before the refinement because an atom of theirs stood more than 2 Å inside its contact distance to another residue (within about 1.3 Å of one of its atoms) (a copy's or a symmetry mate's), as molecular replacement leaves a predicted model's; the log names them | +| `REFINE_SIDE_CHAIN_FIELD_MOVED` | Side chains the side-chain field (§15.7) put in another rotamer, where the free reflections kept the step from the field (0 where they did not); the log names them with their chi angles before and after | +| `REFINE_SIDE_CHAIN_FIELD_PRUNED` | Side chains the field pruned to CB (their atoms left at occupancy 0), under the same condition | | `REFINE_XRAY_WEIGHT` | *dev*. The X-ray weight of the positions that was kept | | `REFINE_NCS_GROUPS` | Molecules present more than once in the model (copies found by sequence alignment, §15.5 of the refinement reference). Present only where there are any, as are the other `REFINE_NCS_` keys | | `REFINE_NCS_COPIES` | The chains of those molecules | diff --git a/image_analysis/structure_refinement/CMakeLists.txt b/image_analysis/structure_refinement/CMakeLists.txt index ecc12bbef..18dc18273 100644 --- a/image_analysis/structure_refinement/CMakeLists.txt +++ b/image_analysis/structure_refinement/CMakeLists.txt @@ -28,6 +28,9 @@ ADD_LIBRARY(JFJochStructureRefinement STATIC SolventSites.cpp SolventSites.h SolventBuild.cpp SolventBuild.h ClashRelief.cpp ClashRelief.h + SideChainRotamers.cpp SideChainRotamers.h + SideChainField.cpp SideChainField.h + $<$:SideChainFieldGPU.cu> $<$:RefinementGPU.cpp> $<$:RefinementGPU.cu> RefinementGPU.h RefinementGPUEngine.h diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index bef8159eb..353b3aa27 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -43,6 +43,7 @@ #include "RefinementMinimiser.h" #include "RefinementNcs.h" #include "RefinementXray.h" +#include "SideChainField.h" #ifdef JFJOCH_USE_CUDA #include "../../common/CUDAWrapper.h" #include "RefinementGPU.h" @@ -3173,6 +3174,166 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vectorFcalc(ctx.Model(), ctx.fcalc); + FieldReflections fr; + for (size_t m = 0; m < ctx.refl.Size(); m++) { + const gemmi::Miller &h = ctx.refl.hkl[m]; + fr.hkl.push_back({static_cast(h[0]), static_cast(h[1]), static_cast(h[2])}); + fr.stol2.push_back(static_cast(ctx.refl.inv_d2[m] / 4.0)); + fr.fo.push_back(ctx.refl.fo[m]); + fr.d.push_back(ctx.rows.d[m]); + fr.var.push_back(ctx.rows.var[m]); + fr.centric.push_back(ctx.refl.centric[m]); + fr.work.push_back(!ctx.refl.free[m]); + fr.k_total.push_back(ctx.rows.k_total[m]); + fr.fmask.push_back(ctx.rows.fmask[m]); + } + // the form factors of the model's elements, each once + std::vector elements; + std::vector element_list; + std::vector element_of(ctx.atoms.size()); + for (size_t i = 0; i < ctx.atoms.size(); i++) { + const gemmi::El el = ctx.atoms[i]->element.elem; + auto it = std::find(element_list.begin(), element_list.end(), el); + if (it == element_list.end()) { + element_list.push_back(el); + elements.push_back(gemmi::IT92::get(el, 0).coefs); + it = element_list.end() - 1; + } + element_of[i] = static_cast(it - element_list.begin()); + } + std::unique_ptr evaluator; +#ifdef JFJOCH_USE_CUDA + if (ctx.engine->OnGPU()) + evaluator = MakeSideChainFieldEvaluatorGPU(ctx.device, fr, FieldOperators(sg), elements); +#endif + if (!evaluator) + evaluator = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(sg), elements, ctx.nthreads); + std::vector> f_total(ctx.fcalc.begin(), ctx.fcalc.begin() + ctx.refl.Size()); + evaluator->SetTotal(f_total); + // the copies of each residue, from the NCS anchors (corresponding C-alpha atoms): per residue, numbered + // over the model in order, the lowest ordinal among its copies as the group + std::vector ncs_group; + if (ncs_p != nullptr) { + std::vector residue_of; + int ordinal = 0; + for (const gemmi::Chain &ch : ctx.Model().chains) + for (const gemmi::Residue &r : ch.residues) { + residue_of.insert(residue_of.end(), r.atoms.size(), ordinal); + ordinal++; + } + ncs_group.assign(ordinal, -1); + for (const std::vector> &pairs : ncs.anchors) + for (const std::array &pair : pairs) { + const int a = residue_of[pair[0]], b = residue_of[pair[1]]; + const int ga = ncs_group[a] < 0 ? a : ncs_group[a], gb = ncs_group[b] < 0 ? b : ncs_group[b]; + const int g = std::min(ga, gb); + for (int &x : ncs_group) + if (x == ga || x == gb) + x = g; + ncs_group[a] = ncs_group[b] = g; + } + } + const SideChainFieldResult field = OptimiseSideChainField(ctx.st, element_of, topo.refinable, ncs_group, + state.trial.weight, *evaluator); + evaluator.reset(); + ctx.nthreads = slot_threads; + result.side_chain_field_candidates = field.candidates; + auto chis = [](const std::vector &v) { + std::string s; + for (double x : v) + s += fmt::format("{}{:.0f}", s.empty() ? "" : "/", x); + return s.empty() ? "absent" : s; + }; + std::string names; + int pruned = 0; + for (const SideChainFieldMove &m : field.moves) { + names += fmt::format("{}{} {} -> {} ({:.0f})", names.empty() ? "" : ", ", m.name, chis(m.chi_before), + chis(m.chi_after), m.energy_gain); + pruned += m.absent; + } + for (const SideChainFieldAlternative &a : field.alternatives) + if (a.gain > 1) + logger.Debug("Side-chain field: {} as given {}, best alternative {} would gain {:.1f}", a.name, + chis(a.chi_given), chis(a.chi_best), a.gain); + logger.Info("Side-chain field: {} residues searched, {} moved ({} pruned) in {} sweep{}, energy {:.1f} -> " + "{:.1f}, {:.2f} s{}{}", field.candidates, field.moves.size(), pruned, field.sweeps, + field.sweeps == 1 ? "" : "s", field.energy_before, field.energy_after, + std::chrono::duration(std::chrono::steady_clock::now() - field_start).count(), + names.empty() ? "" : ": ", names); + if (!field.moves.empty()) { + // one step at the trajectory's weights, from the field and from the model as it is + auto one_step = [&](const std::vector &pos) { + std::vector o = + RunTrials(trial_slots, {{state.trial.weight, state.trial.adp_weight, state.trial.ncs_sigma}}, + refinable, pos, state.b, state.aniso, nullptr, settings.macro_cycles, settings, ncs_p); + return std::move(o[0]); + }; + std::vector moved = state.pos; + std::vector pruned_atoms; + for (const SideChainFieldMove &m : field.moves) { + for (const auto &[a, p] : m.atoms) + moved[a] = p; + pruned_atoms.insert(pruned_atoms.end(), m.pruned.begin(), m.pruned.end()); + } + const TrialOutcome unchanged = one_step(state.pos); + // the pruned atoms absent in every slot: no density, no mask, no contact, not refined + std::vector topo_before(trial_slots.size()); + for (size_t sl = 0; sl < trial_slots.size(); sl++) { + topo_before[sl] = *trial_slots[sl].topo; + for (int i : pruned_atoms) { + trial_slots[sl].ctx->atoms[i]->occ = 0; + trial_slots[sl].topo->refinable[i] = 0; + } + } + const std::vector refinable_before = refinable; + refinable.clear(); + for (size_t i = 0; i < ctx.atoms.size(); i++) + if (topo.refinable[i]) + refinable.push_back(static_cast(i)); + TrialOutcome after = one_step(moved); + const bool kept = after.ran && after.trial.within_bounds && after.trial.r_free < unchanged.trial.r_free && + after.trial.nll_free < unchanged.trial.nll_free; + logger.Info("Side-chain field: refined from the field R-free {:.4f}, -LL free {:.1f}; from the model as it " + "was R-free {:.4f}, -LL free {:.1f}: {}", after.trial.r_free, after.trial.nll_free, + unchanged.trial.r_free, unchanged.trial.nll_free, kept ? "kept" : "not kept"); + if (kept) { + result.side_chain_field_kept = true; + result.side_chain_field_moved = static_cast(field.moves.size()) - pruned; + result.side_chain_field_pruned = pruned; + state = std::move(after); + from_best.clear(); + } else { + for (size_t sl = 0; sl < trial_slots.size(); sl++) { + for (int i : pruned_atoms) + trial_slots[sl].ctx->atoms[i]->occ = 1; + *trial_slots[sl].topo = topo_before[sl]; + } + refinable = refinable_before; + } + } + } + // 3. The final weights: a short ladder from where the trajectory ended, tighter geometry first, its choice // kept where it displaces that. A rung the step from there tried already is taken from that step. if (state.ran && !ctx.Cancelled()) { diff --git a/image_analysis/structure_refinement/Refinement.h b/image_analysis/structure_refinement/Refinement.h index 790e94281..4145c5532 100644 --- a/image_analysis/structure_refinement/Refinement.h +++ b/image_analysis/structure_refinement/Refinement.h @@ -70,6 +70,10 @@ struct RefinementSettings { // solvent audited (SolventSites.h); off (the default, rugnux --refine-solvent turns it on) the solvent stays // as given and is not audited. bool solvent = false; + // After the weight trajectory (and the solvent build), the side chains searched as a discrete field against the + // likelihood of the working reflections (SideChainField.h), and one step from the field kept where the free + // reflections say so. + bool side_chain_field = true; // After the weights are chosen, the displacement model decided on the free set: the isotropic B refined on, TLS // groups (the model's own, one per chain, the chains cut into TLSMD segments) with a residual B per atom, and an // anisotropic U per atom with rigid-bond and similarity restraints (false: the isotropic B, or the model's U held). @@ -197,6 +201,10 @@ struct RefinementResult { int restrained_residues = 0, fixed_residues = 0; std::vector fixed_names; int side_chains_turned = 0; // out of an overlap with another residue before the refinement (ClashRelief.h) + // The side-chain field (settings.side_chain_field): residues searched, moved to another rotamer or pruned to CB + // by the field, and whether the free reflections kept the step from it (the counts are 0 where they did not). + int side_chain_field_candidates = 0, side_chain_field_moved = 0, side_chain_field_pruned = 0; + bool side_chain_field_kept = false; double k_sol = 0, b_sol = 0; double mean_fom = 0; diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp new file mode 100644 index 000000000..e87c7de45 --- /dev/null +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -0,0 +1,451 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include "SideChainField.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "../../common/ParallelFor.h" +#include "RefinementMath.h" +#include "SideChainRotamers.h" + +namespace { + +constexpr double TWO_PI = 6.283185307179586; +constexpr double REPULSION_SIGMA = 0.2; // A, the refinement's (RefinementRestraints.cpp) +constexpr double REACH = 4.0; // A: the farthest pair that can overlap (OverlapOf) +constexpr int MAX_SWEEPS = 6; +// The amide of Asn and Gln either way round fits the density alike at the resolutions of protein crystals: the +// hydrogen bonds its O and N can make decide, as PDB-REDO's flips do (Joosten et al. (2011) Bioinformatics 27, +// 3392-3398). Per partner within HBOND_REACH: a donor to the O or an acceptor to the N counts for the state, two +// donors or two acceptors against it, HBOND_WEIGHT each in the units of -ln L. +constexpr double HBOND_REACH = 3.3; // A +constexpr double HBOND_WEIGHT = 2.0; + +enum class HBond { None, Donor, Acceptor, Both }; + +HBond HBondType(const gemmi::Residue &res, const gemmi::Atom &a) { + if (res.is_water()) + return HBond::Both; + const std::string &n = a.name; + if (n == "N") + return res.name == "PRO" ? HBond::None : HBond::Donor; + if (n == "O" || n == "OXT") + return HBond::Acceptor; + if (res.name == "ASP" || res.name == "GLU" || res.name == "ASN" || res.name == "GLN") + return a.element == gemmi::El::O ? HBond::Acceptor : a.element == gemmi::El::N ? HBond::Donor : HBond::None; + if (res.name == "LYS" || res.name == "ARG" || res.name == "TRP") + return a.element == gemmi::El::N ? HBond::Donor : HBond::None; + if (res.name == "SER" || res.name == "THR" || res.name == "TYR" || res.name == "HIS" || res.name == "CYS") + return a.element == gemmi::El::O || a.element == gemmi::El::N || a.element == gemmi::El::S ? HBond::Both : HBond::None; + if (res.name == "MET") + return a.element == gemmi::El::S ? HBond::Acceptor : HBond::None; + return a.element == gemmi::El::O || a.element == gemmi::El::N ? HBond::Both : HBond::None; +} + +// The hydrogen bonds the amide of an Asn or Gln at positions `p` can make, less those it cannot: partners of its O +// and N in the rest of the model within HBOND_REACH, by type. 0 for any other residue. +int AmideHBonds(gemmi::NeighborSearch &ns, const gemmi::Model &model, const gemmi::UnitCell &cell, + const gemmi::Residue &res, int chain_idx, int residue_idx, const std::vector &p) { + if (res.name != "ASN" && res.name != "GLN") + return 0; + int score = 0; + for (size_t i = 0; i < p.size(); i++) { + const HBond own = HBondType(res, res.atoms[i]); + if (SideChainDepth(res.atoms[i].name) < 3 || (own != HBond::Donor && own != HBond::Acceptor)) + continue; + ns.for_each(p[i], '\0', HBOND_REACH, [&](const gemmi::NeighborSearch::Mark &m, double) { + if (m.image_idx == 0 && m.chain_idx == chain_idx && m.residue_idx == residue_idx) + return; + const gemmi::Residue &other = model.chains[m.chain_idx].residues[m.residue_idx]; + const gemmi::Atom &b = other.atoms[m.atom_idx]; + if (!(b.occ > 0)) + return; + const HBond type = HBondType(other, b); + if (type == HBond::None || cell.find_nearest_pbc_image(p[i], b.pos, m.image_idx).dist() > HBOND_REACH) + return; + if (type == HBond::Both || type != own) + score++; + else + score--; + }); + } + return score; +} + +class EvaluatorCPU : public SideChainFieldEvaluator { +public: + EvaluatorCPU(const FieldReflections &refl, const std::vector> &ops, + const std::vector &elements, size_t nthreads) + : refl_(refl), elements_(elements), nthreads_(nthreads), n_ops_(ops.size()) { + // per row and operator: R^T h and h . t, so that the phase of an atom at x is 2 pi (R^T h . x + h . t) + hop_.resize(refl.Size() * ops.size()); + for (size_t m = 0; m < refl.Size(); m++) { + const std::array &h = refl.hkl[m]; + for (size_t k = 0; k < ops.size(); k++) { + const std::array &op = ops[k]; + hop_[m * ops.size() + k] = {op[0] * h[0] + op[3] * h[1] + op[6] * h[2], + op[1] * h[0] + op[4] * h[1] + op[7] * h[2], + op[2] * h[0] + op[5] * h[1] + op[8] * h[2], + op[9] * h[0] + op[10] * h[1] + op[11] * h[2]}; + } + } + } + + void SetTotal(const std::vector> &f) override { total_ = f; } + + std::vector> Contribution(const FieldStates &states, int s) override { + std::vector> f(refl_.Size()); + const int n = static_cast(refl_.Size()); + ParallelChunks(n, ThreadsForWork(n, nthreads_, 1024), [&](int lo, int hi) { + for (int m = lo; m < hi; m++) + f[m] = Sum(states, s, m); + }); + return f; + } + + std::vector Evaluate(const FieldStates &states) override { + const std::vector>> df = All(states); + std::vector nll(states.Combos(), 0.0); + for (size_t c = 0; c < states.Combos(); c++) { + double sum = 0; + for (size_t m = 0; m < refl_.Size(); m++) { + if (!refl_.work[m]) + continue; + std::complex f = total_[m]; + for (int s : states.current) + f -= df[s][m]; + for (int k = states.combo_start[c]; k < states.combo_start[c + 1]; k++) + f += df[states.combo_states[k]][m]; + sum += Term(m, f); + } + nll[c] = sum; + } + return nll; + } + + void Move(const FieldStates &states, int to) override { + const std::vector>> df = All(states); + for (size_t m = 0; m < refl_.Size(); m++) { + for (int s : states.current) + total_[m] -= df[s][m]; + for (int k = states.combo_start[to]; k < states.combo_start[to + 1]; k++) + total_[m] += df[states.combo_states[k]][m]; + } + } + +private: + std::vector>> All(const FieldStates &states) { + std::vector>> df; + for (size_t s = 0; s < states.Count(); s++) + df.push_back(Contribution(states, static_cast(s))); + return df; + } + std::complex Sum(const FieldStates &states, int s, size_t m) const { + double re = 0, im = 0; + for (int a = states.start[s]; a < states.start[s + 1]; a++) { + const FieldAtom &at = states.atoms[a]; + const FormFactor &ff = elements_[at.element]; + const double stol2 = refl_.stol2[m]; + double f = ff[8]; + for (int i = 0; i < 4; i++) + f += ff[i] * std::exp(-ff[4 + i] * stol2); + f *= at.occ * std::exp(-at.b * stol2); + for (size_t k = 0; k < n_ops_; k++) { + const std::array &h = hop_[m * n_ops_ + k]; + const double phase = TWO_PI * (h[0] * at.x + h[1] * at.y + h[2] * at.z + h[3]); + re += f * std::cos(phase); + im += f * std::sin(phase); + } + } + return {static_cast(re), static_cast(im)}; + } + double Term(size_t m, std::complex f_atoms) const { + const double fc = std::abs(refl_.k_total[m] * (f_atoms + refl_.fmask[m])); + double dfc = 0; + return refine_math::AmplitudeNegLogLikelihood(refl_.fo[m], fc, refl_.d[m], refl_.var[m], + refl_.centric[m], dfc); + } + + FieldReflections refl_; + std::vector elements_; + size_t nthreads_, n_ops_; + std::vector> hop_; + std::vector> total_; +}; + +// The residues the field searches. +struct Candidate { + int chain, residue; // indices in the model + int ordinal; // the residue's number over the model in order + int first_atom; // index of the residue's first atom in the model's atom list + SideChain sc; + std::string name; +}; + +std::vector Candidates(const gemmi::Structure &st, const std::vector &refinable) { + const gemmi::Model &model = st.models[0]; + std::vector> links; + for (const gemmi::Connection &conn : st.connections) { + const gemmi::const_CRA a = model.find_cra(conn.partner1), b = model.find_cra(conn.partner2); + if (a.atom != nullptr && b.atom != nullptr) + links.emplace_back(a.atom, b.atom); + } + std::vector out; + int n = 0, ordinal = 0; + for (size_t ci = 0; ci < model.chains.size(); ci++) + for (size_t ri = 0; ri < model.chains[ci].residues.size(); ri++, ordinal++) { + const gemmi::Residue &res = model.chains[ci].residues[ri]; + const int first = n; + n += static_cast(res.atoms.size()); + std::optional sc = MakeSideChain(res); + if (!sc) + continue; + bool ok = true; + for (size_t i = 0; i < res.atoms.size(); i++) + if (!res.atoms[i].is_hydrogen() && (!refinable[first + i] || !(res.atoms[i].occ >= 1.0f))) + ok = false; + for (const auto &[a, b] : links) + for (const gemmi::Atom &x : res.atoms) + if (&x == a || &x == b) + ok = false; // a side chain a LINK binds stays + if (!ok) + continue; + out.push_back({static_cast(ci), static_cast(ri), ordinal, first, std::move(*sc), + res.name + " " + model.chains[ci].name + res.seqid.str()}); + } + return out; +} + +// The field's variables: a residue, or the copies of one the NCS relates (ncs_group: per residue ordinal, -1 for +// none) - which move together or one at a time; small side chains first, arginine last. +std::vector> Variables(const std::vector &candidates, const gemmi::Model &model, + const std::vector &ncs_group) { + std::vector> out; + std::map of_group; + for (size_t i = 0; i < candidates.size(); i++) { + const int g = ncs_group.empty() ? -1 : ncs_group[candidates[i].ordinal]; + const gemmi::Residue &res = model.chains[candidates[i].chain].residues[candidates[i].residue]; + if (g >= 0 && of_group.count(g) && + model.chains[candidates[out[of_group[g]][0]].chain].residues[candidates[out[of_group[g]][0]].residue].name == + res.name) { + out[of_group[g]].push_back(static_cast(i)); + continue; + } + if (g >= 0) + of_group[g] = out.size(); + out.push_back({static_cast(i)}); + } + std::stable_sort(out.begin(), out.end(), [&](const std::vector &a, const std::vector &b) { + auto key = [&](const Candidate &c) { + return model.chains[c.chain].residues[c.residue].name == "ARG" ? 100 : static_cast(c.sc.chi.size()); + }; + return key(candidates[a[0]]) < key(candidates[b[0]]); + }); + return out; +} + +} // namespace + +std::vector> FieldOperators(const gemmi::SpaceGroup &sg) { + std::vector> out; + for (const gemmi::Op &op : sg.operations()) { + std::array o{}; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) + o[3 * i + j] = static_cast(op.rot[i][j]) / static_cast(gemmi::Op::DEN); + o[9 + i] = static_cast(op.tran[i]) / static_cast(gemmi::Op::DEN); + } + out.push_back(o); + } + return out; +} + +std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements, + size_t nthreads) { + return std::make_unique(refl, ops, elements, nthreads); +} + +std::vector FieldAtomsOf(const gemmi::Residue &res, const SideChain &sc, const std::vector &p, + const gemmi::UnitCell &cell, const std::vector &element_of, int first_atom) { + std::vector out; + for (size_t i = 0; i < res.atoms.size(); i++) + if (sc.depth[i] >= 2) { + const gemmi::Fractional f = cell.fractionalize(p[i]); + out.push_back({static_cast(f.x), static_cast(f.y), static_cast(f.z), res.atoms[i].b_iso, + 1.0f, element_of[first_atom + i]}); + } + return out; +} + +SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const std::vector &element_of, + const std::vector &refinable, const std::vector &ncs_group, + double weight, SideChainFieldEvaluator &evaluator) { + SideChainFieldResult result; + gemmi::Structure st = given; + gemmi::Model &model = st.models[0]; + gemmi::UnitCell cell = st.cell; + cell.set_cell_images_from_spacegroup(st.find_spacegroup()); + std::vector candidates = Candidates(st, refinable); + result.candidates = static_cast(candidates.size()); + if (candidates.empty()) + return result; + const std::vector> variables = Variables(candidates, model, ncs_group); + // the repulsion term of the refinement's target, in the units of -ln L: ((d0 - d) / sigma)^2 summed, over the + // X-ray weight + const double repulsion = 1.0 / (REPULSION_SIGMA * REPULSION_SIGMA * weight); + gemmi::NeighborSearch ns(model, cell, REACH + 1); + ns.populate(/*include_h=*/false); + std::map moved; // candidate -> entry in result.moves + // the residues near a move of the last sweep: what the next sweep visits again (the first visits every residue) + std::set> near, near_last; + auto note_near = [&](const gemmi::Position &pos) { + ns.for_each(pos, '\0', REACH + 2, [&](const gemmi::NeighborSearch::Mark &m, double) { + near.insert({m.chain_idx, m.residue_idx}); + }); + }; + + for (int sweep = 1; sweep <= MAX_SWEEPS; sweep++) { + bool any = false; + near_last.swap(near); + near.clear(); + for (const std::vector &var : variables) { + const size_t k = var.size(); + if (sweep > 1 && std::none_of(var.begin(), var.end(), [&](int i) { + return near_last.count({candidates[i].chain, candidates[i].residue}) > 0; + })) + continue; + // per copy: its positions as it is and in each rotamer (the same list for every copy: one residue + // type), the chi angles of each, and the geometry's share of each one's energy: its overlap with the + // rest of the model, and the hydrogen bonds of an amide + std::vector>> positions(k); + std::vector> overlap(k); + std::vector> chis; + for (size_t c = 0; c < k; c++) { + Candidate &cand = candidates[var[c]]; + const gemmi::Residue &res = model.chains[cand.chain].residues[cand.residue]; + cand.sc.pos.clear(); + for (const gemmi::Atom &a : res.atoms) + cand.sc.pos.push_back(a.pos); + positions[c] = {cand.sc.pos}; + if (c == 0) + chis = {cand.sc.Chis()}; + for (const std::vector &chi : SideChainRotamers(res.name, cand.sc.chi.size())) { + positions[c].push_back(cand.sc.Build(chi)); + if (c == 0) + chis.push_back(chi); + } + for (const std::vector &p : positions[c]) + overlap[c].push_back(repulsion * OverlapOf(ns, model, cell, res, cand.chain, cand.residue, cand.sc, p, {}).sum - + HBOND_WEIGHT * AmideHBonds(ns, model, cell, res, cand.chain, cand.residue, p)); + overlap[c].push_back(0); // absent + } + const int n_pos = static_cast(positions[0].size()), absent = n_pos; + // the states: per copy its positions (c * n_pos + p); the combinations as assignments, per copy the index + // of its positions (absent beyond them): as it is, the copies together in each rotamer and absent, then + // each copy alone in each + FieldStates states; + for (size_t c = 0; c < k; c++) { + const Candidate &cand = candidates[var[c]]; + for (int p = 0; p < n_pos; p++) + states.Add(FieldAtomsOf(model.chains[cand.chain].residues[cand.residue], cand.sc, positions[c][p], cell, + element_of, cand.first_atom)); + states.current.push_back(static_cast(c) * n_pos); + } + std::vector> assign = {std::vector(k, 0)}; + for (int p = 1; p <= absent; p++) + assign.emplace_back(k, p); + if (k > 1) + for (size_t c = 0; c < k; c++) + for (int p = 1; p <= absent; p++) { + assign.emplace_back(k, 0); + assign.back()[c] = p; + } + for (const std::vector &a : assign) { + std::vector combo; + for (size_t c = 0; c < k; c++) + if (a[c] != absent) + combo.push_back(static_cast(c) * n_pos + a[c]); + states.AddCombo(combo); + } + std::vector energy = evaluator.Evaluate(states); + // the alternatives are many and the best of them is chosen: a move must beat the state as given by the + // log of their number, what the best of as many noise draws would gain + const double occam = std::log(static_cast(assign.size() - 1)); + for (size_t s = 0; s < assign.size(); s++) { + for (size_t c = 0; c < k; c++) + energy[s] += overlap[c][assign[s][c]]; + if (s > 0) + energy[s] += occam; + } + const int best = static_cast(std::min_element(energy.begin(), energy.end()) - energy.begin()); + if (sweep == 1 && &var == &variables.front()) + result.energy_before = energy[0]; + if (sweep == 1) { + const int alt = static_cast(std::min_element(energy.begin() + 1, energy.end()) - energy.begin()); + result.alternatives.push_back({candidates[var[0]].name + (k > 1 ? fmt::format(" x{}", k) : ""), + energy[0] - energy[alt] + occam, chis[0], + assign[alt][0] == absent ? std::vector() : chis[assign[alt][0]]}); + } + if (best == 0 || !(energy[best] < energy[0])) + continue; + // moved: the field, the model and the contacts the others see + evaluator.Move(states, best); + int changed = 0; + for (size_t c = 0; c < k; c++) + changed += assign[best][c] != 0; + for (size_t c = 0; c < k; c++) { + const int p = assign[best][c]; + if (p == 0) + continue; + Candidate &cand = candidates[var[c]]; + gemmi::Residue &res = model.chains[cand.chain].residues[cand.residue]; + SideChainFieldMove &m = moved.count(var[c]) ? result.moves[moved[var[c]]] : result.moves.emplace_back(); + if (m.name.empty()) { + moved[var[c]] = result.moves.size() - 1; + m.name = cand.name; + m.chi_before = cand.sc.Chis(); + } + m.energy_gain += (energy[0] - energy[best]) / changed; + m.absent = p == absent; + m.chi_after = m.absent ? std::vector() : chis[p]; + m.atoms.clear(); + m.pruned.clear(); + for (size_t i = 0; i < res.atoms.size(); i++) + if (cand.sc.depth[i] >= 2) { + if (!res.atoms[i].is_hydrogen()) { + note_near(res.atoms[i].pos); + note_near(positions[c][p == absent ? 0 : p][i]); + } + if (m.absent) { + res.atoms[i].occ = 0; + m.pruned.push_back(cand.first_atom + static_cast(i)); + } else { + res.atoms[i].occ = 1; + res.atoms[i].pos = positions[c][p][i]; + m.atoms.emplace_back(cand.first_atom + static_cast(i), positions[c][p][i]); + } + } + } + ns = gemmi::NeighborSearch(model, cell, REACH + 1); + ns.populate(/*include_h=*/false); + any = true; + } + result.sweeps = sweep; + if (!any) + break; + } + result.energy_after = result.energy_before; + for (const SideChainFieldMove &m : result.moves) + result.energy_after -= m.energy_gain; + return result; +} diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h new file mode 100644 index 000000000..c8cc05b7b --- /dev/null +++ b/image_analysis/structure_refinement/SideChainField.h @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#pragma once + +// The side chains of a model as a discrete field, each residue in one of its rotamers (SideChainRotamers.h: the +// staggered chi angles, the planar ends every 30 degrees), as it was given, or absent (pruned to CB), searched +// against the refinement's own target in reciprocal space. The structure factors are linear in the atoms: with +// F_model = F_rest + dF_r(state_r), dF_r the direct-summation structure factors of residue r's side chain in that +// state, a residue's change of state moves F by dF_new - dF_old, and -ln L over the working reflections - the +// scale, the solvent and the likelihood's parameters held where the refinement fitted them - is re-evaluated +// at every reflection in one pass, a few microseconds on the GPU. The energy of the field is that -ln L plus the +// repulsion term of the refinement's geometry (the overlaps with the rest of the model, symmetry mates and the other +// copies included) at the refinement's own X-ray weight. The field is minimised by iterated conditional modes: the +// residues in turn, each put in the state of least energy given the others, small residues first and arginine last +// (the order PDB-REDO rebuilds in: Joosten et al. (2011) Bioinformatics 27, 3392-3398), until a sweep changes +// nothing. The free reflections are never seen: the caller refines from the field and judges on them. +// Side chains as a discrete field and ICM: following Desmet et al. (1992) Nature 356, 539-542 and +// Besag (1986) J. R. Statist. Soc. B48, 259-279. + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "SideChainRotamers.h" + +// What the evaluator needs of the data: the observed rows at the scale, the solvent and the likelihood's parameters +// of the last fit. F_model(row) = k_total (F_atoms + fmask). +struct FieldReflections { + std::vector> hkl; + std::vector stol2; // (sin theta / lambda)^2 + std::vector fo, d, var; + std::vector centric, work; // work: not a free reflection + std::vector k_total; + std::vector> fmask; // scaled + size_t Size() const { return hkl.size(); } +}; + +// An atom of a state: fractional position, isotropic B, occupancy, and its element's entry in the form-factor table. +struct FieldAtom { + float x, y, z, b, occ; + int element; +}; + +// A set of side-chain states, each a list of atoms (CSR: state s is atoms[start[s]..start[s+1])), and the +// combinations evaluated: each the states (one per copy of a residue) put in place of the current ones, F = +// F_total - sum dF(current) + sum dF(combination). +struct FieldStates { + std::vector atoms; + std::vector start = {0}; + std::vector current; + std::vector combo_states; + std::vector combo_start = {0}; + void Add(const std::vector &a) { + atoms.insert(atoms.end(), a.begin(), a.end()); + start.push_back(static_cast(atoms.size())); + } + void AddCombo(const std::vector &states) { + combo_states.insert(combo_states.end(), states.begin(), states.end()); + combo_start.push_back(static_cast(combo_states.size())); + } + size_t Count() const { return start.size() - 1; } + size_t Combos() const { return combo_start.size() - 1; } +}; + +// The IT92 form factor of an element: c + sum a_i exp(-b_i stol2), as a0..a3 b0..b3 c. +using FormFactor = std::array; + +// -ln L over the working reflections of F_total with one residue's side chain (or the copies of one) taken from +// its current states to each combination of states, by direct summation of the states' atoms over the group's +// operators. +class SideChainFieldEvaluator { +public: + virtual ~SideChainFieldEvaluator() = default; + // F of the model's atoms at the rows, as the refinement's engine computes them: what the field starts from. + virtual void SetTotal(const std::vector> &f) = 0; + // -ln L (work) per combination of `states`. + virtual std::vector Evaluate(const FieldStates &states) = 0; + // The field moved to combination `to`: F_total += sum dF(to) - sum dF(current). + virtual void Move(const FieldStates &states, int to) = 0; + // The direct-summation F of one state's atoms at the rows (for checking the sum against the engine). + virtual std::vector> Contribution(const FieldStates &states, int s) = 0; +}; + +// The group's operators as the evaluator takes them: rotation (row-major) and translation, fractional. +std::vector> FieldOperators(const gemmi::SpaceGroup &sg); + +std::unique_ptr MakeSideChainFieldEvaluatorCPU(const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements, + size_t nthreads); +#ifdef JFJOCH_USE_CUDA +// On `device`; throws JFJochException on a CUDA failure. +std::unique_ptr MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements); +#endif + +// A residue the field moved: its atoms (index in the model's atom list) at their new positions, or absent. +struct SideChainFieldMove { + std::string name; // "LYS A87" + std::vector chi_before, chi_after; // degrees (empty after: absent) + bool absent = false; + std::vector> atoms; // every atom of the residue beyond CB, hydrogens included + std::vector pruned; // the atoms put to occupancy 0 (absent only) + double energy_gain = 0; // -ln L (work) units, the field's energy lowered by the move +}; + +// What a residue's best alternative would gain over the state as given, before the margin for the number of +// alternatives (for the log: a gain below the margin moves nothing). +struct SideChainFieldAlternative { + std::string name; // "LYS A87", "ASN A104 x3" for the copies + double gain = 0; // -ln L (work) units + std::vector chi_given, chi_best; // degrees (empty best: absent); the first copy's +}; + +struct SideChainFieldResult { + int candidates = 0; // residues searched + std::vector alternatives; // per variable, in the order searched + int sweeps = 0; // the first over every residue, the others over those near a move + double energy_before = 0, energy_after = 0; + std::vector moves; +}; + +// `st`: the model as it is (hydrogens riding), with its cell and group; `element_of`: per atom in model order its +// entry in the form-factor table the evaluator was made with; `refinable`: per atom (RefinementTopology's); +// `ncs_group`: per residue (numbered over the model in order) the NCS group of its copies, -1 where it has none +// (or empty): the copies of a residue are one variable, searched in each rotamer together - the NCS restraints of +// the step after would pull a copy moved alone back - and each alone, for the copy whose environment differs (a +// crystal contact); `weight`: the X-ray term's weight in the refinement's target, which the repulsion term is +// divided by. The residues searched: amino acids with a complete side chain, no alternative conformations, every +// atom at full occupancy, refinable and not bound by a LINK. `st` is left as it was; the moves say what changed. +SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &st, const std::vector &element_of, + const std::vector &refinable, const std::vector &ncs_group, + double weight, SideChainFieldEvaluator &evaluator); + +// The state of the field's atoms of one residue (depth 2 and beyond, hydrogens included) at positions `p`, at full +// occupancy. +std::vector FieldAtomsOf(const gemmi::Residue &res, const SideChain &sc, const std::vector &p, + const gemmi::UnitCell &cell, const std::vector &element_of, int first_atom); diff --git a/image_analysis/structure_refinement/SideChainFieldGPU.cu b/image_analysis/structure_refinement/SideChainFieldGPU.cu new file mode 100644 index 000000000..37f042cc4 --- /dev/null +++ b/image_analysis/structure_refinement/SideChainFieldGPU.cu @@ -0,0 +1,284 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +// The side-chain field's evaluator on the GPU (SideChainField.h): the direct summation of a state's atoms over the +// group's operators at every row, and -ln L over the working rows of the total moved by it. Every sum runs in a +// fixed order: the same states give the same energies, bit for bit. + +#include + +#include "../../common/JFJochException.h" +#include "RefinementMath.h" +#include "SideChainField.h" + +namespace { + +constexpr float TWO_PI = 6.283185307179586f; +constexpr int THREADS = 256; + +void cuda_err(cudaError_t val) { + if (val != cudaSuccess) + throw JFJochException(JFJochExceptionCategory::GPUCUDAError, cudaGetErrorString(val)); +} + +class DeviceGuard { +public: + explicit DeviceGuard(int device) { + cuda_err(cudaGetDevice(&previous_)); + cuda_err(cudaSetDevice(device)); + } + ~DeviceGuard() { cudaSetDevice(previous_); } + DeviceGuard(const DeviceGuard &) = delete; + DeviceGuard &operator=(const DeviceGuard &) = delete; + +private: + int previous_ = 0; +}; + +struct Atom { + float x, y, z, b, occ; + int element; +}; + +// dF of every state at every row: thread per row, block row y the state. `hop`: per row and operator R^T h and +// h . t; `ff`: the form-factor table, 9 per element. +__global__ void contribution_kernel(int rows, int n_ops, const float4 *hop, const float *stol2, const float *ff, + const Atom *atoms, const int *start, float2 *df) { + const int m = blockIdx.x * blockDim.x + threadIdx.x; + const int s = blockIdx.y; + if (m >= rows) + return; + const float st = stol2[m]; + float re = 0, im = 0; + for (int a = start[s]; a < start[s + 1]; a++) { + const Atom at = atoms[a]; + const float *c = ff + 9 * at.element; + float f = c[8]; + for (int i = 0; i < 4; i++) + f += c[i] * expf(-c[4 + i] * st); + f *= at.occ * expf(-at.b * st); + for (int k = 0; k < n_ops; k++) { + const float4 h = hop[m * n_ops + k]; + float sn, cs; + sincosf(TWO_PI * (h.x * at.x + h.y * at.y + h.z * at.z + h.w), &sn, &cs); + re += f * cs; + im += f * sn; + } + } + df[s * rows + m] = make_float2(re, im); +} + +// -ln L over the working rows per combination: one block per combination, its threads over the rows in strides, +// reduced in a fixed order. F = total - sum of the current states' dF + sum of the combination's. +__global__ void nll_kernel(int rows, const float2 *total, const float2 *df, const int *current, int n_current, + const int *combo_states, const int *combo_start, const float *fo, const float *d, + const float *var, const char *centric, const char *work, const float *k_total, + const float2 *fmask, double *nll) { + __shared__ double partial[THREADS]; + const int c = blockIdx.x; + double sum = 0; + for (int m = threadIdx.x; m < rows; m += THREADS) { + if (!work[m]) + continue; + float re = total[m].x + fmask[m].x, im = total[m].y + fmask[m].y; + for (int i = 0; i < n_current; i++) { + re -= df[current[i] * rows + m].x; + im -= df[current[i] * rows + m].y; + } + for (int k = combo_start[c]; k < combo_start[c + 1]; k++) { + re += df[combo_states[k] * rows + m].x; + im += df[combo_states[k] * rows + m].y; + } + const double fc = k_total[m] * sqrt(static_cast(re) * re + static_cast(im) * im); + double dfc; + sum += refine_math::AmplitudeNegLogLikelihood(static_cast(fo[m]), fc, static_cast(d[m]), + static_cast(var[m]), centric[m] != 0, dfc); + } + partial[threadIdx.x] = sum; + __syncthreads(); + for (int stride = THREADS / 2; stride > 0; stride /= 2) { + if (threadIdx.x < stride) + partial[threadIdx.x] += partial[threadIdx.x + stride]; + __syncthreads(); + } + if (threadIdx.x == 0) + nll[c] = partial[0]; +} + +__global__ void move_kernel(int rows, const float2 *df, const int *current, int n_current, const int *combo_states, + int from, int to, float2 *total) { + const int m = blockIdx.x * blockDim.x + threadIdx.x; + if (m >= rows) + return; + float re = total[m].x, im = total[m].y; + for (int i = 0; i < n_current; i++) { + re -= df[current[i] * rows + m].x; + im -= df[current[i] * rows + m].y; + } + for (int k = from; k < to; k++) { + re += df[combo_states[k] * rows + m].x; + im += df[combo_states[k] * rows + m].y; + } + total[m] = make_float2(re, im); +} + +class EvaluatorGPU : public SideChainFieldEvaluator { +public: + EvaluatorGPU(int device, const FieldReflections &refl, const std::vector> &ops, + const std::vector &elements) + : device_(device), rows_(static_cast(refl.Size())), n_ops_(static_cast(ops.size())) { + DeviceGuard guard(device_); + std::vector hop(refl.Size() * ops.size()); + for (size_t m = 0; m < refl.Size(); m++) { + const std::array &h = refl.hkl[m]; + for (size_t k = 0; k < ops.size(); k++) { + const std::array &op = ops[k]; + hop[m * ops.size() + k] = make_float4(op[0] * h[0] + op[3] * h[1] + op[6] * h[2], + op[1] * h[0] + op[4] * h[1] + op[7] * h[2], + op[2] * h[0] + op[5] * h[1] + op[8] * h[2], + op[9] * h[0] + op[10] * h[1] + op[11] * h[2]); + } + } + std::vector ff; + for (const FormFactor &e : elements) + ff.insert(ff.end(), e.begin(), e.end()); + cuda_err(cudaStreamCreate(&stream_)); + hop_ = Upload(hop); + stol2_ = Upload(refl.stol2); + ff_ = Upload(ff); + fo_ = Upload(refl.fo); + d_ = Upload(refl.d); + var_ = Upload(refl.var); + centric_ = Upload(refl.centric); + work_ = Upload(refl.work); + k_total_ = Upload(refl.k_total); + fmask_ = reinterpret_cast(Upload(refl.fmask)); + cuda_err(cudaMalloc(&total_, rows_ * sizeof(float2))); + } + ~EvaluatorGPU() override { + cudaSetDevice(device_); + for (void *p : {static_cast(hop_), static_cast(stol2_), static_cast(ff_), + static_cast(fo_), static_cast(d_), static_cast(var_), + static_cast(centric_), static_cast(work_), static_cast(k_total_), + static_cast(fmask_), static_cast(total_), static_cast(atoms_), + static_cast(start_), static_cast(df_), static_cast(nll_), + static_cast(combo_states_), static_cast(combo_start_)}) + cudaFree(p); + cudaStreamDestroy(stream_); + } + + void SetTotal(const std::vector> &f) override { + DeviceGuard guard(device_); + cuda_err(cudaMemcpyAsync(total_, f.data(), rows_ * sizeof(float2), cudaMemcpyHostToDevice, stream_)); + cuda_err(cudaStreamSynchronize(stream_)); + } + + std::vector Evaluate(const FieldStates &states) override { + DeviceGuard guard(device_); + Contributions(states); + const int n = static_cast(states.Combos()); + nll_kernel<<>>(rows_, total_, df_, current_, static_cast(states.current.size()), + combo_states_, combo_start_, fo_, d_, var_, centric_, work_, k_total_, + fmask_, nll_); + cuda_err(cudaGetLastError()); + std::vector nll(n); + cuda_err(cudaMemcpyAsync(nll.data(), nll_, n * sizeof(double), cudaMemcpyDeviceToHost, stream_)); + cuda_err(cudaStreamSynchronize(stream_)); + return nll; + } + + void Move(const FieldStates &states, int to) override { + DeviceGuard guard(device_); + Contributions(states); + move_kernel<<<(rows_ + THREADS - 1) / THREADS, THREADS, 0, stream_>>>( + rows_, df_, current_, static_cast(states.current.size()), combo_states_, states.combo_start[to], + states.combo_start[to + 1], total_); + cuda_err(cudaGetLastError()); + cuda_err(cudaStreamSynchronize(stream_)); + } + + std::vector> Contribution(const FieldStates &states, int s) override { + DeviceGuard guard(device_); + Contributions(states); + std::vector> f(rows_); + cuda_err(cudaMemcpyAsync(f.data(), df_ + static_cast(s) * rows_, rows_ * sizeof(float2), + cudaMemcpyDeviceToHost, stream_)); + cuda_err(cudaStreamSynchronize(stream_)); + return f; + } + +private: + template + T *Upload(const std::vector &v) { + T *p = nullptr; + cuda_err(cudaMalloc(&p, v.size() * sizeof(T))); + cuda_err(cudaMemcpyAsync(p, v.data(), v.size() * sizeof(T), cudaMemcpyHostToDevice, stream_)); + cuda_err(cudaStreamSynchronize(stream_)); + return p; + } + // dF of every state of `states` into df_ (grown as needed). + void Contributions(const FieldStates &states) { + const int n = static_cast(states.Count()); + if (states.atoms.size() > atoms_capacity_) { + cudaFree(atoms_); + atoms_capacity_ = states.atoms.size(); + cuda_err(cudaMalloc(&atoms_, atoms_capacity_ * sizeof(Atom))); + } + if (static_cast(n) > states_capacity_) { + cudaFree(start_); + cudaFree(df_); + states_capacity_ = n; + cuda_err(cudaMalloc(&start_, (states_capacity_ + 1) * sizeof(int))); + cuda_err(cudaMalloc(&df_, states_capacity_ * rows_ * sizeof(float2))); + } + const size_t combos = states.Combos(), lists = states.combo_states.size() + states.current.size(); + if (combos + 1 > combos_capacity_) { + cudaFree(combo_start_); + cudaFree(nll_); + combos_capacity_ = combos + 1; + cuda_err(cudaMalloc(&combo_start_, combos_capacity_ * sizeof(int))); + cuda_err(cudaMalloc(&nll_, combos_capacity_ * sizeof(double))); + } + if (lists > lists_capacity_) { + cudaFree(combo_states_); + lists_capacity_ = lists; + cuda_err(cudaMalloc(&combo_states_, lists_capacity_ * sizeof(int))); + } + current_ = combo_states_ + states.combo_states.size(); + if (!states.combo_states.empty()) + cuda_err(cudaMemcpyAsync(combo_states_, states.combo_states.data(), states.combo_states.size() * sizeof(int), + cudaMemcpyHostToDevice, stream_)); + if (!states.current.empty()) + cuda_err(cudaMemcpyAsync(current_, states.current.data(), states.current.size() * sizeof(int), + cudaMemcpyHostToDevice, stream_)); + cuda_err(cudaMemcpyAsync(combo_start_, states.combo_start.data(), (combos + 1) * sizeof(int), + cudaMemcpyHostToDevice, stream_)); + static_assert(sizeof(Atom) == sizeof(FieldAtom)); + if (!states.atoms.empty()) + cuda_err(cudaMemcpyAsync(atoms_, states.atoms.data(), states.atoms.size() * sizeof(Atom), + cudaMemcpyHostToDevice, stream_)); + cuda_err(cudaMemcpyAsync(start_, states.start.data(), (n + 1) * sizeof(int), cudaMemcpyHostToDevice, stream_)); + const dim3 grid((rows_ + THREADS - 1) / THREADS, n); + contribution_kernel<<>>(rows_, n_ops_, hop_, stol2_, ff_, atoms_, start_, df_); + cuda_err(cudaGetLastError()); + } + + int device_, rows_, n_ops_; + cudaStream_t stream_ = nullptr; + float4 *hop_ = nullptr; + float *stol2_ = nullptr, *ff_ = nullptr, *fo_ = nullptr, *d_ = nullptr, *var_ = nullptr, *k_total_ = nullptr; + char *centric_ = nullptr, *work_ = nullptr; + float2 *fmask_ = nullptr, *total_ = nullptr, *df_ = nullptr; + Atom *atoms_ = nullptr; + int *start_ = nullptr, *combo_states_ = nullptr, *current_ = nullptr, *combo_start_ = nullptr; + double *nll_ = nullptr; + size_t atoms_capacity_ = 0, states_capacity_ = 0, combos_capacity_ = 0, lists_capacity_ = 0; +}; + +} // namespace + +std::unique_ptr MakeSideChainFieldEvaluatorGPU(int device, const FieldReflections &refl, + const std::vector> &ops, + const std::vector &elements) { + return std::make_unique(device, refl, ops, elements); +} diff --git a/rugnux/ResultReport.cpp b/rugnux/ResultReport.cpp index b331b29da..99c759703 100644 --- a/rugnux/ResultReport.cpp +++ b/rugnux/ResultReport.cpp @@ -728,6 +728,8 @@ namespace { Add(s, KeyReal("REFINE_RMSD_ANGLES_START", rf.geometry_start.rmsd_angles, "{:.2f}", true)); Add(s, KeyInt("REFINE_CLASHES", rf.geometry.clashes)); Add(s, KeyInt("REFINE_SIDE_CHAINS_TURNED", rf.side_chains_turned)); + Add(s, KeyInt("REFINE_SIDE_CHAIN_FIELD_MOVED", rf.side_chain_field_moved)); + Add(s, KeyInt("REFINE_SIDE_CHAIN_FIELD_PRUNED", rf.side_chain_field_pruned)); Add(s, KeyReal("REFINE_XRAY_WEIGHT", rf.weight, "{:.4g}", true)); if (rf.ncs_groups > 0) { Add(s, KeyInt("REFINE_NCS_GROUPS", rf.ncs_groups)); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0bc1bafd5..74e2376fd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -198,6 +198,7 @@ ADD_EXECUTABLE(jfjoch_test SolventSitesTest.cpp DensityBlobsTest.cpp SolventBuildTest.cpp + SideChainFieldTest.cpp SearchSpaceGroupTest.cpp SearchSpaceGroupTwinTest.cpp SyntheticMergedReflections.h diff --git a/tests/SideChainFieldTest.cpp b/tests/SideChainFieldTest.cpp new file mode 100644 index 000000000..9016c8562 --- /dev/null +++ b/tests/SideChainFieldTest.cpp @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: 2026 Filip Leonarski, Paul Scherrer Institute +// SPDX-License-Identifier: GPL-3.0-only + +#include + +#include +#include +#include + +#include "gemmi/it92.hpp" +#include "gemmi/sfcalc.hpp" +#include "gemmi/symmetry.hpp" +#include "../image_analysis/structure_refinement/SideChainField.h" +#ifdef JFJOCH_USE_CUDA +#include +#endif + +namespace { + +constexpr double EDGE = 24.0; // A, a P 21 21 21 box: the sum runs over four operators +constexpr double D_MIN = 2.0; +constexpr double DEG = 3.14159265358979323846 / 180.0; + +// The atom bonded to c at `bond`, at `angle` b-c-x and the torsion a-b-c-x (degrees). +gemmi::Position place(const gemmi::Position &a, const gemmi::Position &b, const gemmi::Position &c, double bond, + double angle, double torsion) { + const gemmi::Vec3 bc = (c - b).normalized(); + const gemmi::Vec3 n = (b - a).cross(bc).normalized(); + const gemmi::Vec3 m = n.cross(bc); + const double x = -bond * std::cos(angle * DEG); + const double y = bond * std::sin(angle * DEG) * std::cos(torsion * DEG); + const double z = bond * std::sin(angle * DEG) * std::sin(torsion * DEG); + return gemmi::Position(gemmi::Vec3(c) + bc * x + m * y + n * z); +} + +// A lysine at these chi angles, its main chain near the middle of the box. +gemmi::Residue lysine(double chi1, double chi2, double chi3, double chi4) { + gemmi::Residue r; + r.name = "LYS"; + r.seqid = gemmi::SeqId(1, ' '); + const gemmi::Position n(10, 10, 10), ca(11.46, 10, 10); + const gemmi::Position c = place(gemmi::Position(10, 11, 10), n, ca, 1.52, 111, 180); + const gemmi::Position o = place(n, ca, c, 1.23, 121, 0); + const gemmi::Position cb = place(c, n, ca, 1.53, 110.5, 122.5); + const gemmi::Position cg = place(n, ca, cb, 1.52, 114, chi1); + const gemmi::Position cd = place(ca, cb, cg, 1.52, 111, chi2); + const gemmi::Position ce = place(cb, cg, cd, 1.52, 111, chi3); + const gemmi::Position nz = place(cg, cd, ce, 1.49, 112, chi4); + const std::pair atoms[] = {{"N", n}, {"CA", ca}, {"C", c}, {"O", o}, {"CB", cb}, + {"CG", cg}, {"CD", cd}, {"CE", ce}, {"NZ", nz}}; + for (const auto &[name, pos] : atoms) { + gemmi::Atom a; + a.name = name; + a.element = gemmi::Element(name[0] == 'N' ? gemmi::El::N : name[0] == 'O' ? gemmi::El::O : gemmi::El::C); + a.pos = pos; + a.occ = 1.0f; + a.b_iso = 20.0f; + r.atoms.push_back(a); + } + return r; +} + +gemmi::Structure box(const gemmi::Residue &residue) { + gemmi::Structure st; + st.cell = gemmi::UnitCell(EDGE, EDGE, EDGE, 90, 90, 90); + st.spacegroup_hm = "P 21 21 21"; + st.setup_cell_images(); + st.models.emplace_back(1); + st.models[0].chains.emplace_back("A"); + st.models[0].chains[0].residues.push_back(residue); + return st; +} + +// The reflections of the box to D_MIN, one per orbit, every one a working reflection, at D = 1 and a small variance. +FieldReflections reflections(const gemmi::Structure &st) { + const gemmi::SpaceGroup &sg = *st.find_spacegroup(); + const gemmi::GroupOps gops = sg.operations(); + const gemmi::ReciprocalAsu asu(&sg); + FieldReflections fr; + const int hmax = static_cast(EDGE / D_MIN); + for (int h = -hmax; h <= hmax; h++) + for (int k = -hmax; k <= hmax; k++) + for (int l = -hmax; l <= hmax; l++) { + const gemmi::Miller hkl{h, k, l}; + const double inv_d2 = st.cell.calculate_1_d2(hkl); + if ((h == 0 && k == 0 && l == 0) || inv_d2 > 1.0 / (D_MIN * D_MIN) || !asu.is_in(hkl)) + continue; + fr.hkl.push_back({static_cast(h), static_cast(k), static_cast(l)}); + fr.stol2.push_back(static_cast(inv_d2 / 4.0)); + fr.fo.push_back(0); + fr.d.push_back(1.0f); + fr.var.push_back(4.0f); + fr.centric.push_back(gops.is_reflection_centric(hkl)); + fr.work.push_back(1); + fr.k_total.push_back(1.0f); + fr.fmask.push_back(0); + } + return fr; +} + +std::vector elements() { + return {gemmi::IT92::get(gemmi::El::C, 0).coefs, gemmi::IT92::get(gemmi::El::N, 0).coefs, + gemmi::IT92::get(gemmi::El::O, 0).coefs}; +} +int element_index(gemmi::El el) { return el == gemmi::El::C ? 0 : el == gemmi::El::N ? 1 : 2; } + +// Every atom of the residue as a state. +std::vector all_atoms(const gemmi::Structure &st) { + std::vector out; + for (const gemmi::Atom &a : st.models[0].chains[0].residues[0].atoms) { + const gemmi::Fractional f = st.cell.fractionalize(a.pos); + out.push_back({static_cast(f.x), static_cast(f.y), static_cast(f.z), a.b_iso, a.occ, + element_index(a.element.elem)}); + } + return out; +} + +bool gpu_present() { +#ifdef JFJOCH_USE_CUDA + int n = 0; + return cudaGetDeviceCount(&n) == cudaSuccess && n > 0; +#else + return false; +#endif +} + +} // namespace + +TEST_CASE("SideChainField_direct_sum_matches_gemmi", "[Refinement]") { + const gemmi::Structure st = box(lysine(-60, 180, 180, 60)); + const FieldReflections fr = reflections(st); + REQUIRE(fr.Size() > 500); + FieldStates states; + states.Add(all_atoms(st)); + auto cpu = MakeSideChainFieldEvaluatorCPU(fr, FieldOperators(*st.find_spacegroup()), elements(), 2); + const std::vector> f = cpu->Contribution(states, 0); + gemmi::StructureFactorCalculator> calc(st.cell); + double num = 0, den = 0; + for (size_t m = 0; m < fr.Size(); m++) { + const gemmi::Miller hkl{static_cast(fr.hkl[m][0]), static_cast(fr.hkl[m][1]), static_cast(fr.hkl[m][2])}; + const std::complex ref = calc.calculate_sf_from_model(st.models[0], hkl); + num += std::norm(std::complex(f[m]) - ref); + den += std::norm(ref); + } + CHECK(std::sqrt(num / den) < 1e-4); + if (gpu_present()) { +#ifdef JFJOCH_USE_CUDA + auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, FieldOperators(*st.find_spacegroup()), elements()); + const std::vector> g = gpu->Contribution(states, 0); + double gnum = 0; + for (size_t m = 0; m < fr.Size(); m++) + gnum += std::norm(std::complex(f[m]) - std::complex(g[m])); + CHECK(std::sqrt(gnum / den) < 1e-4); +#endif + } +} + +TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") { + // the data: |F| of the lysine at one rotamer; the model starts at another + const gemmi::Structure truth = box(lysine(180, 180, 180, 180)); + FieldReflections fr = reflections(truth); + FieldStates truth_state; + truth_state.Add(all_atoms(truth)); + const std::vector ff = elements(); + const std::vector> ops = FieldOperators(*truth.find_spacegroup()); + { + auto e = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2); + const std::vector> f = e->Contribution(truth_state, 0); + for (size_t m = 0; m < fr.Size(); m++) + fr.fo[m] = std::abs(f[m]); + } + const gemmi::Structure start = box(lysine(-60, 180, 60, 180)); + const std::vector element_of = {1, 0, 0, 2, 0, 0, 0, 0, 1}; + const std::vector refinable(9, 1); + auto run = [&](SideChainFieldEvaluator &e) { + FieldStates s; + s.Add(all_atoms(start)); + e.SetTotal(e.Contribution(s, 0)); + return OptimiseSideChainField(start, element_of, refinable, {}, 1.0, e); + }; + auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2); + const SideChainFieldResult r = run(*cpu); + REQUIRE(r.candidates == 1); + REQUIRE(r.moves.size() == 1); + REQUIRE(!r.moves[0].absent); + REQUIRE(r.moves[0].chi_after.size() == 4); + for (double chi : r.moves[0].chi_after) + CHECK(std::abs(std::remainder(chi - 180.0, 360.0)) < 1.0); + CHECK(r.energy_after < r.energy_before); + if (gpu_present()) { +#ifdef JFJOCH_USE_CUDA + auto gpu = MakeSideChainFieldEvaluatorGPU(0, fr, ops, ff); + const SideChainFieldResult g = run(*gpu); + REQUIRE(g.moves.size() == 1); + CHECK(g.moves[0].chi_after == r.moves[0].chi_after); + CHECK(g.energy_after == Catch::Approx(r.energy_after).epsilon(1e-4)); +#endif + } +} From e71161d59a9e27f76f43289be60c46a61baff716 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 15:47:37 +0100 Subject: [PATCH 4/7] Side-chain field: a copy pruned by an earlier sweep has no atoms in its current state Revisited, its rotamer states were evaluated with the pruned atoms taken out of a total that no longer held them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- .../structure_refinement/SideChainField.cpp | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp index e87c7de45..6e4ab350b 100644 --- a/image_analysis/structure_refinement/SideChainField.cpp +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -188,6 +188,7 @@ struct Candidate { int first_atom; // index of the residue's first atom in the model's atom list SideChain sc; std::string name; + bool absent = false; // pruned to CB by an earlier sweep }; std::vector Candidates(const gemmi::Structure &st, const std::vector &refinable) { @@ -328,7 +329,7 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const // type), the chi angles of each, and the geometry's share of each one's energy: its overlap with the // rest of the model, and the hydrogen bonds of an amide std::vector>> positions(k); - std::vector> overlap(k); + std::vector> geometry(k); std::vector> chis; for (size_t c = 0; c < k; c++) { Candidate &cand = candidates[var[c]]; @@ -345,20 +346,24 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const chis.push_back(chi); } for (const std::vector &p : positions[c]) - overlap[c].push_back(repulsion * OverlapOf(ns, model, cell, res, cand.chain, cand.residue, cand.sc, p, {}).sum - - HBOND_WEIGHT * AmideHBonds(ns, model, cell, res, cand.chain, cand.residue, p)); - overlap[c].push_back(0); // absent + geometry[c].push_back(repulsion * OverlapOf(ns, model, cell, res, cand.chain, cand.residue, cand.sc, p, {}).sum - + HBOND_WEIGHT * AmideHBonds(ns, model, cell, res, cand.chain, cand.residue, p)); + geometry[c].push_back(0); // absent + if (cand.absent) + geometry[c][0] = 0; } const int n_pos = static_cast(positions[0].size()), absent = n_pos; - // the states: per copy its positions (c * n_pos + p); the combinations as assignments, per copy the index - // of its positions (absent beyond them): as it is, the copies together in each rotamer and absent, then - // each copy alone in each + // the states: per copy its positions (c * n_pos + p; as it is: no atoms where it was pruned); the + // combinations as assignments, per copy the index of its positions (absent beyond them): as it is, the + // copies together in each rotamer and absent, then each copy alone in each FieldStates states; for (size_t c = 0; c < k; c++) { const Candidate &cand = candidates[var[c]]; for (int p = 0; p < n_pos; p++) - states.Add(FieldAtomsOf(model.chains[cand.chain].residues[cand.residue], cand.sc, positions[c][p], cell, - element_of, cand.first_atom)); + states.Add(p == 0 && cand.absent + ? std::vector() + : FieldAtomsOf(model.chains[cand.chain].residues[cand.residue], cand.sc, + positions[c][p], cell, element_of, cand.first_atom)); states.current.push_back(static_cast(c) * n_pos); } std::vector> assign = {std::vector(k, 0)}; @@ -383,7 +388,7 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const const double occam = std::log(static_cast(assign.size() - 1)); for (size_t s = 0; s < assign.size(); s++) { for (size_t c = 0; c < k; c++) - energy[s] += overlap[c][assign[s][c]]; + energy[s] += geometry[c][assign[s][c]]; if (s > 0) energy[s] += occam; } @@ -416,7 +421,7 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const m.chi_before = cand.sc.Chis(); } m.energy_gain += (energy[0] - energy[best]) / changed; - m.absent = p == absent; + m.absent = cand.absent = p == absent; m.chi_after = m.absent ? std::vector() : chis[p]; m.atoms.clear(); m.pruned.clear(); From 77dabe12c1f6be6b3969c3327211d79797f909a8 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 16:41:45 +0100 Subject: [PATCH 5/7] Refinement: a population of blurred, kicked members crossed residue by residue after the weight trajectory A single descent from a placed prediction sees the basin it starts in. After the weight trajectory, twelve members refine side by side in one wave, each a trial a macro-cycle longer than a step at the trajectory's weights but on a path of its own: the positions as they are or kicked at random by up to 0.2-0.3 A per coordinate, on the data as they are or blurred along a schedule (60 then 20 A^2 - the landscape of 4 A, then 2.2 A -, 100/30, 30/10, then the resolution as it is). The blur is a continuation on the resolution with no engine change: the likelihood's variance of each reflection raised by exp(B_h s^2/2), which the Rice likelihood reads as Fo and F_model both blurred, with the X-ray weight balanced again for the gradient left (TrialWeights::blur, kick, seed; RunTrial takes the TrialWeights). The crossover is made on one 2mFo-DFc map averaged over the members with the free reflections left out (PopulationMap): each residue is taken from the member whose atoms sit in the most density, the junctions are mended by a macro-cycle at the ladder's tightest rung (at the state's weight the chimera ends out of the geometry bounds), and the child is refined by one trial at the state's weights. The child displaces the state by the rule every trial is held to - within bounds, R-free lower beyond the tie; the members are never chosen among on the free set, whose best of twelve would be the best of twelve (RefinementSettings::population, 0 for none). Measured: on 13 predictions placed by MR (af-bench) the child was kept on 4 - R-free 0.011, 0.010, 0.007 and 0.005 lower - and the other 9 come out as before; on 22 refined models (the battery's refine arm) one came out 0.009 lower and none worse. On the live cytochrome C set the gain, where the child is kept, comes in the rounds that follow it (0.42 -> 0.387 on two of four jittered starts, nothing on the other two, and not repeated with other kick seeds): the population alone, with the rounds off, takes 0.003 off. Six members, or members a step long, did not reach that basin. Blurred rungs tried as members of the first ladder were never chosen on any of the 13 sets (the blur's own R-free at the rung is worse; its gain comes later in the path) and are not kept; forced on every rung they took 0.004-0.006 off four sets and put 0.02 on one. Cost: the members' evaluations are GPU-bound, so the stage costs 1.2 s per refinement pass on a 5000-atom model and 4-5 s on 13k atoms on one card (a quarter of that on four); refinement time on af-bench +15-45%. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/ACKNOWLEDGEMENT.md | 15 ++ docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 25 ++- .../structure_refinement/Refinement.cpp | 187 +++++++++++++++++- .../structure_refinement/Refinement.h | 5 + 4 files changed, 221 insertions(+), 11 deletions(-) diff --git a/docs/ACKNOWLEDGEMENT.md b/docs/ACKNOWLEDGEMENT.md index 824ebffc3..a98136668 100644 --- a/docs/ACKNOWLEDGEMENT.md +++ b/docs/ACKNOWLEDGEMENT.md @@ -684,6 +684,21 @@ displacement model a refinement keeps is decided on the free set rather than by resolution: E. A. Merritt, "To B or not to B: a question of resolution?" (2012), Acta Cryst. D68, 468-477 [doi:10.1107/S0907444911028320](https://doi.org/10.1107/S0907444911028320). +**Population and crossover** — the members' refinement on blurred data is a continuation (homotopy) on the +resolution, the deformation of the landscape of L. Piela, J. Kostrowicki and H. A. Scheraga, "On the +multiple-minima problem in the conformational analysis of molecules: deformation of the potential energy +hypersurface by the diffusion equation method" (1989), J. Phys. Chem. 93, 3339-3346 +[doi:10.1021/j100345a090](https://doi.org/10.1021/j100345a090), applied as the "low resolution first" of +A. Jack and M. Levitt, "Refinement of large structures by simultaneous minimization of energy and R factor" +(1978), Acta Cryst. A34, 931-935 [doi:10.1107/S0567739478001904](https://doi.org/10.1107/S0567739478001904). +The kicked members and the map averaged over them follow J. Praznikar, P. V. Afonine, G. Guncar, P. D. Adams +and D. Turk, "Averaged kick maps: less noise, more signal... and probably less bias" (2009), Acta Cryst. D65, +921-931 [doi:10.1107/S0907444909021933](https://doi.org/10.1107/S0907444909021933). Taking each residue from +the member that fits best is the combination of T. C. Terwilliger et al., "Improved crystallographic +structures using extensive combinatorial refinement" (2013), Structure 21, 1923-1930 +[doi:10.1016/j.str.2013.07.025](https://doi.org/10.1016/j.str.2013.07.025), made on one map inside the +refinement rather than offline on whole models. + **NCS restraints** — copies found by sequence alignment with REFMAC5's criterion (Murshudov et al. 2011, above) and restrained locally on pairs of corresponding interatomic distances under the Geman-McClure loss, as REFMAC5's local NCS restraints are, with the distance range of diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 612f4cdf6..2109c5a92 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -29,7 +29,7 @@ statistics and its map coefficients go out; the restraint dictionary is compiled The model is taken as the refinement can use it: the first model of the file only, its hydrogens removed, and every B inside the range the refinement allows (2 Ų up to twice the model's largest B, within 100-300 Ų; a model that arrives with B above 300 Ų - a structure refined at 4-5 Å - gets a ceiling of 1.5 times its -largest B instead, so that it is not flattened; a later round of the weights (§15.1, 4) raises the ceiling from +largest B instead, so that it is not flattened; a later round of the weights (§15.7, 5) raises the ceiling from the refined model). The model is refined in the data's space group - except where its own group is a proper subgroup of it: a @@ -352,9 +352,24 @@ are: sweep alone). The log names each side chain moved with its chi angles before and after and the energy the move took off (`REFINE_SIDE_CHAIN_FIELD_MOVED`, `REFINE_SIDE_CHAIN_FIELD_PRUNED`). Twinned data and a model with anisotropic atoms are refined without the field. -3. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved +3. **The population**, from where the trajectory ended: twelve members refined side by side, each a trial at the + trajectory's weights but on its own path - the positions as they are or kicked at random by up to 0.2-0.3 Å + in each coordinate, on the data as they are or **blurred** along a schedule, 60 then 20 Ų (the landscape of + 4 Å, then of 2.2 Å), 100 then 30, or 30 then 10. The blur raises the likelihood's variance of each reflection + by $e^{B_h s^2/2}$, which is $F_o$ and $F_\mathrm{model}$ both blurred by $e^{-B_h s^2/4}$ (the Rice + likelihood reads a common scale as nothing else), and the X-ray weight is balanced again for the gradient + left; the B factors refine on the data as they are. A single descent sees the basin it starts in; the + members see those around it. Their **crossover** is made on one 2mFo-DFc map averaged over the members with + the free reflections left out: each residue is taken from the member whose atoms sit in the most density, + the junctions are mended by a macro-cycle at the ladder's tightest rung, and the child is refined by one + trial at the trajectory's weights. The child displaces the state by the rule every trial is held to; the + members themselves are never chosen among on the free set, whose best of twelve would be the best of twelve. On + 13 predictions placed by MR the child was kept on four (R-free 0.005-0.011 lower) and on one of 22 refined + models (0.009 lower, the rest unchanged); the gain, where there is one, comes mostly in the rounds that + follow from the child. +4. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved was never kept on 55 refinements, and taking it away changed no choice). -4. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U +5. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U held), each trial as long as a step on the positions where the weights ended, held - only the displacement parameters refine, so the trials differ in the displacement model alone (where the geometry is out of bounds there, which held positions cannot mend, each trial refines the positions too, a macro-cycle longer than a @@ -384,7 +399,7 @@ are: R-free beyond the tie and undone otherwise; where its ladder and trajectory leave the model where it was, the hypotheses would repeat the same trials, and it ends there - from a model refined before, a round is one ladder and one step. No round runs where the refinement does not displace the model as given. -5. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the +6. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the rule every trial is held to (its geometry within bounds and R-free lower beyond the tie), the model is written as it came, scaled to the data, and the solvent build's edits are undone (`REFINE_AS_GIVEN`). What comes out is never worse on the free set than what went in. This holds only where the free set is free to the model as given: a model @@ -708,6 +723,8 @@ What the refinement does not do yet - each a known gap, not a design choice: ## References - R. C. Agarwal, "A new least-squares refinement technique based on the fast Fourier transform algorithm", *Acta Cryst.* **A34** (1978), 791-809 (the gradient by the FFT method, §15.2). +- L. Piela, J. Kostrowicki & H. A. Scheraga, "On the multiple-minima problem in the conformational analysis of molecules: deformation of the potential energy hypersurface by the diffusion equation method", *J. Phys. Chem.* **93** (1989), 3339-3346, and A. Jack & M. Levitt, "Refinement of large structures by simultaneous minimization of energy and R factor", *Acta Cryst.* **A34** (1978), 931-935 (the members refined on blurred data, §15.7). +- J. Praznikar, P. V. Afonine, G. Guncar, P. D. Adams & D. Turk, "Averaged kick maps: less noise, more signal... and probably less bias", *Acta Cryst.* **D65** (2009), 921-931, and T. C. Terwilliger et al., "Improved crystallographic structures using extensive combinatorial refinement", *Structure* **21** (2013), 1923-1930 (the kicked members, their averaged map and the crossover residue by residue, §15.7). - N. S. Pannu & R. J. Read, "Improved structure refinement through maximum likelihood", *Acta Cryst.* **A52** (1996), 659-668, and G. N. Murshudov, A. A. Vagin & E. J. Dodson, "Refinement of macromolecular structures by the maximum-likelihood method", *Acta Cryst.* **D53** (1997), 240-255 (the maximum-likelihood amplitude target and its figure of merit, §15.2, §15.8). - G. N. Murshudov, P. Skubák, A. A. Lebedev, N. S. Pannu, R. A. Steiner, R. A. Nicholls, M. D. Winn, F. Long & A. A. Vagin, "REFMAC5 for the refinement of macromolecular crystal structures", *Acta Cryst.* **D67** (2011), 355-367 (D from the working set and the variance from the free, §15.3; the Kullback-Leibler form of the B restraints, §15.5; the twinned refinement §15.12 simplifies, its twin laws pruned by fraction and its fractions refined by least squares). - R. J. Read, "Improved Fourier coefficients for maps using phases from partial structures with errors", *Acta Cryst.* **A42** (1986), 140-149 (the m, D weighting of the map coefficients, §15.8). diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 353b3aa27..ff05ba7c2 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1723,6 +1723,14 @@ std::vector OwnSlotPlaces(const Context &main, int wanted, const Refi // none). struct TrialWeights { double w = 0, adp = 0, ncs = 0; + // A member of the population (RefineOnce, 2b.): the positions refined on the data blurred by B_h per macro-cycle + // (A^2; empty or 0: not blurred) - the likelihood's variance raised by exp(B_h s^2 / 2), which is Fo and F_model + // both blurred by exp(-B_h s^2 / 4): the landscape of a lower resolution, which a position 1-2 A off still + // descends - with the X-ray weight balanced again for it; and before the first macro-cycle every refinable atom + // displaced by up to `kick` (A) in each coordinate, at random from `seed`. + std::vector blur; + double kick = 0; + unsigned seed = 0; }; // A displacement model a trial refines (RefinementSettings::adp_hypotheses): the B factors as they are (isotropic, or @@ -1752,15 +1760,34 @@ struct TrialOutcome { int evaluations = 0; }; +// The X-ray target's gradient on the positions, its squared norm over `refinable`, for the atoms as they are. +double XrayGradientNorm(Context &ctx, const std::vector &refinable) { + std::vector> g; + ctx.engine->Evaluate(ctx.Model(), &g); + double n = 0; + for (int i : refinable) + for (int c = 0; c < 3; c++) + n += g[i][c] * g[i][c]; + return n; +} + TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const std::vector &pos0, const std::vector &b0, const std::vector &aniso0, const Context::Fit *fit, - double w, double adp_w, int cycles, const RefinementSettings &settings, const NcsCopies *ncs, - double ncs_sigma, const AdpHypothesis *hyp = nullptr, bool hold_positions = false) { + const TrialWeights &tw, int cycles, const RefinementSettings &settings, const NcsCopies *ncs, + const AdpHypothesis *hyp = nullptr, bool hold_positions = false) { Context &ctx = *slot.ctx; RefinementTopology &topo = *slot.topo; + const double w = tw.w, adp_w = tw.adp, ncs_sigma = tw.ncs; const RefinementStage stage(hyp != nullptr ? "rung.adp" : adp_w > 0 ? "rung.b" : "rung.xyz"); const int evaluations_before = ctx.evaluations; ctx.SetPositions(pos0); + if (tw.kick > 0) { + std::mt19937 rng(tw.seed); + std::uniform_real_distribution shift(-tw.kick, tw.kick); + for (int i : refinable) + if (!ctx.atoms[i]->is_hydrogen()) + ctx.atoms[i]->pos += gemmi::Position(shift(rng), shift(rng), shift(rng)); + } ctx.SetB(b0); ctx.SetAniso(aniso0); ctx.tls.clear(); @@ -1814,7 +1841,21 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s ctx.engine->SetRestraints(0, set); } const RefinementStage s2("minimise.xyz"); - Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser); + const double b_h = cycle < static_cast(tw.blur.size()) ? tw.blur[cycle] : 0.0; + if (b_h > 0) { + // the continuation of Piela, Kostrowicki & Scheraga (1989) J. Phys. Chem. 93, 3339-3346, on the + // resolution: the data blurred (TrialWeights::blur), the weight keeping the gradients' balance + const double n0 = XrayGradientNorm(ctx, refinable); + XrayRowModel blurred = ctx.rows; + for (size_t m = 0; m < ctx.refl.Size(); m++) + blurred.var[m] *= static_cast(std::exp(b_h * ctx.refl.inv_d2[m] / 2.0)); + ctx.engine->SetRowModel(blurred); + const double nh = XrayGradientNorm(ctx, refinable); + const AtomTarget xyz_h(ctx, AtomTarget::Mode::Xyz, refinable, nh > 0 ? w * std::sqrt(n0 / nh) : w, 0, topo); + Minimise(ctx, xyz_h, settings.xyz_iterations, xyz_minimiser); + ctx.engine->SetRowModel(ctx.rows); + } else + Minimise(ctx, xyz, settings.xyz_iterations, xyz_minimiser); } if (kind == AdpHypothesis::Kind::Tls) { ctx.ApplyTls(); // the bodies' U at the atoms as they are @@ -1881,7 +1922,7 @@ TrialOutcome RunTrial(TrialSlot slot, const std::vector &refinable, const s t.nll_free = ctx.Score().nll_free; out.line = fmt::format("{}X-ray weight {:.3g}, {} {}{}: R-work {:.4f}, R-free {:.4f}{}, rmsd bonds {:.4f} A, " "angles {:.2f} deg{}{}", - hyp != nullptr ? hyp->name + ", " : std::string(), w, + std::string(tw.blur.empty() ? "" : "blurred, ") + (hyp != nullptr ? hyp->name + ", " : ""), w, kind == AdpHypothesis::Kind::Anisotropic ? "U" : kind == AdpHypothesis::Kind::Tls ? "residual B" : "B", adp_w > 0 ? fmt::format("refined at weight {:.3g}", adp_w) : std::string("held"), ncs == nullptr || ncs->Empty() ? std::string() @@ -1938,9 +1979,8 @@ std::vector RunTrials(const std::vector &trial_slots, #endif for (size_t j = s; j < trials.size(); j = next++) if (const size_t i = order[j]; !trial_slots[s].ctx->Cancelled()) - out[i] = RunTrial(trial_slots[s], refinable, pos0, b0, aniso0, fit, trials[i].w, trials[i].adp, - cycles, settings, ncs, trials[i].ncs, hyps.empty() ? nullptr : hyps[i], - hold_positions); + out[i] = RunTrial(trial_slots[s], refinable, pos0, b0, aniso0, fit, trials[i], cycles, settings, ncs, + hyps.empty() ? nullptr : hyps[i], hold_positions); } catch (...) { errors[s] = std::current_exception(); } @@ -1957,6 +1997,58 @@ std::vector RunTrials(const std::vector &trial_slots, return out; } +// The 2mFo-DFc map of a population (RefineOnce, 2b.): every member's coefficients, at the scale, the solvent and the +// likelihood parameters fitted where it ended, averaged over the members - the averaged kick map of Praznikar et al. +// (2009) Acta Cryst. D65, 921-931 - with the free reflections left out, so that what is chosen on the map is chosen +// without them. Leaves ctx at the last member. +gemmi::Grid PopulationMap(Context &ctx, const std::vector &members) { + gemmi::AsuData> fwt; + fwt.unit_cell_ = ctx.cell; + fwt.spacegroup_ = ctx.sg; + for (size_t m = 0; m < ctx.refl.Size(); m++) + fwt.v.push_back({ctx.refl.hkl[m], {0.0f, 0.0f}}); + int n = 0; + for (const TrialOutcome &o : members) { + if (!o.ran) + continue; + ctx.SetPositions(o.pos); + ctx.SetB(o.b); + ctx.SetAniso(o.aniso); + ctx.Restore(o.fit); + const std::vector> fmodel = ctx.FModel(); + for (size_t m = 0; m < ctx.refl.Size(); m++) { + if (ctx.refl.free[m]) + continue; + double fo = ctx.refl.fo[m], fc = std::abs(fmodel[m]); + if (ctx.refl.Twinned()) { + const double own = std::abs(ctx.rows.k_total[m] * (ctx.fcalc[m] + ctx.rows.fmask[m])); + if (fc > 0) + fo *= own / fc; + fc = own; + } + const double d = ctx.rows.d[m]; + const double mw = Fom(2.0 * fo * d * fc / ctx.rows.var[m], ctx.refl.centric[m]); + const double coef = ctx.refl.centric[m] ? mw * fo : 2 * mw * fo - d * fc; + fwt.v[m].value += std::polar(static_cast(coef), static_cast(std::arg(fmodel[m]))); + } + n++; + } + for (gemmi::HklValue> &v : fwt.v) + v.value /= static_cast(n); + const std::array size = gemmi::get_size_for_hkl(fwt, {{0, 0, 0}}, 3.0); + return MapFromFPhi(gemmi::get_f_phi_on_grid(fwt, size, true)); +} + +// Per atom the residue it belongs to, numbered through the model in the atoms' order (AtomList). +std::vector ResidueOfAtom(gemmi::Model &model) { + std::vector out; + int r = 0; + for (gemmi::Chain &ch : model.chains) + for (gemmi::Residue &res : ch.residues) + out.insert(out.end(), res.atoms.size(), r++); + return out; +} + } // namespace namespace { @@ -2916,6 +3008,87 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector 0 && state.ran && !ctx.Cancelled()) { + const RefinementStage pop_stage("population"); + const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma; + // the members, the most different first: the positions as they are and kicked, on the data as they are + // and blurred to 4 A then 2.2 A, to 5 A then 2.8 A, to 2.8 A then 2 A, each a macro-cycle longer than a + // step so that every member settles on the data as they are before the crossover (members a step long + // left the basin a step longer reached on a placed prediction) + const std::vector, double>> variants = { + {{}, 0.0}, {{60, 20}, 0.0}, {{60, 20}, 0.3}, {{100, 30}, 0.2}, {{30, 10}, 0.3}, {{}, 0.2}, + {{100, 30}, 0.0}, {{30, 10}, 0.0}, {{60, 20}, 0.2}, {{100, 30}, 0.3}, {{30, 10}, 0.2}, {{}, 0.3}}; + std::vector members; + for (const auto &[blur, kick] : variants) + if (members.size() < static_cast(settings.population)) + members.push_back({w, adp, ncs_s, blur, kick, static_cast(members.size() + 1)}); + std::vector outcomes = RunTrials(trial_slots, members, refinable, state.pos, state.b, state.aniso, + &state.fit, settings.macro_cycles + 1, settings, ncs_p); + int best = -1; + for (size_t k = 0; k < outcomes.size(); k++) { + if (!outcomes[k].ran) + continue; + std::string blur; + for (double b : members[k].blur) + blur += fmt::format("{}{:.0f}", blur.empty() ? "" : "/", b); + logger.Info("Refinement, population: blur {}, kick {:.1f} A: {}", blur.empty() ? "none" : blur, + members[k].kick, outcomes[k].line); + if (outcomes[k].trial.within_bounds && (best < 0 || outcomes[k].trial.nll_free < outcomes[best].trial.nll_free)) + best = static_cast(k); + } + if (best >= 0) { + const gemmi::Grid map = PopulationMap(ctx, outcomes); + const std::vector residue = ResidueOfAtom(ctx.Model()); + const int n_res = residue.back() + 1; + std::vector> density(outcomes.size(), std::vector(n_res, 0.0)); + for (size_t k = 0; k < outcomes.size(); k++) + if (outcomes[k].ran) + for (size_t i = 0; i < ctx.atoms.size(); i++) + if (!ctx.atoms[i]->is_hydrogen() && ctx.atoms[i]->occ > 0) + density[k][residue[i]] += map.interpolate_value(outcomes[k].pos[i]); + std::vector pick(n_res, -1); + for (int r = 0; r < n_res; r++) + for (size_t k = 0; k < outcomes.size(); k++) + if (outcomes[k].ran && (pick[r] < 0 || density[k][r] > density[pick[r]][r])) + pick[r] = static_cast(k); + std::vector child_pos = state.pos; + std::vector child_b = state.b; + std::set used; + for (size_t i = 0; i < ctx.atoms.size(); i++) { + child_pos[i] = outcomes[pick[residue[i]]].pos[i]; + child_b[i] = outcomes[pick[residue[i]]].b[i]; + used.insert(pick[residue[i]]); + } + // the junctions between residues of different members mended by one macro-cycle at the ladder's + // tightest rung, B held (at the state's weight the chimera ends out of the geometry bounds), then the + // step at the state's weights + std::vector child = RunTrials(trial_slots, {{w0 * settings.weight_ladder.front(), 0, ncs_s}}, + refinable, child_pos, child_b, state.aniso, nullptr, 1, settings, + ncs_p); + child = RunTrials(trial_slots, {{w, adp, ncs_s}}, refinable, child[0].pos, child[0].b, state.aniso, + &child[0].fit, settings.macro_cycles, settings, ncs_p); + const RefinementWeightTrial &c = child[0].trial; + const bool kept = child[0].ran && displaces(c, state.trial); + logger.Info("Refinement, population: {} members, the best on the free set R-free {:.4f}, -LL free {:.1f}; " + "crossover from {} members: {}; {} (state R-free {:.4f}, -LL free {:.1f})", outcomes.size(), + outcomes[best].trial.r_free, outcomes[best].trial.nll_free, used.size(), child[0].line, + kept ? "kept" : "not kept", state.trial.r_free, state.trial.nll_free); + if (kept) { + state = std::move(child[0]); + from_best.clear(); + } + } + } + // 2a. The solvent build (SolventBuild.h), from where the trajectory ended: waters put in at the difference // map's peaks, taken out where the maps no longer hold them, and waters and ions relabelled, on the maps // and the solvent audit of that model; then the solvent alone refined, the rest held, at the weights the diff --git a/image_analysis/structure_refinement/Refinement.h b/image_analysis/structure_refinement/Refinement.h index 4145c5532..3f2cf3f91 100644 --- a/image_analysis/structure_refinement/Refinement.h +++ b/image_analysis/structure_refinement/Refinement.h @@ -80,6 +80,11 @@ struct RefinementSettings { bool adp_hypotheses = true; // Rounds of the weights and the displacement model decided again from the refined model (RefineOnce, 4b.). int rounds = 3; + // The population (RefineOnce, 2b.): this many members at most, refined from where the weight trajectory ended in + // one wave, each on the data blurred along a schedule of its own and its positions kicked or not, then crossed + // residue by residue on their averaged map and refined once more; the child kept where the free set prefers it. + // 0: no population. + int population = 12; // A refinement against the data as twinned by each twin law the lattice admits (RefinementTwinLaws), beside the // untwinned one, and the model of whichever the evidence favours (RefineModel). bool twin = true; From 1689745325379b357874f4d45a8263ea5cb5c3fe Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 18:02:30 +0100 Subject: [PATCH 6/7] Refinement: the population and the side-chain field together, each capped in cost The population (now step 2a) runs before the solvent build (2b) and the side-chain field (2c), and the documentation lists them in that order. Two caps keep the stages affordable on large models: - The side-chain field searches the copies of a residue each alone only where a sweep stays within 4e8 likelihood evaluations (states times reflections, about 3 s on one card); beyond that the copies are searched together only, which is where a model of many copies spent its time. - The population has at most as many members as there are trial slots, so it runs in one wave; with fewer than two slots it is left out. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 53 ++++++++++--------- .../structure_refinement/Refinement.cpp | 19 ++++--- .../structure_refinement/Refinement.h | 4 +- .../structure_refinement/SideChainField.cpp | 17 +++++- .../structure_refinement/SideChainField.h | 6 +-- tests/SideChainFieldTest.cpp | 2 +- 6 files changed, 60 insertions(+), 41 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 2109c5a92..1a68aac39 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -331,28 +331,8 @@ are: stays within 0.001 of the best so far, and the trajectory ends at its best step. On a deposited model, whose free set the depositor's refinement has already fitted, R-work falls while R-free rises, and the trajectory ends where it began. - Then, from where the trajectory (and the solvent build, §15.10) ended, **the side-chain field**: every side - chain the search can take - a complete amino-acid side chain at full occupancy, refinable, without alternative - conformations and not bound by a LINK - is a discrete variable whose states are its staggered rotamers (the - terminal planar group every 30°), the conformation it has, and *absent* (pruned to CB, its atoms at occupancy 0: - the honest answer where no rotamer fits). The energy of a state is the refinement's own target with the rest of - the model held: $-\ln L$ over the working reflections at the scale, the solvent and the likelihood parameters - as they stand, plus the repulsion term against the rest of the model (symmetry mates and the other copies - included) over the X-ray weight. The structure factors are linear in the atoms, so with the side chain's own - contribution computed by direct summation - a dozen atoms over the group's operators - a change of state - moves F by $dF_{new} - dF_{old}$ and the target is re-evaluated at every reflection for every state in one pass - on the GPU, microseconds per state, no map and no gradient. The field is minimised by iterated conditional - modes: the residues in turn, each put in its state of least energy given the others, small side chains first - and arginine last, until a sweep changes nothing. An FFT gradient cannot carry a side chain 1.5 Å at 1.7 Å - resolution; the field, a search, can, and it is scored by the likelihood rather than by a map, which on an - incomplete model rewards whatever fills unmodelled density. The free reflections are never seen by the field: - one step at the trajectory's weights is then taken from it and judged against the same step from the model as - it was, kept only where both its R-free and its $-\ln L$ of the free reflections are lower and the geometry - within bounds, undone otherwise (a model whose side chains the data already hold moves nothing and costs the - sweep alone). The log names each side chain moved with its chi angles before and after and the energy the move - took off (`REFINE_SIDE_CHAIN_FIELD_MOVED`, `REFINE_SIDE_CHAIN_FIELD_PRUNED`). Twinned data and a model with - anisotropic atoms are refined without the field. -3. **The population**, from where the trajectory ended: twelve members refined side by side, each a trial at the +3. **The population**, from where the trajectory ended: twelve members (at most as many as the trial slots, so + that they run in one wave; none below two) refined side by side, each a trial at the trajectory's weights but on its own path - the positions as they are or kicked at random by up to 0.2-0.3 Å in each coordinate, on the data as they are or **blurred** along a schedule, 60 then 20 Ų (the landscape of 4 Å, then of 2.2 Å), 100 then 30, or 30 then 10. The blur raises the likelihood's variance of each reflection @@ -367,9 +347,32 @@ are: 13 predictions placed by MR the child was kept on four (R-free 0.005-0.011 lower) and on one of 22 refined models (0.009 lower, the rest unchanged); the gain, where there is one, comes mostly in the rounds that follow from the child. -4. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved +4. **The side-chain field**, from where the population and the solvent build (§15.10) ended: every side + chain the search can take - a complete amino-acid side chain at full occupancy, refinable, without alternative + conformations and not bound by a LINK - is a discrete variable whose states are its staggered rotamers (the + terminal planar group every 30°), the conformation it has, and *absent* (pruned to CB, its atoms at occupancy 0: + the honest answer where no rotamer fits). The energy of a state is the refinement's own target with the rest of + the model held: $-\ln L$ over the working reflections at the scale, the solvent and the likelihood parameters + as they stand, plus the repulsion term against the rest of the model (symmetry mates and the other copies + included) over the X-ray weight. The structure factors are linear in the atoms, so with the side chain's own + contribution computed by direct summation - a dozen atoms over the group's operators - a change of state + moves F by $dF_{new} - dF_{old}$ and the target is re-evaluated at every reflection for every state in one pass + on the GPU, microseconds per state, no map and no gradient. The field is minimised by iterated conditional + modes: the residues in turn, each put in its state of least energy given the others, small side chains first + and arginine last, until a sweep changes nothing. The copies of a residue related by NCS are one variable, + searched in each rotamer together and each alone - together only where a sweep would otherwise take more than + $4 \times 10^8$ evaluations (states times reflections, about 3 s on one card). An FFT gradient cannot carry a side chain 1.5 Å at 1.7 Å + resolution; the field, a search, can, and it is scored by the likelihood rather than by a map, which on an + incomplete model rewards whatever fills unmodelled density. The free reflections are never seen by the field: + one step at the trajectory's weights is then taken from it and judged against the same step from the model as + it was, kept only where both its R-free and its $-\ln L$ of the free reflections are lower and the geometry + within bounds, undone otherwise (a model whose side chains the data already hold moves nothing and costs the + sweep alone). The log names each side chain moved with its chi angles before and after and the energy the move + took off (`REFINE_SIDE_CHAIN_FIELD_MOVED`, `REFINE_SIDE_CHAIN_FIELD_PRUNED`). Twinned data and a model with + anisotropic atoms are refined without the field. +5. **The final ladder**: the trajectory's weights times $\{1, 2\}$, kept where one displaces it (the weight halved was never kept on 55 refinements, and taking it away changed no choice). -5. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U +6. **The displacement model**, from where the weights ended (the B factors as they are: isotropic, or the model's U held), each trial as long as a step on the positions where the weights ended, held - only the displacement parameters refine, so the trials differ in the displacement model alone (where the geometry is out of bounds there, which held positions cannot mend, each trial refines the positions too, a macro-cycle longer than a @@ -399,7 +402,7 @@ are: R-free beyond the tie and undone otherwise; where its ladder and trajectory leave the model where it was, the hypotheses would repeat the same trials, and it ends there - from a model refined before, a round is one ladder and one step. No round runs where the refinement does not displace the model as given. -6. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the +7. **The model as given** is the first candidate in order of trust: where the refinement does not displace it by the rule every trial is held to (its geometry within bounds and R-free lower beyond the tie), the model is written as it came, scaled to the data, and the solvent build's edits are undone (`REFINE_AS_GIVEN`). What comes out is never worse on the free set than what went in. This holds only where the free set is free to the model as given: a model diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index ff05ba7c2..759a55a54 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -1723,7 +1723,7 @@ std::vector OwnSlotPlaces(const Context &main, int wanted, const Refi // none). struct TrialWeights { double w = 0, adp = 0, ncs = 0; - // A member of the population (RefineOnce, 2b.): the positions refined on the data blurred by B_h per macro-cycle + // A member of the population (RefineOnce, 2a.): the positions refined on the data blurred by B_h per macro-cycle // (A^2; empty or 0: not blurred) - the likelihood's variance raised by exp(B_h s^2 / 2), which is Fo and F_model // both blurred by exp(-B_h s^2 / 4): the landscape of a lower resolution, which a position 1-2 A off still // descends - with the X-ray weight balanced again for it; and before the first macro-cycle every refinable atom @@ -1997,7 +1997,7 @@ std::vector RunTrials(const std::vector &trial_slots, return out; } -// The 2mFo-DFc map of a population (RefineOnce, 2b.): every member's coefficients, at the scale, the solvent and the +// The 2mFo-DFc map of a population (RefineOnce, 2a.): every member's coefficients, at the scale, the solvent and the // likelihood parameters fitted where it ended, averaged over the members - the averaged kick map of Praznikar et al. // (2009) Acta Cryst. D65, 921-931 - with the free reflections left out, so that what is chosen on the map is chosen // without them. Leaves ctx at the last member. @@ -3008,7 +3008,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector 0 && state.ran && !ctx.Cancelled()) { + // The members are at most as many as the trial slots, one wave: on a large model, where the card holds + // few, more members would cost a wave each. + const size_t population = std::min(settings.population, trial_slots.size()); + if (population > 1 && state.ran && !ctx.Cancelled()) { const RefinementStage pop_stage("population"); const double w = state.trial.weight, adp = state.trial.adp_weight, ncs_s = state.trial.ncs_sigma; // the members, the most different first: the positions as they are and kicked, on the data as they are @@ -3029,7 +3032,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector members; for (const auto &[blur, kick] : variants) - if (members.size() < static_cast(settings.population)) + if (members.size() < population) members.push_back({w, adp, ncs_s, blur, kick, static_cast(members.size() + 1)}); std::vector outcomes = RunTrials(trial_slots, members, refinable, state.pos, state.b, state.aniso, &state.fit, settings.macro_cycles + 1, settings, ncs_p); @@ -3089,7 +3092,7 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector FieldAtomsOf(const gemmi::Residue &res, const SideChain & SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const std::vector &element_of, const std::vector &refinable, const std::vector &ncs_group, - double weight, SideChainFieldEvaluator &evaluator) { + double weight, size_t rows, SideChainFieldEvaluator &evaluator) { SideChainFieldResult result; gemmi::Structure st = given; gemmi::Model &model = st.models[0]; @@ -301,6 +304,16 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const if (candidates.empty()) return result; const std::vector> variables = Variables(candidates, model, ncs_group); + // each copy alone in each rotamer is what makes a sweep over many copies costly: dropped where it would take a + // sweep over the budget + double sweep_cost = 0; + for (const std::vector &var : variables) { + const Candidate &cand = candidates[var[0]]; + const double n_pos = SideChainRotamers(model.chains[cand.chain].residues[cand.residue].name, + cand.sc.chi.size()).size() + 1; + sweep_cost += (1 + n_pos * (var.size() > 1 ? var.size() + 1 : 1)) * static_cast(rows); + } + const bool copies_alone = sweep_cost <= SWEEP_BUDGET; // the repulsion term of the refinement's target, in the units of -ln L: ((d0 - d) / sigma)^2 summed, over the // X-ray weight const double repulsion = 1.0 / (REPULSION_SIGMA * REPULSION_SIGMA * weight); @@ -369,7 +382,7 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const std::vector> assign = {std::vector(k, 0)}; for (int p = 1; p <= absent; p++) assign.emplace_back(k, p); - if (k > 1) + if (k > 1 && copies_alone) for (size_t c = 0; c < k; c++) for (int p = 1; p <= absent; p++) { assign.emplace_back(k, 0); diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h index c8cc05b7b..c08d46f81 100644 --- a/image_analysis/structure_refinement/SideChainField.h +++ b/image_analysis/structure_refinement/SideChainField.h @@ -133,12 +133,12 @@ struct SideChainFieldResult { // `ncs_group`: per residue (numbered over the model in order) the NCS group of its copies, -1 where it has none // (or empty): the copies of a residue are one variable, searched in each rotamer together - the NCS restraints of // the step after would pull a copy moved alone back - and each alone, for the copy whose environment differs (a -// crystal contact); `weight`: the X-ray term's weight in the refinement's target, which the repulsion term is -// divided by. The residues searched: amino acids with a complete side chain, no alternative conformations, every +// crystal contact) where a sweep stays within its budget; `weight`: the X-ray term's weight in the refinement's target, which the repulsion term is +// divided by; `rows`: the evaluator's reflections. The residues searched: amino acids with a complete side chain, no alternative conformations, every // atom at full occupancy, refinable and not bound by a LINK. `st` is left as it was; the moves say what changed. SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &st, const std::vector &element_of, const std::vector &refinable, const std::vector &ncs_group, - double weight, SideChainFieldEvaluator &evaluator); + double weight, size_t rows, SideChainFieldEvaluator &evaluator); // The state of the field's atoms of one residue (depth 2 and beyond, hydrogens included) at positions `p`, at full // occupancy. diff --git a/tests/SideChainFieldTest.cpp b/tests/SideChainFieldTest.cpp index 9016c8562..0a050764a 100644 --- a/tests/SideChainFieldTest.cpp +++ b/tests/SideChainFieldTest.cpp @@ -176,7 +176,7 @@ TEST_CASE("SideChainField_recovers_the_rotamer_the_data_hold", "[Refinement]") { FieldStates s; s.Add(all_atoms(start)); e.SetTotal(e.Contribution(s, 0)); - return OptimiseSideChainField(start, element_of, refinable, {}, 1.0, e); + return OptimiseSideChainField(start, element_of, refinable, {}, 1.0, fr.Size(), e); }; auto cpu = MakeSideChainFieldEvaluatorCPU(fr, ops, ff, 2); const SideChainFieldResult r = run(*cpu); From 066a0deb125f78c6003536121bb6b3b575bf208c Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 14 Nov 2026 23:14:43 +0100 Subject: [PATCH 7/7] Side-chain field: copies searched together only are one state each, not one per copy Where the sweep budget leaves the copies of a residue together only, each position is now a single state holding every copy's atoms, so a variable of k copies takes a k-th of the states, of the contribution work and of the card memory. With five copies, 305k reflections and up to 325 positions for a side chain with a planar end, the states of one copy each took ~4 GB beside the refinement's pool and the refinement failed on the GPU (out of memory). The energies are the same sums as before. The log says when the copies were searched together only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UqXYdJDrM65gxsCn2PrbfK --- docs/CPU_DATA_ANALYSIS_REFINEMENT.md | 5 +- .../structure_refinement/Refinement.cpp | 5 +- .../structure_refinement/SideChainField.cpp | 66 ++++++++++++------- .../structure_refinement/SideChainField.h | 1 + 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md index 1a68aac39..2951ec81e 100644 --- a/docs/CPU_DATA_ANALYSIS_REFINEMENT.md +++ b/docs/CPU_DATA_ANALYSIS_REFINEMENT.md @@ -360,8 +360,9 @@ are: on the GPU, microseconds per state, no map and no gradient. The field is minimised by iterated conditional modes: the residues in turn, each put in its state of least energy given the others, small side chains first and arginine last, until a sweep changes nothing. The copies of a residue related by NCS are one variable, - searched in each rotamer together and each alone - together only where a sweep would otherwise take more than - $4 \times 10^8$ evaluations (states times reflections, about 3 s on one card). An FFT gradient cannot carry a side chain 1.5 Å at 1.7 Å + searched in each rotamer together and each alone - together only, one state holding every copy's atoms, where a + sweep would otherwise take more than $4 \times 10^8$ evaluations (states times reflections, about 3 s on one + card). An FFT gradient cannot carry a side chain 1.5 Å at 1.7 Å resolution; the field, a search, can, and it is scored by the likelihood rather than by a map, which on an incomplete model rewards whatever fills unmodelled density. The free reflections are never seen by the field: one step at the trajectory's weights is then taken from it and judged against the same step from the model as diff --git a/image_analysis/structure_refinement/Refinement.cpp b/image_analysis/structure_refinement/Refinement.cpp index 759a55a54..caf274933 100644 --- a/image_analysis/structure_refinement/Refinement.cpp +++ b/image_analysis/structure_refinement/Refinement.cpp @@ -3452,8 +3452,9 @@ RefinementResult RefineOnce(const gemmi::Structure &placed, const std::vector 1) logger.Debug("Side-chain field: {} as given {}, best alternative {} would gain {:.1f}", a.name, chis(a.chi_given), chis(a.chi_best), a.gain); - logger.Info("Side-chain field: {} residues searched, {} moved ({} pruned) in {} sweep{}, energy {:.1f} -> " - "{:.1f}, {:.2f} s{}{}", field.candidates, field.moves.size(), pruned, field.sweeps, + logger.Info("Side-chain field: {} residues searched{}, {} moved ({} pruned) in {} sweep{}, energy {:.1f} -> " + "{:.1f}, {:.2f} s{}{}", field.candidates, field.copies_together ? " (copies together)" : "", + field.moves.size(), pruned, field.sweeps, field.sweeps == 1 ? "" : "s", field.energy_before, field.energy_after, std::chrono::duration(std::chrono::steady_clock::now() - field_start).count(), names.empty() ? "" : ": ", names); diff --git a/image_analysis/structure_refinement/SideChainField.cpp b/image_analysis/structure_refinement/SideChainField.cpp index 152a97845..ad2d428f4 100644 --- a/image_analysis/structure_refinement/SideChainField.cpp +++ b/image_analysis/structure_refinement/SideChainField.cpp @@ -314,6 +314,7 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const sweep_cost += (1 + n_pos * (var.size() > 1 ? var.size() + 1 : 1)) * static_cast(rows); } const bool copies_alone = sweep_cost <= SWEEP_BUDGET; + result.copies_together = !copies_alone; // the repulsion term of the refinement's target, in the units of -ln L: ((d0 - d) / sigma)^2 summed, over the // X-ray weight const double repulsion = 1.0 / (REPULSION_SIGMA * REPULSION_SIGMA * weight); @@ -366,34 +367,53 @@ SideChainFieldResult OptimiseSideChainField(const gemmi::Structure &given, const geometry[c][0] = 0; } const int n_pos = static_cast(positions[0].size()), absent = n_pos; - // the states: per copy its positions (c * n_pos + p; as it is: no atoms where it was pruned); the - // combinations as assignments, per copy the index of its positions (absent beyond them): as it is, the - // copies together in each rotamer and absent, then each copy alone in each - FieldStates states; - for (size_t c = 0; c < k; c++) { + auto atoms_of = [&](size_t c, int p) { const Candidate &cand = candidates[var[c]]; - for (int p = 0; p < n_pos; p++) - states.Add(p == 0 && cand.absent - ? std::vector() - : FieldAtomsOf(model.chains[cand.chain].residues[cand.residue], cand.sc, - positions[c][p], cell, element_of, cand.first_atom)); - states.current.push_back(static_cast(c) * n_pos); - } + if (p == 0 && cand.absent) // as it is, pruned: no atoms + return std::vector(); + return FieldAtomsOf(model.chains[cand.chain].residues[cand.residue], cand.sc, positions[c][p], cell, + element_of, cand.first_atom); + }; + // the combinations as assignments, per copy the index of its positions (absent beyond them): as it is, + // the copies together in each rotamer and absent, then each copy alone in each std::vector> assign = {std::vector(k, 0)}; for (int p = 1; p <= absent; p++) assign.emplace_back(k, p); - if (k > 1 && copies_alone) - for (size_t c = 0; c < k; c++) - for (int p = 1; p <= absent; p++) { - assign.emplace_back(k, 0); - assign.back()[c] = p; + FieldStates states; + if (copies_alone) { + // the states: per copy its positions (c * n_pos + p) + for (size_t c = 0; c < k; c++) { + for (int p = 0; p < n_pos; p++) + states.Add(atoms_of(c, p)); + states.current.push_back(static_cast(c) * n_pos); + } + if (k > 1) + for (size_t c = 0; c < k; c++) + for (int p = 1; p <= absent; p++) { + assign.emplace_back(k, 0); + assign.back()[c] = p; + } + for (const std::vector &a : assign) { + std::vector combo; + for (size_t c = 0; c < k; c++) + if (a[c] != absent) + combo.push_back(static_cast(c) * n_pos + a[c]); + states.AddCombo(combo); + } + } else { + // the copies together only: one state per position, every copy's atoms in it (p), a k-th of the + // states and of the memory they take on the card + for (int p = 0; p < n_pos; p++) { + std::vector atoms; + for (size_t c = 0; c < k; c++) { + const std::vector a = atoms_of(c, p); + atoms.insert(atoms.end(), a.begin(), a.end()); } - for (const std::vector &a : assign) { - std::vector combo; - for (size_t c = 0; c < k; c++) - if (a[c] != absent) - combo.push_back(static_cast(c) * n_pos + a[c]); - states.AddCombo(combo); + states.Add(atoms); + } + states.current.push_back(0); + for (const std::vector &a : assign) + states.AddCombo(a[0] == absent ? std::vector() : std::vector{a[0]}); } std::vector energy = evaluator.Evaluate(states); // the alternatives are many and the best of them is chosen: a move must beat the state as given by the diff --git a/image_analysis/structure_refinement/SideChainField.h b/image_analysis/structure_refinement/SideChainField.h index c08d46f81..2385051c9 100644 --- a/image_analysis/structure_refinement/SideChainField.h +++ b/image_analysis/structure_refinement/SideChainField.h @@ -124,6 +124,7 @@ struct SideChainFieldResult { int candidates = 0; // residues searched std::vector alternatives; // per variable, in the order searched int sweeps = 0; // the first over every residue, the others over those near a move + bool copies_together = false; // the copies of a residue searched together only (the sweep budget) double energy_before = 0, energy_after = 0; std::vector moves; };